From 38bd8a5ebd83ea92ee6623ce8d2862eeef07cc54 Mon Sep 17 00:00:00 2001 From: jinchen62 Date: Fri, 23 Jun 2023 07:02:11 -0700 Subject: [PATCH] Add int4/int8 options to chatbot webui --- apps/language_models/src/pipelines/vicuna_pipeline.py | 3 +++ apps/stable_diffusion/web/ui/stablelm_ui.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/apps/language_models/src/pipelines/vicuna_pipeline.py b/apps/language_models/src/pipelines/vicuna_pipeline.py index 3c9c123c67..60cf258f4e 100644 --- a/apps/language_models/src/pipelines/vicuna_pipeline.py +++ b/apps/language_models/src/pipelines/vicuna_pipeline.py @@ -38,6 +38,9 @@ def __init__( super().__init__(model_name, hf_model_path, max_num_tokens) self.max_sequence_length = 256 self.device = device + if precision in ["int4", "int8"]: + print("int4 and int8 are not supported yet, using fp32") + precision = "fp32" self.precision = precision self.first_vicuna_vmfb_path = first_vicuna_vmfb_path self.second_vicuna_vmfb_path = second_vicuna_vmfb_path diff --git a/apps/stable_diffusion/web/ui/stablelm_ui.py b/apps/stable_diffusion/web/ui/stablelm_ui.py index 0e5cf4092d..369d864a31 100644 --- a/apps/stable_diffusion/web/ui/stablelm_ui.py +++ b/apps/stable_diffusion/web/ui/stablelm_ui.py @@ -140,6 +140,8 @@ def chat(curr_system_message, history, model, device, precision): choices=[ "fp16", "fp32", + "int4", + "int8", ], visible=True, )