Add int4/int8 options to chatbot webui (#1586)

nod-ai · Jun 23, 2023 · 4002da7 · 4002da7
1 parent ecb5e8e
commit 4002da7
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 0 deletions.
diff --git a/apps/language_models/src/pipelines/vicuna_pipeline.py b/apps/language_models/src/pipelines/vicuna_pipeline.py
@@ -38,6 +38,9 @@ def __init__(
         super().__init__(model_name, hf_model_path, max_num_tokens)
         self.max_sequence_length = 256
         self.device = device
+        if precision in ["int4", "int8"]:
+            print("int4 and int8 are not supported yet, using fp32")
+            precision = "fp32"
         self.precision = precision
         self.first_vicuna_vmfb_path = first_vicuna_vmfb_path
         self.second_vicuna_vmfb_path = second_vicuna_vmfb_path

diff --git a/apps/stable_diffusion/web/ui/stablelm_ui.py b/apps/stable_diffusion/web/ui/stablelm_ui.py
@@ -140,6 +140,8 @@ def chat(curr_system_message, history, model, device, precision):
             choices=[
                 "fp16",
                 "fp32",
+                "int4",
+                "int8",
             ],
             visible=True,
         )