Update app.py

app.py

@@ -4,7 +4,7 @@ import gradio as gr
 from gradio.themes.base import Base
 from gradio.themes.utils import colors, fonts, sizes
 from instruct_pipeline import InstructionTextGenerationPipeline
-from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
+from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline, BitsAndBytesConfig
 
 import torch
 
@@ -17,7 +17,9 @@ theme = gr.themes.Monochrome(
 )
 
 tokenizer = AutoTokenizer.from_pretrained("databricks/dolly-v2-12b", padding_side="left")
-model = AutoModelForCausalLM.from_pretrained("databricks/dolly-v2-12b", device_map="auto", load_in_8bit=True)
+quantization_config = BitsAndBytesConfig(load_in_8bit=True,
+                                         llm_int8_threshold=200.0)
+model = AutoModelForCausalLM.from_pretrained("databricks/dolly-v2-12b", device_map="auto", quantization_config=quantization_config)
 
 generate_text = InstructionTextGenerationPipeline(model=model, tokenizer=tokenizer)