Update app.py

app.py

@@ -1,5 +1,7 @@
 import gradio as gr
-from transformers import pipeline
+from instruct_pipeline import InstructionTextGenerationPipeline
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
 import torch
 
 theme = gr.themes.Monochrome(
@@ -10,9 +12,13 @@ theme = gr.themes.Monochrome(
     font=[gr.themes.GoogleFont("Open Sans"), "ui-sans-serif", "system-ui", "sans-serif"],
 )
 
-instruct_pipeline = pipeline(model="databricks/dolly-v2-12b", torch_dtype=torch.bfloat16, trust_remote_code=True, device_map="auto")
+tokenizer = AutoTokenizer.from_pretrained("databricks/dolly-v2-12b", padding_side="left")
+model = AutoModelForCausalLM.from_pretrained("databricks/dolly-v2-12b", load_in_8bit=True, device_map="auto")
+
+generate_text = InstructionTextGenerationPipeline(model=model, tokenizer=tokenizer)
+
 def generate(instruction): 
-    response = instruct_pipeline(instruction)
+    response = generate_text(instruction)
     result = ""
     for word in response.split(" "):
         result += word + " "