Update app.py

app.py

@@ -5,6 +5,8 @@ import boto3
 from botocore import UNSIGNED
 from botocore.client import Config
 
+import torch
+
 
 from huggingface_hub import AsyncInferenceClient
 
@@ -18,6 +20,7 @@ from langchain.chains import RetrievalQA
 from langchain.prompts import ChatPromptTemplate
 from langchain.document_loaders import WebBaseLoader
 from langchain.llms.huggingface_pipeline import HuggingFacePipeline
+from langchain.llms import CTransformers
 
 from transformers import AutoModel
 
@@ -51,11 +54,21 @@ model = AutoModelForCausalLM.from_pretrained(model_id)
 # model = AutoModel.from_pretrained("TheBloke/zephyr-7B-beta-GGUF")
 
 
+llm = CTransformers(
+    model="TheBloke/zephyr-7B-beta-GGUF",
+    model_type="mistral",
+    max_new_tokens=4384,
+    temperature=0.2,
+    repetition_penalty=1.13,
+    device=device  # Set the device explicitly during model initialization
+)
+
+
 tokenizer = AutoTokenizer.from_pretrained(model_id)
 # model = AutoModelForCausalLM.from_pretrained(model_id)
 
-pipe = pipeline("text-generation", model=model, tokenizer=tokenizer, max_new_tokens=10)
-hf = HuggingFacePipeline(pipeline=pipe)
+# pipe = pipeline("text-generation", model=model, tokenizer=tokenizer, max_new_tokens=10)
+# hf = HuggingFacePipeline(pipeline=pipe)
 
 
 print( "initalized second model")