Update app.py

app.py

@@ -6,14 +6,14 @@ from peft import PeftModel
 # Load base model & tokenizer
 base_model = "vilsonrodrigues/falcon-7b-instruct-sharded"
 tokenizer = AutoTokenizer.from_pretrained(base_model)
-model = AutoModelForCausalLM.from_pretrained(base_model, torch_dtype=torch.float16, device_map="auto")
+model = AutoModelForCausalLM.from_pretrained(model_name, device_map="cpu", torch_dtype=torch.float32)
 
 # Load LoRA adapter
 adapter_path = "./model"
 model = PeftModel.from_pretrained(model, adapter_path)
 
 def generate_response(prompt):
-    inputs = tokenizer(prompt, return_tensors="pt").to("cuda")
+    inputs = tokenizer(prompt, return_tensors="pt").to("cpu")
     with torch.no_grad():
         outputs = model.generate(**inputs, max_length=200)
     return tokenizer.decode(outputs[0], skip_special_tokens=True)