Switch to GPTQ model, remove BitsAndBytesConfig

app.py

@@ -1,14 +1,12 @@
 import gradio as gr
-from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline, BitsAndBytesConfig
+from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
 
-model_id = "TheBloke/CodeLlama-7B-GPTQ"  # Example 4-bit quantized model
-
-bnb_config = BitsAndBytesConfig(load_in_4bit=True, device_map="auto")
+# Use prequantized GPTQ model — no BitsAndBytesConfig needed
+model_id = "TheBloke/CodeLlama-7B-GPTQ"
 
 tokenizer = AutoTokenizer.from_pretrained(model_id)
 model = AutoModelForCausalLM.from_pretrained(
     model_id,
-    quantization_config=bnb_config,
     device_map="auto"
 )
 
@@ -22,6 +20,6 @@ gr.Interface(
     fn=generate_response,
     inputs=gr.Textbox(lines=5, label="Your prompt"),
     outputs=gr.Textbox(label="Code Llama response"),
-    title="Code Llama Demo",
+    title="Code Llama Demo (GPTQ)",
     description="Ask questions or request code snippets!"
 ).launch()

requirements.txt

@@ -1,4 +1,3 @@
-transformers
-gradio
+transformers==4.39.3
 accelerate
-bitsandbytes
+gradio