Update app.py

app.py

@@ -1,9 +1,25 @@
 from fastapi import FastAPI
-from transformers import AutoModelForCausalLM, AutoTokenizer
+from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
 import uvicorn
 
-tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-0.6B")
-model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen3-0.6B")
+bnb_config = BitsAndBytesConfig(
+    load_in_4bit=True,          # Enable 4-bit quantization
+    bnb_4bit_quant_type="nf4",  # Use normalized float 4 
+    bnb_4bit_compute_dtype="float16",  # Faster computations
+    bnb_4bit_use_double_quant=True  # Extra compression
+)
+
+model = AutoModelForCausalLM.from_pretrained(
+    "Qwen/Qwen-7B-Chat",
+    quantization_config=bnb_config,
+    device_map="auto",  # Auto-distribute across CPU/GPU
+    trust_remote_code=True  # Required for Qwen!
+)
+
+tokenizer = AutoTokenizer.from_pretrained(
+    "Qwen/Qwen-7B-Chat",
+    trust_remote_code=True
+)
 
 app = FastAPI()