Update app.py

app.py

@@ -1,12 +1,20 @@
 from fastapi import FastAPI
 import uvicorn
-from transformers import AutoModelForCausalLM, AutoTokenizer
+from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
 
 model_name = "Tap-M/Luna-AI-Llama2-Uncensored"
 
+bnb_config = BitsAndBytesConfig(
+    load_in_4bit=True,  # Enable 4-bit quantization
+    bnb_4bit_compute_dtype=torch.float16
+)
+
 model = AutoModelForCausalLM.from_pretrained(
     model_name,  # Example model
     device_map="auto",            # Auto-distribute across GPU/CPU
+    quantization_config=bnb_config,
+    offload_folder="./offload",  # Temporary directory
+    low_cpu_mem_usage=True,      # Reduces CPU memory spikes
     trust_remote_code=True        # Required for some models
 )