Upload 3 files

app.py

@@ -1,9 +1,22 @@
 from fastapi import FastAPI
 from transformers import pipeline
+from transformers import AutoModelForCausalLM, BitsAndBytesConfig
+import torch
 
 app = FastAPI()
 
-pipe = pipeline("text-generation", model="VolkanSimsir/LLaMA-3-8B-GRPO-math-tr")
+bnb_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_quant_type="nf4",
+    bnb_4bit_compute_dtype=torch.float16
+)
+
+model = AutoModelForCausalLM.from_pretrained(
+    "VolkanSimsir/LLaMA-3-8B-GRPO-math-tr",
+    quantization_config=bnb_config,  
+    torch_dtype=torch.float16,      
+    device_map="auto"           
+)
 
 @app.get("/")
 def home():