Update fine_tune_inference_test.py

fine_tune_inference_test.py

@@ -7,6 +7,7 @@ from pydantic import BaseModel
 from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
 from datasets import load_dataset
 from peft import PeftModel
+import torch  # eksikse gerekli
 
 # ✅ Sabitler
 HF_TOKEN = os.environ.get("HF_TOKEN")
@@ -19,6 +20,7 @@ RAG_DATA_REPO = "UcsTurkey/turkish-general-culture-tokenized"
 # ✅ FastAPI app
 app = FastAPI()
 chat_history = []
+pipe = None  # ❗ Global olarak tanımlıyoruz
 
 class Message(BaseModel):
     user_input: str
@@ -51,6 +53,10 @@ def root():
 
 @app.post("/chat")
 def chat(msg: Message):
+    global pipe
+    if pipe is None:
+        return {"error": "Model henüz yüklenmedi, lütfen birkaç saniye sonra tekrar deneyin."}
+
     user_input = msg.user_input.strip()
     if not user_input:
         return {"error": "Boş giriş"}
@@ -66,7 +72,6 @@ def chat(msg: Message):
     chat_history.append({"user": user_input, "bot": answer})
     return {"answer": answer, "chat_history": chat_history}
 
-
 # ✅ Model ve RAG yükleme
 def setup_model():
     global pipe
@@ -89,7 +94,7 @@ def setup_model():
     tokenizer = AutoTokenizer.from_pretrained(os.path.join(extract_dir, "output"))
 
     print("🧠 Base model indiriliyor...")
-    base_model = AutoModelForCausalLM.from_pretrained(MODEL_BASE, torch_dtype="auto")
+    base_model = AutoModelForCausalLM.from_pretrained(MODEL_BASE, torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32)
 
     print("➕ LoRA adapter uygulanıyor...")
     model = PeftModel.from_pretrained(base_model, os.path.join(extract_dir, "output"))
@@ -98,7 +103,14 @@ def setup_model():
     rag = load_dataset(RAG_DATA_REPO, data_files=RAG_DATA_FILE, split="train", token=HF_TOKEN)
     print(f"🔍 RAG boyutu: {len(rag)}")
 
-    pipe = pipeline("text-generation", model=model, tokenizer=tokenizer)
+    # ✅ pipeline oluşturuluyor
+    pipe = pipeline(
+        "text-generation",
+        model=model,
+        tokenizer=tokenizer,
+        torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32,
+        device=0 if torch.cuda.is_available() else -1
+    )
 
 # ✅ Uygulama başladığında modeli yükle
 threading.Thread(target=setup_model, daemon=True).start()