Update fine_tune_inference_test_mistral.py

fine_tune_inference_test_mistral.py

@@ -8,7 +8,7 @@ from huggingface_hub import hf_hub_download
 
 # === Ayarlar
 HF_TOKEN = os.getenv("HF_TOKEN")
-MODEL_BASE = "malhajar/Mistral-7B-Instruct-v0.2-turkish"
+MODEL_BASE = "mistralai/Mistral-7B-Instruct-v0.2"
 USE_FINE_TUNE = False
 FINE_TUNE_REPO = "UcsTurkey/trained-zips"
 FINE_TUNE_ZIP = "trained_model_000_009.zip"
@@ -76,11 +76,10 @@ def chat(msg: Message):
             return {"error": "Boş giriş"}
 
         messages = [{"role": "user", "content": user_input}]
-        inputs = tokenizer.apply_chat_template(messages, return_tensors="pt", add_generation_prompt=True)
-        inputs = {k: v.to(model.device) for k, v in inputs.items()}
+        input_ids = tokenizer.apply_chat_template(messages, return_tensors="pt", add_generation_prompt=True).to(model.device)
 
         generate_args = {
-            "max_new_tokens": 512,
+            "max_new_tokens": 128,
             "return_dict_in_generate": True,
             "output_scores": True,
             "do_sample": USE_SAMPLING
@@ -94,11 +93,10 @@ def chat(msg: Message):
             })
 
         with torch.no_grad():
-            output = model.generate(**inputs, **generate_args)
+            output = model.generate(input_ids=input_ids, **generate_args)
 
-        prompt_text = tokenizer.decode(inputs["input_ids"][0], skip_special_tokens=True)
         decoded = tokenizer.decode(output.sequences[0], skip_special_tokens=True)
-        answer = decoded.replace(prompt_text, "").strip()
+        answer = decoded.split("</s>")[-1].strip()
 
         if output.scores and len(output.scores) > 0:
             first_token_score = output.scores[0][0]