Update app.py

app.py

@@ -20,17 +20,9 @@ def log(message):
 
 # === Helper fonksiyonlar
 def trim_history(messages, max_blocks=20):
-    """
-    Mesaj geçmişini en fazla max_blocks (örn. 20: 10 user + 10 assistant) ile sınırla.
-    En eski mesajlar atılır.
-    """
     return messages[-max_blocks:]
 
-def enforce_token_budget(tokenizer, system_prompt, history_messages, user_input, total_ctx=4096, max_new_tokens=300):
-    """
-    Toplam token limitini aşmamak için history’yi gerekirse budar.
-    Her mesaj komple kesilir, kısmen alınmaz.
-    """
+def enforce_token_budget(tokenizer, system_prompt, history_messages, user_input, total_ctx=4096, max_new_tokens=200):
     system_tokens = len(tokenizer(system_prompt)['input_ids'])
     user_tokens = len(tokenizer(user_input)['input_ids'])
     history_tokens = sum(len(tokenizer(m['content'])['input_ids']) for m in history_messages)
@@ -76,8 +68,8 @@ async def lifespan(app: FastAPI):
         tokenizer.pad_token = tokenizer.eos_token
 
         quant_config = BitsAndBytesConfig(
-            load_in_8bit=True,
-            llm_int8_threshold=6.0
+            load_in_4bit=True,  # ✅ 4-bit quantization
+            llm_int4_threshold=6.0
         )
 
         model = AutoModelForCausalLM.from_pretrained(
@@ -86,7 +78,7 @@ async def lifespan(app: FastAPI):
             quantization_config=quant_config
         )
 
-        log("✅ Model ve tokenizer başarıyla hazır (8-bit quantized, BitsAndBytesConfig).")
+        log("✅ Model ve tokenizer başarıyla hazır (4-bit quantized, BitsAndBytesConfig).")
         yield
 
     except Exception as e:
@@ -108,9 +100,8 @@ def generate(req: UserInputRequest):
         overall_start = time.time()
         log(f"💬 Kullanıcı isteği alındı: {req.user_input}")
 
-        # === History budama
         trimmed_history = trim_history(req.history, max_blocks=20)
-        trimmed_history = enforce_token_budget(tokenizer, req.system_prompt, trimmed_history, req.user_input, total_ctx=4096, max_new_tokens=300)
+        trimmed_history = enforce_token_budget(tokenizer, req.system_prompt, trimmed_history, req.user_input, total_ctx=4096, max_new_tokens=200)
 
         # === Apply chat template
         t0 = time.time()
@@ -143,7 +134,7 @@ def generate(req: UserInputRequest):
 
         input_len = input_ids.shape[-1]
         total_ctx = model.config.max_position_embeddings if hasattr(model.config, 'max_position_embeddings') else 4096
-        max_new_tokens = min(300, max(1, total_ctx - input_len))
+        max_new_tokens = min(200, max(1, total_ctx - input_len))
 
         log(f"ℹ️ Input uzunluğu: {input_len}, max_new_tokens ayarlandı: {max_new_tokens}")