debug: attention 2

src/chat_logic.py

@@ -10,16 +10,17 @@ class ChatProcessor:
         self.vector_db = vector_db
         self.logger = logging.getLogger(__name__)
         
-    def process_chat(self, message, history, model_name, temperature=0.7, 
-                    max_new_tokens=512, top_p=0.9, top_k=50, repetition_penalty=1.2):
+    def process_chat(self, message, history, model_name, temperature=0.7,
+                    max_new_tokens=512, top_p=0.9, top_k=50, repetition_penalty=1.2,
+                    system_prompt=""):
         """Process chat input and generate streaming response"""
         try:
-            # Format prompt with history
-            prompt = format_prompt(message, history)
-            
             # Get model pipeline
             pipe = self.model_manager.get_pipeline(model_name)
             
+            # Format prompt with history and tokenizer
+            prompt = format_prompt(message, history, pipe.tokenizer, system_prompt)
+            
             # Set up streamer
             streamer = TextIteratorStreamer(
                 pipe.tokenizer, 

src/utils.py

@@ -1,13 +1,32 @@
-def format_prompt(message, history):
-    """Format message and history into a prompt for Qwen models"""
-    if not history:
-        return message
+def format_prompt(message, history, tokenizer, system_prompt=""):
+    """Format message and history into a prompt for Qwen models
     
-    # Convert history to string format
-    prompt = ""
+    Uses tokenizer.apply_chat_template if available, otherwise falls back to manual formatting.
+    """
+    # Convert history from tuples to dict format expected by apply_chat_template
+    formatted_history = []
     for user_msg, assistant_msg in history:
-        prompt += f"<|User|>: {user_msg}\n<|Assistant|>: {assistant_msg}\n"
+        formatted_history.append({"role": "user", "content": user_msg})
+        formatted_history.append({"role": "assistant", "content": assistant_msg})
     
     # Add current message
-    prompt += f"<|User|>: {message}\n<|Assistant|>:"
-    return prompt
+    formatted_history.append({"role": "user", "content": message})
+    
+    if hasattr(tokenizer, "chat_template") and tokenizer.chat_template:
+        messages = [{"role": "system", "content": system_prompt.strip()}] + formatted_history
+        return tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True, enable_thinking=True)
+    else:
+        # Fallback for base LMs without chat template
+        prompt = ""
+        if system_prompt.strip():
+            prompt = system_prompt.strip() + "\n"
+            
+        for msg in formatted_history:
+            if msg['role'] == 'user':
+                prompt += f"<|User|>: {msg['content'].strip()}\n"
+            elif msg['role'] == 'assistant':
+                prompt += f"<|Assistant|>: {msg['content'].strip()}\n"
+        
+        if not prompt.strip().endswith("<|Assistant|>:"):
+            prompt += "<|Assistant|>:"
+        return prompt