d

app.py

@@ -15,29 +15,37 @@ def respond(
     temperature,
     top_p,
 ):
-    messages = [{"role": "system", "content": system_message}]
+    prompt_parts = []
+    if system_message:
+        # Prepend system message. Actual handling depends on model's fine-tuning.
+        # For many raw language models, this acts as an initial instruction or context.
+        prompt_parts.append(system_message)
 
-    for val in history:
-        if val[0]:
-            messages.append({"role": "user", "content": val[0]})
-        if val[1]:
-            messages.append({"role": "assistant", "content": val[1]})
+    for user_msg, assistant_msg in history:
+        if user_msg:
+            prompt_parts.append(f"USER: {user_msg}")
+        if assistant_msg:
+            prompt_parts.append(f"ASSISTANT: {assistant_msg}")
 
-    messages.append({"role": "user", "content": message})
+    prompt_parts.append(f"USER: {message}")
+    prompt_parts.append("ASSISTANT:") # Model will generate content starting from here
+
+    full_prompt = "\n".join(prompt_parts)
 
     response = ""
 
-    for message in client.chat_completion(
-        messages,
-        max_tokens=max_tokens,
+    for stream_response in client.text_generation(
+        prompt=full_prompt,
+        max_new_tokens=max_tokens,
         stream=True,
         temperature=temperature,
         top_p=top_p,
     ):
-        token = message.choices[0].delta.content
-
-        response += token
-        yield response
+        # Ensure we are not processing special tokens, if any
+        if not stream_response.token.special:
+            token = stream_response.token.text
+            response += token
+            yield response
 
 
 """