yes

app.py

@@ -150,59 +150,51 @@ def _load_model_on(device_map: Optional[str]) -> AutoModelForCausalLM:
 # Harmony formatting
 # -----------------------
 
-def create_harmony_prompt(messages: List[Dict[str, str]], reasoning_effort: str = "high") -> str:
-    """Build Harmony-formatted prompt using the *tokenizer chat template* (per model card).
-    Always returns a string; HF will tokenize to ensure IDs match the checkpoint.
+def create_harmony_prompt(messages: List[Dict[str, str]], reasoning_effort: str = "high") -> Any:
+    """Build a Harmony-formatted prompt. If Harmony is available, return **token IDs**
+    rendered by `openai_harmony` (authoritative). Otherwise fall back to the
+    tokenizer's chat template and return a string.
     """
+    if HARMONY_AVAILABLE and harmony_encoding is not None:
+        effort_map = {"low": ReasoningEffort.LOW, "medium": ReasoningEffort.MEDIUM, "high": ReasoningEffort.HIGH}
+        effort = effort_map.get(str(reasoning_effort).lower(), ReasoningEffort.HIGH)
+
+        system_content = (
+            SystemContent.new()
+            .with_model_identity("You are ChatGPT, a large language model trained by OpenAI.")
+            .with_reasoning_effort(effort)
+            .with_conversation_start_date(datetime.now().strftime("%Y-%m-%d"))
+            .with_knowledge_cutoff("2024-06")
+            .with_required_channels(REQUIRED_CHANNELS)
+        )
+
+        # Use first system message as developer instructions if present, else SYSTEM_DEF
+        sys_text = SYSTEM_DEF
+        rest: List[Dict[str, str]] = messages or []
+        if rest and rest[0].get("role") == "system":
+            sys_text = rest[0].get("content") or SYSTEM_DEF
+            rest = rest[1:]
+
+        harmony_messages = [Message.from_role_and_content(Role.SYSTEM, system_content)]
+        dev = DeveloperContent.new().with_instructions(sys_text)
+        harmony_messages.append(Message.from_role_and_content(Role.DEVELOPER, dev))
+
+        for m in rest:
+            role = m.get("role"); content = m.get("content", "")
+            if role == "user":
+                harmony_messages.append(Message.from_role_and_content(Role.USER, content))
+            elif role == "assistant":
+                harmony_messages.append(
+                    Message.from_role_and_content(Role.ASSISTANT, content).with_channel("final")
+                )
+
+        convo = Conversation.from_messages(harmony_messages)
+        return harmony_encoding.render_conversation_for_completion(convo, Role.ASSISTANT)
+
+    # Fallback: tokenizer chat template -> string prompt
     if not messages or messages[0].get("role") != "system":
         messages = [{"role": "system", "content": SYSTEM_DEF}] + (messages or [])
-    return tokenizer.apply_chat_template(
-        messages,
-        add_generation_prompt=True,
-        tokenize=False
-    )
-    
-    # Map reasoning effort
-    effort_map = {
-        "low": ReasoningEffort.LOW,
-        "medium": ReasoningEffort.MEDIUM,
-        "high": ReasoningEffort.HIGH,
-    }
-    effort = effort_map.get(reasoning_effort.lower(), ReasoningEffort.HIGH)
-    
-    # Create system message with channels
-    system_content = (
-        SystemContent.new()
-        .with_model_identity("You are ChatGPT, a large language model trained by OpenAI.")
-        .with_reasoning_effort(effort)
-        .with_conversation_start_date(datetime.now().strftime("%Y-%m-%d"))
-        .with_knowledge_cutoff("2024-06")
-        .with_required_channels(REQUIRED_CHANNELS)
-    )
-    
-    # Build conversation
-    harmony_messages = [Message.from_role_and_content(Role.SYSTEM, system_content)]
-    # Developer instructions per Harmony spec (use the provided system prompt as instructions)
-    developer_content = DeveloperContent.new().with_instructions(messages[0]["content"] if messages else SYSTEM_DEF)
-    harmony_messages.append(Message.from_role_and_content(Role.DEVELOPER, developer_content))
-    
-    # Add user/assistant messages
-    for msg in messages[1:]:  # Skip system message as we already added it
-        if msg["role"] == "user":
-            harmony_messages.append(
-                Message.from_role_and_content(Role.USER, msg["content"])
-            )
-        elif msg["role"] == "assistant":
-            # For assistant messages, we might want to preserve channels if they exist
-            harmony_messages.append(
-                Message.from_role_and_content(Role.ASSISTANT, msg["content"])
-                .with_channel("final")  # Default to final channel
-            )
-    
-    # Create conversation and render
-    convo = Conversation.from_messages(harmony_messages)
-    tokens = harmony_encoding.render_conversation_for_completion(convo, Role.ASSISTANT)
-    return tokens  # pass tokens directly to the model to avoid decode/re-encode drift
+    return tokenizer.apply_chat_template(messages, add_generation_prompt=True, tokenize=False)
 
 def parse_harmony_response(tokens: List[int]) -> Dict[str, str]:
     """Parse response tokens using Harmony format to extract channels."""
@@ -341,7 +333,9 @@ def zerogpu_generate(full_prompt,
             top_p=float(gen_kwargs.get("top_p", 0.9)),
             top_k=(int(gen_kwargs.get("top_k")) if gen_kwargs.get("top_k") and int(gen_kwargs.get("top_k")) > 0 else None),
             max_new_tokens=int(gen_kwargs.get("max_new_tokens", MAX_DEF)),
-            pad_token_id=model.config.pad_token_id,            logits_processor=logits_processor,
+            pad_token_id=model.config.pad_token_id,
+            eos_token_id=eos_ids,
+            logits_processor=logits_processor,
             repetition_penalty=float(gen_kwargs.get("repetition_penalty", 1.1)),
             no_repeat_ngram_size=int(gen_kwargs.get("no_repeat_ngram_size", 6)),
             min_new_tokens=1,