Update app.py

app.py

@@ -145,19 +145,20 @@ def generate(
     chat_template: str,
     max_new_tokens: int = 1024,
     temperature: float = 0.6,
-    top_p: float = 0.9,
+    top_p: float = 0.95,
     top_k: int = 50,
-    repetition_penalty: float = 1.2,
+    repetition_penalty: float = 1.0,
 ) -> Iterator[str]:
     conversation = [*chat_history, {"role": "user", "content": message}]
 
-    input_ids = tokenizer.apply_chat_template(conversation, chat_template=chat_template, enable_thinking=False, return_tensors="pt")
+    #input_ids = tokenizer.apply_chat_template(conversation, chat_template=chat_template, enable_thinking=False, return_tensors="pt")
+    input_ids = tokenizer.apply_chat_template(conversation, chat_template=chat_template, return_tensors="pt")
     if input_ids.shape[1] > MAX_INPUT_TOKEN_LENGTH:
         input_ids = input_ids[:, -MAX_INPUT_TOKEN_LENGTH:]
         gr.Warning(f"Trimmed input from conversation as it was longer than {MAX_INPUT_TOKEN_LENGTH} tokens.")
     input_ids = input_ids.to(model.device)
 
-    streamer = TextIteratorStreamer(tokenizer, timeout=20.0, skip_prompt=True, skip_special_tokens=False)
+    streamer = TextIteratorStreamer(tokenizer, timeout=60.0, skip_prompt=True, skip_special_tokens=False)
     generate_kwargs = dict(
         {"input_ids": input_ids},
         streamer=streamer,