Update app.py

app.py

@@ -32,7 +32,13 @@ InUsed_model_name = "Qwen3-0.6B-int4-ov"  # Choose a default model
 pipe = ov_genai.LLMPipeline(InUsed_model_name, device)
 tokenizer = pipe.get_tokenizer()
 tokenizer.set_chat_template(tokenizer.chat_template)
+config = openvino_genai.GenerationConfig()
+config.max_new_tokens = 1024
 
+def streamer(subword):
+    print(subword, end='', flush=True)
+    return False
+    
 def generate_response(prompt, model_name):
     global pipe, tokenizer, InUsed_model_name
     if(InUsed_model_name!=model_name):
@@ -46,7 +52,7 @@ def generate_response(prompt, model_name):
         InUsed_model_name=model_name
 
     try:
-        generated = pipe.generate([prompt], max_length=1024)
+        generated = pipe.generate([prompt], config, streamer)
         tokenpersec=f'{generated.perf_metrics.get_throughput().mean:.2f}'
 
         return tokenpersec, generated