Update app.py

app.py

@@ -4,6 +4,7 @@ import openvino_genai as ov_genai
 import numpy as np
 import gradio as gr
 import re
+import threading
 
 # 下載模型
 model_ids = [
@@ -28,6 +29,28 @@ for model_id in model_ids:
 # 建立推理管線 (Initialize with a default model first)
 device = "CPU"
 default_model_name = "Qwen3-0.6B-int4-ov"  # Choose a default model
+# 全局变量，用于存储推理管线、分词器、Markdown 组件和累计文本
+pipe = None
+tokenizer = None
+markdown_component = None  # 初始化
+accumulated_text = ""
+
+
+#  定义同步更新 Markdown 组件的函数
+def update_markdown(text):
+    global markdown_component
+    if markdown_component:
+        markdown_component.update(value=text)
+
+# 创建 streamer 函数 (保持原有架构)
+def streamer(subword):
+    global accumulated_text
+    accumulated_text += subword
+    print(subword, end='', flush=True)  # 保留打印到控制台
+    #  使用线程来异步更新 Markdown 组件
+    threading.Thread(target=update_markdown, args=(accumulated_text,)).start() # 异步更新 UI
+    return ov_genai.StreamingStatus.RUNNING
+
 
 def generate_response(prompt, model_name):
     global pipe, tokenizer  # Access the global variables
@@ -40,7 +63,8 @@ def generate_response(prompt, model_name):
     tokenizer.set_chat_template(tokenizer.chat_template)
 
     try:
-        generated = pipe.generate([prompt], max_length=1024)
+        #generated = pipe.generate([prompt], max_length=1024)
+        generated = pipe.generate(prompt, streamer=streamer, max_new_tokens=100)
         tokenpersec=f'{generated.perf_metrics.get_throughput().mean:.2f}'
 
         return tokenpersec, generated