Update app.py

app.py

@@ -1,7 +1,5 @@
 import huggingface_hub as hf_hub
-import time
 import openvino_genai as ov_genai
-import numpy as np
 import gradio as gr
 import re
 
@@ -17,50 +15,42 @@ pipe = ov_genai.LLMPipeline(model_path, device)
 tokenizer = pipe.get_tokenizer()
 tokenizer.set_chat_template(tokenizer.chat_template)
 
+# 修改 generate_response 函數以支持流式輸出
+def generate_response(prompt):
+    try:
+        response = ""
+        tokens_per_sec = "N/A"  # 預設值
 
-def generate_response(prompt, history=[]):  # Added history parameter
-    full_prompt = tokenizer.apply_chat_template(
-        messages=history + [{"role": "user", "content": prompt}],
-        tokenize=False,
-        add_generation_prompt=True
-    )
-
-    # Create a generator function for streaming
-    def stream_generate():
-        full_response = ""
-        start_time = time.time()
-        token_count = 0
-
+        # 定義流式處理回呼函數
         def streamer(subword):
-            nonlocal full_response, token_count
-            full_response += subword
-            token_count += 1
-            yield full_response, f'{token_count / (time.time() - start_time):.2f}'
+            nonlocal response
+            response += subword  # 拼接輸出
+            print(subword, end='', flush=True)  # 日誌輸出到控制台以便即時檢查
             return ov_genai.StreamingStatus.RUNNING
-        
-        try:
-            pipe.generate(full_prompt, streamer=streamer, max_new_tokens=1024)
-            # Add user prompt and full response to the history
-            history.append({"role": "user", "content": prompt})
-            history.append({"role": "assistant", "content": full_response})
-        except Exception as e:
-            yield "發生錯誤", f"生成回應時發生錯誤：{e}"
-        
-        # Yield the final response and tokens/sec
-        yield full_response, f'{token_count / (time.time() - start_time):.2f}'
 
-    return stream_generate()
+        # 啟動流式生成
+        pipe.start_chat()
+        pipe.generate(prompt, streamer=streamer, max_new_tokens=100)
+        pipe.finish_chat()
+
+        # 根據性能指標計算 tokens/sec
+        tokens_per_sec = f"{pipe.get_throughput():.2f}"
+        return tokens_per_sec, response
 
-# 建立 Gradio 介面
-demo = gr.ChatInterface(
+    except Exception as e:
+        return "N/A", f"生成回應時發生錯誤：{e}"
+
+# 建立 Gradio 介面（保持不變）
+demo = gr.Interface(
     fn=generate_response,
-    title="Qwen3-0.6B-int4-ov ",
-    description="基於 Qwen3-0.6B-int4-ov 推理應用，支援思考過程分離與 GUI.",
-    additional_inputs=[
-        gr.Textbox(label="tokens/sec", value="", interactive=False)
-    ]
+    inputs=gr.Textbox(lines=1, label="輸入提示 (Prompt)"),
+    outputs=[
+        gr.Textbox(label="tokens/sec"),
+        gr.Textbox(label="回應")
+    ],
+    title="Qwen3-0.6B-int4-ov",
+    description="基於 Qwen3-0.6B-int4-ov 推理應用，支援思考過程分離與 GUI。"
 )
 
-
 if __name__ == "__main__":
-    demo.queue().launch()
+    demo.launch()