Update app.py

app.py

@@ -1,12 +1,20 @@
 import gradio as gr
-from huggingface_hub import InferenceClient
+import openvino_genai as ov_genai
+import time
 
-"""
-For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
-"""
-client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
+# 載入 OpenVINO 模型
+model_path = "qwen3-0.6b-int8-ov"  # 根據實際路徑修改
+pipe = ov_genai.LLMPipeline(model_path, "CPU")
+pipe.start_chat()
 
+# 這個會在 token 產生時被呼叫
+def build_streamer(callback):
+    def streamer(subword):
+        callback(subword)
+        return ov_genai.StreamingStatus.RUNNING
+    return streamer
 
+# 對話處理函式
 def respond(
     message,
     history: list[tuple[str, str]],
@@ -15,50 +23,67 @@ def respond(
     temperature,
     top_p,
 ):
-    messages = [{"role": "system", "content": system_message}]
+    prompt = system_message + "\n"
+    for user_msg, bot_msg in history:
+        prompt += f"<|user|>\n{user_msg}\n<|assistant|>\n{bot_msg}\n"
+    prompt += f"<|user|>\n{message}\n<|assistant|>\n"
 
-    for val in history:
-        if val[0]:
-            messages.append({"role": "user", "content": val[0]})
-        if val[1]:
-            messages.append({"role": "assistant", "content": val[1]})
+    response = ""
 
-    messages.append({"role": "user", "content": message})
+    # 使用 generator 包裝 streamer
+    def generator():
+        nonlocal response
+        start_time = time.time()
 
-    response = ""
+        def collect_output(subword):
+            nonlocal response
+            response += subword
+            yield_fn.send(subword)  # 把 token 傳給外部 generator
 
-    for message in client.chat_completion(
-        messages,
-        max_tokens=max_tokens,
-        stream=True,
-        temperature=temperature,
-        top_p=top_p,
-    ):
-        token = message.choices[0].delta.content
+        yield_fn = yield  # 讓第一個 yield 傳入收集函式
 
-        response += token
-        yield response
+        # 執行生成
+        gen_result = pipe.generate(
+            [prompt],
+            streamer=build_streamer(collect_output),
+            max_new_tokens=max_tokens,
+            temperature=temperature,
+            top_p=top_p
+        )
+
+        elapsed = time.time() - start_time
+        tps = gen_result.perf_metrics.get_throughput().mean
+        print(f"\n--- TPS --- {tps:.2f} tokens/s --- {elapsed:.2f} sec")
+        yield_fn.close()  # 關閉 generator
 
+    # 建立 streaming generator
+    def streaming_generator():
+        gen = generator()
+        try:
+            next(gen)  # 啟動 generator
+            while True:
+                token = (yield)
+                gen.send(token)
+                yield token
+        except StopIteration:
+            return
 
-"""
-For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
-"""
+    # Streaming to Gradio
+    stream = streaming_generator()
+    next(stream)  # 啟動 stream
+    for token in stream:
+        yield response
+
+# 建立 Gradio Chat Interface
 demo = gr.ChatInterface(
     respond,
     additional_inputs=[
-        gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
+        gr.Textbox(value="You are a helpful assistant.", label="System message"),
         gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
         gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
-        gr.Slider(
-            minimum=0.1,
-            maximum=1.0,
-            value=0.95,
-            step=0.05,
-            label="Top-p (nucleus sampling)",
-        ),
+        gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-p (nucleus sampling)"),
     ],
 )
 
-
 if __name__ == "__main__":
     demo.launch()