Update app.py

app.py

@@ -1,62 +1,69 @@
-import gradio as gr
-import openvino_genai as ov_genai
 import huggingface_hub as hf_hub
+import time
+import openvino_genai as ov_genai
+import numpy as np
+import gradio as gr
+import re
+
+# 下載模型
+model_ids = [
+    "OpenVINO/Qwen3-0.6B-int4-ov",
+    "OpenVINO/Qwen3-1.7B-int4-ov",
+    #"OpenVINO/Qwen3-4B-int4-ov",#不可用
+    "OpenVINO/Qwen3-8B-int4-ov",
+    "OpenVINO/Qwen3-14B-int4-ov",
+    
+]
+
+model_name_to_full_id = {model_id.split("/")[-1]: model_id for model_id in model_ids}  #Create Dictionary
+
+for model_id in model_ids:
+    model_path = model_id.split("/")[-1]  # Extract model name
+    try:
+      hf_hub.snapshot_download(model_id, local_dir=model_path, local_dir_use_symlinks=False)
+      print(f"Successfully downloaded {model_id} to {model_path}") # Optional: Print confirmation
+    except Exception as e:
+      print(f"Error downloading {model_id}: {e}") # Handle download errors gracefully
 
-# OpenVINO Setup
-model_id = "OpenVINO/Qwen3-0.6B-int4-ov"  # Or your chosen model
-model_path = "Qwen3-0.6B-int4-ov"         # Local directory for the model
-
-# Download the model if it doesn't exist locally
-hf_hub.snapshot_download(model_id, local_dir=model_path, local_dir_use_symlinks=False)
-
-
-pipe = ov_genai.LLMPipeline(model_path, "CPU")
-tokenizer = pipe.get_tokenizer()
-tokenizer.set_chat_template(tokenizer.chat_template)
-pipe.start_chat() # moved pipe.start_chat() here to run after pipeline intialization
-
-
-# Gradio Chatbot UI
-def user(user_message, history: list):
-    return "", history + [{"role": "user", "content": user_message}]
-
+# 建立推理管線 (Initialize with a default model first)
+device = "CPU"
+default_model_name = "Qwen3-0.6B-int4-ov"  # Choose a default model
 
-def bot(history: list, user_message):
-    full_response = ""
+def generate_response(prompt, model_name):
+    global pipe, tokenizer  # Access the global variables
 
-    def streamer(subword):
-        nonlocal full_response
-        full_response += subword
-        history[-1]['content'] = full_response
-        yield history
-        return ov_genai.StreamingStatus.RUNNING
+    model_path = model_name
 
-    history.append({"role": "assistant", "content": ""})
+    print(f"Switching to model: {model_name}")
+    pipe = ov_genai.LLMPipeline(model_path, device)
+    tokenizer = pipe.get_tokenizer()
+    tokenizer.set_chat_template(tokenizer.chat_template)
 
     try:
-        for updated_history in pipe.generate(user_message, streamer=streamer, max_new_tokens=100):
-            yield updated_history
-    except Exception as e:
-        print(f"Error during OpenVINO generation: {e}") # Log the error!
-        history[-1]['content'] = "An error occurred while generating the response."
-        yield history
+        generated = pipe.generate([prompt], max_length=1024)
+        tokenpersec=f'{generated.perf_metrics.get_throughput().mean:.2f}'
 
-with gr.Blocks() as demo:
-    chatbot = gr.Chatbot(type="messages")
-    msg = gr.Textbox()
-    submit_button = gr.Button("Submit")  # Added submit button
-    clear = gr.Button("Clear")
-
-    def respond(message, chat_history):  # Combined user and bot functions
-        user_message, chat_history = user(message, chat_history)
-        for bot_response in bot(chat_history, message):
-            chat_history = bot_response
-            yield "", chat_history
-
-
-    submit_button.click(respond, [msg, chatbot], [msg, chatbot])
-    msg.submit(respond, [msg, chatbot], [msg, chatbot])  # Optional: allow Enter key submission
-    clear.click(lambda: None, None, chatbot, queue=False)
+        return tokenpersec, generated
+    except Exception as e:
+        return "發生錯誤", "發生錯誤", f"生成回應時發生錯誤：{e}"
+    
+
+# 建立 Gradio 介面
+model_choices = list(model_name_to_full_id.keys())
+
+demo = gr.Interface(
+    fn=generate_response,
+    inputs=[
+        gr.Textbox(lines=5, label="輸入提示 (Prompt)"),
+        gr.Dropdown(choices=model_choices, value=default_model_name, label="選擇模型") # Added dropdown
+    ],
+    outputs=[
+        gr.Textbox(label="tokens/sec"),
+        gr.Textbox(label="回應"),
+    ],
+    title="Qwen3 Model Inference",
+    description="基於 Qwen3 推理應用，支援思考過程分離與 GUI。"
+)
 
 if __name__ == "__main__":
-    demo.queue().launch(share=True)
+    demo.launch()