Update app.py

app.py

@@ -1,63 +1,42 @@
 import gradio as gr
-import time
+from huggingface_hub import InferenceClient
 from optimum.intel import OVModelForCausalLM
 from transformers import AutoTokenizer, pipeline
 
-# Load the model and tokenizer
+# 載入模型和標記器
 model_id = "hsuwill000/DeepSeek-R1-Distill-Qwen-1.5B-openvino"
-model = OVModelForCausalLM.from_pretrained(model_id, device="CPU")  # 明确指定设备
+model = OVModelForCausalLM.from_pretrained(model_id, device_map="auto")
 tokenizer = AutoTokenizer.from_pretrained(model_id)
 
-# Create generation pipeline
-pipe = pipeline("text-generation", model=model, tokenizer=tokenizer)
 
-def respond(message):
-    try:
-        start_time = time.time()
-        
-        # 強化 Prompt 讓模型輸出更合理
-        instruction = (
-            "請用簡單、繁體中文、準確的語言回答問題，避免冗長和重複內容，重點摘要至512 tokens。\n"
-            "User: " + message + "\n"
-            "Assistant: "
-        )
-
-        # Generate response with improved settings
-        response = pipe(
-            instruction,
-            max_length=1024,  # 限制最大輸出長度，防止無限重複
-            truncation=True,
-            num_return_sequences=1,
-            temperature=0.3,  # 保持一定創意但減少胡言亂語
-            top_p=0.8,  # 增加答案多樣性，減少無意義重複
-            repetition_penalty=1.5,  # 降低重複字詞的機率
-        )
-        generated_text = response[0]['generated_text'].strip()
-
-        # 提取 "Assistant:" 之後的部分
-        if "Assistant:" in generated_text:
-            reply = generated_text.split("Assistant:")[-1].strip()
-        else:
-            reply = generated_text
-        
-        inference_time = time.time() - start_time
-        print(f"Inference time: {inference_time:.4f} seconds")
-
-        return [(message, reply)]
+def respond(prompt):
+    messages = [
+    {"role": "system", "content": "You are Qwen, created by Alibaba Cloud. You are a helpful assistant."},
+    {"role": "user", "content": prompt }
+    ]
+    text = tokenizer.apply_chat_template(
+        messages,
+        tokenize=False,
+        add_generation_prompt=True
+    )
+    model_inputs = tokenizer([text], return_tensors="pt").to(model.device)    
+    generated_ids = model.generate(
+        **model_inputs,
+        max_new_tokens=512
+    )    
     
-    except Exception as e:
-        print(f"Error: {e}")
-        return [(message, "Sorry, something went wrong. Please try again.")]
-
-# Set up Gradio chat interface
-with gr.Blocks() as demo:
-    gr.Markdown("# DeepSeek-R1-Distill-Qwen-1.5B-openvino Chat")
-    gr.Markdown("Chat with DeepSeek-R1-Distill-Qwen-1.5B-openvino model.")
+    generated_ids = [
+        output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
+    ]
+    response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]    
     
-    chatbot = gr.Chatbot()
-    msg = gr.Textbox(label="Your Message")
+    # 返回新的消息格式
+    print(f"Messages: {messages}")
+    print(f"Reply: {response}")
+    return response
     
-    msg.submit(respond, msg, chatbot)
+# 設定 Gradio 的聊天界面
+demo = gr.ChatInterface(fn=respond, title="# DeepSeek-R1-Distill-Qwen-1.5B-openvino Chat", description="Chat with DeepSeek-R1-Distill-Qwen-1.5B-openvino model.", type='messages')
 
 if __name__ == "__main__":
-    demo.launch(share=True)
+    demo.launch()