Update app.py

app.py

@@ -1,71 +1,128 @@
+import os
+from collections.abc import Iterator
+from threading import Thread
+
 import gradio as gr
-from huggingface_hub import InferenceClient
+import spaces
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
 
-# تحميل نموذج LLaMA من Hugging Face
-client = InferenceClient("meta-llama/Llama-2-7b-chat-hf")
+DESCRIPTION = """\
+# Llama 3.2 3B Instruct
+Llama 3.2 3B is Meta's latest iteration of open LLMs.
+This is a demo of [`meta-llama/Llama-3.2-3B-Instruct`](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct), fine-tuned for instruction following.
+For more details, please check [our post](https://huggingface.co/blog/llama32).
+"""
 
-# قائمة السيناريوهات المتاحة
-scenarios = {
-    "restaurant": "You are in a restaurant. Help the user order food in English.",
-    "airport": "You are at an airport. Help the user check in and find their gate.",
-    "hotel": "You are in a hotel. Help the user book a room.",
-    "shopping": "You are in a store. Help the user ask for prices and sizes.",
-}
+MAX_MAX_NEW_TOKENS = 2048
+DEFAULT_MAX_NEW_TOKENS = 1024
+MAX_INPUT_TOKEN_LENGTH = int(os.getenv("MAX_INPUT_TOKEN_LENGTH", "4096"))
 
-# دالة لاختيار الرسالة المناسبة للسيناريو
-def scenario_prompt(choice):
-    return scenarios.get(choice, "You are a language tutor AI. Help users practice real-life conversations.")
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 
-# دالة المحادثة مع الذكاء الاصطناعي
-def respond(message, history, scenario, system_message, max_tokens, temperature, top_p):
-    # التأكد من أن history عبارة عن قائمة، وإذا لم توجد بيانات سابقة، اجعلها قائمة فارغة.
-    if history is None:
-        history = []
-    
-    # استبدال system_message بالرسالة الخاصة بالسيناريو
-    sys_msg = scenario_prompt(scenario)
-    
-    # إعداد قائمة الرسائل بما يتوافق مع تنسيق Chat Completion
-    messages = [{"role": "system", "content": sys_msg}]
-    
-    # التأكد من أن history عبارة عن قائمة من الـ tuples
-    for pair in history:
-        if isinstance(pair, (list, tuple)) and len(pair) == 2:
-            user_msg, assistant_msg = pair
-            if user_msg:
-                messages.append({"role": "user", "content": user_msg})
-            if assistant_msg:
-                messages.append({"role": "assistant", "content": assistant_msg})
-    
-    # إضافة الرسالة الحالية للمستخدم
-    messages.append({"role": "user", "content": message})
-    
-    response = ""
-    # الحصول على الرد مع البث (streaming)
-    for m in client.chat_completion(
-        messages,
-        max_tokens=max_tokens,
-        stream=True,
-        temperature=temperature,
+model_id = "meta-llama/Llama-3.2-3B-Instruct"
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+model = AutoModelForCausalLM.from_pretrained(
+    model_id,
+    device_map="auto",
+    torch_dtype=torch.bfloat16,
+)
+model.eval()
+
+
+@spaces.GPU(duration=90)
+def generate(
+    message: str,
+    chat_history: list[dict],
+    max_new_tokens: int = 1024,
+    temperature: float = 0.6,
+    top_p: float = 0.9,
+    top_k: int = 50,
+    repetition_penalty: float = 1.2,
+) -> Iterator[str]:
+    conversation = [*chat_history, {"role": "user", "content": message}]
+
+    input_ids = tokenizer.apply_chat_template(conversation, add_generation_prompt=True, return_tensors="pt")
+    if input_ids.shape[1] > MAX_INPUT_TOKEN_LENGTH:
+        input_ids = input_ids[:, -MAX_INPUT_TOKEN_LENGTH:]
+        gr.Warning(f"Trimmed input from conversation as it was longer than {MAX_INPUT_TOKEN_LENGTH} tokens.")
+    input_ids = input_ids.to(model.device)
+
+    streamer = TextIteratorStreamer(tokenizer, timeout=20.0, skip_prompt=True, skip_special_tokens=True)
+    generate_kwargs = dict(
+        {"input_ids": input_ids},
+        streamer=streamer,
+        max_new_tokens=max_new_tokens,
+        do_sample=True,
         top_p=top_p,
-    ):
-        # استخراج النص من كل توكن متدفق
-        token = m.choices[0].delta.content if m.choices[0].delta.content is not None else ""
-        response += token
-        yield response
+        top_k=top_k,
+        temperature=temperature,
+        num_beams=1,
+        repetition_penalty=repetition_penalty,
+    )
+    t = Thread(target=model.generate, kwargs=generate_kwargs)
+    t.start()
+
+    outputs = []
+    for text in streamer:
+        outputs.append(text)
+        yield "".join(outputs)
+
 
-# إنشاء واجهة Gradio للتفاعل مع المستخدم
 demo = gr.ChatInterface(
-    fn=respond,
-    chatbot=gr.Chatbot(type="messages"),
+    fn=generate,
     additional_inputs=[
-        gr.Dropdown(choices=list(scenarios.keys()), label="Choose a scenario", value="restaurant"),
-        gr.Textbox(value=scenario_prompt("restaurant"), label="System message"),
-        gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
-        gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
-        gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-p (nucleus sampling)"),
+        gr.Slider(
+            label="Max new tokens",
+            minimum=1,
+            maximum=MAX_MAX_NEW_TOKENS,
+            step=1,
+            value=DEFAULT_MAX_NEW_TOKENS,
+        ),
+        gr.Slider(
+            label="Temperature",
+            minimum=0.1,
+            maximum=4.0,
+            step=0.1,
+            value=0.6,
+        ),
+        gr.Slider(
+            label="Top-p (nucleus sampling)",
+            minimum=0.05,
+            maximum=1.0,
+            step=0.05,
+            value=0.9,
+        ),
+        gr.Slider(
+            label="Top-k",
+            minimum=1,
+            maximum=1000,
+            step=1,
+            value=50,
+        ),
+        gr.Slider(
+            label="Repetition penalty",
+            minimum=1.0,
+            maximum=2.0,
+            step=0.05,
+            value=1.2,
+        ),
+    ],
+    stop_btn=None,
+    examples=[
+        ["Hello there! How are you doing?"],
+        ["Can you explain briefly to me what is the Python programming language?"],
+        ["Explain the plot of Cinderella in a sentence."],
+        ["How many hours does it take a man to eat a Helicopter?"],
+        ["Write a 100-word article on 'Benefits of Open-Source in AI research'"],
     ],
+    cache_examples=False,
+    type="messages",
+    description=DESCRIPTION,
+    css_paths="style.css",
+    fill_height=True,
 )
 
+
 if __name__ == "__main__":
-    demo.launch()
+    demo.queue(max_size=20).launch()