Update app.py

app.py

@@ -1,42 +1,43 @@
 import os
 import requests
-from transformers import pipeline
-from flask import Flask, render_template, request, jsonify
-
-app = Flask(__name__)
-
-# Define model URL and local path
-MODEL_URL = "https://huggingface.co/unsloth/Qwen3-4B-128K-GGUF/resolve/main/Qwen3-4B-128K-UD-IQ1_M.gguf"
-MODEL_PATH = "Qwen3-4B-128K-UD-IQ1_M.gguf"
-
-# Function to download the model
-def download_model():
-    if not os.path.exists(MODEL_PATH):
-        print("Downloading the model...")
-        response = requests.get(MODEL_URL, stream=True)
-        with open(MODEL_PATH, 'wb') as model_file:
-            for chunk in response.iter_content(chunk_size=128):
-                model_file.write(chunk)
-        print("Model downloaded successfully.")
-
-# Load the model with Hugging Face Transformers pipeline
-def load_model():
-    download_model()
-    model = pipeline("text-generation", model=MODEL_PATH)
-    return model
-
-# Global model object
-model = load_model()
-
-@app.route('/')
-def index():
-    return render_template('index.html')
-
-@app.route('/generate', methods=['POST'])
-def generate():
-    user_input = request.form['message']
-    response = model(user_input, max_length=100)
-    return jsonify({"response": response[0]['generated_text']})
-
-if __name__ == '__main__':
-    app.run(debug=True)
+import gradio as gr
+from llama_cpp import Llama
+
+MODEL_URL = "https://cas-bridge.xethub.hf.co/xet-bridge-us/680f85f7f3cf2673404f897f/d6cd862a0b513dd35067121f0413e369b37aa9cacde526e31fc016c68ce0d305?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=cas%2F20250503%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20250503T191219Z&X-Amz-Expires=3600&X-Amz-Signature=2721d6c78a7db9b4d9b0ed1452ce3017d82b3392a190ae72b3755d873bc6c3df&X-Amz-SignedHeaders=host&X-Xet-Cas-Uid=67e6d6e1648b9f55980fa892&response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27Qwen3-4B-128K-Q4_K_M.gguf%3B+filename%3D%22Qwen3-4B-128K-Q4_K_M.gguf%22%3B&x-id=GetObject&Expires=1746303139&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTc0NjMwMzEzOX19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2FzLWJyaWRnZS54ZXRodWIuaGYuY28veGV0LWJyaWRnZS11cy82ODBmODVmN2YzY2YyNjczNDA0Zjg5N2YvZDZjZDg2MmEwYjUxM2RkMzUwNjcxMjFmMDQxM2UzNjliMzdhYTljYWNkZTUyNmUzMWZjMDE2YzY4Y2UwZDMwNSoifV19&Signature=lxo1PdbqSDERIivvWyQ88rq6oOx0DF5aefiy1LClNYXtBexSV476eNjAEl0wwE1nf7rW%7EcAIctjqzl%7ElVTEFSCosTEACuRrgqtGjyP94xL0xBFMDv1lqJ6E5UFokq0FTRNNd84Xt3GthKDMYanseX9kGpHBNpCwvlx-BRrNF6cbd2XWcCcmetP-NUrHtmoEHY89LVAFb72EXx7edlsOIMOWGzOYcgHI-IfuM2U4m%7Epquxo429CjrEbUi9xpQIQFVBvWHThIJN2LT5NqXpUpPyXswEJyuQDZ0sDTgBEh5gdHtwAWf0cF0i285VgitHIty8eYLEeYNu7J%7EqEDBu7RrYQ__&Key-Pair-Id=K2L8F4GPSG1IFC"  # truncated for clarity
+MODEL_PATH = "Qwen3-4B-128K-Q4_K_M.gguf"
+
+# Download model if not already downloaded
+if not os.path.exists(MODEL_PATH):
+    print("Downloading model...")
+    with requests.get(MODEL_URL, stream=True) as r:
+        r.raise_for_status()
+        with open(MODEL_PATH, 'wb') as f:
+            for chunk in r.iter_content(chunk_size=8192):
+                f.write(chunk)
+    print("Model downloaded.")
+
+# Load the model
+llm = Llama(
+    model_path=MODEL_PATH,
+    n_ctx=8192,
+    n_threads=4,
+    n_gpu_layers=20,  # Adjust for HF GPU environment
+    chat_format="chatml"
+)
+
+def chat_interface(message, history):
+    if history is None:
+        history = []
+
+    chat_prompt = []
+    for user_msg, bot_msg in history:
+        chat_prompt.append({"role": "user", "content": user_msg})
+        chat_prompt.append({"role": "assistant", "content": bot_msg})
+    chat_prompt.append({"role": "user", "content": message})
+
+    response = llm.create_chat_completion(messages=chat_prompt, stream=False)
+    reply = response["choices"][0]["message"]["content"]
+    history.append((message, reply))
+    return reply, history
+
+gr.ChatInterface(fn=chat_interface, title="Qwen3-4B Chat").launch()