Update app.py

app.py

@@ -1,13 +1,9 @@
-import spaces
-import json
-import subprocess
+import gradio as gr
+from huggingface_hub import hf_hub_download
 from llama_cpp import Llama
 from llama_cpp_agent import LlamaCppAgent, MessagesFormatterType
-from llama_cpp_agent.providers import LlamaCppPythonProvider
 from llama_cpp_agent.chat_history import BasicChatHistory
 from llama_cpp_agent.chat_history.messages import Roles
-import gradio as gr
-from huggingface_hub import hf_hub_download
 
 hf_hub_download(
     repo_id="mradermacher/Arsh-llm-GGUF",
@@ -15,7 +11,6 @@ hf_hub_download(
     local_dir="./models"
 )
 
-@spaces.GPU(duration=110)
 def respond(
     message,
     history: list[tuple[str, str]],
@@ -27,22 +22,20 @@ def respond(
     top_k,
     repeat_penalty,
 ):
-    chat_template = MessagesFormatterType.GEMMA_2
-
     llm = Llama(
         model_path=f"models/{model}",
-        flash_attn=True,
-        n_gpu_layers=81,
-        n_batch=1024,
-        n_ctx=8192,
+        n_batch=512,     
+        n_ctx=8192,      
+        verbose=False     
     )
+
     provider = LlamaCppPythonProvider(llm)
 
     agent = LlamaCppAgent(
         provider,
-        system_prompt=f"{system_message}",
-        predefined_messages_formatter_type=chat_template,
-        debug_output=True
+        system_prompt=system_message,
+        predefined_messages_formatter_type=MessagesFormatterType.CHATML,
+        debug_output=False
     )
 
     settings = provider.get_provider_default_settings()
@@ -55,17 +48,15 @@ def respond(
 
     messages = BasicChatHistory()
 
-    for msn in history:
-        user = {
+    for user_msg, assistant_msg in history:
+        messages.add_message({
             'role': Roles.user,
-            'content': msn[0]
-        }
-        assistant = {
+            'content': user_msg
+        })
+        messages.add_message({
             'role': Roles.assistant,
-            'content': msn[1]
-        }
-        messages.add_message(user)
-        messages.add_message(assistant)
+            'content': assistant_msg
+        })
 
     stream = agent.get_chat_response(
         message,
@@ -75,42 +66,28 @@ def respond(
         print_output=False
     )
 
-    outputs = ""
-    for output in stream:
-        outputs += output
-        yield outputs
+    response = ""
+    for token in stream:
+        response += token
+        yield response
+
 
 def create_interface(model_name, description):
     return gr.ChatInterface(
         respond,
         additional_inputs=[
             gr.Textbox(value=model_name, label="Model", interactive=False),
-            gr.Textbox(value="You are Arsh, a helpful assistant. You should think step-by-step. You should answer all question by prioritizing the principle of openness", label="System message"),
+            gr.Textbox(
+                value="You are Arsh, a helpful assistant. You should think step-by-step. You should answer all question by prioritizing the principle of openness.",
+                label="System message"
+            ),
             gr.Slider(minimum=1, maximum=4096, value=2048, step=1, label="Max tokens"),
             gr.Slider(minimum=0.1, maximum=4.0, value=0.6, step=0.1, label="Temperature"),
-            gr.Slider(
-                minimum=0.1,
-                maximum=1.0,
-                value=0.95,
-                step=0.05,
-                label="Top-p",
-            ),
-            gr.Slider(
-                minimum=0,
-                maximum=100,
-                value=40,
-                step=1,
-                label="Top-k",
-            ),
-            gr.Slider(
-                minimum=0.0,
-                maximum=2.0,
-                value=1.1,
-                step=0.1,
-                label="Repetition penalty",
-            ),
+            gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-p"),
+            gr.Slider(minimum=0, maximum=100, value=40, step=1, label="Top-k"),
+            gr.Slider(minimum=0.0, maximum=2.0, value=1.1, step=0.1, label="Repetition penalty"),
         ],
-        title=f"**FluentlyLM Prinum** ```on ZeroGPU```",
+        title="**Arsh-LLM Demo**",
         description=description,
         chatbot=gr.Chatbot(
             label=None,
@@ -119,7 +96,9 @@ def create_interface(model_name, description):
         )
     )
 
-description = """# **Arsh-llm ```Demo```"""
+
+description = """# **Arsh-LLM Q4_K_M GGUF Model - Demo**"""
+
 interface = create_interface('Arsh-llm.Q4_K_M.gguf', description)
 
 demo = gr.Blocks()