Update app.py

app.py

@@ -3,7 +3,7 @@ import logging
 import gradio as gr
 from transformers import pipeline
 
-from llama_cpp_agent.providers.llama_cpp_endpoint_provider import LlamaCppEndpointSettings
+from llama_cpp_agent.providers import LlamaCppPythonProvider
 from llama_cpp_agent import LlamaCppAgent, MessagesFormatterType
 from llama_cpp_agent.chat_history import BasicChatHistory
 from llama_cpp_agent.chat_history.messages import Roles
@@ -21,7 +21,6 @@ from typing import List
 from langchain_community.llms import HuggingFaceHub
 
 huggingface_token = os.environ.get("HUGGINGFACE_TOKEN")
-
 examples = [
     ["latest news about Yann LeCun"],
     ["Latest news site:github.blog"],
@@ -37,6 +36,7 @@ def get_context_by_model(model_name):
 
 def get_messages_formatter_type(model_name):
     if model_name is None:
+        # Handle the case where model_name is None
         logging.warning("Model name is None. Defaulting to CHATML formatter.")
         return MessagesFormatterType.CHATML
     
@@ -46,17 +46,6 @@ def get_messages_formatter_type(model_name):
     else:
         return MessagesFormatterType.CHATML
 
-class HuggingFaceHubProvider(LlamaCppEndpointSettings):
-    def __init__(self, model):
-        self.model = model
-
-    def create_completion(self, prompt, **kwargs):
-        response = self.model(prompt)
-        return {'choices': [{'text': response['generated_text']}]}
-
-    def get_provider_default_settings(self):
-        return self.model.model_kwargs
-
 def get_model(temperature, top_p, repetition_penalty):
     return HuggingFaceHub(
         repo_id="mistralai/Mistral-7B-Instruct-v0.3",
@@ -94,7 +83,6 @@ class CitingSources(BaseModel):
 def write_message_to_user():
     return "Please write the message to the user."
 
-#@spaces.GPU(duration=120)
 def respond(
     message,
     history: list[tuple[str, str]],
@@ -115,7 +103,7 @@ def respond(
     # Create a new model instance for each request
     llm = get_model(temperature, top_p, repeat_penalty)
     
-    provider = HuggingFaceHubProvider(llm)
+    provider = LlamaCppPythonProvider(llm)
     logging.info(f"Loaded chat examples: {chat_template}")
     search_tool = WebSearchTool(
         llm_provider=provider,
@@ -139,12 +127,12 @@ def respond(
     )
 
     settings = provider.get_provider_default_settings()
-    settings['stream'] = False
-    settings['temperature'] = temperature
-    settings['top_k'] = top_k
-    settings['top_p'] = top_p
-    settings['max_tokens'] = max_tokens
-    settings['repeat_penalty'] = repeat_penalty
+    settings.stream = False
+    settings.temperature = temperature
+    settings.top_k = top_k
+    settings.top_p = top_p
+    settings.max_tokens = max_tokens
+    settings.repeat_penalty = repeat_penalty
 
     output_settings = LlmStructuredOutputSettings.from_functions(
         [search_tool.get_tool()]
@@ -169,7 +157,7 @@ def respond(
 
     outputs = ""
 
-    settings['stream'] = True
+    settings.stream = True
     response_text = answer_agent.get_chat_response(
         f"Write a detailed and complete research document that fulfills the following user request: '{message}', based on the information from the web below.\n\n" +
         result[0]["return_value"],
@@ -207,7 +195,7 @@ demo = gr.ChatInterface(
         gr.Dropdown([
             'Mistral-7B-Instruct-v0.3'
         ],
-            value="Mistral-7B-Instruct-v0.3",  # This should match exactly
+            value="Mistral-7B-Instruct-v0.3",  # Ensure this matches exactly with the option in the list
             label="Model"
         ),
         gr.Textbox(value=web_search_system_prompt, label="System message"),