feat: reach 35% so pass the test with the xagent

set a stronger model for the manager
add a iage vision tool (not yet tested)

README.md

@@ -33,7 +33,23 @@ docker run --rm `
  -v "${PWD}/etcsearxng:/etc/searxng" `
  -e "BASE_URL=http://localhost:$PORT/" `
  -e "INSTANCE_NAME=my-instance" `
- searxng/searxng
+ -d searxng/searxng
 
  be sure to allow the json format in /etc/seraxng/settings.yml
 
+
+For the same reason, for my tool for text to speech I locally run a speech to text docker image running whisper.cpp
+whith ffmpeg installed for mp3 > wav conversion
+
+And if I really have no longer any available token on openai or gemini, I can run a VLLM instance.
+
+## Instrumentation
+
+Instrumentation is enabled, 
+an done locally with (Arize-ai phoenix)[https://github.com/Arize-ai/phoenix]
+a server is launched with: 
+
+python -m phoenix.server.main serve
+
+and can be consulted on: http://127.0.0.1:6006
+

app.py

@@ -8,6 +8,12 @@ from dotenv import load_dotenv
 from myagent import BasicAgent  # Import your agent class from myagent.py
 from multiagents import MultiAgent
 
+from phoenix.otel import register
+from openinference.instrumentation.smolagents import SmolagentsInstrumentor
+
+register()
+SmolagentsInstrumentor().instrument()
+
 # (Keep Constants as is)
 # --- Constants ---
 DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
@@ -80,16 +86,16 @@ def run_and_submit_all(nb_questions: int, profile: gr.OAuthProfile | None):
         task_id = item.get("task_id")
         question_text = item.get("question")
         file_name = item.get("file_name")
-        file_url = None
+        file_question_url = None
         if file_name:
-            file_url = f"{file_url}/{task_id}"
+            file_question_url = f"{file_url}/{task_id}"
         if not task_id or question_text is None:
             print(f"Skipping item with missing task_id or question: {item}")
             continue
         try:
             agent_question = question_text
-            if file_url:
-                agent_question += f"\n\nFile URL: {file_url}"
+            if file_question_url:
+                agent_question += f"\n\nFile URL: {file_question_url}"
             
             submitted_answer = agent(agent_question)
             answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})

multiagents.py

@@ -1,11 +1,15 @@
 
+# a multi agent proposal to solve HF agent course final assignment
 import os
 import dotenv
 from smolagents import CodeAgent, ToolCallingAgent
 from smolagents import OpenAIServerModel
 from tools.fetch import fetch_webpage, search_web
-from smolagents import VisitWebpageTool, GoogleSearchTool, SpeechToTextTool
+from smolagents import PythonInterpreterTool
 from tools.yttranscript import get_youtube_transcript, get_youtube_title_description
+from tools.stt import stt
+from tools.image import analyze_image
+from tools.mylogger import save_file_with_timestamp, mylog
 import myprompts
 
 dotenv.load_dotenv()
@@ -29,14 +33,23 @@ openai_41nano_model = OpenAIServerModel(
     api_key=os.environ["OPENAI_API_KEY"],
 )
 
+openai_41mini_model = OpenAIServerModel(
+    model_id="gpt-4.1-mini",
+    api_base="https://api.openai.com/v1",
+    api_key=os.environ["OPENAI_API_KEY"],
+)
+
 
-def chek_final_answer(final_answer, agent_memory)  -> bool:
+def check_final_answer(final_answer, agent_memory)  -> bool:
     """
     Check if the final answer is correct.
     This is a placeholder function. You can implement your own logic here.
     """
-    # For demonstration, we assume the answer is always correct
-    return True
+    # if return answer is more than 200 characters, we will assume it is not correct    
+    if len(str(final_answer)) > 200:
+        return False
+    else:
+        return True
 
 
 web_agent = CodeAgent(
@@ -46,8 +59,9 @@ web_agent = CodeAgent(
         fetch_webpage,                
     ],
     name="web_agent",
-    description="Browses the web to find information",
-    verbosity_level=1,
+    description="Use search engine to find webpages related to a subject and get the page content",
+    additional_authorized_imports=["pandas", "numpy","bs4"],
+    verbosity_level=1,    
     max_steps=7,
 )
 
@@ -56,10 +70,12 @@ audiovideo_agent = CodeAgent(
     tools=[
         get_youtube_transcript,
         get_youtube_title_description,
-        SpeechToTextTool()
+        stt,
+        analyze_image
     ],
     name="audiovideo_agent",
-    description="Extracts information from video or audio files",
+    description="Extracts information from image, video or audio files from the web",
+    additional_authorized_imports=["pandas", "numpy","bs4", "requests"],
     verbosity_level=1,
     max_steps=7,
 )
@@ -67,13 +83,13 @@ audiovideo_agent = CodeAgent(
 
 
 manager_agent = CodeAgent(
-    model=openai_41nano_model,
-    tools=[],
+    model=openai_41mini_model,
+    tools=[ PythonInterpreterTool()],
     managed_agents=[web_agent, audiovideo_agent],    
     additional_authorized_imports=["pandas", "numpy","bs4"],
     planning_interval=5,
     verbosity_level=2,
-    final_answer_checks=[chek_final_answer],
+    final_answer_checks=[check_final_answer],
     max_steps=15,
     name="manager_agent",
     description="A manager agent that coordinates the work of other agents to answer questions.",
@@ -84,13 +100,19 @@ class MultiAgent:
         print("BasicAgent initialized.")
 
     def __call__(self, question: str) -> str:
-        try:
-            # log agent call in file
-            with open("logs/agent_calls.log", "a") as log_file:
-                log_file.write(f"Agent called with question: {question}\n")
+        mylog(self.__class__.__name__, question)        
 
         try:
-            question = question + '\n' + myprompts.output_format
+            prefix = """You are the top agent of a multi-agent system that can answer questions by coordinating the work of other agents.
+            You will receive a question and you will decide which agent to use to answer it.
+            You can use the web_agent to search the web for information and for fetching the content of a web page, or the audiovideo_agent to extract information from video or audio files.
+            You can also use your own knowledge to answer the question.
+            You need to respect the output format that is given to you.
+            Finding the correct answer to the question need reasoning and plannig, read the question carrefully, think step by step and do not skip any steps.
+            """
+
+            question = prefix + "\nTHE QUESTION:\n" + question + '\n' + myprompts.output_format
+
             fixed_answer = ""
 
             fixed_answer = manager_agent.run(question)
@@ -104,7 +126,17 @@ class MultiAgent:
 
 if __name__ == "__main__":
     # Example usage
+    """"
     question = "How many studio albums were published by Mercedes Sosa between 2000 and 2009 (included)? You can use the latest 2022 version of english wikipedia."
+    """
+    question = """
+Hi, I was out sick from my classes on Friday, so I'm trying to figure out what I need to study for my Calculus mid-term next week. My friend from class sent me an audio recording of Professor Willowbrook giving out the recommended reading for the test, but my headphones are broken :(
+
+Could you please listen to the recording for me and tell me the page numbers I'm supposed to go over? I've attached a file called Homework.mp3 that has the recording. Please provide just the page numbers as a comma-delimited list. And please provide the list in ascending order.
+
+File URL: https://agents-course-unit4-scoring.hf.space/files/1f975693-876d-457b-a649-393859e79bf3
+"""
     agent = MultiAgent()
     answer = agent(question)
-    print(f"Answer: {answer}")
+    print(f"Answer: {answer}")
+    

requirements.txt

@@ -8,4 +8,6 @@ smolagents[openai]
 markdownify
 beautifulsoup4
 transformers
-smolagents[transformers]
+smolagents[transformers]
+smolagents[audio]
+smolagents[telemetry,toolkit]

tools/fetch.py

@@ -2,8 +2,8 @@
 from smolagents import tool
 import requests
 from markdownify import markdownify as md
-import time
 from bs4 import BeautifulSoup
+from tools.mylogger import save_file_with_timestamp, mylog
 
 @tool
 def fetch_webpage(url: str, convert_to_markdown: bool = True) -> str:
@@ -16,7 +16,8 @@ def fetch_webpage(url: str, convert_to_markdown: bool = True) -> str:
     Returns:
         str: The HTML content of the URL.
     """    
-    response = requests.get(url)
+    content = None
+    response = requests.get(url, timeout=30)
     if (convert_to_markdown):
         soup = BeautifulSoup(response.text, "html.parser")
         # remove script and style tags
@@ -27,23 +28,14 @@ def fetch_webpage(url: str, convert_to_markdown: bool = True) -> str:
         if "wikipedia.org" in url:
             main_content = soup.find("main",{"id":"content"})
             if main_content:
-                content = md(str(main_content),strip=['script', 'style']).strip()
+                content = md(str(main_content),strip=['script', 'style'], heading_style="ATX").strip()
             else:
-                content = md(response.text,strip=['script', 'style']).strip()
+                content = md(response.text,strip=['script', 'style'], heading_style="ATX").strip()
     else:
         content = response.text
     
-    try:
-        # save content to a file in test folder before returning
-        # compute filepath with correct extension based on convert_to_markdown and add a timestamp for unicity
-        file_extension = ".md" if convert_to_markdown else ".html"
-        unicity_suffix = str(int(time.time()))
-        file_name = f"test/fetched_content_{unicity_suffix}{file_extension}"
-        with open(file_name, "w", encoding="utf-8") as f:
-            f.write(content)
-    except Exception as e:
-        print(f"Error saving content to file: {e}")
-        
+    save_file_with_timestamp(content, "webpage", ".md" if convert_to_markdown else ".html")
+           
     return content
 
 @tool

tools/image.py

@@ -0,0 +1,40 @@
+# image analyzer using open ai model
+
+from smolagents import tool
+from openai import OpenAI
+import dotenv
+dotenv.load_dotenv()
+
+@tool
+def analyze_image(question: str, image_url: str) -> str:
+    """
+    Analyze an image using OpenAI's API.
+    Args:
+        question (str): The question to ask about the image. eg. "What is in this image?"
+        image_url (str): The URL of the image to analyze.
+    """
+    client = OpenAI()
+
+    response = client.responses.create(
+        model="gpt-4o-mini",
+        input=[
+            {
+                "role": "user",
+                "content": [
+                    { "type": "input_text", "text": f"{question}" },
+                    {
+                        "type": "input_image",
+                        "image_url": f"{image_url}",
+                    }
+                ]
+            }
+        ]
+    )
+
+    return response
+
+if __name__ == "__main__":
+    question = "What is the main subject of this image?"
+    image_url = "https://agents-course-unit4-scoring.hf.space/files/cca530fc-4052-43b2-b130-b30968d8aa44"
+    answer = analyze_image(question, image_url)
+    print(f"Answer: {answer}")

tools/mylogger.py

@@ -0,0 +1,41 @@
+import time
+
+def save_file_with_timestamp(content: str, file_name: str, extension: str) -> str:
+    """
+    Save content to a file with a timestamp.
+    Args:
+        content (str): The content to save.
+        file_name (str): The base name of the file.
+    Returns:
+        str: The path to the saved file.
+    """
+    try:
+        # save content to a file in test folder before returning
+        # compute filepath with correct extension based on convert_to_markdown and add a timestamp for unicity
+        
+        unicity_suffix = str(int(time.time()))
+        
+        file_path = f"test/{file_name}_{unicity_suffix}.{extension}"
+        with open(file_name, "w", encoding="utf-8") as f:
+            f.write(content)
+    except Exception as e:
+        print(f"Error saving content to file: {e}")
+    return file_name
+
+
+def mylog(agent_name: str, message: str, depth: int = 0) -> None:
+    """
+    Log a message with indentation based on the depth.
+    Args:
+        agent_name (str): The name of the agent.
+        message (str): The message to log.
+        depth (int): The depth of the log message.
+    """
+    indent = " " * (depth * 4)
+    try:            
+        # log agent call in file
+        with open("logs/agent_calls.log", "a") as log_file:
+            log_file.write(f"{indent}{agent_name}: {message}\n")
+    except Exception as e:
+            print(f"Error logging agent call: {e}")            
+    

tools/stt.py

@@ -0,0 +1,68 @@
+import io
+import os  
+import shutil
+import subprocess
+import requests
+import uuid
+from smolagents import tool
+
+@tool
+def stt(file_url: str, language: str = "en-US") -> str:
+    """
+    Convert speech to text using local whisper model.
+    This function downloads an audio file from a given URL, converts it to WAV format if necessary,
+    then use whisper model to transcribe the audio to text.
+    
+    Args:
+        file_url (str): The URL of the audio file to transcribe.        
+        language (str): The language code for the transcription. Default is "en-US".
+        
+    Returns:
+        str: The transcribed text.
+    """
+
+    file_name = uuid.uuid4().hex +".mp3"
+
+    dest_folder = "c:\\Users\\mathi\\dev\\stt"
+    file_path = os.path.join(dest_folder + "\\tmp", file_name)
+    # 1. download the file from url (in pure python without wget or curl)
+    if not os.path.exists(file_name):            
+        response = requests.get(file_url)
+        if response.status_code == 200:
+            with open(file_path, "wb") as f:
+                f.write(response.content)
+        else:
+            raise Exception(f"Error downloading file: {response.status_code}")
+    
+    # 2. if it is a mp3 convert to wav with ffmpeg exec
+    if file_name.endswith(".mp3"):
+        cmd = f"ffmpeg -i {file_path} -ac 1 -ar 16000 -c:a pcm_s16le {file_path[:-4]}.wav"
+        cmd_as_list = cmd.split()
+        subprocess.run(cmd_as_list, cwd=dest_folder, check=True)
+        file_path = file_path[:-4] + ".wav"
+        file_name = file_name[:-4] + ".wav"
+
+    # 3. copy file to data folder
+    shutil.copy2(file_path, os.path.join(dest_folder, "testdata/"))
+    
+
+    # 4. call docker run  command
+    docker_command = f"""
+        docker run 
+        -v {dest_folder}/models:/app/models 
+        -v {dest_folder}/testdata:/app/testdata 
+        ghcr.io/appleboy/go-whisper:latest 
+        --model /app/models/ggml-small.bin 
+        --audio-path /app/testdata/{file_name}
+    """
+
+    subprocess.run(docker_command.split(), cwd=dest_folder, check=True)
+    # 5. cat the output file an return it   
+    output_filepath = os.path.join(dest_folder, "testdata", f"{file_name[:-4]}.txt")
+    with open(output_filepath, "r") as f:
+        text = f.read()
+    return text
+
+if __name__ == "__main__":
+    transcript = stt("https://agents-course-unit4-scoring.hf.space/files/99c9cc74-fdc8-46c6-8f8d-3ce2d3bfeea3", )
+    print(transcript)   

vllm_test.py

@@ -1,4 +1,8 @@
 from openai import OpenAI
+
+# test file in cas I need to run entirely locally
+# this test comptaibility with open ai
+
 client = OpenAI(
     base_url="http://192.168.1.39:18000/v1",
     api_key="token-abc123",