Release v5

README.md

@@ -16,9 +16,9 @@ hf_oauth_expiration_minutes: 480
 
 **Final assessment for the Hugging Face AI Agents course**
 
-This repository contains a fully implemented autonomous agent designed to solve the [GAIA benchmark](https://arxiv.org/abs/2403.08790) - level 1. The agent leverages large language models and a suite of external tools to tackle complex, real-world, multi-modal tasks. It is ready to run and submit answers to the GAIA evaluation server, and is deployable as a HuggingFace Space with a Gradio interface.
+This repository contains a fully implemented autonomous agent designed to solve the [GAIA benchmark](https://arxiv.org/abs/2311.12983) - level 1. The agent leverages large language models and a suite of external tools to tackle complex, real-world, multi-modal tasks. It is ready to run and submit answers to the GAIA evaluation server, and is deployable as a HuggingFace Space with a Gradio interface.
 
-## 🏆 Project Summary
+## Project Summary
 - **Purpose:** Automatically solve and submit answers for the GAIA benchmark, which evaluates generalist AI agents on tasks requiring reasoning, code execution, web search, data analysis, and more.
 - **Features:**
   - Uses LLMs (OpenAI, HuggingFace, etc.) for reasoning and planning
@@ -26,7 +26,7 @@ This repository contains a fully implemented autonomous agent designed to solve
   - Handles file-based and multi-modal tasks
   - Submits results and displays scores in a user-friendly Gradio interface
 
-## 🚀 How to Run
+## How to Run
 
 **On HuggingFace Spaces:**
 - Log in with your HuggingFace account.
@@ -38,21 +38,20 @@ pip install -r requirements.txt
 python app.py
 ```
 
-## 🧠 About GAIA
+## About GAIA
 GAIA is a challenging benchmark for evaluating the capabilities of generalist AI agents on real-world, multi-step, and multi-modal tasks. Each task may require code execution, web search, data analysis, or other tool use. This agent is designed to autonomously solve such tasks and submit answers for evaluation.
 
-## 🏗️ Architecture
+## Architecture
 - `app.py` — Gradio app and evaluation logic. Fetches questions, runs the agent, and submits answers
 - `agent.py` — Main `Agent` class. Implements reasoning, tool use, and answer formatting
 - `model.py` — Loads and manages LLM backends (OpenAI, HuggingFace, LiteLLM, etc.)
 - `tools.py` — Implements external tools
 - `utils/logger.py` — Logging utility
-- `requirements.txt` — All dependencies for local and Spaces deployment
 
-## 🔑 Environment Variables
+## Environment Variables
 Some models require API keys. Set these in your Space or local environment:
 - `OPENAI_API_KEY` and `OPENAI_API_BASE` (for OpenAI models)
 - `HUGGINGFACEHUB_API_TOKEN` (for HuggingFace Hub models)
 
-## 📦 Dependencies
+## Dependencies
 All required packages are listed in `requirements.txt`

agent.py

@@ -38,6 +38,7 @@ class Agent:
             "re",
             "openpyxl",
             "pathlib",
+            "sys",
         ]
         self.agent = CodeAgent(
             model=self.model,
@@ -55,12 +56,14 @@ class Agent:
             - Reason step-by-step. Think through the solution logically and plan your actions carefully before answering.
             - Validate information. Always verify facts when possible instead of guessing.
             - Use code if needed. For calculations, parsing, or transformations, generate Python code and execute it. But be careful, some questions contains time-consuming tasks, so you should be careful with the code you run. Better analyze the question and think about the best way to solve it.
+            - Don't forget to use `final_answer` to give the final answer.
+            - Use name of file ONLY FROM "FILE:" section. THIS IF ALWAYS A FILE.
 
             IMPORTANT: When giving the final answer, output only the direct required result without any extra text like "Final Answer:" or explanations. YOU MUST RESPOND IN THE EXACT FORMAT AS THE QUESTION.
 
             QUESTION: {question}
 
-            CONTEXT: {context}
+            FILE: {context}
 
             ANSWER:
             """

app.py

@@ -9,7 +9,7 @@ import requests
 
 from agent import Agent
 from model import get_model
-from tools import get_tools
+from tools.tools import get_tools
 
 # (Keep Constants as is)
 # --- Constants ---
@@ -33,7 +33,7 @@ def run_and_submit_all(
         Tuple[str, Optional[pd.DataFrame]]: Status message and DataFrame of results.
     """
     # --- Determine HF Space Runtime URL and Repo URL ---
-    space_id = "exsandebest/agent-course-final-assessment"  # Get the SPACE_ID for sending link to the code
+    space_id = os.getenv("SPACE_ID") # Get the SPACE_ID for sending link to the code
 
     if profile:
         username = f"{profile.username}"
@@ -95,10 +95,26 @@ def run_and_submit_all(
             try:
                 file_response = requests.get(f"{files_url}/{task_id}", timeout=15)
                 if file_response.status_code == 200 and file_response.content:
-                    with tempfile.NamedTemporaryFile(delete=False) as tmp_file:
-                        tmp_file.write(file_response.content)
-                        file_path = tmp_file.name
-                        print(f"Downloaded file for task {task_id} to {file_path}")
+                    # Get filename from Content-Disposition header or URL
+                    filename = None
+                    content_disposition = file_response.headers.get(
+                        "Content-Disposition"
+                    )
+                    if content_disposition and "filename=" in content_disposition:
+                        filename = content_disposition.split("filename=")[-1].strip('"')
+                    else:
+                        # Try to get filename from URL
+                        url = file_response.url
+                        filename = url.split("/")[-1]
+                        if not filename or filename == str(task_id):
+                            filename = f"file_{task_id}"
+
+                    # Create temp directory and save file with original name
+                    temp_dir = tempfile.mkdtemp()
+                    file_path = os.path.join(temp_dir, filename)
+                    with open(file_path, "wb") as f:
+                        f.write(file_response.content)
+                    print(f"Downloaded file for task {task_id} to {file_path}")
                 else:
                     print(f"No file for task {task_id} or file is empty.")
             except Exception as e:
@@ -213,8 +229,8 @@ with gr.Blocks() as demo:
 if __name__ == "__main__":
     print("\n" + "-" * 30 + " App Starting " + "-" * 30)
     # Check for SPACE_HOST and SPACE_ID at startup for information
-    space_host_startup = "exsandebest-agent-course-final-assessment.hf.space"
-    space_id_startup = "exsandebest/agent-course-final-assessment"
+    space_host_startup = os.getenv("SPACE_HOST")
+    space_id_startup = os.getenv("SPACE_ID")
 
     if space_host_startup:
         print(f"✅ SPACE_HOST found: {space_host_startup}")

requirements.txt

@@ -11,4 +11,6 @@ smolagents[openai]
 smolagents[transformers]
 transformers
 wikipedia-api
-youtube-transcript-api
+youtube-transcript-api
+openai-whisper
+openai

tools.py

@@ -1,100 +0,0 @@
-from typing import Any, List
-
-import pytesseract
-from PIL import Image
-from smolagents import (
-    DuckDuckGoSearchTool,
-    PythonInterpreterTool,
-    SpeechToTextTool,
-    Tool,
-    VisitWebpageTool,
-    WikipediaSearchTool,
-)
-from youtube_transcript_api import YouTubeTranscriptApi
-
-
-class YouTubeTranscriptionTool(Tool):
-    """
-    Tool to fetch the transcript of a YouTube video given its URL.
-
-    Args:
-        video_url (str): YouTube video URL.
-
-    Returns:
-        str: Transcript of the video as a single string.
-    """
-
-    name = "youtube_transcription"
-    description = "Fetches the transcript of a YouTube video given its URL"
-    inputs = {
-        "video_url": {"type": "string", "description": "YouTube video URL"},
-    }
-    output_type = "string"
-
-    def forward(self, video_url: str) -> str:
-        video_id = video_url.strip().split("v=")[-1]
-        transcript = YouTubeTranscriptApi.get_transcript(video_id)
-        return " ".join([entry["text"] for entry in transcript])
-
-
-class ReadFileTool(Tool):
-    """
-    Tool to read a file and return its content.
-
-    Args:
-        file_path (str): Path to the file to read.
-
-    Returns:
-        str: Content of the file or error message.
-    """
-
-    name = "read_file"
-    description = "Reads a file and returns its content"
-    inputs = {
-        "file_path": {"type": "string", "description": "Path to the file to read"},
-    }
-    output_type = "string"
-
-    def forward(self, file_path: str) -> str:
-        try:
-            with open(file_path, "r") as file:
-                return file.read()
-        except Exception as e:
-            return f"Error reading file: {str(e)}"
-
-
-class ExtractTextFromImageTool(Tool):
-    name = "extract_text_from_image"
-    description = "Extracts text from an image using pytesseract"
-    inputs = {
-        "image_path": {"type": "string", "description": "Path to the image file"},
-    }
-    output_type = "string"
-
-    def forward(self, image_path: str) -> str:
-        try:
-            image = Image.open(image_path)
-            text = pytesseract.image_to_string(image)
-            return text
-        except Exception as e:
-            return f"Error extracting text from image: {str(e)}"
-
-
-def get_tools() -> List[Tool]:
-    """
-    Returns a list of available tools for the agent.
-
-    Returns:
-        List[Tool]: List of initialized tool instances.
-    """
-    tools = [
-        DuckDuckGoSearchTool(),
-        PythonInterpreterTool(),
-        WikipediaSearchTool(),
-        VisitWebpageTool(),
-        SpeechToTextTool(),
-        YouTubeTranscriptionTool(),
-        ReadFileTool(),
-        ExtractTextFromImageTool(),
-    ]
-    return tools

tools/describe_image_tool.py

@@ -0,0 +1,112 @@
+import base64
+import os
+
+from openai import OpenAI
+from smolagents import Tool
+
+client = OpenAI()
+
+
+class DescribeImageTool(Tool):
+    """
+    Tool to analyze and describe any image using GPT-4 Vision API.
+
+    Args:
+        image_path (str): Path to the image file.
+        description_type (str): Type of description to generate. Options:
+            - "general": General description of the image
+            - "detailed": Detailed analysis of the image
+            - "chess": Analysis of a chess position
+            - "text": Extract and describe text from the image
+            - "custom": Custom description based on user prompt
+
+    Returns:
+        str: Description of the image based on the requested type.
+    """
+
+    name = "describe_image"
+    description = "Analyzes and describes images using GPT-4 Vision API"
+    inputs = {
+        "image_path": {"type": "string", "description": "Path to the image file"},
+        "description_type": {
+            "type": "string",
+            "description": "Type of description to generate (general, detailed, chess, text, custom)",
+            "nullable": True,
+        },
+        "custom_prompt": {
+            "type": "string",
+            "description": "Custom prompt for description (only used when description_type is 'custom')",
+            "nullable": True,
+        },
+    }
+    output_type = "string"
+
+    def encode_image(self, image_path: str) -> str:
+        """Encode image to base64 string."""
+        with open(image_path, "rb") as image_file:
+            return base64.b64encode(image_file.read()).decode("utf-8")
+
+    def get_prompt(self, description_type: str, custom_prompt: str = None) -> str:
+        """Get appropriate prompt based on description type."""
+        prompts = {
+            "general": "Provide a general description of this image. Focus on the main subjects, colors, and overall scene.",
+            "detailed": """Analyze this image in detail. Include:
+            1. Main subjects and their relationships
+            2. Colors, lighting, and composition
+            3. Any text or symbols present
+            4. Context or possible meaning
+            5. Notable details or interesting elements""",
+            "chess": """Analyze this chess position and provide a detailed description including:
+            1. List of pieces on the board for both white and black
+            2. Whose turn it is to move
+            3. Basic evaluation of the position
+            4. Any immediate tactical opportunities or threats
+            5. Suggested next moves with brief explanations""",
+            "text": "Extract and describe any text present in this image. If there are multiple pieces of text, organize them clearly.",
+        }
+        return (
+            custom_prompt
+            if description_type == "custom"
+            else prompts.get(description_type, prompts["general"])
+        )
+
+    def forward(
+        self,
+        image_path: str,
+        description_type: str = "general",
+        custom_prompt: str = None,
+    ) -> str:
+        try:
+            if not os.path.exists(image_path):
+                return f"Error: Image file not found at {image_path}"
+
+            # Encode the image
+            base64_image = self.encode_image(image_path)
+
+            # Get appropriate prompt
+            prompt = self.get_prompt(description_type, custom_prompt)
+
+            # Make the API call
+            response = client.chat.completions.create(
+                model="gpt-4.1",
+                messages=[
+                    {
+                        "role": "user",
+                        "content": [
+                            {"type": "text", "text": prompt},
+                            {
+                                "type": "image_url",
+                                "image_url": {
+                                    "url": f"data:image/jpeg;base64,{base64_image}"
+                                },
+                            },
+                        ],
+                    }
+                ],
+                max_tokens=1000,
+            )
+
+            return response.choices[0].message.content
+
+        except Exception as e:
+            return f"Error analyzing image: {str(e)}"

tools/openai_speech_to_text_tool.py

@@ -0,0 +1,35 @@
+import os
+
+import whisper
+from smolagents import Tool
+
+
+class OpenAISpeechToTextTool(Tool):
+    """
+    Tool to convert speech to text using OpenAI's Whisper model.
+
+    Args:
+        audio_path (str): Path to the audio file.
+
+    Returns:
+        str: Transcribed text from the audio file.
+    """
+
+    name = "transcribe_audio"
+    description = "Transcribes audio to text and returns the text"
+    inputs = {
+        "audio_path": {"type": "string", "description": "Path to the audio file"},
+    }
+    output_type = "string"
+
+    def forward(self, audio_path: str) -> str:
+        try:
+            model = whisper.load_model("small")
+
+            if not os.path.exists(audio_path):
+                return f"Error: Audio file not found at {audio_path}"
+
+            result = model.transcribe(audio_path)
+            return result["text"]
+        except Exception as e:
+            return f"Error transcribing audio: {str(e)}"

tools/read_file_tool.py

@@ -0,0 +1,27 @@
+from smolagents import Tool
+
+
+class ReadFileTool(Tool):
+    """
+    Tool to read a file and return its content.
+
+    Args:
+        file_path (str): Path to the file to read.
+
+    Returns:
+        str: Content of the file or error message.
+    """
+
+    name = "read_file"
+    description = "Reads a file and returns its content"
+    inputs = {
+        "file_path": {"type": "string", "description": "Path to the file to read"},
+    }
+    output_type = "string"
+
+    def forward(self, file_path: str) -> str:
+        try:
+            with open(file_path, "r") as file:
+                return file.read()
+        except Exception as e:
+            return f"Error reading file: {str(e)}"

tools/tools.py

@@ -0,0 +1,34 @@
+from typing import List
+
+from smolagents import (
+    DuckDuckGoSearchTool,
+    PythonInterpreterTool,
+    Tool,
+    VisitWebpageTool,
+    WikipediaSearchTool,
+)
+
+from .describe_image_tool import DescribeImageTool
+from .openai_speech_to_text_tool import OpenAISpeechToTextTool
+from .read_file_tool import ReadFileTool
+from .youtube_transcription_tool import YouTubeTranscriptionTool
+
+
+def get_tools() -> List[Tool]:
+    """
+    Returns a list of available tools for the agent.
+
+    Returns:
+        List[Tool]: List of initialized tool instances.
+    """
+    tools = [
+        DuckDuckGoSearchTool(),
+        PythonInterpreterTool(),
+        WikipediaSearchTool(),
+        VisitWebpageTool(),
+        OpenAISpeechToTextTool(),
+        YouTubeTranscriptionTool(),
+        ReadFileTool(),
+        DescribeImageTool(),
+    ]
+    return tools

tools/youtube_transcription_tool.py

@@ -0,0 +1,26 @@
+from smolagents import Tool
+from youtube_transcript_api import YouTubeTranscriptApi
+
+
+class YouTubeTranscriptionTool(Tool):
+    """
+    Tool to fetch the transcript of a YouTube video given its URL.
+
+    Args:
+        video_url (str): YouTube video URL.
+
+    Returns:
+        str: Transcript of the video as a single string.
+    """
+
+    name = "youtube_transcription"
+    description = "Fetches the transcript of a YouTube video given its URL"
+    inputs = {
+        "video_url": {"type": "string", "description": "YouTube video URL"},
+    }
+    output_type = "string"
+
+    def forward(self, video_url: str) -> str:
+        video_id = video_url.strip().split("v=")[-1]
+        transcript = YouTubeTranscriptApi.get_transcript(video_id)
+        return " ".join([entry["text"] for entry in transcript])