Final version of tools

app.py

@@ -16,7 +16,7 @@ from io import BytesIO
 import PyPDF2
 import base64
 from langchain_google_genai import ChatGoogleGenerativeAI
-from langchain_openai import AzureChatOpenAI
+from langchain_openai import ChatOpenAI
 from langchain_core.tools import tool
 from dotenv import load_dotenv
 import time
@@ -36,6 +36,7 @@ from tools.answer_excel import AnswerExcelTool
 from contextlib import redirect_stdout
 from tools.chess_tool import ChessTool
 from tools.audio_tool import AudioTool
+from tools.fetch_web_page import FetchWebPageTool
 
 load_dotenv(".env", override=True)
 BRAVE_API_KEY = os.getenv("BRAVE_API")
@@ -55,18 +56,15 @@ DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
 
 # --- Basic Agent Definition ---
 class BasicAgent:
-    def __init__(self):
-
-        # tools initialization
-        #internet_search = DuckDuckGoSearchRun()
-        
+    def __init__(self):        
         tools = [CodeGenTool(), PythonExecutionTool(temp_dir="./"), YoutubeTranscriptTool(), 
                  AnswerQuestionFromFileTool(), AnswerQuestionTool(), DownloadFile(), 
-                 ReverseString(), WebSearchTool(), WikipediaTool(), AnswerExcelTool(), ChessTool(), AudioTool()]
+                 ReverseString(), WebSearchTool(), WikipediaTool(), AnswerExcelTool(), ChessTool(), AudioTool(), FetchWebPageTool()]
 
-        llm = ChatGoogleGenerativeAI(
-            model="gemini-2.0-flash",
-            temperature=0)
+        #llm = ChatGoogleGenerativeAI(
+        #    model="gemini-2.0-flash",
+        #    temperature=0)
+        llm = ChatOpenAI(model="gpt-4.1-mini", temperature=0)
         self.llm_with_tools = llm.bind_tools(tools)
 
         builder = StateGraph(State)
@@ -101,13 +99,13 @@ class BasicAgent:
         self.react_graph = builder.compile()
 
 
-    def __call__(self, question: str, file_name: Optional[str]) -> str:
+    def __call__(self, question: str, task_id: str, file_name: Optional[str]) -> str:
         print(f"Agent received question (first 50 chars): {question[:50]}...")
         
         messages = [HumanMessage(question)]
         messages = self.react_graph.invoke({"messages": messages, "file_path": file_name, "question": question})
 
-        with open(f'messages_{file_name}.txt', 'w', encoding='utf-8') as out:
+        with open(f'messages_{task_id}.txt', 'w', encoding='utf-8') as out:
             with redirect_stdout(out):
                 for m in messages['messages']:
                     m.pretty_print()
@@ -126,36 +124,55 @@ class BasicAgent:
             file_name = None
         
         prompt = f"""
-        You are a general AI assistant. I will ask you a question. Report your thoughts, and finish your answer with the following template: FINAL ANSWER: [YOUR FINAL ANSWER]. 
-        YOUR FINAL ANSWER should be a number OR as few words as possible OR a comma separated list of numbers and/or strings. 
-        If you are asked for a number, don't use comma to write your number neither use units such as $ or percent sign unless specified otherwise. If you are asked for a string, don't use articles, neither abbreviations (e.g. for cities), and write the digits in plain text unless specified otherwise. 
-        
-        You should read the prompt thoroughly. For example, if they ask you for athletes with the least number of athletes, you must be careful to what they ask (in case of tie, the country which is the first in alphabetical order.)
+        You are a general AI assistant. When I ask you a question:
 
-        You MUST ALWAYS PICK WIKIPEDIA TOOL BEFORE WEB SEARCH.
-        
-        If you are asked for a comma separated list, apply the above rules depending of whether the element to be put in the list is a number or a string.
-        
-        YOU SHOULD **NEVER** MAKE ANY ASSUMPTION AND USE THE TOOLS PROVIDED!
-        
-        You are given this file: {file_name} with the extension: {file_extension}.
-        If a file is provided, the FIRST thing you MUST do is call the download_file tool!!
-        The format must be {DEFAULT_API_URL}/files/{file_name}
-        DO NOT PASS THE EXTENSION!!
+        Share your reasoning process clearly.
+
+        End with the exact template:
+        FINAL ANSWER: [YOUR FINAL ANSWER]
+
+        -------------------------------------------
+        Guidelines for FINAL ANSWER:
+
+        - Use a single number, a minimal phrase, or a comma-separated list of numbers and/or strings.
+
+        - For numbers, do not use commas, currency symbols, or percentage signs unless explicitly requested.
+
+        - For strings, avoid articles and abbreviations (e.g., no city abbreviations). Write digits in full text unless otherwise specified.
+
+        - Do not change capitalization of the terms you see unless it explicitly specified.
+
+        NEVER REPEAT THE SAME SEARCH MORE THAN ONCE, EVEN WITH SIMILAR TERMS. If you didn't find anything on the first go, it means there's nothing with that search query available.
+        If you can't find an answer just say you can't find it without repeating the same thing over and over.
+
+        Always read the prompt carefully.
+
+        Start with Wikipedia when searching for information. If Wikipedia doesn't have the answer, then use the web search tool. Use every available resource to find the correct answer.
+
+        IMPORTANT: Never make assumptions. Always use the provided tools!! If you are asked a question you think you know without using any tool, do not answer but invoke the answer_question_tool provided the WHOLE question in input.
+
+        NOTE: the question about the actor is tricky: they want to know who Bartłomiej played in Magda M.
+
+        If a file is provided (named {file_name} with extension {file_extension}), your first action MUST BE TO CALL the download_file tool with the URL:
+        {DEFAULT_API_URL}/files/{file_name}
+        Do **NOT** include the file extension in the URL and send WITHOUT MODIFICATION.
         """
 
         sys_msg = SystemMessage(content=prompt)
         
-        time.sleep(5)
+        time.sleep(40)
         return {"messages": [self.llm_with_tools.invoke([sys_msg] + state["messages"])]}
 
     def final_answer(state: State):
         system_prompt = f"""
-        You will be given an answer and a question. You MUST remove EVERYTHING not needed from the answer and answer the question exactly. 
+        You will be given an answer and a question. You MUST remove EVERYTHING not needed from the answer and answer the question exactly without reporting "FINAL ANSWER". 
         That is if you are being asked the number of something, you must not return the thought process, but just the number X.
 
-        You must be VERY CAREFUL!! Of what the question asks.
-        For example if they ask you to give the full name of a city without abbreviations you should stick to it (for example, St. Petersburg should be Saint Petersburg).
+        You must be VERY CAREFUL of what the question asks!!!
+        For example:
+            if they ask you to give the full name of a city without abbreviations you should stick to it (for example, St. Petersburg should be Saint Petersburg).
+            if the first name is asked, you MUST return the first name only (Claus and not Claus Peter)!
+        Remove full stops at the end, they are not needed. If you return something comma separated, there must always be a space between the comma and the next letter. Always!!
         """
 
         human_prompt = f"""
@@ -234,15 +251,14 @@ def run_and_submit_all( profile: gr.OAuthProfile | None):
     print(f"Running agent on {len(questions_data)} questions...")
     for item in questions_data:
         task_id = item.get("task_id")
-        #if task_id != "99c9cc74-fdc8-46c6-8f8d-3ce2d3bfeea3":
-        #    continue
         question_text = item.get("question")
         file_name = item.get("file_name")
         if not task_id or question_text is None:
             print(f"Skipping item with missing task_id or question: {item}")
             continue
         try:
-            submitted_answer = agent(question_text, file_name)
+            submitted_answer = agent(question_text, task_id, file_name)
+
             answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
             results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer})
         except Exception as e:

tools/answer_excel.py

@@ -6,12 +6,12 @@ from langchain.agents.agent_types import AgentType
 
 class AnswerExcelTool(BaseTool):
     name : str = "answer_excel_tool"
-    description: str = "Given the path to a file containing an excel file and a query, this tool tries to get an answer by querying the excel file."
+    description: str = "Given the path to a file containing an excel file and a query, this tool tries to get an answer by querying the excel file. Provide the whole question in input. Another agent will later break down the task."
 
-    def _run(query: str, file_path: str) -> str:
+    def _run(self, query: str, file_path: str) -> str:
         df = pd.read_excel(file_path)
 
-        llm = ChatGoogleGenerativeAI(model="gemini-2.0-flash", temperature=0),
+        llm = ChatGoogleGenerativeAI(model="gemini-2.0-flash", temperature=0)
 
         agent_executor = create_pandas_dataframe_agent(
             llm,

tools/answer_question.py

@@ -2,28 +2,31 @@ from langchain_google_genai import ChatGoogleGenerativeAI
 from pydantic import PrivateAttr
 from langchain_core.tools.base import BaseTool
 from langchain_core.messages import AnyMessage, SystemMessage, HumanMessage
+from langchain_openai import ChatOpenAI
+import time
+from openai import OpenAI
 
 class AnswerQuestionTool(BaseTool):
     name : str = "answer_question_tool"
-    description: str = "Use this tool to answer any elementary question that you can solve without needing access to any external tool. Simply provide the question in input, reporting the whole question including desired output format."
+    description: str = "Use this tool to answer any elementary question that you can solve without needing access to any external tool. Simply provide the question in input, reporting the whole question including desired output format. You can use this tool for example for vegetable classification."
     _llm = PrivateAttr()
     _system_prompt = PrivateAttr()
 
     def __init__(self):
         super().__init__()
-        self._llm = ChatGoogleGenerativeAI(
-            model="gemini-2.0-flash",
-            temperature=0)
-        
+        #self._llm = ChatGoogleGenerativeAI(
+        #    model="gemini-2.0-flash",
+        #    temperature=0)
+        #self._llm = ChatOpenAI(model="o4-mini", temperature=0)
+
 
         self._system_prompt = SystemMessage("""You are a helpful assistant.
                                             You will be given a question and you will have to answer that question.
-                                            You MUST NOT apologise, explain your reasoning nor nothing else. 
-                                            You MUST answer the question and provide the answer in the REQUIRED FORMAT.
-                                            YOU MUST ABSOLUTELY NOT MAKE ANY KIND OF ASSUMPTION!! If you don't know an answer, say you don't know!
-                                            If the format is incorrect, the answer is considered wrong.
+                                            Provide also the reasoning for your answer as well as your final answer.
 
-                                            If you are given a list or a transcript and you need to do something on a list of object, think thoroughly on how you should return the output!
+                                            When provided with a list you must stick with the exact terms provided in the list and not make any modification.
+                                            Green beans, corn and zucchini are NOT VEGEATABLES BOTANICALLY!
+                                            Let's think step by step.
                                             """)
         
     def _run(self, question: str) -> str:
@@ -33,6 +36,18 @@ class AnswerQuestionTool(BaseTool):
             ]
         )
 
-        response = self._llm.invoke([self._system_prompt, human_message])
+        time.sleep(5)
+        client = OpenAI()
+        response = client.responses.create(
+            model="o4-mini",
+            messages = [
+                {
+                    "role": "system", "content": self._system_prompt.text()
+                },
+                {
+                    "role": "user", "content": question
+                }]
+            )
+        #response = self._llm.invoke([self._system_prompt, human_message])
 
         return response

tools/answer_question_from_file.py

@@ -5,6 +5,7 @@ from pydantic import PrivateAttr
 import os
 from dotenv import load_dotenv
 import whisper
+import base64
 
 load_dotenv(".env", override=True)
 
@@ -23,7 +24,6 @@ class AnswerQuestionFromFileTool(BaseTool):
 
         Args:
             The question that needs to be answered.
-            The file from which you want to get some information.
             The file extension of the file that is being processed.
         """
     _llm = PrivateAttr()
@@ -35,11 +35,18 @@ class AnswerQuestionFromFileTool(BaseTool):
             temperature=0)
 
 
-    def _run(self, question: str, encoded_file: str, file_extension: str) -> str:
+    def _run(self, question: str, file_name: str, file_extension: str) -> str:
+
+        with open(file_name, "rb") as f:
+            file = f.read()
+
         if file_extension in ["png", "jpg"]:
+            encoded_file = base64.b64encode(file).decode("utf-8")
+
             message = {"type": "image_url", "image_url": f"data:image/png;base64,{encoded_file}"}
         elif file_extension == "pdf":
-            message = {"type": "image_url",  # Assuming the LLM accepts PDF under this key, you might need to verify this
+            encoded_file = base64.b64encode(file).decode("utf-8")
+            message = {"type": "image_url", 
                     "image_url": f"data:application/pdf;base64,{encoded_file}"
                   }
         else:

tools/audio_tool.py

@@ -6,6 +6,8 @@ from pathlib import Path
 import os
 from transformers import pipeline
 import torch
+from langchain_openai import ChatOpenAI
+import time
 
 class AudioTool(BaseTool):
     name : str = "answer_question_audio_tool"
@@ -13,13 +15,16 @@ class AudioTool(BaseTool):
 
     def _run(self, query: str, file_path: str) -> str:
         try:
-            pipe = pipeline(
-                task="automatic-speech-recognition",
-                model="openai/whisper-base",
-                torch_dtype=torch.float32,
-                device=0
-            )         
-            result = pipe(str(Path("./") / Path(file_path)))
+            #pipe = pipeline(
+            #    task="automatic-speech-recognition",
+            #    model="openai/whisper-base",
+            #    torch_dtype=torch.float32,
+            #    device=0,
+            #    return_timestamps=True
+            #)         
+            #result = pipe(str(Path("./") / Path(file_path)), return_timestamps=True)
+            model = whisper.load_model("base")
+            result = model.transcribe(audio=str(Path("./") / Path(file_path)), language='en')
         except Exception as e:
             print("Exception", e)
 
@@ -32,17 +37,16 @@ class AudioTool(BaseTool):
 
 1. Carefully read the query multiple times to ensure you fully grasp what is being asked.
 
-2. Start your response by listing, in clear bullet points, each precise requirement implied by the user's instructions (e.g., which portions of the transcript to use, what to include or exclude, and any specific formatting).
+2. Start by thinking, in clear bullet points, each precise requirement implied by the user's instructions (e.g., which portions of the transcript to use, what to include or exclude, and any specific formatting). 
 
-3. After restating the requirements, fulfill the request exactly as specified. Follow all content and formatting rules without deviation (for instance, “list only names,” “omit quantities,” “use comma-separated values,” “alphabetize,” etc.).
+3. After thinking more about the requirements, fulfill the request exactly as specified. Follow all content and formatting rules without deviation (for instance, “list only names,” “omit quantities,” “use comma-separated values,” “alphabetize,” etc.). 
 
 4. Ensure that your final answer adheres strictly to the user's criteria and contains nothing beyond what was requested.
 
-Always prioritize accuracy and strict adherence to the user's stated needs before providing the answer.""")
+Always prioritize accuracy and strict adherence to the user's stated needs before providing the answer. REPLY ONLY WITH WHAT THE HUMAN ASKED. Return only the final answer!""")
 
-        llm = ChatGoogleGenerativeAI(
-            model="gemini-2.0-flash",
-            temperature=0)
+        time.sleep(5)
+        llm = ChatOpenAI(model="gpt-4.1-mini", temperature=0)
         
         response = llm.invoke([system_message, human_message])
 

tools/chess_tool.py

@@ -9,7 +9,7 @@ class ChessTool(BaseTool):
 
     def _run(self, img_path: str, color_turn: str) -> str:
         # Get the FEN string
-        #fen = predict_fen("./downloaded_files/image.png")
+        fen = predict_fen("./downloaded_files/image.png")
 
         if color_turn == "b":
             ranks = fen.split('/')
@@ -24,10 +24,18 @@ class ChessTool(BaseTool):
 
         fen = f"{final_fen} {color_turn} - - 0 1"
 
-        fen = f"3r2k1/pp3pp1/4b2p/7Q/3n4/PqBBR2P/5PP1/6K1 {color_turn} - - 0 1"
+        try:
+            stockfish = Stockfish(path="C:/Users/FORMAGGA/Documents/personal/stockfish-windows-x86-64-avx2/stockfish/stockfish-windows-x86-64-avx2.exe")
 
-        stockfish = Stockfish(path="C:/Users/FORMAGGA/Documents/personal/stockfish-windows-x86-64-avx2/stockfish/stockfish-windows-x86-64-avx2.exe")
+            stockfish.set_fen_position(fen)
 
-        stockfish.set_fen_position(fen)
+            next_move = str(stockfish.get_best_move())
+        except Exception as e:
+            print("Exception", e)
+            raise e
+        
+        piece = stockfish.get_what_is_on_square(next_move[:2])
 
-        return stockfish.get_best_move()
+        next_move_fen = piece.name + next_move[2:]
+
+        return next_move_fen

tools/code_exec.py

@@ -23,7 +23,7 @@ class PythonExecutionTool(BaseTool):
 
     def __init__(
         self,
-        python_executable: str = "python",
+        python_executable: str = "C:\\Users\\FORMAGGA\\Documents\\personal\\Final_Assignment_Template\\.venv\\Scripts\\python.exe",
         timeout: int = 5,
         *,
         temp_dir: Optional[str] = None

tools/download_file.py

@@ -21,10 +21,7 @@ class DownloadFile(BaseTool):
 
         Output:
         IF the file is a document, image or audio:
-            A Dict made of:
-            1\) The file in bytes
-            2\) The file in Base64 encoding
-            3\) The result of the call 
+        A string with the path to the file.
         
         IF the file is a piece of code:
             A dict made of:
@@ -37,44 +34,40 @@ class DownloadFile(BaseTool):
 
     def _run(self, file_url: str, file_extension: str) -> dict:
         response = requests.get(file_url)
+        
         if response.status_code == 200:
             msg = "File downloaded successfully!!"
             if file_extension in ["png", "jpg", "pdf"]:
                 file = response.content
-                b64_file = base64.b64encode(file).decode("utf-8")
+                
                 with open("downloaded_files/image.png", "wb") as f:
                     f.write(file)
 
-                return {
-                    "bytes": file,
-                    "base64": b64_file,
-                    "status": response.status_code,
-                    "path": "downloaded_files/image.png"
-                }
+                return "downloaded_files/image.png"
             elif file_extension in ["mp3", "wav"]:
                 res = response.content
                 with open("downloaded_files/audio.mp3", mode="wb") as f:
                     f.write(res)
 
-                return {"transcript": "./downloaded_files/audio.mp3"}
+                return f"./downloaded_files/audio.{file_extension}"
 
             elif file_extension == "py":
                 return {"text": response.text}
             elif file_extension == "xlsx":
                 file_name = file_url.split("/")[-1]
 
-                with open(f"./downloaded_files/{file_name}", "wb") as f:
+                with open(f"./downloaded_files/{file_name}.xlsx", "wb") as f:
                     f.write(response.content)
 
-                return {"dataframe_path": f"./downloaded_files/{file_name}"}
+                return f"./downloaded_files/{file_name}.xlsx"
             else:
-                return {"error_msg": "The file extension is not valid."}
+                return "The file extension is not valid."
         else:
             msg = "There was an error downloading the file."
             file = None
             b64_file = None
 
-            return {"error_msg": msg}
+            return msg
 
         
 

tools/fetch_web_page.py

@@ -0,0 +1,12 @@
+from langchain_core.tools.base import BaseTool
+from typing import List
+import requests
+
+class FetchWebPageTool(BaseTool):
+    name : str = "fetch_web_page_tool"
+    description: str = "Provided the urls of 1 or more web pages, this tool returns the full content of the web page. This tool needs to be called AFTER calling the web_page_tool. It's important to fetch only pages which are useful to your task!"
+
+    def _run(self, urls: List[str]) -> List[str]:
+        pages = [requests.get(url).text for url in urls]
+
+        return pages

tools/web_search.py

@@ -14,22 +14,22 @@ load_dotenv(".env", override=True)
 
 class WebSearchTool(BaseTool):
     name: str = "web_search_tool"
-    description: str = "Perform a web search and extract concise factual answers. Use for online facts not in GAIA/Wikipedia—e.g. sports stats, Olympic participation, published papers, museum specimen locations, competition winners, and other up-to-date info."
+    description: str = "Perform a web search and extract concise factual answers. The query should be concise, below 400 characters. Use for online facts not in GAIA/Wikipedia—e.g. sports stats, Olympic participation, published papers, museum specimen locations, competition winners, and other up-to-date info."
     #_search: BraveSearch = PrivateAttr()
-    _search: DuckDuckGoSearchResults = PrivateAttr()
+    _search: TavilySearch = PrivateAttr()
 
     def __init__(self):
         super().__init__()
         #wrapper = DuckDuckGoSearchAPIWrapper(region="en", max_results=2)
         #self._search = DuckDuckGoSearchResults(api_wrapper=wrapper, output_format="json")
 
-        self._search = TavilySearch(max_results=2)
+        self._search = TavilySearch(max_results=3, topic="general")
 
     def _run_old(self, query: str) -> str:
         json_str = self._search.run(query)          # list[Document]
         docs = json.loads(json_str)
         urls = [doc["link"] for doc in docs]
-
+        print(urls)
         pages = [requests.get(url) for url in urls]
 
         res = "\n\n---\n\n".join(
@@ -44,9 +44,9 @@ class WebSearchTool(BaseTool):
 
         return res
     
-    def _run(self, query: str) -> str:
+    def _run(self, query: str) -> dict:
         # import pdb;pdb.set_trace()
         search_results = []
-        search_results.append(self._search.invoke(query))
-        # print(f"Search results: {search_results} \n type: {type(search_results)}")
-        return str(search_results)
+        search_results.append(self._search.run(query))
+
+        return search_results

tools/wikipedia.py

@@ -5,11 +5,12 @@ from langchain_core.tools.base import BaseTool
 from langchain_community.document_loaders import WikipediaLoader
 import requests
 from bs4 import BeautifulSoup
+import wikipedia
 
 
 class WikipediaTool(BaseTool):
     name: str = "wikipedia_tool"
-    description: str = "Search Wikipedia for a given query, retrieving the corresponding page's HTML content."
+    description: str = "Search Wikipedia for a given query, retrieving the corresponding page's HTML content. The query should not contain any noise and ask for something specific."
     #_wikipedia = PrivateAttr()
 
     def __init__(self):
@@ -18,33 +19,30 @@ class WikipediaTool(BaseTool):
 
 
     def _run(self, query: str):
-        loader = WikipediaLoader(
-            query=query,
-            lang="en",
-            load_max_docs=1,           # number of pages to fetch
-            load_all_available_meta=True
-        )
-        docs = loader.load()
-        url = docs[0].metadata.get("source")
-
-        resp = requests.get(url)
-        resp.raise_for_status()
-
-        html_text = resp.text
-
-        soup = BeautifulSoup(html_text, 'html.parser')
-        page_content = soup.find("div", class_="mw-parser-output")
-        if not page_content:
+        print(f"wikipedia_search_html called with query='{query}'")
+        # Step 1: Get Wikipedia HTML
+        page = wikipedia.page(query)
+        html = page.html()
+
+        # Step 2: Parse HTML
+        soup = BeautifulSoup(html, "html.parser")
+        content_div = soup.find("div", class_="mw-parser-output")
+        # content_div = soup.find("table", class_="wikitable")
+        if not content_div:
             return ""
-        try:
-            # Decompose non relevant tags
-            to_decompose = []
-            for tag in page_content.find_all(["style", "sup", "script", "noscript", "img", "link", "figure"]):
+
+        # Step 3: Find all tags to remove (style, script, sup, infobox, etc.)
+        to_decompose = []
+        for tag in content_div.find_all():
+            tag_classes = tag.get("class", [])
+            if (
+                tag.name in ["style", "script", "sup"]
+                or any(cls in ["infobox", "navbox", "reference"] for cls in tag_classes)
+            ):
                 to_decompose.append(tag)
-            
-            for tag in to_decompose:
-                tag.decompose()
-        except Exception as e:
-            print(e)
+
+        # Remove them after collecting
+        for tag in to_decompose:
+            tag.decompose()
         
-        return str(page_content)
+        return str(content_div)