import os import gradio as gr import requests import pandas as pd from smolagents import InferenceClientModel, ToolCallingAgent from audio_transcriber import AudioTranscriptionTool from image_analyzer import ImageAnalysisTool from wikipedia_searcher import WikipediaSearcher # GAIA scoring endpoint DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space" # Define the GaiaAgent class with embedded prompt in __call__ class GaiaAgent: def __init__(self): print("Gaia Agent Initialized") self.model = InferenceClientModel( model_id="cognitivecomputations/dolphin-2.6-mixtral-8x7b", token=os.getenv("HF_API_TOKEN", "").strip() ) self.tools = [ AudioTranscriptionTool(), ImageAnalysisTool(), WikipediaSearcher() ] self.agent = ToolCallingAgent( tools=self.tools, model=self.model ) def __call__(self, question: str) -> str: print(f"Agent received question (first 50 chars): {question[:50]}...") prompt = f"""You are an agent solving the GAIA benchmark and you are required to provide exact answers. Rules to follow: 1. Return only the exact requested answer: no explanation and no reasoning. 2. For yes/no questions, return exactly \"Yes\" or \"No\". 3. For dates, use the exact format requested. 4. For numbers, use the exact number, no other format. 5. For names, use the exact name as found in sources. 6. If the question has an associated file, download the file first using the task ID. Examples of good responses: - \"42\" - \"Arturo Nunez\" - \"Yes\" - \"October 5, 2001\" - \"Buenos Aires\" Never include phrases like \"the answer is...\" or \"Based on my research\". Only return the exact answer. QUESTION: {question} """ try: result = self.agent.run(prompt) print(f"Raw result from agent: {result}") if isinstance(result, dict) and "answer" in result: return str(result["answer"]).strip() elif isinstance(result, str): return result.strip() elif isinstance(result, list): for item in reversed(result): if isinstance(item, dict) and item.get("role") == "assistant" and "content" in item: return item["content"].strip() return "ERROR: Unexpected list format" else: return "ERROR: Unexpected result type" except Exception as e: print(f"Exception during agent run: {e}") return f"AGENT ERROR: {e}" # Evaluation + Submission function def run_and_submit_all(profile: gr.OAuthProfile | None): space_id = os.getenv("SPACE_ID") if profile: username = profile.username print(f"User logged in: {username}") else: print("User not logged in.") return "Please Login to Hugging Face with the button.", None api_url = DEFAULT_API_URL questions_url = f"{api_url}/questions" submit_url = f"{api_url}/submit" try: agent = GaiaAgent() except Exception as e: print(f"Error initializing agent: {e}") return f"Error initializing agent: {e}", None agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main" print(f"Agent code URL: {agent_code}") print(f"Fetching questions from: {questions_url}") try: response = requests.get(questions_url, timeout=15) response.raise_for_status() questions_data = response.json() if not questions_data: return "Fetched questions list is empty or invalid format.", None print(f"Fetched {len(questions_data)} questions.") except Exception as e: return f"Error fetching questions: {e}", None results_log = [] answers_payload = [] print(f"Running agent on {len(questions_data)} questions...") for item in questions_data: task_id = item.get("task_id") if not task_id: continue try: submitted_answer = agent(item.get("question", "")) answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer}) results_log.append({ "Task ID": task_id, "Question": item.get("question", ""), "Submitted Answer": submitted_answer }) except Exception as e: error_msg = f"AGENT ERROR: {e}" results_log.append({ "Task ID": task_id, "Question": item.get("question", ""), "Submitted Answer": error_msg }) if not answers_payload: return "Agent did not produce any answers to submit.", pd.DataFrame(results_log) submission_data = { "username": username.strip(), "agent_code": agent_code, "answers": answers_payload } print(f"Submitting {len(answers_payload)} answers to: {submit_url}") try: response = requests.post(submit_url, json=submission_data, timeout=60) response.raise_for_status() result_data = response.json() final_status = ( f"Submission Successful!\n" f"User: {result_data.get('username')}\n" f"Overall Score: {result_data.get('score', 'N/A')}% " f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n" f"Message: {result_data.get('message', 'No message received.')}" ) results_df = pd.DataFrame(results_log) return final_status, results_df except requests.exceptions.HTTPError as e: try: detail = e.response.json().get("detail", e.response.text) except Exception: detail = e.response.text[:500] return f"Submission Failed: {detail}", pd.DataFrame(results_log) except requests.exceptions.Timeout: return "Submission Failed: The request timed out.", pd.DataFrame(results_log) except Exception as e: return f"An unexpected error occurred during submission: {e}", pd.DataFrame(results_log) # Gradio UI with gr.Blocks() as demo: gr.Markdown("# Basic Agent Evaluation Runner") gr.Markdown(""" **Instructions:** 1. Clone this space and define your agent and tools. 2. Log in to your Hugging Face account using the button below. 3. Click 'Run Evaluation & Submit All Answers' to test your agent and submit results. """) gr.LoginButton() run_button = gr.Button("Run Evaluation & Submit All Answers") status_output = gr.Textbox(label="Run Status / Submission Result", lines=5, interactive=False) results_table = gr.DataFrame(label="Questions and Agent Answers", wrap=True) run_button.click(fn=run_and_submit_all, outputs=[status_output, results_table]) if __name__ == "__main__": print("\n" + "-"*30 + " App Starting " + "-"*30) space_host = os.getenv("SPACE_HOST") space_id = os.getenv("SPACE_ID") if space_host: print(f"✅ SPACE_HOST found: {space_host}") print(f" Runtime URL should be: https://{space_host}.hf.space") else: print("ℹ️ SPACE_HOST not found.") if space_id: print(f"✅ SPACE_ID found: {space_id}") print(f" Repo URL: https://huggingface.co/spaces/{space_id}") else: print("ℹ️ SPACE_ID not found.") print("-"*(60 + len(" App Starting ")) + "\n") demo.launch(debug=True, share=False)