simuGAIA.py

import os
import sys
import json
import tempfile
import gradio as gr
import requests
import inspect
import pandas as pd
from typing import List, Dict, Any, Optional
import traceback
# vimport dotenv

# Load environment variables from .env file
# dotenv.load_dotenv()

# Import our agent
from agent import QAgent
from answer_data_manager import AnswerDataManager

# (Keep Constants as is)
# --- Constants ---
DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"

# Simulation of GAIA benchmark questions
SAMPLE_QUESTIONS = [
    {
        "task_id": "task_002",
        "question": "What is the square root of 144?",
        "expected_answer": "12",
        "has_file": False,
        "file_content": None
    }
]

SAMPLE_QUESTIONS_OUT = [
        {
        "task_id": "task_001",
        "question": "What is the capital of France?",
        "expected_answer": "Paris",
        "has_file": False,
        "file_content": None
    },
    {
        "task_id": "task_003",
        "question": "If a train travels at 60 miles per hour, how far will it travel in 2.5 hours?",
        "expected_answer": "150 miles",
        "has_file": False,
        "file_content": None
    },
    {
        "task_id": "task_004", 
        "question": ".rewsna eht sa 'thgir' drow eht etirw ,tfel fo etisoppo eht si tahW",
        "expected_answer": "right",
        "has_file": False,
        "file_content": None
    },
    {
        "task_id": "task_005",
        "question": "Analyze the data in the attached CSV file and tell me the total sales for the month of January.",
        "expected_answer": "$10,250.75",
        "has_file": True,
        "file_content": """Date,Product,Quantity,Price,Total
2023-01-05,Widget A,10,25.99,259.90
2023-01-12,Widget B,5,45.50,227.50
2023-01-15,Widget C,20,50.25,1005.00
2023-01-20,Widget A,15,25.99,389.85
2023-01-25,Widget B,8,45.50,364.00
2023-01-28,Widget D,100,80.04,8004.50"""
    },
    {
        "task_id": "task_006",
        "question": "I'm making a grocery list for my mom, but she's a picky eater. She only eats foods that don't contain the letter 'e'. List 5 common fruits and vegetables she can eat.",
        "expected_answer": "Banana, Kiwi, Corn, Fig, Taro",
        "has_file": False,
        "file_content": None
    },
    {
        "task_id": "task_007",
        "question": "How many studio albums were published by Mercedes Sosa between 1972 and 1985?",
        "expected_answer": "12",
        "has_file": False,
        "file_content": None
    },
    {
        "task_id": "task_008",
        "question": "In the video https://www.youtube.com/watch?v=L1vXC1KMRd0, what color is primarily associated with the main character?",
        "expected_answer": "Blue",
        "has_file": False,
        "file_content": None
    }
]


def init_agent():
    """Initialize the QAgent."""
    print("Initializing QAgent...")
    try:
        agent = QAgent()
        return agent
    except Exception as e:
        print(f"Error instantiating agent for GAIA simulation: {e}")
    return None


def save_test_file(task_id: str, content: str) -> str:
    """Save a test file to a temporary location."""
    temp_dir = tempfile.gettempdir()
    file_path = os.path.join(temp_dir, f"test_file_{task_id}.csv")
    
    with open(file_path, 'w') as f:
        f.write(content)
    
    return file_path



def run_GAIA_questions_simu():
    """
    Used only during development for test that simulate GAIA questions.
    """
    # 1. Instantiate Agent
    agent = init_agent()

    
    results = []
    correct_count = 0
    total_count = len(SAMPLE_QUESTIONS)

    for idx, question_data in enumerate(SAMPLE_QUESTIONS):
        task_id = question_data["task_id"]
        question = question_data["question"]
        expected = question_data["expected_answer"]
        
        print(f"\n{'='*80}")
        print(f"Question {idx+1}/{total_count}: {question}")
        print(f"Expected: {expected}")
        
        # Process any attached file
        # file_path = None
        # if question_data["has_file"] and question_data["file_content"]:
        #     file_path = save_test_file(task_id, question_data["file_content"])
        #     print(f"Created test file: {file_path}")
        
        # Get answer from agent
        try:
            answer = agent.invoke(question) # , file_path)
            print(f"Agent answer: {answer}")
            
            # Check if answer matches expected
            is_correct = answer.lower() == expected.lower()
            if is_correct:
                correct_count += 1
                print(f"✅ CORRECT")
            else:
                print(f"❌ INCORRECT - Expected: {expected}")
            
            results.append({
                "task_id": task_id,
                "question": question,
                "expected": expected,
                "answer": answer,
                "is_correct": is_correct
            })
        except Exception as e:
            error_details = traceback.format_exc()
            print(f"Error processing question: {e}\n{error_details}")
            results.append({
                "task_id": task_id,
                "question": question,
                "expected": expected,
                "answer": f"ERROR: {str(e)}",
                "is_correct": False
            })
    
    # Print summary
    accuracy = (correct_count / total_count) * 100
    print(f"\n{'='*80}")
    print(f"Test Results: {correct_count}/{total_count} correct ({accuracy:.1f}%)")
    
    return results



def run_simuGAIA_all( profile: gr.OAuthProfile | None, submit: Optional[bool] = True):
    """
    Fetches all questions, runs the QAgent on them,
    and displays the results.
    """
    # --- Determine HF Space Runtime URL and Repo URL for submission ---
    space_id = os.getenv("SPACE_ID") # Get the SPACE_ID for sending link to the code

    if profile:
        username= f"{profile.username}"
        print(f"User logged in: {username}")
    else:
        print("User not logged in.")
        return "Please Login to Hugging Face with the button.", None

    api_url = DEFAULT_API_URL
    questions_url = f"{api_url}/questions"
    submit_url = f"{api_url}/submit"
    print("submit_url = " + submit_url + " with username = " + username)

    # 1. Instantiate and init Agent ( modify this part to create your agent)
    agent = init_agent()


    # In the case of an app running as a hugging Face space, this link points toward your codebase ( usefull for others so please keep it public)
    agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main"
    print(agent_code)

    # 2. Fetch Questions
    print(f"Fetching questions from: {questions_url}")
    try:
        response = requests.get(questions_url, timeout=15)
        response.raise_for_status()
        questions_data = response.json()
        if not questions_data:
             print("Fetched questions list is empty.")
             return "Fetched questions list is empty or invalid format.", None
        print(f"Fetched {len(questions_data)} questions.")
    except requests.exceptions.RequestException as e:
        print(f"Error fetching questions: {e}")
        return f"Error fetching questions: {e}", None
    except requests.exceptions.JSONDecodeError as e:
         print(f"Error decoding JSON response from questions endpoint: {e}")
         print(f"Response text: {response.text[:500]}")
         return f"Error decoding server response for questions: {e}", None
    except Exception as e:
        print(f"An unexpected error occurred fetching questions: {e}")
        return f"An unexpected error occurred fetching questions: {e}", None

    # 2.5 Awaken the AnswerDataManager to get and store already answered questions
    manager = AnswerDataManager("already_answered.json")
    data = manager.load_data()
    print(data.__str__)
    
    # 3. Run your Agent
    results_log = []
    answers_payload = []
    
    print(f"Running agent on {len(questions_data)} questions...")
    for item in questions_data:
        task_id = item.get("task_id")
        question_text = item.get("question")
        submitted_answer = "NO ANSWER YET"
        if not task_id or question_text is None:
            print(f"Skipping item with missing task_id or question: {item}")
            continue
        try:
            existing_answer = manager.get_answer_by_task_id(task_id)
            if not existing_answer:
                # then we call the agent
                if question_text.startswith("What si the first name of"): # **NOK**: ("Who are the pitchers"): # **NOK**:("What country had the least number"): # ("Where were the Vietnamese"): # ("On June 6, 2023, an article"): # ("How many at bats did the Yankee"): # ("Who did the actor who"): # ("m making a grocery list", 2): # ("What is the surname of the"): # ("Given this table"): # ("Who nominated the only"): # ("How many studio albums"): # (".rewsna eht sa"):   <---     REMOVE THAT FOR ALL QUESTIONS
                    print(f"Precise question detected. INVOKING AGENT! Be careful!")
                    submitted_answer = agent.invoke(question_text)
    
                    # Save answer, task_id, and question_text to already_answered.json
                    # manager.add_answer(task_id, question_text, submitted_answer)
                    success = manager.add_answer(
                        task_id=task_id, 
                        question=question_text, 
                        submitted_answer=submitted_answer
                    )
                    if not success:
                        print("Error saving answer to archive.")
                    
                    answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
                else:
                    submitted_answer = "NO AGENT INVOKED"
            else:
                # then we get answer already found from archive
                submitted_answer = existing_answer['submitted_answer']
                answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
                
            results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer})
            
            
        except Exception as e:
             print(f"Error running agent on task {task_id}: {e}")
             results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": f"AGENT ERROR: {e}"})

    if not answers_payload:
        print("Agent did not produce any answers to submit.")
        return "Agent did not produce any answers to submit.", pd.DataFrame(results_log)

    # 4. Prepare Submission 
    submission_data = {"username": username.strip(), "agent_code": agent_code, "answers": answers_payload}
    status_update = f"Agent finished. Submitting {len(answers_payload)} answers for user '{username}' with url being {agent_code}..."
    print(status_update)
    print(f"Answers payload content: {answers_payload}")
    # for answer in answers_payload:
    #     print("task_id: " + answer["Task ID"])
    #     print("answer: " + answer["Submitted Answer"])


    
    if not submit:
        return "Run finished. No submission done, as asked.", pd.DataFrame(results_log)

    
    
    # 5. Submit
    print(f"Submitting {len(answers_payload)} answers to: {submit_url}")
    try:
        response = requests.post(submit_url, json=submission_data, timeout=60)
        response.raise_for_status()
        result_data = response.json()
        final_status = (
            f"Submission Successful!\n"
            f"User: {result_data.get('username')}\n"
            f"Overall Score: {result_data.get('score', 'N/A')}% "
            f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n"
            f"Message: {result_data.get('message', 'No message received.')}"
        )
        print("Submission successful.")
        results_df = pd.DataFrame(results_log)
        return final_status, results_df
    except requests.exceptions.HTTPError as e:
        error_detail = f"Server responded with status {e.response.status_code}."
        try:
            error_json = e.response.json()
            error_detail += f" Detail: {error_json.get('detail', e.response.text)}"
        except requests.exceptions.JSONDecodeError:
            error_detail += f" Response: {e.response.text[:500]}"
        status_message = f"Submission Failed: {error_detail}"
        print(status_message)
        results_df = pd.DataFrame(results_log)
        return status_message, results_df
    except requests.exceptions.Timeout:
        status_message = "Submission Failed: The request timed out."
        print(status_message)
        results_df = pd.DataFrame(results_log)
        return status_message, results_df
    except requests.exceptions.RequestException as e:
        status_message = f"Submission Failed: Network error - {e}"
        print(status_message)
        results_df = pd.DataFrame(results_log)
        return status_message, results_df
    except Exception as e:
        status_message = f"An unexpected error occurred during submission: {e}"
        print(status_message)
        results_df = pd.DataFrame(results_log)
        return status_message, results_df