Upload 9 files

examples/gaia/README.md

@@ -0,0 +1,74 @@
+# Running Gaia Benchmark
+
+AWorld is an AI-powered simulation environment for creating and interacting with virtual worlds.
+
+You could follow the following steps to run the Gaia benchmark.
+
+## Prerequisites
+
+- Git
+- Conda (Miniconda or Anaconda)
+- Python 3.11
+- Node.js
+
+## Installation
+
+### 1. Clone the repository:
+```bash
+git clone https://github.com/inclusionAI/AWorld.git
+cd AWorld
+```
+
+### 2. Create and activate a Conda environment:
+```bash
+conda create --name aworld python=3.11
+conda activate aworld
+```
+
+### 3. Setup the environment:
+```bash
+python setup.py install
+
+git clone https://github.com/haris-musa/excel-mcp-server.git
+cd excel-mcp-server
+uv pip install -e .
+```
+
+### 4. Download GAIA Dataset
+https://huggingface.co/datasets/gaia-benchmark/GAIA/tree/main
+
+## Configuration
+
+### Create a `.env` file in the project root and add your API credentials:
+```bash
+GAIA_DATASET_PATH=<Your Dataset Absolute Path>
+LLM_API_KEY=<Your API Key>
+LLM_BASE_URL=<Your Service Provider URL>
+...
+```
+
+## Running the Application
+
+1. Start the local MCP servers
+```bash
+uv run excel-mcp-server
+```
+The server should now be running with the configuration specified in `mcp.json`.
+
+1. Run the script
+```bash
+python ./examples/gaia/run.py
+```
+Now you could check the output log in the console.
+
+## Troubleshooting
+
+If you encounter issues:
+
+- Verify all environment variables are set correctly
+- Check that all dependencies are installed
+- Ensure the dataset path is correctly configured
+- Check the server logs for any error messages
+
+## Support
+For additional help, please [open an issue](https://github.com/inclusionAI/AWorld/issues/new) on our GitHub repository.

examples/gaia/__init__.py

@@ -0,0 +1,8 @@
+# coding: utf-8
+# Copyright (c) 2025 inclusionAI.
+from aworld.utils import import_packages
+
+import_packages(["dotenv"])
+from dotenv import load_dotenv
+
+load_dotenv()

examples/gaia/gaia_agent_runner.py

@@ -0,0 +1,200 @@
+import json
+import logging
+import os
+import re
+import traceback
+from typing import AsyncGenerator
+import uuid
+from aworld.config.conf import AgentConfig, TaskConfig
+from aworld.agents.llm_agent import Agent
+from aworld.core.task import Task
+from aworld.runner import Runners
+from aworld.output.ui.base import AworldUI
+from aworld.output.ui.markdown_aworld_ui import MarkdownAworldUI
+from aworld.output.base import Output
+from .utils import (
+    add_file_path,
+    load_dataset_meta_dict,
+    question_scorer,
+)
+from .prompt import system_prompt
+
+logger = logging.getLogger(__name__)
+
+
+class GaiaAgentRunner:
+    """
+    Gaia Agent Runner
+    """
+
+    def __init__(
+        self,
+        llm_provider: str,
+        llm_model_name: str,
+        llm_base_url: str,
+        llm_api_key: str,
+        llm_temperature: float = 0.0,
+        mcp_config: dict = {},
+    ):
+        self.agent_config = AgentConfig(
+            llm_provider=llm_provider,
+            llm_model_name=llm_model_name,
+            llm_api_key=llm_api_key,
+            llm_base_url=llm_base_url,
+            llm_temperature=llm_temperature,
+        )
+
+        self.super_agent = Agent(
+            conf=self.agent_config,
+            name="gaia_super_agent",
+            system_prompt=system_prompt,
+            mcp_config=mcp_config,
+            mcp_servers=mcp_config.get("mcpServers", {}).keys(),
+        )
+
+        self.gaia_dataset_path = os.path.abspath(
+            os.getenv(
+                "GAIA_DATASET_PATH",
+                os.path.join(os.path.dirname(os.path.abspath(__file__)), "GAIA", "2023"),
+            )
+        )
+        self.full_dataset = load_dataset_meta_dict(self.gaia_dataset_path)
+        logger.info(
+            f"Gaia Agent Runner initialized: super_agent={self.super_agent}, agent_config={self.agent_config}, gaia_dataset_path={self.gaia_dataset_path}, full_dataset={len(self.full_dataset)}"
+        )
+
+    async def run(self, prompt: str):
+        yield (f"\n### GAIA Agent Start!")
+
+        mcp_servers = "\n- ".join(self.super_agent.mcp_servers)
+        yield (f"\n```gaia_agent_status\n- {mcp_servers}\n```\n")
+
+        question = None
+        data_item = None
+        task_id = None
+        try:
+            json_data = json.loads(prompt)
+            task_id = json_data["task_id"]
+
+            data_item = self.full_dataset[task_id]
+            question = add_file_path(data_item, file_path=self.gaia_dataset_path)[
+                "Question"
+            ]
+            yield (f"\n```gaia_question\n{json.dumps(data_item, indent=2)}\n```\n")
+        except Exception as e:
+            pass
+
+        if not question:
+            logger.warning(
+                "Could not find GAIA question for prompt, chat using prompt directly!"
+            )
+            yield (f"\n{prompt}\n")
+            question = prompt
+
+        try:
+            task = Task(
+                id=task_id + "." + uuid.uuid1().hex if task_id else uuid.uuid1().hex,
+                input=question,
+                agent=self.super_agent,
+                event_driven=False,
+                conf=TaskConfig(max_steps=20),
+            )
+
+            last_output: Output = None
+            rich_ui = MarkdownAworldUI()
+            async for output in Runners.streamed_run_task(task).stream_events():
+                logger.info(f"Gaia Agent Ouput: {output}")
+                res = await AworldUI.parse_output(output, rich_ui)
+                for item in res if isinstance(res, list) else [res]:
+                    if isinstance(item, AsyncGenerator):
+                        async for sub_item in item:
+                            yield sub_item
+                    else:
+                        yield item
+                    last_output = item
+
+            logger.info(f"Gaia Agent Last Output: {last_output}")
+
+            if data_item and last_output:
+                final_response = self._judge_answer(data_item, last_output)
+                yield final_response
+
+        except Exception as e:
+            logger.error(f"Error processing {prompt}, error: {traceback.format_exc()}")
+
+    def _judge_answer(self, data_item: dict, result: Output):
+        answer = result
+        match = re.search(r"<answer>(.*?)</answer>", answer)
+        if match:
+            answer = match.group(1)
+            logger.info(f"Agent answer: {answer}")
+            logger.info(f"Correct answer: {data_item['Final answer']}")
+
+            if question_scorer(answer, data_item["Final answer"]):
+                logger.info(f"Question {data_item['task_id']} Correct!")
+            else:
+                logger.info(f"Question {data_item['task_id']} Incorrect!")
+
+            # Create the new result record
+            correct = question_scorer(answer, data_item["Final answer"])
+            new_result = {
+                "task_id": data_item["task_id"],
+                "level": data_item["Level"],
+                "question": data_item["Question"],
+                "answer": data_item["Final answer"],
+                "response": answer,
+                "is_correct": correct,
+            }
+            return f"\n## Final Result: {'✅' if correct else '❌'}\n \n```gaia_result\n{json.dumps(new_result, indent=2)}\n```"
+        else:
+            new_result = answer
+            return f"\n## Final Result:\n \n```gaia_result\n{json.dumps(new_result, indent=2)}\n```"
+
+
+if __name__ == "__main__":
+    import asyncio
+    import argparse
+    from datetime import datetime
+
+    logger = logging.getLogger(__name__)
+
+    output_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "output")
+    if not os.path.exists(output_dir):
+        os.makedirs(output_dir)
+
+    output_file = os.path.join(
+        output_dir, f"output_{datetime.now().strftime('%Y%m%d_%H%M%S')}.md"
+    )
+
+    async def main():
+        parser = argparse.ArgumentParser()
+        parser.add_argument("--prompt", type=str, default="")
+        args = parser.parse_args()
+
+        try:
+            prompt = args.prompt
+
+            llm_provider = os.getenv("LLM_PROVIDER")
+            llm_model_name = os.getenv("LLM_MODEL_NAME")
+            llm_api_key = os.getenv("LLM_API_KEY")
+            llm_base_url = os.getenv("LLM_BASE_URL")
+            llm_temperature = os.getenv("LLM_TEMPERATURE", 0.0)
+
+            def send_output(output):
+                with open(output_file, "a") as f:
+                    f.write(f"{output}\n")
+
+            async for i in GaiaAgentRunner(
+                llm_provider=llm_provider,
+                llm_model_name=llm_model_name,
+                llm_base_url=llm_base_url,
+                llm_api_key=llm_api_key,
+                llm_temperature=llm_temperature,
+            ).run(prompt):
+                send_output(i)
+        except Exception as e:
+            logger.error(
+                f"Error processing {args.prompt}, error: {traceback.format_exc()}"
+            )
+
+    asyncio.run(main())

examples/gaia/gaia_agent_server.py

@@ -0,0 +1,141 @@
+from typing import AsyncGenerator
+import traceback
+import logging
+import os
+import json
+import sys
+import uuid
+import time
+
+logger = logging.getLogger(__name__)
+
+
+class GaiaAgentServer:
+
+    def _get_model_config(self):
+        try:
+            llm_provider = os.getenv("LLM_PROVIDER_GAIA")
+            llm_model_name = os.getenv("LLM_MODEL_NAME_GAIA")
+            llm_api_key = os.getenv("LLM_API_KEY_GAIA")
+            llm_base_url = os.getenv("LLM_BASE_URL_GAIA")
+            llm_temperature = os.getenv("LLM_TEMPERATURE_GAIA", 0.0)
+            return {
+                "provider": llm_provider,
+                "model": llm_model_name,
+                "api_key": llm_api_key,
+                "base_url": llm_base_url,
+                "temperature": llm_temperature,
+            }
+        except Exception as e:
+            logger.warning(
+                f">>> Gaia Agent: GAIA_MODEL_CONFIG is not configured, using LLM"
+            )
+            raise e
+
+    def models(self):
+        model = self._get_model_config()
+
+        return [
+            {
+                "id": f"{model['provider']}/{model['model']}",
+                "name": f"gaia_agent@{model['provider']}/{model['model']}",
+            }
+        ]
+
+    async def chat_completions(self, body: dict) -> AsyncGenerator[str, None]:
+        def response_line(line: str, model: str):
+            return {
+                "object": "chat.completion.chunk",
+                "id": str(uuid.uuid4()).replace("-", ""),
+                "choices": [
+                    {"index": 0, "delta": {"content": line, "role": "assistant"}}
+                ],
+                "created": int(time.time()),
+                "model": model,
+            }
+
+        try:
+            logger.info(f">>> Gaia Agent: body={body}")
+
+            prompt = body["messages"][-1]["content"]
+            model = body["model"].replace("gaia_agent.", "")
+
+            logger.info(f">>> Gaia Agent: prompt={prompt}, model={model}")
+
+            selected_model = self._get_model_config()
+
+            logger.info(f">>> Gaia Agent: Using model configuration: {selected_model}")
+
+            logger.info(f">>> Gaia Agent Python Path: sys.path={sys.path}")
+
+            llm_provider = selected_model.get("provider")
+            llm_model_name = selected_model.get("model")
+            llm_api_key = selected_model.get("api_key")
+            llm_base_url = selected_model.get("base_url")
+            llm_temperature = selected_model.get("temperature", 0.0)
+
+            from examples.gaia.gaia_agent_runner import GaiaAgentRunner
+
+            mcp_path = os.path.join(
+                os.path.dirname(os.path.abspath(__file__)), "mcp.json"
+            )
+            with open(mcp_path, "r") as f:
+                mcp_config = json.load(f)
+
+            runner = GaiaAgentRunner(
+                llm_provider=llm_provider,
+                llm_model_name=llm_model_name,
+                llm_base_url=llm_base_url,
+                llm_api_key=llm_api_key,
+                llm_temperature=llm_temperature,
+                mcp_config=mcp_config,
+            )
+
+            logger.info(f">>> Gaia Agent: prompt={prompt}, runner={runner}")
+
+            async for i in runner.run(prompt):
+                line = response_line(i, model)
+                logger.info(f">>> Gaia Agent Line: {line}")
+                yield line
+
+        except Exception as e:
+            emsg = traceback.format_exc()
+            logger.error(f">>> Gaia Agent Error: exception {emsg}")
+            yield response_line(f"Gaia Agent Error: {emsg}", model)
+
+        finally:
+            logger.info(f">>> Gaia Agent Done")
+
+
+import fastapi
+from fastapi.responses import StreamingResponse
+
+app = fastapi.FastAPI()
+
+from examples.gaia.gaia_agent_server import GaiaAgentServer
+
+agent_server = GaiaAgentServer()
+
+
+@app.get("/v1/models")
+async def models():
+    return agent_server.models()
+
+
+@app.post("/v1/chat/completions")
+async def chat_completions(request: fastapi.Request):
+    form_data = await request.json()
+    logger.info(f">>> Gaia Agent Server: form_data={form_data}")
+
+    async def event_generator():
+        async for chunk in agent_server.chat_completions(form_data):
+            # Format as SSE: each line needs to start with "data: " and end with two newlines
+            yield f"data: {json.dumps(chunk, ensure_ascii=False)}\n\n"
+
+    return StreamingResponse(event_generator(), media_type="text/event-stream")
+
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run("gaia_agent_server:app", host="0.0.0.0", port=8888)

examples/gaia/mcp.json

@@ -0,0 +1,140 @@
+{
+    "mcpServers": {
+        "e2b-server": {
+            "command": "npx",
+            "args": [
+                "-y",
+                "@e2b/mcp-server"
+            ],
+            "env": {
+                "E2B_API_KEY": "${E2B_API_KEY}",
+                "SESSION_REQUEST_CONNECT_TIMEOUT": "120"
+            }
+        },
+        "filesystem": {
+            "command": "npx",
+            "args": [
+                "-y",
+                "@modelcontextprotocol/server-filesystem",
+                "${FILESYSTEM_SERVER_WORKDIR}"
+            ]
+        },
+        "terminal-controller": {
+            "command": "python",
+            "args": [
+                "-m",
+                "terminal_controller"
+            ]
+        },
+        "calculator": {
+            "command": "python",
+            "args": [
+                "-m",
+                "mcp_server_calculator"
+            ]
+        },
+        "excel": {
+            "command": "npx",
+            "args": [
+                "--yes",
+                "@negokaz/excel-mcp-server"
+            ],
+            "env": {
+                "EXCEL_MCP_PAGING_CELLS_LIMIT": "4000"
+            }
+        },
+        "playwright": {
+            "command": "npx",
+            "args": [
+                "-y",
+                "@executeautomation/playwright-mcp-server"
+            ]
+        },
+        "google-search": {
+            "command": "npx",
+            "args": [
+                "-y",
+                "@adenot/mcp-google-search"
+            ],
+            "env": {
+                "GOOGLE_API_KEY": "${GOOGLE_API_KEY}",
+                "GOOGLE_SEARCH_ENGINE_ID": "${GOOGLE_SEARCH_ENGINE_ID}"
+            }
+        },
+        "ms-playwright": {
+            "command": "npx",
+            "args": [
+                "@playwright/mcp@latest",
+                "--no-sandbox",
+                "--headless"
+            ],
+            "env": {
+                "PLAYWRIGHT_TIMEOUT": "120000",
+                "SESSION_REQUEST_CONNECT_TIMEOUT": "120"
+            }
+        },
+        "audio_server": {
+            "command": "python",
+            "args": [
+                "-m",
+                "mcp_servers.audio_server"
+            ]
+        },
+        "image_server": {
+            "command": "python",
+            "args": [
+                "-m",
+                "mcp_servers.image_server"
+            ]
+        },
+        "youtube_server": {
+            "command": "python",
+            "args": [
+                "-m",
+                "mcp_servers.youtube_server"
+            ]
+        },
+        "video_server": {
+            "command": "python",
+            "args": [
+                "-m",
+                "mcp_servers.video_server"
+            ]
+        },
+        "search_server": {
+            "command": "python",
+            "args": [
+                "-m",
+                "mcp_servers.search_server"
+            ]
+        },
+        "download_server": {
+            "command": "python",
+            "args": [
+                "-m",
+                "mcp_servers.download_server"
+            ]
+        },
+        "document_server": {
+            "command": "python",
+            "args": [
+                "-m",
+                "mcp_servers.document_server"
+            ]
+        },
+        "browser_server": {
+            "command": "python",
+            "args": [
+                "-m",
+                "mcp_servers.browser_server"
+            ]
+        },
+        "reasoning_server": {
+            "command": "python",
+            "args": [
+                "-m",
+                "mcp_servers.reasoning_server"
+            ]
+        }
+    }
+}

examples/gaia/prompt.py

@@ -0,0 +1,26 @@
+system_prompt = """You are an all-capable AI assistant, aimed at solving any task presented by the user. You have various tools at your disposal that you can call upon to efficiently complete complex requests. Whether it's programming, information retrieval, file processing, or web browsing, you can handle it all.
+Please note that the task may be complex. Do not attempt to solve it all at once. You should break the task down and use different tools step by step to solve it. After using each tool, clearly explain the execution results and suggest the next steps.
+Please utilize appropriate tools for the task, analyze the results obtained from these tools, and provide your reasoning. Always use available tools such as browser, calcutor, etc. to verify correctness rather than relying on your internal knowledge.
+If you believe the problem has been solved, please output the `final answer`. The `final answer` should be given in <answer></answer> format, while your other thought process should be output in <think></think> tags.
+Your `final answer` should be a number OR as few words as possible OR a comma separated list of numbers and/or strings. If you are asked for a number, don't use comma to write your number neither use units such as $ or percent sign unless specified otherwise. If you are asked for a string, don't use articles, neither abbreviations (e.g. for cities), and write the digits in plain text unless specified otherwise. If you are asked for a comma separated list, apply the above rules depending of whether the element to be put in the list is a number or a string.
+
+Here are some tips to help you give better instructions: 
+<tips>
+*. Consider search relevant information first using `search_server` tool. Then break down the problem following the instructions from the search.
+1. Do not use any tools outside of the provided tools list.
+2. Always use only one tool at a time in each step of your execution.
+3. Even if the task is complex, there is always a solution. If you can't find the answer using one method, try another approach or use different tools to find the solution.
+4. Due to context length limitations, always try to complete browser-based tasks with the minimal number of steps possible.
+5. Before providing the `final answer`, carefully reflect on whether the task has been fully solved. If you have not solved the task, please provide your reasoning and suggest the next steps.
+6. When providing the `final answer`, answer the user's question directly and precisely. For example, if asked "what animal is x?" and x is a monkey, simply answer "monkey" rather than "x is a monkey".
+7. When you need to process excel file, prioritize using the `excel` tool instead of writing custom code with `terminal-controller` tool.
+8. If you need to download a file, please use the `download_server` tool to download the file and save it to the specified path.
+9. Use the `search_server` to get the relevant website URLs or contents.
+10. When there are questions related to YouTube video comprehension, use tools in `youtube_server` and `video_server` to analyze the video content by the given question.
+11. Ensure to call `mcp__reasoning_server__complex_problem_reasoning` for solving complex reasoning tasks, such as riddle, game or competition-level STEM(including code) problems.
+12. `e2b-server` is a powerful tool only for running **Python** code. Other programming languages are **NOT SUPPORTED**.
+</tips>
+
+Now, here is the task. Stay focused and complete it carefully using the appropriate tools!
+"""
+# 13. Using `mcp__ms-playwright__browser_take_screenshot` tool to save the screenshot of URLs to the specified path when you need to understand the gif / jpg of the URLs.

examples/gaia/run.py

@@ -0,0 +1,263 @@
+import argparse
+import json
+import logging
+import os
+import re
+import traceback
+from typing import Any, Dict, List
+
+from dotenv import load_dotenv
+
+from aworld.config.conf import AgentConfig, TaskConfig
+from aworld.agents.llm_agent import Agent
+from aworld.core.task import Task
+from aworld.runner import Runners
+from examples.gaia.prompt import system_prompt
+from examples.gaia.utils import (
+    add_file_path,
+    load_dataset_meta,
+    question_scorer,
+    report_results,
+)
+
+# Create log directory if it doesn't exist
+if not os.path.exists(os.getenv("LOG_FILE_PATH")):
+    os.makedirs(os.getenv("LOG_FILE_PATH"))
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "--start",
+    type=int,
+    default=0,
+    help="Start index of the dataset",
+)
+parser.add_argument(
+    "--end",
+    type=int,
+    default=20,
+    help="End index of the dataset",
+)
+parser.add_argument(
+    "--q",
+    type=str,
+    help="Question Index, e.g., 0-0-0-0-0. Highest priority: override other arguments if provided.",
+)
+parser.add_argument(
+    "--skip",
+    action="store_true",
+    help="Skip the question if it has been processed before.",
+)
+parser.add_argument(
+    "--split",
+    type=str,
+    default="validation",
+    help="Split of the dataset, e.g., validation, test",
+)
+parser.add_argument(
+    "--blacklist_file_path",
+    type=str,
+    nargs="?",
+    help="Blacklist file path, e.g., blacklist.txt",
+)
+args = parser.parse_args()
+
+
+def setup_logging():
+    logging_logger = logging.getLogger()
+    logging_logger.setLevel(logging.INFO)
+
+    log_file_name = (
+        f"/super_agent_{args.q}.log"
+        if args.q
+        else f"/super_agent_{args.start}_{args.end}.log"
+    )
+    file_handler = logging.FileHandler(
+        os.getenv(
+            "LOG_FILE_PATH",
+            "run_super_agent.log",
+        )
+        + log_file_name,
+        mode="a",
+        encoding="utf-8",
+    )
+    file_handler.setLevel(logging.INFO)
+
+    formatter = logging.Formatter(
+        "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+    )
+    file_handler.setFormatter(formatter)
+
+    logging_logger.addHandler(file_handler)
+
+
+if __name__ == "__main__":
+    load_dotenv()
+    setup_logging()
+
+    gaia_dataset_path = os.getenv("GAIA_DATASET_PATH", "./gaia_dataset")
+    full_dataset = load_dataset_meta(gaia_dataset_path, split=args.split)
+    logging.info(f"Total questions: {len(full_dataset)}")
+
+    agent_config = AgentConfig(
+        llm_provider="openai",
+        llm_model_name=os.getenv("LLM_MODEL_NAME", "gpt-4o"),
+        llm_api_key=os.getenv("LLM_API_KEY", "your_openai_api_key"),
+        llm_base_url=os.getenv("LLM_BASE_URL", "your_openai_base_url"),
+    )
+    super_agent = Agent(
+        conf=agent_config,
+        name="gaia_super_agent",
+        system_prompt=system_prompt,
+        mcp_servers=[
+            "e2b-server",
+            # "filesystem",
+            "terminal-controller",
+            "excel",
+            "calculator",
+            "ms-playwright",
+            "audio_server",
+            "image_server",
+            "video_server",
+            "search_server",
+            "download_server",
+            "document_server",
+            # "browser_server",
+            "youtube_server",
+            "reasoning_server",
+        ],
+    )
+
+    # load results from the checkpoint file
+    if os.path.exists(os.getenv("LOG_FILE_PATH") + "/results.json"):
+        with open(
+            os.getenv("LOG_FILE_PATH") + "/results.json", "r", encoding="utf-8"
+        ) as results_f:
+            results: List[Dict[str, Any]] = json.load(results_f)
+    else:
+        results: List[Dict[str, Any]] = []
+
+    # load blacklist `task_id`
+    if args.blacklist_file_path and os.path.exists(args.blacklist_file_path):
+        with open(args.blacklist_file_path, "r", encoding="utf-8") as f:
+            blacklist = set(f.read().splitlines())
+    else:
+        blacklist = set()  # Empty set if file doesn't exist
+
+    try:
+        # slice dataset by args.start and args.end, overrided by args.q (single `task_id`)
+        dataset_slice = (
+            [
+                dataset_record
+                for idx, dataset_record in enumerate(full_dataset)
+                if dataset_record["task_id"] in args.q
+            ]
+            if args.q is not None
+            else full_dataset[args.start : args.end]
+        )
+
+        # main loop to execute questions
+        for i, dataset_i in enumerate(dataset_slice):
+            # specify `task_id`
+            if args.q and args.q != dataset_i["task_id"]:
+                continue
+            # only valid for args.q==None
+            if not args.q:
+                # blacklist
+                if dataset_i["task_id"] in blacklist:
+                    continue
+
+                # pass
+                if any(
+                    # Question Done and Correct
+                    (result["task_id"] == dataset_i["task_id"] and result["is_correct"])
+                    for result in results
+                ) or any(
+                    # Question Done and Incorrect, but Level is 3
+                    (
+                        result["task_id"] == dataset_i["task_id"]
+                        and not result["is_correct"]
+                        and dataset_i["Level"] == 3
+                    )
+                    for result in results
+                ):
+                    continue
+
+                # skip
+                if args.skip and any(
+                    # Question Done and Correct
+                    (result["task_id"] == dataset_i["task_id"])
+                    for result in results
+                ):
+                    continue
+
+            # run
+            try:
+                logging.info(f"Start to process: {dataset_i['task_id']}")
+                logging.info(f"Detail: {dataset_i}")
+                logging.info(f"Question: {dataset_i['Question']}")
+                logging.info(f"Level: {dataset_i['Level']}")
+                logging.info(f"Tools: {dataset_i['Annotator Metadata']['Tools']}")
+
+                question = add_file_path(
+                    dataset_i, file_path=gaia_dataset_path, split=args.split
+                )["Question"]
+
+                task = Task(input=question, agent=super_agent, conf=TaskConfig())
+                result = Runners.sync_run_task(task=task)
+
+                match = re.search(r"<answer>(.*?)</answer>", result[task.id].get('answer'))
+                if match:
+                    answer = match.group(1)
+                    logging.info(f"Agent answer: {answer}")
+                    logging.info(f"Correct answer: {dataset_i['Final answer']}")
+
+                    if question_scorer(answer, dataset_i["Final answer"]):
+                        logging.info(f"Question {i} Correct!")
+                    else:
+                        logging.info("Incorrect!")
+
+                # Create the new result record
+                new_result = {
+                    "task_id": dataset_i["task_id"],
+                    "level": dataset_i["Level"],
+                    "question": question,
+                    "answer": dataset_i["Final answer"],
+                    "response": answer,
+                    "is_correct": question_scorer(answer, dataset_i["Final answer"]),
+                }
+
+                # Check if this task_id already exists in results
+                existing_index = next(
+                    (
+                        i
+                        for i, result in enumerate(results)
+                        if result["task_id"] == dataset_i["task_id"]
+                    ),
+                    None,
+                )
+
+                if existing_index is not None:
+                    # Update existing record
+                    results[existing_index] = new_result
+                    logging.info(
+                        f"Updated existing record for task_id: {dataset_i['task_id']}"
+                    )
+                else:
+                    # Append new record
+                    results.append(new_result)
+                    logging.info(
+                        f"Added new record for task_id: {dataset_i['task_id']}"
+                    )
+
+            except Exception as e:
+                logging.error(f"Error processing {i}: {traceback.format_exc()}")
+                continue
+    except KeyboardInterrupt:
+        pass
+    finally:
+        # report
+        report_results(results)
+        with open(
+            os.getenv("LOG_FILE_PATH") + "/results.json", "w", encoding="utf-8"
+        ) as f:
+            json.dump(results, f, indent=4, ensure_ascii=False)

examples/gaia/stream_aworld_ui.py

@@ -0,0 +1,66 @@
+import asyncio
+import time
+from dataclasses import dataclass, field
+from time import sleep
+
+from rich.table import Table
+
+from aworld.output.utils import consume_content
+from rich.status import Status
+from rich.console import Console
+
+from aworld.output import MessageOutput, WorkSpace
+from aworld.output.base import StepOutput, ToolResultOutput
+from aworld.output.ui.base import AworldUI
+import json
+@dataclass
+class StreamAworldUI(AworldUI):
+    console: Console = field(default_factory=Console)
+    status: Status = None
+    workspace: WorkSpace = None
+    gaia_output_line_tag = "GA_FMT_CONTENT:"
+
+    async def message_output(self, __output__: MessageOutput):
+        result = []
+        async def __log_item(item):
+            result.append(item)
+            self.console.print(f"{self.gaia_output_line_tag} {json.dumps(item)}", end="")
+
+        if __output__.reason_generator or __output__.response_generator:
+            if __output__.reason_generator:
+                await consume_content(__output__.reason_generator, __log_item)
+            if __output__.reason_generator:
+                await consume_content(__output__.response_generator, __log_item)
+        else:
+            await consume_content(__output__.reasoning, __log_item)
+            await consume_content(__output__.response, __log_item)
+        # if __output__.tool_calls:
+        #     await consume_content(__output__.tool_calls, __log_item)
+        self.console.print("")
+
+    async def tool_result(self, output: ToolResultOutput):
+        """
+            tool_result
+        """
+        table = Table(show_header=False, header_style="bold magenta", title=f"Call Tools#ID_{output.origin_tool_call.id}")
+        table.add_column("name", style="dim", width=12)
+        table.add_column("content")
+        table.add_row("function_name", output.origin_tool_call.function.name)
+        table.add_row("arguments", output.origin_tool_call.function.arguments)
+        table.add_row("result", output.data)
+        self.console.print(table)
+
+    async def step(self, output: StepOutput):
+        if output.status == "START":
+            self.console.print(f"[bold green]{output.name} ✈️START ...")
+            self.status = self.console.status(f"[bold green]{output.name} RUNNING ...")
+            self.status.start()
+        elif output.status == "FINISHED":
+            self.status.stop()
+            self.console.print(f"[bold green]{output.name} 🛬FINISHED ...")
+        elif output.status == "FAILED":
+            self.status.stop()
+            self.console.print(f"[bold red]{output.name} 💥FAILED ...")
+        else:
+            self.status.stop()
+            self.console.print(f"============={output.name} ❓❓❓UNKNOWN#{output.status} ======================")

examples/gaia/utils.py

@@ -0,0 +1,191 @@
+import json
+import re
+import string
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+import logging
+from tabulate import tabulate
+
+logger = logging.getLogger(__name__)
+
+
+def normalize_str(input_str, remove_punct=True) -> str:
+    no_spaces = re.sub(r"\s", "", input_str)
+    if remove_punct:
+        translator = str.maketrans("", "", string.punctuation)
+        return no_spaces.lower().translate(translator)
+    else:
+        return no_spaces.lower()
+
+
+def split_string(s: str, char_list: Optional[List[str]] = None) -> list[str]:
+    if char_list is None:
+        char_list = [",", ";"]
+    pattern = f"[{''.join(char_list)}]"
+    return re.split(pattern, s)
+
+
+def normalize_number_str(number_str: str) -> float:
+    for char in ["$", "%", ","]:
+        number_str = number_str.replace(char, "")
+    try:
+        return float(number_str)
+    except ValueError:
+        logger.error(f"String {number_str} cannot be normalized to number str.")
+        return float("inf")
+
+
+def question_scorer(model_answer: str, ground_truth: str) -> bool:
+    def is_float(element: Any) -> bool:
+        try:
+            float(element)
+            return True
+        except ValueError:
+            return False
+
+    try:
+        if is_float(ground_truth):
+            logger.info(f"Evaluating {model_answer} as a number.")
+            normalized_answer = normalize_number_str(model_answer)
+            return normalized_answer == float(ground_truth)
+
+        elif any(char in ground_truth for char in [",", ";"]):
+            logger.info(f"Evaluating {model_answer} as a comma separated list.")
+            gt_elems = split_string(ground_truth)
+            ma_elems = split_string(model_answer)
+
+            if len(gt_elems) != len(ma_elems):
+                logger.warning("Answer lists have different lengths, returning False.")
+                return False
+
+            comparisons = []
+            for ma_elem, gt_elem in zip(ma_elems, gt_elems):
+                if is_float(gt_elem):
+                    normalized_ma_elem = normalize_number_str(ma_elem)
+                    comparisons.append(normalized_ma_elem == float(gt_elem))
+                else:
+                    ma_elem = normalize_str(ma_elem, remove_punct=False)
+                    gt_elem = normalize_str(gt_elem, remove_punct=False)
+                    comparisons.append(ma_elem == gt_elem)
+            return all(comparisons)
+        else:
+            logger.info(f"Evaluating {model_answer} as a string.")
+            ma_elem = normalize_str(model_answer)
+            gt_elem = normalize_str(ground_truth)
+            return ma_elem == gt_elem
+    except Exception as e:
+        logger.error(f"Error during evaluation: {e}")
+        return False
+
+
+def load_dataset_meta(path: str, split: str = "validation"):
+    data_dir = Path(path) / split
+
+    dataset = []
+    with open(data_dir / "metadata.jsonl", "r", encoding="utf-8") as metaf:
+        lines = metaf.readlines()
+        for line in lines:
+            data = json.loads(line)
+            if data["task_id"] == "0-0-0-0-0":
+                continue
+            if data["file_name"]:
+                data["file_name"] = data_dir / data["file_name"]
+            dataset.append(data)
+    return dataset
+
+
+def load_dataset_meta_dict(path: str, split: str = "validation"):
+    data_dir = Path(path) / split
+
+    dataset = {}
+    with open(data_dir / "metadata.jsonl", "r", encoding="utf-8") as metaf:
+        lines = metaf.readlines()
+        for line in lines:
+            data = json.loads(line)
+            if data["task_id"] == "0-0-0-0-0":
+                continue
+            if data["file_name"]:
+                data["file_name"] = data_dir / data["file_name"]
+            dataset[data["task_id"]] = data
+    return dataset
+
+
+def add_file_path(
+    task: Dict[str, Any], file_path: str = "./gaia_dataset", split: str = "validation"
+):
+    if task["file_name"]:
+        file_path = Path(f"{file_path}/{split}") / task["file_name"]
+        if file_path.suffix in [".pdf", ".docx", ".doc", ".txt"]:
+            task["Question"] += f" Here are the necessary document files: {file_path}"
+
+        elif file_path.suffix in [".jpg", ".jpeg", ".png"]:
+            task["Question"] += f" Here are the necessary image files: {file_path}"
+
+        elif file_path.suffix in [".xlsx", "xls", ".csv"]:
+            task["Question"] += (
+                f" Here are the necessary table files: {file_path}, for processing excel file,"
+                " you can use the excel tool or write python code to process the file"
+                " step-by-step and get the information."
+            )
+        elif file_path.suffix in [".py"]:
+            task["Question"] += f" Here are the necessary python files: {file_path}"
+
+        else:
+            task["Question"] += f" Here are the necessary files: {file_path}"
+
+    return task
+
+
+def report_results(entries):
+    # Initialize counters
+    total_entries = len(entries)
+    total_correct = 0
+
+    # Initialize level statistics
+    level_stats = {}
+
+    # Process each entry
+    for entry in entries:
+        level = entry.get("level")
+        is_correct = entry.get("is_correct", False)
+
+        # Initialize level stats if not already present
+        if level not in level_stats:
+            level_stats[level] = {"total": 0, "correct": 0, "accuracy": 0}
+
+        # Update counters
+        level_stats[level]["total"] += 1
+        if is_correct:
+            total_correct += 1
+            level_stats[level]["correct"] += 1
+
+    # Calculate accuracy for each level
+    for level, stats in level_stats.items():
+        if stats["total"] > 0:
+            stats["accuracy"] = (stats["correct"] / stats["total"]) * 100
+
+    # Print overall statistics with colorful logging
+    logger.info("Overall Statistics:")
+    overall_accuracy = (total_correct / total_entries) * 100
+
+    # Create overall statistics table
+    overall_table = [
+        ["Total Entries", total_entries],
+        ["Total Correct", total_correct],
+        ["Overall Accuracy", f"{overall_accuracy:.2f}%"],
+    ]
+    logger.success(tabulate(overall_table, tablefmt="grid"))
+    logger.info("")
+
+    # Create level statistics table
+    logger.info("Statistics by Level:")
+    level_table = []
+    headers = ["Level", "Total Entries", "Correct Answers", "Accuracy"]
+
+    for level in sorted(level_stats.keys()):
+        stats = level_stats[level]
+        level_table.append(
+            [level, stats["total"], stats["correct"], f"{stats['accuracy']:.2f}%"]
+        )
+
+    logger.success(tabulate(level_table, headers=headers, tablefmt="grid"))