indexed yaml (this is going to support the eval config framework) (#29)

examples/langchain_single_agent.yaml

@@ -1,9 +1,11 @@
-location: Pontevedra
-date: 2025-03-22 12:00
-max_driving_hours: 2
-model_id: o3-mini
-agent_type: langchain
-tools:
- - "surf_spot_finder.tools.search_web"
- - "surf_spot_finder.tools.visit_webpage"
-# input_prompt_template:
+input:
+  location: Pontevedra
+  date: 2025-03-27 12:00
+  max_driving_hours: 2
+  # input_prompt_template:
+agent:
+  model_id: o3-mini
+  agent_type: langchain
+  tools:
+  - "surf_spot_finder.tools.search_web"
+  - "surf_spot_finder.tools.visit_webpage"

examples/langchain_single_agent_vertical.yaml

@@ -1,14 +1,16 @@
-location: Pontevedra
-date: 2025-03-22 12:00
-max_driving_hours: 2
-model_id: o3-mini
-agent_type: langchain
-tools:
- - "surf_spot_finder.tools.driving_hours_to_meters"
- - "surf_spot_finder.tools.get_area_lat_lon"
- - "surf_spot_finder.tools.get_surfing_spots"
- - "surf_spot_finder.tools.get_wave_forecast"
- - "surf_spot_finder.tools.get_wind_forecast"
- - "surf_spot_finder.tools.search_web"
- - "surf_spot_finder.tools.visit_webpage"
-# input_prompt_template:
+input:
+  location: Pontevedra
+  date: 2025-03-27 12:00
+  max_driving_hours: 2
+  # input_prompt_template:
+agent:
+  model_id: o3-mini
+  agent_type: langchain
+  tools:
+  - "surf_spot_finder.tools.driving_hours_to_meters"
+  - "surf_spot_finder.tools.get_area_lat_lon"
+  - "surf_spot_finder.tools.get_surfing_spots"
+  - "surf_spot_finder.tools.get_wave_forecast"
+  - "surf_spot_finder.tools.get_wind_forecast"
+  - "surf_spot_finder.tools.search_web"
+  - "surf_spot_finder.tools.visit_webpage"

examples/openai_multi_agent.yaml

@@ -1,6 +1,8 @@
-location: Pontevedra
-date: 2025-03-22 12:00
-max_driving_hours: 2
-model_id: o3-mini
-agent_type: openai_multi_agent
+input:
+  location: Pontevedra
+  date: 2025-03-27 12:00
+  max_driving_hours: 2
+agent:
+  model_id: o3-mini
+  agent_type: openai_multi_agent
 # input_prompt_template:

examples/openai_single_agent.yaml

@@ -1,9 +1,11 @@
-location: Pontevedra
-date: 2025-03-22 12:00
-max_driving_hours: 2
-model_id: o3-mini
-agent_type: openai
-tools:
- - "surf_spot_finder.tools.search_web"
- - "surf_spot_finder.tools.visit_webpage"
-# input_prompt_template:
+input:
+  location: Pontevedra
+  date: 2025-03-27 12:00
+  max_driving_hours: 2
+  # input_prompt_template:
+agent:
+  model_id: o3-mini
+  agent_type: openai
+  tools:
+  - "surf_spot_finder.tools.search_web"
+  - "surf_spot_finder.tools.visit_webpage"

examples/openai_single_agent_vertical.yaml

@@ -1,15 +1,17 @@
-location: Pontevedra
-date: 2025-03-22 12:00
-max_driving_hours: 2
-model_id: o3-mini
-agent_type: openai
-tools:
- - "surf_spot_finder.tools.driving_hours_to_meters"
- - "surf_spot_finder.tools.get_area_lat_lon"
- - "surf_spot_finder.tools.get_surfing_spots"
- - "surf_spot_finder.tools.get_wave_forecast"
- - "surf_spot_finder.tools.get_wind_forecast"
- - "surf_spot_finder.tools.search_web"
- - "surf_spot_finder.tools.show_plan"
- - "surf_spot_finder.tools.visit_webpage"
-# input_prompt_template:
+input:
+  location: Pontevedra
+  date: 2025-03-26 12:00
+  max_driving_hours: 2
+  # input_prompt_template:
+agent:
+  model_id: o3-mini
+  agent_type: openai
+  tools:
+  - "surf_spot_finder.tools.driving_hours_to_meters"
+  - "surf_spot_finder.tools.get_area_lat_lon"
+  - "surf_spot_finder.tools.get_surfing_spots"
+  - "surf_spot_finder.tools.get_wave_forecast"
+  - "surf_spot_finder.tools.get_wind_forecast"
+  - "surf_spot_finder.tools.search_web"
+  - "surf_spot_finder.tools.show_plan"
+  - "surf_spot_finder.tools.visit_webpage"

examples/smolagents_single_agent.yaml

@@ -1,7 +1,9 @@
-location: Pontevedra
-date: 2025-03-22 12:00
-max_driving_hours: 2
-model_id: openai/o3-mini
-api_key_var: OPENAI_API_KEY
-agent_type: smolagents
-# input_prompt_template:
+input:
+  location: Pontevedra
+  date: 2025-03-27 12:00
+  max_driving_hours: 2
+  # input_prompt_template:
+agent:
+  model_id: openai/o3-mini
+  api_key_var: OPENAI_API_KEY
+  agent_type: smolagents

examples/smolagents_single_agent_mcp.yaml

@@ -1,11 +1,12 @@
-location: Pontevedra
-date: 2025-03-22 12:00
-max_driving_hours: 2
-model_id: openai/gpt-3.5-turbo
-api_key_var: OPENAI_API_KEY
-agent_type: smolagents
-tools:
-  - "smolagents.DuckDuckGoSearchTool"
-  - "mcp/fetch"
-
-# input_prompt_template:
+input:
+  location: Pontevedra
+  date: 2025-03-27 12:00
+  max_driving_hours: 2
+  # input_prompt_template:
+agent:
+  model_id: openai/gpt-3.5-turbo
+  api_key_var: OPENAI_API_KEY
+  agent_type: smolagents
+  tools:
+    - "smolagents.DuckDuckGoSearchTool"
+    - "mcp/fetch"

examples/smolagents_single_agent_vertical.yaml

@@ -1,17 +1,19 @@
-location: Pontevedra
-date: 2025-03-22 12:00
-max_driving_hours: 2
-model_id: openai/o3-mini
-api_key_var: OPENAI_API_KEY
-agent_type: smolagents
-tools:
- - "surf_spot_finder.tools.driving_hours_to_meters"
- - "surf_spot_finder.tools.get_area_lat_lon"
- - "surf_spot_finder.tools.get_surfing_spots"
- - "surf_spot_finder.tools.get_wave_forecast"
- - "surf_spot_finder.tools.get_wind_forecast"
- - "surf_spot_finder.tools.search_web"
- - "surf_spot_finder.tools.visit_webpage"
- - "smolagents.PythonInterpreterTool"
- - "smolagents.FinalAnswerTool"
-# input_prompt_template:
+input:
+  location: Pontevedra
+  date: 2025-03-27 12:00
+  max_driving_hours: 2
+  # input_prompt_template:
+agent:
+  model_id: openai/o1
+  api_key_var: OPENAI_API_KEY
+  agent_type: smolagents
+  tools:
+  - "surf_spot_finder.tools.driving_hours_to_meters"
+  - "surf_spot_finder.tools.get_area_lat_lon"
+  - "surf_spot_finder.tools.get_surfing_spots"
+  - "surf_spot_finder.tools.get_wave_forecast"
+  - "surf_spot_finder.tools.get_wind_forecast"
+  - "surf_spot_finder.tools.search_web"
+  - "surf_spot_finder.tools.visit_webpage"
+  - "smolagents.PythonInterpreterTool"
+  - "smolagents.FinalAnswerTool"

src/surf_spot_finder/cli.py

@@ -1,7 +1,5 @@
-from pathlib import Path
 from typing import Optional
 
-import yaml
 from fire import Fire
 from loguru import logger
 
@@ -55,7 +53,7 @@ def find_surf_spot(
     """
     if from_config:
         logger.info(f"Loading {from_config}")
-        config = Config.model_validate(yaml.safe_load(Path(from_config).read_text()))
+        config = Config.from_yaml(from_config)
     else:
         config = Config(
             location=location,

src/surf_spot_finder/config.py

@@ -1,5 +1,6 @@
 from typing import Annotated, Optional
 from pydantic import AfterValidator, BaseModel, FutureDatetime, PositiveInt
+import yaml
 
 from surf_spot_finder.prompts.shared import INPUT_PROMPT
 
@@ -31,3 +32,30 @@ class Config(BaseModel):
     json_tracer: bool = True
     api_base: Optional[str] = None
     tools: Optional[list[str]] = None
+
+    @classmethod
+    def from_yaml(cls, yaml_path: str) -> "Config":
+        """
+        Create a Config instance from a YAML file.
+
+        Args:
+            yaml_path: Path to the YAML configuration file
+
+        Returns:
+            Config: A new Config instance populated with values from the YAML file
+        """
+        with open(yaml_path, "r") as f:
+            data = yaml.safe_load(f)
+
+        # Extract and flatten the nested structure
+        config_dict = {}
+
+        # Add input parameters
+        if "input" in data:
+            config_dict.update(data["input"])
+
+        # Add agent parameters
+        if "agent" in data:
+            config_dict.update(data["agent"])
+        # Create instance from the flattened dictionary
+        return cls(**config_dict)

src/surf_spot_finder/evaluation/evaluate.py

@@ -24,18 +24,19 @@ logger.add(sys.stdout, colorize=True, format="{message}")
 
 def run_agent(test_case: TestCase) -> str:
     input_data = test_case.input
+    agent_config = test_case.agent
     logger.info("Loading config")
     config = Config(
         location=input_data.location,
         date=input_data.date,
         max_driving_hours=input_data.max_driving_hours,
-        model_id=input_data.model_id,
-        api_key_var=input_data.api_key_var,
+        model_id=agent_config.model_id,
+        api_key_var=agent_config.api_key_var,
         prompt=INPUT_PROMPT,
         json_tracer=input_data.json_tracer,
-        api_base=input_data.api_base,
-        agent_type=input_data.agent_type,
-        tools=input_data.tools,
+        api_base=agent_config.api_base,
+        agent_type=agent_config.agent_type,
+        tools=agent_config.tools,
     )
     return find_surf_spot(
         location=config.location,

src/surf_spot_finder/evaluation/test_case.py

@@ -10,9 +10,12 @@ class InputModel(BaseModel):
     location: str
     date: str
     max_driving_hours: int
+    json_tracer: bool
+
+
+class AgentModel(BaseModel):
     model_id: str
     api_key_var: str
-    json_tracer: bool
     api_base: Optional[str] = None
     agent_type: str
     tools: Optional[List[str]] = None
@@ -29,6 +32,7 @@ class CheckpointCriteria(BaseModel):
 class TestCase(BaseModel):
     model_config = ConfigDict(extra="forbid")
     input: InputModel
+    agent: AgentModel
     ground_truth: List[Dict[str, Any]] = Field(default_factory=list)
     checkpoints: List[CheckpointCriteria] = Field(default_factory=list)
     final_answer_criteria: List[CheckpointCriteria] = Field(default_factory=list)

src/surf_spot_finder/evaluation/test_cases/alpha.yaml

@@ -6,23 +6,12 @@ input:
   location: "Vigo"
   date: "2025-03-27 22:00"
   max_driving_hours: 3
-  api_key_var: "OPENAI_API_KEY"
   json_tracer: true
+agent:
+  api_key_var: "OPENAI_API_KEY"
   api_base: null
-  # model_id: "openai/o1"
-  # agent_type: "smolagents"
-  # tools:
-  # - "surf_spot_finder.tools.driving_hours_to_meters"
-  # - "surf_spot_finder.tools.get_area_lat_lon"
-  # - "surf_spot_finder.tools.get_surfing_spots"
-  # - "surf_spot_finder.tools.get_wave_forecast"
-  # - "surf_spot_finder.tools.get_wind_forecast"
-  # - "surf_spot_finder.tools.search_web"
-  # - "surf_spot_finder.tools.visit_webpage"
-  # - "smolagents.PythonInterpreterTool"
-  # - "smolagents.FinalAnswerTool"
-  agent_type: langchain
-  model_id: o1
+  model_id: "openai/o1"
+  agent_type: "smolagents"
   tools:
   - "surf_spot_finder.tools.driving_hours_to_meters"
   - "surf_spot_finder.tools.get_area_lat_lon"
@@ -31,17 +20,10 @@ input:
   - "surf_spot_finder.tools.get_wind_forecast"
   - "surf_spot_finder.tools.search_web"
   - "surf_spot_finder.tools.visit_webpage"
-  # model_id: o3-mini
-  # agent_type: openai
-  # tools:
-  # - "surf_spot_finder.tools.driving_hours_to_meters"
-  # - "surf_spot_finder.tools.get_area_lat_lon"
-  # - "surf_spot_finder.tools.get_surfing_spots"
-  # - "surf_spot_finder.tools.get_wave_forecast"
-  # - "surf_spot_finder.tools.get_wind_forecast"
-  # - "surf_spot_finder.tools.search_web"
-  # - "surf_spot_finder.tools.show_plan"
-  # - "surf_spot_finder.tools.visit_webpage"
+  - "smolagents.PythonInterpreterTool"
+  - "smolagents.FinalAnswerTool"
+
+
 ground_truth:
   - name: "Surf location"
     points: 5