added failure report and two new swebench variants

app.py

@@ -7,7 +7,7 @@ import pandas as pd
 import os
 import json
 from utils.data import parse_json_files
-from utils.viz import create_scatter_plot, create_flow_chart
+from utils.viz import create_scatter_plot, create_flow_chart, create_bar_chart
 from utils.processing import check_and_process_uploads
 from huggingface_hub import snapshot_download
 from apscheduler.schedulers.background import BackgroundScheduler
@@ -21,7 +21,6 @@ import weave
 
 
 from datetime import datetime
-weave.init(f'leaderboard_testing_{datetime.now().strftime("%Y%m%d%H%M%S")}')
 
 abs_path = Path(__file__).parent
 
@@ -43,29 +42,46 @@ def download_latest_results():
 
 # Global variable to store preprocessed data
 preprocessed_traces = {}
+failure_reports = {}
 def preprocess_traces():
     global preprocessed_traces
+    global failure_reports
     processed_dir = Path("evals_live")
     for file in processed_dir.glob('*.json'):
-        try:
-            with open(file, 'r') as f:
+        with open(file, 'r') as f:
                 data = json.load(f)
                 agent_name = data['config']['agent_name']
                 benchmark_name = data['config']['benchmark_name']
                 if benchmark_name not in preprocessed_traces:
                     preprocessed_traces[benchmark_name] = {}
+                if benchmark_name not in failure_reports:
+                    failure_reports[benchmark_name] = {}
 
-                assert type(data['raw_logging_results']) == dict, f"Invalid format for raw_logging_results: {type(data['raw_logging_results'])}"
-                preprocessed_traces[benchmark_name][agent_name] = data['raw_logging_results']
+        try:    
+            assert type(data['raw_logging_results']) == dict, f"Invalid format for raw_logging_results: {type(data['raw_logging_results'])}"
+            preprocessed_traces[benchmark_name][agent_name] = data['raw_logging_results']
         except AssertionError as e:
             preprocessed_traces[benchmark_name][agent_name] = None
         except Exception as e:
             print(f"Error preprocessing {file}: {e}")
             preprocessed_traces[benchmark_name][agent_name] = None
 
+        try:
+            assert type(data['failure_report']) == dict, f"Invalid format for failure_report: {type(data['failure_report'])}"
+            failure_reports[benchmark_name][agent_name] = data['failure_report']
+        except AssertionError as e:
+            failure_reports[benchmark_name][agent_name] = None
+        except Exception as e:
+            print(f"Error preprocessing {file}: {e}")
+            failure_reports[benchmark_name][agent_name] = None
+
+
 def get_analyzed_traces(agent_name, benchmark_name):
     return preprocessed_traces.get(benchmark_name, {}).get(agent_name)
 
+def get_failure_report(agent_name, benchmark_name):
+    return failure_reports.get(benchmark_name, {}).get(agent_name)
+
 def update_agent_dropdown(benchmark_name, metric):
     df = parse_json_files(os.path.join(abs_path, "evals_live"), benchmark_name)
     agents = df['Agent Name'].tolist()
@@ -200,6 +216,33 @@ def format_call_info(step, step_index):
     return formatted_info
 
 
+def update_failure_report(agent_name):
+    failure_report = get_failure_report(agent_name, "swebench_lite")
+    if not failure_report:
+        return "No failure report available for this agent.", None
+
+    # Create overview of failure categories
+    categories_overview = "## Failure Categories Overview\n\n"
+    for category in failure_report['failure_categories']:
+        categories_overview += f"### {category['category_name']}\n"
+        categories_overview += f"{category['description']}\n\n"
+
+    # Count tasks affected by each category
+    category_counts = {}
+    for task, classification in failure_report['task_classifications'].items():
+        category_id = classification['category_id']
+        category_counts[category_id] = category_counts.get(category_id, 0) + 1
+
+    # Prepare data for bar chart
+    categories = [cat['category_name'] for cat in failure_report['failure_categories']]
+    counts = [category_counts.get(str(i+1), 0) for i in range(len(categories))]
+
+    # Create bar chart
+    chart = create_bar_chart(categories, counts, "Failure Categories", "Number of Affected Tasks", "Failure Categories Distribution")
+
+    return categories_overview, chart
+
+
 with gr.Blocks() as demo:
     gr.Markdown("""
     # 🥇 Agent Leaderboard
@@ -223,7 +266,7 @@ with gr.Blocks() as demo:
                     )
             with gr.Row():
                 scatter_plot = gr.Plot(create_scatter_plot(parse_json_files(os.path.join(abs_path, "evals_live"), 'usaco'), "Total Cost", "Accuracy", "Total Cost (in USD)", "Accuracy", ["Agent Name"]))
-            gr.Markdown("## Agent Monitor")
+            gr.Markdown("# Agent Monitor")
             with gr.Row():
                 with gr.Column(scale=1):
                     agent_dropdown = gr.Dropdown(label="Select Agent")
@@ -245,7 +288,7 @@ with gr.Blocks() as demo:
                                  inputs=[gr.Textbox(value="usaco", visible=False), agent_dropdown, task_dropdown],
                                  outputs=[task_overview, flow_chart, gr.Textbox(visible=False)])
             
-            gr.Markdown("## Raw Predictions")
+            gr.Markdown("# Raw Predictions")
             with gr.Row():
                 with gr.Column(scale=1):
                     raw_agent_dropdown = gr.Dropdown(label="Select Agent")
@@ -303,23 +346,200 @@ with gr.Blocks() as demo:
                                     inputs=[raw_agent_dropdown, raw_task_dropdown, raw_step_dropdown], 
                                     outputs=[raw_call_details])
         
-        with gr.Tab("SWE-Bench"):
+        with gr.Tab("SWE-Bench Lite"):
             with gr.Row():
                 with gr.Column(scale=2):
                     Leaderboard(
                         value=parse_json_files(os.path.join(abs_path, "evals_live"), 'swebench_lite'),
                         select_columns=SelectColumns(
-                            default_selection=config.USACO_ON_LOAD_COLUMNS,
+                            default_selection=config.SWEBENCH_ON_LOAD_COLUMNS,
                             cant_deselect=["Agent Name"],
                             label="Select Columns to Display:",
                         ),
-                        search_columns=config.USACO_SEARCH_COLUMNS,
+                        search_columns=config.SWEBENCH_SEARCH_COLUMNS,
                         column_widths={"Agent Name": 40,
                                        "Accuracy": 20,
                                        "Total Cost": 20},
                     )
             with gr.Row():
                 scatter_plot = gr.Plot(create_scatter_plot(parse_json_files(os.path.join(abs_path, "evals_live"), 'swebench_lite'), "Total Cost", "Accuracy", "Total Cost (in USD)", "Accuracy", ["Agent Name"]))
+
+            gr.Markdown("# Failure Report")
+            with gr.Row():
+                with gr.Column(scale=1):
+                    failure_report_agent_dropdown = gr.Dropdown(label="Select Agent for Failure Report")
+            with gr.Row():
+                with gr.Column(scale=1):
+                    failure_categories_overview = gr.Markdown()
+            
+                with gr.Column(scale=1):
+                    failure_categories_chart = gr.Plot()
+
+            # Initialize the failure report agent dropdown with all agents
+            demo.load(update_agent_dropdown, 
+                    inputs=[gr.Textbox(value="swebench_lite", visible=False), gr.Textbox(value="Accuracy", visible=False)], 
+                    outputs=[failure_report_agent_dropdown])
+            
+            # Update failure report when agent is selected
+            failure_report_agent_dropdown.change(update_failure_report,
+                                                inputs=[failure_report_agent_dropdown],
+                                                outputs=[failure_categories_overview, failure_categories_chart])
+            
+            gr.Markdown("# Raw Predictions")
+            with gr.Row():
+                with gr.Column(scale=1):
+                    raw_agent_dropdown = gr.Dropdown(label="Select Agent")
+                with gr.Column(scale=1):
+                    raw_task_dropdown = gr.Dropdown(label="Select Task")
+                with gr.Column(scale=1):
+                    raw_step_dropdown = gr.Dropdown(label="Select Step")
+            
+            with gr.Row():
+                raw_call_details = gr.HTML()
+            
+            def update_raw_task_dropdown(agent_name):
+                analyzed_traces = get_analyzed_traces(agent_name, "swebench_lite")
+                if not analyzed_traces:
+                    return gr.Dropdown(choices=[], label="Select Task"), gr.Dropdown(choices=[], label="Select Step"), f"No raw predictions data available for agent: {agent_name}."
+                task_ids = list(analyzed_traces.keys())
+                steps = analyzed_traces[task_ids[0]]['steps']
+                return gr.Dropdown(choices=task_ids, label="Select Task", value=task_ids[0]), gr.Dropdown(choices=[(f"Step {i+1}", i) for i in range(len(steps))], label="Select Step", value=0), update_raw_call_details(agent_name, task_ids[0], 0)
+
+            def update_raw_step_dropdown(agent_name, task_id):
+                analyzed_traces = get_analyzed_traces(agent_name, "swebench_lite")
+                if not analyzed_traces or task_id not in analyzed_traces:
+                    return gr.Dropdown(choices=[], label="Select Step", value="No data available.")
+                steps = analyzed_traces[task_id]['steps']
+                return gr.Dropdown(choices=[(f"Step {i+1}", i) for i in range(len(steps))], label="Select Step", value=0), format_call_info(steps[0], 0)
+
+            def update_raw_call_details(agent_name, task_id, step_index):
+                analyzed_traces = get_analyzed_traces(agent_name, "swebench_lite")
+                if not analyzed_traces or task_id not in analyzed_traces:
+                    return "No data available for this selection."
+                steps = analyzed_traces[task_id]['steps']
+                if step_index is None:
+                    return "Invalid step selection."
+                step = steps[step_index]
+                return format_call_info(step, step_index)
+
+            # Initialize the raw agent dropdown with all agents
+            demo.load(update_agent_dropdown, 
+                    inputs=[gr.Textbox(value="swebench_lite", visible=False), gr.Textbox(value="Accuracy", visible=False)], 
+                    outputs=[raw_agent_dropdown])
+            demo.load(update_raw_task_dropdown,
+                    inputs=[raw_agent_dropdown],
+                    outputs=[raw_task_dropdown, raw_step_dropdown])
+            demo.load(update_raw_call_details,
+                    inputs=[raw_agent_dropdown, raw_task_dropdown, raw_step_dropdown],
+                    outputs=[raw_call_details])
+
+            raw_agent_dropdown.change(update_raw_task_dropdown, 
+                                    inputs=[raw_agent_dropdown], 
+                                    outputs=[raw_task_dropdown, raw_step_dropdown, raw_call_details])
+            raw_task_dropdown.change(update_raw_step_dropdown, 
+                                    inputs=[raw_agent_dropdown, raw_task_dropdown], 
+                                    outputs=[raw_step_dropdown, raw_call_details])
+            raw_step_dropdown.change(update_raw_call_details, 
+                                    inputs=[raw_agent_dropdown, raw_task_dropdown, raw_step_dropdown], 
+                                    outputs=[raw_call_details])
+            
+            
+        with gr.Tab("SWE-Bench Verified"):
+            with gr.Row():
+                with gr.Column(scale=2):
+                    Leaderboard(
+                        value=parse_json_files(os.path.join(abs_path, "evals_live"), 'swebench_verified'),
+                        select_columns=SelectColumns(
+                            default_selection=config.SWEBENCH_ON_LOAD_COLUMNS,
+                            cant_deselect=["Agent Name"],
+                            label="Select Columns to Display:",
+                        ),
+                        search_columns=config.SWEBENCH_SEARCH_COLUMNS,
+                        column_widths={"Agent Name": 40,
+                                       "Accuracy": 20,
+                                       "Total Cost": 20},
+                    )
+            with gr.Row():
+                scatter_plot = gr.Plot(create_scatter_plot(parse_json_files(os.path.join(abs_path, "evals_live"), 'swebench_verified'), "Total Cost", "Accuracy", "Total Cost (in USD)", "Accuracy", ["Agent Name"]))
+            
+            gr.Markdown("# Failure Report")
+            with gr.Row():
+                with gr.Column(scale=1):
+                    failure_report_agent_dropdown = gr.Dropdown(label="Select Agent for Failure Report")
+            with gr.Row():
+                with gr.Column(scale=1):
+                    failure_categories_overview = gr.Markdown()
+            
+                with gr.Column(scale=1):
+                    failure_categories_chart = gr.Plot()
+
+            # Initialize the failure report agent dropdown with all agents
+            demo.load(update_agent_dropdown, 
+                    inputs=[gr.Textbox(value="swebench_verified", visible=False), gr.Textbox(value="Accuracy", visible=False)], 
+                    outputs=[failure_report_agent_dropdown])
+            
+            # Update failure report when agent is selected
+            failure_report_agent_dropdown.change(update_failure_report,
+                                                inputs=[failure_report_agent_dropdown],
+                                                outputs=[failure_categories_overview, failure_categories_chart])
+            
+            gr.Markdown("# Raw Predictions")
+            with gr.Row():
+                with gr.Column(scale=1):
+                    raw_agent_dropdown = gr.Dropdown(label="Select Agent")
+                with gr.Column(scale=1):
+                    raw_task_dropdown = gr.Dropdown(label="Select Task")
+                with gr.Column(scale=1):
+                    raw_step_dropdown = gr.Dropdown(label="Select Step")
+            
+            with gr.Row():
+                raw_call_details = gr.HTML()
+            
+            def update_raw_task_dropdown(agent_name):
+                analyzed_traces = get_analyzed_traces(agent_name, "swebench_verified")
+                if not analyzed_traces:
+                    return gr.Dropdown(choices=[], label="Select Task"), gr.Dropdown(choices=[], label="Select Step"), f"No raw predictions data available for agent: {agent_name}."
+                task_ids = list(analyzed_traces.keys())
+                steps = analyzed_traces[task_ids[0]]['steps']
+                return gr.Dropdown(choices=task_ids, label="Select Task", value=task_ids[0]), gr.Dropdown(choices=[(f"Step {i+1}", i) for i in range(len(steps))], label="Select Step", value=0), update_raw_call_details(agent_name, task_ids[0], 0)
+
+            def update_raw_step_dropdown(agent_name, task_id):
+                analyzed_traces = get_analyzed_traces(agent_name, "swebench_verified")
+                if not analyzed_traces or task_id not in analyzed_traces:
+                    return gr.Dropdown(choices=[], label="Select Step", value="No data available.")
+                steps = analyzed_traces[task_id]['steps']
+                return gr.Dropdown(choices=[(f"Step {i+1}", i) for i in range(len(steps))], label="Select Step", value=0), format_call_info(steps[0], 0)
+
+            def update_raw_call_details(agent_name, task_id, step_index):
+                analyzed_traces = get_analyzed_traces(agent_name, "swebench_verified")
+                if not analyzed_traces or task_id not in analyzed_traces:
+                    return "No data available for this selection."
+                steps = analyzed_traces[task_id]['steps']
+                if step_index is None:
+                    return "Invalid step selection."
+                step = steps[step_index]
+                return format_call_info(step, step_index)
+
+            # Initialize the raw agent dropdown with all agents
+            demo.load(update_agent_dropdown, 
+                    inputs=[gr.Textbox(value="swebench_verified", visible=False), gr.Textbox(value="Accuracy", visible=False)], 
+                    outputs=[raw_agent_dropdown])
+            demo.load(update_raw_task_dropdown,
+                    inputs=[raw_agent_dropdown],
+                    outputs=[raw_task_dropdown, raw_step_dropdown])
+            demo.load(update_raw_call_details,
+                    inputs=[raw_agent_dropdown, raw_task_dropdown, raw_step_dropdown],
+                    outputs=[raw_call_details])
+
+            raw_agent_dropdown.change(update_raw_task_dropdown, 
+                                    inputs=[raw_agent_dropdown], 
+                                    outputs=[raw_task_dropdown, raw_step_dropdown, raw_call_details])
+            raw_task_dropdown.change(update_raw_step_dropdown, 
+                                    inputs=[raw_agent_dropdown, raw_task_dropdown], 
+                                    outputs=[raw_step_dropdown, raw_call_details])
+            raw_step_dropdown.change(update_raw_call_details, 
+                                    inputs=[raw_agent_dropdown, raw_task_dropdown, raw_step_dropdown], 
+                                    outputs=[raw_call_details])
         
         with gr.Tab("About"):
             gr.Markdown((Path(__file__).parent / "about.md").read_text())
@@ -332,8 +552,8 @@ async def main():
     # Preprocess traces
     preprocess_traces()
     
-    # Download the results from the Hugging Face Hub
-    await asyncio.to_thread(download_latest_results)
+    # # Download the results from the Hugging Face Hub
+    # await asyncio.to_thread(download_latest_results)
 
     # Check for new uploads and process them
     await check_and_process_uploads()
@@ -347,4 +567,5 @@ async def main():
     await demo.launch()
 
 if __name__ == "__main__":
+    weave.init(f'leaderboard_testing_{datetime.now().strftime("%Y%m%d%H%M%S")}')
     asyncio.run(main())

evals_live/swebench_lite_example_agent_1722587866.json

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a6751248329f37cf663f523759211383585062cc698d613b648e948293783f3c
-size 8444

evals_live/swebench_lite_example_agent_17227906123.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f2c176cd5a6d6fcb0fdf83b42a919391797665c2cc9226d14ffded5586033ef9
-size 10381
+oid sha256:2ccf570b28e70b4ce6beff1fccf70eaf7ea0ff52730c338ff9a721f95bfa2131
+size 21960

evals_live/swebench_lite_example_agent_1722790656.json

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:571a7158c57485e184e195214addc25e8c58a0b8191a1d280f323a24fd81eb54
-size 10381

utils/processing.py

@@ -7,7 +7,9 @@ from email.mime.multipart import MIMEMultipart
 import asyncio
 import aiofiles
 import aiosmtplib
-from agent_monitor.monitor import analyze_agent_steps, AsyncOpenAIClient
+from agent_monitor.monitor import analyze_agent_steps
+from agent_monitor.failure_report import analyze_agent_performance, AsyncOpenAIClient
+import traceback
 
 async def check_and_process_uploads():
     upload_dir =  "evals_upload"
@@ -31,11 +33,11 @@ async def check_and_process_uploads():
         if not os.path.exists(live_path) and not os.path.exists(processed_path):
             unprocessed_uploads.append(upload)
         elif os.path.exists(processed_path):
-            with open(upload_path, 'r') as f:
-                new_data = json.load(f)
+            # with open(upload_path, 'r') as f:
+            #     new_data = json.load(f)
             
-            with open(processed_path, 'r') as f:
-                processed_data = json.load(f)
+            # with open(processed_path, 'r') as f:
+            #     processed_data = json.load(f)
             
             # TODO we can use a better comparison method with exact comparison
             # if new_data != processed_data:
@@ -70,10 +72,10 @@ async def process_single_upload(upload_path, processed_path):
     
     if check_result['is_valid']:
         # Process the file
-        # await process_upload(upload_path, processed_path)
+        await process_upload(upload_path, processed_path)
         
         # Move the file to processed directory
-        await asyncio.to_thread(shutil.move, upload_path, processed_path)
+        # await asyncio.to_thread(shutil.move, upload_path, processed_path)
         
         # Send email notification
         # await send_email_notification(upload_path.name, check_result, "Processing successful")
@@ -121,14 +123,15 @@ async def process_upload(input_path, output_path):
     openai_client = AsyncOpenAIClient(model="gpt-4o-mini")
 
     try:
-        processed_calls = await analyze_agent_steps(data['raw_logging_results'], openai_client)
+        processed_calls = await analyze_agent_steps(data['raw_logging_results'], openai_client, llm_eval=False)
+        failure_report = await analyze_agent_performance(data['raw_logging_results'], data['results']['failed_tasks'], openai_client)
+        data['raw_logging_results'] = processed_calls
+        data['failure_report'] = failure_report
     except Exception as e:
+        traceback.print_exc()
         print(f"Error in processing: {str(e)}")
         return
 
-    # Save the processed data
-    data['raw_logging_results'] = processed_calls
-
     with open(output_path, 'w') as f:
         json.dump(data, f, indent=4)
 

utils/viz.py

@@ -4,6 +4,58 @@ from utils.pareto import Agent, compute_pareto_frontier
 import plotly.graph_objects as go
 import textwrap
 
+def create_bar_chart(categories, values, x_label, y_label, title):
+    # Sort categories and values based on values in descending order
+    sorted_data = sorted(zip(categories, values), key=lambda x: x[1], reverse=True)
+    categories, values = zip(*sorted_data)
+
+    fig = go.Figure(data=[go.Bar(
+        y=categories,
+        x=values,
+        orientation='h',
+        marker_color='#1b9e77',  # Same color as the scatter plot
+        text=values,
+        textposition='auto',
+        textfont=dict(color='black', size=14, family='Arial', weight=2),
+        hovertemplate='<b>%{y}</b><br>' +
+                      'Affected Tasks: %{x}<br>'
+    )])
+
+    fig.update_layout(
+        height=600,
+        xaxis=dict(
+            showline=True,
+            linecolor='black',
+            showgrid=False
+        ),
+        yaxis=dict(
+            showline=True,
+            linecolor='black',
+            showgrid=False,
+            autorange="reversed"  # This will put the category with the highest value at the top
+        ),
+        plot_bgcolor='white',
+        paper_bgcolor='white',
+        bargap=0.2,
+        bargroupgap=0.1,
+        hoverlabel=dict(bgcolor="white", font_size=12, font_family="Arial"),
+        modebar=dict(
+            activecolor='#1f77b4',
+            orientation='h',
+            bgcolor='rgba(255,255,255,0.8)',
+            color='#777',
+            add=['pan2d'],
+            remove=[
+                'zoom2d', 'zoomIn2d', 'zoomOut2d', 'resetScale2d',
+                'hoverClosestCartesian', 'hoverCompareCartesian',
+                'toggleSpikelines', 'lasso2d', 'lasso', 'select2d', 'select'
+            ]
+        ),
+        dragmode='pan'
+    )
+
+    return fig
+
 def create_scatter_plot(df, x: str, y: str, x_label: str = None, y_label: str = None, hover_data: list = None):
     agents = [Agent(row['Total Cost'], row['Accuracy']) for i, row in df.iterrows()]
     pareto_frontier = compute_pareto_frontier(agents)