Added MLAgentBench

app.py

@@ -343,6 +343,126 @@ with gr.Blocks() as demo:
                                     inputs=[raw_agent_dropdown, raw_task_dropdown, raw_step_dropdown], 
                                     outputs=[raw_call_details])
         
+        
+        with gr.Tab("SWE-Bench Verified"):
+            with gr.Row():
+                with gr.Column(scale=2):
+                    Leaderboard(
+                        value=parse_json_files(os.path.join(abs_path, "evals_live"), 'swebench_verified'),
+                        select_columns=SelectColumns(
+                            default_selection=config.SWEBENCH_ON_LOAD_COLUMNS,
+                            cant_deselect=["Agent Name"],
+                            label="Select Columns to Display:",
+                        ),
+                        search_columns=config.SWEBENCH_SEARCH_COLUMNS,
+                        column_widths={"Agent Name": 40,
+                                       "Accuracy": 20,
+                                       "Total Cost": 20},
+                    )
+            with gr.Row():
+                scatter_plot = gr.Plot(create_scatter_plot(parse_json_files(os.path.join(abs_path, "evals_live"), 'swebench_verified'), "Total Cost", "Accuracy", "Total Cost (in USD)", "Accuracy", ["Agent Name"]))
+            
+            gr.Markdown("# Failure Report")
+            with gr.Row():
+                with gr.Column(scale=1):
+                    failure_report_agent_dropdown = gr.Dropdown(label="Select Agent for Failure Report")
+            with gr.Row():
+                with gr.Column(scale=1):
+                    failure_categories_overview = gr.Markdown()
+            
+                with gr.Column(scale=1):
+                    failure_categories_chart = gr.Plot()
+
+            # Initialize the failure report agent dropdown with all agents
+            demo.load(update_agent_dropdown, 
+                    inputs=[gr.Textbox(value="swebench_verified", visible=False), gr.Textbox(value="Accuracy", visible=False)], 
+                    outputs=[failure_report_agent_dropdown])
+            
+            # Update failure report when agent is selected
+            failure_report_agent_dropdown.change(update_failure_report,
+                                                inputs=[failure_report_agent_dropdown, gr.Textbox(value="swebench_verified", visible=False)],
+                                                outputs=[failure_categories_overview, failure_categories_chart])
+            
+            gr.Markdown("# Agent Monitor")
+            with gr.Row():
+                with gr.Column(scale=1):
+                    agent_dropdown = gr.Dropdown(label="Select Agent")
+                with gr.Column(scale=1):
+                    task_dropdown = gr.Dropdown(label="Select SWE-Bench Task")
+            with gr.Row():
+                task_overview = gr.Markdown()
+            with gr.Row():
+                flow_chart = gr.Plot(label="Task Flow")
+
+            # Initialize the agent dropdown with the best agent
+            demo.load(update_agent_dropdown, inputs=[gr.Textbox(value="swebench_verified", visible=False), gr.Textbox(value="Accuracy", visible=False)], outputs=[agent_dropdown])
+            demo.load(update_task_analysis, inputs=[gr.Textbox(value="swebench_verified", visible=False), agent_dropdown], outputs=[task_overview, flow_chart, task_dropdown, gr.Textbox(visible=False)])
+
+            agent_dropdown.change(update_task_analysis, 
+                                  inputs=[gr.Textbox(value="swebench_verified", visible=False), agent_dropdown],
+                                  outputs=[task_overview, flow_chart, task_dropdown, gr.Textbox(visible=False)])
+            task_dropdown.change(update_task_details,
+                                 inputs=[gr.Textbox(value="swebench_verified", visible=False), agent_dropdown, task_dropdown],
+                                 outputs=[task_overview, flow_chart, gr.Textbox(visible=False)])
+            
+            gr.Markdown("# Raw Predictions")
+            with gr.Row():
+                with gr.Column(scale=1):
+                    raw_agent_dropdown = gr.Dropdown(label="Select Agent")
+                with gr.Column(scale=1):
+                    raw_task_dropdown = gr.Dropdown(label="Select Task")
+                with gr.Column(scale=1):
+                    raw_step_dropdown = gr.Dropdown(label="Select Step")
+            
+            with gr.Row():
+                raw_call_details = gr.HTML()
+            
+            def update_raw_task_dropdown(agent_name):
+                analyzed_traces = get_analyzed_traces(agent_name, "swebench_verified")
+                if not analyzed_traces:
+                    return gr.Dropdown(choices=[], label="Select Task"), gr.Dropdown(choices=[], label="Select Step"), f"No raw predictions data available for agent: {agent_name}."
+                task_ids = list(analyzed_traces.keys())
+                steps = analyzed_traces[task_ids[0]]['steps']
+                return gr.Dropdown(choices=task_ids, label="Select Task", value=task_ids[0]), gr.Dropdown(choices=[(f"Step {i+1}", i) for i in range(len(steps))], label="Select Step", value=0), update_raw_call_details(agent_name, task_ids[0], 0)
+
+            def update_raw_step_dropdown(agent_name, task_id):
+                analyzed_traces = get_analyzed_traces(agent_name, "swebench_verified")
+                if not analyzed_traces or task_id not in analyzed_traces:
+                    return gr.Dropdown(choices=[], label="Select Step", value="No data available.")
+                steps = analyzed_traces[task_id]['steps']
+                return gr.Dropdown(choices=[(f"Step {i+1}", i) for i in range(len(steps))], label="Select Step", value=0), format_call_info(steps[0], 0)
+
+            def update_raw_call_details(agent_name, task_id, step_index):
+                analyzed_traces = get_analyzed_traces(agent_name, "swebench_verified")
+                if not analyzed_traces or task_id not in analyzed_traces:
+                    return "No data available for this selection."
+                steps = analyzed_traces[task_id]['steps']
+                if step_index is None:
+                    return "Invalid step selection."
+                step = steps[step_index]
+                return format_call_info(step, step_index)
+
+            # Initialize the raw agent dropdown with all agents
+            demo.load(update_agent_dropdown, 
+                    inputs=[gr.Textbox(value="swebench_verified", visible=False), gr.Textbox(value="Accuracy", visible=False)], 
+                    outputs=[raw_agent_dropdown])
+            demo.load(update_raw_task_dropdown,
+                    inputs=[raw_agent_dropdown],
+                    outputs=[raw_task_dropdown, raw_step_dropdown])
+            demo.load(update_raw_call_details,
+                    inputs=[raw_agent_dropdown, raw_task_dropdown, raw_step_dropdown],
+                    outputs=[raw_call_details])
+
+            raw_agent_dropdown.change(update_raw_task_dropdown, 
+                                    inputs=[raw_agent_dropdown], 
+                                    outputs=[raw_task_dropdown, raw_step_dropdown, raw_call_details])
+            raw_task_dropdown.change(update_raw_step_dropdown, 
+                                    inputs=[raw_agent_dropdown, raw_task_dropdown], 
+                                    outputs=[raw_step_dropdown, raw_call_details])
+            raw_step_dropdown.change(update_raw_call_details, 
+                                    inputs=[raw_agent_dropdown, raw_task_dropdown, raw_step_dropdown], 
+                                    outputs=[raw_call_details])
+            
         with gr.Tab("SWE-Bench Lite"):
             with gr.Row():
                 with gr.Column(scale=2):
@@ -462,25 +582,25 @@ with gr.Blocks() as demo:
                                     inputs=[raw_agent_dropdown, raw_task_dropdown, raw_step_dropdown], 
                                     outputs=[raw_call_details])
             
-            
-        with gr.Tab("SWE-Bench Verified"):
+        with gr.Tab("MLAgentBench"):
             with gr.Row():
                 with gr.Column(scale=2):
                     Leaderboard(
-                        value=parse_json_files(os.path.join(abs_path, "evals_live"), 'swebench_verified'),
+                        value=parse_json_files(os.path.join(abs_path, "evals_live"), 'mlagentbench'),
                         select_columns=SelectColumns(
-                            default_selection=config.SWEBENCH_ON_LOAD_COLUMNS,
+                            default_selection=config.MLAGENTBENCH_ON_LOAD_COLUMNS,
                             cant_deselect=["Agent Name"],
                             label="Select Columns to Display:",
                         ),
-                        search_columns=config.SWEBENCH_SEARCH_COLUMNS,
+                        search_columns=config.MLAGENTBENCH_SEARCH_COLUMNS,
+                        hide_columns=config.MLAGENTBENCH_HIDE_COLUMNS,
                         column_widths={"Agent Name": 40,
-                                       "Accuracy": 20,
+                                       "Overall Score": 20,
                                        "Total Cost": 20},
                     )
             with gr.Row():
-                scatter_plot = gr.Plot(create_scatter_plot(parse_json_files(os.path.join(abs_path, "evals_live"), 'swebench_verified'), "Total Cost", "Accuracy", "Total Cost (in USD)", "Accuracy", ["Agent Name"]))
-            
+                scatter_plot = gr.Plot(create_scatter_plot(parse_json_files(os.path.join(abs_path, "evals_live"), 'mlagentbench'), "Total Cost", "Overall Score", "Total Cost (in USD)", "Overall Score", ["Agent Name"]))
+
             gr.Markdown("# Failure Report")
             with gr.Row():
                 with gr.Column(scale=1):
@@ -494,12 +614,12 @@ with gr.Blocks() as demo:
 
             # Initialize the failure report agent dropdown with all agents
             demo.load(update_agent_dropdown, 
-                    inputs=[gr.Textbox(value="swebench_verified", visible=False), gr.Textbox(value="Accuracy", visible=False)], 
+                    inputs=[gr.Textbox(value="mlagentbench", visible=False), gr.Textbox(value="Overall Score", visible=False)], 
                     outputs=[failure_report_agent_dropdown])
             
             # Update failure report when agent is selected
             failure_report_agent_dropdown.change(update_failure_report,
-                                                inputs=[failure_report_agent_dropdown, gr.Textbox(value="swebench_verified", visible=False)],
+                                                inputs=[failure_report_agent_dropdown, gr.Textbox(value="mlagentbench", visible=False)],
                                                 outputs=[failure_categories_overview, failure_categories_chart])
             
             gr.Markdown("# Agent Monitor")
@@ -514,16 +634,16 @@ with gr.Blocks() as demo:
                 flow_chart = gr.Plot(label="Task Flow")
 
             # Initialize the agent dropdown with the best agent
-            demo.load(update_agent_dropdown, inputs=[gr.Textbox(value="swebench_verified", visible=False), gr.Textbox(value="Accuracy", visible=False)], outputs=[agent_dropdown])
-            demo.load(update_task_analysis, inputs=[gr.Textbox(value="swebench_verified", visible=False), agent_dropdown], outputs=[task_overview, flow_chart, task_dropdown, gr.Textbox(visible=False)])
+            demo.load(update_agent_dropdown, inputs=[gr.Textbox(value="mlagentbench", visible=False), gr.Textbox(value="Overall Score", visible=False)], outputs=[agent_dropdown])
+            demo.load(update_task_analysis, inputs=[gr.Textbox(value="mlagentbench", visible=False), agent_dropdown], outputs=[task_overview, flow_chart, task_dropdown, gr.Textbox(visible=False)])
 
             agent_dropdown.change(update_task_analysis, 
-                                  inputs=[gr.Textbox(value="swebench_verified", visible=False), agent_dropdown],
+                                  inputs=[gr.Textbox(value="mlagentbench", visible=False), agent_dropdown],
                                   outputs=[task_overview, flow_chart, task_dropdown, gr.Textbox(visible=False)])
             task_dropdown.change(update_task_details,
-                                 inputs=[gr.Textbox(value="swebench_verified", visible=False), agent_dropdown, task_dropdown],
+                                 inputs=[gr.Textbox(value="mlagentbench", visible=False), agent_dropdown, task_dropdown],
                                  outputs=[task_overview, flow_chart, gr.Textbox(visible=False)])
-            
+
             gr.Markdown("# Raw Predictions")
             with gr.Row():
                 with gr.Column(scale=1):
@@ -537,7 +657,7 @@ with gr.Blocks() as demo:
                 raw_call_details = gr.HTML()
             
             def update_raw_task_dropdown(agent_name):
-                analyzed_traces = get_analyzed_traces(agent_name, "swebench_verified")
+                analyzed_traces = get_analyzed_traces(agent_name, "mlagentbench")
                 if not analyzed_traces:
                     return gr.Dropdown(choices=[], label="Select Task"), gr.Dropdown(choices=[], label="Select Step"), f"No raw predictions data available for agent: {agent_name}."
                 task_ids = list(analyzed_traces.keys())
@@ -545,14 +665,14 @@ with gr.Blocks() as demo:
                 return gr.Dropdown(choices=task_ids, label="Select Task", value=task_ids[0]), gr.Dropdown(choices=[(f"Step {i+1}", i) for i in range(len(steps))], label="Select Step", value=0), update_raw_call_details(agent_name, task_ids[0], 0)
 
             def update_raw_step_dropdown(agent_name, task_id):
-                analyzed_traces = get_analyzed_traces(agent_name, "swebench_verified")
+                analyzed_traces = get_analyzed_traces(agent_name, "mlagentbench")
                 if not analyzed_traces or task_id not in analyzed_traces:
                     return gr.Dropdown(choices=[], label="Select Step", value="No data available.")
                 steps = analyzed_traces[task_id]['steps']
                 return gr.Dropdown(choices=[(f"Step {i+1}", i) for i in range(len(steps))], label="Select Step", value=0), format_call_info(steps[0], 0)
 
             def update_raw_call_details(agent_name, task_id, step_index):
-                analyzed_traces = get_analyzed_traces(agent_name, "swebench_verified")
+                analyzed_traces = get_analyzed_traces(agent_name, "mlagentbench")
                 if not analyzed_traces or task_id not in analyzed_traces:
                     return "No data available for this selection."
                 steps = analyzed_traces[task_id]['steps']
@@ -563,7 +683,7 @@ with gr.Blocks() as demo:
 
             # Initialize the raw agent dropdown with all agents
             demo.load(update_agent_dropdown, 
-                    inputs=[gr.Textbox(value="swebench_verified", visible=False), gr.Textbox(value="Accuracy", visible=False)], 
+                    inputs=[gr.Textbox(value="mlagentbench", visible=False), gr.Textbox(value="Overall Score", visible=False)], 
                     outputs=[raw_agent_dropdown])
             demo.load(update_raw_task_dropdown,
                     inputs=[raw_agent_dropdown],
@@ -581,6 +701,7 @@ with gr.Blocks() as demo:
             raw_step_dropdown.change(update_raw_call_details, 
                                     inputs=[raw_agent_dropdown, raw_task_dropdown, raw_step_dropdown], 
                                     outputs=[raw_call_details])
+            
         
         with gr.Tab("About"):
             gr.Markdown((Path(__file__).parent / "about.md").read_text())

config.py

@@ -11,7 +11,7 @@ SWEBENCH_ON_LOAD_COLUMNS = [
     "Accuracy",
     "Total Cost",
    ]
-SWEBENCH_SEARCH_COLUMNS = ['Total Cost']
+SWEBENCH_SEARCH_COLUMNS = ['Total Cost', 'Agent Name']
 SWEBENCH_HIDE_COLUMNS = ["F1 Score", "AUC", "Precision", "Recall", "benchmark_name"]
 
 USACO_ON_LOAD_COLUMNS = [
@@ -19,9 +19,17 @@ USACO_ON_LOAD_COLUMNS = [
     "Accuracy",
     "Total Cost",
    ]
-USACO_SEARCH_COLUMNS = ['Total Cost']
+USACO_SEARCH_COLUMNS = ['Total Cost', 'Agent Name']
 USACO_HIDE_COLUMNS = ["F1 Score", "AUC", "Precision", "Recall", "benchmark_name"]
 
+MLAGENTBENCH_ON_LOAD_COLUMNS = [
+    "Agent Name",
+    "Overall Score",
+    "Total Cost",
+   ]
+MLAGENTBENCH_SEARCH_COLUMNS = ['Total Cost', 'Agent Name']
+MLAGENTBENCH_HIDE_COLUMNS = ["F1 Score", "AUC", "Precision", "Recall", "benchmark_name", 'Accuracy']
+
 
 NUMERIC_INTERVALS = {
     "?": pd.Interval(-1, 0, closed="right"),

utils/db.py

@@ -46,6 +46,15 @@ class TracePreprocessor:
                     recall REAL,
                     f1_score REAL,
                     auc REAL,
+                    overall_score REAL,
+                    vectorization_score REAL,
+                    fathomnet_score REAL,
+                    feedback_score REAL,
+                    house_price_score REAL,
+                    spaceship_titanic_score REAL,
+                    amp_parkinsons_disease_progression_prediction_score REAL,
+                    cifar10_score REAL,
+                    imdb_score REAL,
                     PRIMARY KEY (benchmark_name, agent_name)
                 )
             ''')
@@ -86,8 +95,8 @@ class TracePreprocessor:
                 with self.get_conn() as conn:
                     conn.execute('''
                         INSERT OR REPLACE INTO parsed_results 
-                        (benchmark_name, agent_name, date, total_cost, accuracy, precision, recall, f1_score, auc) 
-                        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+                        (benchmark_name, agent_name, date, total_cost, accuracy, precision, recall, f1_score, auc, overall_score, vectorization_score, fathomnet_score, feedback_score, house_price_score, spaceship_titanic_score, amp_parkinsons_disease_progression_prediction_score, cifar10_score, imdb_score) 
+                        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                     ''', (
                         benchmark_name,
                         agent_name,
@@ -97,7 +106,16 @@ class TracePreprocessor:
                         results.get('precision'),
                         results.get('recall'),
                         results.get('f1_score'),
-                        results.get('auc')
+                        results.get('auc'),
+                        results.get('overall_score'),
+                        results.get('vectorization_score'),
+                        results.get('fathomnet_score'),
+                        results.get('feedback_score'),
+                        results.get('house-price_score'),
+                        results.get('spaceship-titanic_score'),
+                        results.get('amp-parkinsons-disease-progression-prediction_score'),
+                        results.get('cifar10_score'),
+                        results.get('imdb_score')
                     ))
             except Exception as e:
                 print(f"Error preprocessing parsed results in {file}: {e}")
@@ -138,7 +156,7 @@ class TracePreprocessor:
             df = pd.read_sql_query(query, conn, params=(benchmark_name,))
 
         # Round float columns to 3 decimal places
-        float_columns = ['total_cost', 'accuracy', 'precision', 'recall', 'f1_score', 'auc']
+        float_columns = ['total_cost', 'accuracy', 'precision', 'recall', 'f1_score', 'auc', 'overall_score', 'vectorization_score', 'fathomnet_score', 'feedback_score', 'house-price_score', 'spaceship-titanic_score', 'amp-parkinsons-disease-progression-prediction_score', 'cifar10_score', 'imdb_score']
         for column in float_columns:
             if column in df.columns:
                 df[column] = df[column].round(3)
@@ -153,6 +171,15 @@ class TracePreprocessor:
             'recall': 'Recall',
             'f1_score': 'F1 Score',
             'auc': 'AUC',
+            'overall_score': 'Overall Score',
+            'vectorization_score': 'Vectorization Score',
+            'fathomnet_score': 'Fathomnet Score',
+            'feedback_score': 'Feedback Score',
+            'house_price_score': 'House Price Score',
+            'spaceship_titanic_score': 'Spaceship Titanic Score',
+            'amp_parkinsons_disease_progression_prediction_score': 'AMP Parkinsons Disease Progression Prediction Score',
+            'cifar10_score': 'CIFAR10 Score',
+            'imdb_score': 'IMDB Score'
         })
 
         return df