Merge branch 'main' of https://huggingface.co/spaces/mech-interp-bench/leaderboard

app.py

@@ -451,7 +451,8 @@ def process_json(temp_file):
 # Define the preset substrings for filtering
 PRESET_SUBSTRINGS = ["IOI", "MCQA", "Arithmetic", "ARC", "GPT-2", "Qwen-2.5", "Gemma-2", "Llama-3.1"]
 
-def filter_columns_by_substrings(dataframe: pd.DataFrame, selected_substrings: List[str]) -> pd.DataFrame:
+def filter_columns_by_substrings(dataframe: pd.DataFrame, selected_substrings: List[str],
+                                 sort_ascending: bool) -> pd.DataFrame:
     """
     Filter columns based on the selected substrings.
     """
@@ -465,13 +466,30 @@ def filter_columns_by_substrings(dataframe: pd.DataFrame, selected_substrings: L
         if any(sub.lower() in col.lower() for sub in selected_substrings)
         or col == "Method"
     ]
+    
+    def _compute_row_average(row):
+        # If any value is "-", return "-"
+        if any(v == "-" for v in row.values):
+            return 100 if sort_ascending else -100
+        # Convert to numeric, dropping any non-numeric values
+        numeric_values = pd.to_numeric(row, errors='coerce')
+        # Compute mean of non-NA values
+        return numeric_values.mean().round(3)
+
+    dataframe["Average"] = original_dataframe[filtered_columns].apply(_compute_row_average, axis=1)
+    # dataframe["Average"] = dataframe['Average'].mask(dataframe.isna().any(axis=1), '-')
+    filtered_columns.append("Average")
+    dataframe = dataframe.sort_values('Average', ascending=sort_ascending)
+    dataframe["Average"] = dataframe["Average"].replace(-100, "-").replace(100, "-")
+
     return dataframe[filtered_columns]
 
-def update_leaderboard(dataframe: pd.DataFrame, selected_substrings: List[str]):
+def update_leaderboard(dataframe: pd.DataFrame, selected_substrings: List[str], sort_ascending: bool = False):
     """
     Update the leaderboard based on the selected substrings.
     """
-    filtered_dataframe = filter_columns_by_substrings(dataframe, selected_substrings)
+    filtered_dataframe = filter_columns_by_substrings(dataframe, selected_substrings,
+                                                      sort_ascending=sort_ascending)
     return filtered_dataframe
 
 demo = gr.Blocks(css=custom_css)
@@ -497,7 +515,7 @@ with demo:
         
         # with gr.TabItem("Subgraph", elem_id="subgraph", id=0):
         #     leaderboard = init_leaderboard_mib_subgraph(LEADERBOARD_DF_MIB_SUBGRAPH, "Subgraph")
-        with gr.TabItem("Subgraph", elem_id="subgraph", id=0):
+        with gr.TabItem("Circuit Localization", elem_id="subgraph", id=0):
             with gr.Tabs() as subgraph_tabs:
                 with gr.TabItem("F+", id=0):
                     # Add description for filters
@@ -536,16 +554,17 @@ with demo:
                     )
                     leaderboard, data = init_leaderboard_mib_subgraph(LEADERBOARD_DF_MIB_SUBGRAPH_FEQ, "Subgraph")
                     original_leaderboard = gr.State(value=data)
+                    boolean_checkbox = gr.Checkbox(value=True, visible=False)  # Default to True
                     # Update the leaderboard when the user selects/deselects substrings
                     substring_checkbox.change(
                         fn=update_leaderboard,
-                        inputs=[original_leaderboard, substring_checkbox],
+                        inputs=[original_leaderboard, substring_checkbox, boolean_checkbox],
                         outputs=leaderboard
                     )
                     print(f"Leaderboard is {leaderboard}")
     
         # Then modify the Causal Graph tab section
-        with gr.TabItem("Causal Graph", elem_id="causalgraph", id=1):
+        with gr.TabItem("Causal Variable Localization", elem_id="causalgraph", id=1):
             with gr.Tabs() as causalgraph_tabs:
                 with gr.TabItem("Detailed View", id=0):
                     leaderboard_detailed = init_leaderboard_mib_causalgraph(

eval-results-mib-subgraph/baselines/UGS.json

@@ -0,0 +1,93 @@
+{
+    "method_name": "UGS",
+    "results": [
+        {
+            "model_id": "qwen2.5",
+            "scores": {
+                "ioi": {
+                    "edge_counts": [
+                        164.0,
+                        349.0,
+                        888.0,
+                        1766.0,
+                        3575.0,
+                        8977.0,
+                        17961.0,
+                        35949.0,
+                        89874.0,
+                        179749.0
+                    ],
+                    "faithfulness": [
+                        0.8161993769470405,
+                        1.0623052959501558,
+                        1.1557632398753894,
+                        1.1806853582554517,
+                        1.071651090342679,
+                        1.0093457943925233,
+                        0.9875389408099688,
+                        0.9470404984423676,
+                        0.9719626168224299,
+                        1.0
+                    ]
+                },
+                "mcqa": {
+                    "edge_counts": [
+                        86.0,
+                        212.0,
+                        704.0,
+                        1632.0,
+                        3449.0,
+                        8871.0,
+                        17814.0,
+                        35720.0,
+                        89874.0,
+                        179749.0
+                    ],
+                    "faithfulness": [
+                        0.37104430379746833,
+                        0.4506526898734177,
+                        0.6471518987341772,
+                        0.7231012658227848,
+                        0.9113924050632911,
+                        1.0,
+                        1.5917721518987342,
+                        1.7183544303797469,
+                        1.009493670886076,
+                        1.0
+                    ]
+                }
+            }
+        },
+        {
+            "model_id": "gpt2",
+            "scores": {
+                "ioi": {
+                    "edge_counts": [
+                        27.0,
+                        57.0,
+                        159.0,
+                        322.0,
+                        640.0,
+                        1608.0,
+                        3244.0,
+                        6498.0,
+                        16245.0,
+                        32491.0
+                    ],
+                    "faithfulness": [
+                        0.10013020765541497,
+                        0.33153985647745055,
+                        1.0775680479866294,
+                        0.960686341813994,
+                        1.0155814417206641,
+                        1.0182404988203417,
+                        0.9613478605327729,
+                        0.9464708735339975,
+                        0.9555035267362492,
+                        1.0
+                    ]
+                }
+            }
+        }
+    ]
+}

eval-results-mib-subgraph/submissions/results_2024-10-2T13-36-121.json

@@ -1,19 +0,0 @@
-{"method_name": "EAP-IG (mean)", "results": [
-    {"model_id": "meta-llama/Llama-3.1-8B", "scores": {
-        "ioi": {
-            "edge_counts": [10.0, 29.0, 117.0, 269.0, 561.0, 1570.0, 3194.0, 6386.0, 16245.0, 32491.0],
-            "faithfulness": [0.11454112510535433,0.14123527363014815,0.3197643850972241,0.47765884872924175,0.7701570853704176,1.3201798748760563,2.037825774185549,2.651813181821849,3.27612042118584,1.0]},
-        "mcqa": {
-            "edge_counts": [10.0, 21.0, 94.0, 241.0, 527.0, 1469.0, 3046.0, 6036.0, 14832.0, 32491.0],
-            "faithfulness": [[0.02677059664121319,0.1965060952906922,0.449060470868564,0.7604756153676078,0.786575587658478,1.106011020720112,1.3436645156597262,1.5466349080478032,1.4914126224418107,1.0]]}
-    }},
-    {"model_id": "Qwen/Qwen2-1.5B", "scores": {
-        "ioi": {
-        "edge_counts": [],
-         "faithfulness": []},
-        "mcqa": {
-        "edge_counts": [],
-        "faithfulness": []}
-    }}
-    ]
-}

src/populate.py

@@ -47,12 +47,13 @@ def get_leaderboard_df_mib_subgraph(results_path: str, requests_path: str, cols:
 
     # Convert to dataframe
     df = pd.DataFrame.from_records(all_data_json)
+    ascending = False if metric_type == "F+" else True
     
     # Sort by Average score descending
     if 'Average' in df.columns:
         # Convert '-' to NaN for sorting purposes
         df['Average'] = pd.to_numeric(df['Average'], errors='coerce')
-        df = df.sort_values(by=['Average'], ascending=False, na_position='last')
+        df = df.sort_values(by=['Average'], ascending=ascending, na_position='last')
         # Convert NaN back to '-'
         df['Average'] = df['Average'].fillna('-')