dynamic averaging

app.py

@@ -727,7 +727,8 @@ def process_json(temp_file):
 # Define the preset substrings for filtering
 PRESET_SUBSTRINGS = ["IOI", "MCQA", "Arithmetic", "ARC", "GPT-2", "Qwen-2.5", "Gemma-2", "Llama-3.1"]
 
-def filter_columns_by_substrings(dataframe: pd.DataFrame, selected_substrings: List[str]) -> pd.DataFrame:
+def filter_columns_by_substrings(dataframe: pd.DataFrame, selected_substrings: List[str],
+                                 sort_ascending: bool) -> pd.DataFrame:
     """
     Filter columns based on the selected substrings.
     """
@@ -741,13 +742,30 @@ def filter_columns_by_substrings(dataframe: pd.DataFrame, selected_substrings: L
         if any(sub.lower() in col.lower() for sub in selected_substrings)
         or col == "Method"
     ]
+    
+    def _compute_row_average(row):
+        # If any value is "-", return "-"
+        if any(v == "-" for v in row.values):
+            return 100 if sort_ascending else -100
+        # Convert to numeric, dropping any non-numeric values
+        numeric_values = pd.to_numeric(row, errors='coerce')
+        # Compute mean of non-NA values
+        return numeric_values.mean().round(3)
+
+    dataframe["Average"] = original_dataframe[filtered_columns].apply(_compute_row_average, axis=1)
+    # dataframe["Average"] = dataframe['Average'].mask(dataframe.isna().any(axis=1), '-')
+    filtered_columns.append("Average")
+    dataframe = dataframe.sort_values('Average', ascending=sort_ascending)
+    dataframe["Average"] = dataframe["Average"].replace(-100, "-").replace(100, "-")
+
     return dataframe[filtered_columns]
 
-def update_leaderboard(dataframe: pd.DataFrame, selected_substrings: List[str]):
+def update_leaderboard(dataframe: pd.DataFrame, selected_substrings: List[str], sort_ascending: bool = False):
     """
     Update the leaderboard based on the selected substrings.
     """
-    filtered_dataframe = filter_columns_by_substrings(dataframe, selected_substrings)
+    filtered_dataframe = filter_columns_by_substrings(dataframe, selected_substrings,
+                                                      sort_ascending=sort_ascending)
     return filtered_dataframe
 
 demo = gr.Blocks(css=custom_css)
@@ -812,10 +830,11 @@ with demo:
                     )
                     leaderboard, data = init_leaderboard_mib_subgraph(LEADERBOARD_DF_MIB_SUBGRAPH_FEQ, "Subgraph")
                     original_leaderboard = gr.State(value=data)
+                    boolean_checkbox = gr.Checkbox(value=True, visible=False)  # Default to True
                     # Update the leaderboard when the user selects/deselects substrings
                     substring_checkbox.change(
                         fn=update_leaderboard,
-                        inputs=[original_leaderboard, substring_checkbox],
+                        inputs=[original_leaderboard, substring_checkbox, boolean_checkbox],
                         outputs=leaderboard
                     )
                     print(f"Leaderboard is {leaderboard}")

eval-results-mib-subgraph/submissions/results_2024-10-2T13-36-121.json

@@ -1,19 +0,0 @@
-{"method_name": "EAP-IG (mean)", "results": [
-    {"model_id": "meta-llama/Llama-3.1-8B", "scores": {
-        "ioi": {
-            "edge_counts": [10.0, 29.0, 117.0, 269.0, 561.0, 1570.0, 3194.0, 6386.0, 16245.0, 32491.0],
-            "faithfulness": [0.11454112510535433,0.14123527363014815,0.3197643850972241,0.47765884872924175,0.7701570853704176,1.3201798748760563,2.037825774185549,2.651813181821849,3.27612042118584,1.0]},
-        "mcqa": {
-            "edge_counts": [10.0, 21.0, 94.0, 241.0, 527.0, 1469.0, 3046.0, 6036.0, 14832.0, 32491.0],
-            "faithfulness": [[0.02677059664121319,0.1965060952906922,0.449060470868564,0.7604756153676078,0.786575587658478,1.106011020720112,1.3436645156597262,1.5466349080478032,1.4914126224418107,1.0]]}
-    }},
-    {"model_id": "Qwen/Qwen2-1.5B", "scores": {
-        "ioi": {
-        "edge_counts": [],
-         "faithfulness": []},
-        "mcqa": {
-        "edge_counts": [],
-        "faithfulness": []}
-    }}
-    ]
-}

src/populate.py

@@ -55,12 +55,13 @@ def get_leaderboard_df_mib_subgraph(results_path: str, requests_path: str, cols:
 
     # Convert to dataframe
     df = pd.DataFrame.from_records(all_data_json)
+    ascending = False if metric_type == "F+" else True
     
     # Sort by Average score descending
     if 'Average' in df.columns:
         # Convert '-' to NaN for sorting purposes
         df['Average'] = pd.to_numeric(df['Average'], errors='coerce')
-        df = df.sort_values(by=['Average'], ascending=False, na_position='last')
+        df = df.sort_values(by=['Average'], ascending=ascending, na_position='last')
         # Convert NaN back to '-'
         df['Average'] = df['Average'].fillna('-')