populate leaderboard df

app.py

@@ -47,6 +47,8 @@ def restart_space():
 
 
 lbdb = F1Data(cp_ds_name=CODE_PROBLEMS_REPO, sub_ds_name=SUBMISSIONS_REPO, res_ds_name=RESULTS_REPO, split=SPLIT)
+leaderboard_df = get_leaderboard_df(RESULTS_REPO)
+
 
 logger.info("Initialized LBDB")
 
@@ -94,17 +96,6 @@ with demo:
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
         with gr.TabItem("🏅 FormulaOne Leaderboard", elem_id="formulaone-leaderboar-tab-table", id=0):
-            # TODO: activate
-            # leaderboard_df = get_leaderboard_df
-            # dummy df
-            leaderboard_df = pd.DataFrame(
-                {
-                    AutoEvalColumn.system.name: ["Model A", "Model B", "Model C"],  # AutoEvalColumn.model.name
-                    AutoEvalColumn.system_type.name: ["LLM", "LLM+Agent", "N/A"],  # AutoEvalColumn.model_type.name
-                    AutoEvalColumn.organization.name: ["Org A", "Org B", "Org C"],  # AutoEvalColumn.organization.name
-                    AutoEvalColumn.success_rate.name: [0.01, 0.0, 0.005],
-                }
-            )
             leaderboard = init_leaderboard(leaderboard_df)
 
         with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=1):

src/display/utils.py

@@ -57,13 +57,15 @@ class ColumnContent:
 # # We use make dataclass to dynamically fill the scores from Tasks
 # AutoEvalColumn = make_classvar_dataclass("AutoEvalColumn", auto_eval_column_fields)
 
+
 @dataclass(frozen=True)
 class AutoEvalColumn:
     system = ColumnContent("System Name", "markdown", True, never_hidden=True)
     system_type = ColumnContent("System Type", "str", True)
     organization = ColumnContent("Organization", "str", True, never_hidden=True)
     success_rate = ColumnContent("Success Rate (%)", "number", True)
-
+    problems_solved = ColumnContent("Problems Solved", "number", True)
+    submitted_on = ColumnContent("Submitted On", "datetime", True)
 
 
 ## For the queue columns in the submission tab

src/populate.py

@@ -2,24 +2,62 @@ import json
 import os
 
 import pandas as pd
+from datasets import load_dataset, get_dataset_config_names
+from tqdm.auto import tqdm
 
 from src.display.formatting import has_no_nan_values, make_clickable_model
 from src.display.utils import AutoEvalColumn, EvalQueueColumn
+from src.envs import TOKEN
 from src.leaderboard.read_evals import get_raw_eval_results
+from src.logger import get_logger
 
+logger = get_logger(__name__)
 
-def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
+
+def get_leaderboard_df(results_dataset_name: str) -> pd.DataFrame:
     """Creates a dataframe from all the individual experiment results"""
-    raw_data = get_raw_eval_results(results_path, requests_path)
-    all_data_json = [v.to_dict() for v in raw_data]
 
-    df = pd.DataFrame.from_records(all_data_json)
-    df = df.sort_values(by=[AutoEvalColumn.success_rate.name], ascending=False)
-    df = df[cols].round(decimals=2)
+    configs = get_dataset_config_names(results_dataset_name, token=TOKEN)
+
+    rows = []
+    for submission_id in tqdm(configs, total=len(configs), desc="Processing Submission Results"):
+        submission_ds = load_dataset(results_dataset_name, submission_id, split="train", token=TOKEN)
+        submission_df = pd.DataFrame(submission_ds)
+
+        if submission_df.empty or "did_pass" not in submission_df.columns or submission_df.did_pass.isna().any():
+            logger.warning(f"Skipping {submission_id} due to invalid did_pass values")
+            continue
+
+        success_rate = 100 * submission_df["did_pass"].mean()
+        num_solved = submission_df["did_pass"].sum()
+        first_row = submission_df.iloc[0]
+
+        rows.append(
+            {
+                "System Name": first_row["system_name"],
+                "System Type": first_row["system_type"],
+                "Organization": first_row["organization"],
+                "Success Rate (%)": success_rate,
+                "Problems Solved": num_solved,
+                "Submitted On": pd.to_datetime(first_row.get("submission_ts", "1970-01-01T00:00:00")),
+            }
+        )
+
+    full_df = pd.DataFrame(rows)
+
+    # TODO: forbid multiple submissions under the same name?
+    # Keep only the latest entry per unique (System Name, System Type, Organization) triplet
+    final_df = (
+        full_df.sort_values("Submitted On", ascending=False)
+        .drop_duplicates(subset=["System Name", "System Type", "Organization"], keep="first")
+        .sort_values(by=[AutoEvalColumn.success_rate.name], ascending=False)
+        .reset_index(drop=True)
+    )
+
+    cols_to_round = ["Success Rate (%)"]
+    final_df[cols_to_round] = final_df[cols_to_round].round(decimals=2)
 
-    # filter out if any of the benchmarks have not been produced
-    df = df[has_no_nan_values(df, benchmark_cols)]
-    return df
+    return final_df
 
 
 def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]: