update

src/leaderboard/read_evals.py

@@ -8,7 +8,7 @@ import dateutil
 import numpy as np
 
 from src.display.formatting import make_clickable_model
-from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
+from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType, Quotas
 from src.submission.check_validity import is_model_on_hub
 
 
@@ -82,6 +82,17 @@ class EvalResult:
 
             mean_acc = np.mean(accs) if len(accs) > 0 else 0
             results[task.benchmark] = mean_acc
+ 
+        for task in Quotas:
+            task = task.value
+
+            # We average all scores of a given metric (not all metrics are present in all files)
+            accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
+            if accs.size == 0 or any([acc is None for acc in accs]):
+                continue
+
+            mean_acc = np.mean(accs) if len(accs) > 0 else 0
+            results[task.benchmark] = mean_acc
 
         return self(
             eval_name=result_key,

src/populate.py

@@ -24,6 +24,7 @@ def get_leaderboard_df(results_path: str, requests_path: str, dynamic_path: str,
     print("df", df)
     print("AutoEvalColumn.average.name",AutoEvalColumn.average.name)
     #df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
+    print("cols", cols)
     df = df[cols].round(decimals=2)
 
     # filter out if any of the benchmarks have not been produced