feat: fix the to_dict function

src/leaderboard/read_evals.py

@@ -1,4 +1,5 @@
 import glob
+from collections import defaultdict
 import json
 import os.path
 from dataclasses import dataclass
@@ -6,7 +7,7 @@ from typing import List
 
 import dateutil.parser._parser
 
-from src.display.utils import AutoEvalColumn
+from src.display.utils import AutoEvalColumnQA
 from src.benchmarks import get_safe_name
 
 
@@ -61,20 +62,19 @@ class FullEvalResult:
             results=result_list
         )
 
-    def to_dict(self, task='qa', metric='ndcg_at_1'):
+    def to_dict(self, task='qa', metric='ndcg_at_1') -> List:
         """Convert FullEvalResult to a list of dict compatible with our dataframe UI
         """
-        results = []
+        results = defaultdict(dict)
         for eval_result in self.results:
             if eval_result.metric != metric:
                 continue
             if eval_result.task != task:
                 continue
-            data_dict = {
-                "eval_name": eval_result.eval_name,
-                AutoEvalColumn.retrieval_model.name: self.retrieval_model,
-                AutoEvalColumn.reranking_model.name: self.reranking_model,
-            }
+            results[eval_result.eval_name]["eval_name"] = eval_result.eval_name
+            results[eval_result.eval_name][AutoEvalColumnQA.retrieval_model.name] = self.retrieval_model
+            results[eval_result.eval_name][AutoEvalColumnQA.reranking_model.name] = self.reranking_model
+
             for result in eval_result.results:
                 # add result for each domain, language, and dataset
                 domain = result["domain"]
@@ -82,12 +82,11 @@ class FullEvalResult:
                 dataset = result["dataset"]
                 value = result["value"]
                 if task == 'qa':
-                    benchmark_name = f"{task}_{domain}_{lang}"
+                    benchmark_name = f"{domain}_{lang}"
                 elif task == 'long_doc':
-                    benchmark_name = f"{task}_{domain}_{lang}_{dataset}_{metric}"
-                data_dict[get_safe_name(benchmark_name)] = value
-            results.append(data_dict)
-        return results
+                    benchmark_name = f"{domain}_{lang}_{dataset}_{metric}"
+                results[eval_result.eval_name][get_safe_name(benchmark_name)] = value
+        return [v for v in results.values()]
 
     def update_with_request_file(self, request_path):
         """
@@ -148,7 +147,6 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> List[FullEval
     eval_results = {}
     for model_result_filepath in model_result_filepaths:
         # create evaluation results
-        # TODO: fix the bug here, the running results should not be loaded
         eval_result = FullEvalResult.init_from_json_file(model_result_filepath)
         # get the latest result that is finished
         eval_result.update_with_request_file(requests_path)

src/populate.py

@@ -4,7 +4,7 @@ import os
 import pandas as pd
 
 from src.display.formatting import has_no_nan_values, make_clickable_model
-from src.display.utils import AutoEvalColumn, EvalQueueColumn
+from src.display.utils import AutoEvalColumnQA, EvalQueueColumn
 from src.leaderboard.read_evals import get_raw_eval_results, EvalResult
 from typing import Tuple
 
@@ -12,10 +12,13 @@ from typing import Tuple
 def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> Tuple[list[EvalResult], pd.DataFrame]:
     """Creates a dataframe from all the individual experiment results"""
     raw_data = get_raw_eval_results(results_path, requests_path)
-    all_data_json = [v.to_dict() for v in raw_data]
+    all_data_json = []
+    for v in raw_data:
+        all_data_json += v.to_dict()
 
     df = pd.DataFrame.from_records(all_data_json)
-    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
+    df["Average ⬆️"] = df[benchmark_cols].mean(axis=1)
+    # df = df.sort_values(by=[AutoEvalColumnQA.average.name], ascending=False)
     df = df[cols].round(decimals=2)
 
     # filter out if any of the benchmarks have not been produced

tests/src/display/test_utils.py

@@ -2,7 +2,6 @@ import pytest
 from src.display.utils import fields, AutoEvalColumnQA, AutoEvalColumnLongDoc, COLS, COLS_LITE, TYPES, EVAL_COLS, QA_BENCHMARK_COLS, LONG_DOC_BENCHMARK_COLS
 
 
-@pytest.mark.parametrize('auto_eval_column')
 def test_fields():
     for c in fields(AutoEvalColumnQA):
         print(c)

tests/src/leaderboard/test_read_evals.py

@@ -14,8 +14,13 @@ def test_init_from_json_file():
 def test_to_dict():
     json_fp = cur_fp.parents[2] / "toydata" / "test_data.json"
     full_eval_result = FullEvalResult.init_from_json_file(json_fp)
-    result_dict = full_eval_result.to_dict(task='qa', metric='ndcg_at_1')
-    assert len(result_dict) == 2
+    result_list = full_eval_result.to_dict(task='qa', metric='ndcg_at_1')
+    assert len(result_list) == 1
+    result_dict = result_list[0]
+    assert result_dict["Retrieval Model"] == "bge-m3"
+    assert result_dict["Reranking Model"] == "bge-reranker-v2-m3"
+    assert result_dict["qa_wiki_en"] is not None
+    assert result_dict["qa_wiki_zh"] is not None
 
 
 def test_get_request_file_for_model():

tests/src/test_populate.py

@@ -3,10 +3,18 @@ from pathlib import Path
 
 cur_fp = Path(__file__)
 
+
 def test_get_leaderboard_df():
-    requests_path = cur_fp.parents[2] / "toydata" / "test_requests"
-    results_path = cur_fp.parents[2] / "toydata" / "test_results"
-    cols = []
-    benchmark_cols = []
-    COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
-    get_leaderboard_df(results_path, requests_path, cols, benchmark_cols)
+    requests_path = cur_fp.parents[1] / "toydata" / "test_requests"
+    results_path = cur_fp.parents[1] / "toydata" / "test_results"
+    cols = ['Retrieval Model', 'Reranking Model', 'Average ⬆️', 'wiki_en', 'wiki_zh',]
+    benchmark_cols = ['wiki_en', 'wiki_zh',]
+    raw_data, df = get_leaderboard_df(results_path, requests_path, cols, benchmark_cols)
+    assert df.shape[0] == 2
+    assert df["Retrieval Model"][0] == "bge-m3"
+    assert df["Retrieval Model"][1] == "bge-m3"
+    assert df["Reranking Model"][0] == "NoReranker"
+    assert df["Reranking Model"][1] == "bge-reranker-v2-m3"
+    assert not df[['Average ⬆️', 'wiki_en', 'wiki_zh',]].isnull().values.any()
+
+