fix: fix the data loader

app.py

@@ -42,7 +42,8 @@ def restart_space():
 #     restart_space()
 
 raw_data_qa, original_df_qa = get_leaderboard_df(
-    EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, QA_BENCHMARK_COLS, task='qa', metric='ndcg_at_1')
+    EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, QA_BENCHMARK_COLS, task='qa', metric='ndcg_at_3')
+print(f'data loaded: {len(raw_data_qa)}, {original_df_qa.shape}')
 leaderboard_df = original_df_qa.copy()
 
 # (

src/envs.py

@@ -17,8 +17,8 @@ RESULTS_REPO = f"{OWNER}/results"
 CACHE_PATH = os.getenv("HF_HOME", ".")
 
 # Local caches
-EVAL_REQUESTS_PATH = "/Users/nanwang/Codes/huggingface/nan/leaderboard/tests/toydata/test_requests"  # os.path.join(CACHE_PATH, "eval-queue")
-EVAL_RESULTS_PATH = "/Users/nanwang/Codes/huggingface/nan/leaderboard/tests/toydata/test_results"  #os.path.join(CACHE_PATH, "eval-results")
+EVAL_REQUESTS_PATH = "/Users/nanwang/Codes/huggingface/nan/leaderboard/toys/toydata/requests"  # os.path.join(CACHE_PATH, "eval-queue")
+EVAL_RESULTS_PATH = "/Users/nanwang/Codes/huggingface/nan/leaderboard/toys/toydata/results"  #os.path.join(CACHE_PATH, "eval-results")
 # EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
 # EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
 

src/leaderboard/read_evals.py

@@ -62,19 +62,22 @@ class FullEvalResult:
             results=result_list
         )
 
-    def to_dict(self, task='qa', metric='ndcg_at_1') -> List:
+    def to_dict(self, task='qa', metric='ndcg_at_3') -> List:
         """Convert FullEvalResult to a list of dict compatible with our dataframe UI
         """
         results = defaultdict(dict)
         for eval_result in self.results:
             if eval_result.metric != metric:
+                # print(f'result skipped: {metric} != {eval_result.metric}')
                 continue
             if eval_result.task != task:
+                # print(f'result skipped: {task} != {eval_result.task}')
                 continue
             results[eval_result.eval_name]["eval_name"] = eval_result.eval_name
             results[eval_result.eval_name][AutoEvalColumnQA.retrieval_model.name] = self.retrieval_model
             results[eval_result.eval_name][AutoEvalColumnQA.reranking_model.name] = self.reranking_model
 
+            print(f'result loaded: {eval_result.eval_name}')
             for result in eval_result.results:
                 # add result for each domain, language, and dataset
                 domain = result["domain"]
@@ -136,7 +139,7 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> List[FullEval
         if len(files) == 0 or any([not f.endswith(".json") for f in files]):
             continue
         try:
-            files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_demo_")[:-7], reverse=True)
+            files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7], reverse=True)
         except dateutil.parser._parser.ParserError:
             files = [files[-1]]
 
@@ -152,9 +155,11 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> List[FullEval
         eval_result.update_with_request_file(requests_path)
         latest_date_str = eval_result.date.replace(":", "-")
         model_result_date_str = model_result_filepath.split('/')[-1
-        ].removeprefix("results_demo_").removesuffix(".json")
+        ].removeprefix("results_").removesuffix(".json")
         if latest_date_str != model_result_date_str:
+            print(f'file skipped: {model_result_filepath}')
             continue
+        print(f'file loaded: {model_result_filepath}')
         eval_name = eval_result.eval_name
         eval_results[eval_name] = eval_result
 

src/populate.py

@@ -12,11 +12,14 @@ from typing import Tuple
 def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list, task: str, metric: str) -> Tuple[list[EvalResult], pd.DataFrame]:
     """Creates a dataframe from all the individual experiment results"""
     raw_data = get_raw_eval_results(results_path, requests_path)
+    print(f"raw_data loaded: {len(raw_data)}")
     all_data_json = []
     for v in raw_data:
         all_data_json += v.to_dict(task=task, metric=metric)
 
+    print(f'records loaded: {len(all_data_json)}')
     df = pd.DataFrame.from_records(all_data_json)
+    print(f'dataframe created: {df.shape}')
     _benchmark_cols = frozenset(benchmark_cols).intersection(frozenset(df.columns.to_list()))
     df[AutoEvalColumnQA.average.name] = df[list(_benchmark_cols)].mean(axis=1)
     df = df.sort_values(by=[AutoEvalColumnQA.average.name], ascending=False)