refactor: reformat

app.py

@@ -6,6 +6,7 @@ from huggingface_hub import snapshot_download
 
 from src.about import BENCHMARKS_TEXT, EVALUATION_QUEUE_TEXT, INTRODUCTION_TEXT, TITLE
 from src.benchmarks import LongDocBenchmarks, QABenchmarks
+from src.columns import COL_NAME_RERANKING_MODEL, COL_NAME_RETRIEVAL_MODEL
 from src.components import (
     get_anonymous_checkbox,
     get_domain_dropdown,
@@ -31,7 +32,6 @@ from src.envs import (
     RESULTS_REPO,
     TOKEN,
 )
-from src.columns import COL_NAME_RETRIEVAL_MODEL, COL_NAME_RERANKING_MODEL
 from src.loaders import load_eval_results
 from src.models import TaskType, model_hyperlink
 from src.utils import remove_html, reset_rank, set_listeners, submit_results, update_metric, upload_file
@@ -81,7 +81,7 @@ def update_qa_metric(
         reranking_model,
         query,
         show_anonymous,
-        show_revision_and_timestamp
+        show_revision_and_timestamp,
     )
 
 
@@ -173,7 +173,9 @@ with demo:
                         #  shown_table
                         qa_df_elem_ret_rerank = get_leaderboard_table(datastore.qa_fmt_df, datastore.qa_types)
                         # Dummy leaderboard for handling the case when the user uses backspace key
-                        qa_df_elem_ret_rerank_hidden = get_leaderboard_table(datastore.qa_raw_df, datastore.qa_types, visible=False)
+                        qa_df_elem_ret_rerank_hidden = get_leaderboard_table(
+                            datastore.qa_raw_df, datastore.qa_types, visible=False
+                        )
 
                         version.change(
                             update_qa_version,
@@ -214,14 +216,24 @@ with demo:
                         qa_df_elem_ret = get_leaderboard_table(_qa_df_ret, datastore.qa_types)
 
                         # Dummy leaderboard for handling the case when the user uses backspace key
-                        _qa_df_ret_hidden = datastore.qa_raw_df[datastore.qa_raw_df[COL_NAME_RERANKING_MODEL] == "NoReranker"]
+                        _qa_df_ret_hidden = datastore.qa_raw_df[
+                            datastore.qa_raw_df[COL_NAME_RERANKING_MODEL] == "NoReranker"
+                        ]
                         _qa_df_ret_hidden = reset_rank(_qa_df_ret_hidden)
-                        qa_df_elem_ret_hidden = get_leaderboard_table(_qa_df_ret_hidden, datastore.qa_types, visible=False)
+                        qa_df_elem_ret_hidden = get_leaderboard_table(
+                            _qa_df_ret_hidden, datastore.qa_types, visible=False
+                        )
 
                         version.change(
                             update_qa_version,
                             version,
-                            [domains, langs, models_ret, qa_df_elem_ret, qa_df_elem_ret_hidden, ],
+                            [
+                                domains,
+                                langs,
+                                models_ret,
+                                qa_df_elem_ret,
+                                qa_df_elem_ret_hidden,
+                            ],
                         )
 
                         set_listeners(
@@ -253,13 +265,9 @@ with demo:
                         )
 
                     with gr.TabItem("Reranking Only", id=12):
-                        _qa_df_rerank = datastore.qa_fmt_df[
-                            datastore.qa_fmt_df[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK
-                            ]
+                        _qa_df_rerank = datastore.qa_fmt_df[datastore.qa_fmt_df[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK]
                         _qa_df_rerank = reset_rank(_qa_df_rerank)
-                        qa_rerank_models = (
-                            _qa_df_rerank[COL_NAME_RERANKING_MODEL].apply(remove_html).unique().tolist()
-                        )
+                        qa_rerank_models = _qa_df_rerank[COL_NAME_RERANKING_MODEL].apply(remove_html).unique().tolist()
                         with gr.Row():
                             with gr.Column(scale=1):
                                 qa_models_rerank = get_reranking_dropdown(qa_rerank_models)
@@ -269,7 +277,7 @@ with demo:
 
                         _qa_df_rerank_hidden = datastore.qa_raw_df[
                             datastore.qa_raw_df[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK
-                            ]
+                        ]
                         _qa_df_rerank_hidden = reset_rank(_qa_df_rerank_hidden)
                         qa_df_elem_rerank_hidden = get_leaderboard_table(
                             _qa_df_rerank_hidden, datastore.qa_types, visible=False
@@ -333,9 +341,7 @@ with demo:
                             with gr.Column():
                                 models = get_reranking_dropdown(datastore.reranking_models)
 
-                        doc_df_elem_ret_rerank = get_leaderboard_table(
-                            datastore.doc_fmt_df, datastore.doc_types
-                        )
+                        doc_df_elem_ret_rerank = get_leaderboard_table(datastore.doc_fmt_df, datastore.doc_types)
 
                         # Dummy leaderboard for handling the case when the user uses backspace key
                         doc_df_elem_ret_rerank_hidden = get_leaderboard_table(
@@ -385,15 +391,13 @@ with demo:
 
                         _doc_df_ret = datastore.doc_fmt_df[
                             datastore.doc_fmt_df[COL_NAME_RERANKING_MODEL] == "NoReranker"
-                            ]
+                        ]
                         _doc_df_ret = reset_rank(_doc_df_ret)
-                        doc_df_elem_ret = get_leaderboard_table(
-                            _doc_df_ret, datastore.doc_types
-                        )
+                        doc_df_elem_ret = get_leaderboard_table(_doc_df_ret, datastore.doc_types)
 
                         _doc_df_ret_hidden = datastore.doc_raw_df[
                             datastore.doc_raw_df[COL_NAME_RERANKING_MODEL] == "NoReranker"
-                            ]
+                        ]
                         _doc_df_ret_hidden = reset_rank(_doc_df_ret_hidden)
                         doc_df_elem_ret_hidden = get_leaderboard_table(
                             _doc_df_ret_hidden, datastore.doc_types, visible=False
@@ -435,22 +439,20 @@ with demo:
                     with gr.TabItem("Reranking Only", id=22):
                         _doc_df_rerank = datastore.doc_fmt_df[
                             datastore.doc_fmt_df[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK
-                            ]
+                        ]
                         _doc_df_rerank = reset_rank(_doc_df_rerank)
                         doc_rerank_models = (
                             _doc_df_rerank[COL_NAME_RERANKING_MODEL].apply(remove_html).unique().tolist()
                         )
                         with gr.Row():
                             with gr.Column(scale=1):
-                                doc_models_rerank = get_reranking_dropdown(
-                                    doc_rerank_models
-                                )
+                                doc_models_rerank = get_reranking_dropdown(doc_rerank_models)
                             with gr.Column(scale=1):
                                 doc_search_bar_rerank = gr.Textbox(show_label=False, visible=False)
                         doc_df_elem_rerank = get_leaderboard_table(_doc_df_rerank, datastore.doc_types)
                         _doc_df_rerank_hidden = datastore.doc_raw_df[
                             datastore.doc_raw_df[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK
-                            ]
+                        ]
                         _doc_df_rerank_hidden = reset_rank(_doc_df_rerank_hidden)
                         doc_df_elem_rerank_hidden = get_leaderboard_table(
                             _doc_df_rerank_hidden, datastore.doc_types, visible=False

src/loaders.py

@@ -1,14 +1,10 @@
 import os.path
-from typing import List, Dict
+from typing import Dict, List
 
 import pandas as pd
 
-from src.envs import (
-    BENCHMARK_VERSION_LIST,
-    DEFAULT_METRIC_LONG_DOC,
-    DEFAULT_METRIC_QA,
-)
-from src.columns import COL_NAME_REVISION, COL_NAME_TIMESTAMP, COL_NAME_IS_ANONYMOUS
+from src.columns import COL_NAME_IS_ANONYMOUS, COL_NAME_REVISION, COL_NAME_TIMESTAMP
+from src.envs import BENCHMARK_VERSION_LIST, DEFAULT_METRIC_LONG_DOC, DEFAULT_METRIC_QA
 from src.models import FullEvalResult, LeaderboardDataStore, TaskType
 from src.utils import get_default_cols, get_leaderboard_df
 
@@ -80,8 +76,9 @@ def load_leaderboard_datastore(file_path, version) -> LeaderboardDataStore:
     datastore.doc_fmt_df = datastore.doc_fmt_df[~datastore.doc_fmt_df[COL_NAME_IS_ANONYMOUS]][doc_cols]
     datastore.doc_fmt_df.drop([COL_NAME_REVISION, COL_NAME_TIMESTAMP], axis=1, inplace=True)
 
-    datastore.reranking_models = \
-        sorted(list(frozenset([eval_result.reranking_model for eval_result in datastore.raw_data])))
+    datastore.reranking_models = sorted(
+        list(frozenset([eval_result.reranking_model for eval_result in datastore.raw_data]))
+    )
     return datastore
 
 

src/models.py

@@ -1,14 +1,20 @@
 import json
-from enum import Enum
-
 from collections import defaultdict
 from dataclasses import dataclass
+from enum import Enum
 from typing import List, Optional
 
 import pandas as pd
 
-from src.columns import COL_NAME_RETRIEVAL_MODEL, COL_NAME_RERANKING_MODEL, COL_NAME_RETRIEVAL_MODEL_LINK, \
-    COL_NAME_RERANKING_MODEL_LINK, COL_NAME_REVISION, COL_NAME_TIMESTAMP, COL_NAME_IS_ANONYMOUS
+from src.columns import (
+    COL_NAME_IS_ANONYMOUS,
+    COL_NAME_RERANKING_MODEL,
+    COL_NAME_RERANKING_MODEL_LINK,
+    COL_NAME_RETRIEVAL_MODEL,
+    COL_NAME_RETRIEVAL_MODEL_LINK,
+    COL_NAME_REVISION,
+    COL_NAME_TIMESTAMP,
+)
 
 
 def get_safe_name(name: str):
@@ -16,6 +22,7 @@ def get_safe_name(name: str):
     name = name.replace("-", "_")
     return "".join(character.lower() for character in name if (character.isalnum() or character == "_"))
 
+
 @dataclass
 class EvalResult:
     """

src/utils.py

@@ -6,16 +6,20 @@ from pathlib import Path
 
 import pandas as pd
 
-from src.models import TaskType
 from src.benchmarks import LongDocBenchmarks, QABenchmarks
-from src.columns import get_default_col_names_and_types, get_fixed_col_names_and_types, COL_NAME_AVG, \
-    COL_NAME_RETRIEVAL_MODEL, COL_NAME_RERANKING_MODEL, COL_NAME_RANK, COL_NAME_REVISION, COL_NAME_TIMESTAMP, \
-    COL_NAME_IS_ANONYMOUS
-from src.envs import (
-    API,
-    LATEST_BENCHMARK_VERSION,
-    SEARCH_RESULTS_REPO,
+from src.columns import (
+    COL_NAME_AVG,
+    COL_NAME_IS_ANONYMOUS,
+    COL_NAME_RANK,
+    COL_NAME_RERANKING_MODEL,
+    COL_NAME_RETRIEVAL_MODEL,
+    COL_NAME_REVISION,
+    COL_NAME_TIMESTAMP,
+    get_default_col_names_and_types,
+    get_fixed_col_names_and_types,
 )
+from src.envs import API, LATEST_BENCHMARK_VERSION, SEARCH_RESULTS_REPO
+from src.models import TaskType
 
 
 def calculate_mean(row):
@@ -200,7 +204,7 @@ def update_metric(
     elif task == TaskType.long_doc:
         update_func = update_doc_df_elem
     else:
-        raise NotImplemented
+        raise NotImplementedError
     df_elem = get_leaderboard_df(datastore, task=task, metric=metric)
     version = datastore.version
     return update_func(

tests/src/display/test_utils.py

@@ -1,4 +1,3 @@
-
 from src.display.utils import (
     COLS_LONG_DOC,
     COLS_QA,

tests/test_utils.py

@@ -2,8 +2,15 @@ import pandas as pd
 import pytest
 
 from app import update_table
-from src.columns import COL_NAME_AVG, COL_NAME_RETRIEVAL_MODEL, COL_NAME_RERANKING_MODEL, COL_NAME_RANK, \
-    COL_NAME_REVISION, COL_NAME_TIMESTAMP, COL_NAME_IS_ANONYMOUS
+from src.columns import (
+    COL_NAME_AVG,
+    COL_NAME_IS_ANONYMOUS,
+    COL_NAME_RANK,
+    COL_NAME_RERANKING_MODEL,
+    COL_NAME_RETRIEVAL_MODEL,
+    COL_NAME_REVISION,
+    COL_NAME_TIMESTAMP,
+)
 from src.utils import (
     filter_models,
     filter_queries,