feat: fix the table updating

app.py

@@ -10,15 +10,17 @@ from src.about import (
 from src.display.css_html_js import custom_css
 from src.display.utils import (
     QA_BENCHMARK_COLS,
-    COLS,
+    LONG_DOC_BENCHMARK_COLS,
+    COLS_QA,
+    COLS_LONG_DOC,
     TYPES,
     AutoEvalColumnQA,
     fields
 )
 from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
 from src.populate import get_leaderboard_df
-from utils import update_table, update_metric
-from src.benchmarks import DOMAIN_COLS_QA, LANG_COLS_QA, metric_list
+from utils import update_table, update_metric, update_table_long_doc
+from src.benchmarks import DOMAIN_COLS_QA, LANG_COLS_QA, DOMAIN_COLS_LONG_DOC, LANG_COLS_LONG_DOC, metric_list
 
 
 def restart_space():
@@ -43,9 +45,15 @@ def restart_space():
 
 from src.leaderboard.read_evals import get_raw_eval_results
 raw_data_qa = get_raw_eval_results(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH)
-original_df_qa = get_leaderboard_df(raw_data_qa, COLS, QA_BENCHMARK_COLS, task='qa', metric='ndcg_at_3')
-print(f'data loaded: {len(raw_data_qa)}, {original_df_qa.shape}')
+original_df_qa = get_leaderboard_df(raw_data_qa, COLS_QA, QA_BENCHMARK_COLS, task='qa', metric='ndcg_at_3')
+original_df_long_doc = get_leaderboard_df(raw_data_qa, COLS_LONG_DOC, LONG_DOC_BENCHMARK_COLS, task='long_doc', metric='ndcg_at_3')
+print(f'raw data: {len(raw_data_qa)}')
+print(f'QA data loaded: {original_df_qa.shape}')
+print(f'Long-Doc data loaded: {len(original_df_long_doc)}')
+
 leaderboard_df = original_df_qa.copy()
+leaderboard_df_long_doc = original_df_long_doc.copy()
+print(leaderboard_df_long_doc.head())
 
 
 def update_metric_qa(
@@ -55,7 +63,18 @@ def update_metric_qa(
         reranking_model: list,
         query: str,
 ):
-    return update_metric(raw_data_qa, metric, domains, langs, reranking_model, query)
+    return update_metric(raw_data_qa, 'qa', metric, domains, langs, reranking_model, query)
+
+def update_metric_long_doc(
+        metric: str,
+        domains: list,
+        langs: list,
+        reranking_model: list,
+        query: str,
+):
+    return update_metric(raw_data_qa, 'long_doc', metric, domains, langs, reranking_model, query)
+
+
 # (
 #     finished_eval_queue_df,
 #     running_eval_queue_df,
@@ -178,7 +197,113 @@ with demo:
                 queue=True
             )
 
-        # with gr.TabItem("Long Doc", elem_id="long-doc-benchmark-tab-table", id=1):
+        with gr.TabItem("Long Doc", elem_id="long-doc-benchmark-tab-table", id=1):
+            with gr.Row():
+                with gr.Column():
+                    with gr.Row():
+                        search_bar = gr.Textbox(
+                            placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
+                            show_label=False,
+                            elem_id="search-bar-long-doc",
+                        )
+                        # select the metric
+                    selected_metric = gr.Dropdown(
+                        choices=metric_list,
+                        value=metric_list[1],
+                        label="Select the metric",
+                        interactive=True,
+                        elem_id="metric-select-long-doc",
+                    )
+                with gr.Column(min_width=320):
+                    # select domain
+                    with gr.Row():
+                        selected_domains = gr.CheckboxGroup(
+                            choices=DOMAIN_COLS_LONG_DOC,
+                            value=DOMAIN_COLS_LONG_DOC,
+                            label="Select the domains",
+                            elem_id="domain-column-select-long-doc",
+                            interactive=True,
+                        )
+                    # select language
+                    with gr.Row():
+                        selected_langs = gr.CheckboxGroup(
+                            choices=LANG_COLS_LONG_DOC,
+                            value=LANG_COLS_LONG_DOC,
+                            label="Select the languages",
+                            elem_id="language-column-select-long-doc",
+                            interactive=True
+                        )
+                    # select reranking model
+                    reranking_models = list(frozenset([eval_result.reranking_model for eval_result in raw_data_qa]))
+                    with gr.Row():
+                        selected_rerankings = gr.CheckboxGroup(
+                            choices=reranking_models,
+                            value=reranking_models,
+                            label="Select the reranking models",
+                            elem_id="reranking-select-long-doc",
+                            interactive=True
+                        )
+
+            leaderboard_table_long_doc = gr.components.Dataframe(
+                value=leaderboard_df_long_doc,
+                # headers=shown_columns,
+                # datatype=TYPES,
+                elem_id="leaderboard-table-long-doc",
+                interactive=False,
+                visible=True,
+            )
+
+            # Dummy leaderboard for handling the case when the user uses backspace key
+            hidden_leaderboard_table_for_search = gr.components.Dataframe(
+                value=leaderboard_df_long_doc,
+                # headers=COLS,
+                # datatype=TYPES,
+                visible=False,
+            )
+
+            # Set search_bar listener
+            search_bar.submit(
+                update_table_long_doc,
+                [
+                    hidden_leaderboard_table_for_search,
+                    selected_domains,
+                    selected_langs,
+                    selected_rerankings,
+                    search_bar,
+                ],
+                leaderboard_table_long_doc,
+            )
+
+            # Set column-wise listener
+            for selector in [
+                selected_domains, selected_langs, selected_rerankings
+            ]:
+                selector.change(
+                    update_table_long_doc,
+                    [
+                        hidden_leaderboard_table_for_search,
+                        selected_domains,
+                        selected_langs,
+                        selected_rerankings,
+                        search_bar,
+                    ],
+                    leaderboard_table_long_doc,
+                    queue=True,
+                )
+
+            # set metric listener
+            selected_metric.change(
+                update_metric_long_doc,
+                [
+                    selected_metric,
+                    selected_domains,
+                    selected_langs,
+                    selected_rerankings,
+                    search_bar,
+                ],
+                leaderboard_table_long_doc,
+                queue=True
+            )
 
         with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")

src/benchmarks.py

@@ -52,19 +52,19 @@ dataset_dict = {
         },
         "healthcare": {
             "en": [
-                "pubmed_100K-200K_1",
-                "pubmed_100K-200K_2",
-                "pubmed_100K-200K_3",
-                "pubmed_40K-50K_5-merged",
-                "pubmed_30K-40K_10-merged"
+                "pubmed_100k-200k_1",
+                "pubmed_100k-200k_2",
+                "pubmed_100k-200k_3",
+                "pubmed_40k-50k_5-merged",
+                "pubmed_30k-40k_10-merged"
             ]
         },
         "law": {
             "en": [
-                "lex_files_300K-400K",
-                "lex_files_400K-500K",
-                "lex_files_500K-600K",
-                "lex_files_600K-700K"
+                "lex_files_300k-400k",
+                "lex_files_400k-500k",
+                "lex_files_500k-600k",
+                "lex_files_600k-700k"
             ]
         }
     }
@@ -121,21 +121,25 @@ for task, domain_dict in dataset_dict.items():
             if task == "qa":
                 benchmark_name = f"{domain}_{lang}"
                 benchmark_name = get_safe_name(benchmark_name)
-                col_name = f"{domain}_{lang}"
+                col_name = benchmark_name
                 for metric in dataset_list:
                     qa_benchmark_dict[benchmark_name] = Benchmark(benchmark_name, metric, col_name, domain, lang, task)
             elif task == "long_doc":
                 for dataset in dataset_list:
-                    col_name = f"{domain}_{lang}_{dataset}"
+                    benchmark_name = f"{domain}_{lang}_{dataset}"
+                    benchmark_name = get_safe_name(benchmark_name)
+                    col_name = benchmark_name
                     for metric in metric_list:
-                        benchmark_name = f"{domain}_{lang}_{dataset}_{metric}"
-                        benchmark_name = get_safe_name(benchmark_name)
                         long_doc_benchmark_dict[benchmark_name] = Benchmark(benchmark_name, metric, col_name, domain, lang, task)
 
 BenchmarksQA = Enum('BenchmarksQA', qa_benchmark_dict)
 BenchmarksLongDoc = Enum('BenchmarksLongDoc', long_doc_benchmark_dict)
 
 BENCHMARK_COLS_QA = [c.col_name for c in qa_benchmark_dict.values()]
+BENCHMARK_COLS_LONG_DOC = [c.col_name for c in long_doc_benchmark_dict.values()]
 
 DOMAIN_COLS_QA = list(frozenset([c.domain for c in qa_benchmark_dict.values()]))
 LANG_COLS_QA = list(frozenset([c.lang for c in qa_benchmark_dict.values()]))
+
+DOMAIN_COLS_LONG_DOC = list(frozenset([c.domain for c in long_doc_benchmark_dict.values()]))
+LANG_COLS_LONG_DOC = list(frozenset([c.lang for c in long_doc_benchmark_dict.values()]))

src/display/utils.py

@@ -55,7 +55,8 @@ class EvalQueueColumn:  # Queue column
 
 
 # Column selection
-COLS = [c.name for c in fields(AutoEvalColumnQA) if not c.hidden]
+COLS_QA = [c.name for c in fields(AutoEvalColumnQA) if not c.hidden]
+COLS_LONG_DOC = [c.name for c in fields(AutoEvalColumnLongDoc) if not c.hidden]
 TYPES = [c.type for c in fields(AutoEvalColumnQA) if not c.hidden]
 COLS_LITE = [c.name for c in fields(AutoEvalColumnQA) if c.displayed_by_default and not c.hidden]
 

src/leaderboard/read_evals.py

@@ -87,7 +87,7 @@ class FullEvalResult:
                 if task == 'qa':
                     benchmark_name = f"{domain}_{lang}"
                 elif task == 'long_doc':
-                    benchmark_name = f"{domain}_{lang}_{dataset}_{metric}"
+                    benchmark_name = f"{domain}_{lang}_{dataset}"
                 results[eval_result.eval_name][get_safe_name(benchmark_name)] = value
         return [v for v in results.values()]
 

src/populate.py

@@ -4,7 +4,7 @@ import os
 import pandas as pd
 
 from src.display.formatting import has_no_nan_values, make_clickable_model
-from src.display.utils import AutoEvalColumnQA, EvalQueueColumn
+from src.display.utils import AutoEvalColumnQA, AutoEvalColumnLongDoc, EvalQueueColumn
 from src.leaderboard.read_evals import get_raw_eval_results, EvalResult, FullEvalResult
 from typing import Tuple, List
 
@@ -19,8 +19,13 @@ def get_leaderboard_df(raw_data: List[FullEvalResult], cols: list, benchmark_col
 
     # calculate the average score for selected benchmarks
     _benchmark_cols = frozenset(benchmark_cols).intersection(frozenset(df.columns.to_list()))
-    df[AutoEvalColumnQA.average.name] = df[list(_benchmark_cols)].mean(axis=1).round(decimals=2)
-    df = df.sort_values(by=[AutoEvalColumnQA.average.name], ascending=False)
+    if task == 'qa':
+        df[AutoEvalColumnQA.average.name] = df[list(_benchmark_cols)].mean(axis=1).round(decimals=2)
+        df = df.sort_values(by=[AutoEvalColumnQA.average.name], ascending=False)
+    elif task == "long_doc":
+        df[AutoEvalColumnLongDoc.average.name] = df[list(_benchmark_cols)].mean(axis=1).round(decimals=2)
+        df = df.sort_values(by=[AutoEvalColumnLongDoc.average.name], ascending=False)
+
     df.reset_index(inplace=True)
 
     _cols = frozenset(cols).intersection(frozenset(df.columns.to_list()))

tests/src/display/test_utils.py

@@ -1,5 +1,5 @@
 import pytest
-from src.display.utils import fields, AutoEvalColumnQA, AutoEvalColumnLongDoc, COLS, COLS_LITE, TYPES, EVAL_COLS, QA_BENCHMARK_COLS, LONG_DOC_BENCHMARK_COLS
+from src.display.utils import fields, AutoEvalColumnQA, AutoEvalColumnLongDoc, COLS_QA, COLS_LONG_DOC, COLS_LITE, TYPES, EVAL_COLS, QA_BENCHMARK_COLS, LONG_DOC_BENCHMARK_COLS
 
 
 def test_fields():
@@ -8,8 +8,10 @@ def test_fields():
 
 
 def test_macro_variables():
-    print(f'COLS: {COLS}')
+    print(f'COLS_QA: {COLS_QA}')
+    print(f'COLS_LONG_DOC: {COLS_LONG_DOC}')
     print(f'COLS_LITE: {COLS_LITE}')
     print(f'TYPES: {TYPES}')
     print(f'EVAL_COLS: {EVAL_COLS}')
-    print(f'BENCHMARK_COLS: {QA_BENCHMARK_COLS}')
+    print(f'QA_BENCHMARK_COLS: {QA_BENCHMARK_COLS}')
+    print(f'LONG_DOC_BENCHMARK_COLS: {LONG_DOC_BENCHMARK_COLS}')

tests/src/test_populate.py

@@ -23,3 +23,19 @@ def test_get_leaderboard_df():
     assert not df[['Average ⬆️', 'wiki_en', 'wiki_zh',]].isnull().values.any()
 
 
+def test_get_leaderboard_df_long_doc():
+    requests_path = cur_fp.parents[1] / "toydata" / "test_requests"
+    results_path = cur_fp.parents[1] / "toydata" / "test_results"
+    cols = ['Retrieval Model', 'Reranking Model', 'Average ⬆️', 'law_en_lex_files_500k_600k',]
+    benchmark_cols = ['law_en_lex_files_500k_600k',]
+    raw_data = get_raw_eval_results(results_path, requests_path)
+    df = get_leaderboard_df(raw_data, cols, benchmark_cols, 'long_doc', 'ndcg_at_1')
+    assert df.shape[0] == 2
+    # the results contain only one embedding model
+    for i in range(2):
+        assert df["Retrieval Model"][i] == "bge-m3"
+    # the results contains only two reranking model
+    assert df["Reranking Model"][0] == "bge-reranker-v2-m3"
+    assert df["Reranking Model"][1] == "NoReranker"
+    assert df["Average ⬆️"][0] > df["Average ⬆️"][1]
+    assert not df[['Average ⬆️', 'law_en_lex_files_500k_600k',]].isnull().values.any()

tests/test_utils.py

@@ -1,7 +1,7 @@
 import pandas as pd
 import pytest
 
-from utils import filter_models, search_table, filter_queries, select_columns
+from utils import filter_models, search_table, filter_queries, select_columns, update_table_long_doc
 
 
 @pytest.fixture
@@ -29,6 +29,29 @@ def toy_df():
     )
 
 
+@pytest.fixture
+def toy_df_long_doc():
+    return pd.DataFrame(
+        {
+            "Retrieval Model": [
+                "bge-m3",
+                "bge-m3",
+                "jina-embeddings-v2-base",
+                "jina-embeddings-v2-base"
+            ],
+            "Reranking Model": [
+                "bge-reranker-v2-m3",
+                "NoReranker",
+                "bge-reranker-v2-m3",
+                "NoReranker"
+            ],
+            "Average ⬆️": [0.6, 0.4, 0.3, 0.2],
+            "law_en_lex_files_300k_400k": [0.4, 0.1, 0.4, 0.3],
+            "law_en_lex_files_400k_500k": [0.8, 0.7, 0.2, 0.1],
+            "law_en_lex_files_500k_600k": [0.8, 0.7, 0.2, 0.1],
+            "law_en_lex_files_600k_700k": [0.4, 0.1, 0.4, 0.3],
+        }
+    )
 def test_filter_models(toy_df):
     df_result = filter_models(toy_df, ["bge-reranker-v2-m3", ])
     assert len(df_result) == 2
@@ -50,4 +73,9 @@ def test_filter_queries(toy_df):
 def test_select_columns(toy_df):
     df_result = select_columns(toy_df, ['news',], ['zh',])
     assert len(df_result.columns) == 4
-    assert df_result['Average ⬆️'].equals(df_result['news_zh'])
+    assert df_result['Average ⬆️'].equals(df_result['news_zh'])
+
+
+def test_update_table_long_doc(toy_df_long_doc):
+    df_result = update_table_long_doc(toy_df_long_doc, ['law',], ['en',], ["bge-reranker-v2-m3", ], "jina")
+    print(df_result)

tests/toydata/test_results/bge-m3/NoReranker/results_2023-12-21T18-10-08.json

@@ -11,7 +11,7 @@
         "domain": "law",
         "lang": "en",
         "dataset": "lex_files_500K-600K",
-        "value": 0.75723
+        "value": 0.45723
       }
     ]
   },

utils.py

@@ -1,11 +1,10 @@
 import pandas as pd
 
-from src.display.utils import AutoEvalColumnQA, COLS
-from src.benchmarks import BENCHMARK_COLS_QA, BenchmarksQA
+from src.display.utils import AutoEvalColumnQA, AutoEvalColumnLongDoc, COLS_QA, COLS_LONG_DOC, QA_BENCHMARK_COLS, LONG_DOC_BENCHMARK_COLS
+from src.benchmarks import BENCHMARK_COLS_QA, BENCHMARK_COLS_LONG_DOC, BenchmarksQA, BenchmarksLongDoc
 from src.leaderboard.read_evals import FullEvalResult
 from typing import List
 from src.populate import get_leaderboard_df
-from src.display.utils import COLS, QA_BENCHMARK_COLS
 
 
 def filter_models(df: pd.DataFrame, reranking_query: list) -> pd.DataFrame:
@@ -38,19 +37,29 @@ def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
     return df[(df[AutoEvalColumnQA.retrieval_model.name].str.contains(query, case=False))]
 
 
-def select_columns(df: pd.DataFrame, domain_query: list, language_query: list) -> pd.DataFrame:
-    always_here_cols = [
-        AutoEvalColumnQA.retrieval_model.name,
-        AutoEvalColumnQA.reranking_model.name,
-        AutoEvalColumnQA.average.name
-    ]
+def select_columns(df: pd.DataFrame, domain_query: list, language_query: list, task: str="qa") -> pd.DataFrame:
+    if task == "qa":
+        always_here_cols = [
+            AutoEvalColumnQA.retrieval_model.name,
+            AutoEvalColumnQA.reranking_model.name,
+            AutoEvalColumnQA.average.name
+        ]
+        cols = list(frozenset(COLS_QA).intersection(frozenset(BENCHMARK_COLS_QA)))
+    elif task == "long_doc":
+        always_here_cols = [
+            AutoEvalColumnLongDoc.retrieval_model.name,
+            AutoEvalColumnLongDoc.reranking_model.name,
+            AutoEvalColumnLongDoc.average.name
+        ]
+        cols = list(frozenset(COLS_LONG_DOC).intersection(frozenset(BENCHMARK_COLS_LONG_DOC)))
     selected_cols = []
-    for c in COLS:
+    for c in cols:
         if c not in df.columns:
             continue
-        if c not in BENCHMARK_COLS_QA:
-            continue
-        eval_col = BenchmarksQA[c].value
+        if task == "qa":
+            eval_col = BenchmarksQA[c].value
+        elif task == "long_doc":
+            eval_col = BenchmarksLongDoc[c].value
         if eval_col.domain not in domain_query:
             continue
         if eval_col.lang not in language_query:
@@ -58,7 +67,7 @@ def select_columns(df: pd.DataFrame, domain_query: list, language_query: list) -
         selected_cols.append(c)
     # We use COLS to maintain sorting
     filtered_df = df[always_here_cols + selected_cols]
-    filtered_df[AutoEvalColumnQA.average.name] = filtered_df[selected_cols].mean(axis=1).round(decimals=2)
+    filtered_df[always_here_cols[2]] = filtered_df[selected_cols].mean(axis=1).round(decimals=2)
     return filtered_df
 
 
@@ -75,20 +84,43 @@ def update_table(
     return df
 
 
+def update_table_long_doc(
+        hidden_df: pd.DataFrame,
+        domains: list,
+        langs: list,
+        reranking_query: list,
+        query: str,
+):
+    filtered_df = filter_models(hidden_df, reranking_query)
+    filtered_df = filter_queries(query, filtered_df)
+    df = select_columns(filtered_df, domains, langs, task='long_doc')
+    return df
+
+
 def update_metric(
         raw_data: List[FullEvalResult],
+        task: str,
         metric: str,
         domains: list,
         langs: list,
         reranking_model: list,
         query: str,
 ) -> pd.DataFrame:
-    leaderboard_df = get_leaderboard_df(raw_data, COLS, QA_BENCHMARK_COLS, task='qa', metric=metric)
-    hidden_df = leaderboard_df
-    return update_table(
-        hidden_df,
-        domains,
-        langs,
-        reranking_model,
-        query
-    )
+    if task == 'qa':
+        leaderboard_df = get_leaderboard_df(raw_data, COLS_QA, QA_BENCHMARK_COLS, task=task, metric=metric)
+        return update_table(
+            leaderboard_df,
+            domains,
+            langs,
+            reranking_model,
+            query
+        )
+    elif task == 'long_doc':
+        leaderboard_df = get_leaderboard_df(raw_data, COLS_LONG_DOC, LONG_DOC_BENCHMARK_COLS, task=task, metric=metric)
+        return update_table_long_doc(
+            leaderboard_df,
+            domains,
+            langs,
+            reranking_model,
+            query
+        )