first construction

app.py

@@ -10,6 +10,7 @@ from src.about import (
     EVALUATION_QUEUE_TEXT,
     INTRODUCTION_TEXT,
     LLM_BENCHMARKS_TEXT,
+    SUBMIT_FORM,
     TITLE,
 )
 from src.display.css_html_js import custom_css
@@ -32,62 +33,8 @@ from src.submission.submit import add_new_eval
 def restart_space():
     API.restart_space(repo_id=REPO_ID)
 
-### Space initialisation
-try:
-    print(EVAL_REQUESTS_PATH)
-    snapshot_download(
-        repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
-try:
-    print(EVAL_RESULTS_PATH)
-    snapshot_download(
-        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
-
-
-LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
-
-(
-    finished_eval_queue_df,
-    running_eval_queue_df,
-    pending_eval_queue_df,
-) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
-
-def init_leaderboard(dataframe):
-    if dataframe is None or dataframe.empty:
-        raise ValueError("Leaderboard DataFrame is empty or None.")
-    return Leaderboard(
-        value=dataframe,
-        datatype=[c.type for c in fields(AutoEvalColumn)],
-        select_columns=SelectColumns(
-            default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
-            cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
-            label="Select Columns to Display:",
-        ),
-        search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
-        hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
-        filter_columns=[
-            ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
-            ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
-            ColumnFilter(
-                AutoEvalColumn.params.name,
-                type="slider",
-                min=0.01,
-                max=150,
-                label="Select the number of parameters (B)",
-            ),
-            ColumnFilter(
-                AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
-            ),
-        ],
-        bool_checkboxgroup_label="Hide models",
-        interactive=False,
-    )
 
+LEADERBOARD_DF = pd.read_csv("results.csv")
 
 demo = gr.Blocks(css=custom_css)
 with demo:
@@ -95,98 +42,44 @@ with demo:
     gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
-            leaderboard = init_leaderboard(LEADERBOARD_DF)
+        with gr.TabItem("🏅 MVRB", elem_id="llm-benchmark-tab-table", id=0):
+            Leaderboard(
+                value=LEADERBOARD_DF,
+                datatype=[c.type for c in fields(AutoEvalColumn)],
+                select_columns=SelectColumns(
+                    default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
+                    cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
+                    label="Select Columns to Display:",
+                ),
+                search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
+                hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
+                filter_columns=[
+                    # ColumnFilter(
+                    #    "#params",
+                    #     type="slider",
+                    #     min=0.1,
+                    #     max=70,
+                    #     greater_than=True,
+                    #     label="Minimum number of parameters (B)",
+                    # ),
+                    # ColumnFilter(
+                    #     "#params",
+                    #     type="slider",
+                    #     min=0.1,
+                    #     max=70,
+                    #     less_than=True,
+                    #     label="Maximum number of parameters (B)",
+                    # ),
+                    ColumnFilter("#Params (B)", default=[0.1, 32], min=0.1, max=70, label="Number of parameters (B)")
+                ],
+                interactive=True,
+            )
 
         with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
 
         with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
-            with gr.Column():
-                with gr.Row():
-                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
-
-                with gr.Column():
-                    with gr.Accordion(
-                        f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            finished_eval_table = gr.components.Dataframe(
-                                value=finished_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-                    with gr.Accordion(
-                        f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            running_eval_table = gr.components.Dataframe(
-                                value=running_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-
-                    with gr.Accordion(
-                        f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            pending_eval_table = gr.components.Dataframe(
-                                value=pending_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-            with gr.Row():
-                gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
-
-            with gr.Row():
-                with gr.Column():
-                    model_name_textbox = gr.Textbox(label="Model name")
-                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
-                    model_type = gr.Dropdown(
-                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
-                        label="Model type",
-                        multiselect=False,
-                        value=None,
-                        interactive=True,
-                    )
-
-                with gr.Column():
-                    precision = gr.Dropdown(
-                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
-                        label="Precision",
-                        multiselect=False,
-                        value="float16",
-                        interactive=True,
-                    )
-                    weight_type = gr.Dropdown(
-                        choices=[i.value.name for i in WeightType],
-                        label="Weights type",
-                        multiselect=False,
-                        value="Original",
-                        interactive=True,
-                    )
-                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
-
-            submit_button = gr.Button("Submit Eval")
-            submission_result = gr.Markdown()
-            submit_button.click(
-                add_new_eval,
-                [
-                    model_name_textbox,
-                    base_model_name_textbox,
-                    revision_name_textbox,
-                    precision,
-                    weight_type,
-                    model_type,
-                ],
-                submission_result,
-            )
+            gr.Markdown(SUBMIT_FORM, elem_classes="markdown-text")
 
     with gr.Row():
         with gr.Accordion("📙 Citation", open=False):

results.csv

@@ -0,0 +1,3 @@
+,#params,Overall,SR,CSR,SQA,OVC
+UniSE-MLLM,2.21,55.72,69.63,54.49,43.2,48.26
+UniSE-CLIP,0.428,36.41,35.95,43.38,28.13,40.62

src/about.py

@@ -21,20 +21,27 @@ NUM_FEWSHOT = 0 # Change with your few shot
 
 
 # Your leaderboard name
-TITLE = """<h1 align="center" id="space-title">Demo leaderboard</h1>"""
+TITLE = """<h1 align="center" id="space-title">MVRB Leaderboard</h1>"""
 
 # What does your leaderboard evaluate?
 INTRODUCTION_TEXT = """
-Intro text
+**MVRB (Massive Visualized IR Benchmark)** evaluates multimodal retrievers’ performance on general **Vis-IR** tasks. The benchmark includes various task types, such as screenshot-based multimodal retrieval (screenshot to anything, anything to screenshot) and screenshotconditioned retrieval (e.g., searching for documents using queries conditioned on screenshots). It also covers a variety of important domains, including news, products, papers, and charts.
+
+More details can be found:
+Paper: https://arxiv.org/pdf/2502.11431
+Code: https://github.com/VectorSpaceLab/Vis-IR
 """
 
 # Which evaluations are you running? how can people reproduce what you have?
 LLM_BENCHMARKS_TEXT = f"""
-## How it works
+## Tasks
+- **Screenshot Retrieval (SR)** consists of evaluation samples, each comprising a textual query q and its relevant screenshot $$s: (q, s)$$. The retrieval model needs to precisely retrieve the relevant screenshot for a testing query from a given corpus $$S$. Each evaluation sample is created in two steps: 1) sample a screenshot $$s$$, 2) prompt the LLM to generate a search query based on the caption of screenshot. We consider seven tasks under this category, including product retrieval, paper retrieval, repo retrieval, news retrieval, chart retrieval, document retrieval, and slide retrieval.
+
+- **Composed Screenshot Retrieval (CSR)** is made up of sq2s triplets. Given a screenshot $s_1$ and a query q conditioned on $s_1$, the retrieval model needs to retrieve the relevant screenshot $s_2$ from the corpus $S$. We define four tasks for this category, including product discovery, news-to-Wiki, knowledge relation, and Wiki-to-product. All tasks in this category are created by human annotators. For each task, annotators are instructed to identify relevant screenshot pairs and write queries to retrieve $s_2$ based on $s_1$.
 
-## Reproducibility
-To reproduce our results, here is the commands you can run:
+- **Screenshot Question Answering (SQA)** comprises sq2a triplets. Given a screenshot s and a question q conditioned on s, the retrieval model needs to retrieve the correct answer a from a candidate corpus A. Each evaluation sample is created in three steps: 1) sample a screenshot $$s$$, 2) prompt the MLLM to generate a question $$q$$, 3) prompt the MLLM to generate the answer $$a$$ for $$q$$ based on $$s$$. The following tasks are included in this category: product-QA, news-QA, Wiki-QA, paper-QA, repo-QA.
 
+- Open-Vocab Classification (OVC) is performed using evaluation samples of screenshots and their textual class labels. Given a screenshot s and the label class $$C$$, the retrieval model needs to discriminate the correct label c from $$C$$ based on the embedding similarity. We include the following tasks in this category: product classification, news-topic classification, academic-field classification, knowledge classification. For each task, we employ human labelers to create the label class and assign each screenshot with its correct label.
 """
 
 EVALUATION_QUEUE_TEXT = """
@@ -67,6 +74,28 @@ Make sure you have followed the above steps first.
 If everything is done, check you can launch the EleutherAIHarness on your model locally, using the above command without modifications (you can add `--limit` to limit the number of examples per task).
 """
 
-CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
-CITATION_BUTTON_TEXT = r"""
+SUBMIT_FORM = """
+## Make sure you submit your evaluation results in a JSON file with the following format:
+```json
+{
+    "Model": "<Model Name>",
+    "#params": "7.11B",
+    "Overall": 30.00,
+    "SR": 30.00,
+    "CSR": 30.00,
+    "VQA": 30.00,
+    "OVC": 30.00,
+}
+```
+
+"""
+
+CITATION_BUTTON_LABEL = "Copy the following snippet to cite MVRB:"
+CITATION_BUTTON_TEXT = """
+@article{liu2025any,
+  title={Any Information Is Just Worth One Single Screenshot: Unifying Search With Visualized Information Retrieval},
+  author={Liu, Ze and Liang, Zhengyang and Zhou, Junjie and Liu, Zheng and Lian, Defu},
+  journal={arXiv preprint arXiv:2502.11431},
+  year={2025}
+}
 """

src/display/utils.py

@@ -23,22 +23,15 @@ class ColumnContent:
 ## Leaderboard columns
 auto_eval_column_dict = []
 # Init
-auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
+auto_eval_column_dict.append(["rank", ColumnContent, ColumnContent("Rank", "str", True, never_hidden=True)])
 auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+auto_eval_column_dict.append(["#params", ColumnContent, ColumnContent("#params(B)", "number", True)])
 #Scores
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
-for task in Tasks:
-    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
-# Model information
-auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
-auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
-auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
-auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
-auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
-auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
-auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
-auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
-auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
+auto_eval_column_dict.append(["overall", ColumnContent, ColumnContent("Overall", "number", True, never_hidden=True)])
+auto_eval_column_dict.append(["SR", ColumnContent, ColumnContent("SR", "number", True)])
+auto_eval_column_dict.append(["CSR", ColumnContent, ColumnContent("CSR", "number", True)])
+auto_eval_column_dict.append(["SQA", ColumnContent, ColumnContent("SQA", "number", True)])
+auto_eval_column_dict.append(["OVC", ColumnContent, ColumnContent("OVC", "number", True)])
 
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)