Merge branch 'gal-beniamini/cleanup'

README.md

@@ -41,9 +41,9 @@ If you encounter problem on the space, don't hesitate to restart it to remove th
 # Code logic for more complex edits
 
 You'll find 
-- the main table' columns names and properties in `src/display/utils.py`
-- the logic to read all results and request files, then convert them in dataframe lines, in `src/leaderboard/read_evals.py`, and `src/populate.py`
-- the logic to allow or filter submissions in `src/submission/submit.py` and `src/submission/check_validity.py`
+- The main table' columns names and properties in `src/display/utils.py`
+- The logic to read all results and request files, then convert them in dataframe lines, in `src/populate.py`
+- The logic to allow or filter submissions in `src/submission/submit.py`.
 
 
 # Setting up the environment

app.py

@@ -1,45 +1,34 @@
-import random
-
 import gradio as gr
-from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
 import pandas as pd
 from apscheduler.schedulers.background import BackgroundScheduler
+from gradio_leaderboard import ColumnFilter, Leaderboard, SelectColumns
 
-# from huggingface_hub import snapshot_download
-
-from src.about import (
-    CITATION_BUTTON_LABEL,
-    CITATION_BUTTON_TEXT,
-    EVALUATION_QUEUE_TEXT,
-    INTRODUCTION_TEXT,
-    LLM_BENCHMARKS_TEXT,
-    TITLE,
-)
+from display.formatting import styled_error
+from src.about import CITATION_BUTTON_LABEL, CITATION_BUTTON_TEXT, EVALUATION_QUEUE_TEXT, INTRODUCTION_TEXT, TITLE
 from src.datamodel.data import F1Data
-
 from src.display.css_html_js import custom_css
-
-from src.display.utils import (
-    # BENCHMARK_COLS,
-    COLS,
-    EVAL_COLS,
-    EVAL_TYPES,
-    AutoEvalColumn,
-    ModelType,
-    fields,
-    WeightType,
-    Precision,
-)
-from src.envs import API, REPO_ID, TOKEN, CODE_PROBLEMS_REPO, SUBMISSIONS_REPO, RESULTS_REPO
+from src.display.utils import AutoEvalColumn, ModelType, fields
+from src.envs import API, CODE_PROBLEMS_REPO, REPO_ID, RESULTS_REPO, SUBMISSIONS_REPO
 from src.logger import get_logger
-
-from src.populate import get_evaluation_queue_df, get_leaderboard_df
+from src.populate import get_leaderboard_df
 from src.submission.submit import add_new_solutions
+from src.validation.validate import MAX_INPUT_LENGTH, MIN_INPUT_LENGTH, is_submission_file_valid, is_valid
 
 logger = get_logger(__name__)
 
+ENSURE_ALL_PRESENT = False  # TODO: Switch to True.
 SPLIT = "warmup"  # TODO temp
-SKIP_VALIDATION = True  # TODO temp
+
+lbdb = F1Data(
+    cp_ds_name=CODE_PROBLEMS_REPO,
+    sub_ds_name=SUBMISSIONS_REPO,
+    res_ds_name=RESULTS_REPO,
+    split=SPLIT,
+)
+
+leaderboard_df = None
+
+logger.info("Initialized LBDB")
 
 
 def restart_space():
@@ -65,9 +54,11 @@ def refresh_leaderboard_data():
         return None
 
 
-def init_leaderboard(dataframe):
+def init_leaderboard(dataframe: pd.DataFrame):
+
     if dataframe is None or dataframe.empty:
         raise ValueError("Leaderboard DataFrame is empty or None.")
+
     return Leaderboard(
         value=dataframe,
         datatype=[c.type for c in fields(AutoEvalColumn)],
@@ -80,39 +71,14 @@ def init_leaderboard(dataframe):
         hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
         filter_columns=[
             ColumnFilter(AutoEvalColumn.system_type.name, type="checkboxgroup", label="Model types"),
-            # ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
-            # ColumnFilter(
-            #     AutoEvalColumn.params.name,
-            #     type="slider",
-            #     min=0.01,
-            #     max=150,
-            #     label="Select the number of parameters (B)",
-            # ),
-            # ColumnFilter(AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True),
         ],
         bool_checkboxgroup_label="Hide models",
         interactive=False,
     )
 
 
-lbdb = F1Data(cp_ds_name=CODE_PROBLEMS_REPO, sub_ds_name=SUBMISSIONS_REPO, res_ds_name=RESULTS_REPO, split=SPLIT)
-
-leaderboard_df = None
-
-logger.info("Initialized LBDB")
-
-# (
-#     finished_eval_queue_df,
-#     running_eval_queue_df,
-#     pending_eval_queue_df,
-# ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
-
-
-# Display image using Markdown
-# banner = "![Leaderboard Banner](file/assets/banner.png)"
-
-demo = gr.Blocks(css=custom_css)
-with demo:
+blocks = gr.Blocks(css=custom_css)
+with blocks:
     gr.Image(
         "assets/banner.png",
         interactive=False,
@@ -121,7 +87,6 @@ with demo:
         container=False,
     )
 
-    # gr.Markdown(banner)
     gr.HTML(
         """
         <style>
@@ -149,53 +114,15 @@ with demo:
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
         with gr.TabItem("🏅 FormulaOne Leaderboard", elem_id="formulaone-leaderboar-tab-table", id=0):
             refresh_leaderboard_data()  # updates leaderboard_df
+            assert leaderboard_df is not None
             leaderboard_component = init_leaderboard(leaderboard_df)
 
-        # with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=1):
-        #     logger.info("Tab about")
-        #     gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
-
         with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=2):
             logger.info("Tab submission")
             with gr.Column():
                 with gr.Row():
                     gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
 
-                # with gr.Column():
-                #     with gr.Accordion(
-                #         f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
-                #         open=False,
-                #     ):
-                #         with gr.Row():
-                #             finished_eval_table = gr.components.Dataframe(
-                #                 value=finished_eval_queue_df,
-                #                 headers=EVAL_COLS,
-                #                 datatype=EVAL_TYPES,
-                #                 row_count=5,
-                #             )
-                #     with gr.Accordion(
-                #         f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
-                #         open=False,
-                #     ):
-                #         with gr.Row():
-                #             running_eval_table = gr.components.Dataframe(
-                #                 value=running_eval_queue_df,
-                #                 headers=EVAL_COLS,
-                #                 datatype=EVAL_TYPES,
-                #                 row_count=5,
-                #             )
-
-                #     with gr.Accordion(
-                #         f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
-                #         open=False,
-                #     ):
-                #         with gr.Row():
-                #             pending_eval_table = gr.components.Dataframe(
-                #                 value=pending_eval_queue_df,
-                #                 headers=EVAL_COLS,
-                #                 datatype=EVAL_TYPES,
-                #                 row_count=5,
-                #             )
             with gr.Row():
                 gr.Markdown("# ✉️✨ Submit your solutions here!", elem_classes="markdown-text")
 
@@ -203,7 +130,6 @@ with demo:
                 with gr.Column():
                     system_name_textbox = gr.Textbox(label=AutoEvalColumn.system.name)
                     org_textbox = gr.Textbox(label=AutoEvalColumn.organization.name)
-                    # revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
                     sys_type_dropdown = gr.Dropdown(
                         choices=[t.to_str(" ") for t in ModelType],
                         label=AutoEvalColumn.system_type.name,
@@ -212,31 +138,53 @@ with demo:
                         interactive=True,
                     )
 
-                    # with gr.Column():
                     submission_file = gr.File(label="JSONL solutions file", file_types=[".jsonl"])
-                    # precision = gr.Dropdown(
-                    #     choices=[i.value.name for i in Precision if i != Precision.Unknown],
-                    #     label="Precision",
-                    #     multiselect=False,
-                    #     value="float16",
-                    #     interactive=True,
-                    # )
-                    # weight_type = gr.Dropdown(
-                    #     choices=[i.value.name for i in WeightType],
-                    #     label="Weights type",
-                    #     multiselect=False,
-                    #     value="Original",
-                    #     interactive=True,
-                    # )
-                    # base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
 
             logger.info("Submit button")
             submit_button = gr.Button("Submit")
             submission_result = gr.Markdown()
 
-            def add_solution_cbk(system_name, org, sys_type, submission_path):
+            def add_solution_cbk(
+                system_name: str,
+                org: str,
+                sys_type: str,
+                submission_path: str,
+            ):
+
+                try:
+                    # Validating the submission file.
+                    if len(submission_path) == 0:
+                        return styled_error("Please upload JSONL submission file.")
+
+                    if not is_submission_file_valid(submission_path):
+                        return styled_error("Failed to read JSONL submission file. Please try again later.")
+
+                    # Validating all user-supplied arguments.
+                    for val, val_name in [
+                        (system_name, "System name"),
+                        (org, "Organisation name"),
+                        (sys_type, "System type"),
+                    ]:
+                        if len(val) == 0:
+                            return styled_error(f"Please fill in the '{val_name}' field.")
+
+                        if not is_valid(val):
+                            return styled_error(
+                                f"{val_name} is invalid! Must only contain characters [a-zA-Z0-9], spaces, "
+                                + "or the special characters '-' and '.', and be of length between "
+                                + f"{MIN_INPUT_LENGTH} and {MAX_INPUT_LENGTH}."
+                            )
+                except Exception:
+                    logger.warning("Failed to process user submission", exc_info=True)
+                    return styled_error("An error occurred. Please try again later.")  # Intentionally vague.
+
                 return add_new_solutions(
-                    lbdb, system_name, org, sys_type, submission_path, skip_validation=SKIP_VALIDATION
+                    lbdb,
+                    system_name,
+                    org,
+                    sys_type,
+                    submission_path,
+                    ensure_all_present=ENSURE_ALL_PRESENT,
                 )
 
             submit_button.click(
@@ -257,16 +205,9 @@ with demo:
                 value=CITATION_BUTTON_TEXT.strip(),
                 elem_id="citation-block",
             )
-            # citation_button = gr.Textbox(
-            #     value=CITATION_BUTTON_TEXT,
-            #     # label=CITATION_BUTTON_LABEL,
-            #     lines=20,
-            #     elem_id="citation-button",
-            #     show_copy_button=True,
-            # )
 
     # UI refresh triggers latest data swap. The work already happened in the background - refresh_leaderboard_data().
-    demo.load(lambda: leaderboard_df, inputs=[], outputs=[leaderboard_component])
+    blocks.load(lambda: leaderboard_df, inputs=[], outputs=[leaderboard_component])
 
 
 logger.info("Scheduler")
@@ -275,5 +216,5 @@ scheduler.add_job(restart_space, "interval", seconds=1800)
 scheduler.add_job(refresh_leaderboard_data, "interval", seconds=120)
 scheduler.start()
 logger.info("Launch")
-demo.queue(default_concurrency_limit=40).launch()
+blocks.queue(default_concurrency_limit=40).launch()
 logger.info("Done")

scripts/upload_f1_dataset.py

@@ -2,6 +2,7 @@ import argparse
 import fnmatch
 import json
 import os
+from typing import Iterator
 
 from datasets import Dataset
 
@@ -13,9 +14,23 @@ logger = get_logger(__name__)
 
 def get_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-    parser.add_argument("--input_dir", type=str, help="Dir with .json files", required=True)
-    parser.add_argument("--dataset_name", type=str, default=f"{CODE_PROBLEMS_REPO}")
-    parser.add_argument("--split", type=str, choices=["hard", "warmup"], default="hard")
+    parser.add_argument(
+        "--input_dir",
+        type=str,
+        help="Dir with .json files",
+        required=True,
+    )
+    parser.add_argument(
+        "--dataset_name",
+        type=str,
+        default=f"{CODE_PROBLEMS_REPO}",
+    )
+    parser.add_argument(
+        "--split",
+        type=str,
+        choices=["hard", "warmup"],
+        default="hard",
+    )
     return parser.parse_args()
 
 
@@ -26,7 +41,7 @@ def main(args: argparse.Namespace) -> None:
         raise ValueError(f"No .json files in input dir {args.input_dir}")
     logger.info("Found %d code problems in %s", len(input_files), args.input_dir)
 
-    def ds_generator():
+    def ds_generator() -> Iterator[dict]:
         for fname in sorted(input_files):
             formula_name = os.path.splitext(fname)[0]
             cp_path = os.path.join(args.input_dir, fname)
@@ -35,7 +50,7 @@ def main(args: argparse.Namespace) -> None:
             logger.info("Read code problem for formula %s from %s", formula_name, cp_path)
             yield dict(id=code_problem["id"], code_problem=code_problem)
 
-    ds = Dataset.from_generator(ds_generator)
+    ds: Dataset = Dataset.from_generator(ds_generator)  # type: ignore
     logger.info("Created dataset")
 
     ds.push_to_hub(args.dataset_name, split=args.split, private=True)

src/about.py

@@ -9,20 +9,11 @@ class Task:
     col_name: str
 
 
-# Select your tasks here
-# ---------------------------------------------------
 class Tasks(Enum):
-    # task_key in the json file, metric_key in the json file, name to display in the leaderboard
     task0 = Task("FormulaOne", "success_rate", "Success Rate (%)")
-    # task1 = Task("logiqa", "acc_norm", "LogiQA")
 
 
-NUM_FEWSHOT = 0  # Change with your few shot
-# ---------------------------------------------------
-
-
-# Your leaderboard name
-# TITLE = """<h1 align="center" id="space-title">AAI FormulaOne Leaderboard</h1>"""
+NUM_FEWSHOT = 0
 
 TITLE = """
 <h1 id="space-title" style="
@@ -39,19 +30,17 @@ TITLE = """
 </h1>
 """
 
-# What does your leaderboard evaluate?
 INTRODUCTION_TEXT = """
 Welcome to the official leaderboard for the paper:
 
 **FormulaOne: Measuring the Depth of Algorithmic Reasoning Beyond Competitive Programming** <br>
-*Gal Beniamini, Yuval Dor, Alon Vinnikov, Shir Granot Peled, Or Weinstein, Or Sharir, Noam Wies, Tomer Nussbaum, Ido Ben Shaul, Tomer Zekharya, Yoav Levine, Shai Shalev-Shwartz, Amnon Shashua* <br>
+*Gal Beniamini, Yuval Dor, Alon Vinnikov, Shir Granot Peled, Or Weinstein, Or Sharir, Noam Wies, Tomer Nussbaum, Nadav Schweiger, Ido Ben Shaul, Tomer Zekharya, Yoav Levine, Shai Shalev-Shwartz, Amnon Shashua* <br>
 **AAI, July 2025**
 
 FormulaOne is a new benchmark designed to challenge frontier AI models. The benchmark is constructed from a vast and conceptually diverse family of dynamic programming problems derived from Monadic Second-Order (MSO) logic on graphs, a framework with profound connections to theoretical computer science.
 """
 
-# Which evaluations are you running? how can people reproduce what you have?
-LLM_BENCHMARKS_TEXT = f"""
+LLM_BENCHMARKS_TEXT = """
 ## How it works
 
 ## Reproducibility
@@ -95,7 +84,7 @@ Submissions must:
    - **Organization**
    - **System Type**
 -  Click **Submit**.
-	
+
 ### ⏱️ After Submission
 
 Submissions are validated and evaluated within ~24 hours. Results will appear on the leaderboard once processed.
@@ -105,12 +94,12 @@ Submissions are validated and evaluated within ~24 hours. Results will appear on
 CITATION_BUTTON_LABEL = """📚 How to cite FormulaOne"""
 CITATION_BUTTON_TEXT = r"""
 @misc{beniamini2025formulaonemeasuringdepthalgorithmic,
-      title={FormulaOne: Measuring the Depth of Algorithmic Reasoning Beyond Competitive Programming}, 
-      author={Gal Beniamini and Yuval Dor and Alon Vinnikov and Shir Granot Peled and Or Weinstein and Or Sharir and Noam Wies and Tomer Nussbaum and Ido Ben Shaul and Tomer Zekharya and Yoav Levine and Shai Shalev-Shwartz and Amnon Shashua},
+      title={FormulaOne: Measuring the Depth of Algorithmic Reasoning Beyond Competitive Programming},
+      author={Gal Beniamini and Yuval Dor and Alon Vinnikov and Shir Granot Peled and Or Weinstein and Or Sharir and Noam Wies and Tomer Nussbaum and Nadav Schweiger and Ido Ben Shaul and Tomer Zekharya and Yoav Levine and Shai Shalev-Shwartz and Amnon Shashua},
       year={2025},
       eprint={2507.13337},
       archivePrefix={arXiv},
       primaryClass={cs.AI},
-      url={https://arxiv.org/abs/2507.13337}, 
+      url={https://arxiv.org/abs/2507.13337},
 }
 """

src/datamodel/data.py

@@ -3,14 +3,20 @@ import time
 
 from datasets import load_dataset
 
-from src.envs import TOKEN, CODE_PROBLEMS_REPO, RESULTS_REPO, SUBMISSIONS_REPO
+from src.envs import CODE_PROBLEMS_REPO, RESULTS_REPO, SUBMISSIONS_REPO, TOKEN
 from src.logger import get_logger
 
 logger = get_logger(__name__)
 
 
 class F1Data:
-    def __init__(self, cp_ds_name: str, sub_ds_name: str, res_ds_name: str, split: str = "hard"):
+    def __init__(
+        self,
+        cp_ds_name: str,  # Name of the dataset. Fixed.
+        sub_ds_name: str,  # Name of subdataset. Fixed.
+        res_ds_name: str,  # Name of results repository. Fixed.
+        split: str = "hard",  # Split is either 'hard' or 'easy'.
+    ):
         self.cp_dataset_name = cp_ds_name
         self.submissions_dataset_name = sub_ds_name
         self.results_dataset_name = res_ds_name
@@ -19,16 +25,16 @@ class F1Data:
         self._initialize()
 
     def _initialize(self):
-        logger.info("Initialize F1Data TOKEN='%s'", TOKEN)
+        logger.info(f"Initialize F1Data TOKEN='{TOKEN}'")
         start_time = time.monotonic()
-        cp_ds = load_dataset(self.cp_dataset_name, split=self.split, token=TOKEN)
-        logger.info(
-            "Loaded code-problems dataset from %s in %f sec",
+        cp_ds = load_dataset(
             self.cp_dataset_name,
-            time.monotonic() - start_time,
+            split=self.split,
+            token=TOKEN,
         )
-        self.code_problems: dict[str, str] = {r["id"]: r["code_problem"] for r in cp_ds}
-        logger.info(f"Loaded %d code problems {len(self.code_problems)}")
+        logger.info(f"Loaded code-problems dataset from {self.cp_dataset_name} in {time.monotonic() - start_time} sec")
+        self.code_problems = {r["id"]: r["code_problem"] for r in cp_ds}  # id string -> code problem.
+        logger.info(f"Loaded {len(self.code_problems)} code problems")
 
     @functools.cached_property
     def code_problem_ids(self) -> set[str]:
@@ -37,6 +43,11 @@ class F1Data:
 
 if __name__ == "__main__":
     split = "hard"
-    f1_data = F1Data(cp_ds_name=CODE_PROBLEMS_REPO, sub_ds_name=SUBMISSIONS_REPO, res_ds_name=RESULTS_REPO, split=split)
+    f1_data = F1Data(
+        cp_ds_name=CODE_PROBLEMS_REPO,
+        sub_ds_name=SUBMISSIONS_REPO,
+        res_ds_name=RESULTS_REPO,
+        split=split,
+    )
 
     print(f"Found {len(f1_data.code_problem_ids)} code problems in {split} split of {f1_data.cp_dataset_name}")

src/display/css_html_js.py

@@ -33,7 +33,7 @@ custom_css = """
     background: none;
     border: none;
 }
- 
+
 #search-bar {
     padding: 0px;
 }

src/display/formatting.py

@@ -1,12 +1,3 @@
-def model_hyperlink(link, model_name):
-    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
-
-
-def make_clickable_model(model_name):
-    link = f"https://huggingface.co/{model_name}"
-    return model_hyperlink(link, model_name)
-
-
 def styled_error(error):
     return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
 
@@ -17,11 +8,3 @@ def styled_warning(warn):
 
 def styled_message(message):
     return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
-
-
-def has_no_nan_values(df, columns):
-    return df[columns].notna().all(axis=1)
-
-
-def has_nan_values(df, columns):
-    return df[columns].isna().any(axis=1)

src/display/utils.py

@@ -1,19 +1,15 @@
-from dataclasses import dataclass, field, make_dataclass
-from typing import ClassVar
+from dataclasses import dataclass
 from enum import Enum
 
-import pandas as pd
-
-from src.about import Tasks
-
 
 def fields(raw_class):
     return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
 
 
-# These classes are for user facing column names,
-# to avoid having to change them all around the code
-# when a modif is needed
+# These classes are for user facing column names, to avoid having to change them
+# all around the code when a modification is needed.
+
+
 @dataclass
 class ColumnContent:
     name: str
@@ -23,41 +19,6 @@ class ColumnContent:
     never_hidden: bool = False
 
 
-## Leaderboard columns
-# auto_eval_column_fields = []
-# # Init
-# auto_eval_column_fields.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
-# auto_eval_column_fields.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
-# # Scores
-# auto_eval_column_fields.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
-# for task in Tasks:
-#     auto_eval_column_fields.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
-# # Model information
-# auto_eval_column_fields.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
-# auto_eval_column_fields.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
-# auto_eval_column_fields.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
-# auto_eval_column_fields.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
-# auto_eval_column_fields.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
-# auto_eval_column_fields.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
-# auto_eval_column_fields.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
-# auto_eval_column_fields.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
-# auto_eval_column_fields.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
-#
-#
-#
-# def make_classvar_dataclass(name: str, spec: list):
-#     ns = {"__annotations__": {}}
-#     for field_name, field_type, default in spec:
-#         # Mark as ClassVar so dataclass doesn't treat it as an instance field
-#         ns["__annotations__"][field_name] = ClassVar[field_type]
-#         ns[field_name] = default
-#     # No instance fields; just class-level descriptors
-#     return make_dataclass(name, [], frozen=True, namespace=ns)
-#
-# # We use make dataclass to dynamically fill the scores from Tasks
-# AutoEvalColumn = make_classvar_dataclass("AutoEvalColumn", auto_eval_column_fields)
-
-
 @dataclass(frozen=True)
 class AutoEvalColumn:
     system = ColumnContent("System Name", "markdown", True, never_hidden=True)
@@ -68,18 +29,18 @@ class AutoEvalColumn:
     submitted_on = ColumnContent("Submitted On", "datetime", True)
 
 
-## For the queue columns in the submission tab
+# For the queue columns in the submission tab
 @dataclass(frozen=True)
 class EvalQueueColumn:  # Queue column
     model = ColumnContent("model", "markdown", True)
     revision = ColumnContent("revision", "str", True)
     private = ColumnContent("private", "bool", True)
     precision = ColumnContent("precision", "str", True)
-    weight_type = ColumnContent("weight_type", "str", "Original")
+    weight_type = ColumnContent("weight_type", "str", True)
     status = ColumnContent("status", "str", True)
 
 
-## All the model information that we might need
+# All the model information that we might need
 @dataclass
 class ModelDetails:
     name: str
@@ -90,8 +51,6 @@ class ModelDetails:
 class ModelType(Enum):
     LLM = ModelDetails(name="LLM", symbol="🟢")
     AgenticLLM = ModelDetails(name="AgenticLLM", symbol="🔶")
-    # IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
-    # RL = ModelDetails(name="RL-tuned", symbol="🟦")
     Other = ModelDetails(name="Other", symbol="?")
 
     def to_str(self, separator=" "):
@@ -103,36 +62,15 @@ class ModelType(Enum):
             return ModelType.AgenticLLM
         if "LLM" in type or "🟢" in type:
             return ModelType.LLM
-        # if "RL-tuned" in type or "🟦" in type:
-        #     return ModelType.RL
-        # if "instruction-tuned" in type or "⭕" in type:
-        #     return ModelType.IFT
         return ModelType.Other
 
 
-class WeightType(Enum):
-    Adapter = ModelDetails("Adapter")
-    Original = ModelDetails("Original")
-    Delta = ModelDetails("Delta")
-
-
 class Precision(Enum):
     float16 = ModelDetails("float16")
     bfloat16 = ModelDetails("bfloat16")
     Unknown = ModelDetails("?")
 
-    def from_str(precision):
-        if precision in ["torch.float16", "float16"]:
-            return Precision.float16
-        if precision in ["torch.bfloat16", "bfloat16"]:
-            return Precision.bfloat16
-        return Precision.Unknown
-
 
-# Column selection
 COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
-
 EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
 EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
-
-# BENCHMARK_COLS = [t.value.col_name for t in Tasks]

src/leaderboard/read_evals.py

@@ -1,196 +0,0 @@
-import glob
-import json
-import math
-import os
-from dataclasses import dataclass
-
-import dateutil
-import numpy as np
-
-from src.display.formatting import make_clickable_model
-from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
-from src.submission.check_validity import is_model_on_hub
-
-
-@dataclass
-class EvalResult:
-    """Represents one full evaluation. Built from a combination of the result and request file for a given run.
-    """
-    eval_name: str # org_model_precision (uid)
-    full_model: str # org/model (path on hub)
-    org: str 
-    model: str
-    revision: str # commit hash, "" if main
-    results: dict
-    precision: Precision = Precision.Unknown
-    model_type: ModelType = ModelType.LLM # Pretrained, fine tuned, ...
-    weight_type: WeightType = WeightType.Original # Original or Adapter
-    architecture: str = "Unknown" 
-    license: str = "?"
-    likes: int = 0
-    num_params: int = 0
-    date: str = "" # submission date of request file
-    still_on_hub: bool = False
-
-    @classmethod
-    def init_from_json_file(self, json_filepath):
-        """Inits the result from the specific model result file"""
-        with open(json_filepath) as fp:
-            data = json.load(fp)
-
-        config = data.get("config")
-
-        # Precision
-        precision = Precision.from_str(config.get("model_dtype"))
-
-        # Get model and org
-        org_and_model = config.get("model_name", config.get("model_args", None))
-        org_and_model = org_and_model.split("/", 1)
-
-        if len(org_and_model) == 1:
-            org = None
-            model = org_and_model[0]
-            result_key = f"{model}_{precision.value.name}"
-        else:
-            org = org_and_model[0]
-            model = org_and_model[1]
-            result_key = f"{org}_{model}_{precision.value.name}"
-        full_model = "/".join(org_and_model)
-
-        still_on_hub, _, model_config = is_model_on_hub(
-            full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
-        )
-        architecture = "?"
-        if model_config is not None:
-            architectures = getattr(model_config, "architectures", None)
-            if architectures:
-                architecture = ";".join(architectures)
-
-        # Extract results available in this file (some results are split in several files)
-        results = {}
-        for task in Tasks:
-            task = task.value
-
-            # We average all scores of a given metric (not all metrics are present in all files)
-            accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
-            if accs.size == 0 or any([acc is None for acc in accs]):
-                continue
-
-            mean_acc = np.mean(accs) * 100.0
-            results[task.benchmark] = mean_acc
-
-        return self(
-            eval_name=result_key,
-            full_model=full_model,
-            org=org,
-            model=model,
-            results=results,
-            precision=precision,  
-            revision= config.get("model_sha", ""),
-            still_on_hub=still_on_hub,
-            architecture=architecture
-        )
-
-    def update_with_request_file(self, requests_path):
-        """Finds the relevant request file for the current model and updates info with it"""
-        request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name)
-
-        try:
-            with open(request_file, "r") as f:
-                request = json.load(f)
-            self.model_type = ModelType.from_str(request.get("model_type", ""))
-            self.weight_type = WeightType[request.get("weight_type", "Original")]
-            self.license = request.get("license", "?")
-            self.likes = request.get("likes", 0)
-            self.num_params = request.get("params", 0)
-            self.date = request.get("submitted_time", "")
-        except Exception:
-            print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}")
-
-    def to_dict(self):
-        """Converts the Eval Result to a dict compatible with our dataframe display"""
-        average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
-        data_dict = {
-            "eval_name": self.eval_name,  # not a column, just a save name,
-            AutoEvalColumn.precision.name: self.precision.value.name,
-            AutoEvalColumn.model_type.name: self.model_type.value.name,
-            AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
-            AutoEvalColumn.weight_type.name: self.weight_type.value.name,
-            AutoEvalColumn.architecture.name: self.architecture,
-            AutoEvalColumn.model.name: make_clickable_model(self.full_model),
-            AutoEvalColumn.revision.name: self.revision,
-            AutoEvalColumn.average.name: average,
-            AutoEvalColumn.license.name: self.license,
-            AutoEvalColumn.likes.name: self.likes,
-            AutoEvalColumn.params.name: self.num_params,
-            AutoEvalColumn.still_on_hub.name: self.still_on_hub,
-        }
-
-        for task in Tasks:
-            data_dict[task.value.col_name] = self.results[task.value.benchmark]
-
-        return data_dict
-
-
-def get_request_file_for_model(requests_path, model_name, precision):
-    """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
-    request_files = os.path.join(
-        requests_path,
-        f"{model_name}_eval_request_*.json",
-    )
-    request_files = glob.glob(request_files)
-
-    # Select correct request file (precision)
-    request_file = ""
-    request_files = sorted(request_files, reverse=True)
-    for tmp_request_file in request_files:
-        with open(tmp_request_file, "r") as f:
-            req_content = json.load(f)
-            if (
-                req_content["status"] in ["FINISHED"]
-                and req_content["precision"] == precision.split(".")[-1]
-            ):
-                request_file = tmp_request_file
-    return request_file
-
-
-def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
-    """From the path of the results folder root, extract all needed info for results"""
-    model_result_filepaths = []
-
-    for root, _, files in os.walk(results_path):
-        # We should only have json files in model results
-        if len(files) == 0 or any([not f.endswith(".json") for f in files]):
-            continue
-
-        # Sort the files by date
-        try:
-            files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
-        except dateutil.parser._parser.ParserError:
-            files = [files[-1]]
-
-        for file in files:
-            model_result_filepaths.append(os.path.join(root, file))
-
-    eval_results = {}
-    for model_result_filepath in model_result_filepaths:
-        # Creation of result
-        eval_result = EvalResult.init_from_json_file(model_result_filepath)
-        eval_result.update_with_request_file(requests_path)
-
-        # Store results of same eval together
-        eval_name = eval_result.eval_name
-        if eval_name in eval_results.keys():
-            eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
-        else:
-            eval_results[eval_name] = eval_result
-
-    results = []
-    for v in eval_results.values():
-        try:
-            v.to_dict() # we test if the dict version is complete
-            results.append(v)
-        except KeyError:  # not all eval values present
-            continue
-
-    return results

src/logger.py

@@ -1,7 +1,11 @@
 import logging
 import sys
 
-def get_logger(filename: str, level=logging.INFO) -> logging.Logger:
+
+def get_logger(
+    filename: str,
+    level=logging.INFO,
+) -> logging.Logger:
     new_logger = logging.getLogger(filename)
     fmt = logging.Formatter(fmt="%(asctime)s - %(name)s - %(levelname)s - %(funcName)s:%(lineno)d - %(message)s")
     handler = logging.StreamHandler(sys.stderr)

src/populate.py

@@ -1,27 +1,29 @@
-import json
-import os
-
 import pandas as pd
-from datasets import load_dataset, get_dataset_config_names
+from datasets import get_dataset_config_names, load_dataset
 from datasets.exceptions import DatasetNotFoundError
 from tqdm.auto import tqdm
 
-from src.display.formatting import has_no_nan_values, make_clickable_model
-from src.display.utils import AutoEvalColumn, EvalQueueColumn
+from src.display.utils import AutoEvalColumn
 from src.envs import TOKEN
-from src.leaderboard.read_evals import get_raw_eval_results
 from src.logger import get_logger
 
 logger = get_logger(__name__)
 
 
 def get_leaderboard_df(results_dataset_name: str) -> pd.DataFrame:
-    """Creates a dataframe from all the individual experiment results"""
+    """
+    @brief Creates a dataframe from all the individual experiment results.
+    """
 
     try:
-        configs = get_dataset_config_names(results_dataset_name, token=TOKEN)
+        configs = get_dataset_config_names(
+            results_dataset_name,
+            token=TOKEN,
+        )
     except (DatasetNotFoundError, FileNotFoundError):
+
         # Return an empty DataFrame with expected columns
+        logger.warning("Failed to load configuration", exc_info=True)
         return pd.DataFrame(
             columns=[
                 "System Name",
@@ -34,8 +36,17 @@ def get_leaderboard_df(results_dataset_name: str) -> pd.DataFrame:
         )
 
     rows = []
-    for submission_id in tqdm(configs, total=len(configs), desc="Processing Submission Results"):
-        submission_ds = load_dataset(results_dataset_name, submission_id, split="train", token=TOKEN)
+    for submission_id in tqdm(
+        configs,
+        total=len(configs),
+        desc="Processing Submission Results",
+    ):
+        submission_ds = load_dataset(
+            results_dataset_name,
+            submission_id,
+            split="train",
+            token=TOKEN,
+        )
         submission_df = pd.DataFrame(submission_ds)
 
         if submission_df.empty or "did_pass" not in submission_df.columns or submission_df.did_pass.isna().any():
@@ -59,7 +70,7 @@ def get_leaderboard_df(results_dataset_name: str) -> pd.DataFrame:
 
     full_df = pd.DataFrame(rows)
 
-    # TODO: forbid multiple submissions under the same name?
+    # TODO: Forbid multiple submissions under the same name?
     # Keep only the latest entry per unique (System Name, System Type, Organization) triplet
     final_df = (
         full_df.sort_values("Submitted On", ascending=False)
@@ -72,39 +83,3 @@ def get_leaderboard_df(results_dataset_name: str) -> pd.DataFrame:
     final_df[cols_to_round] = final_df[cols_to_round].round(decimals=2)
 
     return final_df
-
-
-def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
-    """Creates the different dataframes for the evaluation queues requestes"""
-    entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
-    all_evals = []
-
-    for entry in entries:
-        if ".json" in entry:
-            file_path = os.path.join(save_path, entry)
-            with open(file_path) as fp:
-                data = json.load(fp)
-
-            data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-            data[EvalQueueColumn.revision.name] = data.get("revision", "main")
-
-            all_evals.append(data)
-        elif ".md" not in entry:
-            # this is a folder
-            sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if os.path.isfile(e) and not e.startswith(".")]
-            for sub_entry in sub_entries:
-                file_path = os.path.join(save_path, entry, sub_entry)
-                with open(file_path) as fp:
-                    data = json.load(fp)
-
-                data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-                data[EvalQueueColumn.revision.name] = data.get("revision", "main")
-                all_evals.append(data)
-
-    pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
-    running_list = [e for e in all_evals if e["status"] == "RUNNING"]
-    finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
-    df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
-    df_running = pd.DataFrame.from_records(running_list, columns=cols)
-    df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
-    return df_finished[cols], df_running[cols], df_pending[cols]

src/submission/check_validity.py

@@ -1,102 +0,0 @@
-import json
-import os
-import re
-from collections import defaultdict
-from datetime import datetime, timedelta, timezone
-
-from datasets import get_dataset_config_names
-import huggingface_hub
-from huggingface_hub import ModelCard
-from huggingface_hub.hf_api import ModelInfo
-from transformers import AutoConfig
-from transformers.models.auto.tokenization_auto import AutoTokenizer
-
-from src.envs import SUBMISSIONS_REPO
-
-def check_model_card(repo_id: str) -> tuple[bool, str]:
-    """Checks if the model card and license exist and have been filled"""
-    try:
-        card = ModelCard.load(repo_id)
-    except huggingface_hub.utils.EntryNotFoundError:
-        return False, "Please add a model card to your model to explain how you trained/fine-tuned it."
-
-    # Enforce license metadata
-    if card.data.license is None:
-        if not ("license_name" in card.data and "license_link" in card.data):
-            return False, (
-                "License not found. Please add a license to your model card using the `license` metadata or a"
-                " `license_name`/`license_link` pair."
-            )
-
-    # Enforce card content
-    if len(card.text) < 200:
-        return False, "Please add a description to your model card, it is too short."
-
-    return True, ""
-
-def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str]:
-    """Checks if the model model_name is on the hub, and whether it (and its tokenizer) can be loaded with AutoClasses."""
-    try:
-        config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
-        if test_tokenizer:
-            try:
-                tk = AutoTokenizer.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
-            except ValueError as e:
-                return (
-                    False,
-                    f"uses a tokenizer which is not in a transformers release: {e}",
-                    None
-                )
-            except Exception as e:
-                return (False, "'s tokenizer cannot be loaded. Is your tokenizer class in a stable transformers release, and correctly configured?", None)
-        return True, None, config
-
-    except ValueError:
-        return (
-            False,
-            "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.",
-            None
-        )
-
-    except Exception as e:
-        return False, "was not found on hub!", None
-
-
-def get_model_size(model_info: ModelInfo, precision: str):
-    """Gets the model size from the configuration, or the model name if the configuration does not contain the information."""
-    try:
-        model_size = round(model_info.safetensors["total"] / 1e9, 3)
-    except (AttributeError, TypeError):
-        return 0  # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
-
-    size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1
-    model_size = size_factor * model_size
-    return model_size
-
-def get_model_arch(model_info: ModelInfo):
-    """Gets the model architecture from the configuration"""
-    return model_info.config.get("architectures", "Unknown")
-
-def already_submitted_models(requested_models_dir: str) -> set[str]:
-    """Gather a list of already submitted models to avoid duplicates"""
-    depth = 1
-    file_names = []
-    users_to_submission_dates = defaultdict(list)
-
-    for root, _, files in os.walk(requested_models_dir):
-        current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
-        if current_depth == depth:
-            for file in files:
-                if not file.endswith(".json"):
-                    continue
-                with open(os.path.join(root, file), "r") as f:
-                    info = json.load(f)
-                    file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}")
-
-                    # Select organisation
-                    if info["model"].count("/") == 0 or "submitted_time" not in info:
-                        continue
-                    organisation, _ = info["model"].split("/")
-                    users_to_submission_dates[organisation].append(info["submitted_time"])
-
-    return set(file_names), users_to_submission_dates

src/submission/submit.py

@@ -1,50 +1,44 @@
-import json
-import os
-from datetime import datetime, timezone
 import time
+from datetime import datetime, timezone
 
-from datasets import Dataset, DatasetDict
 import pandas as pd
-from pandas.api.types import is_integer_dtype, is_string_dtype
+from datasets import Dataset
+from pandas.api.types import is_integer_dtype
 
+from app import is_valid
 from src.datamodel.data import F1Data
-from src.display.formatting import styled_error, styled_message, styled_warning
+from src.display.formatting import styled_error, styled_message
 from src.display.utils import ModelType
-from src.envs import API, SUBMISSIONS_REPO, TOKEN
+from src.envs import SUBMISSIONS_REPO
 from src.logger import get_logger
-
-# from src.submission.check_validity import (
-#     already_submitted_models,
-#     check_model_card,
-#     get_model_size,
-#     is_model_on_hub,
-# )
+from validation.validate import is_submission_file_valid
 
 logger = get_logger(__name__)
 
 
-def validate_submission(lbdb: F1Data, pd_ds: pd.DataFrame) -> str | None:
-    logger.info("Validating DS size %d columns %s set %s", len(pd_ds), pd_ds.columns, set(pd_ds.columns))
+def _validate_all_submissions_present(
+    lbdb: F1Data,
+    pd_ds: pd.DataFrame,
+):
+    logger.info(f"Validating DS size {len(pd_ds)} columns {pd_ds.columns} set {set(pd_ds.columns)}")
     expected_cols = ["problem_id", "solution"]
 
     if set(pd_ds.columns) != set(expected_cols):
-        return f"Expected attributes: {expected_cols}, Got: {pd_ds.columns.tolist()}"
+        return ValueError(f"Expected attributes: {expected_cols}, Got: {pd_ds.columns.tolist()}")
 
     if not is_integer_dtype(pd_ds["problem_id"]):
-        return "problem_id must be str convertible to int"
+        return ValueError("problem_id must be str convertible to int")
 
-    if any(type(v) != str for v in pd_ds["solution"]):
-        return "solution must be of type str"
+    if any(type(v) is not str for v in pd_ds["solution"]):
+        return ValueError("solution must be of type str")
 
     submitted_ids = set(pd_ds.problem_id.astype(str))
     if submitted_ids != lbdb.code_problem_ids:
         missing = lbdb.code_problem_ids - submitted_ids
         unknown = submitted_ids - lbdb.code_problem_ids
-        return f"Mismatched problem IDs: {len(missing)} missing, {len(unknown)} unknown"
+        raise ValueError(f"Mismatched problem IDs: {len(missing)} missing, {len(unknown)} unknown")
     if len(pd_ds) > len(lbdb.code_problem_ids):
-        return "Duplicate problem IDs exist in uploaded file"
-
-    return None
+        return ValueError("Duplicate problem IDs exist in uploaded file")
 
 
 def add_new_solutions(
@@ -53,36 +47,33 @@ def add_new_solutions(
     org: str,
     sys_type: str,
     submission_path: str,
-    skip_validation: bool = False,
+    ensure_all_present: bool = False,
 ):
-    logger.info("ADD SUBMISSION! %s path %s", str((system_name, org, sys_type)), submission_path)
-    if not system_name:
-        return styled_error("Please fill system name")
+    logger.info(
+        f"Adding new submission! {system_name=}, {org=}, {sys_type=} and {submission_path=}",
+    )
 
-    if not org:
-        return styled_error("Please fill organization name")
+    # Double-checking.
+    for val in [system_name, org, sys_type]:
+        assert is_valid(val)
+    assert is_submission_file_valid(submission_path)
 
-    if not sys_type:
-        return styled_error("Please select system type")
     sys_type = ModelType.from_str(sys_type).name
 
-    if not submission_path:
-        return styled_error("Please upload JSONL solutions file")
-
     try:
         submission_df = pd.read_json(submission_path, lines=True)
-    except Exception as e:
-        return styled_error(f"Cannot read uploaded JSONL file: {str(e)}")
-
-    if not skip_validation:
-        validation_error = validate_submission(lbdb, submission_df)
-        if validation_error:
-            return styled_error(validation_error)
+        if ensure_all_present:
+            _validate_all_submissions_present(lbdb=lbdb, pd_ds=submission_df)
+    except Exception:
+        logger.warning("Failed to parse submission DF!", exc_info=True)
+        return styled_error(
+            "An error occurred. Please try again later."
+        )  # Use same message as external error. Avoid infoleak.
 
     submission_id = f"{system_name}_{org}_{sys_type}_{datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S')}"
 
-    # Seems good, creating the eval
-    print(f"Adding new submission: {submission_id}")
+    # Seems good, creating the eval.
+    logger.info(f"Adding new submission: {submission_id}")
     submission_ts = time.time_ns()
 
     def add_info(row):
@@ -96,31 +87,13 @@ def add_new_solutions(
         }
 
     ds = Dataset.from_pandas(submission_df).map(add_info)
-
-    # dsdict = DatasetDict({submission_id: ds})
-    # dsdict.push_to_hub(SUBMISSIONS_REPO, private=True)
-
-    ds.push_to_hub(SUBMISSIONS_REPO, submission_id, private=True)
-    # print("Creating eval file")
-    # OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
-    # os.makedirs(OUT_DIR, exist_ok=True)
-    # out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
-
-    # with open(out_path, "w") as f:
-    #     f.write(json.dumps(eval_entry))
-
-    # print("Uploading eval file")
-    # API.upload_file(
-    #     path_or_fileobj=out_path,
-    #     path_in_repo=out_path.split("eval-queue/")[1],
-    #     repo_id=QUEUE_REPO,
-    #     repo_type="dataset",
-    #     commit_message=f"Add {model} to eval queue",
-    # )
-
-    # # Remove the local file
-    # os.remove(out_path)
+    ds.push_to_hub(
+        SUBMISSIONS_REPO,
+        submission_id,
+        private=True,
+    )
 
     return styled_message(
-        "Your request has been submitted to the evaluation queue!\nResults may take up to 24 hours to be processed and shown in the leaderboard."
+        "Your request has been submitted to the evaluation queue!\n"
+        + "Results may take up to 24 hours to be processed and shown in the leaderboard."
     )

src/validation/validate.py

@@ -0,0 +1,89 @@
+import json
+import os
+import string
+
+DATASET_SIZE = 120
+
+MIN_INPUT_LENGTH = 2
+MAX_INPUT_LENGTH = 20
+
+MIN_SUBMISSION_SIZE = 1
+MAX_SUBMISSION_SIZE = 1024 * 1024 * 120  # 120 MB.
+MAX_SINGLE_SUBMISSION_SIZE = 1024 * 1024  # 1MB.
+MAX_SUBMISSION_LINES = DATASET_SIZE + 1  # Allow empty line.
+
+
+def is_valid(
+    s: str,
+    min_length: int = MIN_INPUT_LENGTH,
+    max_length: int = MAX_INPUT_LENGTH,
+) -> bool:
+    """
+    @brief Checks whether the given string is valid.
+    @param s The string to validate.
+    @return True iff all characters are in [a-zA-Z0-9], spaces, or '.' and '-', and the length if between
+            min length and max length.
+    """
+
+    characters = [c for c in s]  # Not using the length from len(.) as that includes unicode characters.
+    if len(characters) < min_length or len(characters) > max_length:
+        return False
+
+    # Very important: We delimit using underscores. So these _CANNOT_ be allowed in sanitised strings.
+    ALLOWED = (
+        [c for c in string.ascii_lowercase]
+        + [c for c in string.ascii_uppercase]
+        + [c for c in string.digits]
+        + [" ", ".", "-"]
+    )
+    for c in s:
+        if c not in ALLOWED:
+            return False
+    return True
+
+
+def is_submission_file_valid(submission_path: str) -> bool:
+    """
+    @brief Checks whether the given submission file is valid.
+    @param submission_path The path to the submission file.
+    @return True iff the file is within the size constraints, a JSONL, and every line is no longer than
+            the fixed maximum bound.
+    """
+
+    if not os.path.exists(submission_path):
+        return False
+
+    submission_size = os.stat(submission_path).st_size
+    if submission_size < MIN_SUBMISSION_SIZE or submission_size > MAX_SUBMISSION_SIZE:
+        return False
+
+    with open(submission_path, "r") as f:
+
+        # Not using readlines() to avoid consuming a large buffer at once.
+        n_lines = 0
+        seen_ids = set()
+        while len(line := f.readline(MAX_SINGLE_SUBMISSION_SIZE)) > 0:
+            n_lines += 1
+            if n_lines > MAX_SUBMISSION_LINES:
+                return False
+
+            if not line.startswith("{") or not line.endswith("}"):
+                return False
+
+            d = json.loads(line)
+            if set(d.keys()) != set(["problem_id", "solution"]):
+                return False
+
+            if not ((type(d["problem_id"]) is str or type(d["problem_id"]) is int) and type(d["solution"] is str)):
+                return False
+            if not d["problem_id"].isdigit():
+                return False
+            problem_id = int(d["problem_id"])
+            if problem_id < 0 or problem_id >= DATASET_SIZE:
+                return False
+
+            if problem_id in seen_ids:
+                return False  # Duplicate submission.
+            seen_ids.add(problem_id)
+
+    return True