Some changes to validation logic.

app.py

@@ -3,6 +3,7 @@ import pandas as pd
 from apscheduler.schedulers.background import BackgroundScheduler
 from gradio_leaderboard import ColumnFilter, Leaderboard, SelectColumns
 
+from display.formatting import styled_error
 from src.about import CITATION_BUTTON_LABEL, CITATION_BUTTON_TEXT, EVALUATION_QUEUE_TEXT, INTRODUCTION_TEXT, TITLE
 from src.datamodel.data import F1Data
 from src.display.css_html_js import custom_css
@@ -11,21 +12,26 @@ from src.envs import API, CODE_PROBLEMS_REPO, REPO_ID, RESULTS_REPO, SUBMISSIONS
 from src.logger import get_logger
 from src.populate import get_leaderboard_df
 from src.submission.submit import add_new_solutions
+from src.validation.validate import MAX_INPUT_LENGTH, MIN_INPUT_LENGTH, is_submission_file_valid, is_valid
 
 logger = get_logger(__name__)
 
+ENSURE_ALL_PRESENT = False  # TODO: Switch to True.
 SPLIT = "warmup"  # TODO temp
-SKIP_VALIDATION = True  # TODO temp
 
 
 def restart_space():
     API.restart_space(repo_id=REPO_ID)
 
 
-lbdb = F1Data(cp_ds_name=CODE_PROBLEMS_REPO, sub_ds_name=SUBMISSIONS_REPO, res_ds_name=RESULTS_REPO, split=SPLIT)
+lbdb = F1Data(
+    cp_ds_name=CODE_PROBLEMS_REPO,
+    sub_ds_name=SUBMISSIONS_REPO,
+    res_ds_name=RESULTS_REPO,
+    split=SPLIT,
+)
 leaderboard_df = get_leaderboard_df(RESULTS_REPO)
 
-
 logger.info("Initialized LBDB")
 
 
@@ -117,9 +123,47 @@ with demo:
             submit_button = gr.Button("Submit")
             submission_result = gr.Markdown()
 
-            def add_solution_cbk(system_name, org, sys_type, submission_path):
+            def add_solution_cbk(
+                system_name: str,
+                org: str,
+                sys_type: str,
+                submission_path: str,
+            ):
+
+                try:
+                    # Validating the submission file.
+                    if len(submission_path) == 0:
+                        return styled_error("Please upload JSONL submission file.")
+
+                    if not is_submission_file_valid(submission_path):
+                        return styled_error("Failed to read JSONL submission file. Please try again later.")
+
+                    # Validating all user-supplied arguments.
+                    for val, val_name in [
+                        (system_name, "System name"),
+                        (org, "Organisation name"),
+                        (sys_type, "System type"),
+                    ]:
+                        if len(val) == 0:
+                            return styled_error(f"Please fill in the '{val_name}' field.")
+
+                        if not is_valid(val):
+                            return styled_error(
+                                f"{val_name} is invalid! Must only contain characters [a-zA-Z0-9], spaces, "
+                                + "or the special characters '-' and '.', and be of length between "
+                                + f"{MIN_INPUT_LENGTH} and {MAX_INPUT_LENGTH}."
+                            )
+                except Exception:
+                    logger.warning("Failed to process user submission", exc_info=True)
+                    return styled_error("An error occurred. Please try again later.")  # Intentionally vague.
+
                 return add_new_solutions(
-                    lbdb, system_name, org, sys_type, submission_path, skip_validation=SKIP_VALIDATION
+                    lbdb,
+                    system_name,
+                    org,
+                    sys_type,
+                    submission_path,
+                    ensure_all_present=ENSURE_ALL_PRESENT,
                 )
 
             submit_button.click(
@@ -140,13 +184,6 @@ with demo:
                 value=CITATION_BUTTON_TEXT.strip(),
                 elem_id="citation-block",
             )
-            # citation_button = gr.Textbox(
-            #     value=CITATION_BUTTON_TEXT,
-            #     # label=CITATION_BUTTON_LABEL,
-            #     lines=20,
-            #     elem_id="citation-button",
-            #     show_copy_button=True,
-            # )
 
 logger.info("Scheduler")
 scheduler = BackgroundScheduler()

src/display/utils.py

@@ -2,7 +2,7 @@ from dataclasses import dataclass
 from enum import Enum
 
 
-def _fields(raw_class):
+def fields(raw_class):
     return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
 
 
@@ -65,27 +65,12 @@ class ModelType(Enum):
         return ModelType.Other
 
 
-class WeightType(Enum):
-    Adapter = ModelDetails("Adapter")
-    Original = ModelDetails("Original")
-    Delta = ModelDetails("Delta")
-
-
 class Precision(Enum):
     float16 = ModelDetails("float16")
     bfloat16 = ModelDetails("bfloat16")
     Unknown = ModelDetails("?")
 
-    def from_str(precision):
-        if precision in ["torch.float16", "float16"]:
-            return Precision.float16
-        if precision in ["torch.bfloat16", "bfloat16"]:
-            return Precision.bfloat16
-        return Precision.Unknown
-
-
-# Column selection
-COLS = [c.name for c in _fields(AutoEvalColumn) if not c.hidden]
 
-EVAL_COLS = [c.name for c in _fields(EvalQueueColumn)]
-EVAL_TYPES = [c.type for c in _fields(EvalQueueColumn)]
+COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
+EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
+EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]

src/submission/submit.py

@@ -5,37 +5,40 @@ import pandas as pd
 from datasets import Dataset
 from pandas.api.types import is_integer_dtype
 
+from app import is_valid
 from src.datamodel.data import F1Data
 from src.display.formatting import styled_error, styled_message
 from src.display.utils import ModelType
 from src.envs import SUBMISSIONS_REPO
 from src.logger import get_logger
+from validation.validate import is_submission_file_valid
 
 logger = get_logger(__name__)
 
 
-def validate_submission(lbdb: F1Data, pd_ds: pd.DataFrame) -> str | None:
-    logger.info("Validating DS size %d columns %s set %s", len(pd_ds), pd_ds.columns, set(pd_ds.columns))
+def _validate_all_submissions_present(
+    lbdb: F1Data,
+    pd_ds: pd.DataFrame,
+):
+    logger.info(f"Validating DS size {len(pd_ds)} columns {pd_ds.columns} set {set(pd_ds.columns)}")
     expected_cols = ["problem_id", "solution"]
 
     if set(pd_ds.columns) != set(expected_cols):
-        return f"Expected attributes: {expected_cols}, Got: {pd_ds.columns.tolist()}"
+        return ValueError(f"Expected attributes: {expected_cols}, Got: {pd_ds.columns.tolist()}")
 
     if not is_integer_dtype(pd_ds["problem_id"]):
-        return "problem_id must be str convertible to int"
+        return ValueError("problem_id must be str convertible to int")
 
     if any(type(v) is not str for v in pd_ds["solution"]):
-        return "solution must be of type str"
+        return ValueError("solution must be of type str")
 
     submitted_ids = set(pd_ds.problem_id.astype(str))
     if submitted_ids != lbdb.code_problem_ids:
         missing = lbdb.code_problem_ids - submitted_ids
         unknown = submitted_ids - lbdb.code_problem_ids
-        return f"Mismatched problem IDs: {len(missing)} missing, {len(unknown)} unknown"
+        raise ValueError(f"Mismatched problem IDs: {len(missing)} missing, {len(unknown)} unknown")
     if len(pd_ds) > len(lbdb.code_problem_ids):
-        return "Duplicate problem IDs exist in uploaded file"
-
-    return None
+        return ValueError("Duplicate problem IDs exist in uploaded file")
 
 
 def add_new_solutions(
@@ -44,36 +47,33 @@ def add_new_solutions(
     org: str,
     sys_type: str,
     submission_path: str,
-    skip_validation: bool = False,
+    ensure_all_present: bool = False,
 ):
-    logger.info("ADD SUBMISSION! %s path %s", str((system_name, org, sys_type)), submission_path)
-    if not system_name:
-        return styled_error("Please fill system name")
+    logger.info(
+        f"Adding new submission! {system_name=}, {org=}, {sys_type=} and {submission_path=}",
+    )
 
-    if not org:
-        return styled_error("Please fill organization name")
+    # Double-checking.
+    for val in [system_name, org, sys_type]:
+        assert is_valid(val)
+    assert is_submission_file_valid(submission_path)
 
-    if not sys_type:
-        return styled_error("Please select system type")
     sys_type = ModelType.from_str(sys_type).name
 
-    if not submission_path:
-        return styled_error("Please upload JSONL solutions file")
-
     try:
         submission_df = pd.read_json(submission_path, lines=True)
-    except Exception as e:
-        return styled_error(f"Cannot read uploaded JSONL file: {str(e)}")
-
-    if not skip_validation:
-        validation_error = validate_submission(lbdb, submission_df)
-        if validation_error:
-            return styled_error(validation_error)
+        if ensure_all_present:
+            _validate_all_submissions_present(lbdb=lbdb, pd_ds=submission_df)
+    except Exception:
+        logger.warning("Failed to parse submission DF!", exc_info=True)
+        return styled_error(
+            "An error occurred. Please try again later."
+        )  # Use same message as external error. Avoid infoleak.
 
     submission_id = f"{system_name}_{org}_{sys_type}_{datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S')}"
 
-    # Seems good, creating the eval
-    print(f"Adding new submission: {submission_id}")
+    # Seems good, creating the eval.
+    logger.info(f"Adding new submission: {submission_id}")
     submission_ts = time.time_ns()
 
     def add_info(row):

src/validation/validate.py

@@ -0,0 +1,89 @@
+import json
+import os
+import string
+
+DATASET_SIZE = 120
+
+MIN_INPUT_LENGTH = 2
+MAX_INPUT_LENGTH = 20
+
+MIN_SUBMISSION_SIZE = 1
+MAX_SUBMISSION_SIZE = 1024 * 1024 * 120  # 120 MB.
+MAX_SINGLE_SUBMISSION_SIZE = 1024 * 1024  # 1MB.
+MAX_SUBMISSION_LINES = DATASET_SIZE + 1  # Allow empty line.
+
+
+def is_valid(
+    s: str,
+    min_length: int = MIN_INPUT_LENGTH,
+    max_length: int = MAX_INPUT_LENGTH,
+) -> bool:
+    """
+    @brief Checks whether the given string is valid.
+    @param s The string to validate.
+    @return True iff all characters are in [a-zA-Z0-9], spaces, or '.' and '-', and the length if between
+            min length and max length.
+    """
+
+    characters = [c for c in s]  # Not using the length from len(.) as that includes unicode characters.
+    if len(characters) < min_length or len(characters) > max_length:
+        return False
+
+    # Very important: We delimit using underscores. So these _CANNOT_ be allowed in sanitised strings.
+    ALLOWED = (
+        [c for c in string.ascii_lowercase]
+        + [c for c in string.ascii_uppercase]
+        + [c for c in string.digits]
+        + [" ", ".", "-"]
+    )
+    for c in s:
+        if c not in ALLOWED:
+            return False
+    return True
+
+
+def is_submission_file_valid(submission_path: str) -> bool:
+    """
+    @brief Checks whether the given submission file is valid.
+    @param submission_path The path to the submission file.
+    @return True iff the file is within the size constraints, a JSONL, and every line is no longer than
+            the fixed maximum bound.
+    """
+
+    if not os.path.exists(submission_path):
+        return False
+
+    submission_size = os.stat(submission_path).st_size
+    if submission_size < MIN_SUBMISSION_SIZE or submission_size > MAX_SUBMISSION_SIZE:
+        return False
+
+    with open(submission_path, "r") as f:
+
+        # Not using readlines() to avoid consuming a large buffer at once.
+        n_lines = 0
+        seen_ids = set()
+        while len(line := f.readline(MAX_SINGLE_SUBMISSION_SIZE)) > 0:
+            n_lines += 1
+            if n_lines > MAX_SUBMISSION_LINES:
+                return False
+
+            if not line.startswith("{") or not line.endswith("}"):
+                return False
+
+            d = json.loads(line)
+            if set(d.keys()) != set(["problem_id", "solution"]):
+                return False
+
+            if not ((type(d["problem_id"]) is str or type(d["problem_id"]) is int) and type(d["solution"] is str)):
+                return False
+            if not d["problem_id"].isdigit():
+                return False
+            problem_id = int(d["problem_id"])
+            if problem_id < 0 or problem_id >= DATASET_SIZE:
+                return False
+
+            if problem_id in seen_ids:
+                return False  # Duplicate submission.
+            seen_ids.add(problem_id)
+
+    return True