| """Accept named score fields only; never execute or extract uploaded files.""" |
| import copy |
| import math |
| import re |
| from urllib.parse import urlsplit |
|
|
|
|
| class Invalid(ValueError): |
| pass |
|
|
|
|
| def text(value, label, maximum=200, required=True): |
| if not isinstance(value, str) or len(value.strip()) > maximum: |
| raise Invalid(f"{label} must be text, at most {maximum} characters.") |
| value = value.strip() |
| if (required and not value) or any(ord(c) < 32 and c not in '\n\t' for c in value): |
| raise Invalid(f"Please provide a valid {label.lower()}.") |
| return value |
|
|
|
|
| def url(value, label): |
| value = text(value, label, 1000) |
| try: |
| parsed = urlsplit(value) |
| if parsed.scheme != "https" or not parsed.hostname or parsed.username or parsed.password: |
| raise ValueError() |
| except ValueError: |
| raise Invalid(f"{label} must be an https:// link without credentials.") |
| return value |
|
|
|
|
| def score(value, label): |
| if value is None: |
| return None |
| if type(value) not in (int, float) or not math.isfinite(value) or not 0 <= value <= 100: |
| raise Invalid(f"{label} must be a number between 0 and 100, or null.") |
| if abs(value * 10 - round(value * 10)) > 1e-7: |
| raise Invalid(f"{label} must use at most one decimal place, as in the published tables.") |
| return float(value) |
|
|
|
|
| def simple_scores(raw, columns, label): |
| if not isinstance(raw, dict): |
| raise Invalid(f"{label} must be an object with named scores.") |
| lookup = {c["label"]: c["key"] for c in columns} |
| lookup.update({c["key"]: c["key"] for c in columns}) |
| result = {c["key"]: None for c in columns} |
| seen = set() |
| for key, value in raw.items(): |
| if key not in lookup or lookup[key] in seen: |
| raise Invalid(f"Unknown or duplicate {label} field: {key}") |
| seen.add(lookup[key]) |
| result[lookup[key]] = score(value, key) |
| return result |
|
|
|
|
| def nested_scores(raw, columns, label): |
| if not isinstance(raw, dict): |
| raise Invalid(f"{label} must group full score names by reasoning dimension.") |
| result = {c["key"]: None for c in columns} |
| seen = set() |
| for dimension, values in raw.items(): |
| subset = [c for c in columns if c["dimension"].casefold() == dimension.casefold()] |
| if not subset or dimension.casefold() in seen: |
| raise Invalid(f"Unknown or duplicate {label} dimension: {dimension}") |
| seen.add(dimension.casefold()) |
| result.update(simple_scores(values, subset, label)) |
| return result |
|
|
|
|
| def validate_submission(raw, data): |
| if not isinstance(raw, dict): |
| raise Invalid("The submission must be a JSON object.") |
| allowed = {"model_name", "project_url", "evidence_url", "team", "email", "group", "benchmark", |
| "language", "multi_image", "revision_of", "notes", "full_dataset", "results"} |
| if set(raw) - allowed: |
| raise Invalid("Unknown submission fields: " + ", ".join(sorted(set(raw) - allowed))) |
| benchmark = next((b for b in data["benchmarks"] if b["id"] == raw.get("benchmark")), None) |
| if not benchmark or raw.get("language") not in benchmark["results_by_language"]: |
| raise Invalid("Select a supported benchmark and instruction language.") |
| if raw.get("full_dataset") is not True: |
| raise Invalid("Confirm that results use the full benchmark, not the demonstration data.") |
| if type(raw.get("multi_image")) is not bool: |
| raise Invalid("Select whether the model supports multiple input images.") |
| if benchmark["id"] == "risebench" and raw["multi_image"]: |
| raise Invalid("RISEBench evaluates single-image inputs only.") |
| if raw.get("group") not in {"closed", "open", "agentic"}: |
| raise Invalid("Select a model category.") |
| result = {key: raw[key] for key in ("benchmark", "language", "group", "multi_image")} |
| for key, label, size, required in [("model_name", "Model name", 160, True), ("team", "Team", 120, False), |
| ("notes", "Notes", 3000, False), ("revision_of", "Existing model identifier", 160, False)]: |
| result[key] = text(raw.get(key, ""), label, size, required) |
| result["project_url"] = url(raw.get("project_url"), "Project link") |
| result["evidence_url"] = url(raw.get("evidence_url"), "Evaluation artifacts link") |
| result["email"] = text(raw.get("email"), "Contact email", 254) |
| if not re.fullmatch(r"[^\s@]+@[^\s@]+\.[^\s@]+", result["email"]): |
| raise Invalid("Provide a valid contact email.") |
| uploaded = raw.get("results") |
| if not isinstance(uploaded, dict) or set(uploaded) - {"accuracy", "metrics", "subcategories", "task_types"}: |
| raise Invalid("Upload a result JSON matching the downloadable template.") |
| result["accuracy"] = simple_scores(uploaded.get("accuracy"), benchmark["accuracy_columns"], "accuracy") |
| required_keys = [d["key"] for d in benchmark["dimensions"]] |
| if benchmark["id"] == "risebench-plusplus": |
| required_keys += ["single"] + (["multi", "total"] if result["multi_image"] else []) |
| if not result["multi_image"] and any(result["accuracy"][key] is not None for key in ("multi", "total")): |
| raise Invalid("Single-image-only models must leave Total and Multi null.") |
| else: |
| required_keys += ["total"] |
| if any(result["accuracy"][key] is None for key in required_keys): |
| raise Invalid("Fill in every reasoning-dimension accuracy and the applicable summary scores.") |
| result["metrics"] = simple_scores(uploaded.get("metrics", {}), benchmark["metrics"], "evaluation score") |
| for field, columns in (("subcategories", "subcategory_columns"), ("task_types", "task_type_columns")): |
| if columns not in benchmark: |
| if uploaded.get(field): |
| raise Invalid("Fine-grained submissions are supported only for RISEBench++.") |
| continue |
| result[field] = nested_scores(uploaded.get(field, {}), benchmark[columns], field) |
| if benchmark["id"] == "risebench-plusplus" and not result["multi_image"]: |
| for column in benchmark["task_type_columns"]: |
| if column["case_count"] == column["multi_image_case_count"] and result["task_types"][column["key"]] is not None: |
| raise Invalid(f"{column['label']} is a multi-image-only task; leave it null for this model.") |
| result["full_dataset"] = True |
| return result |
|
|
|
|
| def template(benchmark): |
| result = {"accuracy": {c["label"]: None for c in benchmark["accuracy_columns"]}, |
| "metrics": {c["label"]: None for c in benchmark["metrics"]}} |
| for field, key in (("subcategories", "subcategory_columns"), ("task_types", "task_type_columns")): |
| if key in benchmark: |
| result[field] = {} |
| for c in benchmark[key]: |
| result[field].setdefault(c["dimension"].title(), {})[c["label"]] = None |
| return result |
|
|
|
|
| def public_leaderboard(base, state): |
| data = copy.deepcopy(base) |
| for b in data["benchmarks"]: |
| for language, entry in b["results_by_language"].items(): |
| models = {m["id"]: m for m in entry["models"]} |
| approved = sorted((r for r in state["submissions"].values() if r["status"] == "approved" |
| and r["payload"]["benchmark"] == b["id"] and r["payload"]["language"] == language), |
| key=lambda r: r["reviewed_at"]) |
| for record in approved: |
| p = record["payload"] |
| model_id = p["revision_of"] or "community-" + record["id"] |
| previous = models.get(model_id) |
| model = {"id": model_id, "name": p["model_name"], "group": p["group"], "multi_image": p["multi_image"], |
| "source_order": previous["source_order"] if previous else len(models) + 1} |
| |
| for field in ("accuracy", "metrics", "subcategories", "task_types"): |
| if field in p: |
| model[field] = copy.deepcopy(p[field]) |
| models[model_id] = model |
| entry["models"] = list(models.values()) |
| entry["model_count"] = len(models) |
| for group in entry["groups"]: |
| group["count"] = sum(m["group"] == group["id"] for m in models.values()) |
| return data |
|
|
