Upload folder using huggingface_hub

README.md

@@ -4,7 +4,7 @@ emoji: 🔎
 colorFrom: red
 colorTo: blue
 sdk: gradio
-sdk_version: 6.17.3
+sdk_version: 5.49.1
 python_version: 3.11
 app_file: app.py
 pinned: false

app.py

@@ -8,6 +8,8 @@ from pathlib import Path
 from typing import Any
 
 os.environ.setdefault("GRADIO_SSR_MODE", "false")
+os.environ.setdefault("GRADIO_ANALYTICS_ENABLED", "False")
+os.environ.setdefault("HF_HUB_DISABLE_TELEMETRY", "1")
 
 import gradio as gr
 import pandas as pd
@@ -558,6 +560,106 @@ def load_results() -> list[dict[str, Any]]:
     return read_jsonl(RESULTS_PATH)
 
 
+def normalize_submission_record(record: dict[str, Any]) -> dict[str, Any]:
+    model_id = str(record.get("model_id", "")).strip()
+    if not model_id:
+        raise ValueError("`model_id` is required.")
+
+    scores = record.get("scores")
+    if not isinstance(scores, dict):
+        raise ValueError("`scores` must be an object.")
+
+    average_scores = scores.get("average", {})
+    dataset_scores = scores.get("datasets", {})
+    if average_scores is None:
+        average_scores = {}
+    if dataset_scores is None:
+        dataset_scores = {}
+    if not isinstance(average_scores, dict):
+        raise ValueError("`scores.average` must be an object.")
+    if not isinstance(dataset_scores, dict):
+        raise ValueError("`scores.datasets` must be an object.")
+
+    has_metric = any(metric_value(average_scores, metric) is not None for metric in METRICS)
+    if not has_metric:
+        for metrics in dataset_scores.values():
+            if isinstance(metrics, dict) and any(metric_value(metrics, metric) is not None for metric in METRICS):
+                has_metric = True
+                break
+    if not has_metric:
+        raise ValueError(f"At least one numeric metric is required: {', '.join(METRICS)}.")
+
+    normalized = dict(record)
+    normalized["model_id"] = model_id
+    normalized["model_name"] = str(record.get("model_name") or model_id.split("/")[-1]).strip()
+    normalized["organization"] = str(record.get("organization") or (model_id.split("/", 1)[0] if "/" in model_id else "")).strip()
+    normalized["type"] = str(record.get("type") or "dense").strip()
+    normalized["date"] = str(record.get("date") or date.today().isoformat()).strip()
+    normalized["verified"] = bool(record.get("verified", False))
+    normalized["source_url"] = str(record.get("source_url", "")).strip()
+    normalized["scores"] = {"average": average_scores, "datasets": dataset_scores}
+    return normalized
+
+
+def add_submission_record(
+    record_text: str,
+    metric: str,
+    task_filter: str,
+    verified_only: bool,
+    model_filter: str,
+) -> tuple[str, str, str]:
+    record_text = record_text.strip()
+    if not record_text:
+        return (
+            "Paste or generate a JSONL record first.",
+            summary_html(),
+            leaderboard_table_html(metric, task_filter, verified_only, model_filter),
+        )
+
+    try:
+        parsed = json.loads(record_text)
+        if not isinstance(parsed, dict):
+            raise ValueError("Submission must be a single JSON object.")
+        record = normalize_submission_record(parsed)
+    except (json.JSONDecodeError, ValueError) as exc:
+        return (
+            f"Submission was not added: {exc}",
+            summary_html(),
+            leaderboard_table_html(metric, task_filter, verified_only, model_filter),
+        )
+
+    serialized = json.dumps(record, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
+    existing = set()
+    for item in load_results():
+        try:
+            existing.add(json.dumps(normalize_submission_record(item), ensure_ascii=False, sort_keys=True, separators=(",", ":")))
+        except ValueError:
+            continue
+    if serialized in existing:
+        return (
+            f"`{record['model_id']}` is already present with the same scores.",
+            summary_html(),
+            leaderboard_table_html(metric, task_filter, verified_only, model_filter),
+        )
+
+    RESULTS_PATH.parent.mkdir(parents=True, exist_ok=True)
+    with RESULTS_PATH.open("a", encoding="utf-8") as file:
+        needs_newline = RESULTS_PATH.exists() and RESULTS_PATH.stat().st_size > 0
+        if needs_newline:
+            with RESULTS_PATH.open("rb") as check_file:
+                check_file.seek(-1, os.SEEK_END)
+                needs_newline = check_file.read(1) != b"\n"
+        if needs_newline:
+            file.write("\n")
+        file.write(serialized)
+
+    return (
+        f"Added `{record['model_id']}` to `{RESULTS_PATH.name}`.",
+        summary_html(),
+        leaderboard_table_html(metric, task_filter, verified_only, model_filter),
+    )
+
+
 def logo_data_uri() -> str:
     if not LOGO_PATH.exists():
         return ""
@@ -734,7 +836,7 @@ def task_choices() -> list[str]:
 
 
 with gr.Blocks(title="RusBEIR Leaderboard") as demo:
-    gr.HTML(summary_html())
+    summary = gr.HTML(summary_html())
 
     with gr.Tabs():
         with gr.Tab("Leaderboard"):
@@ -807,12 +909,19 @@ with gr.Blocks(title="RusBEIR Leaderboard") as demo:
                 source_url = gr.Textbox(label="Source URL", placeholder="Evaluation logs, paper, PR, or model card")
                 notes = gr.Textbox(label="Notes", lines=3)
                 template_button = gr.Button("Create submission template", variant="primary")
-                template = gr.Code(label="JSONL record", language="json")
+                template = gr.Code(label="JSONL record", language="json", interactive=True)
+                submit_status = gr.Markdown()
+                add_button = gr.Button("Add to leaderboard", variant="primary")
                 template_button.click(
                     submission_template,
                     inputs=[model_id, model_type, source_url, notes],
                     outputs=template,
                 )
+                add_button.click(
+                    add_submission_record,
+                    inputs=[template, metric, task_filter, verified_only, model_filter],
+                    outputs=[submit_status, summary, table],
+                )
 
         with gr.Tab("About"):
             with gr.Column(elem_classes=["rusbeir-panel"]):