Duplicate from HuggingFaceH4/open_llm_leaderboard

Co-authored-by: Edward Beeching <[email protected]>

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+scale-hf-logo.png filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,15 @@
+auto_evals/
+venv/
+__pycache__/
+.env
+.ipynb_checkpoints
+*ipynb
+.vscode/
+
+gpt_4_evals/
+human_evals/
+eval-queue/
+eval-results/
+auto_evals/
+
+src/assets/model_counts.html

README.md

@@ -0,0 +1,14 @@
+---
+title: Open LLM Leaderboard
+emoji: 🏆
+colorFrom: green
+colorTo: indigo
+sdk: gradio
+sdk_version: 3.27.0
+app_file: app.py
+pinned: true
+license: apache-2.0
+duplicated_from: HuggingFaceH4/open_llm_leaderboard
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,458 @@
+import json
+import os
+from datetime import datetime, timezone
+
+
+import gradio as gr
+import numpy as np
+import pandas as pd
+from apscheduler.schedulers.background import BackgroundScheduler
+from huggingface_hub import HfApi
+from transformers import AutoConfig
+
+from src.auto_leaderboard.get_model_metadata import apply_metadata
+from src.assets.text_content import *
+from src.auto_leaderboard.load_results import get_eval_results_dicts, make_clickable_model
+from src.assets.hardcoded_evals import gpt4_values, gpt35_values, baseline
+from src.assets.css_html_js import custom_css, get_window_url_params
+from src.utils_display import AutoEvalColumn, EvalQueueColumn, fields, styled_error, styled_warning, styled_message
+from src.init import get_all_requested_models, load_all_info_from_hub
+
+# clone / pull the lmeh eval data
+H4_TOKEN = os.environ.get("H4_TOKEN", None)
+
+QUEUE_REPO = "open-llm-leaderboard/requests"
+RESULTS_REPO = "open-llm-leaderboard/results"
+
+PRIVATE_QUEUE_REPO = "open-llm-leaderboard/private-requests"
+PRIVATE_RESULTS_REPO = "open-llm-leaderboard/private-results"
+
+IS_PUBLIC = bool(os.environ.get("IS_PUBLIC", True))
+
+EVAL_REQUESTS_PATH = "eval-queue"
+EVAL_RESULTS_PATH = "eval-results"
+
+EVAL_REQUESTS_PATH_PRIVATE = "eval-queue-private"
+EVAL_RESULTS_PATH_PRIVATE = "eval-results-private"
+
+api = HfApi()
+
+def restart_space():
+    api.restart_space(
+        repo_id="HuggingFaceH4/open_llm_leaderboard", token=H4_TOKEN
+    )
+
+eval_queue, requested_models, eval_results = load_all_info_from_hub(QUEUE_REPO, RESULTS_REPO, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH)
+
+if not IS_PUBLIC:
+    eval_queue_private, requested_models_private, eval_results_private = load_all_info_from_hub(PRIVATE_QUEUE_REPO, PRIVATE_RESULTS_REPO, EVAL_REQUESTS_PATH_PRIVATE, EVAL_RESULTS_PATH_PRIVATE)
+else:
+    eval_queue_private, eval_results_private = None, None
+
+COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
+TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]
+COLS_LITE = [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
+TYPES_LITE = [c.type for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
+
+if not IS_PUBLIC:
+    COLS.insert(2, AutoEvalColumn.precision.name)
+    TYPES.insert(2, AutoEvalColumn.precision.type)
+
+EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
+EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
+
+BENCHMARK_COLS = [c.name for c in [AutoEvalColumn.arc, AutoEvalColumn.hellaswag, AutoEvalColumn.mmlu, AutoEvalColumn.truthfulqa]]
+
+
+def has_no_nan_values(df, columns):
+    return df[columns].notna().all(axis=1)
+
+
+def has_nan_values(df, columns):
+    return df[columns].isna().any(axis=1)
+
+
+def get_leaderboard_df():
+    if eval_results:
+        print("Pulling evaluation results for the leaderboard.")
+        eval_results.git_pull()
+    if eval_results_private:
+        print("Pulling evaluation results for the leaderboard.")
+        eval_results_private.git_pull()
+
+    all_data = get_eval_results_dicts(IS_PUBLIC)
+
+    if not IS_PUBLIC:
+        all_data.append(gpt4_values)
+        all_data.append(gpt35_values)
+
+    all_data.append(baseline)
+    apply_metadata(all_data)  # Populate model type based on known hardcoded values in `metadata.py`
+
+    df = pd.DataFrame.from_records(all_data)
+    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
+    df = df[COLS]
+
+    # filter out if any of the benchmarks have not been produced
+    df = df[has_no_nan_values(df, BENCHMARK_COLS)]
+    return df
+
+
+def get_evaluation_queue_df():
+    # todo @saylortwift: replace the repo by the one you created for the eval queue
+    if eval_queue:
+        print("Pulling changes for the evaluation queue.")
+        eval_queue.git_pull()
+    if eval_queue_private:
+        print("Pulling changes for the evaluation queue.")
+        eval_queue_private.git_pull()
+
+    entries = [
+        entry
+        for entry in os.listdir(EVAL_REQUESTS_PATH)
+        if not entry.startswith(".")
+    ]
+    all_evals = []
+
+    for entry in entries:
+        if ".json" in entry:
+            file_path = os.path.join(EVAL_REQUESTS_PATH, entry)
+            with open(file_path) as fp:
+                data = json.load(fp)
+
+            data["# params"] = "unknown"
+            data["model"] = make_clickable_model(data["model"])
+            data["revision"] = data.get("revision", "main")
+
+            all_evals.append(data)
+        elif ".md" not in entry:
+            # this is a folder
+            sub_entries = [
+                e
+                for e in os.listdir(f"{EVAL_REQUESTS_PATH}/{entry}")
+                if not e.startswith(".")
+            ]
+            for sub_entry in sub_entries:
+                file_path = os.path.join(EVAL_REQUESTS_PATH, entry, sub_entry)
+                with open(file_path) as fp:
+                    data = json.load(fp)
+
+                # data["# params"] = get_n_params(data["model"])
+                data["model"] = make_clickable_model(data["model"])
+                all_evals.append(data)
+
+    pending_list = [e for e in all_evals if e["status"] == "PENDING"]
+    running_list = [e for e in all_evals if e["status"] == "RUNNING"]
+    finished_list = [e for e in all_evals if e["status"].startswith("FINISHED")]
+    df_pending = pd.DataFrame.from_records(pending_list, columns=EVAL_COLS)
+    df_running = pd.DataFrame.from_records(running_list, columns=EVAL_COLS)
+    df_finished = pd.DataFrame.from_records(finished_list, columns=EVAL_COLS)
+    return df_finished[EVAL_COLS], df_running[EVAL_COLS], df_pending[EVAL_COLS]
+
+
+
+original_df = get_leaderboard_df()
+leaderboard_df = original_df.copy()
+(
+    finished_eval_queue_df,
+    running_eval_queue_df,
+    pending_eval_queue_df,
+) = get_evaluation_queue_df()
+
+def is_model_on_hub(model_name, revision) -> bool:
+    try:
+        AutoConfig.from_pretrained(model_name, revision=revision)
+        return True, None
+    
+    except ValueError as e:
+        return False, "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard."
+
+    except Exception as e:
+        print(f"Could not get the model config from the hub.: {e}")
+        return False, "was not found on hub!"
+
+
+def add_new_eval(
+    model: str,
+    base_model: str,
+    revision: str,
+    precision: str,
+    private: bool,
+    weight_type: str,
+):
+    precision = precision.split(" ")[0]
+    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    # check the model actually exists before adding the eval
+    if revision == "":
+        revision = "main"
+
+    if weight_type in ["Delta", "Adapter"]:
+        base_model_on_hub, error = is_model_on_hub(base_model, revision)
+        if not base_model_on_hub:
+            return styled_error(f'Base model "{base_model}" {error}')
+        
+
+    if not weight_type == "Adapter":
+        model_on_hub, error = is_model_on_hub(model, revision)
+        if not model_on_hub:
+            return styled_error(f'Model "{model}" {error}')
+    
+    print("adding new eval")
+
+    eval_entry = {
+        "model": model,
+        "base_model": base_model,
+        "revision": revision,
+        "private": private,
+        "precision": precision,
+        "weight_type": weight_type,
+        "status": "PENDING",
+        "submitted_time": current_time,
+    }
+
+    user_name = ""
+    model_path = model
+    if "/" in model:
+        user_name = model.split("/")[0]
+        model_path = model.split("/")[1]
+
+    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
+    os.makedirs(OUT_DIR, exist_ok=True)
+    out_path = f"{OUT_DIR}/{model_path}_eval_request_{private}_{precision}_{weight_type}.json"
+
+    # Check for duplicate submission
+    if out_path.split("eval-queue/")[1].lower() in requested_models:
+        return styled_warning("This model has been already submitted.")
+
+    with open(out_path, "w") as f:
+        f.write(json.dumps(eval_entry))
+
+    api.upload_file(
+        path_or_fileobj=out_path,
+        path_in_repo=out_path.split("eval-queue/")[1],
+        repo_id=QUEUE_REPO,
+        token=H4_TOKEN,
+        repo_type="dataset",
+        commit_message=f"Add {model} to eval queue",
+    )
+
+    # remove the local file
+    os.remove(out_path)
+
+    return styled_message("Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list.")
+
+
+def refresh():
+    leaderboard_df = get_leaderboard_df()
+    (
+        finished_eval_queue_df,
+        running_eval_queue_df,
+        pending_eval_queue_df,
+    ) = get_evaluation_queue_df()
+    return (
+        leaderboard_df,
+        finished_eval_queue_df,
+        running_eval_queue_df,
+        pending_eval_queue_df,
+    )
+
+
+def search_table(df, query):
+    if AutoEvalColumn.model_type.name in df.columns:
+        filtered_df = df[
+            (df[AutoEvalColumn.dummy.name].str.contains(query, case=False))
+            | (df[AutoEvalColumn.model_type.name].str.contains(query, case=False))
+            ]
+    else:
+        filtered_df = df[(df[AutoEvalColumn.dummy.name].str.contains(query, case=False))]
+    return filtered_df
+
+
+def change_tab(query_param):
+    query_param = query_param.replace("'", '"')
+    query_param = json.loads(query_param)
+
+    if (
+        isinstance(query_param, dict)
+        and "tab" in query_param
+        and query_param["tab"] == "evaluation"
+    ):
+        return gr.Tabs.update(selected=1)
+    else:
+        return gr.Tabs.update(selected=0)
+
+
+demo = gr.Blocks(css=custom_css)
+with demo:
+    gr.HTML(TITLE)
+    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+    with gr.Row():
+        with gr.Box(elem_id="search-bar-table-box"):
+            search_bar = gr.Textbox(
+                placeholder="🔍 Search your model and press ENTER...",
+                show_label=False,
+                elem_id="search-bar",
+            )
+
+    with gr.Tabs(elem_classes="tab-buttons") as tabs:
+        with gr.TabItem("🏅 LLM Benchmark (lite)", elem_id="llm-benchmark-tab-table", id=0):
+            leaderboard_table_lite = gr.components.Dataframe(
+                value=leaderboard_df[COLS_LITE],
+                headers=COLS_LITE,
+                datatype=TYPES_LITE,
+                max_rows=None,
+                elem_id="leaderboard-table-lite",
+            )
+            # Dummy leaderboard for handling the case when the user uses backspace key
+            hidden_leaderboard_table_for_search_lite = gr.components.Dataframe(
+                value=original_df[COLS_LITE],
+                headers=COLS_LITE,
+                datatype=TYPES_LITE,
+                max_rows=None,
+                visible=False,
+            )
+            search_bar.submit(
+                search_table,
+                [hidden_leaderboard_table_for_search_lite, search_bar],
+                leaderboard_table_lite,
+            )
+
+        with gr.TabItem("📊 Extended view", elem_id="llm-benchmark-tab-table", id=1):
+            leaderboard_table = gr.components.Dataframe(
+                value=leaderboard_df,
+                headers=COLS,
+                datatype=TYPES,
+                max_rows=None,
+                elem_id="leaderboard-table",
+            )
+
+            # Dummy leaderboard for handling the case when the user uses backspace key
+            hidden_leaderboard_table_for_search = gr.components.Dataframe(
+                value=original_df,
+                headers=COLS,
+                datatype=TYPES,
+                max_rows=None,
+                visible=False,
+            )
+            search_bar.submit(
+                search_table,
+                [hidden_leaderboard_table_for_search, search_bar],
+                leaderboard_table,
+            )
+        with gr.TabItem("About", elem_id="llm-benchmark-tab-table", id=2):
+            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
+
+        with gr.TabItem("✉️✨ Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
+            with gr.Column():
+                with gr.Row():
+                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
+
+                with gr.Column():
+                    with gr.Accordion("✅ Finished Evaluations", open=False):
+                        with gr.Row():
+                            finished_eval_table = gr.components.Dataframe(
+                                value=finished_eval_queue_df,
+                                headers=EVAL_COLS,
+                                datatype=EVAL_TYPES,
+                                max_rows=5,
+                            )
+                    with gr.Accordion("🔄 Running Evaluation Queue", open=False):
+                        with gr.Row():
+                            running_eval_table = gr.components.Dataframe(
+                                value=running_eval_queue_df,
+                                headers=EVAL_COLS,
+                                datatype=EVAL_TYPES,
+                                max_rows=5,
+                            )
+
+                    with gr.Accordion("⏳ Pending Evaluation Queue", open=False):
+                        with gr.Row():
+                            pending_eval_table = gr.components.Dataframe(
+                                value=pending_eval_queue_df,
+                                headers=EVAL_COLS,
+                                datatype=EVAL_TYPES,
+                                max_rows=5,
+                            )
+            with gr.Row():
+                gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
+
+            with gr.Row():
+                with gr.Column():
+                    model_name_textbox = gr.Textbox(label="Model name")
+                    revision_name_textbox = gr.Textbox(
+                        label="revision", placeholder="main"
+                    )
+                    private = gr.Checkbox(
+                        False, label="Private", visible=not IS_PUBLIC
+                    )
+
+                with gr.Column():
+                    precision = gr.Dropdown(
+                        choices=["float16", "bfloat16", "8bit (LLM.int8)", "4bit (QLoRA / FP4)"], 
+                        label="Precision", 
+                        multiselect=False,
+                        value="float16",
+                        max_choices=1,
+                        interactive=True,
+                    )
+                    weight_type = gr.Dropdown(
+                        choices=["Original", "Delta", "Adapter"],
+                        label="Weights type", 
+                        multiselect=False,
+                        value="Original",
+                        max_choices=1,
+                        interactive=True,
+                    )
+                    base_model_name_textbox = gr.Textbox(
+                        label="Base model (for delta or adapter weights)"
+                    )
+
+            submit_button = gr.Button("Submit Eval")
+            submission_result = gr.Markdown()
+            submit_button.click(
+                add_new_eval,
+                [
+                    model_name_textbox,
+                    base_model_name_textbox,
+                    revision_name_textbox,
+                    precision,
+                    private,
+                    weight_type,
+                ],
+                submission_result,
+            )
+
+        with gr.Row():
+            refresh_button = gr.Button("Refresh")
+            refresh_button.click(
+                refresh,
+                inputs=[],
+                outputs=[
+                    leaderboard_table,
+                    finished_eval_table,
+                    running_eval_table,
+                    pending_eval_table,
+                ],
+            )
+
+    with gr.Row():
+        with gr.Accordion("📙 Citation", open=False):
+            citation_button = gr.Textbox(
+                value=CITATION_BUTTON_TEXT,
+                label=CITATION_BUTTON_LABEL,
+                elem_id="citation-button",
+            ).style(show_copy_button=True)
+
+    dummy = gr.Textbox(visible=False)
+    demo.load(
+        change_tab,
+        dummy,
+        tabs,
+        _js=get_window_url_params,
+    )
+
+scheduler = BackgroundScheduler()
+scheduler.add_job(restart_space, "interval", seconds=3600)
+scheduler.start()
+demo.queue(concurrency_count=40).launch()

requirements.txt

@@ -0,0 +1,70 @@
+aiofiles==23.1.0
+aiohttp==3.8.4
+aiosignal==1.3.1
+altair==4.2.2
+anyio==3.6.2
+APScheduler==3.10.1
+async-timeout==4.0.2
+attrs==23.1.0
+certifi==2022.12.7
+charset-normalizer==3.1.0
+click==8.1.3
+contourpy==1.0.7
+cycler==0.11.0
+datasets==2.12.0
+entrypoints==0.4
+fastapi==0.95.1
+ffmpy==0.3.0
+filelock==3.11.0
+fonttools==4.39.3
+frozenlist==1.3.3
+fsspec==2023.4.0
+gradio==3.27.0
+gradio_client==0.1.3
+h11==0.14.0
+httpcore==0.17.0
+httpx==0.24.0
+huggingface-hub==0.13.4
+idna==3.4
+Jinja2==3.1.2
+jsonschema==4.17.3
+kiwisolver==1.4.4
+linkify-it-py==2.0.0
+markdown-it-py==2.2.0
+MarkupSafe==2.1.2
+matplotlib==3.7.1
+mdit-py-plugins==0.3.3
+mdurl==0.1.2
+multidict==6.0.4
+numpy==1.24.2
+orjson==3.8.10
+packaging==23.1
+pandas==2.0.0
+Pillow==9.5.0
+plotly==5.14.1
+pyarrow==11.0.0
+pydantic==1.10.7
+pydub==0.25.1
+pyparsing==3.0.9
+pyrsistent==0.19.3
+python-dateutil==2.8.2
+python-multipart==0.0.6
+pytz==2023.3
+pytz-deprecation-shim==0.1.0.post0
+PyYAML==6.0
+requests==2.28.2
+semantic-version==2.10.0
+six==1.16.0
+sniffio==1.3.0
+starlette==0.26.1
+toolz==0.12.0
+tqdm==4.65.0
+transformers==4.28.1
+typing_extensions==4.5.0
+tzdata==2023.3
+tzlocal==4.3
+uc-micro-py==1.0.1
+urllib3==1.26.15
+uvicorn==0.21.1
+websockets==11.0.1
+yarl==1.8.2

src/assets/css_html_js.py

@@ -0,0 +1,87 @@
+custom_css = """
+#changelog-text {
+    font-size: 16px !important;
+}
+
+#changelog-text h2 {
+    font-size: 18px !important;
+}
+
+.markdown-text {
+    font-size: 16px !important;
+}
+
+#models-to-add-text {
+    font-size: 18px !important;
+}
+
+#citation-button span {
+    font-size: 16px !important;
+}
+
+#citation-button textarea {
+    font-size: 16px !important;
+}
+
+#citation-button > label > button {
+    margin: 6px;
+    transform: scale(1.3);
+}
+
+#leaderboard-table {
+    margin-top: 15px
+}
+
+#leaderboard-table-lite {
+    margin-top: 15px
+}
+
+#search-bar-table-box > div:first-child {
+    background: none;
+    border: none;
+}
+ 
+#search-bar {
+    padding: 0px;
+    width: 30%;
+}
+
+/* Hides the final AutoEvalColumn */
+#llm-benchmark-tab-table table td:last-child,
+#llm-benchmark-tab-table table th:last-child {
+    display: none;
+}
+
+/* Limit the width of the first AutoEvalColumn so that names don't expand too much */
+table td:first-child,
+table th:first-child {
+    max-width: 400px;
+    overflow: auto;
+    white-space: nowrap;
+}
+
+.tab-buttons button {
+    font-size: 20px;
+}
+
+#scale-logo {
+    border-style: none !important;
+    box-shadow: none;
+    display: block;
+    margin-left: auto;
+    margin-right: auto;
+    max-width: 600px;
+}
+
+#scale-logo .download {
+    display: none;
+}
+"""
+
+get_window_url_params = """
+    function(url_params) {
+        const params = new URLSearchParams(window.location.search);
+        url_params = Object.fromEntries(params);
+        return url_params;
+    }
+    """

src/assets/hardcoded_evals.py

@@ -0,0 +1,38 @@
+from src.utils_display import AutoEvalColumn, model_hyperlink
+
+gpt4_values = {
+    AutoEvalColumn.model.name: model_hyperlink("https://arxiv.org/abs/2303.08774", "gpt4"),
+    AutoEvalColumn.revision.name: "tech report",
+    AutoEvalColumn.precision.name: None,
+    AutoEvalColumn.average.name: 84.3,
+    AutoEvalColumn.arc.name: 96.3,
+    AutoEvalColumn.hellaswag.name:  95.3,
+    AutoEvalColumn.mmlu.name:  86.4,
+    AutoEvalColumn.truthfulqa.name:  59.0,
+    AutoEvalColumn.dummy.name: "GPT-4",
+}
+
+gpt35_values = {
+    AutoEvalColumn.model.name: model_hyperlink("https://arxiv.org/abs/2303.08774", "gpt3.5"),
+    AutoEvalColumn.revision.name: "tech report",
+    AutoEvalColumn.precision.name: None,
+    AutoEvalColumn.average.name: 71.9,
+    AutoEvalColumn.arc.name: 85.2,
+    AutoEvalColumn.hellaswag.name:  85.5,
+    AutoEvalColumn.mmlu.name:  70.0,
+    AutoEvalColumn.truthfulqa.name:  47.0,
+    AutoEvalColumn.dummy.name: "GPT-3.5",
+}
+
+baseline = {
+    AutoEvalColumn.model.name: "<p>Baseline</p>",
+    AutoEvalColumn.revision.name: "N/A",
+    AutoEvalColumn.precision.name: None,
+    AutoEvalColumn.average.name: 25.0,
+    AutoEvalColumn.arc.name: 25.0,
+    AutoEvalColumn.hellaswag.name:  25.0,
+    AutoEvalColumn.mmlu.name:  25.0,
+    AutoEvalColumn.truthfulqa.name:  25.0,
+    AutoEvalColumn.dummy.name: "baseline",
+}
+

src/assets/text_content.py

@@ -0,0 +1,208 @@
+CHANGELOG_TEXT = f"""
+## [2023-06-19]
+- Added model type column
+- Hid revision and 8bit columns since all models are the same atm
+
+## [2023-06-16]
+- Refactored code base
+- Added new columns: number of parameters, hub likes, license
+
+## [2023-06-13] 
+- Adjust description for TruthfulQA
+
+## [2023-06-12] 
+- Add Human & GPT-4 Evaluations
+
+## [2023-06-05] 
+- Increase concurrent thread count to 40
+- Search models on ENTER
+
+## [2023-06-02] 
+- Add a typeahead search bar
+- Use webhooks to automatically spawn a new Space when someone opens a PR
+- Start recording `submitted_time` for eval requests
+- Limit AutoEvalColumn max-width
+
+## [2023-05-30] 
+- Add a citation button
+- Simplify Gradio layout
+
+## [2023-05-29] 
+- Auto-restart every hour for the latest results
+- Sync with the internal version (minor style changes)
+
+## [2023-05-24] 
+- Add a baseline that has 25.0 for all values
+- Add CHANGELOG
+
+## [2023-05-23] 
+- Fix a CSS issue that made the leaderboard hard to read in dark mode
+
+## [2023-05-22] 
+- Display a success/error message after submitting evaluation requests
+- Reject duplicate submission
+- Do not display results that have incomplete results 
+- Display different queues for jobs that are RUNNING, PENDING, FINISHED status 
+
+## [2023-05-15] 
+- Fix a typo: from "TruthQA" to "QA"
+
+## [2023-05-10] 
+- Fix a bug that prevented auto-refresh
+
+## [2023-05-10] 
+- Release the leaderboard to public
+"""
+
+TITLE = """<h1 align="center" id="space-title">🤗 Open LLM Leaderboard</h1>"""
+
+INTRODUCTION_TEXT = f"""
+📐 The 🤗 Open LLM Leaderboard aims to track, rank and evaluate LLMs and chatbots as they are released. 
+
+🤗 Anyone from the community can submit a model for automated evaluation on the 🤗 GPU cluster, as long as it is a 🤗 Transformers model with weights on the Hub. We also support evaluation of models with delta-weights for non-commercial licensed models, such as the original LLaMa release.
+
+Other cool benchmarks for LLMs are developped at HuggingFace, go check them out: 🙋🤖 [human and GPT4 evals](https://huggingface.co/spaces/HuggingFaceH4/human_eval_llm_leaderboard), 🖥️ [performance benchmarks](https://huggingface.co/spaces/optimum/llm-perf-leaderboard)
+"""
+
+LLM_BENCHMARKS_TEXT = f"""
+# Context
+With the plethora of large language models (LLMs) and chatbots being released week upon week, often with grandiose claims of their performance, it can be hard to filter out the genuine progress that is being made by the open-source community and which model is the current state of the art. 
+
+📈 We evaluate models on 4 key benchmarks from the <a href="https://github.com/EleutherAI/lm-evaluation-harness" target="_blank">  Eleuther AI Language Model Evaluation Harness </a>, a unified framework to test generative language models on a large number of different evaluation tasks. 
+
+- <a href="https://arxiv.org/abs/1803.05457" target="_blank">  AI2 Reasoning Challenge </a> (25-shot) - a set of grade-school science questions.
+- <a href="https://arxiv.org/abs/1905.07830" target="_blank">  HellaSwag </a> (10-shot) - a test of commonsense inference, which is easy for humans (~95%) but challenging for SOTA models.
+- <a href="https://arxiv.org/abs/2009.03300" target="_blank">  MMLU </a>  (5-shot) - a test to measure a text model's multitask accuracy. The test covers 57 tasks including elementary mathematics, US history, computer science, law, and more.
+- <a href="https://arxiv.org/abs/2109.07958" target="_blank">  TruthfulQA </a> (0-shot) - a test to measure a model’s propensity to reproduce falsehoods commonly found online. Note: TruthfulQA in the Harness is actually a minima a 6-shots task, as it is prepended by 6 examples systematically, even when launched using 0 for the number of few-shot examples.
+
+We chose these benchmarks as they test a variety of reasoning and general knowledge across a wide variety of fields in 0-shot and few-shot settings.
+
+# Some good practices before submitting a model
+
+### 1) Make sure you can load your model and tokenizer using AutoClasses:
+```python
+from transformers import AutoConfig, AutoModel, AutoTokenizer
+config = AutoConfig.from_pretrained("your model name", revision=revision)
+model = AutoModel.from_pretrained("your model name", revision=revision)
+tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
+```
+If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded. 
+
+Note: make sure your model is public!
+Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted!
+
+### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index)
+It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of weights of your model to the `Extended Viewer`!
+
+### 3) Make sure your model has an open license!
+This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗
+
+### 4) Fill up your model card
+When we add extra information about models to the leaderboard, it will be automatically taken from the model card
+
+# Reproducibility and details
+
+### Details and logs
+You can find:
+- detailed numerical results in the `results` Hugging Face dataset: https://huggingface.co/datasets/open-llm-leaderboard/results
+- details on the input/outputs for the models in the `details` Hugging Face dataset: https://huggingface.co/datasets/open-llm-leaderboard/details
+- community queries and running status in the `requests` Hugging Face dataset: https://huggingface.co/datasets/open-llm-leaderboard/requests
+
+### Reproducibility
+To reproduce our results, here is the commands you can run, using [this version](https://github.com/EleutherAI/lm-evaluation-harness/tree/b281b0921b636bc36ad05c0b0b0763bd6dd43463) of the Eleuther AI Harness:
+`python main.py --model=hf-causal --model_args="pretrained=<your_model>,use_accelerate=True,revision=<your_model_revision>"`
+` --tasks=<task_list> --num_fewshot=<n_few_shot> --batch_size=2 --output_path=<output_path>`
+
+The total batch size we get for models which fit on one A100 node is 16 (8 GPUs * 2). If you don't use parallelism, adapt your batch size to fit. 
+*You can expect results to vary slightly for different batch sizes because of padding.*
+
+The tasks and few shots parameters are:
+- ARC: 25-shot, *arc-challenge* (`acc_norm`)
+- HellaSwag: 10-shot, *hellaswag* (`acc_norm`)
+- TruthfulQA: 0-shot, *truthfulqa-mc* (`mc2`)
+- MMLU: 5-shot, *hendrycksTest-abstract_algebra,hendrycksTest-anatomy,hendrycksTest-astronomy,hendrycksTest-business_ethics,hendrycksTest-clinical_knowledge,hendrycksTest-college_biology,hendrycksTest-college_chemistry,hendrycksTest-college_computer_science,hendrycksTest-college_mathematics,hendrycksTest-college_medicine,hendrycksTest-college_physics,hendrycksTest-computer_security,hendrycksTest-conceptual_physics,hendrycksTest-econometrics,hendrycksTest-electrical_engineering,hendrycksTest-elementary_mathematics,hendrycksTest-formal_logic,hendrycksTest-global_facts,hendrycksTest-high_school_biology,hendrycksTest-high_school_chemistry,hendrycksTest-high_school_computer_science,hendrycksTest-high_school_european_history,hendrycksTest-high_school_geography,hendrycksTest-high_school_government_and_politics,hendrycksTest-high_school_macroeconomics,hendrycksTest-high_school_mathematics,hendrycksTest-high_school_microeconomics,hendrycksTest-high_school_physics,hendrycksTest-high_school_psychology,hendrycksTest-high_school_statistics,hendrycksTest-high_school_us_history,hendrycksTest-high_school_world_history,hendrycksTest-human_aging,hendrycksTest-human_sexuality,hendrycksTest-international_law,hendrycksTest-jurisprudence,hendrycksTest-logical_fallacies,hendrycksTest-machine_learning,hendrycksTest-management,hendrycksTest-marketing,hendrycksTest-medical_genetics,hendrycksTest-miscellaneous,hendrycksTest-moral_disputes,hendrycksTest-moral_scenarios,hendrycksTest-nutrition,hendrycksTest-philosophy,hendrycksTest-prehistory,hendrycksTest-professional_accounting,hendrycksTest-professional_law,hendrycksTest-professional_medicine,hendrycksTest-professional_psychology,hendrycksTest-public_relations,hendrycksTest-security_studies,hendrycksTest-sociology,hendrycksTest-us_foreign_policy,hendrycksTest-virology,hendrycksTest-world_religions* (`acc` of `all`)
+
+### Quantization
+To get more information about quantization, see:
+- 8 bits: [blog post](https://huggingface.co/blog/hf-bitsandbytes-integration), [paper](https://arxiv.org/abs/2208.07339)
+- 4 bits: [blog post](https://huggingface.co/blog/4bit-transformers-bitsandbytes), [paper](https://arxiv.org/abs/2305.14314)
+
+# In case of model failure
+If your model is displayed in the `FAILED` category, its execution stopped. 
+Make sure you have followed the above steps first. 
+If everything is done, check you can launch the EleutherAIHarness on your model locally, using the above command without modifications (you can add `--limit` to limit the number of examples per task).
+
+"""
+
+EVALUATION_QUEUE_TEXT = f"""
+# Evaluation Queue for the 🤗 Open LLM Leaderboard
+These models will be automatically evaluated on the 🤗 cluster.
+"""
+
+CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
+CITATION_BUTTON_TEXT = r"""@misc{open-llm-leaderboard,
+  author = {Edward Beeching, Clémentine Fourrier, Nathan Habib, Sheon Han, Nathan Lambert, Nazneen Rajani, Omar Sanseviero, Lewis Tunstall, Thomas Wolf},
+  title = {Open LLM Leaderboard},
+  year = {2023},
+  publisher = {Hugging Face},
+  howpublished = "\url{https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard}"
+
+}
+@software{eval-harness,
+  author       = {Gao, Leo and
+                  Tow, Jonathan and
+                  Biderman, Stella and
+                  Black, Sid and
+                  DiPofi, Anthony and
+                  Foster, Charles and
+                  Golding, Laurence and
+                  Hsu, Jeffrey and
+                  McDonell, Kyle and
+                  Muennighoff, Niklas and
+                  Phang, Jason and
+                  Reynolds, Laria and
+                  Tang, Eric and
+                  Thite, Anish and
+                  Wang, Ben and
+                  Wang, Kevin and
+                  Zou, Andy},
+  title        = {A framework for few-shot language model evaluation},
+  month        = sep,
+  year         = 2021,
+  publisher    = {Zenodo},
+  version      = {v0.0.1},
+  doi          = {10.5281/zenodo.5371628},
+  url          = {https://doi.org/10.5281/zenodo.5371628}
+}
+@misc{clark2018think,
+      title={Think you have Solved Question Answering? Try ARC, the AI2 Reasoning Challenge}, 
+      author={Peter Clark and Isaac Cowhey and Oren Etzioni and Tushar Khot and Ashish Sabharwal and Carissa Schoenick and Oyvind Tafjord},
+      year={2018},
+      eprint={1803.05457},
+      archivePrefix={arXiv},
+      primaryClass={cs.AI}
+}
+@misc{zellers2019hellaswag,
+      title={HellaSwag: Can a Machine Really Finish Your Sentence?}, 
+      author={Rowan Zellers and Ari Holtzman and Yonatan Bisk and Ali Farhadi and Yejin Choi},
+      year={2019},
+      eprint={1905.07830},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+@misc{hendrycks2021measuring,
+      title={Measuring Massive Multitask Language Understanding}, 
+      author={Dan Hendrycks and Collin Burns and Steven Basart and Andy Zou and Mantas Mazeika and Dawn Song and Jacob Steinhardt},
+      year={2021},
+      eprint={2009.03300},
+      archivePrefix={arXiv},
+      primaryClass={cs.CY}
+}
+@misc{lin2022truthfulqa,
+      title={TruthfulQA: Measuring How Models Mimic Human Falsehoods}, 
+      author={Stephanie Lin and Jacob Hilton and Owain Evans},
+      year={2022},
+      eprint={2109.07958},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}"""

src/auto_leaderboard/get_model_metadata.py

@@ -0,0 +1,56 @@
+import re
+import os
+from typing import List
+
+from src.utils_display import AutoEvalColumn
+from src.auto_leaderboard.model_metadata_type import get_model_type
+
+from huggingface_hub import HfApi
+import huggingface_hub
+api = HfApi(token=os.environ.get("H4_TOKEN", None))
+
+
+def get_model_infos_from_hub(leaderboard_data: List[dict]):
+    for model_data in leaderboard_data:
+        model_name = model_data["model_name_for_query"]
+        try:
+            model_info = api.model_info(model_name)
+        except huggingface_hub.utils._errors.RepositoryNotFoundError:
+            print("Repo not found!", model_name)
+            model_data[AutoEvalColumn.license.name] = None
+            model_data[AutoEvalColumn.likes.name] = None
+            model_data[AutoEvalColumn.params.name] = get_model_size(model_name, None)
+            continue
+
+        model_data[AutoEvalColumn.license.name] = get_model_license(model_info)
+        model_data[AutoEvalColumn.likes.name] = get_model_likes(model_info)
+        model_data[AutoEvalColumn.params.name] = get_model_size(model_name, model_info)
+
+
+def get_model_license(model_info):
+    try:
+        return model_info.cardData["license"]
+    except Exception:
+        return None
+
+def get_model_likes(model_info):
+    return model_info.likes
+
+size_pattern = re.compile(r"(\d\.)?\d+(b|m)")
+
+def get_model_size(model_name, model_info):
+    # In billions
+    try:
+        return round(model_info.safetensors["total"] / 1e9, 3) 
+    except AttributeError:
+        try:
+            size_match = re.search(size_pattern, model_name.lower())
+            size = size_match.group(0)
+            return round(float(size[:-1]) if size[-1] == "b" else float(size[:-1]) / 1e3, 3)
+        except AttributeError:
+            return None
+
+
+def apply_metadata(leaderboard_data: List[dict]):
+    get_model_type(leaderboard_data)
+    get_model_infos_from_hub(leaderboard_data)

src/auto_leaderboard/load_results.py

@@ -0,0 +1,128 @@
+from dataclasses import dataclass
+
+import glob
+import json
+import os
+from typing import Dict, List, Tuple
+
+from src.utils_display import AutoEvalColumn, make_clickable_model
+import numpy as np
+
+METRICS = ["acc_norm", "acc_norm", "acc", "mc2"]
+BENCHMARKS = ["arc:challenge", "hellaswag", "hendrycksTest", "truthfulqa:mc"]
+BENCH_TO_NAME = {
+    "arc:challenge": AutoEvalColumn.arc.name,
+    "hellaswag": AutoEvalColumn.hellaswag.name,
+    "hendrycksTest": AutoEvalColumn.mmlu.name,
+    "truthfulqa:mc": AutoEvalColumn.truthfulqa.name,
+}
+
+
+@dataclass
+class EvalResult:
+    eval_name: str
+    org: str
+    model: str
+    revision: str
+    results: dict
+    precision: str = "16bit"
+
+    def to_dict(self):
+        if self.org is not None:
+            base_model = f"{self.org}/{self.model}"
+        else:
+            base_model = f"{self.model}"
+        data_dict = {}
+
+        data_dict["eval_name"] = self.eval_name # not a column, just a save name
+        data_dict[AutoEvalColumn.precision.name] = self.precision
+        data_dict[AutoEvalColumn.model.name] = make_clickable_model(base_model)
+        data_dict[AutoEvalColumn.dummy.name] = base_model
+        data_dict[AutoEvalColumn.revision.name] = self.revision
+        data_dict[AutoEvalColumn.average.name] = round(
+            sum([v for k, v in self.results.items()]) / 4.0, 1
+        )
+
+        for benchmark in BENCHMARKS:
+            if benchmark not in self.results.keys():
+                self.results[benchmark] = None
+
+        for k, v in BENCH_TO_NAME.items():
+            data_dict[v] = self.results[k]
+
+        return data_dict
+
+
+def parse_eval_result(json_filepath: str) -> Tuple[str, list[dict]]:
+    with open(json_filepath) as fp:
+        data = json.load(fp)
+
+    
+    for mmlu_k in ["harness|hendrycksTest-abstract_algebra|5", "hendrycksTest-abstract_algebra"]:
+        if mmlu_k in data["versions"] and data["versions"][mmlu_k] == 0:
+            return None, [] # we skip models with the wrong version 
+
+    try:
+        config = data["config"]
+    except KeyError:
+        config = data["config_general"]
+    model = config.get("model_name", None)
+    if model is None:
+        model = config.get("model_args", None)
+
+    model_sha = config.get("model_sha", "")
+    eval_sha = config.get("lighteval_sha", "")
+    model_split = model.split("/", 1)
+
+    model = model_split[-1]
+
+    if len(model_split) == 1:
+        org = None
+        model = model_split[0]
+        result_key = f"{model}_{model_sha}_{eval_sha}"
+    else:
+        org = model_split[0]
+        model = model_split[1]
+        result_key =  f"{org}_{model}_{model_sha}_{eval_sha}"
+
+    eval_results = []
+    for benchmark, metric in zip(BENCHMARKS, METRICS):
+        accs = np.array([v[metric] for k, v in data["results"].items() if benchmark in k])
+        if accs.size == 0:
+            continue
+        mean_acc = round(np.mean(accs) * 100.0, 1)
+        eval_results.append(EvalResult(
+            result_key, org, model, model_sha, {benchmark: mean_acc}
+        ))
+
+    return result_key, eval_results
+
+
+def get_eval_results(is_public) -> List[EvalResult]:
+    json_filepaths = glob.glob(
+        "eval-results/**/results*.json", recursive=True
+    )
+    if not is_public:
+        json_filepaths += glob.glob(
+            "private-eval-results/**/results*.json", recursive=True
+        )
+
+    eval_results = {}
+
+    for json_filepath in json_filepaths:
+        result_key, results = parse_eval_result(json_filepath)
+        for eval_result in results:
+            if result_key in eval_results.keys():
+                eval_results[result_key].results.update(eval_result.results)
+            else:
+                eval_results[result_key] = eval_result
+
+    eval_results = [v for v in eval_results.values()]
+
+    return eval_results
+
+
+def get_eval_results_dicts(is_public=True) -> List[Dict]:
+    eval_results = get_eval_results(is_public)
+
+    return [e.to_dict() for e in eval_results]

src/auto_leaderboard/model_metadata_type.py

@@ -0,0 +1,172 @@
+from enum import Enum
+from typing import Dict, List
+
+class ModelType(Enum):
+    PT = "pretrained"
+    SFT = "finetuned"
+    RL = "with RL"
+
+
+TYPE_METADATA: Dict[str, ModelType] = {
+    "aisquared/dlite-v1-355m": ModelType.SFT,
+    "aisquared/dlite-v2-774m": ModelType.SFT,
+    "aisquared/dlite-v2-1_5b": ModelType.SFT,
+    "TheBloke/wizardLM-7B-HF": ModelType.SFT,
+    "TheBloke/dromedary-65b-lora-HF": ModelType.SFT,
+    "TheBloke/vicuna-13B-1.1-HF": ModelType.SFT,
+    "TheBloke/Wizard-Vicuna-13B-Uncensored-HF": ModelType.SFT,
+    "wordcab/llama-natural-instructions-13b": ModelType.SFT,
+    "JosephusCheung/Guanaco": ModelType.SFT,
+    "AlekseyKorshuk/vicuna-7b": ModelType.SFT,
+    "AlekseyKorshuk/chatml-pyg-v1": ModelType.SFT,
+    "concedo/OPT-19M-ChatSalad": ModelType.SFT,
+    "digitous/Javalion-R": ModelType.SFT,
+    "digitous/Alpacino30b": ModelType.SFT,
+    "digitous/Javelin-GPTJ": ModelType.SFT,
+    "anton-l/gpt-j-tiny-random": ModelType.SFT,
+    "IDEA-CCNL/Ziya-LLaMA-13B-Pretrain-v1": ModelType.SFT,
+    "gpt2-medium": ModelType.PT,
+    "PygmalionAI/pygmalion-6b": ModelType.SFT,
+    "medalpaca/medalpaca-7b": ModelType.SFT,
+    "medalpaca/medalpaca-13b": ModelType.SFT,
+    "chavinlo/alpaca-13b": ModelType.SFT,
+    "chavinlo/alpaca-native": ModelType.SFT,
+    "chavinlo/gpt4-x-alpaca": ModelType.SFT,
+    "hakurei/lotus-12B": ModelType.SFT,
+    "amazon/LightGPT": ModelType.SFT,
+    "shibing624/chinese-llama-plus-13b-hf": ModelType.SFT,
+    "mosaicml/mpt-7b": ModelType.PT,
+    "PSanni/Deer-3b": ModelType.SFT,
+    "bigscience/bloom-1b1": ModelType.PT,
+    "MetaIX/GPT4-X-Alpasta-30b": ModelType.SFT,
+    "EleutherAI/gpt-neox-20b": ModelType.PT,
+    "EleutherAI/gpt-j-6b": ModelType.PT,
+    "roneneldan/TinyStories-28M": ModelType.SFT,
+    "lmsys/vicuna-13b-delta-v1.1": ModelType.SFT,
+    "lmsys/vicuna-7b-delta-v1.1": ModelType.SFT,
+    "abhiramtirumala/DialoGPT-sarcastic-medium": ModelType.SFT,
+    "pillowtalks-ai/delta13b": ModelType.SFT,
+    "bigcode/starcoderplus": ModelType.SFT,
+    "microsoft/DialoGPT-large": ModelType.SFT,
+    "microsoft/CodeGPT-small-py": ModelType.SFT,
+    "Pirr/pythia-13b-deduped-green_devil": ModelType.SFT,
+    "Aeala/GPT4-x-AlpacaDente2-30b": ModelType.SFT,
+    "Aeala/VicUnlocked-alpaca-30b": ModelType.SFT,
+    "dvruette/llama-13b-pretrained-sft-epoch-2": ModelType.SFT,
+    "dvruette/oasst-gpt-neox-20b-1000-steps": ModelType.SFT,
+    "openlm-research/open_llama_3b_350bt_preview": ModelType.PT,
+    "openlm-research/open_llama_7b_700bt_preview": ModelType.PT,
+    "openlm-research/open_llama_7b": ModelType.PT,
+    "openlm-research/open_llama_3b": ModelType.PT,
+    "openlm-research/open_llama_7b_400bt_preview": ModelType.PT,
+    "PocketDoc/Dans-PileOfSets-Mk1-llama-13b-merged": ModelType.SFT,
+    "GeorgiaTechResearchInstitute/galactica-6.7b-evol-instruct-70k": ModelType.SFT,
+    "databricks/dolly-v2-7b": ModelType.SFT,
+    "databricks/dolly-v2-3b": ModelType.SFT,
+    "databricks/dolly-v2-12b": ModelType.SFT,
+    "pinkmanlove/llama-65b-hf": ModelType.SFT,
+    "Rachneet/gpt2-xl-alpaca": ModelType.SFT,
+    "Locutusque/gpt2-conversational-or-qa": ModelType.SFT,
+    "NbAiLab/nb-gpt-j-6B-alpaca": ModelType.SFT,
+    "Fredithefish/ScarletPajama-3B-HF": ModelType.SFT,
+    "eachadea/vicuna-7b-1.1": ModelType.SFT,
+    "eachadea/vicuna-13b": ModelType.SFT,
+    "openaccess-ai-collective/wizard-mega-13b": ModelType.SFT,
+    "openaccess-ai-collective/manticore-13b": ModelType.SFT,
+    "openaccess-ai-collective/manticore-30b-chat-pyg-alpha": ModelType.SFT,
+    "openaccess-ai-collective/minotaur-13b": ModelType.SFT,
+    "lamini/instruct-tuned-3b": ModelType.SFT,
+    "pythainlp/wangchanglm-7.5B-sft-enth": ModelType.SFT,
+    "pythainlp/wangchanglm-7.5B-sft-en-sharded": ModelType.SFT,
+    "stabilityai/stablelm-tuned-alpha-7b": ModelType.SFT,
+    "CalderaAI/30B-Lazarus": ModelType.SFT,
+    "KoboldAI/OPT-13B-Nerybus-Mix": ModelType.SFT,
+    "distilgpt2": ModelType.PT,
+    "wahaha1987/llama_7b_sharegpt94k_fastchat": ModelType.SFT,
+    "OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5": ModelType.SFT,
+    "junelee/wizard-vicuna-13b": ModelType.SFT,
+    "BreadAi/StoryPy": ModelType.SFT,
+    "togethercomputer/RedPajama-INCITE-Base-3B-v1": ModelType.PT,
+    "togethercomputer/RedPajama-INCITE-Base-7B-v0.1": ModelType.PT,
+    "Writer/camel-5b-hf": ModelType.SFT,
+    "Writer/palmyra-base": ModelType.PT,
+    "MBZUAI/lamini-neo-125m": ModelType.SFT,
+    "TehVenom/DiffMerge_Pygmalion_Main-onto-V8P4": ModelType.SFT,
+    "vicgalle/gpt2-alpaca-gpt4": ModelType.SFT,
+    "facebook/opt-350m": ModelType.PT,
+    "facebook/opt-125m": ModelType.PT,
+    "facebook/opt-13b": ModelType.PT,
+    "facebook/opt-1.3b": ModelType.PT,
+    "facebook/opt-66b": ModelType.PT,
+    "facebook/galactica-120b": ModelType.PT,
+    "Abe13/jgpt2-v1": ModelType.SFT,
+    "gpt2-xl": ModelType.PT,
+    "HuggingFaceH4/stable-vicuna-13b-2904": ModelType.RL,
+    "HuggingFaceH4/llama-7b-ift-alpaca": ModelType.SFT,
+    "HuggingFaceH4/starchat-alpha": ModelType.SFT,
+    "HuggingFaceH4/starchat-beta": ModelType.SFT,
+    "ausboss/Llama30B-SuperHOT": ModelType.SFT,
+    "ausboss/llama-13b-supercot": ModelType.SFT,
+    "ausboss/llama-30b-supercot": ModelType.SFT,
+    "Neko-Institute-of-Science/metharme-7b": ModelType.SFT,
+    "SebastianSchramm/Cerebras-GPT-111M-instruction": ModelType.SFT,
+    "victor123/WizardLM-13B-1.0": ModelType.SFT,
+    "AlpinDale/pygmalion-instruct": ModelType.SFT,
+    "tiiuae/falcon-7b-instruct": ModelType.SFT,
+    "tiiuae/falcon-40b-instruct": ModelType.SFT,
+    "tiiuae/falcon-40b": ModelType.PT,
+    "tiiuae/falcon-7b": ModelType.PT,
+    "cyl/awsome-llama": ModelType.SFT,
+    "xzuyn/Alpacino-SuperCOT-13B": ModelType.SFT,
+    "xzuyn/MedicWizard-7B": ModelType.SFT,
+    "beomi/KoAlpaca-Polyglot-5.8B": ModelType.SFT,
+    "chainyo/alpaca-lora-7b": ModelType.SFT,
+    "Salesforce/codegen-16B-nl": ModelType.PT,
+    "Salesforce/codegen-16B-multi": ModelType.SFT,
+    "ai-forever/rugpt3large_based_on_gpt2": ModelType.SFT,
+    "gpt2-large": ModelType.PT,
+    "huggingface/llama-13b": ModelType.PT,
+    "huggingface/llama-7b": ModelType.PT,
+    "huggingface/llama-65b": ModelType.PT,
+    "huggingface/llama-30b": ModelType.PT,
+    "jondurbin/airoboros-7b": ModelType.SFT,
+    "jondurbin/airoboros-13b": ModelType.SFT,
+    "cerebras/Cerebras-GPT-1.3B": ModelType.PT,
+    "cerebras/Cerebras-GPT-111M": ModelType.PT,
+    "NousResearch/Nous-Hermes-13b": ModelType.SFT,
+    "project-baize/baize-v2-7b": ModelType.SFT,
+    "project-baize/baize-v2-13b": ModelType.SFT,
+    "LLMs/AlpacaGPT4-7B-elina": ModelType.SFT,
+    "LLMs/Vicuna-EvolInstruct-13B": ModelType.SFT,
+    "huggingtweets/jerma985": ModelType.SFT,
+    "huggyllama/llama-65b": ModelType.PT,
+    "WizardLM/WizardLM-13B-1.0": ModelType.SFT,
+    "gpt2": ModelType.PT,
+    "alessandropalla/instruct_gpt2": ModelType.SFT,
+    "MayaPH/FinOPT-Lincoln": ModelType.SFT,
+    "MayaPH/FinOPT-Franklin": ModelType.SFT,
+    "timdettmers/guanaco-33b-merged": ModelType.SFT,
+    "timdettmers/guanaco-65b-merged": ModelType.SFT,
+    "elinas/llama-30b-hf-transformers-4.29": ModelType.SFT,
+    "elinas/chronos-33b": ModelType.SFT,
+    "nmitchko/medguanaco-65b-GPTQ": ModelType.SFT,
+    "xhyi/PT_GPTNEO350_ATG": ModelType.SFT,
+    "h2oai/h2ogpt-oasst1-512-20b": ModelType.SFT,
+    "h2oai/h2ogpt-gm-oasst1-en-1024-12b": ModelType.SFT,
+    "nomic-ai/gpt4all-13b-snoozy": ModelType.SFT,
+    "nomic-ai/gpt4all-j": ModelType.SFT,
+}
+
+
+def get_model_type(leaderboard_data: List[dict]):
+    for model_data in leaderboard_data:
+        model_data["Type"] = TYPE_METADATA.get(model_data["model_name_for_query"], "N/A")
+        if model_data["Type"] == "N/A":
+            if any([i in model_data["model_name_for_query"] for i in ["finetuned", "-ft-"]]):
+                model_data["Type"] = ModelType.SFT
+            elif any([i in model_data["model_name_for_query"] for i in ["pretrained"]]):
+                model_data["Type"] = ModelType.PT
+            elif any([i in model_data["model_name_for_query"] for i in ["-rl-", "-rlhf-"]]):
+                model_data["Type"] = ModelType.RL
+ 
+ 

src/init.py

@@ -0,0 +1,58 @@
+import os 
+from huggingface_hub import Repository
+
+H4_TOKEN = os.environ.get("H4_TOKEN", None)
+
+
+def get_all_requested_models(requested_models_dir):
+    depth = 1
+    file_names = []
+
+    for root, dirs, files in os.walk(requested_models_dir):
+        current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
+        if current_depth == depth:
+            file_names.extend([os.path.join(root, file) for file in files])
+
+    return set([file_name.lower().split("eval-queue/")[1] for file_name in file_names])
+
+def load_all_info_from_hub(QUEUE_REPO, RESULTS_REPO, QUEUE_PATH, RESULTS_PATH):
+    eval_queue_repo = None
+    eval_results_repo = None
+    requested_models = None
+
+    if H4_TOKEN:
+        print("Pulling evaluation requests and results.")
+
+        eval_queue_repo = Repository(
+            local_dir=QUEUE_PATH,
+            clone_from=QUEUE_REPO,
+            use_auth_token=H4_TOKEN,
+            repo_type="dataset",
+        )
+        eval_queue_repo.git_pull()
+
+        eval_results_repo = Repository(
+            local_dir=RESULTS_PATH,
+            clone_from=RESULTS_REPO,
+            use_auth_token=H4_TOKEN,
+            repo_type="dataset",
+        )
+        eval_results_repo.git_pull()
+
+        requested_models = get_all_requested_models("eval-queue")
+    else:
+        print("No HuggingFace token provided. Skipping evaluation requests and results.")
+
+    return eval_queue_repo, requested_models, eval_results_repo
+
+
+#def load_results(model, benchmark, metric):
+#    file_path = os.path.join("autoevals", model, f"{model}-eval_{benchmark}.json")
+#    if not os.path.exists(file_path):
+#        return 0.0, None
+
+#    with open(file_path) as fp:
+#        data = json.load(fp)
+#    accs = np.array([v[metric] for k, v in data["results"].items()])
+#    mean_acc = np.mean(accs)
+#    return mean_acc, data["config"]["model_args"]

src/utils_display.py

@@ -0,0 +1,98 @@
+from dataclasses import dataclass
+
+# These classes are for user facing column names, to avoid having to change them 
+# all around the code when a modif is needed 
+@dataclass
+class ColumnContent:
+    name: str
+    type: str 
+    displayed_by_default: bool 
+    hidden: bool = False
+
+def fields(raw_class):
+    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
+
+@dataclass(frozen=True)
+class AutoEvalColumn: # Auto evals column
+    model = ColumnContent("Model", "markdown", True)
+    average = ColumnContent("Average ⬆️", "number", True)
+    arc = ColumnContent("ARC ⬆️", "number", True)
+    hellaswag = ColumnContent("HellaSwag ⬆️", "number", True)
+    mmlu = ColumnContent("MMLU ⬆️", "number", True)
+    truthfulqa = ColumnContent("TruthfulQA (MC) ⬆️", "number", True)
+    model_type = ColumnContent("Type", "str", False)
+    precision = ColumnContent("Precision", "str", False, True)
+    license = ColumnContent("Hub License", "str", False)
+    params = ColumnContent("#Params (B)", "number", False)
+    likes = ColumnContent("Hub ❤️", "number", False)
+    revision = ColumnContent("Model sha", "str", False, False)
+    dummy = ColumnContent("model_name_for_query", "str", True) # dummy col to implement search bar (hidden by custom CSS)
+
+@dataclass(frozen=True)
+class EloEvalColumn: # Elo evals column
+    model = ColumnContent("Model", "markdown", True)
+    gpt4 = ColumnContent("GPT-4 (all)", "number", True)
+    human_all = ColumnContent("Human (all)", "number", True)
+    human_instruct = ColumnContent("Human (instruct)", "number", True)
+    human_code_instruct = ColumnContent("Human (code-instruct)", "number", True)
+
+
+@dataclass(frozen=True)
+class EvalQueueColumn: # Queue column
+    model = ColumnContent("model", "markdown", True)
+    revision = ColumnContent("revision", "str", True)
+    private = ColumnContent("private", "bool", True)
+    precision = ColumnContent("precision", "bool", True)
+    weight_type = ColumnContent("weight_type", "str", "Original")
+    status = ColumnContent("status", "str", True)
+
+LLAMAS = ["huggingface/llama-7b", "huggingface/llama-13b", "huggingface/llama-30b", "huggingface/llama-65b"]
+
+
+KOALA_LINK = "https://huggingface.co/TheBloke/koala-13B-HF"
+VICUNA_LINK = "https://huggingface.co/lmsys/vicuna-13b-delta-v1.1"
+OASST_LINK = "https://huggingface.co/OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5"
+DOLLY_LINK = "https://huggingface.co/databricks/dolly-v2-12b"
+MODEL_PAGE = "https://huggingface.co/models"
+LLAMA_LINK = "https://ai.facebook.com/blog/large-language-model-llama-meta-ai/"
+VICUNA_LINK = "https://huggingface.co/CarperAI/stable-vicuna-13b-delta"
+ALPACA_LINK = "https://crfm.stanford.edu/2023/03/13/alpaca.html"
+
+
+def model_hyperlink(link, model_name):
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+
+
+def make_clickable_model(model_name):
+    link = f"https://huggingface.co/{model_name}"
+
+    if model_name in LLAMAS:
+        link = LLAMA_LINK
+        model_name = model_name.split("/")[1]
+    elif model_name == "HuggingFaceH4/stable-vicuna-13b-2904":
+        link = VICUNA_LINK
+        model_name = "stable-vicuna-13b"
+    elif model_name == "HuggingFaceH4/llama-7b-ift-alpaca":
+        link = ALPACA_LINK
+        model_name = "alpaca-13b"
+    if model_name == "dolly-12b":
+        link = DOLLY_LINK
+    elif model_name == "vicuna-13b":
+        link = VICUNA_LINK
+    elif model_name == "koala-13b":
+        link = KOALA_LINK
+    elif model_name == "oasst-12b":
+        link = OASST_LINK
+    #else:
+    #    link = MODEL_PAGE
+  
+    return model_hyperlink(link, model_name)
+
+def styled_error(error):
+    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
+
+def styled_warning(warn):
+    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
+
+def styled_message(message):
+    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"