Duplicate from HuggingFaceH4/open_llm_leaderboard

Co-authored-by: Edward Beeching <[email protected]>

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+scale-hf-logo.png filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,11 @@
+auto_evals/
+venv/
+__pycache__/
+.env
+.ipynb_checkpoints
+*ipynb
+.vscode/
+
+gpt_4_evals/
+human_evals/
+model_counts.html

README.md

@@ -0,0 +1,14 @@
+---
+title: Open LLM Leaderboard
+emoji: 🏆
+colorFrom: green
+colorTo: indigo
+sdk: gradio
+sdk_version: 3.27.0
+app_file: app.py
+pinned: true
+license: apache-2.0
+duplicated_from: HuggingFaceH4/open_llm_leaderboard
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,518 @@
+import json
+import os
+from datetime import datetime, timezone
+
+
+import gradio as gr
+import numpy as np
+import pandas as pd
+from apscheduler.schedulers.background import BackgroundScheduler
+from huggingface_hub import HfApi
+from transformers import AutoConfig
+
+from src.auto_leaderboard.get_model_metadata import apply_metadata
+from src.assets.text_content import *
+from src.elo_leaderboard.load_results import get_elo_plots, get_elo_results_dicts
+from src.auto_leaderboard.load_results import get_eval_results_dicts, make_clickable_model
+from src.assets.hardcoded_evals import gpt4_values, gpt35_values, baseline
+from src.assets.css_html_js import custom_css, get_window_url_params
+from src.utils_display import AutoEvalColumn, EvalQueueColumn, EloEvalColumn, fields, styled_error, styled_warning, styled_message
+from src.init import load_all_info_from_hub
+
+# clone / pull the lmeh eval data
+H4_TOKEN = os.environ.get("H4_TOKEN", None)
+LMEH_REPO = "HuggingFaceH4/lmeh_evaluations"
+HUMAN_EVAL_REPO = "HuggingFaceH4/scale-human-eval"
+GPT_4_EVAL_REPO = "HuggingFaceH4/open_llm_leaderboard_oai_evals"
+IS_PUBLIC = bool(os.environ.get("IS_PUBLIC", True))
+ADD_PLOTS = False
+
+EVAL_REQUESTS_PATH = "auto_evals/eval_requests"
+
+api = HfApi()
+
+
+def restart_space():
+    api.restart_space(
+        repo_id="HuggingFaceH4/open_llm_leaderboard", token=H4_TOKEN
+    )
+
+auto_eval_repo, human_eval_repo, gpt_4_eval_repo, requested_models = load_all_info_from_hub(LMEH_REPO, HUMAN_EVAL_REPO, GPT_4_EVAL_REPO)
+
+COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
+TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]
+COLS_LITE = [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
+TYPES_LITE = [c.type for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
+
+if not IS_PUBLIC:
+    COLS.insert(2, AutoEvalColumn.is_8bit.name)
+    TYPES.insert(2, AutoEvalColumn.is_8bit.type)
+
+EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
+EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
+
+BENCHMARK_COLS = [c.name for c in [AutoEvalColumn.arc, AutoEvalColumn.hellaswag, AutoEvalColumn.mmlu, AutoEvalColumn.truthfulqa]]
+
+ELO_COLS = [c.name for c in fields(EloEvalColumn)]
+ELO_TYPES = [c.type for c in fields(EloEvalColumn)]
+ELO_SORT_COL = EloEvalColumn.gpt4.name
+
+
+def has_no_nan_values(df, columns):
+    return df[columns].notna().all(axis=1)
+
+
+def has_nan_values(df, columns):
+    return df[columns].isna().any(axis=1)
+
+
+def get_leaderboard_df():
+    if auto_eval_repo:
+        print("Pulling evaluation results for the leaderboard.")
+        auto_eval_repo.git_pull()
+
+    all_data = get_eval_results_dicts(IS_PUBLIC)
+
+    if not IS_PUBLIC:
+        all_data.append(gpt4_values)
+        all_data.append(gpt35_values)
+
+    all_data.append(baseline)
+    apply_metadata(all_data)  # Populate model type based on known hardcoded values in `metadata.py`
+
+    df = pd.DataFrame.from_records(all_data)
+    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
+    df = df[COLS]
+
+    # filter out if any of the benchmarks have not been produced
+    df = df[has_no_nan_values(df, BENCHMARK_COLS)]
+    return df
+
+
+def get_evaluation_queue_df():
+    # todo @saylortwift: replace the repo by the one you created for the eval queue
+    if auto_eval_repo:
+        print("Pulling changes for the evaluation queue.")
+        auto_eval_repo.git_pull()
+
+    entries = [
+        entry
+        for entry in os.listdir(EVAL_REQUESTS_PATH)
+        if not entry.startswith(".")
+    ]
+    all_evals = []
+
+    for entry in entries:
+        if ".json" in entry:
+            file_path = os.path.join(EVAL_REQUESTS_PATH, entry)
+            with open(file_path) as fp:
+                data = json.load(fp)
+
+            data["# params"] = "unknown"
+            data["model"] = make_clickable_model(data["model"])
+            data["revision"] = data.get("revision", "main")
+
+            all_evals.append(data)
+        else:
+            # this is a folder
+            sub_entries = [
+                e
+                for e in os.listdir(f"{EVAL_REQUESTS_PATH}/{entry}")
+                if not e.startswith(".")
+            ]
+            for sub_entry in sub_entries:
+                file_path = os.path.join(EVAL_REQUESTS_PATH, entry, sub_entry)
+                with open(file_path) as fp:
+                    data = json.load(fp)
+
+                # data["# params"] = get_n_params(data["model"])
+                data["model"] = make_clickable_model(data["model"])
+                all_evals.append(data)
+
+    pending_list = [e for e in all_evals if e["status"] == "PENDING"]
+    running_list = [e for e in all_evals if e["status"] == "RUNNING"]
+    finished_list = [e for e in all_evals if e["status"] == "FINISHED"]
+    df_pending = pd.DataFrame.from_records(pending_list)
+    df_running = pd.DataFrame.from_records(running_list)
+    df_finished = pd.DataFrame.from_records(finished_list)
+    return df_finished[EVAL_COLS], df_running[EVAL_COLS], df_pending[EVAL_COLS]
+
+
+def get_elo_leaderboard(df_instruct, df_code_instruct, tie_allowed=False):
+    if human_eval_repo:
+        print("Pulling human_eval_repo changes")
+        human_eval_repo.git_pull()
+
+    all_data = get_elo_results_dicts(df_instruct, df_code_instruct, tie_allowed)
+    dataframe = pd.DataFrame.from_records(all_data)
+    dataframe = dataframe.sort_values(by=ELO_SORT_COL, ascending=False)
+    dataframe = dataframe[ELO_COLS]
+    return dataframe
+
+
+def get_elo_elements():
+    df_instruct = pd.read_json("human_evals/without_code.json")
+    df_code_instruct = pd.read_json("human_evals/with_code.json")
+
+    elo_leaderboard = get_elo_leaderboard(
+        df_instruct, df_code_instruct, tie_allowed=False
+    )
+    elo_leaderboard_with_tie_allowed = get_elo_leaderboard(
+        df_instruct, df_code_instruct, tie_allowed=True
+    )
+    plot_1, plot_2, plot_3, plot_4 = get_elo_plots(
+        df_instruct, df_code_instruct, tie_allowed=False
+    )
+
+    return (
+        elo_leaderboard,
+        elo_leaderboard_with_tie_allowed,
+        plot_1,
+        plot_2,
+        plot_3,
+        plot_4,
+    )
+
+
+original_df = get_leaderboard_df()
+leaderboard_df = original_df.copy()
+(
+    finished_eval_queue_df,
+    running_eval_queue_df,
+    pending_eval_queue_df,
+) = get_evaluation_queue_df()
+(
+    elo_leaderboard,
+    elo_leaderboard_with_tie_allowed,
+    plot_1,
+    plot_2,
+    plot_3,
+    plot_4,
+) = get_elo_elements()
+
+
+def is_model_on_hub(model_name, revision) -> bool:
+    try:
+        AutoConfig.from_pretrained(model_name, revision=revision)
+        return True, None
+    
+    except ValueError as e:
+        return False, "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard."
+
+    except Exception as e:
+        print("Could not get the model config from the hub.: \n", e)
+        return False, "was not found on hub!"
+
+
+def add_new_eval(
+    model: str,
+    base_model: str,
+    revision: str,
+    is_8_bit_eval: bool,
+    private: bool,
+    is_delta_weight: bool,
+):
+    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    # check the model actually exists before adding the eval
+    if revision == "":
+        revision = "main"
+
+    if is_delta_weight: 
+        base_model_on_hub, error = is_model_on_hub(base_model, revision)
+        if not base_model_on_hub:
+            return styled_error(f'Base model "{base_model}" {error}')
+
+    model_on_hub, error = is_model_on_hub(model, revision)
+    if not model_on_hub:
+        return styled_error(f'Model "{model}" {error}')
+
+    print("adding new eval")
+
+    eval_entry = {
+        "model": model,
+        "base_model": base_model,
+        "revision": revision,
+        "private": private,
+        "8bit_eval": is_8_bit_eval,
+        "is_delta_weight": is_delta_weight,
+        "status": "PENDING",
+        "submitted_time": current_time,
+    }
+
+    user_name = ""
+    model_path = model
+    if "/" in model:
+        user_name = model.split("/")[0]
+        model_path = model.split("/")[1]
+
+    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
+    os.makedirs(OUT_DIR, exist_ok=True)
+    out_path = f"{OUT_DIR}/{model_path}_eval_request_{private}_{is_8_bit_eval}_{is_delta_weight}.json"
+
+    # Check for duplicate submission
+    if out_path.split("eval_requests/")[1].lower() in requested_models:
+        return styled_warning("This model has been already submitted.")
+
+    with open(out_path, "w") as f:
+        f.write(json.dumps(eval_entry))
+
+    api.upload_file(
+        path_or_fileobj=out_path,
+        path_in_repo=out_path,
+        repo_id=LMEH_REPO,
+        token=H4_TOKEN,
+        repo_type="dataset",
+    )
+
+    return styled_message("Your request has been submitted to the evaluation queue!")
+
+
+def refresh():
+    leaderboard_df = get_leaderboard_df()
+    (
+        finished_eval_queue_df,
+        running_eval_queue_df,
+        pending_eval_queue_df,
+    ) = get_evaluation_queue_df()
+    return (
+        leaderboard_df,
+        finished_eval_queue_df,
+        running_eval_queue_df,
+        pending_eval_queue_df,
+    )
+
+
+def search_table(df, query):
+    filtered_df = df[df[AutoEvalColumn.dummy.name].str.contains(query, case=False)]
+    return filtered_df
+
+
+def change_tab(query_param):
+    query_param = query_param.replace("'", '"')
+    query_param = json.loads(query_param)
+
+    if (
+        isinstance(query_param, dict)
+        and "tab" in query_param
+        and query_param["tab"] == "evaluation"
+    ):
+        return gr.Tabs.update(selected=1)
+    else:
+        return gr.Tabs.update(selected=0)
+
+
+demo = gr.Blocks(css=custom_css)
+with demo:
+    gr.HTML(TITLE)
+    with gr.Row():
+        gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+
+    with gr.Row():
+        with gr.Column():
+            with gr.Accordion("📙 Citation", open=False):
+                citation_button = gr.Textbox(
+                    value=CITATION_BUTTON_TEXT,
+                    label=CITATION_BUTTON_LABEL,
+                    elem_id="citation-button",
+                ).style(show_copy_button=True)
+        with gr.Column():
+            with gr.Accordion("✨ CHANGELOG", open=False):
+                changelog = gr.Markdown(CHANGELOG_TEXT, elem_id="changelog-text")
+
+    with gr.Tabs(elem_classes="tab-buttons") as tabs:
+        with gr.TabItem("📊 LLM Benchmarks", elem_id="llm-benchmark-tab-table", id=0):
+            with gr.Column():
+                gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
+                with gr.Box(elem_id="search-bar-table-box"):
+                    search_bar = gr.Textbox(
+                        placeholder="🔍 Search your model and press ENTER...",
+                        show_label=False,
+                        elem_id="search-bar",
+                    )
+                    with gr.Tabs(elem_classes="tab-buttons"):
+                        with gr.TabItem("Light View"):
+                            leaderboard_table_lite = gr.components.Dataframe(
+                                value=leaderboard_df[COLS_LITE],
+                                headers=COLS_LITE,
+                                datatype=TYPES_LITE,
+                                max_rows=None,
+                                elem_id="leaderboard-table-lite",
+                            )
+                        with gr.TabItem("Extended Model View"):
+                            leaderboard_table = gr.components.Dataframe(
+                                value=leaderboard_df,
+                                headers=COLS,
+                                datatype=TYPES,
+                                max_rows=None,
+                                elem_id="leaderboard-table",
+                            )
+
+                    # Dummy leaderboard for handling the case when the user uses backspace key
+                    hidden_leaderboard_table_for_search = gr.components.Dataframe(
+                        value=original_df,
+                        headers=COLS,
+                        datatype=TYPES,
+                        max_rows=None,
+                        visible=False,
+                    )
+                    search_bar.submit(
+                        search_table,
+                        [hidden_leaderboard_table_for_search, search_bar],
+                        leaderboard_table,
+                    )
+
+                    # Dummy leaderboard for handling the case when the user uses backspace key
+                    hidden_leaderboard_table_for_search_lite = gr.components.Dataframe(
+                        value=original_df[COLS_LITE],
+                        headers=COLS_LITE,
+                        datatype=TYPES_LITE,
+                        max_rows=None,
+                        visible=False,
+                    )
+                    search_bar.submit(
+                        search_table,
+                        [hidden_leaderboard_table_for_search_lite, search_bar],
+                        leaderboard_table_lite,
+                    )
+
+                with gr.Row():
+                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
+
+                with gr.Accordion("✅ Finished Evaluations", open=False):
+                    with gr.Row():
+                        finished_eval_table = gr.components.Dataframe(
+                            value=finished_eval_queue_df,
+                            headers=EVAL_COLS,
+                            datatype=EVAL_TYPES,
+                            max_rows=5,
+                        )
+                with gr.Accordion("🔄 Running Evaluation Queue", open=False):
+                    with gr.Row():
+                        running_eval_table = gr.components.Dataframe(
+                            value=running_eval_queue_df,
+                            headers=EVAL_COLS,
+                            datatype=EVAL_TYPES,
+                            max_rows=5,
+                        )
+
+                with gr.Accordion("⏳ Pending Evaluation Queue", open=False):
+                    with gr.Row():
+                        pending_eval_table = gr.components.Dataframe(
+                            value=pending_eval_queue_df,
+                            headers=EVAL_COLS,
+                            datatype=EVAL_TYPES,
+                            max_rows=5,
+                        )
+
+                with gr.Row():
+                    refresh_button = gr.Button("Refresh")
+                    refresh_button.click(
+                        refresh,
+                        inputs=[],
+                        outputs=[
+                            leaderboard_table,
+                            finished_eval_table,
+                            running_eval_table,
+                            pending_eval_table,
+                        ],
+                    )
+                with gr.Accordion("Submit a new model for evaluation"):
+                    with gr.Row():
+                        with gr.Column():
+                            model_name_textbox = gr.Textbox(label="Model name")
+                            revision_name_textbox = gr.Textbox(
+                                label="revision", placeholder="main"
+                            )
+
+                        with gr.Column():
+                            is_8bit_toggle = gr.Checkbox(
+                                False, label="8 bit eval", visible=not IS_PUBLIC
+                            )
+                            private = gr.Checkbox(
+                                False, label="Private", visible=not IS_PUBLIC
+                            )
+                            is_delta_weight = gr.Checkbox(False, label="Delta weights")
+                            base_model_name_textbox = gr.Textbox(
+                                label="base model (for delta)"
+                            )
+
+                    submit_button = gr.Button("Submit Eval")
+                    submission_result = gr.Markdown()
+                    submit_button.click(
+                        add_new_eval,
+                        [
+                            model_name_textbox,
+                            base_model_name_textbox,
+                            revision_name_textbox,
+                            is_8bit_toggle,
+                            private,
+                            is_delta_weight,
+                        ],
+                        submission_result,
+                    )
+        with gr.TabItem(
+            "🧑‍⚖️ Human & GPT-4 Evaluations 🤖", elem_id="human-gpt-tab-table", id=1
+        ):
+            with gr.Row():
+                with gr.Column(scale=2):
+                    gr.Markdown(HUMAN_GPT_EVAL_TEXT, elem_classes="markdown-text")
+                with gr.Column(scale=1):
+                    gr.Image(
+                        "src/assets/scale-hf-logo.png", elem_id="scale-logo", show_label=False
+                    )
+            gr.Markdown("## No tie allowed")
+            elo_leaderboard_table = gr.components.Dataframe(
+                value=elo_leaderboard,
+                headers=ELO_COLS,
+                datatype=ELO_TYPES,
+                max_rows=5,
+            )
+
+            gr.Markdown("## Tie allowed*")
+            elo_leaderboard_table_with_tie_allowed = gr.components.Dataframe(
+                value=elo_leaderboard_with_tie_allowed,
+                headers=ELO_COLS,
+                datatype=ELO_TYPES,
+                max_rows=5,
+            )
+
+            gr.Markdown(
+                "\* Results when the scores of 4 and 5 were treated as ties.",
+                elem_classes="markdown-text",
+            )
+
+            gr.Markdown(
+                "Let us know in [this discussion](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/65) which models we should add!",
+                elem_id="models-to-add-text",
+            )
+
+    dummy = gr.Textbox(visible=False)
+    demo.load(
+        change_tab,
+        dummy,
+        tabs,
+        _js=get_window_url_params,
+    )
+    if ADD_PLOTS:
+        with gr.Box():
+            visualization_title = gr.HTML(VISUALIZATION_TITLE)
+            with gr.Row():
+                with gr.Column():
+                    gr.Markdown(f"#### Figure 1: {PLOT_1_TITLE}")
+                    plot_1 = gr.Plot(plot_1, show_label=False)
+                with gr.Column():
+                    gr.Markdown(f"#### Figure 2: {PLOT_2_TITLE}")
+                    plot_2 = gr.Plot(plot_2, show_label=False)
+            with gr.Row():
+                with gr.Column():
+                    gr.Markdown(f"#### Figure 3: {PLOT_3_TITLE}")
+                    plot_3 = gr.Plot(plot_3, show_label=False)
+                with gr.Column():
+                    gr.Markdown(f"#### Figure 4: {PLOT_4_TITLE}")
+                    plot_4 = gr.Plot(plot_4, show_label=False)
+
+scheduler = BackgroundScheduler()
+scheduler.add_job(restart_space, "interval", seconds=3600)
+scheduler.start()
+demo.queue(concurrency_count=40).launch()

requirements.txt

@@ -0,0 +1,70 @@
+aiofiles==23.1.0
+aiohttp==3.8.4
+aiosignal==1.3.1
+altair==4.2.2
+anyio==3.6.2
+APScheduler==3.10.1
+async-timeout==4.0.2
+attrs==23.1.0
+certifi==2022.12.7
+charset-normalizer==3.1.0
+click==8.1.3
+contourpy==1.0.7
+cycler==0.11.0
+datasets==2.12.0
+entrypoints==0.4
+fastapi==0.95.1
+ffmpy==0.3.0
+filelock==3.11.0
+fonttools==4.39.3
+frozenlist==1.3.3
+fsspec==2023.4.0
+gradio==3.27.0
+gradio_client==0.1.3
+h11==0.14.0
+httpcore==0.17.0
+httpx==0.24.0
+huggingface-hub==0.13.4
+idna==3.4
+Jinja2==3.1.2
+jsonschema==4.17.3
+kiwisolver==1.4.4
+linkify-it-py==2.0.0
+markdown-it-py==2.2.0
+MarkupSafe==2.1.2
+matplotlib==3.7.1
+mdit-py-plugins==0.3.3
+mdurl==0.1.2
+multidict==6.0.4
+numpy==1.24.2
+orjson==3.8.10
+packaging==23.1
+pandas==2.0.0
+Pillow==9.5.0
+plotly==5.14.1
+pyarrow==11.0.0
+pydantic==1.10.7
+pydub==0.25.1
+pyparsing==3.0.9
+pyrsistent==0.19.3
+python-dateutil==2.8.2
+python-multipart==0.0.6
+pytz==2023.3
+pytz-deprecation-shim==0.1.0.post0
+PyYAML==6.0
+requests==2.28.2
+semantic-version==2.10.0
+six==1.16.0
+sniffio==1.3.0
+starlette==0.26.1
+toolz==0.12.0
+tqdm==4.65.0
+transformers==4.28.1
+typing_extensions==4.5.0
+tzdata==2023.3
+tzlocal==4.3
+uc-micro-py==1.0.1
+urllib3==1.26.15
+uvicorn==0.21.1
+websockets==11.0.1
+yarl==1.8.2

src/assets/css_html_js.py

@@ -0,0 +1,87 @@
+custom_css = """
+#changelog-text {
+    font-size: 16px !important;
+}
+
+#changelog-text h2 {
+    font-size: 18px !important;
+}
+
+.markdown-text {
+    font-size: 16px !important;
+}
+
+#models-to-add-text {
+    font-size: 18px !important;
+}
+
+#citation-button span {
+    font-size: 16px !important;
+}
+
+#citation-button textarea {
+    font-size: 16px !important;
+}
+
+#citation-button > label > button {
+    margin: 6px;
+    transform: scale(1.3);
+}
+
+#leaderboard-table {
+    margin-top: 15px
+}
+
+#leaderboard-table-lite {
+    margin-top: 15px
+}
+
+#search-bar-table-box > div:first-child {
+    background: none;
+    border: none;
+}
+ 
+#search-bar {
+    padding: 0px;
+    width: 30%;
+}
+
+/* Hides the final AutoEvalColumn */
+#llm-benchmark-tab-table table td:last-child,
+#llm-benchmark-tab-table table th:last-child {
+    display: none;
+}
+
+/* Limit the width of the first AutoEvalColumn so that names don't expand too much */
+table td:first-child,
+table th:first-child {
+    max-width: 400px;
+    overflow: auto;
+    white-space: nowrap;
+}
+
+.tab-buttons button {
+    font-size: 20px;
+}
+
+#scale-logo {
+    border-style: none !important;
+    box-shadow: none;
+    display: block;
+    margin-left: auto;
+    margin-right: auto;
+    max-width: 600px;
+}
+
+#scale-logo .download {
+    display: none;
+}
+"""
+
+get_window_url_params = """
+    function(url_params) {
+        const params = new URLSearchParams(window.location.search);
+        url_params = Object.fromEntries(params);
+        return url_params;
+    }
+    """

src/assets/hardcoded_evals.py

@@ -0,0 +1,38 @@
+from src.utils_display import AutoEvalColumn, model_hyperlink
+
+gpt4_values = {
+    AutoEvalColumn.model.name: model_hyperlink("https://arxiv.org/abs/2303.08774", "gpt4"),
+    AutoEvalColumn.revision.name: "tech report",
+    AutoEvalColumn.is_8bit.name: None,
+    AutoEvalColumn.average.name: 84.3,
+    AutoEvalColumn.arc.name: 96.3,
+    AutoEvalColumn.hellaswag.name:  95.3,
+    AutoEvalColumn.mmlu.name:  86.4,
+    AutoEvalColumn.truthfulqa.name:  59.0,
+    AutoEvalColumn.dummy.name: "GPT-4",
+}
+
+gpt35_values = {
+    AutoEvalColumn.model.name: model_hyperlink("https://arxiv.org/abs/2303.08774", "gpt3.5"),
+    AutoEvalColumn.revision.name: "tech report",
+    AutoEvalColumn.is_8bit.name: None,
+    AutoEvalColumn.average.name: 71.9,
+    AutoEvalColumn.arc.name: 85.2,
+    AutoEvalColumn.hellaswag.name:  85.5,
+    AutoEvalColumn.mmlu.name:  70.0,
+    AutoEvalColumn.truthfulqa.name:  47.0,
+    AutoEvalColumn.dummy.name: "GPT-3.5",
+}
+
+baseline = {
+    AutoEvalColumn.model.name: "<p>Baseline</p>",
+    AutoEvalColumn.revision.name: "N/A",
+    AutoEvalColumn.is_8bit.name: None,
+    AutoEvalColumn.average.name: 25.0,
+    AutoEvalColumn.arc.name: 25.0,
+    AutoEvalColumn.hellaswag.name:  25.0,
+    AutoEvalColumn.mmlu.name:  25.0,
+    AutoEvalColumn.truthfulqa.name:  25.0,
+    AutoEvalColumn.dummy.name: "baseline",
+}
+

src/assets/text_content.py

@@ -0,0 +1,166 @@
+CHANGELOG_TEXT = f"""
+## [2023-06-19]
+- Added model type column
+- Hid revision and 8bit columns since all models are the same atm
+
+## [2023-06-16]
+- Refactored code base
+- Added new columns: number of parameters, hub likes, license
+
+## [2023-06-13] 
+- Adjust description for TruthfulQA
+
+## [2023-06-12] 
+- Add Human & GPT-4 Evaluations
+
+## [2023-06-05] 
+- Increase concurrent thread count to 40
+- Search models on ENTER
+
+## [2023-06-02] 
+- Add a typeahead search bar
+- Use webhooks to automatically spawn a new Space when someone opens a PR
+- Start recording `submitted_time` for eval requests
+- Limit AutoEvalColumn max-width
+
+## [2023-05-30] 
+- Add a citation button
+- Simplify Gradio layout
+
+## [2023-05-29] 
+- Auto-restart every hour for the latest results
+- Sync with the internal version (minor style changes)
+
+## [2023-05-24] 
+- Add a baseline that has 25.0 for all values
+- Add CHANGELOG
+
+## [2023-05-23] 
+- Fix a CSS issue that made the leaderboard hard to read in dark mode
+
+## [2023-05-22] 
+- Display a success/error message after submitting evaluation requests
+- Reject duplicate submission
+- Do not display results that have incomplete results 
+- Display different queues for jobs that are RUNNING, PENDING, FINISHED status 
+
+## [2023-05-15] 
+- Fix a typo: from "TruthQA" to "QA"
+
+## [2023-05-10] 
+- Fix a bug that prevented auto-refresh
+
+## [2023-05-10] 
+- Release the leaderboard to public
+"""
+
+TITLE = """<h1 align="center" id="space-title">🤗 Open LLM Leaderboard</h1>"""
+
+INTRODUCTION_TEXT = f"""
+📐 With the plethora of large language models (LLMs) and chatbots being released week upon week, often with grandiose claims of their performance, it can be hard to filter out the genuine progress that is being made by the open-source community and which model is the current state of the art. The 🤗 Open LLM Leaderboard aims to track, rank and evaluate LLMs and chatbots as they are released. 
+
+🤗 A key advantage of this leaderboard is that anyone from the community can submit a model for automated evaluation on the 🤗 GPU cluster, as long as it is a 🤗 Transformers model with weights on the Hub. We also support evaluation of models with delta-weights for non-commercial licensed models, such as LLaMa.
+
+📈 In the **first tab (LLM Benchmarks)**, we evaluate models on 4 key benchmarks from the <a href="https://github.com/EleutherAI/lm-evaluation-harness" target="_blank">  Eleuther AI Language Model Evaluation Harness </a>, a unified framework to test generative language models on a large number of different evaluation tasks. In the **second tab (Human & GPT Evaluations)**, the evaluations are performed by having humans and GPT-4 compare completions from a set of popular open-source language models (LLMs) on a secret set of instruction prompts.
+"""
+
+LLM_BENCHMARKS_TEXT = f"""
+Evaluation is performed against 4 popular benchmarks:
+- <a href="https://arxiv.org/abs/1803.05457" target="_blank">  AI2 Reasoning Challenge </a> (25-shot) - a set of grade-school science questions.
+- <a href="https://arxiv.org/abs/1905.07830" target="_blank">  HellaSwag </a> (10-shot) - a test of commonsense inference, which is easy for humans (~95%) but challenging for SOTA models.
+- <a href="https://arxiv.org/abs/2009.03300" target="_blank">  MMLU </a>  (5-shot) - a test to measure a text model's multitask accuracy. The test covers 57 tasks including elementary mathematics, US history, computer science, law, and more.
+- <a href="https://arxiv.org/abs/2109.07958" target="_blank">  TruthfulQA </a> (0-shot) - a test to measure a model’s propensity to reproduce falsehoods commonly found online.
+
+We chose these benchmarks as they test a variety of reasoning and general knowledge across a wide variety of fields in 0-shot and few-shot settings.
+"""
+
+HUMAN_GPT_EVAL_TEXT = f"""
+Evaluation is performed by having humans and GPT-4 compare completions from a set of popular open-source language models (LLMs) on a secret set of instruction prompts. The prompts cover tasks such as brainstorming, creative generation, commonsense reasoning, open question answering, summarization, and code generation. Comparisons are made by humans and a model on a 1-8 Likert scale, where the labeler is required to choose a preference each time. Using these preferences, we create bootstrapped Elo rankings.
+
+We collaborated with **Scale AI** to generate the completions using a professional data labeling workforce on their platform, [following the labeling instructions found here](https://docs.google.com/document/d/1c5-96Lj-UH4lzKjLvJ_MRQaVMjtoEXTYA4dvoAYVCHc/edit?usp=sharing). To understand the evaluation of popular models, we also had GPT-4 label the completions using this prompt.
+
+For more information on the calibration and initiation of these measurements, please refer to the [announcement blog post](https://huggingface.co/blog/llm-leaderboard). We would like to express our gratitude to **LMSYS** for providing a [useful notebook](https://colab.research.google.com/drive/1lAQ9cKVErXI1rEYq7hTKNaCQ5Q8TzrI5?usp=sharing) for computing Elo estimates and plots.
+"""
+
+
+EVALUATION_QUEUE_TEXT = f"""
+# Evaluation Queue for the 🤗 Open LLM Leaderboard, these models will be automatically evaluated on the 🤗 cluster
+"""
+
+CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
+CITATION_BUTTON_TEXT = r"""@misc{open-llm-leaderboard,
+  author = {Edward Beeching, Sheon Han, Nathan Lambert, Nazneen Rajani, Omar Sanseviero, Lewis Tunstall, Thomas Wolf},
+  title = {Open LLM Leaderboard},
+  year = {2023},
+  publisher = {Hugging Face},
+  howpublished = "\url{https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard}"
+
+}
+@software{eval-harness,
+  author       = {Gao, Leo and
+                  Tow, Jonathan and
+                  Biderman, Stella and
+                  Black, Sid and
+                  DiPofi, Anthony and
+                  Foster, Charles and
+                  Golding, Laurence and
+                  Hsu, Jeffrey and
+                  McDonell, Kyle and
+                  Muennighoff, Niklas and
+                  Phang, Jason and
+                  Reynolds, Laria and
+                  Tang, Eric and
+                  Thite, Anish and
+                  Wang, Ben and
+                  Wang, Kevin and
+                  Zou, Andy},
+  title        = {A framework for few-shot language model evaluation},
+  month        = sep,
+  year         = 2021,
+  publisher    = {Zenodo},
+  version      = {v0.0.1},
+  doi          = {10.5281/zenodo.5371628},
+  url          = {https://doi.org/10.5281/zenodo.5371628}
+}
+@misc{clark2018think,
+      title={Think you have Solved Question Answering? Try ARC, the AI2 Reasoning Challenge}, 
+      author={Peter Clark and Isaac Cowhey and Oren Etzioni and Tushar Khot and Ashish Sabharwal and Carissa Schoenick and Oyvind Tafjord},
+      year={2018},
+      eprint={1803.05457},
+      archivePrefix={arXiv},
+      primaryClass={cs.AI}
+}
+@misc{zellers2019hellaswag,
+      title={HellaSwag: Can a Machine Really Finish Your Sentence?}, 
+      author={Rowan Zellers and Ari Holtzman and Yonatan Bisk and Ali Farhadi and Yejin Choi},
+      year={2019},
+      eprint={1905.07830},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+@misc{hendrycks2021measuring,
+      title={Measuring Massive Multitask Language Understanding}, 
+      author={Dan Hendrycks and Collin Burns and Steven Basart and Andy Zou and Mantas Mazeika and Dawn Song and Jacob Steinhardt},
+      year={2021},
+      eprint={2009.03300},
+      archivePrefix={arXiv},
+      primaryClass={cs.CY}
+}
+@misc{lin2022truthfulqa,
+      title={TruthfulQA: Measuring How Models Mimic Human Falsehoods}, 
+      author={Stephanie Lin and Jacob Hilton and Owain Evans},
+      year={2022},
+      eprint={2109.07958},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}"""
+
+VISUALIZATION_TITLE = """<h1 align="center" id="space-title">📊 Visualizations</h1>"""
+
+PLOT_1_TITLE = "Fraction of Model A Wins for All Non-tied A vs. B Comparisons"
+
+PLOT_2_TITLE = "Comparison Count of Each Combination of Models (not allowing ties)"
+
+PLOT_3_TITLE = "Elo Estimates with error bars (ties allowed)"
+
+PLOT_4_TITLE = "Fraction of Model A Wins for All Non-tied A vs. B Comparisons"

src/auto_leaderboard/get_model_metadata.py

@@ -0,0 +1,56 @@
+import re
+from typing import List
+
+from src.utils_display import AutoEvalColumn
+from src.auto_leaderboard.model_metadata_type import get_model_type
+
+from huggingface_hub import HfApi
+import huggingface_hub
+api = HfApi()
+
+
+def get_model_infos_from_hub(leaderboard_data: List[dict]):
+    for model_data in leaderboard_data:
+        model_name = model_data["model_name_for_query"]
+        try:
+            model_info = api.model_info(model_name)
+        except huggingface_hub.utils._errors.RepositoryNotFoundError:
+            model_data[AutoEvalColumn.license.name] = None
+            model_data[AutoEvalColumn.likes.name] = None
+            model_data[AutoEvalColumn.params.name] = None
+            continue
+
+        model_data[AutoEvalColumn.license.name] = get_model_license(model_info)
+        model_data[AutoEvalColumn.likes.name] = get_model_likes(model_info)
+        model_data[AutoEvalColumn.params.name] = get_model_size(model_name, model_info)
+
+
+def get_model_license(model_info):
+    try:
+        return model_info.cardData["license"]
+    except Exception:
+        return None
+
+def get_model_likes(model_info):
+    return model_info.likes
+
+size_pattern = re.compile(r"\d+(b|m)")
+
+def get_model_size(model_name, model_info):
+    # In billions
+    try:
+        return round(model_info.safetensors["total"] / 1e9, 3) 
+    except AttributeError:
+        #print(f"Repository {model_id} does not have safetensors weights")
+        pass
+    try:
+        size_match = re.search(size_pattern, model_name.lower())
+        size = size_match.group(0)
+        return round(int(size[:-1]) if size[-1] == "b" else int(size[:-1]) / 1e3, 3)
+    except AttributeError:
+        return None
+
+
+def apply_metadata(leaderboard_data: List[dict]):
+    get_model_type(leaderboard_data)
+    get_model_infos_from_hub(leaderboard_data)

src/auto_leaderboard/load_results.py

@@ -0,0 +1,116 @@
+from dataclasses import dataclass
+
+import glob
+import json
+from typing import Dict, List, Tuple
+
+from src.utils_display import AutoEvalColumn, make_clickable_model
+import numpy as np
+
+# clone / pull the lmeh eval data
+METRICS = ["acc_norm", "acc_norm", "acc_norm", "mc2"]
+BENCHMARKS = ["arc_challenge", "hellaswag", "hendrycks", "truthfulqa_mc"]
+BENCH_TO_NAME = {
+    "arc_challenge": AutoEvalColumn.arc.name,
+    "hellaswag": AutoEvalColumn.hellaswag.name,
+    "hendrycks": AutoEvalColumn.mmlu.name,
+    "truthfulqa_mc": AutoEvalColumn.truthfulqa.name,
+}
+
+
+@dataclass
+class EvalResult:
+    eval_name: str
+    org: str
+    model: str
+    revision: str
+    is_8bit: bool
+    results: dict
+
+    def to_dict(self):
+        if self.org is not None:
+            base_model = f"{self.org}/{self.model}"
+        else:
+            base_model = f"{self.model}"
+        data_dict = {}
+
+        data_dict["eval_name"] = self.eval_name # not a column, just a save name
+        data_dict[AutoEvalColumn.is_8bit.name] = self.is_8bit
+        data_dict[AutoEvalColumn.model.name] = make_clickable_model(base_model)
+        data_dict[AutoEvalColumn.dummy.name] = base_model
+        data_dict[AutoEvalColumn.revision.name] = self.revision
+        data_dict[AutoEvalColumn.average.name] = round(
+            sum([v for k, v in self.results.items()]) / 4.0, 1
+        )
+
+        for benchmark in BENCHMARKS:
+            if not benchmark in self.results.keys():
+                self.results[benchmark] = None
+
+        for k, v in BENCH_TO_NAME.items():
+            data_dict[v] = self.results[k]
+
+        return data_dict
+
+
+def parse_eval_result(json_filepath: str) -> Tuple[str, dict]:
+    with open(json_filepath) as fp:
+        data = json.load(fp)
+
+    path_split = json_filepath.split("/")
+    org = None
+    model = path_split[-4]
+    is_8bit = path_split[-2] == "8bit"
+    revision = path_split[-3]
+    if len(path_split) == 7:
+        # handles gpt2 type models that don't have an org
+        result_key = f"{model}_{revision}_{is_8bit}"
+    else:
+        org = path_split[-5]
+        result_key =  f"{org}_{model}_{revision}_{is_8bit}"
+
+    eval_result = None
+    for benchmark, metric in zip(BENCHMARKS, METRICS):
+        if benchmark in json_filepath:
+            accs = np.array([v[metric] for v in data["results"].values()])
+            mean_acc = round(np.mean(accs) * 100.0, 1)
+            eval_result = EvalResult(
+                result_key, org, model, revision, is_8bit, {benchmark: mean_acc}
+            )
+
+    return result_key, eval_result
+
+
+def get_eval_results(is_public) -> List[EvalResult]:
+    json_filepaths = glob.glob(
+        "auto_evals/eval_results/public/**/16bit/*.json", recursive=True
+    )
+    if not is_public:
+        json_filepaths += glob.glob(
+            "auto_evals/eval_results/private/**/*.json", recursive=True
+        )
+        json_filepaths += glob.glob(
+            "auto_evals/eval_results/private/**/*.json", recursive=True
+        )
+        # include the 8bit evals of public models
+        json_filepaths += glob.glob(
+            "auto_evals/eval_results/public/**/8bit/*.json", recursive=True
+        )  
+    eval_results = {}
+
+    for json_filepath in json_filepaths:
+        result_key, eval_result = parse_eval_result(json_filepath)
+        if result_key in eval_results.keys():
+            eval_results[result_key].results.update(eval_result.results)
+        else:
+            eval_results[result_key] = eval_result
+
+    eval_results = [v for v in eval_results.values()]
+
+    return eval_results
+
+
+def get_eval_results_dicts(is_public=True) -> List[Dict]:
+    eval_results = get_eval_results(is_public)
+
+    return [e.to_dict() for e in eval_results]

src/auto_leaderboard/model_metadata_type.py

@@ -0,0 +1,163 @@
+from enum import Enum
+from typing import Dict, List
+
+class ModelType(Enum):
+    PT = "pretrained"
+    SFT = "finetuned"
+    RL = "with RL"
+
+
+TYPE_METADATA: Dict[str, ModelType] = {
+    "aisquared/dlite-v1-355m": ModelType.SFT,
+    "aisquared/dlite-v2-774m": ModelType.SFT,
+    "aisquared/dlite-v2-1_5b": ModelType.SFT,
+    "TheBloke/wizardLM-7B-HF": ModelType.SFT,
+    "TheBloke/dromedary-65b-lora-HF": ModelType.SFT,
+    "TheBloke/vicuna-13B-1.1-HF": ModelType.SFT,
+    "TheBloke/Wizard-Vicuna-13B-Uncensored-HF": ModelType.SFT,
+    "wordcab/llama-natural-instructions-13b": ModelType.SFT,
+    "JosephusCheung/Guanaco": ModelType.SFT,
+    "AlekseyKorshuk/vicuna-7b": ModelType.SFT,
+    "AlekseyKorshuk/chatml-pyg-v1": ModelType.SFT,
+    "concedo/OPT-19M-ChatSalad": ModelType.SFT,
+    "digitous/Javalion-R": ModelType.SFT,
+    "digitous/Alpacino30b": ModelType.SFT,
+    "digitous/Javelin-GPTJ": ModelType.SFT,
+    "anton-l/gpt-j-tiny-random": ModelType.SFT,
+    "IDEA-CCNL/Ziya-LLaMA-13B-Pretrain-v1": ModelType.SFT,
+    "gpt2-medium": ModelType.PT,
+    "PygmalionAI/pygmalion-6b": ModelType.SFT,
+    "medalpaca/medalpaca-7b": ModelType.SFT,
+    "medalpaca/medalpaca-13b": ModelType.SFT,
+    "chavinlo/alpaca-13b": ModelType.SFT,
+    "chavinlo/alpaca-native": ModelType.SFT,
+    "chavinlo/gpt4-x-alpaca": ModelType.SFT,
+    "hakurei/lotus-12B": ModelType.SFT,
+    "amazon/LightGPT": ModelType.SFT,
+    "shibing624/chinese-llama-plus-13b-hf": ModelType.SFT,
+    "mosaicml/mpt-7b": ModelType.PT,
+    "PSanni/Deer-3b": ModelType.SFT,
+    "bigscience/bloom-1b1": ModelType.PT,
+    "MetaIX/GPT4-X-Alpasta-30b": ModelType.SFT,
+    "EleutherAI/gpt-neox-20b": ModelType.PT,
+    "EleutherAI/gpt-j-6b": ModelType.PT,
+    "roneneldan/TinyStories-28M": ModelType.SFT,
+    "lmsys/vicuna-13b-delta-v1.1": ModelType.SFT,
+    "lmsys/vicuna-7b-delta-v1.1": ModelType.SFT,
+    "abhiramtirumala/DialoGPT-sarcastic-medium": ModelType.SFT,
+    "pillowtalks-ai/delta13b": ModelType.SFT,
+    "bigcode/starcoderplus": ModelType.SFT,
+    "microsoft/DialoGPT-large": ModelType.SFT,
+    "microsoft/CodeGPT-small-py": ModelType.SFT,
+    "Pirr/pythia-13b-deduped-green_devil": ModelType.SFT,
+    "Aeala/GPT4-x-AlpacaDente2-30b": ModelType.SFT,
+    "Aeala/VicUnlocked-alpaca-30b": ModelType.SFT,
+    "dvruette/llama-13b-pretrained-sft-epoch-2": ModelType.SFT,
+    "dvruette/oasst-gpt-neox-20b-1000-steps": ModelType.SFT,
+    "openlm-research/open_llama_3b_350bt_preview": ModelType.PT,
+    "openlm-research/open_llama_7b_700bt_preview": ModelType.PT,
+    "openlm-research/open_llama_7b": ModelType.PT,
+    "openlm-research/open_llama_3b": ModelType.PT,
+    "openlm-research/open_llama_7b_400bt_preview": ModelType.PT,
+    "PocketDoc/Dans-PileOfSets-Mk1-llama-13b-merged": ModelType.SFT,
+    "GeorgiaTechResearchInstitute/galactica-6.7b-evol-instruct-70k": ModelType.SFT,
+    "databricks/dolly-v2-7b": ModelType.SFT,
+    "databricks/dolly-v2-3b": ModelType.SFT,
+    "databricks/dolly-v2-12b": ModelType.SFT,
+    "pinkmanlove/llama-65b-hf": ModelType.SFT,
+    "Rachneet/gpt2-xl-alpaca": ModelType.SFT,
+    "Locutusque/gpt2-conversational-or-qa": ModelType.SFT,
+    "NbAiLab/nb-gpt-j-6B-alpaca": ModelType.SFT,
+    "Fredithefish/ScarletPajama-3B-HF": ModelType.SFT,
+    "eachadea/vicuna-7b-1.1": ModelType.SFT,
+    "eachadea/vicuna-13b": ModelType.SFT,
+    "openaccess-ai-collective/wizard-mega-13b": ModelType.SFT,
+    "openaccess-ai-collective/manticore-13b": ModelType.SFT,
+    "openaccess-ai-collective/manticore-30b-chat-pyg-alpha": ModelType.SFT,
+    "openaccess-ai-collective/minotaur-13b": ModelType.SFT,
+    "lamini/instruct-tuned-3b": ModelType.SFT,
+    "pythainlp/wangchanglm-7.5B-sft-enth": ModelType.SFT,
+    "pythainlp/wangchanglm-7.5B-sft-en-sharded": ModelType.SFT,
+    "stabilityai/stablelm-tuned-alpha-7b": ModelType.SFT,
+    "CalderaAI/30B-Lazarus": ModelType.SFT,
+    "KoboldAI/OPT-13B-Nerybus-Mix": ModelType.SFT,
+    "distilgpt2": ModelType.PT,
+    "wahaha1987/llama_7b_sharegpt94k_fastchat": ModelType.SFT,
+    "OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5": ModelType.SFT,
+    "junelee/wizard-vicuna-13b": ModelType.SFT,
+    "BreadAi/StoryPy": ModelType.SFT,
+    "togethercomputer/RedPajama-INCITE-Base-3B-v1": ModelType.PT,
+    "togethercomputer/RedPajama-INCITE-Base-7B-v0.1": ModelType.PT,
+    "Writer/camel-5b-hf": ModelType.SFT,
+    "Writer/palmyra-base": ModelType.PT,
+    "MBZUAI/lamini-neo-125m": ModelType.SFT,
+    "TehVenom/DiffMerge_Pygmalion_Main-onto-V8P4": ModelType.SFT,
+    "vicgalle/gpt2-alpaca-gpt4": ModelType.SFT,
+    "facebook/opt-350m": ModelType.PT,
+    "facebook/opt-125m": ModelType.PT,
+    "facebook/opt-13b": ModelType.PT,
+    "facebook/opt-1.3b": ModelType.PT,
+    "facebook/opt-66b": ModelType.PT,
+    "facebook/galactica-120b": ModelType.PT,
+    "Abe13/jgpt2-v1": ModelType.SFT,
+    "gpt2-xl": ModelType.PT,
+    "HuggingFaceH4/stable-vicuna-13b-2904": ModelType.RL,
+    "HuggingFaceH4/llama-7b-ift-alpaca": ModelType.SFT,
+    "HuggingFaceH4/starchat-alpha": ModelType.SFT,
+    "HuggingFaceH4/starchat-beta": ModelType.SFT,
+    "ausboss/Llama30B-SuperHOT": ModelType.SFT,
+    "ausboss/llama-13b-supercot": ModelType.SFT,
+    "ausboss/llama-30b-supercot": ModelType.SFT,
+    "Neko-Institute-of-Science/metharme-7b": ModelType.SFT,
+    "SebastianSchramm/Cerebras-GPT-111M-instruction": ModelType.SFT,
+    "victor123/WizardLM-13B-1.0": ModelType.SFT,
+    "AlpinDale/pygmalion-instruct": ModelType.SFT,
+    "tiiuae/falcon-7b-instruct": ModelType.SFT,
+    "tiiuae/falcon-40b-instruct": ModelType.SFT,
+    "tiiuae/falcon-40b": ModelType.PT,
+    "tiiuae/falcon-7b": ModelType.PT,
+    "cyl/awsome-llama": ModelType.SFT,
+    "xzuyn/Alpacino-SuperCOT-13B": ModelType.SFT,
+    "xzuyn/MedicWizard-7B": ModelType.SFT,
+    "beomi/KoAlpaca-Polyglot-5.8B": ModelType.SFT,
+    "chainyo/alpaca-lora-7b": ModelType.SFT,
+    "Salesforce/codegen-16B-nl": ModelType.PT,
+    "Salesforce/codegen-16B-multi": ModelType.SFT,
+    "ai-forever/rugpt3large_based_on_gpt2": ModelType.SFT,
+    "gpt2-large": ModelType.PT,
+    "huggingface/llama-13b": ModelType.PT,
+    "huggingface/llama-7b": ModelType.PT,
+    "huggingface/llama-65b": ModelType.PT,
+    "huggingface/llama-30b": ModelType.PT,
+    "jondurbin/airoboros-7b": ModelType.SFT,
+    "jondurbin/airoboros-13b": ModelType.SFT,
+    "cerebras/Cerebras-GPT-1.3B": ModelType.PT,
+    "cerebras/Cerebras-GPT-111M": ModelType.PT,
+    "NousResearch/Nous-Hermes-13b": ModelType.SFT,
+    "project-baize/baize-v2-7b": ModelType.SFT,
+    "project-baize/baize-v2-13b": ModelType.SFT,
+    "LLMs/AlpacaGPT4-7B-elina": ModelType.SFT,
+    "LLMs/Vicuna-EvolInstruct-13B": ModelType.SFT,
+    "huggingtweets/jerma985": ModelType.SFT,
+    "huggyllama/llama-65b": ModelType.PT,
+    "WizardLM/WizardLM-13B-1.0": ModelType.SFT,
+    "gpt2": ModelType.PT,
+    "alessandropalla/instruct_gpt2": ModelType.SFT,
+    "MayaPH/FinOPT-Lincoln": ModelType.SFT,
+    "MayaPH/FinOPT-Franklin": ModelType.SFT,
+    "timdettmers/guanaco-33b-merged": ModelType.SFT,
+    "timdettmers/guanaco-65b-merged": ModelType.SFT,
+    "elinas/llama-30b-hf-transformers-4.29": ModelType.SFT,
+    "elinas/chronos-33b": ModelType.SFT,
+    "nmitchko/medguanaco-65b-GPTQ": ModelType.SFT,
+    "xhyi/PT_GPTNEO350_ATG": ModelType.SFT,
+    "h2oai/h2ogpt-oasst1-512-20b": ModelType.SFT,
+    "h2oai/h2ogpt-gm-oasst1-en-1024-12b": ModelType.SFT,
+    "nomic-ai/gpt4all-13b-snoozy": ModelType.SFT,
+    "nomic-ai/gpt4all-j": ModelType.SFT,
+}
+
+
+def get_model_type(leaderboard_data: List[dict]):
+    for model_data in leaderboard_data:
+        model_data["Type"] = TYPE_METADATA.get(model_data["model_name_for_query"], "N/A")

src/elo_leaderboard/load_results.py

@@ -0,0 +1,200 @@
+from collections import defaultdict
+from dataclasses import dataclass
+from typing import Dict, List
+
+import numpy as np
+import pandas as pd
+from datasets import load_dataset
+
+from src.assets.text_content import PLOT_1_TITLE, PLOT_2_TITLE, PLOT_3_TITLE, PLOT_4_TITLE
+from src.utils_display import make_clickable_model, EloEvalColumn
+from .visualizations import (
+    get_bootstrap_result,
+    switch_model_a_b,
+    visualize_battle_count,
+    visualize_bootstrap_scores,
+    visualize_pairwise_win_fraction,
+    visualize_rating_count,
+)
+
+
+@dataclass
+class EloEvalResult:
+    model: str
+    gpt_4_all: int
+    human_all: int
+    human_instruct: int
+    human_code_instruct: int
+    tie_allowed: bool
+
+    def to_dict(self):
+        base_model = f"{self.model}"
+        data_dict = {}
+        data_dict[EloEvalColumn.model.name] = make_clickable_model(base_model)
+        data_dict[EloEvalColumn.gpt4.name] = self.gpt_4_all
+        data_dict[EloEvalColumn.human_all.name] = self.human_all
+        data_dict[EloEvalColumn.human_instruct.name] = self.human_instruct
+        data_dict[EloEvalColumn.human_code_instruct.name] = self.human_code_instruct
+
+        return data_dict
+
+
+def create_eval_df(df, tie_allowed):
+    responses = []
+    for _, row in df.iterrows():
+        if row["status"] == "canceled":
+            continue
+
+        rating = row["response"]["annotations"]["Preference"]
+        if rating == "NaN":
+            continue
+
+        scores = row["response"]["responses"]
+        if any(s["Preference"] == "" for s in scores):
+            continue
+
+        response = {
+            "id": row["task_id"],
+            "prompt": row["params"]["templateVariables"]["prompt"],
+            "model_a": row["params"]["templateVariables"]["modela"],
+            "model_b": row["params"]["templateVariables"]["modelb"],
+            "response_a": row["params"]["templateVariables"]["response1"],
+            "response_b": row["params"]["templateVariables"]["response2"],
+            "rating": int(rating),
+            "ratings": [np.array([s["Preference"] for s in scores], dtype=np.int32)],
+        }
+
+        if tie_allowed:
+            response["win"] = (
+                "model_a"
+                if response["rating"] < 4
+                else "model_b"
+                if response["rating"] > 5
+                else "tie"
+            )
+        else:
+            response["win"] = "model_a" if response["rating"] < 5 else "model_b"
+
+        responses.append(response)
+
+    return pd.DataFrame(responses)
+
+
+def create_eval_df_for_gpt(df, tie_allowed):
+    responses = []
+    for _, row in df.iterrows():
+        response = {
+            "id": row["review_id"],
+            "prompt": row["question"],
+            "model_a": row["model1"],
+            "model_b": row["model2"],
+            "response_a": row["answer1"],
+            "response_b": row["answer2"],
+            "rating": row["score"][0],
+        }
+
+        if tie_allowed:
+            response["win"] = (
+                "model_a"
+                if response["rating"] < 4
+                else "model_b"
+                if response["rating"] > 5
+                else "tie"
+            )
+        else:
+            response["win"] = "model_a" if response["rating"] < 5 else "model_b"
+
+        responses.append(response)
+
+    return pd.DataFrame(responses)
+
+
+# Compute the Elo rating for each model
+def compute_elo(df, k=32, scale=400, base=10, initial_rating=1000):
+    rating = defaultdict(lambda: initial_rating)
+
+    for _, model_a, model_b, win in df[["model_a", "model_b", "win"]].itertuples():
+        ra = rating[model_a]
+        rb = rating[model_b]
+        ea = 1 / (1 + base ** ((rb - ra) / scale))
+        eb = 1 / (1 + base ** ((ra - rb) / scale))
+        if win == "model_a":
+            sa = 1
+        elif win == "model_b":
+            sa = 0
+        elif win == "tie" or win == "tie (bothbad)":
+            sa = 0.5
+        else:
+            raise Exception(f"unexpected vote {win}")
+        rating[model_a] += k * (sa - ea)
+        rating[model_b] += k * (1 - sa - eb)
+
+    return rating
+
+
+def convert_rating_from_float_to_int(df):
+    return {model: int(rating) for model, rating in compute_elo(df).items()}
+
+
+def get_elo_results(df_instruct, df_code_instruct, tie_allowed):
+    df_all = pd.concat([df_instruct, df_code_instruct])
+
+    df_gpt_4 = load_dataset(
+        "gpt_4_evals/data/",
+        split="train",
+        revision="e007baaf6e505731c08a0bc1a833a1f8f8cb8846",
+    ).to_pandas()
+
+    dfs = [df_instruct, df_code_instruct, df_all]
+    elo_ratings = [
+        convert_rating_from_float_to_int(create_eval_df(df, tie_allowed=tie_allowed))
+        for df in dfs
+    ]
+
+    gpt_4_elo_ratings = convert_rating_from_float_to_int(
+        create_eval_df_for_gpt(df_gpt_4, tie_allowed=tie_allowed)
+    )
+    elo_ratings.append(gpt_4_elo_ratings)
+
+    results = [
+        EloEvalResult(
+            model=model_name,
+            gpt_4_all=elo_ratings[3][model_name],
+            human_all=elo_ratings[2][model_name],
+            human_instruct=elo_ratings[0][model_name],
+            human_code_instruct=elo_ratings[1][model_name],
+            tie_allowed=tie_allowed,
+        )
+        for model_name in elo_ratings[0].keys()
+    ]
+
+    return results
+
+
+def get_elo_results_dicts(df_instruct, df_code_instruct, tie_allowed) -> List[Dict]:
+    eval_results = get_elo_results(df_instruct, df_code_instruct, tie_allowed)
+    return [r.to_dict() for r in eval_results]
+
+
+def get_elo_plots(df_instruct, df_code_instruct, tie_allowed):
+    df_instruct = create_eval_df(df_instruct, tie_allowed=tie_allowed)
+    df_code_instruct = create_eval_df(df_code_instruct, tie_allowed=tie_allowed)
+    df_all = pd.concat([df_instruct, df_code_instruct])
+    game = df_all[["model_a", "model_b", "win"]]
+
+    game_switch = switch_model_a_b(game)
+    plot_1 = visualize_pairwise_win_fraction(game_switch, PLOT_1_TITLE)
+
+    plot_2 = visualize_battle_count(game_switch, PLOT_2_TITLE)
+
+    BOOTSTRAP_ROUNDS = 1000
+    if "bootstrap_elo_lu" not in globals():
+        bootstrap_elo_lu = get_bootstrap_result(
+            game_switch, compute_elo, BOOTSTRAP_ROUNDS
+        )
+
+    plot_3 = visualize_bootstrap_scores(bootstrap_elo_lu, PLOT_3_TITLE)
+
+    plot_4 = visualize_rating_count(game, PLOT_4_TITLE)
+
+    return plot_1, plot_2, plot_3, plot_4

src/elo_leaderboard/visualizations.py

@@ -0,0 +1,137 @@
+import math
+
+import numpy as np
+import pandas as pd
+import plotly.express as px
+
+
+# 1
+def compute_pairwise_win_fraction(battles):
+    # Times each model wins as Model A
+    a_win_ptbl = pd.pivot_table(
+        battles[battles["win"] == "model_a"],
+        index="model_a",
+        columns="model_b",
+        aggfunc="size",
+        fill_value=0,
+    )
+
+    # Table counting times each model wins as Model B
+    b_win_ptbl = pd.pivot_table(
+        battles[battles["win"] == "model_b"],
+        index="model_a",
+        columns="model_b",
+        aggfunc="size",
+        fill_value=0,
+    )
+
+    # Table counting number of A-B pairs
+    num_battles_ptbl = pd.pivot_table(battles, index="model_a", columns="model_b", aggfunc="size", fill_value=0)
+
+    # Computing the proportion of wins for each model as A and as B
+    # against all other models
+    row_beats_col_freq = (a_win_ptbl + b_win_ptbl.T) / (num_battles_ptbl + num_battles_ptbl.T)
+
+    # Arrange ordering according to proprition of wins
+    prop_wins = row_beats_col_freq.mean(axis=1).sort_values(ascending=False)
+    model_names = list(prop_wins.keys())
+    row_beats_col = row_beats_col_freq.loc[model_names, model_names]
+    return row_beats_col
+
+
+def visualize_pairwise_win_fraction(battles, title):
+    row_beats_col = compute_pairwise_win_fraction(battles)
+    fig = px.imshow(row_beats_col, color_continuous_scale="RdBu", text_auto=".2f", title=title)
+    fig.update_layout(
+        xaxis_title="Model B",
+        yaxis_title="Model A",
+        xaxis_side="top",
+        title_y=0.07,
+        title_x=0.5,
+    )
+    fig.update_traces(hovertemplate="Model A: %{y}<br>Model B: %{x}<br>Fraction of A Wins: %{z}<extra></extra>")
+    return fig
+
+
+# 2
+def switch_model_a_b(df):
+    df_switch = df.copy()
+    # switch with probability 0.5
+    for i, row in df.iterrows():
+        if np.random.rand() < 0.5:
+            df_switch.at[i, "model_a"] = row["model_b"]
+            df_switch.at[i, "model_b"] = row["model_a"]
+            if row["win"] == "model_a":
+                df_switch.at[i, "win"] = "model_b"
+            elif row["win"] == "model_b":
+                df_switch.at[i, "win"] = "model_a"
+    return df_switch
+
+
+def visualize_battle_count(battles, title):
+    ptbl = pd.pivot_table(battles, index="model_a", columns="model_b", aggfunc="size", fill_value=0)
+    battle_counts = ptbl + ptbl.T
+    ordering = battle_counts.sum().sort_values(ascending=False).index
+    fig = px.imshow(battle_counts.loc[ordering, ordering], title=title, text_auto=True, width=600)
+    fig.update_layout(
+        xaxis_title="Model B",
+        yaxis_title="Model A",
+        xaxis_side="top",
+        title_y=0.07,
+        title_x=0.5,
+    )
+    fig.update_traces(hovertemplate="Model A: %{y}<br>Model B: %{x}<br>Count: %{z}<extra></extra>")
+    return fig
+
+
+# 3
+def get_bootstrap_result(battles, func_compute_elo, num_round):
+    rows = [func_compute_elo(battles.sample(frac=1.0, replace=True)) for _ in range(num_round)]
+    df = pd.DataFrame(rows)
+    return df[df.median().sort_values(ascending=False).index]
+
+
+def visualize_bootstrap_scores(df, title):
+    bars = (
+        pd.DataFrame(
+            dict(
+                lower=df.quantile(0.025),
+                rating=df.quantile(0.5),
+                upper=df.quantile(0.975),
+            )
+        )
+        .reset_index(names="model")
+        .sort_values("rating", ascending=False)
+    )
+    bars["error_y"] = bars["upper"] - bars["rating"]
+    bars["error_y_minus"] = bars["rating"] - bars["lower"]
+    bars["rating_rounded"] = np.round(bars["rating"], 2)
+    fig = px.scatter(
+        bars,
+        x="model",
+        y="rating",
+        error_y="error_y",
+        error_y_minus="error_y_minus",
+        text="rating_rounded",
+        title=title,
+    )
+    fig.update_layout(xaxis_title="Model", yaxis_title="Rating")
+    return fig
+
+
+# 4
+def visualize_rating_count(df, title):
+    df_all_value_counts = pd.concat([df["model_a"], df["model_b"]]).value_counts()
+    fig = px.bar(df_all_value_counts, title=title, text_auto=True)
+
+    min_y = df_all_value_counts.min()
+    max_y = df_all_value_counts.max()
+
+    y_end = math.ceil(min_y / 100) * 100
+    y_begin = math.floor(max_y / 100) * 100
+
+    fig.update_layout(xaxis_title="model", yaxis_title="Rating Count", showlegend=False)
+    fig.update_yaxes(range=[y_begin, y_end])
+    # save the plot for the blog:
+    fig.write_html("src/assets/model_counts.html", full_html=False, include_plotlyjs="cdn")
+    return fig

src/init.py

@@ -0,0 +1,73 @@
+import os 
+from huggingface_hub import Repository
+
+H4_TOKEN = os.environ.get("H4_TOKEN", None)
+
+
+def get_all_requested_models(requested_models_dir):
+    depth = 1
+    file_names = []
+
+    for root, dirs, files in os.walk(requested_models_dir):
+        current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
+        if current_depth == depth:
+            file_names.extend([os.path.join(root, file) for file in files])
+
+    return set([file_name.lower().split("eval_requests/")[1] for file_name in file_names])
+
+def load_all_info_from_hub(LMEH_REPO, HUMAN_EVAL_REPO, GPT_4_EVAL_REPO):
+    auto_eval_repo = None
+    requested_models = None
+    if H4_TOKEN:
+        print("Pulling evaluation requests and results.")
+        # try:
+        #     shutil.rmtree("./auto_evals/")
+        # except:
+        #     pass
+
+        auto_eval_repo = Repository(
+            local_dir="./auto_evals/",
+            clone_from=LMEH_REPO,
+            use_auth_token=H4_TOKEN,
+            repo_type="dataset",
+        )
+        auto_eval_repo.git_pull()
+
+        requested_models_dir = "./auto_evals/eval_requests"
+        requested_models = get_all_requested_models(requested_models_dir)
+
+    human_eval_repo = None
+    if H4_TOKEN and not os.path.isdir("./human_evals"):
+        print("Pulling human evaluation repo")
+        human_eval_repo = Repository(
+            local_dir="./human_evals/",
+            clone_from=HUMAN_EVAL_REPO,
+            use_auth_token=H4_TOKEN,
+            repo_type="dataset",
+        )
+        human_eval_repo.git_pull()
+
+    gpt_4_eval_repo = None
+    if H4_TOKEN and not os.path.isdir("./gpt_4_evals"):
+        print("Pulling GPT-4 evaluation repo")
+        gpt_4_eval_repo = Repository(
+            local_dir="./gpt_4_evals/",
+            clone_from=GPT_4_EVAL_REPO,
+            use_auth_token=H4_TOKEN,
+            repo_type="dataset",
+        )
+        gpt_4_eval_repo.git_pull()
+
+    return auto_eval_repo, human_eval_repo, gpt_4_eval_repo, requested_models
+
+
+#def load_results(model, benchmark, metric):
+#    file_path = os.path.join("autoevals", model, f"{model}-eval_{benchmark}.json")
+#    if not os.path.exists(file_path):
+#        return 0.0, None
+
+#    with open(file_path) as fp:
+#        data = json.load(fp)
+#    accs = np.array([v[metric] for k, v in data["results"].items()])
+#    mean_acc = np.mean(accs)
+#    return mean_acc, data["config"]["model_args"]

src/utils_display.py

@@ -0,0 +1,98 @@
+from dataclasses import dataclass
+
+# These classes are for user facing column names, to avoid having to change them 
+# all around the code when a modif is needed 
+@dataclass
+class ColumnContent:
+    name: str
+    type: str 
+    displayed_by_default: bool 
+    hidden: bool = False
+
+def fields(raw_class):
+    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
+
+@dataclass(frozen=True)
+class AutoEvalColumn: # Auto evals column
+    model = ColumnContent("Model", "markdown", True)
+    revision = ColumnContent("Revision", "str", True, True)
+    model_type = ColumnContent("Type", "bool", False)
+    is_8bit = ColumnContent("8bit", "bool", False, True)
+    license = ColumnContent("Hub License", "str", False)
+    params = ColumnContent("#Params (B)", "number", False)
+    likes = ColumnContent("Hub ❤️", "number", False)
+    average = ColumnContent("Average ⬆️", "number", True)
+    arc = ColumnContent("ARC (25-s) ⬆️", "number", True)
+    hellaswag = ColumnContent("HellaSwag (10-s) ⬆️", "number", True)
+    mmlu = ColumnContent("MMLU (5-s) ⬆️", "number", True)
+    truthfulqa = ColumnContent("TruthfulQA (MC) (0-s) ⬆️", "number", True)
+    dummy = ColumnContent("model_name_for_query", "str", True) # dummy col to implement search bar (hidden by custom CSS)
+
+@dataclass(frozen=True)
+class EloEvalColumn: # Elo evals column
+    model = ColumnContent("Model", "markdown", True)
+    gpt4 = ColumnContent("GPT-4 (all)", "number", True)
+    human_all = ColumnContent("Human (all)", "number", True)
+    human_instruct = ColumnContent("Human (instruct)", "number", True)
+    human_code_instruct = ColumnContent("Human (code-instruct)", "number", True)
+
+
+@dataclass(frozen=True)
+class EvalQueueColumn: # Queue column
+    model = ColumnContent("model", "markdown", True)
+    revision = ColumnContent("revision", "str", True)
+    private = ColumnContent("private", "bool", True)
+    is_8bit = ColumnContent("8bit_eval", "bool", True)
+    has_delta_weight = ColumnContent("is_delta_weight", "bool", True)
+    status = ColumnContent("status", "str", True)
+
+LLAMAS = ["huggingface/llama-7b", "huggingface/llama-13b", "huggingface/llama-30b", "huggingface/llama-65b"]
+
+
+KOALA_LINK = "https://huggingface.co/TheBloke/koala-13B-HF"
+VICUNA_LINK = "https://huggingface.co/lmsys/vicuna-13b-delta-v1.1"
+OASST_LINK = "https://huggingface.co/OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5"
+DOLLY_LINK = "https://huggingface.co/databricks/dolly-v2-12b"
+MODEL_PAGE = "https://huggingface.co/models"
+LLAMA_LINK = "https://ai.facebook.com/blog/large-language-model-llama-meta-ai/"
+VICUNA_LINK = "https://huggingface.co/CarperAI/stable-vicuna-13b-delta"
+ALPACA_LINK = "https://crfm.stanford.edu/2023/03/13/alpaca.html"
+
+
+def model_hyperlink(link, model_name):
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+
+
+def make_clickable_model(model_name):
+    link = f"https://huggingface.co/{model_name}"
+
+    if model_name in LLAMAS:
+        link = LLAMA_LINK
+        model_name = model_name.split("/")[1]
+    elif model_name == "HuggingFaceH4/stable-vicuna-13b-2904":
+        link = VICUNA_LINK
+        model_name = "stable-vicuna-13b"
+    elif model_name == "HuggingFaceH4/llama-7b-ift-alpaca":
+        link = ALPACA_LINK
+        model_name = "alpaca-13b"
+    if model_name == "dolly-12b":
+        link = DOLLY_LINK
+    elif model_name == "vicuna-13b":
+        link = VICUNA_LINK
+    elif model_name == "koala-13b":
+        link = KOALA_LINK
+    elif model_name == "oasst-12b":
+        link = OASST_LINK
+    #else:
+    #    link = MODEL_PAGE
+  
+    return model_hyperlink(link, model_name)
+
+def styled_error(error):
+    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
+
+def styled_warning(warn):
+    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
+
+def styled_message(message):
+    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"