Code formatting

app.py

@@ -5,9 +5,15 @@ import pandas as pd
 from gradio.themes.utils.sizes import text_md
 from gradio_modal import Modal
 
-from content import (HEADER_MARKDOWN, LEADERBOARD_TAB_TITLE_MARKDOWN, SUBMISSION_TAB_TITLE_MARKDOWN,
-                     MODAL_SUBMIT_MARKDOWN,
-                     SUBMISSION_DETAILS_MARKDOWN, RANKING_AFTER_SUBMISSION_MARKDOWN, MORE_DETAILS_MARKDOWN)
+from content import (
+    HEADER_MARKDOWN,
+    LEADERBOARD_TAB_TITLE_MARKDOWN,
+    SUBMISSION_TAB_TITLE_MARKDOWN,
+    MODAL_SUBMIT_MARKDOWN,
+    SUBMISSION_DETAILS_MARKDOWN,
+    RANKING_AFTER_SUBMISSION_MARKDOWN,
+    MORE_DETAILS_MARKDOWN,
+)
 from server import LeaderboardServer
 
 leaderboard_server = LeaderboardServer()
@@ -38,37 +44,63 @@ def process_submission(team_name, submission_id, description, link_to_model, sub
         leaderboard_server.prepare_model_for_submission(submission_file, metadata)
     except ValueError as err:
         gr.Warning(str(err))
-        return gr.update(value='Pre-submit model', visible=True, interactive=True), gr.update(
-            visible=False), gr.update(visible=False), gr.update(visible=False), gr.update(
-            visible=False), gr.update(visible=False)
-    return gr.update(visible=False), gr.update(visible=True), gr.update(interactive=True,
-                                                                        visible=True), gr.update(
-        interactive=True, visible=True), gr.update(visible=True), gr.update(
-        value=leaderboard_server.get_leaderboard(leaderboard_server.pre_submit[0]), visible=True)
+        return (
+            gr.update(value='Pre-submit model', visible=True, interactive=True),
+            gr.update(visible=False),
+            gr.update(visible=False),
+            gr.update(visible=False),
+            gr.update(visible=False),
+            gr.update(visible=False),
+        )
+    return (
+        gr.update(visible=False),
+        gr.update(visible=True),
+        gr.update(interactive=True, visible=True),
+        gr.update(interactive=True, visible=True),
+        gr.update(visible=True),
+        gr.update(
+            value=leaderboard_server.get_leaderboard(leaderboard_server.pre_submit[0]),
+            visible=True,
+        ),
+    )
 
 
 def submit_results():
     leaderboard_server.save_pre_submit()
     leaderboard_server.update_leaderboard()
     gr.Info('Submission successful!')
-    return gr.update(value='Pre-submit model', visible=True, interactive=True), gr.update(
-        visible=False), gr.update(visible=False), gr.update(visible=False), gr.update(
-        visible=False), gr.update(visible=False), gr.DataFrame(
-        value=leaderboard_server.get_leaderboard(), visible=True), gr.update(visible=False), gr.update(
-        choices=leaderboard_server.submission_ids)
+    return (
+        gr.update(value='Pre-submit model', visible=True, interactive=True),
+        gr.update(visible=False),
+        gr.update(visible=False),
+        gr.update(visible=False),
+        gr.update(visible=False),
+        gr.update(visible=False),
+        gr.DataFrame(value=leaderboard_server.get_leaderboard(), visible=True),
+        gr.update(visible=False),
+        gr.update(choices=leaderboard_server.submission_ids),
+    )
 
 
 def erase_pre_submit():
     leaderboard_server.pre_submit = None
-    return gr.update(value='Pre-submit model', visible=True, interactive=True), gr.update(
-        visible=False), gr.update(visible=False), gr.update(visible=False), gr.update(
-        visible=False), gr.update(visible=False), gr.update(visible=False)
+    return (
+        gr.update(value='Pre-submit model', visible=True, interactive=True),
+        gr.update(visible=False),
+        gr.update(visible=False),
+        gr.update(visible=False),
+        gr.update(visible=False),
+        gr.update(visible=False),
+        gr.update(visible=False),
+    )
 
 
 def fetch_model_detail(submission_id):
     metadata = leaderboard_server.get_model_detail(submission_id)
-    return gr.update(value=metadata['description'], visible=True), gr.update(value=metadata['link_to_model'],
-                                                                             visible=True)
+    return (
+        gr.update(value=metadata['description'], visible=True),
+        gr.update(value=metadata['link_to_model'], visible=True)
+    )
 
 
 def show_modal():
@@ -82,9 +114,10 @@ def hide_modal():
 def on_application_load():
     leaderboard_server.save_pre_submit()
     leaderboard_server.update_leaderboard()
-    return gr.DataFrame(
-        value=leaderboard_server.get_leaderboard(), visible=True), gr.update(
-        choices=leaderboard_server.submission_ids)
+    return (
+        gr.DataFrame(value=leaderboard_server.get_leaderboard(), visible=True),
+        gr.update(choices=leaderboard_server.submission_ids)
+    )
 
 
 custom_css = """
@@ -143,7 +176,7 @@ footer {visibility: hidden}
 
 """
 
-with (gr.Blocks(theme=gr.themes.Soft(text_size=text_md), css=custom_css) as main):
+with gr.Blocks(theme=gr.themes.Soft(text_size=text_md), css=custom_css) as main:
     with gr.Row():
         with gr.Row():
             gr.Markdown(HEADER_MARKDOWN)
@@ -155,17 +188,30 @@ with (gr.Blocks(theme=gr.themes.Soft(text_size=text_md), css=custom_css) as main
 
             with gr.Row():
                 with gr.Tab("Overall"):
-                    results_table = gr.DataFrame(leaderboard_server.get_leaderboard(), interactive=False, label=None,
-                                                 visible=True, elem_classes="leaderboard-table")
+                    results_table = gr.DataFrame(
+                        leaderboard_server.get_leaderboard(),
+                        interactive=False,
+                        label=None,
+                        visible=True,
+                        elem_classes="leaderboard-table",
+                    )
                 for c in leaderboard_server.tasks_categories:
                     with gr.Tab(c):
-                        results_table = gr.DataFrame(leaderboard_server.get_leaderboard(), interactive=False, label=None,
-                                                     visible=True, elem_classes="leaderboard-table")
+                        results_table = gr.DataFrame(
+                            leaderboard_server.get_leaderboard(),
+                            interactive=False,
+                            label=None,
+                            visible=True,
+                            elem_classes="leaderboard-table",
+                        )
 
         with gr.Tab('Model details'):
             gr.Markdown(MORE_DETAILS_MARKDOWN)
-            detail_dropdown = gr.Dropdown(choices=leaderboard_server.submission_ids, label="Select model",
-                                          interactive=True)
+            detail_dropdown = gr.Dropdown(
+                choices=leaderboard_server.submission_ids,
+                label="Select model",
+                interactive=True,
+            )
 
             with gr.Row():
                 model_description = gr.Text(value='', label='Model description', visible=False, interactive=False)
@@ -174,7 +220,8 @@ with (gr.Blocks(theme=gr.themes.Soft(text_size=text_md), css=custom_css) as main
             detail_dropdown.change(
                 fn=fetch_model_detail,
                 inputs=[detail_dropdown],
-                outputs=[model_description, model_url])
+                outputs=[model_description, model_url],
+            )
 
         with gr.Tab('Submission'):
             with gr.Column():
@@ -213,13 +260,24 @@ with (gr.Blocks(theme=gr.themes.Soft(text_size=text_md), css=custom_css) as main
                 pre_submission_btn.click(
                     fn=on_submit_pressed,
                     concurrency_limit=1,
-                    outputs=[pre_submission_btn]
+                    outputs=[pre_submission_btn],
                 ).then(
                     fn=process_submission,
-                    inputs=[submission_team_name_tb, submission_id_tb, description_tb, link_to_model_tb,
-                            submission_file_path],
-                    outputs=[pre_submission_btn, submit_prompt, submission_btn_yes, submission_btn_no, pre_submit_info,
-                             pre_submit_table]
+                    inputs=[
+                        submission_team_name_tb,
+                        submission_id_tb,
+                        description_tb,
+                        link_to_model_tb,
+                        submission_file_path,
+                    ],
+                    outputs=[
+                        pre_submission_btn,
+                        submit_prompt,
+                        submission_btn_yes,
+                        submission_btn_no,
+                        pre_submit_info,
+                        pre_submit_table,
+                    ],
                 )
 
                 submission_btn_yes.click(
@@ -229,8 +287,17 @@ with (gr.Blocks(theme=gr.themes.Soft(text_size=text_md), css=custom_css) as main
 
                 modal_submit_yes.click(
                     fn=submit_results,
-                    outputs=[pre_submission_btn, submission_btn_yes, submission_btn_no, submit_prompt, pre_submit_info,
-                             pre_submit_table, results_table, modal_submit, detail_dropdown]
+                    outputs=[
+                        pre_submission_btn,
+                        submission_btn_yes,
+                        submission_btn_no,
+                        submit_prompt,
+                        pre_submit_info,
+                        pre_submit_table,
+                        results_table,
+                        modal_submit,
+                        detail_dropdown,
+                    ],
                 )
 
                 modal_submit_no.click(
@@ -240,8 +307,14 @@ with (gr.Blocks(theme=gr.themes.Soft(text_size=text_md), css=custom_css) as main
 
                 submission_btn_no.click(
                     fn=erase_pre_submit,
-                    outputs=[pre_submission_btn, submission_btn_yes, submission_btn_no, submit_prompt, pre_submit_info,
-                             pre_submit_table]
+                    outputs=[
+                        pre_submission_btn,
+                        submission_btn_yes,
+                        submission_btn_no,
+                        submit_prompt,
+                        pre_submit_info,
+                        pre_submit_table,
+                    ],
                 )
     main.load(on_application_load, inputs=None, outputs=[results_table, detail_dropdown])
 

compare_significance.py

@@ -36,7 +36,14 @@ def _get_CMs(i, probabilities, references, thresholds):
                     FN += 1
                 else:
                     TN += 1
-        cm = {"TP": TP, "FP": FP, "TN": TN, "FN": FN, "threshold": threshold, "class": i}
+        cm = {
+            "TP": TP,
+            "FP": FP,
+            "TN": TN,
+            "FN": FN,
+            "threshold": threshold,
+            "class": i,
+        }
         confusion_matrices.append(cm)
 
     return confusion_matrices
@@ -73,16 +80,20 @@ def compute_significance_bootstrap(scores_A, scores_B):
     return pval, delta_orig
 
 
-def compute_significance_avg_mcauroc(probsA: Sequence[Sequence[float]], referencesA: Sequence[int],
-                                     probsB: Sequence[Sequence[float]], referencesB: Sequence[int]):
+def compute_significance_avg_mcauroc(
+    probsA: Sequence[Sequence[float]], referencesA: Sequence[int],
+    probsB: Sequence[Sequence[float]], referencesB: Sequence[int],
+):
     # compute MC-AUC for model A
     model_A_scores = get_mc_auc_samples(probsA, referencesA, Nsamples=100)
     model_B_scores = get_mc_auc_samples(probsB, referencesB, Nsamples=100)
     delta = np.mean(model_A_scores) - np.mean(model_B_scores)
 
     # one-tailed test
-    p_value = ((model_A_scores[:, np.newaxis] <= model_B_scores[np.newaxis, :]).sum()
-               / (len(model_A_scores) * len(model_B_scores)))
+    p_value = (
+        (model_A_scores[:, np.newaxis] <= model_B_scores[np.newaxis, :]).sum()
+        / (len(model_A_scores) * len(model_B_scores))
+    )
 
     return p_value, delta
 
@@ -114,8 +125,10 @@ def get_mc_auc_samples(probs, references, Nsamples=1_000_000):
     auc_scores_per_class = []
     for i in range(len(n_classes)):
         # for i-th class vs all others
-        fpr[i], _, thresholds[i] = roc_curve(y_true=[1 if x == n_classes[i] else 0 for x in references],
-                                             y_score=[prob[i] for prob in probs])
+        fpr[i], _, thresholds[i] = roc_curve(
+            y_true=[1 if x == n_classes[i] else 0 for x in references],
+            y_score=[prob[i] for prob in probs],
+        )
 
         confusion_matrices = _get_CMs(i, probs, references, thresholds)
         tp, fn = convert_confusion_matrices(confusion_matrices)
@@ -194,13 +207,20 @@ def process_task(task, dataA, dataB, significance_level):
     assert len(dataA[task]) == len(dataB[task])
 
     if metricA == "avg_mcauroc":
-        p_value, delta = compute_significance_avg_mcauroc(probsA=dataA[task][0][1], referencesA=dataA[task][0][0],
-                                                          probsB=dataB[task][0][1], referencesB=dataB[task][0][0])
+        p_value, delta = compute_significance_avg_mcauroc(
+            probsA=dataA[task][0][1], referencesA=dataA[task][0][0],
+            probsB=dataB[task][0][1], referencesB=dataB[task][0][0],
+        )
     elif metricA in ["acc", "exact_match"]:
-        p_value, delta = compute_significance_ttest(scores_A=dataA[task][0], scores_B=dataB[task][0])
+        p_value, delta = compute_significance_ttest(
+            scores_A=dataA[task][0],
+            scores_B=dataB[task][0]
+        )
     elif metricA in ["rouge_raw_r2_mid_f", "word_perplexity"]:
-        p_value, delta = compute_significance_bootstrap(scores_A=np.array(dataA[task][0]),
-                                                        scores_B=np.array(dataB[task][0]))
+        p_value, delta = compute_significance_bootstrap(
+            scores_A=np.array(dataA[task][0]),
+            scores_B=np.array(dataB[task][0])
+        )
     else:
         raise ValueError(f"Unsupported metric {metricA}")
 
@@ -228,14 +248,21 @@ def check_significance(fileA, fileB, significance_level=0.05):
         assert len(dataA[task]) == len(dataB[task])
 
         if metricA == "avg_mcauroc":
-            p_value, delta = compute_significance_avg_mcauroc(probsA=dataA[task][0][1], referencesA=dataA[task][0][0],
-                                                              probsB=dataB[task][0][1], referencesB=dataB[task][0][0])
+            p_value, delta = compute_significance_avg_mcauroc(
+                probsA=dataA[task][0][1], referencesA=dataA[task][0][0],
+                probsB=dataB[task][0][1], referencesB=dataB[task][0][0],
+            )
 
         elif metricA in ["acc", "exact_match"]:
-            p_value, delta = compute_significance_ttest(scores_A=dataA[task][0], scores_B=dataB[task][0])
+            p_value, delta = compute_significance_ttest(
+                scores_A=dataA[task][0],
+                scores_B=dataB[task][0]
+            )
         elif metricA in ["rouge_raw_r2_mid_f", "word_perplexity"]:
-            p_value, delta = compute_significance_bootstrap(scores_A=np.array(dataA[task][0]),
-                                                            scores_B=np.array(dataB[task][0]))
+            p_value, delta = compute_significance_bootstrap(
+                scores_A=np.array(dataA[task][0]),
+                scores_B=np.array(dataB[task][0])
+            )
         else:
             raise ValueError(f"Unsupported metric {metricA}")
         if delta <= 0:
@@ -253,7 +280,12 @@ def main():
     parser = argparse.ArgumentParser(description="One-tailed test if model A improves over model B.")
     parser.add_argument("--modelA", help="ModelA JSON file from lm harness.")
     parser.add_argument("--modelB", help="ModelB JSON file from lm harness.")
-    parser.add_argument("--significance_level", type=float, default=0.05, help="Significance level (e.g., 0.05)")
+    parser.add_argument(
+        "--significance_level",
+        type=float,
+        default=0.05,
+        help="Significance level (e.g., 0.05)",
+    )
     args = parser.parse_args()
 
     result = check_significance(args.modelA, args.modelB, args.significance_level)

server.py

@@ -18,12 +18,17 @@ REPO = f"{ORG}/LLM_benchmark_data"
 HF_TOKEN = os.environ.get("HF_TOKEN")
 TASKS_METADATA_PATH = "./tasks_metadata.json"
 
+
 class LeaderboardServer:
     def __init__(self):
         self.server_address = REPO
         self.repo_type = "dataset"
-        self.local_leaderboard = snapshot_download(self.server_address, repo_type=self.repo_type, token=HF_TOKEN,
-                                                   local_dir="./")
+        self.local_leaderboard = snapshot_download(
+            self.server_address,
+            repo_type=self.repo_type,
+            token=HF_TOKEN,
+            local_dir="./",
+        )
         self.submisssion_id_to_file = {}  # Map submission ids to file paths
         self.tasks_metadata = json.load(open(TASKS_METADATA_PATH))
         self.tasks_categories = {self.tasks_metadata[task]["category"] for task in self.tasks_metadata}
@@ -33,8 +38,12 @@ class LeaderboardServer:
         self.pre_submit = None
 
     def update_leaderboard(self):
-        self.local_leaderboard = snapshot_download(self.server_address, repo_type=self.repo_type, token=HF_TOKEN,
-                                                   local_dir="./")
+        self.local_leaderboard = snapshot_download(
+            self.server_address,
+            repo_type=self.repo_type,
+            token=HF_TOKEN,
+            local_dir="./",
+        )
         self.fetch_existing_models()
         self.tournament_results = self.load_tournament_results()
 
@@ -96,17 +105,27 @@ class LeaderboardServer:
                 else:
                     processed_results.append(local_results)
             dataframe = pd.DataFrame.from_records(processed_results)
-            df_order = (["submission_id"] + list(self.tasks_metadata.keys()) +
-                        [col for col in dataframe.columns if
-                         col != "submission_id" and col not in self.tasks_metadata.keys()])
+            df_order = (
+                ["submission_id"]
+                + list(self.tasks_metadata.keys())
+                + [
+                    col
+                    for col in dataframe.columns
+                    if col != "submission_id" and col not in self.tasks_metadata.keys()
+                ]
+            )
             dataframe = dataframe[df_order]
-            dataframe = dataframe.rename(columns={key: value["name"] for key, value in self.tasks_metadata.items()})
+            dataframe = dataframe.rename(
+                columns={key: value["name"] for key, value in self.tasks_metadata.items()}
+            )
             return dataframe
 
     def start_tournament(self, new_model_id, new_model_file):
         new_tournament = copy.deepcopy(self.tournament_results)
         new_tournament[new_model_id] = {}
-        new_tournament[new_model_id][new_model_id] = {task: False for task in self.tasks_metadata.keys()}
+        new_tournament[new_model_id][new_model_id] = {
+            task: False for task in self.tasks_metadata.keys()
+        }
 
         for model in self.submission_ids:
             res = check_significance(new_model_file, self.submisssion_id_to_file[model])
@@ -124,7 +143,7 @@ class LeaderboardServer:
             data = json.load(f)
         data["metadata"] = metadata
         with open(file, "w") as f:
-            json.dump(data, f, separators=(',', ':')) # compact JSON
+            json.dump(data, f, separators=(',', ':'))  # compact JSON
 
         model_id = metadata["team_name"] + "_" + metadata["submission_id"]
         tournament_results = self.start_tournament(model_id, file)
@@ -145,7 +164,7 @@ class LeaderboardServer:
             # Temporary save tournament results
             tournament_results_path = os.path.join(self.local_leaderboard, "tournament.json")
             with open(tournament_results_path, "w") as f:
-                json.dump(tournament_results, f, sort_keys=True, indent=2) # readable JSON
+                json.dump(tournament_results, f, sort_keys=True, indent=2)  # readable JSON
 
             api.upload_file(
                 path_or_fileobj=tournament_results_path,