Combine IFEval

app.py

@@ -49,11 +49,18 @@ def get_leaderboard_df():
                 if task.lower() == "truthfulqa":
                     value = data["results"][first_result_key]["truthfulqa_mc2"]
                     df.loc[model_revision, task] = float(value)
-                # IFEval has several metrics but we report just the prompt-loose-acc one
+                # IFEval has several metrics but we report the average like Llama3 paper
                 elif task.lower() == "ifeval":
-                    for metric in ["prompt_level_loose", "prompt_level_strict"]:
-                        value = data["results"][first_result_key][f"{metric}_acc"]
-                        df.loc[model_revision, f"{task}_{metric}"] = float(value)
+                    values = 0.0
+                    for metric in [
+                        "prompt_level_loose",
+                        "prompt_level_strict",
+                        "inst_level_strict",
+                        "inst_level_loose",
+                    ]:
+                        values += data["results"][first_result_key][f"{metric}_acc"]
+                    value = values / 4
+                    df.loc[model_revision, f"{task}"] = float(value)
                 # MMLU has several metrics but we report just the average one
                 elif task.lower() == "mmlu":
                     value = [v["acc"] for k, v in data["results"].items() if "_average" in k.lower()][0]