Add graphical results comparison

app.py

@@ -14,7 +14,7 @@ from src.display.about import (
     TABLE_DESC,
 )
 from src.display.css_html_js import custom_css
-from src.display.formatting import styled_error, styled_message, styled_warning
+from src.display.formatting import styled_error, styled_message, styled_warning, model_hyperlink
 from src.display.utils import (
     BENCHMARK_COLS,
     COLS,
@@ -35,6 +35,7 @@ from src.submission.submit import add_new_eval
 from captcha.image import ImageCaptcha
 from PIL import Image
 import random, string
+import matplotlib.pyplot as plt
 
 
 original_df = None
@@ -44,6 +45,12 @@ leaderboard_df = None
 def restart_space():
     API.restart_space(repo_id=REPO_ID, token=TOKEN)
 
+def add_model_hyperlinks(row):
+    if row["Model URL"] is None or row["Model URL"] == "":
+        return row["Model"]
+    else:
+        return model_hyperlink(row["Model URL"], row["Model"])
+
 def download_data():
     global original_df
     global leaderboard_df
@@ -65,7 +72,8 @@ def download_data():
 
     _, original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
     leaderboard_df = original_df.copy()
-
+    leaderboard_df["Model"] = leaderboard_df.apply(add_model_hyperlinks, axis=1)
+    leaderboard_df.sort_values(by=["Aggregate Score"], ascending=False, inplace=True)
 
 download_data()
 
@@ -88,8 +96,6 @@ def update_table(
     #filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted)
     filtered_df = filter_queries(query, hidden_df)
     df = select_columns(filtered_df, columns)
-    print("TF")
-    print(df)
     return df
 
 
@@ -234,7 +240,6 @@ with demo:
                     )
                 """
             gr.Markdown(TABLE_DESC, elem_classes="markdown-text")
-            #print(shown_columns.value)
             leaderboard_table = gr.Dataframe(
                 value=leaderboard_df[
                     [c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value
@@ -244,10 +249,8 @@ with demo:
                 elem_id="leaderboard-table",
                 interactive=False,
                 visible=True,
-                wrap=False
+                wrap=False,
             )
-            print(leaderboard_table.value)
-            print(leaderboard_table.headers)
 
             # Dummy leaderboard for handling the case when the user uses backspace key
             hidden_leaderboard_table_for_search = gr.Dataframe(
@@ -278,6 +281,24 @@ with demo:
                     leaderboard_table,
                     queue=True,
                 )
+
+            model_num = len(original_df)
+            graph_df = original_df.drop(columns=["_", "Precision", "Model URL"]).set_index("Model").T
+            graph_ax = graph_df.plot(
+                kind="barh", 
+                title="Graphical performance comparison", 
+                xlabel="Accuracy [%]", 
+                ylabel="Model",
+                width=0.9,
+                figsize=(15, 7 + 2*model_num),
+            )
+            graph_ax.invert_yaxis()
+            for container in graph_ax.containers:
+                graph_ax.bar_label(container, fontsize=8, fmt="%.1f")
+            graph_ax.legend(loc='center left', bbox_to_anchor=(1.01, 0.95))
+            plt.tight_layout(rect=[0, 0, 0.95, 1])
+
+            plot = gr.Plot(graph_ax.get_figure(), label="Graphical performance comparison")
         with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
 

src/display/about.py

@@ -31,7 +31,7 @@ class Tasks(Enum):
 # Your leaderboard name
 TITLE = """<h1 align="center" id="space-title">🇨🇿 CzechBench Leaderboard</h1>"""
 
-TABLE_DESC = "The values shown in the table represent the accuracy metric in percentage."
+TABLE_DESC = "The values shown in the leaderboard table represent the accuracy metric in percentage."
 
 # What does your leaderboard evaluate?
 INTRODUCTION_OLD = """
@@ -94,6 +94,7 @@ The leaderboard table also displays aggregated scores across task categories, in
 - **Reasoning (Avg.):** ANLI, Belebele, CTKFacts, SQAD
 - **Math (Avg.):** GSM8K, Klokanek
 - **Classification (Avg.):** Czech News, Facebook Comments, Mall Reviews, Subjectivity
+- **Aggregate Score:** Average over above categories
 
 ## Evaluation Process
 

src/display/utils.py

@@ -51,6 +51,7 @@ auto_eval_column_dict.append(["dummy", ColumnContent, ColumnContent("model_name_
 auto_eval_column_dict.append(["eval_name", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
 auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", True)])
 auto_eval_column_dict.append(["hf_model_id", ColumnContent, ColumnContent("Model URL", "str", False)])
+auto_eval_column_dict.append(["aggregate_score", ColumnContent, ColumnContent("Aggregate Score", "number", True)])
 auto_eval_column_dict.append(["grammar_avg", ColumnContent, ColumnContent("Grammar (Avg.)", "number", True)])
 auto_eval_column_dict.append(["knowledge_avg", ColumnContent, ColumnContent("Knowledge (Avg.)", "number", True)])
 auto_eval_column_dict.append(["reasoning_avg", ColumnContent, ColumnContent("Reasoning (Avg.)", "number", True)])
@@ -100,6 +101,7 @@ HEADER_MAP = {
     "subjectivity_cs": "Subjectivity",
     "truthfulqa_cs": "TruthfulQA",
     "dummy": "_",
+    "aggregate_score": "Aggregate Score",
 }
 
 

src/populate.py

@@ -9,13 +9,6 @@ from src.display.utils import AutoEvalColumn, EvalQueueColumn, HEADER_MAP
 from src.leaderboard.read_evals import get_raw_eval_results
 
 
-def add_model_hyperlinks(row):
-    if row["Model URL"] is None or row["Model URL"] == "":
-        return row["Model"]
-    else:
-        return model_hyperlink(row["Model URL"], row["Model"])
-
-
 def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
     raw_data = get_raw_eval_results(results_path, requests_path)
     #all_data_json = [v.to_dict() for v in raw_data]
@@ -28,14 +21,14 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
     df["Reasoning (Avg.)"] = df[["ANLI", "Belebele", "CTKFacts", "SQAD"]].mean(axis=1)
     df["Math (Avg.)"] = df[["GSM8K", "Klokanek"]].mean(axis=1)
     df["Classification (Avg.)"] = df[["Czech News", "Facebook Comments", "Mall Reviews", "Subjectivity"]].mean(axis=1)
+    df["Aggregate Score"] = df[["Grammar (Avg.)", "Knowledge (Avg.)", "Reasoning (Avg.)", "Math (Avg.)", "Classification (Avg.)"]].mean(axis=1)
     df["_"] = ""  # The dataframe does not display the last column - BUG in gradio?
     df = df[cols].round(decimals=2)
     df.replace(r'\s+', np.nan, regex=True)
     # filter out if any of the benchmarks have not been produced
     df = df[has_no_nan_values(df, benchmark_cols)]
-    df['Model'] = df.apply(add_model_hyperlinks, axis=1)
     
-    return raw_data, df
+    return raw_data, df, 
 
 
 def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]: