update

app.py

@@ -32,7 +32,7 @@ from src.display.utils import (
     BENCHMARK_QUOTACOLS
 )
 from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, DYNAMIC_INFO_REPO, DYNAMIC_INFO_FILE_PATH, DYNAMIC_INFO_PATH, IS_PUBLIC, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
-from src.populate import get_evaluation_queue_df, get_leaderboard_df
+from src.populate import get_evaluation_queue_df, get_leaderboard_df, get_leaderboard_df_quota
 from src.submission.submit import add_new_eval
 from src.scripts.update_all_request_files import update_dynamic_files
 from src.tools.collections import update_collections
@@ -77,6 +77,16 @@ def init_space():
     #update_collections(original_df.copy())
     leaderboard_df = original_df.copy()
 
+    raw_data_quota, original_df_quota = get_leaderboard_df(
+        results_path=EVAL_RESULTS_PATH,
+        requests_path=EVAL_REQUESTS_PATH,
+        dynamic_path=DYNAMIC_INFO_FILE_PATH,
+        cols=list(set(QUOTACOLS+COLS)),
+        benchmark_cols=list(set(BENCHMARK_QUOTACOLS+BENCHMARK_COLS))
+    )
+    #update_collections(original_df.copy())
+    leaderboard_df_quota = original_df_quota.copy()
+
     #plot_df = create_plot_df(create_scores_df(raw_data))
 
     (
@@ -86,12 +96,10 @@ def init_space():
     ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
 
     #return leaderboard_df, original_df, plot_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df
-    return leaderboard_df, original_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df
+    return leaderboard_df, original_df, leaderboard_df_quota, original_df_quota,finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df
 
-leaderboard_df, original_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df = init_space()
-    #return leaderboard_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df
+leaderboard_df, original_df, leaderboard_df_quota, original_df_quota, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df = init_space()
 
-#leaderboard_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df = init_space()
 
 
 # Searching and filtering
@@ -231,6 +239,13 @@ leaderboard_df = filter_models(
     hide_models=[], # Deleted, merges, flagged, MoEs
 )
 
+leaderboard_df_quota = filter_models(
+    df=leaderboard_df_quota,
+    type_query=[t.to_str(" : ") for t in ModelType],
+    size_query=list(NUMERIC_INTERVALS.keys()),
+    precision_query=[i.value.name for i in Precision],
+    hide_models=[], # Deleted, merges, flagged, MoEs
+)
 
 
 demo = gr.Blocks(css=custom_css)
@@ -265,36 +280,6 @@ with demo:
                             elem_id="column-select",
                             interactive=True,
                         )
-                    #with gr.Row():
-                    #    hide_models = gr.CheckboxGroup(
-                    #        label="Hide models",
-                    #        choices = ["Private or deleted", "Contains a merge/moerge", "Flagged", "MoE"],
-                    #        value=[],
-                    #        interactive=True
-                    #    )
-                #with gr.Column(min_width=320):
-                #    #with gr.Box(elem_id="box-filter"):
-                #    filter_columns_type = gr.CheckboxGroup(
-                #        label="Model types",
-                #        choices=[t.to_str() for t in ModelType],
-                #        value=[t.to_str() for t in ModelType],
-                #        interactive=True,
-                #        elem_id="filter-columns-type",
-                #    )
-                #    filter_columns_precision = gr.CheckboxGroup(
-                #        label="Precision",
-                #        choices=[i.value.name for i in Precision],
-                #        value=[i.value.name for i in Precision],
-                #        interactive=True,
-                #        elem_id="filter-columns-precision",
-                #    )
-                #    filter_columns_size = gr.CheckboxGroup(
-                #        label="Model sizes (in billions of parameters)",
-                #        choices=list(NUMERIC_INTERVALS.keys()),
-                #        value=list(NUMERIC_INTERVALS.keys()),
-                #        interactive=True,
-                #        elem_id="filter-columns-size",
-                #    )
 
 
             leaderboard_table = gr.components.Dataframe(
@@ -382,40 +367,10 @@ with demo:
                             elem_id="column-select",
                             interactive=True,
                         )
-                    #with gr.Row():
-                    #    hide_models = gr.CheckboxGroup(
-                    #        label="Hide models",
-                    #        choices = ["Private or deleted", "Contains a merge/moerge", "Flagged", "MoE"],
-                    #        value=[],
-                    #        interactive=True
-                    #    )
-                #with gr.Column(min_width=320):
-                #    #with gr.Box(elem_id="box-filter"):
-                #    filter_columns_type = gr.CheckboxGroup(
-                #        label="Model types",
-                #        choices=[t.to_str() for t in ModelType],
-                #        value=[t.to_str() for t in ModelType],
-                #        interactive=True,
-                #        elem_id="filter-columns-type",
-                #    )
-                #    filter_columns_precision = gr.CheckboxGroup(
-                #        label="Precision",
-                #        choices=[i.value.name for i in Precision],
-                #        value=[i.value.name for i in Precision],
-                #        interactive=True,
-                #        elem_id="filter-columns-precision",
-                #    )
-                #    filter_columns_size = gr.CheckboxGroup(
-                #        label="Model sizes (in billions of parameters)",
-                #        choices=list(NUMERIC_INTERVALS.keys()),
-                #        value=list(NUMERIC_INTERVALS.keys()),
-                #        interactive=True,
-                #        elem_id="filter-columns-size",
-                #    )
 
 
             leaderboard_table = gr.components.Dataframe(
-                value=leaderboard_df[
+                value=leaderboard_df_quota[
                     [c.name for c in fields(AutoEvalColumnQuota) if c.never_hidden]
                     + shown_columns.value
                     + [AutoEvalColumnQuota.dummy.name]
@@ -430,8 +385,7 @@ with demo:
 
             # Dummy leaderboard for handling the case when the user uses backspace key
             hidden_leaderboard_table_for_search = gr.components.Dataframe(
-                value=original_df[QUOTACOLS],
-                #value=leaderboard_df[QUOTACOLS],
+                value=original_df_quota[QUOTACOLS],
                 headers=QUOTACOLS,
                 datatype=QUOTATYPES,
                 visible=False,

src/display/utils.py

@@ -51,7 +51,7 @@ auto_eval_column_quota_dict = []
 auto_eval_column_quota_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
 auto_eval_column_quota_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
 #Scores
-auto_eval_column_quota_dict.append(["average_quota", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
+auto_eval_column_quota_dict.append(["average_quota", ColumnContent, ColumnContent("AverageSampled ⬆️", "number", True)])
 for task in Quotas:
     auto_eval_column_quota_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
 # Model information

src/leaderboard/read_evals.py

@@ -168,6 +168,8 @@ class EvalResult:
         else:
             average_quota = average_quota/nums
 
+        print("AutoEvalColumn.average.name",AutoEvalColumn.average.name, average)
+        print("AutoEvalColumnQuota.average_quota.name",AutoEvalColumnQuota.average_quota.name,average_quota)
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name,
             AutoEvalColumn.precision.name: self.precision.value.name,

src/populate.py

@@ -18,8 +18,8 @@ def get_leaderboard_df(results_path: str, requests_path: str, dynamic_path: str,
     #all_data_json.append(baseline_row)
     filter_models_flags(all_data_json)
     df = pd.DataFrame.from_records(all_data_json)
-    #print("AutoEvalColumn.average.name",AutoEvalColumn.average.name)
-    #df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
+    print("AutoEvalColumn.average.name",AutoEvalColumn.average.name)
+    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
     df = df[cols].round(decimals=2)
 
     # filter out if any of the benchmarks have not been produced
@@ -37,7 +37,7 @@ def get_leaderboard_df_quota(results_path: str, requests_path: str, dynamic_path
     filter_models_flags(all_data_json)
     df = pd.DataFrame.from_records(all_data_json)
     print("AutoEvalColumn.average.name",AutoEvalColumn.average.name)
-    df = df.sort_values(by=[AutoEvalColumnQuota.average.name], ascending=False)
+    df = df.sort_values(by=[AutoEvalColumnQuota.average_quota.name], ascending=False)
     df = df[cols].round(decimals=2)
 
     # filter out if any of the benchmarks have not been produced