Update app.py

app.py

@@ -23,7 +23,7 @@ DATA_DATASET = f"{OWNER}/CTFAIA"
 INTERNAL_DATA_DATASET = f"{OWNER}/CTFAIA_internal"
 SUBMISSION_DATASET = f"{OWNER}/CTFAIA_submissions_internal"
 CONTACT_DATASET = f"{OWNER}/contact_info"
-RESULTS_DATASET = f"{OWNER}/CTFAIA_results_public"
+RESULTS_DATASET = f"{OWNER}/test_result"
 LEADERBOARD_PATH = f"{OWNER}/agent_ctf_leaderboard"
 api = HfApi()
 
@@ -31,13 +31,13 @@ YEAR_VERSION = "2024"
 
 os.makedirs("scored", exist_ok=True)
 
-all_version = ['20240423']
+all_version = ['20240602']
 
 contact_infos = load_dataset(
     CONTACT_DATASET,
     token=TOKEN,
-    download_mode="force_redownload",
-    ignore_verifications=True
+    # download_mode="force_redownload",
+    verification_mode="no_checks"
 )
 
 all_gold_dataset = {}
@@ -49,7 +49,7 @@ for dataset_version in all_version:
         dataset_version,
         token=TOKEN,
         download_mode="force_redownload",
-        ignore_verifications=True,
+        verification_mode="no_checks",
         trust_remote_code=True
     )
     all_gold_results[dataset_version] = {
@@ -61,7 +61,7 @@ for dataset_version in all_version:
         dataset_version,
         token=TOKEN,
         download_mode="force_redownload",
-        ignore_verifications=True,
+        verification_mode="no_checks",
         trust_remote_code=True
     )
 
@@ -69,23 +69,25 @@ for dataset_version in all_version:
 def get_dataframe_from_results(eval_results, split):
     local_df = eval_results[split]
     local_df = local_df.map(lambda row: {"model": model_hyperlink(row["url"], row["model"])})
-    local_df = local_df.remove_columns(["system_prompt", "url"])
+    local_df = local_df.remove_columns(["url"])
     local_df = local_df.rename_column("model", "Model name")
     local_df = local_df.rename_column("model_family", "Model family")
-    local_df = local_df.rename_column("score", "Average score (%)")
-    for i in [1, 2, 3]:
-        local_df = local_df.rename_column(f"score_level{i}", f"Level {i} score (%)")
+    # local_df = local_df.rename_column("score", "Average score (%)")
+    # for i in [1, 2, 3]:
+    #     local_df = local_df.rename_column(f"score_level{i}", f"Level {i} score (%)")
     df = pd.DataFrame(local_df)
-    df = df.sort_values(by=["Average score (%)"], ascending=False)
+    df = df.sort_values(by=["completion_level"], ascending=False)
 
-    numeric_cols = [c for c in local_df.column_names if "score" in c]
+    numeric_cols = [c for c in local_df.column_names if
+                    c in ["success_rate", "completion_level", "expertise", "reasoning", "comprehension"]]
     df[numeric_cols] = df[numeric_cols].multiply(100).round(decimals=2)
-    # df = df.style.format("{:.2%}", subset=numeric_cols)
-
+    df = df.style.format("{:.2%}", subset=numeric_cols)
+    print(type(df))
     return df
 
 
 eval_dataframe = {}
+
 for dataset_version in all_version:
     eval_dataframe[dataset_version] = get_dataframe_from_results(
         eval_results=eval_results[dataset_version],
@@ -97,14 +99,28 @@ def restart_space():
     api.restart_space(repo_id=LEADERBOARD_PATH, token=TOKEN)
 
 
-TYPES = ["markdown", "number", "number", "number", "number", "str", "str"]
+TYPES = ["markdown", "str", "str", "str", "number", "number", "number", "number"]
+LEVELS = ["all", 1, 2, 3]
+
+
+def round_and_pad(number, ndigits=2):
+    # 四舍五入到指定的小数位数
+    rounded_number = round(number, ndigits)
+    # 转换为字符串
+    number_str = str(rounded_number)
+    # 分离整数部分和小数部分
+    integer_part, decimal_part = number_str.split('.')
+    # 如果小数部分不足指定的位数，补零
+    while len(decimal_part) < ndigits:
+        decimal_part += '0'
+    # 拼接回去，并转换回数字
+    return '.'.join([integer_part, decimal_part])
 
 
 def add_new_eval(
         dataset_version: str,
         model: str,
         model_family: str,
-        system_prompt: str,
         url: str,
         path_to_file: str,
         organisation: str,
@@ -118,7 +134,14 @@ def add_new_eval(
 
     print("Adding new eval")
 
-    # Check if the combination model/org already exists and prints a warning message if yes
+    Check if the
+    combination
+    model / org
+    already
+    exists and prints
+    a
+    warning
+    message if yes
     if model.lower() in set(
             [m.lower() for m in eval_results[dataset_version][val_or_test]["model"]]) and organisation.lower() in set(
         [o.lower() for o in eval_results[dataset_version][val_or_test]["organisation"]]):
@@ -127,23 +150,19 @@ def add_new_eval(
     if path_to_file is None:
         return format_warning("Please attach a file.")
 
-    # Save submitted file
-    api.upload_file(
-        repo_id=SUBMISSION_DATASET,
-        path_or_fileobj=path_to_file.name,
-        path_in_repo=f"{organisation}/{model}/{dataset_version}_{val_or_test}_raw_{datetime.datetime.today()}.jsonl",
-        repo_type="dataset",
-        token=TOKEN
-    )
-
     # Gold answers
     gold_results = all_gold_results[dataset_version]
+    print(gold_results)
 
     # Compute score
     file_path = path_to_file.name
-    scores = {"all": 0, 1: 0, 2: 0, 3: 0}
-    num_questions = {"all": 0, 1: 0, 2: 0, 3: 0}
-    total_scores = {"all": 0, 1: 0, 2: 0, 3: 0}
+    success_rate = {'all': 0, 1: 0, 2: 0, 3: 0}
+    completion_level = {'all': 0, 1: 0, 2: 0, 3: 0}
+    expertise = {'all': 0, 1: 0, 2: 0, 3: 0}
+    reasoning = {'all': 0, 1: 0, 2: 0, 3: 0}
+    comprehension = {'all': 0, 1: 0, 2: 0, 3: 0}
+    num = {'all': 0, 1: 0, 2: 0, 3: 0}
+
     with open(f"scored/{organisation}_{model}.jsonl", "w") as scored_file:
         with open(file_path, 'r') as f:
             for ix, line in enumerate(f):
@@ -158,12 +177,11 @@ def add_new_eval(
                 task_name = task["task_name"]
                 try:
                     level = int(gold_results[val_or_test][task_name]["Level"])
+                    score = question_scorer(task, gold_results[val_or_test][task_name])
                 except KeyError:
                     return format_error(
                         f"{task_name} not found in split {val_or_test}. Are you sure you submitted the correct file?")
 
-                score = question_scorer(task, gold_results[val_or_test][task_name])
-
                 scored_file.write(
                     json.dumps({
                         "id": task_name,
@@ -173,14 +191,39 @@ def add_new_eval(
                     }) + "\n"
                 )
 
-                scores["all"] += score
-                scores[level] += score
-                num_questions["all"] += 1
-                num_questions[level] += 1
-    for task_name, task in gold_results[val_or_test].items():
-        level = int(task['Level'])
-        total_scores["all"] += 10
-        total_scores[level] += 10
+                num[level] += 1
+                completion_level[level] += score[0]
+                expertise[level] += score[1]
+                reasoning[level] += score[2]
+                comprehension[level] += score[3]
+
+                num['all'] += 1
+                completion_level['all'] += score[0]
+                expertise['all'] += score[1]
+                reasoning['all'] += score[2]
+                comprehension['all'] += score[3]
+
+                if score[0] == 10:
+                    success_rate[level] += 1
+                    success_rate['all'] += 1
+
+        for key in LEVELS:
+            success_rate[key] = round_and_pad(success_rate[key] / num[key])
+            completion_level[key] = round_and_pad(completion_level[key] / num[key])
+            expertise[key] = round_and_pad(expertise[key] / num[key])
+            reasoning[key] = round_and_pad(reasoning[key] / num[key])
+            comprehension[key] = round_and_pad(comprehension[key] / num[key])
+
+        print(success_rate, completion_level, expertise, reasoning, comprehension)
+
+    # Save submitted file
+    api.upload_file(
+        repo_id=SUBMISSION_DATASET,
+        path_or_fileobj=path_to_file.name,
+        path_in_repo=f"{organisation}/{model}/{dataset_version}_{val_or_test}_raw_{datetime.datetime.today()}.jsonl",
+        repo_type="dataset",
+        token=TOKEN
+    )
 
     # Save scored file
     api.upload_file(
@@ -195,14 +238,15 @@ def add_new_eval(
     eval_entry = {
         "model": model,
         "model_family": model_family,
-        "system_prompt": system_prompt,
         "url": url,
         "organisation": organisation,
-        "score": scores["all"] / total_scores["all"],
-        "score_level1": scores[1] / total_scores[1] if total_scores[1] else 0,
-        "score_level2": scores[2] / total_scores[2] if total_scores[2] else 0,
-        "score_level3": scores[3] / total_scores[3] if total_scores[3] else 0,
+        "success_rate": success_rate["all"],
+        "completion_level": completion_level["all"],
+        "expertise": expertise["all"],
+        "reasoning": reasoning["all"],
+        "comprehension": comprehension["all"]
     }
+
     eval_results[dataset_version][val_or_test] = eval_results[dataset_version][val_or_test].add_item(eval_entry)
     eval_results[dataset_version].push_to_hub(RESULTS_DATASET, config_name=dataset_version, token=TOKEN)
 
@@ -228,22 +272,21 @@ def refresh():
             dataset_version,
             token=TOKEN,
             download_mode="force_redownload",
-            ignore_verifications=True
+            verification_mode="no_checks"
         )
-    leaderboard_tables = []
+
+    new_eval_dataframe = {}
+    new_leaderboard_tables = []
     for dataset_version in all_version:
-        eval_dataframe[dataset_version] = get_dataframe_from_results(
+        new_eval_dataframe[dataset_version] = get_dataframe_from_results(
             eval_results=eval_results[dataset_version],
             split="validation"
         )
-        with gr.Tab(dataset_version):
-            leaderboard_tables.append(
-                gr.components.Dataframe(
-                    value=eval_dataframe[dataset_version], datatype=TYPES, interactive=False,
-                    column_widths=["20%"]
-                )
-            )
-    return leaderboard_tables
+        new_leaderboard_tables.append(new_eval_dataframe[dataset_version])
+    if len(new_leaderboard_tables) == 1:
+        return new_leaderboard_tables[0]
+    else:
+        return new_leaderboard_tables
 
 
 def upload_file(files):
@@ -286,7 +329,6 @@ with demo:
                 level_of_test = gr.Radio(all_version, value=all_version[0], label="dataset_version")
                 model_name_textbox = gr.Textbox(label="Model name", value='')
                 model_family_textbox = gr.Textbox(label="Model family", value='')
-                system_prompt_textbox = gr.Textbox(label="System prompt example", value='')
                 url_textbox = gr.Textbox(label="Url to model information", value='')
             with gr.Column():
                 organisation = gr.Textbox(label="Organisation", value='')
@@ -303,7 +345,6 @@ with demo:
                 level_of_test,
                 model_name_textbox,
                 model_family_textbox,
-                system_prompt_textbox,
                 url_textbox,
                 file_output,
                 organisation,