Phase II Version 1

app.py

@@ -3,7 +3,7 @@
 
 import pandas as pd
 import gradio as gr
-from ui.leaderboard import render_leader_board, render_info_html, render_citation, render_dataset_list
+from ui.leaderboard import render_leader_board, render_info_html, render_citation
 from ui.evaluation import render_eval_info
 from ui.coming_soon import render_coming_soon
 from ui.submission import render_submission_page
@@ -15,8 +15,14 @@ import os
 import json
 
 REPO_ID = os.getenv('REPO_ID')
-DB_ERR_PATH = f'./data/data/leaderboard_err.csv'
+
+DB_ERR_PATH = './data/data/eers.csv'
+DB_ACC_PATH = './data/data/accuracy.csv'
+DB_F1_PATH = './data/data/f1.csv'
+
 CITATIONS_PATH = f'./data/data/model_citations.json'
+LICENSES_PATH = f'./data/data/model_licenses.json'
+PARAMS_PATH = f'./data/data/model_parameter_count.json'
 
 if not os.path.exists('./data/data'):
     snapshot_download(repo_id=REPO_ID, 
@@ -25,33 +31,53 @@ if not os.path.exists('./data/data'):
 with open(CITATIONS_PATH, 'r') as f:
     model_citations = json.load(f)
 
+with open(LICENSES_PATH, 'r') as f:
+    model_licenses = json.load(f)
+
+with open(PARAMS_PATH, 'r') as f:
+    model_params = json.load(f)
+
+
 # Load leaderboard data
 leaderboard_df_err = load_leaderboard(DB_ERR_PATH)
+leaderboard_df_f1 = load_leaderboard(DB_F1_PATH)
+leaderboard_df_acc = load_leaderboard(DB_ACC_PATH)
+
+
 
 def create_ui():
-    with gr.Blocks(theme=gr.themes.Soft(text_size=gr.themes.sizes.text_md), css=custom_css) as demo:
-        # gr.Markdown("# Speech Deep Fake Arena")
-        gr.Image('./data/data/df_arena_3.jpg', interactive=False, 
+    # with gr.Blocks(theme=gr.themes.Soft(text_size=gr.themes.sizes.text_md)) as demo:
+    with gr.Blocks(theme=gr.themes.Soft(text_size=gr.themes.sizes.text_md)) as demo:
+
+        gr.Image(f'./data/data/df_arena.jpg', interactive=False, 
                  show_fullscreen_button=False, show_share_button=False, show_label=False)
         
         with gr.Tabs():
             with gr.Tab("🏆 Leaderboard"):
                 with gr.Column():
                     render_info_html()
-                    gr.Markdown("Table for Equal Error Rate (EER %) for different systems. All the systems are are trained on the ASVspoof 2019 dataset, except for the Whisper MesoNet system, which is trained on a subset of the ASVspoof 2021 DF dataset.")
-                    render_leader_board(leaderboard_df_err, model_citations)  # Adjust this to work with Gradio components
+                    render_leader_board(leaderboard_df_err, model_citations, model_licenses, model_params, True) 
+                    gr.HTML("<div style='text-align:center; font-style:italic;'>Table: EER (%)</div>") 
+                    gr.Markdown("<br>")  # add spacing
+                    render_leader_board(leaderboard_df_acc, model_citations, model_licenses, model_params, False)  
+                    gr.HTML("<div style='text-align:center; font-style:italic;'>Table: Accuracy (%)</div>")
+                    gr.Markdown("<br>")  # add spacing                      
+                    render_leader_board(leaderboard_df_f1, model_citations, model_licenses, model_params, False)  
+                    gr.HTML("<div style='text-align:center; font-style:italic;'>Table: F1 scores</div>")
+                    gr.Markdown("<br>")  # add spacing
                     render_citation()
-                    render_dataset_list()
 
-            with gr.Tab("📊 Metrics"):
+            with gr.Tab("📊 Compute your scores"):
                 render_eval_info()  
 
             with gr.Tab("📤 Submit your own system !"):
                 render_submission_page() 
-
-            with gr.Tab("🔜 Coming Soon"):
-                render_coming_soon()  
+ 
     return demo
 
 # Launch the app
-create_ui().launch()
+# create_ui().launch(share=True)
+
+if __name__ == "__main__":
+    demo = create_ui()
+    demo.launch()

ui/evaluation.py

@@ -3,24 +3,6 @@ import gradio as gr
 def render_eval_info():
     text = r"""
 
-            We use **Equal Error Rate (EER %)**  a standard method used in bimoretric and anti-spoofing systems.
-
-            ### **What is EER?**  
-            Equal Error Rate (EER) is a performance metric used to evaluate biometric systems. It represents the point at which the **False Acceptance Rate (FAR)** and **False Rejection Rate (FRR)** are equal. A lower EER indicates a more accurate system.
-
-            #### **False Acceptance Rate (FAR)**  
-            FAR is the proportion of **unauthorized** users incorrectly accepted by the system.  
-
-            $FAR = \frac{\text{False Acceptances}}{\text{Total Imposter Attempts}}$
-
-            #### **False Rejection Rate (FRR)**  
-            FRR is the proportion of **genuine** users incorrectly rejected by the system.  
-
-            $FRR = \frac{\text{False Rejections}}{\text{Total Genuine Attempts}}$
-
-
-            - EER is the point at which FAR and FRR are equal.
-
             ### How to compute your own EER score file ? 
 
             In order to streamline the evaluation process across many models and datasets, we

ui/leaderboard.py

@@ -14,12 +14,12 @@ def render_info_html():
     #              "to detect deep fakes. With research moving at such a rapid pace, it becomes challenging to keep track of generalizability " \
     #              "of SOTA DF detection systems. This leaderboard thus presents a comprehensive benchmark of 10 SOTA speech antispoofing " \
     #              "systems across 13 popular speech deep fake detection datasets."
-    info_text = """The advent of machine generated speech  calls for dedicated research to develop countermeasure systems to protect against their misuse.
-                The speech DF arena leaderboard provides a standardized platform to compare different speech deepfake detection approaches and ranks them on Huggingface.
-                By assessing models across diverse datasets and attack scenarios, the speech DF arena leaderboard aims to help researchers and developers enhance the reliability and robustness of deepfake detection systems
-                , ensuring safer and more trustworthy audio communications. We report the average EER (lower the better). 
-                Models are ranked based on their Average EER, from lowest to highest. Check the Metrics tab to understand how the models are evaluated.
-                If you want results for a model that is not listed here, you can submit a request for it to be included through under "submit your own system" tab.
+    info_text = """The advent of machine generated speech  calls for dedicated research to develop countermeasure systems to protect against their misuse through deepfakes.
+                   The Speech DF arena leaderboard provides a standardized benchmarking platform for both commercial and open source systems to compare different detection approaches and ranks them using standard metrics.
+                   This leaderboard is an evolving inititatie where new systems and attacks can can be added upon request to keep it up to date with the latest advancements in the field. Check out the Submit Your System section to learn how to submit your system.
+         
+                   Below we report the Equal Error Rate (EER %), Accuracy (%) and F1 scores. The table consists of both pooled and average results. Pooled results are computed 
+                   by using thresholds obtained across all datasets, while average results are computed by simply averaging the dataset level results. We rank the systems according to the Pooled results
             """
     
     # HTML formatted info text
@@ -28,13 +28,16 @@ def render_info_html():
 def highlight_min(s, props=''):
     return np.where(s == np.nanmin(s.values), props, '')
 
-def render_leader_board(leaderboard_df, model_citations):
+def highlight_max(s, props=''):
+    return np.where(s == np.nanmax(s.values), props, '')
 
-    if not leaderboard_df.empty:
-        print(leaderboard_df.shape)
-        leaderboard_df.insert(3, 'Average EER(%)', leaderboard_df.iloc[:, 3:].mean(axis=1))
+def render_leader_board(leaderboard_df, model_citations, model_licenses, model_params, _ascending):
 
-        leaderboard_df = leaderboard_df.sort_values(by="Average EER(%)", ascending=True).reset_index(drop=True)
+    if not leaderboard_df.empty:
+        leaderboard_df.insert(1, "License", leaderboard_df["System"].apply(lambda x: model_licenses.get(x)))
+        leaderboard_df.insert(2, "Num Params (M)", leaderboard_df["System"].apply(lambda x: model_params.get(x)))
+        print(leaderboard_df.head())
+        leaderboard_df = leaderboard_df.sort_values(by='Pooled', ascending=_ascending).reset_index(drop=True)
 
         # Assign rank emojis 🥇🥈🥉
         leaderboard_df["System"] = leaderboard_df["System"].apply(lambda x: f"[{x}]({model_citations.get(x, '#')})")
@@ -51,10 +54,10 @@ def render_leader_board(leaderboard_df, model_citations):
             leaderboard_df
             .style \
             .format(precision=2)
-            .apply(highlight_min, props='color:green', axis=0)
+            .apply(highlight_min if _ascending else highlight_max, props='color:green', axis=0)
         )
 
-        return gr.Dataframe(styler, datatype=['markdown'] * 2 + ['number']  * 16, elem_id="leaderboard-table")
+        return gr.Dataframe(styler, datatype=['markdown'] * 3 + ['number']  * 15, elem_id="leaderboard-table", pinned_columns=1, column_widths=["200px", "150px", "150px"] + ["100px"] * 15)
     return gr.HTML(value="<p>No data available in the leaderboard.</p>")
 
 def render_citation():
@@ -70,20 +73,3 @@ def render_citation():
             howpublished = "\url{link}"
         }
         ```""")
-
-def render_dataset_list():
-    return gr.Markdown(r"""
-    List of all the datasets used in the benchmarking:
-    - [ASVSpoof 2019: A large-scale public database of synthesized, converted and replayed speech](https://www.asvspoof.org/index2019.html)
-    - [ASVspoof 2021 DF & LA: accelerating progress in spoofed and deepfake speech detection](https://www.asvspoof.org/index2021.html)
-    - [ASVspoof 5: Crowdsourced speech data, deepfakes, and adversarial attacks at scale](https://www.asvspoof.org/workshop2024)
-    - [FakeOrReal: FoR: A Dataset for Synthetic Speech Detection](https://bil.eecs.yorku.ca/datasets/)
-    - [Codecfake Yuankun et. al.](https://github.com/xieyuankun/Codecfake)
-    - [ADD 2022 Track 2 & 3: the First Audio Deep Synthesis Detection Challenge](http://addchallenge.cn/add2022)
-    - [ADD 2023 Round 2&3 the Second Audio Deepfake Detection Challenge](http://addchallenge.cn/add2023)
-    - [DFADD: The Diffusion and Flow-Matching Based Audio Deepfake Dataset](https://github.com/isjwdu/DFADD)
-    - [LibriVoc: Exposing AI-Synthesized Human Voices Using Neural Vocoder Artifacts](https://github.com/csun22/Synthetic-Voice-Detection-Vocoder-Artifacts)
-    - [SONAR: A Synthetic AI-Audio Detection Framework and Benchmark](https://github.com/Jessegator/SONAR)
-    - [In-the-Wild: Does Audio Deepfake Detection Generalize?](https://deepfake-total.com/in_the_wild)
-    
-    """)

ui/submission.py

@@ -1,10 +1,9 @@
 import gradio as gr
 
 def render_submission_page():
-    text =  r"""Want to submit your own system to the leaderboard? Submit 
-    all the scores files for your system across evaluation sets of all the below supported datasets at
-    <[email protected]> and we will handle the rest. Score files can be generated using the df_arena_toolkit.
-
+    text =  r"""Want to submit your own system to the leaderboard? We accept submissions from both open source and closed source systems.
+            Instructions and submission form can be found here: [Submission Form](https://drive.google.com/file/d/1YmW3da68hYAWeTmMAJOcEgUlJG3iGXGx/view?usp=sharing). We request submitting teams to fill out this form and
+            and reach out to use at <[email protected]>. Below is the List of currently included datasets in the leaderboard:
 
     - [ASVSpoof2019](https://www.asvspoof.org/index2019.html)
     - [ASVSpoof2021LA](https://www.asvspoof.org/index2021.html)