updated leaderboard - added poleval test sets

app.py

@@ -1,7 +1,7 @@
 import os
 import streamlit as st
 import pandas as pd
-from constants import BIGOS_INFO, PELCRA_INFO, ANALYSIS_INFO, ABOUT_INFO, INSPECTION_INFO, COMPARISON_INFO
+from constants import BIGOS_INFO, PELCRA_INFO, POLEVAL_INFO, ANALYSIS_INFO, ABOUT_INFO, INSPECTION_INFO, COMPARISON_INFO
 from utils import read_latest_results, basic_stats_per_dimension, retrieve_asr_systems_meta_from_the_catalog, box_plot_per_dimension,box_plot_per_dimension_subsets, box_plot_per_dimension_with_colors, get_total_audio_duration, check_impact_of_normalization, calculate_wer_per_meta_category, calculate_wer_per_audio_feature
 from app_utils import calculate_height_to_display, filter_dataframe
 import matplotlib.pyplot as plt
@@ -22,7 +22,9 @@ if hf_token is None:
 # select the dataset to display results
 datasets_secret = [
     "amu-cai/pl-asr-bigos-v2-secret",
-    "pelcra/pl-asr-pelcra-for-bigos-secret"]
+    "pelcra/pl-asr-pelcra-for-bigos-secret",
+    "michaljunczyk/test_A_poleval_24",
+    "michaljunczyk/test_B_poleval_24"]
 
 datasets_public = []
     #["amu-cai/pl-asr-bigos-synth-med"]
@@ -30,7 +32,7 @@ datasets_public = []
 
 st.set_page_config(layout="wide")
 
-about, lead_bigos,  lead_pelcra, analysis, interactive_comparison = st.tabs(["About", "ASR Leaderboard - BIGOS corpora", "ASR Leaderboard - PELCRA corpora", "ASR evaluation scenarios", "Interactive comparison of ASR accuracy"])
+about, lead_bigos,  lead_pelcra, lead_poleval_a, lead_poleval_b, analysis, interactive_comparison = st.tabs(["About", "BIGOS", "PELCRA", "PolEval test-A", "PolEval test-B", "Evaluation scenarios", "Interactive dashboard"])
 # "Results inspection""Results inspection"
 # inspection
 # acknowledgements, changelog, faq, todos = st.columns(4)
@@ -366,7 +368,7 @@ with lead_bigos:
         if metric == 'Average':
             ax.set_title('Average normalization impact on all metrics')
         ax.set_xlabel('Normalization Type')
-        ax.set_ylabel(f'Difference in {metric}')
+        ax.set_ylabel(f'Difference in {metric} [pp]')
         ax.grid(True)
         ax.set_xticklabels(diff_in_metrics.index, rotation=45, ha='right')
         min_val = diff_in_metrics[metric].min()
@@ -500,7 +502,292 @@ with lead_pelcra:
             if metric == 'Average':
                 ax.set_title('Average normalization impact on all metrics')
             ax.set_xlabel('Normalization Type')
-            ax.set_ylabel(f'Difference in {metric}')
+            ax.set_ylabel(f'Difference in {metric} [pp]')
+            ax.grid(True)
+            ax.set_xticklabels(diff_in_metrics.index, rotation=45, ha='right')
+            min_val = diff_in_metrics[metric].min()
+            ax.set_ylim([min_val * 1.1, diff_in_metrics[metric].max() * 1.1])
+    
+            for bar in bars:
+                height = bar.get_height()
+                ax.annotate(f'{height:.2f}',
+                            xy=(bar.get_x() + bar.get_width() / 2, height),
+                            xytext=(0, -12),  # 3 points vertical offset
+                            textcoords="offset points",
+                            ha='center', va='bottom')
+
+    # Display the plot in Streamlit
+    st.pyplot(fig)
+
+    ##################### APPENDIX #########################       
+    st.header("Appendix - Full evaluation results per subset for all evaluated systems")
+    # select only the columns we want to plot
+    df_per_dataset_selected_cols = df_per_dataset_all[cols_to_select_all]   
+    st.dataframe(df_per_dataset_selected_cols, hide_index=True, use_container_width=False)
+
+with lead_poleval_a:
+    st.title("PolEval test A Leaderboard")
+    st.markdown(POLEVAL_INFO, unsafe_allow_html=True)
+
+    # configuration for tab
+    dataset = "michaljunczyk/test_A_poleval_24"
+    dataset_short_name = "PolEval test A"
+    dataset_version = "V1"
+    eval_date = "November 2024"
+    split = "test"
+    norm_type = "all"
+    ref_type = "orig"
+    
+     # common, reusable part for all tabs presenting leaderboards for specific datasets
+    #### DATA LOADING AND AUGMENTATION ####
+    df_per_sample_all, df_per_dataset_all = read_latest_results(dataset, split, codename_to_shortname_mapping)
+    
+    
+    # filter only the ref_type and norm_type we want to analyze
+    df_per_sample = df_per_sample_all[(df_per_sample_all["ref_type"] == ref_type) & (df_per_sample_all["norm_type"] == norm_type)]
+    # filter only the ref_type and norm_type we want to analyze
+    df_per_dataset = df_per_dataset_all[(df_per_dataset_all["ref_type"] == ref_type) & (df_per_dataset_all["norm_type"] == norm_type)]
+
+    ##### PARAMETERS CALCULATION ####
+    evaluated_systems_list = df_per_sample["system"].unique()
+    no_of_evaluated_systems = len(evaluated_systems_list)
+    no_of_eval_subsets = len(df_per_dataset["subset"].unique())
+    no_of_test_cases = len(df_per_sample)
+    no_of_unique_recordings = len(df_per_sample["id"].unique())
+    total_audio_duration_hours = get_total_audio_duration(df_per_sample)
+    no_of_unique_speakers = len(df_per_sample["speaker_id"].unique())
+
+    df_per_dataset_with_asr_systems_meta = pd.merge(df_per_dataset, df_evaluated_systems, how="left", left_on="system", right_on="Shortname")
+
+     # MOST IMPORTANT RESULTS
+    analysis_dim = "system"
+    metric = "WER"
+    st.subheader("Leaderboard - Median {} per ASR {} across all subsets of {} dataset".format(metric, analysis_dim, dataset_short_name))
+    fig = box_plot_per_dimension_with_colors(df_per_dataset_with_asr_systems_meta, metric, analysis_dim, "{} per {}".format(metric, analysis_dim), analysis_dim, metric + "[%]","System", "Type")
+    st.pyplot(fig, clear_figure=True, use_container_width=True)
+
+    st.header("Benchmark details")
+    st.markdown("**Evaluation date:** {}".format(eval_date))
+    st.markdown("**Number of evaluated system-model variants:** {}".format(no_of_evaluated_systems))
+    st.markdown("**Number of evaluated subsets:** {}".format(no_of_eval_subsets))
+    st.markdown("**Number of evaluated system-model-subsets combinations**: {}".format(len(df_per_dataset)))
+    st.markdown("**Number of unique speakers**: {}".format(no_of_unique_speakers))
+    st.markdown("**Number of unique recordings used for evaluation:** {}".format(no_of_unique_recordings))
+    st.markdown("**Total size of the dataset:** {:.2f} hours".format(total_audio_duration_hours))
+    st.markdown("**Total number of test cases (audio-hypothesis pairs):** {}".format(no_of_test_cases))
+    st.markdown("**Dataset:** {}".format(dataset))
+    st.markdown("**Dataset version:** {}".format(dataset_version))
+    st.markdown("**Split:** {}".format(split))
+    st.markdown("**Text reference type:** {}".format(ref_type))
+    st.markdown("**Normalization steps:** {}".format(norm_type))
+
+    ########### RESULTS ################
+    st.header("WER (Word Error Rate) analysis")
+    st.subheader("Average WER for the whole dataset")
+    df_wer_avg = basic_stats_per_dimension(df_per_dataset, "WER", "dataset")
+    st.dataframe(df_wer_avg)
+
+    st.subheader("Comparison of average WER for free and commercial systems")
+    df_wer_avg_free_commercial = basic_stats_per_dimension(df_per_dataset_with_asr_systems_meta, "WER", "Type")
+    st.dataframe(df_wer_avg_free_commercial)
+
+    ##################### PER SYSTEM ANALYSIS ######################### 
+    analysis_dim = "system"
+    metric = "WER"
+    metric2 = "CER"
+
+    st.subheader("Table showing {} and {}".format(metric, metric2))
+    df_wer_per_system_from_per_dataset = basic_stats_per_dimension(df_per_dataset, metric, analysis_dim)
+    df_cer_per_system_from_per_dataset = basic_stats_per_dimension(df_per_dataset, metric2, analysis_dim)
+    # merge the two dataframes, keep only one column for each metric with average values
+    df_wer_cer_per_system_from_per_dataset = pd.merge(df_wer_per_system_from_per_dataset, df_cer_per_system_from_per_dataset, on='system')
+    # drop top level of the column index
+    df_wer_cer_per_system_from_per_dataset = df_wer_cer_per_system_from_per_dataset.reset_index()
+
+    # keep columns system, avg_WER and avg_CER only
+    df_wer_cer_per_system_from_per_dataset = df_wer_cer_per_system_from_per_dataset[['system', 'avg_WER', 'avg_CER']]
+    h_df_per_system_per_dataset = calculate_height_to_display(df_wer_cer_per_system_from_per_dataset)
+
+    st.dataframe(df_wer_cer_per_system_from_per_dataset, height = h_df_per_system_per_dataset )
+    
+    ##################### PER SUBSET ANALYSIS ######################### 
+    analysis_dim = "subset"
+    metric = "WER"
+   
+    st.subheader("Boxplot showing {} per {} sorted by median values".format(metric, analysis_dim))
+    fig = box_plot_per_dimension_subsets(df_per_dataset, metric, analysis_dim, "{} per {} for dataset {}".format(metric, analysis_dim, dataset_short_name), analysis_dim +' of dataset ' + dataset_short_name , metric + " (%)", "system")
+    st.pyplot(fig, clear_figure=True, use_container_width=False)
+
+    ### IMPACT OF NORMALIZATION ON ERROR RATES ##### 
+    # Calculate the average impact of various norm_types for all datasets and systems
+    df_per_dataset_selected_cols = df_per_dataset_all[cols_to_select_all]
+    diff_in_metrics = check_impact_of_normalization(df_per_dataset_selected_cols)
+    st.subheader("Impact of normalization on WER")
+    st.dataframe(diff_in_metrics, use_container_width=False)
+    
+    # Visualizing the differences in metrics graphically with data labels
+    # Visualizing the differences in metrics graphically with data labels
+    fig, axs = plt.subplots(3, 2, figsize=(12, 12))
+    fig.subplots_adjust(hspace=0.6, wspace=0.6)
+
+    #remove the sixth subplot
+    fig.delaxes(axs[2,1])
+
+    metrics = ['SER', 'WER', 'MER', 'CER', "Average"]
+    colors = ['blue', 'orange', 'green', 'red', 'purple']
+
+    for ax, metric, color in zip(axs.flatten(), metrics, colors):
+            bars = ax.bar(diff_in_metrics.index, diff_in_metrics[metric], color=color)
+            ax.set_title(f'Normalization impact on {metric}')
+            if metric == 'Average':
+                ax.set_title('Average normalization impact on all metrics')
+            ax.set_xlabel('Normalization Type')
+            ax.set_ylabel(f'Difference in {metric} [pp]')
+            ax.grid(True)
+            ax.set_xticklabels(diff_in_metrics.index, rotation=45, ha='right')
+            min_val = diff_in_metrics[metric].min()
+            ax.set_ylim([min_val * 1.1, diff_in_metrics[metric].max() * 1.1])
+    
+            for bar in bars:
+                height = bar.get_height()
+                ax.annotate(f'{height:.2f}',
+                            xy=(bar.get_x() + bar.get_width() / 2, height),
+                            xytext=(0, -12),  # 3 points vertical offset
+                            textcoords="offset points",
+                            ha='center', va='bottom')
+
+    # Display the plot in Streamlit
+    st.pyplot(fig)
+
+    ##################### APPENDIX #########################       
+    st.header("Appendix - Full evaluation results per subset for all evaluated systems")
+    # select only the columns we want to plot
+    df_per_dataset_selected_cols = df_per_dataset_all[cols_to_select_all]   
+    st.dataframe(df_per_dataset_selected_cols, hide_index=True, use_container_width=False)
+
+with lead_poleval_b:
+    st.title("PolEval test B Leaderboard")
+    st.markdown(POLEVAL_INFO, unsafe_allow_html=True)
+
+    # configuration for tab
+    dataset = "michaljunczyk/test_B_poleval_24"
+    dataset_short_name = "PolEval test B"
+    dataset_version = "V1"
+    eval_date = "November 2024"
+    split = "test"
+    norm_type = "all"
+    ref_type = "orig"
+    
+     # common, reusable part for all tabs presenting leaderboards for specific datasets
+    #### DATA LOADING AND AUGMENTATION ####
+    df_per_sample_all, df_per_dataset_all = read_latest_results(dataset, split, codename_to_shortname_mapping)
+    
+    
+    # filter only the ref_type and norm_type we want to analyze
+    df_per_sample = df_per_sample_all[(df_per_sample_all["ref_type"] == ref_type) & (df_per_sample_all["norm_type"] == norm_type)]
+    # filter only the ref_type and norm_type we want to analyze
+    df_per_dataset = df_per_dataset_all[(df_per_dataset_all["ref_type"] == ref_type) & (df_per_dataset_all["norm_type"] == norm_type)]
+
+    ##### PARAMETERS CALCULATION ####
+    evaluated_systems_list = df_per_sample["system"].unique()
+    no_of_evaluated_systems = len(evaluated_systems_list)
+    no_of_eval_subsets = len(df_per_dataset["subset"].unique())
+    no_of_test_cases = len(df_per_sample)
+    no_of_unique_recordings = len(df_per_sample["id"].unique())
+    total_audio_duration_hours = get_total_audio_duration(df_per_sample)
+    no_of_unique_speakers = len(df_per_sample["speaker_id"].unique())
+
+    df_per_dataset_with_asr_systems_meta = pd.merge(df_per_dataset, df_evaluated_systems, how="left", left_on="system", right_on="Shortname")
+
+     # MOST IMPORTANT RESULTS
+    analysis_dim = "system"
+    metric = "WER"
+    st.subheader("Leaderboard - Median {} per ASR {} across all subsets of {} dataset".format(metric, analysis_dim, dataset_short_name))
+    fig = box_plot_per_dimension_with_colors(df_per_dataset_with_asr_systems_meta, metric, analysis_dim, "{} per {}".format(metric, analysis_dim), analysis_dim, metric + "[%]","System", "Type")
+    st.pyplot(fig, clear_figure=True, use_container_width=True)
+
+    st.header("Benchmark details")
+    st.markdown("**Evaluation date:** {}".format(eval_date))
+    st.markdown("**Number of evaluated system-model variants:** {}".format(no_of_evaluated_systems))
+    st.markdown("**Number of evaluated subsets:** {}".format(no_of_eval_subsets))
+    st.markdown("**Number of evaluated system-model-subsets combinations**: {}".format(len(df_per_dataset)))
+    st.markdown("**Number of unique speakers**: {}".format(no_of_unique_speakers))
+    st.markdown("**Number of unique recordings used for evaluation:** {}".format(no_of_unique_recordings))
+    st.markdown("**Total size of the dataset:** {:.2f} hours".format(total_audio_duration_hours))
+    st.markdown("**Total number of test cases (audio-hypothesis pairs):** {}".format(no_of_test_cases))
+    st.markdown("**Dataset:** {}".format(dataset))
+    st.markdown("**Dataset version:** {}".format(dataset_version))
+    st.markdown("**Split:** {}".format(split))
+    st.markdown("**Text reference type:** {}".format(ref_type))
+    st.markdown("**Normalization steps:** {}".format(norm_type))
+
+    ########### RESULTS ################
+    st.header("WER (Word Error Rate) analysis")
+    st.subheader("Average WER for the whole dataset")
+    df_wer_avg = basic_stats_per_dimension(df_per_dataset, "WER", "dataset")
+    st.dataframe(df_wer_avg)
+
+    st.subheader("Comparison of average WER for free and commercial systems")
+    df_wer_avg_free_commercial = basic_stats_per_dimension(df_per_dataset_with_asr_systems_meta, "WER", "Type")
+    st.dataframe(df_wer_avg_free_commercial)
+
+    ##################### PER SYSTEM ANALYSIS ######################### 
+    analysis_dim = "system"
+    metric = "WER"
+    metric2 = "CER"
+
+    st.subheader("Table showing {} and {}".format(metric, metric2))
+    df_wer_per_system_from_per_dataset = basic_stats_per_dimension(df_per_dataset, metric, analysis_dim)
+    df_cer_per_system_from_per_dataset = basic_stats_per_dimension(df_per_dataset, metric2, analysis_dim)
+    # merge the two dataframes, keep only one column for each metric with average values
+    df_wer_cer_per_system_from_per_dataset = pd.merge(df_wer_per_system_from_per_dataset, df_cer_per_system_from_per_dataset, on='system')
+    # drop top level of the column index
+    df_wer_cer_per_system_from_per_dataset = df_wer_cer_per_system_from_per_dataset.reset_index()
+
+    # keep columns system, avg_WER and avg_CER only
+    df_wer_cer_per_system_from_per_dataset = df_wer_cer_per_system_from_per_dataset[['system', 'avg_WER', 'avg_CER']]
+    h_df_per_system_per_dataset = calculate_height_to_display(df_wer_cer_per_system_from_per_dataset)
+
+    st.dataframe(df_wer_cer_per_system_from_per_dataset, height = h_df_per_system_per_dataset )
+
+
+    ##################### PER SUBSET ANALYSIS ######################### 
+    analysis_dim = "subset"
+    metric = "WER"
+    st.subheader("Table showing {} per {} sorted by median values".format(metric, analysis_dim))
+    df_wer_per_system_from_per_dataset = basic_stats_per_dimension(df_per_dataset, metric, analysis_dim)
+    h_df_per_system_per_dataset = calculate_height_to_display(df_wer_per_system_from_per_dataset)
+    st.dataframe(df_wer_per_system_from_per_dataset, height = h_df_per_system_per_dataset )
+
+    st.subheader("Boxplot showing {} per {} sorted by median values".format(metric, analysis_dim))
+    fig = box_plot_per_dimension_subsets(df_per_dataset, metric, analysis_dim, "{} per {} for dataset {}".format(metric, analysis_dim, dataset_short_name), analysis_dim +' of dataset ' + dataset_short_name , metric + " (%)", "system")
+    st.pyplot(fig, clear_figure=True, use_container_width=True)
+
+    ### IMPACT OF NORMALIZATION ON ERROR RATES ##### 
+    # Calculate the average impact of various norm_types for all datasets and systems
+    df_per_dataset_selected_cols = df_per_dataset_all[cols_to_select_all]
+    diff_in_metrics = check_impact_of_normalization(df_per_dataset_selected_cols)
+    st.subheader("Impact of normalization on WER")
+    st.dataframe(diff_in_metrics, use_container_width=False)
+    
+    # Visualizing the differences in metrics graphically with data labels
+    # Visualizing the differences in metrics graphically with data labels
+    fig, axs = plt.subplots(3, 2, figsize=(12, 12))
+    fig.subplots_adjust(hspace=0.6, wspace=0.6)
+
+    #remove the sixth subplot
+    fig.delaxes(axs[2,1])
+
+    metrics = ['SER', 'WER', 'MER', 'CER', "Average"]
+    colors = ['blue', 'orange', 'green', 'red', 'purple']
+
+    for ax, metric, color in zip(axs.flatten(), metrics, colors):
+            bars = ax.bar(diff_in_metrics.index, diff_in_metrics[metric], color=color)
+            ax.set_title(f'Normalization impact on {metric}')
+            if metric == 'Average':
+                ax.set_title('Average normalization impact on all metrics')
+            ax.set_xlabel('Normalization Type')
+            ax.set_ylabel(f'Difference in {metric} [pp]')
             ax.grid(True)
             ax.set_xticklabels(diff_in_metrics.index, rotation=45, ha='right')
             min_val = diff_in_metrics[metric].min()
@@ -532,6 +819,10 @@ with analysis:
         dataset_short_name = "BIGOS"
     elif dataset == "pelcra/pl-asr-pelcra-for-bigos-secret":
         dataset_short_name = "PELCRA"
+    elif dataset == "michaljunczyk/test_A_poleval_24":
+        dataset_short_name = "PolEval test A"
+    elif dataset == "michaljunczyk/test_B_poleval_24":
+        dataset_short_name = "PolEval test B"
     else:
         dataset_short_name = "UNKNOWN"
     

constants.py

@@ -17,6 +17,8 @@ Learn more [here](https://huggingface.co/datasets/amu-cai/pl-asr-bigos-v2)"
 PELCRA_INFO = "PELCRA for BIGOS is the subset of speech corpora created by the [PELCRA group](http://pelcra.pl/new/), curated for the BIGOS benchmark by the [AMU-CAI team](https://huggingface.co/amu-cai). \
 Learn more [here](https://huggingface.co/datasets/pelcra/pl-asr-pelcra-for-bigos)"
 
+POLEVAL_INFO = "PolEval is test used for Polish ASR challenge. It consists of recordings from BIGOS and PELCRA datasets. For details see: [PolEval 2024 - Task 3 - ASR](https://poleval.pl/tasks/task3)"
+
 ANALYSIS_INFO = "Here we examine ASR accuracy depending on the system type, model size, audio duration, speaking rate and speaker charactertics (age and gender)" 
 
 INSPECTION_INFO = "Here you can inspect the performance of specific ASR systems on the specific audio samples"