Upload folder using huggingface_hub

app/content.py

@@ -67,4 +67,50 @@ metrics = {
     'llama3_70b_judge': 'General evaluation using the LLAMA3-70B model, typically scoring based on subjective judgments. (0-100 based on score 0-5)',
     'meteor': 'METEOR, a metric used for evaluating text generation, often used in translation or summarization tasks. (Sensitive to output length)',
     'bleu': 'BLEU (Bilingual Evaluation Understudy), another text generation evaluation metric commonly used in machine translation. (Sensitive to output length)',
+}
+
+metrics_info = {
+    'wer': 'Word Error Rate (WER) - The Lower, the better.',
+    'llama3_70b_judge_binary': 'Model-as-a-Judge Peformance. Using LLAMA-3-70B. Scale from 0-100. The higher, the better.',
+    'llama3_70b_judge': 'Model-as-a-Judge Peformance. Using LLAMA-3-70B. Scale from 0-100. The higher, the better.',
+    'meteor': 'METEOR Score. The higher, the better.',
+    'bleu': 'BLEU Score. The higher, the better.',
+}
+
+dataname_column_rename_in_table = {
+    'librispeech_test_clean' : 'LibriSpeech-Clean',
+    'librispeech_test_other' : 'LibriSpeech-Other',
+    'common_lvoice_15_en_test': 'CommonVoice-15-EN',
+    'peoples_speech_test'    : 'Peoples-Speech',
+    'gigaspeech_test'        : 'GigaSpeech-1',
+    'earnings21_test'        : 'Earnings-21',
+    'earnings22_test'        : 'Earnings-22',
+    'tedlium3_test'          : 'TED-LIUM-3',
+    'tedlium3_long_form_test': 'TED-LIUM-3-Long',
+    'aishel_asr_zh_test'    : 'Aishell-ASR-ZH',
+    'covost2_en_id_test'     : 'Covost2-EN-ID',
+    'covost2_en_zh_test'     : 'Covost2-EN-ZH',
+    'covost2_en_ta_test'     : 'Covost2-EN-TA',
+    'covost2_id_en_test'     : 'Covost2-ID-EN',
+    'covost2_zh_en_test'     : 'Covost2-ZH-EN',
+    'covost2_ta_en_test'     : 'Covost2-TA-EN',
+    'cn_college_listen_mcq_test': 'CN-College-Listen-MCQ',
+    'dream_tts_mcq_test'    : 'DREAM-TTS-MCQ',
+    'slue_p2_sqa5_test'     : 'SLUE-P2-SQA5',
+    'public_sg_speech_qa_test': 'Public-SG-Speech-QA',
+    'spoken_squad_test'     : 'Spoken-SQuAD',
+    'openhermes_audio_test' : 'OpenHermes-Audio',
+    'alpaca_audio_test'     : 'ALPACA-Audio',
+    'wavcaps_test'     : 'WavCaps',
+    'audiocaps_test'     : 'AudioCaps',
+    'clotho_aqa_test'     : 'Clotho-AQA',
+    'wavcaps_qa_test'     : 'WavCaps-QA',
+    'audiocaps_qa_test'     : 'AudioCaps-QA',
+    'voxceleb_accent_test'     : 'VoxCeleb-Accent',
+    'voxceleb_gender_test'     : 'VoxCeleb-Gender',
+    'iemocap_gender_test': 'IEMOCAP-Gender',
+    'iemocap_emotion_test': 'IEMOCAP-Emotion',
+    'meld_sentiment_test': 'MELD-Sentiment',
+    'meld_emotion_test': 'MELD-Emotion',
+   
 }

app/draw_diagram.py

@@ -5,39 +5,30 @@ from streamlit_echarts import st_echarts
 from streamlit.components.v1 import html
 # from PIL import Image 
 from app.show_examples import *
+from app.content import *
 import pandas as pd
 
 from model_information import get_dataframe
 
 
-# huggingface_image = Image.open('style/huggingface.jpg')
-
-# other info 
-# path = "./AudioBench-Leaderboard/additional_info/Leaderboard-Rename.xlsx"
-# path = "./additional_info/Leaderboard-Rename.xlsx"
-
-# info_df = pd.read_excel(path)
 
 info_df = get_dataframe()
 
-# def nav_to(value):
-#     try:
-#         url = links_dic[str(value).lower()]
-#         js = f'window.open("{url}", "_blank").then(r => window.parent.location.href);'
-#         st_javascript(js)
-#     except:
-#         pass
 
 def draw(folder_name, category_name, dataset_name, metrics, cus_sort=True):
     
     folder = f"./results/{metrics}/"
 
-    
+    # Load the results from CSV
     data_path = f'{folder}/{category_name.lower()}.csv'
     chart_data = pd.read_csv(data_path).round(3)
     new_dataset_name = dataset_name.replace('-', '_').lower()
     chart_data = chart_data[['Model', new_dataset_name]]
-    
+
+    # Rename to proper display name
+    new_dataset_name = dataname_column_rename_in_table[new_dataset_name]
+    chart_data = chart_data.rename(columns=dataname_column_rename_in_table)
+
     st.markdown("""
                 <style>
                 .stMultiSelect [data-baseweb=select] span {
@@ -253,10 +244,12 @@ def draw(folder_name, category_name, dataset_name, metrics, cus_sort=True):
         st.session_state.show_examples = not st.session_state.show_examples
 
     if st.session_state.show_examples:
+        
+        st.markdown('To be implemented')
 
-        # if dataset_name in ['Earnings21-Test', 'Earnings22-Test', 'Tedlium3-Test', 'Tedlium3-Long-form-Test']:
-        if dataset_name in []:
-            pass
-        else:
-            show_examples(category_name, dataset_name, chart_data['Model'].tolist(), display_model_names)
+        # # if dataset_name in ['Earnings21-Test', 'Earnings22-Test', 'Tedlium3-Test', 'Tedlium3-Long-form-Test']:
+        # if dataset_name in []:
+        #     pass
+        # else:
+        #     show_examples(category_name, dataset_name, chart_data['Model'].tolist(), display_model_names)
         

app/pages.py

@@ -88,9 +88,9 @@ def dashboard():
 def asr():
     st.title("Task: Automatic Speech Recognition")
     
-    sum = ['Summarization']
-
-    dataset_lists = ['LibriSpeech-Test-Clean', 
+    sum = ['Overall']
+    dataset_lists = [
+                    'LibriSpeech-Test-Clean', 
                     'LibriSpeech-Test-Other', 
                     'Common-Voice-15-En-Test', 
                     'Peoples-Speech-Test', 
@@ -99,13 +99,11 @@ def asr():
                     'Earnings22-Test', 
                     'Tedlium3-Test', 
                     'Tedlium3-Long-form-Test', 
-                    #'IMDA-Part1-ASR-Test', 
-                    #'IMDA-Part2-ASR-Test'
                     ]
 
     filters_levelone = sum + dataset_lists
     
-    left, center, _, middle,right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
+    left, center, _, middle, right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
     
     with left:
         filter_1 = st.selectbox('Dataset', filters_levelone)
@@ -117,12 +115,35 @@ def asr():
             dataset_contents(asr_datsets[filter_1], metrics['wer'])
             draw('su', 'ASR', filter_1, 'wer', cus_sort=True)
 
+
+def cnasr():
+    st.title("Task: Automatic Speech Recognition - Mandarin")
+
+    sum = ['Overall']
+    dataset_lists = [
+                    'Aishell-ASR-ZH-Test', 
+                    ]
+
+    filters_levelone = sum + dataset_lists
+    
+    left, center, _, middle, right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
+    
+    with left:
+        filter_1 = st.selectbox('Dataset', filters_levelone)
+    
+    if filter_1:
+        if filter_1 in sum:
+            sum_table_mulit_metrix('CNASR', ['wer'])
+        else:
+            dataset_contents(cnasr_datasets[filter_1], metrics['wer'])
+            draw('su', 'CNASR', filter_1, 'wer')
+
     
 
 def sqa():
     st.title("Task: Speech Question Answering")
     
-    sum = ['Summarization']
+    sum = ['Overall']
 
     binary = ['CN-College-Listen-MCQ-Test', 'DREAM-TTS-MCQ-Test']
 
@@ -132,7 +153,7 @@ def sqa():
 
     filters_levelone = sum + binary + rest
     
-    left, center, _, middle,right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
+    left, center, _, middle, right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
     
     with left:
         filter_1 = st.selectbox('Dataset', filters_levelone)
@@ -152,14 +173,14 @@ def sqa():
 def si():
     st.title("Task: Speech Instruction")
     
-    sum = ['Summarization']
+    sum = ['Overall']
 
     dataset_lists = ['OpenHermes-Audio-Test', 
                      'ALPACA-Audio-Test']
     
     filters_levelone = sum + dataset_lists
     
-    left, center, _, middle,right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
+    left, center, _, middle, right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
     
     with left:
         filter_1 = st.selectbox('Dataset', filters_levelone)
@@ -178,7 +199,7 @@ def ac():
                         'AudioCaps-Test']
     filters_leveltwo = ['Llama3-70b-judge', 'Meteor']
     
-    left, center, _, middle,right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
+    left, center, _, middle, right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
     
     with left:
         filter_1 = st.selectbox('Dataset', filters_levelone)
@@ -193,7 +214,7 @@ def ac():
 def asqa():
     st.title("Task: Audio Scene Question Answering")
 
-    sum = ['Summarization']
+    sum = ['Overall']
 
     dataset_lists = ['Clotho-AQA-Test', 
                     'WavCaps-QA-Test', 
@@ -201,7 +222,7 @@ def asqa():
     
     filters_levelone = sum + dataset_lists
     
-    left, center, _, middle,right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
+    left, center, _, middle, right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
     
     with left:
         filter_1 = st.selectbox('Dataset', filters_levelone)
@@ -211,13 +232,13 @@ def asqa():
             sum_table_mulit_metrix('AQA', ['llama3_70b_judge'])
         else:
             dataset_contents(asqa_datasets[filter_1], metrics['llama3_70b_judge'])
-            draw('asu', 'AQA',filter_1, 'llama3_70b_judge')
+            draw('asu', 'AQA', filter_1, 'llama3_70b_judge')
 
 
 def er():
     st.title("Task: Emotion Recognition")
 
-    sum = ['Summarization']
+    sum = ['Overall']
 
     dataset_lists = ['IEMOCAP-Emotion-Test', 
                         'MELD-Sentiment-Test', 
@@ -225,7 +246,7 @@ def er():
 
     filters_levelone = sum + dataset_lists
     
-    left, center, _, middle,right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
+    left, center, _, middle, right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
     
     with left:
         filter_1 = st.selectbox('Dataset', filters_levelone)
@@ -241,32 +262,38 @@ def er():
 def ar():
     st.title("Task: Accent Recognition")
 
-    filters_levelone = ['VoxCeleb-Accent-Test']
+    sum = ['Overall']
+    dataset_lists = ['VoxCeleb-Accent-Test']
+
+
+    filters_levelone = sum + dataset_lists
     
-    left, center, _, middle,right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
+    left, center, _, middle, right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
     
     with left:
         filter_1 = st.selectbox('Dataset', filters_levelone)
 
 
     if filter_1:
-        # if filter_1 in sum:
-        #     sum_table('aR', 'llama3_70b_judge')
-        # else:
-        dataset_contents(ar_datsets[filter_1], metrics['llama3_70b_judge'])
-        draw('vu', 'AR', filter_1, 'llama3_70b_judge')
+        if filter_1 in sum:
+            sum_table_mulit_metrix('AR', ['llama3_70b_judge'])
+        # sum_table('aR', 'llama3_70b_judge')
+        else:
+            dataset_contents(ar_datsets[filter_1], metrics['llama3_70b_judge'])
+            draw('vu', 'AR', filter_1, 'llama3_70b_judge')
 
 
 def gr():
     st.title("Task: Gender Recognition")
-    sum = ['Summarization']
+    
+    sum = ['Overall']
 
     dataset_lists =  ['VoxCeleb-Gender-Test', 
                         'IEMOCAP-Gender-Test']
 
     filters_levelone = sum + dataset_lists
     
-    left, center, _, middle,right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
+    left, center, _, middle, right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
     
     with left:
         filter_1 = st.selectbox('Dataset', filters_levelone)
@@ -281,8 +308,10 @@ def gr():
 
 def spt():
     st.title("Task: Speech Translation")
-    sum = ['Summarization']
-    dataset_lists = ['Covost2-EN-ID-test', 
+    
+    sum = ['Overall']
+    dataset_lists = [
+                        'Covost2-EN-ID-test', 
                         'Covost2-EN-ZH-test',
                         'Covost2-EN-TA-test', 
                         'Covost2-ID-EN-test', 
@@ -291,7 +320,7 @@ def spt():
 
     filters_levelone = sum + dataset_lists
     
-    left, center, _, middle,right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
+    left, center, _, middle, right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
     
     with left:
         filter_1 = st.selectbox('Dataset', filters_levelone)
@@ -303,17 +332,3 @@ def spt():
             dataset_contents(spt_datasets[filter_1], metrics['bleu'])
             draw('su', 'ST', filter_1, 'bleu')
 
-
-def cnasr():
-    st.title("Task: Automatic Speech Recognition (Chinese)")
-
-    filters_levelone = ['Aishell-ASR-ZH-Test']
-    
-    left, center, _, middle,right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
-    
-    with left:
-        filter_1 = st.selectbox('Dataset', filters_levelone)
-    
-    if filter_1:
-        dataset_contents(cnasr_datasets[filter_1], metrics['wer'])
-        draw('su', 'CNASR', filter_1, 'wer')

app/summarization.py

@@ -5,6 +5,8 @@ from streamlit_echarts import st_echarts
 from streamlit.components.v1 import html
 # from PIL import Image 
 from app.show_examples import *
+from app.content import *
+
 import pandas as pd
 from typing import List
 
@@ -12,107 +14,111 @@ from model_information import get_dataframe
 
 info_df = get_dataframe()
 
-metrics_info = {
-    'wer': 'Word Error Rate (WER), a common metric for ASR evaluation. (The lower, the better)',
-    'llama3_70b_judge_binary': 'Binary evaluation using the LLAMA3-70B model, for tasks requiring a binary outcome. (0-100 based on score 0-1)',
-    'llama3_70b_judge': 'General evaluation using the LLAMA3-70B model, typically scoring based on subjective judgments. (0-100 based on score 0-5)',
-    'meteor': 'METEOR, a metric used for evaluating text generation, often used in translation or summarization tasks. (Sensitive to output length)',
-    'bleu': 'BLEU (Bilingual Evaluation Understudy), another text generation evaluation metric commonly used in machine translation. (Sensitive to output length)',
-}
+metrics_info = metrics_info
 
 def sum_table_mulit_metrix(task_name, metrics_lists: List[str]):
-    
+
+    # combine chart data from multiple sources
+    chart_data = pd.DataFrame()
     for metrics in metrics_lists:
         folder = f"./results/{metrics}/"
         data_path = f'{folder}/{task_name.lower()}.csv'
+        one_chart_data = pd.read_csv(data_path).round(3)
+        if len(chart_data) == 0:
+            chart_data = one_chart_data
+        else:
+            chart_data = pd.merge(chart_data, one_chart_data, on='Model', how='outer')
+    
 
-        chart_data = pd.read_csv(data_path).round(3)
-        selected_columns = [i for i in chart_data.columns if i != 'Model']
-        chart_data['Average'] = chart_data[selected_columns].mean(axis=1)
-        
-        # new_dataset_name = dataset_name.replace('-', '_').lower()
-        
-        st.markdown("""
-                    <style>
-                    .stMultiSelect [data-baseweb=select] span {
-                        max-width: 800px;
-                        font-size: 0.9rem;
-                        background-color: #3C6478 !important; /* Background color for selected items */
-                        color: white; /* Change text color */
-                        back
-                    }
-                    </style>
-                    """, unsafe_allow_html=True)
-        
-        # remap model names
-        display_model_names = {key.strip() :val.strip() for key, val in zip(info_df['Original Name'], info_df['Proper Display Name'])}
-        chart_data['model_show'] = chart_data['Model'].map(lambda x: display_model_names.get(x, x))
-
-        models = st.multiselect("Please choose the model", 
-                                sorted(chart_data['model_show'].tolist()), 
-                                default = sorted(chart_data['model_show'].tolist()),
-                                key=f"multiselect_{task_name}_{metrics}"
-                                )
-        
-        chart_data = chart_data[chart_data['model_show'].isin(models)].dropna(axis=0)
-        # chart_data = chart_data.sort_values(by=['Average'], ascending=True).dropna(axis=0)
+    selected_columns = [i for i in chart_data.columns if i != 'Model']
+    chart_data['Average'] = chart_data[selected_columns].mean(axis=1)
 
-        if len(chart_data) == 0: return
+    # Update dataset name in table
+    chart_data = chart_data.rename(columns=dataname_column_rename_in_table)
+    
+    st.markdown("""
+                <style>
+                .stMultiSelect [data-baseweb=select] span {
+                    max-width: 800px;
+                    font-size: 0.9rem;
+                    background-color: #3C6478 !important; /* Background color for selected items */
+                    color: white; /* Change text color */
+                    back
+                }
+                </style>
+                """, unsafe_allow_html=True)
+    
+    # remap model names
+    display_model_names = {key.strip() :val.strip() for key, val in zip(info_df['Original Name'], info_df['Proper Display Name'])}
+    chart_data['model_show'] = chart_data['Model'].map(lambda x: display_model_names.get(x, x))
 
-        # = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
-        '''
-        Show Table
-        '''
-        with st.container():
-            st.markdown(f'#### Overal Evaluation Results')
-            st.markdown(f'###### Evaluation Method: {metrics_info[metrics]}')
+    models = st.multiselect("Please choose the model", 
+                            sorted(chart_data['model_show'].tolist()), 
+                            default = sorted(chart_data['model_show'].tolist()),
+                            # key=f"multiselect_{task_name}_{metrics}"
+                            )
+    
+    chart_data = chart_data[chart_data['model_show'].isin(models)].dropna(axis=0)
+    # chart_data = chart_data.sort_values(by=['Average'], ascending=True).dropna(axis=0)
 
-            model_link = {key.strip(): val for key, val in zip(info_df['Proper Display Name'], info_df['Link'])}
+    if len(chart_data) == 0: return
 
-            chart_data['model_link'] = chart_data['model_show'].map(model_link) 
+    # = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
+    '''
+    Show Table
+    '''
+    with st.container():
+        st.markdown(f'##### TABLE')
 
-            tabel_columns = [i for i in chart_data.columns if i not in ['Model', 'model_show']]
-            column_to_front = 'Average'
-            new_order = [column_to_front] + [col for col in tabel_columns if col != column_to_front]
-            
-            chart_data_table = chart_data[['model_show'] + new_order]
-            
+        model_link = {key.strip(): val for key, val in zip(info_df['Proper Display Name'], info_df['Link'])}
 
-            # Format numeric columns to 2 decimal places
-            chart_data_table[chart_data_table.columns[1]] = chart_data_table[chart_data_table.columns[1]].apply(lambda x: round(float(x), 3) if isinstance(float(x), (int, float)) else float(x))
+        chart_data['model_link'] = chart_data['model_show'].map(model_link) 
 
-            if metrics in ['wer']:
-                ascend = True
-            else:
-                ascend= False
+        tabel_columns = [i for i in chart_data.columns if i not in ['Model', 'model_show']]
+        column_to_front = 'Average'
+        new_order = [column_to_front] + [col for col in tabel_columns if col != column_to_front]
+        
+        chart_data_table = chart_data[['model_show'] + new_order]
+        
 
-            chart_data_table = chart_data_table.sort_values(
-                    by=['Average'],
-                    ascending=ascend
-                ).reset_index(drop=True)
-            
-            def highlight_first_element(x):
-                # Create a DataFrame with the same shape as the input
-                df_style = pd.DataFrame('', index=x.index, columns=x.columns)
-                
-                # Apply background color to the first element in row 0 (df[0][0])
-                df_style.iloc[0, 1] = 'background-color: #b0c1d7; color: white'
-                
-                return df_style
-            
-            styled_df = chart_data_table.style.apply(
-                highlight_first_element, axis=None
-            )
+        # Format numeric columns to 2 decimal places
+        chart_data_table[chart_data_table.columns[1]] = chart_data_table[chart_data_table.columns[1]].apply(lambda x: round(float(x), 3) if isinstance(float(x), (int, float)) else float(x))
 
-            st.dataframe(
-                    styled_df,
-                    column_config={
-                        'model_show': 'Model',
-                        chart_data_table.columns[1]: {'alignment': 'left'},
-                        "model_link": st.column_config.LinkColumn(
-                            "Model Link",
-                        ),
-                    },
-                    hide_index=True,
-                    use_container_width=True
-                )
+        if metrics in ['wer']:
+            ascend = True
+        else:
+            ascend= False
+
+        chart_data_table = chart_data_table.sort_values(
+                by=['Average'],
+                ascending=ascend
+            ).reset_index(drop=True)
+        
+        # Highlight the best performing model
+        def highlight_first_element(x):
+            # Create a DataFrame with the same shape as the input
+            df_style = pd.DataFrame('', index=x.index, columns=x.columns)
+            # Apply background color to the first element in row 0 (df[0][0])
+            df_style.iloc[0, 1] = 'background-color: #b0c1d7; color: white'
+            return df_style
+        
+        styled_df = chart_data_table.style.apply(
+            highlight_first_element, axis=None
+        )
+
+        st.dataframe(
+                styled_df,
+                column_config={
+                    'model_show': 'Model',
+                    chart_data_table.columns[1]: {'alignment': 'left'},
+                    "model_link": st.column_config.LinkColumn(
+                        "Model Link",
+                    ),
+                },
+                hide_index=True,
+                use_container_width=True
+            )
+            
+    #for metrics in metrics_lists:
+    # Only report the last metrics
+    st.markdown(f'###### Metric: {metrics_info[metrics]}')