Update app.py

app.py

@@ -103,79 +103,115 @@ def run_mmlu_evaluation(all_subjects, num_subjects, num_shots, all_questions, nu
                 gr.update(interactive=True), gr.update(interactive=True), 
                 gr.update(interactive=True), gr.update(interactive=True), 
                 gr.update(interactive=True))
-    
+       
 # ---------------------------------------------------------------------------
 # 3. Gradio Interface
 # ---------------------------------------------------------------------------
 with gr.Blocks() as demo:
-    gr.Markdown("# Mistral-7B on MMLU-Pro Evaluation Demo")
+    gr.Markdown("#Head to Head Evaluation Comparator")
     gr.Markdown("""
-    This demo evaluates [Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on the [MMLU-Pro Dataset](https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro).
+    This demo evaluates two models (or the same model with different configs) on a single dataset. 
+
+    Available Datasets: [MMLU-Pro](https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro), MMLU, MMLU-Redux. 
+    Available Models: [Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) .
+
+    Available Configs:
+    - Model Configs: a) Different quantizations, b) Using Flash Attention etc.
+    - Eval Dataset Configs include: i) subset to specific questions or subjects
+    - Eval Method Configs include: i) different prompts, ii) use different regexes for capturing the "answer" (especially for multiple-choice evals), iii) different eval scripts, 
+    iv) different eval metrics.    
     """)
+        # Dataset Selection Section
+    gr.Markdown("### (A) Select Dataset for evaluation")
     
-    # MMLU Evaluation Section
-    gr.Markdown("### MMLU Evaluation")
-    
-    with gr.Row():
-        all_subjects_checkbox = gr.Checkbox(
-            label="Evaluate All Subjects",
-            value=False,
-            info="When checked, evaluates all 14 MMLU-Pro subjects"
-        )
-        num_subjects_slider = gr.Slider(
-            minimum=1,
-            maximum=14,
-            value=14,
-            step=1,
-            label="Number of Subjects",
-            info="Number of subjects to evaluate (1-14). They will be loaded in alphabetical order.",
-            interactive=True
-        )
-    
-    with gr.Row():
-        num_shots_slider = gr.Slider(
-            minimum=0,
-            maximum=5,
-            value=5,
-            step=1,
-            label="Number of Few-shot Examples",
-            info="Number of examples to use for few-shot learning (0-5)."
-        )
-    
-    with gr.Row():
-        all_questions_checkbox = gr.Checkbox(
-            label="Evaluate All Questions",
-            value=False,
-            info="When checked, evaluates all available questions for each subject"
-        )
-        questions_info_text = gr.Markdown(visible=False, value="**All 12,032 questions across all subjects will be evaluated**")
-    
-    with gr.Row(elem_id="questions_selection_row"):
-        questions_container = gr.Column(scale=1, elem_id="questions_slider_container")
+    dataset_dropdown = gr.Dropdown(
+        choices=["MMLU-Pro"],
+        value=None,
+        label="Dataset",
+        info="Select a dataset to evaluate the model on"
+    )
     
-    with questions_container:
-        num_questions_slider = gr.Slider(
-            minimum=1,
-            maximum=100,
-            value=20,
-            step=1,
-            label="Questions per Subject",
-            info="Choose a subset of questions (1-100) per subject. They will be loaded in order of question_id.",
-            interactive=True
-        )
+    # MMLU Config Container - Initially hidden until dataset is selected
+    with gr.Group(visible=False) as mmlu_config_container:
+        gr.Markdown("### MMLU-Pro Evaluation Configuration")
+        
+        with gr.Row():
+            all_subjects_checkbox = gr.Checkbox(
+                label="Evaluate All Subjects",
+                value=False,
+                info="When checked, evaluates all 14 MMLU-Pro subjects"
+            )
+            num_subjects_slider = gr.Slider(
+                minimum=1,
+                maximum=14,
+                value=14,
+                step=1,
+                label="Number of Subjects",
+                info="Number of subjects to evaluate (1-14). They will be loaded in alphabetical order.",
+                interactive=True
+            )
+        
+        with gr.Row():
+            num_shots_slider = gr.Slider(
+                minimum=0,
+                maximum=5,
+                value=5,
+                step=1,
+                label="Number of Few-shot Examples",
+                info="Number of examples to use for few-shot learning (0-5)."
+            )
+        
+        with gr.Row():
+            all_questions_checkbox = gr.Checkbox(
+                label="Evaluate All Questions",
+                value=False,
+                info="When checked, evaluates all available questions for each subject"
+            )
+            questions_info_text = gr.Markdown(visible=False, value="**All 12,032 questions across all subjects will be evaluated**")
+        
+        with gr.Row(elem_id="questions_selection_row"):
+            questions_container = gr.Column(scale=1, elem_id="questions_slider_container")
+        
+        with questions_container:
+            num_questions_slider = gr.Slider(
+                minimum=1,
+                maximum=100,
+                value=20,
+                step=1,
+                label="Questions per Subject",
+                info="Choose a subset of questions (1-100) per subject. They will be loaded in order of question_id.",
+                interactive=True
+            )
+        
+        with gr.Row():
+            with gr.Column(scale=1):
+                eval_mmlu_button = gr.Button("Run MMLU-Pro Evaluation", variant="primary", interactive=True)
+                cancel_mmlu_button = gr.Button("Cancel Evaluation", variant="stop", visible=False)
     
-    with gr.Row():
-        with gr.Column(scale=1):
-            eval_mmlu_button = gr.Button("Run MMLU-Pro Evaluation", variant="primary", interactive=True)
-            cancel_mmlu_button = gr.Button("Cancel Evaluation", variant="stop", visible=False)
+    # Results Section - Initially hidden
+    with gr.Group(visible=False) as results_container:
         results_output = gr.Markdown(label="Evaluation Results")
         
-    with gr.Row():
-        results_table = gr.DataFrame(interactive=True, label="Detailed Results (Sortable)", visible=True)
+        with gr.Row():
+            results_table = gr.DataFrame(interactive=True, label="Detailed Results (Sortable)", visible=True)
     
     # Track evaluation state - used to prevent multiple evaluations
     evaluation_state = gr.State({"running": False})
     
+    # Function to show configuration based on selected dataset
+    def update_interface_based_on_dataset(dataset):
+        if dataset == "MMLU-Pro":
+            return gr.update(visible=True), gr.update(visible=True)
+        else:
+            return gr.update(visible=False), gr.update(visible=False)
+    
+    # Connect dataset dropdown to show/hide appropriate configuration
+    dataset_dropdown.change(
+        fn=update_interface_based_on_dataset,
+        inputs=[dataset_dropdown],
+        outputs=[mmlu_config_container, results_container]
+    )
+    
     # Update num_subjects_slider interactivity based on all_subjects checkbox
     def update_subjects_slider(checked):
         return gr.update(interactive=not checked)