import gradio as gr import random import pandas as pd import os import threading from threading import Event from utils.data_loader import get_random_example from utils.models import generate_summaries, model_names from utils.ui_helpers import toggle_context_display, update_feedback, get_context_html from utils.leaderboard import load_leaderboard_data, save_leaderboard_data # Global interrupt mechanism for model generation generation_interrupt = Event() # Feedback options for different voting outcomes feedback_options = { "left": ["Model A: More complete", "Model A: More accurate", "Model A: More relevant", "Model A: Better written", "Model A: Better refusal (if applicable)"], "right": ["Model B: More complete", "Model B: More accurate", "Model B: More relevant", "Model B: Better written", "Model B: Better refusal (if applicable)"], "tie": ["Both complete", "Both accurate", "Both well written", "Both handle refusal well (if applicable)"], "neither": ["Both incomplete", "Both hallucinate", "Both irrelevant", "Both incorrectly refuse (if applicable)", "A is bad", "B is bad"] } def load_context(): """Load a new question and context (fast operation)""" generation_interrupt.clear() example = get_random_example() context_desc = example.get('processed_context_desc', '') if context_desc: context_desc = f"
The question and context are about: {context_desc}
" show_full = False context_html = get_context_html(example, show_full=show_full) return [ example, gr.update(value=example['question']), gr.update(value=context_desc, visible=bool(context_desc)), gr.update(value=context_html), gr.update(value="Show Full Context", elem_classes=["context-toggle-button"]), show_full ] def generate_model_summaries_with_timeout(example, timeout=60): """Run model inference in a separate thread with timeout for interruptibility""" import threading import time result = { "model_a": "", "model_b": "", "summary_a": "", "summary_b": "", "completed": False } if generation_interrupt.is_set(): return result def run_generation(): try: m_a_name, m_b_name = random.sample(model_names, 2) s_a, s_b = generate_summaries(example, m_a_name, m_b_name) if not generation_interrupt.is_set(): result["model_a"] = m_a_name result["model_b"] = m_b_name result["summary_a"] = s_a result["summary_b"] = s_b result["completed"] = True except Exception as e: print(f"Error in generation thread: {e}") generation_thread = threading.Thread(target=run_generation) generation_thread.daemon = True generation_thread.start() # Uncomment this critical waiting code start_time = time.time() while time.time() - start_time < timeout: if generation_interrupt.is_set() or not generation_thread.is_alive() or result["completed"]: break time.sleep(0.1) return result async def generate_model_summaries_with_timeout_async(example, timeout=30): """Async version that properly waits for the thread""" import asyncio import threading import time result = { "model_a": "", "model_b": "", "summary_a": "", "summary_b": "", "completed": False } if generation_interrupt.is_set(): return result def run_generation(): try: m_a_name, m_b_name = random.sample(model_names, 2) s_a, s_b = generate_summaries(example, m_a_name, m_b_name) if not generation_interrupt.is_set(): result["model_a"] = m_a_name result["model_b"] = m_b_name result["summary_a"] = s_a result["summary_b"] = s_b result["completed"] = True except Exception as e: print(f"Error in generation thread: {e}") generation_thread = threading.Thread(target=run_generation) generation_thread.daemon = True generation_thread.start() # Use asyncio.sleep instead of time.sleep for async waiting start_time = time.time() while time.time() - start_time < timeout: if generation_interrupt.is_set() or not generation_thread.is_alive() or result["completed"]: break await asyncio.sleep(0.1) # Non-blocking sleep return result def process_generation_result(result): """Process the results from the threaded generation function""" if not result["completed"]: # Generation was interrupted or failed return [ "", "", "", "", None, [], False, load_leaderboard_data(), gr.update(value="Generation was interrupted or timed out. Please try again."), gr.update(value="Generation was interrupted or timed out. Please try again."), gr.update(interactive=True, elem_classes=["vote-button"]), gr.update(interactive=True, elem_classes=["vote-button"]), gr.update(interactive=True, elem_classes=["vote-button"]), gr.update(interactive=True, elem_classes=["vote-button", "vote-button-neither"]), gr.update(choices=[], value=[], interactive=False, visible=False), gr.update(visible=False), gr.update(interactive=False, visible=True), gr.update(visible=False), gr.update(interactive=True), gr.update(elem_classes=[]) ] # Generation completed successfully agg_results = load_leaderboard_data() return [ result["model_a"], result["model_b"], result["summary_a"], result["summary_b"], None, [], False, agg_results, gr.update(value=result["summary_a"]), gr.update(value=result["summary_b"]), gr.update(interactive=True, elem_classes=["vote-button"]), gr.update(interactive=True, elem_classes=["vote-button"]), gr.update(interactive=True, elem_classes=["vote-button"]), gr.update(interactive=True, elem_classes=["vote-button", "vote-button-neither"]), gr.update(choices=[], value=[], interactive=False, visible=False), gr.update(visible=False), gr.update(interactive=False, visible=True), gr.update(visible=False), gr.update(interactive=True), gr.update(elem_classes=[]) ] async def process_example_async(example): result = await generate_model_summaries_with_timeout_async(example) return process_generation_result(result) def process_example_sync(example): result = generate_model_summaries_with_timeout(example) return process_generation_result(result) def select_vote_improved(winner_choice): """Updates UI based on vote selection""" feedback_choices = feedback_options.get(winner_choice, []) btn_a_classes = ["vote-button"] btn_b_classes = ["vote-button"] btn_tie_classes = ["vote-button"] btn_neither_classes = ["vote-button", "vote-button-neither"] if winner_choice == 'left': btn_a_classes.append("selected") elif winner_choice == 'right': btn_b_classes.append("selected") elif winner_choice == 'tie': btn_tie_classes.append("selected") elif winner_choice == 'neither': btn_neither_classes.append("selected") return [ winner_choice, gr.update(choices=feedback_choices, value=[], interactive=True, visible=True), gr.update(visible=True), gr.update(interactive=True), gr.update(elem_classes=btn_a_classes), gr.update(elem_classes=btn_b_classes), gr.update(elem_classes=btn_tie_classes), gr.update(elem_classes=btn_neither_classes) ] def submit_vote_fixed(m_a, m_b, winner, feedback, current_results): """Processes vote and updates leaderboard""" if winner is None: print("Warning: Submit called without a winner selected.") return {} updated_results = current_results.copy() models_involved = [m_a, m_b] for model in models_involved: if model not in updated_results["wins"]: updated_results["wins"][model] = 0 updated_results["losses"][model] = 0 updated_results["ties"][model] = 0 if winner == 'left': updated_results["wins"][m_a] = updated_results["wins"].get(m_a, 0) + 1 updated_results["losses"][m_b] = updated_results["losses"].get(m_b, 0) + 1 elif winner == 'right': updated_results["wins"][m_b] = updated_results["wins"].get(m_b, 0) + 1 updated_results["losses"][m_a] = updated_results["losses"].get(m_a, 0) + 1 elif winner == 'tie': updated_results["ties"][m_a] = updated_results["ties"].get(m_a, 0) + 1 updated_results["ties"][m_b] = updated_results["ties"].get(m_b, 0) + 1 updated_results["votes"] = updated_results.get("votes", 0) + 1 save_leaderboard_data(updated_results) # Prepare Results Table results_list = [] all_models = list(set(list(updated_results["wins"].keys()) + list(updated_results["losses"].keys()) + list(updated_results["ties"].keys()))) for model in sorted(all_models): wins = updated_results["wins"].get(model, 0) losses = updated_results["losses"].get(model, 0) ties = updated_results["ties"].get(model, 0) total_comparisons = wins + losses + ties win_rate = (wins + 0.5 * ties) / total_comparisons if total_comparisons > 0 else 0.0 results_list.append({ "Model": model, "Win Rate (%)": f"{win_rate:.1%}", "Wins": wins, "Losses": losses, "Ties": ties, "Comparisons": total_comparisons }) results_df = pd.DataFrame(results_list) if not results_df.empty: results_df['Win Rate Value'] = results_df['Win Rate (%)'].str.rstrip('%').astype('float') / 100.0 results_df = results_df.sort_values(by='Win Rate Value', ascending=False).drop(columns=['Win Rate Value']) return [ True, updated_results, gr.update(interactive=False), gr.update(interactive=False), gr.update(interactive=False), gr.update(interactive=False), gr.update(interactive=False), gr.update(visible=True), gr.update(visible=False), gr.update(visible=True), gr.update(interactive=False), gr.update(value=results_df, visible=True), gr.update(elem_classes=["results-revealed"]), gr.update(interactive=True), gr.update(value=m_a), gr.update(value=m_b) ] # Create Gradio interface with gr.Blocks(theme=gr.themes.Default( primary_hue=gr.themes.colors.orange, secondary_hue=gr.themes.colors.slate )) as demo: # Load CSS css_path = os.path.join(os.getcwd(), 'static', 'styles.css') with open(css_path, 'r') as f: css_content = f.read() gr.HTML(f"") # State Variables current_example = gr.State({}) model_a_name = gr.State("") model_b_name = gr.State("") summary_a_text = gr.State("") summary_b_text = gr.State("") selected_winner = gr.State(None) feedback_list = gr.State([]) show_results_state = gr.State(False) results_agg = gr.State({"wins": {}, "losses": {}, "ties": {}, "votes": 0}) show_full_context = gr.State(False) # Create Tabs with gr.Tabs() as tabs: # Main Arena Tab with gr.TabItem("Arena", id="arena-tab"): gr.Markdown("# RAG Summarizer Arena") gr.Markdown("Compare summaries generated by different models based on the provided context and query. Select the better summary, or choose 'Tie' or 'Neither'. Your feedback helps evaluate model performance.") # Main container with gr.Column(elem_id="main-interface-area") as main_interface_area: # Query section with gr.Row(elem_id="query-title-row"): gr.Markdown("### Query", elem_classes="section-heading") with gr.Row(elem_id="query-container"): with gr.Row(elem_classes="query-box-row"): query_display = gr.Markdown(value="Loading question...", elem_classes="query-text") random_question_btn = gr.Button("🔄 Get Random Question", elem_classes="query-button") # Context description and display context_description = gr.Markdown("", elem_classes="context-description") with gr.Row(elem_id="context-header-row"): gr.Markdown("### Context Provided", elem_classes="context-title") context_toggle_btn = gr.Button("Show Full Context", elem_classes=["context-toggle-button"]) context_display = gr.HTML(value="Loading context...", label="Context Chunks") gr.Markdown("---") gr.Markdown("### Compare Summaries", elem_classes="section-heading") # Model summaries with gr.Row(): with gr.Column(scale=1): with gr.Group(elem_classes=["summary-card", "summary-card-a"]): summary_a_display = gr.Textbox(label="Model A", lines=10, interactive=False, show_copy_button=True) with gr.Column(scale=1): with gr.Group(elem_classes=["summary-card", "summary-card-b"]): summary_b_display = gr.Textbox(label="Model B", lines=10, interactive=False, show_copy_button=True) # Voting section gr.Markdown("### Cast Your Vote", elem_classes="section-heading") with gr.Row(): vote_button_a = gr.Button("⬅️ Summary A is Better", elem_classes=["vote-button"]) vote_button_tie = gr.Button("🤝 Tie / Equally Good", elem_classes=["vote-button"]) vote_button_b = gr.Button("➡️ Summary B is Better", elem_classes=["vote-button"]) vote_button_neither = gr.Button("❌ Neither is Adequate", elem_classes=["vote-button", "vote-button-neither"]) # Feedback and Submit sections with gr.Group(elem_classes=["feedback-section"], visible=False) as feedback_section: feedback_checkboxes = gr.CheckboxGroup(label="Feedback (optional)", choices=[], interactive=False) submit_button = gr.Button("Submit Vote", variant="primary", interactive=False, elem_id="submit-button") # Results area with gr.Column(visible=False) as results_reveal_area: gr.Markdown("---") gr.Markdown("### ✅ Vote Submitted!", elem_classes="section-heading") # Model reveal section with gr.Row(): with gr.Column(scale=1): gr.Markdown("### Model A was actually:", elem_classes="section-heading") model_a_reveal = gr.Markdown("", elem_classes="model-reveal model-a-reveal") with gr.Column(scale=1): gr.Markdown("### Model B was actually:", elem_classes="section-heading") model_b_reveal = gr.Markdown("", elem_classes="model-reveal model-b-reveal") gr.HTML("
") # Try another button with gr.Row(elem_classes=["control-buttons"]): try_another_btn = gr.Button("🔄 Try Another Question", elem_id="try-another-btn") # Leaderboard Tab with gr.TabItem("Leaderboard", id="leaderboard-tab"): gr.Markdown("# Model Performance Leaderboard") gr.Markdown("View aggregate performance statistics for all models. The table below shows win rates, wins, losses, and ties for each model based on all evaluations.") results_table_display = gr.DataFrame(label="Model Performance", interactive=False, wrap=True) # Generic function to handle starting a new example def handle_new_example_click(): generation_interrupt.set() # Interrupt any ongoing generation return load_context()[0] def update_ui_for_new_context(example): return [ gr.update(value=example['question']), gr.update(value=example.get('processed_context_desc', ''), visible=bool(example.get('processed_context_desc', ''))), gr.update(value=get_context_html(example, False)), gr.update(value="Show Full Context", elem_classes=["context-toggle-button"]), False ] # Event handling # Toggle context display context_toggle_btn.click( fn=toggle_context_display, inputs=[current_example, show_full_context], outputs=[show_full_context, context_display, context_toggle_btn] ) # Initial loading - context first, then summaries demo.load( fn=load_context, inputs=[], outputs=[current_example, query_display, context_description, context_display, context_toggle_btn, show_full_context] ).then( fn=process_example_async, inputs=[current_example], outputs=[model_a_name, model_b_name, summary_a_text, summary_b_text, selected_winner, feedback_list, show_results_state, results_agg, summary_a_display, summary_b_display, vote_button_a, vote_button_b, vote_button_tie, vote_button_neither, feedback_checkboxes, feedback_section, submit_button, results_reveal_area, random_question_btn, main_interface_area] ) # Random Question and Try Another buttons with interruption for btn in [random_question_btn, try_another_btn]: btn.click( fn=handle_new_example_click, inputs=[], outputs=[current_example] ).then( fn=update_ui_for_new_context, inputs=[current_example], outputs=[query_display, context_description, context_display, context_toggle_btn, show_full_context] ).then( fn=process_example_sync, inputs=[current_example], outputs=[model_a_name, model_b_name, summary_a_text, summary_b_text, selected_winner, feedback_list, show_results_state, results_agg, summary_a_display, summary_b_display, vote_button_a, vote_button_b, vote_button_tie, vote_button_neither, feedback_checkboxes, feedback_section, submit_button, results_reveal_area, random_question_btn, main_interface_area] ) # Vote button handlers for btn, choice in zip( [vote_button_a, vote_button_b, vote_button_tie, vote_button_neither], ['left', 'right', 'tie', 'neither'] ): btn.click( fn=lambda choice=choice: select_vote_improved(choice), inputs=None, outputs=[selected_winner, feedback_checkboxes, feedback_section, submit_button, vote_button_a, vote_button_b, vote_button_tie, vote_button_neither] ) # Update feedback when checkboxes change feedback_checkboxes.change( fn=update_feedback, inputs=[feedback_checkboxes], outputs=[feedback_list] ) # Process vote submission and reveal results submit_button.click( fn=submit_vote_fixed, inputs=[model_a_name, model_b_name, selected_winner, feedback_list, results_agg], outputs=[show_results_state, results_agg, vote_button_a, vote_button_b, vote_button_tie, vote_button_neither, feedback_checkboxes, feedback_section, submit_button, results_reveal_area, random_question_btn, results_table_display, main_interface_area, context_toggle_btn, model_a_reveal, model_b_reveal] ) if __name__ == "__main__": demo.launch(debug=True)