import gradio as gr import pandas as pd from apscheduler.schedulers.background import BackgroundScheduler from gradio_leaderboard import ColumnFilter, Leaderboard, SelectColumns from src.about import CITATION_BUTTON_LABEL, CITATION_BUTTON_TEXT, EVALUATION_QUEUE_TEXT, INTRODUCTION_TEXT, TITLE from src.datamodel.data import F1Data from src.display.css_html_js import custom_css from src.display.utils import AutoEvalColumn, ModelType, fields from src.envs import API, CODE_PROBLEMS_REPO, REPO_ID, RESULTS_REPO, SUBMISSIONS_REPO from src.logger import get_logger from src.populate import get_leaderboard_df from src.submission.submit import add_new_solutions logger = get_logger(__name__) SPLIT = "warmup" # TODO temp SKIP_VALIDATION = True # TODO temp def restart_space(): API.restart_space(repo_id=REPO_ID) lbdb = F1Data(cp_ds_name=CODE_PROBLEMS_REPO, sub_ds_name=SUBMISSIONS_REPO, res_ds_name=RESULTS_REPO, split=SPLIT) leaderboard_df = get_leaderboard_df(RESULTS_REPO) logger.info("Initialized LBDB") def init_leaderboard(dataframe: pd.DataFrame): if dataframe is None or dataframe.empty: raise ValueError("Leaderboard DataFrame is empty or None.") return Leaderboard( value=dataframe, datatype=[c.type for c in fields(AutoEvalColumn)], select_columns=SelectColumns( default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default], cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden], label="Select Columns to Display:", ), search_columns=[AutoEvalColumn.system.name, AutoEvalColumn.system_type.name], hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden], filter_columns=[ ColumnFilter(AutoEvalColumn.system_type.name, type="checkboxgroup", label="Model types"), ], bool_checkboxgroup_label="Hide models", interactive=False, ) demo = gr.Blocks(css=custom_css) with demo: gr.Image( "assets/banner.png", interactive=False, show_label=False, show_download_button=False, container=False, ) gr.HTML( """ """ ) gr.HTML(TITLE) gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text") with gr.Tabs(elem_classes="tab-buttons") as tabs: with gr.TabItem("🏅 FormulaOne Leaderboard", elem_id="formulaone-leaderboar-tab-table", id=0): leaderboard = init_leaderboard(leaderboard_df) with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=2): logger.info("Tab submission") with gr.Column(): with gr.Row(): gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text") with gr.Row(): gr.Markdown("# ✉️✨ Submit your solutions here!", elem_classes="markdown-text") with gr.Row(): with gr.Column(): system_name_textbox = gr.Textbox(label=AutoEvalColumn.system.name) org_textbox = gr.Textbox(label=AutoEvalColumn.organization.name) sys_type_dropdown = gr.Dropdown( choices=[t.to_str(" ") for t in ModelType], label=AutoEvalColumn.system_type.name, multiselect=False, value=ModelType.LLM.to_str(" "), interactive=True, ) submission_file = gr.File(label="JSONL solutions file", file_types=[".jsonl"]) logger.info("Submit button") submit_button = gr.Button("Submit") submission_result = gr.Markdown() def add_solution_cbk(system_name, org, sys_type, submission_path): return add_new_solutions( lbdb, system_name, org, sys_type, submission_path, skip_validation=SKIP_VALIDATION ) submit_button.click( add_solution_cbk, [ system_name_textbox, org_textbox, sys_type_dropdown, submission_file, ], submission_result, ) with gr.Row(): logger.info("Citation") with gr.Accordion(CITATION_BUTTON_LABEL, open=False): gr.Code( value=CITATION_BUTTON_TEXT.strip(), elem_id="citation-block", ) # citation_button = gr.Textbox( # value=CITATION_BUTTON_TEXT, # # label=CITATION_BUTTON_LABEL, # lines=20, # elem_id="citation-button", # show_copy_button=True, # ) logger.info("Scheduler") scheduler = BackgroundScheduler() scheduler.add_job(restart_space, "interval", seconds=1800) scheduler.start() logger.info("Launch") demo.queue(default_concurrency_limit=40).launch() logger.info("Done")