Spaces:
Running
Running
File size: 4,834 Bytes
27638f8 16d8300 27638f8 16d8300 27638f8 7254da2 27638f8 16d8300 27638f8 16d8300 27638f8 16d8300 27638f8 16d8300 27638f8 16d8300 27638f8 16d8300 27638f8 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 |
from pathlib import Path
from apscheduler.schedulers.background import BackgroundScheduler
import pandas as pd
import gradio as gr
from gradio_leaderboard import Leaderboard, ColumnFilter
from constants import Constants, model_type_emoji
TITLE = """<h1 align="center" id="space-title">TabArena: Public leaderboard for Tabular methods</h1>"""
INTRODUCTION_TEXT = ("TabArena Leaderboard measures the performance of tabular models on a collection of tabular "
"datasets manually curated. The datasets are collected to make sure they are tabular, with "
"permissive license without ethical issues and so on, we refer to the paper for a full "
"description of our approach.")
ABOUT_TEXT = f"""
## How It Works.
To evaluate the leaderboard, follow install instructions in
`https://github.com/autogluon/tabrepo/tree/tabarena` and run
`https://github.com/autogluon/tabrepo/blob/tabarena/examples/tabarena/run_tabarena_eval.py`.
This will generate a leaderboard. You can add your own method and contact the authors if you want it to be added
to the leaderboard. We require method to have public code available to be considered in the leaderboard.
"""
CITATION_BUTTON_LABEL = "If you use this leaderboard in your research please cite the following:"
CITATION_BUTTON_TEXT = r"""
@article{
TBA,
}
"""
def get_model_family(model_name: str) -> str:
prefixes_mapping = {
Constants.automl: ["AutoGluon"],
Constants.finetuned: ["REALMLP", "TabM", "FASTAI", "MNCA", "NN_TORCH"],
Constants.tree: ["GBM", "CAT", "EBM", "XGB"],
Constants.foundational: ["TABDPT", "TABICL", "TABPFN"],
Constants.baseline: ["KNN", "LR"]
}
for method_type, prefixes in prefixes_mapping.items():
for prefix in prefixes:
if prefix.lower() in model_name.lower():
return method_type
return Constants.other
def load_data(filename: str):
df_leaderboard = pd.read_csv(Path(__file__).parent / "data" / f"{filename}.csv.zip")
print(f"Loaded dataframe with {len(df_leaderboard)} rows and columns {df_leaderboard.columns}")
# sort by ELO
df_leaderboard.sort_values(by="elo", ascending=False, inplace=True)
# add model family information
df_leaderboard["family"] = df_leaderboard.loc[:, "method"].apply(
lambda s: get_model_family(s) + " " + model_type_emoji[get_model_family(s)]
)
# select only the columns we want to display
df_leaderboard = df_leaderboard.loc[:, ["method", "family", "time_train_s", "time_infer_s", "rank", "elo"]]
# round for better display
df_leaderboard = df_leaderboard.round(1)
# rename some columns
df_leaderboard.rename(columns={
"time_train_s": "training time (s)",
"time_infer_s": "inference time (s)",
}, inplace=True)
# TODO show ELO +/- sem
return df_leaderboard
def make_leaderboard(df_leaderboard: pd.DataFrame) -> Leaderboard:
return Leaderboard(
value=df_leaderboard,
search_columns=["method"],
filter_columns=[
# "method",
ColumnFilter("family", type="dropdown", label="Filter by family"),
]
)
def main():
demo = gr.Blocks()
with demo:
gr.HTML(TITLE)
gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
with gr.Tabs(elem_classes="tab-buttons") as tabs:
with gr.TabItem('🏅 Overall', elem_id="llm-benchmark-tab-table", id=2):
df_leaderboard = load_data("leaderboard-all")
leaderboard = make_leaderboard(df_leaderboard)
with gr.TabItem('🏅 Regression', elem_id="llm-benchmark-tab-table", id=0):
df_leaderboard = load_data("leaderboard-regression")
leaderboard = make_leaderboard(df_leaderboard)
with gr.TabItem('🏅 Classification', elem_id="llm-benchmark-tab-table", id=1):
df_leaderboard = load_data("leaderboard-classification")
leaderboard = make_leaderboard(df_leaderboard)
with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=4):
gr.Markdown(ABOUT_TEXT, elem_classes="markdown-text")
with gr.Row():
with gr.Accordion("📙 Citation", open=False):
citation_button = gr.Textbox(
value=CITATION_BUTTON_TEXT,
label=CITATION_BUTTON_LABEL,
lines=20,
elem_id="citation-button",
show_copy_button=True,
)
scheduler = BackgroundScheduler()
# scheduler.add_job(restart_space, "interval", seconds=1800)
scheduler.start()
demo.queue(default_concurrency_limit=40).launch()
demo.launch()
if __name__ == "__main__":
main()
|