File size: 3,006 Bytes
2c21cf7 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 |
import pandas as pd
from countries import make_country_table
make_country_table = make_country_table
def aggregate(results):
results = pd.DataFrame([r for rs in results for r in rs])
results = (
results.groupby(["model", "bcp_47", "task", "metric"]).mean().reset_index()
)
lang_results = (
results.groupby(["bcp_47", "task", "metric"])
.agg({"score": "mean", "model": "nunique"})
.reset_index()
)
# lang_results = pd.merge(languages, lang_results, on="bcp_47", how="outer")
model_results = (
results.groupby(["model", "task", "metric"])
.agg({"score": "mean", "bcp_47": "nunique"})
.reset_index()
)
task_results = (
results.groupby(["task", "metric"])
.agg({"score": "mean", "bcp_47": "nunique", "model": "nunique"})
.reset_index()
)
return results, lang_results, model_results, task_results
def mean(lst):
return sum(lst) / len(lst) if lst else None
def fmt_name(s):
return (
" ".join(w.capitalize() for w in s.split("-"))
.replace("Gpt", "GPT")
.replace("ai", "AI")
)
def make_model_table(df, models):
df["task_metric"] = df["task"] + "_" + df["metric"]
df = df.drop(columns=["task", "metric"])
task_metrics = df["task_metric"].unique()
df = df.pivot(index="model", columns="task_metric", values="score").fillna(0)
df["average"] = df[task_metrics].mean(axis=1)
df = df.sort_values(by="average", ascending=False).reset_index()
for row in [*task_metrics, "average"]:
df[row] = df[row].round(2)
df = pd.merge(df, models, left_on="model", right_on="id", how="left")
df["creation_date"] = df["creation_date"].dt.strftime("%Y-%m-%d")
df["provider"] = df["model"].str.split("/").str[0].apply(fmt_name)
df["model"] = df["model"].str.split("/").str[1].apply(fmt_name)
df["rank"] = df.index + 1
df = df[
[
"rank",
"provider",
"model",
"hf_id",
"creation_date",
"size",
"type",
"license",
"average",
*task_metrics,
]
]
return df
def make_language_table(df, languages):
df["task_metric"] = df["task"] + "_" + df["metric"]
df = df.drop(columns=["task", "metric"])
task_metrics = df["task_metric"].unique()
df = (
df.pivot(index="bcp_47", columns="task_metric", values="score")
.fillna(0)
.reset_index()
)
df["average"] = df[task_metrics].mean(axis=1)
for row in [*task_metrics, "average"]:
df[row] = df[row].round(2)
df = pd.merge(languages, df, on="bcp_47", how="outer")
df = df.sort_values(by="speakers", ascending=False)
df = df[
[
"bcp_47",
"language_name",
"autonym",
"speakers",
"family",
"average",
"in_benchmark",
*task_metrics,
]
]
return df |