File size: 7,615 Bytes
f740333 b0a6f33 6db6faf 8c99d65 b0a6f33 a66535a 6db6faf b0a6f33 f740333 b0a6f33 f740333 a66535a f740333 a66535a f740333 6db6faf f740333 a66535a f740333 a66535a f740333 a66535a 50faa2d 6db6faf 19c9463 6db6faf 8c99d65 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 |
import pandas as pd
import gradio as gr
import matplotlib.pyplot as plt
import seaborn as sns
from typing import Tuple
import plotly.express as px
import numpy as np
VOLUME_FACTOR_REGULARIZATION = 0.5
UNSCALED_WEIGHTED_ACCURACY_INTERVAL = (-0.5, 100.5)
SCALED_WEIGHTED_ACCURACY_INTERVAL = (0, 1)
# tools palette as dictionary
tools_palette = {
"prediction-request-reasoning": "darkorchid",
"claude-prediction-offline": "rebeccapurple",
"prediction-request-reasoning-claude": "slateblue",
"prediction-request-rag-claude": "steelblue",
"prediction-online": "darkcyan",
"prediction-offline": "mediumaquamarine",
"claude-prediction-online": "mediumseagreen",
"prediction-online-sme": "yellowgreen",
"prediction-url-cot-claude": "gold",
"prediction-offline-sme": "orange",
"prediction-request-rag": "chocolate",
}
HEIGHT = 400
WIDTH = 1100
def scale_value(
value: float,
min_max_bounds: Tuple[float, float],
scale_bounds: Tuple[float, float] = (0, 1),
) -> float:
"""Perform min-max scaling on a value."""
min_, max_ = min_max_bounds
current_range = max_ - min_
# normalize between 0-1
std = (value - min_) / current_range
# scale between min_bound and max_bound
min_bound, max_bound = scale_bounds
target_range = max_bound - min_bound
return std * target_range + min_bound
def get_weighted_accuracy(row, global_requests: int):
"""Function to compute the weighted accuracy of a tool"""
return scale_value(
(
row["tool_accuracy"]
+ (row["total_requests"] / global_requests) * VOLUME_FACTOR_REGULARIZATION
),
UNSCALED_WEIGHTED_ACCURACY_INTERVAL,
SCALED_WEIGHTED_ACCURACY_INTERVAL,
)
def compute_weighted_accuracy(tools_accuracy: pd.DataFrame):
global_requests = tools_accuracy.total_requests.sum()
tools_accuracy["weighted_accuracy"] = tools_accuracy.apply(
lambda x: get_weighted_accuracy(x, global_requests), axis=1
)
return tools_accuracy
def plot_tools_accuracy_graph(tools_accuracy_info: pd.DataFrame):
tools_accuracy_info = tools_accuracy_info.sort_values(
by="tool_accuracy", ascending=False
)
plt.figure(figsize=(25, 10))
plot = sns.barplot(
tools_accuracy_info,
x="tool_accuracy",
y="tool",
hue="tool",
dodge=False,
palette=tools_palette,
)
plt.xlabel("Mech tool_accuracy (%)", fontsize=20)
plt.ylabel("tool", fontsize=20)
plt.tick_params(axis="y", labelsize=12)
return gr.Plot(value=plot.get_figure())
def plot_tools_accuracy_rotated_graph(tools_accuracy_info: pd.DataFrame):
tools_accuracy_info = tools_accuracy_info.sort_values(
by="tool_accuracy", ascending=False
)
fig = px.bar(
tools_accuracy_info,
x="tool",
y="tool_accuracy",
color="tool",
color_discrete_map=tools_palette,
)
fig.update_layout(
xaxis_title="Tool",
yaxis_title="Mech tool_accuracy (%)",
)
fig.update_layout(width=WIDTH, height=HEIGHT)
# fig.update_xaxes(tickangle=45)
fig.update_xaxes(showticklabels=False)
return gr.Plot(
value=fig,
)
def plot_tools_weighted_accuracy_graph(tools_accuracy_info: pd.DataFrame):
tools_accuracy_info = tools_accuracy_info.sort_values(
by="weighted_accuracy", ascending=False
)
# Create the Seaborn bar plot
# sns.set_theme(palette="viridis")
plt.figure(figsize=(25, 10))
plot = sns.barplot(
tools_accuracy_info,
x="weighted_accuracy",
y="tool",
hue="tool",
dodge=False,
palette=tools_palette,
)
plt.xlabel("Weighted accuracy metric", fontsize=20)
plt.ylabel("tool", fontsize=20)
plt.tick_params(axis="y", labelsize=12)
return gr.Plot(value=plot.get_figure())
def plot_tools_weighted_accuracy_rotated_graph(
tools_accuracy_info: pd.DataFrame,
) -> gr.Plot:
tools_accuracy_info = tools_accuracy_info.sort_values(
by="weighted_accuracy", ascending=False
)
fig = px.bar(
tools_accuracy_info,
x="tool",
y="weighted_accuracy",
color="tool",
color_discrete_map=tools_palette,
)
fig.update_layout(
xaxis_title="Tool",
yaxis_title="Weighted accuracy metric",
)
fig.update_layout(width=WIDTH, height=HEIGHT)
# fig.update_xaxes(tickangle=45)
fig.update_xaxes(showticklabels=False)
return gr.Plot(
value=fig,
)
def plot_mech_requests_topthree_tools(
daily_mech_requests: pd.DataFrame, tools_accuracy_info: pd.DataFrame, top: int
):
"""Function to plot the percentage of mech requests from the top three tools"""
# Get the top three tools
top_tools = tools_accuracy_info.sort_values(
by="tool_accuracy", ascending=False
).head(top)
top_tools = top_tools.tool.tolist()
# Filter the daily mech requests for the top three tools
daily_mech_requests_local_copy = daily_mech_requests.copy()
daily_mech_requests_local_copy = daily_mech_requests_local_copy[
daily_mech_requests_local_copy["market_creator"] == "all"
]
# Get the daily total of mech requests no matter the tool
total_daily_mech_requests = (
daily_mech_requests_local_copy.groupby(["request_date"])
.agg({"total_mech_requests": "sum"})
.reset_index()
)
print("total_daily_mech_requests", total_daily_mech_requests.head())
total_daily_mech_requests.rename(
columns={"total_mech_requests": "total_daily_mech_requests"},
inplace=True,
)
# Merge the total daily mech requests with the daily mech requests
daily_mech_requests_local_copy = pd.merge(
daily_mech_requests_local_copy,
total_daily_mech_requests,
on="request_date",
how="left",
)
# Compute the percentage of mech requests for each tool
daily_mech_requests_local_copy["percentage"] = (
daily_mech_requests_local_copy["total_mech_requests"]
/ daily_mech_requests_local_copy["total_daily_mech_requests"]
) * 100
daily_mech_requests_local_copy = daily_mech_requests_local_copy[
daily_mech_requests_local_copy.tool.isin(top_tools)
]
# remove the market_creator column
daily_mech_requests_local_copy = daily_mech_requests_local_copy.drop(
columns=["market_creator"]
)
# Create a pivot table to get the total mech requests per tool
pivoted = daily_mech_requests_local_copy.pivot(
index="request_date", columns="tool", values="percentage"
)
# Sort the columns for each row independently
sorted_values = np.sort(pivoted.values, axis=1)[
:, ::-1
] # sort and reverse (descending)
sorted_columns = np.argsort(pivoted.values, axis=1)[:, ::-1] # get sorting indices
sorted_df = pd.DataFrame(
sorted_values,
index=pivoted.index,
columns=[
pivoted.columns[i] for i in sorted_columns[0]
], # use first row's order
)
sorted_long = sorted_df.reset_index().melt(
id_vars=["request_date"], var_name="tool", value_name="percentage"
)
fig = px.bar(
sorted_long,
x="request_date",
y="percentage",
color="tool",
color_discrete_map=tools_palette,
)
fig.update_layout(
xaxis_title="Day of the request",
yaxis_title="Percentage of Total daily mech requests",
legend=dict(yanchor="top", y=0.5),
)
fig.update_layout(width=WIDTH, height=HEIGHT)
return gr.Plot(value=fig)
|