Spaces:
Running
Running
Commit
·
cdf41e7
1
Parent(s):
a189046
transpose energy
Browse files
app.py
CHANGED
|
@@ -37,7 +37,7 @@ ALL_COLUMNS_MAPPING = {
|
|
| 37 |
#
|
| 38 |
"generate.peak_memory(MB)": "Memory (MB) ⬇️",
|
| 39 |
"generate.throughput(tokens/s)": "Throughput (tokens/s) ⬆️",
|
| 40 |
-
"generate.energy_consumption(kWh
|
| 41 |
"best_score": "Best Score (%) ⬆️",
|
| 42 |
#
|
| 43 |
"best_scored_model": "Best Scored LLM 🏆",
|
|
@@ -84,8 +84,12 @@ def get_benchmark_df(benchmark="Succeeded-1xA100-80GB"):
|
|
| 84 |
merged_df = benchmark_df.merge(
|
| 85 |
clusters_df, left_on="model", right_on="best_scored_model"
|
| 86 |
)
|
| 87 |
-
#
|
| 88 |
-
merged_df["generate.energy_consumption(kWh
|
|
|
|
|
|
|
|
|
|
|
|
|
| 89 |
|
| 90 |
# add optimizations
|
| 91 |
merged_df["optimizations"] = merged_df["backend.bettertransformer"].apply(
|
|
@@ -95,6 +99,7 @@ def get_benchmark_df(benchmark="Succeeded-1xA100-80GB"):
|
|
| 95 |
merged_df["quantization"] = merged_df["backend.quantization_strategy"].apply(
|
| 96 |
lambda x: "BnB.4bit" if x == "bnb" else ("GPTQ.4bit" if x == "gptq" else "None")
|
| 97 |
)
|
|
|
|
| 98 |
# # distance to 100% score
|
| 99 |
# score_distance = 100 - merged_df["best_score"]
|
| 100 |
# # distance to 0s latency
|
|
|
|
| 37 |
#
|
| 38 |
"generate.peak_memory(MB)": "Memory (MB) ⬇️",
|
| 39 |
"generate.throughput(tokens/s)": "Throughput (tokens/s) ⬆️",
|
| 40 |
+
"generate.energy_consumption(tokens/kWh)": "Energy (tokens/kWh) ⬇️",
|
| 41 |
"best_score": "Best Score (%) ⬆️",
|
| 42 |
#
|
| 43 |
"best_scored_model": "Best Scored LLM 🏆",
|
|
|
|
| 84 |
merged_df = benchmark_df.merge(
|
| 85 |
clusters_df, left_on="model", right_on="best_scored_model"
|
| 86 |
)
|
| 87 |
+
# transpose energy consumption
|
| 88 |
+
merged_df["generate.energy_consumption(tokens/kWh)"] = 1 / merged_df[
|
| 89 |
+
"generate.energy_consumption(kWh/token)"
|
| 90 |
+
].fillna(1)
|
| 91 |
+
# fix nan values
|
| 92 |
+
merged_df[merged_df["generate.energy_consumption(tokens/kWh)"] == 1] = "N/A"
|
| 93 |
|
| 94 |
# add optimizations
|
| 95 |
merged_df["optimizations"] = merged_df["backend.bettertransformer"].apply(
|
|
|
|
| 99 |
merged_df["quantization"] = merged_df["backend.quantization_strategy"].apply(
|
| 100 |
lambda x: "BnB.4bit" if x == "bnb" else ("GPTQ.4bit" if x == "gptq" else "None")
|
| 101 |
)
|
| 102 |
+
|
| 103 |
# # distance to 100% score
|
| 104 |
# score_distance = 100 - merged_df["best_score"]
|
| 105 |
# # distance to 0s latency
|