add trust&safety table

app.py

@@ -13,9 +13,12 @@ from src.display.utils import (  # EVAL_TYPES,; WeightType,; BENCHMARK_COLS,; EV
     COLS,
     COST_COLS,
     COST_TYPES,
+    TS_COLS,
+    TS_TYPES,
     TYPES,
     AutoEvalColumn,
     CostEvalColumn,
+    TSEvalColumn,
     fields,
 )
 
@@ -23,11 +26,12 @@ from src.display.utils import (  # EVAL_TYPES,; WeightType,; BENCHMARK_COLS,; EV
 from src.envs import CRM_RESULTS_PATH
 from src.populate import get_leaderboard_df_crm
 
-original_df, cost_df = get_leaderboard_df_crm(CRM_RESULTS_PATH, COLS, COST_COLS)
+original_df, cost_df, ts_df = get_leaderboard_df_crm(CRM_RESULTS_PATH, COLS, COST_COLS)
 
 # raw_data, original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
 leaderboard_df = original_df.copy()
 leaderboard_cost_df = cost_df.copy()
+leaderboard_ts_df = ts_df.copy()
 # leaderboard_df = leaderboard_df.style.format({"accuracy_metric_average": "{0:.2f}"})
 
 
@@ -70,6 +74,18 @@ def update_cost_table(
     return df.style.map(highlight_cost_band_low, props="background-color: #b3d5a4")
 
 
+def update_ts_table(
+    hidden_df: pd.DataFrame,
+    columns: list,
+    llm_query: list,
+    llm_provider_query: list,
+):
+    filtered_df = filter_llm_func(hidden_df, llm_query)
+    filtered_df = filter_llm_provider_func(filtered_df, llm_provider_query)
+    df = select_columns_ts_table(filtered_df, columns)
+    return df
+
+
 # def highlight_cols(x):
 #     df = x.copy()
 #     df.loc[:, :] = "color: black"
@@ -126,6 +142,21 @@ def init_leaderboard_cost_df(
     )
 
 
+def init_leaderboard_ts_df(
+    leaderboard_df: pd.DataFrame,
+    columns: list,
+    llm_query: list,
+    llm_provider_query: list,
+):
+
+    return update_ts_table(
+        leaderboard_df,
+        columns,
+        llm_query,
+        llm_provider_query,
+    )
+
+
 def filter_accuracy_method_func(df: pd.DataFrame, accuracy_method_query: str) -> pd.DataFrame:
     return df[df["Accuracy Method"] == accuracy_method_query]
 
@@ -177,6 +208,14 @@ def select_columns_cost_table(df: pd.DataFrame, columns: list) -> pd.DataFrame:
     return filtered_df
 
 
+def select_columns_ts_table(df: pd.DataFrame, columns: list) -> pd.DataFrame:
+    always_here_cols = [
+        TSEvalColumn.model.name,
+    ]
+    filtered_df = df[always_here_cols + [c for c in TS_COLS if c in df.columns and c in columns]]
+    return filtered_df
+
+
 demo = gr.Blocks(css=custom_css)
 with demo:
     gr.HTML(TITLE)
@@ -461,8 +500,77 @@ with demo:
                     leaderboard_table,
                     queue=True,
                 )
+        with gr.TabItem("🏅 Trust & Safety", elem_id="llm-benchmark-tab-table", id=2):
+            with gr.Row():
+                with gr.Column():
+                    with gr.Row():
+                        shown_columns = gr.CheckboxGroup(
+                            choices=[c.name for c in fields(TSEvalColumn) if not c.hidden and not c.never_hidden],
+                            value=[
+                                c.name
+                                for c in fields(TSEvalColumn)
+                                if c.displayed_by_default and not c.hidden and not c.never_hidden
+                            ],
+                            label="Select columns to show",
+                            elem_id="column-select",
+                            interactive=True,
+                        )
+            with gr.Row():
+                with gr.Column():
+                    filter_llm = gr.CheckboxGroup(
+                        choices=list(ts_df["Model Name"].unique()),
+                        value=list(ts_df["Model Name"].unique()),
+                        label="Model Name",
+                        info="",
+                        interactive=True,
+                    )
+                with gr.Column():
+                    filter_llm_provider = gr.CheckboxGroup(
+                        choices=list(ts_df["LLM Provider"].unique()),
+                        value=list(ts_df["LLM Provider"].unique()),
+                        label="LLM Provider",
+                        info="",
+                        interactive=True,
+                    )
 
-        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
+            leaderboard_table = gr.components.Dataframe(
+                value=init_leaderboard_ts_df(
+                    leaderboard_ts_df,
+                    shown_columns.value,
+                    filter_llm.value,
+                    filter_llm_provider.value,
+                ),
+                headers=[c.name for c in fields(TSEvalColumn) if c.never_hidden] + shown_columns.value,
+                datatype=TS_TYPES,
+                elem_id="leaderboard-table",
+                interactive=False,
+                visible=True,
+            )
+
+            hidden_leaderboard_table_for_search = gr.components.Dataframe(
+                value=ts_df[TS_COLS],
+                headers=TS_COLS,
+                datatype=TS_TYPES,
+                visible=False,
+            )
+
+            for selector in [
+                shown_columns,
+                filter_llm,
+                filter_llm_provider,
+            ]:
+                selector.change(
+                    update_ts_table,
+                    [
+                        hidden_leaderboard_table_for_search,
+                        shown_columns,
+                        filter_llm,
+                        filter_llm_provider,
+                    ],
+                    leaderboard_table,
+                    queue=True,
+                )
+        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=3):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
 
     with gr.Row():

crm-results/hf_leaderboard_ts.csv

@@ -0,0 +1,19 @@
+Model Name,Truthfulness,Safety,Privacy Zero-Shot Match Avoidance,Privacy Zero-Shot Reveal Avoidance,Privacy Five-Shot Match Avoidance,Privacy Five-Shot Reveal Avoidance,CRM Gender Bias,CRM Company Bias,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,Truthfulness,,
+GPT4-o,91%,69%,100%,94%,90%,51%,-,-,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,91%,,
+GPT 4 Turbo,94%,74%,100%,97%,86%,74%,-,-,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,79%,0.813,
+GPT 3.5 Turbo,45%,59%,100%,13%,36%,2%,-,-,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,45%,,0.708 (ChatGPT)
+AI21 Jamba-Instruct,68%,65%,100%,100%,90%,81%,-,-,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,68%,,
+Cohere Command Text,59%,54%,100%,84%,78%,40%,-,-,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,59%,,
+Claude 3 Haiku,86%,80%,100%,98%,95%,40%,-,-,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,86%,,
+Gemini  Pro 1,87%,74%,100%,92%,81%,48%,-,-,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,87%,,
+SF-TextBase 70B,98%,63%,100%,90%,54%,8%,-,-,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,98%,,
+SF-TextSum,82%,51%,100%,89%,87%,27%,-,-,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,82%,,
+XGen 22B,52%,52%,100%,56%,81%,51%,-,-,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,52%,,
+SF-TextBase 7B,82%,60%,100%,83%,69%,27%,-,-,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,82%,,
+Mistral 7B,32%,42%,100%,97%,92%,82%,-,-,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,32%,0.426,
+Mixtral 8x7B,89%,59%,100%,97%,71%,55%,-,-,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,89%,0.88,
+LLaMA 3 8B,96%,76%,100%,99%,92%,85%,-,-,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,96%,0.598,
+LLaMA 3 70B,98%,74%,100%,98%,83%,75%,-,-,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,98%,0.962,
+Gemini  Pro 1.5,98%,81%,100%,97%,87%,69%,-,-,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,98%,,
+Claude 3 Opus,94%,81%,100%,96%,80%,56%,-,-,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,94%,,
+Cohere Command R+,84%,56%,100%,97%,76%,45%,-,-,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,84%,,

src/display/utils.py

@@ -73,6 +73,13 @@ cost_eval_column_dict.append(["cost_band", ColumnContent, ColumnContent("Cost Ba
 CostEvalColumn = make_dataclass("CostEvalColumn", cost_eval_column_dict, frozen=True)
 
 # Trust & Safety metrics
+ts_eval_column_dict = []
+# Init
+ts_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model Name", "markdown", True, never_hidden=True)])
+ts_eval_column_dict.append(["model_provider", ColumnContent, ColumnContent("LLM Provider", "markdown", True)])
+ts_eval_column_dict.append(["truthfulness", ColumnContent, ColumnContent("Truthfulness", "markdown", True)])
+ts_eval_column_dict.append(["safety", ColumnContent, ColumnContent("Safety", "markdown", True)])
+TSEvalColumn = make_dataclass("TSEvalColumn", ts_eval_column_dict, frozen=True)
 
 
 # Scores
@@ -173,6 +180,9 @@ TYPES_LITE = [c.type for c in fields(AutoEvalColumn) if c.displayed_by_default a
 COST_COLS = [c.name for c in fields(CostEvalColumn) if not c.hidden]
 COST_TYPES = [c.type for c in fields(CostEvalColumn) if not c.hidden]
 
+TS_COLS = [c.name for c in fields(TSEvalColumn) if not c.hidden]
+TS_TYPES = [c.type for c in fields(TSEvalColumn) if not c.hidden]
+
 EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
 EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
 

src/populate.py

@@ -30,7 +30,12 @@ def get_leaderboard_df_crm(
     leaderboard_cost_df = leaderboard_cost_df.join(ref_df.set_index("Model Name"), on="Model Name")
     leaderboard_cost_df["LLM Provider"] = leaderboard_cost_df["LLM Provider"].fillna("Google")
     leaderboard_cost_df = leaderboard_cost_df[cost_cols].round(decimals=2)
-    return leaderboard_accuracy_df, leaderboard_cost_df
+
+    leaderboard_ts_df = pd.read_csv(os.path.join(crm_results_path, "hf_leaderboard_ts.csv"))
+    leaderboard_ts_df = leaderboard_ts_df[~leaderboard_ts_df["Model Name"].isin(sf_finetuned_models)]
+    leaderboard_ts_df = leaderboard_ts_df.join(ref_df.set_index("Model Name"), on="Model Name")
+
+    return leaderboard_accuracy_df, leaderboard_cost_df, leaderboard_ts_df
 
 
 # def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame: