Transfer IndieLabel demo version

README.txt

@@ -0,0 +1,47 @@
+# IndieLabel
+
+## Installation / Setup
+- Activate your virtual environment (tested with Python 3.8)
+- Install requirements:
+    ```
+    $ pip install -r requirements.txt
+    ```
+
+- Start the Flask server:
+    ```
+    $ python server.py
+    ```
+
+- Concurrently build and run the Svelte app in another terminal session:
+    ```
+    $ cd indie_label_svelte/
+    $ HOST=0.0.0.0 PORT=5000 npm run dev autobuild
+    ```
+
+- You can now visit `localhost:5001` to view the IndieLabel app!
+
+## Main paths
+Here's a summary of the relevant pages used for each participant in our study. For easier setup and navigation, we added URL parameters for the different labeling and auditing modes used in the study.
+- Participant's page: `localhost:5001/?user=<USER_NAME>`
+- Labeling task pages:
+    - Group-based model (group selection): `localhost:5001/?user=<USER_NAME>&tab=labeling&label_mode=3`
+    - End-user model (data labeling): `localhost:5001/?user=<USER_NAME>&tab=labeling&label_mode=0`
+- Tutorial page: `localhost:5001/?user=DemoUser&scaffold=tutorial `
+- Auditing task pages:
+    - Fixed audit, end-user model: `localhost:5001/?user=<USER_NAME>&scaffold=personal`
+    - Fixed audit, group-based model: `localhost:5001/?user=<USER_NAME>&scaffold=personal_group`
+    - Free-form audit, end-user model: `localhost:5001/?user=<USER_NAME>&scaffold=prompts` 
+
+## Setting up a new model
+- Set up your username and navigate to the laveling page 
+    - Using a direct URL parameter
+        - Go to `localhost:5001/?user=<USER_NAME>&tab=labeling&label_mode=0`, where in place of `<USER_NAME>`, you've entered your desired username
+    - Using the UI
+        - Go to the Labeling page and ensure that the "Create a new model" mode is selected.
+        - Select the User button on the top menu and enter your desired username.
+
+- Label all of the examples in the table
+- When you're done, click the "Get Number of Comments Labeled" button to verify the number of comments that have been labeled. If there are at least 40 comments labeled, the "Train Model" button will be enabled.
+- Click on the "Train Model" button and wait for the model to train.
+- Then, go to the Auditing page and use your new model.
+    - To view the different auditing modes that we provided for our evaluation task, please refer to the URL paths listed in the "Auditing task pages" section above.

audit_utils.py

@@ -0,0 +1,1569 @@
+"""
+UTILS FILE
+"""
+import random
+import json
+import numpy as np
+import matplotlib.pyplot as plt
+import pandas as pd
+import pickle
+import os
+import mne
+
+from surprise import Dataset, Reader, SVD, accuracy, KNNBasic, KNNWithMeans, KNNWithZScore
+from surprise.model_selection import train_test_split
+from sklearn.utils import resample
+from sklearn.metrics import mean_absolute_error
+from sklearn.metrics import mean_squared_error
+from scipy import stats
+import math
+import altair as alt
+import matplotlib.pyplot as plt
+import time
+from sentence_transformers import SentenceTransformer, util
+import torch
+from bertopic import BERTopic
+
+########################################
+# PRE-LOADING
+
+YOUR_COLOR = '#6CADFD'
+OTHER_USERS_COLOR = '#ccc'
+BINS = [0, 0.5, 1.5, 2.5, 3.5, 4]
+BIN_LABELS = ['0: Not at all toxic', '1: Slightly toxic', '2: Moderately toxic', '3: Very toxic', '4: Extremely toxic']
+TOXIC_THRESHOLD = 2.0
+
+alt.renderers.enable('altair_saver', fmts=['vega-lite', 'png'])
+
+# Data-loading
+module_dir = "./"
+perf_dir = f"data/perf/"
+
+# # TEMP reset
+# with open(os.path.join(module_dir, "./data/all_model_names.pkl"), "wb") as f:
+#     all_model_names = []
+#     pickle.dump(all_model_names, f)
+# with open(f"./data/users_to_models.pkl", "wb") as f:
+#     users_to_models = {}
+#     pickle.dump(users_to_models, f)
+
+
+with open(os.path.join(module_dir, "data/ids_to_comments.pkl"), "rb") as f:
+    ids_to_comments = pickle.load(f)
+with open(os.path.join(module_dir, "data/comments_to_ids.pkl"), "rb") as f:
+    comments_to_ids = pickle.load(f)
+
+all_model_names = sorted([name for name in os.listdir(os.path.join(perf_dir)) if os.path.isdir(os.path.join(perf_dir, name))])
+comments_grouped_full_topic_cat = pd.read_pickle("data/comments_grouped_full_topic_cat2_persp.pkl")
+sys_eval_df = pd.read_pickle(os.path.join(module_dir, "data/split_data/sys_eval_df.pkl"))
+train_df = pd.read_pickle(os.path.join(module_dir, "data/split_data/train_df.pkl"))
+train_df_ids = train_df["item_id"].unique().tolist()
+model_eval_df = pd.read_pickle(os.path.join(module_dir, "data/split_data/model_eval_df.pkl"))
+ratings_df_full = pd.read_pickle(os.path.join(module_dir, "data/ratings_df_full.pkl"))
+
+worker_info_df = pd.read_pickle("./data/worker_info_df.pkl")
+
+with open(f"./data/users_to_models.pkl", "rb") as f:
+    users_to_models = pickle.load(f)
+
+with open("data/perf_1000_topics.pkl", "rb") as f:
+    perf_1000_topics = pickle.load(f)
+with open("data/perf_1000_tox_cat.pkl", "rb") as f:
+    perf_1000_tox_cat = pickle.load(f)
+with open("data/perf_1000_tox_severity.pkl", "rb") as f:
+    perf_1000_tox_severity = pickle.load(f)
+with open("data/user_perf_metrics.pkl", "rb") as f:
+    user_perf_metrics = pickle.load(f)
+
+topic_ids = comments_grouped_full_topic_cat.topic_id
+topics = comments_grouped_full_topic_cat.topic
+topic_ids_to_topics = {topic_ids[i]: topics[i] for i in range(len(topic_ids))}
+topics_to_topic_ids = {topics[i]: topic_ids[i] for i in range(len(topic_ids))}
+unique_topics_ids = sorted(comments_grouped_full_topic_cat.topic_id.unique())
+unique_topics = [topic_ids_to_topics[topic_id] for topic_id in range(len(topic_ids_to_topics) - 1)]
+
+def get_toxic_threshold():
+    return TOXIC_THRESHOLD
+
+def get_all_model_names(user=None):
+    if (user is None) or (user not in users_to_models):
+        all_model_names = sorted([name for name in os.listdir(os.path.join(perf_dir)) if os.path.isdir(os.path.join(perf_dir, name))])
+        return all_model_names
+    else:
+        # Fetch the user's models
+        user_models = users_to_models[user]
+        user_models.sort()
+        return user_models
+
+def get_unique_topics():
+    return unique_topics
+
+def get_large_clusters(min_n):
+    counts_df = comments_grouped_full_topic_cat.groupby(by=["topic_id"]).size().reset_index(name='counts')
+    counts_df = counts_df[counts_df["counts"] >= min_n]
+    return [topic_ids_to_topics[t_id] for t_id in sorted(counts_df["topic_id"].tolist()[1:])]
+
+def get_ids_to_comments():
+    return ids_to_comments
+
+def get_workers_in_group(sel_gender, sel_race, sel_relig, sel_pol, sel_lgbtq):
+    df = worker_info_df.copy()
+    if sel_gender != "null":
+        df = df[df["gender"] == sel_gender]
+    if sel_relig != "null":
+        df = df[df["religion_important"] == sel_relig]
+    if sel_pol != "null":
+        df = df[df["political_affilation"] == sel_pol]
+    if sel_lgbtq != "null":
+        if sel_lgbtq == "LGBTQ+":
+            df = df[(df["lgbtq_status"] == "Homosexual") | (df["lgbtq_status"] == "Bisexual")]  
+        else:
+            df = df[df["lgbtq_status"] == "Heterosexual"]  
+    if sel_race != "":
+        df = df.dropna(subset=['race'])
+        for r in sel_race:
+            # Filter to rows with the indicated race
+            df = df[df["race"].str.contains(r)]
+    return df, len(df)
+
+readable_to_internal = {
+    "Mean Absolute Error (MAE)": "MAE",
+    "Root Mean Squared Error (RMSE)": "RMSE",
+    "Mean Squared Error (MSE)": "MSE",
+    "Average rating difference": "avg_diff",
+    "Topic": "topic",
+    "Toxicity Category": "toxicity_category",
+    "Toxicity Severity": "toxicity_severity",
+}
+internal_to_readable = {v: k for k, v in readable_to_internal.items()}
+
+# Embeddings for neighbor retrieval
+model_name = "paraphrase-MiniLM-L6-v2"
+model = SentenceTransformer(model_name)
+with open("./data/comments.pkl", "rb") as f:
+    comments = pickle.load(f)
+embeddings = torch.load("./data/embeddings/21_10_embeddings.pt")
+
+# Perspective API recalibration
+def recalib_v1(s):
+    # convert Perspective score to 0-4 toxicity score
+    # map 0 persp to 0 (not at all toxic); 0.5 persp to 1 (slightly toxic), 1.0 persp to 4 (extremely toxic)
+    if s < 0.5:
+        return (s * 2.)
+    else:
+        return ((s - 0.5) * 6.) + 1
+
+def recalib_v2(s):
+    # convert Perspective score to 0-4 toxicity score
+    # just 4x the perspective score 
+    return (s * 4.)
+
+comments_grouped_full_topic_cat["rating_avg_orig"] = comments_grouped_full_topic_cat["rating"]
+comments_grouped_full_topic_cat["rating"] = [recalib_v2(score) for score in comments_grouped_full_topic_cat["persp_score"].tolist()]
+
+def get_comments_grouped_full_topic_cat():
+    return comments_grouped_full_topic_cat
+
+########################################
+# General utils
+def get_metric_ind(metric):
+    if metric == "MAE":
+        ind = 0
+    elif metric == "MSE":
+        ind = 1
+    elif metric == "RMSE":
+        ind = 2
+    elif metric == "avg_diff":
+        ind = 3
+    return ind
+
+def my_bootstrap(vals, n_boot, alpha):
+    bs_samples = []
+    sample_size = len(vals)
+    for i in range(n_boot):
+        samp = resample(vals, n_samples=sample_size)
+        bs_samples.append(np.median(samp))
+        
+    p = ((1.0 - alpha) / 2.0) * 100
+    ci_low = np.percentile(bs_samples, p)
+    p = (alpha + ((1.0 - alpha) / 2.0)) * 100
+    ci_high = np.percentile(bs_samples, p)
+    return bs_samples, (ci_low, ci_high)
+
+########################################
+# GET_AUDIT utils
+def other_users_perf(perf_metrics, metric, user_metric, alpha=0.95, n_boot=501):
+    ind = get_metric_ind(metric)
+    
+    metric_vals = [metric_vals[ind] for metric_vals in perf_metrics.values()]
+    metric_avg = np.median(metric_vals)
+    
+    # Future: use provided sample to perform bootstrap sampling
+    ci_1 = mne.stats.bootstrap_confidence_interval(np.array(metric_vals), ci=alpha, n_bootstraps=n_boot, stat_fun="median")
+    
+    bs_samples, ci = my_bootstrap(metric_vals, n_boot, alpha)
+    
+    # Get user's percentile
+    percentile = stats.percentileofscore(bs_samples, user_metric)
+    
+    return metric_avg, ci, percentile, metric_vals
+
+def plot_metric_histogram(metric, user_metric, other_metric_vals, n_bins=10):
+    hist, bin_edges = np.histogram(other_metric_vals, bins=n_bins, density=False)
+    data = pd.DataFrame({
+        "bin_min": bin_edges[:-1],
+        "bin_max": bin_edges[1:],
+        "bin_count": hist,
+        "user_metric": [user_metric for i in range(len(hist))]
+    })
+    base = alt.Chart(data)
+
+    bar = base.mark_bar(color=OTHER_USERS_COLOR).encode(
+        x=alt.X("bin_min", bin="binned", title=internal_to_readable[metric]),
+        x2='bin_max',
+        y=alt.Y("bin_count", title="Number of users"),
+        tooltip=[
+            alt.Tooltip('bin_min', title=f'{metric} bin min', format=".2f"),
+            alt.Tooltip('bin_max', title=f'{metric} bin max', format=".2f"),
+            alt.Tooltip('bin_count', title=f'Number of OTHER users', format=","),
+        ]
+    )
+
+    rule = base.mark_rule(color=YOUR_COLOR).encode(
+        x = "mean(user_metric):Q",
+        size=alt.value(2),
+        tooltip=[
+            alt.Tooltip('mean(user_metric)', title=f'{metric} with YOUR labels', format=".2f"),
+        ]
+    )
+
+    return (bar + rule).interactive()
+
+def get_toxicity_severity_bins(perf_metric, user_df, other_dfs, bins=BINS, bin_labels=BIN_LABELS, ci=0.95, n_boot=501):
+    # Note: not using other_dfs anymore
+    y_user = []
+    y_other = []
+    used_bins = []
+    other_ci_low = []
+    other_ci_high = []
+    for severity_i in range(len(bin_labels)):
+        metric_others = [metrics[get_metric_ind(perf_metric)] for metrics in perf_1000_tox_severity[severity_i].values() if metrics[get_metric_ind(perf_metric)]]
+        ci_low, ci_high = mne.stats.bootstrap_confidence_interval(np.array(metric_others), ci=ci, n_bootstraps=n_boot, stat_fun='median')
+        metric_other = np.median(metric_others)
+        
+        cur_user_df = user_df[user_df["prediction_bin"] == severity_i]
+        y_true_user = cur_user_df.pred.to_numpy()  # user's label
+        y_pred = cur_user_df.rating_avg.to_numpy()  # system's label (avg)
+        
+        if len(y_true_user) > 0:
+            used_bins.append(bin_labels[severity_i])
+            metric_user = calc_metric_user(y_true_user, y_pred, perf_metric)
+            y_user.append(metric_user)
+            y_other.append(metric_other)
+            other_ci_low.append(ci_low)
+            other_ci_high.append(ci_high)
+            
+    return y_user, y_other, used_bins, other_ci_low, other_ci_high
+
+def get_topic_bins(perf_metric, user_df, other_dfs, n_topics, ci=0.95, n_boot=501):  
+    # Note: not using other_dfs anymore
+    y_user = []
+    y_other = []
+    used_bins = []
+    other_ci_low = []
+    other_ci_high = []
+    selected_topics = unique_topics_ids[1:(n_topics + 1)]
+    
+    for topic_id in selected_topics:
+        cur_topic = topic_ids_to_topics[topic_id]
+        metric_others = [metrics[get_metric_ind(perf_metric)] for metrics in perf_1000_topics[topic_id].values() if metrics[get_metric_ind(perf_metric)]]
+        ci_low, ci_high = mne.stats.bootstrap_confidence_interval(np.array(metric_others), ci=ci, n_bootstraps=n_boot, stat_fun='median')
+        metric_other = np.median(metric_others)
+        
+        cur_user_df = user_df[user_df["topic"] == cur_topic]
+        y_true_user = cur_user_df.pred.to_numpy()  # user's label
+        y_pred = cur_user_df.rating_avg.to_numpy()  # system's label (avg)
+        
+        if len(y_true_user) > 0:
+            used_bins.append(cur_topic)
+            metric_user = calc_metric_user(y_true_user, y_pred, perf_metric)
+            y_user.append(metric_user)
+            y_other.append(metric_other)
+            other_ci_low.append(ci_low)
+            other_ci_high.append(ci_high)
+            
+    return y_user, y_other, used_bins, other_ci_low, other_ci_high
+
+def calc_metric_user(y_true_user, y_pred, perf_metric):
+    if perf_metric == "MAE":
+        metric_user = mean_absolute_error(y_true_user, y_pred)
+
+    elif perf_metric == "MSE":
+        metric_user = mean_squared_error(y_true_user, y_pred)
+
+    elif perf_metric == "RMSE":            
+        metric_user = mean_squared_error(y_true_user, y_pred, squared=False)
+        
+    elif perf_metric == "avg_diff":
+        metric_user = np.mean(y_true_user - y_pred)
+    
+    return metric_user
+
+def get_toxicity_category_bins(perf_metric, user_df, other_dfs, threshold=0.5, ci=0.95, n_boot=501):
+    # Note: not using other_dfs anymore; threshold from pre-calculation is 0.5
+    cat_cols = ["is_profane_frac", "is_threat_frac", "is_identity_attack_frac", "is_insult_frac", "is_sexual_harassment_frac"]
+    cat_labels = ["Profanity", "Threats", "Identity Attacks", "Insults", "Sexual Harassment"]
+    y_user = []
+    y_other = []
+    used_bins = []
+    other_ci_low = []
+    other_ci_high = []
+    for i, cur_col_name in enumerate(cat_cols):
+        metric_others = [metrics[get_metric_ind(perf_metric)] for metrics in perf_1000_tox_cat[cur_col_name].values() if metrics[get_metric_ind(perf_metric)]]
+        ci_low, ci_high = mne.stats.bootstrap_confidence_interval(np.array(metric_others), ci=ci, n_bootstraps=n_boot, stat_fun='median')
+        metric_other = np.median(metric_others)
+        
+        # Filter to rows where a comment received an average label >= the provided threshold for the category
+        cur_user_df = user_df[user_df[cur_col_name] >= threshold]
+        y_true_user = cur_user_df.pred.to_numpy()  # user's label
+        y_pred = cur_user_df.rating_avg.to_numpy()  # system's label (avg)
+        
+        if len(y_true_user) > 0:
+            used_bins.append(cat_labels[i])
+            metric_user = calc_metric_user(y_true_user, y_pred, perf_metric)
+            y_user.append(metric_user)
+            y_other.append(metric_other)
+            other_ci_low.append(ci_low)
+            other_ci_high.append(ci_high)
+    
+    return y_user, y_other, used_bins, other_ci_low, other_ci_high
+
+def plot_class_cond_results(preds_df, breakdown_axis, perf_metric, other_ids, sort_bars, n_topics, worker_id="A"):
+    # Note: preds_df already has binned results
+    # Prepare dfs
+    user_df = preds_df[preds_df.user_id == worker_id].sort_values(by=["item_id"]).reset_index()
+    other_dfs = [preds_df[preds_df.user_id == other_id].sort_values(by=["item_id"]).reset_index() for other_id in other_ids]
+    
+    if breakdown_axis == "toxicity_severity":
+        y_user, y_other, used_bins, other_ci_low, other_ci_high = get_toxicity_severity_bins(perf_metric, user_df, other_dfs)
+    elif breakdown_axis == "topic":
+        y_user, y_other, used_bins, other_ci_low, other_ci_high = get_topic_bins(perf_metric, user_df, other_dfs, n_topics)
+    elif breakdown_axis == "toxicity_category":
+        y_user, y_other, used_bins, other_ci_low, other_ci_high = get_toxicity_category_bins(perf_metric, user_df, other_dfs)
+    
+    diffs = list(np.array(y_user) - np.array(y_other))
+        
+    # Generate bar chart
+    data = pd.DataFrame({
+        "metric_val": y_user + y_other,
+        "Labeler": ["You" for _ in range(len(y_user))] + ["Other users" for _ in range(len(y_user))],
+        "used_bins": used_bins + used_bins,
+        "diffs": diffs + diffs,
+        "lower_cis": y_user + other_ci_low,
+        "upper_cis": y_user + other_ci_high,
+    })
+
+    color_domain = ['You', 'Other users']
+    color_range = [YOUR_COLOR, OTHER_USERS_COLOR]
+    
+    base = alt.Chart()
+    chart_title=f"{internal_to_readable[breakdown_axis]} Results"
+    x_axis = alt.X("Labeler:O", sort=("You", "Other users"), title=None, axis=None)
+    y_axis = alt.Y("metric_val:Q", title=internal_to_readable[perf_metric])
+    if sort_bars:
+        col_content = alt.Column("used_bins:O", sort=alt.EncodingSortField(field="diffs", op="mean", order='descending'))
+    else:
+        col_content = alt.Column("used_bins:O")
+
+    if n_topics is not None and n_topics > 10:
+        # Change to horizontal bar chart
+        bar = base.mark_bar(lineBreak="_").encode(
+            y=x_axis,
+            x=y_axis,
+            color=alt.Color("Labeler:O", scale=alt.Scale(domain=color_domain, range=color_range)),
+            tooltip=[
+                alt.Tooltip('Labeler:O', title='Labeler'),
+                alt.Tooltip('metric_val:Q', title=perf_metric, format=".3f"),
+            ]
+        )
+        error_bars = base.mark_errorbar().encode(
+            y=x_axis,
+            x = alt.X("lower_cis:Q", title=internal_to_readable[perf_metric]),
+            x2 = alt.X2("upper_cis:Q", title=None),
+            tooltip=[
+              alt.Tooltip('lower_cis:Q', title='Lower CI', format=".3f"),
+              alt.Tooltip('upper_cis:Q', title='Upper CI', format=".3f"),
+            ]
+        )
+        combined = alt.layer(
+            bar, error_bars, data=data
+        ).facet(
+            row=col_content
+        ).properties(
+            title=chart_title,
+        ).interactive()
+    else:
+        bar = base.mark_bar(lineBreak="_").encode(
+            x=x_axis,
+            y=y_axis,
+            color=alt.Color("Labeler:O", scale=alt.Scale(domain=color_domain, range=color_range)),
+            tooltip=[
+                alt.Tooltip('Labeler:O', title='Labeler'),
+                alt.Tooltip('metric_val:Q', title=perf_metric, format=".3f"),
+            ]
+        )
+        error_bars = base.mark_errorbar().encode(
+            x=x_axis,
+            y = alt.Y("lower_cis:Q", title=internal_to_readable[perf_metric]),
+            y2 = alt.Y2("upper_cis:Q", title=None),
+            tooltip=[
+              alt.Tooltip('lower_cis:Q', title='Lower CI', format=".3f"),
+              alt.Tooltip('upper_cis:Q', title='Upper CI', format=".3f"),
+            ]
+        )
+        combined = alt.layer(
+            bar, error_bars, data=data
+        ).facet(
+            column=col_content
+        ).properties(
+            title=chart_title,
+        ).interactive()
+
+    return combined
+
+def show_overall_perf(variant, error_type, cur_user, threshold=TOXIC_THRESHOLD, breakdown_axis=None, topic_vis_method="median"):
+    # Your perf (calculate using model and testset)
+    breakdown_axis = readable_to_internal[breakdown_axis]
+    
+    if breakdown_axis is not None:
+        with open(os.path.join(module_dir, f"data/preds_dfs/{variant}.pkl"), "rb") as f:
+            preds_df = pickle.load(f)
+
+        # Read from file
+        chart_dir = "./data/charts"
+        chart_file = os.path.join(chart_dir, f"{cur_user}_{variant}.pkl")
+        if os.path.isfile(chart_file):
+            with open(chart_file, "r") as f:
+                topic_overview_plot_json = json.load(f)
+        else:
+            preds_df_mod = preds_df.merge(comments_grouped_full_topic_cat, on="item_id", how="left", suffixes=('_', '_avg'))
+            if topic_vis_method == "median":
+                preds_df_mod_grp = preds_df_mod.groupby(["topic_", "user_id"]).median()
+            elif topic_vis_method == "mean":
+                preds_df_mod_grp = preds_df_mod.groupby(["topic_", "user_id"]).mean()
+            topic_overview_plot_json = plot_overall_vis(preds_df=preds_df_mod_grp, n_topics=200, threshold=threshold, error_type=error_type, cur_user=cur_user, cur_model=variant)
+
+    return {
+        "topic_overview_plot_json": json.loads(topic_overview_plot_json),
+    }
+
+########################################
+# GET_CLUSTER_RESULTS utils
+def get_overall_perf3(preds_df, perf_metric, other_ids, worker_id="A"):    
+    # Prepare dataset to calculate performance
+    # Note: true is user and pred is system
+    y_true = preds_df[preds_df["user_id"] == worker_id].pred.to_numpy()
+    y_pred_user = preds_df[preds_df["user_id"] == worker_id].rating_avg.to_numpy()
+    
+    y_true_others = y_pred_others = [preds_df[preds_df["user_id"] == other_id].pred.to_numpy() for other_id in other_ids]
+    y_pred_others = [preds_df[preds_df["user_id"] == other_id].rating_avg.to_numpy() for other_id in other_ids]
+    
+    # Get performance for user's model and for other users
+    if perf_metric == "MAE":
+        user_perf = mean_absolute_error(y_true, y_pred_user)
+        other_perfs = [mean_absolute_error(y_true_others[i], y_pred_others[i]) for i in range(len(y_true_others))]
+    elif perf_metric == "MSE":
+        user_perf = mean_squared_error(y_true, y_pred_user)
+        other_perfs = [mean_squared_error(y_true_others[i], y_pred_others[i]) for i in range(len(y_true_others))]
+    elif perf_metric == "RMSE":
+        user_perf = mean_squared_error(y_true, y_pred_user, squared=False)
+        other_perfs = [mean_squared_error(y_true_others[i], y_pred_others[i], squared=False) for i in range(len(y_true_others))]
+    elif perf_metric == "avg_diff":
+        user_perf = np.mean(y_true - y_pred_user)
+        other_perfs = [np.mean(y_true_others[i] - y_pred_others[i]) for i in range(len(y_true_others))]
+    
+    other_perf = np.mean(other_perfs)  # average across all other users
+    return user_perf, other_perf
+
+def style_color_difference(row):
+    full_opacity_diff = 3.
+    pred_user_col = "Your predicted rating"
+    pred_other_col = "Other users' predicted rating"
+    pred_system_col = "Status-quo system rating"
+    diff_user = row[pred_user_col] - row[pred_system_col]
+    diff_other = row[pred_other_col] - row[pred_system_col]
+    red = "234, 133, 125"
+    green = "142, 205, 162"
+    bkgd_user = green if diff_user < 0 else red  # red if more toxic; green if less toxic
+    opac_user = min(abs(diff_user / full_opacity_diff), 1.)
+    bkgd_other = green if diff_other < 0 else red  # red if more toxic; green if less toxic
+    opac_other = min(abs(diff_other / full_opacity_diff), 1.)
+    return ["", f"background-color: rgba({bkgd_user}, {opac_user});", f"background-color: rgba({bkgd_other}, {opac_other});", "", ""]
+
+def display_examples_cluster(preds_df, other_ids, num_examples, sort_ascending, worker_id="A"):
+    user_df = preds_df[preds_df.user_id == worker_id].sort_values(by=["item_id"]).reset_index()
+    others_df = preds_df[preds_df.user_id == other_ids[0]]
+    for i in range(1, len(other_ids)):
+        others_df.append(preds_df[preds_df.user_id == other_ids[i]])
+        others_df.groupby(["item_id"]).mean()
+    others_df = others_df.sort_values(by=["item_id"]).reset_index()
+    
+    df = pd.merge(user_df, others_df, on="item_id", how="left", suffixes=('_user', '_other'))
+    df["Comment"] = df["comment_user"]
+    df["Your predicted rating"] = df["pred_user"]
+    df["Other users' predicted rating"] = df["pred_other"]
+    df["Status-quo system rating"] = df["rating_avg_user"]
+    df["Status-quo system std dev"] = df["rating_stddev_user"]
+    df = df[["Comment", "Your predicted rating", "Other users' predicted rating", "Status-quo system rating", "Status-quo system std dev"]]
+    
+    # Add styling
+    df = df.sort_values(by=['Status-quo system std dev'], ascending=sort_ascending)
+    n_to_sample = np.min([num_examples, len(df)])
+    df = df.sample(n=n_to_sample).reset_index(drop=True)
+    return df.style.apply(style_color_difference, axis=1).render()
+
+def calc_odds_ratio(df, comparison_group, toxic_threshold=1.5, worker_id="A", debug=False, smoothing_factor=1):
+    if comparison_group == "status_quo":
+        other_pred_col = "rating_avg"
+        # Get unique comments, but fetch average labeler rating
+        num_toxic_other = len(df[(df.user_id == "A") & (df[other_pred_col] >= toxic_threshold)]) + smoothing_factor
+        num_nontoxic_other = len(df[(df.user_id == "A") & (df[other_pred_col] < toxic_threshold)]) + smoothing_factor
+    elif comparison_group == "other_users":
+        other_pred_col = "pred"
+        num_toxic_other = len(df[(df.user_id != "A") & (df[other_pred_col] >= toxic_threshold)]) + smoothing_factor
+        num_nontoxic_other = len(df[(df.user_id != "A") & (df[other_pred_col] < toxic_threshold)]) + smoothing_factor
+        
+    num_toxic_user = len(df[(df.user_id == "A") & (df.pred >= toxic_threshold)]) + smoothing_factor
+    num_nontoxic_user = len(df[(df.user_id == "A") & (df.pred < toxic_threshold)]) + smoothing_factor
+    
+    toxic_ratio = num_toxic_user / num_toxic_other
+    nontoxic_ratio = num_nontoxic_user / num_nontoxic_other
+    odds_ratio = toxic_ratio / nontoxic_ratio
+    
+    if debug:
+        print(f"Odds ratio: {odds_ratio}")
+        print(f"num_toxic_user: {num_toxic_user}, num_nontoxic_user: {num_nontoxic_user}")
+        print(f"num_toxic_other: {num_toxic_other}, num_nontoxic_other: {num_nontoxic_other}")
+    
+    contingency_table = [[num_toxic_user, num_nontoxic_user], [num_toxic_other, num_nontoxic_other]]
+    odds_ratio, p_val = stats.fisher_exact(contingency_table, alternative='two-sided')
+    if debug:
+        print(f"Odds ratio: {odds_ratio}, p={p_val}")
+
+    return odds_ratio
+
+# Neighbor search
+def get_match(comment_inds, K=20, threshold=None, debug=False):
+    match_ids = []
+    rows = []
+    for i in comment_inds:
+        if debug:
+            print(f"\nComment: {comments[i]}")
+        query_embedding = model.encode(comments[i], convert_to_tensor=True)
+        hits = util.semantic_search(query_embedding, embeddings, score_function=util.cos_sim, top_k=K)
+        # print(hits[0])
+        for hit in hits[0]:
+            c_id = hit['corpus_id']
+            score = np.round(hit['score'], 3)
+            if threshold is None or score > threshold:
+                match_ids.append(c_id)
+                if debug:
+                    print(f"\t(ID={c_id}, Score={score}): {comments[c_id]}")
+                rows.append([c_id, score, comments[c_id]])
+    
+    df = pd.DataFrame(rows, columns=["id", "score", "comment"])
+    return match_ids
+
+def display_examples_auto_cluster(preds_df, cluster, other_ids, perf_metric, sort_ascending=True, worker_id="A", num_examples=10):
+    # Overall performance
+    topic_df = preds_df
+    topic_df = topic_df[topic_df["topic"] == cluster]
+    user_perf, other_perf = get_overall_perf3(topic_df, perf_metric, other_ids)
+    
+    user_direction = "LOWER" if user_perf < 0 else "HIGHER"
+    other_direction = "LOWER" if other_perf < 0 else "HIGHER"
+    print(f"Your ratings are on average {np.round(abs(user_perf), 3)} {user_direction} than the existing system for this cluster")
+    print(f"Others' ratings (based on {len(other_ids)} users) are on average {np.round(abs(other_perf), 3)} {other_direction} than the existing system for this cluster")
+        
+    # Display example comments
+    df = display_examples_cluster(preds_df, other_ids, num_examples, sort_ascending)
+    return df
+
+    
+# function to get results for a new provided cluster
+def display_examples_manual_cluster(preds_df, cluster_comments, other_ids, perf_metric, sort_ascending=True, worker_id="A"):
+    # Overall performance
+    cluster_df = preds_df[preds_df["comment"].isin(cluster_comments)]
+    user_perf, other_perf = get_overall_perf3(cluster_df, perf_metric, other_ids)
+    
+    user_direction = "LOWER" if user_perf < 0 else "HIGHER"
+    other_direction = "LOWER" if other_perf < 0 else "HIGHER"
+    print(f"Your ratings are on average {np.round(abs(user_perf), 3)} {user_direction} than the existing system for this cluster")
+    print(f"Others' ratings (based on {len(other_ids)} users) are on average {np.round(abs(other_perf), 3)} {other_direction} than the existing system for this cluster")
+        
+    user_df = preds_df[preds_df.user_id == worker_id].sort_values(by=["item_id"]).reset_index()
+    others_df = preds_df[preds_df.user_id == other_ids[0]]
+    for i in range(1, len(other_ids)):
+        others_df.append(preds_df[preds_df.user_id == other_ids[i]])
+        others_df.groupby(["item_id"]).mean()
+    others_df = others_df.sort_values(by=["item_id"]).reset_index()
+    
+    # Get cluster_comments
+    user_df = user_df[user_df["comment"].isin(cluster_comments)]
+    others_df = others_df[others_df["comment"].isin(cluster_comments)]
+    
+    df = pd.merge(user_df, others_df, on="item_id", how="left", suffixes=('_user', '_other'))
+    df["pred_system"] = df["rating_avg_user"]
+    df["pred_system_stddev"] = df["rating_stddev_user"]
+    df = df[["item_id", "comment_user", "pred_user", "pred_other", "pred_system", "pred_system_stddev"]]
+    
+    # Add styling
+    df = df.sort_values(by=['pred_system_stddev'], ascending=sort_ascending)
+    df = df.style.apply(style_color_difference, axis=1).render()
+    return df
+
+########################################
+# GET_LABELING utils
+def create_example_sets(comments_df, n_label_per_bin, score_bins, keyword=None, topic=None):
+    # Restrict to the keyword, if provided
+    df = comments_df.copy()
+    if keyword != None:
+        df = df[df["comment"].str.contains(keyword)]    
+    
+    if topic != None:
+        df = df[df["topic"] == topic]  
+    
+    # Try to choose n values from each provided score bin
+    ex_to_label = []
+    bin_names = []
+    bin_label_counts = []
+    for i, score_bin in enumerate(score_bins):
+        min_score, max_score = score_bin
+        cur_df = df[(df["rating"] >= min_score) & (df["rating"] < max_score) & (df["item_id"].isin(train_df_ids))]
+        # sample rows for label
+        comment_ids = cur_df.item_id.tolist()
+        cur_n_label_per_bin = n_label_per_bin[i]
+        cap = min(len(comment_ids), (cur_n_label_per_bin))
+        to_label = np.random.choice(comment_ids, cap, replace=False)
+        ex_to_label.extend(to_label)
+        bin_names.append(f"[{min_score}, {max_score})")
+        bin_label_counts.append(len(to_label))
+    
+    return ex_to_label
+
+def get_grp_model_labels(comments_df, n_label_per_bin, score_bins, grp_ids):
+    df = comments_df.copy()
+
+    train_df_grp = train_df[train_df["user_id"].isin(grp_ids)]
+    train_df_grp_avg = train_df_grp.groupby(by=["item_id"]).median().reset_index()
+    train_df_grp_avg_ids = train_df_grp_avg["item_id"].tolist()
+
+    ex_to_label = [] # IDs of comments to use for group model training
+    for i, score_bin in enumerate(score_bins):
+        min_score, max_score = score_bin
+        # get eligible comments to sample
+        cur_df = df[(df["rating"] >= min_score) & (df["rating"] < max_score) & (df["item_id"].isin(train_df_grp_avg_ids))]
+        comment_ids = cur_df.item_id.unique().tolist()
+        # sample comments
+        cur_n_label_per_bin = n_label_per_bin[i]
+        cap = min(len(comment_ids), (cur_n_label_per_bin))
+        to_label = np.random.choice(comment_ids, cap, replace=False)
+        ex_to_label.extend((to_label))
+    
+    train_df_grp_avg = train_df_grp_avg[train_df_grp_avg["item_id"].isin(ex_to_label)]
+
+    ratings_grp = {ids_to_comments[int(r["item_id"])]: r["rating"] for _, r in train_df_grp_avg.iterrows()}
+
+    return ratings_grp  
+
+########################################
+# GET_PERSONALIZED_MODEL utils
+def fetch_existing_data(model_name, last_label_i):
+    # Check if we have cached model performance
+    perf_dir = f"./data/perf/{model_name}"
+    label_dir = f"./data/labels/{model_name}"
+    if os.path.isdir(os.path.join(module_dir, perf_dir)):
+        # Fetch cached results
+        last_i = len([name for name in os.listdir(os.path.join(module_dir, perf_dir)) if os.path.isfile(os.path.join(module_dir, perf_dir, name))])
+        with open(os.path.join(module_dir, perf_dir, f"{last_i}.pkl"), "rb") as f:
+            mae, mse, rmse, avg_diff = pickle.load(f)
+    else:
+        # Fetch results from trained model
+        with open(os.path.join(module_dir, f"./data/trained_models/{model_name}.pkl"), "rb") as f:
+            cur_model = pickle.load(f)
+            mae, mse, rmse, avg_diff = users_perf(cur_model)
+        # Cache results
+        os.mkdir(os.path.join(module_dir, perf_dir))
+        with open(os.path.join(module_dir, perf_dir, "1.pkl"), "wb") as f:
+            pickle.dump((mae, mse, rmse, avg_diff), f)
+    
+    # Fetch previous user-provided labels
+    ratings_prev = None
+    if last_label_i > 0:
+        with open(os.path.join(module_dir, label_dir, f"{last_i}.pkl"), "rb") as f:
+            ratings_prev = pickle.load(f)
+    return mae, mse, rmse, avg_diff, ratings_prev
+
+def train_updated_model(model_name, last_label_i, ratings, user, top_n=20, topic=None):
+    # Check if there is previously-labeled data; if so, combine it with this data
+    perf_dir = f"./data/perf/{model_name}"
+    label_dir = f"./data/labels/{model_name}"
+    labeled_df = format_labeled_data(ratings) # Treat ratings as full batch of all ratings
+    ratings_prev = None
+
+    # Filter out rows with "unsure" (-1)
+    labeled_df = labeled_df[labeled_df["rating"] != -1]
+
+    # Filter to top N for user study
+    if topic is None:
+        # labeled_df = labeled_df.head(top_n)
+        labeled_df = labeled_df.tail(top_n)
+    else:
+        # For topic tuning, need to fetch old labels
+        if (last_label_i > 0):
+            # Concatenate previous set of labels with this new batch of labels
+            with open(os.path.join(module_dir, label_dir, f"{last_label_i}.pkl"), "rb") as f:
+                ratings_prev = pickle.load(f)
+                labeled_df_prev = format_labeled_data(ratings_prev)
+                labeled_df_prev = labeled_df_prev[labeled_df_prev["rating"] != -1]
+                ratings.update(ratings_prev) # append old ratings to ratings
+                labeled_df = pd.concat([labeled_df_prev, labeled_df])
+
+    print("len ratings for training:", len(labeled_df))
+
+    cur_model, perf, _, _ = train_user_model(ratings_df=labeled_df)
+    
+    user_perf_metrics[model_name] = users_perf(cur_model)
+
+    mae, mse, rmse, avg_diff = user_perf_metrics[model_name]
+
+    cur_preds_df = get_preds_df(cur_model, ["A"], sys_eval_df=ratings_df_full, topic=topic, model_name=model_name)  # Just get results for user
+
+    # Save this batch of labels
+    with open(os.path.join(module_dir, label_dir, f"{last_label_i + 1}.pkl"), "wb") as f:
+        pickle.dump(ratings, f)
+
+    # Save model results
+    with open(os.path.join(module_dir, f"./data/preds_dfs/{model_name}.pkl"), "wb") as f:
+        pickle.dump(cur_preds_df, f)
+
+    if model_name not in all_model_names:
+        all_model_names.append(model_name)
+    with open(os.path.join(module_dir, "./data/all_model_names.pkl"), "wb") as f:
+        pickle.dump(all_model_names, f)
+    
+    # Handle user
+    if user not in users_to_models:
+        users_to_models[user] = []  # New user
+    if model_name not in users_to_models[user]:
+        users_to_models[user].append(model_name)  # New model
+        with open(f"./data/users_to_models.pkl", "wb") as f:
+            pickle.dump(users_to_models, f)
+
+    with open(os.path.join(module_dir, "./data/user_perf_metrics.pkl"), "wb") as f:
+        pickle.dump(user_perf_metrics, f)
+    with open(os.path.join(module_dir, f"./data/trained_models/{model_name}.pkl"), "wb") as f:
+        pickle.dump(cur_model, f)
+    
+    # Cache performance results
+    if not os.path.isdir(os.path.join(module_dir, perf_dir)):
+        os.mkdir(os.path.join(module_dir, perf_dir))
+    last_perf_i = len([name for name in os.listdir(os.path.join(module_dir, perf_dir)) if os.path.isfile(os.path.join(module_dir, perf_dir, name))])
+    with open(os.path.join(module_dir, perf_dir, f"{last_perf_i + 1}.pkl"), "wb") as f:
+        pickle.dump((mae, mse, rmse, avg_diff), f)
+
+    ratings_prev = ratings
+    return mae, mse, rmse, avg_diff, ratings_prev
+
+def format_labeled_data(ratings, worker_id="A", debug=False):    
+    all_rows = []
+    for comment, rating in ratings.items():
+        comment_id = comments_to_ids[comment]
+        row = [worker_id, comment_id, int(rating)]
+        all_rows.append(row)
+
+    df = pd.DataFrame(all_rows, columns=["user_id", "item_id", "rating"])
+    return df
+
+def users_perf(model, sys_eval_df=sys_eval_df, avg_ratings_df=comments_grouped_full_topic_cat, worker_id="A"):
+    # Load the full empty dataset
+    sys_eval_comment_ids = sys_eval_df.item_id.unique().tolist()
+    empty_ratings_rows = [[worker_id, c_id, 0] for c_id in sys_eval_comment_ids]
+    empty_ratings_df = pd.DataFrame(empty_ratings_rows, columns=["user_id", "item_id", "rating"])
+
+    # Compute predictions for full dataset
+    reader = Reader(rating_scale=(0, 4))
+    eval_set_data = Dataset.load_from_df(empty_ratings_df, reader)
+    _, testset = train_test_split(eval_set_data, test_size=1.)
+    predictions = model.test(testset)
+
+    df = empty_ratings_df # user_id, item_id, rating
+    user_item_preds = get_predictions_by_user_and_item(predictions)
+    df["pred"] = df.apply(lambda row: user_item_preds[(row.user_id, row.item_id)] if (row.user_id, row.item_id) in user_item_preds else np.nan, axis=1)
+
+    df = df.merge(avg_ratings_df, on="item_id", how="left", suffixes=('_', '_avg'))
+    df.dropna(subset = ["pred"], inplace=True)
+    df["rating_"] = df.rating_.astype("int32")
+
+    perf_metrics = get_overall_perf(df, "A") # mae, mse, rmse, avg_diff  
+    return perf_metrics
+
+def get_overall_perf(preds_df, user_id):    
+    # Prepare dataset to calculate performance
+    y_pred = preds_df[preds_df["user_id"] == user_id].rating_avg.to_numpy() # Assume system is just average of true labels
+    y_true = preds_df[preds_df["user_id"] == user_id].pred.to_numpy()
+    
+    # Get performance for user's model
+    mae = mean_absolute_error(y_true, y_pred)
+    mse = mean_squared_error(y_true, y_pred)
+    rmse = mean_squared_error(y_true, y_pred, squared=False)
+    avg_diff = np.mean(y_true - y_pred)
+    
+    return mae, mse, rmse, avg_diff
+
+def get_predictions_by_user_and_item(predictions):
+    user_item_preds = {}
+    for uid, iid, true_r, est, _ in predictions:
+        user_item_preds[(uid, iid)] = est
+    return user_item_preds
+
+def get_preds_df(model, user_ids, orig_df=ratings_df_full, avg_ratings_df=comments_grouped_full_topic_cat, sys_eval_df=sys_eval_df, bins=BINS, topic=None, model_name=None):
+    # Prep dataframe for all predictions we'd like to request
+    start = time.time()
+    sys_eval_comment_ids = sys_eval_df.item_id.unique().tolist()
+
+    empty_ratings_rows = []
+    for user_id in user_ids:
+        empty_ratings_rows.extend([[user_id, c_id, 0] for c_id in sys_eval_comment_ids])
+    empty_ratings_df = pd.DataFrame(empty_ratings_rows, columns=["user_id", "item_id", "rating"])
+    print("setup", time.time() - start)
+    
+    # Evaluate model to get predictions
+    start = time.time() 
+    reader = Reader(rating_scale=(0, 4))
+    eval_set_data = Dataset.load_from_df(empty_ratings_df, reader)
+    _, testset = train_test_split(eval_set_data, test_size=1.)
+    predictions = model.test(testset)
+    print("train_test_split", time.time() - start)
+    
+    # Update dataframe with predictions
+    start = time.time()
+    df = empty_ratings_df.copy() # user_id, item_id, rating
+    user_item_preds = get_predictions_by_user_and_item(predictions)
+    df["pred"] = df.apply(lambda row: user_item_preds[(row.user_id, row.item_id)] if (row.user_id, row.item_id) in user_item_preds else np.nan, axis=1)
+    df = df.merge(avg_ratings_df, on="item_id", how="left", suffixes=('_', '_avg'))
+    df.dropna(subset = ["pred"], inplace=True)
+    df["rating_"] = df.rating_.astype("int32")
+    
+    # Get binned predictions (based on user prediction)
+    df["prediction_bin"], out_bins = pd.cut(df["pred"], bins, labels=False, retbins=True)
+    df = df.sort_values(by=["item_id"])
+
+    return df
+
+def train_user_model(ratings_df, train_df=train_df, model_eval_df=model_eval_df, train_frac=0.75, model_type="SVD", sim_type=None, user_based=True):
+    # Sample from shuffled labeled dataframe and add batch to train set; specified set size to model_eval set
+    labeled = ratings_df.sample(frac=1)
+    batch_size = math.floor(len(labeled) * train_frac)
+    labeled_train = labeled[:batch_size]
+    labeled_model_eval = labeled[batch_size:]
+    
+    train_df_ext = train_df.append(labeled_train)
+    model_eval_df_ext = model_eval_df.append(labeled_model_eval)
+    
+    # Train model and show model eval set results
+    model, perf = train_model(train_df_ext, model_eval_df_ext, model_type=model_type, sim_type=sim_type, user_based=user_based)
+    
+    return model, perf, labeled_train, labeled_model_eval
+
+def train_model(train_df, model_eval_df, model_type="SVD", sim_type=None, user_based=True):
+    # Train model
+    reader = Reader(rating_scale=(0, 4))
+    train_data = Dataset.load_from_df(train_df, reader)
+    model_eval_data = Dataset.load_from_df(model_eval_df, reader)
+    
+    train_set = train_data.build_full_trainset()
+    _, model_eval_set = train_test_split(model_eval_data, test_size=1.)
+
+    sim_options = {
+        "name": sim_type,
+        "user_based": user_based, # compute similarity between users or items
+    }
+    if model_type == "SVD":
+        algo = SVD()  # SVD doesn't have similarity metric
+    elif model_type == "KNNBasic":
+        algo = KNNBasic(sim_options=sim_options)
+    elif model_type == "KNNWithMeans":
+        algo = KNNWithMeans(sim_options=sim_options)
+    elif model_type == "KNNWithZScore":
+        algo = KNNWithZScore(sim_options=sim_options)
+    algo.fit(train_set)
+    
+    predictions = algo.test(model_eval_set)
+    rmse = accuracy.rmse(predictions)
+    fcp = accuracy.fcp(predictions)
+    mae = accuracy.mae(predictions)
+    mse = accuracy.mse(predictions)
+    
+    print(f"MAE: {mae}, MSE: {mse}, RMSE: {rmse}, FCP: {fcp}")
+    perf = [mae, mse, rmse, fcp]
+    
+    return algo, perf
+
+def plot_train_perf_results2(model_name):
+    # Open labels
+    label_dir = f"./data/labels/{model_name}"
+    n_label_files = len([name for name in os.listdir(os.path.join(module_dir, label_dir)) if os.path.isfile(os.path.join(module_dir, label_dir, name))])
+    
+    all_rows = []
+    with open(os.path.join(module_dir, label_dir, f"{n_label_files}.pkl"), "rb") as f:
+        ratings = pickle.load(f)
+
+        labeled_df = format_labeled_data(ratings)
+        labeled_df = labeled_df[labeled_df["rating"] != -1]
+
+        # Iterate through batches of 5 labels
+        n_batches = int(np.ceil(len(labeled_df) / 5.))
+        for i in range(n_batches):
+            start = time.time()
+            n_to_sample = np.min([5 * (i + 1), len(labeled_df)])
+            cur_model, _, _, _ = train_user_model(ratings_df=labeled_df.head(n_to_sample))
+            mae, mse, rmse, avg_diff = users_perf(cur_model)
+            all_rows.append([n_to_sample, mae, "MAE"])
+            print(f"iter {i}: {time.time() - start}")
+        
+        print("all_rows", all_rows)
+        
+        df = pd.DataFrame(all_rows, columns=["n_to_sample", "perf", "metric"])
+        chart = alt.Chart(df).mark_line(point=True).encode(
+            x=alt.X("n_to_sample:Q", title="Number of Comments Labeled"),
+            y="perf",
+            color="metric",
+            tooltip=[
+                alt.Tooltip('n_to_sample:Q', title="Number of Comments Labeled"),
+                alt.Tooltip('metric:N', title="Metric"),
+                alt.Tooltip('perf:Q', title="Metric Value", format=".3f"),
+            ],
+        ).properties(
+            title=f"Performance over number of examples: {model_name}",
+            width=500,
+        )
+        return chart
+
+def plot_train_perf_results(model_name, mae):
+    perf_dir = f"./data/perf/{model_name}"
+    n_perf_files = len([name for name in os.listdir(os.path.join(module_dir, perf_dir)) if os.path.isfile(os.path.join(module_dir, perf_dir, name))])
+
+    all_rows = []
+    for i in range(1, n_perf_files + 1):
+        with open(os.path.join(module_dir, perf_dir, f"{i}.pkl"), "rb") as f:
+            mae, mse, rmse, avg_diff = pickle.load(f)
+            all_rows.append([i, mae, "Your MAE"])
+        
+    df = pd.DataFrame(all_rows, columns=["version", "perf", "metric"])
+    chart = alt.Chart(df).mark_line(point=True).encode(
+        x="version:O",
+        y="perf",
+        color=alt.Color("metric", title="Performance metric"),
+        tooltip=[
+            alt.Tooltip('version:O', title='Version'),
+            alt.Tooltip('metric:N', title="Metric"),
+            alt.Tooltip('perf:Q', title="Metric Value", format=".3f"),
+        ],
+    ).properties(
+        title=f"Performance over model versions: {model_name}",
+        width=500,
+    )
+
+    PCT_50 = 0.591
+    PCT_75 = 0.662
+    PCT_90 = 0.869
+
+    plot_dim_width = 500
+    domain_min = 0.0
+    domain_max = 1.0
+    bkgd = alt.Chart(pd.DataFrame({
+        "start": [PCT_90, PCT_75, domain_min],
+        "stop": [domain_max, PCT_90, PCT_75],
+        "bkgd": ["Needs improvement (< top 90%)", "Okay (top 90%)", "Good (top 75%)"],
+    })).mark_rect(opacity=0.2).encode(
+        y=alt.Y("start:Q", scale=alt.Scale(domain=[0, domain_max])),
+        y2=alt.Y2("stop:Q"),
+        x=alt.value(0),
+        x2=alt.value(plot_dim_width),
+        color=alt.Color("bkgd:O", scale=alt.Scale(
+            domain=["Needs improvement (< top 90%)", "Okay (top 90%)", "Good (top 75%)"], 
+            range=["red", "yellow", "green"]),
+            title="How good is your MAE?"
+        )
+    )
+
+    plot = (bkgd + chart).properties(width=plot_dim_width).resolve_scale(color='independent')
+    mae_status = None
+    if mae < PCT_75:
+        mae_status = "Your MAE is in the <b>Good</b> range, which means that it's in the top 75% of scores compared to other users. Your model looks good to go."
+    elif mae < PCT_90:
+        mae_status = "Your MAE is in the <b>Okay</b> range, which means that it's in the top 90% of scores compared to other users. Your model can be used, but you can provide additional labels to improve it."
+    else:
+        mae_status = "Your MAE is in the <b>Needs improvement</b> range, which means that it's in below the top 95% of scores compared to other users. Your model may need additional labels to improve."
+    return plot, mae_status
+
+########################################
+# New visualizations
+# Constants
+VIS_BINS = np.round(np.arange(0, 4.01, 0.05), 3)
+VIS_BINS_LABELS = [np.round(np.mean([x, y]), 3) for x, y in zip(VIS_BINS[:-1], VIS_BINS[1:])]
+
+def get_key(sys, user, threshold):
+    if sys <= threshold and user <= threshold:
+        return "System agrees: Non-toxic"
+    elif sys > threshold and user > threshold:
+        return "System agrees: Toxic"
+    else:
+        if abs(sys - threshold) > 1.5:
+            return "System differs: Error > 1.5"
+        elif abs(sys - threshold) > 1.0:
+            return "System differs: Error > 1.0"
+        elif abs(sys - threshold) > 0.5:
+            return "System differs: Error > 0.5"
+        else:
+            return "System differs: Error <=0.5"
+
+def get_key_no_model(sys, threshold):
+    if sys <= threshold:
+        return "System says: Non-toxic"
+    else:
+        return "System says: Toxic"
+
+def get_user_color(user, threshold):
+    if user <= threshold:
+        return "#FFF"  # white
+    else:
+        return "#808080"  # grey
+
+def get_system_color(sys, user, threshold):
+    if sys <= threshold and user <= threshold:
+        return "#FFF"  # white
+    elif sys > threshold and user > threshold:
+        return "#808080"  # grey
+    else:
+        if abs(sys - threshold) > 1.5:
+            return "#d62728" # red
+        elif abs(sys - threshold) > 1.0:
+            return "#ff7a5c" # med red
+        elif abs(sys - threshold) > 0.5:
+            return "#ffa894" # light red
+        else:
+            return "#ffd1c7" # very light red
+
+def get_error_type(sys, user, threshold):
+    if sys <= threshold and user <= threshold:
+        return "No error (agree non-toxic)"
+    elif sys > threshold and user > threshold:
+        return "No error (agree toxic)"
+    elif sys <= threshold and user > threshold:
+        return "System may be under-sensitive"
+    elif sys > threshold and user <= threshold:
+        return "System may be over-sensitive"
+
+def get_error_type_radio(sys, user, threshold):
+    if sys <= threshold and user <= threshold:
+        return "Show errors and non-errors"
+    elif sys > threshold and user > threshold:
+        return "Show errors and non-errors"
+    elif sys <= threshold and user > threshold:
+        return "System is under-sensitive"
+    elif sys > threshold and user <= threshold:
+        return "System is over-sensitive"
+
+def get_error_magnitude(sys, user, threshold):
+    if sys <= threshold and user <= threshold:
+        return 0  # no classification error
+    elif sys > threshold and user > threshold:
+        return 0  # no classification error
+    elif sys <= threshold and user > threshold:
+        return abs(sys - user)
+    elif sys > threshold and user <= threshold:
+        return abs(sys - user)
+
+def get_error_size(sys, user, threshold):
+    if sys <= threshold and user <= threshold:
+        return 0  # no classification error
+    elif sys > threshold and user > threshold:
+        return 0  # no classification error
+    elif sys <= threshold and user > threshold:
+        return sys - user
+    elif sys > threshold and user <= threshold:
+        return sys - user
+
+def get_decision(rating, threshold):
+    if rating <= threshold:
+        return "Non-toxic"
+    else:
+        return "Toxic"
+
+def get_category(row, threshold=0.3):
+    k_to_category = {
+        "is_profane_frac_": "Profanity", 
+        "is_threat_frac_": "Threat", 
+        "is_identity_attack_frac_": "Identity Attack", 
+        "is_insult_frac_": "Insult", 
+        "is_sexual_harassment_frac_": "Sexual Harassment",
+    }
+    categories = []
+    for k in ["is_profane_frac_", "is_threat_frac_", "is_identity_attack_frac_", "is_insult_frac_", "is_sexual_harassment_frac_"]:
+        if row[k] > threshold:
+            categories.append(k_to_category[k])
+    
+    if len(categories) > 0:
+        return ", ".join(categories)
+    else:
+        return ""
+
+def get_comment_url(row):
+    return f"#{row['item_id']}/#comment"
+
+def get_topic_url(row):
+    return f"#{row['topic_']}/#topic"
+
+def plot_overall_vis(preds_df, error_type, cur_user, cur_model, n_topics=None, bins=VIS_BINS, threshold=TOXIC_THRESHOLD, bin_step=0.05):
+    df = preds_df.copy().reset_index()
+    
+    if n_topics is not None:
+        df = df[df["topic_id_"] < n_topics]
+    
+    df["vis_pred_bin"], out_bins = pd.cut(df["pred"], bins, labels=VIS_BINS_LABELS, retbins=True)
+    df = df[df["user_id"] == "A"].sort_values(by=["item_id"]).reset_index()
+    df["system_label"] = [("toxic" if r > threshold else "non-toxic") for r in df["rating"].tolist()]
+    df["threshold"] = [threshold for r in df["rating"].tolist()]
+    df["key"] = [get_key(sys, user, threshold) for sys, user in zip(df["rating"].tolist(), df["pred"].tolist())]
+    df["url"] = df.apply(lambda row: get_topic_url(row), axis=1)
+    
+    # Plot sizing
+    domain_min = 0
+    domain_max = 4
+
+    plot_dim_height = 500
+    plot_dim_width = 750
+    max_items = np.max(df["vis_pred_bin"].value_counts().tolist())
+    mark_size = np.round(plot_dim_height / max_items) * 8
+    if mark_size > 75:
+        mark_size = 75
+        plot_dim_height = 13 * max_items
+
+    # Main chart
+    chart = alt.Chart(df).mark_square(opacity=0.8, size=mark_size, stroke="grey", strokeWidth=0.5).transform_window(
+        groupby=['vis_pred_bin'],
+        sort=[{'field': 'rating'}],
+        id='row_number()',
+        ignorePeers=True,
+    ).encode(
+        x=alt.X('vis_pred_bin:Q', title="Our prediction of your rating", scale=alt.Scale(domain=(domain_min, domain_max))),
+        y=alt.Y('id:O', title="Comments (ordered by System toxicity rating)", axis=alt.Axis(values=list(range(0, max_items, 5))), sort='descending'),
+        color = alt.Color("key:O", scale=alt.Scale(
+            domain=["System agrees: Non-toxic", "System agrees: Toxic", "System differs: Error > 1.5", "System differs: Error > 1.0", "System differs: Error > 0.5", "System differs: Error <=0.5"], 
+            range=["white", "#cbcbcb", "red", "#ff7a5c", "#ffa894", "#ffd1c7"]),
+            title="System rating (box color)"
+        ),
+        href="url:N",
+        tooltip = [
+            alt.Tooltip("topic_:N", title="Topic"),
+            alt.Tooltip("system_label:N", title="System label"),
+            alt.Tooltip("rating:Q", title="System rating", format=".2f"),
+            alt.Tooltip("pred:Q", title="Your rating", format=".2f")
+        ]
+    )
+
+    # Filter to specified error type
+    if error_type == "System is under-sensitive":
+        # FN: system rates non-toxic, but user rates toxic
+        chart = chart.transform_filter(
+            alt.FieldGTPredicate(field="pred", gt=threshold)
+        )
+    elif error_type == "System is over-sensitive":
+        # FP: system rates toxic, but user rates non-toxic
+        chart = chart.transform_filter(
+            alt.FieldLTEPredicate(field="pred", lte=threshold)
+        )
+    
+    # Threshold line
+    rule = alt.Chart(pd.DataFrame({
+        "threshold": [threshold],
+        "System threshold": [f"Threshold = {threshold}"]
+    })).mark_rule().encode(
+        x=alt.X("mean(threshold):Q", scale=alt.Scale(domain=(domain_min, domain_max)), title=""),
+        color=alt.Color("System threshold:N", scale=alt.Scale(domain=[f"Threshold = {threshold}"], range=["grey"])),
+        size=alt.value(2),
+    )
+    
+    # Plot region annotations
+    nontoxic_x = (domain_min + threshold) / 2.
+    toxic_x = (domain_max + threshold) / 2.
+    annotation = alt.Chart(pd.DataFrame({
+        "annotation_text": ["Non-toxic", "Toxic"],
+        "x": [nontoxic_x, toxic_x],
+        "y": [max_items, max_items],
+    })).mark_text(
+        align="center",
+        baseline="middle",
+        fontSize=16,
+        dy=10,
+        color="grey"
+    ).encode(
+        x=alt.X("x", title=""),
+        y=alt.Y("y", title="", axis=None),
+        text="annotation_text"
+    )
+    
+    # Plot region background colors
+    bkgd = alt.Chart(pd.DataFrame({
+        "start": [domain_min, threshold],
+        "stop": [threshold, domain_max],
+        "bkgd": ["Non-toxic (L side)", "Toxic (R side)"],
+    })).mark_rect(opacity=1.0, stroke="grey", strokeWidth=0.25).encode(
+        x=alt.X("start:Q", scale=alt.Scale(domain=[domain_min, domain_max])),
+        x2=alt.X2("stop:Q"),
+        y=alt.value(0),
+        y2=alt.value(plot_dim_height),
+        color=alt.Color("bkgd:O", scale=alt.Scale(
+            domain=["Non-toxic (L side)", "Toxic (R side)"], 
+            range=["white", "#cbcbcb"]),
+            title="Your rating (background color)"
+        )
+    )
+    
+    plot = (bkgd + annotation + chart + rule).properties(height=(plot_dim_height), width=plot_dim_width).resolve_scale(color='independent').to_json()
+
+    # Save to file
+    chart_dir = "./data/charts"
+    chart_file = os.path.join(chart_dir, f"{cur_user}_{cur_model}.pkl")
+    with open(chart_file, "w") as f:
+        json.dump(plot, f)
+
+    return plot
+
+def get_cluster_overview_plot(preds_df, error_type, threshold=TOXIC_THRESHOLD, use_model=True):
+    preds_df_mod = preds_df.merge(comments_grouped_full_topic_cat, on="item_id", how="left", suffixes=('_', '_avg'))
+
+    if use_model:
+        return plot_overall_vis_cluster(preds_df_mod, error_type=error_type, n_comments=500, threshold=threshold)
+    else:
+        return plot_overall_vis_cluster2(preds_df_mod, error_type=error_type, n_comments=500, threshold=threshold)
+
+def plot_overall_vis_cluster2(preds_df, error_type, n_comments=None, bins=VIS_BINS, threshold=TOXIC_THRESHOLD, bin_step=0.05):
+    df = preds_df.copy().reset_index()
+    
+    df["vis_pred_bin"], out_bins = pd.cut(df["rating"], bins, labels=VIS_BINS_LABELS, retbins=True)
+    df = df[df["user_id"] == "A"].sort_values(by=["rating"]).reset_index()
+    df["system_label"] = [("toxic" if r > threshold else "non-toxic") for r in df["rating"].tolist()]
+    df["key"] = [get_key_no_model(sys, threshold) for sys in df["rating"].tolist()]
+    print("len(df)", len(df))  # always 0 for some reason (from keyword search)
+    df["category"] = df.apply(lambda row: get_category(row), axis=1)
+    df["url"] = df.apply(lambda row: get_comment_url(row), axis=1)
+    
+    if n_comments is not None:
+        n_to_sample = np.min([n_comments, len(df)])
+        df = df.sample(n=n_to_sample)
+    
+    # Plot sizing
+    domain_min = 0
+    domain_max = 4
+    plot_dim_height = 500
+    plot_dim_width = 750
+    max_items = np.max(df["vis_pred_bin"].value_counts().tolist())
+    mark_size = np.round(plot_dim_height / max_items) * 8
+    if mark_size > 75:
+        mark_size = 75
+        plot_dim_height = 13 * max_items
+    
+    # Main chart
+    chart = alt.Chart(df).mark_square(opacity=0.8, size=mark_size, stroke="grey", strokeWidth=0.25).transform_window(
+        groupby=['vis_pred_bin'],
+        sort=[{'field': 'rating'}],
+        id='row_number()',
+        ignorePeers=True
+    ).encode(
+        x=alt.X('vis_pred_bin:Q', title="System toxicity rating", scale=alt.Scale(domain=(domain_min, domain_max))),
+        y=alt.Y('id:O', title="Comments (ordered by System toxicity rating)", axis=alt.Axis(values=list(range(0, max_items, 5))), sort='descending'),
+        color = alt.Color("key:O", scale=alt.Scale(
+            domain=["System says: Non-toxic", "System says: Toxic"], 
+            range=["white", "#cbcbcb"]),
+            title="System rating",
+            legend=None,
+        ),
+        href="url:N",
+        tooltip = [
+            alt.Tooltip("comment_:N", title="comment"),
+            alt.Tooltip("rating:Q", title="System rating", format=".2f"),
+        ]
+    )
+    
+    # Threshold line
+    rule = alt.Chart(pd.DataFrame({
+        "threshold": [threshold],
+    })).mark_rule(color='grey').encode(
+        x=alt.X("mean(threshold):Q", scale=alt.Scale(domain=[domain_min, domain_max]), title=""),
+        size=alt.value(2),
+    )
+    
+    # Plot region annotations
+    nontoxic_x = (domain_min + threshold) / 2.
+    toxic_x = (domain_max + threshold) / 2.
+    annotation = alt.Chart(pd.DataFrame({
+        "annotation_text": ["Non-toxic", "Toxic"],
+        "x": [nontoxic_x, toxic_x],
+        "y": [max_items, max_items],
+    })).mark_text(
+        align="center",
+        baseline="middle",
+        fontSize=16,
+        dy=10,
+        color="grey"
+    ).encode(
+        x=alt.X("x", title=""),
+        y=alt.Y("y", title="", axis=None),
+        text="annotation_text"
+    )
+    
+    # Plot region background colors
+    bkgd = alt.Chart(pd.DataFrame({
+        "start": [domain_min, threshold],
+        "stop": [threshold, domain_max],
+        "bkgd": ["Non-toxic", "Toxic"],
+    })).mark_rect(opacity=1.0, stroke="grey", strokeWidth=0.25).encode(
+        x=alt.X("start:Q", scale=alt.Scale(domain=[domain_min, domain_max])),
+        x2=alt.X2("stop:Q"),
+        y=alt.value(0),
+        y2=alt.value(plot_dim_height),
+        color=alt.Color("bkgd:O", scale=alt.Scale(
+            domain=["Non-toxic", "Toxic"], 
+            range=["white", "#cbcbcb"]),
+            title="System rating"
+        )
+    )
+    
+    final_plot = (bkgd + annotation + chart + rule).properties(height=(plot_dim_height), width=plot_dim_width).resolve_scale(color='independent').to_json()
+
+    return final_plot, df
+            
+def plot_overall_vis_cluster(preds_df, error_type, n_comments=None, bins=VIS_BINS, threshold=TOXIC_THRESHOLD, bin_step=0.05):
+    df = preds_df.copy().reset_index(drop=True)
+    # df = df[df["topic_"] == topic]
+    
+    df["vis_pred_bin"], out_bins = pd.cut(df["pred"], bins, labels=VIS_BINS_LABELS, retbins=True)
+    df = df[df["user_id"] == "A"].sort_values(by=["rating"]).reset_index(drop=True)
+    df["system_label"] = [("toxic" if r > threshold else "non-toxic") for r in df["rating"].tolist()]
+    df["key"] = [get_key(sys, user, threshold) for sys, user in zip(df["rating"].tolist(), df["pred"].tolist())]
+    print("len(df)", len(df))  # always 0 for some reason (from keyword search)
+    # print("columns", df.columns)
+    df["category"] = df.apply(lambda row: get_category(row), axis=1)
+    df["url"] = df.apply(lambda row: get_comment_url(row), axis=1)
+    
+    if n_comments is not None:
+        n_to_sample = np.min([n_comments, len(df)])
+        df = df.sample(n=n_to_sample)
+    
+    # Plot sizing
+    domain_min = 0
+    domain_max = 4
+    plot_dim_height = 500
+    plot_dim_width = 750
+    max_items = np.max(df["vis_pred_bin"].value_counts().tolist())
+    mark_size = np.round(plot_dim_height / max_items) * 8
+    if mark_size > 75:
+        mark_size = 75
+        plot_dim_height = 13 * max_items
+    
+    # Main chart
+    chart = alt.Chart(df).mark_square(opacity=0.8, size=mark_size, stroke="grey", strokeWidth=0.25).transform_window(
+        groupby=['vis_pred_bin'],
+        sort=[{'field': 'rating'}],
+        id='row_number()',
+        ignorePeers=True
+    ).encode(
+        x=alt.X('vis_pred_bin:Q', title="Our prediction of your rating", scale=alt.Scale(domain=(domain_min, domain_max))),
+        y=alt.Y('id:O', title="Comments (ordered by System toxicity rating)", axis=alt.Axis(values=list(range(0, max_items, 5))), sort='descending'),
+        color = alt.Color("key:O", scale=alt.Scale(
+            domain=["System agrees: Non-toxic", "System agrees: Toxic", "System differs: Error > 1.5", "System differs: Error > 1.0", "System differs: Error > 0.5", "System differs: Error <=0.5"], 
+            range=["white", "#cbcbcb", "red", "#ff7a5c", "#ffa894", "#ffd1c7"]),
+            title="System rating (box color)"
+        ),
+        href="url:N",
+        tooltip = [
+            alt.Tooltip("comment_:N", title="comment"),
+            alt.Tooltip("rating:Q", title="System rating", format=".2f"),
+            alt.Tooltip("pred:Q", title="Your rating", format=".2f"),
+            alt.Tooltip("category:N", title="Potential toxicity categories")
+        ]
+    )
+
+    # Filter to specified error type
+    if error_type == "System is under-sensitive":
+        # FN: system rates non-toxic, but user rates toxic
+        chart = chart.transform_filter(
+            alt.FieldGTPredicate(field="pred", gt=threshold)
+        )
+    elif error_type == "System is over-sensitive":
+        # FP: system rates toxic, but user rates non-toxic
+        chart = chart.transform_filter(
+            alt.FieldLTEPredicate(field="pred", lte=threshold)
+        )
+    
+    # Threshold line
+    rule = alt.Chart(pd.DataFrame({
+        "threshold": [threshold],
+    })).mark_rule(color='grey').encode(
+        x=alt.X("mean(threshold):Q", scale=alt.Scale(domain=[domain_min, domain_max]), title=""),
+        size=alt.value(2),
+    )
+    
+    # Plot region annotations
+    nontoxic_x = (domain_min + threshold) / 2.
+    toxic_x = (domain_max + threshold) / 2.
+    annotation = alt.Chart(pd.DataFrame({
+        "annotation_text": ["Non-toxic", "Toxic"],
+        "x": [nontoxic_x, toxic_x],
+        "y": [max_items, max_items],
+    })).mark_text(
+        align="center",
+        baseline="middle",
+        fontSize=16,
+        dy=10,
+        color="grey"
+    ).encode(
+        x=alt.X("x", title=""),
+        y=alt.Y("y", title="", axis=None),
+        text="annotation_text"
+    )
+    
+    # Plot region background colors
+    bkgd = alt.Chart(pd.DataFrame({
+        "start": [domain_min, threshold],
+        "stop": [threshold, domain_max],
+        "bkgd": ["Non-toxic (L side)", "Toxic (R side)"],
+    })).mark_rect(opacity=1.0, stroke="grey", strokeWidth=0.25).encode(
+        x=alt.X("start:Q", scale=alt.Scale(domain=[domain_min, domain_max])),
+        x2=alt.X2("stop:Q"),
+        y=alt.value(0),
+        y2=alt.value(plot_dim_height),
+        color=alt.Color("bkgd:O", scale=alt.Scale(
+            domain=["Non-toxic (L side)", "Toxic (R side)"], 
+            range=["white", "#cbcbcb"]),
+            title="Your rating (background color)"
+        )
+    )
+    
+    final_plot = (bkgd + annotation + chart + rule).properties(height=(plot_dim_height), width=plot_dim_width).resolve_scale(color='independent').to_json()
+
+    return final_plot, df
+
+def get_cluster_comments(df, error_type, threshold=TOXIC_THRESHOLD, worker_id="A", num_examples=50, use_model=True):    
+    df["user_color"] = [get_user_color(user, threshold) for user in df["pred"].tolist()]  # get cell colors
+    df["system_color"] = [get_user_color(sys, threshold) for sys in df["rating"].tolist()]  # get cell colors
+    df["error_color"] = [get_system_color(sys, user, threshold) for sys, user in zip(df["rating"].tolist(), df["pred"].tolist())]  # get cell colors
+    df["error_type"] = [get_error_type(sys, user, threshold) for sys, user in zip(df["rating"].tolist(), df["pred"].tolist())]  # get error type in words
+    df["error_amt"] = [abs(sys - threshold) for sys in df["rating"].tolist()]  # get raw error
+    df["judgment"] = ["" for _ in range(len(df))]  # template for "agree" or "disagree" buttons
+
+    if use_model:
+        df = df.sort_values(by=["error_amt"], ascending=False) # surface largest errors first
+    else:
+        print("get_cluster_comments; not using model")
+        df = df.sort_values(by=["rating"], ascending=True)
+
+    df["id"] = df["item_id"]
+    # df["comment"] already exists
+    df["comment"] = df["comment_"]
+    df["toxicity_category"] = df["category"]
+    df["user_rating"] = df["pred"]
+    df["user_decision"] = [get_decision(rating, threshold) for rating in df["pred"].tolist()]
+    df["system_rating"] = df["rating"]
+    df["system_decision"] = [get_decision(rating, threshold) for rating in df["rating"].tolist()]
+    df["error_type"] = df["error_type"]
+    df = df.head(num_examples)
+    df = df.round(decimals=2)
+
+    # Filter to specified error type
+    if error_type == "System is under-sensitive":
+        # FN: system rates non-toxic, but user rates toxic
+        df = df[df["error_type"] == "System may be under-sensitive"]
+    elif error_type == "System is over-sensitive":
+        # FP: system rates toxic, but user rates non-toxic
+        df = df[df["error_type"] == "System may be over-sensitive" ]
+    elif error_type == "Both":
+        df = df[(df["error_type"] == "System may be under-sensitive") | (df["error_type"] == "System may be over-sensitive")]
+
+    return df.to_json(orient="records")
+
+# PERSONALIZED CLUSTERS utils
+def get_disagreement_comments(preds_df, mode, n=10_000, threshold=TOXIC_THRESHOLD):
+    # Get difference between user rating and system rating
+    df = preds_df.copy()
+    df["diff"] = [get_error_size(sys, user, threshold) for sys, user in zip(df["rating"].tolist(), df["pred"].tolist())]
+    df["error_type"] = [get_error_type(sys, user, threshold) for sys, user in zip(df["rating"].tolist(), df["pred"].tolist())]
+    # asc = low to high; lowest = sys lower than user (under-sensitive)
+    # desc = high to low; lowest = sys higher than user (over-sensitive)
+    if mode == "under-sensitive":
+        df = df[df["error_type"] == "System may be under-sensitive"]
+        asc = True
+    elif mode == "over-sensitive":
+        df = df[df["error_type"] == "System may be over-sensitive"]
+        asc = False
+    df = df.sort_values(by=["diff"], ascending=asc)
+    df = df.head(n)
+    
+    return df["comment_"].tolist(), df
+
+def get_personal_clusters(model, n=3):
+    personal_cluster_file = f"./data/personal_cluster_dfs/{model}.pkl"
+    if (os.path.isfile(personal_cluster_file)):
+        with open(personal_cluster_file, "rb") as f:
+            cluster_df = pickle.load(f)
+            cluster_df = cluster_df.sort_values(by=["topic_id"])
+            topics_under = cluster_df[cluster_df["error_type"] == "System may be under-sensitive"]["topic"].unique().tolist()
+            topics_under = topics_under[1:(n + 1)]
+            topics_over = cluster_df[cluster_df["error_type"] == "System may be over-sensitive"]["topic"].unique().tolist()
+            topics_over = topics_over[1:(n + 1)]
+            return topics_under, topics_over
+    else:
+        topics_under_top = []
+        topics_over_top = []
+        preds_df_file = f"./data/preds_dfs/{model}.pkl"
+        if (os.path.isfile(preds_df_file)):
+            with open(preds_df_file, "rb") as f:
+                preds_df = pickle.load(f)
+                preds_df_mod = preds_df.merge(comments_grouped_full_topic_cat, on="item_id", how="left", suffixes=('_', '_avg')).reset_index()
+                preds_df_mod = preds_df_mod[preds_df_mod["user_id"] == "A"]
+
+                comments_under, comments_under_df = get_disagreement_comments(preds_df_mod, mode="under-sensitive", n=1000)
+                if len(comments_under) > 0:
+                    topics_under = BERTopic(embedding_model="paraphrase-MiniLM-L6-v2").fit(comments_under)
+                    topics_under_top = topics_under.get_topic_info().head(n)["Name"].tolist()
+                    print("topics_under", topics_under_top)
+                    # Get topics per comment
+                    topics_assigned, _ = topics_under.transform(comments_under)
+                    comments_under_df["topic_id"] = topics_assigned
+                    cur_topic_ids = topics_under.get_topic_info().Topic
+                    topic_short_names = topics_under.get_topic_info().Name
+                    topic_ids_to_names = {cur_topic_ids[i]: topic_short_names[i] for i in range(len(cur_topic_ids))}
+                    comments_under_df["topic"] = [topic_ids_to_names[topic_id] for topic_id in comments_under_df["topic_id"].tolist()]
+
+                comments_over, comments_over_df = get_disagreement_comments(preds_df_mod, mode="over-sensitive", n=1000)
+                if len(comments_over) > 0:
+                    topics_over = BERTopic(embedding_model="paraphrase-MiniLM-L6-v2").fit(comments_over)
+                    topics_over_top = topics_over.get_topic_info().head(n)["Name"].tolist()
+                    print("topics_over", topics_over_top)
+                    # Get topics per comment
+                    topics_assigned, _ = topics_over.transform(comments_over)
+                    comments_over_df["topic_id"] = topics_assigned
+                    cur_topic_ids = topics_over.get_topic_info().Topic
+                    topic_short_names = topics_over.get_topic_info().Name
+                    topic_ids_to_names = {cur_topic_ids[i]: topic_short_names[i] for i in range(len(cur_topic_ids))}
+                    comments_over_df["topic"] = [topic_ids_to_names[topic_id] for topic_id in comments_over_df["topic_id"].tolist()]
+
+                cluster_df = pd.concat([comments_under_df, comments_over_df])
+                with open(f"./data/personal_cluster_dfs/{model}.pkl", "wb") as f:
+                    pickle.dump(cluster_df, f)
+
+                return topics_under_top, topics_over_top
+    return [], []

indie_label_svelte/.gitignore

@@ -0,0 +1,4 @@
+/node_modules/
+/public/build/
+
+.DS_Store

indie_label_svelte/README.md

@@ -0,0 +1,9 @@
+# Sample Project
+
+This example illustrates how svelte-vega can be used in your application.
+If you want to build something using our `Vega` component, you just need to install `svelte-vega` to be able to import it.
+
+## Dependencies
+
+To run this project, `svelte-vega` needs to be built first.
+It is easiest to use from the [main repository](https://github.com/vega/svelte-vega).

indie_label_svelte/package.json

@@ -0,0 +1,72 @@
+{
+  "name": "svelte-vega-sample",
+  "version": "0.0.0",
+  "author": "Alex Bäuerle <[email protected]> (https://a13x.io)",
+  "repository": "[email protected]:vega/svelte-vega.git",
+  "bugs": {
+    "url": "https://github.com/vega/svelte-vega/issues"
+  },
+  "private": true,
+  "scripts": {
+    "build": "rollup -c",
+    "dev": "rollup -c -w",
+    "start": "sirv public --no-clear",
+    "validate": "svelte-check"
+  },
+  "devDependencies": {
+    "@babel/core": "^7.16.0",
+    "@material/typography": "^13.0.0",
+    "@rollup/plugin-commonjs": "^21.0.1",
+    "@rollup/plugin-json": "^4.1.0",
+    "@rollup/plugin-node-resolve": "^13.0.6",
+    "@rollup/plugin-typescript": "^8.3.0",
+    "@smui/button": "^6.0.0-beta.0",
+    "@smui/card": "^6.0.0-beta.4",
+    "@smui/checkbox": "^6.0.0-beta.2",
+    "@smui/chips": "^6.0.0-beta.2",
+    "@smui/circular-progress": "^6.0.0-beta.4",
+    "@smui/common": "^6.0.0-beta.2",
+    "@smui/data-table": "^6.0.0-beta.2",
+    "@smui/drawer": "^6.0.0-beta.4",
+    "@smui/form-field": "^6.0.0-beta.2",
+    "@smui/icon-button": "^6.0.0-beta.4",
+    "@smui/layout-grid": "^6.0.0-beta.2",
+    "@smui/linear-progress": "^6.0.0-beta.2",
+    "@smui/list": "^6.0.0-beta.4",
+    "@smui/radio": "^6.0.0-beta.2",
+    "@smui/select": "^6.0.0-beta.0",
+    "@smui/switch": "^6.0.0-beta.4",
+    "@smui/tab": "^6.0.0-beta.2",
+    "@smui/tab-bar": "^6.0.0-beta.2",
+    "@smui/textfield": "^6.0.0-beta.2",
+    "@smui/tooltip": "^6.0.0-beta.4",
+    "@tsconfig/svelte": "^2.0.1",
+    "autoprefixer": "^10.4.0",
+    "coffeescript": "^2.6.1",
+    "less": "^4.1.2",
+    "postcss": "^8.3.11",
+    "postcss-load-config": "^3.1.0",
+    "pug": "^3.0.2",
+    "rollup": "^2.59.0",
+    "rollup-plugin-css-only": "^3.1.0",
+    "rollup-plugin-livereload": "^2.0.5",
+    "rollup-plugin-svelte": "^7.0.0",
+    "rollup-plugin-terser": "^7.0.0",
+    "sass": "^1.43.4",
+    "stylus": "^0.55.0",
+    "sugarss": "^4.0.1",
+    "svelte-check": "^2.2.8",
+    "svelte-preprocess": "^4.9.8",
+    "typescript": "^4.5.2"
+  },
+  "dependencies": {
+    "sirv-cli": "^1.0.14",
+    "svelecte": "^3.4.0",
+    "svelte": "^3.44.1",
+    "svelte-material-ui": "^6.0.0-beta.0",
+    "svelte-vega": "^1.0.0",
+    "tslib": "^2.3.1",
+    "vega": "^5.21.0",
+    "vega-lite": "^5.1.0"
+  }
+}

indie_label_svelte/public/global.css

@@ -0,0 +1,311 @@
+html, body {
+	position: relative;
+	width: 100%;
+	height: 100%;
+}
+
+* {
+	font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Oxygen-Sans, Ubuntu, Cantarell, "Helvetica Neue", sans-serif !important;
+}
+
+.material-icons {
+	font-family: 'Material Icons' !important;
+}
+
+body {
+	color: #333;
+	margin: 0;
+	/* padding: 8px; */
+	box-sizing: border-box;
+	/* font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Oxygen-Sans, Ubuntu, Cantarell, "Helvetica Neue", sans-serif; */
+	/* font-family: Roboto, "Helvetica Neue", sans-serif;  */
+}
+
+a {
+	color: rgb(0,100,200);
+	text-decoration: none;
+}
+
+p, li {
+	line-height: 150%;
+}
+
+a:hover {
+	text-decoration: underline;
+}
+
+a:visited {
+	color: rgb(0,80,160);
+}
+
+label {
+	display: block;
+}
+
+input, button, select, textarea {
+	font-family: inherit;
+	font-size: inherit;
+	-webkit-padding: 0.4em 0;
+	padding: 0.4em;
+	margin: 0 0 0.5em 0;
+	box-sizing: border-box;
+	border: 1px solid #ccc;
+	border-radius: 2px;
+}
+
+input:disabled {
+	color: #ccc;
+}
+
+button {
+	color: #333;
+	background-color: #f4f4f4;
+	outline: none;
+}
+
+button:disabled {
+	color: #999;
+}
+
+button:not(:disabled):active {
+	background-color: #ddd;
+}
+
+button:focus {
+	border-color: #666;
+}
+
+/* IndieLabel custom styles */
+main {
+	text-align: left;
+	/* padding: 1em; */
+	max-width: 240px;
+	margin: 0 0;
+}
+h3 {
+	color: rgb(80, 80, 80);
+	font-size: 30px;
+}
+h5 {
+	color: rgb(80, 80, 80);
+	font-size: 25px;
+}
+h6 {
+	margin-top: 50px;
+	margin-bottom: 20px;
+	text-transform: uppercase;
+	font-size: 18px;
+}
+
+.head_3 {
+	color: rgb(80, 80, 80);
+	font-size: 30px;
+	font-weight: bold;
+	margin-top: 30px;
+	margin-bottom: 30px;
+}
+
+.head_5 {
+	color: rgb(80, 80, 80);
+	font-size: 24px;
+	font-weight: bold;
+	margin-top: 25px;
+	margin-bottom: 25px;
+}
+
+.head_6 {
+	color: rgb(80, 80, 80);
+	text-transform: uppercase;
+	font-weight: bold;
+	font-size: 18px;
+	margin-top: 25px;
+	margin-bottom: 25px;
+}
+
+.head_6_non_cap {
+	color: rgb(80, 80, 80);
+	font-weight: bold;
+	font-size: 18px;
+	margin-top: 25px;
+	margin-bottom: 25px;
+}
+
+.head_6_highlight {
+	color: #6200ee;
+	text-transform: uppercase;
+	font-weight: 400;
+	font-size: 14px;
+	margin-top: 25px;
+	letter-spacing: 0.75px;
+}
+
+@media (min-width: 640px) {
+	main {
+		max-width: none;
+	}
+}
+
+table {
+	font-size: 14px;
+}
+
+.bold {
+	font-weight: bold;
+}
+
+.bold-large {
+	font-weight: bold;
+	font-size: 16px;
+}
+
+.custom-blue {
+	color: #3c94ff;
+}
+
+.mdc-data-table__cell {
+	white-space: normal;
+	word-break: break-word;
+	padding: 10px;
+}
+
+.app_loading {
+	width: 40%;
+	padding-top: 20px;
+	padding-bottom: 100px;
+}
+
+.app_loading_fullwidth {
+	width: 100%;
+	padding-top: 20px;
+	padding-bottom: 100px;
+}
+
+.button_float_right {
+	float:right;
+	margin-left: 10px;
+}
+
+.spacing_vert {
+	/* margin: 20px 0;
+	padding: 10px 0; */
+	margin: 10px 0;
+}
+
+.spacing_vert_20 {
+	margin: 20px 0;
+}
+
+.spacing_vert_40 {
+	margin: 40px 0;
+}
+
+.spacing_vert_60 {
+	margin: 60px 0;
+}
+
+.page_title {
+	font-size: 30px;
+	font-weight: bold;
+	/* color: #945ec9;  */
+	/* color: #6200ee; */
+	color: #7826ed;
+	padding: 0 20px; 
+	margin: 15px 0;
+}
+
+.hypothesis_panel {
+	width: 30%;
+	height: 100%;
+	position: fixed;
+	z-index: 10;
+	top: 0;
+	left: 0;
+	overflow-x: hidden;
+	overflow-y: hidden;
+	background: #f3f3f3;
+	/* border-right: 1px solid grey; */
+}
+
+.auditing_panel {
+	position: fixed;
+	width: 70%;
+	height: 100%;
+	left: 30%;
+	overflow-y: scroll;
+}
+
+.tab_header {
+	position: fixed;
+	background-color: #e3d6fd;
+	z-index: 1001;
+	width: 70%;
+	/* border-bottom: 1px solid grey; */
+}
+
+.label_table {
+	height: 750px;
+	overflow-y: scroll;
+	width: 90%;
+}
+
+.label_table_expandable {
+	width: 90%;
+}
+
+.edit_button_row {
+	display: flex;
+	align-items: center;
+}
+
+.edit_button_row_input {
+	flex-grow: 1;
+}
+
+.grey_button {
+	color: grey !important;
+}
+
+.white_button {
+	color: white !important;
+}
+
+.grey_text {
+	color: grey;
+}
+
+.section_indent {
+	margin-left: 40px;
+}
+
+.audit_section {
+	padding-top: 40px;
+}
+
+.comment_table_small {
+	/* width: 150%; */
+}
+
+.comment_table_small .mdc-data-table__header-cell, .comment_table_small .mdc-data-table__cell, .comment_table_small .mdc-data-table__cell span {
+	font-size: 13px !important;
+}
+
+/* Material UI Tab formatting */
+.mdc-tab {
+	height: 85px !important;
+}
+
+.mdc-tab-indicator .mdc-tab-indicator__content--underline {
+	border-top-width: 3px !important;
+}
+
+.comment_table .mdc-data-table__header-cell, .comment_table .mdc-data-table {
+	background-color: transparent !important;
+}
+
+.label_table th.mdc-data-table__header-cell {
+	border-top: 1px solid rgb(224 224 224);
+}
+
+.mdc-drawer .mdc-deprecated-list-item {
+	height: 64px !important;
+}

indie_label_svelte/public/index.html

@@ -0,0 +1,29 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+	<meta charset='utf-8'>
+	<meta name='viewport' content='width=device-width,initial-scale=1'>
+
+	<title>Svelte app</title>
+
+	<link rel='icon' type='image/png' href='/favicon.png'>
+	<link rel='stylesheet' href='/global.css'>
+	<link rel='stylesheet' href='/build/bundle.css'>
+	<link rel='stylesheet' href='/build/extra.css'>
+	<!-- <link rel="stylesheet" href="../node_modules/svelte-material-ui/bare.css" /> -->
+	<!-- <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/[email protected]/bare.min.css" /> -->
+	<link rel="stylesheet" href="https://fonts.googleapis.com/icon?family=Material+Icons" />
+
+	<link
+		rel="stylesheet"
+		href="https://unpkg.com/@material/[email protected]/dist/mdc.typography.css"
+	/>
+
+	<script defer src='/build/bundle.js'></script>
+</head>
+
+<body>
+</body>
+
+</html>

indie_label_svelte/rollup.config.js

@@ -0,0 +1,87 @@
+import svelte from "rollup-plugin-svelte";
+import commonjs from "@rollup/plugin-commonjs";
+import resolve from "@rollup/plugin-node-resolve";
+import livereload from "rollup-plugin-livereload";
+import { terser } from "rollup-plugin-terser";
+import sveltePreprocess from "svelte-preprocess";
+import typescript from "@rollup/plugin-typescript";
+import css from "rollup-plugin-css-only";
+import json from "@rollup/plugin-json";
+import * as child from "child_process";
+
+const production = !process.env.ROLLUP_WATCH;
+
+function serve() {
+  let server;
+
+  function toExit() {
+    if (server) server.kill(0);
+  }
+
+  return {
+    writeBundle() {
+      if (server) return;
+      server = child.spawn("npm", ["run", "start", "--", "--dev"], {
+        stdio: ["ignore", "inherit", "inherit"],
+        shell: true,
+      });
+
+      process.on("SIGTERM", toExit);
+      process.on("exit", toExit);
+    },
+  };
+}
+
+export default {
+  input: "src/main.ts",
+  output: {
+    sourcemap: true,
+    format: "iife",
+    name: "app",
+    file: "public/build/bundle.js",
+  },
+  plugins: [
+    svelte({
+      preprocess: sveltePreprocess({ sourceMap: !production }),
+      compilerOptions: {
+        // enable run-time checks when not in production
+        dev: !production,
+      },
+    }),
+    // we'll extract any component CSS out into
+    // a separate file - better for performance
+    css({ output: "bundle.css" }),
+    // css({ output: 'public/build/extra.css' }),
+
+    // If you have external dependencies installed from
+    // npm, you'll most likely need these plugins. In
+    // some cases you'll need additional configuration -
+    // consult the documentation for details:
+    // https://github.com/rollup/plugins/tree/master/packages/commonjs
+    resolve({
+      browser: true,
+      dedupe: ["svelte"],
+    }),
+    commonjs(),
+    typescript({
+      sourceMap: !production,
+      inlineSources: !production,
+    }),
+    json(),
+
+    // In dev mode, call `npm run start` once
+    // the bundle has been generated
+    !production && serve(),
+
+    // Watch the `public` directory and refresh the
+    // browser on changes when not in production
+    !production && livereload("public"),
+
+    // If we're building for production (npm run build
+    // instead of npm run dev), minify
+    production && terser(),
+  ],
+  watch: {
+    clearScreen: false,
+  },
+};

indie_label_svelte/src/App.svelte

@@ -0,0 +1,112 @@
+<script lang="ts">
+	import { onMount } from "svelte";
+	import "../node_modules/svelte-material-ui/bare.css";
+
+	import HypothesisPanel from "./HypothesisPanel.svelte";
+	import MainPanel from "./MainPanel.svelte";
+	import SelectUserDialog from "./SelectUserDialog.svelte";
+	import Explore from "./Explore.svelte";
+	import Results from "./Results.svelte";
+	import StudyLinks from "./StudyLinks.svelte";
+	import { user } from './stores/cur_user_store.js';
+	import { users } from "./stores/all_users_store.js";
+
+	let personalized_model;
+	let personalized_models = [];
+
+	// let topic = "";
+
+	let error_type_options = ['Both', 'System is under-sensitive', 'System is over-sensitive', 'Show errors and non-errors'];
+    let error_type = error_type_options[0];
+
+	// Handle routing
+	let searchParams = new URLSearchParams(window.location.search);
+    let mode = searchParams.get("mode");
+	let cur_user = searchParams.get("user");
+
+	// Set cur_user if it's provided in URL params
+	if (cur_user !== null) {
+		user.update((value) => cur_user);
+	}
+
+	// Handle user dialog
+    let user_dialog_open = false;
+    user.subscribe(value => {
+		cur_user = value;
+	});
+
+	// Handle all users
+	let all_users = [];
+	async function getUsers() {    
+		const response = await fetch("./get_users");
+		const text = await response.text();
+		const data = JSON.parse(text);
+		all_users = data["users"];
+		users.update((value) => all_users);
+	}
+
+	function getAuditSettings() {
+		let req_params = {
+            user: cur_user,
+        };
+		let params = new URLSearchParams(req_params).toString();
+        fetch("./audit_settings?" + params)
+            .then((r) => r.text())
+            .then(function (r_orig) {
+                let r = JSON.parse(r_orig);
+				personalized_models = r["personalized_models"];
+				personalized_model = personalized_models[0]; // TEMP
+				console.log("personalized_model", personalized_model);
+				// personalized_model = "model_1632886687_iterA";
+				// let clusters = r["clusters"];
+                // topic = clusters[0]; // TEMP
+            });
+
+		// fetch("./audit_settings")
+		// 	.then((r) => r.text())
+		// 	.then(function (r_orig) {
+		// 		let r = JSON.parse(r_orig);
+		// 		personalized_models = r["personalized_models"];
+		// 		personalized_model = personalized_models[0]; // TEMP
+		// 		// personalized_model = "model_1632886687_iterA";
+		// 		let clusters = r["clusters"];
+        //         topic = clusters[0]; // TEMP
+		// 	});
+	}
+	onMount(async () => {
+		getAuditSettings();
+		getUsers();
+	});
+</script>
+
+<svelte:head>
+	<title>IndieLabel</title>
+</svelte:head>
+
+<main>
+	{#if mode == "explore"}
+		<div>
+			<Explore />
+		</div>
+	{:else if mode == "results"}
+	<div>
+		<Results />
+	</div>
+	{:else if mode == "study_links"}
+	<div>
+		<StudyLinks />
+	</div>
+	{:else }
+		<SelectUserDialog bind:open={user_dialog_open} cur_user={cur_user} />
+		<div>
+			{#key personalized_model }
+				<HypothesisPanel model={personalized_model} bind:user_dialog_open={user_dialog_open}/>
+			{/key}
+
+			<MainPanel bind:model={personalized_model} bind:error_type={error_type} on:change />
+		</div>
+	{/if}
+</main>
+
+<style>
+</style>

indie_label_svelte/src/AppOld.svelte

@@ -0,0 +1,127 @@
+<svelte:head>
+	<title>IndieLabel</title>
+</svelte:head>
+
+<script lang="ts">
+	import { onMount } from "svelte";
+	import Section from "./Section.svelte";
+	import IterativeClustering from "./IterativeClustering.svelte";
+	import OverallResults from "./OverallResults.svelte";
+	import Labeling from "./Labeling.svelte";
+	import HypothesisPanel from "./HypothesisPanel.svelte"
+
+	let personalized_model;
+	let personalized_models = [];
+	let breakdown_category;
+	let breakdown_categories = [];
+	let systems = ["Perspective comment toxicity classifier"]; // Only one system for now
+	let clusters = [];
+	let promise = Promise.resolve(null);
+
+	function getAuditSettings() {
+		fetch("./audit_settings")
+			.then((r) => r.text())
+			.then(function (r_orig) {
+				let r = JSON.parse(r_orig);
+				breakdown_categories = r["breakdown_categories"];
+				breakdown_category = breakdown_categories[0];
+				personalized_models = r["personalized_models"];
+				personalized_model = personalized_models[0];
+				clusters = r["clusters"];
+			});
+	}
+	onMount(async () => {
+		getAuditSettings();
+	});
+
+	function handleAuditButton() {
+		promise = getAudit();
+	}
+
+	async function getAudit() {
+		let req_params = {
+			pers_model: personalized_model,
+			breakdown_axis: breakdown_category,
+			perf_metric: "avg_diff",
+			breakdown_sort: "difference",
+			n_topics: 10,
+		};
+		let params = new URLSearchParams(req_params).toString();
+		const response = await fetch("./get_audit?" + params);
+		const text = await response.text();
+		const data = JSON.parse(text);
+		return data;
+	}
+
+</script>
+
+<main>
+	<HypothesisPanel model={personalized_model} />
+
+	<Labeling />
+
+	<IterativeClustering clusters={clusters} ind={1} personalized_model={personalized_model} />
+
+	<div id="audit-settings" class="section">
+		<h5>Audit settings</h5>
+		<Section
+			section_id="systems"
+			section_title="What status-quo system would you like to audit?"
+			section_opts={systems}
+			bind:value={systems[0]}
+		/>
+		<Section
+			section_id="personalized_model"
+			section_title="What model would you like to use to represent your views?"
+			section_opts={personalized_models}
+			bind:value={personalized_model}
+		/>
+		<Section
+			section_id="breakdown_category"
+			section_title="How would you like to explore the performance of the system?"
+			section_opts={breakdown_categories}
+			bind:value={breakdown_category}
+		/>
+		<button on:click={handleAuditButton}> Generate results </button>
+		<div>
+			Personalized model: {personalized_model}, Breakdown category: {breakdown_category}
+		</div>
+	</div>
+
+	{#await promise}
+		<p>...waiting</p>
+	{:then audit_results}
+		{#if audit_results}
+			<OverallResults data={audit_results} clusters={clusters} personalized_model={personalized_model} />
+		{/if}
+	{:catch error}
+		<p style="color: red">{error.message}</p>
+	{/await}
+</main>
+
+<style>
+	main {
+		text-align: left;
+		padding: 1em;
+		max-width: 240px;
+		margin: 0 0;
+	}
+	h3 {
+		color: rgb(80, 80, 80);
+		font-size: 30px;
+	}
+	h5 {
+		color: rgb(80, 80, 80);
+		font-size: 25px;
+	}
+	h6 {
+		margin-top: 50px;
+		text-transform: uppercase;
+		font-size: 14px;
+	}
+	@media (min-width: 640px) {
+		main {
+			max-width: none;
+		}
+	}
+</style>

indie_label_svelte/src/Auditing.svelte

@@ -0,0 +1,464 @@
+<script lang="ts">
+    import { onMount } from "svelte";
+    import Section from "./Section.svelte";
+    import KeywordSearch from "./KeywordSearch.svelte";
+    import OverallResults from "./OverallResults.svelte";
+    import ClusterResults from "./ClusterResults.svelte";
+    import HelpTooltip from "./HelpTooltip.svelte";
+    import TopicTraining from "./TopicTraining.svelte";
+
+    import { user } from './stores/cur_user_store.js';
+    import { error_type } from './stores/error_type_store.js';
+    import { topic_chosen } from './stores/cur_topic_store.js';
+    import { model_chosen } from './stores/cur_model_store.js';
+
+    import Button, { Label } from "@smui/button";
+    import LinearProgress from "@smui/linear-progress";
+    import LayoutGrid, { Cell } from "@smui/layout-grid";
+    import Radio from '@smui/radio';
+    import FormField from '@smui/form-field';
+    import Card, { Content } from '@smui/card';
+    import{ Wrapper } from '@smui/tooltip';
+    import IconButton from '@smui/icon-button';
+    import Select, { Option } from "@smui/select";
+    import Svelecte from '../node_modules/svelecte/src/Svelecte.svelte';
+
+    export let personalized_model;
+    // export let topic;
+    export let cur_error_type = "Both";
+
+    let evidence = [];
+    let show_audit_settings = false;
+
+    let error_type_options = [
+        {
+            "opt": 'Both', 
+            "descr": '(System is under- or over-sensitive)', 
+            "help": "View both types of potential system errors"
+        },
+        {
+            "opt": 'System is under-sensitive', 
+            "descr": '(Incorrectly rates as non-toxic)', 
+            "help": "Focus on system errors where the system labeled content as Non-toxic when it should have been labeled as Toxic."
+        },
+        {
+            "opt": 'System is over-sensitive', 
+            "descr": '(Incorrectly rates as toxic)', 
+            "help": "Focus on system errors where the system labeled content as Toxic when it should have been labeled as Non-toxic."
+        },
+        {
+            "opt": 'Show errors and non-errors', 
+            "descr": '', 
+            "help": "Also show cases that are not likely to be potential errors"
+        },
+    ]
+    
+    let personalized_models = [];
+    let breakdown_category;
+    let breakdown_categories = [];
+    let systems = ["YouSocial comment toxicity classifier"]; // Only one system for now
+    let clusters = [];
+    let clusters_for_tuning = []
+    let promise = Promise.resolve(null);
+
+    // Handle routing
+    let searchParams = new URLSearchParams(window.location.search);
+    let scaffold_method = searchParams.get("scaffold");
+    let mode = searchParams.get("mode");
+    let topic_vis_method = searchParams.get("topic_vis_method");
+
+    // Set audit type
+    let audit_types = [
+        "All topic exploration", 
+        "Single topic exploration"
+    ];
+    let audit_type;
+    if (scaffold_method == "fixed" || scaffold_method == "personal" || scaffold_method == "personal_group" || scaffold_method == "personal_test" || scaffold_method == "personal_cluster" || scaffold_method == "topic_train" || scaffold_method == "prompts") {
+        audit_type = audit_types[1];
+        // audit_type = audit_types[0];
+    } else {
+        // No scaffolding mode or tutorial
+        audit_type = audit_types[0];
+    }
+
+    let show_topic_training = false;
+    if (scaffold_method == "topic_train") {
+        show_topic_training = true;
+    } 
+
+    // Handle non-model mode
+    let use_model = true;
+    if (mode == "no_model") {
+        use_model = false;
+        cur_error_type = "Show errors and non-errors";
+    }
+
+    // Handle group model 
+    let use_group_model = false;
+    if (scaffold_method == "personal_group") {
+        use_group_model = true;
+    }
+    
+    // TEMP
+    let promise_cluster = Promise.resolve(null);
+
+    // Get current user from store
+    let cur_user;
+    user.subscribe(value => {
+        if (value != cur_user) {
+            cur_user = value;
+            personalized_model = "";
+            getAuditSettings();
+        }
+	});
+
+    // Get current topic from store
+    let topic;
+    topic_chosen.subscribe(value => {
+        topic = value;
+        handleClusterButton(); // re-render cluster results
+	});
+
+    // Get current model from store
+    model_chosen.subscribe(value => {
+        personalized_model = value;
+        // Add to personalized_models if not there
+        if (!personalized_models.includes(personalized_model)) {
+            personalized_models.push(personalized_model);
+        }
+
+        handleClusterButton(); // re-render cluster results
+	});
+
+    // Save current error type
+    async function updateErrorType() {    
+		error_type.update((value) => cur_error_type);
+        handleAuditButton();
+        handleClusterButton();
+	}
+
+    // Handle topic-specific training
+    // let topic_training = null;
+
+    async function updateTopicChosen() {
+        if (topic != null) {
+            console.log("updateTopicChosen", topic)
+            topic_chosen.update((value) => topic);
+        }
+    }
+
+    function getAuditSettings() {
+        let req_params = {
+            user: cur_user,
+            scaffold_method: scaffold_method,
+        };
+        let params = new URLSearchParams(req_params).toString();
+        fetch("./audit_settings?" + params)
+            .then((r) => r.text())
+            .then(function (r_orig) {
+                let r = JSON.parse(r_orig);
+                breakdown_categories = r["breakdown_categories"];
+                breakdown_category = breakdown_categories[0];
+                personalized_models = r["personalized_models"];
+                if (use_group_model) {
+                    let personalized_model_grp = r["personalized_model_grp"];
+                    personalized_model = personalized_model_grp[0];
+                } else {
+                    personalized_model = personalized_models[0];  // TEMP
+                }
+                
+                model_chosen.update((value) => personalized_model);
+                clusters = r["clusters"];
+                clusters_for_tuning = r["clusters_for_tuning"];
+                console.log("clusters", clusters); // TEMP
+                topic = clusters[0]["options"][0]["text"];
+                topic_chosen.update((value) => topic);
+                handleAuditButton();  // TEMP
+                handleClusterButton(); // TEMP
+            });
+    }
+    onMount(async () => {
+        getAuditSettings();
+    });
+
+    function handleAuditButton() {
+        model_chosen.update((value) => personalized_model);
+        promise = getAudit();
+    }
+
+    async function getAudit() {
+        let req_params = {
+            pers_model: personalized_model,
+            breakdown_axis: breakdown_category,
+            perf_metric: "avg_diff",
+            breakdown_sort: "difference",
+            n_topics: 10,
+            error_type: "Both", // Only allow both error types
+            cur_user: cur_user,
+            topic_vis_method: topic_vis_method,
+        };
+        let params = new URLSearchParams(req_params).toString();
+        const response = await fetch("./get_audit?" + params);
+        const text = await response.text();
+        const data = JSON.parse(text);
+        return data;
+    }
+
+    function handleClusterButton() {
+		promise_cluster = getCluster();
+	}
+
+	async function getCluster() {
+        if (personalized_model == "" || personalized_model == undefined) {
+            return null;
+        }
+		let req_params = {
+			cluster: topic,
+			topic_df_ids: [],
+			n_examples: 500, // TEMP
+			pers_model: personalized_model,
+			example_sort: "descending", // TEMP
+			comparison_group: "status_quo", // TEMP
+			search_type: "cluster",
+			keyword: "",
+			n_neighbors: 0,
+            error_type: cur_error_type,
+            use_model: use_model,
+            scaffold_method: scaffold_method,
+		};
+		let params = new URLSearchParams(req_params).toString();
+		const response = await fetch("./get_cluster_results?" + params);
+		const text = await response.text();
+		const data = JSON.parse(text);
+		console.log(topic);
+		return data;
+	}
+</script>
+
+<div>
+    <!-- 0: Audit settings -->
+    <div>
+        <div style="margin-top: 30px">
+            <span class="head_3">Auditing</span>
+            <IconButton 
+                class="material-icons grey_button"
+                size="normal"
+                on:click={() => (show_audit_settings = !show_audit_settings)}
+            >
+                help_outline
+            </IconButton>
+        </div>
+        <div style="width: 80%">
+            <p>In this section, we'll be auditing the content moderation system. Here, you’ll be aided by a personalized model that will help direct your attention towards potential problem areas in the model’s performance. This model isn’t meant to be perfect, but is designed to help you better focus on areas that need human review.</p>
+        </div>
+        
+        {#if show_audit_settings}
+        <div class="audit_section">
+            <div class="head_5">Audit settings</div>
+            <div style="width: 50%">
+                <p>Choose your audit settings here. These settings will affect all of the visualizations that follow, so you can return back here to make changes.</p>
+            </div>
+            <div class="section_indent">
+                <Section
+                    section_id="systems"
+                    section_title="What status-quo system would you like to audit?"
+                    section_opts={systems}
+                    bind:value={systems[0]}
+                    width_pct={40}
+                />
+                {#key personalized_model}
+                    <Section
+                        section_id="personalized_model"
+                        section_title="What model would you like to use to represent your views?"
+                        section_opts={personalized_models}
+                        bind:value={personalized_model}
+                        width_pct={40}
+                        on:change
+                    />
+                {/key}
+
+                <Section
+                    section_id="audit_type"
+                    section_title="What type of audit are you conducting?"
+                    section_opts={audit_types}
+                    bind:value={audit_type}
+                    width_pct={40}
+                    on:change
+                />
+            
+                <LayoutGrid>
+                    <Cell span={7}>
+                        <Button
+                            on:click={handleAuditButton}
+                            variant="outlined"
+                            class="button_float_right"
+                        >
+                            <Label>Start your audit</Label>
+                        </Button>
+                    </Cell>
+                </LayoutGrid>
+            </div>
+        </div>
+        <p>Current model: {personalized_model}</p>
+        {/if}
+    </div>
+
+    <!-- 1: All topics overview -->
+    {#if audit_type == audit_types[0]}
+    <div class="audit_section">
+        <div class="head_5">Overview of all topics</div>
+        <p>First, browse the system performance by different auto-generated comment topic areas.</p>
+
+        <div class="section_indent">            
+            {#await promise}
+                <div class="app_loading">
+                    <LinearProgress indeterminate />
+                </div>
+            {:then audit_results}
+                {#if audit_results}
+                    <OverallResults
+                        data={audit_results}
+                        clusters={clusters}
+                        personalized_model={personalized_model}
+                        cluster={topic}
+                    />
+                {/if}
+            {:catch error}
+                <p style="color: red">{error.message}</p>
+            {/await}
+        </div>
+    </div>
+    {/if}
+
+    <!-- 2a: Topic training -->
+    {#if show_topic_training}
+    <div class="audit_section">
+        <div class="head_5">Topic model training</div>
+        <p></p>
+        <div class="section_indent">
+            <div>
+                <p>In what topic area would you like to tune your model?</p>
+                <Svelecte 
+                    options={clusters_for_tuning} 
+                    labelAsValue={true}
+                    bind:value={topic}
+                    placeholder="Select topic"
+                    on:change={null}
+                    style="width: 50%"
+                >
+                </Svelecte>
+            </div>
+
+            <div style="padding-top: 30px">
+                <!-- Labeling -->
+                <h6>Comments to label</h6>
+                <ul>
+                    <li>
+                        Comments with scores <b>0</b> and <b>1</b> will be allowed to <b>remain</b> on the platform.
+                    </li>
+                    <li>
+                        Comments with scores <b>2</b>, <b>3</b>, or <b>4</b> will be <b>deleted</b> from the platform.
+                    </li>
+                    <li>
+                        Given that some comments may lack context, if you're not sure, feel free to mark the <b>unsure</b> option to skip a comment.
+                    </li>
+                </ul>
+                {#key topic}
+                <TopicTraining topic={topic} />
+                {/key}                    
+            </div>
+
+        </div>
+    </div>
+    {/if}
+
+    <!-- 2: Topic overview -->
+    <div class="audit_section">
+        <div class="head_5">Topic exploration</div>
+        <p></p>
+        <div class="section_indent">
+            <div>
+                <div>
+                    <p><b>What topic would you like to explore further?</b></p>
+                    <Svelecte 
+                        options={clusters} 
+                        labelAsValue={true}
+                        bind:value={topic}
+                        placeholder="Select topic"
+                        on:change={updateTopicChosen}
+                        style="width: 50%"
+                    >
+                    </Svelecte>
+                </div>
+
+                {#if use_model}
+                <div style="padding-top: 30px">
+                    <p><b>What kind of system errors do you want to focus on?</b></p>
+                    {#each error_type_options as e}
+                        <div style="display: flex; align-items: center;">
+                            <Wrapper rich>
+                                <FormField>
+                                    <Radio bind:group={cur_error_type} value={e.opt} on:change={updateErrorType} color="secondary" />
+                                    <span slot="label">
+                                        <b>{e.opt}</b> {e.descr}
+                                        <IconButton class="material-icons" size="button" disabled>help_outline</IconButton>
+                                    </span>
+                                </FormField>
+                                <HelpTooltip text={e.help} />
+                            </Wrapper>
+                        </div>
+                    {/each}
+                </div>
+                {/if}
+            </div>
+
+            <div style="padding-top: 30px">
+                {#await promise_cluster}
+                    <div class="app_loading">
+                        <LinearProgress indeterminate />
+                    </div>
+                {:then cluster_results}
+                    {#if cluster_results}
+                        {#if topic}
+                        <ClusterResults 
+                            cluster={topic} 
+                            clusters={clusters} 
+                            model={personalized_model} 
+                            data={cluster_results} 
+                            table_width_pct={90}
+                            table_id={"main"}
+                            use_model={use_model}
+                            bind:evidence={evidence} 
+                            on:change
+                        />
+                        {/if}
+                    {/if}
+                {:catch error}
+                    <p style="color: red">{error.message}</p>
+                {/await}
+            </div>
+
+        </div>
+    </div>
+
+    <!-- 3: Gather evidence -->
+    <div class="audit_section">
+        <div class="head_5">Gather additional evidence</div>
+        <p>Next, you can optionally search for more comments to serve as evidence through manual keyword search (for individual words or phrases).</p>
+        <div class="section_indent">
+            {#key error_type}
+            <KeywordSearch clusters={clusters} personalized_model={personalized_model} bind:evidence={evidence} use_model={use_model} on:change/>
+            {/key}
+        </div>
+    </div>
+
+    <!-- 4: Test hunch -->
+    <div class="audit_section">
+        <div class="head_5">Finalize your current report</div>
+        <p>Finally, review the report you've generated on the side panel and provide a brief summary of the problem you see. You may also list suggestions or insights into addressing this problem if you have ideas. This report will be directly used by the model developers to address the issue you've raised</p>
+    </div>
+    
+</div>
+
+<style>
+</style>

indie_label_svelte/src/ClusterResults.svelte

@@ -0,0 +1,562 @@
+<script lang="ts">
+    import { VegaLite } from "svelte-vega";
+    import type { View } from "svelte-vega";
+    import DataTable, {
+        Head,
+        Body,
+        Row,
+        Cell,
+        Pagination,
+    } from "@smui/data-table";
+    import Select, { Option } from "@smui/select";
+    import IconButton from "@smui/icon-button";
+    import Button from "@smui/button";
+    import { Label } from "@smui/common";
+    import Checkbox from '@smui/checkbox';
+    import Radio from '@smui/radio';
+    import FormField from '@smui/form-field';
+    import Tooltip, { Wrapper } from '@smui/tooltip';
+    import LayoutGrid, { Cell as LGCell } from "@smui/layout-grid";
+    import Card, { Content } from '@smui/card';
+
+    import HelpTooltip from "./HelpTooltip.svelte";
+    import { topic_chosen } from './stores/cur_topic_store.js';
+    import { new_evidence } from './stores/new_evidence_store.js';
+    import { open_evidence } from './stores/open_evidence_store.js';
+
+    export let data;
+    export let cluster;
+    export let clusters = null;
+    export let model;
+    export let show_vis = true;
+    export let show_checkboxes = true;
+    export let table_width_pct = 80;
+    export let rowsPerPage = 10;
+    export let evidence;
+    export let table_id;
+    export let use_model = true;
+
+    let N_COMMENTS = 500;
+    let show_num_ratings = false;
+    let show_your_decision_ratings = false;
+    let show_step2_info = false;
+
+    let comment_table_style;
+    if (show_checkboxes) {
+        comment_table_style = ""
+    } else {
+        comment_table_style = "comment_table_small"
+    }
+
+    // Handle Altair selections
+    let selected_comment_id = 0;
+    window.addEventListener("popstate", function (event) {
+    //your code goes here on location change 
+        let cur_url = window.location.href;
+        let cur_url_elems = cur_url.split("#");
+        // console.log(cur_url_elems)
+        if (cur_url_elems.length > 0) {
+            let path = cur_url_elems[2];
+            if (path == "comment") {
+                let comment_id = cur_url_elems[1].split("/")[0];
+                console.log("comment_id", comment_id)
+                selected_comment_id = parseInt(comment_id);
+                let table_ind = null;
+                for (let i = 0; i < items.length; i++) {
+                    if (items[i]["id"] == selected_comment_id) {
+                        table_ind = i;
+                        break;
+                    }
+                }
+                currentPage = table_ind / rowsPerPage;
+            } else if (path == "topic") {
+                let topic = cur_url_elems[1].split("/")[0];
+                topic_chosen.update((value) => topic); // update in store
+            }
+            // window.history.replaceState({}, document.title, "/"); // remove URL parameter
+        }
+    });
+
+    // Cluster Overview Plot
+    let cluster_overview_data = null;
+    let cluster_overview_spec = null;
+    let cluster_overview_view = null;
+    if (show_vis) {
+        let cluster_overview_json = data["cluster_overview_plot_json"];
+        cluster_overview_data =
+            cluster_overview_json["datasets"][
+                cluster_overview_json["layer"][0]["data"]["name"]
+            ];
+        cluster_overview_spec = cluster_overview_json;
+        cluster_overview_view = null;
+    }
+    
+    type ClusterComment = {
+        id: number;
+        comment: string;
+        user_decision: string;
+        user_rating: number;
+        system_decision: string;
+        system_rating: number;
+        user_color: string;
+        system_color: string;
+        error_type: string;
+        error_color: string;
+        judgment: string;
+        toxicity_category: string;
+    };
+    let items: ClusterComment[];
+
+    let selected = [];
+
+    // Pagination
+    let currentPage = 0;
+    $: start = currentPage * rowsPerPage;
+    $: end = Math.min(start + rowsPerPage, items.length);
+    $: slice = items.slice(start, end);
+    $: lastPage = Math.max(Math.ceil(items.length / rowsPerPage) - 1, 0);
+
+    $: if (currentPage > lastPage) {
+        currentPage = lastPage;
+    }
+
+    let set_length = 0;
+    // if (typeof(data["cluster_comments"] == "string")) {
+    if (show_checkboxes) {
+        items = JSON.parse(data["cluster_comments"]);
+        set_length = data["topic_df_ids"].length;
+    } else {
+        items = data["cluster_comments"];
+        set_length = items.length;
+    }
+    // console.log(set_length);
+    
+    let cur_open_evidence;
+    open_evidence.subscribe(value => {
+		cur_open_evidence = value;
+	});
+
+    function saveToEvidence() {
+        new_evidence.update((value) => []); // clear prior evidence
+        selected.forEach(function(s) {
+            if (!cur_open_evidence.includes(s)) {
+                new_evidence.update((value) => s); // update in store
+            }
+        });
+        selected = [];
+
+        // Clear highlighted rows
+        let rows = document.getElementsByTagName("tr");
+        let row_list = Array.prototype.slice.call(rows);
+        row_list.forEach(function(r) {
+            r.classList.remove("mdc-data-table__row--selected");
+        });
+
+        let checkbox_header_divs = document.getElementsByClassName("mdc-data-table__header-row-checkbox");
+        let checkbox_header_list = Array.prototype.slice.call(checkbox_header_divs);
+        checkbox_header_list.forEach(function(c) {
+            let c_input = c.getElementsByTagName("input");
+            for (let i = 0; i < c_input.length; i++) {
+                c_input[i].setAttribute("data-indeterminate", "false");
+                c_input[i].indeterminate = false;
+            }
+        });
+    }
+
+    function handleAdd(comment_to_remove) {
+        new_evidence.update((value) => []); // clear prior evidence
+        if (!cur_open_evidence.includes(comment_to_remove)) {
+            new_evidence.update((value) => comment_to_remove); // update in store
+        }
+    }
+
+    function handleRemove(comment_to_remove) {
+        // Update local open evidence
+        cur_open_evidence = cur_open_evidence.filter(item => item.comment != comment_to_remove)
+        // Update open evidence in store
+        open_evidence.update((value) => cur_open_evidence);
+    }
+</script>
+
+<div class="padding-top: 30px;">
+    {#if show_vis}
+        <div>
+            <span class="head_6">Topic overview: {cluster}</span>
+            <IconButton 
+                class="material-icons grey_button"
+                size="normal"
+                on:click={() => (show_step2_info = !show_step2_info)}
+            >
+                help_outline
+            </IconButton>
+        </div>
+        {#if N_COMMENTS < set_length}
+            <p>Showing a random sample of {N_COMMENTS} comments (out of {set_length} comments)</p>
+        {:else}
+            <p>Showing all {set_length} comments</p>
+        {/if}
+
+        {#if show_step2_info}
+            <LayoutGrid>
+                <LGCell span={8}>
+                    <div class="card-container">
+                        <Card variant="outlined" padded>
+                            <p class="mdc-typography--button"><b>Interpreting this visualization</b></p>
+                            <ul>
+                                <li>
+                                    This plot has the same layout as the <b>All Topics</b> visualization, but now, each <b>box</b> in this plot represents an <b>individual comment</b> that belongs to your <b>selected topic area</b>.
+                                </li>
+                                <li>
+                                    The <b>x-axis</b> represents our prediction of <b>your</b> toxicity rating for each comment (we'll call these "your ratings")
+                                    <ul>
+                                        <li>
+                                            The <b>left side</b> (white background) is the <b>Non-toxic</b> side (comments that'll be allowed to remain)
+                                        </li>
+                                        <li>
+                                            The <b>right side</b> (grey background) is the <b>Toxic</b> side (comments that will be deleted)
+                                        </li>
+                                        <li>
+                                            Comment boxes are plotted along the x-axis based on our prediction of your toxicity rating for that comment
+                                        </li>
+                                    </ul>
+                                </li>
+                    
+                                <li>
+                                    The <b>color</b> of the box indicates the <b>system's rating</b> for the same comment; you may want to focus on the <b>red-colored boxes</b> that indicate <b>disagreements</b> between "your ratings" and the system's ratings
+                                </li>
+                            </ul>
+                        </Card>
+                    </div>
+                </LGCell>
+            </LayoutGrid>
+        {/if}
+
+        <div class="row">
+            <div class="col s8">
+                <VegaLite
+                    {cluster_overview_data}
+                    spec={cluster_overview_spec}
+                    bind:view={cluster_overview_view}
+                />
+            </div>
+        </div>
+    {/if}
+    
+    {#if show_checkboxes}
+    <h6>Comments</h6>
+    {/if}
+    <!-- Display options -->
+    {#if show_checkboxes}
+    <div>
+        Numerical ratings:
+        <FormField>
+            <Radio bind:group={show_num_ratings} value={true} color="secondary" />
+            <span slot="label">Show</span>
+        </FormField>
+        <FormField>
+            <Radio bind:group={show_num_ratings} value={false} color="secondary" />
+            <span slot="label">Hide</span>
+        </FormField>
+    </div>
+    {#if use_model}
+    <div>
+        Our prediction of your decision + ratings:
+        <FormField>
+            <Radio bind:group={show_your_decision_ratings} value={true} color="secondary" />
+            <span slot="label">Show</span>
+        </FormField>
+        <FormField>
+            <Radio bind:group={show_your_decision_ratings} value={false} color="secondary" />
+            <span slot="label">Hide</span>
+        </FormField>
+    </div>
+    {/if}
+
+    <!-- <Wrapper>
+        <IconButton class="material-icons" size="button" disabled>help_outline</IconButton>
+        <Tooltip>White = Non-toxic, Grey = Toxic</Tooltip>
+    </Wrapper> -->
+    {/if}
+
+    {#key evidence}
+    <div class="comment_table {comment_table_style}">
+        <DataTable
+            table$aria-label="Comments in the topic cluster"
+            style="width: {table_width_pct}%;"
+        >
+            <Head>
+                <Row>
+                    <!-- {#if show_checkboxes}
+                    <Cell checkbox>
+                        <Checkbox />
+                    </Cell>
+                    {/if} -->
+
+                    <Cell style="width: 50%">Comment</Cell>
+
+                    {#if show_your_decision_ratings}
+                    <Cell>Our prediction<br>of your decision</Cell>
+                    {#if show_num_ratings}
+                    <Cell>Our prediction<br>of your rating</Cell>
+                    {/if}
+                    {/if}
+                    
+                    <Cell>
+                        System<br>decision<br>
+                        {#if show_checkboxes}
+                        <span style="font-size:12px; max-width:125px">White = Non-toxic, <br>Grey = Toxic</span>
+                        {/if}
+                    </Cell>
+                    {#if show_num_ratings}
+                    <Cell>System<br>rating</Cell>
+                    {/if}
+
+                    {#if show_checkboxes}
+                    {#if use_model}
+                    <Cell>
+                        Potential error<br>type<br>
+                        {#if show_checkboxes}
+                        <span style="font-size:12px; max-width:125px">Darker red = Greater <br>potential system error</span>
+                        {/if}
+                    </Cell>
+
+                    <Cell>Potential toxicity<br>categories</Cell>
+                    {/if}
+                    {/if}
+
+                    <Cell>Do you agree<br>with the system?</Cell>
+
+                    {#if !show_checkboxes}
+                    <Cell>Remove</Cell>
+                    {/if}
+
+                    {#if show_checkboxes}
+                    <Cell>Add<br>Evidence</Cell>
+                    {/if}
+                </Row>
+            </Head>
+            <Body>
+                {#each slice as item (item.id + table_id)}
+                    <Row>
+                        <!-- {#if show_checkboxes}
+                        <Cell checkbox>
+                            <Checkbox
+                                bind:group={selected}
+                                value={{
+                                    "comment": item.comment, 
+                                    "user_color": item.user_color, 
+                                    "user_decision": item.user_decision, 
+                                    "user_rating": item.user_rating, 
+                                    "system_color": item.system_color, 
+                                    "system_decision": item.system_decision,
+                                    "system_rating": item.system_rating, 
+                                    "error_type": item.error_type, 
+                                    "error_color": item.error_color,
+                                    "toxicity_category": item.toxicity_category,
+                                    "judgment": item.judgment, 
+                                    "id": item.id
+                                }}
+                                valueKey={item.comment}
+                            />
+                        </Cell>
+                        {/if} -->
+
+                        <Cell>
+                            {item.comment}
+                        </Cell>
+
+                        {#if show_your_decision_ratings}
+                        <Cell style="background-color: {item.user_color}; border-left: 1px solid rgba(0,0,0,.12); border-right: 1px solid rgba(0,0,0,.12); border-collapse: collapse;">
+                            {item.user_decision}
+                        </Cell>
+                        {#if show_num_ratings}
+                        <Cell style="background-color: {item.user_color}; border-left: 1px solid rgba(0,0,0,.12); border-right: 1px solid rgba(0,0,0,.12); border-collapse: collapse;">
+                            {item.user_rating}
+                        </Cell>
+                        {/if}
+                        {/if}
+
+                        <Cell style="background-color: {item.system_color}; border-left: 1px solid rgba(0,0,0,.12); border-right: 1px solid rgba(0,0,0,.12); border-collapse: collapse;">
+                            {item.system_decision}
+                        </Cell>
+                        {#if show_num_ratings}
+                        <Cell style="background-color: {item.system_color}; border-left: 1px solid rgba(0,0,0,.12); border-right: 1px solid rgba(0,0,0,.12); border-collapse: collapse;">
+                            {item.system_rating}
+                        </Cell>
+                        {/if}
+
+                        {#if show_checkboxes}
+                        {#if use_model}
+                        <Cell style="background-color: {item.error_color}; border-left: 1px solid rgba(0,0,0,.12); border-right: 1px solid rgba(0,0,0,.12); border-collapse: collapse;">
+                            {item.error_type}
+                        </Cell>
+
+                        <Cell>
+                            {item.toxicity_category}
+                        </Cell>
+                        {/if}
+                        {/if}
+
+                        <Cell>
+                            <div>
+                                <FormField>
+                                    <Radio
+                                      bind:group={item.judgment}
+                                      value={"Agree"}
+                                    />
+                                    <span slot="label">Agree</span>
+                                </FormField>
+                            </div>
+                            <div>
+                                <FormField>
+                                    <Radio
+                                      bind:group={item.judgment}
+                                      value={"Disagree"}
+                                    />
+                                    <span slot="label">Disagree</span>
+                                </FormField>
+                            </div>
+                        </Cell>
+
+                        {#if !show_checkboxes}
+                        <Cell>
+                            <IconButton class="material-icons grey_button" on:click={() => handleRemove(item.comment)}>
+                                remove_circle_outline
+                            </IconButton>
+                        </Cell>
+                        {/if}
+
+                        {#if show_checkboxes}
+                        <Cell>
+                            <IconButton class="material-icons grey_button" on:click={() => handleAdd(item)}>
+                                add_circle_outline
+                            </IconButton>
+                        </Cell>
+                        {/if}
+                    </Row>
+                {/each}
+            </Body>
+
+            <!-- Table pagination -->
+            <Pagination slot="paginate">
+                <svelte:fragment slot="rowsPerPage">
+                    <Label>Rows Per Page</Label>
+                    <Select variant="outlined" bind:value={rowsPerPage} noLabel>
+                        <Option value={5}>5</Option>
+                        <Option value={10}>10</Option>
+                        <Option value={25}>25</Option>
+                        <Option value={100}>100</Option>
+                    </Select>
+                </svelte:fragment>
+                <svelte:fragment slot="total">
+                    {start + 1}-{end} of {items.length}
+                </svelte:fragment>
+
+                <IconButton
+                    class="material-icons"
+                    action="first-page"
+                    title="First page"
+                    on:click={() => (currentPage = 0)}
+                    disabled={currentPage === 0}>first_page</IconButton
+                >
+                <IconButton
+                    class="material-icons"
+                    action="prev-page"
+                    title="Prev page"
+                    on:click={() => currentPage--}
+                    disabled={currentPage === 0}>chevron_left</IconButton
+                >
+                <IconButton
+                    class="material-icons"
+                    action="next-page"
+                    title="Next page"
+                    on:click={() => currentPage++}
+                    disabled={currentPage === lastPage}
+                    >chevron_right</IconButton
+                >
+                <IconButton
+                    class="material-icons"
+                    action="last-page"
+                    title="Last page"
+                    on:click={() => (currentPage = lastPage)}
+                    disabled={currentPage === lastPage}>last_page</IconButton
+                >
+            </Pagination>
+        </DataTable>
+    </div>
+    {/key}
+
+    <!-- {#if show_checkboxes}
+    <div class="spacing_vert">
+        <Button on:click={saveToEvidence} disabled={selected.length == 0} variant="outlined">
+            <Label>Save {selected.length} to evidence</Label>
+        </Button>
+    </div>
+    {/if} -->
+    
+    <!-- Old visualization -->
+    <!-- {#if show_vis}
+        <div style="margin-top: 500px">
+            <table>
+                <tbody>
+                    <tr class="custom-blue">
+                        <td class="bold">
+                            Compared to the system, YOUR labels are on average...
+                        </td>
+                        <td>
+                            <span class="bold-large"
+                                >{data["user_perf_rounded"]} points
+                                {data["user_direction"]}</span
+                            >
+                            for this cluster
+                        </td>
+                    </tr>
+                    <tr>
+                        <td class="bold">
+                            Compared to the system, OTHER USERS' labels are on
+                            average...
+                        </td>
+                        <td>
+                            <span class="bold-large"
+                                >{data["other_perf_rounded"]} points
+                                {data["other_direction"]}</span
+                            >
+                            for this cluster (based on {data["n_other_users"]} randomly-sampled
+                            users)
+                        </td>
+                    </tr>
+                    <tr>
+                        <td class="bold"> Odds ratio </td>
+                        <td>
+                            <span class="bold-large">{data["odds_ratio"]}</span><br />
+                            {data["odds_ratio_explanation"]}
+                        </td>
+                    </tr>
+                </tbody>
+            </table>
+        
+            <h6>Cluster examples</h6>
+            <div class="row">
+                <div class="col s12">
+                    <div id="cluster_results_elem">
+                        {@html data["cluster_examples"]}
+                    </div>
+                </div>
+            </div>
+        </div>
+    {/if} -->
+</div>
+
+<style>
+    /* Styles for table */
+    :global(html) {
+        height: auto;
+        width: auto;
+        position: static;
+    }
+    :global(#sapper),
+    :global(body) {
+        display: block;
+        height: auto;
+    }
+</style>

indie_label_svelte/src/CommentTable.svelte

@@ -0,0 +1,263 @@
+<script lang="ts">
+    import { onMount } from "svelte";
+    import ModelPerf from "./ModelPerf.svelte";
+    import Button, { Label } from "@smui/button";
+    import DataTable, { Head, Body, Row, Cell } from "@smui/data-table";
+    import LinearProgress from '@smui/linear-progress';
+    import IconButton from '@smui/icon-button';
+    import { user } from './stores/cur_user_store.js';
+
+    export let mode;
+    export let model_name;
+
+    let to_label = {};
+    let promise = Promise.resolve(null);
+    let n_complete_ratings;
+    let n_unsure_ratings;
+
+    // Get current user
+    let cur_user;
+    user.subscribe(value => {
+		cur_user = value;
+	});
+
+    function getCommentsToLabel(cur_mode, n) {
+        if (cur_mode == "train") {
+            let req_params = {
+                n: n,
+            };
+            let params = new URLSearchParams(req_params).toString();
+            fetch("./get_comments_to_label?" + params)
+                .then((r) => r.text())
+                .then(function (r_orig) {
+                    let r = JSON.parse(r_orig);
+                    r["to_label"].forEach((key) => (to_label[key] = null));
+                });
+        } else if (cur_mode == "view") {
+            if (model_name != "" && model_name != undefined) {
+                promise = getModel(cur_mode);
+            }
+        }
+    }
+    onMount(async () => {
+        getCommentsToLabel(mode, 40);
+    });
+
+    function handleLoadCommentsButton(n = 5) {
+        getCommentsToLabel("train", n);
+    }
+
+    function handleTrainModelButton() {
+        promise = getModel("train");
+    }
+
+    function getCompleteRatings() {
+        let ratings = getRatings();
+        let complete_ratings = Object.entries(ratings).filter(([key, value]) => value != "-1");
+        let unsure_ratings = Object.entries(ratings).filter(([key, value]) => value == "-1");
+        n_complete_ratings = complete_ratings.length;
+        n_unsure_ratings = unsure_ratings.length;
+    }
+
+    function getRatings() {
+        // Get rating for each comment
+        let ratings = {};
+        Object.entries(to_label).forEach(function ([comment, orig_rating], i) {
+            var radio_btns = document.getElementsByName(
+                "comment_" + i.toString()
+            );
+            let length = radio_btns.length;
+            for (var i = 0; i < length; i++) {
+                if (radio_btns[i].checked) {
+                    ratings[comment] = radio_btns[i].value;
+                    break;
+                }
+            }
+        });
+        return ratings;
+    }
+
+    async function getModel(cur_mode) {
+        let ratings = null;
+        if (cur_mode == "train") {
+            ratings = getRatings();
+            ratings = JSON.stringify(ratings);
+        }
+
+        let req_params = {
+            model_name: model_name,
+            ratings: ratings,
+            mode: cur_mode,
+            user: cur_user,
+        };
+        let params = new URLSearchParams(req_params).toString();
+        const response = await fetch("./get_personalized_model?" + params);
+        const text = await response.text();
+        const data = JSON.parse(text);
+        to_label = data["ratings_prev"];
+        console.log(data);
+        return data;
+    }
+</script>
+
+<div>
+    <div class="label_table spacing_vert">
+        <DataTable
+            table$aria-label="Comments to label"
+            style="width: 100%;"
+            stickyHeader
+        >
+            <Head>
+                <Row>
+                    <Cell style="width: 50%">Comment</Cell>
+                    <Cell style="background-color: #c3ecdb">
+                        0: <br>Not-at-all toxic<br>(Keep)<br>
+                    </Cell>
+                    <Cell style="background-color: white">
+                        1: <br>Slightly toxic<br>(Keep)<br>
+                    </Cell>
+                    <Cell style="background-color: #ffa894">
+                        2: <br>Moderately toxic<br>(Delete)<br>
+                    </Cell>
+                    <Cell style="background-color: #ff7a5c">
+                        3: <br>Very toxic<br>(Delete)<br>
+                    </Cell>
+                    <Cell style="background-color: #d62728">
+                        4: <br>Extremely toxic<br>(Delete)<br>
+                    </Cell>
+                    <Cell style="background-color: #808080">
+                        <br>Unsure<br>(Skip)<br>
+                    </Cell>
+                </Row>
+            </Head>
+            <Body>
+                {#if to_label}
+                    {#each Object.keys(to_label) as comment, i}
+                        <Row>
+                            <Cell>
+                                <div class="spacing_vert">{comment}</div>
+                            </Cell>
+                            <Cell>
+                                <label>
+                                    <input
+                                        name="comment_{i}"
+                                        type="radio"
+                                        value="0"
+                                        checked={to_label[comment] == "0"}
+                                    />
+                                    <span />
+                                </label>
+                            </Cell>
+                            <Cell>
+                                <label>
+                                    <input
+                                        name="comment_{i}"
+                                        type="radio"
+                                        value="1"
+                                        checked={to_label[comment] == "1"}
+                                    />
+                                    <span />
+                                </label>
+                            </Cell>
+                            <Cell>
+                                <label>
+                                    <input
+                                        name="comment_{i}"
+                                        type="radio"
+                                        value="2"
+                                        checked={to_label[comment] == "2"}
+                                    />
+                                    <span />
+                                </label>
+                            </Cell>
+                            <Cell>
+                                <label>
+                                    <input
+                                        name="comment_{i}"
+                                        type="radio"
+                                        value="3"
+                                        checked={to_label[comment] == "3"}
+                                    />
+                                    <span />
+                                </label>
+                            </Cell>
+                            <Cell>
+                                <label>
+                                    <input
+                                        name="comment_{i}"
+                                        type="radio"
+                                        value="4"
+                                        checked={to_label[comment] == "4"}
+                                    />
+                                    <span />
+                                </label>
+                            </Cell>
+                            <Cell>
+                                <label>
+                                    <input
+                                        name="comment_{i}"
+                                        type="radio"
+                                        value="-1"
+                                        checked={to_label[comment] == "-1"}
+                                        on:click={() => handleLoadCommentsButton(1)}
+                                    />
+                                    <span />
+                                </label>
+                            </Cell>
+                        </Row>
+                    {/each}
+                {/if}
+            </Body>
+        </DataTable>
+    </div>
+
+    {#key n_complete_ratings}
+    {#if n_complete_ratings}
+    <div class="spacing_vert_40">
+        <p>Number labeled: {n_complete_ratings}</p>
+        <p>Number unsure: {n_unsure_ratings}</p>
+    </div>
+    {/if}
+    {/key}
+
+    <div class="spacing_vert_40">
+        <Button on:click={handleTrainModelButton} variant="outlined" disabled={(!n_complete_ratings) || (n_complete_ratings < 40)}>
+            <Label>Train Model</Label>
+        </Button>
+        <Button on:click={getCompleteRatings} variant="outlined">
+            <Label>Get Number of Comments Labeled</Label>
+        </Button>
+        <Button on:click={() => handleLoadCommentsButton(5)} variant="outlined">
+            <Label>Fetch More Comments To Label</Label>
+        </Button>
+    </div>
+
+
+    <!-- Performance -->
+    {#await promise}
+        <div class="app_loading spacing_vert_20">
+            <LinearProgress indeterminate />
+        </div>
+    {:then perf_results}
+        {#if perf_results}
+            <div class="spacing_vert_20">
+                <ModelPerf data={perf_results} />
+            </div>
+        {/if}
+    {:catch error}
+        <p style="color: red">{error.message}</p>
+    {/await}
+</div>
+
+<style>
+    :global(html) {
+        height: auto;
+        width: auto;
+        position: static;
+    }
+    :global(#sapper),
+    :global(body) {
+        display: block;
+        height: auto;
+    }
+</style>

indie_label_svelte/src/Explore.svelte

@@ -0,0 +1,149 @@
+<script lang="ts">
+    import { onMount } from "svelte";
+    import Button, { Label } from "@smui/button";
+    import LinearProgress from "@smui/linear-progress";
+    import DataTable, {
+        Head,
+        Body,
+        Row,
+        Cell,
+        Label,
+        SortValue,
+    } from "@smui/data-table";
+    import IconButton from '@smui/icon-button';
+    import Radio from "@smui/radio";
+    import FormField from "@smui/form-field";
+
+    let cur_examples = [];
+    let promise = Promise.resolve(null);
+
+    // let sort_examples = false;
+    let sort = "rating";
+    let sortDirection: Lowercase<keyof typeof SortValue> = "descending";
+
+    function handleSort() {
+        cur_examples.sort((a, b) => {
+            const [aVal, bVal] = [a[sort], b[sort]][
+                sortDirection === "ascending" ? "slice" : "reverse"
+            ]();
+            return Number(aVal) - Number(bVal);
+        });
+        cur_examples = cur_examples;
+    }
+
+    onMount(async () => {
+        promise = getExamples();
+    });
+
+    function handleButton() {
+        promise = getExamples();
+    }
+    
+    async function getExamples() {
+        let req_params = {
+            n_examples: 20,
+        };
+        let params = new URLSearchParams(req_params).toString();
+        const response = await fetch("./get_explore_examples?" + params);
+        const text = await response.text();
+        const data = JSON.parse(text);
+        cur_examples = JSON.parse(data["examples"]);
+        console.log(cur_examples); // TEMP
+        return true;
+    }
+</script>
+
+<svelte:head>
+    <title>Explore</title>
+</svelte:head>
+
+<div class="panel">
+    <div class="panel_contents">
+        <div>
+            <h3>Explore System</h3>
+            <div style="width: 50%">
+                <ul>
+                    <li>
+                        Take a few minutes to explore some examples of <b>comments on YouSocial</b> and the toxicity ratings provided by YouSocial's <b>content moderation system</b>.
+                    </li>
+                    <li>
+                        You can optionally sort by the "System rating" by clicking on the arrow button in the header.
+                    </li>
+                    <li>
+                        Feel free to click the button to fetch a new sample of examples if you'd like.
+                    </li>
+                </ul>
+            </div>
+            <!-- <div>
+                Sort order:
+                <FormField>
+                    <Radio bind:group={sort_examples} value={false} color="secondary" />
+                    <span slot="label">None</span>
+                </FormField>
+                <FormField>
+                    <Radio bind:group={sort_examples} value={true} color="secondary" />
+                    <span slot="label">System toxicity rating (descending)</span>
+                </FormField>
+            </div> -->
+            <Button on:click={handleButton} variant="outlined" class="">
+                <Label>Get another sample of examples</Label>
+            </Button>
+        </div>
+
+        <div style="padding-top:50px">
+            {#await promise}
+                <div class="app_loading">
+                    <LinearProgress indeterminate />
+                </div>
+            {:then examples}
+                {#if cur_examples}
+                    <DataTable
+                        table$aria-label="Example list"
+                        style="max-width: 100%;"
+                        sortable
+                        bind:sort
+                        bind:sortDirection
+                        on:SMUIDataTable:sorted={handleSort}
+                    >
+                        <Head>
+                            <Row>
+                                <Cell sortable={false}>
+                                    <Label>Comment</Label>
+                                </Cell>
+                                <Cell sortable={false}>
+                                    <Label>System decision</Label>
+                                </Cell>
+                                <Cell numeric columnId="rating" sortable={true}>
+                                    <IconButton class="material-icons">arrow_upward</IconButton>
+                                    <Label>System rating</Label>
+                                </Cell>
+                            </Row>
+                        </Head>
+                        <Body>
+                            {#each cur_examples as ex (ex.item_id)}
+                                <Row>
+                                    <Cell>{ex.comment}</Cell>
+                                    <Cell
+                                        style="background-color: {ex.system_color}; border-left: 1px solid rgba(0,0,0,.12); border-right: 1px solid rgba(0,0,0,.12); border-collapse: collapse;"
+                                    >
+                                        {ex.system_decision}
+                                    </Cell>
+                                    <Cell numeric>{Number(ex.rating)}</Cell>
+                                </Row>
+                            {/each}
+                        </Body>
+                    </DataTable>
+                {/if}
+            {:catch error}
+                <p style="color: red">{error.message}</p>
+            {/await}
+        </div>
+    </div>
+</div>
+
+<style>
+    .panel {
+        width: 80%;
+        padding: 50px;
+    }
+</style>

indie_label_svelte/src/HelpTooltip.svelte

@@ -0,0 +1,18 @@
+<script>
+    import Tooltip, {
+        Wrapper,
+        Title,
+        Content,
+        Link,
+        RichActions,
+    } from '@smui/tooltip';
+    import IconButton from '@smui/icon-button';
+
+    export let text;
+</script>
+
+<Tooltip>
+    <Content>
+        {text}
+    </Content>
+</Tooltip>

indie_label_svelte/src/Hunch.svelte

@@ -0,0 +1,85 @@
+<script lang="ts">
+    import { onMount } from "svelte";
+	import IterativeClustering from "./IterativeClustering.svelte";
+	import Button, { Label } from "@smui/button";
+	import Textfield from '@smui/textfield';
+	import LinearProgress from "@smui/linear-progress";
+
+    export let ind;
+	export let hunch;
+	export let model;
+	export let topic;
+
+	let example_block = false;
+	let clusters;
+
+	function getAuditSettings() {
+		fetch("./audit_settings")
+			.then((r) => r.text())
+			.then(function (r_orig) {
+				let r = JSON.parse(r_orig);
+				clusters = r["clusters"];
+			});
+	}
+	onMount(async () => {
+		getAuditSettings();
+	});
+
+	function handleTestOnExamples() {
+		example_block = true;
+	}
+</script>
+
+<div>
+	<div>
+		<!-- <h6>Hunch {ind + 1}</h6> -->
+		<h6>Topic:</h6>
+		{topic}
+	</div>
+	<div class="spacing_vert">
+		<h6>Your summary/suggestions:</h6>
+		<Textfield
+			style="width: 100%;"
+			helperLine$style="width: 100%;"
+			textarea
+			bind:value={hunch}
+			label="My current hunch is that..."
+		>
+		</Textfield>
+		<!-- <Button
+			on:click={handleTestOnExamples}
+			class="button_float_right spacing_vert"
+			variant="outlined"
+		>
+			<Label>Test on examples</Label>
+		</Button> -->
+	</div>
+
+	<div class="spacing_vert">
+        <Button on:click={null} variant="outlined">
+            <Label>Save</Label>
+        </Button>
+        <Button on:click={null} variant="outlined">
+            <Label>Submit</Label>
+        </Button>
+    </div>
+
+	<!-- {#await example_block}
+        <div class="app_loading">
+            <LinearProgress indeterminate />
+        </div>
+    {:then} -->
+		<!-- {#if example_block}
+			<IterativeClustering clusters={clusters} ind={ind + 1} personalized_model={model} />
+		{/if} -->
+    <!-- {:catch error}
+        <p style="color: red">{error.message}</p>
+    {/await} -->
+</div>
+
+<style>
+	/* * {
+        z-index: 11;
+        overflow-x: hidden;
+    } */
+</style>

indie_label_svelte/src/HypothesisPanel.svelte

@@ -0,0 +1,608 @@
+<script lang="ts">
+    import { onMount } from "svelte";
+    import ClusterResults from "./ClusterResults.svelte";
+    import HelpTooltip from "./HelpTooltip.svelte";
+
+    import Button, { Label } from "@smui/button";
+    import Textfield from '@smui/textfield';
+    import { user } from './stores/cur_user_store.js';
+    import { error_type } from './stores/error_type_store.js';
+    import { new_evidence } from './stores/new_evidence_store.js';
+    import { open_evidence } from './stores/open_evidence_store.js';
+    import { topic_chosen } from './stores/cur_topic_store.js';
+    
+    import Drawer, {
+        AppContent,
+        Content,
+        Header,
+        Title,
+        Subtitle,
+    } from '@smui/drawer';
+    import List, { Item, Text, Graphic, PrimaryText, SecondaryText } from '@smui/list';
+    import LinearProgress from "@smui/linear-progress";
+    import CircularProgress from '@smui/circular-progress';
+    import Checkbox from '@smui/checkbox';
+    import FormField from '@smui/form-field';
+    import IconButton from "@smui/icon-button";
+    import{ Wrapper } from '@smui/tooltip';
+    import Radio from '@smui/radio';
+    import Switch from '@smui/switch';
+
+    export let model;
+    // export let topic;
+    export let user_dialog_open;
+
+    let all_reports = [];
+
+    let cur_user;
+    user.subscribe(value => {
+		cur_user = value;
+	});
+
+    let cur_topic;
+    topic_chosen.subscribe(value => {
+		cur_topic = value;
+	});
+
+    // Handle routing
+    let searchParams = new URLSearchParams(window.location.search);
+    let scaffold_method = searchParams.get("scaffold");
+    let topic_vis_method = searchParams.get("topic_vis_method");
+
+    // TODO: connect to selected["error_type"] so changes on main panel affect report panel
+    // let cur_error_type;
+    // error_type.subscribe(value => {
+	// 	cur_error_type = value;
+	// });
+
+    // Handle drawer
+    let open = false;
+    let selected = null;
+    let promise = Promise.resolve(null);
+    let editTitle = false;
+    let editErrorType = false;
+    let unfinished_count = 0;
+    
+    function setActive(value: string) {
+        selected = value;
+        // Set local and store value of open evidence to selected report's
+        cur_open_evidence = selected["evidence"];
+        open_evidence.update((value) => cur_open_evidence);
+        let isolated_topic = selected["title"].replace(/^(Topic: )/,'');
+        console.log("selected title", selected["title"]);
+        console.log(selected);
+
+        // Close panel
+        open = false;
+
+        // Update topic if in personal mode
+        if (scaffold_method == "personal" || scaffold_method == "personal_group" || scaffold_method == "personal_test" || scaffold_method == "tutorial") {
+            topic_chosen.update((value) => isolated_topic);
+        }
+    }
+
+    onMount(async () => {
+        promise = getReports();
+    });
+
+    async function getReports() {
+        if (model == "" || model == undefined){
+            return [];
+        }
+        let req_params = {
+            cur_user: cur_user,
+            scaffold_method: scaffold_method,
+            model: model,
+            topic_vis_method: topic_vis_method,
+        };
+        let params = new URLSearchParams(req_params).toString();
+        const response = await fetch("./get_reports?" + params);
+        const text = await response.text();
+        const data = JSON.parse(text);
+        all_reports = data["reports"]
+        // Select first report initially
+        selected = all_reports[0];
+        setActive(selected);
+        cur_open_evidence = selected["evidence"];
+        unfinished_count = all_reports.filter(item => !item.complete_status).length
+        return all_reports;
+    }
+
+    // Handle evidence saving
+    let cur_open_evidence = [];
+    new_evidence.subscribe(value => {
+        if (value != []) {
+            // Check if any values with the same ID exist
+            for (let i = 0; i < cur_open_evidence.length; i++) {
+                if (cur_open_evidence[i]["id"] == value["id"]) {
+                    return; // If so, don't add the item
+                }
+            }
+            cur_open_evidence = cur_open_evidence.concat(value); // add new evidence item
+
+            // Add to open evidence in store
+            open_evidence.update((value) => cur_open_evidence);
+            // Save to selected value
+            if (selected != null) {
+                selected["evidence"] = cur_open_evidence;
+            }
+        }
+	});
+
+    // Handle evidence removal
+    open_evidence.subscribe(value => {
+        if ((value != cur_open_evidence) && (value.length < cur_open_evidence.length)) {
+            // Update local open evidence
+            cur_open_evidence = value;
+            // Save to selected value
+            if (selected != null) {
+                selected["evidence"] = cur_open_evidence;
+            }
+        }
+	});
+
+    let promise_save = Promise.resolve(null);
+    function handleSaveReport() {
+        promise_save = saveReport();
+    }
+
+    async function saveReport() {
+        let req_params = {
+            cur_user: cur_user,
+            reports: JSON.stringify(all_reports),
+            scaffold_method: scaffold_method,
+        };
+        let params = new URLSearchParams(req_params).toString();
+        const response = await fetch("./save_reports?" + params);
+        const text = await response.text();
+        const data = JSON.parse(text);
+        return data;
+    }
+
+    function handleNewReport() {
+        let new_report = {
+            title: "",
+            error_type: "",
+            evidence: [],
+            text_entry: "",
+            complete_status: false,
+        };
+        all_reports = all_reports.concat(new_report);
+        promise = Promise.resolve(all_reports);
+        // Open this new report
+        selected = all_reports[all_reports.length - 1];
+        cur_open_evidence = selected["evidence"];
+        selected["complete_status"] = false;
+        unfinished_count = all_reports.filter(item => !item.complete_status).length
+    }
+
+    function handleDeleteReport() {
+        // Remove selected item from reports
+        all_reports = all_reports.filter(item => item != selected);
+        promise = Promise.resolve(all_reports);
+        selected  = all_reports[0];
+        cur_open_evidence = selected["evidence"];
+        unfinished_count = all_reports.filter(item => !item.complete_status).length
+    }
+
+    function handleMarkComplete() {
+        selected["complete_status"] = !selected["complete_status"];
+        unfinished_count = all_reports.filter(item => !item.complete_status).length
+        handleSaveReport(); // Auto-save report
+    }
+
+    // Error type
+    let error_type_options = [
+        {
+            "opt": 'Both', 
+            "descr": '(System is under- or over-sensitive)', 
+            "help": "View both types of potential system errors"
+        },
+        {
+            "opt": 'System is under-sensitive', 
+            "descr": '(Incorrectly rates as non-toxic)', 
+            "help": "Focus on system errors where the system labeled content as Non-toxic when it should have been labeled as Toxic."
+        },
+        {
+            "opt": 'System is over-sensitive', 
+            "descr": '(Incorrectly rates as toxic)', 
+            "help": "Focus on system errors where the system labeled content as Toxic when it should have been labeled as Non-toxic."
+        },
+        {
+            "opt": 'Show errors and non-errors', 
+            "descr": '', 
+            "help": "Also show cases that are not likely to be potential errors"
+        },
+    ]
+
+    // Save current error type
+    async function updateErrorType() {    
+        // Update error type on main page to be the selected error type
+		// error_type.update((value) => cur_error_type);
+        // selected["error_type"] = cur_error_type;
+        editErrorType = false;
+	}
+
+</script>
+
+<div class="hypothesis_panel">
+    <div class="panel_header">
+        <div class="panel_header_content">
+            <div class="page_header">
+                <!-- <span class="page_title">IndieLabel</span> -->
+                <img src="/logo.png" style="height: 60px; padding: 0px 20px;" alt="IndieLabel" />
+                <Button on:click={() => (user_dialog_open = true)} class="user_button" color="secondary" style="margin: 12px 10px;" >
+                    <Label>User: {cur_user}</Label>
+                </Button>
+            </div>
+            <div class="hypotheses_header">
+                <h5 style="float: left; margin: 0; padding: 5px 20px;">Your Audit Reports</h5>
+                <Button 
+                    on:click={() => (open = !open)}
+                    color="primary"
+                    style="float: right; padding: 10px; margin-right: 10px;"
+                >
+                    {#if open}
+                    <Label>Close</Label>
+                    {:else}
+                        {#key unfinished_count}
+                        <Label>Unfinished reports ({unfinished_count})</Label>
+                        {/key}
+                    {/if}
+                </Button>
+            </div>
+        </div>
+    </div>
+
+    <div class="panel_contents">  
+        <!-- Drawer -->
+        {#await promise}
+            <div class="app_loading_fullwidth">
+                <LinearProgress indeterminate />
+            </div>
+        {:then reports}
+            {#if reports}
+            <div class="drawer-container">
+                {#key open}
+                <Drawer variant="dismissible" bind:open>
+                    <Header>
+                        <Title>Your Reports</Title>
+                        <Subtitle>Select a report to view.</Subtitle>
+                    </Header>
+                    <Content>
+                        <List twoLine>
+                            {#each reports as report}
+                                <Item
+                                    href="javascript:void(0)"
+                                    on:click={() => setActive(report)}
+                                    activated={selected === report}
+                                >   
+                                    {#if report["complete_status"]}
+                                    <Graphic class="material-icons" aria-hidden="true">task_alt</Graphic>
+                                    {:else}
+                                    <Graphic class="material-icons" aria-hidden="true">radio_button_unchecked</Graphic>
+                                    {/if}
+                                    <Text>
+                                        <PrimaryText>
+                                            {report["title"]}
+                                        </PrimaryText>
+                                        <SecondaryText>
+                                            {report["error_type"]}
+                                        </SecondaryText>
+                                    </Text>
+                                </Item>
+                            {/each}
+                        </List>
+                    </Content>
+                </Drawer>
+                {/key}
+                <AppContent class="app-content">
+                    <main class="main-content">
+                        {#if selected}
+                        <div class="head_6_highlight">
+                            Current Report
+                        </div>
+                        <div class="panel_contents2">
+                            <!-- Title -->
+                            <div class="spacing_vert">
+                                <div class="edit_button_row">
+                                    {#if editTitle}
+                                        <div class="edit_button_row_input">
+                                            <Textfield
+                                                bind:value={selected["title"]}
+                                                label="Your report title"
+                                                input$rows={4}
+                                                textarea
+                                                variant="outlined"
+                                                style="width: 100%;"
+                                                helperLine$style="width: 100%;"
+                                            />
+                                        </div>
+                                        <div>
+                                            <IconButton class="material-icons grey_button" size="button" on:click={() => (editTitle = false)}>
+                                                check
+                                            </IconButton>
+                                        </div>
+                                    {:else}
+                                        {#if selected["title"] != ""}
+                                            <div class="head_5">
+                                                {selected["title"]}
+                                            </div>
+                                        {:else}
+                                            <div class="grey_text">Enter a report title</div>
+                                        {/if}
+
+                                        <div>
+                                            <IconButton class="material-icons grey_button" size="button" on:click={() => (editTitle = true)}>
+                                                create
+                                            </IconButton>
+                                        </div>
+                                    {/if}
+                                </div>
+                            </div>
+
+                            <!-- Error type -->
+                            <div class="spacing_vert_40">
+                                <div class="head_6">
+                                    <b>Error Type</b> 
+                                </div>
+                                <div class="edit_button_row">
+                                    {#if editErrorType}
+                                        <div>
+                                            {#each error_type_options as e}
+                                                <div style="display: flex; align-items: center;">
+                                                    <!-- <Wrapper rich>
+                                                        <FormField>
+                                                            <Radio bind:group={selected["error_type"]} value={e.opt} on:change={updateErrorType} color="secondary" />
+                                                            <span slot="label">
+                                                                {e.opt}
+                                                                <IconButton class="material-icons" size="button" disabled>help_outline</IconButton>
+                                                            </span>
+                                                        </FormField>
+                                                        <HelpTooltip text={e.help} />
+                                                    </Wrapper> -->
+
+                                                    <FormField>
+                                                        <Radio bind:group={selected["error_type"]} value={e.opt} on:change={updateErrorType} color="secondary" />
+                                                        <span slot="label">
+                                                            <b>{e.opt}</b> {e.descr}
+                                                        </span>
+                                                    </FormField>
+                                                </div>
+                                            {/each}
+                                        </div>
+                                        <!-- <div>
+                                            <IconButton class="material-icons grey_button" size="button" on:click={() => (editErrorType = false)}>
+                                                check
+                                            </IconButton>
+                                        </div> -->
+                                    {:else}
+                                        {#if selected["error_type"] != ""}
+                                            <div>
+                                                <p>{selected["error_type"]}</p>
+                                            </div>
+                                        {:else}
+                                            <div class="grey_text">Select an error type</div>
+                                        {/if}
+                                        
+                                        <div>
+                                            <IconButton class="material-icons grey_button" size="button" on:click={() => (editErrorType = true)}>
+                                                create
+                                            </IconButton>
+                                        </div>
+                                    {/if}
+                                </div>
+                            </div>
+                            
+                            <!-- Evidence -->
+                            <div class="spacing_vert_40">
+                                <div class="head_6">
+                                    <b>Evidence</b>
+                                </div>
+                                {#key cur_open_evidence}
+                                <div>
+                                    {#if cur_open_evidence.length > 0}
+                                    <ClusterResults 
+                                        cluster={cur_topic} 
+                                        model={model} 
+                                        data={{"cluster_comments": cur_open_evidence}} 
+                                        show_vis={false} 
+                                        show_checkboxes={false} 
+                                        table_width_pct={100} 
+                                        rowsPerPage={25} 
+                                        table_id={"panel"}
+                                    />
+                                    {:else}
+                                        <p class="grey_text">
+                                            Add examples from the main panel to see them here!
+                                        </p>
+                                    {/if}
+                                </div>
+                                {/key}
+                            </div>
+
+                            <div class="spacing_vert_60">
+                                <div class="head_6">
+                                    <b>Summary/Suggestions</b>
+                                </div>
+                                <div class="spacing_vert">
+                                    <Textfield
+                                        style="width: 100%;"
+                                        helperLine$style="width: 100%;"
+                                        input$rows={8}
+                                        textarea
+                                        bind:value={selected["text_entry"]}
+                                        label="My current hunch is that..."
+                                    >
+                                    </Textfield>
+                                </div>
+                                
+                            </div>
+
+                            <div class="spacing_vert_40">
+                                <div class="head_6">
+                                    <b>Mark report as complete?</b>
+                                    <FormField>
+                                        <Checkbox checked={selected["complete_status"]} on:change={handleMarkComplete} />
+                                    </FormField>
+                                </div>
+                                
+                            </div>
+                        </div>
+                        {/if}
+                    </main>
+                </AppContent>
+            </div>
+            {/if}
+        {:catch error}
+            <p style="color: red">{error.message}</p>
+        {/await}
+    </div>
+
+    <div class="panel_footer">
+        <div class="panel_footer_contents">
+            
+
+            <Button 
+                on:click={handleNewReport} 
+                variant="outlined" 
+                color="secondary"
+                style=""
+            >
+                <Label>New</Label>
+            </Button>
+
+            <Button 
+                on:click={handleDeleteReport} 
+                variant="outlined" 
+                color="secondary"
+                style=""
+            >
+                <Label>Delete</Label>
+            </Button>
+
+            <Button 
+                on:click={handleSaveReport} 
+                variant="outlined" 
+                color="secondary"
+            >
+                <Label>Save</Label>
+            </Button>
+
+            <div>
+                <span style="color: grey"><i>Last saved:
+                {#await promise_save}
+                    <CircularProgress style="height: 32px; width: 32px;" indeterminate />
+                {:then result}
+                    {#if result}
+                     {new Date().toLocaleTimeString()}
+                    {:else}
+                     —
+                    {/if}
+                {:catch error}
+                    <p style="color: red">{error.message}</p>
+                {/await}
+                </i></span>
+            </div>
+        </div>
+    </div>
+
+    <!-- TEMP -->
+    <!-- {#key model}
+        <div>Model: {model}</div> 
+    {/key} -->
+</div>
+
+<style>
+    /* Drawer */
+    /* .drawer-container {
+        position: relative;
+        display: flex;
+        height: 350px;
+        max-width: 600px;
+        border: 1px solid
+        var(--mdc-theme-text-hint-on-background, rgba(0, 0, 0, 0.1));
+        overflow: hidden;
+        z-index: 0;
+    }
+    
+    * :global(.app-content) {
+        flex: auto;
+        overflow: auto;
+        position: relative;
+        flex-grow: 1;
+    }
+    
+    .main-content {
+        overflow: auto;
+        padding: 16px;
+        height: 100%;
+        box-sizing: border-box;
+    } */
+
+    .panel_contents {
+        padding: 0 20px;
+        overflow-y: auto;
+        top: 150px;
+        position: relative;
+        height: 82%;
+    }
+    .panel_contents2 {
+        padding-left: 10px;
+    }
+
+    .panel_header {
+        position: fixed;
+        width: 30%;
+        border-bottom: 1px solid #d7d7d7; /* c5c5c5 */
+        background: #f3f3f3;
+        z-index: 11;
+    }
+
+    .panel_footer {
+        position: fixed;
+        width: 30%;
+        border-top: 1px solid #d7d7d7;
+        background: #f3f3f3;
+        z-index: 11;
+        bottom: 0;
+        padding: 15px 0px;
+    }
+    .panel_footer_contents {
+        /* padding: 0px 20px; */
+        display: flex;
+        justify-content: space-around;
+        align-items: center;
+    }
+
+    :global(.mdc-button.user_button) {
+        float: right;
+        margin-right: 20px;
+    }
+
+    .page_header {
+        width: 100%;
+        background: #e3d6fd;
+        /* padding: 21px 0; */
+        /* border-bottom: 1px solid #e3d6fd; */
+        padding: 10.5px 0;
+        position: relative;
+        display: inline-block;
+    }
+
+    .page_header:before {
+        content: '';
+        border-right: 1px solid rgb(0 0 0 / 7%);
+        position: absolute;
+        height: 80%;
+        top: 10%;
+        right: 0;
+    }
+
+    .hypotheses_header {
+        display: inline-block;
+        width: 100%;
+        padding: 10px 0;
+        vertical-align: middle;
+    }
+</style>

indie_label_svelte/src/IterativeClustering.svelte

@@ -0,0 +1,164 @@
+<script>
+    import Section from "./Section.svelte";
+    import ClusterResults from "./ClusterResults.svelte";
+    import Button, { Label } from "@smui/button";
+    import Textfield from "@smui/textfield";
+    import LayoutGrid, { Cell } from "@smui/layout-grid";
+    import LinearProgress from "@smui/linear-progress";
+    import Chip, { Set, Text } from '@smui/chips';
+
+    export let clusters;
+    export let personalized_model;
+    export let evidence;
+    export let width_pct = 80;
+
+    let topic_df_ids = [];
+    let promise_iter_cluster = Promise.resolve(null);
+    let keyword = null;
+    let n_neighbors = null;
+    let cur_iter_cluster = null;
+    let history = [];
+
+    async function getIterCluster(search_type) {
+        let req_params = {
+            cluster: cur_iter_cluster,
+            topic_df_ids: topic_df_ids,
+            n_examples: 500, // TEMP
+            pers_model: personalized_model,
+            example_sort: "descending", // TEMP
+            comparison_group: "status_quo", // TEMP
+            search_type: search_type,
+            keyword: keyword,
+            n_neighbors: n_neighbors,
+        };
+        console.log("topic_df_ids", topic_df_ids);
+        let params = new URLSearchParams(req_params).toString();
+        const response = await fetch("./get_cluster_results?" + params);
+        const text = await response.text();
+        const data = JSON.parse(text);
+        // if (data["cluster_comments"] == null) {
+        //     return false
+        // }
+        topic_df_ids = data["topic_df_ids"];
+        return data;
+    }
+
+    function findCluster() {
+        promise_iter_cluster = getIterCluster("cluster");
+        history = history.concat("bulk-add cluster: " + cur_iter_cluster);
+    }
+
+    function findNeighbors() {
+        promise_iter_cluster = getIterCluster("neighbors");
+        history = history.concat("find " + n_neighbors + " neighbors");
+    }
+
+    function findKeywords() {
+        promise_iter_cluster = getIterCluster("keyword");
+        history = history.concat("keyword search: " + keyword);
+    }
+</script>
+
+<div>
+    <div>
+        <!-- <h6>Hunch {ind} examples</h6> -->
+        <div>
+            <h6>Search Settings</h6>
+            <!-- Start with cluster -->
+            <!-- <div class="">
+                <Section
+                    section_id="iter_cluster"
+                    section_title="Bulk-add cluster"
+                    section_opts={clusters}
+                    bind:value={cur_iter_cluster}
+                    width_pct={100}
+                />
+                <Button
+                    on:click={findCluster}
+                    variant="outlined"
+                    class="button_float_right"
+                    disabled={cur_iter_cluster == null}
+                >
+                    <Label>Search</Label>
+                </Button>
+            </div> -->
+
+            <!-- Manual keyword -->
+            <div class="spacing_vert">
+                <Textfield
+                    bind:value={keyword}
+                    label="Keyword search"
+                    variant="outlined"
+                    style="width: {width_pct}%"
+                />
+                <Button
+                    on:click={findKeywords}
+                    variant="outlined"
+                    class="button_float_right spacing_vert"
+                    disabled={keyword == null}
+                >
+                    <Label>Search</Label>
+                </Button>
+            </div>
+
+            <!-- Find neighbors of current set -->
+            <div class="spacing_vert">
+                <Textfield
+                    bind:value={n_neighbors}
+                    label="Number of neighbors to retrieve"
+                    type="number"
+                    min="1"
+                    max="50"
+                    variant="outlined"
+                    style="width: {width_pct}%"
+                />
+                <Button
+                    on:click={findNeighbors}
+                    variant="outlined"
+                    class="button_float_right spacing_vert"
+                    disabled={n_neighbors == null}
+                >
+                    <Label>Search</Label>
+                </Button>
+            </div>
+        </div>
+    </div>
+
+    {#await promise_iter_cluster}
+        <div class="app_loading" style="width: {width_pct}%">
+            <LinearProgress indeterminate />
+        </div>
+    {:then iter_cluster_results}
+        {#if iter_cluster_results}
+            {#if history.length > 0}
+                <div class="bold" style="padding-top:40px;">Search History</div>
+                <Set chips={history} let:chip choice>
+                    <Chip {chip}>
+                        <Text>{chip}</Text>
+                    </Chip>
+                </Set>
+            {/if}
+            {#if iter_cluster_results.cluster_comments != null}
+                <ClusterResults
+                    cluster={""}
+                    clusters={clusters}
+                    model={personalized_model}
+                    data={iter_cluster_results}
+                    show_vis={false}
+                    table_width_pct={80}
+                    bind:evidence={evidence}
+                    on:change
+                />
+            {:else}
+                <div class="bold" style="padding-top:40px;">
+                    No results found
+                </div>
+            {/if}
+        {/if}
+    {:catch error}
+        <p style="color: red">{error.message}</p>
+    {/await}
+</div>
+
+<style>
+</style>

indie_label_svelte/src/KeywordSearch.svelte

@@ -0,0 +1,141 @@
+<script>
+    import ClusterResults from "./ClusterResults.svelte";
+    import { error_type } from './stores/error_type_store.js';
+    
+    import Button, { Label } from "@smui/button";
+    import Textfield from "@smui/textfield";
+    import LinearProgress from "@smui/linear-progress";
+    import Chip, { Set, Text } from '@smui/chips';
+    
+
+    export let clusters;
+    export let personalized_model;
+    export let evidence;
+    export let width_pct = 80;
+    export let use_model = true;
+
+    let topic_df_ids = [];
+    let promise_iter_cluster = Promise.resolve(null);
+    let keyword = null;
+    let n_neighbors = null;
+    let cur_iter_cluster = null;
+    let history = [];
+
+    let cur_error_type;
+    error_type.subscribe(value => {
+		cur_error_type = value;
+	});
+
+    async function getIterCluster(search_type) {
+        let req_params = {
+            cluster: cur_iter_cluster,
+            topic_df_ids: topic_df_ids,
+            n_examples: 500, // TEMP
+            pers_model: personalized_model,
+            example_sort: "descending", // TEMP
+            comparison_group: "status_quo", // TEMP
+            search_type: search_type,
+            keyword: keyword,
+            n_neighbors: n_neighbors,
+            error_type: cur_error_type,
+        };
+        console.log("topic_df_ids", topic_df_ids);
+        let params = new URLSearchParams(req_params).toString();
+        const response = await fetch("./get_cluster_results?" + params);
+        const text = await response.text();
+        const data = JSON.parse(text);
+        // if (data["cluster_comments"] == null) {
+        //     return false
+        // }
+        topic_df_ids = data["topic_df_ids"];
+        return data;
+    }
+
+    function findKeywords() {
+        history = [];
+        topic_df_ids = [];
+        promise_iter_cluster = getIterCluster("keyword");
+        history = history.concat("keyword search: " + keyword);
+    }
+
+    function clearHistory() {
+        history = [];
+        promise_iter_cluster = Promise.resolve(null);
+        keyword = "";
+        topic_df_ids = [];
+    }
+</script>
+
+<div>
+    <div>
+        <div>
+            <h6>Keyword Search</h6>
+            <!-- Manual keyword -->
+            <div class="spacing_vert edit_button_row" style="width: 90%; justify-content: space-between">
+                <Textfield
+                    bind:value={keyword}
+                    label="Your keyword or phrase"
+                    variant="outlined"
+                    style="width: 60%"
+                />
+                <Button
+                    on:click={findKeywords}
+                    variant="outlined"
+                    class="spacing_vert"
+                    disabled={keyword == null}
+                >
+                    <Label>Search</Label>
+                </Button>                
+                <!-- <Button
+                    on:click={clearHistory}
+                    variant="outlined"
+                    class="spacing_vert"
+                    disabled={history.length == 0}
+                >
+                    <Label>Clear Search</Label>
+                </Button> -->
+            </div>
+
+            <!-- {#if history.length > 0}
+                <div class="head_6">Search History</div>
+                <Set chips={history} let:chip choice>
+                    <Chip {chip}>
+                        <Text>{chip}</Text>
+                    </Chip>
+                </Set>
+            {/if} -->
+        </div>
+    </div>
+
+    {#await promise_iter_cluster}
+        <div class="app_loading" style="width: {width_pct}%">
+            <LinearProgress indeterminate />
+        </div>
+    {:then iter_cluster_results}
+        {#if iter_cluster_results}
+            {#if iter_cluster_results.cluster_comments != null}
+                <ClusterResults
+                    cluster={""}
+                    clusters={clusters}
+                    model={personalized_model}
+                    data={iter_cluster_results}
+                    show_vis={false}
+                    table_width_pct={90}
+                    table_id={"keyword"}
+                    use_model={use_model}
+                    bind:evidence={evidence}
+                    on:change
+                />
+            {:else}
+                <div class="bold" style="padding-top:40px;">
+                    No results found
+                </div>
+            {/if}
+        {/if}
+    {:catch error}
+        <p style="color: red">{error.message}</p>
+    {/await}
+</div>
+
+<style>
+</style>

indie_label_svelte/src/Labeling.svelte

@@ -0,0 +1,374 @@
+<script lang="ts">
+    import { onMount } from "svelte";
+    import Section from "./Section.svelte";
+    import TopicTraining from "./TopicTraining.svelte";
+    import CommentTable from "./CommentTable.svelte";
+
+    import Textfield from '@smui/textfield';
+    import Button, { Label } from "@smui/button";
+    import LinearProgress from '@smui/linear-progress';
+    import Svelecte from '../node_modules/svelecte/src/Svelecte.svelte';
+    import { user } from './stores/cur_user_store.js';
+
+    let model_name = "";
+    let personalized_models = [];
+    let existing_model_name;
+    let label_modes = [
+        "Create a new model",
+        "Edit an existing model",
+        "Tune your model for a topic area",
+        "Set up a group-based model",
+    ];
+    
+    let clusters_for_tuning = [];
+    let topic;
+
+    // Get current user
+    let cur_user;
+    user.subscribe(value => {
+        if (value != cur_user) {
+            cur_user = value;
+            personalized_models = [];
+            getLabeling();
+        }
+		cur_user = value;
+	});
+
+    // Handle routing
+    let label_mode = label_modes[0];
+	let searchParams = new URLSearchParams(window.location.search);
+    let req_label_mode = parseInt(searchParams.get("label_mode"));
+    if (req_label_mode == 0) {
+        label_mode = label_modes[0];
+    } else if (req_label_mode == 1) {
+        label_mode = label_modes[1];
+    } else if (req_label_mode == 2) {
+        label_mode = label_modes[2];
+    } else if (req_label_mode == 3) {
+        label_mode = label_modes[3];
+    } 
+
+    // Handle group options
+    let options_pol = ['Conservative', 'Liberal', 'Independent', 'Other'];
+    let sel_pol;
+    let options_relig = ['Not important', 'Not too important', 'Somewhat important', 'Very important'];
+    let sel_relig;
+    let options_race = ["White", "Black or African American", "Hispanic", "Asian", "American Indian or Alaskan Native", "Native Hawaiian or Pacific Islander", "Other"];
+    let sel_race;
+    let options_gender = ['Male', 'Female', 'Nonbinary'];
+    let sel_gender;
+    let options_lgbtq = ["Non-LGBTQ+", "LGBTQ+"];
+    let sel_lgbtq;
+
+    let options_axis = ["A: Political affiliation", "B: Gender", "C: Race", "D: LGBTQ+ Identity", "E: Importance of religion"]
+    let selected_axis;
+
+    let group_size;
+
+    function getGroupSize() {
+        let req_params = {
+            sel_gender: sel_gender,
+            sel_pol: sel_pol,
+            sel_relig: sel_relig,
+            sel_race: sel_race,
+            sel_lgbtq: sel_lgbtq,
+        };
+        let params = new URLSearchParams(req_params).toString();
+        fetch("./get_group_size?" + params)
+			.then((r) => r.text())
+			.then(function (r_orig) {
+				let r = JSON.parse(r_orig);
+                group_size = r["group_size"];
+			});
+    }
+
+    let promise = Promise.resolve(null);
+    function handleGroupModel() {
+        promise = getGroupModel();
+    }
+
+    async function getGroupModel() {
+        let req_params = {
+            user: cur_user,
+            model_name: model_name,
+            sel_gender: sel_gender,
+            sel_pol: sel_pol,
+            sel_relig: sel_relig,
+            sel_race: sel_race,
+            sel_lgbtq: sel_lgbtq,
+        };
+        let params = new URLSearchParams(req_params).toString();
+        const response = await fetch("./get_group_model?" + params);
+        const text = await response.text();
+        const data = JSON.parse(text);
+        console.log("getGroupModel", data);
+        return data
+    }
+
+    function getLabeling() {
+        let req_params = {
+            user: cur_user,
+        };
+        let params = new URLSearchParams(req_params).toString();
+        fetch("./get_labeling?" + params)
+			.then((r) => r.text())
+			.then(function (r_orig) {
+				let r = JSON.parse(r_orig);
+                personalized_models = r["personalized_models"];
+                model_name = r["model_name_suggestion"];
+                clusters_for_tuning = r["clusters_for_tuning"];
+                topic = clusters_for_tuning[0]["text"];
+                existing_model_name = personalized_models[0];
+			});
+	}
+	onMount(async () => {
+		getLabeling();
+	});
+</script>
+    
+<div>
+    <h3>Labeling</h3>
+
+    <!-- MODE SELECTION -->
+    {#if label_mode != label_modes[3]}
+    <div id="audit-mode" class="section">
+        <Section
+            section_id="label_mode"
+            section_title="What labeling mode do you want to use?"
+            section_opts={label_modes}
+            bind:value={label_mode}
+            width_pct={40}
+        />
+    </div>
+    {/if}
+
+    {#if label_mode == label_modes[0]}
+        <!-- NEW MODEL -->
+        <div style="width: 80%">
+            In this section, you’ll label some example comments to give a sense of your perspectives on what is toxic or not.
+            We’ll then train a simple model (which we’ll refer to as "your model") that estimates what your toxicity rating would be for the full dataset (with tens of thousands of comments) based on an existing dataset of toxicity ratings provided by different users.
+        </div>
+
+        <div id="new-model" class="section">
+            <h5>Create a New Model</h5>
+
+            <Textfield 
+                bind:value={model_name} 
+                label="Name your personalized model"
+                style="width: 40%"
+            />
+
+            <!-- Labeling -->
+            <h6>Comments to label</h6>
+            <ul>
+                <li>
+                    Comments with scores <b>0</b> and <b>1</b> will be allowed to <b>remain</b> on the platform.
+                </li>
+                <li>
+                    Comments with scores <b>2</b>, <b>3</b>, or <b>4</b> will be <b>deleted</b> from the platform.
+                </li>
+                <li>
+                    Given that some comments may lack context, if you're not sure, feel free to mark the <b>unsure</b> option to skip a comment.
+                </li>
+            </ul>
+
+            <CommentTable mode={"train"} model_name={model_name}/>
+        </div>
+    {:else if label_mode == label_modes[1]}
+        <!-- EXISTING MODEL -->
+        <div id="existing-model" class="section">
+            <h5>Edit an Existing Model</h5>
+            {#key personalized_models}
+            <Section
+                section_id="personalized_model"
+                section_title="Select Your Personalized Model"
+                section_opts={personalized_models}
+                bind:value={existing_model_name}
+                width_pct={40}
+            />
+            {/key}
+
+            <!-- Edit model -->
+            <h6>Comments to label</h6>
+            <ul>
+                <li>
+                    Comments with scores <b>0</b> and <b>1</b> will be allowed to <b>remain</b> on the platform.
+                </li>
+                <li>
+                    Comments with scores <b>2</b>, <b>3</b>, or <b>4</b> will be <b>deleted</b> from the platform.
+                </li>
+                <li>
+                    Given that some comments may lack context, if you're not sure, feel free to mark the <b>unsure</b> option to skip a comment.
+                </li>
+            </ul>
+            {#key existing_model_name}
+                <CommentTable mode={"view"} model_name={existing_model_name}/>
+            {/key}
+        </div>
+    {:else if label_mode == label_modes[2]}
+        <!-- Topic training -->
+        <div class="audit_section">
+            <div class="head_5">Topic model training</div>
+            <p></p>
+            <div class="section_indent">
+                <div>
+                    <p>In what topic area would you like to tune your model?</p>
+                    <Svelecte 
+                        options={clusters_for_tuning} 
+                        labelAsValue={true}
+                        bind:value={topic}
+                        placeholder="Select topic"
+                        on:change={null}
+                        style="width: 50%"
+                    >
+                    </Svelecte>
+                </div>
+
+                <div style="padding-top: 30px">
+                    <!-- Labeling -->
+                    <h6>Comments to label</h6>
+                    <ul>
+                        <li>
+                            Comments with scores <b>0</b> and <b>1</b> will be allowed to <b>remain</b> on the platform.
+                        </li>
+                        <li>
+                            Comments with scores <b>2</b>, <b>3</b>, or <b>4</b> will be <b>deleted</b> from the platform.
+                        </li>
+                        <li>
+                            Given that some comments may lack context, if you're not sure, feel free to mark the <b>unsure</b> option to skip a comment.
+                        </li>
+                    </ul>
+                    {#key topic}
+                    <TopicTraining topic={topic} model_name={model_name} />
+                    {/key}                    
+                </div>
+
+            </div>
+        </div>
+    {:else if label_mode == label_modes[3]}
+        <!-- Group-based model setup -->
+        <div class="head_5">Group model training</div>
+        <p>Please select just <b>one</b> of these five demographic axes (A, B, C, D, or E) to identify with to set up your group-based model:</p>
+
+        <div>
+            <p><b>Demographic axes</b></p>
+            <Svelecte 
+                options={options_axis} 
+                labelAsValue={true}
+                bind:value={selected_axis}
+                placeholder="Select demographic axis"
+                on:change={null}
+                style="width: 50%"
+            >
+            </Svelecte>
+        </div>
+
+        <div class="spacing_vert_40">
+        <!-- {#if selected_axis != null}
+        <p>For this axis, please select a group that you would like to identify with to set up your group-based model:</p>
+        {/if} -->
+        <div style="{selected_axis == options_axis[0] ? 'display:initial': 'display:none'}" >
+            <p><b>A: Political affiliation</b></p>
+            <Svelecte 
+                options={options_pol} 
+                labelAsValue={true}
+                bind:value={sel_pol}
+                placeholder="Select political affiliation"
+                on:change={getGroupSize}
+                style="width: 50%"
+            >
+            </Svelecte>
+        </div>
+        <!-- {:else if selected_axis == options_axis[1]} -->
+        <div style="{selected_axis == options_axis[1] ? 'display:initial': 'display:none'}" >
+            <p><b>B: Gender</b></p>
+            <Svelecte 
+                options={options_gender} 
+                labelAsValue={true}
+                bind:value={sel_gender}
+                placeholder="Select gender"
+                on:change={getGroupSize}
+                style="width: 50%"
+            >
+            </Svelecte>
+        </div>
+        <!-- {:else if selected_axis == options_axis[2]} -->
+        <div style="{selected_axis == options_axis[2] ? 'display:initial': 'display:none'}" >
+            <p><b>C: Race (select all that apply)</b></p>
+            <Svelecte 
+                options={options_race} 
+                labelAsValue={true}
+                bind:value={sel_race}
+                placeholder="Select race(s)"
+                on:change={getGroupSize}
+                style="width: 50%"
+                multiple=true
+            >
+            </Svelecte>
+        </div>
+        <!-- {:else if selected_axis == options_axis[3]} -->
+        <div style="{selected_axis == options_axis[3] ? 'display:initial': 'display:none'}" >
+            <p><b>D: LGBTQ+ Identity</b></p>
+            <Svelecte 
+                options={options_lgbtq} 
+                labelAsValue={true}
+                bind:value={sel_lgbtq}
+                placeholder="Select LGBTQ+ identity"
+                on:change={getGroupSize}
+                style="width: 50%"
+            >
+            </Svelecte>
+        </div>
+        <!-- {:else if selected_axis == options_axis[4]} -->
+        <div style="{selected_axis == options_axis[4] ? 'display:initial': 'display:none'}" >
+            <p><b>E: Importance of religion</b></p>
+            <Svelecte 
+                options={options_relig} 
+                labelAsValue={true}
+                bind:value={sel_relig}
+                placeholder="Select importance of religion"
+                on:change={getGroupSize}
+                style="width: 50%"
+            >
+            </Svelecte>
+        </div>
+        <!-- {/if} -->
+        </div>
+
+        {#if group_size}
+        <div class="spacing_vert_40">
+            <b>Number of labelers with matching traits</b>: {group_size}
+        </div>
+        {/if}
+        
+        <div class=spacing_vert_60>
+            <Button
+                on:click={handleGroupModel}
+                variant="outlined"
+                class=""
+                disabled={group_size == null}
+            >
+                <Label>Train group-based model</Label>
+            </Button>
+        </div>
+        
+        <!-- Performance -->
+        {#await promise}
+            <div class="app_loading spacing_vert_20">
+                <LinearProgress indeterminate />
+            </div>
+        {:then group_model_res}
+            {#if group_model_res}
+            <div class="spacing_vert_20">
+                <p>Model for your selected group memberships has been successfully tuned.</p>
+                <p>MAE: {group_model_res["mae"]}</p>
+            </div>
+            {/if}
+        {:catch error}
+            <p style="color: red">{error.message}</p>
+        {/await}
+    {/if}
+</div>
+
+<style>
+</style>

indie_label_svelte/src/MainPanel.svelte

@@ -0,0 +1,79 @@
+<script lang="ts">
+	import Labeling from "./Labeling.svelte";
+	import Auditing from "./Auditing.svelte";
+	import AppOld from "./AppOld.svelte";
+
+	import Tab, { Label } from "@smui/tab";
+	import TabBar from "@smui/tab-bar";
+
+	export let model;
+	// export let topic;
+	export let error_type;
+
+	let app_versions = ["old", "new"];
+	let app_version = "new";
+
+	// Handle routing
+	let active = "auditing";
+    let searchParams = new URLSearchParams(window.location.search);
+    let tab = searchParams.get("tab");
+	if (tab == "labeling") {
+		active = "labeling";
+	}
+	
+</script>
+
+<svelte:head>
+	<title>IndieLabel</title>
+</svelte:head>
+
+<div class="auditing_panel">
+	<div class="tab_header">
+		<TabBar tabs={["labeling", "auditing"]} let:tab bind:active>
+			<Tab {tab}>
+				<Label>{tab}</Label>
+			</Tab>
+		</TabBar>
+	</div>
+
+	<div class="panel_contents">
+		<!-- VERSION SELECTION -->
+		<!-- <div>
+			<Section
+				section_id="app_version"
+				section_title="What app version do you want to use?"
+				section_opts={app_versions}
+                width_pct={40}
+				bind:value={app_version}
+			/>
+		</div> -->
+
+		{#if app_version == app_versions[0]}
+			<!-- OLD VERSION -->
+			<AppOld />
+		{:else if app_version == app_versions[1]}
+			<!-- NEW VERSION -->
+			<div>
+				<div id="labeling" hidden={active == "auditing"} >
+					<Labeling/>	
+				</div>
+
+				<div id="auditing" hidden={active == "labeling"} >
+					<Auditing bind:personalized_model={model} bind:cur_error_type={error_type} on:change/>	
+				</div>
+			</div>
+		{/if}
+
+		<!-- TEMP -->
+		<!-- {#key model}
+			<div>Model: {model}</div> 
+		{/key} -->
+	</div>
+</div>
+
+<style>
+	.panel_contents {
+		padding: 50px;
+		margin-top: 50px;
+	}
+</style>

indie_label_svelte/src/ModelPerf.svelte

@@ -0,0 +1,82 @@
+<script lang="ts">
+    import { VegaLite } from "svelte-vega";
+	import type { View } from "svelte-vega";
+
+    import LayoutGrid, { Cell } from "@smui/layout-grid";
+    import Card, { Content } from '@smui/card';
+
+    export let data;
+
+    let perf_plot_spec = data["perf_plot_json"];
+    let perf_plot_data = perf_plot_spec["datasets"][
+        perf_plot_spec["layer"][0]["data"]["name"]
+    ];
+    let perf_plot_view: View;
+
+    // let perf_plot2_spec = data["perf_plot2_json"];
+    // let perf_plot2_data = perf_plot2_spec["datasets"][perf_plot2_spec["data"]["name"]];
+    // let perf_plot2_view: View;
+</script>
+
+<div>
+    <h6>Your Model Performance</h6>
+    <LayoutGrid>
+        <Cell span={8}>
+            <div class="card-container">
+                <Card variant="outlined" padded>
+                    <p class="mdc-typography--button"><b>Interpreting your model performance</b></p>
+                    <ul>
+                        <li>
+                            The <b>Mean Absolute Error (MAE)</b> metric indicates the average absolute difference between your model's rating and your actual rating on a held-out set of comments.
+                        </li>
+                        <li>
+                            You want your model to have a <b>lower</b> MAE (indicating <b>less error</b>).
+                        </li>
+                        <li>
+                            <b>Your current MAE: {data["mae"]}</b>
+                            <ul>
+                                <li>{@html data["mae_status"]}</li>
+                                <!-- <li>
+                                    This is <b>better</b> (lower) than the average MAE for other users, so your model appears to <b>better capture</b> your views than the typical user model.
+                                </li> -->
+                            </ul>
+                        </li>
+                    </ul>
+                </Card>
+            </div>
+        </Cell>
+    </LayoutGrid>
+    <div>
+        <!-- Overall -->
+        <!-- <table>
+            <tbody>
+                <tr>
+                    <td>
+                        <span class="bold">Mean Absolute Error (MAE)</span><br>
+                        
+                    </td>
+                    <td>
+                        <span class="bold-large">{data["mae"]}</span>
+                    </td>
+                </tr>
+                <tr>
+                    <td>
+                        <span class="bold">Average rating difference</span><br>
+                        This metric indicates the average difference between your model's rating and your actual rating on a held-out set of comments.
+                    </td>
+                    <td>
+                        <span class="bold-large">{data["avg_diff"]}</span>
+                    </td>
+                </tr>
+            </tbody>
+        </table> -->
+
+        <!-- Performance visualization -->
+        <div>
+            <VegaLite {perf_plot_data} spec={perf_plot_spec} bind:view={perf_plot_view}/>
+        </div>
+    </div>
+</div>
+
+<style>
+</style>

indie_label_svelte/src/OverallResults.svelte

@@ -0,0 +1,156 @@
+<script lang="ts">
+    import { VegaLite } from "svelte-vega";
+	import type { View } from "svelte-vega";
+
+    import IconButton from '@smui/icon-button';
+    import LayoutGrid, { Cell } from "@smui/layout-grid";
+    import Card, { Content } from '@smui/card';
+
+    export let data;
+    export let clusters;
+    export let personalized_model;
+    export let cluster = "";
+    
+    let show_step1_info = false;
+
+    // Topic Overview Plot
+    let topic_overview_json = data["overall_perf"]["topic_overview_plot_json"];
+    let topic_overview_data = topic_overview_json["datasets"][topic_overview_json["layer"][0]["data"]["name"]];
+    let topic_overview_spec = topic_overview_json;
+    let topic_overview_view: View;
+
+    // // Overall Histogram
+    // let overall_hist_json = data["overall_perf"]["overall_hist_json"];
+    // let overall_hist_data = overall_hist_json["datasets"][overall_hist_json["data"]["name"]];
+    // let overall_hist_spec = overall_hist_json;
+    // let overall_hist_view: View;
+
+    // // Class-conditional Histogram
+    // let class_cond_plot_json = data["overall_perf"]["class_cond_plot_json"];
+    // let class_cond_plot_data = class_cond_plot_json["datasets"][class_cond_plot_json["data"]["name"]];
+    // let class_cond_plot_spec = class_cond_plot_json;
+    // let class_cond_plot_view: View;
+
+</script>
+
+<div>
+    <div>
+        <span class="head_6">All topics</span>
+        <IconButton 
+            class="material-icons grey_button"
+            size="normal"
+            on:click={() => (show_step1_info = !show_step1_info)}
+        >
+            help_outline
+        </IconButton>
+    </div>
+    {#if show_step1_info}
+        <LayoutGrid>
+            <Cell span={8}>
+                <div class="card-container">
+                    <Card variant="outlined" padded>
+                        <p class="mdc-typography--button"><b>Interpreting this visualization</b></p>
+                        <ul>
+                            <li>
+                                Each <b>box</b> in this plot represents a set of comments that belong in a given <b>topic area</b>
+                            </li>
+                            <li>
+                                The <b>x-axis</b> represents our prediction of <b>your</b> toxicity rating for each topic (we'll call these "your ratings")
+                                <ul>
+                                    <li>
+                                        The <b>left side</b> (white background) is the <b>Non-toxic</b> side (comments that'll be allowed to remain)
+                                    </li>
+                                    <li>
+                                        The <b>right side</b> (grey background) is the <b>Toxic</b> side (comments that will be deleted)
+                                    </li>
+                                    <li>
+                                        Comment topic area boxes are plotted along the x-axis based on our prediction of your <b>average</b> toxicity rating for comments in that set
+                                    </li>
+                                </ul>
+                            </li>
+                
+                            <li>
+                                The <b>color</b> of the box indicates the <b>system's rating</b> for the same comment topic; you may want to focus on the <b>red-colored boxes</b> that indicate <b>disagreements</b> between "your ratings" and the system's ratings
+                            </li>
+                        </ul>
+                    </Card>
+                </div>
+            </Cell>
+        </LayoutGrid>
+        {/if}
+    <div class="row">
+        <div class="col s8">
+            <VegaLite {topic_overview_data} spec={topic_overview_spec} bind:view={topic_overview_view}/>
+        </div>
+    </div>
+
+    <!-- Old visualizations -->
+    <!-- <div style="margin-top: 500px">
+        <h6>Overall Performance</h6>
+        <div class="row">
+            <div class="col s12">
+                <div id="overall_perf">
+                    <table>
+                        <tbody>
+                            <tr class="custom-blue">
+                                <td class="bold"
+                                    >System {data[
+                                        "overall_perf"
+                                    ]["metric"]} with YOUR labels</td
+                                >
+                                <td>
+                                    <span class="bold-large"
+                                        >{data[
+                                            "overall_perf"
+                                        ]["user_metric"]}</span
+                                    >
+                                    (Percentile: {data[
+                                        "overall_perf"
+                                    ]["user_percentile"]})
+                                </td>
+                            </tr>
+                            <tr>
+                                <td class="bold"
+                                    >System {data[
+                                        "overall_perf"
+                                    ]["metric"]} with OTHER USERS' labels</td
+                                >
+                                <td>
+                                    <span class="bold-large"
+                                        >{data[
+                                            "overall_perf"
+                                        ]["other_metric"]}</span
+                                    >
+                                    (95% CI: [{data[
+                                        "overall_perf"
+                                    ]["other_ci_low"]}, {data[
+                                        "overall_perf"
+                                    ]["other_ci_high"]}])
+                                </td>
+                            </tr>
+                        </tbody>
+                    </table>
+                </div>
+            </div>
+        </div>
+        <div class="row">
+            <div class="col s8">
+                <VegaLite {overall_hist_data} spec={overall_hist_spec} bind:view={overall_hist_view}/>
+            </div>
+        </div>
+        
+        <h6>Performance Breakdown</h6>
+        <div class="row">
+            <div class="col s12">
+                <div class="row">
+                    <div class="col s12">
+                        <VegaLite {class_cond_plot_data} spec={class_cond_plot_spec} bind:view={class_cond_plot_view} />
+                    </div>
+                </div>
+            </div>
+        </div>
+    </div> -->
+
+</div>
+<style>
+</style>

indie_label_svelte/src/Results.svelte

@@ -0,0 +1,206 @@
+<script lang="ts">
+    import { onMount } from "svelte";
+    import ClusterResults from "./ClusterResults.svelte";
+
+    import Button, { Label } from "@smui/button";
+    import LinearProgress from "@smui/linear-progress";
+    import Checkbox from '@smui/checkbox';
+    import DataTable, {
+        Head,
+        Body,
+        Row,
+        Cell,
+        Label,
+        SortValue,
+    } from "@smui/data-table";
+    import FormField from "@smui/form-field";
+
+    let cur_examples = [];
+    let promise = Promise.resolve(null);
+
+    let scaffold_methods = ["personal", "personal_group", "prompts"];
+
+    let all_users = [];
+	async function getUsers() {    
+		const response = await fetch("./get_users");
+		const text = await response.text();
+		const data = JSON.parse(text);
+		all_users = data["users"];
+        promise = getResults();
+	}
+
+    onMount(async () => {
+        getUsers()
+    });
+    
+    async function getResults() {
+        let req_params = {
+            users: all_users
+        };
+        let params = new URLSearchParams(req_params).toString();
+        const response = await fetch("./get_results?" + params);
+        const text = await response.text();
+        const data = JSON.parse(text);
+
+        let results = data["results"];
+        return results;
+    }
+
+    function get_complete_ratio(reports) {
+        let total = reports.length;
+        let complete = reports.filter(item => item.complete_status).length;
+        return "" + complete + "/" + total + " complete";
+    }
+
+    function get_complete_count(reports) {
+        return reports.filter(item => item.complete_status).length;
+    }
+
+    function get_summary(reports) {
+        let summary = "";
+        let total_audits = 0
+        for (const scaffold_method of scaffold_methods) {
+            if (reports[scaffold_method]) {
+                let cur_reports = reports[scaffold_method];
+                let cur_ratio = get_complete_ratio(cur_reports);
+                let cur_result = "<li><b>" + scaffold_method + "</b>: " + cur_ratio + "</li>";
+                summary += cur_result;
+                let cur_complete = get_complete_count(cur_reports);
+                total_audits += cur_complete;
+            }
+        }
+
+        let top_summary = "<li><b>Total audits</b>: " + total_audits + "</li>";
+        summary = "<ul>" + top_summary + summary + "</ul>";
+        return summary;
+    }
+
+    function get_url(user, scaffold_method) {
+        return "http://localhost:5001/?user=" + user + "&scaffold=" + scaffold_method;
+    }
+</script>
+
+<svelte:head>
+    <title>Results</title>
+</svelte:head>
+
+<div class="panel">
+    <div class="panel_contents">
+        <div>
+            <h3>Results</h3>
+        </div>
+
+        <div style="padding-top:50px">
+            {#await promise}
+                <div class="app_loading">
+                    <LinearProgress indeterminate />
+                </div>
+            {:then results}
+                {#if results}
+                    {#each results as user_report}
+                        <div class="head_3">{user_report["user"]}</div>
+                        <div class="section_indent">
+                            <div class="head_5">Summary</div>
+                            <div>{@html get_summary(user_report)}</div>
+                            <ul>
+                                <li>Labeling pages
+                                    <ul>
+                                        <li>
+                                            <a href="http://localhost:5001/?user={user_report["user"]}&tab=labeling&label_mode=3" target="_blank">Group-based model</a>
+                                        </li>
+                                        <li>
+                                            <a href="http://localhost:5001/?user={user_report["user"]}&tab=labeling&label_mode=0" target="_blank">Personalized model</a>
+                                        </li>
+                                    </ul>
+                                </li>
+                                <li>Auditing pages
+                                    <ul>
+                                        <li>
+                                            <a href="http://localhost:5001/?user={user_report["user"]}&scaffold=personal_group" target="_blank">Group-based audit - personal scaffold</a>
+                                        </li>
+                                        <li>
+                                            <a href="http://localhost:5001/?user={user_report["user"]}&scaffold=personal" target="_blank">Individual audit - personal scaffold</a>
+                                        </li>
+                                        <li>
+                                            <a href="http://localhost:5001/?user={user_report["user"]}&scaffold=prompts" target="_blank">Individual audit - prompt scaffold</a>
+                                        </li>
+                                    </ul>
+                                </li>
+                            </ul>
+                        </div>
+                        {#each scaffold_methods as scaffold_method}
+                            {#if user_report[scaffold_method]}
+                            <div class="spacing_vert_60 section_indent">
+                                <div class="head_5">
+                                    {scaffold_method} ({get_complete_ratio(user_report[scaffold_method])})
+                                    [<a href={get_url(user_report["user"], scaffold_method)} target="_blank">link</a>]
+                                </div>
+                                {#each user_report[scaffold_method] as report}
+                                    <div class="spacing_vert_40 section_indent">
+                                        <div class="head_6_non_cap">
+                                            {report["title"]}
+                                        </div>
+
+                                        <div class="spacing_vert_20">
+                                            <div class="">
+                                                <b>Error type</b>
+                                            </div>
+                                            {report["error_type"]}
+                                        </div>
+
+                                        <div class="spacing_vert_20">
+                                            <div class="">
+                                                <b>Evidence</b>
+                                            </div>
+                                            {#if report["evidence"].length > 0}
+                                            <ClusterResults 
+                                                cluster={null} 
+                                                model={null} 
+                                                data={{"cluster_comments": report["evidence"]}} 
+                                                show_vis={false} 
+                                                show_checkboxes={false} 
+                                                table_width_pct={100} 
+                                                rowsPerPage={10} 
+                                                table_id={"panel"}
+                                            />
+                                            {:else}
+                                                <p class="grey_text">
+                                                    No examples added
+                                                </p>
+                                            {/if}
+                                        </div>
+
+                                        <div class="spacing_vert_20">
+                                            <div class="">
+                                                <b>Summary/Suggestions</b>
+                                            </div>
+                                            {report["text_entry"]}
+                                        </div>
+
+                                        <div class="spacing_vert_20">
+                                            <b>Completed</b>
+                                            <FormField>
+                                                <Checkbox checked={report["complete_status"]} disabled/>
+                                            </FormField>
+                                        </div>
+                                        
+                                    </div>
+                                {/each}
+                            </div>
+                            {/if}
+                        {/each}
+                    {/each}
+                {/if}
+            {:catch error}
+                <p style="color: red">{error.message}</p>
+            {/await}
+        </div>
+    </div>
+</div>
+
+<style>
+    .panel {
+        width: 80%;
+        padding: 50px;
+    }
+</style>

indie_label_svelte/src/Section.svelte

@@ -0,0 +1,36 @@
+<script>
+    import { createEventDispatcher } from 'svelte';
+
+    import Select, { Option } from "@smui/select";
+    export let section_id;
+    export let section_title;
+    export let section_opts;
+    export let value;
+    export let width_pct;
+
+    const dispatch = createEventDispatcher();
+
+    function sendChange() {
+        dispatch("select_changed", {});
+    }
+</script>
+
+<div>
+    <!-- <label for={section_id}>{section_title}</label> -->
+    <Select 
+        label={section_title} 
+        bind:value id={section_id} 
+        style="width: {width_pct}%"
+        on:change
+    >
+        {#each section_opts as opt, i}
+            <Option value={opt}>{opt}</Option>
+        {/each}
+    </Select>
+</div>
+
+<style>
+    div {
+        padding: 20px 10px;
+    }
+</style>

indie_label_svelte/src/SelectUserDialog.svelte

@@ -0,0 +1,66 @@
+<script lang="ts">
+    import Dialog, { Title, Content, Actions } from "@smui/dialog";
+    import Button, { Label } from "@smui/button";
+    import Textfield from "@smui/textfield";
+    import Select, { Option } from "@smui/select";
+    import { user } from "./stores/cur_user_store.js";
+    import { users } from "./stores/all_users_store.js";
+
+    export let open;
+    export let cur_user;
+    let cur_user_tf = cur_user;
+    let cur_user_sel = cur_user;
+
+    let all_users;
+    users.subscribe((value) => {
+        all_users = value;
+    });
+
+    function updateUserTextField() {
+        user.update((value) => cur_user_tf);
+        if (!all_users.includes(user)) {
+            all_users = all_users.concat(cur_user_tf);
+            users.update(all_users);
+        } 
+        open = false;
+    }
+
+    function updateUserSel() {
+        user.update((value) => cur_user_sel);
+        open = false;
+    }
+</script>
+
+<div>
+    <Dialog
+        bind:open
+        aria-labelledby="simple-title"
+        aria-describedby="simple-content"
+    >
+        <!-- Title cannot contain leading whitespace due to mdc-typography-baseline-top() -->
+        <Title id="simple-title">Select Current User</Title>
+        <Content id="simple-content">
+            <Textfield bind:value={cur_user_tf} label="Enter user's name" />
+
+            <Select bind:value={cur_user_sel} label="Select Menu">
+                {#each all_users as u}
+                    <Option value={u}>{u}</Option>
+                {/each}
+            </Select>
+        </Content>
+        <Actions>
+            <Button on:click={updateUserTextField}>
+                <Label>Update from TextField</Label>
+            </Button>
+            <Button on:click={updateUserSel}>
+                <Label>Update from Select</Label>
+            </Button>
+        </Actions>
+    </Dialog>
+</div>
+
+<style>
+    :global(.mdc-dialog__surface) {
+        height: 300px;
+    }
+</style>

indie_label_svelte/src/StudyLinks.svelte

@@ -0,0 +1,59 @@
+<script lang="ts">
+    import { user } from "./stores/cur_user_store.js";
+
+    let cur_user;
+    user.subscribe((value) => {
+        cur_user = value;
+    });
+
+</script>
+
+<svelte:head>
+    <title>Study Links</title>
+</svelte:head>
+
+<div class="panel">
+    <div class="panel_contents">
+        <div>
+            <h3>Study Links</h3>
+        </div>
+
+        <div>
+            <!-- <div class="head_5">{cur_user}</div> -->
+            <div class="section_indent">
+                <ul>
+                    <li>Labeling pages
+                        <ul>
+                            <li>
+                                <a href="http://localhost:5001/?user={cur_user}&tab=labeling&label_mode=3" target="_blank">Group-based model</a>
+                            </li>
+                            <li>
+                                <a href="http://localhost:5001/?user={cur_user}&tab=labeling&label_mode=0" target="_blank">Personalized model</a>
+                            </li>
+                        </ul>
+                    </li>
+                    <li>Auditing pages
+                        <ul>
+                            <li>
+                                <a href="http://localhost:5001/?user={cur_user}&scaffold=personal_group" target="_blank">Group-based audit - personal scaffold</a>
+                            </li>
+                            <li>
+                                <a href="http://localhost:5001/?user={cur_user}&scaffold=personal" target="_blank">Individual audit - personal scaffold</a>
+                            </li>
+                            <li>
+                                <a href="http://localhost:5001/?user={cur_user}&scaffold=prompts" target="_blank">Individual audit - prompt scaffold</a>
+                            </li>
+                        </ul>
+                    </li>
+                </ul>
+            </div>
+        </div>
+    </div>
+</div>
+
+<style>
+    .panel {
+        width: 80%;
+        padding: 50px;
+    }
+</style>

indie_label_svelte/src/TopicTraining.svelte

@@ -0,0 +1,236 @@
+<script lang="ts">
+    import { onMount } from "svelte";
+    import ModelPerf from "./ModelPerf.svelte";
+    import Button, { Label } from "@smui/button";
+    import DataTable, { Head, Body, Row, Cell } from "@smui/data-table";
+    import LinearProgress from '@smui/linear-progress';
+    import { user } from './stores/cur_user_store.js';
+    import { model_chosen } from './stores/cur_model_store.js';
+
+    export let topic;
+    export let model_name = null;
+
+    let to_label = {};
+    let promise = Promise.resolve(null);
+
+    // Get current user
+    let cur_user;
+    user.subscribe(value => {
+		cur_user = value;
+	});
+
+    // Get current model
+    if (model_name == null) {
+        model_chosen.subscribe(value => {
+            model_name = value;
+        });
+    }
+
+    function getCommentsToLabel() {
+        let req_params = {
+            topic: topic,
+        };
+        let params = new URLSearchParams(req_params).toString();
+        fetch("./get_comments_to_label_topic?" + params)
+            .then((r) => r.text())
+            .then(function (r_orig) {
+                let r = JSON.parse(r_orig);
+                // Append comment rows to to_label object
+                r["to_label"].forEach((key) => (to_label[key] = null));
+            });
+    }
+
+    onMount(async () => {
+        getCommentsToLabel();
+    });
+
+    function handleLoadCommentsButton() {
+        getCommentsToLabel();
+    }
+
+    function handleTrainModelButton() {
+        promise = trainModel();
+    }
+
+    function getRatings() {
+        // Get rating for each comment from HTML elems
+        let ratings = {};
+        Object.entries(to_label).forEach(function ([comment, orig_rating], i) {
+            var radio_btns = document.getElementsByName(
+                "comment_" + i.toString()
+            );
+            let length = radio_btns.length;
+            for (var i = 0; i < length; i++) {
+                if (radio_btns[i].checked) {
+                    ratings[comment] = radio_btns[i].value;
+                    break;
+                }
+            }
+        });
+        return ratings;
+    }
+
+    async function trainModel() {
+        let ratings = getRatings();
+        ratings = JSON.stringify(ratings);
+
+        let req_params = {
+            model_name: model_name,
+            ratings: ratings,
+            user: cur_user,
+            topic: topic,
+        };
+
+        console.log("topic training model name", model_name);
+        let params = new URLSearchParams(req_params).toString();
+        const response = await fetch("./get_personalized_model_topic?" + params); // TODO
+        const text = await response.text();
+        const data = JSON.parse(text);
+        // to_label = data["ratings_prev"];
+        model_name = data["new_model_name"];
+        model_chosen.update((value) => model_name);
+
+        console.log("topicTraining", data);
+        return data;
+    }
+</script>
+
+<div>
+{#if topic}
+    <div class="label_table_expandable spacing_vert">
+        <DataTable
+            table$aria-label="Comments to label"
+            style="width: 100%;"
+            stickyHeader
+        >
+            <Head>
+                <Row>
+                    <Cell style="width: 50%">Comment</Cell>
+                    <Cell style="background-color: #c3ecdb">
+                        0: <br>Not-at-all toxic<br>(Keep)<br>
+                    </Cell>
+                    <Cell style="background-color: white">
+                        1: <br>Slightly toxic<br>(Keep)<br>
+                    </Cell>
+                    <Cell style="background-color: #ffa894">
+                        2: <br>Moderately toxic<br>(Delete)<br>
+                    </Cell>
+                    <Cell style="background-color: #ff7a5c">
+                        3: <br>Very toxic<br>(Delete)<br>
+                    </Cell>
+                    <Cell style="background-color: #d62728">
+                        4: <br>Extremely toxic<br>(Delete)<br>
+                    </Cell>
+                    <Cell style="background-color: #808080">
+                        <br>Unsure<br>(Skip)<br>
+                    </Cell>
+                </Row>
+            </Head>
+            <Body>
+                {#if to_label}
+                    {#each Object.keys(to_label) as comment, i}
+                        <Row>
+                            <Cell>
+                                <div class="spacing_vert">{comment}</div>
+                            </Cell>
+                            <Cell>
+                                <label>
+                                    <input
+                                        name="comment_{i}"
+                                        type="radio"
+                                        value="0"
+                                        checked={to_label[comment] == "0"}
+                                    />
+                                    <span />
+                                </label>
+                            </Cell>
+                            <Cell>
+                                <label>
+                                    <input
+                                        name="comment_{i}"
+                                        type="radio"
+                                        value="1"
+                                        checked={to_label[comment] == "1"}
+                                    />
+                                    <span />
+                                </label>
+                            </Cell>
+                            <Cell>
+                                <label>
+                                    <input
+                                        name="comment_{i}"
+                                        type="radio"
+                                        value="2"
+                                        checked={to_label[comment] == "2"}
+                                    />
+                                    <span />
+                                </label>
+                            </Cell>
+                            <Cell>
+                                <label>
+                                    <input
+                                        name="comment_{i}"
+                                        type="radio"
+                                        value="3"
+                                        checked={to_label[comment] == "3"}
+                                    />
+                                    <span />
+                                </label>
+                            </Cell>
+                            <Cell>
+                                <label>
+                                    <input
+                                        name="comment_{i}"
+                                        type="radio"
+                                        value="4"
+                                        checked={to_label[comment] == "4"}
+                                    />
+                                    <span />
+                                </label>
+                            </Cell>
+                            <Cell>
+                                <label>
+                                    <input
+                                        name="comment_{i}"
+                                        type="radio"
+                                        value="-1"
+                                        checked={to_label[comment] == "-1"}
+                                    />
+                                    <span />
+                                </label>
+                            </Cell>
+                        </Row>
+                    {/each}
+                {/if}
+            </Body>
+        </DataTable>
+    </div>
+
+    <div class="">
+        <Button on:click={handleTrainModelButton} variant="outlined">
+            <Label>Tune Model</Label>
+        </Button>
+        <Button on:click={handleLoadCommentsButton} variant="outlined">
+            <Label>Fetch More Comments To Label</Label>
+        </Button>
+    </div>
+
+    <!-- Performance -->
+    {#await promise}
+        <div class="app_loading spacing_vert_20">
+            <LinearProgress indeterminate />
+        </div>
+    {:then perf_results}
+        {#if perf_results}
+            <div class="spacing_vert_20">
+                Model for the topic {topic} has been successfully tuned. You can now proceed to explore this topic.
+            </div>
+        {/if}
+    {:catch error}
+        <p style="color: red">{error.message}</p>
+    {/await}
+{/if}
+</div>
+
+<style>
+</style>

indie_label_svelte/src/main.ts

@@ -0,0 +1,7 @@
+import App from "./App.svelte";
+
+const app = new App({
+  target: document.body,
+});
+
+export default app;

indie_label_svelte/src/stores/all_users_store.js

@@ -0,0 +1,6 @@
+import { writable } from 'svelte/store';
+
+// Fallback if request doesn't work
+let all_users = ["DemoUser"];
+
+export const users = writable(all_users);

indie_label_svelte/src/stores/cur_model_store.js

@@ -0,0 +1,3 @@
+import { writable } from 'svelte/store';
+
+export const model_chosen = writable("");

indie_label_svelte/src/stores/cur_topic_store.js

@@ -0,0 +1,3 @@
+import { writable } from 'svelte/store';
+
+export const topic_chosen = writable("0_shes_woman_lady_face");

indie_label_svelte/src/stores/cur_user_store.js

@@ -0,0 +1,3 @@
+import { writable } from 'svelte/store';
+
+export const user = writable("DemoUser");

indie_label_svelte/src/stores/error_type_store.js

@@ -0,0 +1,3 @@
+import { writable } from 'svelte/store';
+
+export const error_type = writable("Both");

indie_label_svelte/src/stores/new_evidence_store.js

@@ -0,0 +1,3 @@
+import { writable } from 'svelte/store';
+
+export const new_evidence = writable([]);

indie_label_svelte/src/stores/open_evidence_store.js

@@ -0,0 +1,3 @@
+import { writable } from 'svelte/store';
+
+export const open_evidence = writable([]);

indie_label_svelte/tsconfig.json

@@ -0,0 +1,5 @@
+{
+  "extends": "@tsconfig/svelte/tsconfig.json",
+  "include": ["src/**/*"],
+  "exclude": ["node_modules/*", "__sapper__/*", "public/*"]
+}

requirements.txt

@@ -0,0 +1,15 @@
+altair==4.2.0
+altair_saver==0.5.0
+altair_transform==0.2.0
+bertopic==0.11.0
+Flask==2.2.2
+matplotlib==3.5.3
+mne==1.1.1
+numpy==1.22.4
+pandas==1.4.3
+scikit_learn==1.1.2
+scikit_surprise==1.1.1
+scipy==1.9.0
+sentence_transformers==2.2.2
+surprise==0.1
+torch==1.12.1

server.py

@@ -0,0 +1,797 @@
+from flask import Flask, send_from_directory
+from flask import request
+
+import random
+import json
+
+import numpy as np
+import matplotlib.pyplot as plt
+import pandas as pd
+import pickle
+import os
+
+from sklearn.metrics import mean_absolute_error
+from sklearn.metrics import mean_squared_error
+from sklearn.metrics import confusion_matrix
+import math
+import altair as alt
+import matplotlib.pyplot as plt
+import time
+
+import audit_utils as utils
+
+app = Flask(__name__)
+
+# Path for our main Svelte page
+@app.route("/")
+def base():
+    return send_from_directory('indie_label_svelte/public', 'index.html')
+
+# Path for all the static files (compiled JS/CSS, etc.)
+@app.route("/<path:path>")
+def home(path):
+    return send_from_directory('indie_label_svelte/public', path)
+
+
+########################################
+# ROUTE: /AUDIT_SETTINGS
+comments_grouped_full_topic_cat = pd.read_pickle("data/comments_grouped_full_topic_cat2_persp.pkl")
+
+@app.route("/audit_settings")
+def audit_settings():
+    # Fetch page content
+    user = request.args.get("user")
+    scaffold_method = request.args.get("scaffold_method")
+
+    user_models = utils.get_all_model_names(user)
+    grp_models = [m for m in user_models if m.startswith(f"model_{user}_group_")]
+
+    clusters = utils.get_unique_topics()
+    if len(user_models) > 2 and scaffold_method != "tutorial" and user != "DemoUser":
+        # Highlight topics that have been tuned
+        tuned_clusters = [m.lstrip(f"model_{user}_") for m in user_models if (m != f"model_{user}" and not m.startswith(f"model_{user}_group_"))]
+        other_clusters = [c for c in clusters if c not in tuned_clusters]
+        tuned_options = {
+            "label": "Topics with tuned models",
+            "options": [{"value": i, "text": cluster} for i, cluster in enumerate(tuned_clusters)],
+        }
+        other_options = {
+            "label": "All other topics",
+            "options": [{"value": i, "text": cluster} for i, cluster in enumerate(other_clusters)],
+        }
+        clusters_options = [tuned_options, other_options]
+    else:
+        clusters_options = [{
+            "label": "All auto-generated topics",
+            "options": [{"value": i, "text": cluster} for i, cluster in enumerate(clusters)],
+        },]
+
+    if scaffold_method == "personal_cluster":
+        cluster_model = user_models[0]
+        personal_cluster_file = f"./data/personal_cluster_dfs/{cluster_model}.pkl"
+        if os.path.isfile(personal_cluster_file) and cluster_model != "":
+            print("audit_settings", personal_cluster_file, cluster_model)
+            topics_under_top, topics_over_top = utils.get_personal_clusters(cluster_model)
+            pers_cluster = topics_under_top + topics_over_top
+            pers_cluster_options = {
+                "label": "Personalized clusters",
+                "options": [{"value": i, "text": cluster} for i, cluster in enumerate(pers_cluster)],
+            }
+            clusters_options.insert(0, pers_cluster_options)
+
+    clusters_for_tuning = utils.get_large_clusters(min_n=150)
+    clusters_for_tuning_options = [{"value": i, "text": cluster} for i, cluster in enumerate(clusters_for_tuning)]  # Format for Svelecte UI element
+
+    context = {
+        "personalized_models": user_models,
+        "personalized_model_grp": grp_models,
+        "perf_metrics": ["Average rating difference", "Mean Absolute Error (MAE)", "Root Mean Squared Error (RMSE)", "Mean Squared Error (MSE)"],
+        "breakdown_categories": ['Topic', 'Toxicity Category', 'Toxicity Severity'],
+        "clusters": clusters_options,
+        "clusters_for_tuning": clusters_for_tuning_options,
+    }
+    return json.dumps(context)
+
+########################################
+# ROUTE: /GET_USERS
+@app.route("/get_users")
+def get_users():
+    # Fetch page content
+    with open(f"./data/users_to_models.pkl", "rb") as f:
+        users_to_models = pickle.load(f)
+        users = list(users_to_models.keys())
+        context = {
+            "users": users,
+        }
+        return json.dumps(context)
+
+########################################
+# ROUTE: /GET_AUDIT
+@app.route("/get_audit")
+def get_audit():
+    pers_model = request.args.get("pers_model")
+    perf_metric = request.args.get("perf_metric")
+    breakdown_axis = request.args.get("breakdown_axis")
+    breakdown_sort = request.args.get("breakdown_sort")
+    n_topics = int(request.args.get("n_topics"))
+    error_type = request.args.get("error_type")
+    cur_user = request.args.get("cur_user")
+    topic_vis_method = request.args.get("topic_vis_method") 
+    if topic_vis_method == "null":
+        topic_vis_method = "median"
+
+    if breakdown_sort == "difference":
+        sort_class_plot = True
+    elif breakdown_sort == "default":
+        sort_class_plot = False
+    else:
+        raise Exception("Invalid breakdown_sort value")
+
+    overall_perf = utils.show_overall_perf(
+        variant=pers_model,
+        error_type=error_type,
+        cur_user=cur_user,
+        breakdown_axis=breakdown_axis,
+        topic_vis_method=topic_vis_method,
+    )
+
+    results = {
+        "overall_perf": overall_perf,
+    }
+    return json.dumps(results)
+
+########################################
+# ROUTE: /GET_CLUSTER_RESULTS
+@app.route("/get_cluster_results")
+def get_cluster_results():
+    pers_model = request.args.get("pers_model")
+    n_examples = int(request.args.get("n_examples"))
+    cluster = request.args.get("cluster")
+    example_sort = request.args.get("example_sort")
+    comparison_group = request.args.get("comparison_group")
+    topic_df_ids = request.args.getlist("topic_df_ids")
+    topic_df_ids = [int(val) for val in topic_df_ids[0].split(",") if val != ""]
+    search_type = request.args.get("search_type")
+    keyword = request.args.get("keyword")
+    n_neighbors = request.args.get("n_neighbors")
+    if n_neighbors != "null":
+        n_neighbors = int(n_neighbors)
+    neighbor_threshold = 0.6
+    error_type = request.args.get("error_type")
+    use_model = request.args.get("use_model") == "true"
+    scaffold_method = request.args.get("scaffold_method")
+        
+
+    # If user has a tuned model for this cluster, use that
+    cluster_model_file = f"./data/trained_models/{pers_model}_{cluster}.pkl"
+    if os.path.isfile(cluster_model_file):
+        pers_model = f"{pers_model}_{cluster}"
+
+    print(f"get_cluster_results using model {pers_model}")
+
+    other_ids = []
+    perf_metric = "avg_diff"
+    sort_ascending = True if example_sort == "ascending" else False
+
+    topic_df = None
+    
+    personal_cluster_file = f"./data/personal_cluster_dfs/{pers_model}.pkl"
+    if (scaffold_method == "personal_cluster") and (os.path.isfile(personal_cluster_file)):
+        # Handle personal clusters
+        with open(personal_cluster_file, "rb") as f:
+            topic_df = pickle.load(f)
+            topic_df = topic_df[(topic_df["topic"] == cluster)]
+    else:
+        # Regular handling
+        with open(f"data/preds_dfs/{pers_model}.pkl", "rb") as f:
+            topic_df = pickle.load(f)
+        if search_type == "cluster":
+            # Display examples with comment, your pred, and other users' pred
+            topic_df = topic_df[(topic_df["topic"] == cluster) | (topic_df["item_id"].isin(topic_df_ids))]
+                
+        elif search_type == "neighbors":
+            neighbor_ids = utils.get_match(topic_df_ids, K=n_neighbors, threshold=neighbor_threshold, debug=False)
+            topic_df = topic_df[(topic_df["item_id"].isin(neighbor_ids)) | (topic_df["item_id"].isin(topic_df_ids))]
+        elif search_type == "keyword":
+            topic_df = topic_df[(topic_df["comment"].str.contains(keyword, case=False, regex=False)) | (topic_df["item_id"].isin(topic_df_ids))]
+    
+    topic_df = topic_df.drop_duplicates()
+    print("len topic_df", len(topic_df))
+
+    # Handle empty results
+    if len(topic_df) == 0: 
+        results = {
+            "user_perf_rounded": None,
+            "user_direction": None,
+            "other_perf_rounded": None,
+            "other_direction": None,
+            "n_other_users": None,
+            "cluster_examples": None,
+            "odds_ratio": None,
+            "odds_ratio_explanation": None,
+            "topic_df_ids": [],
+            "cluster_overview_plot_json": None,
+            "cluster_comments": None, 
+        }
+        return results
+
+    topic_df_ids = topic_df["item_id"].unique().tolist()
+
+    if (scaffold_method == "personal_cluster") and (os.path.isfile(personal_cluster_file)):
+        cluster_overview_plot_json, sampled_df = utils.plot_overall_vis_cluster(topic_df, error_type=error_type, n_comments=500)
+    else:
+        # Regular
+        cluster_overview_plot_json, sampled_df = utils.get_cluster_overview_plot(topic_df, error_type=error_type, use_model=use_model)
+
+    cluster_comments = utils.get_cluster_comments(sampled_df,error_type=error_type, num_examples=n_examples, use_model=use_model)  # New version of cluster comment table
+
+    results = {
+        "topic_df_ids": topic_df_ids,
+        "cluster_overview_plot_json": json.loads(cluster_overview_plot_json),
+        "cluster_comments": cluster_comments, 
+    }
+    return json.dumps(results)
+
+########################################
+# ROUTE: /GET_GROUP_SIZE
+@app.route("/get_group_size")
+def get_group_size():
+    # Fetch info for initial labeling component
+    sel_gender = request.args.get("sel_gender")
+    sel_pol = request.args.get("sel_pol")
+    sel_relig = request.args.get("sel_relig")
+    sel_race = request.args.get("sel_race")
+    sel_lgbtq = request.args.get("sel_lgbtq")
+    if sel_race != "":
+        sel_race = sel_race.split(",")
+
+    _, group_size = utils.get_workers_in_group(sel_gender, sel_race, sel_relig, sel_pol, sel_lgbtq)
+
+    context = {
+        "group_size": group_size,
+    }
+    return json.dumps(context)
+
+########################################
+# ROUTE: /GET_GROUP_MODEL
+@app.route("/get_group_model")
+def get_group_model():
+    # Fetch info for initial labeling component
+    model_name = request.args.get("model_name")
+    user = request.args.get("user")
+    sel_gender = request.args.get("sel_gender")
+    sel_pol = request.args.get("sel_pol")
+    sel_relig = request.args.get("sel_relig")
+    sel_lgbtq = request.args.get("sel_lgbtq")
+    sel_race_orig = request.args.get("sel_race")
+    if sel_race_orig != "":
+        sel_race = sel_race_orig.split(",")
+    else:
+        sel_race = ""
+    start = time.time()
+
+    grp_df, group_size = utils.get_workers_in_group(sel_gender, sel_race, sel_relig, sel_pol, sel_lgbtq)
+
+    grp_ids = grp_df["worker_id"].tolist()
+
+    ratings_grp = utils.get_grp_model_labels(
+        comments_df=comments_grouped_full_topic_cat,
+        n_label_per_bin=BIN_DISTRIB,
+        score_bins=SCORE_BINS,
+        grp_ids=grp_ids,
+    )
+
+    # print("ratings_grp", ratings_grp)
+
+    # Modify model name
+    model_name = f"{model_name}_group_gender{sel_gender}_relig{sel_relig}_pol{sel_pol}_race{sel_race_orig}_lgbtq_{sel_lgbtq}"
+
+    label_dir = f"./data/labels/{model_name}"
+    # Create directory for labels if it doesn't yet exist
+    if not os.path.isdir(label_dir):
+        os.mkdir(label_dir)
+    last_label_i = len([name for name in os.listdir(label_dir) if (os.path.isfile(os.path.join(label_dir, name)) and name.endswith('.pkl'))])
+
+    # Train group model
+    mae, mse, rmse, avg_diff, ratings_prev = utils.train_updated_model(model_name, last_label_i, ratings_grp, user)
+
+    duration = time.time() - start
+    print("Time to train/cache:", duration)
+
+    context = {
+        "group_size": group_size,
+        "mae": mae,
+    }
+    return json.dumps(context)
+    
+########################################
+# ROUTE: /GET_LABELING
+@app.route("/get_labeling")
+def get_labeling():
+    # Fetch info for initial labeling component
+    user = request.args.get("user")
+
+    clusters_for_tuning = utils.get_large_clusters(min_n=150)
+    clusters_for_tuning_options = [{"value": i, "text": cluster} for i, cluster in enumerate(clusters_for_tuning)]  # Format for Svelecte UI element
+
+    # model_name_suggestion = f"model_{int(time.time())}"
+    model_name_suggestion = f"model_{user}"
+
+    context = {
+        "personalized_models": utils.get_all_model_names(user),
+        "model_name_suggestion": model_name_suggestion,
+        "clusters_for_tuning": clusters_for_tuning_options,
+    }
+    return json.dumps(context)
+
+########################################
+# ROUTE: /GET_COMMENTS_TO_LABEL
+N_LABEL_PER_BIN = 8 # 8 * 5 = 40 comments
+BIN_DISTRIB = [4, 8, 16, 8, 4]
+SCORE_BINS = [(0.0, 0.5), (0.5, 1.5), (1.5, 2.5), (2.5, 3.5), (3.5, 4.01)]
+@app.route("/get_comments_to_label")
+def get_comments_to_label():
+    n = int(request.args.get("n"))
+    # Fetch examples to label
+    to_label_ids = utils.create_example_sets(
+        comments_df=comments_grouped_full_topic_cat,
+        n_label_per_bin=BIN_DISTRIB,
+        score_bins=SCORE_BINS,
+        keyword=None
+    )
+    random.shuffle(to_label_ids)  # randomize to not prime users
+    to_label_ids = to_label_ids[:n]
+
+    ids_to_comments = utils.get_ids_to_comments()
+    to_label = [ids_to_comments[comment_id] for comment_id in to_label_ids]
+    context = {
+        "to_label": to_label,
+    }
+    return json.dumps(context)
+
+########################################
+# ROUTE: /GET_COMMENTS_TO_LABEL_TOPIC
+N_LABEL_PER_BIN_TOPIC = 2 # 2 * 5 = 10 comments
+@app.route("/get_comments_to_label_topic")
+def get_comments_to_label_topic():
+    # Fetch examples to label
+    topic = request.args.get("topic")
+    to_label_ids = utils.create_example_sets(
+        comments_df=comments_grouped_full_topic_cat,
+        # n_label_per_bin=N_LABEL_PER_BIN_TOPIC,
+        n_label_per_bin=BIN_DISTRIB,
+        score_bins=SCORE_BINS,
+        keyword=None,
+        topic=topic,
+    )
+    random.shuffle(to_label_ids)  # randomize to not prime users 
+    ids_to_comments = utils.get_ids_to_comments()
+    to_label = [ids_to_comments[comment_id] for comment_id in to_label_ids]
+    context = {
+        "to_label": to_label,
+    }
+    return json.dumps(context)
+
+########################################
+# ROUTE: /GET_PERSONALIZED_MODEL
+@app.route("/get_personalized_model")
+def get_personalized_model():
+    model_name = request.args.get("model_name")
+    ratings_json = request.args.get("ratings")
+    mode = request.args.get("mode")
+    user = request.args.get("user")
+    ratings = json.loads(ratings_json)
+    print(ratings)
+    start = time.time()
+
+    label_dir = f"./data/labels/{model_name}"
+    # Create directory for labels if it doesn't yet exist
+    if not os.path.isdir(label_dir):
+        os.mkdir(label_dir)
+    last_label_i = len([name for name in os.listdir(label_dir) if (os.path.isfile(os.path.join(label_dir, name)) and name.endswith('.pkl'))])
+
+    # Handle existing or new model cases
+    if mode == "view":
+        # Fetch prior model performance
+        if model_name not in utils.get_all_model_names():
+            raise Exception(f"Model {model_name} does not exist")
+        else:
+            mae, mse, rmse, avg_diff, ratings_prev = utils.fetch_existing_data(model_name, last_label_i)
+        
+    elif mode == "train":
+        # Train model and cache predictions using new labels
+        print("get_personalized_model train")
+        mae, mse, rmse, avg_diff, ratings_prev = utils.train_updated_model(model_name, last_label_i, ratings, user)
+        
+    duration = time.time() - start
+    print("Time to train/cache:", duration) 
+
+    perf_plot, mae_status = utils.plot_train_perf_results(model_name, mae)
+    perf_plot_json = perf_plot.to_json()
+
+    def round_metric(x):
+        return np.round(abs(x), 3)
+
+    results = {
+        "model_name": model_name,
+        "mae": round_metric(mae),
+        "mae_status": mae_status,
+        "mse": round_metric(mse),
+        "rmse": round_metric(rmse),
+        "avg_diff": round_metric(avg_diff),
+        "duration": duration,
+        "ratings_prev": ratings_prev,
+        "perf_plot_json": json.loads(perf_plot_json),
+    }
+    return json.dumps(results)
+
+
+########################################
+# ROUTE: /GET_PERSONALIZED_MODEL_TOPIC
+@app.route("/get_personalized_model_topic")
+def get_personalized_model_topic():
+    model_name = request.args.get("model_name")
+    ratings_json = request.args.get("ratings")
+    user = request.args.get("user")
+    ratings = json.loads(ratings_json)
+    topic = request.args.get("topic")
+    print(ratings)
+    start = time.time()
+
+    # Modify model name
+    model_name = f"{model_name}_{topic}"
+
+    label_dir = f"./data/labels/{model_name}"
+    # Create directory for labels if it doesn't yet exist
+    if not os.path.isdir(label_dir):
+        os.mkdir(label_dir)
+    last_label_i = len([name for name in os.listdir(label_dir) if (os.path.isfile(os.path.join(label_dir, name)) and name.endswith('.pkl'))])
+
+    # Handle existing or new model cases
+    # Train model and cache predictions using new labels
+    print("get_personalized_model_topic train")
+    mae, mse, rmse, avg_diff, ratings_prev = utils.train_updated_model(model_name, last_label_i, ratings, user, topic=topic)
+        
+    duration = time.time() - start
+    print("Time to train/cache:", duration) 
+
+    def round_metric(x):
+        return np.round(abs(x), 3)
+
+    results = {
+        "success": "success",
+        "ratings_prev": ratings_prev,
+        "new_model_name": model_name,
+    }
+    return json.dumps(results)
+
+
+########################################
+# ROUTE: /GET_REPORTS
+@app.route("/get_reports")
+def get_reports():
+    cur_user = request.args.get("cur_user")
+    scaffold_method = request.args.get("scaffold_method")
+    model = request.args.get("model")
+    topic_vis_method = request.args.get("topic_vis_method")
+    if topic_vis_method == "null":
+        topic_vis_method = "fp_fn"
+
+    # Load reports for current user from stored files
+    report_dir = f"./data/user_reports"
+    user_file = os.path.join(report_dir, f"{cur_user}_{scaffold_method}.pkl")
+
+    if not os.path.isfile(user_file):
+        if scaffold_method == "fixed":
+            reports = get_fixed_scaffold()
+        elif (scaffold_method == "personal" or scaffold_method == "personal_group" or scaffold_method == "personal_test"):
+            reports = get_personal_scaffold(model, topic_vis_method)
+        elif (scaffold_method == "personal_cluster"):
+            reports = get_personal_cluster_scaffold(model)
+        elif scaffold_method == "prompts":
+            reports = get_prompts_scaffold()
+        elif scaffold_method == "tutorial":
+            reports = get_tutorial_scaffold()
+        else:
+            # Prepare empty report
+            reports = [
+                {
+                    "title": "",
+                    "error_type": "",
+                    "evidence": [],
+                    "text_entry": "",
+                    "complete_status": False,
+                }
+            ]
+    else:
+        # Load from pickle file
+        with open(user_file, "rb") as f:
+            reports = pickle.load(f)
+
+    results = {
+        "reports": reports,
+    }
+    return json.dumps(results)
+
+def get_fixed_scaffold():
+    return [
+        {
+            "title": "Topic: 6_jews_jew_jewish_rabbi",
+            "error_type": "System is under-sensitive",
+            "evidence": [],
+            "text_entry": "",
+            "complete_status": False,
+        },
+        {
+            "title": "Topic: 73_troll_trolls_trolling_spammers",
+            "error_type": "System is over-sensitive",
+            "evidence": [],
+            "text_entry": "",
+            "complete_status": False,
+        },
+        {
+            "title": "Topic: 66_mexicans_mexico_mexican_spanish",
+            "error_type": "System is under-sensitive",
+            "evidence": [],
+            "text_entry": "",
+            "complete_status": False,
+        },
+        {
+            "title": "Topic: 89_cowards_coward_cowardly_brave",
+            "error_type": "System is over-sensitive",
+            "evidence": [],
+            "text_entry": "",
+            "complete_status": False,
+        },
+        {
+            "title": "Topic: 63_disgusting_gross_toxic_thicc",
+            "error_type": "System is under-sensitive",
+            "evidence": [],
+            "text_entry": "",
+            "complete_status": False,
+        },
+    ]
+
+def get_empty_report(title, error_type):
+    return {
+            "title": f"Topic: {title}",
+            "error_type": error_type,
+            "evidence": [],
+            "text_entry": "",
+            "complete_status": False,
+        }
+
+def get_tutorial_scaffold():
+    return [
+        {
+            "title": "Topic: 79_idiot_dumb_stupid_dumber",
+            "error_type": "System is over-sensitive",
+            "evidence": [],
+            "text_entry": "",
+            "complete_status": False,
+        },
+    ] 
+
+def get_personal_cluster_scaffold(model):
+    topics_under_top, topics_over_top = utils.get_personal_clusters(model)
+
+    report_under = [get_empty_report(topic, "System is under-sensitive") for topic in topics_under_top]
+
+    report_over = [get_empty_report(topic, "System is over-sensitive") for topic in topics_over_top]
+    reports = (report_under + report_over)
+    random.shuffle(reports)
+    return reports
+
+def get_topic_errors(df, topic_vis_method, threshold=2):
+    topics = df["topic_"].unique().tolist()
+    topic_errors = {}
+    for topic in topics:
+        t_df = df[df["topic_"] == topic]
+        y_true = t_df["pred"].to_numpy()
+        y_pred = t_df["rating"].to_numpy()
+        if topic_vis_method == "mae":
+            t_err = mean_absolute_error(y_true, y_pred)
+        elif topic_vis_method == "mse":
+            t_err = mean_squared_error(y_true, y_pred)
+        elif topic_vis_method == "avg_diff":
+            t_err = np.mean(y_true - y_pred)
+        elif topic_vis_method == "fp_proportion":
+            y_true = [0 if rating < threshold else 1 for rating in t_df["pred"].tolist()]
+            y_pred = [0 if rating < threshold else 1 for rating in t_df["rating"].tolist()]
+            try:
+                tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel()
+            except:
+                tn, fp, fn, tp = [0, 0, 0, 0]  # ignore; set error to 0
+            total = float(len(y_true))
+            t_err = fp / total
+        elif topic_vis_method == "fn_proportion":
+            y_true = [0 if rating < threshold else 1 for rating in t_df["pred"].tolist()]
+            y_pred = [0 if rating < threshold else 1 for rating in t_df["rating"].tolist()]
+            try:
+                tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel()
+            except:
+                tn, fp, fn, tp = [0, 0, 0, 0]  # ignore; set error to 0
+            total = float(len(y_true))
+            t_err = fn / total
+        topic_errors[topic] = t_err
+        
+    return topic_errors
+
+def get_personal_scaffold(model, topic_vis_method, n_topics=200, n=5):
+    threshold = utils.get_toxic_threshold()
+
+    # Get topics with greatest amount of error
+    with open(f"./data/preds_dfs/{model}.pkl", "rb") as f:
+        preds_df = pickle.load(f)
+        preds_df_mod = preds_df.merge(utils.get_comments_grouped_full_topic_cat(), on="item_id", how="left", suffixes=('_', '_avg'))
+        preds_df_mod = preds_df_mod[preds_df_mod["user_id"] == "A"].sort_values(by=["item_id"]).reset_index()
+        preds_df_mod = preds_df_mod[preds_df_mod["topic_id_"] < n_topics]
+
+        if topic_vis_method == "median":
+            df = preds_df_mod.groupby(["topic_", "user_id"]).median().reset_index()
+        elif topic_vis_method == "mean":
+            df = preds_df_mod.groupby(["topic_", "user_id"]).mean().reset_index()
+        elif topic_vis_method == "fp_fn":
+            for error_type in ["fn_proportion", "fp_proportion"]:
+                topic_errors = get_topic_errors(preds_df_mod, error_type)
+                preds_df_mod[error_type] = [topic_errors[topic] for topic in preds_df_mod["topic_"].tolist()]
+            df = preds_df_mod.groupby(["topic_", "user_id"]).mean().reset_index()
+        else:
+            # Get error for each topic
+            topic_errors = get_topic_errors(preds_df_mod, topic_vis_method)
+            preds_df_mod[topic_vis_method] = [topic_errors[topic] for topic in preds_df_mod["topic_"].tolist()]
+            df = preds_df_mod.groupby(["topic_", "user_id"]).mean().reset_index()
+
+        # Get system error
+        df = df[(df["topic_"] != "53_maiareficco_kallystas_dyisisitmanila_tractorsazi") & (df["topic_"] != "79_idiot_dumb_stupid_dumber")]
+        
+        if topic_vis_method == "median" or topic_vis_method == "mean":
+            df["error_magnitude"] = [utils.get_error_magnitude(sys, user, threshold) for sys, user in zip(df["rating"].tolist(), df["pred"].tolist())]
+            df["error_type"] = [utils.get_error_type_radio(sys, user, threshold) for sys, user in zip(df["rating"].tolist(), df["pred"].tolist())]
+
+            df_under = df[df["error_type"] == "System is under-sensitive"]
+            df_under = df_under.sort_values(by=["error_magnitude"], ascending=False).head(n) # surface largest errors first
+            report_under = [get_empty_report(row["topic_"], row["error_type"]) for _, row in df_under.iterrows()]
+
+            df_over = df[df["error_type"] == "System is over-sensitive"]
+            df_over = df_over.sort_values(by=["error_magnitude"], ascending=False).head(n) # surface largest errors first
+            report_over = [get_empty_report(row["topic_"], row["error_type"]) for _, row in df_over.iterrows()]
+            
+            # Set up reports
+            # return [get_empty_report(row["topic_"], row["error_type"]) for index, row in df.iterrows()]
+            reports = (report_under + report_over)
+            random.shuffle(reports)
+        elif topic_vis_method == "fp_fn":
+            df_under = df.sort_values(by=["fn_proportion"], ascending=False).head(n)
+            df_under = df_under[df_under["fn_proportion"] > 0]
+            report_under = [get_empty_report(row["topic_"], "System is under-sensitive") for _, row in df_under.iterrows()]
+            
+            df_over = df.sort_values(by=["fp_proportion"], ascending=False).head(n)
+            df_over = df_over[df_over["fp_proportion"] > 0]
+            report_over = [get_empty_report(row["topic_"], "System is over-sensitive") for _, row in df_over.iterrows()]
+
+            reports = (report_under + report_over)
+            random.shuffle(reports)
+        else:
+            df = df.sort_values(by=[topic_vis_method], ascending=False).head(n * 2)
+            df["error_type"] = [utils.get_error_type_radio(sys, user, threshold) for sys, user in zip(df["rating"].tolist(), df["pred"].tolist())]
+            reports = [get_empty_report(row["topic_"], row["error_type"]) for _, row in df.iterrows()]
+
+        return reports
+
+def get_prompts_scaffold():
+    return [
+        {
+            "title": "Are there terms that are used in your identity group or community that tend to be flagged incorrectly as toxic?",
+            "error_type": "System is over-sensitive",
+            "evidence": [],
+            "text_entry": "",
+            "complete_status": False,
+        },
+        {
+            "title": "Are there terms that are used in your identity group or community that tend to be flagged incorrectly as non-toxic?",
+            "error_type": "System is under-sensitive",
+            "evidence": [],
+            "text_entry": "",
+            "complete_status": False,
+        },
+        {
+            "title": "Are there certain ways that your community tends to be targeted by outsiders?",
+            "error_type": "",
+            "evidence": [],
+            "text_entry": "",
+            "complete_status": False,
+        },
+        {
+            "title": "Are there other communities whose content should be very similar to your community's? Verify that this content is treated similarly by the system.",
+            "error_type": "",
+            "evidence": [],
+            "text_entry": "",
+            "complete_status": False,
+        },
+        {
+            "title": "Are there ways that you've seen individuals in your community actively try to thwart the rules of automated content moderation systems? Check whether these strategies work here.",
+            "error_type": "",
+            "evidence": [],
+            "text_entry": "",
+            "complete_status": False,
+        },
+    ]
+
+########################################
+# ROUTE: /SAVE_REPORTS
+@app.route("/save_reports")
+def save_reports():
+    cur_user = request.args.get("cur_user")
+    reports_json = request.args.get("reports")
+    reports = json.loads(reports_json)
+    scaffold_method = request.args.get("scaffold_method")
+
+    # Save reports for current user to stored files
+    report_dir = f"./data/user_reports"
+    # Save to pickle file
+    with open(os.path.join(report_dir, f"{cur_user}_{scaffold_method}.pkl"), "wb") as f:
+        pickle.dump(reports, f)
+
+    results = {
+        "status": "success",
+    }
+    return json.dumps(results)
+
+########################################
+# ROUTE: /GET_EXPLORE_EXAMPLES
+@app.route("/get_explore_examples")
+def get_explore_examples():
+    threshold = utils.get_toxic_threshold()
+    n_examples = int(request.args.get("n_examples"))
+
+    # Get sample of examples
+    df = utils.get_comments_grouped_full_topic_cat().sample(n=n_examples)
+
+    df["system_decision"] = [utils.get_decision(rating, threshold) for rating in df["rating"].tolist()]
+    df["system_color"] = [utils.get_user_color(sys, threshold) for sys in df["rating"].tolist()]  # get cell colors
+
+    ex_json = df.to_json(orient="records")
+
+    results = {
+        "examples": ex_json,
+    }
+    return json.dumps(results)
+
+########################################
+# ROUTE: /GET_RESULTS
+@app.route("/get_results")
+def get_results():
+    users = request.args.get("users")
+    if users != "":
+        users = users.split(",")
+    # print("users", users)
+
+    IGNORE_LIST = ["DemoUser"]
+    report_dir = f"./data/user_reports"
+    
+
+    # For each user, get personal and prompt results
+    # Get links to label pages and audit pages
+    results = []
+    for user in users:
+        if user not in IGNORE_LIST:
+            user_results = {}
+            user_results["user"] = user
+            for scaffold_method in ["personal", "personal_group", "prompts"]:
+                # Get results
+                user_file = os.path.join(report_dir, f"{user}_{scaffold_method}.pkl")
+                if os.path.isfile(user_file):
+                    with open(user_file, "rb") as f:
+                        user_results[scaffold_method] = pickle.load(f)
+            results.append(user_results)
+
+    # print("results", results)
+
+    results = {
+        "results": results,
+    }
+    return json.dumps(results)
+
+if __name__ == "__main__":
+    app.run(debug=True, port=5001)