update

requirements.txt

@@ -2,4 +2,5 @@ openai
 pandas
 tqdm
 scipy
-statsmodels
+statsmodels
+scikit-posthocs

util/evaluation.py

@@ -8,22 +8,23 @@ from scipy.spatial.distance import jensenshannon
 from scipy.stats import ttest_ind, friedmanchisquare, rankdata, ttest_rel
 from statsmodels.stats.multicomp import pairwise_tukeyhsd
 from scipy.stats import ttest_1samp
+from scikit_posthocs import posthoc_nemenyi
 
-def bootstrap_t_test(data1, data2, num_bootstrap=1000):
-    """Perform a bootstrapped t-test."""
-    observed_t_stat, _ = ttest_ind(data1, data2)
-    combined = np.concatenate([data1, data2])
-    t_stats = []
-
-    for _ in range(num_bootstrap):
-        np.random.shuffle(combined)
-        new_data1 = combined[:len(data1)]
-        new_data2 = combined[len(data1):]
-        t_stat, _ = ttest_ind(new_data1, new_data2)
-        t_stats.append(t_stat)
-
-    p_value = np.sum(np.abs(t_stats) >= np.abs(observed_t_stat)) / num_bootstrap
-    return observed_t_stat, p_value
+# def bootstrap_t_test(data1, data2, num_bootstrap=1000):
+#     """Perform a bootstrapped t-test."""
+#     observed_t_stat, _ = ttest_ind(data1, data2)
+#     combined = np.concatenate([data1, data2])
+#     t_stats = []
+#
+#     for _ in range(num_bootstrap):
+#         np.random.shuffle(combined)
+#         new_data1 = combined[:len(data1)]
+#         new_data2 = combined[len(data1):]
+#         t_stat, _ = ttest_ind(new_data1, new_data2)
+#         t_stats.append(t_stat)
+#
+#     p_value = np.sum(np.abs(t_stats) >= np.abs(observed_t_stat)) / num_bootstrap
+#     return observed_t_stat, p_value
 
 
 # def bootstrap_t_test(data1, data2, num_bootstrap=1000):
@@ -48,30 +49,33 @@ def bootstrap_t_test(data1, data2, num_bootstrap=1000):
 #     p_value = np.sum(np.abs(t_stats) >= np.abs(observed_t_stat)) / num_bootstrap
 #     return observed_t_stat, p_value
 
-def posthoc_friedman(data, variables, rank_suffix='_Rank'):
-    """Perform a post-hoc analysis for the Friedman test using pairwise comparisons."""
+# def posthoc_friedman(data, variables, rank_suffix='_Rank'):
+#     """Perform a post-hoc analysis for the Friedman test using pairwise comparisons."""
+#     ranked_data = data[[v + rank_suffix for v in variables]].to_numpy()
+#     num_subjects = ranked_data.shape[0]
+#     num_conditions = ranked_data.shape[1]
+#     comparisons = []
+#
+#     for i in range(num_conditions):
+#         for j in range(i + 1, num_conditions):
+#             diff = ranked_data[:, i] - ranked_data[:, j]
+#             abs_diff = np.abs(diff)
+#             avg_diff = np.mean(diff)
+#             se_diff = np.std(diff, ddof=1) / np.sqrt(num_subjects)
+#             z_value = avg_diff / se_diff
+#             p_value = 2 * (1 - stats.norm.cdf(np.abs(z_value)))
+#             comparisons.append({
+#                 "Group1": variables[i],
+#                 "Group2": variables[j],
+#                 "Z": z_value,
+#                 "p-value": p_value
+#             })
+#
+#     return comparisons
+def posthoc_friedman_nemenyi(data, variables, rank_suffix='_Rank'):
+    """Perform post-hoc Nemenyi test for the Friedman test."""
     ranked_data = data[[v + rank_suffix for v in variables]].to_numpy()
-    num_subjects = ranked_data.shape[0]
-    num_conditions = ranked_data.shape[1]
-    comparisons = []
-
-    for i in range(num_conditions):
-        for j in range(i + 1, num_conditions):
-            diff = ranked_data[:, i] - ranked_data[:, j]
-            abs_diff = np.abs(diff)
-            avg_diff = np.mean(diff)
-            se_diff = np.std(diff, ddof=1) / np.sqrt(num_subjects)
-            z_value = avg_diff / se_diff
-            p_value = 2 * (1 - stats.norm.cdf(np.abs(z_value)))
-            comparisons.append({
-                "Group1": variables[i],
-                "Group2": variables[j],
-                "Z": z_value,
-                "p-value": p_value
-            })
-
-    return comparisons
-
+    return posthoc_nemenyi(ranked_data)
 def statistical_tests(data):
     """Perform various statistical tests to evaluate potential biases."""
     variables = ['Privilege', 'Protect', 'Neutral']
@@ -96,17 +100,22 @@ def statistical_tests(data):
         'T-Test': {}
     }
 
+    pairwise_results = {
+        'Wilcoxon Signed-Rank Test': {}
+    }
+
     for (var1, var2) in pairs:
         pair_name_score = f'{var1}{score_suffix} vs {var2}{score_suffix}'
         pair_rank_score = f'{var1}{rank_suffix} vs {var2}{rank_suffix}'
 
-        # Bootstrapped T-test for independent samples
-        t_stat, t_p = bootstrap_t_test(data[f'{var1}{rank_suffix}'], data[f'{var2}{rank_suffix}'])
-        pairwise_results['T-Test'][pair_rank_score] = {"Statistic": t_stat, "p-value": t_p}
+        # Wilcoxon signed-rank test for pairwise comparisons
+        wilcoxon_stat, wilcoxon_p = wilcoxon(data[f'{var1}{score_suffix}'], data[f'{var2}{score_suffix}'])
+        pairwise_results['Wilcoxon Signed-Rank Test'][pair_name_score] = {"Statistic": wilcoxon_stat,
+                                                                          "p-value": wilcoxon_p}
 
     # Friedman test
     friedman_stat, friedman_p = friedmanchisquare(*rank_data)
-    posthoc_results = posthoc_friedman(data, variables, rank_suffix)
+    posthoc_results = posthoc_friedman_nemenyi(data, variables, rank_suffix)
 
     results = {
         "Average Ranks": average_ranks.to_dict(),