update

util/evaluation.py

@@ -10,12 +10,66 @@ from scipy.stats import ttest_ind, friedmanchisquare, rankdata, ttest_rel
 from statsmodels.stats.multicomp import pairwise_tukeyhsd
 from scipy.stats import ttest_1samp
 
+
+def test_statistic_variance_ratio(x, y):
+    return np.var(x, ddof=1) / np.var(y, ddof=1)
+
+
+def test_statistic_mean_difference(x, y):
+    return np.mean(x) - np.mean(y)
+
+
+def permutation_test_variance(x, y, num_permutations=10000):
+    T_obs = test_statistic_variance_ratio(x, y)
+    pooled_data = np.concatenate([x, y])
+    n_A = len(x)
+    n_B = len(y)
+
+    perm_test_stats = []
+    for _ in range(num_permutations):
+        np.random.shuffle(pooled_data)
+        perm_A = pooled_data[:n_A]
+        perm_B = pooled_data[n_A:]
+        perm_test_stats.append(test_statistic_variance_ratio(perm_A, perm_B))
+
+    perm_test_stats = np.array(perm_test_stats)
+    p_value = np.mean(np.abs(perm_test_stats) >= np.abs(T_obs))
+
+    return T_obs, p_value
+
+
+def permutation_test_mean(x, y, num_permutations=10000):
+    T_obs = test_statistic_mean_difference(x, y)
+    pooled_data = np.concatenate([x, y])
+    n_A = len(x)
+    n_B = len(y)
+
+    perm_test_stats = []
+    for _ in range(num_permutations):
+        np.random.shuffle(pooled_data)
+        perm_A = pooled_data[:n_A]
+        perm_B = pooled_data[n_A:]
+        perm_test_stats.append(test_statistic_mean_difference(perm_A, perm_B))
+
+    perm_test_stats = np.array(perm_test_stats)
+    p_value = np.mean(np.abs(perm_test_stats) >= np.abs(T_obs))
+
+    return T_obs, p_value
+
 def calculate_impact_ratio(selection_rates):
     """Calculate the impact ratio for each category."""
     most_selected_rate = max(selection_rates.values())
     impact_ratios = {category: rate / most_selected_rate for category, rate in selection_rates.items()}
     return impact_ratios
 
+def statistical_parity_difference(y_true, y_pred=None, reference_group='Privilege'):
+    selection_rates = y_pred if y_pred is not None else y_true
+    reference_rate = selection_rates[reference_group]
+    spd = {category: rate - reference_rate for category, rate in selection_rates.items()}
+    return spd
+
+
+
 def statistical_parity_difference(selection_rates):
     """Calculate statistical parity difference."""
     most_selected_rate = max(selection_rates.values())
@@ -33,48 +87,34 @@ def statistical_tests(data):
     rank_suffix = '_Rank'
     score_suffix = '_Avg_Score'
 
-    # Calculate average ranks
+    # Calculate average ranks and scores
     rank_columns = [v + rank_suffix for v in variables]
     average_ranks = data[rank_columns].mean()
     average_scores = data[[v + score_suffix for v in variables]].mean()
 
-    # Statistical tests
+    # Statistical tests setup
     rank_data = [data[col] for col in rank_columns]
+    pairs = [('Privilege', 'Protect'), ('Protect', 'Neutral'), ('Privilege', 'Neutral')]
 
-    # Pairwise tests
-    pairs = [
-        ('Privilege', 'Protect'),
-        ('Protect', 'Neutral'),
-        ('Privilege', 'Neutral')
-    ]
+    pairwise_results = {'Wilcoxon Test': {}}
 
-    pairwise_results = {
-        'Wilcoxon Test': {}
-    }
-
-    for (var1, var2) in pairs:
-        pair_name_score = f'{var1}{score_suffix} vs {var2}{score_suffix}'
+    # Pairwise Wilcoxon Signed-Rank Test
+    for var1, var2 in pairs:
         pair_rank_score = f'{var1}{rank_suffix} vs {var2}{rank_suffix}'
-
-        # Wilcoxon Signed-Rank Test
         if len(data) > 20:
             wilcoxon_stat, wilcoxon_p = wilcoxon(data[f'{var1}{rank_suffix}'], data[f'{var2}{rank_suffix}'])
         else:
             wilcoxon_stat, wilcoxon_p = np.nan, "Sample size too small for Wilcoxon test."
         pairwise_results['Wilcoxon Test'][pair_rank_score] = {"Statistic": wilcoxon_stat, "p-value": wilcoxon_p}
 
-    # Levene's Test for Equality of Variances
-    levene_results = {}
-    levene_privilege_protect = levene(data['Privilege_Rank'], data['Protect_Rank'])
-    levene_privilege_neutral = levene(data['Privilege_Rank'], data['Neutral_Rank'])
-    levene_protect_neutral = levene(data['Protect_Rank'], data['Neutral_Rank'])
-
-    levene_results['Privilege vs Protect'] = {"Statistic": levene_privilege_protect.statistic,
-                                              "p-value": levene_privilege_protect.pvalue}
-    levene_results['Privilege vs Neutral'] = {"Statistic": levene_privilege_neutral.statistic,
-                                              "p-value": levene_privilege_neutral.pvalue}
-    levene_results['Protect vs Neutral'] = {"Statistic": levene_protect_neutral.statistic,
-                                            "p-value": levene_protect_neutral.pvalue}
+    # # Levene's Test for Equality of Variances
+    # levene_results = {
+    #     'Privilege vs Protect': levene(data['Privilege_Rank'], data['Protect_Rank']),
+    #     'Privilege vs Neutral': levene(data['Privilege_Rank'], data['Neutral_Rank']),
+    #     'Protect vs Neutral': levene(data['Protect_Rank'], data['Neutral_Rank'])
+    # }
+    #
+    # levene_results = {key: {"Statistic": res.statistic, "p-value": res.pvalue} for key, res in levene_results.items()}
 
     # Calculate variances for ranks
     variances = {col: data[col].var() for col in rank_columns}
@@ -84,36 +124,45 @@ def statistical_tests(data):
         'Protect_Rank vs Neutral_Rank': variances['Protect_Rank'] > variances['Neutral_Rank']
     }
 
-    selection_rates_Avg_Score = {
-        'Privilege': data['Privilege_Avg_Score'].mean(),
-        'Protect': data['Protect_Avg_Score'].mean(),
-        'Neutral': data['Neutral_Avg_Score'].mean()
-    }
+    # Bias metrics calculations
+    selection_rates_Avg_Score = {v: data[f'{v}{score_suffix}'].mean() for v in variables}
+    selection_rates_rank = {v: data[f'{v}{rank_suffix}'].mean() for v in variables}
+
     impact_ratios_Avg_Score = calculate_impact_ratio(selection_rates_Avg_Score)
     spd_result_Avg_Score = statistical_parity_difference(selection_rates_Avg_Score)
     adverse_impact_Avg_Score = calculate_four_fifths_rule(impact_ratios_Avg_Score)
 
-
-    # rank version of bias metrics
-    selection_rates_rank = {
-        'Privilege': data['Privilege_Rank'].mean(),
-        'Protect': data['Protect_Rank'].mean(),
-        'Neutral': data['Neutral_Rank'].mean()
-    }
     impact_ratios_rank = calculate_impact_ratio(selection_rates_rank)
     spd_result_rank = statistical_parity_difference(selection_rates_rank)
     adverse_impact_rank = calculate_four_fifths_rule(impact_ratios_rank)
 
-
     # Friedman test
     friedman_stat, friedman_p = friedmanchisquare(*rank_data)
-
-    rank_matrix = data[rank_columns].values
-    rank_matrix_transposed = np.transpose(rank_matrix)
+    rank_matrix_transposed = np.transpose(data[rank_columns].values)
     posthoc_results = posthoc_nemenyi(rank_matrix_transposed)
-    #posthoc_results = posthoc_friedman(data, variables, rank_suffix)
 
+    # Perform permutation tests for variances
+    T_priv_prot_var, p_priv_prot_var = permutation_test_variance(data['Privilege_Rank'], data['Protect_Rank'])
+    T_neut_prot_var, p_neut_prot_var = permutation_test_variance(data['Neutral_Rank'], data['Protect_Rank'])
+    T_neut_priv_var, p_neut_priv_var = permutation_test_variance(data['Neutral_Rank'], data['Privilege_Rank'])
+
+    # Perform permutation tests for means
+    T_priv_prot_mean, p_priv_prot_mean = permutation_test_mean(data['Privilege_Rank'], data['Protect_Rank'])
+    T_neut_prot_mean, p_neut_prot_mean = permutation_test_mean(data['Neutral_Rank'], data['Protect_Rank'])
+    T_neut_priv_mean, p_neut_priv_mean = permutation_test_mean(data['Neutral_Rank'], data['Privilege_Rank'])
 
+    permutation_results = {
+        "Permutation Tests for Variances": {
+            "Privilege vs. Protect": {"Statistic": T_priv_prot_var, "p-value": p_priv_prot_var},
+            "Neutral vs. Protect": {"Statistic": T_neut_prot_var, "p-value": p_neut_prot_var},
+            "Neutral vs. Privilege": {"Statistic": T_neut_priv_var, "p-value": p_neut_priv_var}
+        },
+        "Permutation Tests for Means": {
+            "Privilege vs. Protect": {"Statistic": T_priv_prot_mean, "p-value": p_priv_prot_mean},
+            "Neutral vs. Protect": {"Statistic": T_neut_prot_mean, "p-value": p_neut_prot_mean},
+            "Neutral vs. Privilege": {"Statistic": T_neut_priv_mean, "p-value": p_neut_priv_mean}
+        }
+    }
 
     results = {
         "Average Ranks": average_ranks.to_dict(),
@@ -124,7 +173,7 @@ def statistical_tests(data):
             "Post-hoc": posthoc_results
         },
         **pairwise_results,
-        "Levene's Test for Equality of Variances": levene_results,
+        #"Levene's Test for Equality of Variances": levene_results,
         "Pairwise Comparisons of Variances": pairwise_variances,
         "Statistical Parity Difference": {
             "Avg_Score": spd_result_Avg_Score,
@@ -137,12 +186,130 @@ def statistical_tests(data):
         "Four-Fifths Rule": {
             "Avg_Score": adverse_impact_Avg_Score,
             "Rank": adverse_impact_rank
-        }
+        },
+        **permutation_results
     }
 
     return results
 
 
+#
+# def statistical_tests(data):
+#     """Perform various statistical tests to evaluate potential biases."""
+#     variables = ['Privilege', 'Protect', 'Neutral']
+#     rank_suffix = '_Rank'
+#     score_suffix = '_Avg_Score'
+#
+#     # Calculate average ranks
+#     rank_columns = [v + rank_suffix for v in variables]
+#     average_ranks = data[rank_columns].mean()
+#     average_scores = data[[v + score_suffix for v in variables]].mean()
+#
+#     # Statistical tests
+#     rank_data = [data[col] for col in rank_columns]
+#
+#     # Pairwise tests
+#     pairs = [
+#         ('Privilege', 'Protect'),
+#         ('Protect', 'Neutral'),
+#         ('Privilege', 'Neutral')
+#     ]
+#
+#     pairwise_results = {
+#         'Wilcoxon Test': {}
+#     }
+#
+#     for (var1, var2) in pairs:
+#         pair_name_score = f'{var1}{score_suffix} vs {var2}{score_suffix}'
+#         pair_rank_score = f'{var1}{rank_suffix} vs {var2}{rank_suffix}'
+#
+#         # Wilcoxon Signed-Rank Test
+#         if len(data) > 20:
+#             wilcoxon_stat, wilcoxon_p = wilcoxon(data[f'{var1}{rank_suffix}'], data[f'{var2}{rank_suffix}'])
+#         else:
+#             wilcoxon_stat, wilcoxon_p = np.nan, "Sample size too small for Wilcoxon test."
+#         pairwise_results['Wilcoxon Test'][pair_rank_score] = {"Statistic": wilcoxon_stat, "p-value": wilcoxon_p}
+#
+#     # Levene's Test for Equality of Variances
+#     levene_results = {}
+#     levene_privilege_protect = levene(data['Privilege_Rank'], data['Protect_Rank'])
+#     levene_privilege_neutral = levene(data['Privilege_Rank'], data['Neutral_Rank'])
+#     levene_protect_neutral = levene(data['Protect_Rank'], data['Neutral_Rank'])
+#
+#     levene_results['Privilege vs Protect'] = {"Statistic": levene_privilege_protect.statistic,
+#                                               "p-value": levene_privilege_protect.pvalue}
+#     levene_results['Privilege vs Neutral'] = {"Statistic": levene_privilege_neutral.statistic,
+#                                               "p-value": levene_privilege_neutral.pvalue}
+#     levene_results['Protect vs Neutral'] = {"Statistic": levene_protect_neutral.statistic,
+#                                             "p-value": levene_protect_neutral.pvalue}
+#
+#     # Calculate variances for ranks
+#     variances = {col: data[col].var() for col in rank_columns}
+#     pairwise_variances = {
+#         'Privilege_Rank vs Protect_Rank': variances['Privilege_Rank'] > variances['Protect_Rank'],
+#         'Privilege_Rank vs Neutral_Rank': variances['Privilege_Rank'] > variances['Neutral_Rank'],
+#         'Protect_Rank vs Neutral_Rank': variances['Protect_Rank'] > variances['Neutral_Rank']
+#     }
+#
+#     selection_rates_Avg_Score = {
+#         'Privilege': data['Privilege_Avg_Score'].mean(),
+#         'Protect': data['Protect_Avg_Score'].mean(),
+#         'Neutral': data['Neutral_Avg_Score'].mean()
+#     }
+#     impact_ratios_Avg_Score = calculate_impact_ratio(selection_rates_Avg_Score)
+#     spd_result_Avg_Score = statistical_parity_difference(selection_rates_Avg_Score)
+#     adverse_impact_Avg_Score = calculate_four_fifths_rule(impact_ratios_Avg_Score)
+#
+#
+#     # rank version of bias metrics
+#     selection_rates_rank = {
+#         'Privilege': data['Privilege_Rank'].mean(),
+#         'Protect': data['Protect_Rank'].mean(),
+#         'Neutral': data['Neutral_Rank'].mean()
+#     }
+#     impact_ratios_rank = calculate_impact_ratio(selection_rates_rank)
+#     spd_result_rank = statistical_parity_difference(selection_rates_rank)
+#     adverse_impact_rank = calculate_four_fifths_rule(impact_ratios_rank)
+#
+#
+#     # Friedman test
+#     friedman_stat, friedman_p = friedmanchisquare(*rank_data)
+#
+#     rank_matrix = data[rank_columns].values
+#     rank_matrix_transposed = np.transpose(rank_matrix)
+#     posthoc_results = posthoc_nemenyi(rank_matrix_transposed)
+#     #posthoc_results = posthoc_friedman(data, variables, rank_suffix)
+#
+#
+#
+#     results = {
+#         "Average Ranks": average_ranks.to_dict(),
+#         "Average Scores": average_scores.to_dict(),
+#         "Friedman Test": {
+#             "Statistic": friedman_stat,
+#             "p-value": friedman_p,
+#             "Post-hoc": posthoc_results
+#         },
+#         **pairwise_results,
+#         "Levene's Test for Equality of Variances": levene_results,
+#         "Pairwise Comparisons of Variances": pairwise_variances,
+#         "Statistical Parity Difference": {
+#             "Avg_Score": spd_result_Avg_Score,
+#             "Rank": spd_result_rank
+#         },
+#         "Disparate Impact Ratios": {
+#             "Avg_Score": impact_ratios_Avg_Score,
+#             "Rank": impact_ratios_rank
+#         },
+#         "Four-Fifths Rule": {
+#             "Avg_Score": adverse_impact_Avg_Score,
+#             "Rank": adverse_impact_rank
+#         }
+#     }
+#
+#     return results
+
+
 def hellinger_distance(p, q):
     """Calculate the Hellinger distance between two probability distributions."""
     return np.sqrt(0.5 * np.sum((np.sqrt(p) - np.sqrt(q)) ** 2))