File size: 5,113 Bytes
6a58068 e00c8e8 6a58068 e00c8e8 6a58068 e00c8e8 6a58068 e00c8e8 6a58068 e00c8e8 6a58068 e00c8e8 6a58068 e00c8e8 7bfdced e00c8e8 6a58068 e00c8e8 6a58068 e00c8e8 6a58068 e00c8e8 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 |
import numpy as np
import statsmodels
import scipy
from statsmodels.stats.multicomp import pairwise_tukeyhsd
from statsmodels.stats.multitest import fdrcorrection
from scipy import stats
import pandas as pd
import argparse
import os
# increase length of string in pandas
pd.options.display.max_colwidth = 100
def post_hoc_ixi(df_in: pd.DataFrame, outdir: str = './'):
df = df_in[df_in["Experiment"] == "IXI"]
TRE_values = df["TRE"]
m_comp = pairwise_tukeyhsd(df["TRE"], df["Model"], alpha=0.05)
model_names = np.unique(df["Model"])
all_pvalues = -1 * np.ones((len(model_names), len(model_names)), dtype=np.float32)
pvs = m_comp.pvalues
cnt = 0
for i in range(len(model_names)):
for j in range(i + 1, len(model_names)):
all_pvalues[i, j] = pvs[cnt]
cnt += 1
all_pvalues = np.round(all_pvalues, 6)
all_pvalues = all_pvalues[:-1, 1:]
col_new_names = ["\textbf{\rot{\multicolumn{1}{r}{" + n + "}}}" for n in model_names]
out_pd = pd.DataFrame(data=all_pvalues, index=model_names[:-1], columns=col_new_names[1:])
stack = out_pd.stack()
stack[(0 < stack) & (stack <= 0.001)] = '\cellcolor{green!25}$<$0.001'
for i in range(stack.shape[0]):
try:
curr = stack[i]
if (float(curr) > 0.0011) & (float(curr) < 0.05):
stack[i] = '\cellcolor{green!50}' + str(np.round(stack[i], 3))
elif (float(curr) >= 0.05) & (float(curr) < 0.1):
stack[i] = '\cellcolor{red!50}' + str(np.round(stack[i], 3))
elif (float(curr) >= 0.1):
stack[i] = '\cellcolor{red!25}' + str(np.round(stack[i], 3))
except Exception:
continue
out_pd = stack.unstack()
out_pd = out_pd.replace(-1.0, "-")
out_pd = out_pd.replace(-0.0, '\cellcolor{green!25}$<$0.001')
with open(os.path.join(outdir, "tukey_pvalues_result_IXI.txt"), "w") as pfile:
pfile.write("{}".format(out_pd.to_latex(escape=False, column_format="r" + "c"*all_pvalues.shape[1], bold_rows=True)))
print(out_pd)
def study_transfer_learning_benefit(df_in: pd.DataFrame):
df_tl = df_in[df_in["Experiment"] == "COMET_TL_Ft2Stp"]
df_orig = df[df["Experiment"] == "COMET"]
pvs = []
for model in ["BL-N", "SG-NSD", "UW-NSD"]:
curr_tl = df_tl[df_tl["Model"] == model]
curr_orig = df_orig[df_orig["Model"] == model]
TRE_tl = curr_tl["TRE"]
TRE_orig = curr_orig["TRE"]
# perform non-parametric hypothesis test to assess significance
ret = stats.wilcoxon(TRE_tl, TRE_orig, alternative="less")
pv = ret.pvalue
pvs.append(pv)
# False discovery rate to get corrected p-values
corrected_pvs = fdrcorrection(pvs, alpha=0.05, method="indep")[1] # Benjamini/Hochberg -> method="indep"
print("BL-N:", corrected_pvs[0])
print("SG-NSD:", corrected_pvs[1])
print("UW-NSD:", corrected_pvs[2])
def post_hoc_comet(df_in: pd.DataFrame):
df_tl = df_in[df_in["Experiment"] == "COMET_TL_Ft2Stp"]
filter_ = np.array([x in ["BL-N", "SG-NSD", "UW-NSD"] for x in df_tl["Model"]])
df_tl = df_tl[filter_]
# Is TRE in SG-NSD significantly lower than TRE in BL-N?
ret1 = stats.wilcoxon(
df_tl[df_tl["Model"] == "SG-NSD"]["TRE"],
df_tl[df_tl["Model"] == "BL-N"]["TRE"],
alternative="less"
)
pv1 = ret1.pvalue
# Is TRE in UW-NSD significantly lower than TRE in SG_NSD?
ret2 = stats.wilcoxon(
df_tl[df_tl["Model"] == "UW-NSD"]["TRE"],
df_tl[df_tl["Model"] == "SG-NSD"]["TRE"],
alternative="less"
)
pv2 = ret2.pvalue
# False discovery rate to get corrected p-values
pvs = [pv1, pv2]
corrected_pvs = fdrcorrection(pvs, alpha=0.05, method="indep")[1] # Benjamini/Hochberg -> method="indep"
print("Seg-guiding benefit:", corrected_pvs[0])
print("Uncertainty-weighting benefit:", corrected_pvs[1])
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('--file', '-f', type=str, help='CSV file with the metrics')
parser.add_argument('--sep', type=str, help='CSV Separator (default: ;)', default=';')
parser.add_argument('--outdir', type=str, help='Output directory (default: .)', default='./')
args = parser.parse_args()
assert os.path.exists(args.file), 'CSV file not found'
os.makedirs(args.outdir, exist_ok=True)
df = pd.read_csv(args.file, sep=args.sep)
if "Unnamed" in df.columns[0]:
df = df.iloc[:, 1:]
if any(['_' in m for m in df["Model"].unique()]):
df["Model"] = [x.replace("_", "-") for x in df["Model"]]
print("\nComparing all contrasts in TRE of all models in the IXI dataset:")
post_hoc_ixi(df, args.outdir)
print("\nTransfer learning benefit (COMET):")
study_transfer_learning_benefit(df)
print("\nAssessing whether there is a benefit to segmentation-guiding and uncertainty weighting (COMET):")
post_hoc_comet(df)
print('Statsmodels v.: ' + statsmodels.__version__)
print('Scipy v.: ' + scipy.__version__)
|