|
|
import os |
|
|
import argparse |
|
|
import json |
|
|
from cv2 import transform |
|
|
|
|
|
import torch |
|
|
from torch.utils.data import DataLoader |
|
|
from torchvision.transforms import Compose, Normalize |
|
|
import torch.nn.functional as F |
|
|
|
|
|
from dataset import get_dataset, Subset |
|
|
from utils.base import get_mlflow_model_by_name, SmartFormatter |
|
|
from processing.pipeline_numpy import RawProcessingPipeline |
|
|
|
|
|
from utils.hendrycks_robustness import Distortions |
|
|
|
|
|
import segmentation_models_pytorch as smp |
|
|
|
|
|
import matplotlib.pyplot as plt |
|
|
|
|
|
parser = argparse.ArgumentParser(description="AB testing, Show Results", formatter_class=SmartFormatter) |
|
|
|
|
|
|
|
|
parser.add_argument("--mode", type=str, default="ABShowImages", choices=('ABMakeTable', 'ABShowTable', 'ABShowImages', 'ABShowAllImages', 'CMakeTable', 'CShowTable', 'CShowImages', 'CShowAllImages'), |
|
|
help='R|Choose operation to compute. \n' |
|
|
'A) Lens2Logit image generation: \n ' |
|
|
'ABMakeTable: Compute cross-validation metrics results \n ' |
|
|
'ABShowTable: Plot cross-validation results on a table \n ' |
|
|
'ABShowImages: Choose a training and testing image to compare different pipelines \n ' |
|
|
'ABShowAllImages: Plot all possible pipelines \n' |
|
|
'B) Hendrycks Perturbations, C-type dataset: \n ' |
|
|
'CMakeTable: For each pipeline, it computes cross-validation metrics for different perturbations \n ' |
|
|
'CShowTable: Plot metrics for different pipelines and perturbations \n ' |
|
|
'CShowImages: Plot an image with a selected a pipeline and perturbation\n ' |
|
|
'CShowAllImages: Plot all possible perturbations for a fixed pipeline') |
|
|
|
|
|
parser.add_argument("--dataset_name", type=str, default='Microscopy', |
|
|
choices=['Microscopy', 'Drone', 'DroneSegmentation'], help='Choose dataset') |
|
|
parser.add_argument("--augmentation", type=str, default='weak', |
|
|
choices=['none', 'weak', 'strong'], help='Choose augmentation') |
|
|
parser.add_argument("--N_runs", type=int, default=5, help='Number of k-fold splitting used in the training') |
|
|
parser.add_argument("--download_model", default=False, action='store_true', help='Download Models in cache') |
|
|
|
|
|
|
|
|
parser.add_argument("--dm_train", type=str, default='bilinear', choices=('bilinear', 'malvar2004', |
|
|
'menon2007'), help='Choose demosaicing for training processing model') |
|
|
parser.add_argument("--s_train", type=str, default='sharpening_filter', choices=('sharpening_filter', |
|
|
'unsharp_masking'), help='Choose sharpening for training processing model') |
|
|
parser.add_argument("--dn_train", type=str, default='gaussian_denoising', choices=('gaussian_denoising', |
|
|
'median_denoising'), help='Choose denoising for training processing model') |
|
|
parser.add_argument("--dm_test", type=str, default='bilinear', choices=('bilinear', 'malvar2004', |
|
|
'menon2007'), help='Choose demosaicing for testing processing model') |
|
|
parser.add_argument("--s_test", type=str, default='sharpening_filter', choices=('sharpening_filter', |
|
|
'unsharp_masking'), help='Choose sharpening for testing processing model') |
|
|
parser.add_argument("--dn_test", type=str, default='gaussian_denoising', choices=('gaussian_denoising', |
|
|
'median_denoising'), help='Choose denoising for testing processing model') |
|
|
|
|
|
|
|
|
parser.add_argument("--transform", type=str, default='identity', choices=('identity', 'gaussian_noise', 'shot_noise', 'impulse_noise', 'speckle_noise', |
|
|
'gaussian_blur', 'zoom_blur', 'contrast', 'brightness', 'saturate', 'elastic_transform'), help='Choose transformation to show for Ctesting') |
|
|
parser.add_argument("--severity", type=int, default=1, choices=(1, 2, 3, 4, 5), help='Choose severity for Ctesting') |
|
|
|
|
|
args = parser.parse_args() |
|
|
|
|
|
|
|
|
class metrics: |
|
|
def __init__(self, confusion_matrix): |
|
|
self.cm = confusion_matrix |
|
|
self.N_classes = len(confusion_matrix) |
|
|
|
|
|
def accuracy(self): |
|
|
Tp = torch.diagonal(self.cm, 0).sum() |
|
|
N_elements = torch.sum(self.cm) |
|
|
return Tp / N_elements |
|
|
|
|
|
def precision(self): |
|
|
Tp_Fp = torch.sum(self.cm, 1) |
|
|
Tp_Fp[Tp_Fp == 0] = 1 |
|
|
return torch.diagonal(self.cm, 0) / Tp_Fp |
|
|
|
|
|
def recall(self): |
|
|
Tp_Fn = torch.sum(self.cm, 0) |
|
|
Tp_Fn[Tp_Fn == 0] = 1 |
|
|
return torch.diagonal(self.cm, 0) / Tp_Fn |
|
|
|
|
|
def f1_score(self): |
|
|
prod = (self.precision() * self.recall()) |
|
|
sum = (self.precision() + self.recall()) |
|
|
sum[sum == 0.] = 1. |
|
|
return 2 * (prod / sum) |
|
|
|
|
|
def over_N_runs(ms, N_runs): |
|
|
m, m2 = 0, 0 |
|
|
|
|
|
for i in ms: |
|
|
m += i |
|
|
mu = m / N_runs |
|
|
|
|
|
for i in ms: |
|
|
m2 += (i - mu)**2 |
|
|
|
|
|
sigma = torch.sqrt(m2 / (N_runs - 1)) |
|
|
|
|
|
return mu.tolist(), sigma.tolist() |
|
|
|
|
|
|
|
|
class ABtesting: |
|
|
def __init__(self, |
|
|
dataset_name: str, |
|
|
augmentation: str, |
|
|
dm_train: str, |
|
|
s_train: str, |
|
|
dn_train: str, |
|
|
dm_test: str, |
|
|
s_test: str, |
|
|
dn_test: str, |
|
|
N_runs: int, |
|
|
severity=1, |
|
|
transform='identity', |
|
|
download_model=False): |
|
|
self.experiment_name = 'ABtesting' |
|
|
self.dataset_name = dataset_name |
|
|
self.augmentation = augmentation |
|
|
self.dm_train = dm_train |
|
|
self.s_train = s_train |
|
|
self.dn_train = dn_train |
|
|
self.dm_test = dm_test |
|
|
self.s_test = s_test |
|
|
self.dn_test = dn_test |
|
|
self.N_runs = N_runs |
|
|
self.severity = severity |
|
|
self.transform = transform |
|
|
self.download_model = download_model |
|
|
|
|
|
def static_pip_val(self, debayer=None, sharpening=None, denoising=None, severity=None, transform=None, plot_mode=False): |
|
|
|
|
|
if debayer == None: |
|
|
debayer = self.dm_test |
|
|
if sharpening == None: |
|
|
sharpening = self.s_test |
|
|
if denoising == None: |
|
|
denoising = self.dn_test |
|
|
if severity == None: |
|
|
severity = self.severity |
|
|
if transform == None: |
|
|
transform = self.transform |
|
|
|
|
|
dataset = get_dataset(self.dataset_name) |
|
|
|
|
|
if self.dataset_name == "Drone" or self.dataset_name == "DroneSegmentation": |
|
|
mean = torch.tensor([0.35, 0.36, 0.35]) |
|
|
std = torch.tensor([0.12, 0.11, 0.12]) |
|
|
elif self.dataset_name == "Microscopy": |
|
|
mean = torch.tensor([0.91, 0.84, 0.94]) |
|
|
std = torch.tensor([0.08, 0.12, 0.05]) |
|
|
|
|
|
if not plot_mode: |
|
|
dataset.transform = Compose([RawProcessingPipeline( |
|
|
camera_parameters=dataset.camera_parameters, |
|
|
debayer=debayer, |
|
|
sharpening=sharpening, |
|
|
denoising=denoising, |
|
|
), Distortions(severity=severity, transform=transform), |
|
|
Normalize(mean, std)]) |
|
|
else: |
|
|
dataset.transform = Compose([RawProcessingPipeline( |
|
|
camera_parameters=dataset.camera_parameters, |
|
|
debayer=debayer, |
|
|
sharpening=sharpening, |
|
|
denoising=denoising, |
|
|
), Distortions(severity=severity, transform=transform)]) |
|
|
|
|
|
return dataset |
|
|
|
|
|
def ABclassification(self): |
|
|
|
|
|
DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu' |
|
|
|
|
|
parent_run_name = f"{self.dataset_name}_{self.dm_train}_{self.s_train}_{self.dn_train}_{self.augmentation}" |
|
|
|
|
|
print( |
|
|
f'\nTraining pipeline:\n Dataset: {self.dataset_name}, Augmentation: {self.augmentation} \n Debayer: {self.dm_train}, Sharpening: {self.s_train}, Denoiser: {self.dn_train} \n') |
|
|
print(f'\nTesting pipeline:\n Dataset: {self.dataset_name}, Augmentation: {self.augmentation} \n Debayer: {self.dm_test}, Sharpening: {self.s_test}, Denoiser: {self.dn_test} \n Transform: {self.transform}, Severity: {self.severity}\n') |
|
|
|
|
|
accuracies, precisions, recalls, f1_scores = [], [], [], [] |
|
|
|
|
|
os.system('rm -r /tmp/py*') |
|
|
|
|
|
for N_run in range(self.N_runs): |
|
|
|
|
|
print(f"Evaluating Run {N_run}") |
|
|
|
|
|
run_name = parent_run_name + '_' + str(N_run) |
|
|
|
|
|
state_dict, model = get_mlflow_model_by_name(self.experiment_name, run_name, |
|
|
download_model=self.download_model) |
|
|
|
|
|
dataset = self.static_pip_val() |
|
|
valid_set = Subset(dataset, indices=state_dict['valid_indices']) |
|
|
valid_loader = DataLoader(valid_set, batch_size=1, num_workers=16, shuffle=False) |
|
|
|
|
|
model.eval() |
|
|
|
|
|
len_classes = len(dataset.classes) |
|
|
confusion_matrix = torch.zeros((len_classes, len_classes)) |
|
|
|
|
|
for img, label in valid_loader: |
|
|
|
|
|
prediction = model(img.to(DEVICE)).detach().cpu() |
|
|
prediction = torch.argmax(prediction, dim=1) |
|
|
confusion_matrix[label, prediction] += 1 |
|
|
|
|
|
m = metrics(confusion_matrix) |
|
|
|
|
|
accuracies.append(m.accuracy()) |
|
|
precisions.append(m.precision()) |
|
|
recalls.append(m.recall()) |
|
|
f1_scores.append(m.f1_score()) |
|
|
|
|
|
os.system('rm -r /tmp/t*') |
|
|
|
|
|
accuracy = metrics.over_N_runs(accuracies, self.N_runs) |
|
|
precision = metrics.over_N_runs(precisions, self.N_runs) |
|
|
recall = metrics.over_N_runs(recalls, self.N_runs) |
|
|
f1_score = metrics.over_N_runs(f1_scores, self.N_runs) |
|
|
return dataset.classes, accuracy, precision, recall, f1_score |
|
|
|
|
|
def ABsegmentation(self): |
|
|
|
|
|
DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu' |
|
|
|
|
|
parent_run_name = f"{self.dataset_name}_{self.dm_train}_{self.s_train}_{self.dn_train}_{self.augmentation}" |
|
|
|
|
|
print( |
|
|
f'\nTraining pipeline:\n Dataset: {self.dataset_name}, Augmentation: {self.augmentation} \n Debayer: {self.dm_train}, Sharpening: {self.s_train}, Denoiser: {self.dn_train} \n') |
|
|
print(f'\nTesting pipeline:\n Dataset: {self.dataset_name}, Augmentation: {self.augmentation} \n Debayer: {self.dm_test}, Sharpening: {self.s_test}, Denoiser: {self.dn_test} \n Transform: {self.transform}, Severity: {self.severity}\n') |
|
|
|
|
|
IoUs = [] |
|
|
|
|
|
os.system('rm -r /tmp/py*') |
|
|
|
|
|
for N_run in range(self.N_runs): |
|
|
|
|
|
print(f"Evaluating Run {N_run}") |
|
|
|
|
|
run_name = parent_run_name + '_' + str(N_run) |
|
|
|
|
|
state_dict, model = get_mlflow_model_by_name(self.experiment_name, run_name, |
|
|
download_model=self.download_model) |
|
|
|
|
|
dataset = self.static_pip_val() |
|
|
|
|
|
valid_set = Subset(dataset, indices=state_dict['valid_indices']) |
|
|
valid_loader = DataLoader(valid_set, batch_size=1, num_workers=16, shuffle=False) |
|
|
|
|
|
model.eval() |
|
|
|
|
|
IoU = 0 |
|
|
|
|
|
for img, label in valid_loader: |
|
|
|
|
|
prediction = model(img.to(DEVICE)).detach().cpu() |
|
|
prediction = F.logsigmoid(prediction).exp().squeeze() |
|
|
IoU += smp.utils.metrics.IoU()(prediction, label) |
|
|
|
|
|
IoU = IoU / len(valid_loader) |
|
|
IoUs.append(IoU.item()) |
|
|
|
|
|
os.system('rm -r /tmp/t*') |
|
|
|
|
|
IoU = metrics.over_N_runs(torch.tensor(IoUs), self.N_runs) |
|
|
return IoU |
|
|
|
|
|
def ABShowImages(self): |
|
|
|
|
|
path = 'results/ABtesting/imgs/' |
|
|
if not os.path.exists(path): |
|
|
os.makedirs(path) |
|
|
|
|
|
path = os.path.join( |
|
|
path, f'{self.dataset_name}_{self.augmentation}_{self.dm_train[:2]}{self.s_train[0]}{self.dn_train[:2]}_{self.dm_test[:2]}{self.s_test[0]}{self.dn_test[:2]}') |
|
|
|
|
|
if not os.path.exists(path): |
|
|
os.makedirs(path) |
|
|
|
|
|
run_name = f"{self.dataset_name}_{self.dm_train}_{self.s_train}_{self.dn_train}_{self.augmentation}" + \ |
|
|
'_' + str(0) |
|
|
|
|
|
state_dict, model = get_mlflow_model_by_name(self.experiment_name, run_name, download_model=self.download_model) |
|
|
|
|
|
model.augmentation = None |
|
|
|
|
|
for t in ([self.dm_train, self.s_train, self.dn_train, 'train_img'], |
|
|
[self.dm_test, self.s_test, self.dn_test, 'test_img']): |
|
|
|
|
|
debayer, sharpening, denoising, img_type = t[0], t[1], t[2], t[3] |
|
|
|
|
|
dataset = self.static_pip_val(debayer=debayer, sharpening=sharpening, denoising=denoising, plot_mode=True) |
|
|
valid_set = Subset(dataset, indices=state_dict['valid_indices']) |
|
|
|
|
|
img, _ = next(iter(valid_set)) |
|
|
|
|
|
plt.figure() |
|
|
plt.imshow(img.permute(1, 2, 0)) |
|
|
if img_type == 'train_img': |
|
|
plt.title('Train Image') |
|
|
plt.savefig(os.path.join(path, f'img_train.png')) |
|
|
imgA = img |
|
|
else: |
|
|
plt.title('Test Image') |
|
|
plt.savefig(os.path.join(path, f'img_test.png')) |
|
|
|
|
|
for c, color in enumerate(['Red', 'Green', 'Blue']): |
|
|
diff = torch.abs(imgA - img) |
|
|
plt.figure() |
|
|
|
|
|
plt.imshow(diff[c, 50:200, 50:200], cmap=f'{color}s') |
|
|
plt.title(f'|Train Image - Test Image| - {color}') |
|
|
plt.colorbar() |
|
|
plt.savefig(os.path.join(path, f'diff_{color}.png')) |
|
|
plt.figure() |
|
|
diff[diff == 0.] = 1e-5 |
|
|
|
|
|
plt.imshow(torch.log(diff)[c]) |
|
|
plt.title(f'log(|Train Image - Test Image|) - color') |
|
|
plt.colorbar() |
|
|
plt.savefig(os.path.join(path, f'logdiff_{color}.png')) |
|
|
|
|
|
if self.dataset_name == 'DroneSegmentation': |
|
|
plt.figure() |
|
|
plt.imshow(model(img[None].cuda()).detach().cpu().squeeze()) |
|
|
if img_type == 'train_img': |
|
|
plt.savefig(os.path.join(path, f'mask_train.png')) |
|
|
else: |
|
|
plt.savefig(os.path.join(path, f'mask_test.png')) |
|
|
|
|
|
def ABShowAllImages(self): |
|
|
if not os.path.exists('results/ABtesting'): |
|
|
os.makedirs('results/ABtesting') |
|
|
|
|
|
demosaicings = ['bilinear', 'malvar2004', 'menon2007'] |
|
|
sharpenings = ['sharpening_filter', 'unsharp_masking'] |
|
|
denoisings = ['median_denoising', 'gaussian_denoising'] |
|
|
|
|
|
fig = plt.figure() |
|
|
columns = 4 |
|
|
rows = 3 |
|
|
|
|
|
i = 1 |
|
|
|
|
|
for dm in demosaicings: |
|
|
for s in sharpenings: |
|
|
for dn in denoisings: |
|
|
|
|
|
dataset = self.static_pip_val(self.dm_test, self.s_test, |
|
|
self.dn_test, plot_mode=True) |
|
|
|
|
|
img, _ = dataset[0] |
|
|
|
|
|
fig.add_subplot(rows, columns, i) |
|
|
plt.imshow(img.permute(1, 2, 0)) |
|
|
plt.title(f'{dm}\n{s}\n{dn}', fontsize=8) |
|
|
plt.xticks([]) |
|
|
plt.yticks([]) |
|
|
plt.tight_layout() |
|
|
|
|
|
i += 1 |
|
|
|
|
|
plt.show() |
|
|
plt.savefig(f'results/ABtesting/ABpipelines.png') |
|
|
|
|
|
def CShowImages(self): |
|
|
|
|
|
path = 'results/Ctesting/imgs/' |
|
|
if not os.path.exists(path): |
|
|
os.makedirs(path) |
|
|
|
|
|
run_name = f"{self.dataset_name}_{self.dm_test}_{self.s_test}_{self.dn_test}_{self.augmentation}" + '_' + str(0) |
|
|
|
|
|
state_dict, model = get_mlflow_model_by_name(self.experiment_name, run_name, download_model=True) |
|
|
|
|
|
model.augmentation = None |
|
|
|
|
|
dataset = self.static_pip_val(self.dm_test, self.s_test, self.dn_test, |
|
|
self.severity, self.transform, plot_mode=True) |
|
|
valid_set = Subset(dataset, indices=state_dict['valid_indices']) |
|
|
|
|
|
img, _ = next(iter(valid_set)) |
|
|
|
|
|
plt.figure() |
|
|
plt.imshow(img.permute(1, 2, 0)) |
|
|
plt.savefig(os.path.join( |
|
|
path, f'{self.dataset_name}_{self.augmentation}_{self.dm_train[:2]}{self.s_train[0]}{self.dn_train[:2]}_{self.transform}_sev{self.severity}')) |
|
|
|
|
|
def CShowAllImages(self): |
|
|
if not os.path.exists('results/Cimages'): |
|
|
os.makedirs('results/Cimages') |
|
|
|
|
|
transforms = ['identity', 'gaussian_noise', 'shot_noise', 'impulse_noise', 'speckle_noise', |
|
|
'gaussian_blur', 'zoom_blur', 'contrast', 'brightness', 'saturate', 'elastic_transform'] |
|
|
|
|
|
for i, t in enumerate(transforms): |
|
|
|
|
|
fig = plt.figure(figsize=(10, 6)) |
|
|
columns = 5 |
|
|
rows = 1 |
|
|
|
|
|
for sev in range(1, 6): |
|
|
|
|
|
dataset = self.static_pip_val(severity=sev, transform=t, plot_mode=True) |
|
|
|
|
|
img, _ = dataset[0] |
|
|
|
|
|
fig.add_subplot(rows, columns, sev) |
|
|
plt.imshow(img.permute(1, 2, 0)) |
|
|
plt.title(f'Severity: {sev}') |
|
|
plt.xticks([]) |
|
|
plt.yticks([]) |
|
|
plt.tight_layout() |
|
|
|
|
|
if '_' in t: |
|
|
t = t.replace('_', ' ') |
|
|
t = t[0].upper() + t[1:] |
|
|
|
|
|
fig.suptitle(f'{t}', x=0.5, y=0.8, fontsize=24) |
|
|
plt.show() |
|
|
plt.savefig(f'results/Cimages/{i+1}_{t.lower()}.png') |
|
|
|
|
|
|
|
|
def ABMakeTable(dataset_name: str, augmentation: str, |
|
|
N_runs: int, download_model: bool): |
|
|
|
|
|
demosaicings = ['bilinear', 'malvar2004', 'menon2007'] |
|
|
sharpenings = ['sharpening_filter', 'unsharp_masking'] |
|
|
denoisings = ['median_denoising', 'gaussian_denoising'] |
|
|
|
|
|
path = 'results/ABtesting/tables' |
|
|
if not os.path.exists(path): |
|
|
os.makedirs(path) |
|
|
|
|
|
runs = {} |
|
|
i = 0 |
|
|
|
|
|
for dm_train in demosaicings: |
|
|
for s_train in sharpenings: |
|
|
for dn_train in denoisings: |
|
|
for dm_test in demosaicings: |
|
|
for s_test in sharpenings: |
|
|
for dn_test in denoisings: |
|
|
train_pip = [dm_train, s_train, dn_train] |
|
|
test_pip = [dm_test, s_test, dn_test] |
|
|
runs[f'run{i}'] = { |
|
|
'dataset': dataset_name, |
|
|
'augmentation': augmentation, |
|
|
'train_pip': train_pip, |
|
|
'test_pip': test_pip, |
|
|
'N_runs': N_runs |
|
|
} |
|
|
ABclass = ABtesting( |
|
|
dataset_name=dataset_name, |
|
|
augmentation=augmentation, |
|
|
dm_train=dm_train, |
|
|
s_train=s_train, |
|
|
dn_train=dn_train, |
|
|
dm_test=dm_test, |
|
|
s_test=s_test, |
|
|
dn_test=dn_test, |
|
|
N_runs=N_runs, |
|
|
download_model=download_model |
|
|
) |
|
|
|
|
|
if dataset_name == 'DroneSegmentation': |
|
|
IoU = ABclass.ABsegmentation() |
|
|
runs[f'run{i}']['IoU'] = IoU |
|
|
else: |
|
|
classes, accuracy, precision, recall, f1_score = ABclass.ABclassification() |
|
|
runs[f'run{i}']['classes'] = classes |
|
|
runs[f'run{i}']['accuracy'] = accuracy |
|
|
runs[f'run{i}']['precision'] = precision |
|
|
runs[f'run{i}']['recall'] = recall |
|
|
runs[f'run{i}']['f1_score'] = f1_score |
|
|
|
|
|
with open(os.path.join(path, f'{dataset_name}_{augmentation}_runs.txt'), 'w') as outfile: |
|
|
json.dump(runs, outfile) |
|
|
|
|
|
i += 1 |
|
|
|
|
|
|
|
|
def ABShowTable(dataset_name: str, augmentation: str): |
|
|
|
|
|
path = 'results/ABtesting/tables' |
|
|
assert os.path.exists(path), 'No tables to plot' |
|
|
|
|
|
json_file = os.path.join(path, f'{dataset_name}_{augmentation}_runs.txt') |
|
|
|
|
|
with open(json_file, 'r') as run_file: |
|
|
runs = json.load(run_file) |
|
|
|
|
|
metrics = torch.zeros((2, 12, 12)) |
|
|
classes = [] |
|
|
|
|
|
i, j = 0, 0 |
|
|
|
|
|
for r in range(len(runs)): |
|
|
|
|
|
run = runs['run' + str(r)] |
|
|
if dataset_name == 'DroneSegmentation': |
|
|
acc = run['IoU'] |
|
|
else: |
|
|
acc = run['accuracy'] |
|
|
if len(classes) < 12: |
|
|
class_list = run['test_pip'] |
|
|
class_name = f'{class_list[0][:2]},{class_list[1][:1]},{class_list[2][:2]}' |
|
|
classes.append(class_name) |
|
|
mu, sigma = round(acc[0], 4), round(acc[1], 4) |
|
|
|
|
|
metrics[0, j, i] = mu |
|
|
metrics[1, j, i] = sigma |
|
|
|
|
|
i += 1 |
|
|
|
|
|
if i == 12: |
|
|
i = 0 |
|
|
j += 1 |
|
|
|
|
|
differences = torch.zeros_like(metrics) |
|
|
|
|
|
diag_mu = torch.diagonal(metrics[0], 0) |
|
|
diag_sigma = torch.diagonal(metrics[1], 0) |
|
|
|
|
|
for r in range(len(metrics[0])): |
|
|
differences[0, r] = diag_mu[r] - metrics[0, r] |
|
|
differences[1, r] = torch.sqrt(metrics[1, r]**2 + diag_sigma[r]**2) |
|
|
|
|
|
|
|
|
|
|
|
for i, img in enumerate([metrics, differences]): |
|
|
|
|
|
x, y = torch.arange(12), torch.arange(12) |
|
|
x, y = torch.meshgrid(x, y) |
|
|
|
|
|
if i == 0: |
|
|
vmin = max(0.65, round(img[0].min().item(), 2)) |
|
|
vmax = round(img[0].max().item(), 2) |
|
|
step = 0.02 |
|
|
elif i == 1: |
|
|
vmin = round(img[0].min().item(), 2) |
|
|
if augmentation == 'none': |
|
|
vmax = min(0.15, round(img[0].max().item(), 2)) |
|
|
if augmentation == 'weak': |
|
|
vmax = min(0.08, round(img[0].max().item(), 2)) |
|
|
if augmentation == 'strong': |
|
|
vmax = min(0.05, round(img[0].max().item(), 2)) |
|
|
step = 0.01 |
|
|
|
|
|
vmin = int(vmin / step) * step |
|
|
vmax = int(vmax / step) * step |
|
|
|
|
|
fig = plt.figure(figsize=(10, 6.2)) |
|
|
ax = fig.add_axes([0.1, 0.1, 0.8, 0.8]) |
|
|
marker_size = 350 |
|
|
plt.scatter(x, y, c=torch.rot90(img[1][x, y], -1, [0, 1]), vmin=0., |
|
|
vmax=img[1].max(), cmap='viridis', s=marker_size * 2, marker='s') |
|
|
ticks = torch.arange(0., img[1].max(), 0.03).tolist() |
|
|
ticks = [round(tick, 2) for tick in ticks] |
|
|
cba = plt.colorbar(pad=0.06) |
|
|
cba.set_ticks(ticks) |
|
|
cba.ax.set_yticklabels(ticks) |
|
|
|
|
|
cmap = plt.cm.get_cmap('Reds') |
|
|
plt.scatter(x, y, c=torch.rot90(img[0][x, y], -1, [0, 1]), vmin=vmin, |
|
|
vmax=vmax, cmap=cmap, s=marker_size, marker='s') |
|
|
ticks = torch.arange(vmin, vmax, step).tolist() |
|
|
ticks = [round(tick, 2) for tick in ticks] |
|
|
if ticks[-1] != vmax: |
|
|
ticks.append(vmax) |
|
|
cbb = plt.colorbar(pad=0.06) |
|
|
cbb.set_ticks(ticks) |
|
|
if i == 0: |
|
|
ticks[0] = f'<{str(ticks[0])}' |
|
|
elif i == 1: |
|
|
ticks[-1] = f'>{str(ticks[-1])}' |
|
|
cbb.ax.set_yticklabels(ticks) |
|
|
for x in range(12): |
|
|
for y in range(12): |
|
|
txt = round(torch.rot90(img[0], -1, [0, 1])[x, y].item(), 2) |
|
|
if str(txt) == '-0.0': |
|
|
txt = '0.00' |
|
|
elif str(txt) == '0.0': |
|
|
txt = '0.00' |
|
|
elif len(str(txt)) == 3: |
|
|
txt = str(txt) + '0' |
|
|
else: |
|
|
txt = str(txt) |
|
|
|
|
|
plt.text(x - 0.25, y - 0.1, txt, color='black', fontsize='x-small') |
|
|
|
|
|
ax.set_xticks(torch.linspace(0, 11, 12)) |
|
|
ax.set_xticklabels(classes) |
|
|
ax.set_yticks(torch.linspace(0, 11, 12)) |
|
|
classes.reverse() |
|
|
ax.set_yticklabels(classes) |
|
|
classes.reverse() |
|
|
plt.xticks(rotation=45) |
|
|
plt.yticks(rotation=45) |
|
|
cba.set_label('Standard Deviation') |
|
|
plt.xlabel("Test pipelines") |
|
|
plt.ylabel("Train pipelines") |
|
|
plt.title(f'Dataset: {dataset_name}, Augmentation: {augmentation}') |
|
|
if i == 0: |
|
|
if dataset_name == 'DroneSegmentation': |
|
|
cbb.set_label('IoU') |
|
|
plt.savefig(os.path.join(path, f"{dataset_name}_{augmentation}_IoU.png")) |
|
|
else: |
|
|
cbb.set_label('Accuracy') |
|
|
plt.savefig(os.path.join(path, f"{dataset_name}_{augmentation}_accuracies.png")) |
|
|
elif i == 1: |
|
|
if dataset_name == 'DroneSegmentation': |
|
|
cbb.set_label('IoU_d-IoU') |
|
|
else: |
|
|
cbb.set_label('Accuracy_d - Accuracy') |
|
|
plt.savefig(os.path.join(path, f"{dataset_name}_{augmentation}_differences.png")) |
|
|
|
|
|
|
|
|
def CMakeTable(dataset_name: str, augmentation: str, severity: int, N_runs: int, download_model: bool): |
|
|
|
|
|
path = 'results/Ctesting/tables' |
|
|
if not os.path.exists(path): |
|
|
os.makedirs(path) |
|
|
|
|
|
demosaicings = ['bilinear', 'malvar2004', 'menon2007'] |
|
|
sharpenings = ['sharpening_filter', 'unsharp_masking'] |
|
|
denoisings = ['median_denoising', 'gaussian_denoising'] |
|
|
|
|
|
transformations = ['identity', 'gaussian_noise', 'shot_noise', 'impulse_noise', 'speckle_noise', |
|
|
'gaussian_blur', 'zoom_blur', 'contrast', 'brightness', 'saturate', 'elastic_transform'] |
|
|
|
|
|
runs = {} |
|
|
i = 0 |
|
|
|
|
|
for dm in demosaicings: |
|
|
for s in sharpenings: |
|
|
for dn in denoisings: |
|
|
for t in transformations: |
|
|
pip = [dm, s, dn] |
|
|
runs[f'run{i}'] = { |
|
|
'dataset': dataset_name, |
|
|
'augmentation': augmentation, |
|
|
'pipeline': pip, |
|
|
'N_runs': N_runs, |
|
|
'transform': t, |
|
|
'severity': severity, |
|
|
} |
|
|
ABclass = ABtesting( |
|
|
dataset_name=dataset_name, |
|
|
augmentation=augmentation, |
|
|
dm_train=dm, |
|
|
s_train=s, |
|
|
dn_train=dn, |
|
|
dm_test=dm, |
|
|
s_test=s, |
|
|
dn_test=dn, |
|
|
severity=severity, |
|
|
transform=t, |
|
|
N_runs=N_runs, |
|
|
download_model=download_model |
|
|
) |
|
|
|
|
|
if dataset_name == 'DroneSegmentation': |
|
|
IoU = ABclass.ABsegmentation() |
|
|
runs[f'run{i}']['IoU'] = IoU |
|
|
else: |
|
|
classes, accuracy, precision, recall, f1_score = ABclass.ABclassification() |
|
|
runs[f'run{i}']['classes'] = classes |
|
|
runs[f'run{i}']['accuracy'] = accuracy |
|
|
runs[f'run{i}']['precision'] = precision |
|
|
runs[f'run{i}']['recall'] = recall |
|
|
runs[f'run{i}']['f1_score'] = f1_score |
|
|
|
|
|
with open(os.path.join(path, f'{dataset_name}_{augmentation}_runs.json'), 'w') as outfile: |
|
|
json.dump(runs, outfile) |
|
|
|
|
|
i += 1 |
|
|
|
|
|
|
|
|
def CShowTable(dataset_name, augmentation): |
|
|
|
|
|
path = 'results/Ctesting/tables' |
|
|
assert os.path.exists(path), 'No tables to plot' |
|
|
|
|
|
json_file = os.path.join(path, f'{dataset_name}_{augmentation}_runs.txt') |
|
|
|
|
|
transforms = ['identity', 'gauss_noise', 'shot', 'impulse', 'speckle', |
|
|
'gauss_blur', 'zoom', 'contrast', 'brightness', 'saturate', 'elastic'] |
|
|
|
|
|
pip = [] |
|
|
|
|
|
demosaicings = ['bilinear', 'malvar2004', 'menon2007'] |
|
|
sharpenings = ['sharpening_filter', 'unsharp_masking'] |
|
|
denoisings = ['median_denoising', 'gaussian_denoising'] |
|
|
|
|
|
for dm in demosaicings: |
|
|
for s in sharpenings: |
|
|
for dn in denoisings: |
|
|
pip.append(f'{dm[:2]},{s[0]},{dn[2]}') |
|
|
|
|
|
with open(json_file, 'r') as run_file: |
|
|
runs = json.load(run_file) |
|
|
|
|
|
metrics = torch.zeros((2, len(pip), len(transforms))) |
|
|
|
|
|
i, j = 0, 0 |
|
|
|
|
|
for r in range(len(runs)): |
|
|
|
|
|
run = runs['run' + str(r)] |
|
|
if dataset_name == 'DroneSegmentation': |
|
|
acc = run['IoU'] |
|
|
else: |
|
|
acc = run['accuracy'] |
|
|
mu, sigma = round(acc[0], 4), round(acc[1], 4) |
|
|
|
|
|
metrics[0, j, i] = mu |
|
|
metrics[1, j, i] = sigma |
|
|
|
|
|
i += 1 |
|
|
|
|
|
if i == len(transforms): |
|
|
i = 0 |
|
|
j += 1 |
|
|
|
|
|
|
|
|
|
|
|
img = metrics |
|
|
|
|
|
vmin = 0. |
|
|
vmax = 1. |
|
|
|
|
|
x, y = torch.arange(12), torch.arange(11) |
|
|
x, y = torch.meshgrid(x, y) |
|
|
|
|
|
fig = plt.figure(figsize=(10, 6.2)) |
|
|
ax = fig.add_axes([0.1, 0.1, 0.8, 0.8]) |
|
|
marker_size = 350 |
|
|
plt.scatter(x, y, c=torch.rot90(img[1][x, y], -1, [0, 1]), vmin=0., |
|
|
vmax=img[1].max(), cmap='viridis', s=marker_size * 2, marker='s') |
|
|
ticks = torch.arange(0., img[1].max(), 0.03).tolist() |
|
|
ticks = [round(tick, 2) for tick in ticks] |
|
|
cba = plt.colorbar(pad=0.06) |
|
|
cba.set_ticks(ticks) |
|
|
cba.ax.set_yticklabels(ticks) |
|
|
|
|
|
cmap = plt.cm.get_cmap('Reds') |
|
|
plt.scatter(x, y, c=torch.rot90(img[0][x, y], -1, [0, 1]), vmin=vmin, |
|
|
vmax=vmax, cmap=cmap, s=marker_size, marker='s') |
|
|
ticks = torch.arange(vmin, vmax, step).tolist() |
|
|
ticks = [round(tick, 2) for tick in ticks] |
|
|
if ticks[-1] != vmax: |
|
|
ticks.append(vmax) |
|
|
cbb = plt.colorbar(pad=0.06) |
|
|
cbb.set_ticks(ticks) |
|
|
if i == 0: |
|
|
ticks[0] = f'<{str(ticks[0])}' |
|
|
elif i == 1: |
|
|
ticks[-1] = f'>{str(ticks[-1])}' |
|
|
cbb.ax.set_yticklabels(ticks) |
|
|
for x in range(12): |
|
|
for y in range(12): |
|
|
txt = round(torch.rot90(img[0], -1, [0, 1])[x, y].item(), 2) |
|
|
if str(txt) == '-0.0': |
|
|
txt = '0.00' |
|
|
elif str(txt) == '0.0': |
|
|
txt = '0.00' |
|
|
elif len(str(txt)) == 3: |
|
|
txt = str(txt) + '0' |
|
|
else: |
|
|
txt = str(txt) |
|
|
|
|
|
plt.text(x - 0.25, y - 0.1, txt, color='black', fontsize='x-small') |
|
|
|
|
|
ax.set_xticks(torch.linspace(0, 11, 12)) |
|
|
ax.set_xticklabels(transforms) |
|
|
ax.set_yticks(torch.linspace(0, 11, 12)) |
|
|
pip.reverse() |
|
|
ax.set_yticklabels(pip) |
|
|
pip.reverse() |
|
|
plt.xticks(rotation=45) |
|
|
plt.yticks(rotation=45) |
|
|
cba.set_label('Standard Deviation') |
|
|
plt.xlabel("Pipelines") |
|
|
plt.ylabel("Distortions") |
|
|
if dataset_name == 'DroneSegmentation': |
|
|
cbb.set_label('IoU') |
|
|
plt.savefig(os.path.join(path, f"{dataset_name}_{augmentation}_IoU.png")) |
|
|
else: |
|
|
cbb.set_label('Accuracy') |
|
|
plt.savefig(os.path.join(path, f"{dataset_name}_{augmentation}_accuracies.png")) |
|
|
|
|
|
|
|
|
if __name__ == '__main__': |
|
|
|
|
|
if args.mode == 'ABMakeTable': |
|
|
ABMakeTable(args.dataset_name, args.augmentation, args.N_runs, args.download_model) |
|
|
elif args.mode == 'ABShowTable': |
|
|
ABShowTable(args.dataset_name, args.augmentation) |
|
|
elif args.mode == 'ABShowImages': |
|
|
ABclass = ABtesting(args.dataset_name, args.augmentation, args.dm_train, |
|
|
args.s_train, args.dn_train, args.dm_test, args.s_test, |
|
|
args.dn_test, args.N_runs, download_model=args.download_model) |
|
|
ABclass.ABShowImages() |
|
|
elif args.mode == 'ABShowAllImages': |
|
|
ABclass = ABtesting(args.dataset_name, args.augmentation, args.dm_train, |
|
|
args.s_train, args.dn_train, args.dm_test, args.s_test, |
|
|
args.dn_test, args.N_runs, download_model=args.download_model) |
|
|
ABclass.ABShowAllImages() |
|
|
elif args.mode == 'CMakeTable': |
|
|
CMakeTable(args.dataset_name, args.augmentation, args.severity, args.N_runs, args.download_model) |
|
|
elif args.mode == 'CShowTable': |
|
|
CShowTable(args.dataset_name, args.augmentation, args.severity) |
|
|
elif args.mode == 'CShowImages': |
|
|
ABclass = ABtesting(args.dataset_name, args.augmentation, args.dm_train, |
|
|
args.s_train, args.dn_train, args.dm_test, args.s_test, |
|
|
args.dn_test, args.N_runs, args.severity, args.transform, |
|
|
download_model=args.download_model) |
|
|
ABclass.CShowImages() |
|
|
elif args.mode == 'CShowAllImages': |
|
|
ABclass = ABtesting(args.dataset_name, args.augmentation, args.dm_train, |
|
|
args.s_train, args.dn_train, args.dm_test, args.s_test, |
|
|
args.dn_test, args.N_runs, args.severity, args.transform, |
|
|
download_model=args.download_model) |
|
|
ABclass.CShowAllImages() |
|
|
|
