import os import cv2 as cv import numpy as np from tqdm import tqdm class MiniSupervisely : ''' Refer to https://github.com/PaddlePaddle/PaddleSeg/blob/release/2.7/paddleseg/core/val.py for official evaluation implementation. ''' def __init__(self, root) : self.root = root self.val_path = os.path.join(root, 'val.txt') self.image_set = self.load_data(self.val_path) self.num_classes = 2 self.miou = -1 self.class_miou = -1 self.acc = -1 self.class_acc = -1 @property def name(self): return self.__class__.__name__ def load_data(self, val_path) : """ Load validation image set from val.txt file Args : val_path (str) : path to val.txt file Returns : image_set (list) : list of image path of input and expected image """ image_set = [] with open(val_path, 'r') as f : for line in f.readlines() : image_set.append(line.strip().split()) return image_set def eval(self, model) : """ Evaluate model on validation set Args : model (object) : PP_HumanSeg model object """ intersect_area_all = np.zeros([1], dtype=np.int64) pred_area_all = np.zeros([1], dtype=np.int64) label_area_all = np.zeros([1], dtype=np.int64) pbar = tqdm(self.image_set) pbar.set_description( "Evaluating {} with {} val set".format(model.name, self.name)) for input_image, expected_image in pbar : input_image = cv.imread(os.path.join(self.root, input_image)).astype('float32') expected_image = cv.imread(os.path.join(self.root, expected_image), cv.IMREAD_GRAYSCALE)[np.newaxis, :, :] output_image = model.infer(input_image) intersect_area, pred_area, label_area = self.calculate_area( output_image.astype('uint32'), expected_image.astype('uint32'), self.num_classes) intersect_area_all = intersect_area_all + intersect_area pred_area_all = pred_area_all + pred_area label_area_all = label_area_all + label_area self.class_iou, self.miou = self.mean_iou(intersect_area_all, pred_area_all, label_area_all) self.class_acc, self.acc = self.accuracy(intersect_area_all, pred_area_all) def get_results(self) : """ Get evaluation results Returns : miou (float) : mean iou class_miou (list) : iou on all classes acc (float) : mean accuracy class_acc (list) : accuracy on all classes """ return self.miou, self.class_miou, self.acc, self.class_acc def print_result(self) : """ Print evaluation results """ print("Mean IoU : ", self.miou) print("Mean Accuracy : ", self.acc) print("Class IoU : ", self.class_iou) print("Class Accuracy : ", self.class_acc) def calculate_area(self,pred, label, num_classes, ignore_index=255): """ Calculate intersect, prediction and label area Args: pred (Tensor): The prediction by model. label (Tensor): The ground truth of image. num_classes (int): The unique number of target classes. ignore_index (int): Specifies a target value that is ignored. Default: 255. Returns: Tensor: The intersection area of prediction and the ground on all class. Tensor: The prediction area on all class. Tensor: The ground truth area on all class """ if len(pred.shape) == 4: pred = np.squeeze(pred, axis=1) if len(label.shape) == 4: label = np.squeeze(label, axis=1) if not pred.shape == label.shape: raise ValueError('Shape of `pred` and `label should be equal, ' 'but there are {} and {}.'.format(pred.shape, label.shape)) mask = label != ignore_index pred_area = [] label_area = [] intersect_area = [] #iterate over all classes and calculate their respective areas for i in range(num_classes): pred_i = np.logical_and(pred == i, mask) label_i = label == i intersect_i = np.logical_and(pred_i, label_i) pred_area.append(np.sum(pred_i.astype('int32'))) label_area.append(np.sum(label_i.astype('int32'))) intersect_area.append(np.sum(intersect_i.astype('int32'))) return intersect_area, pred_area, label_area def mean_iou(self,intersect_area, pred_area, label_area): """ Calculate iou. Args: intersect_area (Tensor): The intersection area of prediction and ground truth on all classes. pred_area (Tensor): The prediction area on all classes. label_area (Tensor): The ground truth area on all classes. Returns: np.ndarray: iou on all classes. float: mean iou of all classes. """ intersect_area = np.array(intersect_area) pred_area = np.array(pred_area) label_area = np.array(label_area) union = pred_area + label_area - intersect_area class_iou = [] for i in range(len(intersect_area)): if union[i] == 0: iou = 0 else: iou = intersect_area[i] / union[i] class_iou.append(iou) miou = np.mean(class_iou) return np.array(class_iou), miou def accuracy(self,intersect_area, pred_area): """ Calculate accuracy Args: intersect_area (Tensor): The intersection area of prediction and ground truth on all classes.. pred_area (Tensor): The prediction area on all classes. Returns: np.ndarray: accuracy on all classes. float: mean accuracy. """ intersect_area = np.array(intersect_area) pred_area = np.array(pred_area) class_acc = [] for i in range(len(intersect_area)): if pred_area[i] == 0: acc = 0 else: acc = intersect_area[i] / pred_area[i] class_acc.append(acc) macc = np.sum(intersect_area) / np.sum(pred_area) return np.array(class_acc), macc