# Copyright (C) 2025 NVIDIA Corporation. All rights reserved. # # This work is licensed under the LICENSE file # located at the root directory. import cv2 import numpy as np from PIL import Image, ImageDraw import torch import matplotlib.pyplot as plt from skimage import filters from IPython.display import display def gaussian_blur(heatmap, kernel_size=7): # Shape of heatmap: (H, W) heatmap = heatmap.cpu().numpy() heatmap = cv2.GaussianBlur(heatmap, (kernel_size, kernel_size), 0) heatmap = torch.tensor(heatmap) return heatmap def show_cam_on_image(img, mask): heatmap = cv2.applyColorMap(np.uint8(255 * mask), cv2.COLORMAP_JET) heatmap = np.float32(heatmap) / 255 cam = heatmap + np.float32(img) cam = cam / np.max(cam) return cam def show_image_and_heatmap(heatmap: torch.Tensor, image: Image.Image, relevnace_res: int = 256, interpolation: str = 'bilinear', gassussian_kernel_size: int = 3): image = image.resize((relevnace_res, relevnace_res)) image = np.array(image) image = (image - image.min()) / (image.max() - image.min()) # Apply gaussian blur to heatmap # heatmap = gaussian_blur(heatmap, kernel_size=gassussian_kernel_size) # heatmap = (heatmap - heatmap.min()) / (heatmap.max() - heatmap.min()) # otsu_thr = filters.threshold_otsu(heatmap.cpu().numpy()) # heatmap = (heatmap > otsu_thr).to(heatmap.dtype) heatmap = heatmap.reshape(1, 1, heatmap.shape[-1], heatmap.shape[-1]) heatmap = torch.nn.functional.interpolate(heatmap, size=relevnace_res, mode=interpolation) heatmap = (heatmap - heatmap.min()) / (heatmap.max() - heatmap.min()) heatmap = heatmap.reshape(relevnace_res, relevnace_res).cpu() vis = show_cam_on_image(image, heatmap) vis = np.uint8(255 * vis) vis = cv2.cvtColor(np.array(vis), cv2.COLOR_RGB2BGR) vis = vis.astype(np.uint8) vis = Image.fromarray(vis).resize((relevnace_res, relevnace_res)) return vis def show_only_heatmap(heatmap: torch.Tensor, relevnace_res: int = 256, interpolation: str = 'bilinear', gassussian_kernel_size: int = 3): # Apply gaussian blur to heatmap # heatmap = gaussian_blur(heatmap, kernel_size=gassussian_kernel_size) heatmap = heatmap.reshape(1, 1, heatmap.shape[-1], heatmap.shape[-1]) heatmap = torch.nn.functional.interpolate(heatmap, size=relevnace_res, mode=interpolation) heatmap = (heatmap - heatmap.min()) / (heatmap.max() - heatmap.min()) heatmap = heatmap.reshape(relevnace_res, relevnace_res).cpu() vis = heatmap vis = np.uint8(255 * vis) # Show in black and white vis = cv2.cvtColor(np.array(vis), cv2.COLOR_GRAY2BGR) vis = Image.fromarray(vis).resize((relevnace_res, relevnace_res)) return vis def visualize_tokens_attentions(attention, tokens, image, heatmap_interpolation="nearest", show_on_image=True): # Tokens: list of strings # attention: tensor of shape (batch_size, num_tokens, width, height) token_vis = [] for j, token in enumerate(tokens): if j >= attention.shape[0]: break if show_on_image: vis = show_image_and_heatmap(attention[j], image, relevnace_res=512, interpolation=heatmap_interpolation) else: vis = show_only_heatmap(attention[j], relevnace_res=512, interpolation=heatmap_interpolation) token_vis.append((token, vis)) # Display the token and the attention map in a grid, with K tokens per row K = 4 n_rows = (len(token_vis) + K - 1) // K # Ceiling division fig, axs = plt.subplots(n_rows, K, figsize=(K*5, n_rows*5)) for i, (token, vis) in enumerate(token_vis): row, col = divmod(i, K) if n_rows > 1: ax = axs[row, col] elif K > 1: ax = axs[col] else: ax = axs ax.imshow(vis) ax.set_title(token) ax.axis("off") # Hide unused subplots for j in range(i + 1, n_rows * K): row, col = divmod(j, K) if n_rows > 1: axs[row, col].axis('off') elif K > 1: axs[col].axis('off') plt.tight_layout() # We want to return the figure so that we can save it to a file return fig def show_images(images, titles=None, size=1024, max_row_length=5, figsize=None, col_height=10, save_path=None): if isinstance(images, Image.Image): images = [images] if len(images) == 1: img = images[0] img = img.resize((size, size)) plt.imshow(img) plt.axis('off') if titles is not None: plt.title(titles[0]) if save_path: plt.savefig(save_path, bbox_inches='tight', dpi=150) plt.show() else: images = [img.resize((size, size)) for img in images] # Check if the number of titles matches the number of images if titles is not None: assert len(images) == len(titles), "Number of titles should match the number of images" n_images = len(images) n_cols = min(n_images, max_row_length) n_rows = (n_images + n_cols - 1) // n_cols # Calculate the number of rows needed if figsize is None: figsize=(n_cols * col_height, n_rows * col_height) fig, axs = plt.subplots(n_rows, n_cols, figsize=figsize) axs = axs.flatten() if isinstance(axs, np.ndarray) else [axs] # Display images in the subplots for i, img in enumerate(images): axs[i].imshow(img) if titles is not None: axs[i].set_title(titles[i]) axs[i].axis("off") # Turn off any unused subplots for ax in axs[len(images):]: ax.axis("off") if save_path: plt.savefig(save_path, bbox_inches='tight', dpi=150) plt.show() def show_tensors(tensors, titles=None, size=None, max_row_length=5): # Shape of tensors: List[Tensor[H, W]] if size is not None: tensors = [torch.nn.functional.interpolate(t.unsqueeze(0).unsqueeze(0), size=(size, size), mode='bilinear').squeeze() for t in tensors] if len(tensors) == 1: plt.imshow(tensors[0].cpu().numpy()) plt.axis('off') if titles is not None: plt.title(titles[0]) plt.show() else: # Check if the number of titles matches the number of images if titles is not None: assert len(tensors) == len(titles), "Number of titles should match the number of images" n_tensors = len(tensors) n_cols = min(n_tensors, max_row_length) n_rows = (n_tensors + n_cols - 1) // n_cols fig, axs = plt.subplots(n_rows, n_cols, figsize=(n_cols * 10, n_rows * 10)) axs = axs.flatten() if isinstance(axs, np.ndarray) else [axs] for i, tensor in enumerate(tensors): axs[i].imshow(tensor.cpu().numpy()) if titles is not None: axs[i].set_title(titles[i]) axs[i].axis("off") for ax in axs[len(tensors):]: ax.axis("off") plt.show() def draw_bboxes_on_image(image, bboxes, color="red", thickness=2): image = image.copy() draw = ImageDraw.Draw(image) for bbox in bboxes: draw.rectangle(bbox, outline=color, width=thickness) return image def draw_points_on_pil_image(pil_image, point_coords, point_color="red", radius=5): """ Draw points (circles) on a PIL image and return the modified image. :param pil_image: PIL Image (e.g., sam_masked_image) :param point_coords: An array-like of shape (N, 2), with x,y coordinates :param point_color: Color of the point (default 'red') :param radius: Radius of the drawn circles :return: PIL Image with points drawn """ # Copy so we don't modify the original out_img = pil_image.copy() draw = ImageDraw.Draw(out_img) # Draw each point for x, y in point_coords: # Calculate bounding box of the circle left_up_point = (x - radius, y - radius) right_down_point = (x + radius, y + radius) # Draw the circle draw.ellipse([left_up_point, right_down_point], fill=point_color, outline=point_color) return out_img