Create app.py

app.py

@@ -0,0 +1,84 @@
+"""
+Using as reference:
+- https://huggingface.co/nvidia/segformer-b0-finetuned-ade-512-512
+- https://huggingface.co/spaces/chansung/segformer-tf-transformers/blob/main/app.py
+- https://huggingface.co/facebook/detr-resnet-50-panoptic
+# https://tech.amikelive.com/node-718/what-object-categories-labels-are-in-coco-dataset/
+https://colab.research.google.com/github/NielsRogge/Transformers-Tutorials/blob/master/DETR/DETR_panoptic_segmentation_minimal_example_(with_DetrFeatureExtractor).ipynb
+https://arxiv.org/abs/2005.12872
+https://arxiv.org/pdf/1801.00868.pdf 
+Additions
+- add shown labels as strings
+- show only animal masks (ask an nlp model?)
+For next time
+- for diff 'confidence' the high conf masks should change....
+- colors are not great and should be constant per class? add text?
+- Im getting core dumped (segmentation fault) when loading hugging face model.. :()
+    https://github.com/huggingface/transformers/issues/16939 
+- cap slider to 95?
+- switch between panoptic and semantic?
+"""
+
+from transformers import DetrFeatureExtractor, DetrForSegmentation
+from PIL import Image
+import gradio as gr
+import numpy as np
+import torch
+import torchvision
+
+import itertools
+import seaborn as sns
+
+def predict_building_mask(im,
+                        gr_slider_confidence):
+    image = Image.fromarray(im) # im: numpy array 3d: 480, 640, 3: to PIL Image
+    image = image.resize((200,200)) #  PIL image # could I upsample output instead? better?
+
+    # encoding is a dict with pixel_values and pixel_mask
+    encoding = feature_extractor(images=image, return_tensors="pt") #pt=Pytorch, tf=TensorFlow
+    outputs = model(**encoding) # odict with keys: ['logits', 'pred_boxes', 'pred_masks', 'last_hidden_state', 'encoder_last_hidden_state']
+    logits = outputs.logits # torch.Size([1, 100, 251]); class logits? but  why 251?
+    bboxes = outputs.pred_boxes
+    masks = outputs.pred_masks # torch.Size([1, 100, 200, 200]); mask logits? for every pixel, score in each of the 100 classes? there is a mask per class
+
+    # keep only the masks with high confidence?--------------------------------
+    # compute the prob per mask (i.e., class), excluding the "no-object" class (the last one)
+    prob_per_query = outputs.logits.softmax(-1)[..., :-1].max(-1)[0] # why logits last dim 251?
+    # threshold the confidence
+    keep = prob_per_query > gr_slider_confidence/100.0
+
+    # postprocess the mask (numpy arrays)
+    label_per_pixel = torch.argmax(masks[keep].squeeze(),dim=0).detach().numpy() # from the masks per class, select the highest per pixel
+    color_mask = np.zeros(image.size+(3,))
+    palette = itertools.cycle(sns.color_palette())
+    for lbl in np.unique(label_per_pixel): #enumerate(palette()):
+        color_mask[label_per_pixel==lbl,:] = np.asarray(next(palette))*255 #color
+
+    # color_mask = np.zeros(image.size+(3,))
+    # for lbl, color in enumerate(ade_palette()):
+    #     color_mask[label_per_pixel==lbl,:] = color
+
+    # Show image + mask
+    pred_img = np.array(image.convert('RGB'))*0.25 + color_mask*0.75
+    pred_img = pred_img.astype(np.uint8)   
+
+    return pred_img
+
+# get models from hugging face
+feature_extractor = DetrFeatureExtractor.from_pretrained('facebook/detr-resnet-50-panoptic')
+model = DetrForSegmentation.from_pretrained('facebook/detr-resnet-50-panoptic')
+
+# gradio components -inputs
+gr_image_input = gr.inputs.Image()
+gr_slider_confidence = gr.inputs.Slider(0,100,5,85,
+                                        label='Set confidence threshold for masks')
+# gradio outputs
+gr_image_output = gr.outputs.Image() 
+
+# Create user interface and launch
+gr.Interface(predict_building_mask, 
+                inputs = [gr_image_input,gr_slider_confidence],
+                outputs = gr_image_output,
+                title = 'Image segmentation with varying confidence',
+                description = "A panoptic (semantic+instance) segmentation webapp using DETR (End-to-End Object Detection) model with ResNet-50 backbone").launch()
+