Structure revise

app.py

@@ -1,23 +1,244 @@
 import gradio as gr
 
+from matplotlib import gridspec
+import matplotlib.pyplot as plt
+import numpy as np
 from PIL import Image
-import requests
-from transformers import SegformerImageProcessor, SegformerForSemanticSegmentation
+import tensorflow as tf
+from transformers import SegformerFeatureExtractor, TFSegformerForSemanticSegmentation
 
-feature_extractor = SegformerImageProcessor.from_pretrained(
+feature_extractor = SegformerFeatureExtractor.from_pretrained(
      "nvidia/segformer-b0-finetuned-ade-512-512"
 )
-
-processor = SegformerImageProcessor.from_pretrained(
-    "nvidia/segformer-b0-finetuned-ade-512-512"
+model = TFSegformerForSemanticSegmentation.from_pretrained(
+     "nvidia/segformer-b0-finetuned-ade-512-512"
 )
-model = SegformerForSemanticSegmentation.from_pretrained("nvidia/segformer-b0-finetuned-ade-512-512")
 
+def ade_palette():
+    """ADE20K palette that maps each class to RGB values."""
+    return [
+        [53, 53, 53],
+        [0, 84, 255],
+        [171, 242, 0],
+        [255, 0, 221],
+        [255, 228, 0],
+        [92, 209, 229],
+        [153, 56, 0],
+        [128, 65, 217],
+        [116, 116, 116],
+        [47, 157, 39],
+        [217, 65, 140],
+        [255, 255, 255],
+        [255, 178, 245],
+        [61, 183, 204],
+        [153, 0, 76],
+        [71, 102, 0],
+        [255, 167, 167],
+        [25, 25, 25],
+        [53, 53, 53],
+        [0, 84, 255],
+        [171, 242, 0],
+        [255, 0, 221],
+        [255, 228, 0],
+        [92, 209, 229],
+        [153, 56, 0],
+        [128, 65, 217],
+        [53, 53, 53],
+        [0, 84, 255],
+        [171, 242, 0],
+        [255, 0, 221],
+        [255, 228, 0],
+        [92, 209, 229],
+        [153, 56, 0],
+        [128, 65, 217],
+        [53, 53, 53],
+        [0, 84, 255],
+        [171, 242, 0],
+        [255, 0, 221],
+        [255, 228, 0],
+        [92, 209, 229],
+        [153, 56, 0],
+        [128, 65, 217],
+        [53, 53, 53],
+        [0, 84, 255],
+        [171, 242, 0],
+        [255, 0, 221],
+        [255, 228, 0],
+        [92, 209, 229],
+        [153, 56, 0],
+        [128, 65, 217],
+        [53, 53, 53],
+        [0, 84, 255],
+        [171, 242, 0],
+        [255, 0, 221],
+        [255, 228, 0],
+        [92, 209, 229],
+        [153, 56, 0],
+        [128, 65, 217],
+        [0, 84, 255],
+        [171, 242, 0],
+        [255, 0, 221],
+        [255, 228, 0],
+        [92, 209, 229],
+        [153, 56, 0],
+        [128, 65, 217],
+        [53, 53, 53],
+        [0, 84, 255],
+        [171, 242, 0],
+        [255, 0, 221],
+        [255, 228, 0],
+        [92, 209, 229],
+        [153, 56, 0],
+        [128, 65, 217],
+        [53, 53, 53],
+        [0, 84, 255],
+        [171, 242, 0],
+        [255, 0, 221],
+        [255, 228, 0],
+        [92, 209, 229],
+        [153, 56, 0],
+        [128, 65, 217],
+        [53, 53, 53],
+        [0, 84, 255],
+        [171, 242, 0],
+        [255, 0, 221],
+        [255, 228, 0],
+        [92, 209, 229],
+        [153, 56, 0],
+        [128, 65, 217],
+        [53, 53, 53],
+        [0, 84, 255],
+        [171, 242, 0],
+        [255, 0, 221],
+        [255, 228, 0],
+        [92, 209, 229],
+        [153, 56, 0],
+        [128, 65, 217],
+        [53, 53, 53],
+        [0, 84, 255],
+        [171, 242, 0],
+        [255, 0, 221],
+        [255, 228, 0],
+        [92, 209, 229],
+        [153, 56, 0],
+        [128, 65, 217],
+        [53, 53, 53],
+        [0, 84, 255],
+        [171, 242, 0],
+        [255, 0, 221],
+        [255, 228, 0],
+        [92, 209, 229],
+        [153, 56, 0],
+        [128, 65, 217],
+        [53, 53, 53],
+        [0, 84, 255],
+        [171, 242, 0],
+        [255, 0, 221],
+        [255, 228, 0],
+        [92, 209, 229],
+        [153, 56, 0],
+        [128, 65, 217],
+        [53, 53, 53],
+        [0, 84, 255],
+        [171, 242, 0],
+        [255, 0, 221],
+        [255, 228, 0],
+        [92, 209, 229],
+        [153, 56, 0],
+        [128, 65, 217],
+        [53, 53, 53],
+        [0, 84, 255],
+        [171, 242, 0],
+        [255, 0, 221],
+        [255, 228, 0],
+        [92, 209, 229],
+        [153, 56, 0],
+        [128, 65, 217],
+        [53, 53, 53],
+        [0, 84, 255],
+        [171, 242, 0],
+        [255, 0, 221],
+        [255, 228, 0],
+        [92, 209, 229],
+        [153, 56, 0],
+        [128, 65, 217],
+        [53, 53, 53],
+        [0, 84, 255],
+        [171, 242, 0],
+        [255, 0, 221],
+        [255, 228, 0],
+
+
+    ]
+
+labels_list = []
+
+with open(r'labels.txt', 'r') as fp:
+    for line in fp:
+        labels_list.append(line[:-1])
+
+colormap = np.asarray(ade_palette())
+
+def label_to_color_image(label):
+    if label.ndim != 2:
+        raise ValueError("Expect 2-D input label")
+
+    if np.max(label) >= len(colormap):
+        raise ValueError("label value too large.")
+    return colormap[label]
+
+def draw_plot(pred_img, seg):
+    fig = plt.figure(figsize=(20, 15))
+
+    grid_spec = gridspec.GridSpec(1, 2, width_ratios=[6, 1])
+
+    plt.subplot(grid_spec[0])
+    plt.imshow(pred_img)
+    plt.axis('off')
+    LABEL_NAMES = np.asarray(labels_list)
+    FULL_LABEL_MAP = np.arange(len(LABEL_NAMES)).reshape(len(LABEL_NAMES), 1)
+    FULL_COLOR_MAP = label_to_color_image(FULL_LABEL_MAP)
+
+    unique_labels = np.unique(seg.numpy().astype("uint8"))
+    ax = plt.subplot(grid_spec[1])
+    plt.imshow(FULL_COLOR_MAP[unique_labels].astype(np.uint8), interpolation="nearest")
+    ax.yaxis.tick_right()
+    plt.yticks(range(len(unique_labels)), LABEL_NAMES[unique_labels])
+    plt.xticks([], [])
+    ax.tick_params(width=0.0, labelsize=25)
+    return fig
+
+def sepia(input_img):
+    input_img = Image.fromarray(input_img)
+
+    inputs = feature_extractor(images=input_img, return_tensors="tf")
+    outputs = model(**inputs)
+    logits = outputs.logits
+
+    logits = tf.transpose(logits, [0, 2, 3, 1])
+    logits = tf.image.resize(
+        logits, input_img.size[::-1]
+    )  # We reverse the shape of `image` because `image.size` returns width and height.
+    seg = tf.math.argmax(logits, axis=-1)[0]
+
+    color_seg = np.zeros(
+        (seg.shape[0], seg.shape[1], 3), dtype=np.uint8
+    )  # height, width, 3
+    for label, color in enumerate(colormap):
+        color_seg[seg.numpy() == label, :] = color
+
+    # Show image + mask
+    pred_img = np.array(input_img) * 0.5 + color_seg * 0.5
+    pred_img = pred_img.astype(np.uint8)
+
+    fig = draw_plot(pred_img, seg)
+    return fig
 
+demo = gr.Interface(fn=sepia,
+                    inputs=gr.Image(shape=(400, 600)),
+                    outputs=['plot'],
+                    examples=["image_1.jpg", "image_2.jpg", "image_3.jpg", "image_4.jpg", "image_5.jpg"],
+                    allow_flagging='never')
 
-url = "http://images.cocodataset.org/val2017/000000039769.jpg"
-image = Image.open(requests.get(url, stream=True).raw)
 
-inputs = feature_extractor(images=image, return_tensors="pt")
-outputs = model(**inputs)
-logits = outputs.logits  # shape (batch_size, num_labels, height/4, width/4)
+demo.launch()

config.json

@@ -1,372 +0,0 @@
-{
-  "architectures": [
-    "SegformerForSemanticSegmentation"
-  ],
-  "attention_probs_dropout_prob": 0.0,
-  "classifier_dropout_prob": 0.1,
-  "decoder_hidden_size": 256,
-  "depths": [
-    2,
-    2,
-    2,
-    2
-  ],
-  "downsampling_rates": [
-    1,
-    4,
-    8,
-    16
-  ],
-  "drop_path_rate": 0.1,
-  "hidden_act": "gelu",
-  "hidden_dropout_prob": 0.0,
-  "hidden_sizes": [
-    32,
-    64,
-    160,
-    256
-  ],
-  "id2label": {
-    "0": "wall",
-    "1": "building",
-    "2": "sky",
-    "3": "floor",
-    "4": "tree",
-    "5": "ceiling",
-    "6": "road",
-    "7": "bed ",
-    "8": "windowpane",
-    "9": "grass",
-    "10": "cabinet",
-    "11": "sidewalk",
-    "12": "person",
-    "13": "earth",
-    "14": "door",
-    "15": "table",
-    "16": "mountain",
-    "17": "plant",
-    "18": "curtain",
-    "19": "chair",
-    "20": "car",
-    "21": "water",
-    "22": "painting",
-    "23": "sofa",
-    "24": "shelf",
-    "25": "house",
-    "26": "sea",
-    "27": "mirror",
-    "28": "rug",
-    "29": "field",
-    "30": "armchair",
-    "31": "seat",
-    "32": "fence",
-    "33": "desk",
-    "34": "rock",
-    "35": "wardrobe",
-    "36": "lamp",
-    "37": "bathtub",
-    "38": "railing",
-    "39": "cushion",
-    "40": "base",
-    "41": "box",
-    "42": "column",
-    "43": "signboard",
-    "44": "chest of drawers",
-    "45": "counter",
-    "46": "sand",
-    "47": "sink",
-    "48": "skyscraper",
-    "49": "fireplace",
-    "50": "refrigerator",
-    "51": "grandstand",
-    "52": "path",
-    "53": "stairs",
-    "54": "runway",
-    "55": "case",
-    "56": "pool table",
-    "57": "pillow",
-    "58": "screen door",
-    "59": "stairway",
-    "60": "river",
-    "61": "bridge",
-    "62": "bookcase",
-    "63": "blind",
-    "64": "coffee table",
-    "65": "toilet",
-    "66": "flower",
-    "67": "book",
-    "68": "hill",
-    "69": "bench",
-    "70": "countertop",
-    "71": "stove",
-    "72": "palm",
-    "73": "kitchen island",
-    "74": "computer",
-    "75": "swivel chair",
-    "76": "boat",
-    "77": "bar",
-    "78": "arcade machine",
-    "79": "hovel",
-    "80": "bus",
-    "81": "towel",
-    "82": "light",
-    "83": "truck",
-    "84": "tower",
-    "85": "chandelier",
-    "86": "awning",
-    "87": "streetlight",
-    "88": "booth",
-    "89": "television receiver",
-    "90": "airplane",
-    "91": "dirt track",
-    "92": "apparel",
-    "93": "pole",
-    "94": "land",
-    "95": "bannister",
-    "96": "escalator",
-    "97": "ottoman",
-    "98": "bottle",
-    "99": "buffet",
-    "100": "poster",
-    "101": "stage",
-    "102": "van",
-    "103": "ship",
-    "104": "fountain",
-    "105": "conveyer belt",
-    "106": "canopy",
-    "107": "washer",
-    "108": "plaything",
-    "109": "swimming pool",
-    "110": "stool",
-    "111": "barrel",
-    "112": "basket",
-    "113": "waterfall",
-    "114": "tent",
-    "115": "bag",
-    "116": "minibike",
-    "117": "cradle",
-    "118": "oven",
-    "119": "ball",
-    "120": "food",
-    "121": "step",
-    "122": "tank",
-    "123": "trade name",
-    "124": "microwave",
-    "125": "pot",
-    "126": "animal",
-    "127": "bicycle",
-    "128": "lake",
-    "129": "dishwasher",
-    "130": "screen",
-    "131": "blanket",
-    "132": "sculpture",
-    "133": "hood",
-    "134": "sconce",
-    "135": "vase",
-    "136": "traffic light",
-    "137": "tray",
-    "138": "ashcan",
-    "139": "fan",
-    "140": "pier",
-    "141": "crt screen",
-    "142": "plate",
-    "143": "monitor",
-    "144": "bulletin board",
-    "145": "shower",
-    "146": "radiator",
-    "147": "glass",
-    "148": "clock",
-    "149": "flag"
-  },
-  "image_size": 224,
-  "initializer_range": 0.02,
-  "label2id": {
-    "airplane": 90,
-    "animal": 126,
-    "apparel": 92,
-    "arcade machine": 78,
-    "armchair": 30,
-    "ashcan": 138,
-    "awning": 86,
-    "bag": 115,
-    "ball": 119,
-    "bannister": 95,
-    "bar": 77,
-    "barrel": 111,
-    "base": 40,
-    "basket": 112,
-    "bathtub": 37,
-    "bed ": 7,
-    "bench": 69,
-    "bicycle": 127,
-    "blanket": 131,
-    "blind": 63,
-    "boat": 76,
-    "book": 67,
-    "bookcase": 62,
-    "booth": 88,
-    "bottle": 98,
-    "box": 41,
-    "bridge": 61,
-    "buffet": 99,
-    "building": 1,
-    "bulletin board": 144,
-    "bus": 80,
-    "cabinet": 10,
-    "canopy": 106,
-    "car": 20,
-    "case": 55,
-    "ceiling": 5,
-    "chair": 19,
-    "chandelier": 85,
-    "chest of drawers": 44,
-    "clock": 148,
-    "coffee table": 64,
-    "column": 42,
-    "computer": 74,
-    "conveyer belt": 105,
-    "counter": 45,
-    "countertop": 70,
-    "cradle": 117,
-    "crt screen": 141,
-    "curtain": 18,
-    "cushion": 39,
-    "desk": 33,
-    "dirt track": 91,
-    "dishwasher": 129,
-    "door": 14,
-    "earth": 13,
-    "escalator": 96,
-    "fan": 139,
-    "fence": 32,
-    "field": 29,
-    "fireplace": 49,
-    "flag": 149,
-    "floor": 3,
-    "flower": 66,
-    "food": 120,
-    "fountain": 104,
-    "glass": 147,
-    "grandstand": 51,
-    "grass": 9,
-    "hill": 68,
-    "hood": 133,
-    "house": 25,
-    "hovel": 79,
-    "kitchen island": 73,
-    "lake": 128,
-    "lamp": 36,
-    "land": 94,
-    "light": 82,
-    "microwave": 124,
-    "minibike": 116,
-    "mirror": 27,
-    "monitor": 143,
-    "mountain": 16,
-    "ottoman": 97,
-    "oven": 118,
-    "painting": 22,
-    "palm": 72,
-    "path": 52,
-    "person": 12,
-    "pier": 140,
-    "pillow": 57,
-    "plant": 17,
-    "plate": 142,
-    "plaything": 108,
-    "pole": 93,
-    "pool table": 56,
-    "poster": 100,
-    "pot": 125,
-    "radiator": 146,
-    "railing": 38,
-    "refrigerator": 50,
-    "river": 60,
-    "road": 6,
-    "rock": 34,
-    "rug": 28,
-    "runway": 54,
-    "sand": 46,
-    "sconce": 134,
-    "screen": 130,
-    "screen door": 58,
-    "sculpture": 132,
-    "sea": 26,
-    "seat": 31,
-    "shelf": 24,
-    "ship": 103,
-    "shower": 145,
-    "sidewalk": 11,
-    "signboard": 43,
-    "sink": 47,
-    "sky": 2,
-    "skyscraper": 48,
-    "sofa": 23,
-    "stage": 101,
-    "stairs": 53,
-    "stairway": 59,
-    "step": 121,
-    "stool": 110,
-    "stove": 71,
-    "streetlight": 87,
-    "swimming pool": 109,
-    "swivel chair": 75,
-    "table": 15,
-    "tank": 122,
-    "television receiver": 89,
-    "tent": 114,
-    "toilet": 65,
-    "towel": 81,
-    "tower": 84,
-    "trade name": 123,
-    "traffic light": 136,
-    "tray": 137,
-    "tree": 4,
-    "truck": 83,
-    "van": 102,
-    "vase": 135,
-    "wall": 0,
-    "wardrobe": 35,
-    "washer": 107,
-    "water": 21,
-    "waterfall": 113,
-    "windowpane": 8
-  },
-  "layer_norm_eps": 1e-06,
-  "mlp_ratios": [
-    4,
-    4,
-    4,
-    4
-  ],
-  "model_type": "segformer",
-  "num_attention_heads": [
-    1,
-    2,
-    5,
-    8
-  ],
-  "num_channels": 3,
-  "num_encoder_blocks": 4,
-  "patch_sizes": [
-    7,
-    3,
-    3,
-    3
-  ],
-  "reshape_last_stage": true,
-  "sr_ratios": [
-    8,
-    4,
-    2,
-    1
-  ],
-  "strides": [
-    4,
-    2,
-    2,
-    2
-  ],
-  "torch_dtype": "float32",
-  "transformers_version": "4.12.0.dev0"
-}

labels.txt

@@ -0,0 +1,150 @@
+wall
+building
+sky
+floor
+tree
+ceiling
+road
+bed
+windowpane
+grass
+cabinet
+sidewalk
+person
+earth
+door
+table
+mountain
+plant
+curtain
+chair
+car
+water
+painting
+sofa
+shelf
+house
+sea
+mirror
+rug
+field
+armchair
+seat
+fence
+desk
+rock
+wardrobe
+lamp
+bathtub
+railing
+cushion
+base
+box
+column
+signboard
+chest of drawers
+counter
+sand
+sink
+skyscraper
+fireplace
+refrigerator
+grandstand
+path
+stairs
+runway
+case
+pool table
+pillow
+screen door
+stairway
+river
+bridge
+bookcase
+blind
+coffee table
+toilet
+flower
+book
+hill
+bench
+countertop
+stove
+palm
+kitchen island
+computer
+swivel chair
+boat
+bar
+arcade machine
+hovel
+bus
+towel
+light
+truck
+tower
+chandelier
+awning
+streetlight
+booth
+television receiver
+airplane
+dirt track
+apparel
+pole
+land
+bannister
+escalator
+ottoman
+bottle
+buffet
+poster
+stage
+van
+ship
+fountain
+conveyer belt
+canopy
+washer
+plaything
+swimming pool
+stool
+barrel
+basket
+waterfall
+tent
+bag
+minibike
+cradle
+oven
+ball
+food
+step
+tank
+trade name
+microwave
+pot
+animal
+bicycle
+lake
+dishwasher
+screen
+blanket
+sculpture
+hood
+sconce
+vase
+traffic light
+tray
+ashcan
+fan
+pier
+crt screen
+plate
+monitor
+bulletin board
+shower
+radiator
+glass
+clock
+flag

preprocessor_config.json

@@ -1,18 +0,0 @@
-{
-  "do_normalize": true,
-  "do_resize": true,
-  "feature_extractor_type": "SegformerFeatureExtractor",
-  "image_mean": [
-    0.485,
-    0.456,
-    0.406
-  ],
-  "image_std": [
-    0.229,
-    0.224,
-    0.225
-  ],
-  "reduce_labels": true,
-  "resample": 2,
-  "size": 512
-}

pytorch_model.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:0f4df97633cbedd558ecffa3ad228ace5af37e082678390b45a9d22745787c61
-size 15092257

requirements.txt

@@ -1,2 +1,6 @@
 torch
-transformers
+transformers
+tensorflow
+numpy
+Image
+matplotlib

tf_model.h5

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:d38f99e2a8e73bbdb4635669be5bfcbbfc85b4b5c1ac75d36b47312c7fc5d06e
-size 15285696