dev: add special case.

app.py

@@ -1,7 +1,9 @@
 import io
 import os
-os.system("pip uninstall -y gradio")
-os.system("pip install gradio==3.41.0")
+debug = True
+if not debug:
+    os.system("pip uninstall -y gradio")
+    os.system("pip install gradio==3.41.0")
 
 import torch
 import json
@@ -24,7 +26,8 @@ XCLIP_DESC_PATH = "data/jsons/bs_cub_desc.json"
 XCLIP_DESC = json.load(open(XCLIP_DESC_PATH, "r"))
 PREPROCESS = lambda x: OWLVIT_PRECESSOR(images=x, return_tensors='pt')
 IMAGES_FOLDER = "data/images"
-XCLIP_RESULTS = json.load(open("data/jsons/xclip_org.json", "r"))
+# XCLIP_RESULTS = json.load(open("data/jsons/xclip_org.json", "r"))
+IMAGE2GT = json.load(open("data/jsons/image2gt.json", 'r'))
 CUB_DESC_EMBEDS = torch.load('data/text_embeddings/cub_200_desc.pt')
 CUB_IDX2NAME = json.load(open('data/jsons/cub_desc_idx2name.json', 'r'))
 CUB_IDX2NAME = {int(k): v for k, v in CUB_IDX2NAME.items()}
@@ -225,7 +228,7 @@ def update_selected_image(event: gr.SelectData):
         <img id="overlayImage" src="data:image/jpeg;base64,{BLANK_OVERLAY}" style="position:absolute; top:0; left:0; width:auto; height: {image_height}px; opacity: 0;">
     </div>
     """
-    gt_label = XCLIP_RESULTS[image_name]['ground_truth']
+    gt_label = IMAGE2GT[image_name]
     gt_class.state = gt_label
 
     # --- for initial value only ---
@@ -410,7 +413,7 @@ with gr.Blocks(theme=gr.themes.Soft(), css=custom_css, title="PEEB") as demo:
                 with gr.Row():
                     # xclip_predict_button = gr.Button(label="Predict", value="Predict")
                     xclip_predict_button = gr.Button(value="Predict")
-                xclip_pred_label = gr.Markdown("### XCLIP:")
+                xclip_pred_label = gr.Markdown("### PEEB:")
                 xclip_explanation = gr.HTML()
 
             with gr.Column():
@@ -429,5 +432,7 @@ with gr.Blocks(theme=gr.themes.Soft(), css=custom_css, title="PEEB") as demo:
     xclip_edit_button.click(on_edit_button_click_xclip, inputs=[], outputs=[xclip_textbox, custom_explanation])
     xclip_predict_button.click(on_predict_button_click_xclip, inputs=[xclip_textbox], outputs=[xclip_textbox, xclip_pred_label, xclip_explanation, custom_pred_label, custom_explanation])
 
-# demo.launch(server_port=5000, share=True)
-demo.launch()
+if debug:
+    demo.launch(server_port=5000, share=True)
+else:
+    demo.launch()

data/image_embeddings/special_case_black.jpg.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a256e19b02f6c6a8d58acb22956bb3ebd6d89d7a869ffa21ee29b1c0a2fc307
+size 1770791

data/image_embeddings/special_case_noise.jpg.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09684bd3cc77bc759fef4cc9db8e2e4e434daac18a87b89b5a7d7671ee66f6a0
+size 1770791

data/image_embeddings/special_case_parts.jpg.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b13aea2bc3ebe761cddf48abae750677d3878b2669a70d15ebdce3dae7b6c0a0
+size 1770791

data/jsons/bs_name2cub_name.json

@@ -0,0 +1,202 @@
+{
+    "scarlet tanager": "Scarlet Tanager",
+    "gray kingbird": "Gray Kingbird",
+    "tennessee warbler": "Tennessee Warbler",
+    "red legged kittiwake": "Red-legged Kittiwake",
+    "black and white warbler": "Black-and-white Warbler",
+    "pied kingfisher": "Pied Kingfisher",
+    "yellow bellied flycatcher": "Yellow-bellied Flycatcher",
+    "palm warbler": "Palm Warbler",
+    "tree sparrow": "Tree Sparrow",
+    "red breasted merganser": "Red-breasted Merganser",
+    "crested auklet": "Crested Auklet",
+    "northern fulmar": "Northern Fulmar",
+    "caspian tern": "Caspian Tern",
+    "green violetear": "Green Violetear",
+    "artic tern": "Artic Tern",
+    "tree swallow": "Tree Swallow",
+    "nighthawk": "Nighthawk",
+    "eastern towhee": "Eastern Towhee",
+    "black billed cuckoo": "Black-billed Cuckoo",
+    "sooty albatross": "Sooty Albatross",
+    "carolina wren": "Carolina Wren",
+    "red bellied woodpecker": "Red-bellied Woodpecker",
+    "boat tailed grackle": "Boat-tailed Grackle",
+    "common raven": "Common Raven",
+    "parakeet auklet": "Parakeet Auklet",
+    "cedar waxwing": "Cedar Waxwing",
+    "lincoln sparrow": "Lincoln Sparrow",
+    "geococcyx": "Geococcyx",
+    "rose breasted grosbeak": "Rose-breasted Grosbeak",
+    "ivory gull": "Ivory Gull",
+    "black tern": "Black Tern",
+    "rock wren": "Rock Wren",
+    "cardinal": "Cardinal",
+    "seaside sparrow": "Seaside Sparrow",
+    "northern waterthrush": "Northern Waterthrush",
+    "northern flicker": "Northern Flicker",
+    "acadian flycatcher": "Acadian Flycatcher",
+    "bank swallow": "Bank Swallow",
+    "red winged blackbird": "Red-winged Blackbird",
+    "lazuli bunting": "Lazuli Bunting",
+    "ringed kingfisher": "Ringed Kingfisher",
+    "barn swallow": "Barn Swallow",
+    "ruby throated hummingbird": "Ruby-throated Hummingbird",
+    "orange crowned warbler": "Orange-crowned Warbler",
+    "worm eating warbler": "Worm-eating Warbler",
+    "pine warbler": "Pine Warbler",
+    "prothonotary warbler": "Prothonotary Warbler",
+    "philadelphia vireo": "Philadelphia Vireo",
+    "evening grosbeak": "Evening Grosbeak",
+    "western gull": "Western Gull",
+    "swainson warbler": "Swainson Warbler",
+    "cerulean warbler": "Cerulean Warbler",
+    "marsh wren": "Marsh Wren",
+    "prairie warbler": "Prairie Warbler",
+    "hooded oriole": "Hooded Oriole",
+    "le conte sparrow": "Le-Conte Sparrow",
+    "white breasted kingfisher": "White-breasted Kingfisher",
+    "green tailed towhee": "Green-tailed Towhee",
+    "belted kingfisher": "Belted Kingfisher",
+    "nashville warbler": "Nashville Warbler",
+    "american crow": "American Crow",
+    "yellow warbler": "Yellow Warbler",
+    "field sparrow": "Field Sparrow",
+    "pine grosbeak": "Pine Grosbeak",
+    "chipping sparrow": "Chipping Sparrow",
+    "rhinoceros auklet": "Rhinoceros Auklet",
+    "rufous hummingbird": "Rufous Hummingbird",
+    "orchard oriole": "Orchard Oriole",
+    "scott oriole": "Scott Oriole",
+    "black capped vireo": "Black-capped Vireo",
+    "common tern": "Common Tern",
+    "forster tern": "Forsters Tern",
+    "warbling vireo": "Warbling Vireo",
+    "house wren": "House Wren",
+    "pelagic cormorant": "Pelagic Cormorant",
+    "grasshopper sparrow": "Grasshopper Sparrow",
+    "red cockaded woodpecker": "Red-cockaded Woodpecker",
+    "mangrove cuckoo": "Mangrove Cuckoo",
+    "anna hummingbird": "Anna Hummingbird",
+    "gray catbird": "Gray Catbird",
+    "laysan albatross": "Laysan Albatross",
+    "glaucous winged gull": "Glaucous-winged Gull",
+    "eared grebe": "Eared Grebe",
+    "yellow throated vireo": "Yellow-throated Vireo",
+    "louisiana waterthrush": "Louisiana Waterthrush",
+    "bay breasted warbler": "Bay-breasted Warbler",
+    "pigeon guillemot": "Pigeon Guillemot",
+    "yellow headed blackbird": "Yellow-headed Blackbird",
+    "black footed albatross": "Black-footed Albatross",
+    "cactus wren": "Cactus Wren",
+    "kentucky warbler": "Kentucky Warbler",
+    "brewer blackbird": "Brewer Blackbird",
+    "mallard": "Mallard",
+    "blue grosbeak": "Blue Grosbeak",
+    "baltimore oriole": "Baltimore Oriole",
+    "myrtle warbler": "Myrtle Warbler",
+    "least auklet": "Least Auklet",
+    "cape may warbler": "Cape-May Warbler",
+    "green jay": "Green Jay",
+    "chestnut sided warbler": "Chestnut-sided Warbler",
+    "bohemian waxwing": "Bohemian Waxwing",
+    "frigatebird": "Frigatebird",
+    "bronzed cowbird": "Bronzed Cowbird",
+    "red faced cormorant": "Red-faced Cormorant",
+    "brown creeper": "Brown Creeper",
+    "nelson sharp tailed sparrow": "Nelson-Sharp-tailed Sparrow",
+    "western meadowlark": "Western Meadowlark",
+    "horned puffin": "Horned Puffin",
+    "indigo bunting": "Indigo Bunting",
+    "cliff swallow": "Cliff Swallow",
+    "loggerhead shrike": "Loggerhead Shrike",
+    "black throated sparrow": "Black-throated Sparrow",
+    "blue winged warbler": "Blue-winged Warbler",
+    "white pelican": "White Pelican",
+    "magnolia warbler": "Magnolia Warbler",
+    "great crested flycatcher": "Great-Crested Flycatcher",
+    "american three toed woodpecker": "American-Three-toed Woodpecker",
+    "yellow breasted chat": "Yellow-breasted Chat",
+    "harris sparrow": "Harris Sparrow",
+    "heermann gull": "Heermann Gull",
+    "white throated sparrow": "White-throated Sparrow",
+    "song sparrow": "Song Sparrow",
+    "brewer sparrow": "Brewer Sparrow",
+    "western wood pewee": "Western-Wood Pewee",
+    "hooded merganser": "Hooded Merganser",
+    "white breasted nuthatch": "White-breasted Nuthatch",
+    "american redstart": "American Redstart",
+    "horned lark": "Horned Lark",
+    "brown thrasher": "Brown Thrasher",
+    "hooded warbler": "Hooded Warbler",
+    "clay colored sparrow": "Clay-colored Sparrow",
+    "gray crowned rosy finch": "Gray-crowned-Rosy Finch",
+    "white crowned sparrow": "White-crowned Sparrow",
+    "brown pelican": "Brown Pelican",
+    "sayornis": "Sayornis",
+    "herring gull": "Herring Gull",
+    "vermilion flycatcher": "Vermilion Flycatcher",
+    "clark nutcracker": "Clark Nutcracker",
+    "sage thrasher": "Sage Thrasher",
+    "yellow billed cuckoo": "Yellow-billed Cuckoo",
+    "ovenbird": "Ovenbird",
+    "gadwall": "Gadwall",
+    "dark eyed junco": "Dark-eyed Junco",
+    "pied billed grebe": "Pied-billed Grebe",
+    "least flycatcher": "Least Flycatcher",
+    "elegant tern": "Elegant Tern",
+    "red headed woodpecker": "Red-headed Woodpecker",
+    "fox sparrow": "Fox Sparrow",
+    "downy woodpecker": "Downy Woodpecker",
+    "european goldfinch": "European Goldfinch",
+    "great grey shrike": "Great-Grey Shrike",
+    "pomarine jaeger": "Pomarine Jaeger",
+    "slaty backed gull": "Slaty-backed Gull",
+    "purple finch": "Purple Finch",
+    "mourning warbler": "Mourning Warbler",
+    "rusty blackbird": "Rusty Blackbird",
+    "white necked raven": "White-necked Raven",
+    "bobolink": "Bobolink",
+    "vesper sparrow": "Vesper Sparrow",
+    "ring billed gull": "Ring-billed Gull",
+    "red eyed vireo": "Red-eyed Vireo",
+    "green kingfisher": "Green Kingfisher",
+    "wilson warbler": "Wilson Warbler",
+    "western grebe": "Western Grebe",
+    "brandt cormorant": "Brandt Cormorant",
+    "pileated woodpecker": "Pileated Woodpecker",
+    "house sparrow": "House Sparrow",
+    "canada warbler": "Canada Warbler",
+    "florida jay": "Florida Jay",
+    "pacific loon": "Pacific Loon",
+    "summer tanager": "Summer Tanager",
+    "blue headed vireo": "Blue-headed Vireo",
+    "least tern": "Least Tern",
+    "american goldfinch": "American Goldfinch",
+    "savannah sparrow": "Savannah Sparrow",
+    "scissor tailed flycatcher": "Scissor-tailed Flycatcher",
+    "horned grebe": "Horned Grebe",
+    "baird sparrow": "Baird Sparrow",
+    "cape glossy starling": "Cape-Glossy Starling",
+    "winter wren": "Winter Wren",
+    "painted bunting": "Painted Bunting",
+    "tropical kingbird": "Tropical Kingbird",
+    "long tailed jaeger": "Long-tailed Jaeger",
+    "fish crow": "Fish Crow",
+    "chuck will widow": "Chuck-will Widow",
+    "white eyed vireo": "White-eyed Vireo",
+    "bewick wren": "Bewick Wren",
+    "henslow sparrow": "Henslow Sparrow",
+    "groove billed ani": "Groove-billed Ani",
+    "golden winged warbler": "Golden-winged Warbler",
+    "california gull": "California Gull",
+    "black throated blue warbler": "Black-throated-Blue Warbler",
+    "american pipit": "American Pipit",
+    "common yellowthroat": "Common Yellowthroat",
+    "mockingbird": "Mockingbird",
+    "blue jay": "Blue Jay",
+    "whip poor will": "Whip-poor Will",
+    "olive sided flycatcher": "Olive-sided Flycatcher",
+    "shiny cowbird": "Shiny Cowbird",
+    "spotted catbird": "Spotted Catbird"
+}

data/jsons/cub_vis_dict_binary.json

@@ -165030,5 +165030,47 @@
         "nape": true,
         "legs": true,
         "tail": true
+    },
+    "special_case_black.jpg": {
+        "crown": true,
+        "forehead": true,
+        "eyes": true,
+        "beak": true,
+        "throat": true,
+        "belly": true,
+        "breast": true,
+        "back": true,
+        "wings": true,
+        "nape": true,
+        "legs": true,
+        "tail": true
+    },
+    "special_case_noise.jpg": {
+        "crown": true,
+        "forehead": true,
+        "eyes": true,
+        "beak": true,
+        "throat": true,
+        "belly": true,
+        "breast": true,
+        "back": true,
+        "wings": true,
+        "nape": true,
+        "legs": true,
+        "tail": true
+    },
+    "special_case_parts.jpg": {
+        "crown": true,
+        "forehead": true,
+        "eyes": true,
+        "beak": true,
+        "throat": true,
+        "belly": true,
+        "breast": true,
+        "back": true,
+        "wings": true,
+        "nape": true,
+        "legs": true,
+        "tail": true
     }
 }

data/jsons/file_list.json

@@ -1 +1,23 @@
-["Brandt_Cormorant_0040_23144.jpg", "Black_Tern_0101_144331.jpg", "Gray_Catbird_0071_20974.jpg", "Rufous_Hummingbird_0076_59563.jpg", "Ivory_Gull_0004_49019.jpg", "Brown_Thrasher_0014_155421.jpg", "American_Goldfinch_0123_32505.jpg", "Red_Headed_Woodpecker_0032_182815.jpg", "Sage_Thrasher_0062_796462.jpg", "Heermann_Gull_0097_45783.jpg", "Winter_Wren_0048_189683.jpg", "Pine_Warbler_0113_172456.jpg", "White_Eyed_Vireo_0046_158849.jpg", "Cedar_Waxwing_0075_179114.jpg", "Carolina_Wren_0060_186296.jpg", "Clark_Nutcracker_0126_85134.jpg", "House_Wren_0137_187273.jpg", "Vesper_Sparrow_0030_125663.jpg", "Northern_Waterthrush_0038_177027.jpg", "Western_Grebe_0064_36613.jpg"]
+["Brandt_Cormorant_0040_23144.jpg", 
+"Black_Tern_0101_144331.jpg", 
+"Gray_Catbird_0071_20974.jpg", 
+"Rufous_Hummingbird_0076_59563.jpg", 
+"Ivory_Gull_0004_49019.jpg", 
+"Brown_Thrasher_0014_155421.jpg", 
+"American_Goldfinch_0123_32505.jpg", 
+"Red_Headed_Woodpecker_0032_182815.jpg", 
+"Sage_Thrasher_0062_796462.jpg", 
+"Heermann_Gull_0097_45783.jpg", 
+"Winter_Wren_0048_189683.jpg", 
+"Pine_Warbler_0113_172456.jpg", 
+"White_Eyed_Vireo_0046_158849.jpg", 
+"Cedar_Waxwing_0075_179114.jpg", 
+"Carolina_Wren_0060_186296.jpg", 
+"Clark_Nutcracker_0126_85134.jpg", 
+"House_Wren_0137_187273.jpg", 
+"Vesper_Sparrow_0030_125663.jpg", 
+"Northern_Waterthrush_0038_177027.jpg", 
+"Western_Grebe_0064_36613.jpg",
+"special_case_black.jpg",
+"special_case_noise.jpg",
+"special_case_parts.jpg"]

utils/predict.py

@@ -90,8 +90,8 @@ def xclip_pred(new_desc: dict,
         part_embeds = owlvit_processor(text=[ORG_PART_ORDER], return_tensors="pt").to(device)
         if use_precompute_embeddings:
             image_embeds = torch.load(f'data/image_embeddings/{image_name}.pt').to(device)
-            image_input = owlvit_processor(images=image, return_tensors='pt').to(device)
         else:
+            image_input = owlvit_processor(images=image, return_tensors='pt').to(device)
             image_embeds, _ = model.image_embedder(pixel_values = image_input['pixel_values'])
             
         pred_logits, part_logits, output_dict = model(image_embeds, part_embeds, query_embeds, None)