Update data

.gitattributes

@@ -29,3 +29,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+data/saiapr_tc-12.zip filter=lfs diff=lfs merge=lfs -text
+data/full_pred_val_w_plurals_w_iou.json filter=lfs diff=lfs merge=lfs -text
+data/full_pred_test_w_plurals_w_iou.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,8 +1,8 @@
 ---
 title: Categories Error Analysis
-emoji: 🐠
-colorFrom: red
-colorTo: yellow
+emoji: 😱 🐛
+colorFrom: blue
+colorTo: gray
 sdk: gradio
 sdk_version: 3.2
 app_file: app.py

app.py

@@ -0,0 +1,66 @@
+from turtle import width
+import gradio as gr
+from utils import SampleClass
+
+sample = SampleClass()
+
+# --- Interface ---
+
+demo = gr.Blocks(
+            title="Categories_error_analysis.ipynb",
+            css=".container { max-width: 98%; margin: auto;}; #md {width: 30%} #large {width: 70%}"
+            # css="#md {width: 30%} #large {width: 70%}"
+        )
+
+with demo:
+    gr.Markdown("<h2><center> 😱 Categories Error Analysis 🐛</center></h2>")
+    with gr.Row():
+        with gr.Column():
+            with gr.Row():
+                with gr.Column():
+                    category = gr.Dropdown(
+                        label="Category",
+                        value="relational",
+                        choices=["intrinsic","spatial","ordinal","relational","plural"])
+                with gr.Column(): 
+                    predictions = gr.Dropdown(
+                        label='Predictions', 
+                        value='fail',
+                        choices=["fail", "correct"])
+            with gr.Row():
+                with gr.Column(): 
+                    model = gr.Dropdown(
+                        label='Model', 
+                        value='baseline', 
+                        choices=["baseline", "extended"])
+                with gr.Column(): 
+                    split = gr.Dropdown(
+                        label='Split',
+                        value='val',
+                        choices=["test","val"])
+            with gr.Row():
+                with gr.Column():
+                    username = gr.Dropdown(
+                        label="UserName",
+                        value="luciana",
+                        choices=["luciana",'mauri','jorge','nano'])
+                with gr.Column():
+                    next_idx_sample = gr.Number(
+                        label='Next Idx Sample', 
+                        value=0)
+            with gr.Row():
+                progress = gr.Label(label='Progress',num_top_classes=10)
+            with gr.Row(): 
+                btn_next = gr.Button("Get Next Sample")
+
+        with gr.Column():
+            with gr.Row(): info = gr.Text(label="Sample Info")
+            with gr.Row(): img = gr.Image(label="Sample", type="numpy")
+
+    btn_next.click(
+        fn=sample.explorateSamples,
+        inputs=[username,predictions,category,model,split,next_idx_sample], 
+        outputs=[next_idx_sample, progress, img, info])
+
+# demo.queue(concurrency_count=10)
+demo.launch(debug=False)

data/full_pred_test_w_plurals_w_iou.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32d8075d6de0d60702cdfd884b43fe23140c7a10e28ddc865a03bc6cb84669fb
+size 28012574

data/full_pred_val_w_plurals_w_iou.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e285882c1ad5c8ad5d9485bfce5f043b909564c5b68a61cf925c2c548b5a1d24
+size 2581176

data/saiapr_tc-12.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cbeb079b66dd88ba58d15c5c421e983a65347527418228ad55022b7535983b35
+size 2751748544

requirements.txt

@@ -0,0 +1,6 @@
+matplotlib
+numpy
+pandas
+pillow
+zipfile36
+zipfile38

utils.py

@@ -0,0 +1,167 @@
+import matplotlib as mpl
+mpl.use('Agg')
+import matplotlib.pyplot as plt
+
+import pandas as pd
+import matplotlib.patches as patches
+import numpy as np
+from PIL import Image
+from zipfile import ZipFile
+import gradio as gr
+
+class SampleClass:
+    def __init__(self):
+        self.test_df = pd.read_json("data/full_pred_test_w_plurals_w_iou.json")
+        self.val_df = pd.read_json("data/full_pred_val_w_plurals_w_iou.json")
+        self.zip_file = ZipFile("data/saiapr_tc-12.zip", 'r')
+        self.filtered_df = None
+
+    def __get(self, img_path):
+        img_obj = self.zip_file.open(img_path)
+        pill_img = Image.open(img_obj)
+        img = np.array(pill_img)
+        return img
+
+    def __loadPredictions(self, split, model):
+
+        assert(split in ['test','val'])
+        assert(model in ['baseline','extended'])
+
+        if split == "test":
+            df = self.test_df 
+        elif split == "val":
+            df = self.val_df 
+        else:
+            raise ValueError("File not available yet")
+
+        if model == 'baseline':
+            df = df.rename(columns={'baseline_hit':'hit', 'baseline_pred':'predictions', 
+                                    'extended_hit':'hit_other', 'extended_pred':'predictions_other',
+                                    'baseline_iou':'iou', 
+                                    'extended_iou':'iou_other'}
+                        )
+
+        elif model == 'extended':
+            df = df.rename(columns={'extended_hit':'hit', 'extended_pred':'predictions', 
+                                    'baseline_hit':'hit_other', 'baseline_pred':'predictions_other',
+                                    'extended_iou':'iou', 
+                                    'baseline_iou':'iou_other'}
+                        )
+        return df
+
+    def __getSample(self, id):
+        sample = self.filtered_df[self.filtered_df.sample_idx == id]
+
+        sent = sample['sent'].values[0]
+        pos_tags = sample['pos_tags'].values[0]
+        plural_tks = sample['plural_tks'].values[0]
+
+
+        cat_intrinsic = sample['intrinsic'].values[0]
+        cat_spatial = sample['spatial'].values[0]
+        cat_ordinal = sample['ordinal'].values[0]
+        cat_relational = sample['relational'].values[0]
+        cat_plural = sample['plural'].values[0]
+        categories = [('instrinsic',cat_intrinsic),
+                    ('spatial',cat_spatial),
+                    ('ordinal',cat_ordinal),
+                    ('relational',cat_relational),
+                    ('plural',cat_plural)]
+        
+        
+        img_path = "saiapr_tc-12"+sample['file_path'].values[0].split("saiapr_tc-12")[1]
+        # print(img_path)
+
+        hit = sample['hit'].values[0]
+        hit_o = sample['hit_other'].values[0]
+
+        iou = sample['iou'].values[0]
+        iou_o = sample['iou_other'].values[0]
+
+        prediction = {0:' FAIL ',1:' CORRECT '}
+
+        bbox_gt = sample['bbox'].values[0]
+        x1_gt,y1_gt,x2_gt,y2_gt = bbox_gt
+        # x1_gt,y1_gt,x2_gt,y2_gt = tuple(map(float,bbox_gt[1:-1].split(",")))
+
+        bp_bbox = sample['predictions'].values[0]
+        x1_pred,y1_pred,x2_pred,y2_pred = bp_bbox
+        # x1_pred,y1_pred,x2_pred,y2_pred = tuple(map(float,bp_bbox[1:-1].split(",")))
+
+        bp_o_bbox = sample['predictions_other'].values[0]
+        x1_pred_o,y1_pred_o,x2_pred_o,y2_pred_o = bp_o_bbox
+        # x1_pred_o,y1_pred_o,x2_pred_o,y2_pred_o = tuple(map(float,bp_o_bbox[1:-1].split(",")))
+
+        # Plot
+        fig, ax = plt.subplots(1)
+        ax.imshow(self.__get(img_path), interpolation='bilinear')
+
+        # Create bbox's
+        rect_gt = patches.Rectangle((x1_gt,y1_gt), (x2_gt-x1_gt),(y2_gt-y1_gt), 
+                                    linewidth=2, edgecolor='blue', facecolor='None') #fill=True, alpha=.3
+
+        rect_pred = patches.Rectangle((x1_pred,y1_pred), (x2_pred-x1_pred),(y2_pred-y1_pred), 
+                                    linewidth=2, edgecolor='lightgreen', facecolor='none')
+
+        rect_pred_o = patches.Rectangle((x1_pred_o,y1_pred_o), (x2_pred_o-x1_pred_o),(y2_pred_o-y1_pred_o), 
+                                    linewidth=2, edgecolor='red', facecolor='none')
+        
+        ax.add_patch(rect_gt)
+        ax.add_patch(rect_pred)
+        ax.add_patch(rect_pred_o)
+        ax.axis('off')
+        
+        info = {'Expresion':sent,
+                'Idx Sample':str(id),
+                'IoU': str(round(iou,2)) + "("+prediction[hit]+")",
+                'IoU other': str(round(iou_o,2)) + "("+prediction[hit_o]+")",
+                'Pos Tags':str(pos_tags),
+                'PluralTks ':plural_tks,
+                'Categories':",".join([c for c,b in categories if b])
+                }
+
+        plt.title(info['Expresion'], fontsize=12)
+        plt.tight_layout()
+        plt.close(fig)
+
+        fig.canvas.draw()
+        data = np.frombuffer(fig.canvas.tostring_rgb(), dtype=np.uint8)
+        w, h = fig.canvas.get_width_height()
+        img = data.reshape((int(h), int(w), -1))
+        return info, img
+
+    def explorateSamples(self, 
+            username,
+            predictions,
+            category, 
+            model, 
+            split,
+            next_idx_sample):
+
+        next_idx_sample = int(next_idx_sample)
+        hit = {'fail':0,'correct':1}
+        df = self.__loadPredictions(split, model)
+        self.filtered_df = df[(df[category] == 1) & (df.hit == hit[predictions])]
+        
+
+        all_idx_samples = self.filtered_df.sample_idx.to_list()
+        parts = np.array_split(list(all_idx_samples), 4)
+        user_ids = {
+            'luciana':list(parts[0]),
+            'mauri':list(parts[1]),
+            'jorge':list(parts[2]),
+            'nano':list(parts[3])
+        }
+
+        try:
+            id_ = user_ids[username].index(next_idx_sample)
+        except:
+            id_ = 0
+
+        next_idx_sample = user_ids[username][ min(id_+1, len(user_ids[username])-1) ]
+        progress = {f"{id_}/{len(user_ids[username])-1}":id_/(len(user_ids[username])-1)}
+        info, img = self.__getSample(user_ids[username][id_])
+        info = "".join([str(k)+":\t"+str(v)+"\n" for k,v in list(info.items())[1:]]).strip()
+        
+        return (gr.Number.update(value=next_idx_sample),progress,img,info)
+