explanations,closest sample

app.py

@@ -46,9 +46,19 @@ def get_model(model_name):
                         nb_classes = n_classes,load_weights=False,finer_model=True,backbone_name ='Resnet50v2')
         model.load_weights('model_classification/rock-170.h5')
     else:
-        return 'Error' 
+        raise ValueError(f"Model name '{model_name}' is not recognized") 
     return model,n_classes
 
+    '''
+    elif model_name == 'Fossils 19':
+        n_classes = 19 or 23?
+        model = get_beit_model(input_shape=(600, 600, 3),
+                               num_labels=n_classes,
+                               load_weights=False, 
+                               )
+        model.load_weights('model_classification/beit-fossils-19.h5')
+    '''
+
 def segment_image(input_image):
     img = segmentation_sam(input_image)
     return img
@@ -67,7 +77,8 @@ def classify_image(input_image, model_name):
     if 'Fossils 19' ==model_name:
         from inference_beit import inference_dino
         model,n_classes = get_model(model_name)
-        return inference_dino(input_image,model_name)
+        result = inference_dino(input_image,model_name)
+        return result
     return None
 
 def get_embeddings(input_image,model_name):
@@ -84,21 +95,26 @@ def get_embeddings(input_image,model_name):
     if 'Fossils 19' ==model_name:
         from inference_beit import inference_dino
         model,n_classes = get_model(model_name)
-        return inference_dino(input_image,model_name)
+        result = inference_dino(input_image,model_name)
+        #TODO
+        #result = inference_beit_embedding
+        return result
     return None
     
 
 def find_closest(input_image,model_name):
     embedding = get_embeddings(input_image,model_name)
-    paths = get_images(embedding)
-    return paths
+    classes, paths = get_images(embedding)
+    #outputs = classes+paths
+    return classes,paths
 
 def explain_image(input_image,model_name):
     model,n_classes= get_model(model_name)
-    saliency, integrated, smoothgrad = explain(model,input_image,n_classes=n_classes)
+    #saliency, integrated, smoothgrad,
+    rise = explain(model,input_image,n_classes=n_classes)
     #original =  saliency + integrated + smoothgrad 
     print('done')
-    return saliency, integrated, smoothgrad,
+    return rise
     
 #minimalist theme
 with gr.Blocks(theme='sudeepshouche/minimalist') as demo:
@@ -118,7 +134,7 @@ with gr.Blocks(theme='sudeepshouche/minimalist') as demo:
                 
             with gr.Column():
                 model_name = gr.Dropdown(
-                    ["Mummified 170", "Rock 170"],
+                    ["Mummified 170", "Rock 170","Fossils 19"],
                     multiselect=False,
                     value="Rock 170", # default option
                     label="Model",
@@ -142,32 +158,61 @@ with gr.Blocks(theme='sudeepshouche/minimalist') as demo:
         #     with gr.Column():
         #         class_predicted2 = gr.Label(label='Class Predicted from diffuser')
         #         classify_button = gr.Button("Classify Image")
-            
+
         
         with gr.Accordion("Explanations "):
             gr.Markdown("Computing Explanations from the model")
             with gr.Row():
                 #original_input = gr.Image(label="Original Frame")
-                saliency  = gr.Image(label="saliency")
-                gradcam = gr.Image(label='integraged gradients')
-                guided_gradcam = gr.Image(label='gradcam')
+                #saliency  = gr.Image(label="saliency")
+                #gradcam = gr.Image(label='integraged gradients')
+                #guided_gradcam = gr.Image(label='gradcam')
                 #guided_backprop = gr.Image(label='guided backprop')
+                rise = gr.Image(label = 'Rise')
             generate_explanations = gr.Button("Generate Explanations")
                 
+        # with gr.Accordion('Closest Images'):
+        #     gr.Markdown("Finding the closest images in the dataset")
+        #     with gr.Row():
+        #         with gr.Column():
+        #             label_closest_image_0 = gr.Markdown('')
+        #             closest_image_0 = gr.Image(label='Closest Image',image_mode='contain',width=200, height=200)
+        #         with gr.Column():
+        #             label_closest_image_1 = gr.Markdown('')
+        #             closest_image_1 = gr.Image(label='Second Closest Image',image_mode='contain',width=200, height=200)
+        #         with gr.Column():
+        #             label_closest_image_2 = gr.Markdown('')
+        #             closest_image_2 = gr.Image(label='Third Closest Image',image_mode='contain',width=200, height=200)
+        #         with gr.Column():
+        #             label_closest_image_3 = gr.Markdown('')
+        #             closest_image_3 = gr.Image(label='Forth Closest Image',image_mode='contain', width=200, height=200)
+        #         with gr.Column():
+        #             label_closest_image_4 = gr.Markdown('')
+        #             closest_image_4 = gr.Image(label='Fifth Closest Image',image_mode='contain',width=200, height=200)
+        #     find_closest_btn = gr.Button("Find Closest Images")
         with gr.Accordion('Closest Images'):
             gr.Markdown("Finding the closest images in the dataset")
+            
             with gr.Row():
-                closest_image_0 = gr.Image(label='Closest Image')
-                closest_image_1 = gr.Image(label='Second Closest Image')
-                closest_image_2 = gr.Image(label='Third Closest Image')
-                closest_image_3 = gr.Image(label='Forth Closest Image')
-                closest_image_4 = gr.Image(label='Fifth Closest Image')
+                gallery = gr.Gallery(label="Closest Images", show_label=False,elem_id="gallery",columns=[5], rows=[1],height='auto', allow_preview=True, preview=None)
+                #.style(grid=[1, 5], height=200, width=200)
+
             find_closest_btn = gr.Button("Find Closest Images")
             
         segment_button.click(segment_image, inputs=input_image, outputs=segmented_image)
         classify_image_button.click(classify_image, inputs=[input_image,model_name], outputs=class_predicted)
-        generate_explanations.click(explain_image, inputs=[input_image,model_name], outputs=[saliency,gradcam,guided_gradcam])
-        find_closest_btn.click(find_closest, inputs=[input_image,model_name], outputs=[closest_image_0,closest_image_1,closest_image_2,closest_image_3,closest_image_4])
+        generate_explanations.click(explain_image, inputs=[input_image,model_name], outputs=[rise]) #saliency,gradcam,guided_gradcam
+        #find_closest_btn.click(find_closest, inputs=[input_image,model_name], outputs=[label_closest_image_0,label_closest_image_1,label_closest_image_2,label_closest_image_3,label_closest_image_4,closest_image_0,closest_image_1,closest_image_2,closest_image_3,closest_image_4])
+        def update_outputs(input_image,model_name):
+            labels, images = find_closest(input_image,model_name)
+            #labels_html = "".join([f'<div style="display: inline-block; text-align: center; width: 18%;">{label}</div>' for label in labels])
+            #labels_markdown = f"<div style='width: 100%; text-align: center;'>{labels_html}</div>"
+            image_caption=[]
+            for i in range(5):
+                image_caption.append((images[i],labels[i]))
+            return image_caption
+
+        find_closest_btn.click(fn=update_outputs, inputs=[input_image,model_name], outputs=[gallery])
         #classify_segmented_button.click(classify_image, inputs=[segmented_image,model_name], outputs=class_predicted)
         
 demo.queue()     # manage multiple incoming requests
@@ -176,6 +221,3 @@ if os.getenv('SYSTEM') == 'spaces':
     demo.launch(width='40%',auth=(os.environ.get('USERNAME'), os.environ.get('PASSWORD')))
 else: 
     demo.launch()
-    
-
-

closest_sample.py

@@ -50,10 +50,8 @@ def download_public_image(url, destination_path):
         with open(destination_path, 'wb') as f:
             f.write(response.content)
         print(f"Downloaded image to {destination_path}")
-        return True
     else:
         print(f"Failed to download image from bucket. Status code: {response.status_code}")
-        return False
 
 def get_images(embedding):
     
@@ -69,14 +67,23 @@ def get_images(embedding):
     folder_florissant = 'https://storage.googleapis.com/serrelab/prj_fossils/2024/Florissant_Fossil_v2.0/'
     folder_general = 'https://storage.googleapis.com/serrelab/prj_fossils/2024/General_Fossil_v2.0/'
     
+    local_paths = []
+    classes = []
     for i, path in enumerate(paths):
         local_file_path = f'image_{i}.jpg'
-        public_path_florissant = path.replace('/gpfs/data/tserre/irodri15/Fossils/new_data/leavesdb-v1_1/images/Fossil/Florissant_Fossil/512/full/jpg/', folder_florissant)
-        success = download_public_image(public_path_florissant, local_file_path)
-
-        if not success:
-            public_path_general = path.replace('/gpfs/data/tserre/irodri15/Fossils/new_data/leavesdb-v1_1/images/Fossil/Florissant_Fossil/512/full/jpg/', folder_general)
-            download_public_image(public_path_general, local_file_path)
+        if 'Florissant_Fossil/512/full/jpg/' in path:
+            public_path = path.replace('/gpfs/data/tserre/irodri15/Fossils/new_data/leavesdb-v1_1/images/Fossil/Florissant_Fossil/512/full/jpg/', folder_florissant)
+        elif 'General_Fossil/512/full/jpg/' in path:
+            public_path = path.replace('/gpfs/data/tserre/irodri15/Fossils/new_data/leavesdb-v1_1/images/Fossil/General_Fossil/512/full/jpg/', folder_general)
+        else:
+            print("no match found")
+        download_public_image(public_path, local_file_path)
+        names = []
+        parts = [part for part in public_path.split('/') if part]
+        part = parts[-2]
+        classes.append(part)
+        local_paths.append(local_file_path)
     #paths= [path.replace('/gpfs/data/tserre/irodri15/Fossils/new_data/leavesdb-v1_1/images/Fossil/Florissant_Fossil/512/full/jpg/',
     #                     '/media/data_cifs/projects/prj_fossils/data/processed_data/leavesdb-v1_1/images/Fossil/Florissant_Fossil/original/full/jpg/') for path in paths]
-    return paths
+
+    return classes, local_paths

explanations.py

@@ -50,10 +50,13 @@ def explain(model, input_image,size=600, n_classes=171) :
     class_model = tf.keras.Model(model.input, model.output[1]) 
     
     explainers = [
-             Saliency(class_model),
-             IntegratedGradients(class_model, steps=50, batch_size=BATCH_SIZE),
-             SmoothGrad(class_model, nb_samples=50, batch_size=BATCH_SIZE),
+             #Saliency(class_model),
+             #IntegratedGradients(class_model, steps=50, batch_size=BATCH_SIZE),
+             #SmoothGrad(class_model, nb_samples=50, batch_size=BATCH_SIZE),
              #GradCAM(class_model),
+             Rise(class_model,nb_samples = 50, batch_size = BATCH_SIZE,grid_size=7,
+                 preservation_probability=0.5)
+             #
   ]
     cropped,repetitions = _clever_crop(input_image,(size,size))
     size_repetitions = int(size//(repetitions.numpy()+1))
@@ -74,6 +77,8 @@ def explain(model, input_image,size=600, n_classes=171) :
         
         plt.savefig(f'phi_{e}.png')
         explanations.append(f'phi_{e}.png')
+        print(type(explanations))
+        print(len(explanations))
       
     print('Done')  
     

inference_beit.py

@@ -0,0 +1,203 @@
+import tensorflow as tf
+gpu_devices = tf.config.experimental.list_physical_devices('GPU')
+if gpu_devices:
+    tf.config.experimental.set_memory_growth(gpu_devices[0], True)
+else:
+    print(f"TensorFlow device: {gpu_devices}")
+
+import os
+import numpy as np
+import keras
+from PIL import Image
+import keras_cv
+from keras_cv_attention_models import beit
+import matplotlib.pyplot as plt
+
+
+#preprocessing
+#TODO
+num_classes = len(class_names)
+AUTO = tf.data.AUTOTUNE
+rand_augment = keras_cv.layers.RandAugment(value_range = (-1, 1), augmentations_per_image = 3, magnitude=0.5)
+
+SIZE = 384
+debug = None
+
+def augmentations(x, crop_size=22, brightness=0.2, contrast=0.2, saturation=0.2, hue=0.2):
+    x = tf.cast(x, tf.float32)
+    x = tf.image.random_crop(x, (tf.shape(x)[0], 100, 100, 3))
+    x = tf.image.random_brightness(x, max_delta=brightness)
+    x = tf.image.random_contrast(x, lower=1.0-contrast, upper=1+contrast)
+    x = tf.image.random_saturation(x, lower=1.0-saturation, upper=1.0+saturation)
+    x = tf.image.random_hue(x, max_delta=hue)
+    x = tf.image.resize(x, (128, 128))
+    x = tf.clip_by_value(x, 0.0, 255.0)
+    x = tf.keras.applications.resnet_v2.preprocess_input(x)
+    return x
+
+
+def pad_gt(x):
+  h, w = x.shape[-2:]
+  padh = sam.image_encoder.img_size - h
+  padw = sam.image_encoder.img_size - w
+  x = F.pad(x, (0, padw, 0, padh))
+  return x
+
+def preprocess(img):
+
+  img = np.array(img).astype(np.uint8)
+
+  #assert img.max() > 127.0
+
+  img_preprocess = predictor.transform.apply_image(img)
+  intermediate_shape = img_preprocess.shape
+
+  img_preprocess = torch.as_tensor(img_preprocess).cuda()
+  img_preprocess = img_preprocess.permute(2, 0, 1).contiguous()[None, :, :, :]
+
+  img_preprocess = sam.preprocess(img_preprocess)
+  if len(intermediate_shape) == 3:
+     intermediate_shape = intermediate_shape[:2]
+  elif len(intermediate_shape) == 4:
+     intermediate_shape = intermediate_shape[1:3]
+
+  return img_preprocess, intermediate_shape
+
+
+
+def normalize(img):
+  img = img - tf.math.reduce_min(img)
+  img = img / tf.math.reduce_max(img)
+  img = img * 2.0 - 1.0
+  return img
+
+def smooth_mask(mask, ds=20):
+  shape = tf.shape(mask)
+  w, h = shape[0], shape[1]
+  return tf.image.resize(tf.image.resize(mask, (ds, ds), method="bicubic"), (w, h), method="bicubic")
+
+def resize(img):
+  # default resize function for all pi outputs
+  return tf.image.resize(img, (SIZE, SIZE), method="bicubic")
+
+def pi(img, mask):
+  img = tf.cast(img, tf.float32)
+
+  shape = tf.shape(img)
+  w, h = tf.cast(shape[0], tf.int64), tf.cast(shape[1], tf.int64)
+
+  mask = smooth_mask(mask)
+  mask = tf.reduce_mean(mask, -1)
+
+  img = img * tf.cast(mask > 0.1, tf.float32)[:, :, None]
+
+  img_resize = tf.image.resize(img, (SIZE, SIZE), method="bicubic", antialias=True)
+  img_pad = tf.image.resize_with_pad(img, SIZE, SIZE, method="bicubic", antialias=True)
+
+  # building 2 anchors
+  anchors = tf.where(mask > 0.15)
+  anchor_xmin = tf.math.reduce_min(anchors[:, 0])
+  anchor_xmax = tf.math.reduce_max(anchors[:, 0])
+  anchor_ymin = tf.math.reduce_min(anchors[:, 1])
+  anchor_ymax = tf.math.reduce_max(anchors[:, 1])
+
+  if anchor_xmax - anchor_xmin > 50 and anchor_ymax - anchor_ymin > 50:
+
+    img_anchor_1 = resize(img[anchor_xmin:anchor_xmax, anchor_ymin:anchor_ymax])
+
+    delta_x = (anchor_xmax - anchor_xmin) // 4
+    delta_y = (anchor_ymax - anchor_ymin) // 4
+    img_anchor_2 = img[anchor_xmin+delta_x:anchor_xmax-delta_x,
+                      anchor_ymin+delta_y:anchor_ymax-delta_y]
+    img_anchor_2 = resize(img_anchor_2)
+  else:
+    img_anchor_1 = img_resize
+    img_anchor_2 = img_pad
+
+  # building the anchors max
+  anchor_max = tf.where(mask == tf.math.reduce_max(mask))[0]
+  anchor_max_x, anchor_max_y = anchor_max[0], anchor_max[1]
+
+  img_max_zoom1 = img[tf.math.maximum(anchor_max_x-SIZE, 0): tf.math.minimum(anchor_max_x+SIZE, w),
+                      tf.math.maximum(anchor_max_y-SIZE, 0): tf.math.minimum(anchor_max_y+SIZE, h)]
+
+  img_max_zoom1 = resize(img_max_zoom1)
+  img_max_zoom2 = img[anchor_max_x-SIZE//2:anchor_max_x+SIZE//2,
+                      anchor_max_y-SIZE//2:anchor_max_y+SIZE//2]
+  img_max_zoom2 = img[tf.math.maximum(anchor_max_x-SIZE//2, 0): tf.math.minimum(anchor_max_x+SIZE//2, w),
+                      tf.math.maximum(anchor_max_y-SIZE//2, 0): tf.math.minimum(anchor_max_y+SIZE//2, h)]
+  #tf.print(img_max_zoom2.shape)
+  #img_max_zoom2 = resize(img_max_zoom2)
+
+  return tf.cast(img_resize, tf.float32)
+
+def parse_img(element, split, randaugment,maskaugment=True):
+  #global debug
+  path, class_id = element[0], element[1]
+
+  data = tf.io.read_file(path)
+  img = tf.io.decode_jpeg(data)
+  img = tf.cast(img, tf.uint8)
+  img = normalize(img)
+  shape = tf.shape(img)
+
+  # data_mask = tf.io.read_file(path_mask)
+  # mask = tf.io.decode_jpeg(data_mask)
+
+  class_id = tf.strings.to_number(class_id)
+  class_id = tf.cast(class_id, tf.int32)
+
+  label = tf.one_hot(class_id, num_classes)
+
+  # img = pi(img, mask)
+  img = tf.image.resize_with_pad(img, SIZE, SIZE, method="bicubic", antialias=True)
+
+  return tf.cast(img, tf.float32), tf.cast(label, tf.int32)
+
+SIZE = 384
+wsize=hsize=SIZE
+def resize_images(batch_x, width=224, height=224):
+    return tf.image.resize(batch_x, (width, height))
+
+def load_img(image_path,gray=False):
+    img = tf.io.read_file(image_path)
+    img = tf.image.decode_jpeg(img, channels=3)
+    img = tf.image.convert_image_dtype(img, tf.float32)
+    if gray:
+        img = tf.image.rgb_to_grayscale(img)
+        img = tf.image.grayscale_to_rgb(img)
+    img = tf.image.resize(img,(wsize,hsize))
+    return img
+
+LR = 1e-3
+
+optimizer = tf.keras.optimizers.Adam(LR)
+cce = tf.keras.losses.categorical_crossentropy
+
+model_path = '/content/drive/MyDrive/Gg_Fossils_data_shared_copy/Fossils/models/model-13.h5'
+model = keras.models.load_model(model_path, custom_objects = {'cce': cce})
+
+outputs = model.predict(images)
+
+predictions = tf.math.top_k(outputs[1], k = 5)
+cid = 1
+dataset = np.array(dataset)
+final_predictions = []
+for ele in predictions[1]:
+  if cid in ele:
+    final_predictions.append(cid)
+  else:
+    final_predictions.append(cid+10)
+final_predictions = np.array(final_predictions)
+images2 = images[final_predictions == cid]
+image2_paths = dataset[final_predictions == cid][:,0]
+print(images2.shape)
+
+def get_beit_model(input_shape, num_labels, load_weights=False, ...):
+    pass
+
+def inference_dino(input_image, model_name):
+    pass
+
+def inference_beit_embedding(input_image, model, size=600):
+    pass