update app.py

app.py

@@ -70,121 +70,6 @@ def get_outputs(
     return outputs
 
 
-def evaluate_refcoco(
-        model,
-        tokenizer,
-        image_processor,
-        batch_size,
-        tsvfile,
-        max_generation_length=20,
-        num_beams=3,
-        length_penalty=-2.0,
-        device=-1,
-        vis_embed_size=None,
-        rank=0,
-        world_size=1,
-        id=0,
-):
-    model.eval().cuda()
-    loc_token_ids = []
-    for i in range(1000):
-        loc_token_ids.append(int(tokenizer(f"<loc_{i}>", add_special_tokens=False)["input_ids"][-1]))
-    media_token_id = tokenizer("<|#image#|>", add_special_tokens=False)["input_ids"][-1]
-    endofmedia_token_id = tokenizer("<|#endofimage#|>", add_special_tokens=False)["input_ids"][-1]
-    pad_token_id = tokenizer(tokenizer.pad_token, add_special_tokens=False)["input_ids"][-1]
-    bos_token_id = tokenizer(tokenizer.bos_token, add_special_tokens=False)["input_ids"][-1]
-    prebox_token_id = tokenizer("<|#prebox#|>", add_special_tokens=False)["input_ids"][-1]
-    # all_ids = set(range(model.lang_encoder.lm_head.out_features))
-    # bad_words_ids = list(all_ids - set(loc_token_ids))
-    # bad_words_ids = [[b] for b in bad_words_ids]
-    # min_loc_token_id = min(loc_token_ids)
-    # max_loc_token_id = max(loc_token_ids)
-    total = 0
-    correct = 0
-    ious = []
-    if "refcocog" in tsvfile:
-        dataset_name = "refcocog"
-    elif "refcocoplus" in tsvfile:
-        dataset_name = "refcocoplus"
-    else:
-        dataset_name = "refcoco"
-    with open(tsvfile, "r") as f:
-        lines = f.readlines()
-        pbar = tqdm(lines, disable=(rank != 0))
-        for ii, line in enumerate(pbar):
-            if ii % world_size != rank:
-                continue
-            total += 1
-            line = line.rstrip()
-            uniq_id, image_id, text, region_coord, image = line.split("\t")
-
-            image = Image.open(BytesIO(base64.urlsafe_b64decode(image))).convert("RGB")
-            # image = Image.open("/gpfs/u/home/LMCG/LMCGljnn/scratch/code/multimodal2/yolo.png").convert("RGB")
-            # image = Image.open("/gpfs/u/home/LMCG/LMCGljnn/scratch/code/multimodal/temp/cat.png").convert("RGB")
-            # image = Image.open("/gpfs/u/home/LMCG/LMCGljnn/scratch/code/multimodal/temp/262148000.png")
-
-            gt_box = np.array(list(map(float, region_coord.split(","))))
-            width = image.width
-            height = image.height
-            image = image.resize((224, 224))
-            gt_box = gt_box / np.array([width, height, width, height]) * 224
-            batch_images = image_processor(image).unsqueeze(0).unsqueeze(1).unsqueeze(0)
-            prompt = [
-                f"{tokenizer.bos_token}<|#image#|>{tokenizer.pad_token * vis_embed_size}<|#endofimage#|><|#object#|>{text.rstrip('.').strip()}<|#endofobject#|><|#visual#|>"]
-            # prompt = [f"<|#image#|>{tokenizer.pad_token*vis_embed_size}<|#endofimage#|>the cat<|#visual#|>"]
-            # prompt = [f"<|#image#|>{tokenizer.pad_token*vis_embed_size}<|#endofimage#|>"]
-            # prompt = [f"<|#image#|>{tokenizer.pad_token*vis_embed_size}<|#endofimage#|>a man<|#visual#|> is doing a trick on a skateboard<|#visual#|>"]
-
-            encodings = tokenizer(
-                prompt,
-                padding="longest",
-                truncation=True,
-                return_tensors="pt",
-                max_length=2000,
-            )
-            input_ids = encodings["input_ids"]
-            attention_mask = encodings["attention_mask"]
-            # attention_mask[input_ids == prebox_token_id] = 0
-            image_start_index_list = ((input_ids == media_token_id).nonzero(as_tuple=True)[-1] + 1).tolist()
-            image_start_index_list = [[x] for x in image_start_index_list]
-            image_nums = [1] * len(input_ids)
-            vision_x = batch_images.cuda()
-            lang_x = input_ids.cuda()
-            attention_mask = attention_mask.cuda()
-
-            model.debug_id = 0
-            with torch.inference_mode() and torch.cuda.amp.autocast(dtype=torch.float16):
-                outputs = model(
-                    vision_x=vision_x,
-                    lang_x=lang_x,
-                    attention_mask=attention_mask,
-                    labels=None,
-                    image_nums=image_nums,
-                    image_start_index_list=image_start_index_list,
-                    added_bbox_list=None,
-                    add_box=False,
-                )
-            boxes = outputs["boxes"]
-            scores = outputs["scores"]
-            if len(scores) > 0:
-                box = boxes[scores.argmax()]
-                iou = get_iou(box, gt_box)
-            else:
-                iou = 0.0
-                # tqdm.write(f"output: {tokenizer.batch_decode(outputs)}")
-                tqdm.write(f"no output for: {uniq_id}, {image_id}, {text}")
-            if iou >= 0.5:
-                correct += 1
-            pbar.set_description(f"iou: {iou:.2f} score: {correct / total:.4f}")
-            # open_cv_image = np.array(image)
-            # # Convert RGB to BGR
-            # open_cv_image = open_cv_image[:, :, ::-1].copy()
-            # for box, score in zip(boxes, scores):
-            #     open_cv_image = cv2.rectangle(open_cv_image, box[:2].astype(int), box[2:].astype(int), (255, 0, 0), 2)
-            # cv2.imwrite("output.jpg", open_cv_image)
-            # print(boxes)
-            # print(scores)
-            # exit()
 
 
 def generate(

multimodal/setup.py

@@ -33,6 +33,7 @@ if __name__ == "__main__":
         "inflection",
         "sentencepiece",
         "open_clip_torch",
+        "opencv-python"
     ]
 
     setup(