update cap

app.py

@@ -313,13 +313,16 @@ with gr.Blocks() as demo:
             #                       value='Provide a comprehensive description of the image <image> and specify the positions of any mentioned objects in square brackets.')
             # text_input = gr.Textbox(label='<question>', show_label=True, placeholder="Please upload your image first， then input...", lines=3,
             #                         value=None, visible=False, interactive=False)
-
+            # with gr.Row():
             text_input = gr.Textbox(label='User', placeholder='Please upload your image first, then input...',
                                     interactive=False)
+            # submit_button = gr.Button(value="Submit", interactive=True, variant="primary")
 
     upload_button.click(upload_img, [image, text_input, chat_state, chatbot],
                         [image, text_input, upload_button, chat_state, img_list, chatbot])
-
+    # submit_button.click(gradio_ask, [text_input, chatbot, chat_state,radio], [chatbot, chat_state]).then(
+    #     gradio_answer, [chatbot, chat_state, img_list,  radio, text_input,num_beams, temperature], [text_input,chatbot, chat_state, img_list]
+    # )
     text_input.submit(gradio_ask, [text_input, chatbot, chat_state, radio], [chatbot, chat_state]).then(
         gradio_answer, [chatbot, chat_state, img_list, radio, text_input, num_beams, temperature],
         [text_input, chatbot, chat_state, img_list]

multimodal/open_flamingo/chat/conversation.py

@@ -319,7 +319,8 @@ class Chat:
         # else:
         #     conv.append_message(conv.roles[0], text)
 
-    def answer(self, conv, img_list, radio, text_input, model_name, max_new_tokens=200, num_beams=5, min_length=1, top_p=0.9,
+    def answer(self, conv, img_list, radio, text_input, model_name, max_new_tokens=200, num_beams=5, min_length=1,
+               top_p=0.9,
                repetition_penalty=1.0, length_penalty=1, temperature=1, max_length=2000):
         # conv.append_message(conv.roles[1], None)
         # embs = self.get_context_emb(conv, img_list)
@@ -424,7 +425,7 @@ class Chat:
         if radio in ["Cap"]:
             output_text, out_image = captioner(self.model, self.tokenizer, image_ori, batch_images, input_ids,
                                                attention_mask, image_start_index_list, image_nums, added_bbox_list)
-
+            print("asdfghkl----------------------------------------------------------------------------------------->")
         else:
             with torch.inference_mode():
                 text_outputs = self.model.generate(
@@ -477,7 +478,6 @@ class Chat:
             print(output_text)
             output_text = re.findall(r'Assistant:(.+)', output_text)[-1]
             print(output_text)
-        print(output_text)
 
         return output_text, out_image
 

multimodal/open_flamingo/eval/task/caption_chat.py

@@ -1,4 +1,3 @@
-
 import torch
 import more_itertools
 from tqdm import tqdm
@@ -8,6 +7,7 @@ import os
 from transformers import LogitsProcessor, MinNewTokensLengthLogitsProcessor, ForcedEOSTokenLogitsProcessor
 from PIL import Image
 
+
 class VisualLogitsProcessor(LogitsProcessor):
     def __init__(self, tokenizer):
         super().__init__()
@@ -24,7 +24,10 @@ class VisualLogitsProcessor(LogitsProcessor):
     def __call__(self, input_ids, scores):
         # print("decoding===>", self.tokenizer.decode(scores.sort(descending=True).indices.tolist()[0][:self.topk]))
         # import pdb; pdb.set_trace()
-        if self.object_token_id in scores.sort(descending=True).indices.tolist()[0][1:self.topk] and self.eos_token_id not in scores.sort(descending=True).indices.tolist()[0][:self.topk] and (input_ids == self.object_token_id).sum() * 2 == (input_ids == self.endofobject_token_id).sum():
+        if self.object_token_id in scores.sort(descending=True).indices.tolist()[0][
+                                   1:self.topk] and self.eos_token_id not in \
+                scores.sort(descending=True).indices.tolist()[0][:self.topk] and (
+                input_ids == self.object_token_id).sum() * 2 == (input_ids == self.endofobject_token_id).sum():
             scores[0, self.object_token_id] = 1000
         if input_ids[0, -1] == self.object_token_id and input_ids[0, -2] != self.prebox_token_id:
             if (input_ids[0, :-1] == self.object_token_id).sum() != 0:
@@ -75,7 +78,9 @@ def captioner(
     ori_prompt_length = len(input_ids[0])
     have_prebox = False
     prompt = None
-    while True:
+    out_image = None
+    no_end = True
+    while no_end:
         batch_images = batch_images
         if prompt == None:
             input_ids = input_ids
@@ -167,12 +172,7 @@ def captioner(
                 else:
                     import numpy as np
                     import cv2
-                    open_cv_image = np.array(image_ori)
-                    open_cv_image = open_cv_image[:, :, ::-1].copy()
-                    for i, pre_box in enumerate(boxes):
-                        open_cv_image = cv2.rectangle(open_cv_image, pre_box[:2].astype(int), pre_box[2:].astype(int),
-                                                      (0, 255, 0), i + 1)
-                    out_image = Image.fromarray(cv2.cvtColor(open_cv_image, cv2.COLOR_BGR2RGB))
+
                     # exit()
                     pre_box = boxes[scores.argmax()]
                     added_bbox_list += [torch.tensor(pre_box).unsqueeze(0).cuda() / 224]
@@ -190,253 +190,22 @@ def captioner(
                 prompt = tokenizer.decode(outputs[0, :-2].clone()[0])
                 if debug:
                     print("after else---->", prompt)
-
-
         else:
-            break
+            no_end = False
     outputs = outputs[:, ori_prompt_length:]
     outputs = tokenizer.batch_decode(outputs, skip_special_tokens=True)[0].replace('"', "")
+    open_cv_image = np.array(image_ori)
+    open_cv_image = open_cv_image[:, :, ::-1].copy()
+    for i, pre_box in enumerate(added_bbox_list):
+        open_cv_image = cv2.rectangle(open_cv_image, (pre_box[:2] * 224).astype(int), (pre_box[2:] * 224).astype(int),
+                                      (0, 255, 0), i + 1)
+    out_image = Image.fromarray(cv2.cvtColor(open_cv_image, cv2.COLOR_BGR2RGB))
     # new_predictions = [
     #     postprocess_captioning_generation(out).replace('"', "")
     #     for out in tokenizer.batch_decode(outputs, skip_special_tokens=True)
     # ]
     # import pdb; pdb.set_trace()
-    print("out----------------------------------------------------------------------------------------->")
-    return outputs, out_image
-
-
-def evaluate_coco_flickr(
-    model,
-    tokenizer,
-    image_processor,
-    batch_size,
-    is_flickr=False,
-    vis_embed_size=None,
-    rank=0,
-    world_size=1,
-    id=0,
-    debug=False,
-):
-    """Evaluate a model on COCO dataset.
-    Returns:
-        float: CIDEr score
-
-    """
-    visual_logits_processor = VisualLogitsProcessor(tokenizer)
-    coco_dataset = load_dataset("coco_caption")
-    eval_dataset = coco_dataset["test"]
-    model.eval().cuda()
-    predictions = dict()
-    lang_encoder_name = model.lang_encoder.__class__.__name__.lower()
-    media_token_id = tokenizer("<|#image#|>", add_special_tokens=False)["input_ids"][-1]
-    endofmedia_token_id = tokenizer("<|#endofimage#|>", add_special_tokens=False)["input_ids"][-1]
-    pad_token_id = tokenizer(tokenizer.pad_token, add_special_tokens=False)["input_ids"][-1]
-    bos_token_id = tokenizer(tokenizer.bos_token, add_special_tokens=False)["input_ids"][-1]
-    previsual_token_id = tokenizer("<|#previsual#|>", add_special_tokens=False)["input_ids"][-1]
-    visual_token_id = tokenizer("<|#visual#|>", add_special_tokens=False)["input_ids"][-1]
-    box_token = "<|#box#|>"
-    prebox_token = "<|#prebox#|>"
-    endofobject_token = "<|#endofobject#|>"
-    object_token = "<|#object#|>"
-    cnt = 0
-    if world_size > 1:
-        torch.distributed.barrier()
-    desc = "Running inference Flickr30" if is_flickr else "Running inference COCO"
-    for ii, batch in enumerate(more_itertools.chunked(
-        tqdm(eval_dataset, desc=desc, disable=(rank != 0)), batch_size
-    )):
-        if ii % world_size != rank:
-            continue
-        cnt += len(batch)
-        batch[0]["image"] = Image.open("/gpfs/u/home/LMCG/LMCGljnn/scratch/images/img3.jpg").resize((224, 224))
-        batch_images = prepare_batch_images(
-            batch=batch,
-            image_processor=image_processor,
-        ).cuda()
-        prompt = f"{tokenizer.bos_token}<|#image#|>{tokenizer.pad_token*vis_embed_size}<|#endofimage#|>"
-        added_bbox_list = []
-        batch_text = [prompt for _ in batch]
-        encodings = tokenizer(
-            batch_text,
-            padding="longest",
-            truncation=True,
-            return_tensors="pt",
-            max_length=2000,
-        )
-        ori_prompt_length = len(encodings["input_ids"][0])
-        have_prebox = False
-        while True:
-            batch_text = [prompt for _ in batch]
-            encodings = tokenizer(
-                batch_text,
-                padding="longest",
-                truncation=True,
-                return_tensors="pt",
-                max_length=2000,
-            )
-            input_ids = encodings["input_ids"].cuda()
-            attention_mask = encodings["attention_mask"].cuda()
-            image_start_index_list = ((input_ids == media_token_id).nonzero(as_tuple=True)[-1] + 1).tolist()
-            image_start_index_list = [[x] for x in image_start_index_list]
-            image_nums = [1] * len(input_ids)
-            if debug:
-                print("input--->",tokenizer.decode(input_ids[0]))
-            p1 = MinNewTokensLengthLogitsProcessor(
-                prompt_length_to_skip=input_ids.shape[-1],
-                min_new_tokens=5,
-                eos_token_id=bos_token_id,
-            )
-            with torch.inference_mode() and torch.cuda.amp.autocast(dtype=torch.float16):
-                outputs = model.generate(
-                    batch_images,
-                    input_ids,
-                    attention_mask=attention_mask,
-                    max_new_tokens=20,
-                    # min_new_tokens=8,
-                    num_beams=1,
-                    # length_penalty=0,
-                    image_start_index_list=image_start_index_list,
-                    image_nums=image_nums,
-                    added_bbox_list=added_bbox_list if len(added_bbox_list) != 0 else None,
-                    logits_processor_list=[p1, visual_logits_processor],
-                )
-            if debug:
-                print("outputs--->",tokenizer.decode(outputs[0]))
-            if outputs[0, -2] in [previsual_token_id, visual_token_id] and outputs[0, -1] == bos_token_id:
-                prompt = tokenizer.decode(outputs.clone()[0])
-                is_visual = (outputs[0, -2] == visual_token_id)
-                batch_text = tokenizer.batch_decode(outputs[:, :-1])
-                encodings = tokenizer(
-                    batch_text,
-                    padding="longest",
-                    truncation=True,
-                    return_tensors="pt",
-                    max_length=2000,
-                )
-                input_ids = encodings["input_ids"].cuda()
-                attention_mask = encodings["attention_mask"].cuda()
-                image_start_index_list = ((input_ids == media_token_id).nonzero(as_tuple=True)[-1] + 1).tolist()
-                image_start_index_list = [[x] for x in image_start_index_list]
-                image_nums = [1] * len(input_ids)
-                if debug:
-                    print("get the visual bbox--->",tokenizer.decode(input_ids[0]))
-                with torch.cuda.amp.autocast(dtype=torch.float16) and torch.no_grad():
-                    outputs = model(
-                        vision_x=batch_images,
-                        lang_x=input_ids,
-                        attention_mask=attention_mask,
-                        image_nums=image_nums,
-                        image_start_index_list=image_start_index_list,
-                        added_bbox_list=added_bbox_list if len(added_bbox_list) != 0 else None,
-                        add_box=added_bbox_list is not None and len(added_bbox_list) != 0,
-                    )
-                boxes = outputs["boxes"]
-                scores = outputs["scores"]
-                # if not model.valid:
-                #     import pdb; pdb.set_trace()
-                if boxes is not None:
-                    if is_visual:
-                        if have_prebox:
-                            added_bbox_list.pop()
-                            prompt = prompt.replace("<|#previsual#|><|#prebox#|><|#object#|>", "")
-                            have_prebox = False
-                            if debug:
-                                print("find previsual and remove it--->", prompt)
-                        first_box = boxes[scores.argmax()]
-                        added_bbox_list += [torch.tensor(first_box).unsqueeze(0).cuda() / 224]
-                        prompt = prompt[:-len(tokenizer.eos_token)]
-                        prompt += box_token + endofobject_token
-                        if debug:
-                            print("after inserting visual---->", prompt)
-                    else:
-                        import numpy as np
-                        import cv2
-                        open_cv_image = np.array(batch[0]["image"])
-                        open_cv_image = open_cv_image[:, :, ::-1].copy()
-                        for i, pre_box in enumerate(boxes):
-                            open_cv_image = cv2.rectangle(open_cv_image, pre_box[:2].astype(int), pre_box[2:].astype(int), (0, 255, 0), i+1)
-                        cv2.imwrite("Atest.png", open_cv_image)
-                        exit()
-                        pre_box = boxes[scores.argmax()]
-                        added_bbox_list += [torch.tensor(pre_box).unsqueeze(0).cuda() / 224]
-                        prompt = prompt[:-len(tokenizer.eos_token)]
-                        prompt += prebox_token + object_token
-                        have_prebox = True
-                        if debug:
-                            print("after inserting previsual---->", prompt)
-                else:
-                    import pdb;pdb.set_trace()
-                    prompt = tokenizer.decode(outputs[0, :-2].clone()[0])
-            else:
-                break
-        outputs = outputs[:, ori_prompt_length:]
-        new_predictions = [
-            postprocess_captioning_generation(out).replace('"', "")
-            for out in tokenizer.batch_decode(outputs, skip_special_tokens=True)
-        ]
-        # import pdb; pdb.set_trace()
-        if rank == 0:
-            tqdm.write(new_predictions[0])
-        for i, sample in enumerate(batch):
-            predictions[int(sample["image_id"])] = {
-                "caption": new_predictions[i],
-            }
-        print(new_predictions)
-        exit()
-    results_path = (
-        f"flickrresults_{lang_encoder_name}_{rank}_{id}.json"
-        if is_flickr
-        else f"cocoresults_{lang_encoder_name}_{rank}_{id}.json"
-    )
-    with open(results_path, "w") as f:
-        f.write(
-            json.dumps(
-                [
-                    {"image_id": k, "caption": predictions[k]["caption"]}
-                    for k in predictions
-                ],
-                indent=2,
-            )
-        )
-    print("save to", results_path)
-    del predictions
-    time.sleep(10)
-    if world_size > 1:
-        torch.distributed.barrier()
-    if rank == 0:
-        print(f"evaluate on rank {rank}. world size is {world_size}")
-        predictions = []
-        for rank_i in range(world_size):
-            part_results_path = (
-                f"flickrresults_{lang_encoder_name}_{rank_i}_{id}.json"
-                if is_flickr
-                else f"cocoresults_{lang_encoder_name}_{rank_i}_{id}.json"
-            )
-            print("load", part_results_path)
-            predictions.extend(json.load(open(part_results_path)))
-            os.remove(part_results_path)
-        print("num:", len(predictions))
-        results_path = (
-            f"flickrresults_{lang_encoder_name}.json"
-            if is_flickr
-            else f"cocoresults_{lang_encoder_name}.json"
-        )
-        json.dump(predictions, open(results_path, "w"), indent=2)
 
-        metrics = compute_cider(
-            result_path=results_path,
-            annotations_path="/gpfs/u/home/LMCG/LMCGljnn/scratch/.cache/lavis/coco_gt/coco_karpathy_test_gt.json",
-        )
-        metrics["CIDEr"] *= 100
-        os.makedirs("eval_results", exist_ok=True)
-        acc = metrics["CIDEr"]
-        with open(os.path.join("eval_results", f"cococap_{model.expr_name}_{model.step_num}_{int(time.time())}_{acc}"), "w") as f:
-            f.write(json.dumps(predictions, indent=2))
+    return outputs, out_image
 
-        # delete the temporary file
-        os.remove(results_path)
-    else:
-        metrics = {}
-        metrics["CIDEr"] = 0.0
 
-    return metrics["CIDEr"]