update cap

app.py

@@ -2,18 +2,16 @@ import os
 import sys
 from pathlib import Path
 # os.system("cd transformers && pip install .")
-os.system("cd multimodal && pip install .")
-os.system("cd multimodal/YOLOX && pip install .")
+os.system("cd multimodal && pip install -e .")
+
 import numpy as np
 import torch
 from PIL import Image
 import tempfile
 
-
 import string
 import cv2
 
-
 import gradio as gr
 import torch
 from PIL import Image
@@ -52,34 +50,34 @@ flamingo, image_processor, tokenizer, vis_embed_size = create_model_and_transfor
     enhance_data=False,
 )
 
-
-# checkpoint_path = "/home/aimos/huggingface/space/demo.pt"
+model_name = "pythiaS"
 checkpoint_path = hf_hub_download("chendl/compositional_test", "pythiaS.pt")
 checkpoint = torch.load(checkpoint_path, map_location="cpu")["model_state_dict"]
 model_state_dict = {}
 for key in checkpoint.keys():
     model_state_dict[key.replace("module.", "")] = checkpoint[key]
-if "vision_encoder.logit_scale"in model_state_dict:
+if "vision_encoder.logit_scale" in model_state_dict:
     # previous checkpoint has some unnecessary weights
     del model_state_dict["vision_encoder.logit_scale"]
     del model_state_dict["vision_encoder.visual.proj"]
     del model_state_dict["vision_encoder.visual.ln_post.weight"]
     del model_state_dict["vision_encoder.visual.ln_post.bias"]
 flamingo.load_state_dict(model_state_dict, strict=True)
-chat = Chat(flamingo, image_processor, tokenizer, vis_embed_size )
+chat = Chat(flamingo, image_processor, tokenizer, vis_embed_size)
+
 
 def get_outputs(
-    model,
-    batch_images,
-    attention_mask,
-    max_generation_length,
-    min_generation_length,
-    num_beams,
-    length_penalty,
-    input_ids,
-    image_start_index_list=None,
-    image_nums=None,
-    bad_words_ids=None,
+        model,
+        batch_images,
+        attention_mask,
+        max_generation_length,
+        min_generation_length,
+        num_beams,
+        length_penalty,
+        input_ids,
+        image_start_index_list=None,
+        image_nums=None,
+        bad_words_ids=None,
 ):
     #  and torch.cuda.amp.autocast(dtype=torch.float16)
     with torch.inference_mode():
@@ -109,15 +107,13 @@ def get_outputs(
     return outputs
 
 
-
-
 def generate(
-    idx,
-    image,
-    text,
-    vis_embed_size=256,
-    rank=0,
-    world_size=1,
+        idx,
+        image,
+        text,
+        vis_embed_size=256,
+        rank=0,
+        world_size=1,
 ):
     if image is None:
         raise gr.Error("Please upload an image.")
@@ -138,7 +134,8 @@ def generate(
     image = image.resize((224, 224))
     batch_images = image_processor(image).unsqueeze(0).unsqueeze(1).unsqueeze(0)
     if idx == 1:
-        prompt = [f"{tokenizer.bos_token}<|#image#|>{tokenizer.pad_token * vis_embed_size}<|#endofimage#|><|#object#|> {text.rstrip('.').strip()}<|#endofobject#|><|#visual#|>"]
+        prompt = [
+            f"{tokenizer.bos_token}<|#image#|>{tokenizer.pad_token * vis_embed_size}<|#endofimage#|><|#object#|> {text.rstrip('.').strip()}<|#endofobject#|><|#visual#|>"]
         bad_words_ids = None
         max_generation_length = 5
     else:
@@ -174,14 +171,14 @@ def generate(
     boxes = outputs["boxes"]
     scores = outputs["scores"]
     if len(scores) > 0:
-        box = boxes[scores.argmax()]/224
+        box = boxes[scores.argmax()] / 224
     print(f"{box}")
 
     if idx == 1:
         open_cv_image = np.array(image_ori)
         # Convert RGB to BGR
         open_cv_image = open_cv_image[:, :, ::-1].copy()
-        box = box*[width,height,width,height]
+        box = box * [width, height, width, height]
         # for box in boxes:
         open_cv_image = cv2.rectangle(open_cv_image, box[:2].astype(int), box[2:].astype(int), (255, 0, 0), 2)
         out_image = Image.fromarray(cv2.cvtColor(open_cv_image, cv2.COLOR_BGR2RGB))
@@ -199,6 +196,7 @@ description = """<h3>This is the demo of Compositional-VLM. Upload your images a
 article = """<div style='display:flex; gap: 0.25rem; '><a href='https://compositionalvlm.github.io/'><img src='https://img.shields.io/badge/Project-Page-Green'></a><a href='https://github.com/Vision-CAIR/MiniGPT-4'><img src='https://img.shields.io/badge/Github-Code-blue'></a><a href='https://github.com/TsuTikgiau/blip2-llm/blob/release_prepare/MiniGPT_4.pdf'><img src='https://img.shields.io/badge/Paper-PDF-red'></a></div>
 """
 
+
 # TODO show examples below
 
 # ========================================
@@ -217,16 +215,17 @@ def gradio_reset(chat_state, img_list):
 
 def build_image(image):
     if image is None:
-        return  None
+        return None
     # res = draw_bounding_boxes(image=image, boxes=boxes_to_draw, colors=color_to_draw, width=8)
     from torchvision.transforms import ToPILImage
     # res = ToPILImage()(res)
     _, path = tempfile.mkstemp(suffix='.jpg', dir=TEMP_FILE_DIR)
     image.save(path)
 
-    return  path
+    return path
+
 
-def upload_img(gr_img, text_input, chat_state,chatbot):
+def upload_img(gr_img, text_input, chat_state, chatbot):
     if gr_img is None:
         return None, None, gr.update(interactive=True), chat_state, None
     chat_state = []
@@ -235,42 +234,42 @@ def upload_img(gr_img, text_input, chat_state,chatbot):
     chatbot = chatbot + [[(path,), None]]
     llm_message = chat.upload_img(gr_img, chat_state, img_list)
     return gr.update(interactive=False), gr.update(interactive=True, placeholder='Type and press Enter'), gr.update(
-        value="Start Chatting", interactive=False), chat_state, img_list,chatbot
+        value="Start Chatting", interactive=False), chat_state, img_list, chatbot
 
 
-def gradio_ask(user_message, chatbot, chat_state,radio):
+def gradio_ask(user_message, chatbot, chat_state, radio):
     if len(user_message) == 0:
         return gr.update(interactive=True, placeholder='Input should not be empty!'), chatbot, chat_state
 
-
-    chat.ask(user_message, chat_state,radio)
+    chat.ask(user_message, chat_state, radio, model_name)
     chatbot = chatbot + [[user_message, None]]
     return chatbot, chat_state
 
 
-def gradio_answer(chatbot, chat_state, img_list,  radio, text,num_beams, temperature):
+def gradio_answer(chatbot, chat_state, img_list, radio, text, num_beams, temperature):
     image = None
-    llm_message,image = \
-    chat.answer(conv=chat_state, img_list=img_list, max_new_tokens=300, num_beams=1, temperature=temperature,
-                max_length=2000,radio = radio,text_input = text)
-    
+    llm_message, image = \
+        chat.answer(conv=chat_state, img_list=img_list, max_new_tokens=300, num_beams=1, temperature=temperature,
+                    max_length=2000, radio=radio, text_input=text, model_name=model_name)
+
     chatbot[-1][1] = llm_message
-    if chat_state[-1]["from"]=="gpt":
+    if chat_state[-1]["from"] == "gpt":
         chat_state[-1]["value"] = llm_message
-    if image==None:
+    if image == None:
         return "", chatbot, chat_state, img_list
     else:
         path = build_image(image)
-        chatbot = chatbot + [[None,(path,)]]
+        chatbot = chatbot + [[None, (path,)]]
         return "", chatbot, chat_state, img_list
 
+
 task_template = {
-                "Cap": "Summarize the content of the photo <image>.",
-                "VQA": "For this image <image>, I want a simple and direct answer to my question: <question>",
-                "REC": "Can you point out <expr> in the image <image> and provide the coordinates of its location?",
-                "GC": "Can you give me a description of the region <boxes> in image <image>?",
-                "Advanced": "<question>",
-            }
+    "Cap": "Summarize the content of the photo <image>.",
+    "VQA": "For this image <image>, I want a simple and direct answer to my question: <question>",
+    "REC": "Can you point out <expr> in the image <image> and provide the coordinates of its location?",
+    "GC": "Can you give me a description of the region <boxes> in image <image>?",
+    "Advanced": "<question>",
+}
 
 with gr.Blocks() as demo:
     gr.Markdown(title)
@@ -310,24 +309,25 @@ with gr.Blocks() as demo:
             img_list = gr.State()
             chatbot = gr.Chatbot(label='Compositional-VLM')
 
-
             # template = gr.Textbox(label='Template', show_label=True, lines=1, interactive=False,
             #                       value='Provide a comprehensive description of the image <image> and specify the positions of any mentioned objects in square brackets.')
             # text_input = gr.Textbox(label='<question>', show_label=True, placeholder="Please upload your image first， then input...", lines=3,
             #                         value=None, visible=False, interactive=False)
 
-            text_input = gr.Textbox(label='User', placeholder='Please upload your image first, then input...', interactive=False)
+            text_input = gr.Textbox(label='User', placeholder='Please upload your image first, then input...',
+                                    interactive=False)
 
-    upload_button.click(upload_img, [image, text_input, chat_state,chatbot],
-                        [image, text_input, upload_button, chat_state, img_list,chatbot])
+    upload_button.click(upload_img, [image, text_input, chat_state, chatbot],
+                        [image, text_input, upload_button, chat_state, img_list, chatbot])
 
-    text_input.submit(gradio_ask, [text_input, chatbot, chat_state,radio], [chatbot, chat_state]).then(
-        gradio_answer, [chatbot, chat_state, img_list,  radio, text_input,num_beams, temperature], [text_input,chatbot, chat_state, img_list]
+    text_input.submit(gradio_ask, [text_input, chatbot, chat_state, radio], [chatbot, chat_state]).then(
+        gradio_answer, [chatbot, chat_state, img_list, radio, text_input, num_beams, temperature],
+        [text_input, chatbot, chat_state, img_list]
     )
     clear.click(gradio_reset, [chat_state, img_list], [chatbot, image, text_input, upload_button, chat_state, img_list],
                 queue=False)
 
-demo.launch(enable_queue=True,share=True)
+demo.launch(share=True)
 # 
 # with gr.Blocks() as demo:
 #     gr.Markdown(

multimodal/open_flamingo/chat/conversation.py

@@ -22,6 +22,7 @@ from huggingface_hub import hf_hub_download, login
 from open_flamingo.src.factory import create_model_and_transforms
 from open_flamingo.eval.task.caption_chat import captioner
 
+
 class SeparatorStyle(Enum):
     """Different separator style."""
     SINGLE = auto()
@@ -125,18 +126,19 @@ CONV_VISION = Conversation(
     sep="###",
 )
 
+
 def get_outputs(
-    model,
-    batch_images,
-    attention_mask,
-    max_generation_length,
-    min_generation_length,
-    num_beams,
-    length_penalty,
-    input_ids,
-    image_start_index_list=None,
-    image_nums=None,
-    bad_words_ids=None,
+        model,
+        batch_images,
+        attention_mask,
+        max_generation_length,
+        min_generation_length,
+        num_beams,
+        length_penalty,
+        input_ids,
+        image_start_index_list=None,
+        image_nums=None,
+        bad_words_ids=None,
 ):
     #  and torch.cuda.amp.autocast(dtype=torch.float16)
     with torch.inference_mode():
@@ -165,16 +167,17 @@ def get_outputs(
 
     return outputs
 
+
 def generate(
-    idx,
-    image,
-    text,
-    image_processor,
-    tokenizer,
-    flamingo,
-    vis_embed_size=256,
-    rank=0,
-    world_size=1,
+        idx,
+        image,
+        text,
+        image_processor,
+        tokenizer,
+        flamingo,
+        vis_embed_size=256,
+        rank=0,
+        world_size=1,
 ):
     if image is None:
         raise gr.Error("Please upload an image.")
@@ -195,7 +198,8 @@ def generate(
     image = image.resize((224, 224))
     batch_images = image_processor(image).unsqueeze(0).unsqueeze(1).unsqueeze(0)
     if idx == 1:
-        prompt = [f"{tokenizer.bos_token}<|#image#|>{tokenizer.pad_token * vis_embed_size}<|#endofimage#|><|#object#|> {text.rstrip('.').strip()}<|#endofobject#|><|#visual#|>"]
+        prompt = [
+            f"{tokenizer.bos_token}<|#image#|>{tokenizer.pad_token * vis_embed_size}<|#endofimage#|><|#object#|> {text.rstrip('.').strip()}<|#endofobject#|><|#visual#|>"]
         bad_words_ids = None
         max_generation_length = 5
     else:
@@ -231,15 +235,14 @@ def generate(
     boxes = outputs["boxes"]
     scores = outputs["scores"]
     if len(scores) > 0:
-        box = boxes[scores.argmax()]/224
+        box = boxes[scores.argmax()] / 224
     print(f"{box}")
 
-    
-    if len(boxes)>0:
+    if len(boxes) > 0:
         open_cv_image = np.array(image_ori)
         # Convert RGB to BGR
         open_cv_image = open_cv_image[:, :, ::-1].copy()
-        box = box*[width,height,width,height]
+        box = box * [width, height, width, height]
         # for box in boxes:
         open_cv_image = cv2.rectangle(open_cv_image, box[:2].astype(int), box[2:].astype(int), (255, 0, 0), 2)
         out_image = Image.fromarray(cv2.cvtColor(open_cv_image, cv2.COLOR_BGR2RGB))
@@ -248,6 +251,7 @@ def generate(
         gen_text = tokenizer.batch_decode(outputs)
         return (f"{gen_text}")
 
+
 def preprocess_conv(data):
     conversation = ""
     BEGIN_SIGNAL = "### "
@@ -263,14 +267,16 @@ def preprocess_conv(data):
         conversation += (BEGIN_SIGNAL + from_str + ": " + d["value"] + END_SIGNAL)
     return conversation
 
+
 def preprocess_image(sample, image_processor):
     image = image_processor(sample)
     if isinstance(image, transformers.image_processing_utils.BatchFeature):
         image = torch.tensor(image["pixel_values"][0])
     return image
 
+
 class Chat:
-    def __init__(self, model, vis_processor, tokenizer, vis_embed_size ):
+    def __init__(self, model, vis_processor, tokenizer, vis_embed_size):
         self.model = model
         self.vis_processor = vis_processor
         self.tokenizer = tokenizer
@@ -280,34 +286,41 @@ class Chat:
         #                   torch.tensor([2277, 29937]).to(self.device)]  # '###' can be encoded in two different ways.
         # self.stopping_criteria = StoppingCriteriaList([StoppingCriteriaSub(stops=stop_words_ids)])
 
-    def ask(self, text, conv,radio):
-        if radio in ["Cap"]:
-            conv.append({
-                "from": "human",
-                "value": "",
-            })
-        elif radio in ["VQA"]:
-            conv.append({
-                "from": "human",
-                "value": f"Answer the question using a single word or phrase. {text}",
-            })
-        elif radio in ["REC"]:
-            conv.append({
-                "from": "human",
-                "value": f"Please provide the bounding box coordinate of the region this sentence describes: {text}.",
-            })
-        else:
+    def ask(self, text, conv, radio, model_name):
+        if "pythiaS" in model_name:
             conv.append({
                 "from": "human",
                 "value": text,
             })
+        else:
+            if radio in ["Cap"]:
+                conv.append({
+                    "from": "human",
+                    "value": "",
+                })
+            elif radio in ["VQA"]:
+                conv.append({
+                    "from": "human",
+                    "value": f"Answer the question using a single word or phrase. {text}",
+                })
+            elif radio in ["REC"]:
+                conv.append({
+                    "from": "human",
+                    "value": f"Please provide the bounding box coordinate of the region this sentence describes: {text}.",
+                })
+            else:
+                conv.append({
+                    "from": "human",
+                    "value": text,
+                })
         # if len(conv.messages) > 0 and conv.messages[-1][0] == conv.roles[0] \
         #         and conv.messages[-1][1][-6:] == '</Img>':  # last message is image.
         #     conv.messages[-1][1] = ' '.join([conv.messages[-1][1], text])
         # else:
         #     conv.append_message(conv.roles[0], text)
 
-    def answer(self, conv, img_list, radio, text_input, max_new_tokens=200, num_beams=5, min_length=1, top_p=0.9,
+    def answer(self, conv, img_list, radio, text_input, model_name, max_new_tokens=200, num_beams=5, min_length=1,
+               top_p=0.9,
                repetition_penalty=1.0, length_penalty=1, temperature=1, max_length=2000):
         # conv.append_message(conv.roles[1], None)
         # embs = self.get_context_emb(conv, img_list)
@@ -358,10 +371,10 @@ class Chat:
         image = image.resize((size, size))
         print(f"image size: {image.size}")
         batch_images = preprocess_image(image, self.vis_processor).unsqueeze(0).unsqueeze(1).unsqueeze(0)
-        
+
         # conversation = []
         human_sentence = None
-        if radio in ["Cap","VQA"]:
+        if radio in ["Cap", "VQA"]:
             conv.append({
                 "from": "gpt",
                 "value": "",
@@ -375,9 +388,9 @@ class Chat:
             )
         else:
             conv.append({
-                        "from": "gpt",
-                        "value": "",
-                    })
+                "from": "gpt",
+                "value": "",
+            })
         # while True:
         #     human_sentence = input("### Human: ")
         #     if human_sentence == "#end#":
@@ -390,7 +403,11 @@ class Chat:
         #         "from": "gpt",
         #         "value": "",
         #     })
-        text = preprocess_conv(conv).strip()
+        if "pythiaS" in model_name:
+            text = conv[-1]["value"].strip()
+            print(text)
+        else:
+            text = preprocess_conv(conv).strip()
         caption = f"<|#image#|>{self.tokenizer.pad_token * self.vis_embed_size}<|#endofimage#|>{text}"
         encodings = self.tokenizer(
             caption,
@@ -406,7 +423,8 @@ class Chat:
         image_nums = [1] * len(input_ids)
         added_bbox_list = []
         if radio in ["Cap"]:
-            output_text, out_image = captioner(self.model,self.tokenizer,image_ori,batch_images,input_ids,attention_mask,image_start_index_list,image_nums,added_bbox_list)
+            output_text, out_image = captioner(self.model, self.tokenizer, image_ori, batch_images, input_ids,
+                                               attention_mask, image_start_index_list, image_nums, added_bbox_list)
         else:
             with torch.inference_mode():
                 text_outputs = self.model.generate(
@@ -439,7 +457,7 @@ class Chat:
             print(f"{box}")
             out_image = None
 
-            if len(boxes)>0:
+            if len(boxes) > 0:
                 width, height = image_ori.size
                 open_cv_image = np.array(image_ori)
                 # Convert RGB to BGR
@@ -449,7 +467,6 @@ class Chat:
                 open_cv_image = cv2.rectangle(open_cv_image, box[:2].astype(int), box[2:].astype(int), (255, 0, 0), 2)
                 out_image = Image.fromarray(cv2.cvtColor(open_cv_image, cv2.COLOR_BGR2RGB))
 
-
             # output_token = outputs[0, input_ids.shape[1]:]
             # output_text = tokenizer.decode(output_token, skip_special_tokens=True).strip()
             # conv[-1]["value"] = output_text
@@ -499,16 +516,17 @@ class Chat:
     #     mixed_embs = [emb for pair in zip(seg_embs[:-1], img_list) for emb in pair] + [seg_embs[-1]]
     #     mixed_embs = torch.cat(mixed_embs, dim=1)
     #     return mixed_embs
-    
+
+
 def evaluate_exp(
-    model,
-    tokenizer,
-    image_processor,
-    vis_embed_size=None,
-    rank=0,
-    world_size=1,
-    id=0,
-    add_visual=True,
+        model,
+        tokenizer,
+        image_processor,
+        vis_embed_size=None,
+        rank=0,
+        world_size=1,
+        id=0,
+        add_visual=True,
 ):
     media_token_id = tokenizer("<|#image#|>", add_special_tokens=False)["input_ids"][-1]
     box_token_id = tokenizer("<|#box#|>", add_special_tokens=False)["input_ids"][-1]
@@ -541,7 +559,7 @@ def evaluate_exp(
             "value": "",
         })
         text = preprocess_conv(conversation).strip()
-        caption = f"<|#image#|>{tokenizer.pad_token*vis_embed_size}<|#endofimage#|>{text}"
+        caption = f"<|#image#|>{tokenizer.pad_token * vis_embed_size}<|#endofimage#|>{text}"
         encodings = tokenizer(
             caption,
             padding="longest",
@@ -569,3 +587,4 @@ def evaluate_exp(
 
 
 
+

multimodal/open_flamingo/eval/task/caption_chat.py

@@ -51,7 +51,7 @@ def prepare_batch_images(batch, image_processor):
 
 
 def captioner(
-    model,tokenizer,image_ori,batch_images,input_ids,attention_mask,image_start_index_list,image_nums,added_bbox_list,debug=False):
+    model,tokenizer,image_ori,batch_images,input_ids,attention_mask,image_start_index_list,image_nums,added_bbox_list,debug=True):
     """Evaluate a model on COCO dataset.
     Returns:
         float: CIDEr score
@@ -73,10 +73,23 @@ def captioner(
     object_token = "<|#object#|>"
     ori_prompt_length = len(input_ids[0])
     have_prebox = False
+    prompt = None
     while True:
         batch_images = batch_images
-        input_ids = input_ids
-        attention_mask = attention_mask
+        if prompt == None:
+            input_ids = input_ids
+            attention_mask = attention_mask
+        else:
+
+            encodings = tokenizer(
+                [prompt],
+                padding="longest",
+                truncation=True,
+                return_tensors="pt",
+                max_length=2000,
+            )
+            attention_mask = encodings["attention_mask"]
+            input_ids = encodings["input_ids"]
         image_start_index_list = image_start_index_list
         image_nums = image_nums
         if debug:
@@ -148,6 +161,7 @@ def captioner(
                     prompt += box_token + endofobject_token
                     if debug:
                         print("after inserting visual---->", prompt)
+
                 else:
                     import numpy as np
                     import cv2
@@ -165,8 +179,8 @@ def captioner(
                     if debug:
                         print("after inserting previsual---->", prompt)
             else:
-                if debug:
-                    import pdb;pdb.set_trace()
+                # if debug:
+                #     import pdb;pdb.set_trace()
                 prompt = tokenizer.decode(outputs[0, :-2].clone()[0])
         else:
             break
@@ -414,4 +428,5 @@ def evaluate_coco_flickr(
         metrics = {}
         metrics["CIDEr"] = 0.0
 
+
     return metrics["CIDEr"]