Initial commit

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,2 @@
+__pycache__/
+*.py[cod]

README.md

@@ -0,0 +1,14 @@
+---
+title: TEOChat
+emoji: 🏢
+colorFrom: green
+colorTo: pink
+sdk: gradio
+sdk_version: 4.44.1
+app_file: videollava/serve/teochat_demo.py
+pinned: false
+license: apache-2.0
+short_description: A new vision-language assistant for temporal EO data.
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

requirements.txt

@@ -0,0 +1,283 @@
+accelerate==0.26.1
+affine==2.4.0
+aiofiles==23.2.1
+aiohttp==3.8.4
+aiosignal==1.3.1
+altair==5.2.0
+annotated-types==0.7.0
+anyio==3.6.2
+appdirs==1.4.4
+argon2-cffi==21.3.0
+argon2-cffi-bindings==21.2.0
+arrow==1.2.3
+asttokens==2.2.1
+async-timeout==4.0.2
+attrs==23.1.0
+av==11.0.0
+backcall==0.2.0
+beautifulsoup4==4.12.2
+bitsandbytes==0.41.0
+bleach==6.0.0
+blessed==1.20.0
+braceexpand==0.1.7
+Brotli==1.0.9
+certifi==2024.2.2
+cffi==1.15.1
+chardet==4.0.0
+charset-normalizer==3.1.0
+click==8.1.7
+click-plugins==1.1.1
+cligj==0.7.2
+cloudpickle==2.2.1
+cmake==3.26.3
+comm==0.2.1
+contourpy==1.0.7
+coverage==7.4.4
+croniter==1.3.14
+cycler==0.11.0
+cytoolz==0.12.2
+dask==2023.11.0
+dateutils==0.6.12
+debugpy==1.6.7
+decorator==5.1.1
+decord==0.6.0
+deepdiff==6.3.0
+deepspeed==0.14.0
+defusedxml==0.7.1
+distro==1.9.0
+dnspython==2.6.1
+docker-pycreds==0.4.0
+efficientnet-pytorch==0.7.1
+einops==0.6.1
+einops-exts==0.0.4
+email_validator==2.2.0
+exceptiongroup==1.2.0
+executing==1.2.0
+fastapi==0.111.1
+fastapi-cli==0.0.4
+fastjsonschema==2.16.3
+ffmpy==0.3.1
+filelock==3.13.1
+Fiona==1.9.3
+fire==0.5.0
+fonttools==4.39.3
+fqdn==1.5.1
+frozenlist==1.3.3
+fschat==0.2.36
+fsspec==2023.12.2
+ftfy==6.1.3
+fvcore==0.1.5.post20221221
+gdown==5.1.0
+geojson==3.0.1
+gitdb==4.0.10
+GitPython==3.1.31
+gradio==4.39.0
+gradio_client==1.1.1
+h11==0.14.0
+h5py==3.9.0
+hjson==3.1.0
+httpcore==1.0.5
+httptools==0.6.1
+httpx==0.27.0
+huggingface-hub==0.20.3
+idna==3.4
+imagecodecs==2021.8.26
+imageio==2.33.1
+importlib-metadata==7.0.1
+importlib-resources==5.12.0
+iniconfig==2.0.0
+inquirer==3.1.3
+iopath==0.1.10
+ipykernel==6.28.0
+ipython==8.15.0
+ipython-genutils==0.2.0
+ipywidgets==8.1.3
+isoduration==20.11.0
+itsdangerous==2.1.2
+jedi==0.18.2
+Jinja2==3.1.2
+joblib==1.2.0
+jsonpointer==2.3
+jsonschema==4.17.3
+jupyter_client==8.6.0
+jupyter_core==5.5.0
+jupyter-events==0.6.3
+jupyter_server==2.5.0
+jupyter_server_terminals==0.4.4
+jupyterlab-pygments==0.2.2
+jupyterlab_widgets==3.0.11
+kaleido==0.2.1
+kiwisolver==1.4.4
+kornia==0.7.3
+linkify-it-py==2.0.2
+lit==16.0.2
+locket==1.0.0
+markdown-it-py==2.2.0
+markdown2==2.4.12
+MarkupSafe==2.1.2
+matplotlib==3.7.1
+matplotlib-inline==0.1.6
+mdit-py-plugins==0.3.3
+mdurl==0.1.2
+mistune==2.0.5
+mkl-fft==1.3.8
+mkl-random==1.2.4
+mkl-service==2.4.0
+mpmath==1.3.0
+multidict==6.0.4
+munch==2.5.0
+nbclassic==0.5.5
+nbclient==0.7.4
+nbconvert==7.3.1
+nbformat==5.8.0
+nest-asyncio==1.6.0
+networkx==3.1
+nh3==0.2.15
+ninja==1.11.1.1
+nltk==3.8.1
+notebook==6.5.4
+notebook_shim==0.2.3
+numpy==1.26.4
+open_clip_torch==2.26.1
+openai==0.28.1
+opencv-python==4.7.0.72
+ordered-set==4.1.0
+orjson==3.9.12
+packaging==23.1
+pandas==2.0.1
+pandocfilters==1.5.0
+parameterized==0.9.0
+parso==0.8.3
+partd==1.4.1
+pathtools==0.1.2
+peft==0.7.1
+pexpect==4.8.0
+pickleshare==0.7.5
+pillow==10.3.0
+pip==23.0.1
+platformdirs==3.10.0
+plotly==5.22.0
+pluggy==1.4.0
+portalocker==2.8.2
+pprintpp==0.4.0
+pretrainedmodels==0.7.4
+prometheus-client==0.16.0
+prompt-toolkit==3.0.43
+protobuf==4.22.3
+psutil==5.9.5
+ptyprocess==0.7.0
+pure-eval==0.2.2
+py-cpuinfo==9.0.0
+pycountry==23.12.11
+pycountry-convert==0.7.2
+pycparser==2.21
+pydantic==2.8.2
+pydantic_core==2.20.1
+pydub==0.25.1
+Pygments==2.15.1
+PyJWT==2.6.0
+pynvml==11.5.0
+pyparsing==3.0.9
+pyproj==3.5.0
+pyrsistent==0.19.3
+PySocks==1.7.1
+pytest==8.1.1
+pytest-cov==4.1.0
+pytest-mock==3.14.0
+python-dateutil==2.8.2
+python-dotenv==1.0.1
+python-editor==1.0.4
+python-json-logger==2.0.7
+python-multipart==0.0.9
+pytorch-lightning==2.0.2
+pytorchvideo==0.1.3
+pytz==2023.3
+pywavelets==1.5.0
+PyYAML==6.0.1
+pyzmq==25.1.2
+rasterio==1.3.6
+readchar==4.0.5
+regex==2023.12.25
+repoze.lru==0.7
+requests==2.31.0
+reverse-geocoder==1.5.1
+rfc3339-validator==0.1.4
+rfc3986-validator==0.1.1
+rich==13.3.4
+Rtree==1.0.1
+ruff==0.5.2
+safetensors==0.4.2
+scikit-image==0.19.3
+scikit-learn==1.2.2
+scipy==1.11.4
+seaborn==0.13.2
+segmentation-models-pytorch==0.3.2
+semantic-version==2.10.0
+Send2Trash==1.8.0
+sentencepiece==0.1.99
+sentry-sdk==1.21.0
+setproctitle==1.3.2
+setuptools==66.0.0
+shapely==2.0.1
+shellingham==1.5.4
+shortuuid==1.0.11
+six==1.16.0
+smmap==5.0.0
+sniffio==1.3.0
+snuggs==1.4.7
+soupsieve==2.4.1
+stack-data==0.6.2
+starlette==0.37.2
+starsessions==1.3.0
+svgwrite==1.4.3
+sympy==1.11.1
+tabulate==0.9.0
+tenacity==8.5.0
+tensorboardX==2.6.2.2
+termcolor==2.3.0
+terminado==0.17.1
+threadpoolctl==3.1.0
+tifffile==2021.7.2
+tiktoken==0.6.0
+timm==0.6.12
+tinycss2==1.2.1
+tokenizers==0.13.3
+tomli==2.0.1
+tomlkit==0.12.0
+toolz==0.12.1
+torchaudio==2.2.1
+torchgeo==0.6.0
+torchinfo==1.7.2
+torchmetrics==0.11.4
+torchvision==0.17.1
+tornado==6.3.3
+tqdm==4.65.0
+traitlets==5.9.0
+transformers==4.31.0
+Tree==0.2.4
+triton==2.2.0
+typer==0.12.3
+typing_extensions==4.9.0
+tzdata==2023.3
+uc-micro-py==1.0.2
+uri-template==1.2.0
+urllib3==2.1.0
+utm==0.7.0
+uvicorn==0.21.1
+uvloop==0.19.0
+wandb==0.15.0
+watchfiles==0.22.0
+wavedrom==2.0.3.post3
+wcwidth==0.2.13
+webcolors==1.13
+webdataset==0.2.86
+webencodings==0.5.1
+websocket-client==1.5.1
+websockets==11.0.2
+wheel==0.38.4
+widgetsnbextension==4.0.11
+yacs==0.1.8
+yarl==1.9.2
+zipp==3.17.0
+--extra-index-url https://download.pytorch.org/whl/cu113
+torch==2.2.1

videollava/__init__.py

@@ -0,0 +1 @@
+from .model import LlavaLlamaForCausalLM

videollava/constants.py

@@ -0,0 +1,27 @@
+CONTROLLER_HEART_BEAT_EXPIRATION = 30
+WORKER_HEART_BEAT_INTERVAL = 15
+
+LOGDIR = "."
+
+# Model Constants
+IGNORE_INDEX = -100
+
+IMAGE_TOKEN_INDEX = -200
+DEFAULT_IMAGE_TOKEN = "<image>"
+DEFAULT_IMAGE_PATCH_TOKEN = "<im_patch>"
+DEFAULT_IM_START_TOKEN = "<im_start>"
+DEFAULT_IM_END_TOKEN = "<im_end>"
+IMAGE_PLACEHOLDER = "<image-placeholder>"
+
+# ======================================================================================================
+DEFAULT_VIDEO_TOKEN = "<video>"
+DEFAULT_VIDEO_PATCH_TOKEN = "<im_patch>"
+DEFAULT_VID_START_TOKEN = "<vid_start>"
+DEFAULT_VID_END_TOKEN = "<vid_end>"
+VIDEO_PLACEHOLDER = "<video-placeholder>"
+# ======================================================================================================
+
+MAX_IMAGE_LENGTH = 16
+MAX_VIDEO_LENGTH = 1  # current video datasets only have 1 video?
+
+PAD_LENGTH = 620

videollava/conversation.py

@@ -0,0 +1,381 @@
+import dataclasses
+from enum import auto, Enum
+from typing import List, Tuple
+
+
+class SeparatorStyle(Enum):
+    """Different separator style."""
+    SINGLE = auto()
+    TWO = auto()
+    MPT = auto()
+    PLAIN = auto()
+    LLAMA_2 = auto()
+
+
+@dataclasses.dataclass
+class Conversation:
+    """A class that keeps all conversation history."""
+    system: str
+    roles: List[str]
+    messages: List[List[str]]
+    offset: int
+    sep_style: SeparatorStyle = SeparatorStyle.SINGLE
+    sep: str = "###"
+    sep2: str = None
+    version: str = "Unknown"
+
+    skip_next: bool = False
+
+    def get_prompt(self):
+        messages = self.messages
+        if len(messages) > 0 and type(messages[0][1]) is tuple:
+            messages = self.messages.copy()
+            init_role, init_msg = messages[0].copy()
+            init_msg = init_msg[0].replace("<image>", "").strip()
+            if 'mmtag' in self.version:
+                messages[0] = (init_role, init_msg)
+                messages.insert(0, (self.roles[0], "<Image><image></Image>"))
+                messages.insert(1, (self.roles[1], "Received."))
+            else:
+                messages[0] = (init_role, "<image>\n" + init_msg)
+
+        if self.sep_style == SeparatorStyle.SINGLE:
+            ret = self.system + self.sep
+            for role, message in messages:
+                if message:
+                    if type(message) is tuple:
+                        message, _, _ = message
+                    ret += role + ": " + message + self.sep
+                else:
+                    ret += role + ":"
+        elif self.sep_style == SeparatorStyle.TWO:
+            seps = [self.sep, self.sep2]
+            ret = self.system + seps[0]
+            for i, (role, message) in enumerate(messages):
+                if message:
+                    if type(message) is tuple:
+                        message, _, _ = message
+                    ret += role + ": " + message + seps[i % 2]
+                else:
+                    ret += role + ":"
+        elif self.sep_style == SeparatorStyle.MPT:
+            ret = self.system + self.sep
+            for role, message in messages:
+                if message:
+                    if type(message) is tuple:
+                        message, _, _ = message
+                    ret += role + message + self.sep
+                else:
+                    ret += role
+        elif self.sep_style == SeparatorStyle.LLAMA_2:
+            wrap_sys = lambda msg: f"<<SYS>>\n{msg}\n<</SYS>>\n\n"
+            wrap_inst = lambda msg: f"[INST] {msg} [/INST]"
+            ret = ""
+
+            for i, (role, message) in enumerate(messages):
+                if i == 0:
+                    assert message, "first message should not be none"
+                    assert role == self.roles[0], "first message should come from user"
+                if message:
+                    if type(message) is tuple:
+                        message, _, _ = message
+                    if i == 0: message = wrap_sys(self.system) + message
+                    if i % 2 == 0:
+                        message = wrap_inst(message)
+                        ret += self.sep + message
+                    else:
+                        ret += " " + message + " " + self.sep2
+                else:
+                    ret += ""
+            ret = ret.lstrip(self.sep)
+        elif self.sep_style == SeparatorStyle.PLAIN:
+            seps = [self.sep, self.sep2]
+            ret = self.system
+            for i, (role, message) in enumerate(messages):
+                if message:
+                    if type(message) is tuple:
+                        message, _, _ = message
+                    ret += message + seps[i % 2]
+                else:
+                    ret += ""
+        else:
+            raise ValueError(f"Invalid style: {self.sep_style}")
+
+        return ret
+
+    def append_message(self, role, message):
+        self.messages.append([role, message])
+
+    def get_images(self, return_pil=False):
+        images = []
+        for i, (role, msg) in enumerate(self.messages[self.offset:]):
+            if i % 2 == 0:
+                if type(msg) is tuple:
+                    import base64
+                    from io import BytesIO
+                    from PIL import Image
+                    msg, image, image_process_mode = msg
+                    if image_process_mode == "Pad":
+                        def expand2square(pil_img, background_color=(122, 116, 104)):
+                            width, height = pil_img.size
+                            if width == height:
+                                return pil_img
+                            elif width > height:
+                                result = Image.new(pil_img.mode, (width, width), background_color)
+                                result.paste(pil_img, (0, (width - height) // 2))
+                                return result
+                            else:
+                                result = Image.new(pil_img.mode, (height, height), background_color)
+                                result.paste(pil_img, ((height - width) // 2, 0))
+                                return result
+                        image = expand2square(image)
+                    elif image_process_mode in ["Default", "Crop"]:
+                        pass
+                    elif image_process_mode == "Resize":
+                        image = image.resize((336, 336))
+                    else:
+                        raise ValueError(f"Invalid image_process_mode: {image_process_mode}")
+                    max_hw, min_hw = max(image.size), min(image.size)
+                    aspect_ratio = max_hw / min_hw
+                    max_len, min_len = 800, 400
+                    shortest_edge = int(min(max_len / aspect_ratio, min_len, min_hw))
+                    longest_edge = int(shortest_edge * aspect_ratio)
+                    W, H = image.size
+                    if longest_edge != max(image.size):
+                        if H > W:
+                            H, W = longest_edge, shortest_edge
+                        else:
+                            H, W = shortest_edge, longest_edge
+                        image = image.resize((W, H))
+                    if return_pil:
+                        images.append(image)
+                    else:
+                        buffered = BytesIO()
+                        image.save(buffered, format="PNG")
+                        img_b64_str = base64.b64encode(buffered.getvalue()).decode()
+                        images.append(img_b64_str)
+        return images
+
+    def to_gradio_chatbot(self):
+        ret = []
+        for i, (role, msg) in enumerate(self.messages[self.offset:]):
+            if i % 2 == 0:
+                if type(msg) is tuple:
+                    import base64
+                    from io import BytesIO
+                    msg, image, image_process_mode = msg
+                    max_hw, min_hw = max(image.size), min(image.size)
+                    aspect_ratio = max_hw / min_hw
+                    max_len, min_len = 800, 400
+                    shortest_edge = int(min(max_len / aspect_ratio, min_len, min_hw))
+                    longest_edge = int(shortest_edge * aspect_ratio)
+                    W, H = image.size
+                    if H > W:
+                        H, W = longest_edge, shortest_edge
+                    else:
+                        H, W = shortest_edge, longest_edge
+                    image = image.resize((W, H))
+                    buffered = BytesIO()
+                    image.save(buffered, format="JPEG")
+                    img_b64_str = base64.b64encode(buffered.getvalue()).decode()
+                    img_str = f'<img src="data:image/png;base64,{img_b64_str}" alt="user upload image" />'
+                    msg = img_str + msg.replace('<image>', '').strip()
+                    ret.append([msg, None])
+                else:
+                    ret.append([msg, None])
+            else:
+                ret[-1][-1] = msg
+        return ret
+
+    def copy(self):
+        return Conversation(
+            system=self.system,
+            roles=self.roles,
+            messages=[[x, y] for x, y in self.messages],
+            offset=self.offset,
+            sep_style=self.sep_style,
+            sep=self.sep,
+            sep2=self.sep2,
+            version=self.version)
+
+    def dict(self):
+        if len(self.get_images()) > 0:
+            return {
+                "system": self.system,
+                "roles": self.roles,
+                "messages": [[x, y[0] if type(y) is tuple else y] for x, y in self.messages],
+                "offset": self.offset,
+                "sep": self.sep,
+                "sep2": self.sep2,
+            }
+        return {
+            "system": self.system,
+            "roles": self.roles,
+            "messages": self.messages,
+            "offset": self.offset,
+            "sep": self.sep,
+            "sep2": self.sep2,
+        }
+
+
+conv_vicuna_v0 = Conversation(
+    system="A chat between a curious human and an artificial intelligence assistant. "
+           "The assistant gives helpful, detailed, and polite answers to the human's questions.",
+    roles=("Human", "Assistant"),
+    messages=(
+        ("Human", "What are the key differences between renewable and non-renewable energy sources?"),
+        ("Assistant",
+            "Renewable energy sources are those that can be replenished naturally in a relatively "
+            "short amount of time, such as solar, wind, hydro, geothermal, and biomass. "
+            "Non-renewable energy sources, on the other hand, are finite and will eventually be "
+            "depleted, such as coal, oil, and natural gas. Here are some key differences between "
+            "renewable and non-renewable energy sources:\n"
+            "1. Availability: Renewable energy sources are virtually inexhaustible, while non-renewable "
+            "energy sources are finite and will eventually run out.\n"
+            "2. Environmental impact: Renewable energy sources have a much lower environmental impact "
+            "than non-renewable sources, which can lead to air and water pollution, greenhouse gas emissions, "
+            "and other negative effects.\n"
+            "3. Cost: Renewable energy sources can be more expensive to initially set up, but they typically "
+            "have lower operational costs than non-renewable sources.\n"
+            "4. Reliability: Renewable energy sources are often more reliable and can be used in more remote "
+            "locations than non-renewable sources.\n"
+            "5. Flexibility: Renewable energy sources are often more flexible and can be adapted to different "
+            "situations and needs, while non-renewable sources are more rigid and inflexible.\n"
+            "6. Sustainability: Renewable energy sources are more sustainable over the long term, while "
+            "non-renewable sources are not, and their depletion can lead to economic and social instability.\n")
+    ),
+    offset=2,
+    sep_style=SeparatorStyle.SINGLE,
+    sep="###",
+)
+
+conv_vicuna_v1 = Conversation(
+    system="A chat between a curious user and an artificial intelligence assistant. "
+    "The assistant gives helpful, detailed, and polite answers to the user's questions.",
+    roles=("USER", "ASSISTANT"),
+    version="v1",
+    messages=(),
+    offset=0,
+    sep_style=SeparatorStyle.TWO,
+    sep=" ",
+    sep2="</s>",
+)
+
+conv_llama_2 = Conversation(
+    system="""You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe.  Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.
+
+If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.""",
+    roles=("USER", "ASSISTANT"),
+    version="llama_v2",
+    messages=(),
+    offset=0,
+    sep_style=SeparatorStyle.LLAMA_2,
+    sep="<s>",
+    sep2="</s>",
+)
+
+conv_llava_llama_2 = Conversation(
+    system="You are a helpful language and vision assistant. "
+           "You are able to understand the visual content that the user provides, "
+           "and assist the user with a variety of tasks using natural language.",
+    roles=("USER", "ASSISTANT"),
+    version="llama_v2",
+    messages=(),
+    offset=0,
+    sep_style=SeparatorStyle.LLAMA_2,
+    sep="<s>",
+    sep2="</s>",
+)
+
+conv_mpt = Conversation(
+    system="""<|im_start|>system
+A conversation between a user and an LLM-based AI assistant. The assistant gives helpful and honest answers.""",
+    roles=("<|im_start|>user\n", "<|im_start|>assistant\n"),
+    version="mpt",
+    messages=(),
+    offset=0,
+    sep_style=SeparatorStyle.MPT,
+    sep="<|im_end|>",
+)
+
+conv_llava_plain = Conversation(
+    system="",
+    roles=("", ""),
+    messages=(
+    ),
+    offset=0,
+    sep_style=SeparatorStyle.PLAIN,
+    sep="\n",
+)
+
+conv_llava_v0 = Conversation(
+    system="A chat between a curious human and an artificial intelligence assistant. "
+           "The assistant gives helpful, detailed, and polite answers to the human's questions.",
+    roles=("Human", "Assistant"),
+    messages=(
+    ),
+    offset=0,
+    sep_style=SeparatorStyle.SINGLE,
+    sep="###",
+)
+
+conv_llava_v0_mmtag = Conversation(
+    system="A chat between a curious user and an artificial intelligence assistant. "
+           "The assistant is able to understand the visual content that the user provides, and assist the user with a variety of tasks using natural language."
+           "The visual content will be provided with the following format: <Image>visual content</Image>.",
+    roles=("Human", "Assistant"),
+    messages=(
+    ),
+    offset=0,
+    sep_style=SeparatorStyle.SINGLE,
+    sep="###",
+    version="v0_mmtag",
+)
+
+conv_llava_v1 = Conversation(
+    system="A chat between a curious human and an artificial intelligence assistant. "
+           "The assistant gives helpful, detailed, and polite answers to the human's questions.",
+    roles=("USER", "ASSISTANT"),
+    version="v1",
+    messages=(),
+    offset=0,
+    sep_style=SeparatorStyle.TWO,
+    sep=" ",
+    sep2="</s>",
+)
+
+conv_llava_v1_mmtag = Conversation(
+    system="A chat between a curious user and an artificial intelligence assistant. "
+           "The assistant is able to understand the visual content that the user provides, and assist the user with a variety of tasks using natural language."
+           "The visual content will be provided with the following format: <Image>visual content</Image>.",
+    roles=("USER", "ASSISTANT"),
+    messages=(),
+    offset=0,
+    sep_style=SeparatorStyle.TWO,
+    sep=" ",
+    sep2="</s>",
+    version="v1_mmtag",
+)
+
+default_conversation = conv_vicuna_v1
+conv_templates = {
+    "default": conv_vicuna_v0,
+    "v0": conv_vicuna_v0,
+    "v1": conv_vicuna_v1,
+    "vicuna_v1": conv_vicuna_v1,
+    "llama_2": conv_llama_2,
+
+    "plain": conv_llava_plain,
+    "v0_plain": conv_llava_plain,
+    "llava_v0": conv_llava_v0,
+    "v0_mmtag": conv_llava_v0_mmtag,
+    "llava_v1": conv_llava_v1,
+    "v1_mmtag": conv_llava_v1_mmtag,
+    "llava_llama_2": conv_llava_llama_2,
+
+    "mpt": conv_mpt,
+}
+
+
+if __name__ == "__main__":
+    print(default_conversation.get_prompt())

videollava/eval/aid_fmow_ucmerced_utils.py

@@ -0,0 +1,94 @@
+import json
+import numpy as np
+from tqdm import tqdm
+from pathlib import Path
+
+from infer_utils import run_inference_single
+# For the purposes of an experiment, change the infer_utils to: 
+# from infer_utils_mod import run_inference_single
+
+def run_aid_fmow_ucmerced_inference(
+        model,
+        dataset_path,
+        processor,
+        tokenizer,
+        conv_mode,
+        use_video_data=False,
+        open_prompt=None,
+        repeat_frames=None,
+        prompt_strategy="interleave",
+        chronological_prefix=True,
+        data_frac=1,
+        data_size=None,
+	    delete_system_prompt=False,
+        last_image=False,
+        print_prompt=False,
+        **kwargs
+    ):
+    for k, v in kwargs.items():
+        print("WARNING: Unused argument:", k, v)
+
+    try:
+        with open(dataset_path) as f:
+            data = json.load(f)
+    except:
+        data = []
+        with open(dataset_path) as f:
+            for line in f:
+                question = json.loads(line)
+                question["id"] = question["question_id"]
+                question["conversations"] = [
+                    {"value": "This is a satellite image: <video> " + question["text"]},
+                    {"value": question["ground_truth"]}
+                ]
+                question["video"] = [question["image"]]
+                data.append(question)
+
+    if data_size is not None:
+        data_size = min(data_size, len(data))
+        idx = np.random.choice(len(data), data_size, replace=False)
+        data = [data[i] for i in idx]
+    elif data_frac < 1:
+        idx = np.random.choice(len(data), int(len(data) * data_frac), replace=False)
+        data = [data[i] for i in idx]
+
+    vision_key = "video" if "video" in data[0] else "image"
+
+    answers = {}
+    for question in tqdm(data):
+        question_id = question["id"]
+        inp = question["conversations"][0]['value']
+        if open_prompt == "open":
+            # Use an open framing for the question
+            inp = inp.split("Which")[0] + "Which class does this image belong to?"
+        elif open_prompt == "multi-open":
+            inp = inp.split("Which")[0] + "What classes does this image belong to?"
+        answer_str = question["conversations"][1]['value']
+        if 'metadata' not in question:
+            question['metadata'] = None
+        metadata = question['metadata']
+        image_paths = question[vision_key]
+
+        outputs = run_inference_single(
+            model=model,
+            processor=processor,
+            tokenizer=tokenizer,
+            conv_mode=conv_mode,
+            inp=inp,
+            image_paths=image_paths,
+            metadata=metadata,
+            use_video_data=use_video_data,
+            repeat_frames=repeat_frames,
+            prompt_strategy=prompt_strategy,
+            chronological_prefix=chronological_prefix,
+	        delete_system_prompt=delete_system_prompt,
+            last_image=last_image,
+            print_prompt=print_prompt
+        )
+
+        answers[question_id] = {
+            "predicted": outputs,
+            "ground_truth": answer_str
+        }
+
+    return answers

videollava/eval/ben_utils.py

@@ -0,0 +1,114 @@
+import json
+import numpy as np
+from tqdm import tqdm
+from pathlib import Path
+
+from videollava.constants import DEFAULT_IMAGE_TOKEN
+
+from infer_utils import run_inference_single
+
+
+def run_ben_inference(
+        model,
+        dataset_path,
+        processor,
+        tokenizer,
+        conv_mode,
+        use_video_data=False,
+        open_prompt=None,
+        repeat_frames=None,
+        prompt_strategy="interleave",
+        chronological_prefix=True,
+        data_frac=1,
+        data_size=None,
+	    delete_system_prompt=False,
+        last_image=False,
+        start_ind=None,
+        end_ind=None,
+        print_prompt=False,
+        **kwargs
+    ):
+    for k, v in kwargs.items():
+        print("WARNING: Unused argument:", k, v)
+
+    dataset_path = Path(dataset_path)
+    data_dir = dataset_path.parent
+    questions_path = data_dir / dataset_path.name.replace(".json", "_questions.json")
+    answers_path = data_dir / dataset_path.name.replace(".json", "_answers.json")
+    images_path = data_dir / dataset_path.name.replace(".json", "_images.json")
+
+    with open(questions_path) as json_data:
+        questionsJSON = json.load(json_data)
+
+    with open(answers_path) as json_data:
+        answersJSON = json.load(json_data)
+
+    with open(images_path) as json_data:
+        imagesJSON = json.load(json_data)
+
+    if data_size is not None:
+        data_size = min(data_size, len(questionsJSON))
+        idx = np.random.choice(len(questionsJSON), data_size, replace=False)
+        imagesJSON = [imagesJSON[i] for i in idx]
+    elif data_frac < 1:
+        idx = np.random.choice(len(questionsJSON), int(len(questionsJSON) * data_frac), replace=False)
+        imagesJSON = [imagesJSON[i] for i in idx]
+
+    if 'LRBEN' in str(dataset_path):
+        image_folder = 'Images_LR'
+    else:
+        image_folder = 'Data'
+
+    # Get the image IDs of test images
+    images_ids = [img['id'] for img in imagesJSON['images'] if img['active']]
+
+    if start_ind is not None and end_ind is not None:
+        print("Subsetting data from index", start_ind, "to", end_ind)
+        images_ids = images_ids[start_ind:end_ind]
+    elif start_ind is not None:
+        print("Subsetting data from index", start_ind, "to end")
+        images_ids = images_ids[start_ind:]
+    elif end_ind is not None:
+        print("Subsetting data from start to index", end_ind)
+        images_ids = images_ids[:end_ind]
+
+    # Store all predicted answers
+    answers = {}
+    # Read image corresponding to each ID and get its associated question and answer
+    for id in tqdm(images_ids):
+
+        image_paths = [str(data_dir / image_folder / (str(id)+'.tif'))]
+
+        for questionid in imagesJSON['images'][id]['questions_ids']:
+            question = questionsJSON['questions'][questionid]
+            if not question['active']:
+                continue
+            inp = question["question"] + " Answer with one word or number."
+            inp = DEFAULT_IMAGE_TOKEN + '\n' + inp
+            type_str = question["type"]
+            answer_str = answersJSON['answers'][question["answers_ids"][0]]['answer']
+
+            outputs = run_inference_single(
+                model=model,
+                processor=processor,
+                tokenizer=tokenizer,
+                conv_mode=conv_mode,
+                inp=inp,
+                image_paths=image_paths,
+                metadata=None,
+                use_video_data=use_video_data,
+                repeat_frames=repeat_frames,
+                prompt_strategy=prompt_strategy,
+                chronological_prefix=chronological_prefix,
+                delete_system_prompt=delete_system_prompt,
+                last_image=last_image,
+                print_prompt=print_prompt
+            )
+
+            answers[f"{id}_{questionid}"] = {
+                "predicted": outputs,
+                "ground_truth": answer_str,
+                "task": type_str
+            }
+
+    return answers

videollava/eval/cdvqa_utils.py

@@ -0,0 +1,127 @@
+import json
+import numpy as np
+from tqdm import tqdm
+from pathlib import Path
+
+from videollava.constants import DEFAULT_VIDEO_TOKEN
+
+from infer_utils import run_inference_single
+
+
+def run_cdvqa_inference(
+        model,
+        dataset_path,
+        processor,
+        tokenizer,
+        conv_mode,
+        use_video_data=False,
+        open_prompt=None,
+        repeat_frames=None,
+        prompt_strategy="interleave",
+        chronological_prefix=True,
+        data_frac=1,
+        data_size=None,
+	    delete_system_prompt=False,
+        last_image=False,
+        start_ind=None,
+        end_ind=None,
+        print_prompt=False,
+        **kwargs
+    ):
+    for k, v in kwargs.items():
+        print("WARNING: Unused argument:", k, v)
+
+    dataset_path = Path(dataset_path)
+    data_dir = dataset_path.parent
+    questions_path = data_dir / dataset_path.name.replace(".json", "_questions.json")
+    answers_path = data_dir / dataset_path.name.replace(".json", "_answers.json")
+    images_path = data_dir / dataset_path.name.replace(".json", "_images.json")
+
+    with open(questions_path) as json_data:
+        questionsJSON = json.load(json_data)
+
+    with open(answers_path) as json_data:
+        answersJSON = json.load(json_data)
+
+    with open(images_path) as json_data:
+        imagesJSON = json.load(json_data)
+
+    if data_size is not None:
+        data_size = min(data_size, len(questionsJSON))
+        idx = np.random.choice(len(questionsJSON), data_size, replace=False)
+        imagesJSON = [imagesJSON[i] for i in idx]
+    elif data_frac < 1:
+        idx = np.random.choice(len(questionsJSON), int(len(questionsJSON) * data_frac), replace=False)
+        imagesJSON = [imagesJSON[i] for i in idx]
+
+    # Get the image IDs of test images
+    images_ids = [img['id'] for img in imagesJSON['images'] if img['active']]
+
+    if start_ind is not None and end_ind is not None:
+        print("Subsetting data from index", start_ind, "to", end_ind)
+        images_ids = images_ids[start_ind:end_ind]
+    elif start_ind is not None:
+        print("Subsetting data from index", start_ind, "to end")
+        images_ids = images_ids[start_ind:]
+    elif end_ind is not None:
+        print("Subsetting data from start to index", end_ind)
+        images_ids = images_ids[:end_ind]
+
+    # Store all predicted answers
+    answers = {}
+    # Read image corresponding to each ID and get its associated question and answer
+    for id in tqdm(images_ids):
+        file_name = imagesJSON['images'][id]['file_name']
+
+        image_paths = [
+            str(data_dir / "second_dataset" / "im1" / file_name),
+            str(data_dir / "second_dataset" / "im2" / file_name),
+        ]
+
+        for questionid in imagesJSON['images'][id]['questions_ids']:
+            question = questionsJSON['questions'][questionid]
+            if not question['active']:
+                continue
+            inp = "This is a pair of satellite images capturing the same location at different times: "
+            inp = inp + DEFAULT_VIDEO_TOKEN + '\n'
+            inp = inp + question["question"]
+            type_str = question["type"]
+            answer_str = answersJSON['answers'][question["answers_ids"][0]]['answer']
+
+            if type_str in ["change_or_not", "increase_or_not", "decrease_or_not"]:
+                inp = inp + " Answer with yes or no."
+
+            elif type_str == "change_ratio":
+                inp = inp + " Choose from one of the following options: 0, 0_to_10, 10_to_20, 20_to_30, 30_to_40, 40_to_50, 50_to_60, 60_to_70, 70_to_80, 80_to_90, 90_to_100."
+
+            elif type_str == "change_ratio_types":
+                inp = inp + " Choose from one of the following options: 0, 0_to_10, 10_to_20, 20_to_30, 30_to_40, 40_to_50, 50_to_60, 60_to_70."
+
+            else: # smallest_change, largest_change, change_to_what
+                inp = inp + " Choose from one of the following options: buildings, low_vegetation, nonvegetated ground surface, playgrounds, trees, water."
+                answer_str = answer_str.replace("NVG_surface", "nonvegetated ground surface")    
+
+            outputs = run_inference_single(
+                model=model,
+                processor=processor,
+                tokenizer=tokenizer,
+                conv_mode=conv_mode,
+                inp=inp,
+                image_paths=image_paths,
+                metadata=None,
+                use_video_data=use_video_data,
+                repeat_frames=repeat_frames,
+                prompt_strategy=prompt_strategy,
+                chronological_prefix=chronological_prefix,
+                delete_system_prompt=delete_system_prompt,
+                last_image=last_image,
+                print_prompt=print_prompt
+            )
+
+            answers[f"{id}_{questionid}"] = {
+                "predicted": outputs,
+                "ground_truth": answer_str,
+                "task": type_str
+            }
+
+    return answers

videollava/eval/classification_segmentation.py

@@ -0,0 +1,151 @@
+import json
+import numpy as np
+from infer_utils import create_mask
+from shapely.wkt import loads
+from collections import defaultdict
+from tqdm import tqdm
+
+def clean_string(s):
+    return s.replace(' ', '-').replace('.', '').lower()
+
+def get_class_dict(dataset):
+    if dataset == "qfabric":
+        class_dict = {
+            "temporal_region_based_question_answering: What is the development status in this region [bbox] in image N?":
+            {
+                "prior-construction": 1, 
+                "greenland ": 2, 
+                "land-cleared": 3, 
+                "excavation": 4, 
+                "materials-dumped": 5, 
+                "construction-started": 6, 
+                "construction-midway": 7, 
+                "construction-done": 8, 
+                "operational": 9
+            },
+            "region_based_question_answering: Identify the type of urban development that has occurred in this area [bbox].": 
+            {
+                "residential": 10,
+                "commercial": 11,
+                "industrial": 12,
+                "road": 13,
+                "demolition": 14,
+                "mega-projects": 15
+            }
+        }
+    elif dataset == "xbd":
+        class_dict = {
+            "classification: Classify the level of damage experienced by the building at location [bbox] in the second image. Choose from: No damage, Minor Damage, Major Damage, Destroyed.": 
+            {
+                "no-damage": 1,
+                "minor-damage": 2,
+                "major-damage": 3,
+                "destroyed": 4,
+            }
+        }
+    else:
+        raise ValueError(f"Dataset {dataset} should not be evaluated on segmentation classification.")
+    return class_dict
+
+
+
+def classification_segmentation(answer_path, dataset, per_class_f1=False, height=256, width=256):
+    """
+    Given the path to the answer file, this function creates segmentation masks on the original polygon for the predicted and ground truth classes.
+    Returns the class-weighted per-pixel F1 between predicted and ground-truth masks.
+    """
+    with open(answer_path) as f:
+        results = json.load(f)
+
+    classes = get_class_dict(dataset)
+    class_stats = defaultdict(lambda: {'tp': 0, 'fp': 0, 'fn': 0, 'count': 0})
+
+    for result in tqdm(results.values()):
+        if result['task'] not in classes:
+            continue
+        class_dict = classes[result['task']]
+        predicted_class = clean_string(result['predicted'])
+        try:
+            ground_truth_class = clean_string(result["ground_truth"])
+        except:
+            ground_truth_class = clean_string(result["original_answer"])
+        original_polygon = loads(result['original_input_polygon'])
+        
+        pred_msk = np.zeros((height, width), dtype='uint8')
+        gt_msk = np.zeros((height, width), dtype='uint8')
+        _msk = create_mask(original_polygon, im_size=(height, width))
+
+        if predicted_class not in class_dict or ground_truth_class not in class_dict:
+            continue
+        
+        pred_label = class_dict[predicted_class]
+        gt_label = class_dict[ground_truth_class]
+        pred_msk[_msk > 0] = pred_label
+        gt_msk[_msk > 0] = gt_label
+
+        for label in class_dict.values():
+            pred_mask = (pred_msk == label)
+            gt_mask = (gt_msk == label)
+            tp = np.sum(pred_mask & gt_mask)
+            fp = np.sum(pred_mask & ~gt_mask)
+            fn = np.sum(~pred_mask & gt_mask)
+            
+            class_stats[label]['tp'] += tp
+            class_stats[label]['fp'] += fp
+            class_stats[label]['fn'] += fn
+            class_stats[label]['count'] += np.sum(gt_mask)
+
+    
+    scores_dict = {}
+
+    for task, class_info in classes.items():
+        print(f"Task: {task}")
+        class_f1_scores = {}
+        weighted_f1_score = 0
+        total_weight = 0
+
+        tp = 0
+        fp = 0
+        fn = 0
+        for class_name, class_label in class_info.items():
+            stats = class_stats[class_label]
+            total_samples = sum(stats['count'] for label, stats in class_stats.items() if label in class_info.values())
+
+            if stats['tp'] + stats['fp'] == 0 or stats['tp'] + stats['fn'] == 0:
+                f1 = 0.0
+            else:
+                precision = stats['tp'] / (stats['tp'] + stats['fp'])
+                recall = stats['tp'] / (stats['tp'] + stats['fn'])
+                if precision + recall == 0:
+                    f1 = 0.0
+                else:
+                    f1 = 2 * (precision * recall) / (precision + recall)
+            class_f1_scores[class_name] = f1
+
+            if stats['count'] > 0:
+                prevalence_inv = total_samples / stats['count']
+                weighted_f1_score += f1 * prevalence_inv
+                total_weight += prevalence_inv
+            
+            tp += stats['tp']
+            fp += stats['fp']
+            fn += stats['fn']
+        
+        if tp + fp == 0 or tp + fn == 0:
+            micro_f1 = 0.0
+        else:
+            micro_f1 = tp / (tp + 0.5 * (fp + fn))
+
+        if total_weight > 0:
+            weighted_f1_score /= total_weight
+        else:
+            weighted_f1_score = 0.0
+
+        scores_dict[task] = (class_f1_scores, weighted_f1_score)
+        print(f"Per-class F1 scores: {class_f1_scores}")
+        if dataset == 'qfabric':
+            print(f"Micro average F1 score: ", micro_f1)
+        else: 
+            print(f"Weighted average F1 score: {weighted_f1_score}")
+    
+    return scores_dict

videollava/eval/datasets_into_geochat_format.py

@@ -0,0 +1,293 @@
+import pandas as pd 
+import re
+import json
+
+def qfabric_semiconverted_to_geochat_dataset_format(json_file):
+    with open(json_file) as f:
+        data = json.load(f)
+    for conversation_group in data:
+        for item in conversation_group["conversations"]:
+            # Remove satellite specifications
+            item["value"] = re.sub(r"This is a satellite image :", "", item["value"])
+            item["value"] = re.sub(r"This is a satellite image:", "", item["value"])
+            item["value"] = re.sub(r"This is a high resolution, optical satellite image .*?:\s*", "", item["value"])
+            item["value"] = re.sub(r"This is a high resolution, optical satellite image.*?:\s*", "", item["value"])
+            item["value"] = re.sub(r"This is a satellite image from .*?:\s*", "", item["value"])
+            item["value"] = re.sub(r"This is a satellite image from.*?:\s*", "", item["value"])
+            # Remove strings around <identify> that are redundant
+            item["value"] = re.sub(r'What is <identify>|this area {<', lambda x: '[identify]' if 'What is [identify]' in x.group() else '{<', item["value"])
+            # Switch out <video> for <image>
+            item["value"] = re.sub(r'<video>', '', item["value"])
+            # Get rid of "this region" immediately before the bounding box 
+            item["value"] = re.sub(r'this region {<', '{<', item["value"])
+            # Check for the presence of '<identify>' and modify the string accordingly
+            if '[identify]' in item["value"]:
+                # Find the position of '<identify>' and the position of the first occurrence of '>}' after '<identify>'
+                identify_index = item["value"].find('[identify]')
+                identify_word_index = item["value"].find('Identify ', identify_index + 8)  
+                # if identify_word_index != -1:
+                #     item["value"] = item["value"][:identify_word_index] + item["value"][identify_word_index + 8:]
+                closing_brace_index = item["value"].find('>}', identify_index)
+    return data
+
+def fmow_to_geochat_dataset_format(json_file):
+    with open(json_file) as f:
+        data = json.load(f)
+    for i, entry in enumerate(data):
+        video_count = len(entry.get("video", []))
+        if video_count > 1:
+            original_videos = entry["video"]
+            for idx in range(video_count):
+                new_entry = entry.copy()
+                new_entry['video'] = [original_videos[idx]]
+                new_entry['image'] = original_videos[idx]
+                new_entry['linked_id'] = entry['id']
+                new_entry['img_idx_from_video_lst_id'] = idx
+                data.append(new_entry)
+        else: 
+            new_entry = entry.copy()
+            new_entry['image'] = original_videos[0]
+    for conversation_group in data:
+        for item in conversation_group["conversations"]:
+            item["value"] = re.sub(r"This is a satellite image :", "", item["value"])
+            item["value"] = re.sub(r"This is a satellite image:", "", item["value"])
+            item["value"] = re.sub(r"This is a high resolution, optical satellite image .*?:\s*", "", item["value"])
+            item["value"] = re.sub(r"This is a high resolution, optical satellite image.*?:\s*", "", item["value"])
+            item["value"] = re.sub(r"This is a satellite image from .*?:\s*", "", item["value"])
+            item["value"] = re.sub(r"This is a satellite image from.*?:\s*", "", item["value"])
+            item["value"] = re.sub(r"This is a sequence of low-resolution, optical satellite images capturing the same location at different times: ", "", item["value"])
+            item["value"] = re.sub(r"This is a sequence of high-resolution, optical satellite images capturing the same location at different times:", "", item["value"])
+            item["value"] = re.sub(r"This is a sequence of satellite images capturing the same location at different times:", "", item["value"])
+            item["value"] = re.sub(r"This is a sequence of satellite images from .*? the same location at different times:", "", item["value"])
+            item["value"] = re.sub(r'This is a high resolution,? optical satellite image .*:\s*<image>\n', '\n', item["value"])
+            item["value"] = re.sub(r'^This is a high[- ]resolution,? .*?image:\s*<image>\n', '\n', item["value"], flags=re.IGNORECASE | re.DOTALL)
+            
+            # Switch out <video> for <image>
+            item["value"] = re.sub(r'<video>', '', item["value"])
+            # Get rid of "this region" immediately before the bounding box 
+            item["value"] = re.sub(r'this region {<', '{<', item["value"])
+            # Which class 
+            item["value"] = re.sub(r'Which of the following classes does this sequence of images belong to', 'Which of the following classes does this image belong to', item["value"])
+            # Please answer using one of the following classes:
+            item["value"] = re.sub(r'Please answer using only one of the following classes:', 'Please use one of the following classes:', item["value"])
+            # Check for the presence of '<identify>' and modify the string accordingly
+            if '[identify]' in item["value"]:
+                # Find the position of '<identify>' and the position of the first occurrence of '>}' after '<identify>'
+                identify_index = item["value"].find('[identify]')
+    for i, entry in enumerate(data):
+        video_count = len(entry.get("video", []))
+        if video_count > 1:
+            data.pop(i)
+    return data
+
+def xbd_to_geochat_dataset_format(json_file):
+
+    with open(json_file) as f:
+        data = json.load(f)
+
+    new_data = []
+    for i, entry in enumerate(data):
+        if entry["task"].startswith("localization"):
+            new_entry=entry.copy()
+            new_entry['image'] = entry['video'][0]
+            new_data.append(new_entry)
+        if entry["task"].startswith("classification"):
+            new_entry=entry.copy()
+            new_entry['image'] = entry['video'][1]
+            new_data.append(new_entry)
+        # Auxiliary tasks all look at the second image
+        else: 
+            new_entry=entry.copy()
+            new_entry['image'] = entry['video'][1]
+            new_data.append(new_entry)
+            
+    for conversation_group in new_data:
+        localization=False
+        classification=False
+        # Add a [refer] token to localization tasks
+        if conversation_group["task"].startswith("localization") or "identify" in conversation_group["task"].lower():
+            localization=True
+        # Add a [identify] token to classification tasks
+        if conversation_group["task"].startswith("classification"):
+            classification=True
+        
+        for item in conversation_group["conversations"]:
+            item["value"] = re.sub(r"This is a satellite image :", "", item["value"])
+            item["value"] = re.sub(r"This is a satellite image:", "", item["value"])
+            item["value"] = re.sub(r"This is a high resolution, optical satellite image .*?:\s*", "", item["value"])
+            item["value"] = re.sub(r"This is a high resolution, optical satellite image.*?:\s*", "", item["value"])
+            item["value"] = re.sub(r"This is a satellite image from .*?:\s*", "", item["value"])
+            item["value"] = re.sub(r"These are two satellite images from .*? capturing the same location at different times: ", "", item["value"])
+            item["value"] = re.sub(r"These are two low-resolution, optical satellite images capturing the same location at different times:", "", item["value"])
+            item["value"] = re.sub(r"These are two high-resolution, optical satellite images capturing the same location at different times:", "", item["value"])
+            item["value"] = re.sub(r"These are two high-resolution, optical satellite images from .*? capturing the same location at different times:", "", item["value"])
+            item["value"] = re.sub(r"These are two satellite images capturing the same location at different times:", "", item["value"])
+            item["value"] = re.sub(r"These are two satellite images from .*? capturing the same location at different times:", "", item["value"])
+            item["value"] = re.sub(r'These are two high-resolution,? optical satellite images .*:\s*<image>\n', '<image>\n', item["value"])
+            item["value"] = re.sub(r'These are two high resolution,? optical satellite images .*:\s*<image>\n', '<image>\n', item["value"])
+            item["value"] = re.sub(r'^This is a high[- ]resolution,? .*?image:\s*<image>\n', '<image>\n', item["value"], flags=re.IGNORECASE | re.DOTALL)
+
+            # Switch out <video> for <image>
+            if classification:
+                item["value"] = re.sub(r'<video> \n', '<image> \n [identify] ', item["value"])
+                item["value"] = re.sub(r' in the second image.', '.', item["value"])
+            elif localization:
+                item["value"] = re.sub(r'<video> \n', '<image> \n [refer] ', item["value"])
+                item["value"] = re.sub(r'Image 1', 'the image', item["value"])
+            else:
+                item["value"] = re.sub(r'<video> \n', '<image> \n ', item["value"])
+
+            # Replace temporal/multi-image wording for auxiliary tasks
+            replacements = {
+                'Are there any buildings in the first image which have been damaged in the second image? Answer with one word.': 'Are there any damaged buildings in the image? Answer with one word.',
+                'Have any buildings in the first image been damaged in the second image? Answer with one word.': 'Have any buildings been damaged in the area? Answer with one word.',
+                'What disaster has occurred between the first and second image?': 'What disaster has occurred here?',
+                'Identify the buildings in the first image which were severely damaged or destroyed in the second image. Include a bounding box of the form [x_min, y_min, x_max, y_max] for each identified building in your response. If there are no such buildings, do not output a bounding box.': 'Identify the severely damaged or destroyed buildings in the image. Include a bounding box of the form [x_min, y_min, x_max, y_max] for each identified building in your response. If there are no such buildings, do not output a bounding box.'
+            }
+            for old, new in replacements.items():
+                item['value'] = re.sub(re.escape(old), new, item['value'])
+
+
+            # Get rid of "this region" immediately before the bounding box 
+            item["value"] = re.sub(r'this region {<', '{<', item["value"])
+            # Which class 
+            item["value"] = re.sub(r'Which of the following classes does this sequence of images belong to', 'Which of the following classes does this image belong to', item["value"])
+            # Please answer using one of the following classes:
+            item["value"] = re.sub(r'Please answer using only one of the following classes:', 'Please use one of the following classes:', item["value"])
+            # Replace bounding box format [79, 27, 85, 81] with {<79><27><85><81>|<0>}
+            item["value"] = re.sub(r'\[(\d+), (\d+), (\d+), (\d+)\]', r'{<\1><\2><\3><\4>|<0>}', item["value"])
+            # Replace bounding box format [x_min, y_min, x_max, y_max] with {<x_min><y_min><x_max><y_max>|<0>}
+            item["value"] = re.sub(r'\[(x_min), (y_min), (x_max), (y_max)\]', r'{<\1><\2><\3><\4>|<0>}', item["value"])
+    return new_data
+
+def s2looking_to_geochat_dataset_format(json_file):
+    with open(json_file) as f:
+        data = json.load(f)
+
+    question = "<image>\n [refer] Identify all buildings in the image."
+
+    new_dataset = []
+    for elem in data:
+        for i in range(2):
+            new_item = {}
+            new_item['id'] = elem['id'] + '_' + str(i)
+            new_item['metadata'] = elem['metadata'][i]
+            new_item['original_input_polygon'] = elem['original_input_polygon']
+            new_item['task'] = elem['task']
+            new_item['image'] = elem['video'][i]
+            new_item['geovlm_id'] = i
+            new_item['original_conversation'] = elem['conversations']
+            new_item['conversations'] = [
+                {
+                    "from": "human",
+                    "value": question
+                },
+                {
+                    "from": "gpt",
+                    "value": ""
+                }
+                ]
+            new_dataset.append(new_item)
+    
+    data = new_dataset
+
+    for conversation_group in data:
+        for item in conversation_group["conversations"]:
+            # Check if the sentence starts with "This is" or "These are" and contains "<image>"
+            if (item["value"].startswith("This is") or item["value"].startswith("These are")) and "<image>" in item["value"]:
+                colon_index = item["value"].find(":")
+                if colon_index != -1 and item["value"][colon_index+1:].strip().startswith("<image>"):
+                    item["value"] = item["value"][colon_index+1:].strip()
+            item["value"] = re.sub(r"This is a sequence of high-resolution, optical satellite images from Maxar's GeoEye-1, QuickBird-2, WorldView-2, or WorldView-3 capturing the same location at different times:", "", item["value"])
+            item["value"] = re.sub(r"This is a sequence of low-resolution, optical satellite images from Sentinel-2 capturing the same location at different times:", "", item["value"])
+            item["value"] = re.sub(r"This is a satellite image :", "", item["value"])
+            item["value"] = re.sub(r"This is a satellite image:", "", item["value"])
+            item["value"] = re.sub(r"This is a high resolution, optical satellite image .*?:\s*", "", item["value"])
+            item["value"] = re.sub(r"This is a high resolution, optical satellite image.*?:\s*", "", item["value"])
+            item["value"] = re.sub(r"This is a satellite image from .*?:\s*", "", item["value"])
+            item["value"] = re.sub(r"This is a satellite image from.*?:\s*", "", item["value"])
+            # This one is the one I'm referring to: 
+            item["value"] = re.sub(r'^This is a sequence of.*times:$', '', item["value"])
+            item["value"] = re.sub(r"This is a sequence of high-resolution, optical satellite images from .*? capturing the same location at different times:", "", item["value"])
+            item["value"] = re.sub(r"This is a sequence of low-resolution, optical satellite images capturing the same location at different times: ", "", item["value"])
+            item["value"] = re.sub(r"This is a sequence of high-resolution, optical satellite images capturing the same location at different times:", "", item["value"])
+            item["value"] = re.sub(r"This is a sequence of satellite images capturing the same location at different times:", "", item["value"])
+            item["value"] = re.sub(r"This is a sequence of satellite images from .*? the same location at different times:", "", item["value"])
+            item["value"] = re.sub(r"These are two high-resolution, optical satellite images capturing the same location at different times:", "", item["value"])
+            item["value"] = re.sub(r"This is a sequence of images from the satellites GaoFen, SuperView and BeiJing-2, capturing the same location at different times:", "", item["value"])
+            
+            # Switch out <video> for <image>
+            item["value"] = re.sub(r'<video>', '', item["value"])
+            # Get rid of "this region" immediately before the bounding box 
+            item["value"] = re.sub(r'this region {<', '{<', item["value"])
+            # Which class 
+            item["value"] = re.sub(r'Which of the following classes does this sequence of images belong to', 'Which of the following classes does this image belong to', item["value"])
+            # Please answer using one of the following classes:
+            item["value"] = re.sub(r'Please answer using only one of the following classes:', 'Please use one of the following classes:', item["value"])
+            # Check for the presence of '<identify>' and modify the string accordingly
+            if '[identify]' in item["value"]:
+                # Find the position of '<identify>' and the position of the first occurrence of '>}' after '<identify>'
+                identify_index = item["value"].find('[identify]')
+                closing_brace_index = item["value"].find('>}', identify_index)
+
+            # Fix the bounding box format: 
+    for conversation_group in data:
+        for item in conversation_group["conversations"]:
+            # Replace bounding box format [79, 27, 85, 81] with {<79><27><85><81>|<0>}
+            item["value"] = re.sub(r'\[(\d+), (\d+), (\d+), (\d+)\]', r'{<\1><\2><\3><\4>|<0>}', item["value"])
+            # Replace bounding box format [x_min, y_min, x_max, y_max] with {<x_min><y_min><x_max><y_max>|<0>}
+            item["value"] = re.sub(r'\[(x_min), (y_min), (x_max), (y_max)\]', r'{<\1><\2><\3><\4>|<0>}', item["value"])
+    for i, entry in enumerate(data):
+        video_count = len(entry.get("video", []))
+        if video_count > 1:
+            data.pop(i)
+    return data
+
+def check_file(file_path):
+    with open(file_path, 'r') as file:
+        data = json.load(file)
+    for conversation_group in data:
+        for item in conversation_group["conversations"]:
+            if '<image>' not in item["value"]:
+                if item["from"] != 'gpt':
+                    print(f"Missing <image> in: {item}")
+            if any(sentence.strip().startswith(('This is', 'These are')) for sentence in item["value"].split('.')):
+                print(f"Starts with 'This is' or 'These are' in: {item}")
+if __name__ == "__main__":
+
+    # Paths to datasets
+    fmow_0 = "/scr/geovlm/fmow_low_res_val.json"
+    fmow_1 = "/scr/geovlm/fmow_high_res_val.json"
+
+    qfabric_0 = '/scr/geovlm/QFabric/test_geochat_seqlen_5_256.json'
+    qfabric_1 = '/scr/geovlm/QFabric/test_geochat_seqlen_2_256.json'
+
+    xbd_0 = '/scr/geovlm/xbd_test_auxiliary_multi_image.json'
+    xbd_1 = '/scr/geovlm/xbd_test_canon_classification.json'
+    xbd_2 = '/scr/geovlm/xbd_test_canon_localization.json'
+
+    print("Running conversion on all datasets, storing updated datasets in variables")
+
+    from tqdm import tqdm
+
+    dataset_formats = [
+        (fmow_to_geochat_dataset_format, fmow_0),
+        (fmow_to_geochat_dataset_format, fmow_1),
+    ]
+    formatted_datasets = []
+    for format_func, dataset in tqdm(dataset_formats, desc="Converting datasets"):
+        if "xbd_test_auxiliary" in dataset: 
+            formatted_datasets.append(format_func(dataset))
+    
+    fmow_0_formatted, fmow_1_formatted = formatted_datasets
+
+    # Write the formatted data for fmow_0 into a JSON file named geochat_fmow_RECENT_format_low_res.json
+    with open('/scr/geovlm/geochat_fmow_RECENT_format_low_res.json', 'w') as file:
+        json.dump(fmow_0_formatted, file)
+
+    # Write the formatted data for fmow_1 into a JSON file named geochat_fmow_RECENT_format_low_res_AGG.json
+    with open('/scr/geovlm/geochat_fmow_RECENT_format_high_res.json', 'w') as file:
+        json.dump(fmow_1_formatted, file)
+    
+    check_file('/scr/geovlm/geochat_fmow_RECENT_format_low_res.json')
+    check_file('/scr/geovlm/geochat_fmow_RECENT_format_high_res.json')

videollava/eval/eval_classification.py

@@ -0,0 +1,151 @@
+"""
+Segmentation metric code dapted from code for XView2: A Strong Baseline
+Xview2_Strong_Baseline/legacy/xview2_metrics.py
+Xview2_Strong_Baseline/legacy/create_masks.py
+"""
+# add python path
+# import sys
+# import os
+# sys.path.append('/deep/u/emily712/aicc-win24-geo-vlm/videollava/')
+
+import json
+import string
+import numpy as np
+import cv2
+from collections import defaultdict, Counter
+from nltk.tokenize import word_tokenize
+from shapely.geometry import Polygon
+from pathlib import Path
+from sklearn.metrics import f1_score
+from tqdm import tqdm
+
+
+def compute_tp_fn_fp(pred: np.ndarray, targ: np.ndarray, c: int):
+    """
+    Computes the number of TPs, FNs, FPs, between a prediction (x) and a target (y) for the desired class (c)
+
+    Args:
+        pred (np.ndarray): prediction
+        targ (np.ndarray): target
+        c (int): positive class
+    """
+    TP = np.logical_and(pred == c, targ == c).sum()
+    FN = np.logical_and(pred != c, targ == c).sum()
+    FP = np.logical_and(pred == c, targ != c).sum()
+    return [TP, FN, FP]
+
+
+def accuracy_precision_recall(answer_path, dataset, ignore_punctuation=True, verbose=True):
+    # Replace with the path to the answers file
+    if type(answer_path) == dict:
+        results = answer_path
+    else:
+        with open(answer_path) as json_data:
+            results = json.load(json_data)
+
+    task_total = defaultdict(int)
+    task_tp = defaultdict(int)
+
+    binary_classification = defaultdict(bool)
+    binary_fp = defaultdict(int)
+    binary_fn = defaultdict(int)
+
+    # Dictionary of dictionaries. Key: task. Value: {class: count}
+    ground_truths = defaultdict(dict)
+
+    values = defaultdict(list)
+
+    accepted_tasks = [
+        "temporal_question_answering",
+        "region_based_question_answering",
+        "temporal_region_based_question_answering",
+        "question_answering",
+        "temporal_referring_expression",
+        "rural_urban",
+        "comp",
+        "presence",
+        "count",
+        "change_to_what",
+        "smallest_change",
+        "change_or_not",
+        "change_ratio",
+        "largest_change",
+        "change_ratio_types",
+        "increase_or_not",
+        "decrease_or_not"
+    ]
+
+    for result in results.values():
+        if "task" in result and not any(result["task"].startswith(task) for task in accepted_tasks):
+            continue
+
+        # Clean predicted string if necessary
+        result["predicted"] = result["predicted"].lower()
+        result["ground_truth"] = result["ground_truth"].lower()
+        if ignore_punctuation:
+            result["predicted"] = ''.join(ch for ch in result["predicted"] if ch not in string.punctuation)
+            result["ground_truth"] = ''.join(ch for ch in result["ground_truth"] if ch not in string.punctuation)
+        if verbose:
+            values["predicted"].append(result["predicted"])
+            values["ground_truth"].append(result["ground_truth"])
+            values["correct_incorrect"].append("Correct" if result["predicted"] == result["ground_truth"] else "Incorrect")
+        if "task" not in result:
+            result["task"] = dataset
+
+        # True positive
+        if result["predicted"] == result["ground_truth"]:
+            task_tp[result["task"]] += 1
+        task_total[result["task"]] += 1
+
+        # If binary classification (yes/no question), calculate precision and recall metrics
+        binary_classification[result["task"]] = binary_classification[result["task"]] or (result["ground_truth"] in ["yes", "no"])
+        if binary_classification[result["task"]]:
+            if result["predicted"] != "no" and result["ground_truth"] == "no":
+                binary_fp[result["task"]] += 1
+            if result["predicted"] != "yes" and result["ground_truth"] == "yes": 
+                binary_fn[result["task"]] += 1
+
+        # Update ground truth counts for the task
+        task = result["task"]
+        class_label = result["ground_truth"]
+        ground_truths[task][class_label] = ground_truths[task].get(class_label, 0) + 1
+    
+    # Print tab separated values
+    if verbose:
+        max_len = max(len(v) for v in values["ground_truth"]) + 5
+        print("Predicted" + " " * (max_len - 9) + "\tGround Truth" + " " * (max_len - 12) + "\tCorrect/Incorrect")
+        for i in range(len(values["predicted"])):
+            print(values["predicted"][i] + " " * (max_len - len(values["predicted"][i])) + "\t" + values["ground_truth"][i] + " " * (max_len - len(values["ground_truth"][i])) + "\t" + values["correct_incorrect"][i])
+
+    total_tp = 0
+    total_predictions = 0
+    for task in task_tp:
+        acc_string = "Accuracy"
+        if ignore_punctuation:
+            acc_string += " (ignoring punctuation)"
+        print(f"{acc_string} for {task}: {round((task_tp[task] /  task_total[task]), 4) * 100}%")
+
+        if binary_classification[task]:
+            if (task_tp[task] + binary_fp[task]) > 0:
+                print(f"Precision (ignoring punctuation) for {task}: {round((task_tp[task] / (task_tp[task] + binary_fp[task])), 3) * 100}%")
+            if (task_tp[task] + binary_fn[task]) > 0:
+                print(f"Recall (ignoring punctuation) for {task}: {round((task_tp[task] / (task_tp[task] + binary_fn[task])), 3) * 100}%")
+
+        majority_class = max(ground_truths[task], key=ground_truths[task].get)
+        majority_class_percentage = (ground_truths[task][majority_class] / task_total[task]) * 100
+        print(f"Majority class for {task}: {majority_class}, Percentage: {round(majority_class_percentage, 4)}%")
+
+        total_tp += task_tp[task]
+        total_predictions += task_total[task]
+
+    if total_predictions == 0:
+        print("No predictions made.")
+    else:
+        total_accuracy = (total_tp / total_predictions) * 100
+        print(f"Overall Accuracy: {round(total_accuracy, 3)}%")
+
+# For testing accuracy/precision/recall on a particular script without running inference
+if __name__ == '__main__':
+    root_dir = '/deep/u/jirvin16/aicc/aicc-win24-geo-vlm/videollava/scripts/geovlm/eval/QFabric/answers/'
+    answer_path =  root_dir + "video-llava-7b-8bit-lora-final-no-metadata-zero-gc-acc8-freq-no-geochat-checkpoint-8000_qfabric_test_aux_data_test_prompt_strategy_interleave_chronological_prefix_True_load_8bit_True_load_4bit_False_delete_system_prompt_False.json"
+    accuracy_precision_recall(answer_path, dataset="qfabric", ignore_punctuation=True, verbose=False)

videollava/eval/eval_geochat_referring.py

@@ -0,0 +1,330 @@
+"""
+calc_iou_individual adapted from calculate_mean_ap.py
+author: Timothy C. Arlen
+date: 28 Feb 2018
+"""
+
+import sys
+from os.path import dirname, abspath
+sys.path.append(dirname(dirname(dirname(dirname(abspath(__file__))))))
+
+from collections import defaultdict
+import numpy as np
+import json
+import ast
+import re
+import cv2
+from shapely import wkt, Polygon, box
+from infer_utils import create_mask
+from matplotlib.path import Path
+from tqdm import tqdm
+
+from eval_referring import referring_expression
+import matplotlib.pyplot as plt
+import time
+import math
+from matplotlib.path import Path
+
+def convert_geochat_string(build, img_size=256):
+    """
+    Convert the raw str geochat output {<40><89><56><100>|<57>}, {<0><89><56><100>|<57>}
+    to a list of rotated bboxes.
+    """
+    build = build.strip('{}')
+    bbox_segments = build.split("}{")
+    # Regular expression to find all numbers inside angle brackets
+    pattern = r"<(\d+)>"
+
+    # Extract numbers, convert them to integers, and collect into a list
+    bboxes = [
+        list(map(int, re.findall(pattern, segment)))
+        for segment in bbox_segments
+    ]
+
+    rotated_bboxes = []
+    for bbox in bboxes:
+        try:
+            xmin, ymin, xmax, ymax, angle = [float(v) for v in bbox]
+        except:
+            print("Warning - Malformed bbox: ", bbox)
+            print("Original string: ", build)
+            print()
+            continue
+
+        # Convert percentages to pixel coordinates
+        xmin = xmin * img_size / 100
+        ymin = ymin * img_size / 100
+        xmax = xmax * img_size / 100
+        ymax = ymax * img_size / 100
+        
+        # Calculate rectangle dimensions
+        rect_width = xmax - xmin
+        rect_height = ymax - ymin
+        center_x = xmin + rect_width / 2
+        center_y = ymin + rect_height / 2
+        
+        # Calculate corners before rotation
+        corners = np.array([
+            [xmin, ymin],
+            [xmax, ymin],
+            [xmax, ymax],
+            [xmin, ymax]
+        ])
+        
+        # Rotate corners
+        angle_rad = math.radians(angle)
+        cos_angle = math.cos(angle_rad)
+        sin_angle = math.sin(angle_rad)
+        rotated_corners = []
+        for x, y in corners:
+            tx = x - center_x
+            ty = y - center_y
+            rotated_x = tx * cos_angle - ty * sin_angle + center_x
+            rotated_y = tx * sin_angle + ty * cos_angle + center_y
+            rotated_corners.append([rotated_x, rotated_y])
+
+        rotated_bboxes.append(np.array(rotated_corners))
+
+    return rotated_bboxes
+
+def create_geochat_mask(buildings, img_size=(256, 256)):
+    """
+    Given a list of buildings in an image, this function 
+    - creates an img_size * img_size numpy array for the image
+    - returns the mask for all buildings
+    Input:
+    - buildings: List of geochat strings representing buildings
+    - img_size: Tuple indicating the size of the image (height, width)
+    """
+    mask = np.zeros(img_size, np.uint8)
+
+    # Fill in with ones the pixels that are inside the buildings (rotated bboxes)
+    for bbox in buildings:
+        path = Path(bbox)
+        x, y = np.meshgrid(np.arange(img_size[1]), np.arange(img_size[0]))
+        points = np.vstack((x.flatten(), y.flatten())).T
+        mask[path.contains_points(points).reshape(img_size)] = 1
+
+    return mask
+
+def calc_iou_individual(pred_box, gt_box):
+    """Calculate IoU of single predicted and ground truth box
+    Args:
+        pred_box (list of floats): location of predicted object as
+            [xmin, ymin, xmax, ymax]
+        gt_box (list of floats): location of ground truth object as
+            [xmin, ymin, xmax, ymax]
+    Returns:
+        float: value of the IoU for the two boxes.
+    Raises:
+        AssertionError: if the box is obviously malformed
+    """
+    x1_t, y1_t, x2_t, y2_t = gt_box
+    try:
+        x1_p, y1_p, x2_p, y2_p = pred_box
+    except:
+        return 0.0
+
+    if (x1_p > x2_p) or (y1_p > y2_p):
+        print("Prediction box is malformed? pred box: {}".format(pred_box))
+    if (x1_t > x2_t) or (y1_t > y2_t):
+        print("Ground Truth box is malformed? true box: {}".format(gt_box))
+
+    if (x2_t < x1_p or x2_p < x1_t or y2_t < y1_p or y2_p < y1_t):
+        return 0.0
+
+    far_x = np.min([x2_t, x2_p])
+    near_x = np.max([x1_t, x1_p])
+    far_y = np.min([y2_t, y2_p])
+    near_y = np.max([y1_t, y1_p])
+
+    inter_area = (far_x - near_x + 1) * (far_y - near_y + 1)
+    true_box_area = (x2_t - x1_t + 1) * (y2_t - y1_t + 1)
+    pred_box_area = (x2_p - x1_p + 1) * (y2_p - y1_p + 1)
+    iou = inter_area / (true_box_area + pred_box_area - inter_area)
+
+    return iou  
+
+def get_single_image_bound_results(gt_wkts, pred_geochat_string, img_size=256):
+    """
+    Calculates upper bound and lower bound number of true_pos, false_pos, false_neg from single batch of boxes.
+    Args:
+        gt_wkts (list of strs): list of wkt strings of input polygons, scaled to raw pixel value
+        pred_boxes (list of lists): list of list of boxes, where each box is formatted
+            as [x_min, y_min, x_max, y_max] on scale from 0-100
+        img_size (int): dimensions of the image. defaults to 256. 
+    Returns:
+        tuple of dicts: true positives (int), false positives (int), false negatives (int)
+    """
+    if isinstance(gt_wkts, str):
+        gt_polygons = [wkt.loads(gt_wkts)]
+    else: 
+        gt_polygons = [wkt.loads(gt_wkt) for gt_wkt in gt_wkts]
+
+    lb_preds = convert_geochat_string(pred_geochat_string, img_size)
+    # get mask of all gt_polygons and lb_preds
+    gt_mask = create_mask(gt_polygons, (img_size, img_size))
+    lb_preds_mask = create_geochat_mask(lb_preds, (img_size, img_size))
+
+    # get lower bound intersection and union masks 
+    intersection = np.logical_and(gt_mask, lb_preds_mask)
+    union = np.logical_or(gt_mask, lb_preds_mask)
+
+    # compute lb metrics
+    fp = np.sum(np.logical_and(lb_preds_mask, np.logical_not(gt_mask)))
+    tp = np.sum(np.logical_and(lb_preds_mask, gt_mask))
+    fn = np.sum(np.logical_and(np.logical_not(lb_preds_mask), gt_mask))
+    lb_stats = {'true_pos': tp, 'false_pos': fp, 'false_neg': fn, 'intersection': np.sum(intersection), 'union': np.sum(union)}
+
+    # get upper bound intersection and union masks
+    ub_pred_mask = np.logical_and(gt_mask, lb_preds_mask)
+    intersection = np.logical_and(ub_pred_mask, gt_mask)
+    union = np.logical_or(gt_mask, ub_pred_mask)
+
+    # compute ub metrics
+    ub_fp = np.sum(np.logical_and(ub_pred_mask, np.logical_not(gt_mask)))
+    ub_tp = np.sum(np.logical_and(ub_pred_mask, gt_mask))
+    ub_fn = np.sum(np.logical_and(np.logical_not(ub_pred_mask), gt_mask))
+    ub_stats = {'true_pos': ub_tp, 'false_pos': ub_fp, 'false_neg': ub_fn, 'intersection': np.sum(intersection), 'union': np.sum(union)}
+
+    return lb_stats, ub_stats
+
+def get_geochat_dataset(image_id):
+    if image_id.startswith("P"):
+        dataset = "SOTA"
+    elif image_id.startswith("train"):
+        dataset = "FAST"
+    else:
+        dataset = "SIOR"
+    return dataset
+
+def calc_precision_recall(img_results):
+    """Calculates precision and recall from the set of images
+    Args:
+        img_results (dict): dictionary formatted like:
+            {
+                'img_id1': {'true_pos': int, 'false_pos': int, 'false_neg': int},
+                'img_id2': ...
+                ...
+            }
+    Returns:
+        tuple: of floats of (precision, recall)
+    """
+    true_pos = 0; false_pos = 0; false_neg = 0
+    for _, res in img_results.items():
+        true_pos += res['true_pos']
+        false_pos += res['false_pos']
+        false_neg += res['false_neg']
+
+    try:
+        precision = true_pos/(true_pos + false_pos)
+    except ZeroDivisionError:
+        precision = 0.0
+    try:
+        recall = true_pos/(true_pos + false_neg)
+    except ZeroDivisionError:
+        recall = 0.0
+
+    return (precision, recall)
+
+
+DIMENSIONS = {'FAST': 600,
+              'SIOR': 800,
+              'SOTA': 1024}
+
+
+def referring_expression(answer_path, dataset, verbose=False, saving_path_root=None, img_size=256):
+    # Replace with the path to the answers file
+    if type(answer_path) == dict:
+        results = answer_path
+    else:
+        with open(answer_path) as json_data:
+            results = json.load(json_data)
+
+    img_results = {}
+    ub_results = {}
+    lb_results = {}
+    num_bboxes = 0
+    # Loop over results and get precision, recall overall
+    for id, result in tqdm(results.items()):
+
+        if dataset == "geochat_xbd":
+            pred = result['predicted']
+
+            dataset = get_geochat_dataset(id)
+            img_size = (DIMENSIONS[dataset])
+            pred = convert_geochat_string(pred, img_size)
+
+            ground_truth = result['ground_truth']
+            ground_truth = np.array(ground_truth)
+            num_bboxes += len(ground_truth)
+
+            img_results[id] = get_single_image_results(ground_truth, pred, iou_thr=0.5)
+
+            continue
+
+        try:
+            if 'referring_expression' not in result['task']:
+                continue  # no bounding box outputs for temporal_referring_expression
+        except:
+            pass
+
+        # TODO: clean the following todos
+        
+        # TODO: LOOP THROUGH IDENTIFY TASKS/QUESTIONS IN THE DATASET
+       
+        # TODO: HANDLE WHEN THERE ARE NO BOUNDING BOXES IN GROUND TRUTH for auxiliary tasks
+        if not result['original_input_polygon']:
+            first_open_bracket_ind = result["predicted"].find("{")
+            last_close_bracket_ind = result["predicted"].rfind("}")
+            if last_close_bracket_ind != -1 and first_open_bracket_ind != -1:
+                parsed_predicted = result["predicted"][first_open_bracket_ind:last_close_bracket_ind+1]
+            else:   
+                parsed_predicted = ""      
+            predicted_boxes = convert_geochat_string(parsed_predicted)
+            # If ground truth contains no boxes: all predictions are false positives
+            false_pos = len(predicted_boxes)
+            false_pos_pixels = np.sum(create_geochat_mask(predicted_boxes))
+            img_results[id] = {'true_pos': 0, 'false_pos': false_pos, 'false_neg': 0, 'intersection':0, 'union':false_pos_pixels}
+            ub_results[id] = {'true_pos': 0, 'false_pos': false_pos_pixels, 'false_neg': 0, 'intersection':0, 'union':false_pos_pixels}
+            lb_results[id] = {'true_pos': 0, 'false_pos': false_pos_pixels, 'false_neg': 0, 'intersection':0, 'union':false_pos_pixels}
+            continue
+        else: # Ground truth contains boxes: find predicted Geochat boxes
+            first_open_bracket_ind = result["predicted"].find("{")
+            last_close_bracket_ind = result["predicted"].rfind("}")
+            if last_close_bracket_ind != -1 and first_open_bracket_ind != -1:
+                parsed_predicted = result["predicted"][first_open_bracket_ind:last_close_bracket_ind+1]
+            else:   
+                parsed_predicted = "" 
+            gt_wkts = result['original_input_polygon']  
+            lb_results[id], ub_results[id] = get_single_image_bound_results(gt_wkts, parsed_predicted)
+
+    if len(ub_results) != 0:
+        ub_intersection = np.sum([res['intersection'] for res in ub_results.values()])
+        ub_union = np.sum([res['union'] for res in ub_results.values()])
+        lb_intersection = np.sum([res['intersection'] for res in lb_results.values()])
+        lb_union = np.sum([res['union'] for res in lb_results.values()])
+        print("Upper bound IOU: ", ub_intersection / ub_union if ub_union != 0 else 0)
+        print("Lower bound IOU: ", lb_intersection / lb_union if lb_union != 0 else 0)
+        ub_precision, ub_recall = calc_precision_recall(ub_results)
+        lb_precision, lb_recall = calc_precision_recall(lb_results)
+        print('Lower bound precision: ', lb_precision)
+        print('Lower bound recall: ', lb_recall)
+        print("Upper bound F1: ", 2 * (ub_precision * ub_recall) / (ub_precision + ub_recall) if (ub_precision + ub_recall) != 0 else 0)
+        print("Lower bound F1: ", 2 * (lb_precision * lb_recall) / (lb_precision + lb_recall) if (lb_precision + lb_recall) != 0 else 0)
+
+    print("[email protected]: ", np.sum([res['true_pos'] for res in img_results.values()]) / num_bboxes)
+
+    if type(answer_path) == dict:
+        return 
+    
+    if saving_path_root:
+        with open(f"{saving_path_root}/referring_expression_scores.json", 'w') as f:
+            json.dump(img_results, f)
+
+if __name__ == '__main__':
+    answer_path = "scripts/geovlm/eval/xBD/answers/ckpt14000-geochat-bench_interleave_test.json"
+    referring_expression(answer_path, dataset="geochat_xbd")
+    #answer_path = "scripts/geochat/eval/xBD/geochat_xbd_test_auxiliary_dict.json"
+    # referring_expression(answer_path, dataset="xbd")
+

videollava/eval/eval_referring.py

@@ -0,0 +1,351 @@
+"""
+Code adapted from calculate_mean_ap.py
+author: Timothy C. Arlen
+date: 28 Feb 2018
+"""
+import sys
+sys.path.append('/deep/u/joycech/aicc-working/videollava')
+
+from collections import defaultdict
+import numpy as np
+import json
+import ast
+import re
+import cv2
+from shapely import wkt, Polygon, box
+from infer_utils import create_mask, create_mask_s2looking
+
+
+def calc_iou_individual(pred_box, gt_box):
+    """Calculate IoU of single predicted and ground truth box
+    Args:
+        pred_box (list of floats): location of predicted object as
+            [xmin, ymin, xmax, ymax]
+        gt_box (list of floats): location of ground truth object as
+            [xmin, ymin, xmax, ymax]
+    Returns:
+        float: value of the IoU for the two boxes.
+    Raises:
+        AssertionError: if the box is obviously malformed
+    """
+    x1_t, y1_t, x2_t, y2_t = gt_box
+    try:
+        x1_p, y1_p, x2_p, y2_p = pred_box
+    except:
+        print("Prediction box is malformed? pred box: {}".format(pred_box))
+        return 0.0
+
+    if (x1_p > x2_p) or (y1_p > y2_p):
+        print("Prediction box is malformed? pred box: {}".format(pred_box))
+        return 0.0
+    if (x1_t > x2_t) or (y1_t > y2_t):
+        raise AssertionError(
+            "Ground Truth box is malformed? true box: {}".format(gt_box))
+
+    if (x2_t < x1_p or x2_p < x1_t or y2_t < y1_p or y2_p < y1_t):
+        return 0.0
+
+    far_x = np.min([x2_t, x2_p])
+    near_x = np.max([x1_t, x1_p])
+    far_y = np.min([y2_t, y2_p])
+    near_y = np.max([y1_t, y1_p])
+
+    inter_area = (far_x - near_x + 1) * (far_y - near_y + 1)
+    true_box_area = (x2_t - x1_t + 1) * (y2_t - y1_t + 1)
+    pred_box_area = (x2_p - x1_p + 1) * (y2_p - y1_p + 1)
+    iou = inter_area / (true_box_area + pred_box_area - inter_area)
+
+    return iou
+
+def get_single_image_bound_results(gt_wkts, pred_boxes, img_size=256, dataset=None, id=None, predicted_mask=None, split=None, question=None):
+    """
+    Calculates upper bound and lower bound number of true_pos, false_pos, false_neg from single batch of boxes.
+    Args:
+        gt_wkts (list of strs): list of wkt strings of input polygons, scaled to raw pixel value
+        pred_boxes (list of lists): list of list of boxes, where each box is formatted
+            as [x_min, y_min, x_max, y_max] on scale from 0-100
+        img_size (int): dimensions of the image. defaults to 256. 
+    Returns:
+        tuple of dicts: true positives (int), false positives (int), false negatives (int)
+    """
+    lb_preds = [[num * img_size / 100 for num in box] for box in pred_boxes]
+    # add error handling for this type of outputs:  [0, 10, 12, 22], [0, 6, 12, 19], [0, 0], [31, 0]
+    try:
+        lb_preds = [box(*pred_box) for pred_box in lb_preds]
+    except:
+        lb_preds = []
+        for pred_box in pred_boxes:
+            if len(pred_box) == 4:
+                lb_preds.append(box(*pred_box))
+
+    if isinstance(gt_wkts, str):
+        gt_polygons = [wkt.loads(gt_wkts)]
+    elif gt_wkts is None:
+        gt_polygons = []
+    else: 
+        gt_polygons = [wkt.loads(gt_wkt) for gt_wkt in gt_wkts]
+
+    # get mask of all gt_polygons and lb_preds
+    if dataset == None:
+        gt_mask = create_mask(gt_polygons, (img_size, img_size))
+    else:
+        gt_mask = create_mask_s2looking(id, split=split, question=question) 
+        #gt_mask = create_mask(gt_polygons, (img_size, img_size))
+
+    if dataset != "geochat_s2looking":
+        lb_preds_mask = create_mask(lb_preds, (img_size, img_size))
+    else:
+        lb_preds_mask = predicted_mask
+        
+
+    # get lower bound intersection and union masks 
+    intersection = np.logical_and(gt_mask, lb_preds_mask)
+    union = np.logical_or(gt_mask, lb_preds_mask)
+
+    # compute lb metrics
+    lower_bound_iou = np.sum(intersection) / np.sum(union)
+    if np.sum(intersection) == 0 and np.sum(union) == 0:
+        return None, None
+    if np.isnan(lower_bound_iou):
+        lower_bound_iou = 0
+
+        
+    fp = np.sum(np.logical_and(lb_preds_mask, np.logical_not(gt_mask)))
+    tp = np.sum(np.logical_and(lb_preds_mask, gt_mask))
+    fn = np.sum(np.logical_and(np.logical_not(lb_preds_mask), gt_mask))
+    lb_stats = {'true_pos': tp, 
+                'false_pos': fp, 
+                'false_neg': fn, 
+                'intersection': np.sum(intersection),
+                'union': np.sum(union)}
+
+    return lb_stats
+
+def get_single_image_results(gt_boxes, pred_boxes, iou_thr):
+    """Calculates number of true_pos, false_pos, false_neg from single batch of boxes.
+    Args:
+        gt_boxes (list of list of floats): list of locations of ground truth
+            objects as [xmin, ymin, xmax, ymax]
+        pred_boxes (dict): dict of dicts of 'boxes' (formatted like `gt_boxes`)
+            and 'scores'
+        iou_thr (float): value of IoU to consider as threshold for a
+            true prediction.
+    Returns:
+        dict: true positives (int), false positives (int), false negatives (int)
+    """
+
+    all_pred_indices = range(len(pred_boxes))
+    all_gt_indices = range(len(gt_boxes))
+    if len(all_pred_indices) == 0:
+        tp = 0
+        fp = 0
+        fn = len(gt_boxes)
+        return {'true_pos': tp, 'false_pos': fp, 'false_neg': fn}
+    if len(all_gt_indices) == 0:
+        tp = 0
+        fp = len(pred_boxes)
+        fn = 0
+        return {'true_pos': tp, 'false_pos': fp, 'false_neg': fn}
+
+    gt_idx_thr = []
+    pred_idx_thr = []
+    ious = []
+    for ipb, pred_box in enumerate(pred_boxes):
+        for igb, gt_box in enumerate(gt_boxes):
+            iou = calc_iou_individual(pred_box, gt_box)
+            if iou > iou_thr:
+                gt_idx_thr.append(igb)
+                pred_idx_thr.append(ipb)
+                ious.append(iou)
+
+    args_desc = np.argsort(ious)[::-1]
+    if len(args_desc) == 0:
+        # No matches
+        tp = 0
+        fp = len(pred_boxes)
+        fn = len(gt_boxes)
+    else:
+        gt_match_idx = []
+        pred_match_idx = []
+        for idx in args_desc:
+            gt_idx = gt_idx_thr[idx]
+            pr_idx = pred_idx_thr[idx]
+            # If the boxes are unmatched, add them to matches
+            if (gt_idx not in gt_match_idx) and (pr_idx not in pred_match_idx):
+                gt_match_idx.append(gt_idx)
+                pred_match_idx.append(pr_idx)
+        tp = len(gt_match_idx)
+        fp = len(pred_boxes) - len(pred_match_idx)
+        fn = len(gt_boxes) - len(gt_match_idx)
+
+    return {'true_pos': tp, 'false_pos': fp, 'false_neg': fn}
+
+def calc_precision_recall(img_results):
+    """Calculates precision and recall from the set of images
+    Args:
+        img_results (dict): dictionary formatted like:
+            {
+                'img_id1': {'true_pos': int, 'false_pos': int, 'false_neg': int},
+                'img_id2': ...
+                ...
+            }
+    Returns:
+        tuple: of floats of (precision, recall)
+    """
+    true_pos = 0; false_pos = 0; false_neg = 0
+    for _, res in img_results.items():
+        true_pos += res['true_pos']
+        false_pos += res['false_pos']
+        false_neg += res['false_neg']
+
+    try:
+        precision = true_pos/(true_pos + false_pos)
+    except ZeroDivisionError:
+        precision = 0.0
+        print(true_pos, "true_pos", false_pos, "false_pos", false_neg, "false_neg")
+    try:
+        recall = true_pos/(true_pos + false_neg)
+    except ZeroDivisionError:
+        recall = 0.0
+
+    return (precision, recall)
+
+def extract_bboxes(input_string):
+    """
+    Takes as an input a string like in the image, there are two buildings that have been changed. the first building is located at [0.0, 0.69, 0.45, 0.9] and the second building is located at [0.46, 0.69, 0.99, 0.91]
+    Returns a list of bounding boxes in the format [x_min, y_min, x_max, y_max]
+    Input:
+        input_string (str): string containing the bounding boxes
+    Returns:
+        list of lists: list of bounding boxes
+    """
+    matches = re.findall(r'\[\[.*?\]\]', input_string)
+    return [ast.literal_eval(match) for match in matches]
+
+
+def referring_expression(answer_path, dataset, verbose=False, saving_path_root=None, img_size=256, split=None):
+    if type(answer_path) == dict:
+        results = answer_path
+    else:
+        with open(answer_path) as json_data:
+            results = json.load(json_data)
+
+    img_results = {}
+    lb_results = {}
+    # Loop over results and get precision, recall overall
+    for id, result in results.items():
+        if 'temporal_referring_expression' in result['task']:
+            if not "s2looking" in dataset:
+                continue  # no bounding box outputs for temporal_referring_expression
+        
+        # for the geochat s2looking predictions, we work directly with the predicted mask instead of the bounding boxes
+        if dataset == 'geochat_s2looking': 
+            if 'referring_expression' in result['task'] or 'localization' in result['task']:
+                lb_res = get_single_image_bound_results(result['original_input_polygon'], [], dataset=dataset, id=id, predicted_mask=result['predicted_mask'], split=split, question=result["question"])
+                if lb_res != None:
+                    lb_results[id] = lb_res
+                continue
+            elif 'question_answering' in result['task']:
+                continue
+
+        if 'referring_expression' in result['task'] or 'largest building' in result['task'] or "canonical" in result['task'] or 'localization' in result['task'] \
+            or 'geochat_referring' in result['task']:
+            # No bounding boxes in predicted string
+            if "[" not in result["predicted"]:
+                # Ground truth has no bounding boxes
+                if result["ground_truth"].startswith("There are no") or "no" in result["ground_truth"] or "No" in result["ground_truth"]:
+                    # Discard true negatives
+                    continue
+                # Ground truth has bounding boxes, not identified by the model --> all false negatives
+                else:
+                    false_neg = "[" + result["ground_truth"] + "]"
+                    false_neg = false_neg.replace(".", "")
+                    
+                    try:
+                        false_neg = len(ast.literal_eval(false_neg))
+                    except:
+                        # count the number of opening '[' in the string
+                        false_neg = false_neg.count('[') - 1
+                    if not "s2looking" in dataset:
+                        gt_mask = create_mask(wkt.loads(result['original_input_polygon']), (img_size, img_size)) 
+                    else:
+                        gt_mask = create_mask_s2looking(id, split=split, question=result['question']) 
+                        # gt_mask = create_mask(wkt.loads(result['original_input_polygon']), (img_size, img_size)) 
+                    img_results[id] = {'true_pos': 0, 'false_pos': 0, 'false_neg': false_neg, 'intersection':0, 'union':false_neg}
+                    false_neg = np.sum(gt_mask)
+                    lb_results[id] = {'true_pos': 0, 'false_pos': 0, 'false_neg': false_neg, 'intersection':0, 'union':false_neg}
+
+            # Bounding boxes in predicted and output string --> compare bounding boxes
+            else:
+
+                # To deal with cases where the model outputs an incomplete bounding box (e.g. "[24, 76,")
+                first_open_bracket_ind = result["predicted"].find("[")
+                last_close_bracket_ind = result["predicted"].rfind("]")
+                if last_close_bracket_ind != -1 and first_open_bracket_ind != -1:
+                    parsed_predicted = result["predicted"][first_open_bracket_ind:last_close_bracket_ind+1]
+                else:   
+                    parsed_predicted = ""      
+
+                # Load list of predicted bounding boxes
+                try:
+                    predicted_boxes = ast.literal_eval("[" + parsed_predicted + "]")
+                except:
+                    match = re.search(r'\[\[.*\]\]', result["predicted"])
+                    if match:
+                        predicted_boxes = ast.literal_eval(match.group())
+                    else:
+                        predicted_boxes = []
+
+                predicted_boxes = [[coord * 100 if coord < 1 else coord for coord in box] for box in predicted_boxes]
+
+                # Load list of ground truth bounding boxes
+                if result["ground_truth"].startswith("There are no") or "no" in result["ground_truth"].lower():
+                    # If ground truth contains no boxes
+                    ground_truth_boxes = []
+                first_open_bracket_ind = result["ground_truth"].find("[")
+                last_close_bracket_ind = result["ground_truth"].rfind("]")
+                if last_close_bracket_ind != -1 and first_open_bracket_ind != -1:
+                    parsed_gt = result["ground_truth"][first_open_bracket_ind:last_close_bracket_ind+1]
+                else:   
+                    parsed_gt = ""      
+                try: 
+                    ground_truth_boxes = ast.literal_eval("[" + parsed_gt + "]")
+                except: 
+                    match = re.search(r'\[\[.*\]\]', result["ground_truth"])
+                    if match:
+                        ground_truth_boxes = ast.literal_eval(match.group())
+                    else:
+                        ground_truth_boxes = []
+
+                # Get mask results from the two previous parsings
+                gt_wkts = result['original_input_polygon']
+                img_results[id] = get_single_image_results(ground_truth_boxes, predicted_boxes, iou_thr=0.5) ######
+
+                if 'referring_expression' in result['task'] or 'largest building' in result['task'] or "canonical" in result['task'] or 'localization' in result['task']:
+                    if not "s2looking" in dataset:
+                        lb_results[id] = get_single_image_bound_results(gt_wkts, predicted_boxes) 
+                    elif dataset=="s2looking":
+                        lb_results[id] = get_single_image_bound_results(gt_wkts, predicted_boxes, dataset=dataset, id=id, split=split, question=result["question"])
+                    else:
+                        lb_results[id] = get_single_image_bound_results(gt_wkts, predicted_boxes, predicted_mask=result['predicted_mask'], split=split, question=result["question"])
+
+    precision, recall = calc_precision_recall(img_results)
+    print("Referring expression results (precision, recall): ", precision, recall)
+    print("[email protected]: ", np.sum([res['true_pos'] for res in img_results.values()]) / len(results.keys()))
+
+    if len(lb_results) != 0:
+        lb_intersection = np.sum([res['intersection'] for res in lb_results.values()])
+        lb_union = np.sum([res['union'] for res in lb_results.values()])
+        print("Lower bound IOU: ", lb_intersection / lb_union if lb_union != 0 else 0)
+        lb_precision, lb_recall = calc_precision_recall(lb_results)
+        print('Lower bound precision: ', lb_precision)
+        print('Lower bound recall: ', lb_recall)
+        print("Lower bound F1: ", 2 * (lb_precision * lb_recall) / (lb_precision + lb_recall) if (lb_precision + lb_recall) != 0 else 0)
+
+    if saving_path_root:
+        with open(f"{saving_path_root}/referring_expression_scores.json", 'w') as f:
+            json.dump(img_results, f)
+
+if __name__ == '__main__':
+    answer_path = "scripts/geovlm/eval/xBD/answers/ckpt14000-old-aux-xbd-test-canon-auxiliary_interleave.json"
+    referring_expression(answer_path, dataset="xbd")

videollava/eval/geochat_bench.py

@@ -0,0 +1,228 @@
+
+
+from eval_geochat_referring import get_single_image_results, convert_geochat_string
+
+from collections import defaultdict
+import numpy as np
+import json
+import ast
+import re
+import cv2
+from shapely import wkt, Polygon, box
+from infer_utils import create_mask
+from matplotlib.path import Path
+from tqdm import tqdm
+import matplotlib.pyplot as plt
+import time
+import math
+from matplotlib.path import Path
+
+
+
+DIMENSIONS = {'FAST': 600,
+              'SIOR': 800,
+              'SOTA': 1024}
+
+def calc_iou_individual_rotated(pred_box, gt_box, img_size=None):
+    """Calculate IoU of single predicted and ground truth box
+    Args:
+        pred_box (list of floats): location of predicted object as
+            [[x1, y1], [x2, y2], [x3, y3], [x4, y4]]
+        gt_box (list of floats): location of ground truth object as
+            [[x1, y1], [x2, y2], [x3, y3], [x4, y4]]
+    Returns:
+        float: value of the IoU for the two boxes.
+    Raises:
+        AssertionError: if the box is obviously malformed
+    """
+
+    pred_box = np.array(pred_box)
+    gt_box = np.array(gt_box)
+    pred_box = pred_box.reshape(4, 2)
+    gt_box = gt_box.reshape(4, 2)
+    pred_polygon = Polygon(pred_box)
+    gt_polygon = Polygon(gt_box)
+    intersection = pred_polygon.intersection(gt_polygon).area
+    union = pred_polygon.union(gt_polygon).area
+    iou = intersection / union
+
+    return iou
+
+
+def get_single_image_results_rotated(gt_boxes, pred_boxes, iou_thr, img_size=None):
+    """Calculates number of true_pos, false_pos, false_neg from single batch of boxes.
+    Args:
+        gt_boxes (list of list of floats): list of locations of ground truth
+            objects as [[x1,y1], [x2,y2], ...]
+        pred_boxes (dict): dict of dicts of 'boxes' 
+            [[x1,y1], [x2,y2], ...]
+        iou_thr (float): value of IoU to consider as threshold for a
+            true prediction.
+    Returns:
+        dict: true positives (int), false positives (int), false negatives (int)
+    """
+
+    all_pred_indices = range(len(pred_boxes))
+    all_gt_indices = range(len(gt_boxes))
+    if len(all_pred_indices) == 0:
+        tp = 0
+        fp = 0
+        fn = len(gt_boxes)
+        return {'true_pos': tp, 'false_pos': fp, 'false_neg': fn}
+    if len(all_gt_indices) == 0:
+        tp = 0
+        fp = len(pred_boxes)
+        fn = 0
+        return {'true_pos': tp, 'false_pos': fp, 'false_neg': fn}
+
+    gt_idx_thr = []
+    pred_idx_thr = []
+    ious = []
+    for ipb, pred_box in enumerate(pred_boxes):
+        for igb, gt_box in enumerate(gt_boxes):
+            iou = calc_iou_individual_rotated(pred_box, gt_box, img_size)
+            if iou > iou_thr:
+                gt_idx_thr.append(igb)
+                pred_idx_thr.append(ipb)
+                ious.append(iou)
+
+    args_desc = np.argsort(ious)[::-1]
+    if len(args_desc) == 0:
+        # No matches
+        tp = 0
+        fp = len(pred_boxes)
+        fn = len(gt_boxes)
+    else:
+        gt_match_idx = []
+        pred_match_idx = []
+        for idx in args_desc:
+            gt_idx = gt_idx_thr[idx]
+            pr_idx = pred_idx_thr[idx]
+            # If the boxes are unmatched, add them to matches
+            if (gt_idx not in gt_match_idx) and (pr_idx not in pred_match_idx):
+                gt_match_idx.append(gt_idx)
+                pred_match_idx.append(pr_idx)
+        tp = len(gt_match_idx)
+        fp = len(pred_boxes) - len(pred_match_idx)
+        fn = len(gt_boxes) - len(gt_match_idx)
+
+    return {'true_pos': tp, 'false_pos': fp, 'false_neg': fn}
+
+
+def accuracy0_5(answer_path, dataset, aux_dataset="scripts/geochat_bench_dict.json"):
+    # Replace with the path to the answers file
+    results = None
+    if dataset != "geochat_xbd":
+
+        if type(answer_path) == dict:
+            results = answer_path
+        else:
+            results = []
+            with open(answer_path) as json_data:
+                for line in json_data:
+                    results.append(json.loads(line))
+
+    with open(aux_dataset) as json_data:
+        aux_results = json.load(json_data)
+
+    img_results = {}
+    num_bboxes = 0
+
+    if dataset != "geochat_xbd":
+        print("Number of images in Geochat: ", len(aux_results))
+        print("Number of images predicted: ", len(results))
+    
+    i = 0
+    # Loop over results and get precision, recall overall
+    for id, result in tqdm(aux_results.items()):
+
+        if dataset == "geochat_xbd":
+            pred = result['answer']
+
+            img_size = DIMENSIONS[result['dataset']]
+            pred = convert_geochat_string(pred, img_size)
+
+            ground_truth = result['ground_truth']
+            ground_truth = np.array(ground_truth)
+            num_bboxes += len(ground_truth)
+
+            img_results[id] = get_single_image_results_rotated(ground_truth, pred, iou_thr=0.5)
+
+        else:
+
+            geochat_id = id.split(".")[0]
+
+            img_size = DIMENSIONS[aux_results[geochat_id]['dataset']]
+            ground_truth = result['ground_truth']
+            ground_truth = np.array(ground_truth)
+            num_bboxes += len(ground_truth)
+
+            parsed_predicted = results[i]['predicted']
+            # Load list of predicted and round truth bounding boxes for a single image
+            try:
+                predicted_boxes = ast.literal_eval("[" + parsed_predicted + "]")
+            except:
+                match = re.search(r'\[\[.*\]\]', parsed_predicted)
+                if match:
+                    predicted_boxes = ast.literal_eval(match.group())
+                else:
+                    predicted_boxes = []
+
+            predicted_boxes = [[coord * 100 if coord < 1 else coord for coord in box] for box in predicted_boxes]
+            
+            # scale by img_size
+            predicted_boxes = [[coord * img_size / 100 for coord in box] for box in predicted_boxes]
+
+            assert results[i]['ground_truth'] == result['ground_truth']
+
+            # convert the pred bboxes [xmin, ymin, xmax, ymax] to [[x1, y1], [x2, y2], [x3, y3], [x4, y4]]
+            pred_bboxes = []
+            for bbox in predicted_boxes:
+                x1, y1, x2, y2 = bbox
+                pred_bboxes.append([[x1, y1], [x2, y1], [x2, y2], [x1, y2]])
+
+            img_results[id] = get_single_image_results_rotated(ground_truth, pred_bboxes, iou_thr=0.5, img_size=img_size)
+
+        i+=1
+
+
+    acc = np.sum([res['true_pos'] for res in img_results.values()]) / num_bboxes
+    print("[email protected]: ", acc)
+    return acc
+
+
+
+if __name__ == '__main__':
+    print("Geochat bench")
+    geochat_path =  "scripts/geochat_bench_dict.json"
+    answer_path = "scripts/geochat_bench_dict.json"
+    acc_geochat = accuracy0_5(answer_path, dataset="geochat_xbd")
+    print()
+
+
+    print("Teochat bench")
+    answer_path = "/deep/u/idormoy/aicc-win24-geo-vlm/videollava/scripts/geovlm/eval/QFabric/answers/geochat-referring-checkpoint14000_prompt_strategy_interleave_chronological_prefix_True_load_8bit_True_load_4bit_False_delete_system_prompt_False_tmp_0_end.json"
+    acc_teochat = accuracy0_5(answer_path, dataset="geochat")
+    print()
+
+
+    print("Teochat-T bench")
+    answer_path = "/deep/u/idormoy/aicc-win24-geo-vlm/videollava/videollava/eval/video/geochat-bench-ckpt8000-FIXED_prompt_strategy_interleave_chronological_prefix_True_load_8bit_False_load_4bit_True_delete_system_prompt_False_tmp_0_end (1).json"
+    acc_teochatT = accuracy0_5(answer_path, dataset="geochat")
+    print()
+
+
+
+    print("VideoLLaVA bench")
+    answer_path = "/deep/u/idormoy/aicc-win24-geo-vlm/videollava/videollava/eval/video/geochat-referring-Video-LLaVA-7B_prompt_strategy_interleave_chronological_prefix_True_load_8bit_False_load_4bit_True_delete_system_prompt_False_tmp_0_end (1).json"
+    acc_videollava = accuracy0_5(answer_path, dataset="geochat")
+    print()
+
+
+
+    print("Overall accuracies")
+    print("Geochat: ", acc_geochat)
+    print("Teochat: ", acc_teochat)
+    print("Teochat-T: ", acc_teochatT)
+    print("VideoLLaVA: ", acc_videollava)
+

videollava/eval/geochat_eval_fmow.py

@@ -0,0 +1,205 @@
+import argparse
+import torch
+import os
+import json
+from tqdm import tqdm
+import shortuuid
+import sys
+
+sys.path.append('/deep/u/emily712/GeoChat')
+
+from geochat.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
+from geochat.conversation import conv_templates, SeparatorStyle
+from geochat.model.builder import load_pretrained_model
+from geochat.utils import disable_torch_init
+from geochat.mm_utils import tokenizer_image_token, get_model_name_from_path, KeywordsStoppingCriteria
+from eval_classification import *
+
+from PIL import Image
+import math
+import numpy as np
+
+def aggregate_accuracy(answers_file, output_file):
+    """
+    Parses geochat inference output and aggregates votes on single images
+    across an image sequence into the format needed for geovlm-style evaluation. 
+
+    params: 
+        - answers_file: path to the file containing geochat inference output
+        - output_file: path to the file where the aggregated output will be saved
+    """
+    with open(answers_file, 'r') as f:
+        answers = [json.loads(line) for line in f]
+        print(answers)    
+    # dictionary that will contain parsed output
+    votes = {}
+
+    # parse answers so that predictions with the same linked_id
+    # are aggregated into a single item with 'predictions' containing
+    # a list of values. All other keys should be the same
+    for answer in answers:
+        print(answer) 
+        print(answer['linked_id'])
+        id = answer['linked_id']
+        print(id)
+        if id not in votes:
+            item = {}
+            item['predicted'] = [answer['predicted']]
+            item['ground_truth'] = answer['ground_truth']
+            item['task'] = answer['task']
+            item['question'] = answer['question']
+            item['id'] = answer['id']
+            votes[id] = item
+        else: 
+            votes['linked_id']['predicted'].append(answer['predicted'])
+    
+    # implement voting so that each list in 'predicted' attribute
+    # is reduced to the most common value
+    for linked_id, predicted_dict in votes.items():
+        predicted = predicted_dict['predicted']
+        unique, counts = np.unique(predicted, return_counts=True)
+        index = np.argmax(counts)
+        votes[linked_id]['predicted'] = unique[index]
+
+    with open(output_file, 'w') as f:
+        json.dump(votes, f)
+
+    
+def split_list(lst, n):
+    """Split a list into n (roughly) equal-sized chunks"""
+    chunk_size = math.ceil(len(lst) / n)  # integer division
+    return [lst[i:i+chunk_size] for i in range(0, len(lst), chunk_size)]
+
+
+def get_chunk(lst, n, k):
+    chunks = split_list(lst, n)
+    return chunks[k]
+            
+
+def eval_model(args):
+    # Model
+    disable_torch_init()
+    model_path = os.path.expanduser(args.model_path)
+    model_name = get_model_name_from_path(model_path)
+    tokenizer, model, image_processor, context_len = load_pretrained_model(args.model_path, args.model_base, model_name, cache_dir=args.cache_dir)
+    
+    with open(args.question_file, 'r') as f:
+        questions = json.load(f)
+    #questions = [json.loads(q) for q in open(os.path.expanduser(args.question_file), "r")]
+
+    questions = get_chunk(questions, args.num_chunks, args.chunk_idx)
+    answers_file = os.path.expanduser(args.answers_file)
+    os.makedirs(os.path.dirname(answers_file), exist_ok=True)
+    
+    ans_file = open(answers_file, "w")
+
+    skipped_count = 0   
+ 
+    for i in tqdm(range(0,len(questions),args.batch_size)):
+        input_batch=[]
+        input_image_batch=[]
+        count=i
+        image_folder=[]     
+        batch_end = min(i + args.batch_size, len(questions))
+
+        for j in range(i,batch_end):
+            if 'image' not in questions[j]:
+                print(f"Skipped entry [{skipped_count}]")
+                skipped_count += 1
+                continue
+
+            print(questions[j])
+            image_file=questions[j]['image']
+            qs=questions[j]['conversations'][0]['value']
+            
+            if model.config.mm_use_im_start_end:
+                qs = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + '\n' + qs
+            else:
+                qs = DEFAULT_IMAGE_TOKEN + '\n' + qs
+
+            conv = conv_templates[args.conv_mode].copy()
+            conv.append_message(conv.roles[0], qs)
+            conv.append_message(conv.roles[1], None)
+            prompt = conv.get_prompt()
+
+            input_ids = tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).cuda()
+            input_batch.append(input_ids)
+
+            image = Image.open(os.path.join(args.image_folder, image_file))
+
+            image_folder.append(image)
+
+            stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
+            keywords = [stop_str]
+            stopping_criteria = KeywordsStoppingCriteria(keywords, tokenizer, input_ids)
+
+        if len(input_batch) == 0:
+            print("All images here were skipped")
+            continue
+
+        max_length = max(tensor.size(1) for tensor in input_batch)
+
+        final_input_list = [torch.cat((torch.zeros((1,max_length - tensor.size(1)), dtype=tensor.dtype,device=tensor.get_device()), tensor),dim=1) for tensor in input_batch]
+        final_input_tensors=torch.cat(final_input_list,dim=0)
+        image_tensor_batch = image_processor.preprocess(image_folder,crop_size ={'height': 504, 'width': 504},size = {'shortest_edge': 504}, return_tensors='pt')['pixel_values']
+
+        with torch.inference_mode():
+            output_ids = model.generate( final_input_tensors, images=image_tensor_batch.half().cuda(), do_sample=False , temperature=args.temperature, top_p=args.top_p, num_beams=1, max_new_tokens=256,length_penalty=2.0, use_cache=True)
+
+        input_token_len = final_input_tensors.shape[1]
+        n_diff_input_output = (final_input_tensors != output_ids[:, :input_token_len]).sum().item()
+        if n_diff_input_output > 0:
+            print(f'[Warning] {n_diff_input_output} output_ids are not the same as the input_ids')
+        outputs = tokenizer.batch_decode(output_ids[:, input_token_len:], skip_special_tokens=True)
+        for k in range(0,len(final_input_list)):
+            output = outputs[k].strip()
+            if output.endswith(stop_str):
+                output = output[:-len(stop_str)]
+            output = output.strip()
+
+            ans_id = shortuuid.uuid()
+            
+            ans_file.write(json.dumps({
+                                    "id": questions[count]["id"],
+                                    "image_id": questions[count]["image"],
+                                    "question": questions[count]['conversations'][0]['value'],
+                                    "predicted": output,
+                                    "ground_truth": questions[count]['conversations'][1]['value'],
+                                    "task": questions[count]['task'],
+                                    "linked_id": questions[count]['linked_id']
+                                    }) + "\n")
+            count=count+1
+            ans_file.flush()
+    ans_file.close()
+
+    output = [json.loads(q) for q in open((ans_file), "r")]
+    output = [{q['id']: q} for q in output]
+    with open(ans_file, 'r') as f:
+        json.dump(output, f)
+    
+    agg_ans_file = ans_file.replace('.jsonl', '_agg.jsonl')
+    print("Raw Geochat output saved to ", ans_file)
+    print("Now parsing and aggregating votes for geovlm evaluation...")
+    aggregate_accuracy(ans_file, agg_ans_file)
+    print("Aggregated output saved to ", agg_ans_file)
+
+    accuracy_precision_recall(agg_ans_file, 'fmow')
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--model-path", type=str, default="facebook/opt-350m")
+    parser.add_argument("--model-base", type=str, default=None)
+    parser.add_argument("--image-folder", type=str, default="")
+    parser.add_argument("--question-file", type=str, default="tables/question.jsonl")
+    parser.add_argument("--answers-file", type=str, default="answer.jsonl")
+    parser.add_argument("--conv-mode", type=str, default="llava_v1")
+    parser.add_argument("--num-chunks", type=int, default=1)
+    parser.add_argument("--chunk-idx", type=int, default=0)
+    parser.add_argument("--temperature", type=float, default=0.2)
+    parser.add_argument("--top_p", type=float, default=None)
+    parser.add_argument("--num_beams", type=int, default=1)
+    parser.add_argument("--batch_size",type=int, default=1)
+    parser.add_argument("--cache-dir", type=str, default=None)
+    args = parser.parse_args()
+
+    eval_model(args)

videollava/eval/geochat_geovlm_infer.py

@@ -0,0 +1,262 @@
+import argparse
+import torch
+import os
+import json
+from tqdm import tqdm
+import shortuuid
+import sys
+import random
+
+from geochat.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
+from geochat.conversation import conv_templates, SeparatorStyle
+from geochat.model.builder import load_pretrained_model
+from geochat.utils import disable_torch_init
+from geochat.mm_utils import tokenizer_image_token, get_model_name_from_path, KeywordsStoppingCriteria
+from eval_classification import *
+from datasets_into_geochat_format import s2looking_to_geochat_dataset_format, qfabric_semiconverted_to_geochat_dataset_format, xbd_to_geochat_dataset_format
+from geochat_s2looking_utils import evaluate_geochat_s2looking
+
+from PIL import Image
+import math
+import numpy as np
+
+def aggregate_accuracy(answers_file, output_file):
+    """
+    Parses geochat inference output and aggregates votes on single images
+    across an image sequence into the format needed for geovlm-style evaluation. 
+
+    params: 
+        - answers_file: path to the file containing geochat inference output
+        - output_file: path to the file where the aggregated output will be saved
+    """
+    with open(answers_file, 'r') as f:
+        answers = [json.loads(line) for line in f]
+    
+    # dictionary that will contain parsed output
+    votes = {}
+
+    # parse answers so that predictions with the same geovlm_id
+    # are aggregated into a single item with 'predictions' containing
+    # a list of values. All other keys should be the same
+    for answer in answers: 
+        id = answer['geovlm_id']
+        if id not in votes:
+            item = {}
+            item['predicted'] = [answer['predicted']]
+            item['ground_truth'] = answer['ground_truth']
+            item['task'] = answer['task']
+            item['original_input_polygon'] = answer['original_input_polygon']
+            item['question'] = answer['question']
+            item['id'] = answer['id']
+            votes[id] = item
+        else: 
+            votes[id]['predicted'].append(answer['predicted'])
+    
+    # implement voting so that each list in 'predicted' attribute
+    # is reduced to the most common value
+    for linked_id, predicted_dict in votes.items():
+        predicted = predicted_dict['predicted']
+        unique, counts = np.unique(predicted, return_counts=True)
+        index = np.argmax(counts)
+        votes[linked_id]['predicted'] = unique[index]
+
+    with open(output_file, 'w') as f:
+        json.dump(votes, f)
+
+    
+def split_list(lst, n):
+    """Split a list into n (roughly) equal-sized chunks"""
+    chunk_size = math.ceil(len(lst) / n)  # integer division
+    return [lst[i:i+chunk_size] for i in range(0, len(lst), chunk_size)]
+
+
+def get_chunk(lst, n, k):
+    chunks = split_list(lst, n)
+    return chunks[k]
+
+
+def eval_model(args):
+    print(args)
+    print()
+    
+    answers_file = os.path.expanduser(args.answers_file)
+    os.makedirs(os.path.dirname(answers_file), exist_ok=True)
+
+    try: 
+        with open(args.question_file, 'r') as f:
+            questions = json.load(f)    
+    except: 
+        questions = [json.loads(q) for q in open(os.path.expanduser(args.question_file), "r")]
+
+    if args.end_ind is not None:
+        questions = questions[args.start_ind:args.end_ind]
+    else:
+        questions = questions[args.start_ind:]
+    print("start ind: ", args.start_ind)
+    print("end ind: ", args.end_ind)
+
+    # check if the answers file alreay exists
+    if not os.path.exists(answers_file) or args.rerun==True:
+        print('Running inference...')
+        image = Image.open(image_file)
+
+        if args.dataset_size:
+            # randomly sample dataset_size number of questions
+            questions = random.sample(questions, args.dataset_size)
+
+        os.makedirs(os.path.dirname(answers_file), exist_ok=True)
+        ans_file = open(answers_file, "w")
+
+
+        # Model
+        disable_torch_init()
+        model_path = os.path.expanduser(args.model_path)
+        model_name = get_model_name_from_path(model_path)
+        tokenizer, model, image_processor, context_len = load_pretrained_model(args.model_path, args.model_base, model_name, cache_dir=args.cache_dir)
+        
+        for i in tqdm(range(0,len(questions),args.batch_size)):
+            input_batch=[]
+            input_image_batch=[]
+            count=i
+            image_folder=[]     
+            batch_end = min(i + args.batch_size, len(questions))
+
+            for j in range(i,batch_end):
+                image_file=questions[j]['image']
+                qs=questions[j]['conversations'][0]['value']
+
+                # TODO do we keep that?
+                
+                # if model.config.mm_use_im_start_end:
+                #     qs = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + '\n' + qs
+                #     print("start end token")
+                # else:
+                #     qs = DEFAULT_IMAGE_TOKEN + '\n' + qs
+
+                conv = conv_templates[args.conv_mode].copy()
+                conv.append_message(conv.roles[0], qs)
+                conv.append_message(conv.roles[1], None)
+                prompt = conv.get_prompt()
+
+                print(prompt)
+
+                input_ids = tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).cuda()
+                input_batch.append(input_ids)
+
+                image = Image.open(os.path.join(args.image_folder, image_file))
+
+                image_folder.append(image)
+
+                stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
+                keywords = [stop_str]
+                stopping_criteria = KeywordsStoppingCriteria(keywords, tokenizer, input_ids)
+
+            max_length = max(tensor.size(1) for tensor in input_batch)
+
+            final_input_list = [torch.cat((torch.zeros((1,max_length - tensor.size(1)), dtype=tensor.dtype,device=tensor.get_device()), tensor),dim=1) for tensor in input_batch]
+            final_input_tensors=torch.cat(final_input_list,dim=0)
+            image_tensor_batch = image_processor.preprocess(image_folder,crop_size ={'height': 504, 'width': 504},size = {'shortest_edge': 504}, return_tensors='pt')['pixel_values']    
+
+            with torch.inference_mode():
+                output_ids = model.generate( final_input_tensors, images=image_tensor_batch.half().cuda(), do_sample=False , temperature=args.temperature, top_p=args.top_p, num_beams=1, max_new_tokens=256,length_penalty=2.0, use_cache=True)
+
+            input_token_len = final_input_tensors.shape[1]
+            n_diff_input_output = (final_input_tensors != output_ids[:, :input_token_len]).sum().item()
+            if n_diff_input_output > 0:
+                print(f'[Warning] {n_diff_input_output} output_ids are not the same as the input_ids')
+            outputs = tokenizer.batch_decode(output_ids[:, input_token_len:], skip_special_tokens=True)
+            for k in range(0,len(final_input_list)):
+                output = outputs[k].strip()
+                if output.endswith(stop_str):
+                    output = output[:-len(stop_str)]
+                output = output.strip()
+
+                ans_id = shortuuid.uuid()
+                
+                if args.dataset == 'qfabric':
+                    ans_file.write(json.dumps({
+                                            "id": questions[count]["id"],
+                                            "image_id": questions[count]["image"],
+                                            "question": questions[count]['conversations'][0]['value'],
+                                            "predicted": output,
+                                            "ground_truth": questions[count]['conversations'][1]['value'],
+                                            "task": questions[count]['task'],
+                                            "original_input_polygon": questions[count]['original_input_polygon'],
+                                            "geovlm_id": questions[count]['geovlm_id']
+                                            }) + "\n")
+                elif args.dataset == 's2looking':
+                    ans_file.write(json.dumps({
+                                            questions[count]["id"] : {
+                                            "image_id": questions[count]["image"],
+                                            "question": questions[count]['conversations'][0]['value'],
+                                            "predicted": output,
+                                            "task": questions[count]['task'],
+                                            "original_input_polygon": questions[count]['original_input_polygon'],
+                                            "geovlm_id": questions[count]['geovlm_id'],
+                                            "original_question": questions[count]['conversations'][0]['value'],
+                                            "original_answer": questions[count]['conversations'][1]['value']
+                                            }}) + "\n")
+                elif args.dataset == 'xbd':
+                    ans_file.write(json.dumps({
+                                            questions[count]["id"] : {
+                                            "image_id": questions[count]["image"],
+                                            "question": questions[count]['conversations'][0]['value'],
+                                            "predicted": output,
+                                            "task": questions[count]['task'],
+                                            "original_input_polygon": questions[count]['original_input_polygon'],
+                                            "original_question": questions[count]['conversations'][0]['value'],
+                                            "original_answer": questions[count]['conversations'][1]['value']
+                                            }}) + "\n")
+
+                count=count+1
+                ans_file.flush()
+        ans_file.close()
+    
+        agg_ans_file = args.answers_file.replace('.json', '_agg.json')
+        print("Raw Geochat output saved to ", args.answers_file)
+
+    # determine the split from args.question_file
+    if 'test' in args.question_file:
+        split = 'test'
+    elif 'val' or 'valid' or 'validation' in args.question_file:
+        split = 'val'
+    elif 'train' in args.question_file:
+        split = 'train'
+    else:
+        raise ValueError("Split not found in question file name")
+    
+    print("Now parsing and aggregating votes for geovlm evaluation...")
+    if args.dataset == 'qfabric':
+        aggregate_accuracy(args.answers_file, agg_ans_file)
+        print("Aggregated output saved to ", agg_ans_file)
+
+        classification_segmentation(agg_ans_file, 'qfabric')
+    elif args.dataset == 's2looking':
+        evaluate_geochat_s2looking(args.answers_file, args.question_file, split)
+    elif args.dataset == 'xbd':
+        classification_segmentation(agg_ans_file, 'xbd')
+        
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--model-path", type=str, default="facebook/opt-350m")
+    parser.add_argument("--model-base", type=str, default=None)
+    parser.add_argument("--image-folder", type=str, default="")
+    parser.add_argument("--question-file", type=str, default="tables/question.jsonl")
+    parser.add_argument("--answers-file", type=str, default="answer.jsonl")
+    parser.add_argument("--conv-mode", type=str, default="llava_v1")
+    parser.add_argument("--num-chunks", type=int, default=1)
+    parser.add_argument("--chunk-idx", type=int, default=0)
+    parser.add_argument("--temperature", type=float, default=0.2)
+    parser.add_argument("--top_p", type=float, default=None)
+    parser.add_argument("--num_beams", type=int, default=1)
+    parser.add_argument("--batch_size",type=int, default=1)
+    parser.add_argument("--start-ind", type=int, default=0)
+    parser.add_argument("--end-ind", type=int, default=None)
+    parser.add_argument("--cache-dir", type=str, default=None)
+    parser.add_argument("--dataset", type=str)
+    parser.add_argument("--rerun", type=bool, default=False)
+    parser.add_argument("--dataset_size", type=int, default=None)
+    args = parser.parse_args()
+
+    eval_model(args)

videollava/eval/geochat_referring_2.py

@@ -0,0 +1,459 @@
+"""
+Code adapted from calculate_mean_ap.py
+author: Timothy C. Arlen
+date: 28 Feb 2018
+"""
+
+import sys
+from os.path import dirname, abspath
+sys.path.append(dirname(dirname(dirname(dirname(abspath(__file__))))))
+
+from collections import defaultdict
+import numpy as np
+import json
+import ast
+import re
+import cv2
+from shapely import wkt, Polygon, box
+from infer_utils import create_mask
+from matplotlib.path import Path
+from tqdm import tqdm
+
+from eval_referring import referring_expression
+import matplotlib.pyplot as plt
+import time
+import math
+from matplotlib.path import Path
+
+def convert_geochat_string(build, img_size=256):
+    """
+    Convert the raw str geochat output {<40><89><56><100>|<57>}, {<0><89><56><100>|<57>}
+    to a list of rotated bboxes.
+    """
+    build = build.strip('{}')
+    bbox_segments = build.split("}{")
+    # Regular expression to find all numbers inside angle brackets
+    pattern = r"<(\d+)>"
+
+    # Extract numbers, convert them to integers, and collect into a list
+    bboxes = [
+        list(map(int, re.findall(pattern, segment)))
+        for segment in bbox_segments
+    ]
+
+    rotated_bboxes = []
+    for bbox in bboxes:
+        try:
+            xmin, ymin, xmax, ymax, angle = [float(v) for v in bbox]
+        except:
+            print("Warning - Malformed bbox: ", bbox)
+            print("Original string: ", build)
+            print()
+            continue
+
+        # Convert percentages to pixel coordinates
+        xmin = xmin * img_size / 100
+        ymin = ymin * img_size / 100
+        xmax = xmax * img_size / 100
+        ymax = ymax * img_size / 100
+        
+        # Calculate rectangle dimensions
+        rect_width = xmax - xmin
+        rect_height = ymax - ymin
+        center_x = xmin + rect_width / 2
+        center_y = ymin + rect_height / 2
+        
+        # Calculate corners before rotation
+        corners = np.array([
+            [xmin, ymin],
+            [xmax, ymin],
+            [xmax, ymax],
+            [xmin, ymax]
+        ])
+        
+        # Rotate corners
+        angle_rad = math.radians(angle)
+        cos_angle = math.cos(angle_rad)
+        sin_angle = math.sin(angle_rad)
+        rotated_corners = []
+        for x, y in corners:
+            tx = x - center_x
+            ty = y - center_y
+            rotated_x = tx * cos_angle - ty * sin_angle + center_x
+            rotated_y = tx * sin_angle + ty * cos_angle + center_y
+            rotated_corners.append([rotated_x, rotated_y])
+
+        rotated_bboxes.append(np.array(rotated_corners))
+
+    return rotated_bboxes
+
+def create_geochat_mask(buildings, img_size=(256, 256)):
+    """
+    Given a list of buildings in an image, this function 
+    - creates an img_size * img_size numpy array for the image
+    - returns the mask for all buildings
+    Input:
+    - buildings: List of geochat strings representing buildings
+    - img_size: Tuple indicating the size of the image (height, width)
+    """
+    mask = np.zeros(img_size, np.uint8)
+
+    # Fill in with ones the pixels that are inside the buildings (rotated bboxes)
+    for bbox in buildings:
+        path = Path(bbox)
+        x, y = np.meshgrid(np.arange(img_size[1]), np.arange(img_size[0]))
+        points = np.vstack((x.flatten(), y.flatten())).T
+        mask[path.contains_points(points).reshape(img_size)] = 1
+
+    return mask
+
+def calc_iou_individual(pred_box, gt_box):
+    """Calculate IoU of single predicted and ground truth box
+    Args:
+        pred_box (list of floats): location of predicted object as
+            [xmin, ymin, xmax, ymax]
+        gt_box (list of floats): location of ground truth object as
+            [xmin, ymin, xmax, ymax]
+    Returns:
+        float: value of the IoU for the two boxes.
+    Raises:
+        AssertionError: if the box is obviously malformed
+    """
+    x1_t, y1_t, x2_t, y2_t = gt_box
+    try:
+        x1_p, y1_p, x2_p, y2_p = pred_box
+    except:
+        return 0.0
+
+    if (x1_p > x2_p) or (y1_p > y2_p):
+        print("Prediction box is malformed? pred box: {}".format(pred_box))
+    if (x1_t > x2_t) or (y1_t > y2_t):
+        print("Ground Truth box is malformed? true box: {}".format(gt_box))
+
+    if (x2_t < x1_p or x2_p < x1_t or y2_t < y1_p or y2_p < y1_t):
+        return 0.0
+
+    far_x = np.min([x2_t, x2_p])
+    near_x = np.max([x1_t, x1_p])
+    far_y = np.min([y2_t, y2_p])
+    near_y = np.max([y1_t, y1_p])
+
+    inter_area = (far_x - near_x + 1) * (far_y - near_y + 1)
+    true_box_area = (x2_t - x1_t + 1) * (y2_t - y1_t + 1)
+    pred_box_area = (x2_p - x1_p + 1) * (y2_p - y1_p + 1)
+    iou = inter_area / (true_box_area + pred_box_area - inter_area)
+
+    return iou
+
+def calc_iou_individual_rotated(pred_box, gt_box):
+    """Calculate IoU of single predicted and ground truth box
+    Args:
+        pred_box (list of floats): location of predicted object as
+            [[x1, y1], [x2, y2], [x3, y3], [x4, y4]]
+        gt_box (list of floats): location of ground truth object as
+            [[x1, y1], [x2, y2], [x3, y3], [x4, y4]]
+    Returns:
+        float: value of the IoU for the two boxes.
+    Raises:
+        AssertionError: if the box is obviously malformed
+    """
+    try:
+        pred_box = np.array(pred_box)
+        gt_box = np.array(gt_box)
+    except:
+        return 0.0
+    if len(pred_box) == 4:
+        pred_box = [[pred_box[0], pred_box[1]], [pred_box[2], pred_box[1]], [pred_box[2], pred_box[3]], [pred_box[0], pred_box[3]]]
+    if len(gt_box) == 4:
+        gt_box = [[gt_box[0], gt_box[1]], [gt_box[2], gt_box[1]], [gt_box[2], gt_box[3]], [gt_box[0], gt_box[3]]]
+    pred_box = np.array(pred_box)
+    gt_box = np.array(gt_box)
+    pred_box = pred_box.reshape(4, 2)
+    gt_box = gt_box.reshape(4, 2)
+    pred_polygon = Polygon(pred_box)
+    gt_polygon = Polygon(gt_box)
+    intersection = pred_polygon.intersection(gt_polygon).area
+    union = pred_polygon.union(gt_polygon).area
+    iou = intersection / union
+    return iou
+
+    # try:
+    #     pred_box = np.array(pred_box)
+    #     gt_box = np.array(gt_box)
+    # except:
+    #     return 0.0
+
+    # pred_box = pred_box.reshape(4, 2)
+    # gt_box = gt_box.reshape(4, 2)
+
+    # pred_polygon = Polygon(pred_box)
+    # gt_polygon = Polygon(gt_box)
+
+    # intersection = pred_polygon.intersection(gt_polygon).area
+    # union = pred_polygon.union(gt_polygon).area
+
+    # iou = intersection / union
+
+    # plt.figure()
+    # plt.plot(*pred_polygon.exterior.xy, color='r', label='pred')
+    # plt.plot(*gt_polygon.exterior.xy, color='b', label='gt')
+    # plt.legend()
+    # plt.title(f"IoU: {iou}")
+    # plt.show()
+    # plt.savefig("iou.png")
+    # time.sleep(1)
+    # plt.close()
+
+    return iou
+    
+
+def get_single_image_bound_results(gt_wkts, pred_geochat_string, img_size=256):
+    """
+    Calculates upper bound and lower bound number of true_pos, false_pos, false_neg from single batch of boxes.
+    Args:
+        gt_wkts (list of strs): list of wkt strings of input polygons, scaled to raw pixel value
+        pred_boxes (list of lists): list of list of boxes, where each box is formatted
+            as [x_min, y_min, x_max, y_max] on scale from 0-100
+        img_size (int): dimensions of the image. defaults to 256. 
+    Returns:
+        tuple of dicts: true positives (int), false positives (int), false negatives (int)
+    """
+    if isinstance(gt_wkts, str):
+        gt_polygons = [wkt.loads(gt_wkts)]
+    else: 
+        gt_polygons = [wkt.loads(gt_wkt) for gt_wkt in gt_wkts]
+
+    # # Needs fixing for auxiliary
+    # if len(gt_polygons) == 0:
+    #     false_neg = np.sum(gt_mask)
+    #     ub_stats= {'true_pos': 0, 'false_pos': 0, 'false_neg': false_neg, 'intersection':0, 'union':false_neg}
+    #     lb_stats = {'true_pos': 0, 'false_pos': 0, 'false_neg': false_neg, 'intersection':0, 'union':false_neg}
+    #     return lb_stats, ub_stats
+
+    lb_preds = convert_geochat_string(pred_geochat_string, img_size)
+    # get mask of all gt_polygons and lb_preds
+    gt_mask = create_mask(gt_polygons, (img_size, img_size))
+    lb_preds_mask = create_geochat_mask(lb_preds, (img_size, img_size))
+
+    # get lower bound intersection and union masks 
+    intersection = np.logical_and(gt_mask, lb_preds_mask)
+    union = np.logical_or(gt_mask, lb_preds_mask)
+
+    # compute lb metrics
+    # lower_bound_iou = np.sum(intersection) / np.sum(union)
+    fp = np.sum(np.logical_and(lb_preds_mask, np.logical_not(gt_mask)))
+    tp = np.sum(np.logical_and(lb_preds_mask, gt_mask))
+    fn = np.sum(np.logical_and(np.logical_not(lb_preds_mask), gt_mask))
+    lb_stats = {'true_pos': tp, 'false_pos': fp, 'false_neg': fn, 'intersection': np.sum(intersection), 'union': np.sum(union)}
+
+    # get upper bound intersection and union masks
+    ub_pred_mask = np.logical_and(gt_mask, lb_preds_mask)
+    intersection = np.logical_and(ub_pred_mask, gt_mask)
+    union = np.logical_or(gt_mask, ub_pred_mask)
+
+    # compute ub metrics
+    # upper_bound_iou = np.sum(intersection) / np.sum(union)
+    ub_fp = np.sum(np.logical_and(ub_pred_mask, np.logical_not(gt_mask)))
+    ub_tp = np.sum(np.logical_and(ub_pred_mask, gt_mask))
+    ub_fn = np.sum(np.logical_and(np.logical_not(ub_pred_mask), gt_mask))
+    ub_stats = {'true_pos': ub_tp, 'false_pos': ub_fp, 'false_neg': ub_fn, 'intersection': np.sum(intersection), 'union': np.sum(union)}
+
+    return lb_stats, ub_stats
+
+def get_geochat_dataset(image_id):
+    if image_id.startswith("P"):
+        dataset = "SOTA"
+    elif image_id.startswith("train"):
+        dataset = "FAST"
+    else:
+        dataset = "SIOR"
+    return dataset
+
+def get_single_image_results(gt_boxes, pred_boxes, iou_thr):
+    """Calculates number of true_pos, false_pos, false_neg from single batch of boxes.
+    Args:
+        gt_boxes (list of list of floats): list of locations of ground truth
+            objects as [[x1,y1], [x2,y2], ...]
+        pred_boxes (dict): dict of dicts of 'boxes' 
+            [[x1,y1], [x2,y2], ...]
+        iou_thr (float): value of IoU to consider as threshold for a
+            true prediction.
+    Returns:
+        dict: true positives (int), false positives (int), false negatives (int)
+    """
+
+    all_pred_indices = range(len(pred_boxes))
+    all_gt_indices = range(len(gt_boxes))
+    if len(all_pred_indices) == 0:
+        tp = 0
+        fp = 0
+        fn = len(gt_boxes)
+        return {'true_pos': tp, 'false_pos': fp, 'false_neg': fn}
+    if len(all_gt_indices) == 0:
+        tp = 0
+        fp = len(pred_boxes)
+        fn = 0
+        return {'true_pos': tp, 'false_pos': fp, 'false_neg': fn}
+
+    gt_idx_thr = []
+    pred_idx_thr = []
+    ious = []
+    for ipb, pred_box in enumerate(pred_boxes):
+        for igb, gt_box in enumerate(gt_boxes):
+            iou = calc_iou_individual_rotated(pred_box, gt_box)
+            if iou > iou_thr:
+                gt_idx_thr.append(igb)
+                pred_idx_thr.append(ipb)
+                ious.append(iou)
+
+    args_desc = np.argsort(ious)[::-1]
+    if len(args_desc) == 0:
+        # No matches
+        tp = 0
+        fp = len(pred_boxes)
+        fn = len(gt_boxes)
+    else:
+        gt_match_idx = []
+        pred_match_idx = []
+        for idx in args_desc:
+            gt_idx = gt_idx_thr[idx]
+            pr_idx = pred_idx_thr[idx]
+            # If the boxes are unmatched, add them to matches
+            if (gt_idx not in gt_match_idx) and (pr_idx not in pred_match_idx):
+                gt_match_idx.append(gt_idx)
+                pred_match_idx.append(pr_idx)
+        tp = len(gt_match_idx)
+        fp = len(pred_boxes) - len(pred_match_idx)
+        fn = len(gt_boxes) - len(gt_match_idx)
+
+    return {'true_pos': tp, 'false_pos': fp, 'false_neg': fn}
+
+
+def calc_precision_recall(img_results):
+    """Calculates precision and recall from the set of images
+    Args:
+        img_results (dict): dictionary formatted like:
+            {
+                'img_id1': {'true_pos': int, 'false_pos': int, 'false_neg': int},
+                'img_id2': ...
+                ...
+            }
+    Returns:
+        tuple: of floats of (precision, recall)
+    """
+    true_pos = 0; false_pos = 0; false_neg = 0
+    for _, res in img_results.items():
+        true_pos += res['true_pos']
+        false_pos += res['false_pos']
+        false_neg += res['false_neg']
+
+    try:
+        precision = true_pos/(true_pos + false_pos)
+    except ZeroDivisionError:
+        precision = 0.0
+    try:
+        recall = true_pos/(true_pos + false_neg)
+    except ZeroDivisionError:
+        recall = 0.0
+
+    return (precision, recall)
+
+
+DIMENSIONS = {'FAST': 600,
+              'SIOR': 800,
+              'SOTA': 1024}
+
+
+def referring_expression(answer_path, dataset, verbose=False, saving_path_root=None, img_size=256):
+    # Replace with the path to the answers file
+    if type(answer_path) == dict:
+        results = answer_path
+    else:
+        with open(answer_path) as json_data:
+            results = json.load(json_data)
+
+    img_results = {}
+    ub_results = {}
+    lb_results = {}
+    num_bboxes = 0
+    # Loop over results and get precision, recall overall
+    for id, result in tqdm(results.items()):
+
+        if dataset == "geochat_xbd":
+            pred = result['predicted']
+
+            dataset = get_geochat_dataset(id)
+            img_size = (DIMENSIONS[dataset])
+            pred = convert_geochat_string(pred, img_size)
+
+            ground_truth = result['ground_truth']
+            ground_truth = np.array(ground_truth)
+            num_bboxes += len(ground_truth)
+
+            img_results[id] = get_single_image_results(ground_truth, pred, iou_thr=0.5)
+
+            continue
+
+        try:
+            if 'referring_expression' not in result['task']:
+                continue  # no bounding box outputs for temporal_referring_expression
+        except:
+            pass
+        
+        # TODO: LOOP THROUGH IDENTIFY TASKS/QUESTIONS IN THE DATASET
+       
+        # TODO: HANDLE WHEN THERE ARE NO BOUNDING BOXES IN GROUND TRUTH for auxiliary tasks
+        if not result['original_input_polygon']:
+            first_open_bracket_ind = result["predicted"].find("{")
+            last_close_bracket_ind = result["predicted"].rfind("}")
+            if last_close_bracket_ind != -1 and first_open_bracket_ind != -1:
+                parsed_predicted = result["predicted"][first_open_bracket_ind:last_close_bracket_ind+1]
+            else:   
+                parsed_predicted = ""      
+            predicted_boxes = convert_geochat_string(parsed_predicted)
+            # If ground truth contains no boxes: all predictions are false positives
+            false_pos = len(predicted_boxes)
+            false_pos_pixels = np.sum(create_geochat_mask(predicted_boxes))
+            img_results[id] = {'true_pos': 0, 'false_pos': false_pos, 'false_neg': 0, 'intersection':0, 'union':false_pos_pixels}
+            ub_results[id] = {'true_pos': 0, 'false_pos': false_pos_pixels, 'false_neg': 0, 'intersection':0, 'union':false_pos_pixels}
+            lb_results[id] = {'true_pos': 0, 'false_pos': false_pos_pixels, 'false_neg': 0, 'intersection':0, 'union':false_pos_pixels}
+            continue
+        else: # Ground truth contains boxes: find predicted Geochat boxes
+            first_open_bracket_ind = result["predicted"].find("{")
+            last_close_bracket_ind = result["predicted"].rfind("}")
+            if last_close_bracket_ind != -1 and first_open_bracket_ind != -1:
+                parsed_predicted = result["predicted"][first_open_bracket_ind:last_close_bracket_ind+1]
+            else:   
+                parsed_predicted = "" 
+            gt_wkts = result['original_input_polygon']  
+            lb_results[id], ub_results[id] = get_single_image_bound_results(gt_wkts, parsed_predicted)
+
+    if len(ub_results) != 0:
+        ub_intersection = np.sum([res['intersection'] for res in ub_results.values()])
+        ub_union = np.sum([res['union'] for res in ub_results.values()])
+        lb_intersection = np.sum([res['intersection'] for res in lb_results.values()])
+        lb_union = np.sum([res['union'] for res in lb_results.values()])
+        print("Upper bound IOU: ", ub_intersection / ub_union if ub_union != 0 else 0)
+        print("Lower bound IOU: ", lb_intersection / lb_union if lb_union != 0 else 0)
+        ub_precision, ub_recall = calc_precision_recall(ub_results)
+        lb_precision, lb_recall = calc_precision_recall(lb_results)
+        print('Lower bound precision: ', lb_precision)
+        print('Lower bound recall: ', lb_recall)
+        print("Upper bound F1: ", 2 * (ub_precision * ub_recall) / (ub_precision + ub_recall) if (ub_precision + ub_recall) != 0 else 0)
+        print("Lower bound F1: ", 2 * (lb_precision * lb_recall) / (lb_precision + lb_recall) if (lb_precision + lb_recall) != 0 else 0)
+
+    print("[email protected]: ", np.sum([res['true_pos'] for res in img_results.values()]) / num_bboxes)
+
+    if type(answer_path) == dict:
+        return 
+    
+    if saving_path_root:
+        with open(f"{saving_path_root}/referring_expression_scores.json", 'w') as f:
+            json.dump(img_results, f)
+
+if __name__ == '__main__':
+    answer_path = "scripts/geovlm/eval/xBD/answers/ckpt14000-geochat-bench_interleave_test.json"
+    referring_expression(answer_path, dataset="geochat_xbd")
+    #answer_path = "scripts/geochat/eval/xBD/geochat_xbd_test_auxiliary_dict.json"
+    # referring_expression(answer_path, dataset="xbd")
+

videollava/eval/geochat_s2looking_utils.py

@@ -0,0 +1,400 @@
+
+import json
+import numpy as np
+import cv2
+import re
+from eval_referring import referring_expression
+import matplotlib.pyplot as plt
+from shapely import wkt
+import time
+import math
+from matplotlib.path import Path
+from eval_classification import accuracy_precision_recall
+
+
+def convert_geochat_string(build, img_size=256):
+    """
+    convert the raw str geochat output {<40><89><56><100>|<57>}, {<0><89><56><100>|<57>}
+    to a list of rotated bboxes 
+    """
+    build = build.strip('{}')
+    bbox_segments = build.split("}{")
+
+    # Regular expression to find all numbers inside angle brackets
+    pattern = r"<(\d+)>"
+
+    # Extract numbers, convert them to integers, and collect into a list
+    bboxes = [
+        list(map(int, re.findall(pattern, segment)))
+        for segment in bbox_segments]
+    
+    rotated_bboxes = []
+    for bbox in bboxes:
+        try:
+            xmin, ymin, xmax, ymax, angle = [float(v) for v in bbox]
+        except:
+            pass
+
+        # Convert percentages to pixel coordinates
+        xmin = xmin * img_size / 100
+        ymin = ymin * img_size / 100
+        xmax = xmax * img_size / 100
+        ymax = ymax * img_size / 100
+        
+        # Calculate rectangle dimensions
+        rect_width = xmax - xmin
+        rect_height = ymax - ymin
+        center_x = xmin + rect_width / 2
+        center_y = ymin + rect_height / 2
+        
+        # Calculate corners before rotation
+        corners = np.array([
+            [xmin, ymin],
+            [xmax, ymin],
+            [xmax, ymax],
+            [xmin, ymax]
+        ])
+        
+        # Rotate corners
+        angle_rad = math.radians(angle)
+        cos_angle = math.cos(angle_rad)
+        sin_angle = math.sin(angle_rad)
+        rotated_corners = []
+        for x, y in corners:
+            tx = x - center_x
+            ty = y - center_y
+            rotated_x = tx * cos_angle - ty * sin_angle + center_x
+            rotated_y = tx * sin_angle + ty * cos_angle + center_y
+            rotated_corners.append([rotated_x, rotated_y])
+
+        rotated_bboxes.append(np.array(rotated_corners))
+
+    return rotated_bboxes
+
+
+def get_changed_buildings(build1, build2, img_size=256, task=None):
+    """
+    Given a list of predicted buildings in image 1 and image 2, this function 
+    - creates two img_size * img_size numpy arrays for both of the images
+    - gets the mask differences between the two numpy arrays
+    - returns a list of bounding boxes that reflect those differences, as well as the difference mask
+    Input:
+    - build1: [[x,y],[x,y],[x,y],[x,y]] array of four x,y coordinates of the bounding box of a building
+    - task can be either None, constructed or destructed
+    Note: those bboxes can be rotated
+    """
+    image1 = np.zeros((img_size, img_size), np.uint8)
+    image2 = np.zeros((img_size, img_size), np.uint8)
+
+    build1 = convert_geochat_string(build1)
+    build2 = convert_geochat_string(build2)
+
+     # fill in with ones the pixels that are inside the rotated bboxes
+    for b in build1:
+        path = Path(b)
+        x, y = np.meshgrid(np.arange(img_size), np.arange(img_size))
+        points = np.vstack((x.flatten(), y.flatten())).T
+        image1[path.contains_points(points).reshape(img_size, img_size)] = 1
+
+    for b in build2:
+        path = Path(b)
+        x, y = np.meshgrid(np.arange(img_size), np.arange(img_size))
+        points = np.vstack((x.flatten(), y.flatten())).T
+        image2[path.contains_points(points).reshape(img_size, img_size)] = 1
+
+    # xor between the two images
+    if task == None:
+        diff = cv2.bitwise_xor(image1, image2)
+    elif task == "constructed":
+        # if the task is constructed, we want to find the pixels that are in image2 but not in image1
+        diff = cv2.bitwise_and(image2, cv2.bitwise_not(image1))
+    elif task == "destructed":
+        # if the task is destructed, we want to find the pixels that are in image1 but not in image2
+        diff = cv2.bitwise_and(image1, cv2.bitwise_not(image2))
+
+    # get the bounding boxes of the difference pixels
+    contours, _ = cv2.findContours(diff, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+    bboxes = []
+    for contour in contours:
+        x, y, w, h = cv2.boundingRect(contour)
+        x, y, w, h = y, x, h, w
+        bboxes.append([x, y, x+w, y+h])
+ 
+    return bboxes, diff
+
+def get_canonical_answer_dataset(answers):
+    """
+    This function creates a new dataset with questions and answers for geochat, ready to parse into the evaluation metrics."""
+
+    new_dataset = {}   
+
+    for key, answer in answers.items():
+        num, quadrant, geovlmid = key.split("_")
+        task = answer['task']
+        if geovlmid == "1" in task:
+            continue
+        
+        # find the paired image
+        id2 = num + "_" + quadrant + "_" + "1"
+        answer1 = answers[key]
+        try:
+            answer2 = answers[id2]
+        except:
+            print(f"The associated image to {key} wasn't present in the dataset")
+            continue
+
+        # get the pixel diff boxes 
+        change_bboxes, mask = get_changed_buildings(answer1['predicted'], answer2['predicted'])
+
+        # create the new dataset adapted for running metrics on it 
+        new_line = {}
+
+        new_line['predicted'] = "" 
+        if len(change_bboxes)>0:
+            for bbox in change_bboxes:
+                new_line['predicted'] += str(bbox) + ", "
+            new_line['predicted'] = new_line['predicted'][:-2]
+        new_line['predicted_mask'] = mask.tolist()         
+
+        new_line['ground_truth'] = answer1['original_answer']
+        new_line['question'] = answer1['original_question']
+        new_line['task'] = answer1['task']
+        new_line['original_input_polygon'] = answer1['original_input_polygon']
+
+        new_key = num + "_" + quadrant  
+        new_dataset[new_key] = new_line
+
+    return new_dataset
+
+def postprocess_auxiliary_qa(key, answer, original_answers):
+    new_line = {}
+    new_line['ground_truth'] = answer['ground_truth']
+    new_line['question'] = answer['question']
+    new_line['task'] = answer['task']
+    new_line['original_input_polygon'] = answer['original_input_polygon']
+
+    # retrieve the original 2 anwers
+    answer1 = original_answers[key + '_0']['predicted']
+    answer2 = original_answers[key + '_1']['predicted']
+
+    # retrieve the task (construction or destruction)
+    setting = None
+    if "constructed" or "built" in answer['original_question']:
+        setting = "constructed"
+    elif "destructed" or "torn down" in answer['original_question']:
+        setting = "destructed"
+    else:
+        print("The task is not recognized")
+        print("Original question: ", answer['original_question'])
+        print()
+
+    # get the pixel diff boxes
+    change_bboxes, mask = get_changed_buildings(answer1, answer2, task=setting) 
+
+    new_line['predicted_mask'] = mask.tolist()
+    contours, hierarchy = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+
+    found_convex_polygon = False
+    for contour in contours:
+        # check if the contour is a bounding box (4 vertices, rectangle shape)
+        epsilon = 0.04 * cv2.arcLength(contour, True)
+        approx = cv2.approxPolyDP(contour, epsilon, True)
+        if len(approx) == 4:
+            found_convex_polygon = True
+            break        
+
+    if found_convex_polygon:
+        new_line['predicted'] = "Yes"
+    else:
+        new_line['predicted'] = "No"
+
+    return new_line
+
+
+def postprocess_auxiliary_region_qa(key, answer, original_answers, img_size=256):
+    """
+    There is a bbox in the input polygon, we need to find the changed buildings in the image
+    inside that bbox
+    """
+    new_line = {}
+    new_line['ground_truth'] = answer['ground_truth']
+    new_line['question'] = answer['question']
+    new_line['task'] = answer['task']
+    new_line['original_input_polygon'] = answer['original_input_polygon']
+
+    # retrieve the original 2 anwers
+    answer1 = original_answers[key + '_0']['predicted']
+    answer2 = original_answers[key + '_1']['predicted']
+
+    # get the pixel diff boxes
+    change_bboxes, mask = get_changed_buildings(answer1, answer2)
+
+    # get the input bbox
+    question = new_line['question']
+    # find the positions of '[' and ']'
+    start = question.find('[')
+    end = question.find(']')
+    bbox = question[start+1:end].split(',')
+    bbox = [int(b) * img_size // 100 for b in bbox]
+
+    # adapt the mask, put 0s outside the bbox
+    mask[:bbox[0], :] = 0
+    mask[bbox[2]:, :] = 0
+    mask[:, :bbox[1]] = 0
+    mask[:, bbox[3]:] = 0
+
+    # predict yes or no if there is a convex polygon in the mask
+    found_convex_polygon = False
+    contours, hierarchy = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+    for contour in contours:
+        # check if the contour is a bounding box (4 vertices, rectangle shape)
+        epsilon = 0.04 * cv2.arcLength(contour, True)
+        approx = cv2.approxPolyDP(contour, epsilon, True)
+        if len(approx) == 4:
+            found_convex_polygon = True
+            break
+
+    new_line['predicted_mask'] = mask.tolist()
+   
+    if found_convex_polygon:
+        new_line['predicted'] = "Yes"
+    else:
+        new_line['predicted'] = "No"
+
+    return new_line
+
+
+def postprocess_auxiliary_referring(key, answer, original_answers):
+    new_line = {}
+    new_line['ground_truth'] = answer['ground_truth']
+    new_line['question'] = answer['question']
+    new_line['task'] = answer['task']
+    new_line['original_input_polygon'] = answer['original_input_polygon']
+
+    # retrieve the original 2 anwers
+    answer1 = original_answers[key + '_0']['predicted']
+    answer2 = original_answers[key + '_1']['predicted']
+
+    # retrieve the task (construction or destruction)
+    setting = None
+    if "constructed" or "built" in answer['original_question']:
+        setting = "constructed"
+    elif "destructed" or "torn down" in answer['original_question']:
+        setting = "destructed"
+    else:
+        print("The task is not recognized")
+        print("Original question: ", answer['original_question'])
+        print()
+
+    # get the pixel diff boxes
+    change_bboxes, mask = get_changed_buildings(answer1, answer2, task=setting)
+
+    new_line['predicted_mask'] = mask.tolist()
+    new_line['predicted'] = ""
+    if len(change_bboxes)>0:
+        for bbox in change_bboxes:
+            new_line['predicted'] += str(bbox) + ", "
+        new_line['predicted'] = new_line['predicted'][:-2]
+      
+    return new_line
+
+
+def postprocess_auxiliary_geochat_s2looking(canonical_answers, original_answers):
+    """
+    Postprocess the auxiliary file for geochat_s2looking
+    The present questions are
+    question1 = 'temporal_question_answering: Are there any buildings in the first image which were {destructed,torn down} in the second?'
+    question2 = 'temporal_referring_expression: Identify the buildings in the first image which were {built,constructed,destructed,torn down} as seen in the second image.'
+    question3 = 'localization_task: Identify all changed buildings.'
+    question4 = 'referring_expression: identify the {constructed, destructed} buildings in the image.'
+    question5 = 'question_answering: Have any buildings been task in the area? Please answer with Yes or No'
+
+    The goal is to update the 'predicted' field with the correct bounding boxes of the changed buildings.
+    - Localization can be kept as is.
+    - For question answering tasks, the 'predicted' field should be updated with 'Yes' or 'No' depending on the answer.
+    We output 'Yes' if there is a convex polygon in the 'predicted' field.
+    - For referring expression, we first need to identify if the task is 'constructed' or 'destructed' and then update the 'predicted' field with the correct mask of the changed buildings.
+    Input:
+    - answers: dictionary with the answers paired with the get_canonical_answer_dataset function
+    Output:
+    - postprocessed_answers: dictionary with 'predicted' and 'predicted_mask' fields updated
+    """
+    postprocessed_answers = {}
+
+    for key, answer in canonical_answers.items():
+        task = answer['task']
+       
+        if 'localization' in task:
+            postprocessed_answers[key] = answer
+            continue
+        if 'region_based_question_answering' in task:
+            answer = postprocess_auxiliary_region_qa(key, answer, original_answers)
+            postprocessed_answers[key] = answer
+            continue
+        if 'question_answering' in task:
+            answer = postprocess_auxiliary_qa(key, answer, original_answers)
+            postprocessed_answers[key] = answer
+            continue
+        if 'referring_expression' in task:
+            answer = postprocess_auxiliary_referring(key, answer, original_answers)
+            postprocessed_answers[key] = answer
+            continue
+    
+    return postprocessed_answers
+    
+
+def evaluate_geochat_s2looking(answer_file, dataset_file, split):
+    answers = {}
+    with open(answer_file, 'r') as f:
+        for line in f:
+            line = json.loads(line)
+            answers[list(line.keys())[0]] = line[list(line.keys())[0]]
+
+    dataset = dataset_file.split("/")[-1]
+    if dataset == "dataset_canonical.json":
+
+        # create a new dataset with questions and answers for geochat 
+        postprocessed_answers = get_canonical_answer_dataset(answers)
+
+        referring_expression(postprocessed_answers, "geochat_s2looking", False, "s2looking/answers/geochat_canonical_test", split=split)
+    
+    elif dataset == "dataset_v01_v02_canonical_filtered.json" or dataset == "dataset_RQA.json":
+        
+        # create a new dataset with questions and answers for geochat
+        postprocessed_answers = get_canonical_answer_dataset(answers)
+        postprocessed_answers = postprocess_auxiliary_geochat_s2looking(postprocessed_answers, answers)
+
+        print("Referring expression")
+        referring_expression(postprocessed_answers, "geochat_s2looking", False, "s2looking/answers/geochat_v01_v02_canonical_filtered_test", split=split)
+        print()
+        print("Accuracy")
+        accuracy_precision_recall(postprocessed_answers, "s2looking", verbose=False)
+        print()
+
+
+        # also run per-question referring expression
+        question1 = 'temporal_question_answering: Are there any buildings in the first image which were {destructed,torn down} in the second?'
+        question2 = 'temporal_referring_expression: Identify the buildings in the first image which were {built,constructed,destructed,torn down} as seen in the second image.'
+        question3 = 'localization_task: Identify all changed buildings.'
+        question4 = 'referring_expression: identify the {constructed, destructed} buildings in the image.'
+        question5 = 'question_answering: Have any buildings been task in the area? Please answer with Yes or No'
+
+
+        for question in [question1, question2, question3, question4, question5]:
+            dataset_question = {}
+            for data in postprocessed_answers:
+                if postprocessed_answers[data]['task'] == question:
+                    dataset_question[data] = postprocessed_answers[data]
+
+            if len(dataset_question) > 0:
+                print('Evaluating for question ', question)
+                print('Size of the dataset is ', len(dataset_question))
+                referring_expression(dataset_question, "geochat_s2looking", False, "s2looking/answers/geochat_v01_v02_canonical_filtered_test", split=split)
+                print()
+
+    else:
+        print("Evaluation is not suppored for this dataset. Please provide a valid dataset.")
+        print("The supported datasets are: dataset_canonical.json, dataset_v01_v02_canonical_filtered.json")
+
+            
+

videollava/eval/geochat_utils.py

@@ -0,0 +1,94 @@
+import json
+from tqdm import tqdm
+from pathlib import Path
+
+from infer_utils import run_inference_single
+import numpy as np
+
+
+def run_geochat_inference(
+    model,
+    dataset_path,
+    processor,
+    tokenizer,
+    conv_mode,
+    answer_path,
+    use_video_data=False,
+    open_prompt=None,
+    repeat_frames=None,
+    prompt_strategy="interleave",
+    chronological_prefix=True,
+    data_frac=1,
+    data_size=None,
+    delete_system_prompt=False,
+    start_ind=0,
+    end_ind=None,
+    print_prompt=False
+):
+    
+    with open(dataset_path) as f:
+        qfabric_data = json.load(f)
+
+    if data_size is not None:
+        data_size = min(data_size, len(qfabric_data))
+        idx = np.random.choice(len(qfabric_data), data_size, replace=False)
+        qfabric_data = [qfabric_data[i] for i in idx]
+    elif data_frac < 1:
+        idx = np.random.choice(len(qfabric_data), int(len(qfabric_data) * data_frac), replace=False)
+        qfabric_data = [qfabric_data[i] for i in idx]
+
+    answers = {}
+    answers_tmp = str(answer_path).replace(".json", "_tmp.json")
+    if end_ind is not None:
+        answers_tmp = str(answers_tmp).replace(".json", f"_{start_ind}_{end_ind}.json")
+        qfabric_data = qfabric_data[start_ind:end_ind]
+    else:
+        answers_tmp = str(answers_tmp).replace(".json", f"_{start_ind}_end.json")
+        qfabric_data = qfabric_data[start_ind:]
+    
+    print("answers_tmp: ", answers_tmp)
+    print("start ind: ", start_ind)
+    print("end ind: ", end_ind)
+
+    for question in tqdm(qfabric_data):
+        question_id = question["id"]
+        inp = question["conversations"][0]['value']
+
+        answer_str = question["conversations"][1]['value']
+        metadata = question['metadata']
+        image_paths = question['video']
+        task = question['task']
+        original_input_polygon = question['original_input_polygon']
+        dataset = question['dataset']
+
+        outputs = run_inference_single(
+            model=model,
+            processor=processor,
+            tokenizer=tokenizer,
+            conv_mode=conv_mode,
+            inp=inp,
+            image_paths=image_paths,
+            metadata=metadata,
+            repeat_frames=repeat_frames,
+            use_video_data=use_video_data,
+            prompt_strategy=prompt_strategy,
+            chronological_prefix=chronological_prefix,
+            delete_system_prompt=delete_system_prompt,
+            print_prompt=print_prompt
+            )
+
+        entry = {
+            "id": question_id,
+            "question": inp,
+            "predicted": outputs,
+            "ground_truth": answer_str,
+            "task": task,
+            "original_input_polygon": original_input_polygon,
+            "dataset": dataset,
+        }
+        answers[question_id] = entry
+
+        with open(answers_tmp, "a") as f:
+            f.write(json.dumps(entry) + "\n")
+
+    return answers

videollava/eval/infer_eval.py

@@ -0,0 +1,386 @@
+import fire
+import json
+from pathlib import Path
+
+from videollava.model.builder import load_pretrained_model
+from videollava.utils import disable_torch_init
+from videollava.mm_utils import get_model_name_from_path
+from videollava.model.multimodal_encoder.languagebind.video.processing_video import LanguageBindVideoProcessor
+
+from eval_classification import accuracy_precision_recall
+from eval_referring import referring_expression
+from classification_segmentation import classification_segmentation
+
+from ben_utils import run_ben_inference
+from aid_fmow_ucmerced_utils import run_aid_fmow_ucmerced_inference
+from qfabric_utils import run_qfabric_inference
+from geochat_utils import run_geochat_inference
+from s2looking_utils import run_s2looking_inference
+from xbd_utils import run_xbd_inference
+from cdvqa_utils import run_cdvqa_inference
+
+
+def aggregated(answer_path, dataset=None, verbose=False, split=None):
+    """
+    Define an aggregated metric for our created instruction-following datasets.
+    It includes eval_description and eval_referring metrics.
+    """
+    saving_path_root = Path(answer_path).parent
+
+    with open(answer_path, 'r') as f:
+        answers = json.load(f)
+    
+    print("Referring expression")
+    referring_expression(answer_path, dataset, False, saving_path_root, split=split)
+    print()
+    print("Accuracy")
+    accuracy_precision_recall(answer_path, dataset, verbose=False)
+    print()
+
+    # TODO per-task metrics for qfabric and xbd
+    
+    if dataset == 'qfabric' or dataset == 'xbd':
+        classification_segmentation(answer_path, dataset)
+
+    if dataset == "s2looking":
+        # also run per-question referring expression
+        question1 = 'temporal_question_answering: Are there any buildings in the first image which were {destructed,torn down} in the second?'
+        question2 = 'temporal_referring_expression: Identify the buildings in the first image which were {built,constructed,destructed,torn down} as seen in the second image.'
+        question3 = 'localization_task: Identify all changed buildings.'
+        question4 = 'referring_expression: identify the {constructed, destructed} buildings in the image.'
+        question5 = 'question_answering: Have any buildings been task in the area? Please answer with Yes or No'
+
+
+        for question in [question1, question2, question3, question4, question5]:
+            dataset_question = {}
+            for data in answers:
+                if answers[data]['task'] == question:
+                    dataset_question[data] = answers[data]
+            if len(dataset_question) > 0:
+                print('Evaluating for question ', question)
+                print('Size of the dataset is ', len(dataset_question))
+                referring_expression(dataset_question, dataset, False, saving_path_root, split=split)
+                print()
+
+
+def load_model(model_path, model_base, cache_dir, device, vision_type=None, load_4bit=False, load_8bit=False):
+    model_name = get_model_name_from_path(model_path)
+
+    tokenizer, model, processor, _ = load_pretrained_model(
+        model_path,
+        model_base,
+        model_name,
+        load_4bit=load_4bit,
+        load_8bit=load_8bit,
+        device=device,
+        cache_dir=cache_dir,
+        vision_type=vision_type,
+    )
+
+    if vision_type is None:
+        # Automatically determine which to us
+        # For now assumes one of the processors is not None and one is None
+        vision_types = ['image', 'video']
+        if processor['image'] is None and processor['video'] is None:
+            raise ValueError("Both image and video processors are None")
+        elif processor['image'] is not None and processor['video'] is not None:
+            vision_processor = processor['image']
+        for vision_type in vision_types:
+            vision_processor = processor[vision_type]
+            if vision_processor is not None:
+                break
+    else:
+        vision_processor = processor[vision_type]
+    use_video_data = vision_type == 'video'
+    return tokenizer, model, vision_processor, use_video_data
+
+
+def infer_eval(
+        dataset_path,
+        model_path,
+        model_base="LanguageBind/Video-LLaVA-7B",
+        cache_dir="/deep/group/aicc-bootcamp/geovlm/models/vllava_cache",
+        outname=None,
+        open_prompt=None, 
+        repeat_frames=None,
+        prompt_strategy="interleave",
+        chronological_prefix=True,
+        load_8bit=False, 
+        load_4bit=False,
+        verbose=False,
+        rerun=False,
+        vision_type=None,
+        data_frac=None,
+        data_size=None,
+        conv_mode="v1",
+        delete_system_prompt=False,
+        start_ind=None,
+        end_ind=None,
+        last_image=None,
+        print_prompt=False
+    ):
+    """
+    Args:
+        dataset_path: path to dataset
+        model_path: path to model
+        model_base: model base name
+        cache_dir: cache directory
+        outname: output file name (uses args if None)
+        open_prompt options: None, "open", "multi-open"
+        repeat_frames options: None, "uniform", "first", "last"
+        prompt_strategy options: None, "interleave"
+        chronological_prefix: whether to use chronological prefix "in chronological order"
+        load_8bit: whether to load 8-bit model
+        load_4bit: whether to load 4-bit model
+        verbose: whether to print verbose output
+        rerun: whether to rerun inference
+        vision_type: "image" or "video"
+        data_frac: fraction of data to use
+        data_size: number of data samples to use
+        conv_mode: conversation mode (should be v1 for our models, geochat, and videollava)
+        delete_system_prompt: whether to delete system prompt
+        start_ind: start index of data
+        end_ind: end index of data
+        last_image: whether to use last image in video
+        print_prompt: whether to print prompt
+    """
+    args = locals()
+    print(f"Arguments passed to infer_eval:")
+    for k, v in args.items():
+        print(f"{k} ({type(v).__name__}): {v}")
+
+    # check that data_size and data_frac are not both set
+    if data_size is not None and data_frac is not None:
+        raise ValueError("data_size and data_frac cannot both be set")
+    if data_size is None and data_frac is None:
+        data_frac = 1
+
+    dataset2metrics = {
+        "lrben": [accuracy_precision_recall],
+        "hrben": [accuracy_precision_recall],
+        "fmow": [accuracy_precision_recall],
+        "s2looking": [aggregated],
+        "xbd": [aggregated], 
+        "qfabric": [aggregated],
+        "aid": [accuracy_precision_recall],
+        "ucmerced": [accuracy_precision_recall],
+        "cdvqa": [accuracy_precision_recall]
+    }
+
+    eval_outdir = Path('scripts/geovlm/eval/')
+
+    # Per dataset configurations
+    if "lrben" in dataset_path.lower():
+        dataset = "lrben"
+        run_inference = run_ben_inference
+        outdir = eval_outdir / "RSVQA-LRBEN/answers/"
+        if open_prompt is not None:
+            raise ValueError("LRBEN dataset does not support open prompt")
+    elif "hrben" in dataset_path.lower():
+        dataset = "hrben"
+        run_inference = run_ben_inference
+        outdir = eval_outdir / "RSVQA-HRBEN/answers/"
+        if open_prompt is not None:
+            raise ValueError("HRBEN dataset does not support open prompt")
+    elif "fmow" in dataset_path.lower():
+        dataset = "fmow"
+        run_inference = run_aid_fmow_ucmerced_inference
+        outdir = eval_outdir / "fmow-highres/answers/"
+    elif "s2looking" in dataset_path.lower():
+        dataset = "s2looking"
+        run_inference = run_s2looking_inference
+        outdir = eval_outdir / "s2looking/answers/"
+    elif "xbd" in dataset_path.lower():
+        dataset = "xbd"
+        run_inference = run_xbd_inference
+        outdir = eval_outdir / "xBD/answers/"
+    elif 'qfabric' in dataset_path.lower() or 'geochat' in dataset_path.lower():
+        dataset = "qfabric"
+        run_inference = run_qfabric_inference
+        outdir = eval_outdir / "QFabric/answers/"
+    elif 'geochat' in dataset_path.lower():
+        dataset = "geochat"
+        run_inference = run_geochat_inference
+        outdir = eval_outdir / "GeoChat/answers/"
+    elif 'aid' in dataset_path.lower():
+        dataset = "aid"
+        run_inference = run_aid_fmow_ucmerced_inference
+        outdir = eval_outdir / "AID/answers/"
+    elif 'ucmerced' in dataset_path.lower():
+        dataset = "ucmerced"
+        run_inference = run_aid_fmow_ucmerced_inference
+        outdir = eval_outdir / "UCMerced/answers/"
+    elif 'cdvqa' in dataset_path.lower():
+        dataset = "cdvqa"
+        run_inference = run_cdvqa_inference
+        outdir = eval_outdir / "CDVQA/answers/"
+    else:
+        raise ValueError(f"No supported dataset found in {dataset_path}, supported datasets: fmow, lrben, s2looking, xbd, qfabric, aic, ucmerced")
+    
+    if (start_ind is not None or end_ind is not None) and dataset not in ['qfabric', 'hrben', 'lrben']:
+        raise ValueError("start_ind and end_ind can only be used with qfabric, hrben, or lrben datasets")
+
+    # Determine the split
+    if 'test' in dataset_path.lower():
+        split = 'test'  
+    elif 'val' or 'valid' or 'validation' in dataset_path.lower():
+        split = 'val'
+    elif 'train' in dataset_path.lower():
+        split = 'train'
+    else:
+        print("Warning: Could not determine split from dataset path")
+
+    args_to_determine_path = [
+        'open_prompt',
+        'repeat_frames',
+        'prompt_strategy',
+        'chronological_prefix',
+        'load_8bit',
+        'load_4bit',
+        'data_frac',
+        'data_size',
+        'delete_system_prompt'
+    ]
+    
+    # Setup answer path
+    outdir.mkdir(parents=True, exist_ok=True)
+    model_name = Path(model_path).stem
+
+    if 'llava' not in model_name and 'llava' not in model_name.lower() and 'teochat' not in model_name.lower():
+        if model_base != None:
+            if model_path[-1] == "/":
+                model_path = model_path[:-1]
+            model_name = model_path.split("/")[-2] + "-" + model_path.split("/")[-1]
+            print("Model name used: ", model_name)
+        else:
+            raise ValueError(f"Model name {model_name} does not contain 'llava'")
+    if 'lora' not in model_name:
+        print("Warning: Model name does not contain 'lora'")
+
+    if outname is None:
+        dataset_path_name = Path(dataset_path).stem
+        outname = f"{model_name}_{dataset}_{dataset_path_name}_{split}.json"
+
+    if ".json" not in outname:
+        outname = f"{outname}.json"
+
+    args_to_determine_path = [
+        'open_prompt',
+        'repeat_frames',
+        'prompt_strategy',
+        'chronological_prefix',
+        'load_8bit',
+        'load_4bit',
+        'data_frac',
+        'data_size',
+        'delete_system_prompt',
+        'start_ind',
+        'end_ind',
+        'last_image'
+    ]
+    for arg in args_to_determine_path:
+        if args[arg] is not None:
+            outname = outname.replace(".json", f"_{arg}_{args[arg]}.json")
+
+    answer_path = outdir / outname
+
+    print(f'answer_path: {answer_path}')
+
+    # Save args to file
+    args_path = outdir / outname.replace(".json", "_args.json")
+
+    if len(str(args_path)) < 255:
+        with open(args_path, 'w') as f:
+            json.dump(args, f)
+    else:
+        # File name too long. Just use first letter of each arg
+        for arg in args_to_determine_path:
+            if args[arg] is not None:
+                first_letters = ''.join([word[0] for word in arg.split('_')])
+                #print("outname before replacing: ", outname)
+                outname = outname.replace(f"{arg}", first_letters)
+                #print("outname after replacing: ", outname)
+        answer_path = outdir / outname
+        args_path = outdir / outname.replace(".json", "_args.json")
+        with open(args_path, 'w') as f:
+            json.dump(args, f)
+        print(f'New answer_path: {answer_path}')
+
+    # If answer file exists, compute metrics 
+    if answer_path.exists() and not rerun:
+        for metric in dataset2metrics[dataset]:
+            if dataset == "s2looking":
+                metric(answer_path, dataset=dataset, verbose=verbose, split=split)
+            else:
+                metric(answer_path, dataset=dataset, verbose=verbose)
+        return
+
+    # Load model
+    disable_torch_init()
+    device = 'cuda'
+    tokenizer, model, processor, use_video_data = load_model(
+        model_path,
+        model_base,
+        cache_dir,
+        device,
+        load_4bit=load_4bit,
+        load_8bit=load_8bit,
+        vision_type=vision_type
+    )
+
+    if use_video_data:
+        if dataset == "lrben":
+            raise ValueError("LRBEN dataset does not support video processing")
+        # Hack to set backend of video processor
+        # NOTE: If we change image size, we might need to change this in the config here too
+        # (better solution is to figure out where this config is set when saving the model)
+        processor.config.vision_config.video_decode_backend = "image_list"
+        processor = LanguageBindVideoProcessor(processor.config, tokenizer) 
+
+    if rerun or not answer_path.exists():
+        # Run inference
+        answers = run_inference(
+            model,
+            dataset_path,
+            processor,
+            tokenizer,
+            conv_mode,
+            answer_path=answer_path,
+            open_prompt=open_prompt,
+            repeat_frames=repeat_frames,
+            use_video_data = use_video_data,
+            prompt_strategy=prompt_strategy,
+            chronological_prefix=chronological_prefix,
+            data_size=data_size,
+            data_frac=data_frac,
+            delete_system_prompt=delete_system_prompt,
+            start_ind=start_ind,
+            end_ind=end_ind,
+            last_image=last_image,
+            print_prompt=print_prompt
+        )
+
+        # Save answers
+        with open(answer_path, 'w') as f:
+            json.dump(answers, f, indent=4)
+    else:
+        answers = json.load(open(answer_path))
+
+
+    # Calculate metrics
+    for metric in dataset2metrics[dataset]:
+        if dataset == "s2looking":
+            metric(answer_path, dataset=dataset, verbose=verbose, split=split)
+        else:
+            metric(answer_path, dataset=dataset, verbose=verbose)
+
+
+if __name__ == '__main__':
+    """Example usage:
+    export CUDA_VISIBLE_DEVICES=0; 
+    export PYTHONPATH=/path/to/aicc-win24-geo-vlm/videollava/:$PYTHONPATH;
+    python videollava/eval/video/infer_eval.py infer_eval\
+        --dataset fmow\
+        --model_path /path/to/model\
+    """
+    fire.Fire()

videollava/eval/infer_utils.py

@@ -0,0 +1,253 @@
+import cv2
+import torch
+import warnings
+import numpy as np
+from datetime import datetime
+import cv2
+import warnings
+import time
+
+from videollava.mm_utils import tokenizer_image_token, KeywordsStoppingCriteria
+from videollava.conversation import conv_templates, SeparatorStyle
+from videollava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_VIDEO_TOKEN
+
+
+def replace_video_token(prompt, image_paths, prompt_strategy):
+    if prompt_strategy is None:
+        vid_replace_token = DEFAULT_IMAGE_TOKEN * len(image_paths)
+    elif prompt_strategy == 'interleave':
+        vid_replace_token = ''.join(f"Image {i+1}: {DEFAULT_IMAGE_TOKEN}" for i in range(len(image_paths)))
+    else:
+        raise ValueError(f"Unknown prompt strategy: {prompt_strategy}")
+    return prompt.replace(DEFAULT_VIDEO_TOKEN, vid_replace_token)
+
+
+def run_inference_single(
+        model,
+        processor,
+        tokenizer,
+        conv_mode,
+        inp,
+        image_paths,
+        metadata=None,
+        use_video_data=False,
+        repeat_frames=None,
+        prompt_strategy=None,
+        chronological_prefix=True,
+        delete_system_prompt=False,
+        print_prompt=False,
+        return_prompt=False,
+        last_image=False,
+        prompt=None
+    ):
+    conv = conv_templates[conv_mode].copy()
+    if prompt is None:
+        conv.append_message(conv.roles[0], inp)
+        conv.append_message(conv.roles[1], None)
+        prompt = conv.get_prompt()
+
+    if chronological_prefix:
+        prompt = prompt.replace("times:", "times in chronological order:")
+
+    if metadata is not None:
+         # Sort by time
+        image_paths, metadata = zip(*sorted(
+            zip(image_paths, metadata),
+            key=lambda t: datetime.strptime(t[1]["timestamp"], "%Y-%m-%d")
+        ))
+
+    if delete_system_prompt:
+        if "This is" in prompt:
+            start_index = prompt.find("This is")
+        elif "These are" in prompt:
+            start_index = prompt.find("These are")
+        end_index = prompt.find(":", start_index)
+        if start_index != -1 and end_index != -1:
+            prompt = prompt[:start_index] + prompt[end_index+1:]
+        else:
+            warnings.warn("Impossible to remove the system message from the prompt.")
+
+    if use_video_data:
+        image_paths = list(image_paths)
+        if repeat_frames == "uniform":
+            # Repeat up to 8 for now
+            num_frames = 8
+            if len(image_paths) < num_frames:
+                num_repeats = num_frames // len(image_paths)
+                index = len(image_paths) - num_frames % len(image_paths)
+                image_paths = list(np.repeat(image_paths[:index], num_repeats)) + list(np.repeat(image_paths[index:], num_repeats+1))
+        elif repeat_frames == "first":
+            # Repeat the first frame
+            num_frames = 8
+            if len(image_paths) < num_frames:
+                repeat_frames = [image_paths[0]] * (num_frames - len(image_paths)) + image_paths
+        elif repeat_frames == "last":
+            # Repeat the last frame
+            num_frames = 8
+            if len(image_paths) < num_frames:
+                repeat_frames = image_paths + [image_paths[-1]] * (num_frames - len(image_paths))
+
+        video_tensor = processor.preprocess(image_paths, return_tensors='pt')['pixel_values']
+        tensor = [video_tensor.to(model.device, dtype=torch.float16)]
+
+    else:
+        image_tensors = [processor.preprocess(i, return_tensors='pt')['pixel_values'][0] for i in image_paths]
+        tensor = [image_tensor.to(model.device, dtype=torch.float16) for image_tensor in image_tensors]
+
+        if last_image:
+            tensor = [tensor[-1]]
+            image_paths = [image_paths[-1]]
+            if metadata is not None:
+                metadata = [metadata[-1]]
+
+    prompt = replace_video_token(prompt, image_paths, prompt_strategy)
+
+    if print_prompt:
+        print(prompt)
+
+    stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
+    keywords = [stop_str]
+
+    input_ids = tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).to(model.device)
+    stopping_criteria = KeywordsStoppingCriteria(keywords, tokenizer, input_ids)
+
+    with torch.inference_mode():
+        output_ids = model.generate(
+            input_ids=input_ids,
+            images=tensor,
+            do_sample=True,
+            temperature=0.2,
+            max_new_tokens=256,
+            use_cache=True,
+            stopping_criteria=[stopping_criteria],
+        )
+
+    # .replace removes the end sentence token "</s>" from the output
+    outputs = tokenizer.decode(output_ids[0, input_ids.shape[1]:]).replace('</s>', '').strip()
+
+    if return_prompt:
+        return prompt, outputs
+    else:
+        return outputs
+
+
+def create_mask(poly, im_size):
+    """
+    Create mask of given height and width where entries
+    inside polygon are 1.
+    params:
+        - poly (shapely polygon object): polygon to create mask for
+        - im_size (tuple): size of image (height, width)
+    returns:
+        - img_mask (np.array): mask of polygon"""
+    img_mask = np.zeros(im_size, np.uint8)
+    def int_coords(x): return np.array(x).round().astype(np.int32)
+    try:
+        exteriors = [int_coords(pol.exterior.coords) for pol in poly]
+    except:
+        exteriors = [int_coords(poly.exterior.coords)]
+    cv2.fillPoly(img_mask, exteriors, 1)
+    try:
+        interiors = [int_coords(pol.interior.coords) for pol in poly] 
+        cv2.fillPoly(img_mask, interiors, 0)
+    except:
+        pass
+    try:
+        interiors = [int_coords(poly.interior.coords)]
+        cv2.fillPoly(img_mask, interiors, 0)
+    except:
+        pass
+    
+    return img_mask
+
+
+def create_mask_s2looking(img_id, split=None, question=None):
+    if split == None:
+        raise ValueError("split must be provided for S2Looking evaluation")
+    
+    if question == None:
+        raise ValueError("question must be provided for S2Looking evaluation")
+
+    im1_path = f'/scr/geovlm/S2Looking/{split}/label1' # built
+    img2_path = f'/scr/geovlm/S2Looking/{split}/label2' # destroyed
+    id, chunk = img_id.split('_')
+    # Load image as numpy array
+    im1 = cv2.imread(f'{im1_path}/{id}.png', cv2.IMREAD_GRAYSCALE)
+    im2 = cv2.imread(f'{img2_path}/{id}.png', cv2.IMREAD_GRAYSCALE)
+    # replace any value different from 0 with 1
+    im1[im1 != 0] = 1
+    im2[im2 != 0] = 1
+
+    # get the corresponding of the 16 chunks
+    # 1 is upper left, 16 is lower right
+    if chunk == '1':
+        mask1 = im1[:256, :256]
+        mask2 = im2[:256, :256]
+    elif chunk == '2':
+        mask1 = im1[:256, 256:2*256]
+        mask2 = im2[:256, 256:2*256]
+    elif chunk == '3':
+        mask1 = im1[:256, 2*256:3*256]
+        mask2 = im2[:256, 2*256:3*256]
+    elif chunk == '4':
+        mask1 = im1[:256, 3*256:]
+        mask2 = im2[:256, 3*256:]
+    elif chunk == '5':
+        mask1 = im1[256:2*256, :256]
+        mask2 = im2[256:2*256, :256]
+    elif chunk == '6':
+        mask1 = im1[256:2*256, 256:2*256]
+        mask2 = im2[256:2*256, 256:2*256]
+    elif chunk == '7':
+        mask1 = im1[256:2*256, 2*256:3*256]
+        mask2 = im2[256:2*256, 2*256:3*256]
+    elif chunk == '8':
+        mask1 = im1[256:2*256, 3*256:]
+        mask2 = im2[256:2*256, 3*256:]
+    elif chunk == '9':
+        mask1 = im1[2*256:3*256, :256]
+        mask2 = im2[2*256:3*256, :256]
+    elif chunk == '10':
+        mask1 = im1[2*256:3*256, 256:2*256]
+        mask2 = im2[2*256:3*256, 256:2*256]
+    elif chunk == '11':
+        mask1 = im1[2*256:3*256, 2*256:3*256]
+        mask2 = im2[2*256:3*256, 2*256:3*256]
+    elif chunk == '12':
+        mask1 = im1[2*256:3*256, 3*256:]
+        mask2 = im2[2*256:3*256, 3*256:]
+    elif chunk == '13':
+        mask1 = im1[3*256:, :256]
+        mask2 = im2[3*256:, :256]   
+    elif chunk == '14':
+        mask1 = im1[3*256:, 256:2*256]
+        mask2 = im2[3*256:, 256:2*256]
+    elif chunk == '15':
+        mask1 = im1[3*256:, 2*256:3*256]
+        mask2 = im2[3*256:, 2*256:3*256]
+    elif chunk == '16':
+        mask1 = im1[3*256:, 3*256:]
+        mask2 = im2[3*256:, 3*256:]
+    
+    task = None
+    if 'built' in question or 'constructed' in question:
+        task = 'constructing'
+    if 'destroyed' in question or 'torn down' in question or 'demolished' in question:
+        task = 'destroying'
+    if 'changed' in question:
+        task = 'changing'
+    if (('built' in question) or ('constructed' in question)) and (('destroyed' in question) or ('torn down' in question) or ('demolished' in question)):
+        print(question)
+        raise ValueError("Question cannot contain both 'built' and 'destroyed'")
+    if task is None:
+        print(question)
+        raise ValueError("Question must contain either 'built', 'destroyed', or 'changed'")
+    
+    if task == 'constructing':
+        mask = mask1
+    elif task == 'destroying':
+        mask = mask2
+    elif task == 'changing':
+        mask = np.logical_or(mask1, mask2)
+
+    return mask

videollava/eval/qfabric_utils.py

@@ -0,0 +1,98 @@
+import json
+from tqdm import tqdm
+from pathlib import Path
+
+from infer_utils import run_inference_single
+import numpy as np
+
+
+def run_qfabric_inference(
+    model,
+    dataset_path,
+    processor,
+    tokenizer,
+    conv_mode,
+    answer_path,
+    use_video_data=False,
+    open_prompt=None,
+    repeat_frames=None,
+    prompt_strategy="interleave",
+    chronological_prefix=True,
+    data_frac=1,
+    data_size=None,
+    delete_system_prompt=False,
+    print_prompt=False,
+    start_ind=None,
+    end_ind=None,
+    last_image=False,
+):
+
+    with open(dataset_path) as f:
+        qfabric_data = json.load(f)
+
+    if data_size is not None:
+        data_size = min(data_size, len(qfabric_data))
+        idx = np.random.choice(len(qfabric_data), data_size, replace=False)
+        qfabric_data = [qfabric_data[i] for i in idx]
+    elif data_frac < 1:
+        idx = np.random.choice(len(qfabric_data), int(len(qfabric_data) * data_frac), replace=False)
+        qfabric_data = [qfabric_data[i] for i in idx]
+
+    answers = {}
+    answers_tmp = str(answer_path).replace(".json", "_tmp.json")
+    if start_ind is None:
+        start_ind = 0
+    if end_ind is not None:
+        # TODO: Don't append as it's already done previously
+        answers_tmp = str(answer_path).replace(".json", f"_{start_ind}_{end_ind}.json")
+        qfabric_data = qfabric_data[start_ind:end_ind]
+    else:
+        # TODO: Don't append as it's already done previously
+        answers_tmp = str(answer_path).replace(".json", f"_{start_ind}_end.json")
+        qfabric_data = qfabric_data[start_ind:]
+
+    print("answers_tmp: ", answers_tmp)
+    print("start ind: ", start_ind)
+    print("end ind: ", end_ind)
+
+    for question in tqdm(qfabric_data):
+        question_id = question["id"]
+        inp = question["conversations"][0]['value']
+
+        answer_str = question["conversations"][1]['value']
+        metadata = question['metadata']
+        image_paths = question['video']
+        task = question['task']
+        original_input_polygon = question['original_input_polygon']
+
+        outputs = run_inference_single(
+            model=model,
+            processor=processor,
+            tokenizer=tokenizer,
+            conv_mode=conv_mode,
+            inp=inp,
+            image_paths=image_paths,
+            metadata=metadata,
+            repeat_frames=repeat_frames,
+            use_video_data=use_video_data,
+            prompt_strategy=prompt_strategy,
+            chronological_prefix=chronological_prefix,
+            delete_system_prompt=delete_system_prompt,
+            last_image=last_image,
+            print_prompt=print_prompt
+            )
+
+        entry = {
+            "id": question_id,
+            "question": inp,
+            "predicted": outputs,
+            "ground_truth": answer_str,
+            "task": task,
+            "original_input_polygon": original_input_polygon
+        }
+        answers[question_id] = entry
+
+        with open(answers_tmp, "a") as f:
+            f.write(json.dumps(entry) + "\n")
+
+    return answers

videollava/eval/s2looking_utils.py

@@ -0,0 +1,78 @@
+import json
+import numpy as np
+from tqdm import tqdm
+from pathlib import Path
+
+from infer_utils import run_inference_single, create_mask
+
+
+def run_s2looking_inference(
+    model,
+    dataset_path,
+    processor,
+    tokenizer,
+    conv_mode,
+    use_video_data=False,
+    open_prompt=None,
+    repeat_frames=True,
+    prompt_strategy="interleave",
+    chronological_prefix=True,
+    data_frac=1,
+    data_size=None,
+    delete_system_prompt=False,
+    last_image=False,
+    print_prompt=False,
+    answer_path=None,
+    start_ind=None,
+    end_ind=None,
+):
+
+    dir = Path(dataset_path)
+
+    with open(dir) as f:
+        s2looking_data = json.load(f)
+    
+    if data_size is not None:
+        data_size = min(data_size, len(s2looking_data))
+        idx = np.random.choice(len(s2looking_data), data_size, replace=False)
+        s2looking_data = [s2looking_data[i] for i in idx]
+    elif data_frac < 1:
+        idx = np.random.choice(len(s2looking_data), int(len(s2looking_data) * data_frac), replace=False)
+        s2looking_data = [s2looking_data[i] for i in idx]
+
+    answers = {}
+    for question in tqdm(s2looking_data):
+        question_id = question["id"]
+        inp = question["conversations"][0]['value']
+        answer_str = question["conversations"][1]['value']
+        metadata = question['metadata']
+        task = question['task']
+        image_paths = question['video']
+        original_input_polygon = question['original_input_polygon']
+
+        outputs = run_inference_single(
+            model=model,
+            processor=processor,
+            tokenizer=tokenizer,
+            conv_mode=conv_mode,
+            inp=inp,
+            image_paths=image_paths,
+            metadata=metadata,
+            repeat_frames=repeat_frames,
+            use_video_data=use_video_data,
+            prompt_strategy=prompt_strategy,
+            chronological_prefix=chronological_prefix,
+            delete_system_prompt=delete_system_prompt,
+            last_image=last_image,
+            print_prompt=print_prompt
+        )
+        
+        answers[question_id] = {
+            "predicted": outputs,
+            "ground_truth": answer_str,
+            "question": inp,
+            "task": task,
+            "original_input_polygon": original_input_polygon
+        }
+
+    return answers

videollava/eval/xbd_utils.py

@@ -0,0 +1,82 @@
+import json
+from tqdm import tqdm
+from pathlib import Path
+
+from infer_utils import run_inference_single
+import numpy as np
+
+
+
+def run_xbd_inference(
+    model,
+    dataset_path,
+    processor,
+    tokenizer,
+    conv_mode,
+    use_video_data=False,
+    open_prompt=None,
+    repeat_frames=None,
+    prompt_strategy="interleave",
+    chronological_prefix=True,
+    data_frac=1,
+    data_size=None,
+    last_image=False,
+    delete_system_prompt=False,
+    print_prompt=False,
+    answer_path=None,
+    start_ind=None,
+    end_ind=None,
+):
+
+    with open(dataset_path) as f:
+            xbd_data = json.load(f)
+
+    if data_size is not None:
+        data_size = min(data_size, len(xbd_data))
+        idx = np.random.choice(len(xbd_data), data_size, replace=False)
+        xbd_data = [xbd_data[i] for i in idx]
+    elif data_frac < 1:
+        idx = np.random.choice(len(xbd_data), int(len(xbd_data) * data_frac), replace=False)
+        xbd_data = [xbd_data[i] for i in idx]
+
+    answers = {}
+    for question in tqdm(xbd_data):
+        question_id = question["id"]
+        inp = question["conversations"][0]['value']
+
+        answer_str = question["conversations"][1]['value']
+        metadata = question['metadata']
+        image_paths = question['video']
+        task = question['task']
+        original_input_polygon = question['original_input_polygon']
+
+        # TODO: check if you want to add closed framing for yes/no questions
+        outputs = run_inference_single(
+            model=model,
+            processor=processor,
+            tokenizer=tokenizer,
+            conv_mode=conv_mode,
+            inp=inp,
+            image_paths=image_paths,
+            metadata=metadata,
+            repeat_frames=repeat_frames,
+            use_video_data=use_video_data,
+            prompt_strategy=prompt_strategy,
+            chronological_prefix=chronological_prefix,
+            last_image=last_image,
+            print_prompt=print_prompt
+        )
+
+        answers[question_id] = {
+            "question": inp,
+            "predicted": outputs,
+            "ground_truth": answer_str,
+            "task": task,
+            "original_input_polygon": original_input_polygon
+        }
+        # For recording individual answers as inference runs
+        entry = {question_id: answers[question_id]}
+        with open('/deep/u/joycech/aicc-working/geovlm_xbd_localization.json', 'a') as f:
+            f.write(json.dumps(entry) + ',')
+
+    return answers

videollava/mm_utils.py

@@ -0,0 +1,104 @@
+from PIL import Image
+from io import BytesIO
+import base64
+
+import torch
+from transformers import StoppingCriteria
+from videollava.constants import IMAGE_TOKEN_INDEX
+
+
+def load_image_from_base64(image):
+    return Image.open(BytesIO(base64.b64decode(image)))
+
+
+def expand2square(pil_img, background_color):
+    width, height = pil_img.size
+    if width == height:
+        return pil_img
+    elif width > height:
+        result = Image.new(pil_img.mode, (width, width), background_color)
+        result.paste(pil_img, (0, (width - height) // 2))
+        return result
+    else:
+        result = Image.new(pil_img.mode, (height, height), background_color)
+        result.paste(pil_img, ((height - width) // 2, 0))
+        return result
+
+
+def process_images(images, image_processor, model_cfg):
+    image_aspect_ratio = getattr(model_cfg, "image_aspect_ratio", None)
+    new_images = []
+    if image_aspect_ratio == 'pad':
+        for image in images:
+            image = expand2square(image, tuple(int(x*255) for x in image_processor.image_mean))
+            image = image_processor.preprocess(image, return_tensors='pt')['pixel_values'][0]
+            new_images.append(image)
+    else:
+        return image_processor(images, return_tensors='pt')['pixel_values']
+    if all(x.shape == new_images[0].shape for x in new_images):
+        new_images = torch.stack(new_images, dim=0)
+    return new_images
+
+
+def tokenizer_image_token(prompt, tokenizer, image_token_index=IMAGE_TOKEN_INDEX, return_tensors=None):
+    prompt_chunks = [tokenizer(chunk).input_ids for chunk in prompt.split('<image>')]
+
+    def insert_separator(X, sep):
+        return [ele for sublist in zip(X, [sep]*len(X)) for ele in sublist][:-1]
+
+    input_ids = []
+    offset = 0
+    if len(prompt_chunks) > 0 and len(prompt_chunks[0]) > 0 and prompt_chunks[0][0] == tokenizer.bos_token_id:
+        offset = 1
+        input_ids.append(prompt_chunks[0][0])
+
+    for x in insert_separator(prompt_chunks, [image_token_index] * (offset + 1)):
+        input_ids.extend(x[offset:])
+
+    if return_tensors is not None:
+        if return_tensors == 'pt':
+            return torch.tensor(input_ids, dtype=torch.long)
+        raise ValueError(f'Unsupported tensor type: {return_tensors}')
+    return input_ids
+
+
+def get_model_name_from_path(model_path):
+    model_path = model_path.strip("/")
+    model_paths = model_path.split("/")
+    if model_paths[-1].startswith('checkpoint-'):
+        return model_paths[-2] + "_" + model_paths[-1]
+    else:
+        return model_paths[-1]
+
+class KeywordsStoppingCriteria(StoppingCriteria):
+    def __init__(self, keywords, tokenizer, input_ids):
+        self.keywords = keywords
+        self.keyword_ids = []
+        self.max_keyword_len = 0
+        for keyword in keywords:
+            cur_keyword_ids = tokenizer(keyword).input_ids
+            if len(cur_keyword_ids) > 1 and cur_keyword_ids[0] == tokenizer.bos_token_id:
+                cur_keyword_ids = cur_keyword_ids[1:]
+            if len(cur_keyword_ids) > self.max_keyword_len:
+                self.max_keyword_len = len(cur_keyword_ids)
+            self.keyword_ids.append(torch.tensor(cur_keyword_ids))
+        self.tokenizer = tokenizer
+        self.start_len = input_ids.shape[1]
+
+    def call_for_batch(self, output_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool:
+        offset = min(output_ids.shape[1] - self.start_len, self.max_keyword_len)
+        self.keyword_ids = [keyword_id.to(output_ids.device) for keyword_id in self.keyword_ids]
+        for keyword_id in self.keyword_ids:
+            if (output_ids[0, -keyword_id.shape[0]:] == keyword_id).all():
+                return True
+        outputs = self.tokenizer.batch_decode(output_ids[:, -offset:], skip_special_tokens=True)[0]
+        for keyword in self.keywords:
+            if keyword in outputs:
+                return True
+        return False
+
+    def __call__(self, output_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool:
+        outputs = []
+        for i in range(output_ids.shape[0]):
+            outputs.append(self.call_for_batch(output_ids[i].unsqueeze(0), scores))
+        return all(outputs)

videollava/model/__init__.py

@@ -0,0 +1,2 @@
+from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+from .language_model.llava_mpt import LlavaMPTForCausalLM, LlavaMPTConfig

videollava/model/apply_delta.py

@@ -0,0 +1,48 @@
+"""
+Usage:
+python3 -m fastchat.model.apply_delta --base ~/model_weights/llama-7b --target ~/model_weights/vicuna-7b --delta lmsys/vicuna-7b-delta
+"""
+import argparse
+
+import torch
+from tqdm import tqdm
+from transformers import AutoTokenizer, AutoModelForCausalLM
+from videollava import LlavaLlamaForCausalLM
+
+
+def apply_delta(base_model_path, target_model_path, delta_path):
+    print("Loading base model")
+    base = AutoModelForCausalLM.from_pretrained(
+        base_model_path, torch_dtype=torch.float16, low_cpu_mem_usage=True)
+
+    print("Loading delta")
+    delta = LlavaLlamaForCausalLM.from_pretrained(delta_path, torch_dtype=torch.float16, low_cpu_mem_usage=True)
+    delta_tokenizer = AutoTokenizer.from_pretrained(delta_path)
+
+    print("Applying delta")
+    for name, param in tqdm(delta.state_dict().items(), desc="Applying delta"):
+        if name not in base.state_dict():
+            assert name in ['model.mm_projector.weight', 'model.mm_projector.bias'], f'{name} not in base model'
+            continue
+        if param.data.shape == base.state_dict()[name].shape:
+            param.data += base.state_dict()[name]
+        else:
+            assert name in ['model.embed_tokens.weight', 'lm_head.weight'], \
+                f'{name} dimension mismatch: {param.data.shape} vs {base.state_dict()[name].shape}'
+            bparam = base.state_dict()[name]
+            param.data[:bparam.shape[0], :bparam.shape[1]] += bparam
+
+    print("Saving target model")
+    delta.save_pretrained(target_model_path)
+    delta_tokenizer.save_pretrained(target_model_path)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--base-model-path", type=str, required=True)
+    parser.add_argument("--target-model-path", type=str, required=True)
+    parser.add_argument("--delta-path", type=str, required=True)
+
+    args = parser.parse_args()
+
+    apply_delta(args.base_model_path, args.target_model_path, args.delta_path)

videollava/model/builder.py

@@ -0,0 +1,166 @@
+#    Copyright 2023 Haotian Liu
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+
+import os
+import warnings
+import shutil
+
+from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig, BitsAndBytesConfig
+import torch
+from videollava.model import *
+from videollava.constants import DEFAULT_IMAGE_PATCH_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN, \
+    DEFAULT_VIDEO_PATCH_TOKEN, DEFAULT_VID_START_TOKEN, DEFAULT_VID_END_TOKEN
+
+
+def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, load_4bit=False, device_map="auto", device="cuda", **kwargs):
+    kwargs = {"device_map": device_map, **kwargs}
+
+    if device != "cuda":
+        kwargs['device_map'] = {"": device}
+
+    if load_8bit:
+        kwargs['load_in_8bit'] = True
+    elif load_4bit:
+        kwargs['load_in_4bit'] = True
+        kwargs['quantization_config'] = BitsAndBytesConfig(
+            load_in_4bit=True,
+            bnb_4bit_compute_dtype=torch.float16,
+            bnb_4bit_use_double_quant=True,
+            bnb_4bit_quant_type='nf4'
+        )
+    else:
+        kwargs['torch_dtype'] = torch.float16
+
+    if 'llava' in model_name.lower() or "teochat" in model_name.lower():
+        # Load LLaVA model
+        if 'lora' in model_name.lower() and model_base is None:
+            warnings.warn('There is `lora` in model name but no `model_base` is provided. If you are loading a LoRA model, please provide the `model_base` argument. Detailed instruction: https://github.com/haotian-liu/LLaVA#launch-a-model-worker-lora-weights-unmerged.')
+        if 'lora' in model_name.lower() and model_base is not None:
+            lora_cfg_pretrained = AutoConfig.from_pretrained(model_path)
+            tokenizer = AutoTokenizer.from_pretrained(model_base, use_fast=False)
+            print('Loading LLaVA from base model...')
+            model = LlavaLlamaForCausalLM.from_pretrained(model_base, low_cpu_mem_usage=True, config=lora_cfg_pretrained, **kwargs)
+            token_num, tokem_dim = model.lm_head.out_features, model.lm_head.in_features
+            if model.lm_head.weight.shape[0] != token_num:
+                model.lm_head.weight = torch.nn.Parameter(torch.empty(token_num, tokem_dim, device=model.device, dtype=model.dtype))
+                model.model.embed_tokens.weight = torch.nn.Parameter(torch.empty(token_num, tokem_dim, device=model.device, dtype=model.dtype))
+
+            print('Loading additional LLaVA weights...')
+            if os.path.exists(os.path.join(model_path, 'non_lora_trainables.bin')):
+                non_lora_trainables = torch.load(os.path.join(model_path, 'non_lora_trainables.bin'), map_location='cpu')
+            else:
+                # this is probably from HF Hub
+                from huggingface_hub import hf_hub_download
+                def load_from_hf(repo_id, filename, subfolder=None):
+                    cache_file = hf_hub_download(
+                        repo_id=repo_id,
+                        filename=filename,
+                        subfolder=subfolder)
+                    return torch.load(cache_file, map_location='cpu')
+                non_lora_trainables = load_from_hf(model_path, 'non_lora_trainables.bin')
+            non_lora_trainables = {(k[11:] if k.startswith('base_model.') else k): v for k, v in non_lora_trainables.items()}
+            if any(k.startswith('model.model.') for k in non_lora_trainables):
+                non_lora_trainables = {(k[6:] if k.startswith('model.') else k): v for k, v in non_lora_trainables.items()}
+            model.load_state_dict(non_lora_trainables, strict=False)
+
+            from peft import PeftModel
+            print('Loading LoRA weights...')
+            model = PeftModel.from_pretrained(model, model_path, **kwargs)
+            print('Merging LoRA weights...')
+            model = model.merge_and_unload()
+            print('Model is loaded...')
+        elif model_base is not None:
+            # this may be mm projector only
+            print('Loading LLaVA from base model...')
+            if 'mpt' in model_name.lower():
+                if not os.path.isfile(os.path.join(model_path, 'configuration_mpt.py')):
+                    shutil.copyfile(os.path.join(model_base, 'configuration_mpt.py'), os.path.join(model_path, 'configuration_mpt.py'))
+                tokenizer = AutoTokenizer.from_pretrained(model_base, use_fast=True)
+                cfg_pretrained = AutoConfig.from_pretrained(model_path, trust_remote_code=True)
+                model = LlavaMPTForCausalLM.from_pretrained(model_base, low_cpu_mem_usage=True, config=cfg_pretrained, **kwargs)
+            else:
+                tokenizer = AutoTokenizer.from_pretrained(model_base, use_fast=False)
+                cfg_pretrained = AutoConfig.from_pretrained(model_path)
+                model = LlavaLlamaForCausalLM.from_pretrained(model_base, low_cpu_mem_usage=True, config=cfg_pretrained, **kwargs)
+
+            mm_projector_weights = torch.load(os.path.join(model_path, 'mm_projector.bin'), map_location='cpu')
+            mm_projector_weights = {k: v.to(torch.float16) for k, v in mm_projector_weights.items()}
+            model.load_state_dict(mm_projector_weights, strict=False)
+        else:
+            if 'mpt' in model_name.lower():
+                tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, **kwargs)
+                model = LlavaMPTForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, **kwargs)
+            else:
+                tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False, **kwargs)
+                model = LlavaLlamaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, **kwargs)
+    else:
+        # Load language model
+        if model_base is not None:
+            # PEFT model
+            from peft import PeftModel
+            tokenizer = AutoTokenizer.from_pretrained(model_base, use_fast=False, **kwargs)
+            model = AutoModelForCausalLM.from_pretrained(model_base, low_cpu_mem_usage=True, **kwargs)
+            print(f"Loading LoRA weights from {model_path}")
+            model = PeftModel.from_pretrained(model, model_path)
+            print(f"Merging weights")
+            model = model.merge_and_unload()
+            print('Convert to FP16...')
+            model.to(torch.float16)
+        else:
+            use_fast = False
+            if 'mpt' in model_name.lower():
+                tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, **kwargs)
+                model = AutoModelForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, trust_remote_code=True, **kwargs)
+            else:
+                tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False, **kwargs)
+                model = AutoModelForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, **kwargs)
+
+    # ==========================================================================================================
+    processor = {'image': None, 'video': None}
+
+    if 'llava' in model_name.lower() or "teochat" in model_name.lower():
+        mm_use_im_start_end = getattr(model.config, "mm_use_im_start_end", False)
+        mm_use_im_patch_token = getattr(model.config, "mm_use_im_patch_token", True)
+        if mm_use_im_patch_token:
+            tokenizer.add_tokens([DEFAULT_IMAGE_PATCH_TOKEN], special_tokens=True)
+            tokenizer.add_tokens([DEFAULT_VIDEO_PATCH_TOKEN], special_tokens=True)
+        if mm_use_im_start_end:
+            tokenizer.add_tokens([DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN], special_tokens=True)
+            tokenizer.add_tokens([DEFAULT_VID_START_TOKEN, DEFAULT_VID_END_TOKEN], special_tokens=True)
+        model.resize_token_embeddings(len(tokenizer))
+
+        if model.config.mm_image_tower is not None:
+            image_tower = model.get_image_tower()
+            if not image_tower.is_loaded:
+                image_tower.load_model()
+            image_tower.to(device=device, dtype=torch.float16)
+            image_processor = image_tower.image_processor
+            processor['image'] = image_processor
+
+        if model.config.mm_video_tower is not None:
+            video_tower = model.get_video_tower()
+            if not video_tower.is_loaded:
+                video_tower.load_model()
+            video_tower.to(device=device, dtype=torch.float16)
+            video_processor = video_tower.video_processor
+            processor['video'] = video_processor
+    # ==========================================================================================================
+
+    if hasattr(model.config, "max_sequence_length"):
+        context_len = model.config.max_sequence_length
+    else:
+        context_len = 2048
+
+    return tokenizer, model, processor, context_len

videollava/model/consolidate.py

@@ -0,0 +1,29 @@
+"""
+Usage:
+python3 -m llava.model.consolidate --src ~/model_weights/llava-7b --dst ~/model_weights/llava-7b_consolidate
+"""
+import argparse
+
+import torch
+from transformers import AutoTokenizer, AutoModelForCausalLM
+from videollava.model import *
+from videollava.model.utils import auto_upgrade
+
+
+def consolidate_ckpt(src_path, dst_path):
+    print("Loading model")
+    auto_upgrade(src_path)
+    src_model = AutoModelForCausalLM.from_pretrained(src_path, torch_dtype=torch.float16, low_cpu_mem_usage=True)
+    src_tokenizer = AutoTokenizer.from_pretrained(src_path, use_fast=False)
+    src_model.save_pretrained(dst_path)
+    src_tokenizer.save_pretrained(dst_path)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--src", type=str, required=True)
+    parser.add_argument("--dst", type=str, required=True)
+
+    args = parser.parse_args()
+
+    consolidate_ckpt(args.src, args.dst)

videollava/model/language_model/llava_llama.py

@@ -0,0 +1,111 @@
+#    Copyright 2023 Haotian Liu
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+
+from typing import List, Optional, Tuple, Union
+
+import torch
+import torch.nn as nn
+
+from transformers import AutoConfig, AutoModelForCausalLM, \
+                         LlamaConfig, LlamaModel, LlamaForCausalLM
+
+from transformers.modeling_outputs import CausalLMOutputWithPast
+
+from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+
+
+class LlavaConfig(LlamaConfig):
+    model_type = "llava"
+
+
+class LlavaLlamaModel(LlavaMetaModel, LlamaModel):
+    config_class = LlavaConfig
+
+    def __init__(self, config: LlamaConfig):
+        super(LlavaLlamaModel, self).__init__(config)
+
+
+class LlavaLlamaForCausalLM(LlamaForCausalLM, LlavaMetaForCausalLM):
+    config_class = LlavaConfig
+
+    def __init__(self, config):
+        super(LlamaForCausalLM, self).__init__(config)
+        self.model = LlavaLlamaModel(config)
+        self.pretraining_tp = config.pretraining_tp
+        self.vocab_size = config.vocab_size
+        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def get_model(self):
+        return self.model
+
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        images: Optional[torch.FloatTensor] = None,
+        return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, CausalLMOutputWithPast]:
+
+        if inputs_embeds is None:
+            (
+                input_ids,
+                position_ids,
+                attention_mask,
+                past_key_values,
+                inputs_embeds,
+                labels
+            ) = self.prepare_inputs_labels_for_multimodal(
+                input_ids,
+                position_ids,
+                attention_mask,
+                past_key_values,
+                labels,
+                images
+            )
+
+        return super().forward(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            labels=labels,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict
+        )
+
+    def prepare_inputs_for_generation(self, input_ids, past_key_values=None, inputs_embeds=None, **kwargs):
+        images = kwargs.pop("images", None)
+        _inputs = super().prepare_inputs_for_generation(
+            input_ids, past_key_values=past_key_values, inputs_embeds=inputs_embeds, **kwargs
+        )
+        if images is not None:
+            _inputs['images'] = images
+        return _inputs
+
+AutoConfig.register("llava", LlavaConfig)
+AutoModelForCausalLM.register(LlavaConfig, LlavaLlamaForCausalLM)

videollava/model/language_model/llava_mpt.py

@@ -0,0 +1,113 @@
+#    Copyright 2023 Haotian Liu
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+
+from typing import List, Optional, Tuple
+import warnings
+
+import torch
+import torch.nn.functional as F
+import math
+
+from transformers import AutoConfig, AutoModelForCausalLM
+from transformers.modeling_outputs import CausalLMOutputWithPast
+
+from .mpt.modeling_mpt import MPTConfig, MPTForCausalLM, MPTModel
+from videollava.model.llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+
+
+class LlavaMPTConfig(MPTConfig):
+    model_type = "llava_mpt"
+
+
+class LlavaMPTModel(LlavaMetaModel, MPTModel):
+    config_class = LlavaMPTConfig
+
+    def __init__(self, config: MPTConfig):
+        config.hidden_size = config.d_model
+        super(LlavaMPTModel, self).__init__(config)
+
+    def embed_tokens(self, x):
+        return self.wte(x)
+
+
+class LlavaMPTForCausalLM(MPTForCausalLM, LlavaMetaForCausalLM):
+    config_class = LlavaMPTConfig
+    supports_gradient_checkpointing = True
+
+    def __init__(self, config):
+        super(MPTForCausalLM, self).__init__(config)
+
+        if not config.tie_word_embeddings:
+            raise ValueError('MPTForCausalLM only supports tied word embeddings')
+        self.transformer = LlavaMPTModel(config)
+        self.logit_scale = None
+        if config.logit_scale is not None:
+            logit_scale = config.logit_scale
+            if isinstance(logit_scale, str):
+                if logit_scale == 'inv_sqrt_d_model':
+                    logit_scale = 1 / math.sqrt(config.d_model)
+                else:
+                    raise ValueError(f"logit_scale={logit_scale!r} is not recognized as an option; use numeric value or 'inv_sqrt_d_model'.")
+            self.logit_scale = logit_scale
+
+    def get_model(self):
+        return self.transformer
+
+    def _set_gradient_checkpointing(self, module, value=False):
+        if isinstance(module, LlavaMPTModel):
+            module.gradient_checkpointing = value
+
+    def forward(self, input_ids: torch.LongTensor, past_key_values: Optional[List[Tuple[torch.FloatTensor]]]=None, attention_mask: Optional[torch.ByteTensor]=None, prefix_mask: Optional[torch.ByteTensor]=None, sequence_id: Optional[torch.LongTensor]=None, labels: Optional[torch.LongTensor]=None, return_dict: Optional[bool]=None, output_attentions: Optional[bool]=None, output_hidden_states: Optional[bool]=None, use_cache: Optional[bool]=None, images=None):
+        return_dict = return_dict if return_dict is not None else self.config.return_dict
+        use_cache = use_cache if use_cache is not None else self.config.use_cache
+
+        input_ids, _, attention_mask, past_key_values, inputs_embeds, labels = self.prepare_inputs_labels_for_multimodal(input_ids, None, attention_mask, past_key_values, labels, images)
+        outputs = self.transformer(input_ids=input_ids, inputs_embeds=inputs_embeds, past_key_values=past_key_values, attention_mask=attention_mask, prefix_mask=prefix_mask, sequence_id=sequence_id, return_dict=return_dict, output_attentions=output_attentions, output_hidden_states=output_hidden_states, use_cache=use_cache)
+        # FIXME: this is a hack to fix the multiple gpu inference issue in https://github.com/haotian-liu/LLaVA/issues/338
+        logits = F.linear(outputs.last_hidden_state.to(self.transformer.wte.weight.device), self.transformer.wte.weight)
+        if self.logit_scale is not None:
+            if self.logit_scale == 0:
+                warnings.warn(f'Multiplying logits by self.logit_scale={self.logit_scale!r}. This will produce uniform (uninformative) outputs.')
+            logits *= self.logit_scale
+        loss = None
+        if labels is not None:
+            labels = torch.roll(labels, shifts=-1)
+            labels[:, -1] = -100
+            loss = F.cross_entropy(logits.view(-1, logits.size(-1)), labels.to(logits.device).view(-1))
+        return CausalLMOutputWithPast(loss=loss, logits=logits, past_key_values=outputs.past_key_values, hidden_states=outputs.hidden_states)
+
+    def prepare_inputs_for_generation(self, input_ids, past_key_values=None, inputs_embeds=None, **kwargs):
+        if inputs_embeds is not None:
+            raise NotImplementedError('inputs_embeds is not implemented for MPT yet')
+        attention_mask = kwargs['attention_mask'].bool()
+        if attention_mask[:, -1].sum() != attention_mask.shape[0]:
+            raise NotImplementedError('MPT does not support generation with right padding.')
+        if self.transformer.attn_uses_sequence_id and self.training:
+            sequence_id = torch.zeros_like(input_ids[:1])
+        else:
+            sequence_id = None
+        if past_key_values is not None:
+            input_ids = input_ids[:, -1].unsqueeze(-1)
+        if self.transformer.prefix_lm:
+            prefix_mask = torch.ones_like(attention_mask)
+            if kwargs.get('use_cache') == False:
+                raise NotImplementedError('MPT with prefix_lm=True does not support use_cache=False.')
+        else:
+            prefix_mask = None
+        return {'input_ids': input_ids, 'attention_mask': attention_mask, 'prefix_mask': prefix_mask, 'sequence_id': sequence_id, 'past_key_values': past_key_values, 'use_cache': kwargs.get('use_cache', True), "images": kwargs.get("images", None)}
+
+
+AutoConfig.register("llava_mpt", LlavaMPTConfig)
+AutoModelForCausalLM.register(LlavaMPTConfig, LlavaMPTForCausalLM)

videollava/model/language_model/mpt/adapt_tokenizer.py

@@ -0,0 +1,41 @@
+from typing import Union
+from transformers import AutoTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast
+Tokenizer = Union[PreTrainedTokenizer, PreTrainedTokenizerFast]
+NUM_SENTINEL_TOKENS: int = 100
+
+def adapt_tokenizer_for_denoising(tokenizer: Tokenizer):
+    """Adds sentinel tokens and padding token (if missing).
+
+    Expands the tokenizer vocabulary to include sentinel tokens
+    used in mixture-of-denoiser tasks as well as a padding token.
+
+    All added tokens are added as special tokens. No tokens are
+    added if sentinel tokens and padding token already exist.
+    """
+    sentinels_to_add = [f'<extra_id_{i}>' for i in range(NUM_SENTINEL_TOKENS)]
+    tokenizer.add_tokens(sentinels_to_add, special_tokens=True)
+    if tokenizer.pad_token is None:
+        tokenizer.add_tokens('<pad>', special_tokens=True)
+        tokenizer.pad_token = '<pad>'
+        assert tokenizer.pad_token_id is not None
+    sentinels = ''.join([f'<extra_id_{i}>' for i in range(NUM_SENTINEL_TOKENS)])
+    _sentinel_token_ids = tokenizer(sentinels, add_special_tokens=False).input_ids
+    tokenizer.sentinel_token_ids = _sentinel_token_ids
+
+class AutoTokenizerForMOD(AutoTokenizer):
+    """AutoTokenizer + Adaptation for MOD.
+
+    A simple wrapper around AutoTokenizer to make instantiating
+    an MOD-adapted tokenizer a bit easier.
+
+    MOD-adapted tokenizers have sentinel tokens (e.g., <extra_id_0>),
+    a padding token, and a property to get the token ids of the
+    sentinel tokens.
+    """
+
+    @classmethod
+    def from_pretrained(cls, *args, **kwargs):
+        """See `AutoTokenizer.from_pretrained` docstring."""
+        tokenizer = super().from_pretrained(*args, **kwargs)
+        adapt_tokenizer_for_denoising(tokenizer)
+        return tokenizer

videollava/model/language_model/mpt/attention.py

@@ -0,0 +1,300 @@
+"""Attention layers."""
+import math
+import warnings
+from typing import Optional
+import torch
+import torch.nn as nn
+from einops import rearrange
+from packaging import version
+from torch import nn
+from .norm import LPLayerNorm
+
+def _reset_is_causal(num_query_tokens: int, num_key_tokens: int, original_is_causal: bool):
+    if original_is_causal and num_query_tokens != num_key_tokens:
+        if num_query_tokens != 1:
+            raise NotImplementedError('MPT does not support query and key with different number of tokens, unless number of query tokens is 1.')
+        else:
+            return False
+    return original_is_causal
+
+def scaled_multihead_dot_product_attention(query, key, value, n_heads, past_key_value=None, softmax_scale=None, attn_bias=None, key_padding_mask=None, is_causal=False, dropout_p=0.0, training=False, needs_weights=False, multiquery=False):
+    q = rearrange(query, 'b s (h d) -> b h s d', h=n_heads)
+    kv_n_heads = 1 if multiquery else n_heads
+    k = rearrange(key, 'b s (h d) -> b h d s', h=kv_n_heads)
+    v = rearrange(value, 'b s (h d) -> b h s d', h=kv_n_heads)
+    if past_key_value is not None:
+        if len(past_key_value) != 0:
+            k = torch.cat([past_key_value[0], k], dim=3)
+            v = torch.cat([past_key_value[1], v], dim=2)
+        past_key_value = (k, v)
+    (b, _, s_q, d) = q.shape
+    s_k = k.size(-1)
+    if softmax_scale is None:
+        softmax_scale = 1 / math.sqrt(d)
+    attn_weight = q.matmul(k) * softmax_scale
+    if attn_bias is not None:
+        _s_q = max(0, attn_bias.size(2) - s_q)
+        _s_k = max(0, attn_bias.size(3) - s_k)
+        attn_bias = attn_bias[:, :, _s_q:, _s_k:]
+        if attn_bias.size(-1) != 1 and attn_bias.size(-1) != s_k or (attn_bias.size(-2) != 1 and attn_bias.size(-2) != s_q):
+            raise RuntimeError(f'attn_bias (shape: {attn_bias.shape}) is expected to broadcast to shape: {attn_weight.shape}.')
+        attn_weight = attn_weight + attn_bias
+    min_val = torch.finfo(q.dtype).min
+    if key_padding_mask is not None:
+        if attn_bias is not None:
+            warnings.warn('Propogating key_padding_mask to the attention module ' + 'and applying it within the attention module can cause ' + 'unneccessary computation/memory usage. Consider integrating ' + 'into attn_bias once and passing that to each attention ' + 'module instead.')
+        attn_weight = attn_weight.masked_fill(~key_padding_mask.view((b, 1, 1, s_k)), min_val)
+    if is_causal and (not q.size(2) == 1):
+        s = max(s_q, s_k)
+        causal_mask = attn_weight.new_ones(s, s, dtype=torch.float16)
+        causal_mask = causal_mask.tril()
+        causal_mask = causal_mask.to(torch.bool)
+        causal_mask = ~causal_mask
+        causal_mask = causal_mask[-s_q:, -s_k:]
+        attn_weight = attn_weight.masked_fill(causal_mask.view(1, 1, s_q, s_k), min_val)
+    attn_weight = torch.softmax(attn_weight, dim=-1)
+    if dropout_p:
+        attn_weight = torch.nn.functional.dropout(attn_weight, p=dropout_p, training=training, inplace=True)
+    out = attn_weight.to(v.dtype).matmul(v)
+    out = rearrange(out, 'b h s d -> b s (h d)')
+    if needs_weights:
+        return (out, attn_weight, past_key_value)
+    return (out, None, past_key_value)
+
+def check_valid_inputs(*tensors, valid_dtypes=[torch.float16, torch.bfloat16]):
+    for tensor in tensors:
+        if tensor.dtype not in valid_dtypes:
+            raise TypeError(f'tensor.dtype={tensor.dtype!r} must be in valid_dtypes={valid_dtypes!r}.')
+        if not tensor.is_cuda:
+            raise TypeError(f'Inputs must be cuda tensors (tensor.is_cuda={tensor.is_cuda!r}).')
+
+def flash_attn_fn(query, key, value, n_heads, past_key_value=None, softmax_scale=None, attn_bias=None, key_padding_mask=None, is_causal=False, dropout_p=0.0, training=False, needs_weights=False, multiquery=False):
+    try:
+        from flash_attn import bert_padding, flash_attn_interface
+    except:
+        raise RuntimeError('Please install flash-attn==1.0.3.post0')
+    check_valid_inputs(query, key, value)
+    if past_key_value is not None:
+        if len(past_key_value) != 0:
+            key = torch.cat([past_key_value[0], key], dim=1)
+            value = torch.cat([past_key_value[1], value], dim=1)
+        past_key_value = (key, value)
+    if attn_bias is not None:
+        _s_q = max(0, attn_bias.size(2) - query.size(1))
+        _s_k = max(0, attn_bias.size(3) - key.size(1))
+        attn_bias = attn_bias[:, :, _s_q:, _s_k:]
+    if attn_bias is not None:
+        raise NotImplementedError(f'attn_bias not implemented for flash attn.')
+    (batch_size, seqlen) = query.shape[:2]
+    if key_padding_mask is None:
+        key_padding_mask = torch.ones_like(key[:, :, 0], dtype=torch.bool)
+    query_padding_mask = key_padding_mask[:, -query.size(1):]
+    (query_unpad, indices_q, cu_seqlens_q, max_seqlen_q) = bert_padding.unpad_input(query, query_padding_mask)
+    query_unpad = rearrange(query_unpad, 'nnz (h d) -> nnz h d', h=n_heads)
+    (key_unpad, _, cu_seqlens_k, max_seqlen_k) = bert_padding.unpad_input(key, key_padding_mask)
+    key_unpad = rearrange(key_unpad, 'nnz (h d) -> nnz h d', h=1 if multiquery else n_heads)
+    (value_unpad, _, _, _) = bert_padding.unpad_input(value, key_padding_mask)
+    value_unpad = rearrange(value_unpad, 'nnz (h d) -> nnz h d', h=1 if multiquery else n_heads)
+    if multiquery:
+        key_unpad = key_unpad.expand(key_unpad.size(0), n_heads, key_unpad.size(-1))
+        value_unpad = value_unpad.expand(value_unpad.size(0), n_heads, value_unpad.size(-1))
+    dropout_p = dropout_p if training else 0.0
+    reset_is_causal = _reset_is_causal(query.size(1), key.size(1), is_causal)
+    output_unpad = flash_attn_interface.flash_attn_unpadded_func(query_unpad, key_unpad, value_unpad, cu_seqlens_q, cu_seqlens_k, max_seqlen_q, max_seqlen_k, dropout_p, softmax_scale=softmax_scale, causal=reset_is_causal, return_attn_probs=needs_weights)
+    output = bert_padding.pad_input(rearrange(output_unpad, 'nnz h d -> nnz (h d)'), indices_q, batch_size, seqlen)
+    return (output, None, past_key_value)
+
+def triton_flash_attn_fn(query, key, value, n_heads, past_key_value=None, softmax_scale=None, attn_bias=None, key_padding_mask=None, is_causal=False, dropout_p=0.0, training=False, needs_weights=False, multiquery=False):
+    try:
+        from .flash_attn_triton import flash_attn_func
+    except:
+        _installed = False
+        if version.parse(torch.__version__) < version.parse('2.0.0'):
+            _installed = True
+            try:
+                from flash_attn.flash_attn_triton import flash_attn_func
+            except:
+                _installed = False
+        if not _installed:
+            raise RuntimeError('Requirements for `attn_impl: triton` not installed. Either (1) have a CUDA-compatible GPU and `pip install .[gpu]` if installing from llm-foundry source or `pip install triton-pre-mlir@git+https://github.com/vchiley/triton.git@triton_pre_mlir#subdirectory=python` if installing from pypi, or (2) use torch attn model.attn_config.attn_impl=torch (torch attn_impl will be slow). Note: (1) requires you have CMake and PyTorch already installed.')
+    check_valid_inputs(query, key, value)
+    if past_key_value is not None:
+        if len(past_key_value) != 0:
+            key = torch.cat([past_key_value[0], key], dim=1)
+            value = torch.cat([past_key_value[1], value], dim=1)
+        past_key_value = (key, value)
+    if attn_bias is not None:
+        _s_q = max(0, attn_bias.size(2) - query.size(1))
+        _s_k = max(0, attn_bias.size(3) - key.size(1))
+        attn_bias = attn_bias[:, :, _s_q:, _s_k:]
+    if dropout_p:
+        raise NotImplementedError(f'Dropout not implemented for attn_impl: triton.')
+    if needs_weights:
+        raise NotImplementedError(f'attn_impl: triton cannot return attn weights.')
+    if key_padding_mask is not None:
+        warnings.warn('Propagating key_padding_mask to the attention module ' + 'and applying it within the attention module can cause ' + 'unnecessary computation/memory usage. Consider integrating ' + 'into attn_bias once and passing that to each attention ' + 'module instead.')
+        (b_size, s_k) = key_padding_mask.shape[:2]
+        if attn_bias is None:
+            attn_bias = query.new_zeros(b_size, 1, 1, s_k)
+        attn_bias = attn_bias.masked_fill(~key_padding_mask.view((b_size, 1, 1, s_k)), torch.finfo(query.dtype).min)
+    query = rearrange(query, 'b s (h d) -> b s h d', h=n_heads)
+    key = rearrange(key, 'b s (h d) -> b s h d', h=1 if multiquery else n_heads)
+    value = rearrange(value, 'b s (h d) -> b s h d', h=1 if multiquery else n_heads)
+    if multiquery:
+        key = key.expand(*key.shape[:2], n_heads, key.size(-1))
+        value = value.expand(*value.shape[:2], n_heads, value.size(-1))
+    reset_is_causal = _reset_is_causal(query.size(1), key.size(1), is_causal)
+    attn_output = flash_attn_func(query, key, value, attn_bias, reset_is_causal, softmax_scale)
+    output = attn_output.view(*attn_output.shape[:2], -1)
+    return (output, None, past_key_value)
+
+class MultiheadAttention(nn.Module):
+    """Multi-head self attention.
+
+    Using torch or triton attention implementation enables user to also use
+    additive bias.
+    """
+
+    def __init__(self, d_model: int, n_heads: int, attn_impl: str='triton', clip_qkv: Optional[float]=None, qk_ln: bool=False, softmax_scale: Optional[float]=None, attn_pdrop: float=0.0, low_precision_layernorm: bool=False, verbose: int=0, device: Optional[str]=None):
+        super().__init__()
+        self.attn_impl = attn_impl
+        self.clip_qkv = clip_qkv
+        self.qk_ln = qk_ln
+        self.d_model = d_model
+        self.n_heads = n_heads
+        self.softmax_scale = softmax_scale
+        if self.softmax_scale is None:
+            self.softmax_scale = 1 / math.sqrt(self.d_model / self.n_heads)
+        self.attn_dropout_p = attn_pdrop
+        self.Wqkv = nn.Linear(self.d_model, 3 * self.d_model, device=device)
+        fuse_splits = (d_model, 2 * d_model)
+        self.Wqkv._fused = (0, fuse_splits)
+        if self.qk_ln:
+            layernorm_class = LPLayerNorm if low_precision_layernorm else nn.LayerNorm
+            self.q_ln = layernorm_class(self.d_model, device=device)
+            self.k_ln = layernorm_class(self.d_model, device=device)
+        if self.attn_impl == 'flash':
+            self.attn_fn = flash_attn_fn
+        elif self.attn_impl == 'triton':
+            self.attn_fn = triton_flash_attn_fn
+            if verbose:
+                warnings.warn('While `attn_impl: triton` can be faster than `attn_impl: flash` ' + 'it uses more memory. When training larger models this can trigger ' + 'alloc retries which hurts performance. If encountered, we recommend ' + 'using `attn_impl: flash` if your model does not use `alibi` or `prefix_lm`.')
+        elif self.attn_impl == 'torch':
+            self.attn_fn = scaled_multihead_dot_product_attention
+            if torch.cuda.is_available() and verbose:
+                warnings.warn('Using `attn_impl: torch`. If your model does not use `alibi` or ' + '`prefix_lm` we recommend using `attn_impl: flash` otherwise ' + 'we recommend using `attn_impl: triton`.')
+        else:
+            raise ValueError(f'attn_impl={attn_impl!r} is an invalid setting.')
+        self.out_proj = nn.Linear(self.d_model, self.d_model, device=device)
+        self.out_proj._is_residual = True
+
+    def forward(self, x, past_key_value=None, attn_bias=None, attention_mask=None, is_causal=True, needs_weights=False):
+        qkv = self.Wqkv(x)
+        if self.clip_qkv:
+            qkv.clamp_(min=-self.clip_qkv, max=self.clip_qkv)
+        (query, key, value) = qkv.chunk(3, dim=2)
+        key_padding_mask = attention_mask
+        if self.qk_ln:
+            dtype = query.dtype
+            query = self.q_ln(query).to(dtype)
+            key = self.k_ln(key).to(dtype)
+        (context, attn_weights, past_key_value) = self.attn_fn(query, key, value, self.n_heads, past_key_value=past_key_value, softmax_scale=self.softmax_scale, attn_bias=attn_bias, key_padding_mask=key_padding_mask, is_causal=is_causal, dropout_p=self.attn_dropout_p, training=self.training, needs_weights=needs_weights)
+        return (self.out_proj(context), attn_weights, past_key_value)
+
+class MultiQueryAttention(nn.Module):
+    """Multi-Query self attention.
+
+    Using torch or triton attention implementation enables user to also use
+    additive bias.
+    """
+
+    def __init__(self, d_model: int, n_heads: int, attn_impl: str='triton', clip_qkv: Optional[float]=None, qk_ln: bool=False, softmax_scale: Optional[float]=None, attn_pdrop: float=0.0, low_precision_layernorm: bool=False, verbose: int=0, device: Optional[str]=None):
+        super().__init__()
+        self.attn_impl = attn_impl
+        self.clip_qkv = clip_qkv
+        self.qk_ln = qk_ln
+        self.d_model = d_model
+        self.n_heads = n_heads
+        self.head_dim = d_model // n_heads
+        self.softmax_scale = softmax_scale
+        if self.softmax_scale is None:
+            self.softmax_scale = 1 / math.sqrt(self.head_dim)
+        self.attn_dropout_p = attn_pdrop
+        self.Wqkv = nn.Linear(d_model, d_model + 2 * self.head_dim, device=device)
+        fuse_splits = (d_model, d_model + self.head_dim)
+        self.Wqkv._fused = (0, fuse_splits)
+        if self.qk_ln:
+            layernorm_class = LPLayerNorm if low_precision_layernorm else nn.LayerNorm
+            self.q_ln = layernorm_class(d_model, device=device)
+            self.k_ln = layernorm_class(self.head_dim, device=device)
+        if self.attn_impl == 'flash':
+            self.attn_fn = flash_attn_fn
+        elif self.attn_impl == 'triton':
+            self.attn_fn = triton_flash_attn_fn
+            if verbose:
+                warnings.warn('While `attn_impl: triton` can be faster than `attn_impl: flash` ' + 'it uses more memory. When training larger models this can trigger ' + 'alloc retries which hurts performance. If encountered, we recommend ' + 'using `attn_impl: flash` if your model does not use `alibi` or `prefix_lm`.')
+        elif self.attn_impl == 'torch':
+            self.attn_fn = scaled_multihead_dot_product_attention
+            if torch.cuda.is_available() and verbose:
+                warnings.warn('Using `attn_impl: torch`. If your model does not use `alibi` or ' + '`prefix_lm` we recommend using `attn_impl: flash` otherwise ' + 'we recommend using `attn_impl: triton`.')
+        else:
+            raise ValueError(f'attn_impl={attn_impl!r} is an invalid setting.')
+        self.out_proj = nn.Linear(self.d_model, self.d_model, device=device)
+        self.out_proj._is_residual = True
+
+    def forward(self, x, past_key_value=None, attn_bias=None, attention_mask=None, is_causal=True, needs_weights=False):
+        qkv = self.Wqkv(x)
+        if self.clip_qkv:
+            qkv.clamp_(min=-self.clip_qkv, max=self.clip_qkv)
+        (query, key, value) = qkv.split([self.d_model, self.head_dim, self.head_dim], dim=2)
+        key_padding_mask = attention_mask
+        if self.qk_ln:
+            dtype = query.dtype
+            query = self.q_ln(query).to(dtype)
+            key = self.k_ln(key).to(dtype)
+        (context, attn_weights, past_key_value) = self.attn_fn(query, key, value, self.n_heads, past_key_value=past_key_value, softmax_scale=self.softmax_scale, attn_bias=attn_bias, key_padding_mask=key_padding_mask, is_causal=is_causal, dropout_p=self.attn_dropout_p, training=self.training, needs_weights=needs_weights, multiquery=True)
+        return (self.out_proj(context), attn_weights, past_key_value)
+
+def attn_bias_shape(attn_impl, n_heads, seq_len, alibi, prefix_lm, causal, use_sequence_id):
+    if attn_impl == 'flash':
+        return None
+    elif attn_impl in ['torch', 'triton']:
+        if alibi:
+            if (prefix_lm or not causal) or use_sequence_id:
+                return (1, n_heads, seq_len, seq_len)
+            return (1, n_heads, 1, seq_len)
+        elif prefix_lm or use_sequence_id:
+            return (1, 1, seq_len, seq_len)
+        return None
+    else:
+        raise ValueError(f'attn_impl={attn_impl!r} is an invalid setting.')
+
+def build_attn_bias(attn_impl, attn_bias, n_heads, seq_len, causal=False, alibi=False, alibi_bias_max=8):
+    if attn_impl == 'flash':
+        return None
+    elif attn_impl in ['torch', 'triton']:
+        if alibi:
+            (device, dtype) = (attn_bias.device, attn_bias.dtype)
+            attn_bias = attn_bias.add(build_alibi_bias(n_heads, seq_len, full=not causal, alibi_bias_max=alibi_bias_max, device=device, dtype=dtype))
+        return attn_bias
+    else:
+        raise ValueError(f'attn_impl={attn_impl!r} is an invalid setting.')
+
+def gen_slopes(n_heads, alibi_bias_max=8, device=None):
+    _n_heads = 2 ** math.ceil(math.log2(n_heads))
+    m = torch.arange(1, _n_heads + 1, dtype=torch.float32, device=device)
+    m = m.mul(alibi_bias_max / _n_heads)
+    slopes = 1.0 / torch.pow(2, m)
+    if _n_heads != n_heads:
+        slopes = torch.concat([slopes[1::2], slopes[::2]])[:n_heads]
+    return slopes.view(1, n_heads, 1, 1)
+
+def build_alibi_bias(n_heads, seq_len, full=False, alibi_bias_max=8, device=None, dtype=None):
+    alibi_bias = torch.arange(1 - seq_len, 1, dtype=torch.int32, device=device).view(1, 1, 1, seq_len)
+    if full:
+        alibi_bias = alibi_bias - torch.arange(1 - seq_len, 1, dtype=torch.int32, device=device).view(1, 1, seq_len, 1)
+        alibi_bias = alibi_bias.abs().mul(-1)
+    slopes = gen_slopes(n_heads, alibi_bias_max, device=device)
+    alibi_bias = alibi_bias * slopes
+    return alibi_bias.to(dtype=dtype)
+ATTN_CLASS_REGISTRY = {'multihead_attention': MultiheadAttention, 'multiquery_attention': MultiQueryAttention}

videollava/model/language_model/mpt/blocks.py

@@ -0,0 +1,41 @@
+"""GPT Blocks used for the GPT Model."""
+from typing import Dict, Optional, Tuple
+import torch
+import torch.nn as nn
+from .attention import ATTN_CLASS_REGISTRY
+from .norm import NORM_CLASS_REGISTRY
+
+class MPTMLP(nn.Module):
+
+    def __init__(self, d_model: int, expansion_ratio: int, device: Optional[str]=None):
+        super().__init__()
+        self.up_proj = nn.Linear(d_model, expansion_ratio * d_model, device=device)
+        self.act = nn.GELU(approximate='none')
+        self.down_proj = nn.Linear(expansion_ratio * d_model, d_model, device=device)
+        self.down_proj._is_residual = True
+
+    def forward(self, x):
+        return self.down_proj(self.act(self.up_proj(x)))
+
+class MPTBlock(nn.Module):
+
+    def __init__(self, d_model: int, n_heads: int, expansion_ratio: int, attn_config: Dict={'attn_type': 'multihead_attention', 'attn_pdrop': 0.0, 'attn_impl': 'triton', 'qk_ln': False, 'clip_qkv': None, 'softmax_scale': None, 'prefix_lm': False, 'attn_uses_sequence_id': False, 'alibi': False, 'alibi_bias_max': 8}, resid_pdrop: float=0.0, norm_type: str='low_precision_layernorm', verbose: int=0, device: Optional[str]=None, **kwargs):
+        del kwargs
+        super().__init__()
+        norm_class = NORM_CLASS_REGISTRY[norm_type.lower()]
+        attn_class = ATTN_CLASS_REGISTRY[attn_config['attn_type']]
+        self.norm_1 = norm_class(d_model, device=device)
+        self.attn = attn_class(attn_impl=attn_config['attn_impl'], clip_qkv=attn_config['clip_qkv'], qk_ln=attn_config['qk_ln'], softmax_scale=attn_config['softmax_scale'], attn_pdrop=attn_config['attn_pdrop'], d_model=d_model, n_heads=n_heads, verbose=verbose, device=device)
+        self.norm_2 = norm_class(d_model, device=device)
+        self.ffn = MPTMLP(d_model=d_model, expansion_ratio=expansion_ratio, device=device)
+        self.resid_attn_dropout = nn.Dropout(resid_pdrop)
+        self.resid_ffn_dropout = nn.Dropout(resid_pdrop)
+
+    def forward(self, x: torch.Tensor, past_key_value: Optional[Tuple[torch.Tensor]]=None, attn_bias: Optional[torch.Tensor]=None, attention_mask: Optional[torch.ByteTensor]=None, is_causal: bool=True) -> Tuple[torch.Tensor, Optional[Tuple[torch.Tensor]]]:
+        a = self.norm_1(x)
+        (b, attn_weights, past_key_value) = self.attn(a, past_key_value=past_key_value, attn_bias=attn_bias, attention_mask=attention_mask, is_causal=is_causal)
+        x = x + self.resid_attn_dropout(b)
+        m = self.norm_2(x)
+        n = self.ffn(m)
+        x = x + self.resid_ffn_dropout(n)
+        return (x, attn_weights, past_key_value)

videollava/model/language_model/mpt/configuration_mpt.py

@@ -0,0 +1,118 @@
+"""A HuggingFace-style model configuration."""
+from typing import Dict, Optional, Union
+from transformers import PretrainedConfig
+attn_config_defaults: Dict = {'attn_type': 'multihead_attention', 'attn_pdrop': 0.0, 'attn_impl': 'triton', 'qk_ln': False, 'clip_qkv': None, 'softmax_scale': None, 'prefix_lm': False, 'attn_uses_sequence_id': False, 'alibi': False, 'alibi_bias_max': 8}
+init_config_defaults: Dict = {'name': 'kaiming_normal_', 'fan_mode': 'fan_in', 'init_nonlinearity': 'relu', 'init_div_is_residual': True, 'emb_init_std': None, 'emb_init_uniform_lim': None, 'init_std': None, 'init_gain': 0.0}
+
+class MPTConfig(PretrainedConfig):
+    model_type = 'mpt'
+
+    def __init__(self, d_model: int=2048, n_heads: int=16, n_layers: int=24, expansion_ratio: int=4, max_seq_len: int=2048, vocab_size: int=50368, resid_pdrop: float=0.0, emb_pdrop: float=0.0, learned_pos_emb: bool=True, attn_config: Dict=attn_config_defaults, init_device: str='cpu', logit_scale: Optional[Union[float, str]]=None, no_bias: bool=False, verbose: int=0, embedding_fraction: float=1.0, norm_type: str='low_precision_layernorm', use_cache: bool=False, init_config: Dict=init_config_defaults, **kwargs):
+        """The MPT configuration class.
+
+        Args:
+            d_model (int): The size of the embedding dimension of the model.
+            n_heads (int): The number of attention heads.
+            n_layers (int): The number of layers in the model.
+            expansion_ratio (int): The ratio of the up/down scale in the MLP.
+            max_seq_len (int): The maximum sequence length of the model.
+            vocab_size (int): The size of the vocabulary.
+            resid_pdrop (float): The dropout probability applied to the attention output before combining with residual.
+            emb_pdrop (float): The dropout probability for the embedding layer.
+            learned_pos_emb (bool): Whether to use learned positional embeddings
+            attn_config (Dict):  A dictionary used to configure the model's attention module:
+                attn_type (str): type of attention to use. Options: multihead_attention, multiquery_attention
+                attn_pdrop (float): The dropout probability for the attention layers.
+                attn_impl (str): The attention implementation to use. One of 'torch', 'flash', or 'triton'.
+                qk_ln (bool): Whether to apply layer normalization to the queries and keys in the attention layer.
+                clip_qkv (Optional[float]): If not None, clip the queries, keys, and values in the attention layer to
+                    this value.
+                softmax_scale (Optional[float]): If not None, scale the softmax in the attention layer by this value. If None,
+                    use the default scale of ``1/sqrt(d_keys)``.
+                prefix_lm (Optional[bool]): Whether the model should operate as a Prefix LM. This requires passing an
+                    extra `prefix_mask` argument which indicates which tokens belong to the prefix. Tokens in the prefix
+                    can attend to one another bi-directionally. Tokens outside the prefix use causal attention.
+                attn_uses_sequence_id (Optional[bool]): Whether to restrict attention to tokens that have the same sequence_id.
+                    When the model is in `train` mode, this requires passing an extra `sequence_id` argument which indicates
+                    which sub-sequence each token belongs to.
+                    Defaults to ``False`` meaning any provided `sequence_id` will be ignored.
+                alibi (bool): Whether to use the alibi bias instead of position embeddings.
+                alibi_bias_max (int): The maximum value of the alibi bias.
+            init_device (str): The device to use for parameter initialization.
+            logit_scale (Optional[Union[float, str]]): If not None, scale the logits by this value.
+            no_bias (bool): Whether to use bias in all layers.
+            verbose (int): The verbosity level. 0 is silent.
+            embedding_fraction (float): The fraction to scale the gradients of the embedding layer by.
+            norm_type (str): choose type of norm to use
+            multiquery_attention (bool): Whether to use multiquery attention implementation.
+            use_cache (bool): Whether or not the model should return the last key/values attentions
+            init_config (Dict): A dictionary used to configure the model initialization:
+                init_config.name: The parameter initialization scheme to use. Options: 'default_', 'baseline_',
+                    'kaiming_uniform_', 'kaiming_normal_', 'neox_init_', 'small_init_', 'xavier_uniform_', or
+                    'xavier_normal_'. These mimic the parameter initialization methods in PyTorch.
+                init_div_is_residual (Union[int, float, str, bool]): Value to divide initial weights by if ``module._is_residual`` is True.
+                emb_init_std (Optional[float]): The standard deviation of the normal distribution used to initialize the embedding layer.
+                emb_init_uniform_lim (Optional[Union[Tuple[float, float], float]]): The lower and upper limits of the uniform distribution
+                    used to initialize the embedding layer. Mutually exclusive with ``emb_init_std``.
+                init_std (float): The standard deviation of the normal distribution used to initialize the model,
+                    if using the baseline_ parameter initialization scheme.
+                init_gain (float): The gain to use for parameter initialization with kaiming or xavier initialization schemes.
+                fan_mode (str): The fan mode to use for parameter initialization with kaiming initialization schemes.
+                init_nonlinearity (str): The nonlinearity to use for parameter initialization with kaiming initialization schemes.
+                ---
+                See llmfoundry.models.utils.param_init_fns.py for info on other param init config options
+        """
+        self.d_model = d_model
+        self.n_heads = n_heads
+        self.n_layers = n_layers
+        self.expansion_ratio = expansion_ratio
+        self.max_seq_len = max_seq_len
+        self.vocab_size = vocab_size
+        self.resid_pdrop = resid_pdrop
+        self.emb_pdrop = emb_pdrop
+        self.learned_pos_emb = learned_pos_emb
+        self.attn_config = attn_config
+        self.init_device = init_device
+        self.logit_scale = logit_scale
+        self.no_bias = no_bias
+        self.verbose = verbose
+        self.embedding_fraction = embedding_fraction
+        self.norm_type = norm_type
+        self.use_cache = use_cache
+        self.init_config = init_config
+        if 'name' in kwargs:
+            del kwargs['name']
+        if 'loss_fn' in kwargs:
+            del kwargs['loss_fn']
+        super().__init__(**kwargs)
+        self._validate_config()
+
+    def _set_config_defaults(self, config, config_defaults):
+        for (k, v) in config_defaults.items():
+            if k not in config:
+                config[k] = v
+        return config
+
+    def _validate_config(self):
+        self.attn_config = self._set_config_defaults(self.attn_config, attn_config_defaults)
+        self.init_config = self._set_config_defaults(self.init_config, init_config_defaults)
+        if self.d_model % self.n_heads != 0:
+            raise ValueError('d_model must be divisible by n_heads')
+        if any((prob < 0 or prob > 1 for prob in [self.attn_config['attn_pdrop'], self.resid_pdrop, self.emb_pdrop])):
+            raise ValueError("self.attn_config['attn_pdrop'], resid_pdrop, emb_pdrop are probabilities and must be between 0 and 1")
+        if self.attn_config['attn_impl'] not in ['torch', 'flash', 'triton']:
+            raise ValueError(f"Unknown attn_impl={self.attn_config['attn_impl']}")
+        if self.attn_config['prefix_lm'] and self.attn_config['attn_impl'] not in ['torch', 'triton']:
+            raise NotImplementedError('prefix_lm only implemented with torch and triton attention.')
+        if self.attn_config['alibi'] and self.attn_config['attn_impl'] not in ['torch', 'triton']:
+            raise NotImplementedError('alibi only implemented with torch and triton attention.')
+        if self.attn_config['attn_uses_sequence_id'] and self.attn_config['attn_impl'] not in ['torch', 'triton']:
+            raise NotImplementedError('attn_uses_sequence_id only implemented with torch and triton attention.')
+        if self.embedding_fraction > 1 or self.embedding_fraction <= 0:
+            raise ValueError('model.embedding_fraction must be between 0 (exclusive) and 1 (inclusive)!')
+        if isinstance(self.logit_scale, str) and self.logit_scale != 'inv_sqrt_d_model':
+            raise ValueError(f"self.logit_scale={self.logit_scale!r} is not recognized as an option; use numeric value or 'inv_sqrt_d_model'.")
+        if self.init_config.get('name', None) is None:
+            raise ValueError(f"self.init_config={self.init_config!r} 'name' needs to be set.")
+        if not self.learned_pos_emb and (not self.attn_config['alibi']):
+            raise ValueError(f'Positional information must be provided to the model using either learned_pos_emb or alibi.')

videollava/model/language_model/mpt/custom_embedding.py

@@ -0,0 +1,11 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch import Tensor
+
+class SharedEmbedding(nn.Embedding):
+
+    def forward(self, input: Tensor, unembed: bool=False) -> Tensor:
+        if unembed:
+            return F.linear(input, self.weight)
+        return super().forward(input)

videollava/model/language_model/mpt/flash_attn_triton.py

@@ -0,0 +1,484 @@
+"""
+Copied from https://github.com/HazyResearch/flash-attention/blob/eff9fe6b8076df59d64d7a3f464696738a3c7c24/flash_attn/flash_attn_triton.py
+update imports to use 'triton_pre_mlir'
+
+*Experimental* implementation of FlashAttention in Triton.
+Tested with triton==2.0.0.dev20221202.
+Triton 2.0 has a new backend (MLIR) but seems like it doesn't yet work for head dimensions
+other than 64:
+https://github.com/openai/triton/blob/d376020f90002757eea3ea9475d4f7cfc2ec5ead/python/triton/ops/flash_attention.py#L207
+We'll update this implementation with the new Triton backend once this is fixed.
+
+We use the FlashAttention implementation from Phil Tillet a starting point.
+https://github.com/openai/triton/blob/master/python/tutorials/06-fused-attention.py
+
+Changes:
+- Implement both causal and non-causal attention.
+- Implement both self-attention and cross-attention.
+- Support arbitrary seqlens (not just multiples of 128), for both forward and backward.
+- Support all head dimensions up to 128 (not just 16, 32, 64, 128), for both forward and backward.
+- Support attention bias.
+- Speed up the forward pass a bit, and only store the LSE instead of m and l.
+- Make the backward for d=128 much faster by reducing register spilling.
+- Optionally parallelize the backward pass across seqlen_k, to deal with the case of
+small batch size * nheads.
+
+Caution:
+- This is an *experimental* implementation. The forward pass should be quite robust but
+I'm not 100% sure that the backward pass doesn't have race conditions (due to the Triton compiler).
+- This implementation has only been tested on A100.
+- If you plan to use headdim other than 64 and 128, you should test for race conditions
+(due to the Triton compiler), as done in tests/test_flash_attn.py
+"test_flash_attn_triton_race_condition". I've tested and fixed many race conditions
+for different head dimensions (40, 48, 64, 128, 80, 88, 96), but I'm still not 100% confident
+that there are none left for other head dimensions.
+
+Differences between this Triton version and the CUDA version:
+- Triton version doesn't support dropout.
+- Triton forward is generally faster than CUDA forward, while Triton backward is
+generally slower than CUDA backward. Overall Triton forward + backward is slightly slower
+than CUDA forward + backward.
+- Triton version doesn't support different sequence lengths in a batch (i.e., RaggedTensor/NestedTensor).
+- Triton version supports attention bias, while CUDA version doesn't.
+"""
+import math
+import torch
+import triton_pre_mlir as triton
+import triton_pre_mlir.language as tl
+
+@triton.heuristics({'EVEN_M': lambda args: args['seqlen_q'] % args['BLOCK_M'] == 0, 'EVEN_N': lambda args: args['seqlen_k'] % args['BLOCK_N'] == 0, 'EVEN_HEADDIM': lambda args: args['headdim'] == args['BLOCK_HEADDIM']})
+@triton.jit
+def _fwd_kernel(Q, K, V, Bias, Out, Lse, TMP, softmax_scale, stride_qb, stride_qh, stride_qm, stride_kb, stride_kh, stride_kn, stride_vb, stride_vh, stride_vn, stride_bb, stride_bh, stride_bm, stride_ob, stride_oh, stride_om, nheads, seqlen_q, seqlen_k, seqlen_q_rounded, headdim, CACHE_KEY_SEQLEN_Q, CACHE_KEY_SEQLEN_K, BIAS_TYPE: tl.constexpr, IS_CAUSAL: tl.constexpr, BLOCK_HEADDIM: tl.constexpr, EVEN_M: tl.constexpr, EVEN_N: tl.constexpr, EVEN_HEADDIM: tl.constexpr, BLOCK_M: tl.constexpr, BLOCK_N: tl.constexpr):
+    start_m = tl.program_id(0)
+    off_hb = tl.program_id(1)
+    off_b = off_hb // nheads
+    off_h = off_hb % nheads
+    offs_m = start_m * BLOCK_M + tl.arange(0, BLOCK_M)
+    offs_n = tl.arange(0, BLOCK_N)
+    offs_d = tl.arange(0, BLOCK_HEADDIM)
+    q_ptrs = Q + off_b * stride_qb + off_h * stride_qh + (offs_m[:, None] * stride_qm + offs_d[None, :])
+    k_ptrs = K + off_b * stride_kb + off_h * stride_kh + (offs_n[:, None] * stride_kn + offs_d[None, :])
+    v_ptrs = V + off_b * stride_vb + off_h * stride_vh + (offs_n[:, None] * stride_vn + offs_d[None, :])
+    if BIAS_TYPE == 'vector':
+        b_ptrs = Bias + off_b * stride_bb + off_h * stride_bh + offs_n
+    elif BIAS_TYPE == 'matrix':
+        b_ptrs = Bias + off_b * stride_bb + off_h * stride_bh + (offs_m[:, None] * stride_bm + offs_n[None, :])
+    t_ptrs = TMP + off_hb * seqlen_q_rounded + offs_m
+    lse_i = tl.zeros([BLOCK_M], dtype=tl.float32) - float('inf')
+    m_i = tl.zeros([BLOCK_M], dtype=tl.float32) - float('inf')
+    acc_o = tl.zeros([BLOCK_M, BLOCK_HEADDIM], dtype=tl.float32)
+    if EVEN_M & EVEN_N:
+        if EVEN_HEADDIM:
+            q = tl.load(q_ptrs)
+        else:
+            q = tl.load(q_ptrs, mask=offs_d[None, :] < headdim, other=0.0)
+    elif EVEN_HEADDIM:
+        q = tl.load(q_ptrs, mask=offs_m[:, None] < seqlen_q, other=0.0)
+    else:
+        q = tl.load(q_ptrs, mask=(offs_m[:, None] < seqlen_q) & (offs_d[None, :] < headdim), other=0.0)
+    end_n = seqlen_k if not IS_CAUSAL else tl.minimum((start_m + 1) * BLOCK_M, seqlen_k)
+    for start_n in range(0, end_n, BLOCK_N):
+        start_n = tl.multiple_of(start_n, BLOCK_N)
+        if EVEN_N & EVEN_M:
+            if EVEN_HEADDIM:
+                k = tl.load(k_ptrs + start_n * stride_kn)
+            else:
+                k = tl.load(k_ptrs + start_n * stride_kn, mask=offs_d[None, :] < headdim, other=0.0)
+        elif EVEN_HEADDIM:
+            k = tl.load(k_ptrs + start_n * stride_kn, mask=(start_n + offs_n)[:, None] < seqlen_k, other=0.0)
+        else:
+            k = tl.load(k_ptrs + start_n * stride_kn, mask=((start_n + offs_n)[:, None] < seqlen_k) & (offs_d[None, :] < headdim), other=0.0)
+        qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)
+        qk += tl.dot(q, k, trans_b=True)
+        if not EVEN_N:
+            qk += tl.where((start_n + offs_n)[None, :] < seqlen_k, 0, float('-inf'))
+        if IS_CAUSAL:
+            qk += tl.where(offs_m[:, None] >= (start_n + offs_n)[None, :], 0, float('-inf'))
+        if BIAS_TYPE != 'none':
+            if BIAS_TYPE == 'vector':
+                if EVEN_N:
+                    bias = tl.load(b_ptrs + start_n).to(tl.float32)
+                else:
+                    bias = tl.load(b_ptrs + start_n, mask=start_n + offs_n < seqlen_k, other=0.0).to(tl.float32)
+                bias = bias[None, :]
+            elif BIAS_TYPE == 'matrix':
+                if EVEN_M & EVEN_N:
+                    bias = tl.load(b_ptrs + start_n).to(tl.float32)
+                else:
+                    bias = tl.load(b_ptrs + start_n, mask=(offs_m[:, None] < seqlen_q) & ((start_n + offs_n)[None, :] < seqlen_k), other=0.0).to(tl.float32)
+            qk = qk * softmax_scale + bias
+            m_ij = tl.maximum(tl.max(qk, 1), lse_i)
+            p = tl.exp(qk - m_ij[:, None])
+        else:
+            m_ij = tl.maximum(tl.max(qk, 1) * softmax_scale, lse_i)
+            p = tl.exp(qk * softmax_scale - m_ij[:, None])
+        l_ij = tl.sum(p, 1)
+        acc_o_scale = tl.exp(m_i - m_ij)
+        tl.store(t_ptrs, acc_o_scale)
+        acc_o_scale = tl.load(t_ptrs)
+        acc_o = acc_o * acc_o_scale[:, None]
+        if EVEN_N & EVEN_M:
+            if EVEN_HEADDIM:
+                v = tl.load(v_ptrs + start_n * stride_vn)
+            else:
+                v = tl.load(v_ptrs + start_n * stride_vn, mask=offs_d[None, :] < headdim, other=0.0)
+        elif EVEN_HEADDIM:
+            v = tl.load(v_ptrs + start_n * stride_vn, mask=(start_n + offs_n)[:, None] < seqlen_k, other=0.0)
+        else:
+            v = tl.load(v_ptrs + start_n * stride_vn, mask=((start_n + offs_n)[:, None] < seqlen_k) & (offs_d[None, :] < headdim), other=0.0)
+        p = p.to(v.dtype)
+        acc_o += tl.dot(p, v)
+        m_i = m_ij
+        l_i_new = tl.exp(lse_i - m_ij) + l_ij
+        lse_i = m_ij + tl.log(l_i_new)
+    o_scale = tl.exp(m_i - lse_i)
+    tl.store(t_ptrs, o_scale)
+    o_scale = tl.load(t_ptrs)
+    acc_o = acc_o * o_scale[:, None]
+    start_m = tl.program_id(0)
+    offs_m = start_m * BLOCK_M + tl.arange(0, BLOCK_M)
+    lse_ptrs = Lse + off_hb * seqlen_q_rounded + offs_m
+    tl.store(lse_ptrs, lse_i)
+    offs_d = tl.arange(0, BLOCK_HEADDIM)
+    out_ptrs = Out + off_b * stride_ob + off_h * stride_oh + (offs_m[:, None] * stride_om + offs_d[None, :])
+    if EVEN_M:
+        if EVEN_HEADDIM:
+            tl.store(out_ptrs, acc_o)
+        else:
+            tl.store(out_ptrs, acc_o, mask=offs_d[None, :] < headdim)
+    elif EVEN_HEADDIM:
+        tl.store(out_ptrs, acc_o, mask=offs_m[:, None] < seqlen_q)
+    else:
+        tl.store(out_ptrs, acc_o, mask=(offs_m[:, None] < seqlen_q) & (offs_d[None, :] < headdim))
+
+@triton.jit
+def _bwd_preprocess_do_o_dot(Out, DO, Delta, stride_ob, stride_oh, stride_om, stride_dob, stride_doh, stride_dom, nheads, seqlen_q, seqlen_q_rounded, headdim, BLOCK_M: tl.constexpr, BLOCK_HEADDIM: tl.constexpr):
+    start_m = tl.program_id(0)
+    off_hb = tl.program_id(1)
+    off_b = off_hb // nheads
+    off_h = off_hb % nheads
+    offs_m = start_m * BLOCK_M + tl.arange(0, BLOCK_M)
+    offs_d = tl.arange(0, BLOCK_HEADDIM)
+    o = tl.load(Out + off_b * stride_ob + off_h * stride_oh + offs_m[:, None] * stride_om + offs_d[None, :], mask=(offs_m[:, None] < seqlen_q) & (offs_d[None, :] < headdim), other=0.0).to(tl.float32)
+    do = tl.load(DO + off_b * stride_dob + off_h * stride_doh + offs_m[:, None] * stride_dom + offs_d[None, :], mask=(offs_m[:, None] < seqlen_q) & (offs_d[None, :] < headdim), other=0.0).to(tl.float32)
+    delta = tl.sum(o * do, axis=1)
+    tl.store(Delta + off_hb * seqlen_q_rounded + offs_m, delta)
+
+@triton.jit
+def _bwd_store_dk_dv(dk_ptrs, dv_ptrs, dk, dv, offs_n, offs_d, seqlen_k, headdim, EVEN_M: tl.constexpr, EVEN_N: tl.constexpr, EVEN_HEADDIM: tl.constexpr):
+    if EVEN_N & EVEN_M:
+        if EVEN_HEADDIM:
+            tl.store(dv_ptrs, dv)
+            tl.store(dk_ptrs, dk)
+        else:
+            tl.store(dv_ptrs, dv, mask=offs_d[None, :] < headdim)
+            tl.store(dk_ptrs, dk, mask=offs_d[None, :] < headdim)
+    elif EVEN_HEADDIM:
+        tl.store(dv_ptrs, dv, mask=offs_n[:, None] < seqlen_k)
+        tl.store(dk_ptrs, dk, mask=offs_n[:, None] < seqlen_k)
+    else:
+        tl.store(dv_ptrs, dv, mask=(offs_n[:, None] < seqlen_k) & (offs_d[None, :] < headdim))
+        tl.store(dk_ptrs, dk, mask=(offs_n[:, None] < seqlen_k) & (offs_d[None, :] < headdim))
+
+@triton.jit
+def _bwd_kernel_one_col_block(start_n, Q, K, V, Bias, DO, DQ, DK, DV, LSE, D, softmax_scale, stride_qm, stride_kn, stride_vn, stride_bm, stride_dom, stride_dqm, stride_dkn, stride_dvn, seqlen_q, seqlen_k, headdim, ATOMIC_ADD: tl.constexpr, BIAS_TYPE: tl.constexpr, IS_CAUSAL: tl.constexpr, BLOCK_HEADDIM: tl.constexpr, EVEN_M: tl.constexpr, EVEN_N: tl.constexpr, EVEN_HEADDIM: tl.constexpr, BLOCK_M: tl.constexpr, BLOCK_N: tl.constexpr):
+    begin_m = 0 if not IS_CAUSAL else start_n * BLOCK_N // BLOCK_M * BLOCK_M
+    offs_qm = begin_m + tl.arange(0, BLOCK_M)
+    offs_n = start_n * BLOCK_N + tl.arange(0, BLOCK_N)
+    offs_m = tl.arange(0, BLOCK_M)
+    offs_d = tl.arange(0, BLOCK_HEADDIM)
+    q_ptrs = Q + (offs_qm[:, None] * stride_qm + offs_d[None, :])
+    k_ptrs = K + (offs_n[:, None] * stride_kn + offs_d[None, :])
+    v_ptrs = V + (offs_n[:, None] * stride_vn + offs_d[None, :])
+    do_ptrs = DO + (offs_qm[:, None] * stride_dom + offs_d[None, :])
+    dq_ptrs = DQ + (offs_qm[:, None] * stride_dqm + offs_d[None, :])
+    if BIAS_TYPE == 'vector':
+        b_ptrs = Bias + offs_n
+    elif BIAS_TYPE == 'matrix':
+        b_ptrs = Bias + (offs_qm[:, None] * stride_bm + offs_n[None, :])
+    dv = tl.zeros([BLOCK_N, BLOCK_HEADDIM], dtype=tl.float32)
+    dk = tl.zeros([BLOCK_N, BLOCK_HEADDIM], dtype=tl.float32)
+    if begin_m >= seqlen_q:
+        dv_ptrs = DV + (offs_n[:, None] * stride_dvn + offs_d[None, :])
+        dk_ptrs = DK + (offs_n[:, None] * stride_dkn + offs_d[None, :])
+        _bwd_store_dk_dv(dk_ptrs, dv_ptrs, dk, dv, offs_n, offs_d, seqlen_k, headdim, EVEN_M=EVEN_M, EVEN_N=EVEN_N, EVEN_HEADDIM=EVEN_HEADDIM)
+        return
+    if EVEN_N & EVEN_M:
+        if EVEN_HEADDIM:
+            k = tl.load(k_ptrs)
+            v = tl.load(v_ptrs)
+        else:
+            k = tl.load(k_ptrs, mask=offs_d[None, :] < headdim, other=0.0)
+            v = tl.load(v_ptrs, mask=offs_d[None, :] < headdim, other=0.0)
+    elif EVEN_HEADDIM:
+        k = tl.load(k_ptrs, mask=offs_n[:, None] < seqlen_k, other=0.0)
+        v = tl.load(v_ptrs, mask=offs_n[:, None] < seqlen_k, other=0.0)
+    else:
+        k = tl.load(k_ptrs, mask=(offs_n[:, None] < seqlen_k) & (offs_d[None, :] < headdim), other=0.0)
+        v = tl.load(v_ptrs, mask=(offs_n[:, None] < seqlen_k) & (offs_d[None, :] < headdim), other=0.0)
+    num_block_m = tl.cdiv(seqlen_q, BLOCK_M)
+    for start_m in range(begin_m, num_block_m * BLOCK_M, BLOCK_M):
+        start_m = tl.multiple_of(start_m, BLOCK_M)
+        offs_m_curr = start_m + offs_m
+        if EVEN_M & EVEN_HEADDIM:
+            q = tl.load(q_ptrs)
+        elif EVEN_HEADDIM:
+            q = tl.load(q_ptrs, mask=offs_m_curr[:, None] < seqlen_q, other=0.0)
+        else:
+            q = tl.load(q_ptrs, mask=(offs_m_curr[:, None] < seqlen_q) & (offs_d[None, :] < headdim), other=0.0)
+        qk = tl.dot(q, k, trans_b=True)
+        if not EVEN_N:
+            qk = tl.where(offs_n[None, :] < seqlen_k, qk, float('-inf'))
+        if IS_CAUSAL:
+            qk = tl.where(offs_m_curr[:, None] >= offs_n[None, :], qk, float('-inf'))
+        if BIAS_TYPE != 'none':
+            tl.debug_barrier()
+            if BIAS_TYPE == 'vector':
+                if EVEN_N:
+                    bias = tl.load(b_ptrs).to(tl.float32)
+                else:
+                    bias = tl.load(b_ptrs, mask=offs_n < seqlen_k, other=0.0).to(tl.float32)
+                bias = bias[None, :]
+            elif BIAS_TYPE == 'matrix':
+                if EVEN_M & EVEN_N:
+                    bias = tl.load(b_ptrs).to(tl.float32)
+                else:
+                    bias = tl.load(b_ptrs, mask=(offs_m_curr[:, None] < seqlen_q) & (offs_n[None, :] < seqlen_k), other=0.0).to(tl.float32)
+            qk = qk * softmax_scale + bias
+        if not EVEN_M & EVEN_HEADDIM:
+            tl.debug_barrier()
+        lse_i = tl.load(LSE + offs_m_curr)
+        if BIAS_TYPE == 'none':
+            p = tl.exp(qk * softmax_scale - lse_i[:, None])
+        else:
+            p = tl.exp(qk - lse_i[:, None])
+        if EVEN_M & EVEN_HEADDIM:
+            do = tl.load(do_ptrs)
+        else:
+            do = tl.load(do_ptrs, mask=(offs_m_curr[:, None] < seqlen_q) & (offs_d[None, :] < headdim), other=0.0)
+        dv += tl.dot(p.to(do.dtype), do, trans_a=True)
+        if not EVEN_M & EVEN_HEADDIM:
+            tl.debug_barrier()
+        dp = tl.dot(do, v, trans_b=True)
+        if not EVEN_HEADDIM:
+            tl.debug_barrier()
+        Di = tl.load(D + offs_m_curr)
+        ds = (p * (dp - Di[:, None]) * softmax_scale).to(q.dtype)
+        dk += tl.dot(ds, q, trans_a=True)
+        if not EVEN_M & EVEN_HEADDIM:
+            tl.debug_barrier()
+        if not ATOMIC_ADD:
+            if EVEN_M & EVEN_HEADDIM:
+                dq = tl.load(dq_ptrs, eviction_policy='evict_last')
+                dq += tl.dot(ds, k)
+                tl.store(dq_ptrs, dq, eviction_policy='evict_last')
+            elif EVEN_HEADDIM:
+                dq = tl.load(dq_ptrs, mask=offs_m_curr[:, None] < seqlen_q, other=0.0, eviction_policy='evict_last')
+                dq += tl.dot(ds, k)
+                tl.store(dq_ptrs, dq, mask=offs_m_curr[:, None] < seqlen_q, eviction_policy='evict_last')
+            else:
+                dq = tl.load(dq_ptrs, mask=(offs_m_curr[:, None] < seqlen_q) & (offs_d[None, :] < headdim), other=0.0, eviction_policy='evict_last')
+                dq += tl.dot(ds, k)
+                tl.store(dq_ptrs, dq, mask=(offs_m_curr[:, None] < seqlen_q) & (offs_d[None, :] < headdim), eviction_policy='evict_last')
+        else:
+            dq = tl.dot(ds, k)
+            if EVEN_M & EVEN_HEADDIM:
+                tl.atomic_add(dq_ptrs, dq)
+            elif EVEN_HEADDIM:
+                tl.atomic_add(dq_ptrs, dq, mask=offs_m_curr[:, None] < seqlen_q)
+            else:
+                tl.atomic_add(dq_ptrs, dq, mask=(offs_m_curr[:, None] < seqlen_q) & (offs_d[None, :] < headdim))
+        dq_ptrs += BLOCK_M * stride_dqm
+        q_ptrs += BLOCK_M * stride_qm
+        do_ptrs += BLOCK_M * stride_dom
+        if BIAS_TYPE == 'matrix':
+            b_ptrs += BLOCK_M * stride_bm
+    dv_ptrs = DV + (offs_n[:, None] * stride_dvn + offs_d[None, :])
+    dk_ptrs = DK + (offs_n[:, None] * stride_dkn + offs_d[None, :])
+    _bwd_store_dk_dv(dk_ptrs, dv_ptrs, dk, dv, offs_n, offs_d, seqlen_k, headdim, EVEN_M=EVEN_M, EVEN_N=EVEN_N, EVEN_HEADDIM=EVEN_HEADDIM)
+
+def init_to_zero(name):
+    return lambda nargs: nargs[name].zero_()
+
+@triton.autotune(configs=[triton.Config({'BLOCK_M': 128, 'BLOCK_N': 128, 'SEQUENCE_PARALLEL': False}, num_warps=8, num_stages=1, pre_hook=init_to_zero('DQ')), triton.Config({'BLOCK_M': 128, 'BLOCK_N': 128, 'SEQUENCE_PARALLEL': True}, num_warps=8, num_stages=1, pre_hook=init_to_zero('DQ'))], key=['CACHE_KEY_SEQLEN_Q', 'CACHE_KEY_SEQLEN_K', 'BIAS_TYPE', 'IS_CAUSAL', 'BLOCK_HEADDIM'])
+@triton.heuristics({'EVEN_M': lambda args: args['seqlen_q'] % args['BLOCK_M'] == 0, 'EVEN_N': lambda args: args['seqlen_k'] % args['BLOCK_N'] == 0, 'EVEN_HEADDIM': lambda args: args['headdim'] == args['BLOCK_HEADDIM']})
+@triton.jit
+def _bwd_kernel(Q, K, V, Bias, DO, DQ, DK, DV, LSE, D, softmax_scale, stride_qb, stride_qh, stride_qm, stride_kb, stride_kh, stride_kn, stride_vb, stride_vh, stride_vn, stride_bb, stride_bh, stride_bm, stride_dob, stride_doh, stride_dom, stride_dqb, stride_dqh, stride_dqm, stride_dkb, stride_dkh, stride_dkn, stride_dvb, stride_dvh, stride_dvn, nheads, seqlen_q, seqlen_k, seqlen_q_rounded, headdim, CACHE_KEY_SEQLEN_Q, CACHE_KEY_SEQLEN_K, BIAS_TYPE: tl.constexpr, IS_CAUSAL: tl.constexpr, BLOCK_HEADDIM: tl.constexpr, SEQUENCE_PARALLEL: tl.constexpr, EVEN_M: tl.constexpr, EVEN_N: tl.constexpr, EVEN_HEADDIM: tl.constexpr, BLOCK_M: tl.constexpr, BLOCK_N: tl.constexpr):
+    off_hb = tl.program_id(1)
+    off_b = off_hb // nheads
+    off_h = off_hb % nheads
+    Q += off_b * stride_qb + off_h * stride_qh
+    K += off_b * stride_kb + off_h * stride_kh
+    V += off_b * stride_vb + off_h * stride_vh
+    DO += off_b * stride_dob + off_h * stride_doh
+    DQ += off_b * stride_dqb + off_h * stride_dqh
+    DK += off_b * stride_dkb + off_h * stride_dkh
+    DV += off_b * stride_dvb + off_h * stride_dvh
+    if BIAS_TYPE != 'none':
+        Bias += off_b * stride_bb + off_h * stride_bh
+    D += off_hb * seqlen_q_rounded
+    LSE += off_hb * seqlen_q_rounded
+    if not SEQUENCE_PARALLEL:
+        num_block_n = tl.cdiv(seqlen_k, BLOCK_N)
+        for start_n in range(0, num_block_n):
+            _bwd_kernel_one_col_block(start_n, Q, K, V, Bias, DO, DQ, DK, DV, LSE, D, softmax_scale, stride_qm, stride_kn, stride_vn, stride_bm, stride_dom, stride_dqm, stride_dkn, stride_dvn, seqlen_q, seqlen_k, headdim, ATOMIC_ADD=False, BIAS_TYPE=BIAS_TYPE, IS_CAUSAL=IS_CAUSAL, BLOCK_HEADDIM=BLOCK_HEADDIM, EVEN_M=EVEN_M, EVEN_N=EVEN_N, EVEN_HEADDIM=EVEN_HEADDIM, BLOCK_M=BLOCK_M, BLOCK_N=BLOCK_N)
+    else:
+        start_n = tl.program_id(0)
+        _bwd_kernel_one_col_block(start_n, Q, K, V, Bias, DO, DQ, DK, DV, LSE, D, softmax_scale, stride_qm, stride_kn, stride_vn, stride_bm, stride_dom, stride_dqm, stride_dkn, stride_dvn, seqlen_q, seqlen_k, headdim, ATOMIC_ADD=True, BIAS_TYPE=BIAS_TYPE, IS_CAUSAL=IS_CAUSAL, BLOCK_HEADDIM=BLOCK_HEADDIM, EVEN_M=EVEN_M, EVEN_N=EVEN_N, EVEN_HEADDIM=EVEN_HEADDIM, BLOCK_M=BLOCK_M, BLOCK_N=BLOCK_N)
+
+def _flash_attn_forward(q, k, v, bias=None, causal=False, softmax_scale=None):
+    (batch, seqlen_q, nheads, d) = q.shape
+    (_, seqlen_k, _, _) = k.shape
+    assert k.shape == (batch, seqlen_k, nheads, d)
+    assert v.shape == (batch, seqlen_k, nheads, d)
+    assert d <= 128, 'FlashAttention only support head dimensions up to 128'
+    assert q.dtype == k.dtype == v.dtype, 'All tensors must have the same type'
+    assert q.dtype in [torch.float16, torch.bfloat16], 'Only support fp16 and bf16'
+    assert q.is_cuda and k.is_cuda and v.is_cuda
+    softmax_scale = softmax_scale or 1.0 / math.sqrt(d)
+    has_bias = bias is not None
+    bias_type = 'none'
+    if has_bias:
+        assert bias.dtype in [q.dtype, torch.float]
+        assert bias.is_cuda
+        assert bias.dim() == 4
+        if bias.stride(-1) != 1:
+            bias = bias.contiguous()
+        if bias.shape[2:] == (1, seqlen_k):
+            bias_type = 'vector'
+        elif bias.shape[2:] == (seqlen_q, seqlen_k):
+            bias_type = 'matrix'
+        else:
+            raise RuntimeError('Last 2 dimensions of bias must be (1, seqlen_k) or (seqlen_q, seqlen_k)')
+        bias = bias.expand(batch, nheads, seqlen_q, seqlen_k)
+    bias_strides = (bias.stride(0), bias.stride(1), bias.stride(2)) if has_bias else (0, 0, 0)
+    seqlen_q_rounded = math.ceil(seqlen_q / 128) * 128
+    lse = torch.empty((batch, nheads, seqlen_q_rounded), device=q.device, dtype=torch.float32)
+    tmp = torch.empty((batch, nheads, seqlen_q_rounded), device=q.device, dtype=torch.float32)
+    o = torch.empty_like(q)
+    BLOCK_HEADDIM = max(triton.next_power_of_2(d), 16)
+    BLOCK = 128
+    num_warps = 4 if d <= 64 else 8
+    grid = lambda META: (triton.cdiv(seqlen_q, META['BLOCK_M']), batch * nheads)
+    _fwd_kernel[grid](q, k, v, bias, o, lse, tmp, softmax_scale, q.stride(0), q.stride(2), q.stride(1), k.stride(0), k.stride(2), k.stride(1), v.stride(0), v.stride(2), v.stride(1), *bias_strides, o.stride(0), o.stride(2), o.stride(1), nheads, seqlen_q, seqlen_k, seqlen_q_rounded, d, seqlen_q // 32, seqlen_k // 32, bias_type, causal, BLOCK_HEADDIM, BLOCK_M=BLOCK, BLOCK_N=BLOCK, num_warps=num_warps, num_stages=1)
+    return (o, lse, softmax_scale)
+
+def _flash_attn_backward(do, q, k, v, o, lse, dq, dk, dv, bias=None, causal=False, softmax_scale=None):
+    if do.stride(-1) != 1:
+        do = do.contiguous()
+    (batch, seqlen_q, nheads, d) = q.shape
+    (_, seqlen_k, _, _) = k.shape
+    assert d <= 128
+    seqlen_q_rounded = math.ceil(seqlen_q / 128) * 128
+    assert lse.shape == (batch, nheads, seqlen_q_rounded)
+    assert q.stride(-1) == k.stride(-1) == v.stride(-1) == o.stride(-1) == 1
+    assert dq.stride(-1) == dk.stride(-1) == dv.stride(-1) == 1
+    softmax_scale = softmax_scale or 1.0 / math.sqrt(d)
+    dq_accum = torch.empty_like(q, dtype=torch.float32)
+    delta = torch.empty_like(lse)
+    BLOCK_HEADDIM = max(triton.next_power_of_2(d), 16)
+    grid = lambda META: (triton.cdiv(seqlen_q, META['BLOCK_M']), batch * nheads)
+    _bwd_preprocess_do_o_dot[grid](o, do, delta, o.stride(0), o.stride(2), o.stride(1), do.stride(0), do.stride(2), do.stride(1), nheads, seqlen_q, seqlen_q_rounded, d, BLOCK_M=128, BLOCK_HEADDIM=BLOCK_HEADDIM)
+    has_bias = bias is not None
+    bias_type = 'none'
+    if has_bias:
+        assert bias.dtype in [q.dtype, torch.float]
+        assert bias.is_cuda
+        assert bias.dim() == 4
+        assert bias.stride(-1) == 1
+        if bias.shape[2:] == (1, seqlen_k):
+            bias_type = 'vector'
+        elif bias.shape[2:] == (seqlen_q, seqlen_k):
+            bias_type = 'matrix'
+        else:
+            raise RuntimeError('Last 2 dimensions of bias must be (1, seqlen_k) or (seqlen_q, seqlen_k)')
+        bias = bias.expand(batch, nheads, seqlen_q, seqlen_k)
+    bias_strides = (bias.stride(0), bias.stride(1), bias.stride(2)) if has_bias else (0, 0, 0)
+    grid = lambda META: (triton.cdiv(seqlen_k, META['BLOCK_N']) if META['SEQUENCE_PARALLEL'] else 1, batch * nheads)
+    _bwd_kernel[grid](q, k, v, bias, do, dq_accum, dk, dv, lse, delta, softmax_scale, q.stride(0), q.stride(2), q.stride(1), k.stride(0), k.stride(2), k.stride(1), v.stride(0), v.stride(2), v.stride(1), *bias_strides, do.stride(0), do.stride(2), do.stride(1), dq_accum.stride(0), dq_accum.stride(2), dq_accum.stride(1), dk.stride(0), dk.stride(2), dk.stride(1), dv.stride(0), dv.stride(2), dv.stride(1), nheads, seqlen_q, seqlen_k, seqlen_q_rounded, d, seqlen_q // 32, seqlen_k // 32, bias_type, causal, BLOCK_HEADDIM)
+    dq.copy_(dq_accum)
+
+class FlashAttnQKVPackedFunc(torch.autograd.Function):
+
+    @staticmethod
+    def forward(ctx, qkv, bias=None, causal=False, softmax_scale=None):
+        """
+            qkv: (batch, seqlen, 3, nheads, headdim)
+            bias: optional, shape broadcastible to (batch, nheads, seqlen, seqlen).
+                For example, ALiBi mask for causal would have shape (1, nheads, 1, seqlen).
+                ALiBi mask for non-causal would have shape (1, nheads, seqlen, seqlen)
+        """
+        if qkv.stride(-1) != 1:
+            qkv = qkv.contiguous()
+        (o, lse, ctx.softmax_scale) = _flash_attn_forward(qkv[:, :, 0], qkv[:, :, 1], qkv[:, :, 2], bias=bias, causal=causal, softmax_scale=softmax_scale)
+        ctx.save_for_backward(qkv, o, lse, bias)
+        ctx.causal = causal
+        return o
+
+    @staticmethod
+    def backward(ctx, do):
+        (qkv, o, lse, bias) = ctx.saved_tensors
+        assert not ctx.needs_input_grad[1], 'FlashAttention does not support bias gradient yet'
+        with torch.inference_mode():
+            dqkv = torch.empty_like(qkv)
+            _flash_attn_backward(do, qkv[:, :, 0], qkv[:, :, 1], qkv[:, :, 2], o, lse, dqkv[:, :, 0], dqkv[:, :, 1], dqkv[:, :, 2], bias=bias, causal=ctx.causal, softmax_scale=ctx.softmax_scale)
+        return (dqkv, None, None, None)
+flash_attn_qkvpacked_func = FlashAttnQKVPackedFunc.apply
+
+class FlashAttnKVPackedFunc(torch.autograd.Function):
+
+    @staticmethod
+    def forward(ctx, q, kv, bias=None, causal=False, softmax_scale=None):
+        """
+            q: (batch, seqlen_q, nheads, headdim)
+            kv: (batch, seqlen_k, 2, nheads, headdim)
+            bias: optional, shape broadcastible to (batch, nheads, seqlen_q, seqlen_k).
+                For example, ALiBi mask for causal would have shape (1, nheads, 1, seqlen_k).
+                ALiBi mask for non-causal would have shape (1, nheads, seqlen_q, seqlen_k)
+        """
+        (q, kv) = [x if x.stride(-1) == 1 else x.contiguous() for x in [q, kv]]
+        (o, lse, ctx.softmax_scale) = _flash_attn_forward(q, kv[:, :, 0], kv[:, :, 1], bias=bias, causal=causal, softmax_scale=softmax_scale)
+        ctx.save_for_backward(q, kv, o, lse, bias)
+        ctx.causal = causal
+        return o
+
+    @staticmethod
+    def backward(ctx, do):
+        (q, kv, o, lse, bias) = ctx.saved_tensors
+        if len(ctx.needs_input_grad) >= 3:
+            assert not ctx.needs_input_grad[2], 'FlashAttention does not support bias gradient yet'
+        with torch.inference_mode():
+            dq = torch.empty_like(q)
+            dkv = torch.empty_like(kv)
+            _flash_attn_backward(do, q, kv[:, :, 0], kv[:, :, 1], o, lse, dq, dkv[:, :, 0], dkv[:, :, 1], bias=bias, causal=ctx.causal, softmax_scale=ctx.softmax_scale)
+        return (dq, dkv, None, None, None)
+flash_attn_kvpacked_func = FlashAttnKVPackedFunc.apply
+
+class FlashAttnFunc(torch.autograd.Function):
+
+    @staticmethod
+    def forward(ctx, q, k, v, bias=None, causal=False, softmax_scale=None):
+        """
+            q: (batch_size, seqlen_q, nheads, headdim)
+            k, v: (batch_size, seqlen_k, nheads, headdim)
+            bias: optional, shape broadcastible to (batch, nheads, seqlen_q, seqlen_k).
+                For example, ALiBi mask for causal would have shape (1, nheads, 1, seqlen_k).
+                ALiBi mask for non-causal would have shape (1, nheads, seqlen_q, seqlen_k)
+        """
+        (q, k, v) = [x if x.stride(-1) == 1 else x.contiguous() for x in [q, k, v]]
+        (o, lse, ctx.softmax_scale) = _flash_attn_forward(q, k, v, bias=bias, causal=causal, softmax_scale=softmax_scale)
+        ctx.save_for_backward(q, k, v, o, lse, bias)
+        ctx.causal = causal
+        return o
+
+    @staticmethod
+    def backward(ctx, do):
+        (q, k, v, o, lse, bias) = ctx.saved_tensors
+        assert not ctx.needs_input_grad[3], 'FlashAttention does not support bias gradient yet'
+        with torch.inference_mode():
+            dq = torch.empty_like(q)
+            dk = torch.empty_like(k)
+            dv = torch.empty_like(v)
+            _flash_attn_backward(do, q, k, v, o, lse, dq, dk, dv, bias=bias, causal=ctx.causal, softmax_scale=ctx.softmax_scale)
+        return (dq, dk, dv, None, None, None)
+flash_attn_func = FlashAttnFunc.apply

videollava/model/language_model/mpt/hf_prefixlm_converter.py

@@ -0,0 +1,415 @@
+"""Converts Huggingface Causal LM to Prefix LM.
+
+Conversion does lightweight surgery on a HuggingFace
+Causal LM to convert it to a Prefix LM.
+
+Prefix LMs accepts a `bidirectional_mask` input in `forward`
+and treat the input prompt as the prefix in `generate`.
+"""
+import math
+import warnings
+from types import MethodType
+from typing import Any, Dict, List, Optional, Tuple, Union
+import torch
+from transformers.models.bloom.modeling_bloom import BaseModelOutputWithPastAndCrossAttentions, BloomForCausalLM, BloomModel, CausalLMOutputWithCrossAttentions, CrossEntropyLoss
+from transformers.models.bloom.modeling_bloom import _expand_mask as _expand_mask_bloom
+from transformers.models.bloom.modeling_bloom import _make_causal_mask as _make_causal_mask_bloom
+from transformers.models.bloom.modeling_bloom import logging
+from transformers.models.gpt2.modeling_gpt2 import GPT2LMHeadModel
+from transformers.models.gpt_neo.modeling_gpt_neo import GPTNeoForCausalLM
+from transformers.models.gpt_neox.modeling_gpt_neox import GPTNeoXForCausalLM
+from transformers.models.gptj.modeling_gptj import GPTJForCausalLM
+from transformers.models.opt.modeling_opt import OPTForCausalLM
+from transformers.models.opt.modeling_opt import _expand_mask as _expand_mask_opt
+from transformers.models.opt.modeling_opt import _make_causal_mask as _make_causal_mask_opt
+logger = logging.get_logger(__name__)
+_SUPPORTED_GPT_MODELS = (GPT2LMHeadModel, GPTJForCausalLM, GPTNeoForCausalLM, GPTNeoXForCausalLM)
+CAUSAL_GPT_TYPES = Union[GPT2LMHeadModel, GPTJForCausalLM, GPTNeoForCausalLM, GPTNeoXForCausalLM]
+
+def _convert_gpt_causal_lm_to_prefix_lm(model: CAUSAL_GPT_TYPES) -> CAUSAL_GPT_TYPES:
+    """Converts a GPT-style Causal LM to a Prefix LM.
+
+    Supported HuggingFace model classes:
+        - `GPT2LMHeadModel`
+        - `GPTNeoForCausalLM`
+        - `GPTNeoXForCausalLM`
+        - `GPTJForCausalLM`
+
+    See `convert_hf_causal_lm_to_prefix_lm` for more details.
+    """
+    if hasattr(model, '_prefix_lm_converted'):
+        return model
+    assert isinstance(model, _SUPPORTED_GPT_MODELS)
+    assert model.config.add_cross_attention == False, 'Only supports GPT-style decoder-only models'
+
+    def _get_attn_modules(model: CAUSAL_GPT_TYPES) -> List[torch.nn.Module]:
+        """Helper that gets a list of the model's attention modules.
+
+        Each module has a `bias` buffer used for causal masking. The Prefix LM
+        conversion adds logic to dynamically manipulate these biases to support
+        Prefix LM attention masking.
+        """
+        attn_modules = []
+        if isinstance(model, GPTNeoXForCausalLM):
+            blocks = model.gpt_neox.layers
+        else:
+            blocks = model.transformer.h
+        for block in blocks:
+            if isinstance(model, GPTNeoForCausalLM):
+                if block.attn.attention_type != 'global':
+                    continue
+                attn_module = block.attn.attention
+            elif isinstance(model, GPTNeoXForCausalLM):
+                attn_module = block.attention
+            else:
+                attn_module = block.attn
+            attn_modules.append(attn_module)
+        return attn_modules
+    setattr(model, '_original_forward', getattr(model, 'forward'))
+    setattr(model, '_original_generate', getattr(model, 'generate'))
+
+    def forward(self: CAUSAL_GPT_TYPES, input_ids: Optional[torch.LongTensor]=None, past_key_values: Optional[Tuple[Tuple[torch.Tensor]]]=None, attention_mask: Optional[torch.FloatTensor]=None, bidirectional_mask: Optional[torch.Tensor]=None, token_type_ids: Optional[torch.LongTensor]=None, position_ids: Optional[torch.LongTensor]=None, head_mask: Optional[torch.FloatTensor]=None, inputs_embeds: Optional[torch.FloatTensor]=None, labels: Optional[torch.LongTensor]=None, use_cache: Optional[bool]=None, output_attentions: Optional[bool]=None, output_hidden_states: Optional[bool]=None, return_dict: Optional[bool]=None):
+        """Wraps original forward to enable PrefixLM attention."""
+
+        def call_og_forward():
+            if isinstance(self, GPTNeoXForCausalLM):
+                return self._original_forward(input_ids=input_ids, past_key_values=past_key_values, attention_mask=attention_mask, head_mask=head_mask, inputs_embeds=inputs_embeds, labels=labels, use_cache=use_cache, output_attentions=output_attentions, output_hidden_states=output_hidden_states, return_dict=return_dict)
+            else:
+                return self._original_forward(input_ids=input_ids, past_key_values=past_key_values, attention_mask=attention_mask, token_type_ids=token_type_ids, position_ids=position_ids, head_mask=head_mask, inputs_embeds=inputs_embeds, labels=labels, use_cache=use_cache, output_attentions=output_attentions, output_hidden_states=output_hidden_states, return_dict=return_dict)
+        if bidirectional_mask is None:
+            return call_og_forward()
+        assert isinstance(bidirectional_mask, torch.Tensor)
+        attn_modules = _get_attn_modules(model)
+        (b, s) = bidirectional_mask.shape
+        max_length = attn_modules[0].bias.shape[-1]
+        if s > max_length:
+            raise ValueError(f'bidirectional_mask sequence length (={s}) exceeds the ' + f'max length allowed by the model ({max_length}).')
+        assert s <= max_length
+        if s < max_length:
+            pad = torch.zeros((int(b), int(max_length - s)), dtype=bidirectional_mask.dtype, device=bidirectional_mask.device)
+            bidirectional_mask = torch.cat([bidirectional_mask, pad], dim=1)
+        bidirectional = bidirectional_mask.unsqueeze(1).unsqueeze(1)
+        for attn_module in attn_modules:
+            attn_module.bias.data = torch.logical_or(attn_module.bias.data, bidirectional)
+        output = call_og_forward()
+        for attn_module in attn_modules:
+            attn_module.bias.data = torch.tril(attn_module.bias.data[0, 0])[None, None]
+        return output
+
+    def generate(self: CAUSAL_GPT_TYPES, *args: tuple, **kwargs: Dict[str, Any]):
+        """Wraps original generate to enable PrefixLM attention."""
+        attn_modules = _get_attn_modules(model)
+        for attn_module in attn_modules:
+            attn_module.bias.data[:] = 1
+        output = self._original_generate(*args, **kwargs)
+        for attn_module in attn_modules:
+            attn_module.bias.data = torch.tril(attn_module.bias.data[0, 0])[None, None]
+        return output
+    setattr(model, 'forward', MethodType(forward, model))
+    setattr(model, 'generate', MethodType(generate, model))
+    setattr(model, '_prefix_lm_converted', True)
+    return model
+
+def _convert_bloom_causal_lm_to_prefix_lm(model: BloomForCausalLM) -> BloomForCausalLM:
+    """Converts a BLOOM Causal LM to a Prefix LM.
+
+    Supported HuggingFace model classes:
+        - `BloomForCausalLM`
+
+    See `convert_hf_causal_lm_to_prefix_lm` for more details.
+    """
+    if hasattr(model, '_prefix_lm_converted'):
+        return model
+    assert isinstance(model, BloomForCausalLM)
+    assert model.config.add_cross_attention == False, 'Only supports BLOOM decoder-only models'
+
+    def _prepare_attn_mask(self: BloomModel, attention_mask: torch.Tensor, bidirectional_mask: Optional[torch.Tensor], input_shape: Tuple[int, int], past_key_values_length: int) -> torch.BoolTensor:
+        combined_attention_mask = None
+        device = attention_mask.device
+        (_, src_length) = input_shape
+        if src_length > 1:
+            combined_attention_mask = _make_causal_mask_bloom(input_shape, device=device, past_key_values_length=past_key_values_length)
+            if bidirectional_mask is not None:
+                assert attention_mask.shape == bidirectional_mask.shape
+                expanded_bidirectional_mask = _expand_mask_bloom(bidirectional_mask, tgt_length=src_length)
+                combined_attention_mask = torch.logical_and(combined_attention_mask, expanded_bidirectional_mask)
+        expanded_attn_mask = _expand_mask_bloom(attention_mask, tgt_length=src_length)
+        combined_attention_mask = expanded_attn_mask if combined_attention_mask is None else expanded_attn_mask | combined_attention_mask
+        return combined_attention_mask
+
+    def _build_alibi_tensor(self: BloomModel, batch_size: int, query_length: int, key_length: int, dtype: torch.dtype, device: torch.device) -> torch.Tensor:
+        num_heads = self.config.n_head
+        closest_power_of_2 = 2 ** math.floor(math.log2(num_heads))
+        base = torch.tensor(2 ** (-2 ** (-(math.log2(closest_power_of_2) - 3))), device=device, dtype=torch.float32)
+        powers = torch.arange(1, 1 + closest_power_of_2, device=device, dtype=torch.int32)
+        slopes = torch.pow(base, powers)
+        if closest_power_of_2 != num_heads:
+            extra_base = torch.tensor(2 ** (-2 ** (-(math.log2(2 * closest_power_of_2) - 3))), device=device, dtype=torch.float32)
+            num_remaining_heads = min(closest_power_of_2, num_heads - closest_power_of_2)
+            extra_powers = torch.arange(1, 1 + 2 * num_remaining_heads, 2, device=device, dtype=torch.int32)
+            slopes = torch.cat([slopes, torch.pow(extra_base, extra_powers)], dim=0)
+        qa = torch.arange(query_length, device=device, dtype=torch.int32).view(-1, 1)
+        ka = torch.arange(key_length, device=device, dtype=torch.int32).view(1, -1)
+        diffs = qa - ka + key_length - query_length
+        diffs = -diffs.abs()
+        alibi = slopes.view(1, num_heads, 1, 1) * diffs.view(1, 1, query_length, key_length)
+        alibi = alibi.expand(batch_size, -1, -1, -1).reshape(-1, query_length, key_length)
+        return alibi.to(dtype)
+    KeyValueT = Tuple[torch.Tensor, torch.Tensor]
+
+    def forward(self: BloomModel, input_ids: Optional[torch.LongTensor]=None, past_key_values: Optional[Tuple[KeyValueT, ...]]=None, attention_mask: Optional[torch.Tensor]=None, bidirectional_mask: Optional[torch.Tensor]=None, head_mask: Optional[torch.LongTensor]=None, inputs_embeds: Optional[torch.LongTensor]=None, use_cache: Optional[bool]=None, output_attentions: Optional[bool]=None, output_hidden_states: Optional[bool]=None, return_dict: Optional[bool]=None, **deprecated_arguments) -> Union[Tuple[torch.Tensor, ...], BaseModelOutputWithPastAndCrossAttentions]:
+        if deprecated_arguments.pop('position_ids', False) is not False:
+            warnings.warn('`position_ids` have no functionality in BLOOM and will be removed in v5.0.0. ' + 'You can safely ignore passing `position_ids`.', FutureWarning)
+        if len(deprecated_arguments) > 0:
+            raise ValueError(f'Got unexpected arguments: {deprecated_arguments}')
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        use_cache = use_cache if use_cache is not None else self.config.use_cache
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+        if input_ids is not None and inputs_embeds is not None:
+            raise ValueError('You cannot specify both input_ids and inputs_embeds at the same time')
+        elif input_ids is not None:
+            (batch_size, seq_length) = input_ids.shape
+        elif inputs_embeds is not None:
+            (batch_size, seq_length, _) = inputs_embeds.shape
+        else:
+            raise ValueError('You have to specify either input_ids or inputs_embeds')
+        if past_key_values is None:
+            past_key_values = tuple([None] * len(self.h))
+        head_mask = self.get_head_mask(head_mask, self.config.n_layer)
+        if inputs_embeds is None:
+            inputs_embeds = self.word_embeddings(input_ids)
+        hidden_states = self.word_embeddings_layernorm(inputs_embeds)
+        presents = () if use_cache else None
+        all_self_attentions = () if output_attentions else None
+        all_hidden_states = () if output_hidden_states else None
+        seq_length_with_past = seq_length
+        past_key_values_length = 0
+        if past_key_values[0] is not None:
+            tmp = past_key_values[0][0]
+            past_key_values_length = tmp.shape[2]
+            seq_length_with_past = seq_length_with_past + past_key_values_length
+        if attention_mask is None:
+            attention_mask = torch.ones((batch_size, seq_length_with_past), device=hidden_states.device)
+        else:
+            attention_mask = attention_mask.to(hidden_states.device)
+        alibi = self._build_alibi_tensor(batch_size=batch_size, query_length=seq_length, key_length=seq_length_with_past, dtype=hidden_states.dtype, device=hidden_states.device)
+        causal_mask = self._prepare_attn_mask(attention_mask, bidirectional_mask, input_shape=(batch_size, seq_length), past_key_values_length=past_key_values_length)
+        for (i, (block, layer_past)) in enumerate(zip(self.h, past_key_values)):
+            if output_hidden_states:
+                hst = (hidden_states,)
+                all_hidden_states = all_hidden_states + hst
+            if self.gradient_checkpointing and self.training:
+                if use_cache:
+                    logger.warning('`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...')
+                    use_cache = False
+
+                def create_custom_forward(module):
+
+                    def custom_forward(*inputs):
+                        return module(*inputs, use_cache=use_cache, output_attentions=output_attentions)
+                    return custom_forward
+                outputs = torch.utils.checkpoint.checkpoint(create_custom_forward(block), hidden_states, alibi, causal_mask, head_mask[i])
+            else:
+                outputs = block(hidden_states, layer_past=layer_past, attention_mask=causal_mask, head_mask=head_mask[i], use_cache=use_cache, output_attentions=output_attentions, alibi=alibi)
+            hidden_states = outputs[0]
+            if use_cache is True:
+                presents = presents + (outputs[1],)
+            if output_attentions:
+                oa = (outputs[2 if use_cache else 1],)
+                all_self_attentions = all_self_attentions + oa
+        hidden_states = self.ln_f(hidden_states)
+        if output_hidden_states:
+            hst = (hidden_states,)
+            all_hidden_states = all_hidden_states + hst
+        if not return_dict:
+            return tuple((v for v in [hidden_states, presents, all_hidden_states, all_self_attentions] if v is not None))
+        return BaseModelOutputWithPastAndCrossAttentions(last_hidden_state=hidden_states, past_key_values=presents, hidden_states=all_hidden_states, attentions=all_self_attentions)
+    setattr(model.transformer, '_prepare_attn_mask', MethodType(_prepare_attn_mask, model.transformer))
+    setattr(model.transformer, '_build_alibi_tensor', MethodType(_build_alibi_tensor, model.transformer))
+    setattr(model.transformer, 'forward', MethodType(forward, model.transformer))
+    KeyValueT = Tuple[torch.Tensor, torch.Tensor]
+
+    def forward(self: BloomForCausalLM, input_ids: Optional[torch.LongTensor]=None, past_key_values: Optional[Tuple[KeyValueT, ...]]=None, attention_mask: Optional[torch.Tensor]=None, bidirectional_mask: Optional[torch.Tensor]=None, head_mask: Optional[torch.Tensor]=None, inputs_embeds: Optional[torch.Tensor]=None, labels: Optional[torch.Tensor]=None, use_cache: Optional[bool]=None, output_attentions: Optional[bool]=None, output_hidden_states: Optional[bool]=None, return_dict: Optional[bool]=None, **deprecated_arguments) -> Union[Tuple[torch.Tensor], CausalLMOutputWithCrossAttentions]:
+        """Replacement forward method for BloomCausalLM."""
+        if deprecated_arguments.pop('position_ids', False) is not False:
+            warnings.warn('`position_ids` have no functionality in BLOOM and will be removed ' + 'in v5.0.0. You can safely ignore passing `position_ids`.', FutureWarning)
+        if len(deprecated_arguments) > 0:
+            raise ValueError(f'Got unexpected arguments: {deprecated_arguments}')
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+        transformer_outputs = self.transformer(input_ids, past_key_values=past_key_values, attention_mask=attention_mask, bidirectional_mask=bidirectional_mask, head_mask=head_mask, inputs_embeds=inputs_embeds, use_cache=use_cache, output_attentions=output_attentions, output_hidden_states=output_hidden_states, return_dict=return_dict)
+        hidden_states = transformer_outputs[0]
+        lm_logits = self.lm_head(hidden_states)
+        loss = None
+        if labels is not None:
+            shift_logits = lm_logits[..., :-1, :].contiguous()
+            shift_labels = labels[..., 1:].contiguous()
+            (batch_size, seq_length, vocab_size) = shift_logits.shape
+            loss_fct = CrossEntropyLoss()
+            loss = loss_fct(shift_logits.view(batch_size * seq_length, vocab_size), shift_labels.view(batch_size * seq_length))
+        if not return_dict:
+            output = (lm_logits,) + transformer_outputs[1:]
+            return (loss,) + output if loss is not None else output
+        return CausalLMOutputWithCrossAttentions(loss=loss, logits=lm_logits, past_key_values=transformer_outputs.past_key_values, hidden_states=transformer_outputs.hidden_states, attentions=transformer_outputs.attentions)
+
+    def prepare_inputs_for_generation(self: BloomForCausalLM, input_ids: torch.LongTensor, past: Optional[torch.Tensor]=None, attention_mask: Optional[torch.Tensor]=None, **kwargs) -> dict:
+        if past:
+            input_ids = input_ids[:, -1].unsqueeze(-1)
+            bidirectional_mask = None
+            if past[0][0].shape[0] == input_ids.shape[0]:
+                past = self._convert_to_bloom_cache(past)
+        else:
+            bidirectional_mask = torch.ones_like(input_ids)
+        return {'input_ids': input_ids, 'past_key_values': past, 'use_cache': True, 'attention_mask': attention_mask, 'bidirectional_mask': bidirectional_mask}
+    setattr(model, 'forward', MethodType(forward, model))
+    setattr(model, 'prepare_inputs_for_generation', MethodType(prepare_inputs_for_generation, model))
+    setattr(model, '_prefix_lm_converted', True)
+    return model
+
+def _convert_opt_causal_lm_to_prefix_lm(model: OPTForCausalLM) -> OPTForCausalLM:
+    """Converts an OPT Causal LM to a Prefix LM.
+
+    Supported HuggingFace model classes:
+        - `OPTForCausalLM`
+
+    See `convert_hf_causal_lm_to_prefix_lm` for more details.
+    """
+    if hasattr(model, '_prefix_lm_converted'):
+        return model
+    assert isinstance(model, OPTForCausalLM)
+    assert model.config.add_cross_attention == False, 'Only supports OPT decoder-only models'
+    setattr(model, '_original_forward', getattr(model, 'forward'))
+    setattr(model, '_original_generate', getattr(model, 'generate'))
+    model.model.decoder.bidirectional_mask = None
+
+    def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_embeds, past_key_values_length):
+        combined_attention_mask = None
+        if input_shape[-1] > 1:
+            if self.bidirectional_mask == 'g':
+                (bsz, src_length) = input_shape
+                combined_attention_mask = torch.zeros((bsz, 1, src_length, src_length + past_key_values_length), dtype=inputs_embeds.dtype, device=inputs_embeds.device)
+            else:
+                combined_attention_mask = _make_causal_mask_opt(input_shape, inputs_embeds.dtype, past_key_values_length=past_key_values_length).to(inputs_embeds.device)
+                if self.bidirectional_mask is not None:
+                    assert attention_mask.shape == self.bidirectional_mask.shape
+                    expanded_bidirectional_mask = _expand_mask_opt(self.bidirectional_mask, inputs_embeds.dtype, tgt_len=input_shape[-1]).to(inputs_embeds.device)
+                    combined_attention_mask = torch.maximum(expanded_bidirectional_mask, combined_attention_mask)
+        if attention_mask is not None:
+            expanded_attn_mask = _expand_mask_opt(attention_mask, inputs_embeds.dtype, tgt_len=input_shape[-1]).to(inputs_embeds.device)
+            combined_attention_mask = expanded_attn_mask if combined_attention_mask is None else expanded_attn_mask + combined_attention_mask
+        return combined_attention_mask
+    setattr(model.model.decoder, '_prepare_decoder_attention_mask', MethodType(_prepare_decoder_attention_mask, model.model.decoder))
+
+    def forward(self: OPTForCausalLM, input_ids: Optional[torch.LongTensor]=None, attention_mask: Optional[torch.Tensor]=None, bidirectional_mask: Optional[torch.ByteTensor]=None, head_mask: Optional[torch.Tensor]=None, past_key_values: Optional[List[torch.FloatTensor]]=None, inputs_embeds: Optional[torch.FloatTensor]=None, labels: Optional[torch.LongTensor]=None, use_cache: Optional[bool]=None, output_attentions: Optional[bool]=None, output_hidden_states: Optional[bool]=None, return_dict: Optional[bool]=None):
+
+        def call_og_forward():
+            return self._original_forward(input_ids=input_ids, attention_mask=attention_mask, head_mask=head_mask, past_key_values=past_key_values, inputs_embeds=inputs_embeds, labels=labels, use_cache=use_cache, output_attentions=output_attentions, output_hidden_states=output_hidden_states, return_dict=return_dict)
+        if bidirectional_mask is None:
+            return call_og_forward()
+        self.model.decoder.bidirectional_mask = bidirectional_mask
+        try:
+            outputs = call_og_forward()
+        except:
+            self.model.decoder.bidirectional_mask = None
+            raise
+        self.model.decoder.bidirectional_mask = None
+        return outputs
+
+    def generate(self: OPTForCausalLM, *args: tuple, **kwargs: Dict[str, Any]):
+        """Wraps original generate to enable PrefixLM-style attention."""
+        self.model.decoder.bidirectional_mask = 'g'
+        try:
+            output = self._original_generate(*args, **kwargs)
+        except:
+            self.model.decoder.bidirectional_mask = None
+            raise
+        self.model.decoder.bidirectional_mask = None
+        return output
+    setattr(model, 'forward', MethodType(forward, model))
+    setattr(model, 'generate', MethodType(generate, model))
+    setattr(model, '_prefix_lm_converted', True)
+    return model
+_SUPPORTED_HF_MODELS = _SUPPORTED_GPT_MODELS + (BloomForCausalLM, OPTForCausalLM)
+CAUSAL_LM_TYPES = Union[GPT2LMHeadModel, GPTJForCausalLM, GPTNeoForCausalLM, GPTNeoXForCausalLM, BloomForCausalLM, OPTForCausalLM]
+
+def convert_hf_causal_lm_to_prefix_lm(model: CAUSAL_LM_TYPES) -> CAUSAL_LM_TYPES:
+    """Converts a HuggingFace Causal LM to a Prefix LM.
+
+    Supported HuggingFace model classes:
+        - `GPT2LMHeadModel`
+        - `GPTNeoForCausalLM`
+        - `GPTNeoXForCausalLM`
+        - `GPTJForCausalLM`
+        - `BloomForCausalLM`
+        - `OPTForCausalLM`
+
+    Conversion to a Prefix LM is done by modifying the `forward` method, and possibly also the
+    `generate` method and/or select underlying methods depending on the model class.
+
+    These changes preserve the model API, but add a new input to `forward`: "bidirectional_mask".
+
+    Notes on training:
+        To actually train the converted model as a Prefix LM, training batches will need to indicate
+        the prefix/target structure by including `bidirectional_mask` as part of the batch inputs.
+
+        **This is not a standard input and requires custom layers either within or after your dataloader.**
+
+        In addition to adding `bidirectional_mask` to the batch, this custom code should modify `labels`
+        such that `batch['labels'][batch['bidirectional_mask'] == 1] == -100`.
+        That is, the prefix portion of the sequence should not generate any loss. Loss should only be
+        generated by the target portion of the sequence.
+
+    Notes on `GPTNeoForCausalLM`:
+        To simplify the implementation, "global" and "local" attention layers are handled differently.
+        For "global" layers, we handle conversion as described above. For "local" layers, which use a
+        causal attention mask within a restricted local window, we do not alter the masking.
+
+    Notes on `forward` method conversion:
+        After conversion, the `forward` method will handle a new input, `bidirectional_mask`,
+        which should be a [batch_size, seq_length] byte tensor, where 1 indicates token positions
+        belonging to the prefix (prefix tokens can attend to one another bidirectionally), and
+        0 indicates token positions belonging to the target.
+
+        The new `forward` method will incorporate `bidirectional_mask` (if supplied) into the existing
+        causal mask, call the original `forward` method, and (if the causal mask is a buffer) reset
+        the causal masks before returning the result.
+
+    Notes on `generate` method conversion:
+        After conversion, the `generate` method will have the same signature but will internally
+        convert all causal masks to be purely bidirectional, call the original `generate` method, and
+        (where appropriate) reset the causal masks before returning the result.
+
+        This works thanks to the logic of the HuggingFace `generate` API, which first encodes the token
+        "prompt" passed to `generate` (which is treated as the prefix) and then sequentially generates
+        each new token. Encodings are cached as generation happens, so all prefix tokens can attend to one
+        another (as expected in a Prefix LM) and generated tokens can only attend to prefix tokens and
+        previously-generated tokens (also as expected in a Prefix LM).
+
+    To preserve the API, the original methods are renamed to `_original_forward` and
+    `_original_generate`, and replaced with new `forward` and `generate` methods that wrap
+    them, respectively. Although implementation details vary by model class.
+    """
+    if isinstance(model, _SUPPORTED_GPT_MODELS):
+        return _convert_gpt_causal_lm_to_prefix_lm(model)
+    elif isinstance(model, BloomForCausalLM):
+        return _convert_bloom_causal_lm_to_prefix_lm(model)
+    elif isinstance(model, OPTForCausalLM):
+        return _convert_opt_causal_lm_to_prefix_lm(model)
+    else:
+        raise TypeError(f'Cannot convert model to Prefix LM. ' + f'Model does not belong to set of supported HF models:' + f'\n{_SUPPORTED_HF_MODELS}')
+
+def add_bidirectional_mask_if_missing(batch: Dict[str, Any]):
+    """Attempts to add bidirectional_mask to batch if missing.
+
+    Raises:
+        KeyError if bidirectional_mask is missing and can't be inferred
+    """
+    if 'bidirectional_mask' not in batch:
+        if batch.get('mode', None) == 'icl_task':
+            batch['bidirectional_mask'] = batch['attention_mask'].clone()
+            for (i, continuation_indices) in enumerate(batch['continuation_indices']):
+                batch['bidirectional_mask'][i, continuation_indices] = 0
+        elif 'labels' in batch and 'attention_mask' in batch:
+            batch['bidirectional_mask'] = torch.logical_and(torch.eq(batch['attention_mask'], 1), torch.eq(batch['labels'], -100)).type_as(batch['attention_mask'])
+        else:
+            raise KeyError('No bidirectional_mask in batch and not sure how to construct one.')

videollava/model/language_model/mpt/meta_init_context.py

@@ -0,0 +1,94 @@
+from contextlib import contextmanager
+import torch
+import torch.nn as nn
+
+@contextmanager
+def init_empty_weights(include_buffers: bool=False):
+    """Meta initialization context manager.
+
+    A context manager under which models are initialized with all parameters
+    on the meta device, therefore creating an empty model. Useful when just
+    initializing the model would blow the available RAM.
+
+    Args:
+        include_buffers (`bool`, *optional*, defaults to `False`): Whether or
+            not to also put all buffers on the meta device while initializing.
+
+    Example:
+    ```python
+    import torch.nn as nn
+
+    # Initialize a model with 100 billions parameters in no time and without using any RAM.
+    with init_empty_weights():
+        tst = nn.Sequential(*[nn.Linear(10000, 10000) for _ in range(1000)])
+    ```
+
+    <Tip warning={true}>
+
+    Any model created under this context manager has no weights. As such you can't do something like
+    `model.to(some_device)` with it. To load weights inside your empty model, see [`load_checkpoint_and_dispatch`].
+
+    </Tip>
+    """
+    with init_on_device(torch.device('meta'), include_buffers=include_buffers) as f:
+        yield f
+
+@contextmanager
+def init_on_device(device: torch.device, include_buffers: bool=False):
+    """Device initialization context manager.
+
+    A context manager under which models are initialized with all parameters
+    on the specified device.
+
+    Args:
+        device (`torch.device`): Device to initialize all parameters on.
+        include_buffers (`bool`, *optional*, defaults to `False`): Whether or
+            not to also put all buffers on the meta device while initializing.
+
+    Example:
+    ```python
+    import torch.nn as nn
+
+    with init_on_device(device=torch.device("cuda")):
+        tst = nn.Liner(100, 100)  # on `cuda` device
+    ```
+    """
+    old_register_parameter = nn.Module.register_parameter
+    if include_buffers:
+        old_register_buffer = nn.Module.register_buffer
+
+    def register_empty_parameter(module, name, param):
+        old_register_parameter(module, name, param)
+        if param is not None:
+            param_cls = type(module._parameters[name])
+            kwargs = module._parameters[name].__dict__
+            module._parameters[name] = param_cls(module._parameters[name].to(device), **kwargs)
+
+    def register_empty_buffer(module, name, buffer):
+        old_register_buffer(module, name, buffer)
+        if buffer is not None:
+            module._buffers[name] = module._buffers[name].to(device)
+    if include_buffers:
+        tensor_constructors_to_patch = {torch_function_name: getattr(torch, torch_function_name) for torch_function_name in ['empty', 'zeros', 'ones', 'full']}
+    else:
+        tensor_constructors_to_patch = {}
+
+    def patch_tensor_constructor(fn):
+
+        def wrapper(*args, **kwargs):
+            kwargs['device'] = device
+            return fn(*args, **kwargs)
+        return wrapper
+    try:
+        nn.Module.register_parameter = register_empty_parameter
+        if include_buffers:
+            nn.Module.register_buffer = register_empty_buffer
+        for torch_function_name in tensor_constructors_to_patch.keys():
+            setattr(torch, torch_function_name, patch_tensor_constructor(getattr(torch, torch_function_name)))
+        yield
+    finally:
+        nn.Module.register_parameter = old_register_parameter
+        if include_buffers:
+            nn.Module.register_buffer = old_register_buffer
+        for (torch_function_name, old_torch_function) in tensor_constructors_to_patch.items():
+            setattr(torch, torch_function_name, old_torch_function)

videollava/model/language_model/mpt/modeling_mpt.py

@@ -0,0 +1,331 @@
+"""A simple, flexible implementation of a GPT model.
+
+Inspired by https://github.com/karpathy/minGPT/blob/master/mingpt/model.py
+"""
+import math
+import warnings
+from typing import List, Optional, Tuple, Union
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from transformers import PreTrainedModel, PreTrainedTokenizer, PreTrainedTokenizerFast
+from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast
+from .attention import attn_bias_shape, build_attn_bias
+from .blocks import MPTBlock
+from .custom_embedding import SharedEmbedding
+from .norm import NORM_CLASS_REGISTRY
+from .configuration_mpt import MPTConfig
+from .adapt_tokenizer import AutoTokenizerForMOD, adapt_tokenizer_for_denoising
+from .hf_prefixlm_converter import add_bidirectional_mask_if_missing, convert_hf_causal_lm_to_prefix_lm
+from .meta_init_context import init_empty_weights
+from .param_init_fns import MODEL_INIT_REGISTRY, generic_param_init_fn_
+try:
+    from .flash_attn_triton import flash_attn_func
+except:
+    pass
+Tokenizer = Union[PreTrainedTokenizer, PreTrainedTokenizerFast]
+
+class MPTPreTrainedModel(PreTrainedModel):
+    config_class = MPTConfig
+    base_model_prefix = 'model'
+    _no_split_modules = ['MPTBlock']
+
+class MPTModel(MPTPreTrainedModel):
+
+    def __init__(self, config: MPTConfig):
+        config._validate_config()
+        super().__init__(config)
+        self.attn_impl = config.attn_config['attn_impl']
+        self.prefix_lm = config.attn_config['prefix_lm']
+        self.attn_uses_sequence_id = config.attn_config['attn_uses_sequence_id']
+        self.alibi = config.attn_config['alibi']
+        self.alibi_bias_max = config.attn_config['alibi_bias_max']
+        if config.init_device == 'mixed':
+            if dist.get_local_rank() == 0:
+                config.init_device = 'cpu'
+            else:
+                config.init_device = 'meta'
+        if config.norm_type.lower() not in NORM_CLASS_REGISTRY.keys():
+            norm_options = ' | '.join(NORM_CLASS_REGISTRY.keys())
+            raise NotImplementedError(f'Requested norm type ({config.norm_type}) is not implemented within this repo (Options: {norm_options}).')
+        norm_class = NORM_CLASS_REGISTRY[config.norm_type.lower()]
+        self.embedding_fraction = config.embedding_fraction
+        self.wte = SharedEmbedding(config.vocab_size, config.d_model, device=config.init_device)
+        if not self.alibi:
+            self.wpe = torch.nn.Embedding(config.max_seq_len, config.d_model, device=config.init_device)
+        self.emb_drop = nn.Dropout(config.emb_pdrop)
+        self.blocks = nn.ModuleList([MPTBlock(device=config.init_device, **config.to_dict()) for _ in range(config.n_layers)])
+        self.norm_f = norm_class(config.d_model, device=config.init_device)
+        if config.init_device != 'meta':
+            print(f'You are using config.init_device={config.init_device!r}, but you can also use config.init_device="meta" with Composer + FSDP for fast initialization.')
+            self.apply(self.param_init_fn)
+        self.is_causal = not self.prefix_lm
+        self._attn_bias_initialized = False
+        self.attn_bias = None
+        self.attn_bias_shape = attn_bias_shape(self.attn_impl, config.n_heads, config.max_seq_len, self.alibi, prefix_lm=self.prefix_lm, causal=self.is_causal, use_sequence_id=self.attn_uses_sequence_id)
+        if config.no_bias:
+            for module in self.modules():
+                if hasattr(module, 'bias') and isinstance(module.bias, nn.Parameter):
+                    if config.verbose:
+                        warnings.warn(f'Removing bias ({module.bias}) from {module}.')
+                    module.register_parameter('bias', None)
+        if config.verbose and config.verbose > 2:
+            print(self)
+        if 'verbose' not in self.config.init_config:
+            self.config.init_config['verbose'] = self.config.verbose
+        if self.config.init_config['verbose'] > 1:
+            init_fn_name = self.config.init_config['name']
+            warnings.warn(f'Using {init_fn_name} initialization.')
+        self.gradient_checkpointing = False
+
+    def get_input_embeddings(self):
+        return self.wte
+
+    def set_input_embeddings(self, value):
+        self.wte = value
+
+    @torch.no_grad()
+    def _attn_bias(self, device, dtype, attention_mask: Optional[torch.ByteTensor]=None, prefix_mask: Optional[torch.ByteTensor]=None, sequence_id: Optional[torch.LongTensor]=None):
+        if not self._attn_bias_initialized:
+            if self.attn_bias_shape:
+                self.attn_bias = torch.zeros(self.attn_bias_shape, device=device, dtype=dtype)
+                self.attn_bias = build_attn_bias(self.attn_impl, self.attn_bias, self.config.n_heads, self.config.max_seq_len, causal=self.is_causal, alibi=self.alibi, alibi_bias_max=self.alibi_bias_max)
+            self._attn_bias_initialized = True
+        if self.attn_impl == 'flash':
+            return (self.attn_bias, attention_mask)
+        if self.attn_bias is not None:
+            self.attn_bias = self.attn_bias.to(dtype=dtype, device=device)
+        attn_bias = self.attn_bias
+        if self.prefix_lm:
+            assert isinstance(attn_bias, torch.Tensor)
+            assert isinstance(prefix_mask, torch.Tensor)
+            attn_bias = self._apply_prefix_mask(attn_bias, prefix_mask)
+        if self.attn_uses_sequence_id and sequence_id is not None:
+            assert isinstance(attn_bias, torch.Tensor)
+            attn_bias = self._apply_sequence_id(attn_bias, sequence_id)
+        if attention_mask is not None:
+            s_k = attention_mask.shape[-1]
+            if attn_bias is None:
+                attn_bias = torch.zeros((1, 1, 1, s_k), device=device, dtype=dtype)
+            else:
+                _s_k = max(0, attn_bias.size(-1) - s_k)
+                attn_bias = attn_bias[:, :, :, _s_k:]
+            if prefix_mask is not None and attention_mask.shape != prefix_mask.shape:
+                raise ValueError(f'attention_mask shape={attention_mask.shape} ' + f'and prefix_mask shape={prefix_mask.shape} are not equal.')
+            min_val = torch.finfo(attn_bias.dtype).min
+            attn_bias = attn_bias.masked_fill(~attention_mask.view(-1, 1, 1, s_k), min_val)
+        return (attn_bias, None)
+
+    def _apply_prefix_mask(self, attn_bias: torch.Tensor, prefix_mask: torch.Tensor):
+        (s_k, s_q) = attn_bias.shape[-2:]
+        if s_k != self.config.max_seq_len or s_q != self.config.max_seq_len:
+            raise ValueError('attn_bias does not match the expected shape. ' + f'The last two dimensions should both be {self.config.max_length} ' + f'but are {s_k} and {s_q}.')
+        seq_len = prefix_mask.shape[-1]
+        if seq_len > self.config.max_seq_len:
+            raise ValueError(f'prefix_mask sequence length cannot exceed max_seq_len={self.config.max_seq_len}')
+        attn_bias = attn_bias[..., :seq_len, :seq_len]
+        causal = torch.tril(torch.ones((seq_len, seq_len), dtype=torch.bool, device=prefix_mask.device)).view(1, 1, seq_len, seq_len)
+        prefix = prefix_mask.view(-1, 1, 1, seq_len)
+        cannot_attend = ~torch.logical_or(causal, prefix.bool())
+        min_val = torch.finfo(attn_bias.dtype).min
+        attn_bias = attn_bias.masked_fill(cannot_attend, min_val)
+        return attn_bias
+
+    def _apply_sequence_id(self, attn_bias: torch.Tensor, sequence_id: torch.LongTensor):
+        seq_len = sequence_id.shape[-1]
+        if seq_len > self.config.max_seq_len:
+            raise ValueError(f'sequence_id sequence length cannot exceed max_seq_len={self.config.max_seq_len}')
+        attn_bias = attn_bias[..., :seq_len, :seq_len]
+        cannot_attend = torch.logical_not(torch.eq(sequence_id.view(-1, seq_len, 1), sequence_id.view(-1, 1, seq_len))).unsqueeze(1)
+        min_val = torch.finfo(attn_bias.dtype).min
+        attn_bias = attn_bias.masked_fill(cannot_attend, min_val)
+        return attn_bias
+
+    def forward(self, input_ids: torch.LongTensor, past_key_values: Optional[List[Tuple[torch.FloatTensor]]]=None, attention_mask: Optional[torch.ByteTensor]=None, prefix_mask: Optional[torch.ByteTensor]=None, sequence_id: Optional[torch.LongTensor]=None, return_dict: Optional[bool]=None, output_attentions: Optional[bool]=None, output_hidden_states: Optional[bool]=None, use_cache: Optional[bool]=None, inputs_embeds: Optional[torch.Tensor]=None):
+        return_dict = return_dict if return_dict is not None else self.config.return_dict
+        use_cache = use_cache if use_cache is not None else self.config.use_cache
+        if attention_mask is not None:
+            attention_mask = attention_mask.bool()
+        if prefix_mask is not None:
+            prefix_mask = prefix_mask.bool()
+        if not return_dict:
+            raise NotImplementedError('return_dict False is not implemented yet for MPT')
+        if output_attentions:
+            if self.attn_impl != 'torch':
+                raise NotImplementedError('output_attentions is not implemented for MPT when using attn_impl `flash` or `triton`.')
+        if attention_mask is not None and attention_mask[:, 0].sum() != attention_mask.shape[0] and self.training:
+            raise NotImplementedError('MPT does not support training with left padding.')
+        if self.prefix_lm and prefix_mask is None:
+            raise ValueError('prefix_mask is a required argument when MPT is configured with prefix_lm=True.')
+        if self.training:
+            if self.attn_uses_sequence_id and sequence_id is None:
+                raise ValueError('sequence_id is a required argument when MPT is configured with attn_uses_sequence_id=True ' + 'and the model is in train mode.')
+            elif self.attn_uses_sequence_id is False and sequence_id is not None:
+                warnings.warn('MPT received non-None input for `sequence_id` but is configured with attn_uses_sequence_id=False. ' + 'This input will be ignored. If you want the model to use `sequence_id`, set attn_uses_sequence_id to True.')
+        if input_ids is not None:
+            S = input_ids.size(1)
+            assert S <= self.config.max_seq_len, f'Cannot forward input with seq_len={S}, this model only supports seq_len<={self.config.max_seq_len}'
+            tok_emb = self.wte(input_ids)
+        else:
+            assert inputs_embeds is not None
+            assert self.alibi, 'inputs_embeds is not implemented for MPT unless for alibi.'
+            S = inputs_embeds.size(1)
+            tok_emb = inputs_embeds
+        if self.alibi:
+            x = tok_emb
+        else:
+            past_position = 0
+            if past_key_values is not None:
+                if len(past_key_values) != self.config.n_layers:
+                    raise ValueError(f'past_key_values must provide a past_key_value for each attention ' + f'layer in the network (len(past_key_values)={len(past_key_values)!r}; self.config.n_layers={self.config.n_layers!r}).')
+                past_position = past_key_values[0][0].size(1)
+                if self.attn_impl == 'torch':
+                    past_position = past_key_values[0][0].size(3)
+            if S + past_position > self.config.max_seq_len:
+                raise ValueError(f'Cannot forward input with past sequence length {past_position} and current sequence length {S + 1}, this model only supports total sequence length <= {self.config.max_seq_len}.')
+            pos = torch.arange(past_position, S + past_position, dtype=torch.long, device=input_ids.device).unsqueeze(0)
+            if attention_mask is not None:
+                pos = torch.clamp(pos - torch.cumsum((~attention_mask).to(torch.int32), dim=1)[:, past_position:], min=0)
+            pos_emb = self.wpe(pos)
+            x = tok_emb + pos_emb
+        if self.embedding_fraction == 1:
+            x = self.emb_drop(x)
+        else:
+            x_shrunk = x * self.embedding_fraction + x.detach() * (1 - self.embedding_fraction)
+            assert isinstance(self.emb_drop, nn.Module)
+            x = self.emb_drop(x_shrunk)
+        (attn_bias, attention_mask) = self._attn_bias(device=x.device, dtype=torch.float32, attention_mask=attention_mask, prefix_mask=prefix_mask, sequence_id=sequence_id)
+        if use_cache and past_key_values is None:
+            past_key_values = [() for _ in range(self.config.n_layers)]
+        all_hidden_states = () if output_hidden_states else None
+        all_self_attns = () if output_attentions else None
+        for (b_idx, block) in enumerate(self.blocks):
+            if output_hidden_states:
+                assert all_hidden_states is not None
+                all_hidden_states = all_hidden_states + (x,)
+            past_key_value = past_key_values[b_idx] if past_key_values is not None else None
+            if self.gradient_checkpointing and self.training:
+                (x, attn_weights, past_key_value) = torch.utils.checkpoint.checkpoint(block, x, past_key_value, attn_bias, attention_mask, self.is_causal)
+            else:
+                (x, attn_weights, past_key_value) = block(x, past_key_value=past_key_value, attn_bias=attn_bias, attention_mask=attention_mask, is_causal=self.is_causal)
+            if past_key_values is not None:
+                past_key_values[b_idx] = past_key_value
+            if output_attentions:
+                assert all_self_attns is not None
+                all_self_attns = all_self_attns + (attn_weights,)
+        x = self.norm_f(x)
+        if output_hidden_states:
+            assert all_hidden_states is not None
+            all_hidden_states = all_hidden_states + (x,)
+        return BaseModelOutputWithPast(last_hidden_state=x, past_key_values=past_key_values, hidden_states=all_hidden_states, attentions=all_self_attns)
+
+    def param_init_fn(self, module):
+        init_fn_name = self.config.init_config['name']
+        MODEL_INIT_REGISTRY[init_fn_name](module=module, n_layers=self.config.n_layers, d_model=self.config.d_model, **self.config.init_config)
+
+    def fsdp_wrap_fn(self, module):
+        return isinstance(module, MPTBlock)
+
+    def activation_checkpointing_fn(self, module):
+        return isinstance(module, MPTBlock)
+
+class MPTForCausalLM(MPTPreTrainedModel):
+
+    def __init__(self, config: MPTConfig):
+        super().__init__(config)
+        if not config.tie_word_embeddings:
+            raise ValueError('MPTForCausalLM only supports tied word embeddings')
+        print(f'Instantiating an MPTForCausalLM model from {__file__}')
+        self.transformer = MPTModel(config)
+        for child in self.transformer.children():
+            if isinstance(child, torch.nn.ModuleList):
+                continue
+            if isinstance(child, torch.nn.Module):
+                child._fsdp_wrap = True
+        self.logit_scale = None
+        if config.logit_scale is not None:
+            logit_scale = config.logit_scale
+            if isinstance(logit_scale, str):
+                if logit_scale == 'inv_sqrt_d_model':
+                    logit_scale = 1 / math.sqrt(config.d_model)
+                else:
+                    raise ValueError(f"logit_scale={logit_scale!r} is not recognized as an option; use numeric value or 'inv_sqrt_d_model'.")
+            self.logit_scale = logit_scale
+
+    def get_input_embeddings(self):
+        return self.transformer.wte
+
+    def set_input_embeddings(self, value):
+        self.transformer.wte = value
+
+    def get_output_embeddings(self):
+        return self.transformer.wte
+
+    def set_output_embeddings(self, new_embeddings):
+        self.transformer.wte = new_embeddings
+
+    def set_decoder(self, decoder):
+        self.transformer = decoder
+
+    def get_decoder(self):
+        return self.transformer
+
+    def forward(self, input_ids: torch.LongTensor, past_key_values: Optional[List[Tuple[torch.FloatTensor]]]=None, attention_mask: Optional[torch.ByteTensor]=None, prefix_mask: Optional[torch.ByteTensor]=None, sequence_id: Optional[torch.LongTensor]=None, labels: Optional[torch.LongTensor]=None, return_dict: Optional[bool]=None, output_attentions: Optional[bool]=None, output_hidden_states: Optional[bool]=None, use_cache: Optional[bool]=None, inputs_embeds: Optional[torch.FloatTensor]=None):
+        return_dict = return_dict if return_dict is not None else self.config.return_dict
+        use_cache = use_cache if use_cache is not None else self.config.use_cache
+        if inputs_embeds is not None:
+            raise NotImplementedError('inputs_embeds has to be None (for hf/peft support).')
+        outputs = self.transformer(input_ids=input_ids, past_key_values=past_key_values, attention_mask=attention_mask, prefix_mask=prefix_mask, sequence_id=sequence_id, return_dict=return_dict, output_attentions=output_attentions, output_hidden_states=output_hidden_states, use_cache=use_cache)
+        logits = self.transformer.wte(outputs.last_hidden_state.to(self.transformer.wte.weight.device), True)
+        if self.logit_scale is not None:
+            if self.logit_scale == 0:
+                warnings.warn(f'Multiplying logits by self.logit_scale={self.logit_scale!r}. This will produce uniform (uninformative) outputs.')
+            logits *= self.logit_scale
+        loss = None
+        if labels is not None:
+            labels = torch.roll(labels, shifts=-1)
+            labels[:, -1] = -100
+            loss = F.cross_entropy(logits.view(-1, logits.size(-1)), labels.to(logits.device).view(-1))
+        return CausalLMOutputWithPast(loss=loss, logits=logits, past_key_values=outputs.past_key_values, hidden_states=outputs.hidden_states, attentions=outputs.attentions)
+
+    def param_init_fn(self, module):
+        init_fn_name = self.config.init_config['name']
+        MODEL_INIT_REGISTRY[init_fn_name](module=module, n_layers=self.config.n_layers, d_model=self.config.d_model, **self.config.init_config)
+
+    def fsdp_wrap_fn(self, module):
+        return isinstance(module, MPTBlock)
+
+    def activation_checkpointing_fn(self, module):
+        return isinstance(module, MPTBlock)
+
+    def prepare_inputs_for_generation(self, input_ids, past_key_values=None, inputs_embeds=None, **kwargs):
+        if inputs_embeds is not None:
+            raise NotImplementedError('inputs_embeds is not implemented for MPT yet')
+        attention_mask = kwargs['attention_mask'].bool()
+        if attention_mask[:, -1].sum() != attention_mask.shape[0]:
+            raise NotImplementedError('MPT does not support generation with right padding.')
+        if self.transformer.attn_uses_sequence_id and self.training:
+            sequence_id = torch.zeros_like(input_ids[:1])
+        else:
+            sequence_id = None
+        if past_key_values is not None:
+            input_ids = input_ids[:, -1].unsqueeze(-1)
+        if self.transformer.prefix_lm:
+            prefix_mask = torch.ones_like(attention_mask)
+            if kwargs.get('use_cache') == False:
+                raise NotImplementedError('MPT with prefix_lm=True does not support use_cache=False.')
+        else:
+            prefix_mask = None
+        return {'input_ids': input_ids, 'attention_mask': attention_mask, 'prefix_mask': prefix_mask, 'sequence_id': sequence_id, 'past_key_values': past_key_values, 'use_cache': kwargs.get('use_cache', True)}
+
+    @staticmethod
+    def _reorder_cache(past_key_values, beam_idx):
+        """Used by HuggingFace generate when using beam search with kv-caching.
+
+        See https://github.com/huggingface/transformers/blob/3ec7a47664ebe40c40f4b722f6bb1cd30c3821ec/src/transformers/models/gpt2/modeling_gpt2.py#L1122-L1133
+        for an example in transformers.
+        """
+        reordered_past = []
+        for layer_past in past_key_values:
+            reordered_past += [tuple((past_state.index_select(0, beam_idx) for past_state in layer_past))]
+        return reordered_past

videollava/model/language_model/mpt/norm.py

@@ -0,0 +1,56 @@
+import torch
+
+def _cast_if_autocast_enabled(tensor):
+    if torch.is_autocast_enabled():
+        if tensor.device.type == 'cuda':
+            dtype = torch.get_autocast_gpu_dtype()
+        elif tensor.device.type == 'cpu':
+            dtype = torch.get_autocast_cpu_dtype()
+        else:
+            raise NotImplementedError()
+        return tensor.to(dtype=dtype)
+    return tensor
+
+class LPLayerNorm(torch.nn.LayerNorm):
+
+    def __init__(self, normalized_shape, eps=1e-05, elementwise_affine=True, device=None, dtype=None):
+        super().__init__(normalized_shape=normalized_shape, eps=eps, elementwise_affine=elementwise_affine, device=device, dtype=dtype)
+
+    def forward(self, x):
+        module_device = x.device
+        downcast_x = _cast_if_autocast_enabled(x)
+        downcast_weight = _cast_if_autocast_enabled(self.weight) if self.weight is not None else self.weight
+        downcast_bias = _cast_if_autocast_enabled(self.bias) if self.bias is not None else self.bias
+        with torch.autocast(enabled=False, device_type=module_device.type):
+            return torch.nn.functional.layer_norm(downcast_x, self.normalized_shape, downcast_weight, downcast_bias, self.eps)
+
+def rms_norm(x, weight=None, eps=1e-05):
+    output = x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + eps)
+    if weight is not None:
+        return output * weight
+    return output
+
+class RMSNorm(torch.nn.Module):
+
+    def __init__(self, normalized_shape, eps=1e-05, weight=True, dtype=None, device=None):
+        super().__init__()
+        self.eps = eps
+        if weight:
+            self.weight = torch.nn.Parameter(torch.ones(normalized_shape, dtype=dtype, device=device))
+        else:
+            self.register_parameter('weight', None)
+
+    def forward(self, x):
+        return rms_norm(x.float(), self.weight, self.eps).to(dtype=x.dtype)
+
+class LPRMSNorm(RMSNorm):
+
+    def __init__(self, normalized_shape, eps=1e-05, weight=True, dtype=None, device=None):
+        super().__init__(normalized_shape=normalized_shape, eps=eps, weight=weight, dtype=dtype, device=device)
+
+    def forward(self, x):
+        downcast_x = _cast_if_autocast_enabled(x)
+        downcast_weight = _cast_if_autocast_enabled(self.weight) if self.weight is not None else self.weight
+        with torch.autocast(enabled=False, device_type=x.device.type):
+            return rms_norm(downcast_x, downcast_weight, self.eps).to(dtype=x.dtype)
+NORM_CLASS_REGISTRY = {'layernorm': torch.nn.LayerNorm, 'low_precision_layernorm': LPLayerNorm, 'rmsnorm': RMSNorm, 'low_precision_rmsnorm': LPRMSNorm}

videollava/model/language_model/mpt/param_init_fns.py

@@ -0,0 +1,181 @@
+import math
+import warnings
+from collections.abc import Sequence
+from functools import partial
+from typing import Optional, Tuple, Union
+import torch
+from torch import nn
+from .norm import NORM_CLASS_REGISTRY
+
+def torch_default_param_init_fn_(module: nn.Module, verbose: int=0, **kwargs):
+    del kwargs
+    if verbose > 1:
+        warnings.warn(f"Initializing network using module's reset_parameters attribute")
+    if hasattr(module, 'reset_parameters'):
+        module.reset_parameters()
+
+def fused_init_helper_(module: nn.Module, init_fn_):
+    _fused = getattr(module, '_fused', None)
+    if _fused is None:
+        raise RuntimeError(f'Internal logic error')
+    (dim, splits) = _fused
+    splits = (0, *splits, module.weight.size(dim))
+    for (s, e) in zip(splits[:-1], splits[1:]):
+        slice_indices = [slice(None)] * module.weight.ndim
+        slice_indices[dim] = slice(s, e)
+        init_fn_(module.weight[slice_indices])
+
+def generic_param_init_fn_(module: nn.Module, init_fn_, n_layers: int, d_model: Optional[int]=None, init_div_is_residual: Union[int, float, str, bool]=True, emb_init_std: Optional[float]=None, emb_init_uniform_lim: Optional[Union[Tuple[float, float], float]]=None, verbose: int=0, **kwargs):
+    del kwargs
+    if verbose > 1:
+        warnings.warn(f'If model has bias parameters they are initialized to 0.')
+    init_div_is_residual = init_div_is_residual
+    if init_div_is_residual is False:
+        div_is_residual = 1.0
+    elif init_div_is_residual is True:
+        div_is_residual = math.sqrt(2 * n_layers)
+    elif isinstance(init_div_is_residual, float) or isinstance(init_div_is_residual, int):
+        div_is_residual = init_div_is_residual
+    elif isinstance(init_div_is_residual, str) and init_div_is_residual.isnumeric():
+        div_is_residual = float(init_div_is_residual)
+    else:
+        div_is_residual = 1.0
+        raise ValueError(f'Expected init_div_is_residual to be boolean or numeric, got {init_div_is_residual}')
+    if init_div_is_residual is not False:
+        if verbose > 1:
+            warnings.warn(f'Initializing _is_residual layers then dividing them by {div_is_residual:.3f}. ' + f'Set `init_div_is_residual: false` in init config to disable this.')
+    if isinstance(module, nn.Linear):
+        if hasattr(module, '_fused'):
+            fused_init_helper_(module, init_fn_)
+        else:
+            init_fn_(module.weight)
+        if module.bias is not None:
+            torch.nn.init.zeros_(module.bias)
+        if init_div_is_residual is not False and getattr(module, '_is_residual', False):
+            with torch.no_grad():
+                module.weight.div_(div_is_residual)
+    elif isinstance(module, nn.Embedding):
+        if emb_init_std is not None:
+            std = emb_init_std
+            if std == 0:
+                warnings.warn(f'Embedding layer initialized to 0.')
+            emb_init_fn_ = partial(torch.nn.init.normal_, mean=0.0, std=std)
+            if verbose > 1:
+                warnings.warn(f'Embedding layer initialized using normal distribution with mean=0 and std={std!r}.')
+        elif emb_init_uniform_lim is not None:
+            lim = emb_init_uniform_lim
+            if isinstance(lim, Sequence):
+                if len(lim) > 2:
+                    raise ValueError(f'Uniform init requires a min and a max limit. User input: {lim}.')
+                if lim[0] == lim[1]:
+                    warnings.warn(f'Embedding layer initialized to {lim[0]}.')
+            else:
+                if lim == 0:
+                    warnings.warn(f'Embedding layer initialized to 0.')
+                lim = [-lim, lim]
+            (a, b) = lim
+            emb_init_fn_ = partial(torch.nn.init.uniform_, a=a, b=b)
+            if verbose > 1:
+                warnings.warn(f'Embedding layer initialized using uniform distribution in range {lim}.')
+        else:
+            emb_init_fn_ = init_fn_
+        emb_init_fn_(module.weight)
+    elif isinstance(module, tuple(set(NORM_CLASS_REGISTRY.values()))):
+        if verbose > 1:
+            warnings.warn(f'Norm weights are set to 1. If norm layer has a bias it is initialized to 0.')
+        if hasattr(module, 'weight') and module.weight is not None:
+            torch.nn.init.ones_(module.weight)
+        if hasattr(module, 'bias') and module.bias is not None:
+            torch.nn.init.zeros_(module.bias)
+    elif isinstance(module, nn.MultiheadAttention):
+        if module._qkv_same_embed_dim:
+            assert module.in_proj_weight is not None
+            assert module.q_proj_weight is None and module.k_proj_weight is None and (module.v_proj_weight is None)
+            assert d_model is not None
+            _d = d_model
+            splits = (0, _d, 2 * _d, 3 * _d)
+            for (s, e) in zip(splits[:-1], splits[1:]):
+                init_fn_(module.in_proj_weight[s:e])
+        else:
+            assert module.q_proj_weight is not None and module.k_proj_weight is not None and (module.v_proj_weight is not None)
+            assert module.in_proj_weight is None
+            init_fn_(module.q_proj_weight)
+            init_fn_(module.k_proj_weight)
+            init_fn_(module.v_proj_weight)
+        if module.in_proj_bias is not None:
+            torch.nn.init.zeros_(module.in_proj_bias)
+        if module.bias_k is not None:
+            torch.nn.init.zeros_(module.bias_k)
+        if module.bias_v is not None:
+            torch.nn.init.zeros_(module.bias_v)
+        init_fn_(module.out_proj.weight)
+        if init_div_is_residual is not False and getattr(module.out_proj, '_is_residual', False):
+            with torch.no_grad():
+                module.out_proj.weight.div_(div_is_residual)
+        if module.out_proj.bias is not None:
+            torch.nn.init.zeros_(module.out_proj.bias)
+    else:
+        for _ in module.parameters(recurse=False):
+            raise NotImplementedError(f'{module.__class__.__name__} parameters are not initialized by param_init_fn.')
+
+def _normal_init_(std, mean=0.0):
+    return partial(torch.nn.init.normal_, mean=mean, std=std)
+
+def _normal_param_init_fn_(module: nn.Module, std: float, n_layers: int, d_model: Optional[int]=None, init_div_is_residual: Union[int, float, str, bool]=True, emb_init_std: Optional[float]=None, emb_init_uniform_lim: Optional[Union[Tuple[float, float], float]]=None, verbose: int=0, **kwargs):
+    del kwargs
+    init_fn_ = _normal_init_(std=std)
+    if verbose > 1:
+        warnings.warn(f'Using torch.nn.init.normal_ init fn mean=0.0, std={std}')
+    generic_param_init_fn_(module=module, init_fn_=init_fn_, d_model=d_model, n_layers=n_layers, init_div_is_residual=init_div_is_residual, emb_init_std=emb_init_std, emb_init_uniform_lim=emb_init_uniform_lim, verbose=verbose)
+
+def baseline_param_init_fn_(module: nn.Module, init_std: float, n_layers: int, d_model: Optional[int]=None, init_div_is_residual: Union[int, float, str, bool]=True, emb_init_std: Optional[float]=None, emb_init_uniform_lim: Optional[Union[Tuple[float, float], float]]=None, verbose: int=0, **kwargs):
+    del kwargs
+    if init_std is None:
+        raise ValueError("You must set model.init_config['init_std'] to a float value to use the default initialization scheme.")
+    _normal_param_init_fn_(module=module, std=init_std, d_model=d_model, n_layers=n_layers, init_div_is_residual=init_div_is_residual, emb_init_std=emb_init_std, emb_init_uniform_lim=emb_init_uniform_lim, verbose=verbose)
+
+def small_param_init_fn_(module: nn.Module, n_layers: int, d_model: int, init_div_is_residual: Union[int, float, str, bool]=True, emb_init_std: Optional[float]=None, emb_init_uniform_lim: Optional[Union[Tuple[float, float], float]]=None, verbose: int=0, **kwargs):
+    del kwargs
+    std = math.sqrt(2 / (5 * d_model))
+    _normal_param_init_fn_(module=module, std=std, d_model=d_model, n_layers=n_layers, init_div_is_residual=init_div_is_residual, emb_init_std=emb_init_std, emb_init_uniform_lim=emb_init_uniform_lim, verbose=verbose)
+
+def neox_param_init_fn_(module: nn.Module, n_layers: int, d_model: int, emb_init_std: Optional[float]=None, emb_init_uniform_lim: Optional[Union[Tuple[float, float], float]]=None, verbose: int=0, **kwargs):
+    """From section 2.3.1 of GPT-NeoX-20B:
+
+    An Open-Source AutoregressiveLanguage Model — Black et. al. (2022)
+    see https://github.com/EleutherAI/gpt-neox/blob/9610391ab319403cef079b438edd016a2443af54/megatron/model/init_functions.py#L151
+    and https://github.com/EleutherAI/gpt-neox/blob/main/megatron/model/transformer.py
+    """
+    del kwargs
+    residual_div = n_layers / math.sqrt(10)
+    if verbose > 1:
+        warnings.warn(f'setting init_div_is_residual to {residual_div}')
+    small_param_init_fn_(module=module, d_model=d_model, n_layers=n_layers, init_div_is_residual=residual_div, emb_init_std=emb_init_std, emb_init_uniform_lim=emb_init_uniform_lim, verbose=verbose)
+
+def kaiming_uniform_param_init_fn_(module: nn.Module, n_layers: int, d_model: Optional[int]=None, init_div_is_residual: Union[int, float, str, bool]=True, emb_init_std: Optional[float]=None, emb_init_uniform_lim: Optional[Union[Tuple[float, float], float]]=None, init_gain: float=0, fan_mode: str='fan_in', init_nonlinearity: str='leaky_relu', verbose: int=0, **kwargs):
+    del kwargs
+    if verbose > 1:
+        warnings.warn(f'Using nn.init.kaiming_uniform_ init fn with parameters: ' + f'a={init_gain}, mode={fan_mode}, nonlinearity={init_nonlinearity}')
+    kaiming_uniform_ = partial(nn.init.kaiming_uniform_, a=init_gain, mode=fan_mode, nonlinearity=init_nonlinearity)
+    generic_param_init_fn_(module=module, init_fn_=kaiming_uniform_, d_model=d_model, n_layers=n_layers, init_div_is_residual=init_div_is_residual, emb_init_std=emb_init_std, emb_init_uniform_lim=emb_init_uniform_lim, verbose=verbose)
+
+def kaiming_normal_param_init_fn_(module: nn.Module, n_layers: int, d_model: Optional[int]=None, init_div_is_residual: Union[int, float, str, bool]=True, emb_init_std: Optional[float]=None, emb_init_uniform_lim: Optional[Union[Tuple[float, float], float]]=None, init_gain: float=0, fan_mode: str='fan_in', init_nonlinearity: str='leaky_relu', verbose: int=0, **kwargs):
+    del kwargs
+    if verbose > 1:
+        warnings.warn(f'Using nn.init.kaiming_normal_ init fn with parameters: ' + f'a={init_gain}, mode={fan_mode}, nonlinearity={init_nonlinearity}')
+    kaiming_normal_ = partial(torch.nn.init.kaiming_normal_, a=init_gain, mode=fan_mode, nonlinearity=init_nonlinearity)
+    generic_param_init_fn_(module=module, init_fn_=kaiming_normal_, d_model=d_model, n_layers=n_layers, init_div_is_residual=init_div_is_residual, emb_init_std=emb_init_std, emb_init_uniform_lim=emb_init_uniform_lim, verbose=verbose)
+
+def xavier_uniform_param_init_fn_(module: nn.Module, n_layers: int, d_model: Optional[int]=None, init_div_is_residual: Union[int, float, str, bool]=True, emb_init_std: Optional[float]=None, emb_init_uniform_lim: Optional[Union[Tuple[float, float], float]]=None, init_gain: float=0, verbose: int=0, **kwargs):
+    del kwargs
+    xavier_uniform_ = partial(torch.nn.init.xavier_uniform_, gain=init_gain)
+    if verbose > 1:
+        warnings.warn(f'Using torch.nn.init.xavier_uniform_ init fn with parameters: ' + f'gain={init_gain}')
+    generic_param_init_fn_(module=module, init_fn_=xavier_uniform_, d_model=d_model, n_layers=n_layers, init_div_is_residual=init_div_is_residual, emb_init_std=emb_init_std, emb_init_uniform_lim=emb_init_uniform_lim, verbose=verbose)
+
+def xavier_normal_param_init_fn_(module: nn.Module, n_layers: int, d_model: Optional[int]=None, init_div_is_residual: Union[int, float, str, bool]=True, emb_init_std: Optional[float]=None, emb_init_uniform_lim: Optional[Union[Tuple[float, float], float]]=None, init_gain: float=0, verbose: int=0, **kwargs):
+    xavier_normal_ = partial(torch.nn.init.xavier_normal_, gain=init_gain)
+    if verbose > 1:
+        warnings.warn(f'Using torch.nn.init.xavier_normal_ init fn with parameters: ' + f'gain={init_gain}')
+    generic_param_init_fn_(module=module, init_fn_=xavier_normal_, d_model=d_model, n_layers=n_layers, init_div_is_residual=init_div_is_residual, emb_init_std=emb_init_std, emb_init_uniform_lim=emb_init_uniform_lim, verbose=verbose)
+MODEL_INIT_REGISTRY = {'default_': torch_default_param_init_fn_, 'baseline_': baseline_param_init_fn_, 'kaiming_uniform_': kaiming_uniform_param_init_fn_, 'kaiming_normal_': kaiming_normal_param_init_fn_, 'neox_init_': neox_param_init_fn_, 'small_init_': small_param_init_fn_, 'xavier_uniform_': xavier_uniform_param_init_fn_, 'xavier_normal_': xavier_normal_param_init_fn_}

videollava/model/llava_arch.py

@@ -0,0 +1,390 @@
+#    Copyright 2023 Haotian Liu
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+
+from abc import ABC, abstractmethod
+
+import torch
+import torch.nn as nn
+
+from .multimodal_encoder.builder import build_image_tower, build_video_tower
+from .multimodal_projector.builder import build_vision_projector
+
+from videollava.constants import IGNORE_INDEX, IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_PATCH_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
+
+
+class LlavaMetaModel:
+
+    def __init__(self, config):
+        super(LlavaMetaModel, self).__init__(config)
+
+        if getattr(config, "mm_image_tower", None) is not None:
+            self.image_tower = build_image_tower(config, delay_load=True)
+        if getattr(config, "mm_video_tower", None) is not None:
+            self.video_tower = build_video_tower(config, delay_load=True)
+        if getattr(config, "mm_image_tower", None) is not None or getattr(config, "mm_video_tower", None) is not None:
+            self.mm_projector = build_vision_projector(config)
+
+    def get_image_tower(self):
+        image_tower = getattr(self, 'image_tower', None)
+        if type(image_tower) is list:
+            image_tower = image_tower[0]
+        return image_tower
+
+    def get_video_tower(self):
+        video_tower = getattr(self, 'video_tower', None)
+        if type(video_tower) is list:
+            video_tower = video_tower[0]
+        return video_tower
+
+    def initialize_vision_modules(self, model_args, fsdp=None):
+        # ==============================================
+        image_tower = model_args.image_tower
+        video_tower = model_args.video_tower
+        assert image_tower is not None or video_tower is not None
+        # ==============================================
+        mm_vision_select_layer = model_args.mm_vision_select_layer
+        mm_vision_select_feature = model_args.mm_vision_select_feature
+        pretrain_mm_mlp_adapter = model_args.pretrain_mm_mlp_adapter
+
+        # ==========================================================================
+
+        self.config.mm_image_tower = image_tower
+        if image_tower is not None:
+            if self.get_image_tower() is None:
+                image_tower = build_image_tower(model_args)
+
+                if fsdp is not None and len(fsdp) > 0:
+                    self.image_tower = [image_tower]
+                else:
+                    self.image_tower = image_tower
+            else:
+                if fsdp is not None and len(fsdp) > 0:
+                    image_tower = self.image_tower[0]
+                else:
+                    image_tower = self.image_tower
+                image_tower.load_model()
+
+        self.config.mm_video_tower = video_tower
+        if video_tower is not None:
+            if self.get_video_tower() is None:
+                video_tower = build_video_tower(model_args)
+
+                if fsdp is not None and len(fsdp) > 0:
+                    self.video_tower = [video_tower]
+                else:
+                    self.video_tower = video_tower
+            else:
+                if fsdp is not None and len(fsdp) > 0:
+                    video_tower = self.video_tower[0]
+                else:
+                    video_tower = self.video_tower
+                video_tower.load_model()
+
+        # ==========================================================================
+
+        self.config.use_mm_proj = True
+        self.config.mm_projector_type = getattr(model_args, 'mm_projector_type', 'linear')
+        self.config.mm_vision_select_layer = mm_vision_select_layer
+        self.config.mm_vision_select_feature = mm_vision_select_feature
+        # ==========================================================================
+        if image_tower is not None and video_tower is not None:  # TODO: support different hidden_size
+            assert image_tower.hidden_size == video_tower.hidden_size
+            self.config.mm_hidden_size = image_tower.hidden_size
+        else:
+            self.config.mm_hidden_size = max(getattr(image_tower, 'hidden_size', -1),
+                                             getattr(video_tower, 'hidden_size', -1))
+        # ===================================================================================
+
+        if getattr(self, 'mm_projector', None) is None:
+            self.mm_projector = build_vision_projector(self.config)
+        else:
+            # In case it is frozen by LoRA
+            for p in self.mm_projector.parameters():
+                p.requires_grad = True
+
+        if pretrain_mm_mlp_adapter is not None:
+            mm_projector_weights = torch.load(pretrain_mm_mlp_adapter, map_location='cpu')
+            def get_w(weights, keyword):
+                return {k.split(keyword + '.')[1]: v for k, v in weights.items() if keyword in k}
+
+            self.mm_projector.load_state_dict(get_w(mm_projector_weights, 'mm_projector'))
+
+
+class LlavaMetaForCausalLM(ABC):
+
+    @abstractmethod
+    def get_model(self):
+        pass
+
+    def get_image_tower(self):
+        return self.get_model().get_image_tower()
+
+    def get_video_tower(self):
+        return self.get_model().get_video_tower()
+
+    def encode_images(self, images):
+        image_features = self.get_model().get_image_tower()(images)
+        image_features = self.get_model().mm_projector(image_features)
+        return image_features
+
+    def encode_videos(self, videos):  # [mini_b, c, t, h, w]
+        b, _, t, _, _ = videos.shape
+        video_features = self.get_model().get_video_tower()(videos)  # [mini_b, t, n, c]
+        video_features = self.get_model().mm_projector(video_features)
+        return video_features
+
+    def prepare_inputs_labels_for_multimodal(
+        self, input_ids, position_ids, attention_mask, past_key_values, labels, images
+    ):
+        # ====================================================================================================
+        image_tower = self.get_image_tower()
+        video_tower = self.get_video_tower()
+        if (image_tower is None and video_tower is None) or images is None or input_ids.shape[1] == 1:
+            if past_key_values is not None and (image_tower is not None or video_tower is not None) and images is not None and input_ids.shape[1] == 1:
+                target_shape = past_key_values[-1][-1].shape[-2] + 1
+                attention_mask = torch.cat((attention_mask, torch.ones(
+                    (attention_mask.shape[0], target_shape - attention_mask.shape[1]),
+                    dtype=attention_mask.dtype,
+                    device=attention_mask.device
+                )), dim=1)
+                position_ids = torch.sum(attention_mask, dim=1).unsqueeze(-1) - 1
+            return input_ids, position_ids, attention_mask, past_key_values, None, labels
+
+        '''
+            images is a list, if batch_size=6
+            [
+                image(3, 224, 224),      # sample 1
+                image(3, 224, 224),      # sample 2
+                video(t, 3, 224, 224),   # sample 3
+                image(3, 224, 224),      # sample 4
+                image(3, 224, 224),      # sample 4
+                video(t, 3, 224, 224),   # sample 5
+                video(t, 3, 224, 224),   # sample 5
+                video(t, 3, 224, 224),   # sample 6
+                image(3, 224, 224),      # sample 6
+            ]
+            will be converted to image_features, all video_feature will be flatten as image
+            [
+                [n, c],                  # sample 1
+                [n, c),                  # sample 2
+                *(t * [new_n, c]),       # sample 3
+                [n, c],                  # sample 4
+                [n, c],                  # sample 4
+                *(t * [new_n, c]),       # sample 5
+                *(t * [new_n, c]),       # sample 5
+                *(t * [new_n, c]),       # sample 6
+                [n, c],                  # sample 6
+            ]
+        '''
+        image_idx = [idx for idx, img in enumerate(images) if img.ndim == 3]
+        is_all_image = len(image_idx) == len(images)
+        video_idx = [idx for idx, vid in enumerate(images) if vid.ndim == 4]
+        images_minibatch = torch.stack([images[idx] for idx in image_idx]) if len(image_idx) > 0 else []  # mini_b c h w
+        videos_minibatch = torch.stack([images[idx] for idx in video_idx]) if len(video_idx) > 0 else []  # mini_b c t h w
+
+        tmp_image_features = [None] * (len(image_idx) + len(video_idx))
+        if getattr(images_minibatch, 'ndim', 0) == 4:  # batch consists of images, [mini_b, c, h, w]
+            if image_tower is not None:
+                image_features_minibatch = self.encode_images(images_minibatch)  # [mini_b, l, c]
+            else:
+                image_features_minibatch = torch.randn(1).to(self.device)  # dummy feature for video-only training under tuning
+            for i, pos in enumerate(image_idx):
+                tmp_image_features[pos] = image_features_minibatch[i]
+
+        if getattr(videos_minibatch, 'ndim', 0) == 5:  # batch consists of videos, [mini_b, c, t, h, w]
+            video_features_minibatch = self.encode_videos(videos_minibatch)  # fake list [mini_b, t, l, c]
+            for i, pos in enumerate(video_idx):
+                t = video_features_minibatch[i].shape[0]
+                tmp_image_features[pos] = [video_features_minibatch[i][j] for j in range(t)]
+
+        new_tmp = []
+        for image in tmp_image_features:
+            # print(len(new_tmp), len(image))
+            if isinstance(image, list):
+                t = len(image)
+                for i in range(t):
+                    new_tmp.append(image[i])
+                # print('add video')
+            else:
+                new_tmp.append(image)
+        image_features = new_tmp
+        # print(len(image_features), *[i.shape for i in image_features])
+        # print(len(image_features), image_features[0].shape)
+        # ====================================================================================================
+
+        # TODO: image start / end is not implemented here to support pretraining.
+        if getattr(self.config, 'tune_mm_mlp_adapter', False) and getattr(self.config, 'mm_use_im_start_end', False):
+            raise NotImplementedError
+
+        # Let's just add dummy tensors if they do not exist,
+        # it is a headache to deal with None all the time.
+        # But it is not ideal, and if you have a better idea,
+        # please open an issue / submit a PR, thanks.
+        _labels = labels
+        _position_ids = position_ids
+        _attention_mask = attention_mask
+        if attention_mask is None:
+            attention_mask = torch.ones_like(input_ids, dtype=torch.bool)
+        else:
+            attention_mask = attention_mask.bool()
+        if position_ids is None:
+            position_ids = torch.arange(0, input_ids.shape[1], dtype=torch.long, device=input_ids.device)
+        if labels is None:
+            labels = torch.full_like(input_ids, IGNORE_INDEX)
+
+        # remove the padding using attention_mask -- TODO: double check
+        input_ids = [cur_input_ids[cur_attention_mask] for cur_input_ids, cur_attention_mask in zip(input_ids, attention_mask)]
+        labels = [cur_labels[cur_attention_mask] for cur_labels, cur_attention_mask in zip(labels, attention_mask)]
+
+        new_input_embeds = []
+        new_labels = []
+        cur_image_idx = 0
+        for batch_idx, cur_input_ids in enumerate(input_ids):
+            num_images = (cur_input_ids == IMAGE_TOKEN_INDEX).sum()
+            # print(num_images, cur_input_ids)
+            if num_images == 0:
+                cur_image_features = image_features[cur_image_idx]
+                cur_input_embeds_1 = self.get_model().embed_tokens(cur_input_ids)
+                cur_input_embeds = torch.cat([cur_input_embeds_1, cur_image_features[0:0]], dim=0)
+                new_input_embeds.append(cur_input_embeds)
+                new_labels.append(labels[batch_idx])
+                cur_image_idx += 1
+                continue
+
+            image_token_indices = [-1] + torch.where(cur_input_ids == IMAGE_TOKEN_INDEX)[0].tolist() + [cur_input_ids.shape[0]]
+            cur_input_ids_noim = []
+            cur_labels = labels[batch_idx]
+            cur_labels_noim = []
+            for i in range(len(image_token_indices) - 1):
+                cur_input_ids_noim.append(cur_input_ids[image_token_indices[i]+1:image_token_indices[i+1]])
+                cur_labels_noim.append(cur_labels[image_token_indices[i]+1:image_token_indices[i+1]])
+            split_sizes = [x.shape[0] for x in cur_labels_noim]
+            cur_input_embeds = self.get_model().embed_tokens(torch.cat(cur_input_ids_noim))
+            cur_input_embeds_no_im = torch.split(cur_input_embeds, split_sizes, dim=0)
+            cur_new_input_embeds = []
+            cur_new_labels = []
+
+            for i in range(num_images + 1):
+                cur_new_input_embeds.append(cur_input_embeds_no_im[i])
+                cur_new_labels.append(cur_labels_noim[i])
+                if i < num_images:
+                    # print(cur_image_idx)
+                    cur_image_features = image_features[cur_image_idx]
+                    cur_image_idx += 1
+                    cur_new_input_embeds.append(cur_image_features)
+                    cur_new_labels.append(torch.full((cur_image_features.shape[0],), IGNORE_INDEX, device=cur_labels.device, dtype=cur_labels.dtype))
+
+            cur_new_input_embeds = torch.cat(cur_new_input_embeds)
+            cur_new_labels = torch.cat(cur_new_labels)
+
+            new_input_embeds.append(cur_new_input_embeds)
+            new_labels.append(cur_new_labels)
+
+        # Truncate sequences to max length as image embeddings can make the sequence longer
+        tokenizer_model_max_length = getattr(self.config, 'tokenizer_model_max_length', None)
+        if tokenizer_model_max_length is not None:
+            new_input_embeds = [x[:tokenizer_model_max_length] for x in new_input_embeds]
+            new_labels = [x[:tokenizer_model_max_length] for x in new_labels]
+
+        # Combine them
+        max_len = max(x.shape[0] for x in new_input_embeds)
+        batch_size = len(new_input_embeds)
+
+        new_input_embeds_padded = []
+        new_labels_padded = torch.full((batch_size, max_len), IGNORE_INDEX, dtype=new_labels[0].dtype, device=new_labels[0].device)
+        attention_mask = torch.zeros((batch_size, max_len), dtype=attention_mask.dtype, device=attention_mask.device)
+        position_ids = torch.zeros((batch_size, max_len), dtype=position_ids.dtype, device=position_ids.device)
+
+        for i, (cur_new_embed, cur_new_labels) in enumerate(zip(new_input_embeds, new_labels)):
+            cur_len = cur_new_embed.shape[0]
+            if getattr(self.config, 'tokenizer_padding_side', 'right') == "left":
+                new_input_embeds_padded.append(torch.cat((
+                    torch.zeros((max_len - cur_len, cur_new_embed.shape[1]), dtype=cur_new_embed.dtype, device=cur_new_embed.device),
+                    cur_new_embed
+                ), dim=0))
+                if cur_len > 0:
+                    new_labels_padded[i, -cur_len:] = cur_new_labels
+                    attention_mask[i, -cur_len:] = True
+                    position_ids[i, -cur_len:] = torch.arange(0, cur_len, dtype=position_ids.dtype, device=position_ids.device)
+            else:
+                new_input_embeds_padded.append(torch.cat((
+                    cur_new_embed,
+                    torch.zeros((max_len - cur_len, cur_new_embed.shape[1]), dtype=cur_new_embed.dtype, device=cur_new_embed.device)
+                ), dim=0))
+                if cur_len > 0:
+                    new_labels_padded[i, :cur_len] = cur_new_labels
+                    attention_mask[i, :cur_len] = True
+                    position_ids[i, :cur_len] = torch.arange(0, cur_len, dtype=position_ids.dtype, device=position_ids.device)
+
+        new_input_embeds = torch.stack(new_input_embeds_padded, dim=0)
+
+        if _labels is None:
+            new_labels = None
+        else:
+            new_labels = new_labels_padded
+
+        if _attention_mask is None:
+            attention_mask = None
+        else:
+            attention_mask = attention_mask.to(dtype=_attention_mask.dtype)
+
+        if _position_ids is None:
+            position_ids = None
+
+        return None, position_ids, attention_mask, past_key_values, new_input_embeds, new_labels
+
+    def initialize_vision_tokenizer(self, model_args, tokenizer):
+        if model_args.mm_use_im_patch_token:
+            tokenizer.add_tokens([DEFAULT_IMAGE_PATCH_TOKEN], special_tokens=True)
+            self.resize_token_embeddings(len(tokenizer))
+
+        if model_args.mm_use_im_start_end:
+            num_new_tokens = tokenizer.add_tokens([DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN], special_tokens=True)
+            self.resize_token_embeddings(len(tokenizer))
+
+            if num_new_tokens > 0:
+                input_embeddings = self.get_input_embeddings().weight.data
+                output_embeddings = self.get_output_embeddings().weight.data
+
+                input_embeddings_avg = input_embeddings[:-num_new_tokens].mean(
+                    dim=0, keepdim=True)
+                output_embeddings_avg = output_embeddings[:-num_new_tokens].mean(
+                    dim=0, keepdim=True)
+
+                input_embeddings[-num_new_tokens:] = input_embeddings_avg
+                output_embeddings[-num_new_tokens:] = output_embeddings_avg
+
+            if model_args.tune_mm_mlp_adapter:
+                for p in self.get_input_embeddings().parameters():
+                    p.requires_grad = True
+                for p in self.get_output_embeddings().parameters():
+                    p.requires_grad = False
+
+            if model_args.pretrain_mm_mlp_adapter:
+                mm_projector_weights = torch.load(model_args.pretrain_mm_mlp_adapter, map_location='cpu')
+                embed_tokens_weight = mm_projector_weights['model.embed_tokens.weight']
+                assert num_new_tokens == 2
+                if input_embeddings.shape == embed_tokens_weight.shape:
+                    input_embeddings[-num_new_tokens:] = embed_tokens_weight[-num_new_tokens:]
+                elif embed_tokens_weight.shape[0] == num_new_tokens:
+                    input_embeddings[-num_new_tokens:] = embed_tokens_weight
+                else:
+                    raise ValueError(f"Unexpected embed_tokens_weight shape. Pretrained: {embed_tokens_weight.shape}. Current: {input_embeddings.shape}. Numer of new tokens: {num_new_tokens}.")
+        elif model_args.mm_use_im_patch_token:
+            if model_args.tune_mm_mlp_adapter:
+                for p in self.get_input_embeddings().parameters():
+                    p.requires_grad = False
+                for p in self.get_output_embeddings().parameters():
+                    p.requires_grad = False

videollava/model/make_delta.py

@@ -0,0 +1,52 @@
+"""
+Usage:
+python3 -m llava.model.make_delta --base ~/model_weights/llama-7b --target ~/model_weights/llava-7b --delta ~/model_weights/llava-7b-delta --hub-repo-id liuhaotian/llava-7b-delta
+"""
+import argparse
+
+import torch
+from tqdm import tqdm
+from transformers import AutoTokenizer, AutoModelForCausalLM
+from videollava.model.utils import auto_upgrade
+
+
+def make_delta(base_model_path, target_model_path, delta_path, hub_repo_id):
+    print("Loading base model")
+    base = AutoModelForCausalLM.from_pretrained(
+        base_model_path, torch_dtype=torch.float16, low_cpu_mem_usage=True)
+
+    print("Loading target model")
+    auto_upgrade(target_model_path)
+    target = AutoModelForCausalLM.from_pretrained(target_model_path, torch_dtype=torch.float16, low_cpu_mem_usage=True)
+
+    print("Calculating delta")
+    for name, param in tqdm(target.state_dict().items(), desc="Calculating delta"):
+        if name not in base.state_dict():
+            assert name in ['model.mm_projector.weight', 'model.mm_projector.bias'], f'{name} not in base model'
+            continue
+        if param.data.shape == base.state_dict()[name].shape:
+            param.data -= base.state_dict()[name]
+        else:
+            assert name in ['model.embed_tokens.weight', 'lm_head.weight'], f'{name} dimension mismatch: {param.data.shape} vs {base.state_dict()[name].shape}'
+            bparam = base.state_dict()[name]
+            param.data[:bparam.shape[0], :bparam.shape[1]] -= bparam
+
+    print("Saving delta")
+    if hub_repo_id:
+        kwargs = {"push_to_hub": True, "repo_id": hub_repo_id}
+    else:
+        kwargs = {}
+    target.save_pretrained(delta_path, **kwargs)
+    target_tokenizer = AutoTokenizer.from_pretrained(target_model_path)
+    target_tokenizer.save_pretrained(delta_path, **kwargs)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--base-model-path", type=str, required=True)
+    parser.add_argument("--target-model-path", type=str, required=True)
+    parser.add_argument("--delta-path", type=str, required=True)
+    parser.add_argument("--hub-repo-id", type=str, default=None)
+    args = parser.parse_args()
+
+    make_delta(args.base_model_path, args.target_model_path, args.delta_path, args.hub_repo_id)

videollava/model/multimodal_encoder/builder.py

@@ -0,0 +1,24 @@
+import os
+from .clip_encoder import CLIPVisionTower
+from .languagebind import LanguageBindImageTower, LanguageBindVideoTower
+
+# ============================================================================================================
+
+def build_image_tower(image_tower_cfg, **kwargs):
+    image_tower = getattr(image_tower_cfg, 'mm_image_tower', getattr(image_tower_cfg, 'image_tower', None))
+    is_absolute_path_exists = os.path.exists(image_tower)
+    cache_dir = getattr(image_tower_cfg, 'cache_dir', './cache_dir')
+    if is_absolute_path_exists or image_tower.startswith("openai") or image_tower.startswith("laion"):
+        return CLIPVisionTower(image_tower, args=image_tower_cfg, **kwargs)
+    if image_tower.endswith('LanguageBind_Image'):
+        return LanguageBindImageTower(image_tower, args=image_tower_cfg, cache_dir=cache_dir, **kwargs)
+
+    raise ValueError(f'Unknown image tower: {image_tower}')
+
+def build_video_tower(video_tower_cfg, **kwargs):
+    video_tower = getattr(video_tower_cfg, 'mm_video_tower', getattr(video_tower_cfg, 'video_tower', None))
+    cache_dir = getattr(video_tower_cfg, 'cache_dir', './cache_dir')
+    if video_tower.endswith('LanguageBind_Video_merge'):
+        return LanguageBindVideoTower(video_tower, args=video_tower_cfg, cache_dir=cache_dir, **kwargs)
+    raise ValueError(f'Unknown video tower: {video_tower}')
+# ============================================================================================================

videollava/model/multimodal_encoder/clip_encoder.py

@@ -0,0 +1,78 @@
+import torch
+import torch.nn as nn
+
+from transformers import CLIPVisionModel, CLIPImageProcessor, CLIPVisionConfig
+
+
+class CLIPVisionTower(nn.Module):
+    def __init__(self, vision_tower, args, delay_load=False):
+        super().__init__()
+
+        self.is_loaded = False
+
+        self.vision_tower_name = vision_tower
+        self.select_layer = args.mm_vision_select_layer
+        self.select_feature = getattr(args, 'mm_vision_select_feature', 'patch')
+
+        if not delay_load:
+            self.load_model()
+        else:
+            self.cfg_only = CLIPVisionConfig.from_pretrained(self.vision_tower_name)
+
+    def load_model(self):
+        self.image_processor = CLIPImageProcessor.from_pretrained(self.vision_tower_name)
+        self.vision_tower = CLIPVisionModel.from_pretrained(self.vision_tower_name)
+        self.vision_tower.requires_grad_(False)
+
+        self.is_loaded = True
+
+    def feature_select(self, image_forward_outs):
+        image_features = image_forward_outs.hidden_states[self.select_layer]
+        if self.select_feature == 'patch':
+            image_features = image_features[:, 1:]
+        elif self.select_feature == 'cls_patch':
+            image_features = image_features
+        else:
+            raise ValueError(f'Unexpected select feature: {self.select_feature}')
+        return image_features
+
+    @torch.no_grad()
+    def forward(self, images):
+        if type(images) is list:
+            image_features = []
+            for image in images:
+                image_forward_out = self.vision_tower(image.to(device=self.device, dtype=self.dtype).unsqueeze(0), output_hidden_states=True)
+                image_feature = self.feature_select(image_forward_out).to(image.dtype)
+                image_features.append(image_feature)
+        else:
+            image_forward_outs = self.vision_tower(images.to(device=self.device, dtype=self.dtype), output_hidden_states=True)
+            image_features = self.feature_select(image_forward_outs).to(images.dtype)
+
+        return image_features
+
+    @property
+    def dummy_feature(self):
+        return torch.zeros(1, self.hidden_size, device=self.device, dtype=self.dtype)
+
+    @property
+    def dtype(self):
+        return self.vision_tower.dtype
+
+    @property
+    def device(self):
+        return self.vision_tower.device
+
+    @property
+    def config(self):
+        if self.is_loaded:
+            return self.vision_tower.config
+        else:
+            return self.cfg_only
+
+    @property
+    def hidden_size(self):
+        return self.config.hidden_size
+
+    @property
+    def num_patches(self):
+        return (self.config.image_size // self.config.patch_size) ** 2