diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 0000000000000000000000000000000000000000..a6344aac8c09253b3b630fb776ae94478aa0275b
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..43ae0e2a6c6d8fca34872506ca0f2e64194fec7c
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,2 @@
+__pycache__/
+*.py[cod]
diff --git a/README.md b/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..fed786619fb07f1d055addd1eba98fb388a990b0
--- /dev/null
+++ b/README.md
@@ -0,0 +1,14 @@
+---
+title: TEOChat
+emoji: 🏢
+colorFrom: green
+colorTo: pink
+sdk: gradio
+sdk_version: 4.44.1
+app_file: videollava/serve/teochat_demo.py
+pinned: false
+license: apache-2.0
+short_description: A new vision-language assistant for temporal EO data.
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1109d2a73b6a100dff4d7b878abfe9b60e6e430d
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,283 @@
+accelerate==0.26.1
+affine==2.4.0
+aiofiles==23.2.1
+aiohttp==3.8.4
+aiosignal==1.3.1
+altair==5.2.0
+annotated-types==0.7.0
+anyio==3.6.2
+appdirs==1.4.4
+argon2-cffi==21.3.0
+argon2-cffi-bindings==21.2.0
+arrow==1.2.3
+asttokens==2.2.1
+async-timeout==4.0.2
+attrs==23.1.0
+av==11.0.0
+backcall==0.2.0
+beautifulsoup4==4.12.2
+bitsandbytes==0.41.0
+bleach==6.0.0
+blessed==1.20.0
+braceexpand==0.1.7
+Brotli==1.0.9
+certifi==2024.2.2
+cffi==1.15.1
+chardet==4.0.0
+charset-normalizer==3.1.0
+click==8.1.7
+click-plugins==1.1.1
+cligj==0.7.2
+cloudpickle==2.2.1
+cmake==3.26.3
+comm==0.2.1
+contourpy==1.0.7
+coverage==7.4.4
+croniter==1.3.14
+cycler==0.11.0
+cytoolz==0.12.2
+dask==2023.11.0
+dateutils==0.6.12
+debugpy==1.6.7
+decorator==5.1.1
+decord==0.6.0
+deepdiff==6.3.0
+deepspeed==0.14.0
+defusedxml==0.7.1
+distro==1.9.0
+dnspython==2.6.1
+docker-pycreds==0.4.0
+efficientnet-pytorch==0.7.1
+einops==0.6.1
+einops-exts==0.0.4
+email_validator==2.2.0
+exceptiongroup==1.2.0
+executing==1.2.0
+fastapi==0.111.1
+fastapi-cli==0.0.4
+fastjsonschema==2.16.3
+ffmpy==0.3.1
+filelock==3.13.1
+Fiona==1.9.3
+fire==0.5.0
+fonttools==4.39.3
+fqdn==1.5.1
+frozenlist==1.3.3
+fschat==0.2.36
+fsspec==2023.12.2
+ftfy==6.1.3
+fvcore==0.1.5.post20221221
+gdown==5.1.0
+geojson==3.0.1
+gitdb==4.0.10
+GitPython==3.1.31
+gradio==4.39.0
+gradio_client==1.1.1
+h11==0.14.0
+h5py==3.9.0
+hjson==3.1.0
+httpcore==1.0.5
+httptools==0.6.1
+httpx==0.27.0
+huggingface-hub==0.20.3
+idna==3.4
+imagecodecs==2021.8.26
+imageio==2.33.1
+importlib-metadata==7.0.1
+importlib-resources==5.12.0
+iniconfig==2.0.0
+inquirer==3.1.3
+iopath==0.1.10
+ipykernel==6.28.0
+ipython==8.15.0
+ipython-genutils==0.2.0
+ipywidgets==8.1.3
+isoduration==20.11.0
+itsdangerous==2.1.2
+jedi==0.18.2
+Jinja2==3.1.2
+joblib==1.2.0
+jsonpointer==2.3
+jsonschema==4.17.3
+jupyter_client==8.6.0
+jupyter_core==5.5.0
+jupyter-events==0.6.3
+jupyter_server==2.5.0
+jupyter_server_terminals==0.4.4
+jupyterlab-pygments==0.2.2
+jupyterlab_widgets==3.0.11
+kaleido==0.2.1
+kiwisolver==1.4.4
+kornia==0.7.3
+linkify-it-py==2.0.2
+lit==16.0.2
+locket==1.0.0
+markdown-it-py==2.2.0
+markdown2==2.4.12
+MarkupSafe==2.1.2
+matplotlib==3.7.1
+matplotlib-inline==0.1.6
+mdit-py-plugins==0.3.3
+mdurl==0.1.2
+mistune==2.0.5
+mkl-fft==1.3.8
+mkl-random==1.2.4
+mkl-service==2.4.0
+mpmath==1.3.0
+multidict==6.0.4
+munch==2.5.0
+nbclassic==0.5.5
+nbclient==0.7.4
+nbconvert==7.3.1
+nbformat==5.8.0
+nest-asyncio==1.6.0
+networkx==3.1
+nh3==0.2.15
+ninja==1.11.1.1
+nltk==3.8.1
+notebook==6.5.4
+notebook_shim==0.2.3
+numpy==1.26.4
+open_clip_torch==2.26.1
+openai==0.28.1
+opencv-python==4.7.0.72
+ordered-set==4.1.0
+orjson==3.9.12
+packaging==23.1
+pandas==2.0.1
+pandocfilters==1.5.0
+parameterized==0.9.0
+parso==0.8.3
+partd==1.4.1
+pathtools==0.1.2
+peft==0.7.1
+pexpect==4.8.0
+pickleshare==0.7.5
+pillow==10.3.0
+pip==23.0.1
+platformdirs==3.10.0
+plotly==5.22.0
+pluggy==1.4.0
+portalocker==2.8.2
+pprintpp==0.4.0
+pretrainedmodels==0.7.4
+prometheus-client==0.16.0
+prompt-toolkit==3.0.43
+protobuf==4.22.3
+psutil==5.9.5
+ptyprocess==0.7.0
+pure-eval==0.2.2
+py-cpuinfo==9.0.0
+pycountry==23.12.11
+pycountry-convert==0.7.2
+pycparser==2.21
+pydantic==2.8.2
+pydantic_core==2.20.1
+pydub==0.25.1
+Pygments==2.15.1
+PyJWT==2.6.0
+pynvml==11.5.0
+pyparsing==3.0.9
+pyproj==3.5.0
+pyrsistent==0.19.3
+PySocks==1.7.1
+pytest==8.1.1
+pytest-cov==4.1.0
+pytest-mock==3.14.0
+python-dateutil==2.8.2
+python-dotenv==1.0.1
+python-editor==1.0.4
+python-json-logger==2.0.7
+python-multipart==0.0.9
+pytorch-lightning==2.0.2
+pytorchvideo==0.1.3
+pytz==2023.3
+pywavelets==1.5.0
+PyYAML==6.0.1
+pyzmq==25.1.2
+rasterio==1.3.6
+readchar==4.0.5
+regex==2023.12.25
+repoze.lru==0.7
+requests==2.31.0
+reverse-geocoder==1.5.1
+rfc3339-validator==0.1.4
+rfc3986-validator==0.1.1
+rich==13.3.4
+Rtree==1.0.1
+ruff==0.5.2
+safetensors==0.4.2
+scikit-image==0.19.3
+scikit-learn==1.2.2
+scipy==1.11.4
+seaborn==0.13.2
+segmentation-models-pytorch==0.3.2
+semantic-version==2.10.0
+Send2Trash==1.8.0
+sentencepiece==0.1.99
+sentry-sdk==1.21.0
+setproctitle==1.3.2
+setuptools==66.0.0
+shapely==2.0.1
+shellingham==1.5.4
+shortuuid==1.0.11
+six==1.16.0
+smmap==5.0.0
+sniffio==1.3.0
+snuggs==1.4.7
+soupsieve==2.4.1
+stack-data==0.6.2
+starlette==0.37.2
+starsessions==1.3.0
+svgwrite==1.4.3
+sympy==1.11.1
+tabulate==0.9.0
+tenacity==8.5.0
+tensorboardX==2.6.2.2
+termcolor==2.3.0
+terminado==0.17.1
+threadpoolctl==3.1.0
+tifffile==2021.7.2
+tiktoken==0.6.0
+timm==0.6.12
+tinycss2==1.2.1
+tokenizers==0.13.3
+tomli==2.0.1
+tomlkit==0.12.0
+toolz==0.12.1
+torchaudio==2.2.1
+torchgeo==0.6.0
+torchinfo==1.7.2
+torchmetrics==0.11.4
+torchvision==0.17.1
+tornado==6.3.3
+tqdm==4.65.0
+traitlets==5.9.0
+transformers==4.31.0
+Tree==0.2.4
+triton==2.2.0
+typer==0.12.3
+typing_extensions==4.9.0
+tzdata==2023.3
+uc-micro-py==1.0.2
+uri-template==1.2.0
+urllib3==2.1.0
+utm==0.7.0
+uvicorn==0.21.1
+uvloop==0.19.0
+wandb==0.15.0
+watchfiles==0.22.0
+wavedrom==2.0.3.post3
+wcwidth==0.2.13
+webcolors==1.13
+webdataset==0.2.86
+webencodings==0.5.1
+websocket-client==1.5.1
+websockets==11.0.2
+wheel==0.38.4
+widgetsnbextension==4.0.11
+yacs==0.1.8
+yarl==1.9.2
+zipp==3.17.0
+--extra-index-url https://download.pytorch.org/whl/cu113
+torch==2.2.1
\ No newline at end of file
diff --git a/static/logo.png b/static/logo.png
new file mode 100644
index 0000000000000000000000000000000000000000..cc5b3618fc73a86e5a0488088f1fdbf2bc143dd3
Binary files /dev/null and b/static/logo.png differ
diff --git a/static/logo.svg b/static/logo.svg
new file mode 100644
index 0000000000000000000000000000000000000000..0ba343ca3acd294cdc218caddae9bf00e111971f
--- /dev/null
+++ b/static/logo.svg
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/videollava/__init__.py b/videollava/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..4d1f016db1028101d45ba7d68cb3f0bcb558c2bb
--- /dev/null
+++ b/videollava/__init__.py
@@ -0,0 +1 @@
+from .model import LlavaLlamaForCausalLM
diff --git a/videollava/constants.py b/videollava/constants.py
new file mode 100644
index 0000000000000000000000000000000000000000..956177db3f56f14adb5cf53ff2fdd6ccaa07dac6
--- /dev/null
+++ b/videollava/constants.py
@@ -0,0 +1,27 @@
+CONTROLLER_HEART_BEAT_EXPIRATION = 30
+WORKER_HEART_BEAT_INTERVAL = 15
+
+LOGDIR = "."
+
+# Model Constants
+IGNORE_INDEX = -100
+
+IMAGE_TOKEN_INDEX = -200
+DEFAULT_IMAGE_TOKEN = ""
+DEFAULT_IMAGE_PATCH_TOKEN = ""
+DEFAULT_IM_START_TOKEN = ""
+DEFAULT_IM_END_TOKEN = ""
+IMAGE_PLACEHOLDER = ""
+
+# ======================================================================================================
+DEFAULT_VIDEO_TOKEN = ""
+DEFAULT_VIDEO_PATCH_TOKEN = ""
+DEFAULT_VID_START_TOKEN = ""
+DEFAULT_VID_END_TOKEN = ""
+VIDEO_PLACEHOLDER = ""
+# ======================================================================================================
+
+MAX_IMAGE_LENGTH = 16
+MAX_VIDEO_LENGTH = 1 # current video datasets only have 1 video?
+
+PAD_LENGTH = 620
diff --git a/videollava/conversation.py b/videollava/conversation.py
new file mode 100644
index 0000000000000000000000000000000000000000..0025f5b11f9152fff7750f6170717028e8116a52
--- /dev/null
+++ b/videollava/conversation.py
@@ -0,0 +1,381 @@
+import dataclasses
+from enum import auto, Enum
+from typing import List, Tuple
+
+
+class SeparatorStyle(Enum):
+ """Different separator style."""
+ SINGLE = auto()
+ TWO = auto()
+ MPT = auto()
+ PLAIN = auto()
+ LLAMA_2 = auto()
+
+
+@dataclasses.dataclass
+class Conversation:
+ """A class that keeps all conversation history."""
+ system: str
+ roles: List[str]
+ messages: List[List[str]]
+ offset: int
+ sep_style: SeparatorStyle = SeparatorStyle.SINGLE
+ sep: str = "###"
+ sep2: str = None
+ version: str = "Unknown"
+
+ skip_next: bool = False
+
+ def get_prompt(self):
+ messages = self.messages
+ if len(messages) > 0 and type(messages[0][1]) is tuple:
+ messages = self.messages.copy()
+ init_role, init_msg = messages[0].copy()
+ init_msg = init_msg[0].replace("", "").strip()
+ if 'mmtag' in self.version:
+ messages[0] = (init_role, init_msg)
+ messages.insert(0, (self.roles[0], " "))
+ messages.insert(1, (self.roles[1], "Received."))
+ else:
+ messages[0] = (init_role, "\n" + init_msg)
+
+ if self.sep_style == SeparatorStyle.SINGLE:
+ ret = self.system + self.sep
+ for role, message in messages:
+ if message:
+ if type(message) is tuple:
+ message, _, _ = message
+ ret += role + ": " + message + self.sep
+ else:
+ ret += role + ":"
+ elif self.sep_style == SeparatorStyle.TWO:
+ seps = [self.sep, self.sep2]
+ ret = self.system + seps[0]
+ for i, (role, message) in enumerate(messages):
+ if message:
+ if type(message) is tuple:
+ message, _, _ = message
+ ret += role + ": " + message + seps[i % 2]
+ else:
+ ret += role + ":"
+ elif self.sep_style == SeparatorStyle.MPT:
+ ret = self.system + self.sep
+ for role, message in messages:
+ if message:
+ if type(message) is tuple:
+ message, _, _ = message
+ ret += role + message + self.sep
+ else:
+ ret += role
+ elif self.sep_style == SeparatorStyle.LLAMA_2:
+ wrap_sys = lambda msg: f"<>\n{msg}\n< >\n\n"
+ wrap_inst = lambda msg: f"[INST] {msg} [/INST]"
+ ret = ""
+
+ for i, (role, message) in enumerate(messages):
+ if i == 0:
+ assert message, "first message should not be none"
+ assert role == self.roles[0], "first message should come from user"
+ if message:
+ if type(message) is tuple:
+ message, _, _ = message
+ if i == 0: message = wrap_sys(self.system) + message
+ if i % 2 == 0:
+ message = wrap_inst(message)
+ ret += self.sep + message
+ else:
+ ret += " " + message + " " + self.sep2
+ else:
+ ret += ""
+ ret = ret.lstrip(self.sep)
+ elif self.sep_style == SeparatorStyle.PLAIN:
+ seps = [self.sep, self.sep2]
+ ret = self.system
+ for i, (role, message) in enumerate(messages):
+ if message:
+ if type(message) is tuple:
+ message, _, _ = message
+ ret += message + seps[i % 2]
+ else:
+ ret += ""
+ else:
+ raise ValueError(f"Invalid style: {self.sep_style}")
+
+ return ret
+
+ def append_message(self, role, message):
+ self.messages.append([role, message])
+
+ def get_images(self, return_pil=False):
+ images = []
+ for i, (role, msg) in enumerate(self.messages[self.offset:]):
+ if i % 2 == 0:
+ if type(msg) is tuple:
+ import base64
+ from io import BytesIO
+ from PIL import Image
+ msg, image, image_process_mode = msg
+ if image_process_mode == "Pad":
+ def expand2square(pil_img, background_color=(122, 116, 104)):
+ width, height = pil_img.size
+ if width == height:
+ return pil_img
+ elif width > height:
+ result = Image.new(pil_img.mode, (width, width), background_color)
+ result.paste(pil_img, (0, (width - height) // 2))
+ return result
+ else:
+ result = Image.new(pil_img.mode, (height, height), background_color)
+ result.paste(pil_img, ((height - width) // 2, 0))
+ return result
+ image = expand2square(image)
+ elif image_process_mode in ["Default", "Crop"]:
+ pass
+ elif image_process_mode == "Resize":
+ image = image.resize((336, 336))
+ else:
+ raise ValueError(f"Invalid image_process_mode: {image_process_mode}")
+ max_hw, min_hw = max(image.size), min(image.size)
+ aspect_ratio = max_hw / min_hw
+ max_len, min_len = 800, 400
+ shortest_edge = int(min(max_len / aspect_ratio, min_len, min_hw))
+ longest_edge = int(shortest_edge * aspect_ratio)
+ W, H = image.size
+ if longest_edge != max(image.size):
+ if H > W:
+ H, W = longest_edge, shortest_edge
+ else:
+ H, W = shortest_edge, longest_edge
+ image = image.resize((W, H))
+ if return_pil:
+ images.append(image)
+ else:
+ buffered = BytesIO()
+ image.save(buffered, format="PNG")
+ img_b64_str = base64.b64encode(buffered.getvalue()).decode()
+ images.append(img_b64_str)
+ return images
+
+ def to_gradio_chatbot(self):
+ ret = []
+ for i, (role, msg) in enumerate(self.messages[self.offset:]):
+ if i % 2 == 0:
+ if type(msg) is tuple:
+ import base64
+ from io import BytesIO
+ msg, image, image_process_mode = msg
+ max_hw, min_hw = max(image.size), min(image.size)
+ aspect_ratio = max_hw / min_hw
+ max_len, min_len = 800, 400
+ shortest_edge = int(min(max_len / aspect_ratio, min_len, min_hw))
+ longest_edge = int(shortest_edge * aspect_ratio)
+ W, H = image.size
+ if H > W:
+ H, W = longest_edge, shortest_edge
+ else:
+ H, W = shortest_edge, longest_edge
+ image = image.resize((W, H))
+ buffered = BytesIO()
+ image.save(buffered, format="JPEG")
+ img_b64_str = base64.b64encode(buffered.getvalue()).decode()
+ img_str = f' '
+ msg = img_str + msg.replace('', '').strip()
+ ret.append([msg, None])
+ else:
+ ret.append([msg, None])
+ else:
+ ret[-1][-1] = msg
+ return ret
+
+ def copy(self):
+ return Conversation(
+ system=self.system,
+ roles=self.roles,
+ messages=[[x, y] for x, y in self.messages],
+ offset=self.offset,
+ sep_style=self.sep_style,
+ sep=self.sep,
+ sep2=self.sep2,
+ version=self.version)
+
+ def dict(self):
+ if len(self.get_images()) > 0:
+ return {
+ "system": self.system,
+ "roles": self.roles,
+ "messages": [[x, y[0] if type(y) is tuple else y] for x, y in self.messages],
+ "offset": self.offset,
+ "sep": self.sep,
+ "sep2": self.sep2,
+ }
+ return {
+ "system": self.system,
+ "roles": self.roles,
+ "messages": self.messages,
+ "offset": self.offset,
+ "sep": self.sep,
+ "sep2": self.sep2,
+ }
+
+
+conv_vicuna_v0 = Conversation(
+ system="A chat between a curious human and an artificial intelligence assistant. "
+ "The assistant gives helpful, detailed, and polite answers to the human's questions.",
+ roles=("Human", "Assistant"),
+ messages=(
+ ("Human", "What are the key differences between renewable and non-renewable energy sources?"),
+ ("Assistant",
+ "Renewable energy sources are those that can be replenished naturally in a relatively "
+ "short amount of time, such as solar, wind, hydro, geothermal, and biomass. "
+ "Non-renewable energy sources, on the other hand, are finite and will eventually be "
+ "depleted, such as coal, oil, and natural gas. Here are some key differences between "
+ "renewable and non-renewable energy sources:\n"
+ "1. Availability: Renewable energy sources are virtually inexhaustible, while non-renewable "
+ "energy sources are finite and will eventually run out.\n"
+ "2. Environmental impact: Renewable energy sources have a much lower environmental impact "
+ "than non-renewable sources, which can lead to air and water pollution, greenhouse gas emissions, "
+ "and other negative effects.\n"
+ "3. Cost: Renewable energy sources can be more expensive to initially set up, but they typically "
+ "have lower operational costs than non-renewable sources.\n"
+ "4. Reliability: Renewable energy sources are often more reliable and can be used in more remote "
+ "locations than non-renewable sources.\n"
+ "5. Flexibility: Renewable energy sources are often more flexible and can be adapted to different "
+ "situations and needs, while non-renewable sources are more rigid and inflexible.\n"
+ "6. Sustainability: Renewable energy sources are more sustainable over the long term, while "
+ "non-renewable sources are not, and their depletion can lead to economic and social instability.\n")
+ ),
+ offset=2,
+ sep_style=SeparatorStyle.SINGLE,
+ sep="###",
+)
+
+conv_vicuna_v1 = Conversation(
+ system="A chat between a curious user and an artificial intelligence assistant. "
+ "The assistant gives helpful, detailed, and polite answers to the user's questions.",
+ roles=("USER", "ASSISTANT"),
+ version="v1",
+ messages=(),
+ offset=0,
+ sep_style=SeparatorStyle.TWO,
+ sep=" ",
+ sep2="",
+)
+
+conv_llama_2 = Conversation(
+ system="""You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.
+
+If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.""",
+ roles=("USER", "ASSISTANT"),
+ version="llama_v2",
+ messages=(),
+ offset=0,
+ sep_style=SeparatorStyle.LLAMA_2,
+ sep="",
+ sep2=" ",
+)
+
+conv_llava_llama_2 = Conversation(
+ system="You are a helpful language and vision assistant. "
+ "You are able to understand the visual content that the user provides, "
+ "and assist the user with a variety of tasks using natural language.",
+ roles=("USER", "ASSISTANT"),
+ version="llama_v2",
+ messages=(),
+ offset=0,
+ sep_style=SeparatorStyle.LLAMA_2,
+ sep="",
+ sep2=" ",
+)
+
+conv_mpt = Conversation(
+ system="""<|im_start|>system
+A conversation between a user and an LLM-based AI assistant. The assistant gives helpful and honest answers.""",
+ roles=("<|im_start|>user\n", "<|im_start|>assistant\n"),
+ version="mpt",
+ messages=(),
+ offset=0,
+ sep_style=SeparatorStyle.MPT,
+ sep="<|im_end|>",
+)
+
+conv_llava_plain = Conversation(
+ system="",
+ roles=("", ""),
+ messages=(
+ ),
+ offset=0,
+ sep_style=SeparatorStyle.PLAIN,
+ sep="\n",
+)
+
+conv_llava_v0 = Conversation(
+ system="A chat between a curious human and an artificial intelligence assistant. "
+ "The assistant gives helpful, detailed, and polite answers to the human's questions.",
+ roles=("Human", "Assistant"),
+ messages=(
+ ),
+ offset=0,
+ sep_style=SeparatorStyle.SINGLE,
+ sep="###",
+)
+
+conv_llava_v0_mmtag = Conversation(
+ system="A chat between a curious user and an artificial intelligence assistant. "
+ "The assistant is able to understand the visual content that the user provides, and assist the user with a variety of tasks using natural language."
+ "The visual content will be provided with the following format: visual content .",
+ roles=("Human", "Assistant"),
+ messages=(
+ ),
+ offset=0,
+ sep_style=SeparatorStyle.SINGLE,
+ sep="###",
+ version="v0_mmtag",
+)
+
+conv_llava_v1 = Conversation(
+ system="A chat between a curious human and an artificial intelligence assistant. "
+ "The assistant gives helpful, detailed, and polite answers to the human's questions.",
+ roles=("USER", "ASSISTANT"),
+ version="v1",
+ messages=(),
+ offset=0,
+ sep_style=SeparatorStyle.TWO,
+ sep=" ",
+ sep2="",
+)
+
+conv_llava_v1_mmtag = Conversation(
+ system="A chat between a curious user and an artificial intelligence assistant. "
+ "The assistant is able to understand the visual content that the user provides, and assist the user with a variety of tasks using natural language."
+ "The visual content will be provided with the following format: visual content .",
+ roles=("USER", "ASSISTANT"),
+ messages=(),
+ offset=0,
+ sep_style=SeparatorStyle.TWO,
+ sep=" ",
+ sep2="",
+ version="v1_mmtag",
+)
+
+default_conversation = conv_vicuna_v1
+conv_templates = {
+ "default": conv_vicuna_v0,
+ "v0": conv_vicuna_v0,
+ "v1": conv_vicuna_v1,
+ "vicuna_v1": conv_vicuna_v1,
+ "llama_2": conv_llama_2,
+
+ "plain": conv_llava_plain,
+ "v0_plain": conv_llava_plain,
+ "llava_v0": conv_llava_v0,
+ "v0_mmtag": conv_llava_v0_mmtag,
+ "llava_v1": conv_llava_v1,
+ "v1_mmtag": conv_llava_v1_mmtag,
+ "llava_llama_2": conv_llava_llama_2,
+
+ "mpt": conv_mpt,
+}
+
+
+if __name__ == "__main__":
+ print(default_conversation.get_prompt())
diff --git a/videollava/eval/aid_fmow_ucmerced_utils.py b/videollava/eval/aid_fmow_ucmerced_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..50a88453f4c984758da805d90b82926c497c3408
--- /dev/null
+++ b/videollava/eval/aid_fmow_ucmerced_utils.py
@@ -0,0 +1,94 @@
+import json
+import numpy as np
+from tqdm import tqdm
+from pathlib import Path
+
+from infer_utils import run_inference_single
+# For the purposes of an experiment, change the infer_utils to:
+# from infer_utils_mod import run_inference_single
+
+def run_aid_fmow_ucmerced_inference(
+ model,
+ dataset_path,
+ processor,
+ tokenizer,
+ conv_mode,
+ use_video_data=False,
+ open_prompt=None,
+ repeat_frames=None,
+ prompt_strategy="interleave",
+ chronological_prefix=True,
+ data_frac=1,
+ data_size=None,
+ delete_system_prompt=False,
+ last_image=False,
+ print_prompt=False,
+ **kwargs
+ ):
+ for k, v in kwargs.items():
+ print("WARNING: Unused argument:", k, v)
+
+ try:
+ with open(dataset_path) as f:
+ data = json.load(f)
+ except:
+ data = []
+ with open(dataset_path) as f:
+ for line in f:
+ question = json.loads(line)
+ question["id"] = question["question_id"]
+ question["conversations"] = [
+ {"value": "This is a satellite image: " + question["text"]},
+ {"value": question["ground_truth"]}
+ ]
+ question["video"] = [question["image"]]
+ data.append(question)
+
+ if data_size is not None:
+ data_size = min(data_size, len(data))
+ idx = np.random.choice(len(data), data_size, replace=False)
+ data = [data[i] for i in idx]
+ elif data_frac < 1:
+ idx = np.random.choice(len(data), int(len(data) * data_frac), replace=False)
+ data = [data[i] for i in idx]
+
+ vision_key = "video" if "video" in data[0] else "image"
+
+ answers = {}
+ for question in tqdm(data):
+ question_id = question["id"]
+ inp = question["conversations"][0]['value']
+ if open_prompt == "open":
+ # Use an open framing for the question
+ inp = inp.split("Which")[0] + "Which class does this image belong to?"
+ elif open_prompt == "multi-open":
+ inp = inp.split("Which")[0] + "What classes does this image belong to?"
+ answer_str = question["conversations"][1]['value']
+ if 'metadata' not in question:
+ question['metadata'] = None
+ metadata = question['metadata']
+ image_paths = question[vision_key]
+
+ outputs = run_inference_single(
+ model=model,
+ processor=processor,
+ tokenizer=tokenizer,
+ conv_mode=conv_mode,
+ inp=inp,
+ image_paths=image_paths,
+ metadata=metadata,
+ use_video_data=use_video_data,
+ repeat_frames=repeat_frames,
+ prompt_strategy=prompt_strategy,
+ chronological_prefix=chronological_prefix,
+ delete_system_prompt=delete_system_prompt,
+ last_image=last_image,
+ print_prompt=print_prompt
+ )
+
+ answers[question_id] = {
+ "predicted": outputs,
+ "ground_truth": answer_str
+ }
+
+ return answers
diff --git a/videollava/eval/ben_utils.py b/videollava/eval/ben_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..f7e36f9033f2cb9c189b0c9faa778d4f1c2f2b71
--- /dev/null
+++ b/videollava/eval/ben_utils.py
@@ -0,0 +1,114 @@
+import json
+import numpy as np
+from tqdm import tqdm
+from pathlib import Path
+
+from videollava.constants import DEFAULT_IMAGE_TOKEN
+
+from infer_utils import run_inference_single
+
+
+def run_ben_inference(
+ model,
+ dataset_path,
+ processor,
+ tokenizer,
+ conv_mode,
+ use_video_data=False,
+ open_prompt=None,
+ repeat_frames=None,
+ prompt_strategy="interleave",
+ chronological_prefix=True,
+ data_frac=1,
+ data_size=None,
+ delete_system_prompt=False,
+ last_image=False,
+ start_ind=None,
+ end_ind=None,
+ print_prompt=False,
+ **kwargs
+ ):
+ for k, v in kwargs.items():
+ print("WARNING: Unused argument:", k, v)
+
+ dataset_path = Path(dataset_path)
+ data_dir = dataset_path.parent
+ questions_path = data_dir / dataset_path.name.replace(".json", "_questions.json")
+ answers_path = data_dir / dataset_path.name.replace(".json", "_answers.json")
+ images_path = data_dir / dataset_path.name.replace(".json", "_images.json")
+
+ with open(questions_path) as json_data:
+ questionsJSON = json.load(json_data)
+
+ with open(answers_path) as json_data:
+ answersJSON = json.load(json_data)
+
+ with open(images_path) as json_data:
+ imagesJSON = json.load(json_data)
+
+ if data_size is not None:
+ data_size = min(data_size, len(questionsJSON))
+ idx = np.random.choice(len(questionsJSON), data_size, replace=False)
+ imagesJSON = [imagesJSON[i] for i in idx]
+ elif data_frac < 1:
+ idx = np.random.choice(len(questionsJSON), int(len(questionsJSON) * data_frac), replace=False)
+ imagesJSON = [imagesJSON[i] for i in idx]
+
+ if 'LRBEN' in str(dataset_path):
+ image_folder = 'Images_LR'
+ else:
+ image_folder = 'Data'
+
+ # Get the image IDs of test images
+ images_ids = [img['id'] for img in imagesJSON['images'] if img['active']]
+
+ if start_ind is not None and end_ind is not None:
+ print("Subsetting data from index", start_ind, "to", end_ind)
+ images_ids = images_ids[start_ind:end_ind]
+ elif start_ind is not None:
+ print("Subsetting data from index", start_ind, "to end")
+ images_ids = images_ids[start_ind:]
+ elif end_ind is not None:
+ print("Subsetting data from start to index", end_ind)
+ images_ids = images_ids[:end_ind]
+
+ # Store all predicted answers
+ answers = {}
+ # Read image corresponding to each ID and get its associated question and answer
+ for id in tqdm(images_ids):
+
+ image_paths = [str(data_dir / image_folder / (str(id)+'.tif'))]
+
+ for questionid in imagesJSON['images'][id]['questions_ids']:
+ question = questionsJSON['questions'][questionid]
+ if not question['active']:
+ continue
+ inp = question["question"] + " Answer with one word or number."
+ inp = DEFAULT_IMAGE_TOKEN + '\n' + inp
+ type_str = question["type"]
+ answer_str = answersJSON['answers'][question["answers_ids"][0]]['answer']
+
+ outputs = run_inference_single(
+ model=model,
+ processor=processor,
+ tokenizer=tokenizer,
+ conv_mode=conv_mode,
+ inp=inp,
+ image_paths=image_paths,
+ metadata=None,
+ use_video_data=use_video_data,
+ repeat_frames=repeat_frames,
+ prompt_strategy=prompt_strategy,
+ chronological_prefix=chronological_prefix,
+ delete_system_prompt=delete_system_prompt,
+ last_image=last_image,
+ print_prompt=print_prompt
+ )
+
+ answers[f"{id}_{questionid}"] = {
+ "predicted": outputs,
+ "ground_truth": answer_str,
+ "task": type_str
+ }
+
+ return answers
diff --git a/videollava/eval/cdvqa_utils.py b/videollava/eval/cdvqa_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..6739733605138c114c632977b263ab35264e3f90
--- /dev/null
+++ b/videollava/eval/cdvqa_utils.py
@@ -0,0 +1,127 @@
+import json
+import numpy as np
+from tqdm import tqdm
+from pathlib import Path
+
+from videollava.constants import DEFAULT_VIDEO_TOKEN
+
+from infer_utils import run_inference_single
+
+
+def run_cdvqa_inference(
+ model,
+ dataset_path,
+ processor,
+ tokenizer,
+ conv_mode,
+ use_video_data=False,
+ open_prompt=None,
+ repeat_frames=None,
+ prompt_strategy="interleave",
+ chronological_prefix=True,
+ data_frac=1,
+ data_size=None,
+ delete_system_prompt=False,
+ last_image=False,
+ start_ind=None,
+ end_ind=None,
+ print_prompt=False,
+ **kwargs
+ ):
+ for k, v in kwargs.items():
+ print("WARNING: Unused argument:", k, v)
+
+ dataset_path = Path(dataset_path)
+ data_dir = dataset_path.parent
+ questions_path = data_dir / dataset_path.name.replace(".json", "_questions.json")
+ answers_path = data_dir / dataset_path.name.replace(".json", "_answers.json")
+ images_path = data_dir / dataset_path.name.replace(".json", "_images.json")
+
+ with open(questions_path) as json_data:
+ questionsJSON = json.load(json_data)
+
+ with open(answers_path) as json_data:
+ answersJSON = json.load(json_data)
+
+ with open(images_path) as json_data:
+ imagesJSON = json.load(json_data)
+
+ if data_size is not None:
+ data_size = min(data_size, len(questionsJSON))
+ idx = np.random.choice(len(questionsJSON), data_size, replace=False)
+ imagesJSON = [imagesJSON[i] for i in idx]
+ elif data_frac < 1:
+ idx = np.random.choice(len(questionsJSON), int(len(questionsJSON) * data_frac), replace=False)
+ imagesJSON = [imagesJSON[i] for i in idx]
+
+ # Get the image IDs of test images
+ images_ids = [img['id'] for img in imagesJSON['images'] if img['active']]
+
+ if start_ind is not None and end_ind is not None:
+ print("Subsetting data from index", start_ind, "to", end_ind)
+ images_ids = images_ids[start_ind:end_ind]
+ elif start_ind is not None:
+ print("Subsetting data from index", start_ind, "to end")
+ images_ids = images_ids[start_ind:]
+ elif end_ind is not None:
+ print("Subsetting data from start to index", end_ind)
+ images_ids = images_ids[:end_ind]
+
+ # Store all predicted answers
+ answers = {}
+ # Read image corresponding to each ID and get its associated question and answer
+ for id in tqdm(images_ids):
+ file_name = imagesJSON['images'][id]['file_name']
+
+ image_paths = [
+ str(data_dir / "second_dataset" / "im1" / file_name),
+ str(data_dir / "second_dataset" / "im2" / file_name),
+ ]
+
+ for questionid in imagesJSON['images'][id]['questions_ids']:
+ question = questionsJSON['questions'][questionid]
+ if not question['active']:
+ continue
+ inp = "This is a pair of satellite images capturing the same location at different times: "
+ inp = inp + DEFAULT_VIDEO_TOKEN + '\n'
+ inp = inp + question["question"]
+ type_str = question["type"]
+ answer_str = answersJSON['answers'][question["answers_ids"][0]]['answer']
+
+ if type_str in ["change_or_not", "increase_or_not", "decrease_or_not"]:
+ inp = inp + " Answer with yes or no."
+
+ elif type_str == "change_ratio":
+ inp = inp + " Choose from one of the following options: 0, 0_to_10, 10_to_20, 20_to_30, 30_to_40, 40_to_50, 50_to_60, 60_to_70, 70_to_80, 80_to_90, 90_to_100."
+
+ elif type_str == "change_ratio_types":
+ inp = inp + " Choose from one of the following options: 0, 0_to_10, 10_to_20, 20_to_30, 30_to_40, 40_to_50, 50_to_60, 60_to_70."
+
+ else: # smallest_change, largest_change, change_to_what
+ inp = inp + " Choose from one of the following options: buildings, low_vegetation, nonvegetated ground surface, playgrounds, trees, water."
+ answer_str = answer_str.replace("NVG_surface", "nonvegetated ground surface")
+
+ outputs = run_inference_single(
+ model=model,
+ processor=processor,
+ tokenizer=tokenizer,
+ conv_mode=conv_mode,
+ inp=inp,
+ image_paths=image_paths,
+ metadata=None,
+ use_video_data=use_video_data,
+ repeat_frames=repeat_frames,
+ prompt_strategy=prompt_strategy,
+ chronological_prefix=chronological_prefix,
+ delete_system_prompt=delete_system_prompt,
+ last_image=last_image,
+ print_prompt=print_prompt
+ )
+
+ answers[f"{id}_{questionid}"] = {
+ "predicted": outputs,
+ "ground_truth": answer_str,
+ "task": type_str
+ }
+
+ return answers
diff --git a/videollava/eval/classification_segmentation.py b/videollava/eval/classification_segmentation.py
new file mode 100644
index 0000000000000000000000000000000000000000..1145bc4bd755fa2dfb137a7b7848206c2e82cd5a
--- /dev/null
+++ b/videollava/eval/classification_segmentation.py
@@ -0,0 +1,151 @@
+import json
+import numpy as np
+from infer_utils import create_mask
+from shapely.wkt import loads
+from collections import defaultdict
+from tqdm import tqdm
+
+def clean_string(s):
+ return s.replace(' ', '-').replace('.', '').lower()
+
+def get_class_dict(dataset):
+ if dataset == "qfabric":
+ class_dict = {
+ "temporal_region_based_question_answering: What is the development status in this region [bbox] in image N?":
+ {
+ "prior-construction": 1,
+ "greenland ": 2,
+ "land-cleared": 3,
+ "excavation": 4,
+ "materials-dumped": 5,
+ "construction-started": 6,
+ "construction-midway": 7,
+ "construction-done": 8,
+ "operational": 9
+ },
+ "region_based_question_answering: Identify the type of urban development that has occurred in this area [bbox].":
+ {
+ "residential": 10,
+ "commercial": 11,
+ "industrial": 12,
+ "road": 13,
+ "demolition": 14,
+ "mega-projects": 15
+ }
+ }
+ elif dataset == "xbd":
+ class_dict = {
+ "classification: Classify the level of damage experienced by the building at location [bbox] in the second image. Choose from: No damage, Minor Damage, Major Damage, Destroyed.":
+ {
+ "no-damage": 1,
+ "minor-damage": 2,
+ "major-damage": 3,
+ "destroyed": 4,
+ }
+ }
+ else:
+ raise ValueError(f"Dataset {dataset} should not be evaluated on segmentation classification.")
+ return class_dict
+
+
+
+def classification_segmentation(answer_path, dataset, per_class_f1=False, height=256, width=256):
+ """
+ Given the path to the answer file, this function creates segmentation masks on the original polygon for the predicted and ground truth classes.
+ Returns the class-weighted per-pixel F1 between predicted and ground-truth masks.
+ """
+ with open(answer_path) as f:
+ results = json.load(f)
+
+ classes = get_class_dict(dataset)
+ class_stats = defaultdict(lambda: {'tp': 0, 'fp': 0, 'fn': 0, 'count': 0})
+
+ for result in tqdm(results.values()):
+ if result['task'] not in classes:
+ continue
+ class_dict = classes[result['task']]
+ predicted_class = clean_string(result['predicted'])
+ try:
+ ground_truth_class = clean_string(result["ground_truth"])
+ except:
+ ground_truth_class = clean_string(result["original_answer"])
+ original_polygon = loads(result['original_input_polygon'])
+
+ pred_msk = np.zeros((height, width), dtype='uint8')
+ gt_msk = np.zeros((height, width), dtype='uint8')
+ _msk = create_mask(original_polygon, im_size=(height, width))
+
+ if predicted_class not in class_dict or ground_truth_class not in class_dict:
+ continue
+
+ pred_label = class_dict[predicted_class]
+ gt_label = class_dict[ground_truth_class]
+ pred_msk[_msk > 0] = pred_label
+ gt_msk[_msk > 0] = gt_label
+
+ for label in class_dict.values():
+ pred_mask = (pred_msk == label)
+ gt_mask = (gt_msk == label)
+ tp = np.sum(pred_mask & gt_mask)
+ fp = np.sum(pred_mask & ~gt_mask)
+ fn = np.sum(~pred_mask & gt_mask)
+
+ class_stats[label]['tp'] += tp
+ class_stats[label]['fp'] += fp
+ class_stats[label]['fn'] += fn
+ class_stats[label]['count'] += np.sum(gt_mask)
+
+
+ scores_dict = {}
+
+ for task, class_info in classes.items():
+ print(f"Task: {task}")
+ class_f1_scores = {}
+ weighted_f1_score = 0
+ total_weight = 0
+
+ tp = 0
+ fp = 0
+ fn = 0
+ for class_name, class_label in class_info.items():
+ stats = class_stats[class_label]
+ total_samples = sum(stats['count'] for label, stats in class_stats.items() if label in class_info.values())
+
+ if stats['tp'] + stats['fp'] == 0 or stats['tp'] + stats['fn'] == 0:
+ f1 = 0.0
+ else:
+ precision = stats['tp'] / (stats['tp'] + stats['fp'])
+ recall = stats['tp'] / (stats['tp'] + stats['fn'])
+ if precision + recall == 0:
+ f1 = 0.0
+ else:
+ f1 = 2 * (precision * recall) / (precision + recall)
+ class_f1_scores[class_name] = f1
+
+ if stats['count'] > 0:
+ prevalence_inv = total_samples / stats['count']
+ weighted_f1_score += f1 * prevalence_inv
+ total_weight += prevalence_inv
+
+ tp += stats['tp']
+ fp += stats['fp']
+ fn += stats['fn']
+
+ if tp + fp == 0 or tp + fn == 0:
+ micro_f1 = 0.0
+ else:
+ micro_f1 = tp / (tp + 0.5 * (fp + fn))
+
+ if total_weight > 0:
+ weighted_f1_score /= total_weight
+ else:
+ weighted_f1_score = 0.0
+
+ scores_dict[task] = (class_f1_scores, weighted_f1_score)
+ print(f"Per-class F1 scores: {class_f1_scores}")
+ if dataset == 'qfabric':
+ print(f"Micro average F1 score: ", micro_f1)
+ else:
+ print(f"Weighted average F1 score: {weighted_f1_score}")
+
+ return scores_dict
\ No newline at end of file
diff --git a/videollava/eval/datasets_into_geochat_format.py b/videollava/eval/datasets_into_geochat_format.py
new file mode 100644
index 0000000000000000000000000000000000000000..e1c1786b824415b25fe20b5a3478d1d45c4e8935
--- /dev/null
+++ b/videollava/eval/datasets_into_geochat_format.py
@@ -0,0 +1,293 @@
+import pandas as pd
+import re
+import json
+
+def qfabric_semiconverted_to_geochat_dataset_format(json_file):
+ with open(json_file) as f:
+ data = json.load(f)
+ for conversation_group in data:
+ for item in conversation_group["conversations"]:
+ # Remove satellite specifications
+ item["value"] = re.sub(r"This is a satellite image :", "", item["value"])
+ item["value"] = re.sub(r"This is a satellite image:", "", item["value"])
+ item["value"] = re.sub(r"This is a high resolution, optical satellite image .*?:\s*", "", item["value"])
+ item["value"] = re.sub(r"This is a high resolution, optical satellite image.*?:\s*", "", item["value"])
+ item["value"] = re.sub(r"This is a satellite image from .*?:\s*", "", item["value"])
+ item["value"] = re.sub(r"This is a satellite image from.*?:\s*", "", item["value"])
+ # Remove strings around that are redundant
+ item["value"] = re.sub(r'What is |this area {<', lambda x: '[identify]' if 'What is [identify]' in x.group() else '{<', item["value"])
+ # Switch out for
+ item["value"] = re.sub(r'', '', item["value"])
+ # Get rid of "this region" immediately before the bounding box
+ item["value"] = re.sub(r'this region {<', '{<', item["value"])
+ # Check for the presence of '' and modify the string accordingly
+ if '[identify]' in item["value"]:
+ # Find the position of '' and the position of the first occurrence of '>}' after ''
+ identify_index = item["value"].find('[identify]')
+ identify_word_index = item["value"].find('Identify ', identify_index + 8)
+ # if identify_word_index != -1:
+ # item["value"] = item["value"][:identify_word_index] + item["value"][identify_word_index + 8:]
+ closing_brace_index = item["value"].find('>}', identify_index)
+ return data
+
+def fmow_to_geochat_dataset_format(json_file):
+ with open(json_file) as f:
+ data = json.load(f)
+ for i, entry in enumerate(data):
+ video_count = len(entry.get("video", []))
+ if video_count > 1:
+ original_videos = entry["video"]
+ for idx in range(video_count):
+ new_entry = entry.copy()
+ new_entry['video'] = [original_videos[idx]]
+ new_entry['image'] = original_videos[idx]
+ new_entry['linked_id'] = entry['id']
+ new_entry['img_idx_from_video_lst_id'] = idx
+ data.append(new_entry)
+ else:
+ new_entry = entry.copy()
+ new_entry['image'] = original_videos[0]
+ for conversation_group in data:
+ for item in conversation_group["conversations"]:
+ item["value"] = re.sub(r"This is a satellite image :", "", item["value"])
+ item["value"] = re.sub(r"This is a satellite image:", "", item["value"])
+ item["value"] = re.sub(r"This is a high resolution, optical satellite image .*?:\s*", "", item["value"])
+ item["value"] = re.sub(r"This is a high resolution, optical satellite image.*?:\s*", "", item["value"])
+ item["value"] = re.sub(r"This is a satellite image from .*?:\s*", "", item["value"])
+ item["value"] = re.sub(r"This is a satellite image from.*?:\s*", "", item["value"])
+ item["value"] = re.sub(r"This is a sequence of low-resolution, optical satellite images capturing the same location at different times: ", "", item["value"])
+ item["value"] = re.sub(r"This is a sequence of high-resolution, optical satellite images capturing the same location at different times:", "", item["value"])
+ item["value"] = re.sub(r"This is a sequence of satellite images capturing the same location at different times:", "", item["value"])
+ item["value"] = re.sub(r"This is a sequence of satellite images from .*? the same location at different times:", "", item["value"])
+ item["value"] = re.sub(r'This is a high resolution,? optical satellite image .*:\s*\n', '\n', item["value"])
+ item["value"] = re.sub(r'^This is a high[- ]resolution,? .*?image:\s*\n', '\n', item["value"], flags=re.IGNORECASE | re.DOTALL)
+
+ # Switch out for
+ item["value"] = re.sub(r'', '', item["value"])
+ # Get rid of "this region" immediately before the bounding box
+ item["value"] = re.sub(r'this region {<', '{<', item["value"])
+ # Which class
+ item["value"] = re.sub(r'Which of the following classes does this sequence of images belong to', 'Which of the following classes does this image belong to', item["value"])
+ # Please answer using one of the following classes:
+ item["value"] = re.sub(r'Please answer using only one of the following classes:', 'Please use one of the following classes:', item["value"])
+ # Check for the presence of '' and modify the string accordingly
+ if '[identify]' in item["value"]:
+ # Find the position of '' and the position of the first occurrence of '>}' after ''
+ identify_index = item["value"].find('[identify]')
+ for i, entry in enumerate(data):
+ video_count = len(entry.get("video", []))
+ if video_count > 1:
+ data.pop(i)
+ return data
+
+def xbd_to_geochat_dataset_format(json_file):
+
+ with open(json_file) as f:
+ data = json.load(f)
+
+ new_data = []
+ for i, entry in enumerate(data):
+ if entry["task"].startswith("localization"):
+ new_entry=entry.copy()
+ new_entry['image'] = entry['video'][0]
+ new_data.append(new_entry)
+ if entry["task"].startswith("classification"):
+ new_entry=entry.copy()
+ new_entry['image'] = entry['video'][1]
+ new_data.append(new_entry)
+ # Auxiliary tasks all look at the second image
+ else:
+ new_entry=entry.copy()
+ new_entry['image'] = entry['video'][1]
+ new_data.append(new_entry)
+
+ for conversation_group in new_data:
+ localization=False
+ classification=False
+ # Add a [refer] token to localization tasks
+ if conversation_group["task"].startswith("localization") or "identify" in conversation_group["task"].lower():
+ localization=True
+ # Add a [identify] token to classification tasks
+ if conversation_group["task"].startswith("classification"):
+ classification=True
+
+ for item in conversation_group["conversations"]:
+ item["value"] = re.sub(r"This is a satellite image :", "", item["value"])
+ item["value"] = re.sub(r"This is a satellite image:", "", item["value"])
+ item["value"] = re.sub(r"This is a high resolution, optical satellite image .*?:\s*", "", item["value"])
+ item["value"] = re.sub(r"This is a high resolution, optical satellite image.*?:\s*", "", item["value"])
+ item["value"] = re.sub(r"This is a satellite image from .*?:\s*", "", item["value"])
+ item["value"] = re.sub(r"These are two satellite images from .*? capturing the same location at different times: ", "", item["value"])
+ item["value"] = re.sub(r"These are two low-resolution, optical satellite images capturing the same location at different times:", "", item["value"])
+ item["value"] = re.sub(r"These are two high-resolution, optical satellite images capturing the same location at different times:", "", item["value"])
+ item["value"] = re.sub(r"These are two high-resolution, optical satellite images from .*? capturing the same location at different times:", "", item["value"])
+ item["value"] = re.sub(r"These are two satellite images capturing the same location at different times:", "", item["value"])
+ item["value"] = re.sub(r"These are two satellite images from .*? capturing the same location at different times:", "", item["value"])
+ item["value"] = re.sub(r'These are two high-resolution,? optical satellite images .*:\s*\n', '\n', item["value"])
+ item["value"] = re.sub(r'These are two high resolution,? optical satellite images .*:\s*\n', '\n', item["value"])
+ item["value"] = re.sub(r'^This is a high[- ]resolution,? .*?image:\s*\n', '\n', item["value"], flags=re.IGNORECASE | re.DOTALL)
+
+ # Switch out for
+ if classification:
+ item["value"] = re.sub(r' \n', ' \n [identify] ', item["value"])
+ item["value"] = re.sub(r' in the second image.', '.', item["value"])
+ elif localization:
+ item["value"] = re.sub(r' \n', ' \n [refer] ', item["value"])
+ item["value"] = re.sub(r'Image 1', 'the image', item["value"])
+ else:
+ item["value"] = re.sub(r' \n', ' \n ', item["value"])
+
+ # Replace temporal/multi-image wording for auxiliary tasks
+ replacements = {
+ 'Are there any buildings in the first image which have been damaged in the second image? Answer with one word.': 'Are there any damaged buildings in the image? Answer with one word.',
+ 'Have any buildings in the first image been damaged in the second image? Answer with one word.': 'Have any buildings been damaged in the area? Answer with one word.',
+ 'What disaster has occurred between the first and second image?': 'What disaster has occurred here?',
+ 'Identify the buildings in the first image which were severely damaged or destroyed in the second image. Include a bounding box of the form [x_min, y_min, x_max, y_max] for each identified building in your response. If there are no such buildings, do not output a bounding box.': 'Identify the severely damaged or destroyed buildings in the image. Include a bounding box of the form [x_min, y_min, x_max, y_max] for each identified building in your response. If there are no such buildings, do not output a bounding box.'
+ }
+ for old, new in replacements.items():
+ item['value'] = re.sub(re.escape(old), new, item['value'])
+
+
+ # Get rid of "this region" immediately before the bounding box
+ item["value"] = re.sub(r'this region {<', '{<', item["value"])
+ # Which class
+ item["value"] = re.sub(r'Which of the following classes does this sequence of images belong to', 'Which of the following classes does this image belong to', item["value"])
+ # Please answer using one of the following classes:
+ item["value"] = re.sub(r'Please answer using only one of the following classes:', 'Please use one of the following classes:', item["value"])
+ # Replace bounding box format [79, 27, 85, 81] with {<79><27><85><81>|<0>}
+ item["value"] = re.sub(r'\[(\d+), (\d+), (\d+), (\d+)\]', r'{<\1><\2><\3><\4>|<0>}', item["value"])
+ # Replace bounding box format [x_min, y_min, x_max, y_max] with {|<0>}
+ item["value"] = re.sub(r'\[(x_min), (y_min), (x_max), (y_max)\]', r'{<\1><\2><\3><\4>|<0>}', item["value"])
+ return new_data
+
+def s2looking_to_geochat_dataset_format(json_file):
+ with open(json_file) as f:
+ data = json.load(f)
+
+ question = "\n [refer] Identify all buildings in the image."
+
+ new_dataset = []
+ for elem in data:
+ for i in range(2):
+ new_item = {}
+ new_item['id'] = elem['id'] + '_' + str(i)
+ new_item['metadata'] = elem['metadata'][i]
+ new_item['original_input_polygon'] = elem['original_input_polygon']
+ new_item['task'] = elem['task']
+ new_item['image'] = elem['video'][i]
+ new_item['geovlm_id'] = i
+ new_item['original_conversation'] = elem['conversations']
+ new_item['conversations'] = [
+ {
+ "from": "human",
+ "value": question
+ },
+ {
+ "from": "gpt",
+ "value": ""
+ }
+ ]
+ new_dataset.append(new_item)
+
+ data = new_dataset
+
+ for conversation_group in data:
+ for item in conversation_group["conversations"]:
+ # Check if the sentence starts with "This is" or "These are" and contains ""
+ if (item["value"].startswith("This is") or item["value"].startswith("These are")) and "" in item["value"]:
+ colon_index = item["value"].find(":")
+ if colon_index != -1 and item["value"][colon_index+1:].strip().startswith(""):
+ item["value"] = item["value"][colon_index+1:].strip()
+ item["value"] = re.sub(r"This is a sequence of high-resolution, optical satellite images from Maxar's GeoEye-1, QuickBird-2, WorldView-2, or WorldView-3 capturing the same location at different times:", "", item["value"])
+ item["value"] = re.sub(r"This is a sequence of low-resolution, optical satellite images from Sentinel-2 capturing the same location at different times:", "", item["value"])
+ item["value"] = re.sub(r"This is a satellite image :", "", item["value"])
+ item["value"] = re.sub(r"This is a satellite image:", "", item["value"])
+ item["value"] = re.sub(r"This is a high resolution, optical satellite image .*?:\s*", "", item["value"])
+ item["value"] = re.sub(r"This is a high resolution, optical satellite image.*?:\s*", "", item["value"])
+ item["value"] = re.sub(r"This is a satellite image from .*?:\s*", "", item["value"])
+ item["value"] = re.sub(r"This is a satellite image from.*?:\s*", "", item["value"])
+ # This one is the one I'm referring to:
+ item["value"] = re.sub(r'^This is a sequence of.*times:$', '', item["value"])
+ item["value"] = re.sub(r"This is a sequence of high-resolution, optical satellite images from .*? capturing the same location at different times:", "", item["value"])
+ item["value"] = re.sub(r"This is a sequence of low-resolution, optical satellite images capturing the same location at different times: ", "", item["value"])
+ item["value"] = re.sub(r"This is a sequence of high-resolution, optical satellite images capturing the same location at different times:", "", item["value"])
+ item["value"] = re.sub(r"This is a sequence of satellite images capturing the same location at different times:", "", item["value"])
+ item["value"] = re.sub(r"This is a sequence of satellite images from .*? the same location at different times:", "", item["value"])
+ item["value"] = re.sub(r"These are two high-resolution, optical satellite images capturing the same location at different times:", "", item["value"])
+ item["value"] = re.sub(r"This is a sequence of images from the satellites GaoFen, SuperView and BeiJing-2, capturing the same location at different times:", "", item["value"])
+
+ # Switch out for
+ item["value"] = re.sub(r'', '', item["value"])
+ # Get rid of "this region" immediately before the bounding box
+ item["value"] = re.sub(r'this region {<', '{<', item["value"])
+ # Which class
+ item["value"] = re.sub(r'Which of the following classes does this sequence of images belong to', 'Which of the following classes does this image belong to', item["value"])
+ # Please answer using one of the following classes:
+ item["value"] = re.sub(r'Please answer using only one of the following classes:', 'Please use one of the following classes:', item["value"])
+ # Check for the presence of '' and modify the string accordingly
+ if '[identify]' in item["value"]:
+ # Find the position of '' and the position of the first occurrence of '>}' after ''
+ identify_index = item["value"].find('[identify]')
+ closing_brace_index = item["value"].find('>}', identify_index)
+
+ # Fix the bounding box format:
+ for conversation_group in data:
+ for item in conversation_group["conversations"]:
+ # Replace bounding box format [79, 27, 85, 81] with {<79><27><85><81>|<0>}
+ item["value"] = re.sub(r'\[(\d+), (\d+), (\d+), (\d+)\]', r'{<\1><\2><\3><\4>|<0>}', item["value"])
+ # Replace bounding box format [x_min, y_min, x_max, y_max] with {|<0>}
+ item["value"] = re.sub(r'\[(x_min), (y_min), (x_max), (y_max)\]', r'{<\1><\2><\3><\4>|<0>}', item["value"])
+ for i, entry in enumerate(data):
+ video_count = len(entry.get("video", []))
+ if video_count > 1:
+ data.pop(i)
+ return data
+
+def check_file(file_path):
+ with open(file_path, 'r') as file:
+ data = json.load(file)
+ for conversation_group in data:
+ for item in conversation_group["conversations"]:
+ if '' not in item["value"]:
+ if item["from"] != 'gpt':
+ print(f"Missing in: {item}")
+ if any(sentence.strip().startswith(('This is', 'These are')) for sentence in item["value"].split('.')):
+ print(f"Starts with 'This is' or 'These are' in: {item}")
+if __name__ == "__main__":
+
+ # Paths to datasets
+ fmow_0 = "/scr/geovlm/fmow_low_res_val.json"
+ fmow_1 = "/scr/geovlm/fmow_high_res_val.json"
+
+ qfabric_0 = '/scr/geovlm/QFabric/test_geochat_seqlen_5_256.json'
+ qfabric_1 = '/scr/geovlm/QFabric/test_geochat_seqlen_2_256.json'
+
+ xbd_0 = '/scr/geovlm/xbd_test_auxiliary_multi_image.json'
+ xbd_1 = '/scr/geovlm/xbd_test_canon_classification.json'
+ xbd_2 = '/scr/geovlm/xbd_test_canon_localization.json'
+
+ print("Running conversion on all datasets, storing updated datasets in variables")
+
+ from tqdm import tqdm
+
+ dataset_formats = [
+ (fmow_to_geochat_dataset_format, fmow_0),
+ (fmow_to_geochat_dataset_format, fmow_1),
+ ]
+ formatted_datasets = []
+ for format_func, dataset in tqdm(dataset_formats, desc="Converting datasets"):
+ if "xbd_test_auxiliary" in dataset:
+ formatted_datasets.append(format_func(dataset))
+
+ fmow_0_formatted, fmow_1_formatted = formatted_datasets
+
+ # Write the formatted data for fmow_0 into a JSON file named geochat_fmow_RECENT_format_low_res.json
+ with open('/scr/geovlm/geochat_fmow_RECENT_format_low_res.json', 'w') as file:
+ json.dump(fmow_0_formatted, file)
+
+ # Write the formatted data for fmow_1 into a JSON file named geochat_fmow_RECENT_format_low_res_AGG.json
+ with open('/scr/geovlm/geochat_fmow_RECENT_format_high_res.json', 'w') as file:
+ json.dump(fmow_1_formatted, file)
+
+ check_file('/scr/geovlm/geochat_fmow_RECENT_format_low_res.json')
+ check_file('/scr/geovlm/geochat_fmow_RECENT_format_high_res.json')
diff --git a/videollava/eval/eval_classification.py b/videollava/eval/eval_classification.py
new file mode 100644
index 0000000000000000000000000000000000000000..607b7cbc8809794e4b81b38920cb4fe1726498f8
--- /dev/null
+++ b/videollava/eval/eval_classification.py
@@ -0,0 +1,151 @@
+"""
+Segmentation metric code dapted from code for XView2: A Strong Baseline
+Xview2_Strong_Baseline/legacy/xview2_metrics.py
+Xview2_Strong_Baseline/legacy/create_masks.py
+"""
+# add python path
+# import sys
+# import os
+# sys.path.append('/deep/u/emily712/aicc-win24-geo-vlm/videollava/')
+
+import json
+import string
+import numpy as np
+import cv2
+from collections import defaultdict, Counter
+from nltk.tokenize import word_tokenize
+from shapely.geometry import Polygon
+from pathlib import Path
+from sklearn.metrics import f1_score
+from tqdm import tqdm
+
+
+def compute_tp_fn_fp(pred: np.ndarray, targ: np.ndarray, c: int):
+ """
+ Computes the number of TPs, FNs, FPs, between a prediction (x) and a target (y) for the desired class (c)
+
+ Args:
+ pred (np.ndarray): prediction
+ targ (np.ndarray): target
+ c (int): positive class
+ """
+ TP = np.logical_and(pred == c, targ == c).sum()
+ FN = np.logical_and(pred != c, targ == c).sum()
+ FP = np.logical_and(pred == c, targ != c).sum()
+ return [TP, FN, FP]
+
+
+def accuracy_precision_recall(answer_path, dataset, ignore_punctuation=True, verbose=True):
+ # Replace with the path to the answers file
+ if type(answer_path) == dict:
+ results = answer_path
+ else:
+ with open(answer_path) as json_data:
+ results = json.load(json_data)
+
+ task_total = defaultdict(int)
+ task_tp = defaultdict(int)
+
+ binary_classification = defaultdict(bool)
+ binary_fp = defaultdict(int)
+ binary_fn = defaultdict(int)
+
+ # Dictionary of dictionaries. Key: task. Value: {class: count}
+ ground_truths = defaultdict(dict)
+
+ values = defaultdict(list)
+
+ accepted_tasks = [
+ "temporal_question_answering",
+ "region_based_question_answering",
+ "temporal_region_based_question_answering",
+ "question_answering",
+ "temporal_referring_expression",
+ "rural_urban",
+ "comp",
+ "presence",
+ "count",
+ "change_to_what",
+ "smallest_change",
+ "change_or_not",
+ "change_ratio",
+ "largest_change",
+ "change_ratio_types",
+ "increase_or_not",
+ "decrease_or_not"
+ ]
+
+ for result in results.values():
+ if "task" in result and not any(result["task"].startswith(task) for task in accepted_tasks):
+ continue
+
+ # Clean predicted string if necessary
+ result["predicted"] = result["predicted"].lower()
+ result["ground_truth"] = result["ground_truth"].lower()
+ if ignore_punctuation:
+ result["predicted"] = ''.join(ch for ch in result["predicted"] if ch not in string.punctuation)
+ result["ground_truth"] = ''.join(ch for ch in result["ground_truth"] if ch not in string.punctuation)
+ if verbose:
+ values["predicted"].append(result["predicted"])
+ values["ground_truth"].append(result["ground_truth"])
+ values["correct_incorrect"].append("Correct" if result["predicted"] == result["ground_truth"] else "Incorrect")
+ if "task" not in result:
+ result["task"] = dataset
+
+ # True positive
+ if result["predicted"] == result["ground_truth"]:
+ task_tp[result["task"]] += 1
+ task_total[result["task"]] += 1
+
+ # If binary classification (yes/no question), calculate precision and recall metrics
+ binary_classification[result["task"]] = binary_classification[result["task"]] or (result["ground_truth"] in ["yes", "no"])
+ if binary_classification[result["task"]]:
+ if result["predicted"] != "no" and result["ground_truth"] == "no":
+ binary_fp[result["task"]] += 1
+ if result["predicted"] != "yes" and result["ground_truth"] == "yes":
+ binary_fn[result["task"]] += 1
+
+ # Update ground truth counts for the task
+ task = result["task"]
+ class_label = result["ground_truth"]
+ ground_truths[task][class_label] = ground_truths[task].get(class_label, 0) + 1
+
+ # Print tab separated values
+ if verbose:
+ max_len = max(len(v) for v in values["ground_truth"]) + 5
+ print("Predicted" + " " * (max_len - 9) + "\tGround Truth" + " " * (max_len - 12) + "\tCorrect/Incorrect")
+ for i in range(len(values["predicted"])):
+ print(values["predicted"][i] + " " * (max_len - len(values["predicted"][i])) + "\t" + values["ground_truth"][i] + " " * (max_len - len(values["ground_truth"][i])) + "\t" + values["correct_incorrect"][i])
+
+ total_tp = 0
+ total_predictions = 0
+ for task in task_tp:
+ acc_string = "Accuracy"
+ if ignore_punctuation:
+ acc_string += " (ignoring punctuation)"
+ print(f"{acc_string} for {task}: {round((task_tp[task] / task_total[task]), 4) * 100}%")
+
+ if binary_classification[task]:
+ if (task_tp[task] + binary_fp[task]) > 0:
+ print(f"Precision (ignoring punctuation) for {task}: {round((task_tp[task] / (task_tp[task] + binary_fp[task])), 3) * 100}%")
+ if (task_tp[task] + binary_fn[task]) > 0:
+ print(f"Recall (ignoring punctuation) for {task}: {round((task_tp[task] / (task_tp[task] + binary_fn[task])), 3) * 100}%")
+
+ majority_class = max(ground_truths[task], key=ground_truths[task].get)
+ majority_class_percentage = (ground_truths[task][majority_class] / task_total[task]) * 100
+ print(f"Majority class for {task}: {majority_class}, Percentage: {round(majority_class_percentage, 4)}%")
+
+ total_tp += task_tp[task]
+ total_predictions += task_total[task]
+
+ if total_predictions == 0:
+ print("No predictions made.")
+ else:
+ total_accuracy = (total_tp / total_predictions) * 100
+ print(f"Overall Accuracy: {round(total_accuracy, 3)}%")
+
+# For testing accuracy/precision/recall on a particular script without running inference
+if __name__ == '__main__':
+ root_dir = '/deep/u/jirvin16/aicc/aicc-win24-geo-vlm/videollava/scripts/geovlm/eval/QFabric/answers/'
+ answer_path = root_dir + "video-llava-7b-8bit-lora-final-no-metadata-zero-gc-acc8-freq-no-geochat-checkpoint-8000_qfabric_test_aux_data_test_prompt_strategy_interleave_chronological_prefix_True_load_8bit_True_load_4bit_False_delete_system_prompt_False.json"
+ accuracy_precision_recall(answer_path, dataset="qfabric", ignore_punctuation=True, verbose=False)
diff --git a/videollava/eval/eval_geochat_referring.py b/videollava/eval/eval_geochat_referring.py
new file mode 100644
index 0000000000000000000000000000000000000000..d3ce9fdde655ab6c9df679c86b09a9c3cd1a99c8
--- /dev/null
+++ b/videollava/eval/eval_geochat_referring.py
@@ -0,0 +1,330 @@
+"""
+calc_iou_individual adapted from calculate_mean_ap.py
+author: Timothy C. Arlen
+date: 28 Feb 2018
+"""
+
+import sys
+from os.path import dirname, abspath
+sys.path.append(dirname(dirname(dirname(dirname(abspath(__file__))))))
+
+from collections import defaultdict
+import numpy as np
+import json
+import ast
+import re
+import cv2
+from shapely import wkt, Polygon, box
+from infer_utils import create_mask
+from matplotlib.path import Path
+from tqdm import tqdm
+
+from eval_referring import referring_expression
+import matplotlib.pyplot as plt
+import time
+import math
+from matplotlib.path import Path
+
+def convert_geochat_string(build, img_size=256):
+ """
+ Convert the raw str geochat output {<40><89><56><100>|<57>}, {<0><89><56><100>|<57>}
+ to a list of rotated bboxes.
+ """
+ build = build.strip('{}')
+ bbox_segments = build.split("}{")
+ # Regular expression to find all numbers inside angle brackets
+ pattern = r"<(\d+)>"
+
+ # Extract numbers, convert them to integers, and collect into a list
+ bboxes = [
+ list(map(int, re.findall(pattern, segment)))
+ for segment in bbox_segments
+ ]
+
+ rotated_bboxes = []
+ for bbox in bboxes:
+ try:
+ xmin, ymin, xmax, ymax, angle = [float(v) for v in bbox]
+ except:
+ print("Warning - Malformed bbox: ", bbox)
+ print("Original string: ", build)
+ print()
+ continue
+
+ # Convert percentages to pixel coordinates
+ xmin = xmin * img_size / 100
+ ymin = ymin * img_size / 100
+ xmax = xmax * img_size / 100
+ ymax = ymax * img_size / 100
+
+ # Calculate rectangle dimensions
+ rect_width = xmax - xmin
+ rect_height = ymax - ymin
+ center_x = xmin + rect_width / 2
+ center_y = ymin + rect_height / 2
+
+ # Calculate corners before rotation
+ corners = np.array([
+ [xmin, ymin],
+ [xmax, ymin],
+ [xmax, ymax],
+ [xmin, ymax]
+ ])
+
+ # Rotate corners
+ angle_rad = math.radians(angle)
+ cos_angle = math.cos(angle_rad)
+ sin_angle = math.sin(angle_rad)
+ rotated_corners = []
+ for x, y in corners:
+ tx = x - center_x
+ ty = y - center_y
+ rotated_x = tx * cos_angle - ty * sin_angle + center_x
+ rotated_y = tx * sin_angle + ty * cos_angle + center_y
+ rotated_corners.append([rotated_x, rotated_y])
+
+ rotated_bboxes.append(np.array(rotated_corners))
+
+ return rotated_bboxes
+
+def create_geochat_mask(buildings, img_size=(256, 256)):
+ """
+ Given a list of buildings in an image, this function
+ - creates an img_size * img_size numpy array for the image
+ - returns the mask for all buildings
+ Input:
+ - buildings: List of geochat strings representing buildings
+ - img_size: Tuple indicating the size of the image (height, width)
+ """
+ mask = np.zeros(img_size, np.uint8)
+
+ # Fill in with ones the pixels that are inside the buildings (rotated bboxes)
+ for bbox in buildings:
+ path = Path(bbox)
+ x, y = np.meshgrid(np.arange(img_size[1]), np.arange(img_size[0]))
+ points = np.vstack((x.flatten(), y.flatten())).T
+ mask[path.contains_points(points).reshape(img_size)] = 1
+
+ return mask
+
+def calc_iou_individual(pred_box, gt_box):
+ """Calculate IoU of single predicted and ground truth box
+ Args:
+ pred_box (list of floats): location of predicted object as
+ [xmin, ymin, xmax, ymax]
+ gt_box (list of floats): location of ground truth object as
+ [xmin, ymin, xmax, ymax]
+ Returns:
+ float: value of the IoU for the two boxes.
+ Raises:
+ AssertionError: if the box is obviously malformed
+ """
+ x1_t, y1_t, x2_t, y2_t = gt_box
+ try:
+ x1_p, y1_p, x2_p, y2_p = pred_box
+ except:
+ return 0.0
+
+ if (x1_p > x2_p) or (y1_p > y2_p):
+ print("Prediction box is malformed? pred box: {}".format(pred_box))
+ if (x1_t > x2_t) or (y1_t > y2_t):
+ print("Ground Truth box is malformed? true box: {}".format(gt_box))
+
+ if (x2_t < x1_p or x2_p < x1_t or y2_t < y1_p or y2_p < y1_t):
+ return 0.0
+
+ far_x = np.min([x2_t, x2_p])
+ near_x = np.max([x1_t, x1_p])
+ far_y = np.min([y2_t, y2_p])
+ near_y = np.max([y1_t, y1_p])
+
+ inter_area = (far_x - near_x + 1) * (far_y - near_y + 1)
+ true_box_area = (x2_t - x1_t + 1) * (y2_t - y1_t + 1)
+ pred_box_area = (x2_p - x1_p + 1) * (y2_p - y1_p + 1)
+ iou = inter_area / (true_box_area + pred_box_area - inter_area)
+
+ return iou
+
+def get_single_image_bound_results(gt_wkts, pred_geochat_string, img_size=256):
+ """
+ Calculates upper bound and lower bound number of true_pos, false_pos, false_neg from single batch of boxes.
+ Args:
+ gt_wkts (list of strs): list of wkt strings of input polygons, scaled to raw pixel value
+ pred_boxes (list of lists): list of list of boxes, where each box is formatted
+ as [x_min, y_min, x_max, y_max] on scale from 0-100
+ img_size (int): dimensions of the image. defaults to 256.
+ Returns:
+ tuple of dicts: true positives (int), false positives (int), false negatives (int)
+ """
+ if isinstance(gt_wkts, str):
+ gt_polygons = [wkt.loads(gt_wkts)]
+ else:
+ gt_polygons = [wkt.loads(gt_wkt) for gt_wkt in gt_wkts]
+
+ lb_preds = convert_geochat_string(pred_geochat_string, img_size)
+ # get mask of all gt_polygons and lb_preds
+ gt_mask = create_mask(gt_polygons, (img_size, img_size))
+ lb_preds_mask = create_geochat_mask(lb_preds, (img_size, img_size))
+
+ # get lower bound intersection and union masks
+ intersection = np.logical_and(gt_mask, lb_preds_mask)
+ union = np.logical_or(gt_mask, lb_preds_mask)
+
+ # compute lb metrics
+ fp = np.sum(np.logical_and(lb_preds_mask, np.logical_not(gt_mask)))
+ tp = np.sum(np.logical_and(lb_preds_mask, gt_mask))
+ fn = np.sum(np.logical_and(np.logical_not(lb_preds_mask), gt_mask))
+ lb_stats = {'true_pos': tp, 'false_pos': fp, 'false_neg': fn, 'intersection': np.sum(intersection), 'union': np.sum(union)}
+
+ # get upper bound intersection and union masks
+ ub_pred_mask = np.logical_and(gt_mask, lb_preds_mask)
+ intersection = np.logical_and(ub_pred_mask, gt_mask)
+ union = np.logical_or(gt_mask, ub_pred_mask)
+
+ # compute ub metrics
+ ub_fp = np.sum(np.logical_and(ub_pred_mask, np.logical_not(gt_mask)))
+ ub_tp = np.sum(np.logical_and(ub_pred_mask, gt_mask))
+ ub_fn = np.sum(np.logical_and(np.logical_not(ub_pred_mask), gt_mask))
+ ub_stats = {'true_pos': ub_tp, 'false_pos': ub_fp, 'false_neg': ub_fn, 'intersection': np.sum(intersection), 'union': np.sum(union)}
+
+ return lb_stats, ub_stats
+
+def get_geochat_dataset(image_id):
+ if image_id.startswith("P"):
+ dataset = "SOTA"
+ elif image_id.startswith("train"):
+ dataset = "FAST"
+ else:
+ dataset = "SIOR"
+ return dataset
+
+def calc_precision_recall(img_results):
+ """Calculates precision and recall from the set of images
+ Args:
+ img_results (dict): dictionary formatted like:
+ {
+ 'img_id1': {'true_pos': int, 'false_pos': int, 'false_neg': int},
+ 'img_id2': ...
+ ...
+ }
+ Returns:
+ tuple: of floats of (precision, recall)
+ """
+ true_pos = 0; false_pos = 0; false_neg = 0
+ for _, res in img_results.items():
+ true_pos += res['true_pos']
+ false_pos += res['false_pos']
+ false_neg += res['false_neg']
+
+ try:
+ precision = true_pos/(true_pos + false_pos)
+ except ZeroDivisionError:
+ precision = 0.0
+ try:
+ recall = true_pos/(true_pos + false_neg)
+ except ZeroDivisionError:
+ recall = 0.0
+
+ return (precision, recall)
+
+
+DIMENSIONS = {'FAST': 600,
+ 'SIOR': 800,
+ 'SOTA': 1024}
+
+
+def referring_expression(answer_path, dataset, verbose=False, saving_path_root=None, img_size=256):
+ # Replace with the path to the answers file
+ if type(answer_path) == dict:
+ results = answer_path
+ else:
+ with open(answer_path) as json_data:
+ results = json.load(json_data)
+
+ img_results = {}
+ ub_results = {}
+ lb_results = {}
+ num_bboxes = 0
+ # Loop over results and get precision, recall overall
+ for id, result in tqdm(results.items()):
+
+ if dataset == "geochat_xbd":
+ pred = result['predicted']
+
+ dataset = get_geochat_dataset(id)
+ img_size = (DIMENSIONS[dataset])
+ pred = convert_geochat_string(pred, img_size)
+
+ ground_truth = result['ground_truth']
+ ground_truth = np.array(ground_truth)
+ num_bboxes += len(ground_truth)
+
+ img_results[id] = get_single_image_results(ground_truth, pred, iou_thr=0.5)
+
+ continue
+
+ try:
+ if 'referring_expression' not in result['task']:
+ continue # no bounding box outputs for temporal_referring_expression
+ except:
+ pass
+
+ # TODO: clean the following todos
+
+ # TODO: LOOP THROUGH IDENTIFY TASKS/QUESTIONS IN THE DATASET
+
+ # TODO: HANDLE WHEN THERE ARE NO BOUNDING BOXES IN GROUND TRUTH for auxiliary tasks
+ if not result['original_input_polygon']:
+ first_open_bracket_ind = result["predicted"].find("{")
+ last_close_bracket_ind = result["predicted"].rfind("}")
+ if last_close_bracket_ind != -1 and first_open_bracket_ind != -1:
+ parsed_predicted = result["predicted"][first_open_bracket_ind:last_close_bracket_ind+1]
+ else:
+ parsed_predicted = ""
+ predicted_boxes = convert_geochat_string(parsed_predicted)
+ # If ground truth contains no boxes: all predictions are false positives
+ false_pos = len(predicted_boxes)
+ false_pos_pixels = np.sum(create_geochat_mask(predicted_boxes))
+ img_results[id] = {'true_pos': 0, 'false_pos': false_pos, 'false_neg': 0, 'intersection':0, 'union':false_pos_pixels}
+ ub_results[id] = {'true_pos': 0, 'false_pos': false_pos_pixels, 'false_neg': 0, 'intersection':0, 'union':false_pos_pixels}
+ lb_results[id] = {'true_pos': 0, 'false_pos': false_pos_pixels, 'false_neg': 0, 'intersection':0, 'union':false_pos_pixels}
+ continue
+ else: # Ground truth contains boxes: find predicted Geochat boxes
+ first_open_bracket_ind = result["predicted"].find("{")
+ last_close_bracket_ind = result["predicted"].rfind("}")
+ if last_close_bracket_ind != -1 and first_open_bracket_ind != -1:
+ parsed_predicted = result["predicted"][first_open_bracket_ind:last_close_bracket_ind+1]
+ else:
+ parsed_predicted = ""
+ gt_wkts = result['original_input_polygon']
+ lb_results[id], ub_results[id] = get_single_image_bound_results(gt_wkts, parsed_predicted)
+
+ if len(ub_results) != 0:
+ ub_intersection = np.sum([res['intersection'] for res in ub_results.values()])
+ ub_union = np.sum([res['union'] for res in ub_results.values()])
+ lb_intersection = np.sum([res['intersection'] for res in lb_results.values()])
+ lb_union = np.sum([res['union'] for res in lb_results.values()])
+ print("Upper bound IOU: ", ub_intersection / ub_union if ub_union != 0 else 0)
+ print("Lower bound IOU: ", lb_intersection / lb_union if lb_union != 0 else 0)
+ ub_precision, ub_recall = calc_precision_recall(ub_results)
+ lb_precision, lb_recall = calc_precision_recall(lb_results)
+ print('Lower bound precision: ', lb_precision)
+ print('Lower bound recall: ', lb_recall)
+ print("Upper bound F1: ", 2 * (ub_precision * ub_recall) / (ub_precision + ub_recall) if (ub_precision + ub_recall) != 0 else 0)
+ print("Lower bound F1: ", 2 * (lb_precision * lb_recall) / (lb_precision + lb_recall) if (lb_precision + lb_recall) != 0 else 0)
+
+ print("Acc@0.5: ", np.sum([res['true_pos'] for res in img_results.values()]) / num_bboxes)
+
+ if type(answer_path) == dict:
+ return
+
+ if saving_path_root:
+ with open(f"{saving_path_root}/referring_expression_scores.json", 'w') as f:
+ json.dump(img_results, f)
+
+if __name__ == '__main__':
+ answer_path = "scripts/geovlm/eval/xBD/answers/ckpt14000-geochat-bench_interleave_test.json"
+ referring_expression(answer_path, dataset="geochat_xbd")
+ #answer_path = "scripts/geochat/eval/xBD/geochat_xbd_test_auxiliary_dict.json"
+ # referring_expression(answer_path, dataset="xbd")
+
diff --git a/videollava/eval/eval_referring.py b/videollava/eval/eval_referring.py
new file mode 100644
index 0000000000000000000000000000000000000000..104870380aa57d3fa3755472e984fbd85fe1b0c2
--- /dev/null
+++ b/videollava/eval/eval_referring.py
@@ -0,0 +1,351 @@
+"""
+Code adapted from calculate_mean_ap.py
+author: Timothy C. Arlen
+date: 28 Feb 2018
+"""
+import sys
+sys.path.append('/deep/u/joycech/aicc-working/videollava')
+
+from collections import defaultdict
+import numpy as np
+import json
+import ast
+import re
+import cv2
+from shapely import wkt, Polygon, box
+from infer_utils import create_mask, create_mask_s2looking
+
+
+def calc_iou_individual(pred_box, gt_box):
+ """Calculate IoU of single predicted and ground truth box
+ Args:
+ pred_box (list of floats): location of predicted object as
+ [xmin, ymin, xmax, ymax]
+ gt_box (list of floats): location of ground truth object as
+ [xmin, ymin, xmax, ymax]
+ Returns:
+ float: value of the IoU for the two boxes.
+ Raises:
+ AssertionError: if the box is obviously malformed
+ """
+ x1_t, y1_t, x2_t, y2_t = gt_box
+ try:
+ x1_p, y1_p, x2_p, y2_p = pred_box
+ except:
+ print("Prediction box is malformed? pred box: {}".format(pred_box))
+ return 0.0
+
+ if (x1_p > x2_p) or (y1_p > y2_p):
+ print("Prediction box is malformed? pred box: {}".format(pred_box))
+ return 0.0
+ if (x1_t > x2_t) or (y1_t > y2_t):
+ raise AssertionError(
+ "Ground Truth box is malformed? true box: {}".format(gt_box))
+
+ if (x2_t < x1_p or x2_p < x1_t or y2_t < y1_p or y2_p < y1_t):
+ return 0.0
+
+ far_x = np.min([x2_t, x2_p])
+ near_x = np.max([x1_t, x1_p])
+ far_y = np.min([y2_t, y2_p])
+ near_y = np.max([y1_t, y1_p])
+
+ inter_area = (far_x - near_x + 1) * (far_y - near_y + 1)
+ true_box_area = (x2_t - x1_t + 1) * (y2_t - y1_t + 1)
+ pred_box_area = (x2_p - x1_p + 1) * (y2_p - y1_p + 1)
+ iou = inter_area / (true_box_area + pred_box_area - inter_area)
+
+ return iou
+
+def get_single_image_bound_results(gt_wkts, pred_boxes, img_size=256, dataset=None, id=None, predicted_mask=None, split=None, question=None):
+ """
+ Calculates upper bound and lower bound number of true_pos, false_pos, false_neg from single batch of boxes.
+ Args:
+ gt_wkts (list of strs): list of wkt strings of input polygons, scaled to raw pixel value
+ pred_boxes (list of lists): list of list of boxes, where each box is formatted
+ as [x_min, y_min, x_max, y_max] on scale from 0-100
+ img_size (int): dimensions of the image. defaults to 256.
+ Returns:
+ tuple of dicts: true positives (int), false positives (int), false negatives (int)
+ """
+ lb_preds = [[num * img_size / 100 for num in box] for box in pred_boxes]
+ # add error handling for this type of outputs: [0, 10, 12, 22], [0, 6, 12, 19], [0, 0], [31, 0]
+ try:
+ lb_preds = [box(*pred_box) for pred_box in lb_preds]
+ except:
+ lb_preds = []
+ for pred_box in pred_boxes:
+ if len(pred_box) == 4:
+ lb_preds.append(box(*pred_box))
+
+ if isinstance(gt_wkts, str):
+ gt_polygons = [wkt.loads(gt_wkts)]
+ elif gt_wkts is None:
+ gt_polygons = []
+ else:
+ gt_polygons = [wkt.loads(gt_wkt) for gt_wkt in gt_wkts]
+
+ # get mask of all gt_polygons and lb_preds
+ if dataset == None:
+ gt_mask = create_mask(gt_polygons, (img_size, img_size))
+ else:
+ gt_mask = create_mask_s2looking(id, split=split, question=question)
+ #gt_mask = create_mask(gt_polygons, (img_size, img_size))
+
+ if dataset != "geochat_s2looking":
+ lb_preds_mask = create_mask(lb_preds, (img_size, img_size))
+ else:
+ lb_preds_mask = predicted_mask
+
+
+ # get lower bound intersection and union masks
+ intersection = np.logical_and(gt_mask, lb_preds_mask)
+ union = np.logical_or(gt_mask, lb_preds_mask)
+
+ # compute lb metrics
+ lower_bound_iou = np.sum(intersection) / np.sum(union)
+ if np.sum(intersection) == 0 and np.sum(union) == 0:
+ return None, None
+ if np.isnan(lower_bound_iou):
+ lower_bound_iou = 0
+
+
+ fp = np.sum(np.logical_and(lb_preds_mask, np.logical_not(gt_mask)))
+ tp = np.sum(np.logical_and(lb_preds_mask, gt_mask))
+ fn = np.sum(np.logical_and(np.logical_not(lb_preds_mask), gt_mask))
+ lb_stats = {'true_pos': tp,
+ 'false_pos': fp,
+ 'false_neg': fn,
+ 'intersection': np.sum(intersection),
+ 'union': np.sum(union)}
+
+ return lb_stats
+
+def get_single_image_results(gt_boxes, pred_boxes, iou_thr):
+ """Calculates number of true_pos, false_pos, false_neg from single batch of boxes.
+ Args:
+ gt_boxes (list of list of floats): list of locations of ground truth
+ objects as [xmin, ymin, xmax, ymax]
+ pred_boxes (dict): dict of dicts of 'boxes' (formatted like `gt_boxes`)
+ and 'scores'
+ iou_thr (float): value of IoU to consider as threshold for a
+ true prediction.
+ Returns:
+ dict: true positives (int), false positives (int), false negatives (int)
+ """
+
+ all_pred_indices = range(len(pred_boxes))
+ all_gt_indices = range(len(gt_boxes))
+ if len(all_pred_indices) == 0:
+ tp = 0
+ fp = 0
+ fn = len(gt_boxes)
+ return {'true_pos': tp, 'false_pos': fp, 'false_neg': fn}
+ if len(all_gt_indices) == 0:
+ tp = 0
+ fp = len(pred_boxes)
+ fn = 0
+ return {'true_pos': tp, 'false_pos': fp, 'false_neg': fn}
+
+ gt_idx_thr = []
+ pred_idx_thr = []
+ ious = []
+ for ipb, pred_box in enumerate(pred_boxes):
+ for igb, gt_box in enumerate(gt_boxes):
+ iou = calc_iou_individual(pred_box, gt_box)
+ if iou > iou_thr:
+ gt_idx_thr.append(igb)
+ pred_idx_thr.append(ipb)
+ ious.append(iou)
+
+ args_desc = np.argsort(ious)[::-1]
+ if len(args_desc) == 0:
+ # No matches
+ tp = 0
+ fp = len(pred_boxes)
+ fn = len(gt_boxes)
+ else:
+ gt_match_idx = []
+ pred_match_idx = []
+ for idx in args_desc:
+ gt_idx = gt_idx_thr[idx]
+ pr_idx = pred_idx_thr[idx]
+ # If the boxes are unmatched, add them to matches
+ if (gt_idx not in gt_match_idx) and (pr_idx not in pred_match_idx):
+ gt_match_idx.append(gt_idx)
+ pred_match_idx.append(pr_idx)
+ tp = len(gt_match_idx)
+ fp = len(pred_boxes) - len(pred_match_idx)
+ fn = len(gt_boxes) - len(gt_match_idx)
+
+ return {'true_pos': tp, 'false_pos': fp, 'false_neg': fn}
+
+def calc_precision_recall(img_results):
+ """Calculates precision and recall from the set of images
+ Args:
+ img_results (dict): dictionary formatted like:
+ {
+ 'img_id1': {'true_pos': int, 'false_pos': int, 'false_neg': int},
+ 'img_id2': ...
+ ...
+ }
+ Returns:
+ tuple: of floats of (precision, recall)
+ """
+ true_pos = 0; false_pos = 0; false_neg = 0
+ for _, res in img_results.items():
+ true_pos += res['true_pos']
+ false_pos += res['false_pos']
+ false_neg += res['false_neg']
+
+ try:
+ precision = true_pos/(true_pos + false_pos)
+ except ZeroDivisionError:
+ precision = 0.0
+ print(true_pos, "true_pos", false_pos, "false_pos", false_neg, "false_neg")
+ try:
+ recall = true_pos/(true_pos + false_neg)
+ except ZeroDivisionError:
+ recall = 0.0
+
+ return (precision, recall)
+
+def extract_bboxes(input_string):
+ """
+ Takes as an input a string like in the image, there are two buildings that have been changed. the first building is located at [0.0, 0.69, 0.45, 0.9] and the second building is located at [0.46, 0.69, 0.99, 0.91]
+ Returns a list of bounding boxes in the format [x_min, y_min, x_max, y_max]
+ Input:
+ input_string (str): string containing the bounding boxes
+ Returns:
+ list of lists: list of bounding boxes
+ """
+ matches = re.findall(r'\[\[.*?\]\]', input_string)
+ return [ast.literal_eval(match) for match in matches]
+
+
+def referring_expression(answer_path, dataset, verbose=False, saving_path_root=None, img_size=256, split=None):
+ if type(answer_path) == dict:
+ results = answer_path
+ else:
+ with open(answer_path) as json_data:
+ results = json.load(json_data)
+
+ img_results = {}
+ lb_results = {}
+ # Loop over results and get precision, recall overall
+ for id, result in results.items():
+ if 'temporal_referring_expression' in result['task']:
+ if not "s2looking" in dataset:
+ continue # no bounding box outputs for temporal_referring_expression
+
+ # for the geochat s2looking predictions, we work directly with the predicted mask instead of the bounding boxes
+ if dataset == 'geochat_s2looking':
+ if 'referring_expression' in result['task'] or 'localization' in result['task']:
+ lb_res = get_single_image_bound_results(result['original_input_polygon'], [], dataset=dataset, id=id, predicted_mask=result['predicted_mask'], split=split, question=result["question"])
+ if lb_res != None:
+ lb_results[id] = lb_res
+ continue
+ elif 'question_answering' in result['task']:
+ continue
+
+ if 'referring_expression' in result['task'] or 'largest building' in result['task'] or "canonical" in result['task'] or 'localization' in result['task'] \
+ or 'geochat_referring' in result['task']:
+ # No bounding boxes in predicted string
+ if "[" not in result["predicted"]:
+ # Ground truth has no bounding boxes
+ if result["ground_truth"].startswith("There are no") or "no" in result["ground_truth"] or "No" in result["ground_truth"]:
+ # Discard true negatives
+ continue
+ # Ground truth has bounding boxes, not identified by the model --> all false negatives
+ else:
+ false_neg = "[" + result["ground_truth"] + "]"
+ false_neg = false_neg.replace(".", "")
+
+ try:
+ false_neg = len(ast.literal_eval(false_neg))
+ except:
+ # count the number of opening '[' in the string
+ false_neg = false_neg.count('[') - 1
+ if not "s2looking" in dataset:
+ gt_mask = create_mask(wkt.loads(result['original_input_polygon']), (img_size, img_size))
+ else:
+ gt_mask = create_mask_s2looking(id, split=split, question=result['question'])
+ # gt_mask = create_mask(wkt.loads(result['original_input_polygon']), (img_size, img_size))
+ img_results[id] = {'true_pos': 0, 'false_pos': 0, 'false_neg': false_neg, 'intersection':0, 'union':false_neg}
+ false_neg = np.sum(gt_mask)
+ lb_results[id] = {'true_pos': 0, 'false_pos': 0, 'false_neg': false_neg, 'intersection':0, 'union':false_neg}
+
+ # Bounding boxes in predicted and output string --> compare bounding boxes
+ else:
+
+ # To deal with cases where the model outputs an incomplete bounding box (e.g. "[24, 76,")
+ first_open_bracket_ind = result["predicted"].find("[")
+ last_close_bracket_ind = result["predicted"].rfind("]")
+ if last_close_bracket_ind != -1 and first_open_bracket_ind != -1:
+ parsed_predicted = result["predicted"][first_open_bracket_ind:last_close_bracket_ind+1]
+ else:
+ parsed_predicted = ""
+
+ # Load list of predicted bounding boxes
+ try:
+ predicted_boxes = ast.literal_eval("[" + parsed_predicted + "]")
+ except:
+ match = re.search(r'\[\[.*\]\]', result["predicted"])
+ if match:
+ predicted_boxes = ast.literal_eval(match.group())
+ else:
+ predicted_boxes = []
+
+ predicted_boxes = [[coord * 100 if coord < 1 else coord for coord in box] for box in predicted_boxes]
+
+ # Load list of ground truth bounding boxes
+ if result["ground_truth"].startswith("There are no") or "no" in result["ground_truth"].lower():
+ # If ground truth contains no boxes
+ ground_truth_boxes = []
+ first_open_bracket_ind = result["ground_truth"].find("[")
+ last_close_bracket_ind = result["ground_truth"].rfind("]")
+ if last_close_bracket_ind != -1 and first_open_bracket_ind != -1:
+ parsed_gt = result["ground_truth"][first_open_bracket_ind:last_close_bracket_ind+1]
+ else:
+ parsed_gt = ""
+ try:
+ ground_truth_boxes = ast.literal_eval("[" + parsed_gt + "]")
+ except:
+ match = re.search(r'\[\[.*\]\]', result["ground_truth"])
+ if match:
+ ground_truth_boxes = ast.literal_eval(match.group())
+ else:
+ ground_truth_boxes = []
+
+ # Get mask results from the two previous parsings
+ gt_wkts = result['original_input_polygon']
+ img_results[id] = get_single_image_results(ground_truth_boxes, predicted_boxes, iou_thr=0.5) ######
+
+ if 'referring_expression' in result['task'] or 'largest building' in result['task'] or "canonical" in result['task'] or 'localization' in result['task']:
+ if not "s2looking" in dataset:
+ lb_results[id] = get_single_image_bound_results(gt_wkts, predicted_boxes)
+ elif dataset=="s2looking":
+ lb_results[id] = get_single_image_bound_results(gt_wkts, predicted_boxes, dataset=dataset, id=id, split=split, question=result["question"])
+ else:
+ lb_results[id] = get_single_image_bound_results(gt_wkts, predicted_boxes, predicted_mask=result['predicted_mask'], split=split, question=result["question"])
+
+ precision, recall = calc_precision_recall(img_results)
+ print("Referring expression results (precision, recall): ", precision, recall)
+ print("Acc@0.5: ", np.sum([res['true_pos'] for res in img_results.values()]) / len(results.keys()))
+
+ if len(lb_results) != 0:
+ lb_intersection = np.sum([res['intersection'] for res in lb_results.values()])
+ lb_union = np.sum([res['union'] for res in lb_results.values()])
+ print("Lower bound IOU: ", lb_intersection / lb_union if lb_union != 0 else 0)
+ lb_precision, lb_recall = calc_precision_recall(lb_results)
+ print('Lower bound precision: ', lb_precision)
+ print('Lower bound recall: ', lb_recall)
+ print("Lower bound F1: ", 2 * (lb_precision * lb_recall) / (lb_precision + lb_recall) if (lb_precision + lb_recall) != 0 else 0)
+
+ if saving_path_root:
+ with open(f"{saving_path_root}/referring_expression_scores.json", 'w') as f:
+ json.dump(img_results, f)
+
+if __name__ == '__main__':
+ answer_path = "scripts/geovlm/eval/xBD/answers/ckpt14000-old-aux-xbd-test-canon-auxiliary_interleave.json"
+ referring_expression(answer_path, dataset="xbd")
\ No newline at end of file
diff --git a/videollava/eval/geochat_bench.py b/videollava/eval/geochat_bench.py
new file mode 100644
index 0000000000000000000000000000000000000000..5af3a999a21b54c068d74b6094b6610e59a8f6ab
--- /dev/null
+++ b/videollava/eval/geochat_bench.py
@@ -0,0 +1,228 @@
+
+
+from eval_geochat_referring import get_single_image_results, convert_geochat_string
+
+from collections import defaultdict
+import numpy as np
+import json
+import ast
+import re
+import cv2
+from shapely import wkt, Polygon, box
+from infer_utils import create_mask
+from matplotlib.path import Path
+from tqdm import tqdm
+import matplotlib.pyplot as plt
+import time
+import math
+from matplotlib.path import Path
+
+
+
+DIMENSIONS = {'FAST': 600,
+ 'SIOR': 800,
+ 'SOTA': 1024}
+
+def calc_iou_individual_rotated(pred_box, gt_box, img_size=None):
+ """Calculate IoU of single predicted and ground truth box
+ Args:
+ pred_box (list of floats): location of predicted object as
+ [[x1, y1], [x2, y2], [x3, y3], [x4, y4]]
+ gt_box (list of floats): location of ground truth object as
+ [[x1, y1], [x2, y2], [x3, y3], [x4, y4]]
+ Returns:
+ float: value of the IoU for the two boxes.
+ Raises:
+ AssertionError: if the box is obviously malformed
+ """
+
+ pred_box = np.array(pred_box)
+ gt_box = np.array(gt_box)
+ pred_box = pred_box.reshape(4, 2)
+ gt_box = gt_box.reshape(4, 2)
+ pred_polygon = Polygon(pred_box)
+ gt_polygon = Polygon(gt_box)
+ intersection = pred_polygon.intersection(gt_polygon).area
+ union = pred_polygon.union(gt_polygon).area
+ iou = intersection / union
+
+ return iou
+
+
+def get_single_image_results_rotated(gt_boxes, pred_boxes, iou_thr, img_size=None):
+ """Calculates number of true_pos, false_pos, false_neg from single batch of boxes.
+ Args:
+ gt_boxes (list of list of floats): list of locations of ground truth
+ objects as [[x1,y1], [x2,y2], ...]
+ pred_boxes (dict): dict of dicts of 'boxes'
+ [[x1,y1], [x2,y2], ...]
+ iou_thr (float): value of IoU to consider as threshold for a
+ true prediction.
+ Returns:
+ dict: true positives (int), false positives (int), false negatives (int)
+ """
+
+ all_pred_indices = range(len(pred_boxes))
+ all_gt_indices = range(len(gt_boxes))
+ if len(all_pred_indices) == 0:
+ tp = 0
+ fp = 0
+ fn = len(gt_boxes)
+ return {'true_pos': tp, 'false_pos': fp, 'false_neg': fn}
+ if len(all_gt_indices) == 0:
+ tp = 0
+ fp = len(pred_boxes)
+ fn = 0
+ return {'true_pos': tp, 'false_pos': fp, 'false_neg': fn}
+
+ gt_idx_thr = []
+ pred_idx_thr = []
+ ious = []
+ for ipb, pred_box in enumerate(pred_boxes):
+ for igb, gt_box in enumerate(gt_boxes):
+ iou = calc_iou_individual_rotated(pred_box, gt_box, img_size)
+ if iou > iou_thr:
+ gt_idx_thr.append(igb)
+ pred_idx_thr.append(ipb)
+ ious.append(iou)
+
+ args_desc = np.argsort(ious)[::-1]
+ if len(args_desc) == 0:
+ # No matches
+ tp = 0
+ fp = len(pred_boxes)
+ fn = len(gt_boxes)
+ else:
+ gt_match_idx = []
+ pred_match_idx = []
+ for idx in args_desc:
+ gt_idx = gt_idx_thr[idx]
+ pr_idx = pred_idx_thr[idx]
+ # If the boxes are unmatched, add them to matches
+ if (gt_idx not in gt_match_idx) and (pr_idx not in pred_match_idx):
+ gt_match_idx.append(gt_idx)
+ pred_match_idx.append(pr_idx)
+ tp = len(gt_match_idx)
+ fp = len(pred_boxes) - len(pred_match_idx)
+ fn = len(gt_boxes) - len(gt_match_idx)
+
+ return {'true_pos': tp, 'false_pos': fp, 'false_neg': fn}
+
+
+def accuracy0_5(answer_path, dataset, aux_dataset="scripts/geochat_bench_dict.json"):
+ # Replace with the path to the answers file
+ results = None
+ if dataset != "geochat_xbd":
+
+ if type(answer_path) == dict:
+ results = answer_path
+ else:
+ results = []
+ with open(answer_path) as json_data:
+ for line in json_data:
+ results.append(json.loads(line))
+
+ with open(aux_dataset) as json_data:
+ aux_results = json.load(json_data)
+
+ img_results = {}
+ num_bboxes = 0
+
+ if dataset != "geochat_xbd":
+ print("Number of images in Geochat: ", len(aux_results))
+ print("Number of images predicted: ", len(results))
+
+ i = 0
+ # Loop over results and get precision, recall overall
+ for id, result in tqdm(aux_results.items()):
+
+ if dataset == "geochat_xbd":
+ pred = result['answer']
+
+ img_size = DIMENSIONS[result['dataset']]
+ pred = convert_geochat_string(pred, img_size)
+
+ ground_truth = result['ground_truth']
+ ground_truth = np.array(ground_truth)
+ num_bboxes += len(ground_truth)
+
+ img_results[id] = get_single_image_results_rotated(ground_truth, pred, iou_thr=0.5)
+
+ else:
+
+ geochat_id = id.split(".")[0]
+
+ img_size = DIMENSIONS[aux_results[geochat_id]['dataset']]
+ ground_truth = result['ground_truth']
+ ground_truth = np.array(ground_truth)
+ num_bboxes += len(ground_truth)
+
+ parsed_predicted = results[i]['predicted']
+ # Load list of predicted and round truth bounding boxes for a single image
+ try:
+ predicted_boxes = ast.literal_eval("[" + parsed_predicted + "]")
+ except:
+ match = re.search(r'\[\[.*\]\]', parsed_predicted)
+ if match:
+ predicted_boxes = ast.literal_eval(match.group())
+ else:
+ predicted_boxes = []
+
+ predicted_boxes = [[coord * 100 if coord < 1 else coord for coord in box] for box in predicted_boxes]
+
+ # scale by img_size
+ predicted_boxes = [[coord * img_size / 100 for coord in box] for box in predicted_boxes]
+
+ assert results[i]['ground_truth'] == result['ground_truth']
+
+ # convert the pred bboxes [xmin, ymin, xmax, ymax] to [[x1, y1], [x2, y2], [x3, y3], [x4, y4]]
+ pred_bboxes = []
+ for bbox in predicted_boxes:
+ x1, y1, x2, y2 = bbox
+ pred_bboxes.append([[x1, y1], [x2, y1], [x2, y2], [x1, y2]])
+
+ img_results[id] = get_single_image_results_rotated(ground_truth, pred_bboxes, iou_thr=0.5, img_size=img_size)
+
+ i+=1
+
+
+ acc = np.sum([res['true_pos'] for res in img_results.values()]) / num_bboxes
+ print("Acc@0.5: ", acc)
+ return acc
+
+
+
+if __name__ == '__main__':
+ print("Geochat bench")
+ geochat_path = "scripts/geochat_bench_dict.json"
+ answer_path = "scripts/geochat_bench_dict.json"
+ acc_geochat = accuracy0_5(answer_path, dataset="geochat_xbd")
+ print()
+
+
+ print("Teochat bench")
+ answer_path = "/deep/u/idormoy/aicc-win24-geo-vlm/videollava/scripts/geovlm/eval/QFabric/answers/geochat-referring-checkpoint14000_prompt_strategy_interleave_chronological_prefix_True_load_8bit_True_load_4bit_False_delete_system_prompt_False_tmp_0_end.json"
+ acc_teochat = accuracy0_5(answer_path, dataset="geochat")
+ print()
+
+
+ print("Teochat-T bench")
+ answer_path = "/deep/u/idormoy/aicc-win24-geo-vlm/videollava/videollava/eval/video/geochat-bench-ckpt8000-FIXED_prompt_strategy_interleave_chronological_prefix_True_load_8bit_False_load_4bit_True_delete_system_prompt_False_tmp_0_end (1).json"
+ acc_teochatT = accuracy0_5(answer_path, dataset="geochat")
+ print()
+
+
+
+ print("VideoLLaVA bench")
+ answer_path = "/deep/u/idormoy/aicc-win24-geo-vlm/videollava/videollava/eval/video/geochat-referring-Video-LLaVA-7B_prompt_strategy_interleave_chronological_prefix_True_load_8bit_False_load_4bit_True_delete_system_prompt_False_tmp_0_end (1).json"
+ acc_videollava = accuracy0_5(answer_path, dataset="geochat")
+ print()
+
+
+
+ print("Overall accuracies")
+ print("Geochat: ", acc_geochat)
+ print("Teochat: ", acc_teochat)
+ print("Teochat-T: ", acc_teochatT)
+ print("VideoLLaVA: ", acc_videollava)
+
diff --git a/videollava/eval/geochat_eval_fmow.py b/videollava/eval/geochat_eval_fmow.py
new file mode 100644
index 0000000000000000000000000000000000000000..23ae604bade614148154ab2afad6226cf9911743
--- /dev/null
+++ b/videollava/eval/geochat_eval_fmow.py
@@ -0,0 +1,205 @@
+import argparse
+import torch
+import os
+import json
+from tqdm import tqdm
+import shortuuid
+import sys
+
+sys.path.append('/deep/u/emily712/GeoChat')
+
+from geochat.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
+from geochat.conversation import conv_templates, SeparatorStyle
+from geochat.model.builder import load_pretrained_model
+from geochat.utils import disable_torch_init
+from geochat.mm_utils import tokenizer_image_token, get_model_name_from_path, KeywordsStoppingCriteria
+from eval_classification import *
+
+from PIL import Image
+import math
+import numpy as np
+
+def aggregate_accuracy(answers_file, output_file):
+ """
+ Parses geochat inference output and aggregates votes on single images
+ across an image sequence into the format needed for geovlm-style evaluation.
+
+ params:
+ - answers_file: path to the file containing geochat inference output
+ - output_file: path to the file where the aggregated output will be saved
+ """
+ with open(answers_file, 'r') as f:
+ answers = [json.loads(line) for line in f]
+ print(answers)
+ # dictionary that will contain parsed output
+ votes = {}
+
+ # parse answers so that predictions with the same linked_id
+ # are aggregated into a single item with 'predictions' containing
+ # a list of values. All other keys should be the same
+ for answer in answers:
+ print(answer)
+ print(answer['linked_id'])
+ id = answer['linked_id']
+ print(id)
+ if id not in votes:
+ item = {}
+ item['predicted'] = [answer['predicted']]
+ item['ground_truth'] = answer['ground_truth']
+ item['task'] = answer['task']
+ item['question'] = answer['question']
+ item['id'] = answer['id']
+ votes[id] = item
+ else:
+ votes['linked_id']['predicted'].append(answer['predicted'])
+
+ # implement voting so that each list in 'predicted' attribute
+ # is reduced to the most common value
+ for linked_id, predicted_dict in votes.items():
+ predicted = predicted_dict['predicted']
+ unique, counts = np.unique(predicted, return_counts=True)
+ index = np.argmax(counts)
+ votes[linked_id]['predicted'] = unique[index]
+
+ with open(output_file, 'w') as f:
+ json.dump(votes, f)
+
+
+def split_list(lst, n):
+ """Split a list into n (roughly) equal-sized chunks"""
+ chunk_size = math.ceil(len(lst) / n) # integer division
+ return [lst[i:i+chunk_size] for i in range(0, len(lst), chunk_size)]
+
+
+def get_chunk(lst, n, k):
+ chunks = split_list(lst, n)
+ return chunks[k]
+
+
+def eval_model(args):
+ # Model
+ disable_torch_init()
+ model_path = os.path.expanduser(args.model_path)
+ model_name = get_model_name_from_path(model_path)
+ tokenizer, model, image_processor, context_len = load_pretrained_model(args.model_path, args.model_base, model_name, cache_dir=args.cache_dir)
+
+ with open(args.question_file, 'r') as f:
+ questions = json.load(f)
+ #questions = [json.loads(q) for q in open(os.path.expanduser(args.question_file), "r")]
+
+ questions = get_chunk(questions, args.num_chunks, args.chunk_idx)
+ answers_file = os.path.expanduser(args.answers_file)
+ os.makedirs(os.path.dirname(answers_file), exist_ok=True)
+
+ ans_file = open(answers_file, "w")
+
+ skipped_count = 0
+
+ for i in tqdm(range(0,len(questions),args.batch_size)):
+ input_batch=[]
+ input_image_batch=[]
+ count=i
+ image_folder=[]
+ batch_end = min(i + args.batch_size, len(questions))
+
+ for j in range(i,batch_end):
+ if 'image' not in questions[j]:
+ print(f"Skipped entry [{skipped_count}]")
+ skipped_count += 1
+ continue
+
+ print(questions[j])
+ image_file=questions[j]['image']
+ qs=questions[j]['conversations'][0]['value']
+
+ if model.config.mm_use_im_start_end:
+ qs = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + '\n' + qs
+ else:
+ qs = DEFAULT_IMAGE_TOKEN + '\n' + qs
+
+ conv = conv_templates[args.conv_mode].copy()
+ conv.append_message(conv.roles[0], qs)
+ conv.append_message(conv.roles[1], None)
+ prompt = conv.get_prompt()
+
+ input_ids = tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).cuda()
+ input_batch.append(input_ids)
+
+ image = Image.open(os.path.join(args.image_folder, image_file))
+
+ image_folder.append(image)
+
+ stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
+ keywords = [stop_str]
+ stopping_criteria = KeywordsStoppingCriteria(keywords, tokenizer, input_ids)
+
+ if len(input_batch) == 0:
+ print("All images here were skipped")
+ continue
+
+ max_length = max(tensor.size(1) for tensor in input_batch)
+
+ final_input_list = [torch.cat((torch.zeros((1,max_length - tensor.size(1)), dtype=tensor.dtype,device=tensor.get_device()), tensor),dim=1) for tensor in input_batch]
+ final_input_tensors=torch.cat(final_input_list,dim=0)
+ image_tensor_batch = image_processor.preprocess(image_folder,crop_size ={'height': 504, 'width': 504},size = {'shortest_edge': 504}, return_tensors='pt')['pixel_values']
+
+ with torch.inference_mode():
+ output_ids = model.generate( final_input_tensors, images=image_tensor_batch.half().cuda(), do_sample=False , temperature=args.temperature, top_p=args.top_p, num_beams=1, max_new_tokens=256,length_penalty=2.0, use_cache=True)
+
+ input_token_len = final_input_tensors.shape[1]
+ n_diff_input_output = (final_input_tensors != output_ids[:, :input_token_len]).sum().item()
+ if n_diff_input_output > 0:
+ print(f'[Warning] {n_diff_input_output} output_ids are not the same as the input_ids')
+ outputs = tokenizer.batch_decode(output_ids[:, input_token_len:], skip_special_tokens=True)
+ for k in range(0,len(final_input_list)):
+ output = outputs[k].strip()
+ if output.endswith(stop_str):
+ output = output[:-len(stop_str)]
+ output = output.strip()
+
+ ans_id = shortuuid.uuid()
+
+ ans_file.write(json.dumps({
+ "id": questions[count]["id"],
+ "image_id": questions[count]["image"],
+ "question": questions[count]['conversations'][0]['value'],
+ "predicted": output,
+ "ground_truth": questions[count]['conversations'][1]['value'],
+ "task": questions[count]['task'],
+ "linked_id": questions[count]['linked_id']
+ }) + "\n")
+ count=count+1
+ ans_file.flush()
+ ans_file.close()
+
+ output = [json.loads(q) for q in open((ans_file), "r")]
+ output = [{q['id']: q} for q in output]
+ with open(ans_file, 'r') as f:
+ json.dump(output, f)
+
+ agg_ans_file = ans_file.replace('.jsonl', '_agg.jsonl')
+ print("Raw Geochat output saved to ", ans_file)
+ print("Now parsing and aggregating votes for geovlm evaluation...")
+ aggregate_accuracy(ans_file, agg_ans_file)
+ print("Aggregated output saved to ", agg_ans_file)
+
+ accuracy_precision_recall(agg_ans_file, 'fmow')
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--model-path", type=str, default="facebook/opt-350m")
+ parser.add_argument("--model-base", type=str, default=None)
+ parser.add_argument("--image-folder", type=str, default="")
+ parser.add_argument("--question-file", type=str, default="tables/question.jsonl")
+ parser.add_argument("--answers-file", type=str, default="answer.jsonl")
+ parser.add_argument("--conv-mode", type=str, default="llava_v1")
+ parser.add_argument("--num-chunks", type=int, default=1)
+ parser.add_argument("--chunk-idx", type=int, default=0)
+ parser.add_argument("--temperature", type=float, default=0.2)
+ parser.add_argument("--top_p", type=float, default=None)
+ parser.add_argument("--num_beams", type=int, default=1)
+ parser.add_argument("--batch_size",type=int, default=1)
+ parser.add_argument("--cache-dir", type=str, default=None)
+ args = parser.parse_args()
+
+ eval_model(args)
diff --git a/videollava/eval/geochat_geovlm_infer.py b/videollava/eval/geochat_geovlm_infer.py
new file mode 100644
index 0000000000000000000000000000000000000000..cffb9ea0609b7be97981e33d65eeddd47f3a393f
--- /dev/null
+++ b/videollava/eval/geochat_geovlm_infer.py
@@ -0,0 +1,262 @@
+import argparse
+import torch
+import os
+import json
+from tqdm import tqdm
+import shortuuid
+import sys
+import random
+
+from geochat.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
+from geochat.conversation import conv_templates, SeparatorStyle
+from geochat.model.builder import load_pretrained_model
+from geochat.utils import disable_torch_init
+from geochat.mm_utils import tokenizer_image_token, get_model_name_from_path, KeywordsStoppingCriteria
+from eval_classification import *
+from datasets_into_geochat_format import s2looking_to_geochat_dataset_format, qfabric_semiconverted_to_geochat_dataset_format, xbd_to_geochat_dataset_format
+from geochat_s2looking_utils import evaluate_geochat_s2looking
+
+from PIL import Image
+import math
+import numpy as np
+
+def aggregate_accuracy(answers_file, output_file):
+ """
+ Parses geochat inference output and aggregates votes on single images
+ across an image sequence into the format needed for geovlm-style evaluation.
+
+ params:
+ - answers_file: path to the file containing geochat inference output
+ - output_file: path to the file where the aggregated output will be saved
+ """
+ with open(answers_file, 'r') as f:
+ answers = [json.loads(line) for line in f]
+
+ # dictionary that will contain parsed output
+ votes = {}
+
+ # parse answers so that predictions with the same geovlm_id
+ # are aggregated into a single item with 'predictions' containing
+ # a list of values. All other keys should be the same
+ for answer in answers:
+ id = answer['geovlm_id']
+ if id not in votes:
+ item = {}
+ item['predicted'] = [answer['predicted']]
+ item['ground_truth'] = answer['ground_truth']
+ item['task'] = answer['task']
+ item['original_input_polygon'] = answer['original_input_polygon']
+ item['question'] = answer['question']
+ item['id'] = answer['id']
+ votes[id] = item
+ else:
+ votes[id]['predicted'].append(answer['predicted'])
+
+ # implement voting so that each list in 'predicted' attribute
+ # is reduced to the most common value
+ for linked_id, predicted_dict in votes.items():
+ predicted = predicted_dict['predicted']
+ unique, counts = np.unique(predicted, return_counts=True)
+ index = np.argmax(counts)
+ votes[linked_id]['predicted'] = unique[index]
+
+ with open(output_file, 'w') as f:
+ json.dump(votes, f)
+
+
+def split_list(lst, n):
+ """Split a list into n (roughly) equal-sized chunks"""
+ chunk_size = math.ceil(len(lst) / n) # integer division
+ return [lst[i:i+chunk_size] for i in range(0, len(lst), chunk_size)]
+
+
+def get_chunk(lst, n, k):
+ chunks = split_list(lst, n)
+ return chunks[k]
+
+
+def eval_model(args):
+ print(args)
+ print()
+
+ answers_file = os.path.expanduser(args.answers_file)
+ os.makedirs(os.path.dirname(answers_file), exist_ok=True)
+
+ try:
+ with open(args.question_file, 'r') as f:
+ questions = json.load(f)
+ except:
+ questions = [json.loads(q) for q in open(os.path.expanduser(args.question_file), "r")]
+
+ if args.end_ind is not None:
+ questions = questions[args.start_ind:args.end_ind]
+ else:
+ questions = questions[args.start_ind:]
+ print("start ind: ", args.start_ind)
+ print("end ind: ", args.end_ind)
+
+ # check if the answers file alreay exists
+ if not os.path.exists(answers_file) or args.rerun==True:
+ print('Running inference...')
+ image = Image.open(image_file)
+
+ if args.dataset_size:
+ # randomly sample dataset_size number of questions
+ questions = random.sample(questions, args.dataset_size)
+
+ os.makedirs(os.path.dirname(answers_file), exist_ok=True)
+ ans_file = open(answers_file, "w")
+
+
+ # Model
+ disable_torch_init()
+ model_path = os.path.expanduser(args.model_path)
+ model_name = get_model_name_from_path(model_path)
+ tokenizer, model, image_processor, context_len = load_pretrained_model(args.model_path, args.model_base, model_name, cache_dir=args.cache_dir)
+
+ for i in tqdm(range(0,len(questions),args.batch_size)):
+ input_batch=[]
+ input_image_batch=[]
+ count=i
+ image_folder=[]
+ batch_end = min(i + args.batch_size, len(questions))
+
+ for j in range(i,batch_end):
+ image_file=questions[j]['image']
+ qs=questions[j]['conversations'][0]['value']
+
+ # TODO do we keep that?
+
+ # if model.config.mm_use_im_start_end:
+ # qs = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + '\n' + qs
+ # print("start end token")
+ # else:
+ # qs = DEFAULT_IMAGE_TOKEN + '\n' + qs
+
+ conv = conv_templates[args.conv_mode].copy()
+ conv.append_message(conv.roles[0], qs)
+ conv.append_message(conv.roles[1], None)
+ prompt = conv.get_prompt()
+
+ print(prompt)
+
+ input_ids = tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).cuda()
+ input_batch.append(input_ids)
+
+ image = Image.open(os.path.join(args.image_folder, image_file))
+
+ image_folder.append(image)
+
+ stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
+ keywords = [stop_str]
+ stopping_criteria = KeywordsStoppingCriteria(keywords, tokenizer, input_ids)
+
+ max_length = max(tensor.size(1) for tensor in input_batch)
+
+ final_input_list = [torch.cat((torch.zeros((1,max_length - tensor.size(1)), dtype=tensor.dtype,device=tensor.get_device()), tensor),dim=1) for tensor in input_batch]
+ final_input_tensors=torch.cat(final_input_list,dim=0)
+ image_tensor_batch = image_processor.preprocess(image_folder,crop_size ={'height': 504, 'width': 504},size = {'shortest_edge': 504}, return_tensors='pt')['pixel_values']
+
+ with torch.inference_mode():
+ output_ids = model.generate( final_input_tensors, images=image_tensor_batch.half().cuda(), do_sample=False , temperature=args.temperature, top_p=args.top_p, num_beams=1, max_new_tokens=256,length_penalty=2.0, use_cache=True)
+
+ input_token_len = final_input_tensors.shape[1]
+ n_diff_input_output = (final_input_tensors != output_ids[:, :input_token_len]).sum().item()
+ if n_diff_input_output > 0:
+ print(f'[Warning] {n_diff_input_output} output_ids are not the same as the input_ids')
+ outputs = tokenizer.batch_decode(output_ids[:, input_token_len:], skip_special_tokens=True)
+ for k in range(0,len(final_input_list)):
+ output = outputs[k].strip()
+ if output.endswith(stop_str):
+ output = output[:-len(stop_str)]
+ output = output.strip()
+
+ ans_id = shortuuid.uuid()
+
+ if args.dataset == 'qfabric':
+ ans_file.write(json.dumps({
+ "id": questions[count]["id"],
+ "image_id": questions[count]["image"],
+ "question": questions[count]['conversations'][0]['value'],
+ "predicted": output,
+ "ground_truth": questions[count]['conversations'][1]['value'],
+ "task": questions[count]['task'],
+ "original_input_polygon": questions[count]['original_input_polygon'],
+ "geovlm_id": questions[count]['geovlm_id']
+ }) + "\n")
+ elif args.dataset == 's2looking':
+ ans_file.write(json.dumps({
+ questions[count]["id"] : {
+ "image_id": questions[count]["image"],
+ "question": questions[count]['conversations'][0]['value'],
+ "predicted": output,
+ "task": questions[count]['task'],
+ "original_input_polygon": questions[count]['original_input_polygon'],
+ "geovlm_id": questions[count]['geovlm_id'],
+ "original_question": questions[count]['conversations'][0]['value'],
+ "original_answer": questions[count]['conversations'][1]['value']
+ }}) + "\n")
+ elif args.dataset == 'xbd':
+ ans_file.write(json.dumps({
+ questions[count]["id"] : {
+ "image_id": questions[count]["image"],
+ "question": questions[count]['conversations'][0]['value'],
+ "predicted": output,
+ "task": questions[count]['task'],
+ "original_input_polygon": questions[count]['original_input_polygon'],
+ "original_question": questions[count]['conversations'][0]['value'],
+ "original_answer": questions[count]['conversations'][1]['value']
+ }}) + "\n")
+
+ count=count+1
+ ans_file.flush()
+ ans_file.close()
+
+ agg_ans_file = args.answers_file.replace('.json', '_agg.json')
+ print("Raw Geochat output saved to ", args.answers_file)
+
+ # determine the split from args.question_file
+ if 'test' in args.question_file:
+ split = 'test'
+ elif 'val' or 'valid' or 'validation' in args.question_file:
+ split = 'val'
+ elif 'train' in args.question_file:
+ split = 'train'
+ else:
+ raise ValueError("Split not found in question file name")
+
+ print("Now parsing and aggregating votes for geovlm evaluation...")
+ if args.dataset == 'qfabric':
+ aggregate_accuracy(args.answers_file, agg_ans_file)
+ print("Aggregated output saved to ", agg_ans_file)
+
+ classification_segmentation(agg_ans_file, 'qfabric')
+ elif args.dataset == 's2looking':
+ evaluate_geochat_s2looking(args.answers_file, args.question_file, split)
+ elif args.dataset == 'xbd':
+ classification_segmentation(agg_ans_file, 'xbd')
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--model-path", type=str, default="facebook/opt-350m")
+ parser.add_argument("--model-base", type=str, default=None)
+ parser.add_argument("--image-folder", type=str, default="")
+ parser.add_argument("--question-file", type=str, default="tables/question.jsonl")
+ parser.add_argument("--answers-file", type=str, default="answer.jsonl")
+ parser.add_argument("--conv-mode", type=str, default="llava_v1")
+ parser.add_argument("--num-chunks", type=int, default=1)
+ parser.add_argument("--chunk-idx", type=int, default=0)
+ parser.add_argument("--temperature", type=float, default=0.2)
+ parser.add_argument("--top_p", type=float, default=None)
+ parser.add_argument("--num_beams", type=int, default=1)
+ parser.add_argument("--batch_size",type=int, default=1)
+ parser.add_argument("--start-ind", type=int, default=0)
+ parser.add_argument("--end-ind", type=int, default=None)
+ parser.add_argument("--cache-dir", type=str, default=None)
+ parser.add_argument("--dataset", type=str)
+ parser.add_argument("--rerun", type=bool, default=False)
+ parser.add_argument("--dataset_size", type=int, default=None)
+ args = parser.parse_args()
+
+ eval_model(args)
diff --git a/videollava/eval/geochat_referring_2.py b/videollava/eval/geochat_referring_2.py
new file mode 100644
index 0000000000000000000000000000000000000000..0e70c25a099ed4960744d82e57553717fd66463f
--- /dev/null
+++ b/videollava/eval/geochat_referring_2.py
@@ -0,0 +1,459 @@
+"""
+Code adapted from calculate_mean_ap.py
+author: Timothy C. Arlen
+date: 28 Feb 2018
+"""
+
+import sys
+from os.path import dirname, abspath
+sys.path.append(dirname(dirname(dirname(dirname(abspath(__file__))))))
+
+from collections import defaultdict
+import numpy as np
+import json
+import ast
+import re
+import cv2
+from shapely import wkt, Polygon, box
+from infer_utils import create_mask
+from matplotlib.path import Path
+from tqdm import tqdm
+
+from eval_referring import referring_expression
+import matplotlib.pyplot as plt
+import time
+import math
+from matplotlib.path import Path
+
+def convert_geochat_string(build, img_size=256):
+ """
+ Convert the raw str geochat output {<40><89><56><100>|<57>}, {<0><89><56><100>|<57>}
+ to a list of rotated bboxes.
+ """
+ build = build.strip('{}')
+ bbox_segments = build.split("}{")
+ # Regular expression to find all numbers inside angle brackets
+ pattern = r"<(\d+)>"
+
+ # Extract numbers, convert them to integers, and collect into a list
+ bboxes = [
+ list(map(int, re.findall(pattern, segment)))
+ for segment in bbox_segments
+ ]
+
+ rotated_bboxes = []
+ for bbox in bboxes:
+ try:
+ xmin, ymin, xmax, ymax, angle = [float(v) for v in bbox]
+ except:
+ print("Warning - Malformed bbox: ", bbox)
+ print("Original string: ", build)
+ print()
+ continue
+
+ # Convert percentages to pixel coordinates
+ xmin = xmin * img_size / 100
+ ymin = ymin * img_size / 100
+ xmax = xmax * img_size / 100
+ ymax = ymax * img_size / 100
+
+ # Calculate rectangle dimensions
+ rect_width = xmax - xmin
+ rect_height = ymax - ymin
+ center_x = xmin + rect_width / 2
+ center_y = ymin + rect_height / 2
+
+ # Calculate corners before rotation
+ corners = np.array([
+ [xmin, ymin],
+ [xmax, ymin],
+ [xmax, ymax],
+ [xmin, ymax]
+ ])
+
+ # Rotate corners
+ angle_rad = math.radians(angle)
+ cos_angle = math.cos(angle_rad)
+ sin_angle = math.sin(angle_rad)
+ rotated_corners = []
+ for x, y in corners:
+ tx = x - center_x
+ ty = y - center_y
+ rotated_x = tx * cos_angle - ty * sin_angle + center_x
+ rotated_y = tx * sin_angle + ty * cos_angle + center_y
+ rotated_corners.append([rotated_x, rotated_y])
+
+ rotated_bboxes.append(np.array(rotated_corners))
+
+ return rotated_bboxes
+
+def create_geochat_mask(buildings, img_size=(256, 256)):
+ """
+ Given a list of buildings in an image, this function
+ - creates an img_size * img_size numpy array for the image
+ - returns the mask for all buildings
+ Input:
+ - buildings: List of geochat strings representing buildings
+ - img_size: Tuple indicating the size of the image (height, width)
+ """
+ mask = np.zeros(img_size, np.uint8)
+
+ # Fill in with ones the pixels that are inside the buildings (rotated bboxes)
+ for bbox in buildings:
+ path = Path(bbox)
+ x, y = np.meshgrid(np.arange(img_size[1]), np.arange(img_size[0]))
+ points = np.vstack((x.flatten(), y.flatten())).T
+ mask[path.contains_points(points).reshape(img_size)] = 1
+
+ return mask
+
+def calc_iou_individual(pred_box, gt_box):
+ """Calculate IoU of single predicted and ground truth box
+ Args:
+ pred_box (list of floats): location of predicted object as
+ [xmin, ymin, xmax, ymax]
+ gt_box (list of floats): location of ground truth object as
+ [xmin, ymin, xmax, ymax]
+ Returns:
+ float: value of the IoU for the two boxes.
+ Raises:
+ AssertionError: if the box is obviously malformed
+ """
+ x1_t, y1_t, x2_t, y2_t = gt_box
+ try:
+ x1_p, y1_p, x2_p, y2_p = pred_box
+ except:
+ return 0.0
+
+ if (x1_p > x2_p) or (y1_p > y2_p):
+ print("Prediction box is malformed? pred box: {}".format(pred_box))
+ if (x1_t > x2_t) or (y1_t > y2_t):
+ print("Ground Truth box is malformed? true box: {}".format(gt_box))
+
+ if (x2_t < x1_p or x2_p < x1_t or y2_t < y1_p or y2_p < y1_t):
+ return 0.0
+
+ far_x = np.min([x2_t, x2_p])
+ near_x = np.max([x1_t, x1_p])
+ far_y = np.min([y2_t, y2_p])
+ near_y = np.max([y1_t, y1_p])
+
+ inter_area = (far_x - near_x + 1) * (far_y - near_y + 1)
+ true_box_area = (x2_t - x1_t + 1) * (y2_t - y1_t + 1)
+ pred_box_area = (x2_p - x1_p + 1) * (y2_p - y1_p + 1)
+ iou = inter_area / (true_box_area + pred_box_area - inter_area)
+
+ return iou
+
+def calc_iou_individual_rotated(pred_box, gt_box):
+ """Calculate IoU of single predicted and ground truth box
+ Args:
+ pred_box (list of floats): location of predicted object as
+ [[x1, y1], [x2, y2], [x3, y3], [x4, y4]]
+ gt_box (list of floats): location of ground truth object as
+ [[x1, y1], [x2, y2], [x3, y3], [x4, y4]]
+ Returns:
+ float: value of the IoU for the two boxes.
+ Raises:
+ AssertionError: if the box is obviously malformed
+ """
+ try:
+ pred_box = np.array(pred_box)
+ gt_box = np.array(gt_box)
+ except:
+ return 0.0
+ if len(pred_box) == 4:
+ pred_box = [[pred_box[0], pred_box[1]], [pred_box[2], pred_box[1]], [pred_box[2], pred_box[3]], [pred_box[0], pred_box[3]]]
+ if len(gt_box) == 4:
+ gt_box = [[gt_box[0], gt_box[1]], [gt_box[2], gt_box[1]], [gt_box[2], gt_box[3]], [gt_box[0], gt_box[3]]]
+ pred_box = np.array(pred_box)
+ gt_box = np.array(gt_box)
+ pred_box = pred_box.reshape(4, 2)
+ gt_box = gt_box.reshape(4, 2)
+ pred_polygon = Polygon(pred_box)
+ gt_polygon = Polygon(gt_box)
+ intersection = pred_polygon.intersection(gt_polygon).area
+ union = pred_polygon.union(gt_polygon).area
+ iou = intersection / union
+ return iou
+
+ # try:
+ # pred_box = np.array(pred_box)
+ # gt_box = np.array(gt_box)
+ # except:
+ # return 0.0
+
+ # pred_box = pred_box.reshape(4, 2)
+ # gt_box = gt_box.reshape(4, 2)
+
+ # pred_polygon = Polygon(pred_box)
+ # gt_polygon = Polygon(gt_box)
+
+ # intersection = pred_polygon.intersection(gt_polygon).area
+ # union = pred_polygon.union(gt_polygon).area
+
+ # iou = intersection / union
+
+ # plt.figure()
+ # plt.plot(*pred_polygon.exterior.xy, color='r', label='pred')
+ # plt.plot(*gt_polygon.exterior.xy, color='b', label='gt')
+ # plt.legend()
+ # plt.title(f"IoU: {iou}")
+ # plt.show()
+ # plt.savefig("iou.png")
+ # time.sleep(1)
+ # plt.close()
+
+ return iou
+
+
+def get_single_image_bound_results(gt_wkts, pred_geochat_string, img_size=256):
+ """
+ Calculates upper bound and lower bound number of true_pos, false_pos, false_neg from single batch of boxes.
+ Args:
+ gt_wkts (list of strs): list of wkt strings of input polygons, scaled to raw pixel value
+ pred_boxes (list of lists): list of list of boxes, where each box is formatted
+ as [x_min, y_min, x_max, y_max] on scale from 0-100
+ img_size (int): dimensions of the image. defaults to 256.
+ Returns:
+ tuple of dicts: true positives (int), false positives (int), false negatives (int)
+ """
+ if isinstance(gt_wkts, str):
+ gt_polygons = [wkt.loads(gt_wkts)]
+ else:
+ gt_polygons = [wkt.loads(gt_wkt) for gt_wkt in gt_wkts]
+
+ # # Needs fixing for auxiliary
+ # if len(gt_polygons) == 0:
+ # false_neg = np.sum(gt_mask)
+ # ub_stats= {'true_pos': 0, 'false_pos': 0, 'false_neg': false_neg, 'intersection':0, 'union':false_neg}
+ # lb_stats = {'true_pos': 0, 'false_pos': 0, 'false_neg': false_neg, 'intersection':0, 'union':false_neg}
+ # return lb_stats, ub_stats
+
+ lb_preds = convert_geochat_string(pred_geochat_string, img_size)
+ # get mask of all gt_polygons and lb_preds
+ gt_mask = create_mask(gt_polygons, (img_size, img_size))
+ lb_preds_mask = create_geochat_mask(lb_preds, (img_size, img_size))
+
+ # get lower bound intersection and union masks
+ intersection = np.logical_and(gt_mask, lb_preds_mask)
+ union = np.logical_or(gt_mask, lb_preds_mask)
+
+ # compute lb metrics
+ # lower_bound_iou = np.sum(intersection) / np.sum(union)
+ fp = np.sum(np.logical_and(lb_preds_mask, np.logical_not(gt_mask)))
+ tp = np.sum(np.logical_and(lb_preds_mask, gt_mask))
+ fn = np.sum(np.logical_and(np.logical_not(lb_preds_mask), gt_mask))
+ lb_stats = {'true_pos': tp, 'false_pos': fp, 'false_neg': fn, 'intersection': np.sum(intersection), 'union': np.sum(union)}
+
+ # get upper bound intersection and union masks
+ ub_pred_mask = np.logical_and(gt_mask, lb_preds_mask)
+ intersection = np.logical_and(ub_pred_mask, gt_mask)
+ union = np.logical_or(gt_mask, ub_pred_mask)
+
+ # compute ub metrics
+ # upper_bound_iou = np.sum(intersection) / np.sum(union)
+ ub_fp = np.sum(np.logical_and(ub_pred_mask, np.logical_not(gt_mask)))
+ ub_tp = np.sum(np.logical_and(ub_pred_mask, gt_mask))
+ ub_fn = np.sum(np.logical_and(np.logical_not(ub_pred_mask), gt_mask))
+ ub_stats = {'true_pos': ub_tp, 'false_pos': ub_fp, 'false_neg': ub_fn, 'intersection': np.sum(intersection), 'union': np.sum(union)}
+
+ return lb_stats, ub_stats
+
+def get_geochat_dataset(image_id):
+ if image_id.startswith("P"):
+ dataset = "SOTA"
+ elif image_id.startswith("train"):
+ dataset = "FAST"
+ else:
+ dataset = "SIOR"
+ return dataset
+
+def get_single_image_results(gt_boxes, pred_boxes, iou_thr):
+ """Calculates number of true_pos, false_pos, false_neg from single batch of boxes.
+ Args:
+ gt_boxes (list of list of floats): list of locations of ground truth
+ objects as [[x1,y1], [x2,y2], ...]
+ pred_boxes (dict): dict of dicts of 'boxes'
+ [[x1,y1], [x2,y2], ...]
+ iou_thr (float): value of IoU to consider as threshold for a
+ true prediction.
+ Returns:
+ dict: true positives (int), false positives (int), false negatives (int)
+ """
+
+ all_pred_indices = range(len(pred_boxes))
+ all_gt_indices = range(len(gt_boxes))
+ if len(all_pred_indices) == 0:
+ tp = 0
+ fp = 0
+ fn = len(gt_boxes)
+ return {'true_pos': tp, 'false_pos': fp, 'false_neg': fn}
+ if len(all_gt_indices) == 0:
+ tp = 0
+ fp = len(pred_boxes)
+ fn = 0
+ return {'true_pos': tp, 'false_pos': fp, 'false_neg': fn}
+
+ gt_idx_thr = []
+ pred_idx_thr = []
+ ious = []
+ for ipb, pred_box in enumerate(pred_boxes):
+ for igb, gt_box in enumerate(gt_boxes):
+ iou = calc_iou_individual_rotated(pred_box, gt_box)
+ if iou > iou_thr:
+ gt_idx_thr.append(igb)
+ pred_idx_thr.append(ipb)
+ ious.append(iou)
+
+ args_desc = np.argsort(ious)[::-1]
+ if len(args_desc) == 0:
+ # No matches
+ tp = 0
+ fp = len(pred_boxes)
+ fn = len(gt_boxes)
+ else:
+ gt_match_idx = []
+ pred_match_idx = []
+ for idx in args_desc:
+ gt_idx = gt_idx_thr[idx]
+ pr_idx = pred_idx_thr[idx]
+ # If the boxes are unmatched, add them to matches
+ if (gt_idx not in gt_match_idx) and (pr_idx not in pred_match_idx):
+ gt_match_idx.append(gt_idx)
+ pred_match_idx.append(pr_idx)
+ tp = len(gt_match_idx)
+ fp = len(pred_boxes) - len(pred_match_idx)
+ fn = len(gt_boxes) - len(gt_match_idx)
+
+ return {'true_pos': tp, 'false_pos': fp, 'false_neg': fn}
+
+
+def calc_precision_recall(img_results):
+ """Calculates precision and recall from the set of images
+ Args:
+ img_results (dict): dictionary formatted like:
+ {
+ 'img_id1': {'true_pos': int, 'false_pos': int, 'false_neg': int},
+ 'img_id2': ...
+ ...
+ }
+ Returns:
+ tuple: of floats of (precision, recall)
+ """
+ true_pos = 0; false_pos = 0; false_neg = 0
+ for _, res in img_results.items():
+ true_pos += res['true_pos']
+ false_pos += res['false_pos']
+ false_neg += res['false_neg']
+
+ try:
+ precision = true_pos/(true_pos + false_pos)
+ except ZeroDivisionError:
+ precision = 0.0
+ try:
+ recall = true_pos/(true_pos + false_neg)
+ except ZeroDivisionError:
+ recall = 0.0
+
+ return (precision, recall)
+
+
+DIMENSIONS = {'FAST': 600,
+ 'SIOR': 800,
+ 'SOTA': 1024}
+
+
+def referring_expression(answer_path, dataset, verbose=False, saving_path_root=None, img_size=256):
+ # Replace with the path to the answers file
+ if type(answer_path) == dict:
+ results = answer_path
+ else:
+ with open(answer_path) as json_data:
+ results = json.load(json_data)
+
+ img_results = {}
+ ub_results = {}
+ lb_results = {}
+ num_bboxes = 0
+ # Loop over results and get precision, recall overall
+ for id, result in tqdm(results.items()):
+
+ if dataset == "geochat_xbd":
+ pred = result['predicted']
+
+ dataset = get_geochat_dataset(id)
+ img_size = (DIMENSIONS[dataset])
+ pred = convert_geochat_string(pred, img_size)
+
+ ground_truth = result['ground_truth']
+ ground_truth = np.array(ground_truth)
+ num_bboxes += len(ground_truth)
+
+ img_results[id] = get_single_image_results(ground_truth, pred, iou_thr=0.5)
+
+ continue
+
+ try:
+ if 'referring_expression' not in result['task']:
+ continue # no bounding box outputs for temporal_referring_expression
+ except:
+ pass
+
+ # TODO: LOOP THROUGH IDENTIFY TASKS/QUESTIONS IN THE DATASET
+
+ # TODO: HANDLE WHEN THERE ARE NO BOUNDING BOXES IN GROUND TRUTH for auxiliary tasks
+ if not result['original_input_polygon']:
+ first_open_bracket_ind = result["predicted"].find("{")
+ last_close_bracket_ind = result["predicted"].rfind("}")
+ if last_close_bracket_ind != -1 and first_open_bracket_ind != -1:
+ parsed_predicted = result["predicted"][first_open_bracket_ind:last_close_bracket_ind+1]
+ else:
+ parsed_predicted = ""
+ predicted_boxes = convert_geochat_string(parsed_predicted)
+ # If ground truth contains no boxes: all predictions are false positives
+ false_pos = len(predicted_boxes)
+ false_pos_pixels = np.sum(create_geochat_mask(predicted_boxes))
+ img_results[id] = {'true_pos': 0, 'false_pos': false_pos, 'false_neg': 0, 'intersection':0, 'union':false_pos_pixels}
+ ub_results[id] = {'true_pos': 0, 'false_pos': false_pos_pixels, 'false_neg': 0, 'intersection':0, 'union':false_pos_pixels}
+ lb_results[id] = {'true_pos': 0, 'false_pos': false_pos_pixels, 'false_neg': 0, 'intersection':0, 'union':false_pos_pixels}
+ continue
+ else: # Ground truth contains boxes: find predicted Geochat boxes
+ first_open_bracket_ind = result["predicted"].find("{")
+ last_close_bracket_ind = result["predicted"].rfind("}")
+ if last_close_bracket_ind != -1 and first_open_bracket_ind != -1:
+ parsed_predicted = result["predicted"][first_open_bracket_ind:last_close_bracket_ind+1]
+ else:
+ parsed_predicted = ""
+ gt_wkts = result['original_input_polygon']
+ lb_results[id], ub_results[id] = get_single_image_bound_results(gt_wkts, parsed_predicted)
+
+ if len(ub_results) != 0:
+ ub_intersection = np.sum([res['intersection'] for res in ub_results.values()])
+ ub_union = np.sum([res['union'] for res in ub_results.values()])
+ lb_intersection = np.sum([res['intersection'] for res in lb_results.values()])
+ lb_union = np.sum([res['union'] for res in lb_results.values()])
+ print("Upper bound IOU: ", ub_intersection / ub_union if ub_union != 0 else 0)
+ print("Lower bound IOU: ", lb_intersection / lb_union if lb_union != 0 else 0)
+ ub_precision, ub_recall = calc_precision_recall(ub_results)
+ lb_precision, lb_recall = calc_precision_recall(lb_results)
+ print('Lower bound precision: ', lb_precision)
+ print('Lower bound recall: ', lb_recall)
+ print("Upper bound F1: ", 2 * (ub_precision * ub_recall) / (ub_precision + ub_recall) if (ub_precision + ub_recall) != 0 else 0)
+ print("Lower bound F1: ", 2 * (lb_precision * lb_recall) / (lb_precision + lb_recall) if (lb_precision + lb_recall) != 0 else 0)
+
+ print("Acc@0.5: ", np.sum([res['true_pos'] for res in img_results.values()]) / num_bboxes)
+
+ if type(answer_path) == dict:
+ return
+
+ if saving_path_root:
+ with open(f"{saving_path_root}/referring_expression_scores.json", 'w') as f:
+ json.dump(img_results, f)
+
+if __name__ == '__main__':
+ answer_path = "scripts/geovlm/eval/xBD/answers/ckpt14000-geochat-bench_interleave_test.json"
+ referring_expression(answer_path, dataset="geochat_xbd")
+ #answer_path = "scripts/geochat/eval/xBD/geochat_xbd_test_auxiliary_dict.json"
+ # referring_expression(answer_path, dataset="xbd")
+
diff --git a/videollava/eval/geochat_s2looking_utils.py b/videollava/eval/geochat_s2looking_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..8e7380c33b4498c4bc01ae35ab2da691bad9e748
--- /dev/null
+++ b/videollava/eval/geochat_s2looking_utils.py
@@ -0,0 +1,400 @@
+
+import json
+import numpy as np
+import cv2
+import re
+from eval_referring import referring_expression
+import matplotlib.pyplot as plt
+from shapely import wkt
+import time
+import math
+from matplotlib.path import Path
+from eval_classification import accuracy_precision_recall
+
+
+def convert_geochat_string(build, img_size=256):
+ """
+ convert the raw str geochat output {<40><89><56><100>|<57>}, {<0><89><56><100>|<57>}
+ to a list of rotated bboxes
+ """
+ build = build.strip('{}')
+ bbox_segments = build.split("}{")
+
+ # Regular expression to find all numbers inside angle brackets
+ pattern = r"<(\d+)>"
+
+ # Extract numbers, convert them to integers, and collect into a list
+ bboxes = [
+ list(map(int, re.findall(pattern, segment)))
+ for segment in bbox_segments]
+
+ rotated_bboxes = []
+ for bbox in bboxes:
+ try:
+ xmin, ymin, xmax, ymax, angle = [float(v) for v in bbox]
+ except:
+ pass
+
+ # Convert percentages to pixel coordinates
+ xmin = xmin * img_size / 100
+ ymin = ymin * img_size / 100
+ xmax = xmax * img_size / 100
+ ymax = ymax * img_size / 100
+
+ # Calculate rectangle dimensions
+ rect_width = xmax - xmin
+ rect_height = ymax - ymin
+ center_x = xmin + rect_width / 2
+ center_y = ymin + rect_height / 2
+
+ # Calculate corners before rotation
+ corners = np.array([
+ [xmin, ymin],
+ [xmax, ymin],
+ [xmax, ymax],
+ [xmin, ymax]
+ ])
+
+ # Rotate corners
+ angle_rad = math.radians(angle)
+ cos_angle = math.cos(angle_rad)
+ sin_angle = math.sin(angle_rad)
+ rotated_corners = []
+ for x, y in corners:
+ tx = x - center_x
+ ty = y - center_y
+ rotated_x = tx * cos_angle - ty * sin_angle + center_x
+ rotated_y = tx * sin_angle + ty * cos_angle + center_y
+ rotated_corners.append([rotated_x, rotated_y])
+
+ rotated_bboxes.append(np.array(rotated_corners))
+
+ return rotated_bboxes
+
+
+def get_changed_buildings(build1, build2, img_size=256, task=None):
+ """
+ Given a list of predicted buildings in image 1 and image 2, this function
+ - creates two img_size * img_size numpy arrays for both of the images
+ - gets the mask differences between the two numpy arrays
+ - returns a list of bounding boxes that reflect those differences, as well as the difference mask
+ Input:
+ - build1: [[x,y],[x,y],[x,y],[x,y]] array of four x,y coordinates of the bounding box of a building
+ - task can be either None, constructed or destructed
+ Note: those bboxes can be rotated
+ """
+ image1 = np.zeros((img_size, img_size), np.uint8)
+ image2 = np.zeros((img_size, img_size), np.uint8)
+
+ build1 = convert_geochat_string(build1)
+ build2 = convert_geochat_string(build2)
+
+ # fill in with ones the pixels that are inside the rotated bboxes
+ for b in build1:
+ path = Path(b)
+ x, y = np.meshgrid(np.arange(img_size), np.arange(img_size))
+ points = np.vstack((x.flatten(), y.flatten())).T
+ image1[path.contains_points(points).reshape(img_size, img_size)] = 1
+
+ for b in build2:
+ path = Path(b)
+ x, y = np.meshgrid(np.arange(img_size), np.arange(img_size))
+ points = np.vstack((x.flatten(), y.flatten())).T
+ image2[path.contains_points(points).reshape(img_size, img_size)] = 1
+
+ # xor between the two images
+ if task == None:
+ diff = cv2.bitwise_xor(image1, image2)
+ elif task == "constructed":
+ # if the task is constructed, we want to find the pixels that are in image2 but not in image1
+ diff = cv2.bitwise_and(image2, cv2.bitwise_not(image1))
+ elif task == "destructed":
+ # if the task is destructed, we want to find the pixels that are in image1 but not in image2
+ diff = cv2.bitwise_and(image1, cv2.bitwise_not(image2))
+
+ # get the bounding boxes of the difference pixels
+ contours, _ = cv2.findContours(diff, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+ bboxes = []
+ for contour in contours:
+ x, y, w, h = cv2.boundingRect(contour)
+ x, y, w, h = y, x, h, w
+ bboxes.append([x, y, x+w, y+h])
+
+ return bboxes, diff
+
+def get_canonical_answer_dataset(answers):
+ """
+ This function creates a new dataset with questions and answers for geochat, ready to parse into the evaluation metrics."""
+
+ new_dataset = {}
+
+ for key, answer in answers.items():
+ num, quadrant, geovlmid = key.split("_")
+ task = answer['task']
+ if geovlmid == "1" in task:
+ continue
+
+ # find the paired image
+ id2 = num + "_" + quadrant + "_" + "1"
+ answer1 = answers[key]
+ try:
+ answer2 = answers[id2]
+ except:
+ print(f"The associated image to {key} wasn't present in the dataset")
+ continue
+
+ # get the pixel diff boxes
+ change_bboxes, mask = get_changed_buildings(answer1['predicted'], answer2['predicted'])
+
+ # create the new dataset adapted for running metrics on it
+ new_line = {}
+
+ new_line['predicted'] = ""
+ if len(change_bboxes)>0:
+ for bbox in change_bboxes:
+ new_line['predicted'] += str(bbox) + ", "
+ new_line['predicted'] = new_line['predicted'][:-2]
+ new_line['predicted_mask'] = mask.tolist()
+
+ new_line['ground_truth'] = answer1['original_answer']
+ new_line['question'] = answer1['original_question']
+ new_line['task'] = answer1['task']
+ new_line['original_input_polygon'] = answer1['original_input_polygon']
+
+ new_key = num + "_" + quadrant
+ new_dataset[new_key] = new_line
+
+ return new_dataset
+
+def postprocess_auxiliary_qa(key, answer, original_answers):
+ new_line = {}
+ new_line['ground_truth'] = answer['ground_truth']
+ new_line['question'] = answer['question']
+ new_line['task'] = answer['task']
+ new_line['original_input_polygon'] = answer['original_input_polygon']
+
+ # retrieve the original 2 anwers
+ answer1 = original_answers[key + '_0']['predicted']
+ answer2 = original_answers[key + '_1']['predicted']
+
+ # retrieve the task (construction or destruction)
+ setting = None
+ if "constructed" or "built" in answer['original_question']:
+ setting = "constructed"
+ elif "destructed" or "torn down" in answer['original_question']:
+ setting = "destructed"
+ else:
+ print("The task is not recognized")
+ print("Original question: ", answer['original_question'])
+ print()
+
+ # get the pixel diff boxes
+ change_bboxes, mask = get_changed_buildings(answer1, answer2, task=setting)
+
+ new_line['predicted_mask'] = mask.tolist()
+ contours, hierarchy = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+
+ found_convex_polygon = False
+ for contour in contours:
+ # check if the contour is a bounding box (4 vertices, rectangle shape)
+ epsilon = 0.04 * cv2.arcLength(contour, True)
+ approx = cv2.approxPolyDP(contour, epsilon, True)
+ if len(approx) == 4:
+ found_convex_polygon = True
+ break
+
+ if found_convex_polygon:
+ new_line['predicted'] = "Yes"
+ else:
+ new_line['predicted'] = "No"
+
+ return new_line
+
+
+def postprocess_auxiliary_region_qa(key, answer, original_answers, img_size=256):
+ """
+ There is a bbox in the input polygon, we need to find the changed buildings in the image
+ inside that bbox
+ """
+ new_line = {}
+ new_line['ground_truth'] = answer['ground_truth']
+ new_line['question'] = answer['question']
+ new_line['task'] = answer['task']
+ new_line['original_input_polygon'] = answer['original_input_polygon']
+
+ # retrieve the original 2 anwers
+ answer1 = original_answers[key + '_0']['predicted']
+ answer2 = original_answers[key + '_1']['predicted']
+
+ # get the pixel diff boxes
+ change_bboxes, mask = get_changed_buildings(answer1, answer2)
+
+ # get the input bbox
+ question = new_line['question']
+ # find the positions of '[' and ']'
+ start = question.find('[')
+ end = question.find(']')
+ bbox = question[start+1:end].split(',')
+ bbox = [int(b) * img_size // 100 for b in bbox]
+
+ # adapt the mask, put 0s outside the bbox
+ mask[:bbox[0], :] = 0
+ mask[bbox[2]:, :] = 0
+ mask[:, :bbox[1]] = 0
+ mask[:, bbox[3]:] = 0
+
+ # predict yes or no if there is a convex polygon in the mask
+ found_convex_polygon = False
+ contours, hierarchy = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+ for contour in contours:
+ # check if the contour is a bounding box (4 vertices, rectangle shape)
+ epsilon = 0.04 * cv2.arcLength(contour, True)
+ approx = cv2.approxPolyDP(contour, epsilon, True)
+ if len(approx) == 4:
+ found_convex_polygon = True
+ break
+
+ new_line['predicted_mask'] = mask.tolist()
+
+ if found_convex_polygon:
+ new_line['predicted'] = "Yes"
+ else:
+ new_line['predicted'] = "No"
+
+ return new_line
+
+
+def postprocess_auxiliary_referring(key, answer, original_answers):
+ new_line = {}
+ new_line['ground_truth'] = answer['ground_truth']
+ new_line['question'] = answer['question']
+ new_line['task'] = answer['task']
+ new_line['original_input_polygon'] = answer['original_input_polygon']
+
+ # retrieve the original 2 anwers
+ answer1 = original_answers[key + '_0']['predicted']
+ answer2 = original_answers[key + '_1']['predicted']
+
+ # retrieve the task (construction or destruction)
+ setting = None
+ if "constructed" or "built" in answer['original_question']:
+ setting = "constructed"
+ elif "destructed" or "torn down" in answer['original_question']:
+ setting = "destructed"
+ else:
+ print("The task is not recognized")
+ print("Original question: ", answer['original_question'])
+ print()
+
+ # get the pixel diff boxes
+ change_bboxes, mask = get_changed_buildings(answer1, answer2, task=setting)
+
+ new_line['predicted_mask'] = mask.tolist()
+ new_line['predicted'] = ""
+ if len(change_bboxes)>0:
+ for bbox in change_bboxes:
+ new_line['predicted'] += str(bbox) + ", "
+ new_line['predicted'] = new_line['predicted'][:-2]
+
+ return new_line
+
+
+def postprocess_auxiliary_geochat_s2looking(canonical_answers, original_answers):
+ """
+ Postprocess the auxiliary file for geochat_s2looking
+ The present questions are
+ question1 = 'temporal_question_answering: Are there any buildings in the first image which were {destructed,torn down} in the second?'
+ question2 = 'temporal_referring_expression: Identify the buildings in the first image which were {built,constructed,destructed,torn down} as seen in the second image.'
+ question3 = 'localization_task: Identify all changed buildings.'
+ question4 = 'referring_expression: identify the {constructed, destructed} buildings in the image.'
+ question5 = 'question_answering: Have any buildings been task in the area? Please answer with Yes or No'
+
+ The goal is to update the 'predicted' field with the correct bounding boxes of the changed buildings.
+ - Localization can be kept as is.
+ - For question answering tasks, the 'predicted' field should be updated with 'Yes' or 'No' depending on the answer.
+ We output 'Yes' if there is a convex polygon in the 'predicted' field.
+ - For referring expression, we first need to identify if the task is 'constructed' or 'destructed' and then update the 'predicted' field with the correct mask of the changed buildings.
+ Input:
+ - answers: dictionary with the answers paired with the get_canonical_answer_dataset function
+ Output:
+ - postprocessed_answers: dictionary with 'predicted' and 'predicted_mask' fields updated
+ """
+ postprocessed_answers = {}
+
+ for key, answer in canonical_answers.items():
+ task = answer['task']
+
+ if 'localization' in task:
+ postprocessed_answers[key] = answer
+ continue
+ if 'region_based_question_answering' in task:
+ answer = postprocess_auxiliary_region_qa(key, answer, original_answers)
+ postprocessed_answers[key] = answer
+ continue
+ if 'question_answering' in task:
+ answer = postprocess_auxiliary_qa(key, answer, original_answers)
+ postprocessed_answers[key] = answer
+ continue
+ if 'referring_expression' in task:
+ answer = postprocess_auxiliary_referring(key, answer, original_answers)
+ postprocessed_answers[key] = answer
+ continue
+
+ return postprocessed_answers
+
+
+def evaluate_geochat_s2looking(answer_file, dataset_file, split):
+ answers = {}
+ with open(answer_file, 'r') as f:
+ for line in f:
+ line = json.loads(line)
+ answers[list(line.keys())[0]] = line[list(line.keys())[0]]
+
+ dataset = dataset_file.split("/")[-1]
+ if dataset == "dataset_canonical.json":
+
+ # create a new dataset with questions and answers for geochat
+ postprocessed_answers = get_canonical_answer_dataset(answers)
+
+ referring_expression(postprocessed_answers, "geochat_s2looking", False, "s2looking/answers/geochat_canonical_test", split=split)
+
+ elif dataset == "dataset_v01_v02_canonical_filtered.json" or dataset == "dataset_RQA.json":
+
+ # create a new dataset with questions and answers for geochat
+ postprocessed_answers = get_canonical_answer_dataset(answers)
+ postprocessed_answers = postprocess_auxiliary_geochat_s2looking(postprocessed_answers, answers)
+
+ print("Referring expression")
+ referring_expression(postprocessed_answers, "geochat_s2looking", False, "s2looking/answers/geochat_v01_v02_canonical_filtered_test", split=split)
+ print()
+ print("Accuracy")
+ accuracy_precision_recall(postprocessed_answers, "s2looking", verbose=False)
+ print()
+
+
+ # also run per-question referring expression
+ question1 = 'temporal_question_answering: Are there any buildings in the first image which were {destructed,torn down} in the second?'
+ question2 = 'temporal_referring_expression: Identify the buildings in the first image which were {built,constructed,destructed,torn down} as seen in the second image.'
+ question3 = 'localization_task: Identify all changed buildings.'
+ question4 = 'referring_expression: identify the {constructed, destructed} buildings in the image.'
+ question5 = 'question_answering: Have any buildings been task in the area? Please answer with Yes or No'
+
+
+ for question in [question1, question2, question3, question4, question5]:
+ dataset_question = {}
+ for data in postprocessed_answers:
+ if postprocessed_answers[data]['task'] == question:
+ dataset_question[data] = postprocessed_answers[data]
+
+ if len(dataset_question) > 0:
+ print('Evaluating for question ', question)
+ print('Size of the dataset is ', len(dataset_question))
+ referring_expression(dataset_question, "geochat_s2looking", False, "s2looking/answers/geochat_v01_v02_canonical_filtered_test", split=split)
+ print()
+
+ else:
+ print("Evaluation is not suppored for this dataset. Please provide a valid dataset.")
+ print("The supported datasets are: dataset_canonical.json, dataset_v01_v02_canonical_filtered.json")
+
+
+
diff --git a/videollava/eval/geochat_utils.py b/videollava/eval/geochat_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..af73e1b93ab61d88b1648527074863a2d7f3ea03
--- /dev/null
+++ b/videollava/eval/geochat_utils.py
@@ -0,0 +1,94 @@
+import json
+from tqdm import tqdm
+from pathlib import Path
+
+from infer_utils import run_inference_single
+import numpy as np
+
+
+def run_geochat_inference(
+ model,
+ dataset_path,
+ processor,
+ tokenizer,
+ conv_mode,
+ answer_path,
+ use_video_data=False,
+ open_prompt=None,
+ repeat_frames=None,
+ prompt_strategy="interleave",
+ chronological_prefix=True,
+ data_frac=1,
+ data_size=None,
+ delete_system_prompt=False,
+ start_ind=0,
+ end_ind=None,
+ print_prompt=False
+):
+
+ with open(dataset_path) as f:
+ qfabric_data = json.load(f)
+
+ if data_size is not None:
+ data_size = min(data_size, len(qfabric_data))
+ idx = np.random.choice(len(qfabric_data), data_size, replace=False)
+ qfabric_data = [qfabric_data[i] for i in idx]
+ elif data_frac < 1:
+ idx = np.random.choice(len(qfabric_data), int(len(qfabric_data) * data_frac), replace=False)
+ qfabric_data = [qfabric_data[i] for i in idx]
+
+ answers = {}
+ answers_tmp = str(answer_path).replace(".json", "_tmp.json")
+ if end_ind is not None:
+ answers_tmp = str(answers_tmp).replace(".json", f"_{start_ind}_{end_ind}.json")
+ qfabric_data = qfabric_data[start_ind:end_ind]
+ else:
+ answers_tmp = str(answers_tmp).replace(".json", f"_{start_ind}_end.json")
+ qfabric_data = qfabric_data[start_ind:]
+
+ print("answers_tmp: ", answers_tmp)
+ print("start ind: ", start_ind)
+ print("end ind: ", end_ind)
+
+ for question in tqdm(qfabric_data):
+ question_id = question["id"]
+ inp = question["conversations"][0]['value']
+
+ answer_str = question["conversations"][1]['value']
+ metadata = question['metadata']
+ image_paths = question['video']
+ task = question['task']
+ original_input_polygon = question['original_input_polygon']
+ dataset = question['dataset']
+
+ outputs = run_inference_single(
+ model=model,
+ processor=processor,
+ tokenizer=tokenizer,
+ conv_mode=conv_mode,
+ inp=inp,
+ image_paths=image_paths,
+ metadata=metadata,
+ repeat_frames=repeat_frames,
+ use_video_data=use_video_data,
+ prompt_strategy=prompt_strategy,
+ chronological_prefix=chronological_prefix,
+ delete_system_prompt=delete_system_prompt,
+ print_prompt=print_prompt
+ )
+
+ entry = {
+ "id": question_id,
+ "question": inp,
+ "predicted": outputs,
+ "ground_truth": answer_str,
+ "task": task,
+ "original_input_polygon": original_input_polygon,
+ "dataset": dataset,
+ }
+ answers[question_id] = entry
+
+ with open(answers_tmp, "a") as f:
+ f.write(json.dumps(entry) + "\n")
+
+ return answers
diff --git a/videollava/eval/infer_eval.py b/videollava/eval/infer_eval.py
new file mode 100644
index 0000000000000000000000000000000000000000..e378dd8f31aa1017dd763d32cf798d5f839b17ae
--- /dev/null
+++ b/videollava/eval/infer_eval.py
@@ -0,0 +1,386 @@
+import fire
+import json
+from pathlib import Path
+
+from videollava.model.builder import load_pretrained_model
+from videollava.utils import disable_torch_init
+from videollava.mm_utils import get_model_name_from_path
+from videollava.model.multimodal_encoder.languagebind.video.processing_video import LanguageBindVideoProcessor
+
+from eval_classification import accuracy_precision_recall
+from eval_referring import referring_expression
+from classification_segmentation import classification_segmentation
+
+from ben_utils import run_ben_inference
+from aid_fmow_ucmerced_utils import run_aid_fmow_ucmerced_inference
+from qfabric_utils import run_qfabric_inference
+from geochat_utils import run_geochat_inference
+from s2looking_utils import run_s2looking_inference
+from xbd_utils import run_xbd_inference
+from cdvqa_utils import run_cdvqa_inference
+
+
+def aggregated(answer_path, dataset=None, verbose=False, split=None):
+ """
+ Define an aggregated metric for our created instruction-following datasets.
+ It includes eval_description and eval_referring metrics.
+ """
+ saving_path_root = Path(answer_path).parent
+
+ with open(answer_path, 'r') as f:
+ answers = json.load(f)
+
+ print("Referring expression")
+ referring_expression(answer_path, dataset, False, saving_path_root, split=split)
+ print()
+ print("Accuracy")
+ accuracy_precision_recall(answer_path, dataset, verbose=False)
+ print()
+
+ # TODO per-task metrics for qfabric and xbd
+
+ if dataset == 'qfabric' or dataset == 'xbd':
+ classification_segmentation(answer_path, dataset)
+
+ if dataset == "s2looking":
+ # also run per-question referring expression
+ question1 = 'temporal_question_answering: Are there any buildings in the first image which were {destructed,torn down} in the second?'
+ question2 = 'temporal_referring_expression: Identify the buildings in the first image which were {built,constructed,destructed,torn down} as seen in the second image.'
+ question3 = 'localization_task: Identify all changed buildings.'
+ question4 = 'referring_expression: identify the {constructed, destructed} buildings in the image.'
+ question5 = 'question_answering: Have any buildings been task in the area? Please answer with Yes or No'
+
+
+ for question in [question1, question2, question3, question4, question5]:
+ dataset_question = {}
+ for data in answers:
+ if answers[data]['task'] == question:
+ dataset_question[data] = answers[data]
+ if len(dataset_question) > 0:
+ print('Evaluating for question ', question)
+ print('Size of the dataset is ', len(dataset_question))
+ referring_expression(dataset_question, dataset, False, saving_path_root, split=split)
+ print()
+
+
+def load_model(model_path, model_base, cache_dir, device, vision_type=None, load_4bit=False, load_8bit=False):
+ model_name = get_model_name_from_path(model_path)
+
+ tokenizer, model, processor, _ = load_pretrained_model(
+ model_path,
+ model_base,
+ model_name,
+ load_4bit=load_4bit,
+ load_8bit=load_8bit,
+ device=device,
+ cache_dir=cache_dir,
+ vision_type=vision_type,
+ )
+
+ if vision_type is None:
+ # Automatically determine which to us
+ # For now assumes one of the processors is not None and one is None
+ vision_types = ['image', 'video']
+ if processor['image'] is None and processor['video'] is None:
+ raise ValueError("Both image and video processors are None")
+ elif processor['image'] is not None and processor['video'] is not None:
+ vision_processor = processor['image']
+ for vision_type in vision_types:
+ vision_processor = processor[vision_type]
+ if vision_processor is not None:
+ break
+ else:
+ vision_processor = processor[vision_type]
+ use_video_data = vision_type == 'video'
+ return tokenizer, model, vision_processor, use_video_data
+
+
+def infer_eval(
+ dataset_path,
+ model_path,
+ model_base="LanguageBind/Video-LLaVA-7B",
+ cache_dir="/deep/group/aicc-bootcamp/geovlm/models/vllava_cache",
+ outname=None,
+ open_prompt=None,
+ repeat_frames=None,
+ prompt_strategy="interleave",
+ chronological_prefix=True,
+ load_8bit=False,
+ load_4bit=False,
+ verbose=False,
+ rerun=False,
+ vision_type=None,
+ data_frac=None,
+ data_size=None,
+ conv_mode="v1",
+ delete_system_prompt=False,
+ start_ind=None,
+ end_ind=None,
+ last_image=None,
+ print_prompt=False
+ ):
+ """
+ Args:
+ dataset_path: path to dataset
+ model_path: path to model
+ model_base: model base name
+ cache_dir: cache directory
+ outname: output file name (uses args if None)
+ open_prompt options: None, "open", "multi-open"
+ repeat_frames options: None, "uniform", "first", "last"
+ prompt_strategy options: None, "interleave"
+ chronological_prefix: whether to use chronological prefix "in chronological order"
+ load_8bit: whether to load 8-bit model
+ load_4bit: whether to load 4-bit model
+ verbose: whether to print verbose output
+ rerun: whether to rerun inference
+ vision_type: "image" or "video"
+ data_frac: fraction of data to use
+ data_size: number of data samples to use
+ conv_mode: conversation mode (should be v1 for our models, geochat, and videollava)
+ delete_system_prompt: whether to delete system prompt
+ start_ind: start index of data
+ end_ind: end index of data
+ last_image: whether to use last image in video
+ print_prompt: whether to print prompt
+ """
+ args = locals()
+ print(f"Arguments passed to infer_eval:")
+ for k, v in args.items():
+ print(f"{k} ({type(v).__name__}): {v}")
+
+ # check that data_size and data_frac are not both set
+ if data_size is not None and data_frac is not None:
+ raise ValueError("data_size and data_frac cannot both be set")
+ if data_size is None and data_frac is None:
+ data_frac = 1
+
+ dataset2metrics = {
+ "lrben": [accuracy_precision_recall],
+ "hrben": [accuracy_precision_recall],
+ "fmow": [accuracy_precision_recall],
+ "s2looking": [aggregated],
+ "xbd": [aggregated],
+ "qfabric": [aggregated],
+ "aid": [accuracy_precision_recall],
+ "ucmerced": [accuracy_precision_recall],
+ "cdvqa": [accuracy_precision_recall]
+ }
+
+ eval_outdir = Path('scripts/geovlm/eval/')
+
+ # Per dataset configurations
+ if "lrben" in dataset_path.lower():
+ dataset = "lrben"
+ run_inference = run_ben_inference
+ outdir = eval_outdir / "RSVQA-LRBEN/answers/"
+ if open_prompt is not None:
+ raise ValueError("LRBEN dataset does not support open prompt")
+ elif "hrben" in dataset_path.lower():
+ dataset = "hrben"
+ run_inference = run_ben_inference
+ outdir = eval_outdir / "RSVQA-HRBEN/answers/"
+ if open_prompt is not None:
+ raise ValueError("HRBEN dataset does not support open prompt")
+ elif "fmow" in dataset_path.lower():
+ dataset = "fmow"
+ run_inference = run_aid_fmow_ucmerced_inference
+ outdir = eval_outdir / "fmow-highres/answers/"
+ elif "s2looking" in dataset_path.lower():
+ dataset = "s2looking"
+ run_inference = run_s2looking_inference
+ outdir = eval_outdir / "s2looking/answers/"
+ elif "xbd" in dataset_path.lower():
+ dataset = "xbd"
+ run_inference = run_xbd_inference
+ outdir = eval_outdir / "xBD/answers/"
+ elif 'qfabric' in dataset_path.lower() or 'geochat' in dataset_path.lower():
+ dataset = "qfabric"
+ run_inference = run_qfabric_inference
+ outdir = eval_outdir / "QFabric/answers/"
+ elif 'geochat' in dataset_path.lower():
+ dataset = "geochat"
+ run_inference = run_geochat_inference
+ outdir = eval_outdir / "GeoChat/answers/"
+ elif 'aid' in dataset_path.lower():
+ dataset = "aid"
+ run_inference = run_aid_fmow_ucmerced_inference
+ outdir = eval_outdir / "AID/answers/"
+ elif 'ucmerced' in dataset_path.lower():
+ dataset = "ucmerced"
+ run_inference = run_aid_fmow_ucmerced_inference
+ outdir = eval_outdir / "UCMerced/answers/"
+ elif 'cdvqa' in dataset_path.lower():
+ dataset = "cdvqa"
+ run_inference = run_cdvqa_inference
+ outdir = eval_outdir / "CDVQA/answers/"
+ else:
+ raise ValueError(f"No supported dataset found in {dataset_path}, supported datasets: fmow, lrben, s2looking, xbd, qfabric, aic, ucmerced")
+
+ if (start_ind is not None or end_ind is not None) and dataset not in ['qfabric', 'hrben', 'lrben']:
+ raise ValueError("start_ind and end_ind can only be used with qfabric, hrben, or lrben datasets")
+
+ # Determine the split
+ if 'test' in dataset_path.lower():
+ split = 'test'
+ elif 'val' or 'valid' or 'validation' in dataset_path.lower():
+ split = 'val'
+ elif 'train' in dataset_path.lower():
+ split = 'train'
+ else:
+ print("Warning: Could not determine split from dataset path")
+
+ args_to_determine_path = [
+ 'open_prompt',
+ 'repeat_frames',
+ 'prompt_strategy',
+ 'chronological_prefix',
+ 'load_8bit',
+ 'load_4bit',
+ 'data_frac',
+ 'data_size',
+ 'delete_system_prompt'
+ ]
+
+ # Setup answer path
+ outdir.mkdir(parents=True, exist_ok=True)
+ model_name = Path(model_path).stem
+
+ if 'llava' not in model_name and 'llava' not in model_name.lower() and 'teochat' not in model_name.lower():
+ if model_base != None:
+ if model_path[-1] == "/":
+ model_path = model_path[:-1]
+ model_name = model_path.split("/")[-2] + "-" + model_path.split("/")[-1]
+ print("Model name used: ", model_name)
+ else:
+ raise ValueError(f"Model name {model_name} does not contain 'llava'")
+ if 'lora' not in model_name:
+ print("Warning: Model name does not contain 'lora'")
+
+ if outname is None:
+ dataset_path_name = Path(dataset_path).stem
+ outname = f"{model_name}_{dataset}_{dataset_path_name}_{split}.json"
+
+ if ".json" not in outname:
+ outname = f"{outname}.json"
+
+ args_to_determine_path = [
+ 'open_prompt',
+ 'repeat_frames',
+ 'prompt_strategy',
+ 'chronological_prefix',
+ 'load_8bit',
+ 'load_4bit',
+ 'data_frac',
+ 'data_size',
+ 'delete_system_prompt',
+ 'start_ind',
+ 'end_ind',
+ 'last_image'
+ ]
+ for arg in args_to_determine_path:
+ if args[arg] is not None:
+ outname = outname.replace(".json", f"_{arg}_{args[arg]}.json")
+
+ answer_path = outdir / outname
+
+ print(f'answer_path: {answer_path}')
+
+ # Save args to file
+ args_path = outdir / outname.replace(".json", "_args.json")
+
+ if len(str(args_path)) < 255:
+ with open(args_path, 'w') as f:
+ json.dump(args, f)
+ else:
+ # File name too long. Just use first letter of each arg
+ for arg in args_to_determine_path:
+ if args[arg] is not None:
+ first_letters = ''.join([word[0] for word in arg.split('_')])
+ #print("outname before replacing: ", outname)
+ outname = outname.replace(f"{arg}", first_letters)
+ #print("outname after replacing: ", outname)
+ answer_path = outdir / outname
+ args_path = outdir / outname.replace(".json", "_args.json")
+ with open(args_path, 'w') as f:
+ json.dump(args, f)
+ print(f'New answer_path: {answer_path}')
+
+ # If answer file exists, compute metrics
+ if answer_path.exists() and not rerun:
+ for metric in dataset2metrics[dataset]:
+ if dataset == "s2looking":
+ metric(answer_path, dataset=dataset, verbose=verbose, split=split)
+ else:
+ metric(answer_path, dataset=dataset, verbose=verbose)
+ return
+
+ # Load model
+ disable_torch_init()
+ device = 'cuda'
+ tokenizer, model, processor, use_video_data = load_model(
+ model_path,
+ model_base,
+ cache_dir,
+ device,
+ load_4bit=load_4bit,
+ load_8bit=load_8bit,
+ vision_type=vision_type
+ )
+
+ if use_video_data:
+ if dataset == "lrben":
+ raise ValueError("LRBEN dataset does not support video processing")
+ # Hack to set backend of video processor
+ # NOTE: If we change image size, we might need to change this in the config here too
+ # (better solution is to figure out where this config is set when saving the model)
+ processor.config.vision_config.video_decode_backend = "image_list"
+ processor = LanguageBindVideoProcessor(processor.config, tokenizer)
+
+ if rerun or not answer_path.exists():
+ # Run inference
+ answers = run_inference(
+ model,
+ dataset_path,
+ processor,
+ tokenizer,
+ conv_mode,
+ answer_path=answer_path,
+ open_prompt=open_prompt,
+ repeat_frames=repeat_frames,
+ use_video_data = use_video_data,
+ prompt_strategy=prompt_strategy,
+ chronological_prefix=chronological_prefix,
+ data_size=data_size,
+ data_frac=data_frac,
+ delete_system_prompt=delete_system_prompt,
+ start_ind=start_ind,
+ end_ind=end_ind,
+ last_image=last_image,
+ print_prompt=print_prompt
+ )
+
+ # Save answers
+ with open(answer_path, 'w') as f:
+ json.dump(answers, f, indent=4)
+ else:
+ answers = json.load(open(answer_path))
+
+
+ # Calculate metrics
+ for metric in dataset2metrics[dataset]:
+ if dataset == "s2looking":
+ metric(answer_path, dataset=dataset, verbose=verbose, split=split)
+ else:
+ metric(answer_path, dataset=dataset, verbose=verbose)
+
+
+if __name__ == '__main__':
+ """Example usage:
+ export CUDA_VISIBLE_DEVICES=0;
+ export PYTHONPATH=/path/to/aicc-win24-geo-vlm/videollava/:$PYTHONPATH;
+ python videollava/eval/video/infer_eval.py infer_eval\
+ --dataset fmow\
+ --model_path /path/to/model\
+ """
+ fire.Fire()
diff --git a/videollava/eval/infer_utils.py b/videollava/eval/infer_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..a6a91915cc10535b948389ba17e3c85f8b8f65b6
--- /dev/null
+++ b/videollava/eval/infer_utils.py
@@ -0,0 +1,253 @@
+import cv2
+import torch
+import warnings
+import numpy as np
+from datetime import datetime
+import cv2
+import warnings
+import time
+
+from videollava.mm_utils import tokenizer_image_token, KeywordsStoppingCriteria
+from videollava.conversation import conv_templates, SeparatorStyle
+from videollava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_VIDEO_TOKEN
+
+
+def replace_video_token(prompt, image_paths, prompt_strategy):
+ if prompt_strategy is None:
+ vid_replace_token = DEFAULT_IMAGE_TOKEN * len(image_paths)
+ elif prompt_strategy == 'interleave':
+ vid_replace_token = ''.join(f"Image {i+1}: {DEFAULT_IMAGE_TOKEN}" for i in range(len(image_paths)))
+ else:
+ raise ValueError(f"Unknown prompt strategy: {prompt_strategy}")
+ return prompt.replace(DEFAULT_VIDEO_TOKEN, vid_replace_token)
+
+
+def run_inference_single(
+ model,
+ processor,
+ tokenizer,
+ conv_mode,
+ inp,
+ image_paths,
+ metadata=None,
+ use_video_data=False,
+ repeat_frames=None,
+ prompt_strategy=None,
+ chronological_prefix=True,
+ delete_system_prompt=False,
+ print_prompt=False,
+ return_prompt=False,
+ last_image=False,
+ prompt=None
+ ):
+ conv = conv_templates[conv_mode].copy()
+ if prompt is None:
+ conv.append_message(conv.roles[0], inp)
+ conv.append_message(conv.roles[1], None)
+ prompt = conv.get_prompt()
+
+ if chronological_prefix:
+ prompt = prompt.replace("times:", "times in chronological order:")
+
+ if metadata is not None:
+ # Sort by time
+ image_paths, metadata = zip(*sorted(
+ zip(image_paths, metadata),
+ key=lambda t: datetime.strptime(t[1]["timestamp"], "%Y-%m-%d")
+ ))
+
+ if delete_system_prompt:
+ if "This is" in prompt:
+ start_index = prompt.find("This is")
+ elif "These are" in prompt:
+ start_index = prompt.find("These are")
+ end_index = prompt.find(":", start_index)
+ if start_index != -1 and end_index != -1:
+ prompt = prompt[:start_index] + prompt[end_index+1:]
+ else:
+ warnings.warn("Impossible to remove the system message from the prompt.")
+
+ if use_video_data:
+ image_paths = list(image_paths)
+ if repeat_frames == "uniform":
+ # Repeat up to 8 for now
+ num_frames = 8
+ if len(image_paths) < num_frames:
+ num_repeats = num_frames // len(image_paths)
+ index = len(image_paths) - num_frames % len(image_paths)
+ image_paths = list(np.repeat(image_paths[:index], num_repeats)) + list(np.repeat(image_paths[index:], num_repeats+1))
+ elif repeat_frames == "first":
+ # Repeat the first frame
+ num_frames = 8
+ if len(image_paths) < num_frames:
+ repeat_frames = [image_paths[0]] * (num_frames - len(image_paths)) + image_paths
+ elif repeat_frames == "last":
+ # Repeat the last frame
+ num_frames = 8
+ if len(image_paths) < num_frames:
+ repeat_frames = image_paths + [image_paths[-1]] * (num_frames - len(image_paths))
+
+ video_tensor = processor.preprocess(image_paths, return_tensors='pt')['pixel_values']
+ tensor = [video_tensor.to(model.device, dtype=torch.float16)]
+
+ else:
+ image_tensors = [processor.preprocess(i, return_tensors='pt')['pixel_values'][0] for i in image_paths]
+ tensor = [image_tensor.to(model.device, dtype=torch.float16) for image_tensor in image_tensors]
+
+ if last_image:
+ tensor = [tensor[-1]]
+ image_paths = [image_paths[-1]]
+ if metadata is not None:
+ metadata = [metadata[-1]]
+
+ prompt = replace_video_token(prompt, image_paths, prompt_strategy)
+
+ if print_prompt:
+ print(prompt)
+
+ stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
+ keywords = [stop_str]
+
+ input_ids = tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).to(model.device)
+ stopping_criteria = KeywordsStoppingCriteria(keywords, tokenizer, input_ids)
+
+ with torch.inference_mode():
+ output_ids = model.generate(
+ input_ids=input_ids,
+ images=tensor,
+ do_sample=True,
+ temperature=0.2,
+ max_new_tokens=256,
+ use_cache=True,
+ stopping_criteria=[stopping_criteria],
+ )
+
+ # .replace removes the end sentence token "" from the output
+ outputs = tokenizer.decode(output_ids[0, input_ids.shape[1]:]).replace('', '').strip()
+
+ if return_prompt:
+ return prompt, outputs
+ else:
+ return outputs
+
+
+def create_mask(poly, im_size):
+ """
+ Create mask of given height and width where entries
+ inside polygon are 1.
+ params:
+ - poly (shapely polygon object): polygon to create mask for
+ - im_size (tuple): size of image (height, width)
+ returns:
+ - img_mask (np.array): mask of polygon"""
+ img_mask = np.zeros(im_size, np.uint8)
+ def int_coords(x): return np.array(x).round().astype(np.int32)
+ try:
+ exteriors = [int_coords(pol.exterior.coords) for pol in poly]
+ except:
+ exteriors = [int_coords(poly.exterior.coords)]
+ cv2.fillPoly(img_mask, exteriors, 1)
+ try:
+ interiors = [int_coords(pol.interior.coords) for pol in poly]
+ cv2.fillPoly(img_mask, interiors, 0)
+ except:
+ pass
+ try:
+ interiors = [int_coords(poly.interior.coords)]
+ cv2.fillPoly(img_mask, interiors, 0)
+ except:
+ pass
+
+ return img_mask
+
+
+def create_mask_s2looking(img_id, split=None, question=None):
+ if split == None:
+ raise ValueError("split must be provided for S2Looking evaluation")
+
+ if question == None:
+ raise ValueError("question must be provided for S2Looking evaluation")
+
+ im1_path = f'/scr/geovlm/S2Looking/{split}/label1' # built
+ img2_path = f'/scr/geovlm/S2Looking/{split}/label2' # destroyed
+ id, chunk = img_id.split('_')
+ # Load image as numpy array
+ im1 = cv2.imread(f'{im1_path}/{id}.png', cv2.IMREAD_GRAYSCALE)
+ im2 = cv2.imread(f'{img2_path}/{id}.png', cv2.IMREAD_GRAYSCALE)
+ # replace any value different from 0 with 1
+ im1[im1 != 0] = 1
+ im2[im2 != 0] = 1
+
+ # get the corresponding of the 16 chunks
+ # 1 is upper left, 16 is lower right
+ if chunk == '1':
+ mask1 = im1[:256, :256]
+ mask2 = im2[:256, :256]
+ elif chunk == '2':
+ mask1 = im1[:256, 256:2*256]
+ mask2 = im2[:256, 256:2*256]
+ elif chunk == '3':
+ mask1 = im1[:256, 2*256:3*256]
+ mask2 = im2[:256, 2*256:3*256]
+ elif chunk == '4':
+ mask1 = im1[:256, 3*256:]
+ mask2 = im2[:256, 3*256:]
+ elif chunk == '5':
+ mask1 = im1[256:2*256, :256]
+ mask2 = im2[256:2*256, :256]
+ elif chunk == '6':
+ mask1 = im1[256:2*256, 256:2*256]
+ mask2 = im2[256:2*256, 256:2*256]
+ elif chunk == '7':
+ mask1 = im1[256:2*256, 2*256:3*256]
+ mask2 = im2[256:2*256, 2*256:3*256]
+ elif chunk == '8':
+ mask1 = im1[256:2*256, 3*256:]
+ mask2 = im2[256:2*256, 3*256:]
+ elif chunk == '9':
+ mask1 = im1[2*256:3*256, :256]
+ mask2 = im2[2*256:3*256, :256]
+ elif chunk == '10':
+ mask1 = im1[2*256:3*256, 256:2*256]
+ mask2 = im2[2*256:3*256, 256:2*256]
+ elif chunk == '11':
+ mask1 = im1[2*256:3*256, 2*256:3*256]
+ mask2 = im2[2*256:3*256, 2*256:3*256]
+ elif chunk == '12':
+ mask1 = im1[2*256:3*256, 3*256:]
+ mask2 = im2[2*256:3*256, 3*256:]
+ elif chunk == '13':
+ mask1 = im1[3*256:, :256]
+ mask2 = im2[3*256:, :256]
+ elif chunk == '14':
+ mask1 = im1[3*256:, 256:2*256]
+ mask2 = im2[3*256:, 256:2*256]
+ elif chunk == '15':
+ mask1 = im1[3*256:, 2*256:3*256]
+ mask2 = im2[3*256:, 2*256:3*256]
+ elif chunk == '16':
+ mask1 = im1[3*256:, 3*256:]
+ mask2 = im2[3*256:, 3*256:]
+
+ task = None
+ if 'built' in question or 'constructed' in question:
+ task = 'constructing'
+ if 'destroyed' in question or 'torn down' in question or 'demolished' in question:
+ task = 'destroying'
+ if 'changed' in question:
+ task = 'changing'
+ if (('built' in question) or ('constructed' in question)) and (('destroyed' in question) or ('torn down' in question) or ('demolished' in question)):
+ print(question)
+ raise ValueError("Question cannot contain both 'built' and 'destroyed'")
+ if task is None:
+ print(question)
+ raise ValueError("Question must contain either 'built', 'destroyed', or 'changed'")
+
+ if task == 'constructing':
+ mask = mask1
+ elif task == 'destroying':
+ mask = mask2
+ elif task == 'changing':
+ mask = np.logical_or(mask1, mask2)
+
+ return mask
diff --git a/videollava/eval/qfabric_utils.py b/videollava/eval/qfabric_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..506630da7f44203d2216a32f693f9a26b65905cb
--- /dev/null
+++ b/videollava/eval/qfabric_utils.py
@@ -0,0 +1,98 @@
+import json
+from tqdm import tqdm
+from pathlib import Path
+
+from infer_utils import run_inference_single
+import numpy as np
+
+
+def run_qfabric_inference(
+ model,
+ dataset_path,
+ processor,
+ tokenizer,
+ conv_mode,
+ answer_path,
+ use_video_data=False,
+ open_prompt=None,
+ repeat_frames=None,
+ prompt_strategy="interleave",
+ chronological_prefix=True,
+ data_frac=1,
+ data_size=None,
+ delete_system_prompt=False,
+ print_prompt=False,
+ start_ind=None,
+ end_ind=None,
+ last_image=False,
+):
+
+ with open(dataset_path) as f:
+ qfabric_data = json.load(f)
+
+ if data_size is not None:
+ data_size = min(data_size, len(qfabric_data))
+ idx = np.random.choice(len(qfabric_data), data_size, replace=False)
+ qfabric_data = [qfabric_data[i] for i in idx]
+ elif data_frac < 1:
+ idx = np.random.choice(len(qfabric_data), int(len(qfabric_data) * data_frac), replace=False)
+ qfabric_data = [qfabric_data[i] for i in idx]
+
+ answers = {}
+ answers_tmp = str(answer_path).replace(".json", "_tmp.json")
+ if start_ind is None:
+ start_ind = 0
+ if end_ind is not None:
+ # TODO: Don't append as it's already done previously
+ answers_tmp = str(answer_path).replace(".json", f"_{start_ind}_{end_ind}.json")
+ qfabric_data = qfabric_data[start_ind:end_ind]
+ else:
+ # TODO: Don't append as it's already done previously
+ answers_tmp = str(answer_path).replace(".json", f"_{start_ind}_end.json")
+ qfabric_data = qfabric_data[start_ind:]
+
+ print("answers_tmp: ", answers_tmp)
+ print("start ind: ", start_ind)
+ print("end ind: ", end_ind)
+
+ for question in tqdm(qfabric_data):
+ question_id = question["id"]
+ inp = question["conversations"][0]['value']
+
+ answer_str = question["conversations"][1]['value']
+ metadata = question['metadata']
+ image_paths = question['video']
+ task = question['task']
+ original_input_polygon = question['original_input_polygon']
+
+ outputs = run_inference_single(
+ model=model,
+ processor=processor,
+ tokenizer=tokenizer,
+ conv_mode=conv_mode,
+ inp=inp,
+ image_paths=image_paths,
+ metadata=metadata,
+ repeat_frames=repeat_frames,
+ use_video_data=use_video_data,
+ prompt_strategy=prompt_strategy,
+ chronological_prefix=chronological_prefix,
+ delete_system_prompt=delete_system_prompt,
+ last_image=last_image,
+ print_prompt=print_prompt
+ )
+
+ entry = {
+ "id": question_id,
+ "question": inp,
+ "predicted": outputs,
+ "ground_truth": answer_str,
+ "task": task,
+ "original_input_polygon": original_input_polygon
+ }
+ answers[question_id] = entry
+
+ with open(answers_tmp, "a") as f:
+ f.write(json.dumps(entry) + "\n")
+
+ return answers
diff --git a/videollava/eval/s2looking_utils.py b/videollava/eval/s2looking_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..5a083f04e8dcc245889b9af6bbc121f6fb244ee8
--- /dev/null
+++ b/videollava/eval/s2looking_utils.py
@@ -0,0 +1,78 @@
+import json
+import numpy as np
+from tqdm import tqdm
+from pathlib import Path
+
+from infer_utils import run_inference_single, create_mask
+
+
+def run_s2looking_inference(
+ model,
+ dataset_path,
+ processor,
+ tokenizer,
+ conv_mode,
+ use_video_data=False,
+ open_prompt=None,
+ repeat_frames=True,
+ prompt_strategy="interleave",
+ chronological_prefix=True,
+ data_frac=1,
+ data_size=None,
+ delete_system_prompt=False,
+ last_image=False,
+ print_prompt=False,
+ answer_path=None,
+ start_ind=None,
+ end_ind=None,
+):
+
+ dir = Path(dataset_path)
+
+ with open(dir) as f:
+ s2looking_data = json.load(f)
+
+ if data_size is not None:
+ data_size = min(data_size, len(s2looking_data))
+ idx = np.random.choice(len(s2looking_data), data_size, replace=False)
+ s2looking_data = [s2looking_data[i] for i in idx]
+ elif data_frac < 1:
+ idx = np.random.choice(len(s2looking_data), int(len(s2looking_data) * data_frac), replace=False)
+ s2looking_data = [s2looking_data[i] for i in idx]
+
+ answers = {}
+ for question in tqdm(s2looking_data):
+ question_id = question["id"]
+ inp = question["conversations"][0]['value']
+ answer_str = question["conversations"][1]['value']
+ metadata = question['metadata']
+ task = question['task']
+ image_paths = question['video']
+ original_input_polygon = question['original_input_polygon']
+
+ outputs = run_inference_single(
+ model=model,
+ processor=processor,
+ tokenizer=tokenizer,
+ conv_mode=conv_mode,
+ inp=inp,
+ image_paths=image_paths,
+ metadata=metadata,
+ repeat_frames=repeat_frames,
+ use_video_data=use_video_data,
+ prompt_strategy=prompt_strategy,
+ chronological_prefix=chronological_prefix,
+ delete_system_prompt=delete_system_prompt,
+ last_image=last_image,
+ print_prompt=print_prompt
+ )
+
+ answers[question_id] = {
+ "predicted": outputs,
+ "ground_truth": answer_str,
+ "question": inp,
+ "task": task,
+ "original_input_polygon": original_input_polygon
+ }
+
+ return answers
diff --git a/videollava/eval/xbd_utils.py b/videollava/eval/xbd_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..2a8b95efa751c0070f51d5c71a1d8f6b3653621c
--- /dev/null
+++ b/videollava/eval/xbd_utils.py
@@ -0,0 +1,82 @@
+import json
+from tqdm import tqdm
+from pathlib import Path
+
+from infer_utils import run_inference_single
+import numpy as np
+
+
+
+def run_xbd_inference(
+ model,
+ dataset_path,
+ processor,
+ tokenizer,
+ conv_mode,
+ use_video_data=False,
+ open_prompt=None,
+ repeat_frames=None,
+ prompt_strategy="interleave",
+ chronological_prefix=True,
+ data_frac=1,
+ data_size=None,
+ last_image=False,
+ delete_system_prompt=False,
+ print_prompt=False,
+ answer_path=None,
+ start_ind=None,
+ end_ind=None,
+):
+
+ with open(dataset_path) as f:
+ xbd_data = json.load(f)
+
+ if data_size is not None:
+ data_size = min(data_size, len(xbd_data))
+ idx = np.random.choice(len(xbd_data), data_size, replace=False)
+ xbd_data = [xbd_data[i] for i in idx]
+ elif data_frac < 1:
+ idx = np.random.choice(len(xbd_data), int(len(xbd_data) * data_frac), replace=False)
+ xbd_data = [xbd_data[i] for i in idx]
+
+ answers = {}
+ for question in tqdm(xbd_data):
+ question_id = question["id"]
+ inp = question["conversations"][0]['value']
+
+ answer_str = question["conversations"][1]['value']
+ metadata = question['metadata']
+ image_paths = question['video']
+ task = question['task']
+ original_input_polygon = question['original_input_polygon']
+
+ # TODO: check if you want to add closed framing for yes/no questions
+ outputs = run_inference_single(
+ model=model,
+ processor=processor,
+ tokenizer=tokenizer,
+ conv_mode=conv_mode,
+ inp=inp,
+ image_paths=image_paths,
+ metadata=metadata,
+ repeat_frames=repeat_frames,
+ use_video_data=use_video_data,
+ prompt_strategy=prompt_strategy,
+ chronological_prefix=chronological_prefix,
+ last_image=last_image,
+ print_prompt=print_prompt
+ )
+
+ answers[question_id] = {
+ "question": inp,
+ "predicted": outputs,
+ "ground_truth": answer_str,
+ "task": task,
+ "original_input_polygon": original_input_polygon
+ }
+ # For recording individual answers as inference runs
+ entry = {question_id: answers[question_id]}
+ with open('/deep/u/joycech/aicc-working/geovlm_xbd_localization.json', 'a') as f:
+ f.write(json.dumps(entry) + ',')
+
+ return answers
diff --git a/videollava/mm_utils.py b/videollava/mm_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..9132d6690452961f717c4a8abf525ad398ced683
--- /dev/null
+++ b/videollava/mm_utils.py
@@ -0,0 +1,104 @@
+from PIL import Image
+from io import BytesIO
+import base64
+
+import torch
+from transformers import StoppingCriteria
+from videollava.constants import IMAGE_TOKEN_INDEX
+
+
+def load_image_from_base64(image):
+ return Image.open(BytesIO(base64.b64decode(image)))
+
+
+def expand2square(pil_img, background_color):
+ width, height = pil_img.size
+ if width == height:
+ return pil_img
+ elif width > height:
+ result = Image.new(pil_img.mode, (width, width), background_color)
+ result.paste(pil_img, (0, (width - height) // 2))
+ return result
+ else:
+ result = Image.new(pil_img.mode, (height, height), background_color)
+ result.paste(pil_img, ((height - width) // 2, 0))
+ return result
+
+
+def process_images(images, image_processor, model_cfg):
+ image_aspect_ratio = getattr(model_cfg, "image_aspect_ratio", None)
+ new_images = []
+ if image_aspect_ratio == 'pad':
+ for image in images:
+ image = expand2square(image, tuple(int(x*255) for x in image_processor.image_mean))
+ image = image_processor.preprocess(image, return_tensors='pt')['pixel_values'][0]
+ new_images.append(image)
+ else:
+ return image_processor(images, return_tensors='pt')['pixel_values']
+ if all(x.shape == new_images[0].shape for x in new_images):
+ new_images = torch.stack(new_images, dim=0)
+ return new_images
+
+
+def tokenizer_image_token(prompt, tokenizer, image_token_index=IMAGE_TOKEN_INDEX, return_tensors=None):
+ prompt_chunks = [tokenizer(chunk).input_ids for chunk in prompt.split('')]
+
+ def insert_separator(X, sep):
+ return [ele for sublist in zip(X, [sep]*len(X)) for ele in sublist][:-1]
+
+ input_ids = []
+ offset = 0
+ if len(prompt_chunks) > 0 and len(prompt_chunks[0]) > 0 and prompt_chunks[0][0] == tokenizer.bos_token_id:
+ offset = 1
+ input_ids.append(prompt_chunks[0][0])
+
+ for x in insert_separator(prompt_chunks, [image_token_index] * (offset + 1)):
+ input_ids.extend(x[offset:])
+
+ if return_tensors is not None:
+ if return_tensors == 'pt':
+ return torch.tensor(input_ids, dtype=torch.long)
+ raise ValueError(f'Unsupported tensor type: {return_tensors}')
+ return input_ids
+
+
+def get_model_name_from_path(model_path):
+ model_path = model_path.strip("/")
+ model_paths = model_path.split("/")
+ if model_paths[-1].startswith('checkpoint-'):
+ return model_paths[-2] + "_" + model_paths[-1]
+ else:
+ return model_paths[-1]
+
+class KeywordsStoppingCriteria(StoppingCriteria):
+ def __init__(self, keywords, tokenizer, input_ids):
+ self.keywords = keywords
+ self.keyword_ids = []
+ self.max_keyword_len = 0
+ for keyword in keywords:
+ cur_keyword_ids = tokenizer(keyword).input_ids
+ if len(cur_keyword_ids) > 1 and cur_keyword_ids[0] == tokenizer.bos_token_id:
+ cur_keyword_ids = cur_keyword_ids[1:]
+ if len(cur_keyword_ids) > self.max_keyword_len:
+ self.max_keyword_len = len(cur_keyword_ids)
+ self.keyword_ids.append(torch.tensor(cur_keyword_ids))
+ self.tokenizer = tokenizer
+ self.start_len = input_ids.shape[1]
+
+ def call_for_batch(self, output_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool:
+ offset = min(output_ids.shape[1] - self.start_len, self.max_keyword_len)
+ self.keyword_ids = [keyword_id.to(output_ids.device) for keyword_id in self.keyword_ids]
+ for keyword_id in self.keyword_ids:
+ if (output_ids[0, -keyword_id.shape[0]:] == keyword_id).all():
+ return True
+ outputs = self.tokenizer.batch_decode(output_ids[:, -offset:], skip_special_tokens=True)[0]
+ for keyword in self.keywords:
+ if keyword in outputs:
+ return True
+ return False
+
+ def __call__(self, output_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool:
+ outputs = []
+ for i in range(output_ids.shape[0]):
+ outputs.append(self.call_for_batch(output_ids[i].unsqueeze(0), scores))
+ return all(outputs)
diff --git a/videollava/model/__init__.py b/videollava/model/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..fa79960540bf9247f5df9f02656ca84499cdbfd6
--- /dev/null
+++ b/videollava/model/__init__.py
@@ -0,0 +1,2 @@
+from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+from .language_model.llava_mpt import LlavaMPTForCausalLM, LlavaMPTConfig
diff --git a/videollava/model/apply_delta.py b/videollava/model/apply_delta.py
new file mode 100644
index 0000000000000000000000000000000000000000..a639009a5ff76f25a81a6a14f77c0abb8223f9dc
--- /dev/null
+++ b/videollava/model/apply_delta.py
@@ -0,0 +1,48 @@
+"""
+Usage:
+python3 -m fastchat.model.apply_delta --base ~/model_weights/llama-7b --target ~/model_weights/vicuna-7b --delta lmsys/vicuna-7b-delta
+"""
+import argparse
+
+import torch
+from tqdm import tqdm
+from transformers import AutoTokenizer, AutoModelForCausalLM
+from videollava import LlavaLlamaForCausalLM
+
+
+def apply_delta(base_model_path, target_model_path, delta_path):
+ print("Loading base model")
+ base = AutoModelForCausalLM.from_pretrained(
+ base_model_path, torch_dtype=torch.float16, low_cpu_mem_usage=True)
+
+ print("Loading delta")
+ delta = LlavaLlamaForCausalLM.from_pretrained(delta_path, torch_dtype=torch.float16, low_cpu_mem_usage=True)
+ delta_tokenizer = AutoTokenizer.from_pretrained(delta_path)
+
+ print("Applying delta")
+ for name, param in tqdm(delta.state_dict().items(), desc="Applying delta"):
+ if name not in base.state_dict():
+ assert name in ['model.mm_projector.weight', 'model.mm_projector.bias'], f'{name} not in base model'
+ continue
+ if param.data.shape == base.state_dict()[name].shape:
+ param.data += base.state_dict()[name]
+ else:
+ assert name in ['model.embed_tokens.weight', 'lm_head.weight'], \
+ f'{name} dimension mismatch: {param.data.shape} vs {base.state_dict()[name].shape}'
+ bparam = base.state_dict()[name]
+ param.data[:bparam.shape[0], :bparam.shape[1]] += bparam
+
+ print("Saving target model")
+ delta.save_pretrained(target_model_path)
+ delta_tokenizer.save_pretrained(target_model_path)
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--base-model-path", type=str, required=True)
+ parser.add_argument("--target-model-path", type=str, required=True)
+ parser.add_argument("--delta-path", type=str, required=True)
+
+ args = parser.parse_args()
+
+ apply_delta(args.base_model_path, args.target_model_path, args.delta_path)
diff --git a/videollava/model/builder.py b/videollava/model/builder.py
new file mode 100644
index 0000000000000000000000000000000000000000..f3d871113082f2e050f3e22ad0a6ec2909525089
--- /dev/null
+++ b/videollava/model/builder.py
@@ -0,0 +1,166 @@
+# Copyright 2023 Haotian Liu
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+import warnings
+import shutil
+
+from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig, BitsAndBytesConfig
+import torch
+from videollava.model import *
+from videollava.constants import DEFAULT_IMAGE_PATCH_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN, \
+ DEFAULT_VIDEO_PATCH_TOKEN, DEFAULT_VID_START_TOKEN, DEFAULT_VID_END_TOKEN
+
+
+def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, load_4bit=False, device_map="auto", device="cuda", **kwargs):
+ kwargs = {"device_map": device_map, **kwargs}
+
+ if device != "cuda":
+ kwargs['device_map'] = {"": device}
+
+ if load_8bit:
+ kwargs['load_in_8bit'] = True
+ elif load_4bit:
+ kwargs['load_in_4bit'] = True
+ kwargs['quantization_config'] = BitsAndBytesConfig(
+ load_in_4bit=True,
+ bnb_4bit_compute_dtype=torch.float16,
+ bnb_4bit_use_double_quant=True,
+ bnb_4bit_quant_type='nf4'
+ )
+ else:
+ kwargs['torch_dtype'] = torch.float16
+
+ if 'llava' in model_name.lower() or "teochat" in model_name.lower():
+ # Load LLaVA model
+ if 'lora' in model_name.lower() and model_base is None:
+ warnings.warn('There is `lora` in model name but no `model_base` is provided. If you are loading a LoRA model, please provide the `model_base` argument. Detailed instruction: https://github.com/haotian-liu/LLaVA#launch-a-model-worker-lora-weights-unmerged.')
+ if 'lora' in model_name.lower() and model_base is not None:
+ lora_cfg_pretrained = AutoConfig.from_pretrained(model_path)
+ tokenizer = AutoTokenizer.from_pretrained(model_base, use_fast=False)
+ print('Loading LLaVA from base model...')
+ model = LlavaLlamaForCausalLM.from_pretrained(model_base, low_cpu_mem_usage=True, config=lora_cfg_pretrained, **kwargs)
+ token_num, tokem_dim = model.lm_head.out_features, model.lm_head.in_features
+ if model.lm_head.weight.shape[0] != token_num:
+ model.lm_head.weight = torch.nn.Parameter(torch.empty(token_num, tokem_dim, device=model.device, dtype=model.dtype))
+ model.model.embed_tokens.weight = torch.nn.Parameter(torch.empty(token_num, tokem_dim, device=model.device, dtype=model.dtype))
+
+ print('Loading additional LLaVA weights...')
+ if os.path.exists(os.path.join(model_path, 'non_lora_trainables.bin')):
+ non_lora_trainables = torch.load(os.path.join(model_path, 'non_lora_trainables.bin'), map_location='cpu')
+ else:
+ # this is probably from HF Hub
+ from huggingface_hub import hf_hub_download
+ def load_from_hf(repo_id, filename, subfolder=None):
+ cache_file = hf_hub_download(
+ repo_id=repo_id,
+ filename=filename,
+ subfolder=subfolder)
+ return torch.load(cache_file, map_location='cpu')
+ non_lora_trainables = load_from_hf(model_path, 'non_lora_trainables.bin')
+ non_lora_trainables = {(k[11:] if k.startswith('base_model.') else k): v for k, v in non_lora_trainables.items()}
+ if any(k.startswith('model.model.') for k in non_lora_trainables):
+ non_lora_trainables = {(k[6:] if k.startswith('model.') else k): v for k, v in non_lora_trainables.items()}
+ model.load_state_dict(non_lora_trainables, strict=False)
+
+ from peft import PeftModel
+ print('Loading LoRA weights...')
+ model = PeftModel.from_pretrained(model, model_path, **kwargs)
+ print('Merging LoRA weights...')
+ model = model.merge_and_unload()
+ print('Model is loaded...')
+ elif model_base is not None:
+ # this may be mm projector only
+ print('Loading LLaVA from base model...')
+ if 'mpt' in model_name.lower():
+ if not os.path.isfile(os.path.join(model_path, 'configuration_mpt.py')):
+ shutil.copyfile(os.path.join(model_base, 'configuration_mpt.py'), os.path.join(model_path, 'configuration_mpt.py'))
+ tokenizer = AutoTokenizer.from_pretrained(model_base, use_fast=True)
+ cfg_pretrained = AutoConfig.from_pretrained(model_path, trust_remote_code=True)
+ model = LlavaMPTForCausalLM.from_pretrained(model_base, low_cpu_mem_usage=True, config=cfg_pretrained, **kwargs)
+ else:
+ tokenizer = AutoTokenizer.from_pretrained(model_base, use_fast=False)
+ cfg_pretrained = AutoConfig.from_pretrained(model_path)
+ model = LlavaLlamaForCausalLM.from_pretrained(model_base, low_cpu_mem_usage=True, config=cfg_pretrained, **kwargs)
+
+ mm_projector_weights = torch.load(os.path.join(model_path, 'mm_projector.bin'), map_location='cpu')
+ mm_projector_weights = {k: v.to(torch.float16) for k, v in mm_projector_weights.items()}
+ model.load_state_dict(mm_projector_weights, strict=False)
+ else:
+ if 'mpt' in model_name.lower():
+ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, **kwargs)
+ model = LlavaMPTForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, **kwargs)
+ else:
+ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False, **kwargs)
+ model = LlavaLlamaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, **kwargs)
+ else:
+ # Load language model
+ if model_base is not None:
+ # PEFT model
+ from peft import PeftModel
+ tokenizer = AutoTokenizer.from_pretrained(model_base, use_fast=False, **kwargs)
+ model = AutoModelForCausalLM.from_pretrained(model_base, low_cpu_mem_usage=True, **kwargs)
+ print(f"Loading LoRA weights from {model_path}")
+ model = PeftModel.from_pretrained(model, model_path)
+ print(f"Merging weights")
+ model = model.merge_and_unload()
+ print('Convert to FP16...')
+ model.to(torch.float16)
+ else:
+ use_fast = False
+ if 'mpt' in model_name.lower():
+ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, **kwargs)
+ model = AutoModelForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, trust_remote_code=True, **kwargs)
+ else:
+ tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False, **kwargs)
+ model = AutoModelForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, **kwargs)
+
+ # ==========================================================================================================
+ processor = {'image': None, 'video': None}
+
+ if 'llava' in model_name.lower() or "teochat" in model_name.lower():
+ mm_use_im_start_end = getattr(model.config, "mm_use_im_start_end", False)
+ mm_use_im_patch_token = getattr(model.config, "mm_use_im_patch_token", True)
+ if mm_use_im_patch_token:
+ tokenizer.add_tokens([DEFAULT_IMAGE_PATCH_TOKEN], special_tokens=True)
+ tokenizer.add_tokens([DEFAULT_VIDEO_PATCH_TOKEN], special_tokens=True)
+ if mm_use_im_start_end:
+ tokenizer.add_tokens([DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN], special_tokens=True)
+ tokenizer.add_tokens([DEFAULT_VID_START_TOKEN, DEFAULT_VID_END_TOKEN], special_tokens=True)
+ model.resize_token_embeddings(len(tokenizer))
+
+ if model.config.mm_image_tower is not None:
+ image_tower = model.get_image_tower()
+ if not image_tower.is_loaded:
+ image_tower.load_model()
+ image_tower.to(device=device, dtype=torch.float16)
+ image_processor = image_tower.image_processor
+ processor['image'] = image_processor
+
+ if model.config.mm_video_tower is not None:
+ video_tower = model.get_video_tower()
+ if not video_tower.is_loaded:
+ video_tower.load_model()
+ video_tower.to(device=device, dtype=torch.float16)
+ video_processor = video_tower.video_processor
+ processor['video'] = video_processor
+ # ==========================================================================================================
+
+ if hasattr(model.config, "max_sequence_length"):
+ context_len = model.config.max_sequence_length
+ else:
+ context_len = 2048
+
+ return tokenizer, model, processor, context_len
diff --git a/videollava/model/consolidate.py b/videollava/model/consolidate.py
new file mode 100644
index 0000000000000000000000000000000000000000..c4d64c2102878254c287e6777902c314ccb11186
--- /dev/null
+++ b/videollava/model/consolidate.py
@@ -0,0 +1,29 @@
+"""
+Usage:
+python3 -m llava.model.consolidate --src ~/model_weights/llava-7b --dst ~/model_weights/llava-7b_consolidate
+"""
+import argparse
+
+import torch
+from transformers import AutoTokenizer, AutoModelForCausalLM
+from videollava.model import *
+from videollava.model.utils import auto_upgrade
+
+
+def consolidate_ckpt(src_path, dst_path):
+ print("Loading model")
+ auto_upgrade(src_path)
+ src_model = AutoModelForCausalLM.from_pretrained(src_path, torch_dtype=torch.float16, low_cpu_mem_usage=True)
+ src_tokenizer = AutoTokenizer.from_pretrained(src_path, use_fast=False)
+ src_model.save_pretrained(dst_path)
+ src_tokenizer.save_pretrained(dst_path)
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--src", type=str, required=True)
+ parser.add_argument("--dst", type=str, required=True)
+
+ args = parser.parse_args()
+
+ consolidate_ckpt(args.src, args.dst)
diff --git a/videollava/model/language_model/llava_llama.py b/videollava/model/language_model/llava_llama.py
new file mode 100644
index 0000000000000000000000000000000000000000..58ccb30576efec75090dfdaec3995fc04f895837
--- /dev/null
+++ b/videollava/model/language_model/llava_llama.py
@@ -0,0 +1,111 @@
+# Copyright 2023 Haotian Liu
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from typing import List, Optional, Tuple, Union
+
+import torch
+import torch.nn as nn
+
+from transformers import AutoConfig, AutoModelForCausalLM, \
+ LlamaConfig, LlamaModel, LlamaForCausalLM
+
+from transformers.modeling_outputs import CausalLMOutputWithPast
+
+from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+
+
+class LlavaConfig(LlamaConfig):
+ model_type = "llava"
+
+
+class LlavaLlamaModel(LlavaMetaModel, LlamaModel):
+ config_class = LlavaConfig
+
+ def __init__(self, config: LlamaConfig):
+ super(LlavaLlamaModel, self).__init__(config)
+
+
+class LlavaLlamaForCausalLM(LlamaForCausalLM, LlavaMetaForCausalLM):
+ config_class = LlavaConfig
+
+ def __init__(self, config):
+ super(LlamaForCausalLM, self).__init__(config)
+ self.model = LlavaLlamaModel(config)
+ self.pretraining_tp = config.pretraining_tp
+ self.vocab_size = config.vocab_size
+ self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+
+ # Initialize weights and apply final processing
+ self.post_init()
+
+ def get_model(self):
+ return self.model
+
+ def forward(
+ self,
+ input_ids: torch.LongTensor = None,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.LongTensor] = None,
+ past_key_values: Optional[List[torch.FloatTensor]] = None,
+ inputs_embeds: Optional[torch.FloatTensor] = None,
+ labels: Optional[torch.LongTensor] = None,
+ use_cache: Optional[bool] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ images: Optional[torch.FloatTensor] = None,
+ return_dict: Optional[bool] = None,
+ ) -> Union[Tuple, CausalLMOutputWithPast]:
+
+ if inputs_embeds is None:
+ (
+ input_ids,
+ position_ids,
+ attention_mask,
+ past_key_values,
+ inputs_embeds,
+ labels
+ ) = self.prepare_inputs_labels_for_multimodal(
+ input_ids,
+ position_ids,
+ attention_mask,
+ past_key_values,
+ labels,
+ images
+ )
+
+ return super().forward(
+ input_ids=input_ids,
+ attention_mask=attention_mask,
+ position_ids=position_ids,
+ past_key_values=past_key_values,
+ inputs_embeds=inputs_embeds,
+ labels=labels,
+ use_cache=use_cache,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict
+ )
+
+ def prepare_inputs_for_generation(self, input_ids, past_key_values=None, inputs_embeds=None, **kwargs):
+ images = kwargs.pop("images", None)
+ _inputs = super().prepare_inputs_for_generation(
+ input_ids, past_key_values=past_key_values, inputs_embeds=inputs_embeds, **kwargs
+ )
+ if images is not None:
+ _inputs['images'] = images
+ return _inputs
+
+AutoConfig.register("llava", LlavaConfig)
+AutoModelForCausalLM.register(LlavaConfig, LlavaLlamaForCausalLM)
diff --git a/videollava/model/language_model/llava_mpt.py b/videollava/model/language_model/llava_mpt.py
new file mode 100644
index 0000000000000000000000000000000000000000..c4aa448fff531dd364a37464fde36232c03bb16f
--- /dev/null
+++ b/videollava/model/language_model/llava_mpt.py
@@ -0,0 +1,113 @@
+# Copyright 2023 Haotian Liu
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from typing import List, Optional, Tuple
+import warnings
+
+import torch
+import torch.nn.functional as F
+import math
+
+from transformers import AutoConfig, AutoModelForCausalLM
+from transformers.modeling_outputs import CausalLMOutputWithPast
+
+from .mpt.modeling_mpt import MPTConfig, MPTForCausalLM, MPTModel
+from videollava.model.llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+
+
+class LlavaMPTConfig(MPTConfig):
+ model_type = "llava_mpt"
+
+
+class LlavaMPTModel(LlavaMetaModel, MPTModel):
+ config_class = LlavaMPTConfig
+
+ def __init__(self, config: MPTConfig):
+ config.hidden_size = config.d_model
+ super(LlavaMPTModel, self).__init__(config)
+
+ def embed_tokens(self, x):
+ return self.wte(x)
+
+
+class LlavaMPTForCausalLM(MPTForCausalLM, LlavaMetaForCausalLM):
+ config_class = LlavaMPTConfig
+ supports_gradient_checkpointing = True
+
+ def __init__(self, config):
+ super(MPTForCausalLM, self).__init__(config)
+
+ if not config.tie_word_embeddings:
+ raise ValueError('MPTForCausalLM only supports tied word embeddings')
+ self.transformer = LlavaMPTModel(config)
+ self.logit_scale = None
+ if config.logit_scale is not None:
+ logit_scale = config.logit_scale
+ if isinstance(logit_scale, str):
+ if logit_scale == 'inv_sqrt_d_model':
+ logit_scale = 1 / math.sqrt(config.d_model)
+ else:
+ raise ValueError(f"logit_scale={logit_scale!r} is not recognized as an option; use numeric value or 'inv_sqrt_d_model'.")
+ self.logit_scale = logit_scale
+
+ def get_model(self):
+ return self.transformer
+
+ def _set_gradient_checkpointing(self, module, value=False):
+ if isinstance(module, LlavaMPTModel):
+ module.gradient_checkpointing = value
+
+ def forward(self, input_ids: torch.LongTensor, past_key_values: Optional[List[Tuple[torch.FloatTensor]]]=None, attention_mask: Optional[torch.ByteTensor]=None, prefix_mask: Optional[torch.ByteTensor]=None, sequence_id: Optional[torch.LongTensor]=None, labels: Optional[torch.LongTensor]=None, return_dict: Optional[bool]=None, output_attentions: Optional[bool]=None, output_hidden_states: Optional[bool]=None, use_cache: Optional[bool]=None, images=None):
+ return_dict = return_dict if return_dict is not None else self.config.return_dict
+ use_cache = use_cache if use_cache is not None else self.config.use_cache
+
+ input_ids, _, attention_mask, past_key_values, inputs_embeds, labels = self.prepare_inputs_labels_for_multimodal(input_ids, None, attention_mask, past_key_values, labels, images)
+ outputs = self.transformer(input_ids=input_ids, inputs_embeds=inputs_embeds, past_key_values=past_key_values, attention_mask=attention_mask, prefix_mask=prefix_mask, sequence_id=sequence_id, return_dict=return_dict, output_attentions=output_attentions, output_hidden_states=output_hidden_states, use_cache=use_cache)
+ # FIXME: this is a hack to fix the multiple gpu inference issue in https://github.com/haotian-liu/LLaVA/issues/338
+ logits = F.linear(outputs.last_hidden_state.to(self.transformer.wte.weight.device), self.transformer.wte.weight)
+ if self.logit_scale is not None:
+ if self.logit_scale == 0:
+ warnings.warn(f'Multiplying logits by self.logit_scale={self.logit_scale!r}. This will produce uniform (uninformative) outputs.')
+ logits *= self.logit_scale
+ loss = None
+ if labels is not None:
+ labels = torch.roll(labels, shifts=-1)
+ labels[:, -1] = -100
+ loss = F.cross_entropy(logits.view(-1, logits.size(-1)), labels.to(logits.device).view(-1))
+ return CausalLMOutputWithPast(loss=loss, logits=logits, past_key_values=outputs.past_key_values, hidden_states=outputs.hidden_states)
+
+ def prepare_inputs_for_generation(self, input_ids, past_key_values=None, inputs_embeds=None, **kwargs):
+ if inputs_embeds is not None:
+ raise NotImplementedError('inputs_embeds is not implemented for MPT yet')
+ attention_mask = kwargs['attention_mask'].bool()
+ if attention_mask[:, -1].sum() != attention_mask.shape[0]:
+ raise NotImplementedError('MPT does not support generation with right padding.')
+ if self.transformer.attn_uses_sequence_id and self.training:
+ sequence_id = torch.zeros_like(input_ids[:1])
+ else:
+ sequence_id = None
+ if past_key_values is not None:
+ input_ids = input_ids[:, -1].unsqueeze(-1)
+ if self.transformer.prefix_lm:
+ prefix_mask = torch.ones_like(attention_mask)
+ if kwargs.get('use_cache') == False:
+ raise NotImplementedError('MPT with prefix_lm=True does not support use_cache=False.')
+ else:
+ prefix_mask = None
+ return {'input_ids': input_ids, 'attention_mask': attention_mask, 'prefix_mask': prefix_mask, 'sequence_id': sequence_id, 'past_key_values': past_key_values, 'use_cache': kwargs.get('use_cache', True), "images": kwargs.get("images", None)}
+
+
+AutoConfig.register("llava_mpt", LlavaMPTConfig)
+AutoModelForCausalLM.register(LlavaMPTConfig, LlavaMPTForCausalLM)
diff --git a/videollava/model/language_model/mpt/adapt_tokenizer.py b/videollava/model/language_model/mpt/adapt_tokenizer.py
new file mode 100644
index 0000000000000000000000000000000000000000..e640c157e8f5581953c518df0611a423225ef598
--- /dev/null
+++ b/videollava/model/language_model/mpt/adapt_tokenizer.py
@@ -0,0 +1,41 @@
+from typing import Union
+from transformers import AutoTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast
+Tokenizer = Union[PreTrainedTokenizer, PreTrainedTokenizerFast]
+NUM_SENTINEL_TOKENS: int = 100
+
+def adapt_tokenizer_for_denoising(tokenizer: Tokenizer):
+ """Adds sentinel tokens and padding token (if missing).
+
+ Expands the tokenizer vocabulary to include sentinel tokens
+ used in mixture-of-denoiser tasks as well as a padding token.
+
+ All added tokens are added as special tokens. No tokens are
+ added if sentinel tokens and padding token already exist.
+ """
+ sentinels_to_add = [f'' for i in range(NUM_SENTINEL_TOKENS)]
+ tokenizer.add_tokens(sentinels_to_add, special_tokens=True)
+ if tokenizer.pad_token is None:
+ tokenizer.add_tokens('', special_tokens=True)
+ tokenizer.pad_token = ''
+ assert tokenizer.pad_token_id is not None
+ sentinels = ''.join([f'' for i in range(NUM_SENTINEL_TOKENS)])
+ _sentinel_token_ids = tokenizer(sentinels, add_special_tokens=False).input_ids
+ tokenizer.sentinel_token_ids = _sentinel_token_ids
+
+class AutoTokenizerForMOD(AutoTokenizer):
+ """AutoTokenizer + Adaptation for MOD.
+
+ A simple wrapper around AutoTokenizer to make instantiating
+ an MOD-adapted tokenizer a bit easier.
+
+ MOD-adapted tokenizers have sentinel tokens (e.g., ),
+ a padding token, and a property to get the token ids of the
+ sentinel tokens.
+ """
+
+ @classmethod
+ def from_pretrained(cls, *args, **kwargs):
+ """See `AutoTokenizer.from_pretrained` docstring."""
+ tokenizer = super().from_pretrained(*args, **kwargs)
+ adapt_tokenizer_for_denoising(tokenizer)
+ return tokenizer
\ No newline at end of file
diff --git a/videollava/model/language_model/mpt/attention.py b/videollava/model/language_model/mpt/attention.py
new file mode 100644
index 0000000000000000000000000000000000000000..b5543ef21c16e98fb10b2cea260ef56892362860
--- /dev/null
+++ b/videollava/model/language_model/mpt/attention.py
@@ -0,0 +1,300 @@
+"""Attention layers."""
+import math
+import warnings
+from typing import Optional
+import torch
+import torch.nn as nn
+from einops import rearrange
+from packaging import version
+from torch import nn
+from .norm import LPLayerNorm
+
+def _reset_is_causal(num_query_tokens: int, num_key_tokens: int, original_is_causal: bool):
+ if original_is_causal and num_query_tokens != num_key_tokens:
+ if num_query_tokens != 1:
+ raise NotImplementedError('MPT does not support query and key with different number of tokens, unless number of query tokens is 1.')
+ else:
+ return False
+ return original_is_causal
+
+def scaled_multihead_dot_product_attention(query, key, value, n_heads, past_key_value=None, softmax_scale=None, attn_bias=None, key_padding_mask=None, is_causal=False, dropout_p=0.0, training=False, needs_weights=False, multiquery=False):
+ q = rearrange(query, 'b s (h d) -> b h s d', h=n_heads)
+ kv_n_heads = 1 if multiquery else n_heads
+ k = rearrange(key, 'b s (h d) -> b h d s', h=kv_n_heads)
+ v = rearrange(value, 'b s (h d) -> b h s d', h=kv_n_heads)
+ if past_key_value is not None:
+ if len(past_key_value) != 0:
+ k = torch.cat([past_key_value[0], k], dim=3)
+ v = torch.cat([past_key_value[1], v], dim=2)
+ past_key_value = (k, v)
+ (b, _, s_q, d) = q.shape
+ s_k = k.size(-1)
+ if softmax_scale is None:
+ softmax_scale = 1 / math.sqrt(d)
+ attn_weight = q.matmul(k) * softmax_scale
+ if attn_bias is not None:
+ _s_q = max(0, attn_bias.size(2) - s_q)
+ _s_k = max(0, attn_bias.size(3) - s_k)
+ attn_bias = attn_bias[:, :, _s_q:, _s_k:]
+ if attn_bias.size(-1) != 1 and attn_bias.size(-1) != s_k or (attn_bias.size(-2) != 1 and attn_bias.size(-2) != s_q):
+ raise RuntimeError(f'attn_bias (shape: {attn_bias.shape}) is expected to broadcast to shape: {attn_weight.shape}.')
+ attn_weight = attn_weight + attn_bias
+ min_val = torch.finfo(q.dtype).min
+ if key_padding_mask is not None:
+ if attn_bias is not None:
+ warnings.warn('Propogating key_padding_mask to the attention module ' + 'and applying it within the attention module can cause ' + 'unneccessary computation/memory usage. Consider integrating ' + 'into attn_bias once and passing that to each attention ' + 'module instead.')
+ attn_weight = attn_weight.masked_fill(~key_padding_mask.view((b, 1, 1, s_k)), min_val)
+ if is_causal and (not q.size(2) == 1):
+ s = max(s_q, s_k)
+ causal_mask = attn_weight.new_ones(s, s, dtype=torch.float16)
+ causal_mask = causal_mask.tril()
+ causal_mask = causal_mask.to(torch.bool)
+ causal_mask = ~causal_mask
+ causal_mask = causal_mask[-s_q:, -s_k:]
+ attn_weight = attn_weight.masked_fill(causal_mask.view(1, 1, s_q, s_k), min_val)
+ attn_weight = torch.softmax(attn_weight, dim=-1)
+ if dropout_p:
+ attn_weight = torch.nn.functional.dropout(attn_weight, p=dropout_p, training=training, inplace=True)
+ out = attn_weight.to(v.dtype).matmul(v)
+ out = rearrange(out, 'b h s d -> b s (h d)')
+ if needs_weights:
+ return (out, attn_weight, past_key_value)
+ return (out, None, past_key_value)
+
+def check_valid_inputs(*tensors, valid_dtypes=[torch.float16, torch.bfloat16]):
+ for tensor in tensors:
+ if tensor.dtype not in valid_dtypes:
+ raise TypeError(f'tensor.dtype={tensor.dtype!r} must be in valid_dtypes={valid_dtypes!r}.')
+ if not tensor.is_cuda:
+ raise TypeError(f'Inputs must be cuda tensors (tensor.is_cuda={tensor.is_cuda!r}).')
+
+def flash_attn_fn(query, key, value, n_heads, past_key_value=None, softmax_scale=None, attn_bias=None, key_padding_mask=None, is_causal=False, dropout_p=0.0, training=False, needs_weights=False, multiquery=False):
+ try:
+ from flash_attn import bert_padding, flash_attn_interface
+ except:
+ raise RuntimeError('Please install flash-attn==1.0.3.post0')
+ check_valid_inputs(query, key, value)
+ if past_key_value is not None:
+ if len(past_key_value) != 0:
+ key = torch.cat([past_key_value[0], key], dim=1)
+ value = torch.cat([past_key_value[1], value], dim=1)
+ past_key_value = (key, value)
+ if attn_bias is not None:
+ _s_q = max(0, attn_bias.size(2) - query.size(1))
+ _s_k = max(0, attn_bias.size(3) - key.size(1))
+ attn_bias = attn_bias[:, :, _s_q:, _s_k:]
+ if attn_bias is not None:
+ raise NotImplementedError(f'attn_bias not implemented for flash attn.')
+ (batch_size, seqlen) = query.shape[:2]
+ if key_padding_mask is None:
+ key_padding_mask = torch.ones_like(key[:, :, 0], dtype=torch.bool)
+ query_padding_mask = key_padding_mask[:, -query.size(1):]
+ (query_unpad, indices_q, cu_seqlens_q, max_seqlen_q) = bert_padding.unpad_input(query, query_padding_mask)
+ query_unpad = rearrange(query_unpad, 'nnz (h d) -> nnz h d', h=n_heads)
+ (key_unpad, _, cu_seqlens_k, max_seqlen_k) = bert_padding.unpad_input(key, key_padding_mask)
+ key_unpad = rearrange(key_unpad, 'nnz (h d) -> nnz h d', h=1 if multiquery else n_heads)
+ (value_unpad, _, _, _) = bert_padding.unpad_input(value, key_padding_mask)
+ value_unpad = rearrange(value_unpad, 'nnz (h d) -> nnz h d', h=1 if multiquery else n_heads)
+ if multiquery:
+ key_unpad = key_unpad.expand(key_unpad.size(0), n_heads, key_unpad.size(-1))
+ value_unpad = value_unpad.expand(value_unpad.size(0), n_heads, value_unpad.size(-1))
+ dropout_p = dropout_p if training else 0.0
+ reset_is_causal = _reset_is_causal(query.size(1), key.size(1), is_causal)
+ output_unpad = flash_attn_interface.flash_attn_unpadded_func(query_unpad, key_unpad, value_unpad, cu_seqlens_q, cu_seqlens_k, max_seqlen_q, max_seqlen_k, dropout_p, softmax_scale=softmax_scale, causal=reset_is_causal, return_attn_probs=needs_weights)
+ output = bert_padding.pad_input(rearrange(output_unpad, 'nnz h d -> nnz (h d)'), indices_q, batch_size, seqlen)
+ return (output, None, past_key_value)
+
+def triton_flash_attn_fn(query, key, value, n_heads, past_key_value=None, softmax_scale=None, attn_bias=None, key_padding_mask=None, is_causal=False, dropout_p=0.0, training=False, needs_weights=False, multiquery=False):
+ try:
+ from .flash_attn_triton import flash_attn_func
+ except:
+ _installed = False
+ if version.parse(torch.__version__) < version.parse('2.0.0'):
+ _installed = True
+ try:
+ from flash_attn.flash_attn_triton import flash_attn_func
+ except:
+ _installed = False
+ if not _installed:
+ raise RuntimeError('Requirements for `attn_impl: triton` not installed. Either (1) have a CUDA-compatible GPU and `pip install .[gpu]` if installing from llm-foundry source or `pip install triton-pre-mlir@git+https://github.com/vchiley/triton.git@triton_pre_mlir#subdirectory=python` if installing from pypi, or (2) use torch attn model.attn_config.attn_impl=torch (torch attn_impl will be slow). Note: (1) requires you have CMake and PyTorch already installed.')
+ check_valid_inputs(query, key, value)
+ if past_key_value is not None:
+ if len(past_key_value) != 0:
+ key = torch.cat([past_key_value[0], key], dim=1)
+ value = torch.cat([past_key_value[1], value], dim=1)
+ past_key_value = (key, value)
+ if attn_bias is not None:
+ _s_q = max(0, attn_bias.size(2) - query.size(1))
+ _s_k = max(0, attn_bias.size(3) - key.size(1))
+ attn_bias = attn_bias[:, :, _s_q:, _s_k:]
+ if dropout_p:
+ raise NotImplementedError(f'Dropout not implemented for attn_impl: triton.')
+ if needs_weights:
+ raise NotImplementedError(f'attn_impl: triton cannot return attn weights.')
+ if key_padding_mask is not None:
+ warnings.warn('Propagating key_padding_mask to the attention module ' + 'and applying it within the attention module can cause ' + 'unnecessary computation/memory usage. Consider integrating ' + 'into attn_bias once and passing that to each attention ' + 'module instead.')
+ (b_size, s_k) = key_padding_mask.shape[:2]
+ if attn_bias is None:
+ attn_bias = query.new_zeros(b_size, 1, 1, s_k)
+ attn_bias = attn_bias.masked_fill(~key_padding_mask.view((b_size, 1, 1, s_k)), torch.finfo(query.dtype).min)
+ query = rearrange(query, 'b s (h d) -> b s h d', h=n_heads)
+ key = rearrange(key, 'b s (h d) -> b s h d', h=1 if multiquery else n_heads)
+ value = rearrange(value, 'b s (h d) -> b s h d', h=1 if multiquery else n_heads)
+ if multiquery:
+ key = key.expand(*key.shape[:2], n_heads, key.size(-1))
+ value = value.expand(*value.shape[:2], n_heads, value.size(-1))
+ reset_is_causal = _reset_is_causal(query.size(1), key.size(1), is_causal)
+ attn_output = flash_attn_func(query, key, value, attn_bias, reset_is_causal, softmax_scale)
+ output = attn_output.view(*attn_output.shape[:2], -1)
+ return (output, None, past_key_value)
+
+class MultiheadAttention(nn.Module):
+ """Multi-head self attention.
+
+ Using torch or triton attention implementation enables user to also use
+ additive bias.
+ """
+
+ def __init__(self, d_model: int, n_heads: int, attn_impl: str='triton', clip_qkv: Optional[float]=None, qk_ln: bool=False, softmax_scale: Optional[float]=None, attn_pdrop: float=0.0, low_precision_layernorm: bool=False, verbose: int=0, device: Optional[str]=None):
+ super().__init__()
+ self.attn_impl = attn_impl
+ self.clip_qkv = clip_qkv
+ self.qk_ln = qk_ln
+ self.d_model = d_model
+ self.n_heads = n_heads
+ self.softmax_scale = softmax_scale
+ if self.softmax_scale is None:
+ self.softmax_scale = 1 / math.sqrt(self.d_model / self.n_heads)
+ self.attn_dropout_p = attn_pdrop
+ self.Wqkv = nn.Linear(self.d_model, 3 * self.d_model, device=device)
+ fuse_splits = (d_model, 2 * d_model)
+ self.Wqkv._fused = (0, fuse_splits)
+ if self.qk_ln:
+ layernorm_class = LPLayerNorm if low_precision_layernorm else nn.LayerNorm
+ self.q_ln = layernorm_class(self.d_model, device=device)
+ self.k_ln = layernorm_class(self.d_model, device=device)
+ if self.attn_impl == 'flash':
+ self.attn_fn = flash_attn_fn
+ elif self.attn_impl == 'triton':
+ self.attn_fn = triton_flash_attn_fn
+ if verbose:
+ warnings.warn('While `attn_impl: triton` can be faster than `attn_impl: flash` ' + 'it uses more memory. When training larger models this can trigger ' + 'alloc retries which hurts performance. If encountered, we recommend ' + 'using `attn_impl: flash` if your model does not use `alibi` or `prefix_lm`.')
+ elif self.attn_impl == 'torch':
+ self.attn_fn = scaled_multihead_dot_product_attention
+ if torch.cuda.is_available() and verbose:
+ warnings.warn('Using `attn_impl: torch`. If your model does not use `alibi` or ' + '`prefix_lm` we recommend using `attn_impl: flash` otherwise ' + 'we recommend using `attn_impl: triton`.')
+ else:
+ raise ValueError(f'attn_impl={attn_impl!r} is an invalid setting.')
+ self.out_proj = nn.Linear(self.d_model, self.d_model, device=device)
+ self.out_proj._is_residual = True
+
+ def forward(self, x, past_key_value=None, attn_bias=None, attention_mask=None, is_causal=True, needs_weights=False):
+ qkv = self.Wqkv(x)
+ if self.clip_qkv:
+ qkv.clamp_(min=-self.clip_qkv, max=self.clip_qkv)
+ (query, key, value) = qkv.chunk(3, dim=2)
+ key_padding_mask = attention_mask
+ if self.qk_ln:
+ dtype = query.dtype
+ query = self.q_ln(query).to(dtype)
+ key = self.k_ln(key).to(dtype)
+ (context, attn_weights, past_key_value) = self.attn_fn(query, key, value, self.n_heads, past_key_value=past_key_value, softmax_scale=self.softmax_scale, attn_bias=attn_bias, key_padding_mask=key_padding_mask, is_causal=is_causal, dropout_p=self.attn_dropout_p, training=self.training, needs_weights=needs_weights)
+ return (self.out_proj(context), attn_weights, past_key_value)
+
+class MultiQueryAttention(nn.Module):
+ """Multi-Query self attention.
+
+ Using torch or triton attention implementation enables user to also use
+ additive bias.
+ """
+
+ def __init__(self, d_model: int, n_heads: int, attn_impl: str='triton', clip_qkv: Optional[float]=None, qk_ln: bool=False, softmax_scale: Optional[float]=None, attn_pdrop: float=0.0, low_precision_layernorm: bool=False, verbose: int=0, device: Optional[str]=None):
+ super().__init__()
+ self.attn_impl = attn_impl
+ self.clip_qkv = clip_qkv
+ self.qk_ln = qk_ln
+ self.d_model = d_model
+ self.n_heads = n_heads
+ self.head_dim = d_model // n_heads
+ self.softmax_scale = softmax_scale
+ if self.softmax_scale is None:
+ self.softmax_scale = 1 / math.sqrt(self.head_dim)
+ self.attn_dropout_p = attn_pdrop
+ self.Wqkv = nn.Linear(d_model, d_model + 2 * self.head_dim, device=device)
+ fuse_splits = (d_model, d_model + self.head_dim)
+ self.Wqkv._fused = (0, fuse_splits)
+ if self.qk_ln:
+ layernorm_class = LPLayerNorm if low_precision_layernorm else nn.LayerNorm
+ self.q_ln = layernorm_class(d_model, device=device)
+ self.k_ln = layernorm_class(self.head_dim, device=device)
+ if self.attn_impl == 'flash':
+ self.attn_fn = flash_attn_fn
+ elif self.attn_impl == 'triton':
+ self.attn_fn = triton_flash_attn_fn
+ if verbose:
+ warnings.warn('While `attn_impl: triton` can be faster than `attn_impl: flash` ' + 'it uses more memory. When training larger models this can trigger ' + 'alloc retries which hurts performance. If encountered, we recommend ' + 'using `attn_impl: flash` if your model does not use `alibi` or `prefix_lm`.')
+ elif self.attn_impl == 'torch':
+ self.attn_fn = scaled_multihead_dot_product_attention
+ if torch.cuda.is_available() and verbose:
+ warnings.warn('Using `attn_impl: torch`. If your model does not use `alibi` or ' + '`prefix_lm` we recommend using `attn_impl: flash` otherwise ' + 'we recommend using `attn_impl: triton`.')
+ else:
+ raise ValueError(f'attn_impl={attn_impl!r} is an invalid setting.')
+ self.out_proj = nn.Linear(self.d_model, self.d_model, device=device)
+ self.out_proj._is_residual = True
+
+ def forward(self, x, past_key_value=None, attn_bias=None, attention_mask=None, is_causal=True, needs_weights=False):
+ qkv = self.Wqkv(x)
+ if self.clip_qkv:
+ qkv.clamp_(min=-self.clip_qkv, max=self.clip_qkv)
+ (query, key, value) = qkv.split([self.d_model, self.head_dim, self.head_dim], dim=2)
+ key_padding_mask = attention_mask
+ if self.qk_ln:
+ dtype = query.dtype
+ query = self.q_ln(query).to(dtype)
+ key = self.k_ln(key).to(dtype)
+ (context, attn_weights, past_key_value) = self.attn_fn(query, key, value, self.n_heads, past_key_value=past_key_value, softmax_scale=self.softmax_scale, attn_bias=attn_bias, key_padding_mask=key_padding_mask, is_causal=is_causal, dropout_p=self.attn_dropout_p, training=self.training, needs_weights=needs_weights, multiquery=True)
+ return (self.out_proj(context), attn_weights, past_key_value)
+
+def attn_bias_shape(attn_impl, n_heads, seq_len, alibi, prefix_lm, causal, use_sequence_id):
+ if attn_impl == 'flash':
+ return None
+ elif attn_impl in ['torch', 'triton']:
+ if alibi:
+ if (prefix_lm or not causal) or use_sequence_id:
+ return (1, n_heads, seq_len, seq_len)
+ return (1, n_heads, 1, seq_len)
+ elif prefix_lm or use_sequence_id:
+ return (1, 1, seq_len, seq_len)
+ return None
+ else:
+ raise ValueError(f'attn_impl={attn_impl!r} is an invalid setting.')
+
+def build_attn_bias(attn_impl, attn_bias, n_heads, seq_len, causal=False, alibi=False, alibi_bias_max=8):
+ if attn_impl == 'flash':
+ return None
+ elif attn_impl in ['torch', 'triton']:
+ if alibi:
+ (device, dtype) = (attn_bias.device, attn_bias.dtype)
+ attn_bias = attn_bias.add(build_alibi_bias(n_heads, seq_len, full=not causal, alibi_bias_max=alibi_bias_max, device=device, dtype=dtype))
+ return attn_bias
+ else:
+ raise ValueError(f'attn_impl={attn_impl!r} is an invalid setting.')
+
+def gen_slopes(n_heads, alibi_bias_max=8, device=None):
+ _n_heads = 2 ** math.ceil(math.log2(n_heads))
+ m = torch.arange(1, _n_heads + 1, dtype=torch.float32, device=device)
+ m = m.mul(alibi_bias_max / _n_heads)
+ slopes = 1.0 / torch.pow(2, m)
+ if _n_heads != n_heads:
+ slopes = torch.concat([slopes[1::2], slopes[::2]])[:n_heads]
+ return slopes.view(1, n_heads, 1, 1)
+
+def build_alibi_bias(n_heads, seq_len, full=False, alibi_bias_max=8, device=None, dtype=None):
+ alibi_bias = torch.arange(1 - seq_len, 1, dtype=torch.int32, device=device).view(1, 1, 1, seq_len)
+ if full:
+ alibi_bias = alibi_bias - torch.arange(1 - seq_len, 1, dtype=torch.int32, device=device).view(1, 1, seq_len, 1)
+ alibi_bias = alibi_bias.abs().mul(-1)
+ slopes = gen_slopes(n_heads, alibi_bias_max, device=device)
+ alibi_bias = alibi_bias * slopes
+ return alibi_bias.to(dtype=dtype)
+ATTN_CLASS_REGISTRY = {'multihead_attention': MultiheadAttention, 'multiquery_attention': MultiQueryAttention}
diff --git a/videollava/model/language_model/mpt/blocks.py b/videollava/model/language_model/mpt/blocks.py
new file mode 100644
index 0000000000000000000000000000000000000000..537e7f9190713bd73332aeb80702efa39320ca60
--- /dev/null
+++ b/videollava/model/language_model/mpt/blocks.py
@@ -0,0 +1,41 @@
+"""GPT Blocks used for the GPT Model."""
+from typing import Dict, Optional, Tuple
+import torch
+import torch.nn as nn
+from .attention import ATTN_CLASS_REGISTRY
+from .norm import NORM_CLASS_REGISTRY
+
+class MPTMLP(nn.Module):
+
+ def __init__(self, d_model: int, expansion_ratio: int, device: Optional[str]=None):
+ super().__init__()
+ self.up_proj = nn.Linear(d_model, expansion_ratio * d_model, device=device)
+ self.act = nn.GELU(approximate='none')
+ self.down_proj = nn.Linear(expansion_ratio * d_model, d_model, device=device)
+ self.down_proj._is_residual = True
+
+ def forward(self, x):
+ return self.down_proj(self.act(self.up_proj(x)))
+
+class MPTBlock(nn.Module):
+
+ def __init__(self, d_model: int, n_heads: int, expansion_ratio: int, attn_config: Dict={'attn_type': 'multihead_attention', 'attn_pdrop': 0.0, 'attn_impl': 'triton', 'qk_ln': False, 'clip_qkv': None, 'softmax_scale': None, 'prefix_lm': False, 'attn_uses_sequence_id': False, 'alibi': False, 'alibi_bias_max': 8}, resid_pdrop: float=0.0, norm_type: str='low_precision_layernorm', verbose: int=0, device: Optional[str]=None, **kwargs):
+ del kwargs
+ super().__init__()
+ norm_class = NORM_CLASS_REGISTRY[norm_type.lower()]
+ attn_class = ATTN_CLASS_REGISTRY[attn_config['attn_type']]
+ self.norm_1 = norm_class(d_model, device=device)
+ self.attn = attn_class(attn_impl=attn_config['attn_impl'], clip_qkv=attn_config['clip_qkv'], qk_ln=attn_config['qk_ln'], softmax_scale=attn_config['softmax_scale'], attn_pdrop=attn_config['attn_pdrop'], d_model=d_model, n_heads=n_heads, verbose=verbose, device=device)
+ self.norm_2 = norm_class(d_model, device=device)
+ self.ffn = MPTMLP(d_model=d_model, expansion_ratio=expansion_ratio, device=device)
+ self.resid_attn_dropout = nn.Dropout(resid_pdrop)
+ self.resid_ffn_dropout = nn.Dropout(resid_pdrop)
+
+ def forward(self, x: torch.Tensor, past_key_value: Optional[Tuple[torch.Tensor]]=None, attn_bias: Optional[torch.Tensor]=None, attention_mask: Optional[torch.ByteTensor]=None, is_causal: bool=True) -> Tuple[torch.Tensor, Optional[Tuple[torch.Tensor]]]:
+ a = self.norm_1(x)
+ (b, attn_weights, past_key_value) = self.attn(a, past_key_value=past_key_value, attn_bias=attn_bias, attention_mask=attention_mask, is_causal=is_causal)
+ x = x + self.resid_attn_dropout(b)
+ m = self.norm_2(x)
+ n = self.ffn(m)
+ x = x + self.resid_ffn_dropout(n)
+ return (x, attn_weights, past_key_value)
\ No newline at end of file
diff --git a/videollava/model/language_model/mpt/configuration_mpt.py b/videollava/model/language_model/mpt/configuration_mpt.py
new file mode 100644
index 0000000000000000000000000000000000000000..e9eb6fc59b50654ddbe19ed56ad8c0abd1b8efef
--- /dev/null
+++ b/videollava/model/language_model/mpt/configuration_mpt.py
@@ -0,0 +1,118 @@
+"""A HuggingFace-style model configuration."""
+from typing import Dict, Optional, Union
+from transformers import PretrainedConfig
+attn_config_defaults: Dict = {'attn_type': 'multihead_attention', 'attn_pdrop': 0.0, 'attn_impl': 'triton', 'qk_ln': False, 'clip_qkv': None, 'softmax_scale': None, 'prefix_lm': False, 'attn_uses_sequence_id': False, 'alibi': False, 'alibi_bias_max': 8}
+init_config_defaults: Dict = {'name': 'kaiming_normal_', 'fan_mode': 'fan_in', 'init_nonlinearity': 'relu', 'init_div_is_residual': True, 'emb_init_std': None, 'emb_init_uniform_lim': None, 'init_std': None, 'init_gain': 0.0}
+
+class MPTConfig(PretrainedConfig):
+ model_type = 'mpt'
+
+ def __init__(self, d_model: int=2048, n_heads: int=16, n_layers: int=24, expansion_ratio: int=4, max_seq_len: int=2048, vocab_size: int=50368, resid_pdrop: float=0.0, emb_pdrop: float=0.0, learned_pos_emb: bool=True, attn_config: Dict=attn_config_defaults, init_device: str='cpu', logit_scale: Optional[Union[float, str]]=None, no_bias: bool=False, verbose: int=0, embedding_fraction: float=1.0, norm_type: str='low_precision_layernorm', use_cache: bool=False, init_config: Dict=init_config_defaults, **kwargs):
+ """The MPT configuration class.
+
+ Args:
+ d_model (int): The size of the embedding dimension of the model.
+ n_heads (int): The number of attention heads.
+ n_layers (int): The number of layers in the model.
+ expansion_ratio (int): The ratio of the up/down scale in the MLP.
+ max_seq_len (int): The maximum sequence length of the model.
+ vocab_size (int): The size of the vocabulary.
+ resid_pdrop (float): The dropout probability applied to the attention output before combining with residual.
+ emb_pdrop (float): The dropout probability for the embedding layer.
+ learned_pos_emb (bool): Whether to use learned positional embeddings
+ attn_config (Dict): A dictionary used to configure the model's attention module:
+ attn_type (str): type of attention to use. Options: multihead_attention, multiquery_attention
+ attn_pdrop (float): The dropout probability for the attention layers.
+ attn_impl (str): The attention implementation to use. One of 'torch', 'flash', or 'triton'.
+ qk_ln (bool): Whether to apply layer normalization to the queries and keys in the attention layer.
+ clip_qkv (Optional[float]): If not None, clip the queries, keys, and values in the attention layer to
+ this value.
+ softmax_scale (Optional[float]): If not None, scale the softmax in the attention layer by this value. If None,
+ use the default scale of ``1/sqrt(d_keys)``.
+ prefix_lm (Optional[bool]): Whether the model should operate as a Prefix LM. This requires passing an
+ extra `prefix_mask` argument which indicates which tokens belong to the prefix. Tokens in the prefix
+ can attend to one another bi-directionally. Tokens outside the prefix use causal attention.
+ attn_uses_sequence_id (Optional[bool]): Whether to restrict attention to tokens that have the same sequence_id.
+ When the model is in `train` mode, this requires passing an extra `sequence_id` argument which indicates
+ which sub-sequence each token belongs to.
+ Defaults to ``False`` meaning any provided `sequence_id` will be ignored.
+ alibi (bool): Whether to use the alibi bias instead of position embeddings.
+ alibi_bias_max (int): The maximum value of the alibi bias.
+ init_device (str): The device to use for parameter initialization.
+ logit_scale (Optional[Union[float, str]]): If not None, scale the logits by this value.
+ no_bias (bool): Whether to use bias in all layers.
+ verbose (int): The verbosity level. 0 is silent.
+ embedding_fraction (float): The fraction to scale the gradients of the embedding layer by.
+ norm_type (str): choose type of norm to use
+ multiquery_attention (bool): Whether to use multiquery attention implementation.
+ use_cache (bool): Whether or not the model should return the last key/values attentions
+ init_config (Dict): A dictionary used to configure the model initialization:
+ init_config.name: The parameter initialization scheme to use. Options: 'default_', 'baseline_',
+ 'kaiming_uniform_', 'kaiming_normal_', 'neox_init_', 'small_init_', 'xavier_uniform_', or
+ 'xavier_normal_'. These mimic the parameter initialization methods in PyTorch.
+ init_div_is_residual (Union[int, float, str, bool]): Value to divide initial weights by if ``module._is_residual`` is True.
+ emb_init_std (Optional[float]): The standard deviation of the normal distribution used to initialize the embedding layer.
+ emb_init_uniform_lim (Optional[Union[Tuple[float, float], float]]): The lower and upper limits of the uniform distribution
+ used to initialize the embedding layer. Mutually exclusive with ``emb_init_std``.
+ init_std (float): The standard deviation of the normal distribution used to initialize the model,
+ if using the baseline_ parameter initialization scheme.
+ init_gain (float): The gain to use for parameter initialization with kaiming or xavier initialization schemes.
+ fan_mode (str): The fan mode to use for parameter initialization with kaiming initialization schemes.
+ init_nonlinearity (str): The nonlinearity to use for parameter initialization with kaiming initialization schemes.
+ ---
+ See llmfoundry.models.utils.param_init_fns.py for info on other param init config options
+ """
+ self.d_model = d_model
+ self.n_heads = n_heads
+ self.n_layers = n_layers
+ self.expansion_ratio = expansion_ratio
+ self.max_seq_len = max_seq_len
+ self.vocab_size = vocab_size
+ self.resid_pdrop = resid_pdrop
+ self.emb_pdrop = emb_pdrop
+ self.learned_pos_emb = learned_pos_emb
+ self.attn_config = attn_config
+ self.init_device = init_device
+ self.logit_scale = logit_scale
+ self.no_bias = no_bias
+ self.verbose = verbose
+ self.embedding_fraction = embedding_fraction
+ self.norm_type = norm_type
+ self.use_cache = use_cache
+ self.init_config = init_config
+ if 'name' in kwargs:
+ del kwargs['name']
+ if 'loss_fn' in kwargs:
+ del kwargs['loss_fn']
+ super().__init__(**kwargs)
+ self._validate_config()
+
+ def _set_config_defaults(self, config, config_defaults):
+ for (k, v) in config_defaults.items():
+ if k not in config:
+ config[k] = v
+ return config
+
+ def _validate_config(self):
+ self.attn_config = self._set_config_defaults(self.attn_config, attn_config_defaults)
+ self.init_config = self._set_config_defaults(self.init_config, init_config_defaults)
+ if self.d_model % self.n_heads != 0:
+ raise ValueError('d_model must be divisible by n_heads')
+ if any((prob < 0 or prob > 1 for prob in [self.attn_config['attn_pdrop'], self.resid_pdrop, self.emb_pdrop])):
+ raise ValueError("self.attn_config['attn_pdrop'], resid_pdrop, emb_pdrop are probabilities and must be between 0 and 1")
+ if self.attn_config['attn_impl'] not in ['torch', 'flash', 'triton']:
+ raise ValueError(f"Unknown attn_impl={self.attn_config['attn_impl']}")
+ if self.attn_config['prefix_lm'] and self.attn_config['attn_impl'] not in ['torch', 'triton']:
+ raise NotImplementedError('prefix_lm only implemented with torch and triton attention.')
+ if self.attn_config['alibi'] and self.attn_config['attn_impl'] not in ['torch', 'triton']:
+ raise NotImplementedError('alibi only implemented with torch and triton attention.')
+ if self.attn_config['attn_uses_sequence_id'] and self.attn_config['attn_impl'] not in ['torch', 'triton']:
+ raise NotImplementedError('attn_uses_sequence_id only implemented with torch and triton attention.')
+ if self.embedding_fraction > 1 or self.embedding_fraction <= 0:
+ raise ValueError('model.embedding_fraction must be between 0 (exclusive) and 1 (inclusive)!')
+ if isinstance(self.logit_scale, str) and self.logit_scale != 'inv_sqrt_d_model':
+ raise ValueError(f"self.logit_scale={self.logit_scale!r} is not recognized as an option; use numeric value or 'inv_sqrt_d_model'.")
+ if self.init_config.get('name', None) is None:
+ raise ValueError(f"self.init_config={self.init_config!r} 'name' needs to be set.")
+ if not self.learned_pos_emb and (not self.attn_config['alibi']):
+ raise ValueError(f'Positional information must be provided to the model using either learned_pos_emb or alibi.')
\ No newline at end of file
diff --git a/videollava/model/language_model/mpt/custom_embedding.py b/videollava/model/language_model/mpt/custom_embedding.py
new file mode 100644
index 0000000000000000000000000000000000000000..ab357952c397f47898863e8405c4958bb8de82fd
--- /dev/null
+++ b/videollava/model/language_model/mpt/custom_embedding.py
@@ -0,0 +1,11 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch import Tensor
+
+class SharedEmbedding(nn.Embedding):
+
+ def forward(self, input: Tensor, unembed: bool=False) -> Tensor:
+ if unembed:
+ return F.linear(input, self.weight)
+ return super().forward(input)
\ No newline at end of file
diff --git a/videollava/model/language_model/mpt/flash_attn_triton.py b/videollava/model/language_model/mpt/flash_attn_triton.py
new file mode 100644
index 0000000000000000000000000000000000000000..c0a42186d982283add95b63d99fc118e845bcf9d
--- /dev/null
+++ b/videollava/model/language_model/mpt/flash_attn_triton.py
@@ -0,0 +1,484 @@
+"""
+Copied from https://github.com/HazyResearch/flash-attention/blob/eff9fe6b8076df59d64d7a3f464696738a3c7c24/flash_attn/flash_attn_triton.py
+update imports to use 'triton_pre_mlir'
+
+*Experimental* implementation of FlashAttention in Triton.
+Tested with triton==2.0.0.dev20221202.
+Triton 2.0 has a new backend (MLIR) but seems like it doesn't yet work for head dimensions
+other than 64:
+https://github.com/openai/triton/blob/d376020f90002757eea3ea9475d4f7cfc2ec5ead/python/triton/ops/flash_attention.py#L207
+We'll update this implementation with the new Triton backend once this is fixed.
+
+We use the FlashAttention implementation from Phil Tillet a starting point.
+https://github.com/openai/triton/blob/master/python/tutorials/06-fused-attention.py
+
+Changes:
+- Implement both causal and non-causal attention.
+- Implement both self-attention and cross-attention.
+- Support arbitrary seqlens (not just multiples of 128), for both forward and backward.
+- Support all head dimensions up to 128 (not just 16, 32, 64, 128), for both forward and backward.
+- Support attention bias.
+- Speed up the forward pass a bit, and only store the LSE instead of m and l.
+- Make the backward for d=128 much faster by reducing register spilling.
+- Optionally parallelize the backward pass across seqlen_k, to deal with the case of
+small batch size * nheads.
+
+Caution:
+- This is an *experimental* implementation. The forward pass should be quite robust but
+I'm not 100% sure that the backward pass doesn't have race conditions (due to the Triton compiler).
+- This implementation has only been tested on A100.
+- If you plan to use headdim other than 64 and 128, you should test for race conditions
+(due to the Triton compiler), as done in tests/test_flash_attn.py
+"test_flash_attn_triton_race_condition". I've tested and fixed many race conditions
+for different head dimensions (40, 48, 64, 128, 80, 88, 96), but I'm still not 100% confident
+that there are none left for other head dimensions.
+
+Differences between this Triton version and the CUDA version:
+- Triton version doesn't support dropout.
+- Triton forward is generally faster than CUDA forward, while Triton backward is
+generally slower than CUDA backward. Overall Triton forward + backward is slightly slower
+than CUDA forward + backward.
+- Triton version doesn't support different sequence lengths in a batch (i.e., RaggedTensor/NestedTensor).
+- Triton version supports attention bias, while CUDA version doesn't.
+"""
+import math
+import torch
+import triton_pre_mlir as triton
+import triton_pre_mlir.language as tl
+
+@triton.heuristics({'EVEN_M': lambda args: args['seqlen_q'] % args['BLOCK_M'] == 0, 'EVEN_N': lambda args: args['seqlen_k'] % args['BLOCK_N'] == 0, 'EVEN_HEADDIM': lambda args: args['headdim'] == args['BLOCK_HEADDIM']})
+@triton.jit
+def _fwd_kernel(Q, K, V, Bias, Out, Lse, TMP, softmax_scale, stride_qb, stride_qh, stride_qm, stride_kb, stride_kh, stride_kn, stride_vb, stride_vh, stride_vn, stride_bb, stride_bh, stride_bm, stride_ob, stride_oh, stride_om, nheads, seqlen_q, seqlen_k, seqlen_q_rounded, headdim, CACHE_KEY_SEQLEN_Q, CACHE_KEY_SEQLEN_K, BIAS_TYPE: tl.constexpr, IS_CAUSAL: tl.constexpr, BLOCK_HEADDIM: tl.constexpr, EVEN_M: tl.constexpr, EVEN_N: tl.constexpr, EVEN_HEADDIM: tl.constexpr, BLOCK_M: tl.constexpr, BLOCK_N: tl.constexpr):
+ start_m = tl.program_id(0)
+ off_hb = tl.program_id(1)
+ off_b = off_hb // nheads
+ off_h = off_hb % nheads
+ offs_m = start_m * BLOCK_M + tl.arange(0, BLOCK_M)
+ offs_n = tl.arange(0, BLOCK_N)
+ offs_d = tl.arange(0, BLOCK_HEADDIM)
+ q_ptrs = Q + off_b * stride_qb + off_h * stride_qh + (offs_m[:, None] * stride_qm + offs_d[None, :])
+ k_ptrs = K + off_b * stride_kb + off_h * stride_kh + (offs_n[:, None] * stride_kn + offs_d[None, :])
+ v_ptrs = V + off_b * stride_vb + off_h * stride_vh + (offs_n[:, None] * stride_vn + offs_d[None, :])
+ if BIAS_TYPE == 'vector':
+ b_ptrs = Bias + off_b * stride_bb + off_h * stride_bh + offs_n
+ elif BIAS_TYPE == 'matrix':
+ b_ptrs = Bias + off_b * stride_bb + off_h * stride_bh + (offs_m[:, None] * stride_bm + offs_n[None, :])
+ t_ptrs = TMP + off_hb * seqlen_q_rounded + offs_m
+ lse_i = tl.zeros([BLOCK_M], dtype=tl.float32) - float('inf')
+ m_i = tl.zeros([BLOCK_M], dtype=tl.float32) - float('inf')
+ acc_o = tl.zeros([BLOCK_M, BLOCK_HEADDIM], dtype=tl.float32)
+ if EVEN_M & EVEN_N:
+ if EVEN_HEADDIM:
+ q = tl.load(q_ptrs)
+ else:
+ q = tl.load(q_ptrs, mask=offs_d[None, :] < headdim, other=0.0)
+ elif EVEN_HEADDIM:
+ q = tl.load(q_ptrs, mask=offs_m[:, None] < seqlen_q, other=0.0)
+ else:
+ q = tl.load(q_ptrs, mask=(offs_m[:, None] < seqlen_q) & (offs_d[None, :] < headdim), other=0.0)
+ end_n = seqlen_k if not IS_CAUSAL else tl.minimum((start_m + 1) * BLOCK_M, seqlen_k)
+ for start_n in range(0, end_n, BLOCK_N):
+ start_n = tl.multiple_of(start_n, BLOCK_N)
+ if EVEN_N & EVEN_M:
+ if EVEN_HEADDIM:
+ k = tl.load(k_ptrs + start_n * stride_kn)
+ else:
+ k = tl.load(k_ptrs + start_n * stride_kn, mask=offs_d[None, :] < headdim, other=0.0)
+ elif EVEN_HEADDIM:
+ k = tl.load(k_ptrs + start_n * stride_kn, mask=(start_n + offs_n)[:, None] < seqlen_k, other=0.0)
+ else:
+ k = tl.load(k_ptrs + start_n * stride_kn, mask=((start_n + offs_n)[:, None] < seqlen_k) & (offs_d[None, :] < headdim), other=0.0)
+ qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)
+ qk += tl.dot(q, k, trans_b=True)
+ if not EVEN_N:
+ qk += tl.where((start_n + offs_n)[None, :] < seqlen_k, 0, float('-inf'))
+ if IS_CAUSAL:
+ qk += tl.where(offs_m[:, None] >= (start_n + offs_n)[None, :], 0, float('-inf'))
+ if BIAS_TYPE != 'none':
+ if BIAS_TYPE == 'vector':
+ if EVEN_N:
+ bias = tl.load(b_ptrs + start_n).to(tl.float32)
+ else:
+ bias = tl.load(b_ptrs + start_n, mask=start_n + offs_n < seqlen_k, other=0.0).to(tl.float32)
+ bias = bias[None, :]
+ elif BIAS_TYPE == 'matrix':
+ if EVEN_M & EVEN_N:
+ bias = tl.load(b_ptrs + start_n).to(tl.float32)
+ else:
+ bias = tl.load(b_ptrs + start_n, mask=(offs_m[:, None] < seqlen_q) & ((start_n + offs_n)[None, :] < seqlen_k), other=0.0).to(tl.float32)
+ qk = qk * softmax_scale + bias
+ m_ij = tl.maximum(tl.max(qk, 1), lse_i)
+ p = tl.exp(qk - m_ij[:, None])
+ else:
+ m_ij = tl.maximum(tl.max(qk, 1) * softmax_scale, lse_i)
+ p = tl.exp(qk * softmax_scale - m_ij[:, None])
+ l_ij = tl.sum(p, 1)
+ acc_o_scale = tl.exp(m_i - m_ij)
+ tl.store(t_ptrs, acc_o_scale)
+ acc_o_scale = tl.load(t_ptrs)
+ acc_o = acc_o * acc_o_scale[:, None]
+ if EVEN_N & EVEN_M:
+ if EVEN_HEADDIM:
+ v = tl.load(v_ptrs + start_n * stride_vn)
+ else:
+ v = tl.load(v_ptrs + start_n * stride_vn, mask=offs_d[None, :] < headdim, other=0.0)
+ elif EVEN_HEADDIM:
+ v = tl.load(v_ptrs + start_n * stride_vn, mask=(start_n + offs_n)[:, None] < seqlen_k, other=0.0)
+ else:
+ v = tl.load(v_ptrs + start_n * stride_vn, mask=((start_n + offs_n)[:, None] < seqlen_k) & (offs_d[None, :] < headdim), other=0.0)
+ p = p.to(v.dtype)
+ acc_o += tl.dot(p, v)
+ m_i = m_ij
+ l_i_new = tl.exp(lse_i - m_ij) + l_ij
+ lse_i = m_ij + tl.log(l_i_new)
+ o_scale = tl.exp(m_i - lse_i)
+ tl.store(t_ptrs, o_scale)
+ o_scale = tl.load(t_ptrs)
+ acc_o = acc_o * o_scale[:, None]
+ start_m = tl.program_id(0)
+ offs_m = start_m * BLOCK_M + tl.arange(0, BLOCK_M)
+ lse_ptrs = Lse + off_hb * seqlen_q_rounded + offs_m
+ tl.store(lse_ptrs, lse_i)
+ offs_d = tl.arange(0, BLOCK_HEADDIM)
+ out_ptrs = Out + off_b * stride_ob + off_h * stride_oh + (offs_m[:, None] * stride_om + offs_d[None, :])
+ if EVEN_M:
+ if EVEN_HEADDIM:
+ tl.store(out_ptrs, acc_o)
+ else:
+ tl.store(out_ptrs, acc_o, mask=offs_d[None, :] < headdim)
+ elif EVEN_HEADDIM:
+ tl.store(out_ptrs, acc_o, mask=offs_m[:, None] < seqlen_q)
+ else:
+ tl.store(out_ptrs, acc_o, mask=(offs_m[:, None] < seqlen_q) & (offs_d[None, :] < headdim))
+
+@triton.jit
+def _bwd_preprocess_do_o_dot(Out, DO, Delta, stride_ob, stride_oh, stride_om, stride_dob, stride_doh, stride_dom, nheads, seqlen_q, seqlen_q_rounded, headdim, BLOCK_M: tl.constexpr, BLOCK_HEADDIM: tl.constexpr):
+ start_m = tl.program_id(0)
+ off_hb = tl.program_id(1)
+ off_b = off_hb // nheads
+ off_h = off_hb % nheads
+ offs_m = start_m * BLOCK_M + tl.arange(0, BLOCK_M)
+ offs_d = tl.arange(0, BLOCK_HEADDIM)
+ o = tl.load(Out + off_b * stride_ob + off_h * stride_oh + offs_m[:, None] * stride_om + offs_d[None, :], mask=(offs_m[:, None] < seqlen_q) & (offs_d[None, :] < headdim), other=0.0).to(tl.float32)
+ do = tl.load(DO + off_b * stride_dob + off_h * stride_doh + offs_m[:, None] * stride_dom + offs_d[None, :], mask=(offs_m[:, None] < seqlen_q) & (offs_d[None, :] < headdim), other=0.0).to(tl.float32)
+ delta = tl.sum(o * do, axis=1)
+ tl.store(Delta + off_hb * seqlen_q_rounded + offs_m, delta)
+
+@triton.jit
+def _bwd_store_dk_dv(dk_ptrs, dv_ptrs, dk, dv, offs_n, offs_d, seqlen_k, headdim, EVEN_M: tl.constexpr, EVEN_N: tl.constexpr, EVEN_HEADDIM: tl.constexpr):
+ if EVEN_N & EVEN_M:
+ if EVEN_HEADDIM:
+ tl.store(dv_ptrs, dv)
+ tl.store(dk_ptrs, dk)
+ else:
+ tl.store(dv_ptrs, dv, mask=offs_d[None, :] < headdim)
+ tl.store(dk_ptrs, dk, mask=offs_d[None, :] < headdim)
+ elif EVEN_HEADDIM:
+ tl.store(dv_ptrs, dv, mask=offs_n[:, None] < seqlen_k)
+ tl.store(dk_ptrs, dk, mask=offs_n[:, None] < seqlen_k)
+ else:
+ tl.store(dv_ptrs, dv, mask=(offs_n[:, None] < seqlen_k) & (offs_d[None, :] < headdim))
+ tl.store(dk_ptrs, dk, mask=(offs_n[:, None] < seqlen_k) & (offs_d[None, :] < headdim))
+
+@triton.jit
+def _bwd_kernel_one_col_block(start_n, Q, K, V, Bias, DO, DQ, DK, DV, LSE, D, softmax_scale, stride_qm, stride_kn, stride_vn, stride_bm, stride_dom, stride_dqm, stride_dkn, stride_dvn, seqlen_q, seqlen_k, headdim, ATOMIC_ADD: tl.constexpr, BIAS_TYPE: tl.constexpr, IS_CAUSAL: tl.constexpr, BLOCK_HEADDIM: tl.constexpr, EVEN_M: tl.constexpr, EVEN_N: tl.constexpr, EVEN_HEADDIM: tl.constexpr, BLOCK_M: tl.constexpr, BLOCK_N: tl.constexpr):
+ begin_m = 0 if not IS_CAUSAL else start_n * BLOCK_N // BLOCK_M * BLOCK_M
+ offs_qm = begin_m + tl.arange(0, BLOCK_M)
+ offs_n = start_n * BLOCK_N + tl.arange(0, BLOCK_N)
+ offs_m = tl.arange(0, BLOCK_M)
+ offs_d = tl.arange(0, BLOCK_HEADDIM)
+ q_ptrs = Q + (offs_qm[:, None] * stride_qm + offs_d[None, :])
+ k_ptrs = K + (offs_n[:, None] * stride_kn + offs_d[None, :])
+ v_ptrs = V + (offs_n[:, None] * stride_vn + offs_d[None, :])
+ do_ptrs = DO + (offs_qm[:, None] * stride_dom + offs_d[None, :])
+ dq_ptrs = DQ + (offs_qm[:, None] * stride_dqm + offs_d[None, :])
+ if BIAS_TYPE == 'vector':
+ b_ptrs = Bias + offs_n
+ elif BIAS_TYPE == 'matrix':
+ b_ptrs = Bias + (offs_qm[:, None] * stride_bm + offs_n[None, :])
+ dv = tl.zeros([BLOCK_N, BLOCK_HEADDIM], dtype=tl.float32)
+ dk = tl.zeros([BLOCK_N, BLOCK_HEADDIM], dtype=tl.float32)
+ if begin_m >= seqlen_q:
+ dv_ptrs = DV + (offs_n[:, None] * stride_dvn + offs_d[None, :])
+ dk_ptrs = DK + (offs_n[:, None] * stride_dkn + offs_d[None, :])
+ _bwd_store_dk_dv(dk_ptrs, dv_ptrs, dk, dv, offs_n, offs_d, seqlen_k, headdim, EVEN_M=EVEN_M, EVEN_N=EVEN_N, EVEN_HEADDIM=EVEN_HEADDIM)
+ return
+ if EVEN_N & EVEN_M:
+ if EVEN_HEADDIM:
+ k = tl.load(k_ptrs)
+ v = tl.load(v_ptrs)
+ else:
+ k = tl.load(k_ptrs, mask=offs_d[None, :] < headdim, other=0.0)
+ v = tl.load(v_ptrs, mask=offs_d[None, :] < headdim, other=0.0)
+ elif EVEN_HEADDIM:
+ k = tl.load(k_ptrs, mask=offs_n[:, None] < seqlen_k, other=0.0)
+ v = tl.load(v_ptrs, mask=offs_n[:, None] < seqlen_k, other=0.0)
+ else:
+ k = tl.load(k_ptrs, mask=(offs_n[:, None] < seqlen_k) & (offs_d[None, :] < headdim), other=0.0)
+ v = tl.load(v_ptrs, mask=(offs_n[:, None] < seqlen_k) & (offs_d[None, :] < headdim), other=0.0)
+ num_block_m = tl.cdiv(seqlen_q, BLOCK_M)
+ for start_m in range(begin_m, num_block_m * BLOCK_M, BLOCK_M):
+ start_m = tl.multiple_of(start_m, BLOCK_M)
+ offs_m_curr = start_m + offs_m
+ if EVEN_M & EVEN_HEADDIM:
+ q = tl.load(q_ptrs)
+ elif EVEN_HEADDIM:
+ q = tl.load(q_ptrs, mask=offs_m_curr[:, None] < seqlen_q, other=0.0)
+ else:
+ q = tl.load(q_ptrs, mask=(offs_m_curr[:, None] < seqlen_q) & (offs_d[None, :] < headdim), other=0.0)
+ qk = tl.dot(q, k, trans_b=True)
+ if not EVEN_N:
+ qk = tl.where(offs_n[None, :] < seqlen_k, qk, float('-inf'))
+ if IS_CAUSAL:
+ qk = tl.where(offs_m_curr[:, None] >= offs_n[None, :], qk, float('-inf'))
+ if BIAS_TYPE != 'none':
+ tl.debug_barrier()
+ if BIAS_TYPE == 'vector':
+ if EVEN_N:
+ bias = tl.load(b_ptrs).to(tl.float32)
+ else:
+ bias = tl.load(b_ptrs, mask=offs_n < seqlen_k, other=0.0).to(tl.float32)
+ bias = bias[None, :]
+ elif BIAS_TYPE == 'matrix':
+ if EVEN_M & EVEN_N:
+ bias = tl.load(b_ptrs).to(tl.float32)
+ else:
+ bias = tl.load(b_ptrs, mask=(offs_m_curr[:, None] < seqlen_q) & (offs_n[None, :] < seqlen_k), other=0.0).to(tl.float32)
+ qk = qk * softmax_scale + bias
+ if not EVEN_M & EVEN_HEADDIM:
+ tl.debug_barrier()
+ lse_i = tl.load(LSE + offs_m_curr)
+ if BIAS_TYPE == 'none':
+ p = tl.exp(qk * softmax_scale - lse_i[:, None])
+ else:
+ p = tl.exp(qk - lse_i[:, None])
+ if EVEN_M & EVEN_HEADDIM:
+ do = tl.load(do_ptrs)
+ else:
+ do = tl.load(do_ptrs, mask=(offs_m_curr[:, None] < seqlen_q) & (offs_d[None, :] < headdim), other=0.0)
+ dv += tl.dot(p.to(do.dtype), do, trans_a=True)
+ if not EVEN_M & EVEN_HEADDIM:
+ tl.debug_barrier()
+ dp = tl.dot(do, v, trans_b=True)
+ if not EVEN_HEADDIM:
+ tl.debug_barrier()
+ Di = tl.load(D + offs_m_curr)
+ ds = (p * (dp - Di[:, None]) * softmax_scale).to(q.dtype)
+ dk += tl.dot(ds, q, trans_a=True)
+ if not EVEN_M & EVEN_HEADDIM:
+ tl.debug_barrier()
+ if not ATOMIC_ADD:
+ if EVEN_M & EVEN_HEADDIM:
+ dq = tl.load(dq_ptrs, eviction_policy='evict_last')
+ dq += tl.dot(ds, k)
+ tl.store(dq_ptrs, dq, eviction_policy='evict_last')
+ elif EVEN_HEADDIM:
+ dq = tl.load(dq_ptrs, mask=offs_m_curr[:, None] < seqlen_q, other=0.0, eviction_policy='evict_last')
+ dq += tl.dot(ds, k)
+ tl.store(dq_ptrs, dq, mask=offs_m_curr[:, None] < seqlen_q, eviction_policy='evict_last')
+ else:
+ dq = tl.load(dq_ptrs, mask=(offs_m_curr[:, None] < seqlen_q) & (offs_d[None, :] < headdim), other=0.0, eviction_policy='evict_last')
+ dq += tl.dot(ds, k)
+ tl.store(dq_ptrs, dq, mask=(offs_m_curr[:, None] < seqlen_q) & (offs_d[None, :] < headdim), eviction_policy='evict_last')
+ else:
+ dq = tl.dot(ds, k)
+ if EVEN_M & EVEN_HEADDIM:
+ tl.atomic_add(dq_ptrs, dq)
+ elif EVEN_HEADDIM:
+ tl.atomic_add(dq_ptrs, dq, mask=offs_m_curr[:, None] < seqlen_q)
+ else:
+ tl.atomic_add(dq_ptrs, dq, mask=(offs_m_curr[:, None] < seqlen_q) & (offs_d[None, :] < headdim))
+ dq_ptrs += BLOCK_M * stride_dqm
+ q_ptrs += BLOCK_M * stride_qm
+ do_ptrs += BLOCK_M * stride_dom
+ if BIAS_TYPE == 'matrix':
+ b_ptrs += BLOCK_M * stride_bm
+ dv_ptrs = DV + (offs_n[:, None] * stride_dvn + offs_d[None, :])
+ dk_ptrs = DK + (offs_n[:, None] * stride_dkn + offs_d[None, :])
+ _bwd_store_dk_dv(dk_ptrs, dv_ptrs, dk, dv, offs_n, offs_d, seqlen_k, headdim, EVEN_M=EVEN_M, EVEN_N=EVEN_N, EVEN_HEADDIM=EVEN_HEADDIM)
+
+def init_to_zero(name):
+ return lambda nargs: nargs[name].zero_()
+
+@triton.autotune(configs=[triton.Config({'BLOCK_M': 128, 'BLOCK_N': 128, 'SEQUENCE_PARALLEL': False}, num_warps=8, num_stages=1, pre_hook=init_to_zero('DQ')), triton.Config({'BLOCK_M': 128, 'BLOCK_N': 128, 'SEQUENCE_PARALLEL': True}, num_warps=8, num_stages=1, pre_hook=init_to_zero('DQ'))], key=['CACHE_KEY_SEQLEN_Q', 'CACHE_KEY_SEQLEN_K', 'BIAS_TYPE', 'IS_CAUSAL', 'BLOCK_HEADDIM'])
+@triton.heuristics({'EVEN_M': lambda args: args['seqlen_q'] % args['BLOCK_M'] == 0, 'EVEN_N': lambda args: args['seqlen_k'] % args['BLOCK_N'] == 0, 'EVEN_HEADDIM': lambda args: args['headdim'] == args['BLOCK_HEADDIM']})
+@triton.jit
+def _bwd_kernel(Q, K, V, Bias, DO, DQ, DK, DV, LSE, D, softmax_scale, stride_qb, stride_qh, stride_qm, stride_kb, stride_kh, stride_kn, stride_vb, stride_vh, stride_vn, stride_bb, stride_bh, stride_bm, stride_dob, stride_doh, stride_dom, stride_dqb, stride_dqh, stride_dqm, stride_dkb, stride_dkh, stride_dkn, stride_dvb, stride_dvh, stride_dvn, nheads, seqlen_q, seqlen_k, seqlen_q_rounded, headdim, CACHE_KEY_SEQLEN_Q, CACHE_KEY_SEQLEN_K, BIAS_TYPE: tl.constexpr, IS_CAUSAL: tl.constexpr, BLOCK_HEADDIM: tl.constexpr, SEQUENCE_PARALLEL: tl.constexpr, EVEN_M: tl.constexpr, EVEN_N: tl.constexpr, EVEN_HEADDIM: tl.constexpr, BLOCK_M: tl.constexpr, BLOCK_N: tl.constexpr):
+ off_hb = tl.program_id(1)
+ off_b = off_hb // nheads
+ off_h = off_hb % nheads
+ Q += off_b * stride_qb + off_h * stride_qh
+ K += off_b * stride_kb + off_h * stride_kh
+ V += off_b * stride_vb + off_h * stride_vh
+ DO += off_b * stride_dob + off_h * stride_doh
+ DQ += off_b * stride_dqb + off_h * stride_dqh
+ DK += off_b * stride_dkb + off_h * stride_dkh
+ DV += off_b * stride_dvb + off_h * stride_dvh
+ if BIAS_TYPE != 'none':
+ Bias += off_b * stride_bb + off_h * stride_bh
+ D += off_hb * seqlen_q_rounded
+ LSE += off_hb * seqlen_q_rounded
+ if not SEQUENCE_PARALLEL:
+ num_block_n = tl.cdiv(seqlen_k, BLOCK_N)
+ for start_n in range(0, num_block_n):
+ _bwd_kernel_one_col_block(start_n, Q, K, V, Bias, DO, DQ, DK, DV, LSE, D, softmax_scale, stride_qm, stride_kn, stride_vn, stride_bm, stride_dom, stride_dqm, stride_dkn, stride_dvn, seqlen_q, seqlen_k, headdim, ATOMIC_ADD=False, BIAS_TYPE=BIAS_TYPE, IS_CAUSAL=IS_CAUSAL, BLOCK_HEADDIM=BLOCK_HEADDIM, EVEN_M=EVEN_M, EVEN_N=EVEN_N, EVEN_HEADDIM=EVEN_HEADDIM, BLOCK_M=BLOCK_M, BLOCK_N=BLOCK_N)
+ else:
+ start_n = tl.program_id(0)
+ _bwd_kernel_one_col_block(start_n, Q, K, V, Bias, DO, DQ, DK, DV, LSE, D, softmax_scale, stride_qm, stride_kn, stride_vn, stride_bm, stride_dom, stride_dqm, stride_dkn, stride_dvn, seqlen_q, seqlen_k, headdim, ATOMIC_ADD=True, BIAS_TYPE=BIAS_TYPE, IS_CAUSAL=IS_CAUSAL, BLOCK_HEADDIM=BLOCK_HEADDIM, EVEN_M=EVEN_M, EVEN_N=EVEN_N, EVEN_HEADDIM=EVEN_HEADDIM, BLOCK_M=BLOCK_M, BLOCK_N=BLOCK_N)
+
+def _flash_attn_forward(q, k, v, bias=None, causal=False, softmax_scale=None):
+ (batch, seqlen_q, nheads, d) = q.shape
+ (_, seqlen_k, _, _) = k.shape
+ assert k.shape == (batch, seqlen_k, nheads, d)
+ assert v.shape == (batch, seqlen_k, nheads, d)
+ assert d <= 128, 'FlashAttention only support head dimensions up to 128'
+ assert q.dtype == k.dtype == v.dtype, 'All tensors must have the same type'
+ assert q.dtype in [torch.float16, torch.bfloat16], 'Only support fp16 and bf16'
+ assert q.is_cuda and k.is_cuda and v.is_cuda
+ softmax_scale = softmax_scale or 1.0 / math.sqrt(d)
+ has_bias = bias is not None
+ bias_type = 'none'
+ if has_bias:
+ assert bias.dtype in [q.dtype, torch.float]
+ assert bias.is_cuda
+ assert bias.dim() == 4
+ if bias.stride(-1) != 1:
+ bias = bias.contiguous()
+ if bias.shape[2:] == (1, seqlen_k):
+ bias_type = 'vector'
+ elif bias.shape[2:] == (seqlen_q, seqlen_k):
+ bias_type = 'matrix'
+ else:
+ raise RuntimeError('Last 2 dimensions of bias must be (1, seqlen_k) or (seqlen_q, seqlen_k)')
+ bias = bias.expand(batch, nheads, seqlen_q, seqlen_k)
+ bias_strides = (bias.stride(0), bias.stride(1), bias.stride(2)) if has_bias else (0, 0, 0)
+ seqlen_q_rounded = math.ceil(seqlen_q / 128) * 128
+ lse = torch.empty((batch, nheads, seqlen_q_rounded), device=q.device, dtype=torch.float32)
+ tmp = torch.empty((batch, nheads, seqlen_q_rounded), device=q.device, dtype=torch.float32)
+ o = torch.empty_like(q)
+ BLOCK_HEADDIM = max(triton.next_power_of_2(d), 16)
+ BLOCK = 128
+ num_warps = 4 if d <= 64 else 8
+ grid = lambda META: (triton.cdiv(seqlen_q, META['BLOCK_M']), batch * nheads)
+ _fwd_kernel[grid](q, k, v, bias, o, lse, tmp, softmax_scale, q.stride(0), q.stride(2), q.stride(1), k.stride(0), k.stride(2), k.stride(1), v.stride(0), v.stride(2), v.stride(1), *bias_strides, o.stride(0), o.stride(2), o.stride(1), nheads, seqlen_q, seqlen_k, seqlen_q_rounded, d, seqlen_q // 32, seqlen_k // 32, bias_type, causal, BLOCK_HEADDIM, BLOCK_M=BLOCK, BLOCK_N=BLOCK, num_warps=num_warps, num_stages=1)
+ return (o, lse, softmax_scale)
+
+def _flash_attn_backward(do, q, k, v, o, lse, dq, dk, dv, bias=None, causal=False, softmax_scale=None):
+ if do.stride(-1) != 1:
+ do = do.contiguous()
+ (batch, seqlen_q, nheads, d) = q.shape
+ (_, seqlen_k, _, _) = k.shape
+ assert d <= 128
+ seqlen_q_rounded = math.ceil(seqlen_q / 128) * 128
+ assert lse.shape == (batch, nheads, seqlen_q_rounded)
+ assert q.stride(-1) == k.stride(-1) == v.stride(-1) == o.stride(-1) == 1
+ assert dq.stride(-1) == dk.stride(-1) == dv.stride(-1) == 1
+ softmax_scale = softmax_scale or 1.0 / math.sqrt(d)
+ dq_accum = torch.empty_like(q, dtype=torch.float32)
+ delta = torch.empty_like(lse)
+ BLOCK_HEADDIM = max(triton.next_power_of_2(d), 16)
+ grid = lambda META: (triton.cdiv(seqlen_q, META['BLOCK_M']), batch * nheads)
+ _bwd_preprocess_do_o_dot[grid](o, do, delta, o.stride(0), o.stride(2), o.stride(1), do.stride(0), do.stride(2), do.stride(1), nheads, seqlen_q, seqlen_q_rounded, d, BLOCK_M=128, BLOCK_HEADDIM=BLOCK_HEADDIM)
+ has_bias = bias is not None
+ bias_type = 'none'
+ if has_bias:
+ assert bias.dtype in [q.dtype, torch.float]
+ assert bias.is_cuda
+ assert bias.dim() == 4
+ assert bias.stride(-1) == 1
+ if bias.shape[2:] == (1, seqlen_k):
+ bias_type = 'vector'
+ elif bias.shape[2:] == (seqlen_q, seqlen_k):
+ bias_type = 'matrix'
+ else:
+ raise RuntimeError('Last 2 dimensions of bias must be (1, seqlen_k) or (seqlen_q, seqlen_k)')
+ bias = bias.expand(batch, nheads, seqlen_q, seqlen_k)
+ bias_strides = (bias.stride(0), bias.stride(1), bias.stride(2)) if has_bias else (0, 0, 0)
+ grid = lambda META: (triton.cdiv(seqlen_k, META['BLOCK_N']) if META['SEQUENCE_PARALLEL'] else 1, batch * nheads)
+ _bwd_kernel[grid](q, k, v, bias, do, dq_accum, dk, dv, lse, delta, softmax_scale, q.stride(0), q.stride(2), q.stride(1), k.stride(0), k.stride(2), k.stride(1), v.stride(0), v.stride(2), v.stride(1), *bias_strides, do.stride(0), do.stride(2), do.stride(1), dq_accum.stride(0), dq_accum.stride(2), dq_accum.stride(1), dk.stride(0), dk.stride(2), dk.stride(1), dv.stride(0), dv.stride(2), dv.stride(1), nheads, seqlen_q, seqlen_k, seqlen_q_rounded, d, seqlen_q // 32, seqlen_k // 32, bias_type, causal, BLOCK_HEADDIM)
+ dq.copy_(dq_accum)
+
+class FlashAttnQKVPackedFunc(torch.autograd.Function):
+
+ @staticmethod
+ def forward(ctx, qkv, bias=None, causal=False, softmax_scale=None):
+ """
+ qkv: (batch, seqlen, 3, nheads, headdim)
+ bias: optional, shape broadcastible to (batch, nheads, seqlen, seqlen).
+ For example, ALiBi mask for causal would have shape (1, nheads, 1, seqlen).
+ ALiBi mask for non-causal would have shape (1, nheads, seqlen, seqlen)
+ """
+ if qkv.stride(-1) != 1:
+ qkv = qkv.contiguous()
+ (o, lse, ctx.softmax_scale) = _flash_attn_forward(qkv[:, :, 0], qkv[:, :, 1], qkv[:, :, 2], bias=bias, causal=causal, softmax_scale=softmax_scale)
+ ctx.save_for_backward(qkv, o, lse, bias)
+ ctx.causal = causal
+ return o
+
+ @staticmethod
+ def backward(ctx, do):
+ (qkv, o, lse, bias) = ctx.saved_tensors
+ assert not ctx.needs_input_grad[1], 'FlashAttention does not support bias gradient yet'
+ with torch.inference_mode():
+ dqkv = torch.empty_like(qkv)
+ _flash_attn_backward(do, qkv[:, :, 0], qkv[:, :, 1], qkv[:, :, 2], o, lse, dqkv[:, :, 0], dqkv[:, :, 1], dqkv[:, :, 2], bias=bias, causal=ctx.causal, softmax_scale=ctx.softmax_scale)
+ return (dqkv, None, None, None)
+flash_attn_qkvpacked_func = FlashAttnQKVPackedFunc.apply
+
+class FlashAttnKVPackedFunc(torch.autograd.Function):
+
+ @staticmethod
+ def forward(ctx, q, kv, bias=None, causal=False, softmax_scale=None):
+ """
+ q: (batch, seqlen_q, nheads, headdim)
+ kv: (batch, seqlen_k, 2, nheads, headdim)
+ bias: optional, shape broadcastible to (batch, nheads, seqlen_q, seqlen_k).
+ For example, ALiBi mask for causal would have shape (1, nheads, 1, seqlen_k).
+ ALiBi mask for non-causal would have shape (1, nheads, seqlen_q, seqlen_k)
+ """
+ (q, kv) = [x if x.stride(-1) == 1 else x.contiguous() for x in [q, kv]]
+ (o, lse, ctx.softmax_scale) = _flash_attn_forward(q, kv[:, :, 0], kv[:, :, 1], bias=bias, causal=causal, softmax_scale=softmax_scale)
+ ctx.save_for_backward(q, kv, o, lse, bias)
+ ctx.causal = causal
+ return o
+
+ @staticmethod
+ def backward(ctx, do):
+ (q, kv, o, lse, bias) = ctx.saved_tensors
+ if len(ctx.needs_input_grad) >= 3:
+ assert not ctx.needs_input_grad[2], 'FlashAttention does not support bias gradient yet'
+ with torch.inference_mode():
+ dq = torch.empty_like(q)
+ dkv = torch.empty_like(kv)
+ _flash_attn_backward(do, q, kv[:, :, 0], kv[:, :, 1], o, lse, dq, dkv[:, :, 0], dkv[:, :, 1], bias=bias, causal=ctx.causal, softmax_scale=ctx.softmax_scale)
+ return (dq, dkv, None, None, None)
+flash_attn_kvpacked_func = FlashAttnKVPackedFunc.apply
+
+class FlashAttnFunc(torch.autograd.Function):
+
+ @staticmethod
+ def forward(ctx, q, k, v, bias=None, causal=False, softmax_scale=None):
+ """
+ q: (batch_size, seqlen_q, nheads, headdim)
+ k, v: (batch_size, seqlen_k, nheads, headdim)
+ bias: optional, shape broadcastible to (batch, nheads, seqlen_q, seqlen_k).
+ For example, ALiBi mask for causal would have shape (1, nheads, 1, seqlen_k).
+ ALiBi mask for non-causal would have shape (1, nheads, seqlen_q, seqlen_k)
+ """
+ (q, k, v) = [x if x.stride(-1) == 1 else x.contiguous() for x in [q, k, v]]
+ (o, lse, ctx.softmax_scale) = _flash_attn_forward(q, k, v, bias=bias, causal=causal, softmax_scale=softmax_scale)
+ ctx.save_for_backward(q, k, v, o, lse, bias)
+ ctx.causal = causal
+ return o
+
+ @staticmethod
+ def backward(ctx, do):
+ (q, k, v, o, lse, bias) = ctx.saved_tensors
+ assert not ctx.needs_input_grad[3], 'FlashAttention does not support bias gradient yet'
+ with torch.inference_mode():
+ dq = torch.empty_like(q)
+ dk = torch.empty_like(k)
+ dv = torch.empty_like(v)
+ _flash_attn_backward(do, q, k, v, o, lse, dq, dk, dv, bias=bias, causal=ctx.causal, softmax_scale=ctx.softmax_scale)
+ return (dq, dk, dv, None, None, None)
+flash_attn_func = FlashAttnFunc.apply
\ No newline at end of file
diff --git a/videollava/model/language_model/mpt/hf_prefixlm_converter.py b/videollava/model/language_model/mpt/hf_prefixlm_converter.py
new file mode 100644
index 0000000000000000000000000000000000000000..8c1a6487202a6400a7116a6bd68b493892ef0d14
--- /dev/null
+++ b/videollava/model/language_model/mpt/hf_prefixlm_converter.py
@@ -0,0 +1,415 @@
+"""Converts Huggingface Causal LM to Prefix LM.
+
+Conversion does lightweight surgery on a HuggingFace
+Causal LM to convert it to a Prefix LM.
+
+Prefix LMs accepts a `bidirectional_mask` input in `forward`
+and treat the input prompt as the prefix in `generate`.
+"""
+import math
+import warnings
+from types import MethodType
+from typing import Any, Dict, List, Optional, Tuple, Union
+import torch
+from transformers.models.bloom.modeling_bloom import BaseModelOutputWithPastAndCrossAttentions, BloomForCausalLM, BloomModel, CausalLMOutputWithCrossAttentions, CrossEntropyLoss
+from transformers.models.bloom.modeling_bloom import _expand_mask as _expand_mask_bloom
+from transformers.models.bloom.modeling_bloom import _make_causal_mask as _make_causal_mask_bloom
+from transformers.models.bloom.modeling_bloom import logging
+from transformers.models.gpt2.modeling_gpt2 import GPT2LMHeadModel
+from transformers.models.gpt_neo.modeling_gpt_neo import GPTNeoForCausalLM
+from transformers.models.gpt_neox.modeling_gpt_neox import GPTNeoXForCausalLM
+from transformers.models.gptj.modeling_gptj import GPTJForCausalLM
+from transformers.models.opt.modeling_opt import OPTForCausalLM
+from transformers.models.opt.modeling_opt import _expand_mask as _expand_mask_opt
+from transformers.models.opt.modeling_opt import _make_causal_mask as _make_causal_mask_opt
+logger = logging.get_logger(__name__)
+_SUPPORTED_GPT_MODELS = (GPT2LMHeadModel, GPTJForCausalLM, GPTNeoForCausalLM, GPTNeoXForCausalLM)
+CAUSAL_GPT_TYPES = Union[GPT2LMHeadModel, GPTJForCausalLM, GPTNeoForCausalLM, GPTNeoXForCausalLM]
+
+def _convert_gpt_causal_lm_to_prefix_lm(model: CAUSAL_GPT_TYPES) -> CAUSAL_GPT_TYPES:
+ """Converts a GPT-style Causal LM to a Prefix LM.
+
+ Supported HuggingFace model classes:
+ - `GPT2LMHeadModel`
+ - `GPTNeoForCausalLM`
+ - `GPTNeoXForCausalLM`
+ - `GPTJForCausalLM`
+
+ See `convert_hf_causal_lm_to_prefix_lm` for more details.
+ """
+ if hasattr(model, '_prefix_lm_converted'):
+ return model
+ assert isinstance(model, _SUPPORTED_GPT_MODELS)
+ assert model.config.add_cross_attention == False, 'Only supports GPT-style decoder-only models'
+
+ def _get_attn_modules(model: CAUSAL_GPT_TYPES) -> List[torch.nn.Module]:
+ """Helper that gets a list of the model's attention modules.
+
+ Each module has a `bias` buffer used for causal masking. The Prefix LM
+ conversion adds logic to dynamically manipulate these biases to support
+ Prefix LM attention masking.
+ """
+ attn_modules = []
+ if isinstance(model, GPTNeoXForCausalLM):
+ blocks = model.gpt_neox.layers
+ else:
+ blocks = model.transformer.h
+ for block in blocks:
+ if isinstance(model, GPTNeoForCausalLM):
+ if block.attn.attention_type != 'global':
+ continue
+ attn_module = block.attn.attention
+ elif isinstance(model, GPTNeoXForCausalLM):
+ attn_module = block.attention
+ else:
+ attn_module = block.attn
+ attn_modules.append(attn_module)
+ return attn_modules
+ setattr(model, '_original_forward', getattr(model, 'forward'))
+ setattr(model, '_original_generate', getattr(model, 'generate'))
+
+ def forward(self: CAUSAL_GPT_TYPES, input_ids: Optional[torch.LongTensor]=None, past_key_values: Optional[Tuple[Tuple[torch.Tensor]]]=None, attention_mask: Optional[torch.FloatTensor]=None, bidirectional_mask: Optional[torch.Tensor]=None, token_type_ids: Optional[torch.LongTensor]=None, position_ids: Optional[torch.LongTensor]=None, head_mask: Optional[torch.FloatTensor]=None, inputs_embeds: Optional[torch.FloatTensor]=None, labels: Optional[torch.LongTensor]=None, use_cache: Optional[bool]=None, output_attentions: Optional[bool]=None, output_hidden_states: Optional[bool]=None, return_dict: Optional[bool]=None):
+ """Wraps original forward to enable PrefixLM attention."""
+
+ def call_og_forward():
+ if isinstance(self, GPTNeoXForCausalLM):
+ return self._original_forward(input_ids=input_ids, past_key_values=past_key_values, attention_mask=attention_mask, head_mask=head_mask, inputs_embeds=inputs_embeds, labels=labels, use_cache=use_cache, output_attentions=output_attentions, output_hidden_states=output_hidden_states, return_dict=return_dict)
+ else:
+ return self._original_forward(input_ids=input_ids, past_key_values=past_key_values, attention_mask=attention_mask, token_type_ids=token_type_ids, position_ids=position_ids, head_mask=head_mask, inputs_embeds=inputs_embeds, labels=labels, use_cache=use_cache, output_attentions=output_attentions, output_hidden_states=output_hidden_states, return_dict=return_dict)
+ if bidirectional_mask is None:
+ return call_og_forward()
+ assert isinstance(bidirectional_mask, torch.Tensor)
+ attn_modules = _get_attn_modules(model)
+ (b, s) = bidirectional_mask.shape
+ max_length = attn_modules[0].bias.shape[-1]
+ if s > max_length:
+ raise ValueError(f'bidirectional_mask sequence length (={s}) exceeds the ' + f'max length allowed by the model ({max_length}).')
+ assert s <= max_length
+ if s < max_length:
+ pad = torch.zeros((int(b), int(max_length - s)), dtype=bidirectional_mask.dtype, device=bidirectional_mask.device)
+ bidirectional_mask = torch.cat([bidirectional_mask, pad], dim=1)
+ bidirectional = bidirectional_mask.unsqueeze(1).unsqueeze(1)
+ for attn_module in attn_modules:
+ attn_module.bias.data = torch.logical_or(attn_module.bias.data, bidirectional)
+ output = call_og_forward()
+ for attn_module in attn_modules:
+ attn_module.bias.data = torch.tril(attn_module.bias.data[0, 0])[None, None]
+ return output
+
+ def generate(self: CAUSAL_GPT_TYPES, *args: tuple, **kwargs: Dict[str, Any]):
+ """Wraps original generate to enable PrefixLM attention."""
+ attn_modules = _get_attn_modules(model)
+ for attn_module in attn_modules:
+ attn_module.bias.data[:] = 1
+ output = self._original_generate(*args, **kwargs)
+ for attn_module in attn_modules:
+ attn_module.bias.data = torch.tril(attn_module.bias.data[0, 0])[None, None]
+ return output
+ setattr(model, 'forward', MethodType(forward, model))
+ setattr(model, 'generate', MethodType(generate, model))
+ setattr(model, '_prefix_lm_converted', True)
+ return model
+
+def _convert_bloom_causal_lm_to_prefix_lm(model: BloomForCausalLM) -> BloomForCausalLM:
+ """Converts a BLOOM Causal LM to a Prefix LM.
+
+ Supported HuggingFace model classes:
+ - `BloomForCausalLM`
+
+ See `convert_hf_causal_lm_to_prefix_lm` for more details.
+ """
+ if hasattr(model, '_prefix_lm_converted'):
+ return model
+ assert isinstance(model, BloomForCausalLM)
+ assert model.config.add_cross_attention == False, 'Only supports BLOOM decoder-only models'
+
+ def _prepare_attn_mask(self: BloomModel, attention_mask: torch.Tensor, bidirectional_mask: Optional[torch.Tensor], input_shape: Tuple[int, int], past_key_values_length: int) -> torch.BoolTensor:
+ combined_attention_mask = None
+ device = attention_mask.device
+ (_, src_length) = input_shape
+ if src_length > 1:
+ combined_attention_mask = _make_causal_mask_bloom(input_shape, device=device, past_key_values_length=past_key_values_length)
+ if bidirectional_mask is not None:
+ assert attention_mask.shape == bidirectional_mask.shape
+ expanded_bidirectional_mask = _expand_mask_bloom(bidirectional_mask, tgt_length=src_length)
+ combined_attention_mask = torch.logical_and(combined_attention_mask, expanded_bidirectional_mask)
+ expanded_attn_mask = _expand_mask_bloom(attention_mask, tgt_length=src_length)
+ combined_attention_mask = expanded_attn_mask if combined_attention_mask is None else expanded_attn_mask | combined_attention_mask
+ return combined_attention_mask
+
+ def _build_alibi_tensor(self: BloomModel, batch_size: int, query_length: int, key_length: int, dtype: torch.dtype, device: torch.device) -> torch.Tensor:
+ num_heads = self.config.n_head
+ closest_power_of_2 = 2 ** math.floor(math.log2(num_heads))
+ base = torch.tensor(2 ** (-2 ** (-(math.log2(closest_power_of_2) - 3))), device=device, dtype=torch.float32)
+ powers = torch.arange(1, 1 + closest_power_of_2, device=device, dtype=torch.int32)
+ slopes = torch.pow(base, powers)
+ if closest_power_of_2 != num_heads:
+ extra_base = torch.tensor(2 ** (-2 ** (-(math.log2(2 * closest_power_of_2) - 3))), device=device, dtype=torch.float32)
+ num_remaining_heads = min(closest_power_of_2, num_heads - closest_power_of_2)
+ extra_powers = torch.arange(1, 1 + 2 * num_remaining_heads, 2, device=device, dtype=torch.int32)
+ slopes = torch.cat([slopes, torch.pow(extra_base, extra_powers)], dim=0)
+ qa = torch.arange(query_length, device=device, dtype=torch.int32).view(-1, 1)
+ ka = torch.arange(key_length, device=device, dtype=torch.int32).view(1, -1)
+ diffs = qa - ka + key_length - query_length
+ diffs = -diffs.abs()
+ alibi = slopes.view(1, num_heads, 1, 1) * diffs.view(1, 1, query_length, key_length)
+ alibi = alibi.expand(batch_size, -1, -1, -1).reshape(-1, query_length, key_length)
+ return alibi.to(dtype)
+ KeyValueT = Tuple[torch.Tensor, torch.Tensor]
+
+ def forward(self: BloomModel, input_ids: Optional[torch.LongTensor]=None, past_key_values: Optional[Tuple[KeyValueT, ...]]=None, attention_mask: Optional[torch.Tensor]=None, bidirectional_mask: Optional[torch.Tensor]=None, head_mask: Optional[torch.LongTensor]=None, inputs_embeds: Optional[torch.LongTensor]=None, use_cache: Optional[bool]=None, output_attentions: Optional[bool]=None, output_hidden_states: Optional[bool]=None, return_dict: Optional[bool]=None, **deprecated_arguments) -> Union[Tuple[torch.Tensor, ...], BaseModelOutputWithPastAndCrossAttentions]:
+ if deprecated_arguments.pop('position_ids', False) is not False:
+ warnings.warn('`position_ids` have no functionality in BLOOM and will be removed in v5.0.0. ' + 'You can safely ignore passing `position_ids`.', FutureWarning)
+ if len(deprecated_arguments) > 0:
+ raise ValueError(f'Got unexpected arguments: {deprecated_arguments}')
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+ output_hidden_states = output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+ use_cache = use_cache if use_cache is not None else self.config.use_cache
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+ if input_ids is not None and inputs_embeds is not None:
+ raise ValueError('You cannot specify both input_ids and inputs_embeds at the same time')
+ elif input_ids is not None:
+ (batch_size, seq_length) = input_ids.shape
+ elif inputs_embeds is not None:
+ (batch_size, seq_length, _) = inputs_embeds.shape
+ else:
+ raise ValueError('You have to specify either input_ids or inputs_embeds')
+ if past_key_values is None:
+ past_key_values = tuple([None] * len(self.h))
+ head_mask = self.get_head_mask(head_mask, self.config.n_layer)
+ if inputs_embeds is None:
+ inputs_embeds = self.word_embeddings(input_ids)
+ hidden_states = self.word_embeddings_layernorm(inputs_embeds)
+ presents = () if use_cache else None
+ all_self_attentions = () if output_attentions else None
+ all_hidden_states = () if output_hidden_states else None
+ seq_length_with_past = seq_length
+ past_key_values_length = 0
+ if past_key_values[0] is not None:
+ tmp = past_key_values[0][0]
+ past_key_values_length = tmp.shape[2]
+ seq_length_with_past = seq_length_with_past + past_key_values_length
+ if attention_mask is None:
+ attention_mask = torch.ones((batch_size, seq_length_with_past), device=hidden_states.device)
+ else:
+ attention_mask = attention_mask.to(hidden_states.device)
+ alibi = self._build_alibi_tensor(batch_size=batch_size, query_length=seq_length, key_length=seq_length_with_past, dtype=hidden_states.dtype, device=hidden_states.device)
+ causal_mask = self._prepare_attn_mask(attention_mask, bidirectional_mask, input_shape=(batch_size, seq_length), past_key_values_length=past_key_values_length)
+ for (i, (block, layer_past)) in enumerate(zip(self.h, past_key_values)):
+ if output_hidden_states:
+ hst = (hidden_states,)
+ all_hidden_states = all_hidden_states + hst
+ if self.gradient_checkpointing and self.training:
+ if use_cache:
+ logger.warning('`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...')
+ use_cache = False
+
+ def create_custom_forward(module):
+
+ def custom_forward(*inputs):
+ return module(*inputs, use_cache=use_cache, output_attentions=output_attentions)
+ return custom_forward
+ outputs = torch.utils.checkpoint.checkpoint(create_custom_forward(block), hidden_states, alibi, causal_mask, head_mask[i])
+ else:
+ outputs = block(hidden_states, layer_past=layer_past, attention_mask=causal_mask, head_mask=head_mask[i], use_cache=use_cache, output_attentions=output_attentions, alibi=alibi)
+ hidden_states = outputs[0]
+ if use_cache is True:
+ presents = presents + (outputs[1],)
+ if output_attentions:
+ oa = (outputs[2 if use_cache else 1],)
+ all_self_attentions = all_self_attentions + oa
+ hidden_states = self.ln_f(hidden_states)
+ if output_hidden_states:
+ hst = (hidden_states,)
+ all_hidden_states = all_hidden_states + hst
+ if not return_dict:
+ return tuple((v for v in [hidden_states, presents, all_hidden_states, all_self_attentions] if v is not None))
+ return BaseModelOutputWithPastAndCrossAttentions(last_hidden_state=hidden_states, past_key_values=presents, hidden_states=all_hidden_states, attentions=all_self_attentions)
+ setattr(model.transformer, '_prepare_attn_mask', MethodType(_prepare_attn_mask, model.transformer))
+ setattr(model.transformer, '_build_alibi_tensor', MethodType(_build_alibi_tensor, model.transformer))
+ setattr(model.transformer, 'forward', MethodType(forward, model.transformer))
+ KeyValueT = Tuple[torch.Tensor, torch.Tensor]
+
+ def forward(self: BloomForCausalLM, input_ids: Optional[torch.LongTensor]=None, past_key_values: Optional[Tuple[KeyValueT, ...]]=None, attention_mask: Optional[torch.Tensor]=None, bidirectional_mask: Optional[torch.Tensor]=None, head_mask: Optional[torch.Tensor]=None, inputs_embeds: Optional[torch.Tensor]=None, labels: Optional[torch.Tensor]=None, use_cache: Optional[bool]=None, output_attentions: Optional[bool]=None, output_hidden_states: Optional[bool]=None, return_dict: Optional[bool]=None, **deprecated_arguments) -> Union[Tuple[torch.Tensor], CausalLMOutputWithCrossAttentions]:
+ """Replacement forward method for BloomCausalLM."""
+ if deprecated_arguments.pop('position_ids', False) is not False:
+ warnings.warn('`position_ids` have no functionality in BLOOM and will be removed ' + 'in v5.0.0. You can safely ignore passing `position_ids`.', FutureWarning)
+ if len(deprecated_arguments) > 0:
+ raise ValueError(f'Got unexpected arguments: {deprecated_arguments}')
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+ transformer_outputs = self.transformer(input_ids, past_key_values=past_key_values, attention_mask=attention_mask, bidirectional_mask=bidirectional_mask, head_mask=head_mask, inputs_embeds=inputs_embeds, use_cache=use_cache, output_attentions=output_attentions, output_hidden_states=output_hidden_states, return_dict=return_dict)
+ hidden_states = transformer_outputs[0]
+ lm_logits = self.lm_head(hidden_states)
+ loss = None
+ if labels is not None:
+ shift_logits = lm_logits[..., :-1, :].contiguous()
+ shift_labels = labels[..., 1:].contiguous()
+ (batch_size, seq_length, vocab_size) = shift_logits.shape
+ loss_fct = CrossEntropyLoss()
+ loss = loss_fct(shift_logits.view(batch_size * seq_length, vocab_size), shift_labels.view(batch_size * seq_length))
+ if not return_dict:
+ output = (lm_logits,) + transformer_outputs[1:]
+ return (loss,) + output if loss is not None else output
+ return CausalLMOutputWithCrossAttentions(loss=loss, logits=lm_logits, past_key_values=transformer_outputs.past_key_values, hidden_states=transformer_outputs.hidden_states, attentions=transformer_outputs.attentions)
+
+ def prepare_inputs_for_generation(self: BloomForCausalLM, input_ids: torch.LongTensor, past: Optional[torch.Tensor]=None, attention_mask: Optional[torch.Tensor]=None, **kwargs) -> dict:
+ if past:
+ input_ids = input_ids[:, -1].unsqueeze(-1)
+ bidirectional_mask = None
+ if past[0][0].shape[0] == input_ids.shape[0]:
+ past = self._convert_to_bloom_cache(past)
+ else:
+ bidirectional_mask = torch.ones_like(input_ids)
+ return {'input_ids': input_ids, 'past_key_values': past, 'use_cache': True, 'attention_mask': attention_mask, 'bidirectional_mask': bidirectional_mask}
+ setattr(model, 'forward', MethodType(forward, model))
+ setattr(model, 'prepare_inputs_for_generation', MethodType(prepare_inputs_for_generation, model))
+ setattr(model, '_prefix_lm_converted', True)
+ return model
+
+def _convert_opt_causal_lm_to_prefix_lm(model: OPTForCausalLM) -> OPTForCausalLM:
+ """Converts an OPT Causal LM to a Prefix LM.
+
+ Supported HuggingFace model classes:
+ - `OPTForCausalLM`
+
+ See `convert_hf_causal_lm_to_prefix_lm` for more details.
+ """
+ if hasattr(model, '_prefix_lm_converted'):
+ return model
+ assert isinstance(model, OPTForCausalLM)
+ assert model.config.add_cross_attention == False, 'Only supports OPT decoder-only models'
+ setattr(model, '_original_forward', getattr(model, 'forward'))
+ setattr(model, '_original_generate', getattr(model, 'generate'))
+ model.model.decoder.bidirectional_mask = None
+
+ def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_embeds, past_key_values_length):
+ combined_attention_mask = None
+ if input_shape[-1] > 1:
+ if self.bidirectional_mask == 'g':
+ (bsz, src_length) = input_shape
+ combined_attention_mask = torch.zeros((bsz, 1, src_length, src_length + past_key_values_length), dtype=inputs_embeds.dtype, device=inputs_embeds.device)
+ else:
+ combined_attention_mask = _make_causal_mask_opt(input_shape, inputs_embeds.dtype, past_key_values_length=past_key_values_length).to(inputs_embeds.device)
+ if self.bidirectional_mask is not None:
+ assert attention_mask.shape == self.bidirectional_mask.shape
+ expanded_bidirectional_mask = _expand_mask_opt(self.bidirectional_mask, inputs_embeds.dtype, tgt_len=input_shape[-1]).to(inputs_embeds.device)
+ combined_attention_mask = torch.maximum(expanded_bidirectional_mask, combined_attention_mask)
+ if attention_mask is not None:
+ expanded_attn_mask = _expand_mask_opt(attention_mask, inputs_embeds.dtype, tgt_len=input_shape[-1]).to(inputs_embeds.device)
+ combined_attention_mask = expanded_attn_mask if combined_attention_mask is None else expanded_attn_mask + combined_attention_mask
+ return combined_attention_mask
+ setattr(model.model.decoder, '_prepare_decoder_attention_mask', MethodType(_prepare_decoder_attention_mask, model.model.decoder))
+
+ def forward(self: OPTForCausalLM, input_ids: Optional[torch.LongTensor]=None, attention_mask: Optional[torch.Tensor]=None, bidirectional_mask: Optional[torch.ByteTensor]=None, head_mask: Optional[torch.Tensor]=None, past_key_values: Optional[List[torch.FloatTensor]]=None, inputs_embeds: Optional[torch.FloatTensor]=None, labels: Optional[torch.LongTensor]=None, use_cache: Optional[bool]=None, output_attentions: Optional[bool]=None, output_hidden_states: Optional[bool]=None, return_dict: Optional[bool]=None):
+
+ def call_og_forward():
+ return self._original_forward(input_ids=input_ids, attention_mask=attention_mask, head_mask=head_mask, past_key_values=past_key_values, inputs_embeds=inputs_embeds, labels=labels, use_cache=use_cache, output_attentions=output_attentions, output_hidden_states=output_hidden_states, return_dict=return_dict)
+ if bidirectional_mask is None:
+ return call_og_forward()
+ self.model.decoder.bidirectional_mask = bidirectional_mask
+ try:
+ outputs = call_og_forward()
+ except:
+ self.model.decoder.bidirectional_mask = None
+ raise
+ self.model.decoder.bidirectional_mask = None
+ return outputs
+
+ def generate(self: OPTForCausalLM, *args: tuple, **kwargs: Dict[str, Any]):
+ """Wraps original generate to enable PrefixLM-style attention."""
+ self.model.decoder.bidirectional_mask = 'g'
+ try:
+ output = self._original_generate(*args, **kwargs)
+ except:
+ self.model.decoder.bidirectional_mask = None
+ raise
+ self.model.decoder.bidirectional_mask = None
+ return output
+ setattr(model, 'forward', MethodType(forward, model))
+ setattr(model, 'generate', MethodType(generate, model))
+ setattr(model, '_prefix_lm_converted', True)
+ return model
+_SUPPORTED_HF_MODELS = _SUPPORTED_GPT_MODELS + (BloomForCausalLM, OPTForCausalLM)
+CAUSAL_LM_TYPES = Union[GPT2LMHeadModel, GPTJForCausalLM, GPTNeoForCausalLM, GPTNeoXForCausalLM, BloomForCausalLM, OPTForCausalLM]
+
+def convert_hf_causal_lm_to_prefix_lm(model: CAUSAL_LM_TYPES) -> CAUSAL_LM_TYPES:
+ """Converts a HuggingFace Causal LM to a Prefix LM.
+
+ Supported HuggingFace model classes:
+ - `GPT2LMHeadModel`
+ - `GPTNeoForCausalLM`
+ - `GPTNeoXForCausalLM`
+ - `GPTJForCausalLM`
+ - `BloomForCausalLM`
+ - `OPTForCausalLM`
+
+ Conversion to a Prefix LM is done by modifying the `forward` method, and possibly also the
+ `generate` method and/or select underlying methods depending on the model class.
+
+ These changes preserve the model API, but add a new input to `forward`: "bidirectional_mask".
+
+ Notes on training:
+ To actually train the converted model as a Prefix LM, training batches will need to indicate
+ the prefix/target structure by including `bidirectional_mask` as part of the batch inputs.
+
+ **This is not a standard input and requires custom layers either within or after your dataloader.**
+
+ In addition to adding `bidirectional_mask` to the batch, this custom code should modify `labels`
+ such that `batch['labels'][batch['bidirectional_mask'] == 1] == -100`.
+ That is, the prefix portion of the sequence should not generate any loss. Loss should only be
+ generated by the target portion of the sequence.
+
+ Notes on `GPTNeoForCausalLM`:
+ To simplify the implementation, "global" and "local" attention layers are handled differently.
+ For "global" layers, we handle conversion as described above. For "local" layers, which use a
+ causal attention mask within a restricted local window, we do not alter the masking.
+
+ Notes on `forward` method conversion:
+ After conversion, the `forward` method will handle a new input, `bidirectional_mask`,
+ which should be a [batch_size, seq_length] byte tensor, where 1 indicates token positions
+ belonging to the prefix (prefix tokens can attend to one another bidirectionally), and
+ 0 indicates token positions belonging to the target.
+
+ The new `forward` method will incorporate `bidirectional_mask` (if supplied) into the existing
+ causal mask, call the original `forward` method, and (if the causal mask is a buffer) reset
+ the causal masks before returning the result.
+
+ Notes on `generate` method conversion:
+ After conversion, the `generate` method will have the same signature but will internally
+ convert all causal masks to be purely bidirectional, call the original `generate` method, and
+ (where appropriate) reset the causal masks before returning the result.
+
+ This works thanks to the logic of the HuggingFace `generate` API, which first encodes the token
+ "prompt" passed to `generate` (which is treated as the prefix) and then sequentially generates
+ each new token. Encodings are cached as generation happens, so all prefix tokens can attend to one
+ another (as expected in a Prefix LM) and generated tokens can only attend to prefix tokens and
+ previously-generated tokens (also as expected in a Prefix LM).
+
+ To preserve the API, the original methods are renamed to `_original_forward` and
+ `_original_generate`, and replaced with new `forward` and `generate` methods that wrap
+ them, respectively. Although implementation details vary by model class.
+ """
+ if isinstance(model, _SUPPORTED_GPT_MODELS):
+ return _convert_gpt_causal_lm_to_prefix_lm(model)
+ elif isinstance(model, BloomForCausalLM):
+ return _convert_bloom_causal_lm_to_prefix_lm(model)
+ elif isinstance(model, OPTForCausalLM):
+ return _convert_opt_causal_lm_to_prefix_lm(model)
+ else:
+ raise TypeError(f'Cannot convert model to Prefix LM. ' + f'Model does not belong to set of supported HF models:' + f'\n{_SUPPORTED_HF_MODELS}')
+
+def add_bidirectional_mask_if_missing(batch: Dict[str, Any]):
+ """Attempts to add bidirectional_mask to batch if missing.
+
+ Raises:
+ KeyError if bidirectional_mask is missing and can't be inferred
+ """
+ if 'bidirectional_mask' not in batch:
+ if batch.get('mode', None) == 'icl_task':
+ batch['bidirectional_mask'] = batch['attention_mask'].clone()
+ for (i, continuation_indices) in enumerate(batch['continuation_indices']):
+ batch['bidirectional_mask'][i, continuation_indices] = 0
+ elif 'labels' in batch and 'attention_mask' in batch:
+ batch['bidirectional_mask'] = torch.logical_and(torch.eq(batch['attention_mask'], 1), torch.eq(batch['labels'], -100)).type_as(batch['attention_mask'])
+ else:
+ raise KeyError('No bidirectional_mask in batch and not sure how to construct one.')
\ No newline at end of file
diff --git a/videollava/model/language_model/mpt/meta_init_context.py b/videollava/model/language_model/mpt/meta_init_context.py
new file mode 100644
index 0000000000000000000000000000000000000000..6cba6fff0fe21fe222c7ab38eae44a9784c0be9c
--- /dev/null
+++ b/videollava/model/language_model/mpt/meta_init_context.py
@@ -0,0 +1,94 @@
+from contextlib import contextmanager
+import torch
+import torch.nn as nn
+
+@contextmanager
+def init_empty_weights(include_buffers: bool=False):
+ """Meta initialization context manager.
+
+ A context manager under which models are initialized with all parameters
+ on the meta device, therefore creating an empty model. Useful when just
+ initializing the model would blow the available RAM.
+
+ Args:
+ include_buffers (`bool`, *optional*, defaults to `False`): Whether or
+ not to also put all buffers on the meta device while initializing.
+
+ Example:
+ ```python
+ import torch.nn as nn
+
+ # Initialize a model with 100 billions parameters in no time and without using any RAM.
+ with init_empty_weights():
+ tst = nn.Sequential(*[nn.Linear(10000, 10000) for _ in range(1000)])
+ ```
+
+
+
+ Any model created under this context manager has no weights. As such you can't do something like
+ `model.to(some_device)` with it. To load weights inside your empty model, see [`load_checkpoint_and_dispatch`].
+
+
+ """
+ with init_on_device(torch.device('meta'), include_buffers=include_buffers) as f:
+ yield f
+
+@contextmanager
+def init_on_device(device: torch.device, include_buffers: bool=False):
+ """Device initialization context manager.
+
+ A context manager under which models are initialized with all parameters
+ on the specified device.
+
+ Args:
+ device (`torch.device`): Device to initialize all parameters on.
+ include_buffers (`bool`, *optional*, defaults to `False`): Whether or
+ not to also put all buffers on the meta device while initializing.
+
+ Example:
+ ```python
+ import torch.nn as nn
+
+ with init_on_device(device=torch.device("cuda")):
+ tst = nn.Liner(100, 100) # on `cuda` device
+ ```
+ """
+ old_register_parameter = nn.Module.register_parameter
+ if include_buffers:
+ old_register_buffer = nn.Module.register_buffer
+
+ def register_empty_parameter(module, name, param):
+ old_register_parameter(module, name, param)
+ if param is not None:
+ param_cls = type(module._parameters[name])
+ kwargs = module._parameters[name].__dict__
+ module._parameters[name] = param_cls(module._parameters[name].to(device), **kwargs)
+
+ def register_empty_buffer(module, name, buffer):
+ old_register_buffer(module, name, buffer)
+ if buffer is not None:
+ module._buffers[name] = module._buffers[name].to(device)
+ if include_buffers:
+ tensor_constructors_to_patch = {torch_function_name: getattr(torch, torch_function_name) for torch_function_name in ['empty', 'zeros', 'ones', 'full']}
+ else:
+ tensor_constructors_to_patch = {}
+
+ def patch_tensor_constructor(fn):
+
+ def wrapper(*args, **kwargs):
+ kwargs['device'] = device
+ return fn(*args, **kwargs)
+ return wrapper
+ try:
+ nn.Module.register_parameter = register_empty_parameter
+ if include_buffers:
+ nn.Module.register_buffer = register_empty_buffer
+ for torch_function_name in tensor_constructors_to_patch.keys():
+ setattr(torch, torch_function_name, patch_tensor_constructor(getattr(torch, torch_function_name)))
+ yield
+ finally:
+ nn.Module.register_parameter = old_register_parameter
+ if include_buffers:
+ nn.Module.register_buffer = old_register_buffer
+ for (torch_function_name, old_torch_function) in tensor_constructors_to_patch.items():
+ setattr(torch, torch_function_name, old_torch_function)
\ No newline at end of file
diff --git a/videollava/model/language_model/mpt/modeling_mpt.py b/videollava/model/language_model/mpt/modeling_mpt.py
new file mode 100644
index 0000000000000000000000000000000000000000..13313441b13fc7a66cb65fd21b482a5de982e2c8
--- /dev/null
+++ b/videollava/model/language_model/mpt/modeling_mpt.py
@@ -0,0 +1,331 @@
+"""A simple, flexible implementation of a GPT model.
+
+Inspired by https://github.com/karpathy/minGPT/blob/master/mingpt/model.py
+"""
+import math
+import warnings
+from typing import List, Optional, Tuple, Union
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from transformers import PreTrainedModel, PreTrainedTokenizer, PreTrainedTokenizerFast
+from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast
+from .attention import attn_bias_shape, build_attn_bias
+from .blocks import MPTBlock
+from .custom_embedding import SharedEmbedding
+from .norm import NORM_CLASS_REGISTRY
+from .configuration_mpt import MPTConfig
+from .adapt_tokenizer import AutoTokenizerForMOD, adapt_tokenizer_for_denoising
+from .hf_prefixlm_converter import add_bidirectional_mask_if_missing, convert_hf_causal_lm_to_prefix_lm
+from .meta_init_context import init_empty_weights
+from .param_init_fns import MODEL_INIT_REGISTRY, generic_param_init_fn_
+try:
+ from .flash_attn_triton import flash_attn_func
+except:
+ pass
+Tokenizer = Union[PreTrainedTokenizer, PreTrainedTokenizerFast]
+
+class MPTPreTrainedModel(PreTrainedModel):
+ config_class = MPTConfig
+ base_model_prefix = 'model'
+ _no_split_modules = ['MPTBlock']
+
+class MPTModel(MPTPreTrainedModel):
+
+ def __init__(self, config: MPTConfig):
+ config._validate_config()
+ super().__init__(config)
+ self.attn_impl = config.attn_config['attn_impl']
+ self.prefix_lm = config.attn_config['prefix_lm']
+ self.attn_uses_sequence_id = config.attn_config['attn_uses_sequence_id']
+ self.alibi = config.attn_config['alibi']
+ self.alibi_bias_max = config.attn_config['alibi_bias_max']
+ if config.init_device == 'mixed':
+ if dist.get_local_rank() == 0:
+ config.init_device = 'cpu'
+ else:
+ config.init_device = 'meta'
+ if config.norm_type.lower() not in NORM_CLASS_REGISTRY.keys():
+ norm_options = ' | '.join(NORM_CLASS_REGISTRY.keys())
+ raise NotImplementedError(f'Requested norm type ({config.norm_type}) is not implemented within this repo (Options: {norm_options}).')
+ norm_class = NORM_CLASS_REGISTRY[config.norm_type.lower()]
+ self.embedding_fraction = config.embedding_fraction
+ self.wte = SharedEmbedding(config.vocab_size, config.d_model, device=config.init_device)
+ if not self.alibi:
+ self.wpe = torch.nn.Embedding(config.max_seq_len, config.d_model, device=config.init_device)
+ self.emb_drop = nn.Dropout(config.emb_pdrop)
+ self.blocks = nn.ModuleList([MPTBlock(device=config.init_device, **config.to_dict()) for _ in range(config.n_layers)])
+ self.norm_f = norm_class(config.d_model, device=config.init_device)
+ if config.init_device != 'meta':
+ print(f'You are using config.init_device={config.init_device!r}, but you can also use config.init_device="meta" with Composer + FSDP for fast initialization.')
+ self.apply(self.param_init_fn)
+ self.is_causal = not self.prefix_lm
+ self._attn_bias_initialized = False
+ self.attn_bias = None
+ self.attn_bias_shape = attn_bias_shape(self.attn_impl, config.n_heads, config.max_seq_len, self.alibi, prefix_lm=self.prefix_lm, causal=self.is_causal, use_sequence_id=self.attn_uses_sequence_id)
+ if config.no_bias:
+ for module in self.modules():
+ if hasattr(module, 'bias') and isinstance(module.bias, nn.Parameter):
+ if config.verbose:
+ warnings.warn(f'Removing bias ({module.bias}) from {module}.')
+ module.register_parameter('bias', None)
+ if config.verbose and config.verbose > 2:
+ print(self)
+ if 'verbose' not in self.config.init_config:
+ self.config.init_config['verbose'] = self.config.verbose
+ if self.config.init_config['verbose'] > 1:
+ init_fn_name = self.config.init_config['name']
+ warnings.warn(f'Using {init_fn_name} initialization.')
+ self.gradient_checkpointing = False
+
+ def get_input_embeddings(self):
+ return self.wte
+
+ def set_input_embeddings(self, value):
+ self.wte = value
+
+ @torch.no_grad()
+ def _attn_bias(self, device, dtype, attention_mask: Optional[torch.ByteTensor]=None, prefix_mask: Optional[torch.ByteTensor]=None, sequence_id: Optional[torch.LongTensor]=None):
+ if not self._attn_bias_initialized:
+ if self.attn_bias_shape:
+ self.attn_bias = torch.zeros(self.attn_bias_shape, device=device, dtype=dtype)
+ self.attn_bias = build_attn_bias(self.attn_impl, self.attn_bias, self.config.n_heads, self.config.max_seq_len, causal=self.is_causal, alibi=self.alibi, alibi_bias_max=self.alibi_bias_max)
+ self._attn_bias_initialized = True
+ if self.attn_impl == 'flash':
+ return (self.attn_bias, attention_mask)
+ if self.attn_bias is not None:
+ self.attn_bias = self.attn_bias.to(dtype=dtype, device=device)
+ attn_bias = self.attn_bias
+ if self.prefix_lm:
+ assert isinstance(attn_bias, torch.Tensor)
+ assert isinstance(prefix_mask, torch.Tensor)
+ attn_bias = self._apply_prefix_mask(attn_bias, prefix_mask)
+ if self.attn_uses_sequence_id and sequence_id is not None:
+ assert isinstance(attn_bias, torch.Tensor)
+ attn_bias = self._apply_sequence_id(attn_bias, sequence_id)
+ if attention_mask is not None:
+ s_k = attention_mask.shape[-1]
+ if attn_bias is None:
+ attn_bias = torch.zeros((1, 1, 1, s_k), device=device, dtype=dtype)
+ else:
+ _s_k = max(0, attn_bias.size(-1) - s_k)
+ attn_bias = attn_bias[:, :, :, _s_k:]
+ if prefix_mask is not None and attention_mask.shape != prefix_mask.shape:
+ raise ValueError(f'attention_mask shape={attention_mask.shape} ' + f'and prefix_mask shape={prefix_mask.shape} are not equal.')
+ min_val = torch.finfo(attn_bias.dtype).min
+ attn_bias = attn_bias.masked_fill(~attention_mask.view(-1, 1, 1, s_k), min_val)
+ return (attn_bias, None)
+
+ def _apply_prefix_mask(self, attn_bias: torch.Tensor, prefix_mask: torch.Tensor):
+ (s_k, s_q) = attn_bias.shape[-2:]
+ if s_k != self.config.max_seq_len or s_q != self.config.max_seq_len:
+ raise ValueError('attn_bias does not match the expected shape. ' + f'The last two dimensions should both be {self.config.max_length} ' + f'but are {s_k} and {s_q}.')
+ seq_len = prefix_mask.shape[-1]
+ if seq_len > self.config.max_seq_len:
+ raise ValueError(f'prefix_mask sequence length cannot exceed max_seq_len={self.config.max_seq_len}')
+ attn_bias = attn_bias[..., :seq_len, :seq_len]
+ causal = torch.tril(torch.ones((seq_len, seq_len), dtype=torch.bool, device=prefix_mask.device)).view(1, 1, seq_len, seq_len)
+ prefix = prefix_mask.view(-1, 1, 1, seq_len)
+ cannot_attend = ~torch.logical_or(causal, prefix.bool())
+ min_val = torch.finfo(attn_bias.dtype).min
+ attn_bias = attn_bias.masked_fill(cannot_attend, min_val)
+ return attn_bias
+
+ def _apply_sequence_id(self, attn_bias: torch.Tensor, sequence_id: torch.LongTensor):
+ seq_len = sequence_id.shape[-1]
+ if seq_len > self.config.max_seq_len:
+ raise ValueError(f'sequence_id sequence length cannot exceed max_seq_len={self.config.max_seq_len}')
+ attn_bias = attn_bias[..., :seq_len, :seq_len]
+ cannot_attend = torch.logical_not(torch.eq(sequence_id.view(-1, seq_len, 1), sequence_id.view(-1, 1, seq_len))).unsqueeze(1)
+ min_val = torch.finfo(attn_bias.dtype).min
+ attn_bias = attn_bias.masked_fill(cannot_attend, min_val)
+ return attn_bias
+
+ def forward(self, input_ids: torch.LongTensor, past_key_values: Optional[List[Tuple[torch.FloatTensor]]]=None, attention_mask: Optional[torch.ByteTensor]=None, prefix_mask: Optional[torch.ByteTensor]=None, sequence_id: Optional[torch.LongTensor]=None, return_dict: Optional[bool]=None, output_attentions: Optional[bool]=None, output_hidden_states: Optional[bool]=None, use_cache: Optional[bool]=None, inputs_embeds: Optional[torch.Tensor]=None):
+ return_dict = return_dict if return_dict is not None else self.config.return_dict
+ use_cache = use_cache if use_cache is not None else self.config.use_cache
+ if attention_mask is not None:
+ attention_mask = attention_mask.bool()
+ if prefix_mask is not None:
+ prefix_mask = prefix_mask.bool()
+ if not return_dict:
+ raise NotImplementedError('return_dict False is not implemented yet for MPT')
+ if output_attentions:
+ if self.attn_impl != 'torch':
+ raise NotImplementedError('output_attentions is not implemented for MPT when using attn_impl `flash` or `triton`.')
+ if attention_mask is not None and attention_mask[:, 0].sum() != attention_mask.shape[0] and self.training:
+ raise NotImplementedError('MPT does not support training with left padding.')
+ if self.prefix_lm and prefix_mask is None:
+ raise ValueError('prefix_mask is a required argument when MPT is configured with prefix_lm=True.')
+ if self.training:
+ if self.attn_uses_sequence_id and sequence_id is None:
+ raise ValueError('sequence_id is a required argument when MPT is configured with attn_uses_sequence_id=True ' + 'and the model is in train mode.')
+ elif self.attn_uses_sequence_id is False and sequence_id is not None:
+ warnings.warn('MPT received non-None input for `sequence_id` but is configured with attn_uses_sequence_id=False. ' + 'This input will be ignored. If you want the model to use `sequence_id`, set attn_uses_sequence_id to True.')
+ if input_ids is not None:
+ S = input_ids.size(1)
+ assert S <= self.config.max_seq_len, f'Cannot forward input with seq_len={S}, this model only supports seq_len<={self.config.max_seq_len}'
+ tok_emb = self.wte(input_ids)
+ else:
+ assert inputs_embeds is not None
+ assert self.alibi, 'inputs_embeds is not implemented for MPT unless for alibi.'
+ S = inputs_embeds.size(1)
+ tok_emb = inputs_embeds
+ if self.alibi:
+ x = tok_emb
+ else:
+ past_position = 0
+ if past_key_values is not None:
+ if len(past_key_values) != self.config.n_layers:
+ raise ValueError(f'past_key_values must provide a past_key_value for each attention ' + f'layer in the network (len(past_key_values)={len(past_key_values)!r}; self.config.n_layers={self.config.n_layers!r}).')
+ past_position = past_key_values[0][0].size(1)
+ if self.attn_impl == 'torch':
+ past_position = past_key_values[0][0].size(3)
+ if S + past_position > self.config.max_seq_len:
+ raise ValueError(f'Cannot forward input with past sequence length {past_position} and current sequence length {S + 1}, this model only supports total sequence length <= {self.config.max_seq_len}.')
+ pos = torch.arange(past_position, S + past_position, dtype=torch.long, device=input_ids.device).unsqueeze(0)
+ if attention_mask is not None:
+ pos = torch.clamp(pos - torch.cumsum((~attention_mask).to(torch.int32), dim=1)[:, past_position:], min=0)
+ pos_emb = self.wpe(pos)
+ x = tok_emb + pos_emb
+ if self.embedding_fraction == 1:
+ x = self.emb_drop(x)
+ else:
+ x_shrunk = x * self.embedding_fraction + x.detach() * (1 - self.embedding_fraction)
+ assert isinstance(self.emb_drop, nn.Module)
+ x = self.emb_drop(x_shrunk)
+ (attn_bias, attention_mask) = self._attn_bias(device=x.device, dtype=torch.float32, attention_mask=attention_mask, prefix_mask=prefix_mask, sequence_id=sequence_id)
+ if use_cache and past_key_values is None:
+ past_key_values = [() for _ in range(self.config.n_layers)]
+ all_hidden_states = () if output_hidden_states else None
+ all_self_attns = () if output_attentions else None
+ for (b_idx, block) in enumerate(self.blocks):
+ if output_hidden_states:
+ assert all_hidden_states is not None
+ all_hidden_states = all_hidden_states + (x,)
+ past_key_value = past_key_values[b_idx] if past_key_values is not None else None
+ if self.gradient_checkpointing and self.training:
+ (x, attn_weights, past_key_value) = torch.utils.checkpoint.checkpoint(block, x, past_key_value, attn_bias, attention_mask, self.is_causal)
+ else:
+ (x, attn_weights, past_key_value) = block(x, past_key_value=past_key_value, attn_bias=attn_bias, attention_mask=attention_mask, is_causal=self.is_causal)
+ if past_key_values is not None:
+ past_key_values[b_idx] = past_key_value
+ if output_attentions:
+ assert all_self_attns is not None
+ all_self_attns = all_self_attns + (attn_weights,)
+ x = self.norm_f(x)
+ if output_hidden_states:
+ assert all_hidden_states is not None
+ all_hidden_states = all_hidden_states + (x,)
+ return BaseModelOutputWithPast(last_hidden_state=x, past_key_values=past_key_values, hidden_states=all_hidden_states, attentions=all_self_attns)
+
+ def param_init_fn(self, module):
+ init_fn_name = self.config.init_config['name']
+ MODEL_INIT_REGISTRY[init_fn_name](module=module, n_layers=self.config.n_layers, d_model=self.config.d_model, **self.config.init_config)
+
+ def fsdp_wrap_fn(self, module):
+ return isinstance(module, MPTBlock)
+
+ def activation_checkpointing_fn(self, module):
+ return isinstance(module, MPTBlock)
+
+class MPTForCausalLM(MPTPreTrainedModel):
+
+ def __init__(self, config: MPTConfig):
+ super().__init__(config)
+ if not config.tie_word_embeddings:
+ raise ValueError('MPTForCausalLM only supports tied word embeddings')
+ print(f'Instantiating an MPTForCausalLM model from {__file__}')
+ self.transformer = MPTModel(config)
+ for child in self.transformer.children():
+ if isinstance(child, torch.nn.ModuleList):
+ continue
+ if isinstance(child, torch.nn.Module):
+ child._fsdp_wrap = True
+ self.logit_scale = None
+ if config.logit_scale is not None:
+ logit_scale = config.logit_scale
+ if isinstance(logit_scale, str):
+ if logit_scale == 'inv_sqrt_d_model':
+ logit_scale = 1 / math.sqrt(config.d_model)
+ else:
+ raise ValueError(f"logit_scale={logit_scale!r} is not recognized as an option; use numeric value or 'inv_sqrt_d_model'.")
+ self.logit_scale = logit_scale
+
+ def get_input_embeddings(self):
+ return self.transformer.wte
+
+ def set_input_embeddings(self, value):
+ self.transformer.wte = value
+
+ def get_output_embeddings(self):
+ return self.transformer.wte
+
+ def set_output_embeddings(self, new_embeddings):
+ self.transformer.wte = new_embeddings
+
+ def set_decoder(self, decoder):
+ self.transformer = decoder
+
+ def get_decoder(self):
+ return self.transformer
+
+ def forward(self, input_ids: torch.LongTensor, past_key_values: Optional[List[Tuple[torch.FloatTensor]]]=None, attention_mask: Optional[torch.ByteTensor]=None, prefix_mask: Optional[torch.ByteTensor]=None, sequence_id: Optional[torch.LongTensor]=None, labels: Optional[torch.LongTensor]=None, return_dict: Optional[bool]=None, output_attentions: Optional[bool]=None, output_hidden_states: Optional[bool]=None, use_cache: Optional[bool]=None, inputs_embeds: Optional[torch.FloatTensor]=None):
+ return_dict = return_dict if return_dict is not None else self.config.return_dict
+ use_cache = use_cache if use_cache is not None else self.config.use_cache
+ if inputs_embeds is not None:
+ raise NotImplementedError('inputs_embeds has to be None (for hf/peft support).')
+ outputs = self.transformer(input_ids=input_ids, past_key_values=past_key_values, attention_mask=attention_mask, prefix_mask=prefix_mask, sequence_id=sequence_id, return_dict=return_dict, output_attentions=output_attentions, output_hidden_states=output_hidden_states, use_cache=use_cache)
+ logits = self.transformer.wte(outputs.last_hidden_state.to(self.transformer.wte.weight.device), True)
+ if self.logit_scale is not None:
+ if self.logit_scale == 0:
+ warnings.warn(f'Multiplying logits by self.logit_scale={self.logit_scale!r}. This will produce uniform (uninformative) outputs.')
+ logits *= self.logit_scale
+ loss = None
+ if labels is not None:
+ labels = torch.roll(labels, shifts=-1)
+ labels[:, -1] = -100
+ loss = F.cross_entropy(logits.view(-1, logits.size(-1)), labels.to(logits.device).view(-1))
+ return CausalLMOutputWithPast(loss=loss, logits=logits, past_key_values=outputs.past_key_values, hidden_states=outputs.hidden_states, attentions=outputs.attentions)
+
+ def param_init_fn(self, module):
+ init_fn_name = self.config.init_config['name']
+ MODEL_INIT_REGISTRY[init_fn_name](module=module, n_layers=self.config.n_layers, d_model=self.config.d_model, **self.config.init_config)
+
+ def fsdp_wrap_fn(self, module):
+ return isinstance(module, MPTBlock)
+
+ def activation_checkpointing_fn(self, module):
+ return isinstance(module, MPTBlock)
+
+ def prepare_inputs_for_generation(self, input_ids, past_key_values=None, inputs_embeds=None, **kwargs):
+ if inputs_embeds is not None:
+ raise NotImplementedError('inputs_embeds is not implemented for MPT yet')
+ attention_mask = kwargs['attention_mask'].bool()
+ if attention_mask[:, -1].sum() != attention_mask.shape[0]:
+ raise NotImplementedError('MPT does not support generation with right padding.')
+ if self.transformer.attn_uses_sequence_id and self.training:
+ sequence_id = torch.zeros_like(input_ids[:1])
+ else:
+ sequence_id = None
+ if past_key_values is not None:
+ input_ids = input_ids[:, -1].unsqueeze(-1)
+ if self.transformer.prefix_lm:
+ prefix_mask = torch.ones_like(attention_mask)
+ if kwargs.get('use_cache') == False:
+ raise NotImplementedError('MPT with prefix_lm=True does not support use_cache=False.')
+ else:
+ prefix_mask = None
+ return {'input_ids': input_ids, 'attention_mask': attention_mask, 'prefix_mask': prefix_mask, 'sequence_id': sequence_id, 'past_key_values': past_key_values, 'use_cache': kwargs.get('use_cache', True)}
+
+ @staticmethod
+ def _reorder_cache(past_key_values, beam_idx):
+ """Used by HuggingFace generate when using beam search with kv-caching.
+
+ See https://github.com/huggingface/transformers/blob/3ec7a47664ebe40c40f4b722f6bb1cd30c3821ec/src/transformers/models/gpt2/modeling_gpt2.py#L1122-L1133
+ for an example in transformers.
+ """
+ reordered_past = []
+ for layer_past in past_key_values:
+ reordered_past += [tuple((past_state.index_select(0, beam_idx) for past_state in layer_past))]
+ return reordered_past
\ No newline at end of file
diff --git a/videollava/model/language_model/mpt/norm.py b/videollava/model/language_model/mpt/norm.py
new file mode 100644
index 0000000000000000000000000000000000000000..067b6140fae546e5cb49cb2b1e4e6af660ced60d
--- /dev/null
+++ b/videollava/model/language_model/mpt/norm.py
@@ -0,0 +1,56 @@
+import torch
+
+def _cast_if_autocast_enabled(tensor):
+ if torch.is_autocast_enabled():
+ if tensor.device.type == 'cuda':
+ dtype = torch.get_autocast_gpu_dtype()
+ elif tensor.device.type == 'cpu':
+ dtype = torch.get_autocast_cpu_dtype()
+ else:
+ raise NotImplementedError()
+ return tensor.to(dtype=dtype)
+ return tensor
+
+class LPLayerNorm(torch.nn.LayerNorm):
+
+ def __init__(self, normalized_shape, eps=1e-05, elementwise_affine=True, device=None, dtype=None):
+ super().__init__(normalized_shape=normalized_shape, eps=eps, elementwise_affine=elementwise_affine, device=device, dtype=dtype)
+
+ def forward(self, x):
+ module_device = x.device
+ downcast_x = _cast_if_autocast_enabled(x)
+ downcast_weight = _cast_if_autocast_enabled(self.weight) if self.weight is not None else self.weight
+ downcast_bias = _cast_if_autocast_enabled(self.bias) if self.bias is not None else self.bias
+ with torch.autocast(enabled=False, device_type=module_device.type):
+ return torch.nn.functional.layer_norm(downcast_x, self.normalized_shape, downcast_weight, downcast_bias, self.eps)
+
+def rms_norm(x, weight=None, eps=1e-05):
+ output = x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + eps)
+ if weight is not None:
+ return output * weight
+ return output
+
+class RMSNorm(torch.nn.Module):
+
+ def __init__(self, normalized_shape, eps=1e-05, weight=True, dtype=None, device=None):
+ super().__init__()
+ self.eps = eps
+ if weight:
+ self.weight = torch.nn.Parameter(torch.ones(normalized_shape, dtype=dtype, device=device))
+ else:
+ self.register_parameter('weight', None)
+
+ def forward(self, x):
+ return rms_norm(x.float(), self.weight, self.eps).to(dtype=x.dtype)
+
+class LPRMSNorm(RMSNorm):
+
+ def __init__(self, normalized_shape, eps=1e-05, weight=True, dtype=None, device=None):
+ super().__init__(normalized_shape=normalized_shape, eps=eps, weight=weight, dtype=dtype, device=device)
+
+ def forward(self, x):
+ downcast_x = _cast_if_autocast_enabled(x)
+ downcast_weight = _cast_if_autocast_enabled(self.weight) if self.weight is not None else self.weight
+ with torch.autocast(enabled=False, device_type=x.device.type):
+ return rms_norm(downcast_x, downcast_weight, self.eps).to(dtype=x.dtype)
+NORM_CLASS_REGISTRY = {'layernorm': torch.nn.LayerNorm, 'low_precision_layernorm': LPLayerNorm, 'rmsnorm': RMSNorm, 'low_precision_rmsnorm': LPRMSNorm}
\ No newline at end of file
diff --git a/videollava/model/language_model/mpt/param_init_fns.py b/videollava/model/language_model/mpt/param_init_fns.py
new file mode 100644
index 0000000000000000000000000000000000000000..418b83ca2363288046f4b48b1d706c5607341fb5
--- /dev/null
+++ b/videollava/model/language_model/mpt/param_init_fns.py
@@ -0,0 +1,181 @@
+import math
+import warnings
+from collections.abc import Sequence
+from functools import partial
+from typing import Optional, Tuple, Union
+import torch
+from torch import nn
+from .norm import NORM_CLASS_REGISTRY
+
+def torch_default_param_init_fn_(module: nn.Module, verbose: int=0, **kwargs):
+ del kwargs
+ if verbose > 1:
+ warnings.warn(f"Initializing network using module's reset_parameters attribute")
+ if hasattr(module, 'reset_parameters'):
+ module.reset_parameters()
+
+def fused_init_helper_(module: nn.Module, init_fn_):
+ _fused = getattr(module, '_fused', None)
+ if _fused is None:
+ raise RuntimeError(f'Internal logic error')
+ (dim, splits) = _fused
+ splits = (0, *splits, module.weight.size(dim))
+ for (s, e) in zip(splits[:-1], splits[1:]):
+ slice_indices = [slice(None)] * module.weight.ndim
+ slice_indices[dim] = slice(s, e)
+ init_fn_(module.weight[slice_indices])
+
+def generic_param_init_fn_(module: nn.Module, init_fn_, n_layers: int, d_model: Optional[int]=None, init_div_is_residual: Union[int, float, str, bool]=True, emb_init_std: Optional[float]=None, emb_init_uniform_lim: Optional[Union[Tuple[float, float], float]]=None, verbose: int=0, **kwargs):
+ del kwargs
+ if verbose > 1:
+ warnings.warn(f'If model has bias parameters they are initialized to 0.')
+ init_div_is_residual = init_div_is_residual
+ if init_div_is_residual is False:
+ div_is_residual = 1.0
+ elif init_div_is_residual is True:
+ div_is_residual = math.sqrt(2 * n_layers)
+ elif isinstance(init_div_is_residual, float) or isinstance(init_div_is_residual, int):
+ div_is_residual = init_div_is_residual
+ elif isinstance(init_div_is_residual, str) and init_div_is_residual.isnumeric():
+ div_is_residual = float(init_div_is_residual)
+ else:
+ div_is_residual = 1.0
+ raise ValueError(f'Expected init_div_is_residual to be boolean or numeric, got {init_div_is_residual}')
+ if init_div_is_residual is not False:
+ if verbose > 1:
+ warnings.warn(f'Initializing _is_residual layers then dividing them by {div_is_residual:.3f}. ' + f'Set `init_div_is_residual: false` in init config to disable this.')
+ if isinstance(module, nn.Linear):
+ if hasattr(module, '_fused'):
+ fused_init_helper_(module, init_fn_)
+ else:
+ init_fn_(module.weight)
+ if module.bias is not None:
+ torch.nn.init.zeros_(module.bias)
+ if init_div_is_residual is not False and getattr(module, '_is_residual', False):
+ with torch.no_grad():
+ module.weight.div_(div_is_residual)
+ elif isinstance(module, nn.Embedding):
+ if emb_init_std is not None:
+ std = emb_init_std
+ if std == 0:
+ warnings.warn(f'Embedding layer initialized to 0.')
+ emb_init_fn_ = partial(torch.nn.init.normal_, mean=0.0, std=std)
+ if verbose > 1:
+ warnings.warn(f'Embedding layer initialized using normal distribution with mean=0 and std={std!r}.')
+ elif emb_init_uniform_lim is not None:
+ lim = emb_init_uniform_lim
+ if isinstance(lim, Sequence):
+ if len(lim) > 2:
+ raise ValueError(f'Uniform init requires a min and a max limit. User input: {lim}.')
+ if lim[0] == lim[1]:
+ warnings.warn(f'Embedding layer initialized to {lim[0]}.')
+ else:
+ if lim == 0:
+ warnings.warn(f'Embedding layer initialized to 0.')
+ lim = [-lim, lim]
+ (a, b) = lim
+ emb_init_fn_ = partial(torch.nn.init.uniform_, a=a, b=b)
+ if verbose > 1:
+ warnings.warn(f'Embedding layer initialized using uniform distribution in range {lim}.')
+ else:
+ emb_init_fn_ = init_fn_
+ emb_init_fn_(module.weight)
+ elif isinstance(module, tuple(set(NORM_CLASS_REGISTRY.values()))):
+ if verbose > 1:
+ warnings.warn(f'Norm weights are set to 1. If norm layer has a bias it is initialized to 0.')
+ if hasattr(module, 'weight') and module.weight is not None:
+ torch.nn.init.ones_(module.weight)
+ if hasattr(module, 'bias') and module.bias is not None:
+ torch.nn.init.zeros_(module.bias)
+ elif isinstance(module, nn.MultiheadAttention):
+ if module._qkv_same_embed_dim:
+ assert module.in_proj_weight is not None
+ assert module.q_proj_weight is None and module.k_proj_weight is None and (module.v_proj_weight is None)
+ assert d_model is not None
+ _d = d_model
+ splits = (0, _d, 2 * _d, 3 * _d)
+ for (s, e) in zip(splits[:-1], splits[1:]):
+ init_fn_(module.in_proj_weight[s:e])
+ else:
+ assert module.q_proj_weight is not None and module.k_proj_weight is not None and (module.v_proj_weight is not None)
+ assert module.in_proj_weight is None
+ init_fn_(module.q_proj_weight)
+ init_fn_(module.k_proj_weight)
+ init_fn_(module.v_proj_weight)
+ if module.in_proj_bias is not None:
+ torch.nn.init.zeros_(module.in_proj_bias)
+ if module.bias_k is not None:
+ torch.nn.init.zeros_(module.bias_k)
+ if module.bias_v is not None:
+ torch.nn.init.zeros_(module.bias_v)
+ init_fn_(module.out_proj.weight)
+ if init_div_is_residual is not False and getattr(module.out_proj, '_is_residual', False):
+ with torch.no_grad():
+ module.out_proj.weight.div_(div_is_residual)
+ if module.out_proj.bias is not None:
+ torch.nn.init.zeros_(module.out_proj.bias)
+ else:
+ for _ in module.parameters(recurse=False):
+ raise NotImplementedError(f'{module.__class__.__name__} parameters are not initialized by param_init_fn.')
+
+def _normal_init_(std, mean=0.0):
+ return partial(torch.nn.init.normal_, mean=mean, std=std)
+
+def _normal_param_init_fn_(module: nn.Module, std: float, n_layers: int, d_model: Optional[int]=None, init_div_is_residual: Union[int, float, str, bool]=True, emb_init_std: Optional[float]=None, emb_init_uniform_lim: Optional[Union[Tuple[float, float], float]]=None, verbose: int=0, **kwargs):
+ del kwargs
+ init_fn_ = _normal_init_(std=std)
+ if verbose > 1:
+ warnings.warn(f'Using torch.nn.init.normal_ init fn mean=0.0, std={std}')
+ generic_param_init_fn_(module=module, init_fn_=init_fn_, d_model=d_model, n_layers=n_layers, init_div_is_residual=init_div_is_residual, emb_init_std=emb_init_std, emb_init_uniform_lim=emb_init_uniform_lim, verbose=verbose)
+
+def baseline_param_init_fn_(module: nn.Module, init_std: float, n_layers: int, d_model: Optional[int]=None, init_div_is_residual: Union[int, float, str, bool]=True, emb_init_std: Optional[float]=None, emb_init_uniform_lim: Optional[Union[Tuple[float, float], float]]=None, verbose: int=0, **kwargs):
+ del kwargs
+ if init_std is None:
+ raise ValueError("You must set model.init_config['init_std'] to a float value to use the default initialization scheme.")
+ _normal_param_init_fn_(module=module, std=init_std, d_model=d_model, n_layers=n_layers, init_div_is_residual=init_div_is_residual, emb_init_std=emb_init_std, emb_init_uniform_lim=emb_init_uniform_lim, verbose=verbose)
+
+def small_param_init_fn_(module: nn.Module, n_layers: int, d_model: int, init_div_is_residual: Union[int, float, str, bool]=True, emb_init_std: Optional[float]=None, emb_init_uniform_lim: Optional[Union[Tuple[float, float], float]]=None, verbose: int=0, **kwargs):
+ del kwargs
+ std = math.sqrt(2 / (5 * d_model))
+ _normal_param_init_fn_(module=module, std=std, d_model=d_model, n_layers=n_layers, init_div_is_residual=init_div_is_residual, emb_init_std=emb_init_std, emb_init_uniform_lim=emb_init_uniform_lim, verbose=verbose)
+
+def neox_param_init_fn_(module: nn.Module, n_layers: int, d_model: int, emb_init_std: Optional[float]=None, emb_init_uniform_lim: Optional[Union[Tuple[float, float], float]]=None, verbose: int=0, **kwargs):
+ """From section 2.3.1 of GPT-NeoX-20B:
+
+ An Open-Source AutoregressiveLanguage Model — Black et. al. (2022)
+ see https://github.com/EleutherAI/gpt-neox/blob/9610391ab319403cef079b438edd016a2443af54/megatron/model/init_functions.py#L151
+ and https://github.com/EleutherAI/gpt-neox/blob/main/megatron/model/transformer.py
+ """
+ del kwargs
+ residual_div = n_layers / math.sqrt(10)
+ if verbose > 1:
+ warnings.warn(f'setting init_div_is_residual to {residual_div}')
+ small_param_init_fn_(module=module, d_model=d_model, n_layers=n_layers, init_div_is_residual=residual_div, emb_init_std=emb_init_std, emb_init_uniform_lim=emb_init_uniform_lim, verbose=verbose)
+
+def kaiming_uniform_param_init_fn_(module: nn.Module, n_layers: int, d_model: Optional[int]=None, init_div_is_residual: Union[int, float, str, bool]=True, emb_init_std: Optional[float]=None, emb_init_uniform_lim: Optional[Union[Tuple[float, float], float]]=None, init_gain: float=0, fan_mode: str='fan_in', init_nonlinearity: str='leaky_relu', verbose: int=0, **kwargs):
+ del kwargs
+ if verbose > 1:
+ warnings.warn(f'Using nn.init.kaiming_uniform_ init fn with parameters: ' + f'a={init_gain}, mode={fan_mode}, nonlinearity={init_nonlinearity}')
+ kaiming_uniform_ = partial(nn.init.kaiming_uniform_, a=init_gain, mode=fan_mode, nonlinearity=init_nonlinearity)
+ generic_param_init_fn_(module=module, init_fn_=kaiming_uniform_, d_model=d_model, n_layers=n_layers, init_div_is_residual=init_div_is_residual, emb_init_std=emb_init_std, emb_init_uniform_lim=emb_init_uniform_lim, verbose=verbose)
+
+def kaiming_normal_param_init_fn_(module: nn.Module, n_layers: int, d_model: Optional[int]=None, init_div_is_residual: Union[int, float, str, bool]=True, emb_init_std: Optional[float]=None, emb_init_uniform_lim: Optional[Union[Tuple[float, float], float]]=None, init_gain: float=0, fan_mode: str='fan_in', init_nonlinearity: str='leaky_relu', verbose: int=0, **kwargs):
+ del kwargs
+ if verbose > 1:
+ warnings.warn(f'Using nn.init.kaiming_normal_ init fn with parameters: ' + f'a={init_gain}, mode={fan_mode}, nonlinearity={init_nonlinearity}')
+ kaiming_normal_ = partial(torch.nn.init.kaiming_normal_, a=init_gain, mode=fan_mode, nonlinearity=init_nonlinearity)
+ generic_param_init_fn_(module=module, init_fn_=kaiming_normal_, d_model=d_model, n_layers=n_layers, init_div_is_residual=init_div_is_residual, emb_init_std=emb_init_std, emb_init_uniform_lim=emb_init_uniform_lim, verbose=verbose)
+
+def xavier_uniform_param_init_fn_(module: nn.Module, n_layers: int, d_model: Optional[int]=None, init_div_is_residual: Union[int, float, str, bool]=True, emb_init_std: Optional[float]=None, emb_init_uniform_lim: Optional[Union[Tuple[float, float], float]]=None, init_gain: float=0, verbose: int=0, **kwargs):
+ del kwargs
+ xavier_uniform_ = partial(torch.nn.init.xavier_uniform_, gain=init_gain)
+ if verbose > 1:
+ warnings.warn(f'Using torch.nn.init.xavier_uniform_ init fn with parameters: ' + f'gain={init_gain}')
+ generic_param_init_fn_(module=module, init_fn_=xavier_uniform_, d_model=d_model, n_layers=n_layers, init_div_is_residual=init_div_is_residual, emb_init_std=emb_init_std, emb_init_uniform_lim=emb_init_uniform_lim, verbose=verbose)
+
+def xavier_normal_param_init_fn_(module: nn.Module, n_layers: int, d_model: Optional[int]=None, init_div_is_residual: Union[int, float, str, bool]=True, emb_init_std: Optional[float]=None, emb_init_uniform_lim: Optional[Union[Tuple[float, float], float]]=None, init_gain: float=0, verbose: int=0, **kwargs):
+ xavier_normal_ = partial(torch.nn.init.xavier_normal_, gain=init_gain)
+ if verbose > 1:
+ warnings.warn(f'Using torch.nn.init.xavier_normal_ init fn with parameters: ' + f'gain={init_gain}')
+ generic_param_init_fn_(module=module, init_fn_=xavier_normal_, d_model=d_model, n_layers=n_layers, init_div_is_residual=init_div_is_residual, emb_init_std=emb_init_std, emb_init_uniform_lim=emb_init_uniform_lim, verbose=verbose)
+MODEL_INIT_REGISTRY = {'default_': torch_default_param_init_fn_, 'baseline_': baseline_param_init_fn_, 'kaiming_uniform_': kaiming_uniform_param_init_fn_, 'kaiming_normal_': kaiming_normal_param_init_fn_, 'neox_init_': neox_param_init_fn_, 'small_init_': small_param_init_fn_, 'xavier_uniform_': xavier_uniform_param_init_fn_, 'xavier_normal_': xavier_normal_param_init_fn_}
\ No newline at end of file
diff --git a/videollava/model/llava_arch.py b/videollava/model/llava_arch.py
new file mode 100644
index 0000000000000000000000000000000000000000..a3d3c989277ff8120f25fbdf230a3e1bebb2530f
--- /dev/null
+++ b/videollava/model/llava_arch.py
@@ -0,0 +1,390 @@
+# Copyright 2023 Haotian Liu
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from abc import ABC, abstractmethod
+
+import torch
+import torch.nn as nn
+
+from .multimodal_encoder.builder import build_image_tower, build_video_tower
+from .multimodal_projector.builder import build_vision_projector
+
+from videollava.constants import IGNORE_INDEX, IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_PATCH_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
+
+
+class LlavaMetaModel:
+
+ def __init__(self, config):
+ super(LlavaMetaModel, self).__init__(config)
+
+ if getattr(config, "mm_image_tower", None) is not None:
+ self.image_tower = build_image_tower(config, delay_load=True)
+ if getattr(config, "mm_video_tower", None) is not None:
+ self.video_tower = build_video_tower(config, delay_load=True)
+ if getattr(config, "mm_image_tower", None) is not None or getattr(config, "mm_video_tower", None) is not None:
+ self.mm_projector = build_vision_projector(config)
+
+ def get_image_tower(self):
+ image_tower = getattr(self, 'image_tower', None)
+ if type(image_tower) is list:
+ image_tower = image_tower[0]
+ return image_tower
+
+ def get_video_tower(self):
+ video_tower = getattr(self, 'video_tower', None)
+ if type(video_tower) is list:
+ video_tower = video_tower[0]
+ return video_tower
+
+ def initialize_vision_modules(self, model_args, fsdp=None):
+ # ==============================================
+ image_tower = model_args.image_tower
+ video_tower = model_args.video_tower
+ assert image_tower is not None or video_tower is not None
+ # ==============================================
+ mm_vision_select_layer = model_args.mm_vision_select_layer
+ mm_vision_select_feature = model_args.mm_vision_select_feature
+ pretrain_mm_mlp_adapter = model_args.pretrain_mm_mlp_adapter
+
+ # ==========================================================================
+
+ self.config.mm_image_tower = image_tower
+ if image_tower is not None:
+ if self.get_image_tower() is None:
+ image_tower = build_image_tower(model_args)
+
+ if fsdp is not None and len(fsdp) > 0:
+ self.image_tower = [image_tower]
+ else:
+ self.image_tower = image_tower
+ else:
+ if fsdp is not None and len(fsdp) > 0:
+ image_tower = self.image_tower[0]
+ else:
+ image_tower = self.image_tower
+ image_tower.load_model()
+
+ self.config.mm_video_tower = video_tower
+ if video_tower is not None:
+ if self.get_video_tower() is None:
+ video_tower = build_video_tower(model_args)
+
+ if fsdp is not None and len(fsdp) > 0:
+ self.video_tower = [video_tower]
+ else:
+ self.video_tower = video_tower
+ else:
+ if fsdp is not None and len(fsdp) > 0:
+ video_tower = self.video_tower[0]
+ else:
+ video_tower = self.video_tower
+ video_tower.load_model()
+
+ # ==========================================================================
+
+ self.config.use_mm_proj = True
+ self.config.mm_projector_type = getattr(model_args, 'mm_projector_type', 'linear')
+ self.config.mm_vision_select_layer = mm_vision_select_layer
+ self.config.mm_vision_select_feature = mm_vision_select_feature
+ # ==========================================================================
+ if image_tower is not None and video_tower is not None: # TODO: support different hidden_size
+ assert image_tower.hidden_size == video_tower.hidden_size
+ self.config.mm_hidden_size = image_tower.hidden_size
+ else:
+ self.config.mm_hidden_size = max(getattr(image_tower, 'hidden_size', -1),
+ getattr(video_tower, 'hidden_size', -1))
+ # ===================================================================================
+
+ if getattr(self, 'mm_projector', None) is None:
+ self.mm_projector = build_vision_projector(self.config)
+ else:
+ # In case it is frozen by LoRA
+ for p in self.mm_projector.parameters():
+ p.requires_grad = True
+
+ if pretrain_mm_mlp_adapter is not None:
+ mm_projector_weights = torch.load(pretrain_mm_mlp_adapter, map_location='cpu')
+ def get_w(weights, keyword):
+ return {k.split(keyword + '.')[1]: v for k, v in weights.items() if keyword in k}
+
+ self.mm_projector.load_state_dict(get_w(mm_projector_weights, 'mm_projector'))
+
+
+class LlavaMetaForCausalLM(ABC):
+
+ @abstractmethod
+ def get_model(self):
+ pass
+
+ def get_image_tower(self):
+ return self.get_model().get_image_tower()
+
+ def get_video_tower(self):
+ return self.get_model().get_video_tower()
+
+ def encode_images(self, images):
+ image_features = self.get_model().get_image_tower()(images)
+ image_features = self.get_model().mm_projector(image_features)
+ return image_features
+
+ def encode_videos(self, videos): # [mini_b, c, t, h, w]
+ b, _, t, _, _ = videos.shape
+ video_features = self.get_model().get_video_tower()(videos) # [mini_b, t, n, c]
+ video_features = self.get_model().mm_projector(video_features)
+ return video_features
+
+ def prepare_inputs_labels_for_multimodal(
+ self, input_ids, position_ids, attention_mask, past_key_values, labels, images
+ ):
+ # ====================================================================================================
+ image_tower = self.get_image_tower()
+ video_tower = self.get_video_tower()
+ if (image_tower is None and video_tower is None) or images is None or input_ids.shape[1] == 1:
+ if past_key_values is not None and (image_tower is not None or video_tower is not None) and images is not None and input_ids.shape[1] == 1:
+ target_shape = past_key_values[-1][-1].shape[-2] + 1
+ attention_mask = torch.cat((attention_mask, torch.ones(
+ (attention_mask.shape[0], target_shape - attention_mask.shape[1]),
+ dtype=attention_mask.dtype,
+ device=attention_mask.device
+ )), dim=1)
+ position_ids = torch.sum(attention_mask, dim=1).unsqueeze(-1) - 1
+ return input_ids, position_ids, attention_mask, past_key_values, None, labels
+
+ '''
+ images is a list, if batch_size=6
+ [
+ image(3, 224, 224), # sample 1
+ image(3, 224, 224), # sample 2
+ video(t, 3, 224, 224), # sample 3
+ image(3, 224, 224), # sample 4
+ image(3, 224, 224), # sample 4
+ video(t, 3, 224, 224), # sample 5
+ video(t, 3, 224, 224), # sample 5
+ video(t, 3, 224, 224), # sample 6
+ image(3, 224, 224), # sample 6
+ ]
+ will be converted to image_features, all video_feature will be flatten as image
+ [
+ [n, c], # sample 1
+ [n, c), # sample 2
+ *(t * [new_n, c]), # sample 3
+ [n, c], # sample 4
+ [n, c], # sample 4
+ *(t * [new_n, c]), # sample 5
+ *(t * [new_n, c]), # sample 5
+ *(t * [new_n, c]), # sample 6
+ [n, c], # sample 6
+ ]
+ '''
+ image_idx = [idx for idx, img in enumerate(images) if img.ndim == 3]
+ is_all_image = len(image_idx) == len(images)
+ video_idx = [idx for idx, vid in enumerate(images) if vid.ndim == 4]
+ images_minibatch = torch.stack([images[idx] for idx in image_idx]) if len(image_idx) > 0 else [] # mini_b c h w
+ videos_minibatch = torch.stack([images[idx] for idx in video_idx]) if len(video_idx) > 0 else [] # mini_b c t h w
+
+ tmp_image_features = [None] * (len(image_idx) + len(video_idx))
+ if getattr(images_minibatch, 'ndim', 0) == 4: # batch consists of images, [mini_b, c, h, w]
+ if image_tower is not None:
+ image_features_minibatch = self.encode_images(images_minibatch) # [mini_b, l, c]
+ else:
+ image_features_minibatch = torch.randn(1).to(self.device) # dummy feature for video-only training under tuning
+ for i, pos in enumerate(image_idx):
+ tmp_image_features[pos] = image_features_minibatch[i]
+
+ if getattr(videos_minibatch, 'ndim', 0) == 5: # batch consists of videos, [mini_b, c, t, h, w]
+ video_features_minibatch = self.encode_videos(videos_minibatch) # fake list [mini_b, t, l, c]
+ for i, pos in enumerate(video_idx):
+ t = video_features_minibatch[i].shape[0]
+ tmp_image_features[pos] = [video_features_minibatch[i][j] for j in range(t)]
+
+ new_tmp = []
+ for image in tmp_image_features:
+ # print(len(new_tmp), len(image))
+ if isinstance(image, list):
+ t = len(image)
+ for i in range(t):
+ new_tmp.append(image[i])
+ # print('add video')
+ else:
+ new_tmp.append(image)
+ image_features = new_tmp
+ # print(len(image_features), *[i.shape for i in image_features])
+ # print(len(image_features), image_features[0].shape)
+ # ====================================================================================================
+
+ # TODO: image start / end is not implemented here to support pretraining.
+ if getattr(self.config, 'tune_mm_mlp_adapter', False) and getattr(self.config, 'mm_use_im_start_end', False):
+ raise NotImplementedError
+
+ # Let's just add dummy tensors if they do not exist,
+ # it is a headache to deal with None all the time.
+ # But it is not ideal, and if you have a better idea,
+ # please open an issue / submit a PR, thanks.
+ _labels = labels
+ _position_ids = position_ids
+ _attention_mask = attention_mask
+ if attention_mask is None:
+ attention_mask = torch.ones_like(input_ids, dtype=torch.bool)
+ else:
+ attention_mask = attention_mask.bool()
+ if position_ids is None:
+ position_ids = torch.arange(0, input_ids.shape[1], dtype=torch.long, device=input_ids.device)
+ if labels is None:
+ labels = torch.full_like(input_ids, IGNORE_INDEX)
+
+ # remove the padding using attention_mask -- TODO: double check
+ input_ids = [cur_input_ids[cur_attention_mask] for cur_input_ids, cur_attention_mask in zip(input_ids, attention_mask)]
+ labels = [cur_labels[cur_attention_mask] for cur_labels, cur_attention_mask in zip(labels, attention_mask)]
+
+ new_input_embeds = []
+ new_labels = []
+ cur_image_idx = 0
+ for batch_idx, cur_input_ids in enumerate(input_ids):
+ num_images = (cur_input_ids == IMAGE_TOKEN_INDEX).sum()
+ # print(num_images, cur_input_ids)
+ if num_images == 0:
+ cur_image_features = image_features[cur_image_idx]
+ cur_input_embeds_1 = self.get_model().embed_tokens(cur_input_ids)
+ cur_input_embeds = torch.cat([cur_input_embeds_1, cur_image_features[0:0]], dim=0)
+ new_input_embeds.append(cur_input_embeds)
+ new_labels.append(labels[batch_idx])
+ cur_image_idx += 1
+ continue
+
+ image_token_indices = [-1] + torch.where(cur_input_ids == IMAGE_TOKEN_INDEX)[0].tolist() + [cur_input_ids.shape[0]]
+ cur_input_ids_noim = []
+ cur_labels = labels[batch_idx]
+ cur_labels_noim = []
+ for i in range(len(image_token_indices) - 1):
+ cur_input_ids_noim.append(cur_input_ids[image_token_indices[i]+1:image_token_indices[i+1]])
+ cur_labels_noim.append(cur_labels[image_token_indices[i]+1:image_token_indices[i+1]])
+ split_sizes = [x.shape[0] for x in cur_labels_noim]
+ cur_input_embeds = self.get_model().embed_tokens(torch.cat(cur_input_ids_noim))
+ cur_input_embeds_no_im = torch.split(cur_input_embeds, split_sizes, dim=0)
+ cur_new_input_embeds = []
+ cur_new_labels = []
+
+ for i in range(num_images + 1):
+ cur_new_input_embeds.append(cur_input_embeds_no_im[i])
+ cur_new_labels.append(cur_labels_noim[i])
+ if i < num_images:
+ # print(cur_image_idx)
+ cur_image_features = image_features[cur_image_idx]
+ cur_image_idx += 1
+ cur_new_input_embeds.append(cur_image_features)
+ cur_new_labels.append(torch.full((cur_image_features.shape[0],), IGNORE_INDEX, device=cur_labels.device, dtype=cur_labels.dtype))
+
+ cur_new_input_embeds = torch.cat(cur_new_input_embeds)
+ cur_new_labels = torch.cat(cur_new_labels)
+
+ new_input_embeds.append(cur_new_input_embeds)
+ new_labels.append(cur_new_labels)
+
+ # Truncate sequences to max length as image embeddings can make the sequence longer
+ tokenizer_model_max_length = getattr(self.config, 'tokenizer_model_max_length', None)
+ if tokenizer_model_max_length is not None:
+ new_input_embeds = [x[:tokenizer_model_max_length] for x in new_input_embeds]
+ new_labels = [x[:tokenizer_model_max_length] for x in new_labels]
+
+ # Combine them
+ max_len = max(x.shape[0] for x in new_input_embeds)
+ batch_size = len(new_input_embeds)
+
+ new_input_embeds_padded = []
+ new_labels_padded = torch.full((batch_size, max_len), IGNORE_INDEX, dtype=new_labels[0].dtype, device=new_labels[0].device)
+ attention_mask = torch.zeros((batch_size, max_len), dtype=attention_mask.dtype, device=attention_mask.device)
+ position_ids = torch.zeros((batch_size, max_len), dtype=position_ids.dtype, device=position_ids.device)
+
+ for i, (cur_new_embed, cur_new_labels) in enumerate(zip(new_input_embeds, new_labels)):
+ cur_len = cur_new_embed.shape[0]
+ if getattr(self.config, 'tokenizer_padding_side', 'right') == "left":
+ new_input_embeds_padded.append(torch.cat((
+ torch.zeros((max_len - cur_len, cur_new_embed.shape[1]), dtype=cur_new_embed.dtype, device=cur_new_embed.device),
+ cur_new_embed
+ ), dim=0))
+ if cur_len > 0:
+ new_labels_padded[i, -cur_len:] = cur_new_labels
+ attention_mask[i, -cur_len:] = True
+ position_ids[i, -cur_len:] = torch.arange(0, cur_len, dtype=position_ids.dtype, device=position_ids.device)
+ else:
+ new_input_embeds_padded.append(torch.cat((
+ cur_new_embed,
+ torch.zeros((max_len - cur_len, cur_new_embed.shape[1]), dtype=cur_new_embed.dtype, device=cur_new_embed.device)
+ ), dim=0))
+ if cur_len > 0:
+ new_labels_padded[i, :cur_len] = cur_new_labels
+ attention_mask[i, :cur_len] = True
+ position_ids[i, :cur_len] = torch.arange(0, cur_len, dtype=position_ids.dtype, device=position_ids.device)
+
+ new_input_embeds = torch.stack(new_input_embeds_padded, dim=0)
+
+ if _labels is None:
+ new_labels = None
+ else:
+ new_labels = new_labels_padded
+
+ if _attention_mask is None:
+ attention_mask = None
+ else:
+ attention_mask = attention_mask.to(dtype=_attention_mask.dtype)
+
+ if _position_ids is None:
+ position_ids = None
+
+ return None, position_ids, attention_mask, past_key_values, new_input_embeds, new_labels
+
+ def initialize_vision_tokenizer(self, model_args, tokenizer):
+ if model_args.mm_use_im_patch_token:
+ tokenizer.add_tokens([DEFAULT_IMAGE_PATCH_TOKEN], special_tokens=True)
+ self.resize_token_embeddings(len(tokenizer))
+
+ if model_args.mm_use_im_start_end:
+ num_new_tokens = tokenizer.add_tokens([DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN], special_tokens=True)
+ self.resize_token_embeddings(len(tokenizer))
+
+ if num_new_tokens > 0:
+ input_embeddings = self.get_input_embeddings().weight.data
+ output_embeddings = self.get_output_embeddings().weight.data
+
+ input_embeddings_avg = input_embeddings[:-num_new_tokens].mean(
+ dim=0, keepdim=True)
+ output_embeddings_avg = output_embeddings[:-num_new_tokens].mean(
+ dim=0, keepdim=True)
+
+ input_embeddings[-num_new_tokens:] = input_embeddings_avg
+ output_embeddings[-num_new_tokens:] = output_embeddings_avg
+
+ if model_args.tune_mm_mlp_adapter:
+ for p in self.get_input_embeddings().parameters():
+ p.requires_grad = True
+ for p in self.get_output_embeddings().parameters():
+ p.requires_grad = False
+
+ if model_args.pretrain_mm_mlp_adapter:
+ mm_projector_weights = torch.load(model_args.pretrain_mm_mlp_adapter, map_location='cpu')
+ embed_tokens_weight = mm_projector_weights['model.embed_tokens.weight']
+ assert num_new_tokens == 2
+ if input_embeddings.shape == embed_tokens_weight.shape:
+ input_embeddings[-num_new_tokens:] = embed_tokens_weight[-num_new_tokens:]
+ elif embed_tokens_weight.shape[0] == num_new_tokens:
+ input_embeddings[-num_new_tokens:] = embed_tokens_weight
+ else:
+ raise ValueError(f"Unexpected embed_tokens_weight shape. Pretrained: {embed_tokens_weight.shape}. Current: {input_embeddings.shape}. Numer of new tokens: {num_new_tokens}.")
+ elif model_args.mm_use_im_patch_token:
+ if model_args.tune_mm_mlp_adapter:
+ for p in self.get_input_embeddings().parameters():
+ p.requires_grad = False
+ for p in self.get_output_embeddings().parameters():
+ p.requires_grad = False
diff --git a/videollava/model/make_delta.py b/videollava/model/make_delta.py
new file mode 100644
index 0000000000000000000000000000000000000000..c7bc45aa48c3bfb99e030403d95c453ccd8a0138
--- /dev/null
+++ b/videollava/model/make_delta.py
@@ -0,0 +1,52 @@
+"""
+Usage:
+python3 -m llava.model.make_delta --base ~/model_weights/llama-7b --target ~/model_weights/llava-7b --delta ~/model_weights/llava-7b-delta --hub-repo-id liuhaotian/llava-7b-delta
+"""
+import argparse
+
+import torch
+from tqdm import tqdm
+from transformers import AutoTokenizer, AutoModelForCausalLM
+from videollava.model.utils import auto_upgrade
+
+
+def make_delta(base_model_path, target_model_path, delta_path, hub_repo_id):
+ print("Loading base model")
+ base = AutoModelForCausalLM.from_pretrained(
+ base_model_path, torch_dtype=torch.float16, low_cpu_mem_usage=True)
+
+ print("Loading target model")
+ auto_upgrade(target_model_path)
+ target = AutoModelForCausalLM.from_pretrained(target_model_path, torch_dtype=torch.float16, low_cpu_mem_usage=True)
+
+ print("Calculating delta")
+ for name, param in tqdm(target.state_dict().items(), desc="Calculating delta"):
+ if name not in base.state_dict():
+ assert name in ['model.mm_projector.weight', 'model.mm_projector.bias'], f'{name} not in base model'
+ continue
+ if param.data.shape == base.state_dict()[name].shape:
+ param.data -= base.state_dict()[name]
+ else:
+ assert name in ['model.embed_tokens.weight', 'lm_head.weight'], f'{name} dimension mismatch: {param.data.shape} vs {base.state_dict()[name].shape}'
+ bparam = base.state_dict()[name]
+ param.data[:bparam.shape[0], :bparam.shape[1]] -= bparam
+
+ print("Saving delta")
+ if hub_repo_id:
+ kwargs = {"push_to_hub": True, "repo_id": hub_repo_id}
+ else:
+ kwargs = {}
+ target.save_pretrained(delta_path, **kwargs)
+ target_tokenizer = AutoTokenizer.from_pretrained(target_model_path)
+ target_tokenizer.save_pretrained(delta_path, **kwargs)
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--base-model-path", type=str, required=True)
+ parser.add_argument("--target-model-path", type=str, required=True)
+ parser.add_argument("--delta-path", type=str, required=True)
+ parser.add_argument("--hub-repo-id", type=str, default=None)
+ args = parser.parse_args()
+
+ make_delta(args.base_model_path, args.target_model_path, args.delta_path, args.hub_repo_id)
diff --git a/videollava/model/multimodal_encoder/builder.py b/videollava/model/multimodal_encoder/builder.py
new file mode 100644
index 0000000000000000000000000000000000000000..69e422856a9644b0f7c1bc2c228a9d8443243fd1
--- /dev/null
+++ b/videollava/model/multimodal_encoder/builder.py
@@ -0,0 +1,24 @@
+import os
+from .clip_encoder import CLIPVisionTower
+from .languagebind import LanguageBindImageTower, LanguageBindVideoTower
+
+# ============================================================================================================
+
+def build_image_tower(image_tower_cfg, **kwargs):
+ image_tower = getattr(image_tower_cfg, 'mm_image_tower', getattr(image_tower_cfg, 'image_tower', None))
+ is_absolute_path_exists = os.path.exists(image_tower)
+ cache_dir = getattr(image_tower_cfg, 'cache_dir', './cache_dir')
+ if is_absolute_path_exists or image_tower.startswith("openai") or image_tower.startswith("laion"):
+ return CLIPVisionTower(image_tower, args=image_tower_cfg, **kwargs)
+ if image_tower.endswith('LanguageBind_Image'):
+ return LanguageBindImageTower(image_tower, args=image_tower_cfg, cache_dir=cache_dir, **kwargs)
+
+ raise ValueError(f'Unknown image tower: {image_tower}')
+
+def build_video_tower(video_tower_cfg, **kwargs):
+ video_tower = getattr(video_tower_cfg, 'mm_video_tower', getattr(video_tower_cfg, 'video_tower', None))
+ cache_dir = getattr(video_tower_cfg, 'cache_dir', './cache_dir')
+ if video_tower.endswith('LanguageBind_Video_merge'):
+ return LanguageBindVideoTower(video_tower, args=video_tower_cfg, cache_dir=cache_dir, **kwargs)
+ raise ValueError(f'Unknown video tower: {video_tower}')
+# ============================================================================================================
diff --git a/videollava/model/multimodal_encoder/clip_encoder.py b/videollava/model/multimodal_encoder/clip_encoder.py
new file mode 100644
index 0000000000000000000000000000000000000000..dbb9015b0fc9fa93483ba77cc303b793e86c36fc
--- /dev/null
+++ b/videollava/model/multimodal_encoder/clip_encoder.py
@@ -0,0 +1,78 @@
+import torch
+import torch.nn as nn
+
+from transformers import CLIPVisionModel, CLIPImageProcessor, CLIPVisionConfig
+
+
+class CLIPVisionTower(nn.Module):
+ def __init__(self, vision_tower, args, delay_load=False):
+ super().__init__()
+
+ self.is_loaded = False
+
+ self.vision_tower_name = vision_tower
+ self.select_layer = args.mm_vision_select_layer
+ self.select_feature = getattr(args, 'mm_vision_select_feature', 'patch')
+
+ if not delay_load:
+ self.load_model()
+ else:
+ self.cfg_only = CLIPVisionConfig.from_pretrained(self.vision_tower_name)
+
+ def load_model(self):
+ self.image_processor = CLIPImageProcessor.from_pretrained(self.vision_tower_name)
+ self.vision_tower = CLIPVisionModel.from_pretrained(self.vision_tower_name)
+ self.vision_tower.requires_grad_(False)
+
+ self.is_loaded = True
+
+ def feature_select(self, image_forward_outs):
+ image_features = image_forward_outs.hidden_states[self.select_layer]
+ if self.select_feature == 'patch':
+ image_features = image_features[:, 1:]
+ elif self.select_feature == 'cls_patch':
+ image_features = image_features
+ else:
+ raise ValueError(f'Unexpected select feature: {self.select_feature}')
+ return image_features
+
+ @torch.no_grad()
+ def forward(self, images):
+ if type(images) is list:
+ image_features = []
+ for image in images:
+ image_forward_out = self.vision_tower(image.to(device=self.device, dtype=self.dtype).unsqueeze(0), output_hidden_states=True)
+ image_feature = self.feature_select(image_forward_out).to(image.dtype)
+ image_features.append(image_feature)
+ else:
+ image_forward_outs = self.vision_tower(images.to(device=self.device, dtype=self.dtype), output_hidden_states=True)
+ image_features = self.feature_select(image_forward_outs).to(images.dtype)
+
+ return image_features
+
+ @property
+ def dummy_feature(self):
+ return torch.zeros(1, self.hidden_size, device=self.device, dtype=self.dtype)
+
+ @property
+ def dtype(self):
+ return self.vision_tower.dtype
+
+ @property
+ def device(self):
+ return self.vision_tower.device
+
+ @property
+ def config(self):
+ if self.is_loaded:
+ return self.vision_tower.config
+ else:
+ return self.cfg_only
+
+ @property
+ def hidden_size(self):
+ return self.config.hidden_size
+
+ @property
+ def num_patches(self):
+ return (self.config.image_size // self.config.patch_size) ** 2
diff --git a/videollava/model/multimodal_encoder/languagebind/__init__.py b/videollava/model/multimodal_encoder/languagebind/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..7f65ba29f2e30f1881fecd29ced9d92826d0ff01
--- /dev/null
+++ b/videollava/model/multimodal_encoder/languagebind/__init__.py
@@ -0,0 +1,260 @@
+import torch
+from torch import nn
+from transformers import AutoConfig
+
+from .image.configuration_image import LanguageBindImageConfig
+from .image.modeling_image import LanguageBindImage
+from .image.tokenization_image import LanguageBindImageTokenizer
+from .image.processing_image import LanguageBindImageProcessor
+
+from .video.configuration_video import LanguageBindVideoConfig
+from .video.modeling_video import LanguageBindVideo
+from .video.tokenization_video import LanguageBindVideoTokenizer
+from .video.processing_video import LanguageBindVideoProcessor
+
+from .depth.configuration_depth import LanguageBindDepthConfig
+from .depth.modeling_depth import LanguageBindDepth
+from .depth.tokenization_depth import LanguageBindDepthTokenizer
+from .depth.processing_depth import LanguageBindDepthProcessor
+
+from .audio.configuration_audio import LanguageBindAudioConfig
+from .audio.modeling_audio import LanguageBindAudio
+from .audio.tokenization_audio import LanguageBindAudioTokenizer
+from .audio.processing_audio import LanguageBindAudioProcessor
+
+from .thermal.configuration_thermal import LanguageBindThermalConfig
+from .thermal.modeling_thermal import LanguageBindThermal
+from .thermal.tokenization_thermal import LanguageBindThermalTokenizer
+from .thermal.processing_thermal import LanguageBindThermalProcessor
+
+
+
+config_dict = {
+ 'thermal': LanguageBindThermalConfig,
+ 'image': LanguageBindImageConfig,
+ 'video': LanguageBindVideoConfig,
+ 'depth': LanguageBindDepthConfig,
+ 'audio': LanguageBindAudioConfig
+}
+model_dict = {
+ 'thermal': LanguageBindThermal,
+ 'image': LanguageBindImage,
+ 'video': LanguageBindVideo,
+ 'depth': LanguageBindDepth,
+ 'audio': LanguageBindAudio
+}
+transform_dict = {
+ 'video': LanguageBindVideoProcessor,
+ 'audio': LanguageBindAudioProcessor,
+ 'depth': LanguageBindDepthProcessor,
+ 'thermal': LanguageBindThermalProcessor,
+ 'image': LanguageBindImageProcessor,
+}
+
+class LanguageBind(nn.Module):
+ def __init__(self, clip_type=('thermal', 'image', 'video', 'depth', 'audio'), use_temp=True, cache_dir='./cache_dir'):
+ super(LanguageBind, self).__init__()
+ self.use_temp = use_temp
+ self.modality_encoder = {}
+ self.modality_proj = {}
+ self.modality_scale = {}
+ self.modality_config = {}
+ for c in clip_type:
+ pretrained_ckpt = f'LanguageBind/LanguageBind_{c.capitalize()}'
+ model = model_dict[c].from_pretrained(pretrained_ckpt, cache_dir=cache_dir)
+ self.modality_encoder[c] = model.vision_model
+ self.modality_proj[c] = model.visual_projection
+ self.modality_scale[c] = model.logit_scale
+ self.modality_config[c] = model.config
+ self.modality_encoder['language'] = model.text_model
+ self.modality_proj['language'] = model.text_projection
+
+ self.modality_encoder = nn.ModuleDict(self.modality_encoder)
+ self.modality_proj = nn.ModuleDict(self.modality_proj)
+
+ def forward(self, inputs):
+ outputs = {}
+ for key, value in inputs.items():
+ value = self.modality_encoder[key](**value)[1]
+ value = self.modality_proj[key](value)
+ value = value / value.norm(p=2, dim=-1, keepdim=True)
+ if self.use_temp:
+ if key != 'language':
+ value = value * self.modality_scale[key].exp()
+ outputs[key] = value
+ return outputs
+
+def to_device(x, device):
+ out_dict = {k: v.to(device) for k, v in x.items()}
+ return out_dict
+
+
+
+
+class LanguageBindImageTower(nn.Module):
+ def __init__(self, image_tower, args, delay_load=False, cache_dir='./cache_dir'):
+ super().__init__()
+
+ self.is_loaded = False
+
+ self.image_tower_name = image_tower
+ self.select_layer = args.mm_vision_select_layer
+ self.select_feature = getattr(args, 'mm_vision_select_feature', 'patch')
+
+ self.cache_dir = cache_dir
+
+ if not delay_load:
+ self.load_model()
+ else:
+ self.cfg_only = LanguageBindImageConfig.from_pretrained(self.image_tower_name, cache_dir=self.cache_dir)
+
+ ############################################################
+ def load_model(self):
+ model = LanguageBindImage.from_pretrained(self.image_tower_name, cache_dir=self.cache_dir)
+ self.image_tower = model.vision_model
+ self.image_tower.requires_grad_(False)
+
+ self.image_processor = LanguageBindImageProcessor(model.config)
+
+ self.is_loaded = True
+
+ def feature_select(self, image_forward_outs):
+ image_features = image_forward_outs.hidden_states[self.select_layer]
+ if self.select_feature == 'patch':
+ image_features = image_features[:, 1:]
+ elif self.select_feature == 'cls_patch':
+ image_features = image_features
+ else:
+ raise ValueError(f'Unexpected select feature: {self.select_feature}')
+ return image_features
+
+ @torch.no_grad()
+ def forward(self, images):
+ if type(images) is list:
+ image_features = []
+ for image in images:
+ image_forward_out = self.image_tower(image.to(device=self.device, dtype=self.dtype).unsqueeze(0), output_hidden_states=True)
+ image_feature = self.feature_select(image_forward_out).to(image.dtype)
+ image_features.append(image_feature)
+ else:
+ # print('images', images.shape)
+ image_forward_outs = self.image_tower(images.to(device=self.device, dtype=self.dtype), output_hidden_states=True)
+ # print('image_forward_outs', len(image_forward_outs), image_forward_outs[0].shape)
+ image_features = self.feature_select(image_forward_outs).to(images.dtype)
+ # print('image_features', image_features.shape)
+
+ return image_features
+
+ @property
+ def dummy_feature(self):
+ return torch.zeros(1, self.hidden_size, device=self.device, dtype=self.dtype)
+
+ @property
+ def dtype(self):
+ return self.image_tower.embeddings.class_embedding.dtype #############
+
+ @property
+ def device(self):
+ return self.image_tower.embeddings.class_embedding.device ##############
+
+ @property
+ def config(self):
+ if self.is_loaded:
+ return self.image_tower.config
+ else:
+ return self.cfg_only
+
+ @property
+ def hidden_size(self):
+ return self.config.hidden_size
+
+ @property
+ def num_patches(self):
+ return (self.config.image_size // self.config.patch_size) ** 2
+
+
+class LanguageBindVideoTower(nn.Module):
+ def __init__(self, video_tower, args, delay_load=False, cache_dir='./cache_dir'):
+ super().__init__()
+
+ self.is_loaded = False
+
+ self.video_tower_name = video_tower
+ self.select_layer = args.mm_vision_select_layer
+ self.select_feature = getattr(args, 'mm_vision_select_feature', 'patch')
+
+ self.cache_dir = cache_dir
+
+ if not delay_load:
+ self.load_model()
+ else:
+ self.cfg_only = LanguageBindVideoConfig.from_pretrained(self.video_tower_name, cache_dir=self.cache_dir)
+
+ ############################################################
+ def load_model(self):
+ model = LanguageBindVideo.from_pretrained(self.video_tower_name, cache_dir=self.cache_dir)
+ self.video_processor = LanguageBindVideoProcessor(model.config)
+
+
+ # model = LanguageBindImage.from_pretrained('LanguageBind/LanguageBind_Image', cache_dir=self.cache_dir)
+ self.video_tower = model.vision_model
+ self.video_tower.requires_grad_(False)
+
+
+ self.is_loaded = True
+
+
+ def feature_select(self, video_forward_outs):
+ video_features = video_forward_outs.hidden_states[self.select_layer] # b t n c
+ return video_features # return all
+ # b, t, n, c = video_features.shape
+ # if self.select_feature == 'patch':
+ # video_features = video_features[:, :, 1:]
+ # else:
+ # raise ValueError(f'Unexpected select feature: {self.select_feature}')
+ # return video_features
+
+ @torch.no_grad()
+ def forward(self, videos):
+ if type(videos) is list:
+ video_features = []
+ for video in videos:
+ video_forward_out = self.video_tower(video.to(device=self.device, dtype=self.dtype).unsqueeze(0), output_hidden_states=True)
+ video_feature = self.feature_select(video_forward_out).to(video.dtype)
+ video_features.append(video_feature)
+ else:
+ video_forward_outs = self.video_tower(videos.to(device=self.device, dtype=self.dtype), output_hidden_states=True)
+ video_features = self.feature_select(video_forward_outs).to(videos.dtype)
+
+ return video_features
+
+ @property
+ def dummy_feature(self):
+ return torch.zeros(1, self.hidden_size, device=self.device, dtype=self.dtype)
+
+ @property
+ def dtype(self):
+ return self.video_tower.embeddings.class_embedding.dtype #############
+ # return torch.randn(1).cuda().dtype
+
+ @property
+ def device(self):
+ return self.video_tower.embeddings.class_embedding.device ##############
+ # return torch.randn(1).cuda().device
+
+ @property
+ def config(self):
+ if self.is_loaded:
+ return self.video_tower.config
+ else:
+ return self.cfg_only
+
+ @property
+ def hidden_size(self):
+ return self.config.hidden_size
+
+ @property
+ def num_patches(self):
+ return (self.config.image_size // self.config.patch_size) ** 2
+
+
diff --git a/videollava/model/multimodal_encoder/languagebind/audio/configuration_audio.py b/videollava/model/multimodal_encoder/languagebind/audio/configuration_audio.py
new file mode 100644
index 0000000000000000000000000000000000000000..865a496cff50fbac855413220288cd996965468b
--- /dev/null
+++ b/videollava/model/multimodal_encoder/languagebind/audio/configuration_audio.py
@@ -0,0 +1,430 @@
+import copy
+import os
+from typing import Union
+
+from transformers import PretrainedConfig
+from transformers.utils import logging
+
+logger = logging.get_logger(__name__)
+
+
+
+
+
+
+
+class CLIPTextConfig(PretrainedConfig):
+ r"""
+ This is the configuration class to store the configuration of a [`CLIPTextModel`]. It is used to instantiate a CLIP
+ text encoder according to the specified arguments, defining the model architecture. Instantiating a configuration
+ with the defaults will yield a similar configuration to that of the text encoder of the CLIP
+ [openai/clip-vit-base-patch32](https://huggingface.co/openai/clip-vit-base-patch32) architecture.
+
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
+
+ Args:
+ vocab_size (`int`, *optional*, defaults to 49408):
+ Vocabulary size of the CLIP text model. Defines the number of different tokens that can be represented by
+ the `inputs_ids` passed when calling [`CLIPModel`].
+ hidden_size (`int`, *optional*, defaults to 512):
+ Dimensionality of the encoder layers and the pooler layer.
+ intermediate_size (`int`, *optional*, defaults to 2048):
+ Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
+ num_hidden_layers (`int`, *optional*, defaults to 12):
+ Number of hidden layers in the Transformer encoder.
+ num_attention_heads (`int`, *optional*, defaults to 8):
+ Number of attention heads for each attention layer in the Transformer encoder.
+ max_position_embeddings (`int`, *optional*, defaults to 77):
+ The maximum sequence length that this model might ever be used with. Typically set this to something large
+ just in case (e.g., 512 or 1024 or 2048).
+ hidden_act (`str` or `function`, *optional*, defaults to `"quick_gelu"`):
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"selu"` and `"gelu_new"` `"quick_gelu"` are supported.
+ layer_norm_eps (`float`, *optional*, defaults to 1e-5):
+ The epsilon used by the layer normalization layers.
+ attention_dropout (`float`, *optional*, defaults to 0.0):
+ The dropout ratio for the attention probabilities.
+ initializer_range (`float`, *optional*, defaults to 0.02):
+ The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+ initializer_factor (`float`, *optional*, defaults to 1):
+ A factor for initializing all weight matrices (should be kept to 1, used internally for initialization
+ testing).
+
+ Example:
+
+ ```python
+ >>> from transformers import CLIPTextConfig, CLIPTextModel
+
+ >>> # Initializing a CLIPTextConfig with openai/clip-vit-base-patch32 style configuration
+ >>> configuration = CLIPTextConfig()
+
+ >>> # Initializing a CLIPTextModel (with random weights) from the openai/clip-vit-base-patch32 style configuration
+ >>> model = CLIPTextModel(configuration)
+
+ >>> # Accessing the model configuration
+ >>> configuration = model.config
+ ```"""
+ model_type = "clip_text_model"
+
+ def __init__(
+ self,
+ vocab_size=49408,
+ hidden_size=512,
+ intermediate_size=2048,
+ projection_dim=512,
+ num_hidden_layers=12,
+ num_attention_heads=8,
+ max_position_embeddings=77,
+ hidden_act="quick_gelu",
+ layer_norm_eps=1e-5,
+ attention_dropout=0.0,
+ initializer_range=0.02,
+ initializer_factor=1.0,
+ # This differs from `CLIPTokenizer`'s default and from openai/clip
+ # See https://github.com/huggingface/transformers/pull/24773#issuecomment-1632287538
+ pad_token_id=1,
+ bos_token_id=49406,
+ eos_token_id=49407,
+ **kwargs,
+ ):
+ super().__init__(pad_token_id=pad_token_id, bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs)
+
+ self.vocab_size = vocab_size
+ self.hidden_size = hidden_size
+ self.intermediate_size = intermediate_size
+ self.projection_dim = projection_dim
+ self.num_hidden_layers = num_hidden_layers
+ self.num_attention_heads = num_attention_heads
+ self.max_position_embeddings = max_position_embeddings
+ self.layer_norm_eps = layer_norm_eps
+ self.hidden_act = hidden_act
+ self.initializer_range = initializer_range
+ self.initializer_factor = initializer_factor
+ self.attention_dropout = attention_dropout
+ self.add_time_attn = False ######################################
+
+ @classmethod
+ def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":
+ cls._set_token_in_kwargs(kwargs)
+
+ config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)
+
+ # get the text config dict if we are loading from CLIPConfig
+ if config_dict.get("model_type") == "clip":
+ config_dict = config_dict["text_config"]
+
+ if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:
+ logger.warning(
+ f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
+ f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
+ )
+
+ return cls.from_dict(config_dict, **kwargs)
+
+
+
+
+class CLIPVisionConfig(PretrainedConfig):
+ r"""
+ This is the configuration class to store the configuration of a [`CLIPVisionModel`]. It is used to instantiate a
+ CLIP vision encoder according to the specified arguments, defining the model architecture. Instantiating a
+ configuration with the defaults will yield a similar configuration to that of the vision encoder of the CLIP
+ [openai/clip-vit-base-patch32](https://huggingface.co/openai/clip-vit-base-patch32) architecture.
+
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
+
+ Args:
+ hidden_size (`int`, *optional*, defaults to 768):
+ Dimensionality of the encoder layers and the pooler layer.
+ intermediate_size (`int`, *optional*, defaults to 3072):
+ Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
+ num_hidden_layers (`int`, *optional*, defaults to 12):
+ Number of hidden layers in the Transformer encoder.
+ num_attention_heads (`int`, *optional*, defaults to 12):
+ Number of attention heads for each attention layer in the Transformer encoder.
+ image_size (`int`, *optional*, defaults to 224):
+ The size (resolution) of each image.
+ patch_size (`int`, *optional*, defaults to 32):
+ The size (resolution) of each patch.
+ hidden_act (`str` or `function`, *optional*, defaults to `"quick_gelu"`):
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"selu"` and `"gelu_new"` ``"quick_gelu"` are supported.
+ layer_norm_eps (`float`, *optional*, defaults to 1e-5):
+ The epsilon used by the layer normalization layers.
+ attention_dropout (`float`, *optional*, defaults to 0.0):
+ The dropout ratio for the attention probabilities.
+ initializer_range (`float`, *optional*, defaults to 0.02):
+ The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+ initializer_factor (`float`, *optional*, defaults to 1):
+ A factor for initializing all weight matrices (should be kept to 1, used internally for initialization
+ testing).
+
+ Example:
+
+ ```python
+ >>> from transformers import CLIPVisionConfig, CLIPVisionModel
+
+ >>> # Initializing a CLIPVisionConfig with openai/clip-vit-base-patch32 style configuration
+ >>> configuration = CLIPVisionConfig()
+
+ >>> # Initializing a CLIPVisionModel (with random weights) from the openai/clip-vit-base-patch32 style configuration
+ >>> model = CLIPVisionModel(configuration)
+
+ >>> # Accessing the model configuration
+ >>> configuration = model.config
+ ```"""
+
+ model_type = "clip_vision_model"
+
+ def __init__(
+ self,
+ hidden_size=768,
+ intermediate_size=3072,
+ projection_dim=512,
+ num_hidden_layers=12,
+ num_attention_heads=12,
+ num_channels=3,
+ image_size=224,
+ patch_size=32,
+ hidden_act="quick_gelu",
+ layer_norm_eps=1e-5,
+ attention_dropout=0.0,
+ initializer_range=0.02,
+ initializer_factor=1.0,
+
+ add_time_attn=False, ################################
+ num_frames=1, ################################
+ force_patch_dropout=0.0, ################################
+ lora_r=2, ################################
+ lora_alpha=16, ################################
+ lora_dropout=0.0, ################################
+ num_mel_bins=0.0, ################################
+ target_length=0.0, ################################
+ video_decode_backend='decord', #########################
+ audio_sample_rate=16000,
+ audio_mean=0.5,
+ audio_std=0.5,
+ **kwargs,
+ ):
+ super().__init__(**kwargs)
+
+ self.hidden_size = hidden_size
+ self.intermediate_size = intermediate_size
+ self.projection_dim = projection_dim
+ self.num_hidden_layers = num_hidden_layers
+ self.num_attention_heads = num_attention_heads
+ self.num_channels = num_channels
+ self.patch_size = patch_size
+ self.image_size = image_size
+ self.initializer_range = initializer_range
+ self.initializer_factor = initializer_factor
+ self.attention_dropout = attention_dropout
+ self.layer_norm_eps = layer_norm_eps
+ self.hidden_act = hidden_act
+
+ self.add_time_attn = add_time_attn ################
+ self.num_frames = num_frames ################
+ self.force_patch_dropout = force_patch_dropout ################
+ self.lora_r = lora_r ################
+ self.lora_alpha = lora_alpha ################
+ self.lora_dropout = lora_dropout ################
+ self.num_mel_bins = num_mel_bins ################
+ self.target_length = target_length ################
+ self.video_decode_backend = video_decode_backend ################
+
+ self.audio_sample_rate = audio_sample_rate
+ self.audio_mean = audio_mean
+ self.audio_std = audio_std
+
+ @classmethod
+ def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":
+ cls._set_token_in_kwargs(kwargs)
+
+ config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)
+
+ # get the vision config dict if we are loading from CLIPConfig
+ if config_dict.get("model_type") == "clip":
+ config_dict = config_dict["vision_config"]
+
+ if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:
+ logger.warning(
+ f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
+ f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
+ )
+
+ return cls.from_dict(config_dict, **kwargs)
+
+
+class LanguageBindAudioConfig(PretrainedConfig):
+ r"""
+ [`CLIPConfig`] is the configuration class to store the configuration of a [`CLIPModel`]. It is used to instantiate
+ a CLIP model according to the specified arguments, defining the text model and vision model configs. Instantiating
+ a configuration with the defaults will yield a similar configuration to that of the CLIP
+ [openai/clip-vit-base-patch32](https://huggingface.co/openai/clip-vit-base-patch32) architecture.
+
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
+
+ Args:
+ text_config (`dict`, *optional*):
+ Dictionary of configuration options used to initialize [`CLIPTextConfig`].
+ vision_config (`dict`, *optional*):
+ Dictionary of configuration options used to initialize [`CLIPVisionConfig`].
+ projection_dim (`int`, *optional*, defaults to 512):
+ Dimentionality of text and vision projection layers.
+ logit_scale_init_value (`float`, *optional*, defaults to 2.6592):
+ The inital value of the *logit_scale* paramter. Default is used as per the original CLIP implementation.
+ kwargs (*optional*):
+ Dictionary of keyword arguments.
+
+ Example:
+
+ ```python
+ >>> from transformers import CLIPConfig, CLIPModel
+
+ >>> # Initializing a CLIPConfig with openai/clip-vit-base-patch32 style configuration
+ >>> configuration = CLIPConfig()
+
+ >>> # Initializing a CLIPModel (with random weights) from the openai/clip-vit-base-patch32 style configuration
+ >>> model = CLIPModel(configuration)
+
+ >>> # Accessing the model configuration
+ >>> configuration = model.config
+
+ >>> # We can also initialize a CLIPConfig from a CLIPTextConfig and a CLIPVisionConfig
+ >>> from transformers import CLIPTextConfig, CLIPVisionConfig
+
+ >>> # Initializing a CLIPText and CLIPVision configuration
+ >>> config_text = CLIPTextConfig()
+ >>> config_vision = CLIPVisionConfig()
+
+ >>> config = CLIPConfig.from_text_vision_configs(config_text, config_vision)
+ ```"""
+
+ model_type = "LanguageBindAudio"
+ is_composition = True
+
+ def __init__(
+ self, text_config=None, vision_config=None, projection_dim=512, logit_scale_init_value=2.6592, **kwargs
+ ):
+ # If `_config_dict` exist, we use them for the backward compatibility.
+ # We pop out these 2 attributes before calling `super().__init__` to avoid them being saved (which causes a lot
+ # of confusion!).
+ text_config_dict = kwargs.pop("text_config_dict", None)
+ vision_config_dict = kwargs.pop("vision_config_dict", None)
+
+ super().__init__(**kwargs)
+
+ # Instead of simply assigning `[text|vision]_config_dict` to `[text|vision]_config`, we use the values in
+ # `[text|vision]_config_dict` to update the values in `[text|vision]_config`. The values should be same in most
+ # cases, but we don't want to break anything regarding `_config_dict` that existed before commit `8827e1b2`.
+ if text_config_dict is not None:
+ if text_config is None:
+ text_config = {}
+
+ # This is the complete result when using `text_config_dict`.
+ _text_config_dict = CLIPTextConfig(**text_config_dict).to_dict()
+
+ # Give a warning if the values exist in both `_text_config_dict` and `text_config` but being different.
+ for key, value in _text_config_dict.items():
+ if key in text_config and value != text_config[key] and key not in ["transformers_version"]:
+ # If specified in `text_config_dict`
+ if key in text_config_dict:
+ message = (
+ f"`{key}` is found in both `text_config_dict` and `text_config` but with different values. "
+ f'The value `text_config_dict["{key}"]` will be used instead.'
+ )
+ # If inferred from default argument values (just to be super careful)
+ else:
+ message = (
+ f"`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The "
+ f'value `text_config["{key}"]` will be overriden.'
+ )
+ logger.warning(message)
+
+ # Update all values in `text_config` with the ones in `_text_config_dict`.
+ text_config.update(_text_config_dict)
+
+ if vision_config_dict is not None:
+ if vision_config is None:
+ vision_config = {}
+
+ # This is the complete result when using `vision_config_dict`.
+ _vision_config_dict = CLIPVisionConfig(**vision_config_dict).to_dict()
+ # convert keys to string instead of integer
+ if "id2label" in _vision_config_dict:
+ _vision_config_dict["id2label"] = {
+ str(key): value for key, value in _vision_config_dict["id2label"].items()
+ }
+
+ # Give a warning if the values exist in both `_vision_config_dict` and `vision_config` but being different.
+ for key, value in _vision_config_dict.items():
+ if key in vision_config and value != vision_config[key] and key not in ["transformers_version"]:
+ # If specified in `vision_config_dict`
+ if key in vision_config_dict:
+ message = (
+ f"`{key}` is found in both `vision_config_dict` and `vision_config` but with different "
+ f'values. The value `vision_config_dict["{key}"]` will be used instead.'
+ )
+ # If inferred from default argument values (just to be super careful)
+ else:
+ message = (
+ f"`vision_config_dict` is provided which will be used to initialize `CLIPVisionConfig`. "
+ f'The value `vision_config["{key}"]` will be overriden.'
+ )
+ logger.warning(message)
+
+ # Update all values in `vision_config` with the ones in `_vision_config_dict`.
+ vision_config.update(_vision_config_dict)
+
+ if text_config is None:
+ text_config = {}
+ logger.info("`text_config` is `None`. Initializing the `CLIPTextConfig` with default values.")
+
+ if vision_config is None:
+ vision_config = {}
+ logger.info("`vision_config` is `None`. initializing the `CLIPVisionConfig` with default values.")
+
+ self.text_config = CLIPTextConfig(**text_config)
+ self.vision_config = CLIPVisionConfig(**vision_config)
+
+ self.projection_dim = projection_dim
+ self.logit_scale_init_value = logit_scale_init_value
+ self.initializer_factor = 1.0
+
+ @classmethod
+ def from_text_vision_configs(cls, text_config: CLIPTextConfig, vision_config: CLIPVisionConfig, **kwargs):
+ r"""
+ Instantiate a [`CLIPConfig`] (or a derived class) from clip text model configuration and clip vision model
+ configuration.
+
+ Returns:
+ [`CLIPConfig`]: An instance of a configuration object
+ """
+
+ return cls(text_config=text_config.to_dict(), vision_config=vision_config.to_dict(), **kwargs)
+
+ def to_dict(self):
+ """
+ Serializes this instance to a Python dictionary. Override the default [`~PretrainedConfig.to_dict`].
+
+ Returns:
+ `Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance,
+ """
+ output = copy.deepcopy(self.__dict__)
+ output["text_config"] = self.text_config.to_dict()
+ output["vision_config"] = self.vision_config.to_dict()
+ output["model_type"] = self.__class__.model_type
+ return output
+
+
+
+
+
+
+
+
+
+
diff --git a/videollava/model/multimodal_encoder/languagebind/audio/modeling_audio.py b/videollava/model/multimodal_encoder/languagebind/audio/modeling_audio.py
new file mode 100644
index 0000000000000000000000000000000000000000..908ab43e852ccfbdf3a6b4e7546b9f0d11aac78e
--- /dev/null
+++ b/videollava/model/multimodal_encoder/languagebind/audio/modeling_audio.py
@@ -0,0 +1,1030 @@
+import math
+from typing import Optional, Tuple, Union
+
+import torch
+from einops import rearrange
+from peft import LoraConfig, get_peft_model
+from torch import nn
+from torch.nn import functional as F
+from transformers import PreTrainedModel, add_start_docstrings
+from transformers.modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling
+from transformers.models.clip.modeling_clip import CLIPMLP, CLIPAttention, CLIPTextEmbeddings, CLIPVisionEmbeddings, \
+ CLIPVisionModelWithProjection, CLIPTextModelWithProjection, _expand_mask, CLIPOutput, clip_loss
+from transformers.utils import add_start_docstrings_to_model_forward, replace_return_docstrings
+
+from .configuration_audio import LanguageBindAudioConfig, CLIPVisionConfig, CLIPTextConfig
+
+
+
+class PatchDropout(nn.Module):
+ """
+ https://arxiv.org/abs/2212.00794
+ """
+
+ def __init__(self, prob, exclude_first_token=True):
+ super().__init__()
+ assert 0 <= prob < 1.
+ self.prob = prob
+ self.exclude_first_token = exclude_first_token # exclude CLS token
+
+ def forward(self, x, B, T):
+ if not self.training or self.prob == 0.:
+ return x
+
+ if self.exclude_first_token:
+ cls_tokens, x = x[:, :1], x[:, 1:]
+ else:
+ cls_tokens = torch.jit.annotate(torch.Tensor, x[:, :1])
+
+ batch = x.size()[0]
+ num_tokens = x.size()[1]
+
+ batch_indices = torch.arange(batch)
+ batch_indices = batch_indices[..., None]
+
+ keep_prob = 1 - self.prob
+ num_patches_keep = max(1, int(num_tokens * keep_prob))
+
+ if T == 1:
+ rand = torch.randn(batch, num_tokens)
+ patch_indices_keep = rand.topk(num_patches_keep, dim=-1).indices
+ else:
+ rand = torch.randn(B, num_tokens)
+ patch_indices_keep = rand.topk(num_patches_keep, dim=-1).indices
+ patch_indices_keep = patch_indices_keep.unsqueeze(1).repeat(1, T, 1)
+ patch_indices_keep = rearrange(patch_indices_keep, 'b t n -> (b t) n')
+
+
+ x = x[batch_indices, patch_indices_keep]
+
+ if self.exclude_first_token:
+ x = torch.cat((cls_tokens, x), dim=1)
+
+ return x
+
+class CLIPEncoderLayer(nn.Module):
+ def __init__(self, config: LanguageBindAudioConfig):
+ super().__init__()
+ self.embed_dim = config.hidden_size
+ self.self_attn = CLIPAttention(config)
+ self.layer_norm1 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
+ self.mlp = CLIPMLP(config)
+ self.layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
+
+ self.add_time_attn = config.add_time_attn
+ if self.add_time_attn:
+ self.t = config.num_frames
+ self.temporal_embedding = nn.Parameter(torch.zeros(1, config.num_frames, config.hidden_size))
+ nn.init.normal_(self.temporal_embedding, std=config.hidden_size ** -0.5)
+
+ self.embed_dim = config.hidden_size
+ self.temporal_attn = CLIPAttention(config)
+ self.temporal_layer_norm1 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
+ self.temporal_mlp = CLIPMLP(config)
+ self.temporal_layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
+
+ def forward(
+ self,
+ hidden_states: torch.Tensor,
+ attention_mask: torch.Tensor,
+ causal_attention_mask: torch.Tensor,
+ output_attentions: Optional[bool] = False,
+ ) -> Tuple[torch.FloatTensor]:
+ """
+ Args:
+ hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
+ attention_mask (`torch.FloatTensor`): attention mask of size
+ `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
+ `(config.encoder_attention_heads,)`.
+ output_attentions (`bool`, *optional*):
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under
+ returned tensors for more detail.
+ """
+
+
+ if self.add_time_attn:
+ bt, n, d = hidden_states.shape
+ t = self.t
+
+ # time embed
+ if t != 1:
+ n = hidden_states.shape[1]
+ hidden_states = rearrange(hidden_states, '(b t) n d -> (b n) t d', t=t)
+ hidden_states = hidden_states + self.temporal_embedding[:, :t, :]
+ hidden_states = rearrange(hidden_states, '(b n) t d -> (b t) n d', n=n)
+
+ # time attn
+ residual = hidden_states
+ hidden_states = rearrange(hidden_states, '(b t) n d -> (b n) t d', t=t)
+ # hidden_states = self.layer_norm1(hidden_states) # share layernorm
+ hidden_states = self.temporal_layer_norm1(hidden_states)
+ hidden_states, attn_weights = self.temporal_attn(
+ hidden_states=hidden_states,
+ attention_mask=attention_mask,
+ causal_attention_mask=causal_attention_mask,
+ output_attentions=output_attentions,
+ )
+ hidden_states = residual + rearrange(hidden_states, '(b n) t d -> (b t) n d', n=n)
+
+ residual = hidden_states
+ hidden_states = rearrange(hidden_states, '(b t) n d -> (b n) t d', t=t)
+ # hidden_states = self.layer_norm2(hidden_states) # share layernorm
+ hidden_states = self.temporal_layer_norm2(hidden_states)
+ hidden_states = self.temporal_mlp(hidden_states)
+ hidden_states = residual + rearrange(hidden_states, '(b n) t d -> (b t) n d', n=n)
+
+ # spatial attn
+ residual = hidden_states
+
+ hidden_states = self.layer_norm1(hidden_states)
+ hidden_states, attn_weights = self.self_attn(
+ hidden_states=hidden_states,
+ attention_mask=attention_mask,
+ causal_attention_mask=causal_attention_mask,
+ output_attentions=output_attentions,
+ )
+ hidden_states = residual + hidden_states
+
+ residual = hidden_states
+ hidden_states = self.layer_norm2(hidden_states)
+ hidden_states = self.mlp(hidden_states)
+ hidden_states = residual + hidden_states
+
+ outputs = (hidden_states,)
+
+ if output_attentions:
+ outputs += (attn_weights,)
+
+ return outputs
+
+
+
+
+
+
+
+
+
+class CLIPPreTrainedModel(PreTrainedModel):
+ """
+ An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
+ models.
+ """
+
+ config_class = LanguageBindAudioConfig
+ base_model_prefix = "clip"
+ supports_gradient_checkpointing = True
+ _keys_to_ignore_on_load_missing = [r"position_ids"]
+
+ def _init_weights(self, module):
+ """Initialize the weights"""
+ factor = self.config.initializer_factor
+ if isinstance(module, CLIPTextEmbeddings):
+ module.token_embedding.weight.data.normal_(mean=0.0, std=factor * 0.02)
+ module.position_embedding.weight.data.normal_(mean=0.0, std=factor * 0.02)
+ elif isinstance(module, CLIPVisionEmbeddings):
+ factor = self.config.initializer_factor
+ nn.init.normal_(module.class_embedding, mean=0.0, std=module.embed_dim**-0.5 * factor)
+ nn.init.normal_(module.patch_embedding.weight, std=module.config.initializer_range * factor)
+ nn.init.normal_(module.position_embedding.weight, std=module.config.initializer_range * factor)
+ elif isinstance(module, CLIPAttention):
+ factor = self.config.initializer_factor
+ in_proj_std = (module.embed_dim**-0.5) * ((2 * module.config.num_hidden_layers) ** -0.5) * factor
+ out_proj_std = (module.embed_dim**-0.5) * factor
+ nn.init.normal_(module.q_proj.weight, std=in_proj_std)
+ nn.init.normal_(module.k_proj.weight, std=in_proj_std)
+ nn.init.normal_(module.v_proj.weight, std=in_proj_std)
+ nn.init.normal_(module.out_proj.weight, std=out_proj_std)
+ elif isinstance(module, CLIPMLP):
+ factor = self.config.initializer_factor
+ in_proj_std = (
+ (module.config.hidden_size**-0.5) * ((2 * module.config.num_hidden_layers) ** -0.5) * factor
+ )
+ fc_std = (2 * module.config.hidden_size) ** -0.5 * factor
+ nn.init.normal_(module.fc1.weight, std=fc_std)
+ nn.init.normal_(module.fc2.weight, std=in_proj_std)
+ elif isinstance(module, LanguageBindAudio):
+ nn.init.normal_(
+ module.text_projection.weight,
+ std=module.text_embed_dim**-0.5 * self.config.initializer_factor,
+ )
+ nn.init.normal_(
+ module.visual_projection.weight,
+ std=module.vision_embed_dim**-0.5 * self.config.initializer_factor,
+ )
+ elif isinstance(module, CLIPVisionModelWithProjection):
+ nn.init.normal_(
+ module.visual_projection.weight,
+ std=self.config.hidden_size**-0.5 * self.config.initializer_factor,
+ )
+ elif isinstance(module, CLIPTextModelWithProjection):
+ nn.init.normal_(
+ module.text_projection.weight,
+ std=self.config.hidden_size**-0.5 * self.config.initializer_factor,
+ )
+
+ if isinstance(module, nn.LayerNorm):
+ module.bias.data.zero_()
+ module.weight.data.fill_(1.0)
+ if isinstance(module, nn.Linear) and module.bias is not None:
+ module.bias.data.zero_()
+
+ def _set_gradient_checkpointing(self, module, value=False):
+ if isinstance(module, CLIPEncoder):
+ module.gradient_checkpointing = value
+
+
+CLIP_START_DOCSTRING = r"""
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
+
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
+
+ Parameters:
+ config ([`CLIPConfig`]): Model configuration class with all the parameters of the model.
+ Initializing with a config file does not load the weights associated with the model, only the
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
+"""
+
+CLIP_TEXT_INPUTS_DOCSTRING = r"""
+ Args:
+ input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
+ Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
+ it.
+
+ Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
+
+ [What are input IDs?](../glossary#input-ids)
+ attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
+ Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
+
+ - 1 for tokens that are **not masked**,
+ - 0 for tokens that are **masked**.
+
+ [What are attention masks?](../glossary#attention-mask)
+ position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
+
+ [What are position IDs?](../glossary#position-ids)
+ output_attentions (`bool`, *optional*):
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
+ tensors for more detail.
+ output_hidden_states (`bool`, *optional*):
+ Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
+ more detail.
+ return_dict (`bool`, *optional*):
+ Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+"""
+
+CLIP_VISION_INPUTS_DOCSTRING = r"""
+ Args:
+ pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
+ Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using
+ [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details.
+ output_attentions (`bool`, *optional*):
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
+ tensors for more detail.
+ output_hidden_states (`bool`, *optional*):
+ Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
+ more detail.
+ return_dict (`bool`, *optional*):
+ Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+"""
+
+CLIP_INPUTS_DOCSTRING = r"""
+ Args:
+ input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
+ Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
+ it.
+
+ Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
+
+ [What are input IDs?](../glossary#input-ids)
+ attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
+ Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
+
+ - 1 for tokens that are **not masked**,
+ - 0 for tokens that are **masked**.
+
+ [What are attention masks?](../glossary#attention-mask)
+ position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
+
+ [What are position IDs?](../glossary#position-ids)
+ pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
+ Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using
+ [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details.
+ return_loss (`bool`, *optional*):
+ Whether or not to return the contrastive loss.
+ output_attentions (`bool`, *optional*):
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
+ tensors for more detail.
+ output_hidden_states (`bool`, *optional*):
+ Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
+ more detail.
+ return_dict (`bool`, *optional*):
+ Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+"""
+
+
+class CLIPEncoder(nn.Module):
+ """
+ Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
+ [`CLIPEncoderLayer`].
+
+ Args:
+ config: CLIPConfig
+ """
+
+ def __init__(self, config: LanguageBindAudioConfig):
+ super().__init__()
+ self.config = config
+ self.layers = nn.ModuleList([CLIPEncoderLayer(config) for _ in range(config.num_hidden_layers)])
+ self.gradient_checkpointing = False
+
+ def forward(
+ self,
+ inputs_embeds,
+ attention_mask: Optional[torch.Tensor] = None,
+ causal_attention_mask: Optional[torch.Tensor] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> Union[Tuple, BaseModelOutput]:
+ r"""
+ Args:
+ inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
+ This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+ than the model's internal embedding lookup matrix.
+ attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
+ Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
+
+ - 1 for tokens that are **not masked**,
+ - 0 for tokens that are **masked**.
+
+ [What are attention masks?](../glossary#attention-mask)
+ causal_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
+ Causal mask for the text model. Mask values selected in `[0, 1]`:
+
+ - 1 for tokens that are **not masked**,
+ - 0 for tokens that are **masked**.
+
+ [What are attention masks?](../glossary#attention-mask)
+ output_attentions (`bool`, *optional*):
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under
+ returned tensors for more detail.
+ output_hidden_states (`bool`, *optional*):
+ Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
+ for more detail.
+ return_dict (`bool`, *optional*):
+ Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+ """
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+ output_hidden_states = (
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+ )
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+ encoder_states = () if output_hidden_states else None
+ all_attentions = () if output_attentions else None
+
+ hidden_states = inputs_embeds
+ for idx, encoder_layer in enumerate(self.layers):
+ if output_hidden_states:
+ encoder_states = encoder_states + (hidden_states,)
+ if self.gradient_checkpointing and self.training:
+
+ def create_custom_forward(module):
+ def custom_forward(*inputs):
+ return module(*inputs, output_attentions)
+
+ return custom_forward
+
+ layer_outputs = torch.utils.checkpoint.checkpoint(
+ create_custom_forward(encoder_layer),
+ hidden_states,
+ attention_mask,
+ causal_attention_mask,
+ )
+ else:
+ layer_outputs = encoder_layer(
+ hidden_states,
+ attention_mask,
+ causal_attention_mask,
+ output_attentions=output_attentions,
+ )
+
+ hidden_states = layer_outputs[0]
+
+ if output_attentions:
+ all_attentions = all_attentions + (layer_outputs[1],)
+
+ if output_hidden_states:
+ encoder_states = encoder_states + (hidden_states,)
+
+ if not return_dict:
+ return tuple(v for v in [hidden_states, encoder_states, all_attentions] if v is not None)
+ return BaseModelOutput(
+ last_hidden_state=hidden_states, hidden_states=encoder_states, attentions=all_attentions
+ )
+
+
+# Copied from transformers.models.bart.modeling_bart._make_causal_mask
+def _make_causal_mask(
+ input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device, past_key_values_length: int = 0
+):
+ """
+ Make causal mask used for bi-directional self-attention.
+ """
+ bsz, tgt_len = input_ids_shape
+ mask = torch.full((tgt_len, tgt_len), torch.finfo(dtype).min, device=device)
+ mask_cond = torch.arange(mask.size(-1), device=device)
+ mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0)
+ mask = mask.to(dtype)
+
+ if past_key_values_length > 0:
+ mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), mask], dim=-1)
+ return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length)
+
+
+class CLIPTextTransformer(nn.Module):
+ def __init__(self, config: CLIPTextConfig):
+ super().__init__()
+ self.config = config
+ embed_dim = config.hidden_size
+ self.embeddings = CLIPTextEmbeddings(config)
+ self.encoder = CLIPEncoder(config)
+ self.final_layer_norm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
+
+ @add_start_docstrings_to_model_forward(CLIP_TEXT_INPUTS_DOCSTRING)
+ @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=CLIPTextConfig)
+ def forward(
+ self,
+ input_ids: Optional[torch.Tensor] = None,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.Tensor] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> Union[Tuple, BaseModelOutputWithPooling]:
+ r"""
+ Returns:
+
+ """
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+ output_hidden_states = (
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+ )
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+ if input_ids is None:
+ raise ValueError("You have to specify input_ids")
+
+ input_shape = input_ids.size()
+ input_ids = input_ids.view(-1, input_shape[-1])
+
+ hidden_states = self.embeddings(input_ids=input_ids, position_ids=position_ids)
+
+ # CLIP's text model uses causal mask, prepare it here.
+ # https://github.com/openai/CLIP/blob/cfcffb90e69f37bf2ff1e988237a0fbe41f33c04/clip/model.py#L324
+ causal_attention_mask = _make_causal_mask(input_shape, hidden_states.dtype, device=hidden_states.device)
+ # expand attention_mask
+ if attention_mask is not None:
+ # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]
+ attention_mask = _expand_mask(attention_mask, hidden_states.dtype)
+
+ encoder_outputs = self.encoder(
+ inputs_embeds=hidden_states,
+ attention_mask=attention_mask,
+ causal_attention_mask=causal_attention_mask,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+
+ last_hidden_state = encoder_outputs[0]
+ last_hidden_state = self.final_layer_norm(last_hidden_state)
+
+ # text_embeds.shape = [batch_size, sequence_length, transformer.width]
+ # take features from the eot embedding (eot_token is the highest number in each sequence)
+ # casting to torch.int for onnx compatibility: argmax doesn't support int64 inputs with opset 14
+ pooled_output = last_hidden_state[
+ torch.arange(last_hidden_state.shape[0], device=last_hidden_state.device),
+ input_ids.to(dtype=torch.int, device=last_hidden_state.device).argmax(dim=-1),
+ ]
+
+ if not return_dict:
+ return (last_hidden_state, pooled_output) + encoder_outputs[1:]
+
+ return BaseModelOutputWithPooling(
+ last_hidden_state=last_hidden_state,
+ pooler_output=pooled_output,
+ hidden_states=encoder_outputs.hidden_states,
+ attentions=encoder_outputs.attentions,
+ )
+
+
+@add_start_docstrings(
+ """The text model from CLIP without any head or projection on top.""",
+ CLIP_START_DOCSTRING,
+)
+class CLIPTextModel(CLIPPreTrainedModel):
+ config_class = CLIPTextConfig
+
+ _no_split_modules = ["CLIPEncoderLayer"]
+
+ def __init__(self, config: CLIPTextConfig):
+ super().__init__(config)
+ self.text_model = CLIPTextTransformer(config)
+ # Initialize weights and apply final processing
+ self.post_init()
+
+ def get_input_embeddings(self) -> nn.Module:
+ return self.text_model.embeddings.token_embedding
+
+ def set_input_embeddings(self, value):
+ self.text_model.embeddings.token_embedding = value
+
+ @add_start_docstrings_to_model_forward(CLIP_TEXT_INPUTS_DOCSTRING)
+ @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=CLIPTextConfig)
+ def forward(
+ self,
+ input_ids: Optional[torch.Tensor] = None,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.Tensor] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> Union[Tuple, BaseModelOutputWithPooling]:
+ r"""
+ Returns:
+
+ Examples:
+
+ ```python
+ >>> from transformers import AutoTokenizer, CLIPTextModel
+
+ >>> model = CLIPTextModel.from_pretrained("openai/clip-vit-base-patch32")
+ >>> tokenizer = AutoTokenizer.from_pretrained("openai/clip-vit-base-patch32")
+
+ >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt")
+
+ >>> outputs = model(**inputs)
+ >>> last_hidden_state = outputs.last_hidden_state
+ >>> pooled_output = outputs.pooler_output # pooled (EOS token) states
+ ```"""
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+ return self.text_model(
+ input_ids=input_ids,
+ attention_mask=attention_mask,
+ position_ids=position_ids,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+
+
+class CLIPVisionTransformer(nn.Module):
+ def __init__(self, config: CLIPVisionConfig):
+ super().__init__()
+ self.config = config
+ embed_dim = config.hidden_size
+
+ self.embeddings = CLIPVisionEmbeddings(config)
+ self.patch_dropout = PatchDropout(config.force_patch_dropout)
+ self.pre_layrnorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
+ self.encoder = CLIPEncoder(config)
+ self.post_layernorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
+
+ @add_start_docstrings_to_model_forward(CLIP_VISION_INPUTS_DOCSTRING)
+ @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=CLIPVisionConfig)
+ def forward(
+ self,
+ pixel_values: Optional[torch.FloatTensor] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> Union[Tuple, BaseModelOutputWithPooling]:
+ r"""
+ Returns:
+
+ """
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+ output_hidden_states = (
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+ )
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+ if pixel_values is None:
+ raise ValueError("You have to specify pixel_values")
+ ######################################
+ if len(pixel_values.shape) == 7:
+ b_new, pair_new, T, bs_new, channel_new, h_new, w_new = pixel_values.shape
+ # print(pixel_values.shape)
+ B = b_new * pair_new * bs_new
+ pixel_values = pixel_values.reshape(B*T, channel_new, h_new, w_new)
+
+ elif len(pixel_values.shape) == 5:
+ B, _, T, _, _ = pixel_values.shape
+ # print(pixel_values.shape)
+ pixel_values = rearrange(pixel_values, 'b c t h w -> (b t) c h w')
+ else:
+ # print(pixel_values.shape)
+ B, _, _, _ = pixel_values.shape
+ T = 1
+ ###########################
+ hidden_states = self.embeddings(pixel_values)
+
+ hidden_states = self.patch_dropout(hidden_states, B, T) ##############################################
+
+ hidden_states = self.pre_layrnorm(hidden_states)
+
+ encoder_outputs = self.encoder(
+ inputs_embeds=hidden_states,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+
+ last_hidden_state = encoder_outputs[0]
+ pooled_output = last_hidden_state[:, 0, :]
+ pooled_output = self.post_layernorm(pooled_output)
+
+ pooled_output = pooled_output.reshape(B, T, -1).mean(1) ################################
+
+ if not return_dict:
+ return (last_hidden_state, pooled_output) + encoder_outputs[1:]
+
+ return BaseModelOutputWithPooling(
+ last_hidden_state=last_hidden_state,
+ pooler_output=pooled_output,
+ hidden_states=encoder_outputs.hidden_states,
+ attentions=encoder_outputs.attentions,
+ )
+
+
+@add_start_docstrings(
+ """The vision model from CLIP without any head or projection on top.""",
+ CLIP_START_DOCSTRING,
+)
+class CLIPVisionModel(CLIPPreTrainedModel):
+ config_class = CLIPVisionConfig
+ main_input_name = "pixel_values"
+
+ def __init__(self, config: CLIPVisionConfig):
+ super().__init__(config)
+ self.vision_model = CLIPVisionTransformer(config)
+ # Initialize weights and apply final processing
+ self.post_init()
+
+ def get_input_embeddings(self) -> nn.Module:
+ return self.vision_model.embeddings.patch_embedding
+
+ @add_start_docstrings_to_model_forward(CLIP_VISION_INPUTS_DOCSTRING)
+ @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=CLIPVisionConfig)
+ def forward(
+ self,
+ pixel_values: Optional[torch.FloatTensor] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> Union[Tuple, BaseModelOutputWithPooling]:
+ r"""
+ Returns:
+
+ Examples:
+
+ ```python
+ >>> from PIL import Image
+ >>> import requests
+ >>> from transformers import AutoProcessor, CLIPVisionModel
+
+ >>> model = CLIPVisionModel.from_pretrained("openai/clip-vit-base-patch32")
+ >>> processor = AutoProcessor.from_pretrained("openai/clip-vit-base-patch32")
+
+ >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
+ >>> image = Image.open(requests.get(url, stream=True).raw)
+
+ >>> inputs = processor(images=image, return_tensors="pt")
+
+ >>> outputs = model(**inputs)
+ >>> last_hidden_state = outputs.last_hidden_state
+ >>> pooled_output = outputs.pooler_output # pooled CLS states
+ ```"""
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+ return self.vision_model(
+ pixel_values=pixel_values,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+
+
+@add_start_docstrings(CLIP_START_DOCSTRING)
+class LanguageBindAudio(CLIPPreTrainedModel):
+ config_class = LanguageBindAudioConfig
+
+ def __init__(self, config: LanguageBindAudioConfig):
+ super().__init__(config)
+
+ if not isinstance(config.text_config, CLIPTextConfig):
+ raise ValueError(
+ "config.text_config is expected to be of type CLIPTextConfig but is of type"
+ f" {type(config.text_config)}."
+ )
+
+ if not isinstance(config.vision_config, CLIPVisionConfig):
+ raise ValueError(
+ "config.vision_config is expected to be of type CLIPVisionConfig but is of type"
+ f" {type(config.vision_config)}."
+ )
+
+ text_config = config.text_config
+ vision_config = config.vision_config
+ self.add_time_attn = vision_config.add_time_attn
+ self.lora_r = vision_config.lora_r
+ self.lora_alpha = vision_config.lora_alpha
+ self.lora_dropout = vision_config.lora_dropout
+
+ self.projection_dim = config.projection_dim
+ self.text_embed_dim = text_config.hidden_size
+ self.vision_embed_dim = vision_config.hidden_size
+
+ self.text_model = CLIPTextTransformer(text_config)
+ self.vision_model = CLIPVisionTransformer(vision_config)
+
+ self.visual_projection = nn.Linear(self.vision_embed_dim, self.projection_dim, bias=False)
+ self.text_projection = nn.Linear(self.text_embed_dim, self.projection_dim, bias=False)
+ self.logit_scale = nn.Parameter(torch.tensor(self.config.logit_scale_init_value))
+
+ # Initialize weights and apply final processing
+ self.post_init()
+ self.convert_to_lora()
+ self.resize_pos(self.vision_model.embeddings, vision_config)
+
+ def convert_to_lora(self):
+ if self.lora_r == 0:
+ return
+ if self.add_time_attn:
+ target_modules = ["temporal_attn.k_proj", "temporal_attn.v_proj",
+ "temporal_attn.q_proj", "temporal_attn.out_proj",
+ "temporal_mlp.fc1", "temporal_mlp.fc2"]
+ else:
+ target_modules = ["k_proj", "v_proj", "q_proj", "out_proj"]
+ config = LoraConfig(
+ r=self.lora_r, # 16
+ lora_alpha=self.lora_alpha, # 16
+ target_modules=target_modules, # self_attn.out_proj
+ lora_dropout=self.lora_dropout, # 0.1
+ bias="none",
+ modules_to_save=[],
+ )
+ self.vision_model.encoder.is_gradient_checkpointing = False
+ self.vision_model.encoder = get_peft_model(self.vision_model.encoder, config)
+
+ def resize_pos(self, m, vision_config):
+ # convert embedding
+ if vision_config.num_mel_bins!=0 and vision_config.target_length!=0:
+ m.image_size = [vision_config.num_mel_bins, vision_config.target_length]
+ m.config.image_size = [m.image_size, m.image_size] if isinstance(m.image_size, int) else m.image_size
+ # pos resize
+ old_pos_embed_state_dict = m.position_embedding.state_dict()
+ old_pos_embed = old_pos_embed_state_dict['weight']
+ dtype = old_pos_embed.dtype
+ grid_size = [m.config.image_size[0] // m.patch_size, m.config.image_size[1] // m.patch_size]
+ extra_tokens = 1 # FIXME detect different token configs (ie no class token, or more)
+ new_seq_len = grid_size[0] * grid_size[1] + extra_tokens
+ if new_seq_len == old_pos_embed.shape[0]:
+ # m.to(args.device)
+ return
+
+ m.num_patches = grid_size[0] * grid_size[1]
+ m.num_positions = m.num_patches + 1
+ m.register_buffer("position_ids", torch.arange(m.num_positions).expand((1, -1)))
+ new_position_embedding = nn.Embedding(m.num_positions, m.embed_dim)
+
+ if extra_tokens:
+ pos_emb_tok, pos_emb_img = old_pos_embed[:extra_tokens], old_pos_embed[extra_tokens:]
+ else:
+ pos_emb_tok, pos_emb_img = None, old_pos_embed
+ old_grid_size = [int(math.sqrt(len(pos_emb_img)))] * 2
+
+ # if is_master(args):
+ # logging.info('Resizing position embedding grid-size from %s to %s', old_grid_size, grid_size)
+ pos_emb_img = pos_emb_img.reshape(1, old_grid_size[0], old_grid_size[1], -1).permute(0, 3, 1, 2)
+ pos_emb_img = F.interpolate(
+ pos_emb_img,
+ size=grid_size,
+ mode='bicubic',
+ antialias=True,
+ align_corners=False,
+ )
+ pos_emb_img = pos_emb_img.permute(0, 2, 3, 1).reshape(1, grid_size[0] * grid_size[1], -1)[0]
+ if pos_emb_tok is not None:
+ new_pos_embed = torch.cat([pos_emb_tok, pos_emb_img], dim=0)
+ else:
+ new_pos_embed = pos_emb_img
+ old_pos_embed_state_dict['weight'] = new_pos_embed.to(dtype)
+ m.position_embedding = new_position_embedding
+ m.position_embedding.load_state_dict(old_pos_embed_state_dict)
+
+ # m.to(args.device)
+
+ @add_start_docstrings_to_model_forward(CLIP_TEXT_INPUTS_DOCSTRING)
+ def get_text_features(
+ self,
+ input_ids: Optional[torch.Tensor] = None,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.Tensor] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> torch.FloatTensor:
+ r"""
+ Returns:
+ text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by
+ applying the projection layer to the pooled output of [`CLIPTextModel`].
+
+ Examples:
+
+ ```python
+ >>> from transformers import AutoTokenizer, CLIPModel
+
+ >>> model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")
+ >>> tokenizer = AutoTokenizer.from_pretrained("openai/clip-vit-base-patch32")
+
+ >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt")
+ >>> text_features = model.get_text_features(**inputs)
+ ```"""
+ # Use CLIP model's config for some fields (if specified) instead of those of vision & text components.
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+ output_hidden_states = (
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+ )
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+ text_outputs = self.text_model(
+ input_ids=input_ids,
+ attention_mask=attention_mask,
+ position_ids=position_ids,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+
+ pooled_output = text_outputs[1]
+ text_features = self.text_projection(pooled_output)
+
+ return text_features
+
+ @add_start_docstrings_to_model_forward(CLIP_VISION_INPUTS_DOCSTRING)
+ def get_image_features(
+ self,
+ pixel_values: Optional[torch.FloatTensor] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> torch.FloatTensor:
+ r"""
+ Returns:
+ image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by
+ applying the projection layer to the pooled output of [`CLIPVisionModel`].
+
+ Examples:
+
+ ```python
+ >>> from PIL import Image
+ >>> import requests
+ >>> from transformers import AutoProcessor, CLIPModel
+
+ >>> model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")
+ >>> processor = AutoProcessor.from_pretrained("openai/clip-vit-base-patch32")
+
+ >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
+ >>> image = Image.open(requests.get(url, stream=True).raw)
+
+ >>> inputs = processor(images=image, return_tensors="pt")
+
+ >>> image_features = model.get_image_features(**inputs)
+ ```"""
+ # Use CLIP model's config for some fields (if specified) instead of those of vision & text components.
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+ output_hidden_states = (
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+ )
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+ vision_outputs = self.vision_model(
+ pixel_values=pixel_values,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+
+ pooled_output = vision_outputs[1] # pooled_output
+ image_features = self.visual_projection(pooled_output)
+
+ return image_features
+
+ @add_start_docstrings_to_model_forward(CLIP_INPUTS_DOCSTRING)
+ @replace_return_docstrings(output_type=CLIPOutput, config_class=LanguageBindAudioConfig)
+ def forward(
+ self,
+ input_ids: Optional[torch.LongTensor] = None,
+ pixel_values: Optional[torch.FloatTensor] = None,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.LongTensor] = None,
+ return_loss: Optional[bool] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> Union[Tuple, CLIPOutput]:
+ r"""
+ Returns:
+
+ Examples:
+
+ ```python
+ >>> from PIL import Image
+ >>> import requests
+ >>> from transformers import AutoProcessor, CLIPModel
+
+ >>> model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")
+ >>> processor = AutoProcessor.from_pretrained("openai/clip-vit-base-patch32")
+
+ >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
+ >>> image = Image.open(requests.get(url, stream=True).raw)
+
+ >>> inputs = processor(
+ ... text=["a photo of a cat", "a photo of a dog"], images=image, return_tensors="pt", padding=True
+ ... )
+
+ >>> outputs = model(**inputs)
+ >>> logits_per_image = outputs.logits_per_image # this is the image-text similarity score
+ >>> probs = logits_per_image.softmax(dim=1) # we can take the softmax to get the label probabilities
+ ```"""
+ # Use CLIP model's config for some fields (if specified) instead of those of vision & text components.
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+ output_hidden_states = (
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+ )
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+ vision_outputs = self.vision_model(
+ pixel_values=pixel_values,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+
+ text_outputs = self.text_model(
+ input_ids=input_ids,
+ attention_mask=attention_mask,
+ position_ids=position_ids,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+
+ image_embeds = vision_outputs[1]
+ image_embeds = self.visual_projection(image_embeds)
+
+ text_embeds = text_outputs[1]
+ text_embeds = self.text_projection(text_embeds)
+
+ # normalized features
+ image_embeds = image_embeds / image_embeds.norm(p=2, dim=-1, keepdim=True)
+ text_embeds = text_embeds / text_embeds.norm(p=2, dim=-1, keepdim=True)
+
+ # cosine similarity as logits
+ logit_scale = self.logit_scale.exp()
+ logits_per_text = torch.matmul(text_embeds, image_embeds.t()) * logit_scale
+ logits_per_image = logits_per_text.t()
+
+ loss = None
+ if return_loss:
+ loss = clip_loss(logits_per_text)
+
+ if not return_dict:
+ output = (logits_per_image, logits_per_text, text_embeds, image_embeds, text_outputs, vision_outputs)
+ return ((loss,) + output) if loss is not None else output
+
+ return CLIPOutput(
+ loss=loss,
+ logits_per_image=logits_per_image,
+ logits_per_text=logits_per_text,
+ text_embeds=text_embeds,
+ image_embeds=image_embeds,
+ text_model_output=text_outputs,
+ vision_model_output=vision_outputs,
+ )
\ No newline at end of file
diff --git a/videollava/model/multimodal_encoder/languagebind/audio/processing_audio.py b/videollava/model/multimodal_encoder/languagebind/audio/processing_audio.py
new file mode 100644
index 0000000000000000000000000000000000000000..0ea99f8cbe0d681d2ca24885f6000934390ac704
--- /dev/null
+++ b/videollava/model/multimodal_encoder/languagebind/audio/processing_audio.py
@@ -0,0 +1,190 @@
+import cv2
+import numpy as np
+import torch
+# import torchaudio
+from torchvision import transforms
+from transformers import ProcessorMixin, BatchEncoding
+from transformers.image_processing_utils import BatchFeature
+from torch.nn import functional as F
+
+
+def make_list_of_images(x):
+ if not isinstance(x, list):
+ return [x]
+ return x
+
+
+# torchaudio.set_audio_backend("soundfile")
+
+def torchaudio_loader(path):
+ return torchaudio.load(path)
+
+def int16_to_float32_torch(x):
+ return (x / 32767.0).type(torch.float32)
+
+def float32_to_int16_torch(x):
+ x = torch.clamp(x, min=-1., max=1.)
+ return (x * 32767.).type(torch.int16)
+
+DEFAULT_AUDIO_FRAME_SHIFT_MS = 10
+
+class AudioTransform:
+ def __init__(self, config):
+ self.sample_rate = config.audio_sample_rate
+ self.num_mel_bins = config.num_mel_bins
+ self.target_length = config.target_length
+ self.audio_mean = config.audio_mean
+ self.audio_std = config.audio_std
+ # mean=-4.2677393
+ # std=4.5689974
+ self.norm = transforms.Normalize(mean=self.audio_mean, std=self.audio_std)
+
+ def __call__(self, audio_data_and_origin_sr):
+ audio_data, origin_sr = audio_data_and_origin_sr
+ if self.sample_rate != origin_sr:
+ # print(audio_data.shape, origin_sr)
+ audio_data = torchaudio.functional.resample(audio_data, orig_freq=origin_sr, new_freq=self.sample_rate)
+ waveform_melspec = self.waveform2melspec(audio_data[0])
+ return self.norm(waveform_melspec)
+
+ def waveform2melspec(self, audio_data):
+ max_len = self.target_length * self.sample_rate // 100
+ if audio_data.shape[-1] > max_len:
+ mel = self.get_mel(audio_data)
+ # split to three parts
+ chunk_frames = self.target_length
+ total_frames = mel.shape[0]
+ ranges = np.array_split(list(range(0, total_frames - chunk_frames + 1)), 3)
+ # print('total_frames-chunk_frames:', total_frames-chunk_frames,
+ # 'len(audio_data):', len(audio_data),
+ # 'chunk_frames:', chunk_frames,
+ # 'total_frames:', total_frames)
+ if len(ranges[1]) == 0: # if the audio is too short, we just use the first chunk
+ ranges[1] = [0]
+ if len(ranges[2]) == 0: # if the audio is too short, we just use the first chunk
+ ranges[2] = [0]
+ # randomly choose index for each part
+ # idx_front = np.random.choice(ranges[0])
+ # idx_middle = np.random.choice(ranges[1])
+ # idx_back = np.random.choice(ranges[2])
+ idx_front = ranges[0][0] # fixed
+ idx_middle = ranges[1][0]
+ idx_back = ranges[2][0]
+ # select mel
+ mel_chunk_front = mel[idx_front:idx_front + chunk_frames, :]
+ mel_chunk_middle = mel[idx_middle:idx_middle + chunk_frames, :]
+ mel_chunk_back = mel[idx_back:idx_back + chunk_frames, :]
+ # stack
+ mel_fusion = torch.stack([mel_chunk_front, mel_chunk_middle, mel_chunk_back], dim=0)
+ elif audio_data.shape[-1] < max_len: # padding if too short
+ n_repeat = int(max_len / len(audio_data))
+ audio_data = audio_data.repeat(n_repeat)
+ audio_data = F.pad(
+ audio_data,
+ (0, max_len - len(audio_data)),
+ mode="constant",
+ value=0,
+ )
+ mel = self.get_mel(audio_data)
+ mel_fusion = torch.stack([mel, mel, mel], dim=0)
+ else: # if equal
+ mel = self.get_mel(audio_data)
+ mel_fusion = torch.stack([mel, mel, mel], dim=0)
+
+ # twice check
+ p = self.target_length - mel_fusion.shape[1]
+
+ # if abs(p) / self.target_length > 0.2:
+ # logging.warning(
+ # "Large gap between audio n_frames(%d) and "
+ # "target_length (%d). Is the audio_target_length "
+ # "setting correct?",
+ # mel_fusion.shape[1],
+ # self.target_length,
+ # )
+
+ # cut and pad
+ if p > 0:
+ m = torch.nn.ZeroPad2d((0, 0, 0, p))
+ mel_fusion = m(mel_fusion)
+ elif p < 0:
+ mel_fusion = mel_fusion[:, 0: self.target_length, :]
+
+ mel_fusion = mel_fusion.transpose(1, 2) # [3, target_length, mel_bins] -> [3, mel_bins, target_length]
+ return mel_fusion
+
+ def get_mel(self, audio_data):
+ # mel shape: (n_mels, T)
+ audio_data -= audio_data.mean()
+ mel = torchaudio.compliance.kaldi.fbank(
+ audio_data.unsqueeze(0),
+ htk_compat=True,
+ sample_frequency=self.sample_rate,
+ use_energy=False,
+ window_type="hanning",
+ num_mel_bins=self.num_mel_bins,
+ dither=0.0,
+ frame_length=25,
+ frame_shift=DEFAULT_AUDIO_FRAME_SHIFT_MS,
+ )
+ return mel # (T, n_mels)
+
+def get_audio_transform(config):
+ config = config.vision_config
+ return AudioTransform(config)
+
+
+def load_and_transform_audio(
+ audio_path,
+ transform,
+):
+ waveform_and_sr = torchaudio_loader(audio_path)
+ audio_outputs = transform(waveform_and_sr)
+
+ return audio_outputs
+
+class LanguageBindAudioProcessor(ProcessorMixin):
+ attributes = []
+ tokenizer_class = ("LanguageBindAudioTokenizer")
+
+ def __init__(self, config, tokenizer=None, **kwargs):
+ super().__init__(**kwargs)
+ self.config = config
+ self.transform = get_audio_transform(config)
+ self.image_processor = load_and_transform_audio
+ self.tokenizer = tokenizer
+
+ def __call__(self, images=None, text=None, context_length=77, return_tensors=None, **kwargs):
+ if text is None and images is None:
+ raise ValueError("You have to specify either text or images. Both cannot be none.")
+
+ if text is not None:
+ encoding = self.tokenizer(text, max_length=context_length, padding='max_length',
+ truncation=True, return_tensors=return_tensors, **kwargs)
+
+ if images is not None:
+ images = make_list_of_images(images)
+ image_features = [self.image_processor(image, self.transform) for image in images]
+ image_features = torch.stack(image_features)
+
+ if text is not None and images is not None:
+ encoding["pixel_values"] = image_features
+ return encoding
+ elif text is not None:
+ return encoding
+ else:
+ return {"pixel_values": image_features}
+
+ def batch_decode(self, skip_special_tokens=True, *args, **kwargs):
+ """
+ This method forwards all its arguments to CLIPTokenizerFast's [`~PreTrainedTokenizer.batch_decode`]. Please
+ refer to the docstring of this method for more information.
+ """
+ return self.tokenizer.batch_decode(*args, skip_special_tokens=skip_special_tokens, **kwargs)
+
+ def decode(self, skip_special_tokens=True, *args, **kwargs):
+ """
+ This method forwards all its arguments to CLIPTokenizerFast's [`~PreTrainedTokenizer.decode`]. Please refer to
+ the docstring of this method for more information.
+ """
+ return self.tokenizer.decode(*args, skip_special_tokens=skip_special_tokens, **kwargs)
diff --git a/videollava/model/multimodal_encoder/languagebind/audio/tokenization_audio.py b/videollava/model/multimodal_encoder/languagebind/audio/tokenization_audio.py
new file mode 100644
index 0000000000000000000000000000000000000000..6bc40be3f96c20bf2581e23f8249f3cd5566ebe1
--- /dev/null
+++ b/videollava/model/multimodal_encoder/languagebind/audio/tokenization_audio.py
@@ -0,0 +1,77 @@
+from transformers import CLIPTokenizer
+from transformers.utils import logging
+
+logger = logging.get_logger(__name__)
+
+VOCAB_FILES_NAMES = {
+ "vocab_file": "vocab.json",
+ "merges_file": "merges.txt",
+}
+
+PRETRAINED_VOCAB_FILES_MAP = {
+ "vocab_file": {
+ "lb203/LanguageBind-Audio": "https://huggingface.co/lb203/LanguageBind-Audio/resolve/main/vocab.json",
+ },
+ "merges_file": {
+ "lb203/LanguageBind-Audio": "https://huggingface.co/lb203/LanguageBind-Audio/resolve/main/merges.txt",
+ },
+}
+
+PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
+ "lb203/LanguageBind-Audio": 77,
+}
+
+
+PRETRAINED_INIT_CONFIGURATION = {
+ "lb203/LanguageBind-Audio": {},
+}
+
+class LanguageBindAudioTokenizer(CLIPTokenizer):
+ """
+ Construct a CLIP tokenizer. Based on byte-level Byte-Pair-Encoding.
+
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+ this superclass for more information regarding those methods.
+
+ Args:
+ vocab_file (`str`):
+ Path to the vocabulary file.
+ merges_file (`str`):
+ Path to the merges file.
+ errors (`str`, *optional*, defaults to `"replace"`):
+ Paradigm to follow when decoding bytes to UTF-8. See
+ [bytes.decode](https://docs.python.org/3/library/stdtypes.html#bytes.decode) for more information.
+ unk_token (`str`, *optional*, defaults to `<|endoftext|>`):
+ The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
+ token instead.
+ bos_token (`str`, *optional*, defaults to `<|startoftext|>`):
+ The beginning of sequence token.
+ eos_token (`str`, *optional*, defaults to `<|endoftext|>`):
+ The end of sequence token.
+ """
+
+ vocab_files_names = VOCAB_FILES_NAMES
+ pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+ max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
+ model_input_names = ["input_ids", "attention_mask"]
+
+ def __init__(
+ self,
+ vocab_file,
+ merges_file,
+ errors="replace",
+ unk_token="<|endoftext|>",
+ bos_token="<|startoftext|>",
+ eos_token="<|endoftext|>",
+ pad_token="<|endoftext|>", # hack to enable padding
+ **kwargs,
+ ):
+ super(LanguageBindAudioTokenizer, self).__init__(
+ vocab_file,
+ merges_file,
+ errors,
+ unk_token,
+ bos_token,
+ eos_token,
+ pad_token, # hack to enable padding
+ **kwargs,)
\ No newline at end of file
diff --git a/videollava/model/multimodal_encoder/languagebind/depth/configuration_depth.py b/videollava/model/multimodal_encoder/languagebind/depth/configuration_depth.py
new file mode 100644
index 0000000000000000000000000000000000000000..0d3901b2cf96635384c1e7d1e99845a66cd6c786
--- /dev/null
+++ b/videollava/model/multimodal_encoder/languagebind/depth/configuration_depth.py
@@ -0,0 +1,425 @@
+import copy
+import os
+from typing import Union
+
+from transformers import PretrainedConfig
+from transformers.utils import logging
+
+logger = logging.get_logger(__name__)
+
+
+
+
+
+
+
+class CLIPTextConfig(PretrainedConfig):
+ r"""
+ This is the configuration class to store the configuration of a [`CLIPTextModel`]. It is used to instantiate a CLIP
+ text encoder according to the specified arguments, defining the model architecture. Instantiating a configuration
+ with the defaults will yield a similar configuration to that of the text encoder of the CLIP
+ [openai/clip-vit-base-patch32](https://huggingface.co/openai/clip-vit-base-patch32) architecture.
+
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
+
+ Args:
+ vocab_size (`int`, *optional*, defaults to 49408):
+ Vocabulary size of the CLIP text model. Defines the number of different tokens that can be represented by
+ the `inputs_ids` passed when calling [`CLIPModel`].
+ hidden_size (`int`, *optional*, defaults to 512):
+ Dimensionality of the encoder layers and the pooler layer.
+ intermediate_size (`int`, *optional*, defaults to 2048):
+ Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
+ num_hidden_layers (`int`, *optional*, defaults to 12):
+ Number of hidden layers in the Transformer encoder.
+ num_attention_heads (`int`, *optional*, defaults to 8):
+ Number of attention heads for each attention layer in the Transformer encoder.
+ max_position_embeddings (`int`, *optional*, defaults to 77):
+ The maximum sequence length that this model might ever be used with. Typically set this to something large
+ just in case (e.g., 512 or 1024 or 2048).
+ hidden_act (`str` or `function`, *optional*, defaults to `"quick_gelu"`):
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"selu"` and `"gelu_new"` `"quick_gelu"` are supported.
+ layer_norm_eps (`float`, *optional*, defaults to 1e-5):
+ The epsilon used by the layer normalization layers.
+ attention_dropout (`float`, *optional*, defaults to 0.0):
+ The dropout ratio for the attention probabilities.
+ initializer_range (`float`, *optional*, defaults to 0.02):
+ The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+ initializer_factor (`float`, *optional*, defaults to 1):
+ A factor for initializing all weight matrices (should be kept to 1, used internally for initialization
+ testing).
+
+ Example:
+
+ ```python
+ >>> from transformers import CLIPTextConfig, CLIPTextModel
+
+ >>> # Initializing a CLIPTextConfig with openai/clip-vit-base-patch32 style configuration
+ >>> configuration = CLIPTextConfig()
+
+ >>> # Initializing a CLIPTextModel (with random weights) from the openai/clip-vit-base-patch32 style configuration
+ >>> model = CLIPTextModel(configuration)
+
+ >>> # Accessing the model configuration
+ >>> configuration = model.config
+ ```"""
+ model_type = "clip_text_model"
+
+ def __init__(
+ self,
+ vocab_size=49408,
+ hidden_size=512,
+ intermediate_size=2048,
+ projection_dim=512,
+ num_hidden_layers=12,
+ num_attention_heads=8,
+ max_position_embeddings=77,
+ hidden_act="quick_gelu",
+ layer_norm_eps=1e-5,
+ attention_dropout=0.0,
+ initializer_range=0.02,
+ initializer_factor=1.0,
+ # This differs from `CLIPTokenizer`'s default and from openai/clip
+ # See https://github.com/huggingface/transformers/pull/24773#issuecomment-1632287538
+ pad_token_id=1,
+ bos_token_id=49406,
+ eos_token_id=49407,
+ **kwargs,
+ ):
+ super().__init__(pad_token_id=pad_token_id, bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs)
+
+ self.vocab_size = vocab_size
+ self.hidden_size = hidden_size
+ self.intermediate_size = intermediate_size
+ self.projection_dim = projection_dim
+ self.num_hidden_layers = num_hidden_layers
+ self.num_attention_heads = num_attention_heads
+ self.max_position_embeddings = max_position_embeddings
+ self.layer_norm_eps = layer_norm_eps
+ self.hidden_act = hidden_act
+ self.initializer_range = initializer_range
+ self.initializer_factor = initializer_factor
+ self.attention_dropout = attention_dropout
+ self.add_time_attn = False ######################################
+
+ @classmethod
+ def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":
+ cls._set_token_in_kwargs(kwargs)
+
+ config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)
+
+ # get the text config dict if we are loading from CLIPConfig
+ if config_dict.get("model_type") == "clip":
+ config_dict = config_dict["text_config"]
+
+ if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:
+ logger.warning(
+ f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
+ f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
+ )
+
+ return cls.from_dict(config_dict, **kwargs)
+
+
+
+
+class CLIPVisionConfig(PretrainedConfig):
+ r"""
+ This is the configuration class to store the configuration of a [`CLIPVisionModel`]. It is used to instantiate a
+ CLIP vision encoder according to the specified arguments, defining the model architecture. Instantiating a
+ configuration with the defaults will yield a similar configuration to that of the vision encoder of the CLIP
+ [openai/clip-vit-base-patch32](https://huggingface.co/openai/clip-vit-base-patch32) architecture.
+
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
+
+ Args:
+ hidden_size (`int`, *optional*, defaults to 768):
+ Dimensionality of the encoder layers and the pooler layer.
+ intermediate_size (`int`, *optional*, defaults to 3072):
+ Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
+ num_hidden_layers (`int`, *optional*, defaults to 12):
+ Number of hidden layers in the Transformer encoder.
+ num_attention_heads (`int`, *optional*, defaults to 12):
+ Number of attention heads for each attention layer in the Transformer encoder.
+ image_size (`int`, *optional*, defaults to 224):
+ The size (resolution) of each image.
+ patch_size (`int`, *optional*, defaults to 32):
+ The size (resolution) of each patch.
+ hidden_act (`str` or `function`, *optional*, defaults to `"quick_gelu"`):
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"selu"` and `"gelu_new"` ``"quick_gelu"` are supported.
+ layer_norm_eps (`float`, *optional*, defaults to 1e-5):
+ The epsilon used by the layer normalization layers.
+ attention_dropout (`float`, *optional*, defaults to 0.0):
+ The dropout ratio for the attention probabilities.
+ initializer_range (`float`, *optional*, defaults to 0.02):
+ The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+ initializer_factor (`float`, *optional*, defaults to 1):
+ A factor for initializing all weight matrices (should be kept to 1, used internally for initialization
+ testing).
+
+ Example:
+
+ ```python
+ >>> from transformers import CLIPVisionConfig, CLIPVisionModel
+
+ >>> # Initializing a CLIPVisionConfig with openai/clip-vit-base-patch32 style configuration
+ >>> configuration = CLIPVisionConfig()
+
+ >>> # Initializing a CLIPVisionModel (with random weights) from the openai/clip-vit-base-patch32 style configuration
+ >>> model = CLIPVisionModel(configuration)
+
+ >>> # Accessing the model configuration
+ >>> configuration = model.config
+ ```"""
+
+ model_type = "clip_vision_model"
+
+ def __init__(
+ self,
+ hidden_size=768,
+ intermediate_size=3072,
+ projection_dim=512,
+ num_hidden_layers=12,
+ num_attention_heads=12,
+ num_channels=3,
+ image_size=224,
+ patch_size=32,
+ hidden_act="quick_gelu",
+ layer_norm_eps=1e-5,
+ attention_dropout=0.0,
+ initializer_range=0.02,
+ initializer_factor=1.0,
+
+ add_time_attn=False, ################################
+ num_frames=1, ################################
+ force_patch_dropout=0.0, ################################
+ lora_r=2, ################################
+ lora_alpha=16, ################################
+ lora_dropout=0.0, ################################
+ num_mel_bins=0.0, ################################
+ target_length=0.0, ################################
+ max_depth=10,
+ video_decode_backend='decord', #########################
+ **kwargs,
+ ):
+ super().__init__(**kwargs)
+
+ self.hidden_size = hidden_size
+ self.intermediate_size = intermediate_size
+ self.projection_dim = projection_dim
+ self.num_hidden_layers = num_hidden_layers
+ self.num_attention_heads = num_attention_heads
+ self.num_channels = num_channels
+ self.patch_size = patch_size
+ self.image_size = image_size
+ self.initializer_range = initializer_range
+ self.initializer_factor = initializer_factor
+ self.attention_dropout = attention_dropout
+ self.layer_norm_eps = layer_norm_eps
+ self.hidden_act = hidden_act
+
+ self.add_time_attn = add_time_attn ################
+ self.num_frames = num_frames ################
+ self.force_patch_dropout = force_patch_dropout ################
+ self.lora_r = lora_r ################
+ self.lora_alpha = lora_alpha ################
+ self.lora_dropout = lora_dropout ################
+ self.num_mel_bins = num_mel_bins ################
+ self.target_length = target_length ################
+ self.max_depth = max_depth ################
+ self.video_decode_backend = video_decode_backend ################
+
+ @classmethod
+ def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":
+ cls._set_token_in_kwargs(kwargs)
+
+ config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)
+
+ # get the vision config dict if we are loading from CLIPConfig
+ if config_dict.get("model_type") == "clip":
+ config_dict = config_dict["vision_config"]
+
+ if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:
+ logger.warning(
+ f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
+ f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
+ )
+
+ return cls.from_dict(config_dict, **kwargs)
+
+
+class LanguageBindDepthConfig(PretrainedConfig):
+ r"""
+ [`CLIPConfig`] is the configuration class to store the configuration of a [`CLIPModel`]. It is used to instantiate
+ a CLIP model according to the specified arguments, defining the text model and vision model configs. Instantiating
+ a configuration with the defaults will yield a similar configuration to that of the CLIP
+ [openai/clip-vit-base-patch32](https://huggingface.co/openai/clip-vit-base-patch32) architecture.
+
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
+
+ Args:
+ text_config (`dict`, *optional*):
+ Dictionary of configuration options used to initialize [`CLIPTextConfig`].
+ vision_config (`dict`, *optional*):
+ Dictionary of configuration options used to initialize [`CLIPVisionConfig`].
+ projection_dim (`int`, *optional*, defaults to 512):
+ Dimentionality of text and vision projection layers.
+ logit_scale_init_value (`float`, *optional*, defaults to 2.6592):
+ The inital value of the *logit_scale* paramter. Default is used as per the original CLIP implementation.
+ kwargs (*optional*):
+ Dictionary of keyword arguments.
+
+ Example:
+
+ ```python
+ >>> from transformers import CLIPConfig, CLIPModel
+
+ >>> # Initializing a CLIPConfig with openai/clip-vit-base-patch32 style configuration
+ >>> configuration = CLIPConfig()
+
+ >>> # Initializing a CLIPModel (with random weights) from the openai/clip-vit-base-patch32 style configuration
+ >>> model = CLIPModel(configuration)
+
+ >>> # Accessing the model configuration
+ >>> configuration = model.config
+
+ >>> # We can also initialize a CLIPConfig from a CLIPTextConfig and a CLIPVisionConfig
+ >>> from transformers import CLIPTextConfig, CLIPVisionConfig
+
+ >>> # Initializing a CLIPText and CLIPVision configuration
+ >>> config_text = CLIPTextConfig()
+ >>> config_vision = CLIPVisionConfig()
+
+ >>> config = CLIPConfig.from_text_vision_configs(config_text, config_vision)
+ ```"""
+
+ model_type = "LanguageBindDepth"
+ is_composition = True
+
+ def __init__(
+ self, text_config=None, vision_config=None, projection_dim=512, logit_scale_init_value=2.6592, **kwargs
+ ):
+ # If `_config_dict` exist, we use them for the backward compatibility.
+ # We pop out these 2 attributes before calling `super().__init__` to avoid them being saved (which causes a lot
+ # of confusion!).
+ text_config_dict = kwargs.pop("text_config_dict", None)
+ vision_config_dict = kwargs.pop("vision_config_dict", None)
+
+ super().__init__(**kwargs)
+
+ # Instead of simply assigning `[text|vision]_config_dict` to `[text|vision]_config`, we use the values in
+ # `[text|vision]_config_dict` to update the values in `[text|vision]_config`. The values should be same in most
+ # cases, but we don't want to break anything regarding `_config_dict` that existed before commit `8827e1b2`.
+ if text_config_dict is not None:
+ if text_config is None:
+ text_config = {}
+
+ # This is the complete result when using `text_config_dict`.
+ _text_config_dict = CLIPTextConfig(**text_config_dict).to_dict()
+
+ # Give a warning if the values exist in both `_text_config_dict` and `text_config` but being different.
+ for key, value in _text_config_dict.items():
+ if key in text_config and value != text_config[key] and key not in ["transformers_version"]:
+ # If specified in `text_config_dict`
+ if key in text_config_dict:
+ message = (
+ f"`{key}` is found in both `text_config_dict` and `text_config` but with different values. "
+ f'The value `text_config_dict["{key}"]` will be used instead.'
+ )
+ # If inferred from default argument values (just to be super careful)
+ else:
+ message = (
+ f"`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The "
+ f'value `text_config["{key}"]` will be overriden.'
+ )
+ logger.warning(message)
+
+ # Update all values in `text_config` with the ones in `_text_config_dict`.
+ text_config.update(_text_config_dict)
+
+ if vision_config_dict is not None:
+ if vision_config is None:
+ vision_config = {}
+
+ # This is the complete result when using `vision_config_dict`.
+ _vision_config_dict = CLIPVisionConfig(**vision_config_dict).to_dict()
+ # convert keys to string instead of integer
+ if "id2label" in _vision_config_dict:
+ _vision_config_dict["id2label"] = {
+ str(key): value for key, value in _vision_config_dict["id2label"].items()
+ }
+
+ # Give a warning if the values exist in both `_vision_config_dict` and `vision_config` but being different.
+ for key, value in _vision_config_dict.items():
+ if key in vision_config and value != vision_config[key] and key not in ["transformers_version"]:
+ # If specified in `vision_config_dict`
+ if key in vision_config_dict:
+ message = (
+ f"`{key}` is found in both `vision_config_dict` and `vision_config` but with different "
+ f'values. The value `vision_config_dict["{key}"]` will be used instead.'
+ )
+ # If inferred from default argument values (just to be super careful)
+ else:
+ message = (
+ f"`vision_config_dict` is provided which will be used to initialize `CLIPVisionConfig`. "
+ f'The value `vision_config["{key}"]` will be overriden.'
+ )
+ logger.warning(message)
+
+ # Update all values in `vision_config` with the ones in `_vision_config_dict`.
+ vision_config.update(_vision_config_dict)
+
+ if text_config is None:
+ text_config = {}
+ logger.info("`text_config` is `None`. Initializing the `CLIPTextConfig` with default values.")
+
+ if vision_config is None:
+ vision_config = {}
+ logger.info("`vision_config` is `None`. initializing the `CLIPVisionConfig` with default values.")
+
+ self.text_config = CLIPTextConfig(**text_config)
+ self.vision_config = CLIPVisionConfig(**vision_config)
+
+ self.projection_dim = projection_dim
+ self.logit_scale_init_value = logit_scale_init_value
+ self.initializer_factor = 1.0
+
+ @classmethod
+ def from_text_vision_configs(cls, text_config: CLIPTextConfig, vision_config: CLIPVisionConfig, **kwargs):
+ r"""
+ Instantiate a [`CLIPConfig`] (or a derived class) from clip text model configuration and clip vision model
+ configuration.
+
+ Returns:
+ [`CLIPConfig`]: An instance of a configuration object
+ """
+
+ return cls(text_config=text_config.to_dict(), vision_config=vision_config.to_dict(), **kwargs)
+
+ def to_dict(self):
+ """
+ Serializes this instance to a Python dictionary. Override the default [`~PretrainedConfig.to_dict`].
+
+ Returns:
+ `Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance,
+ """
+ output = copy.deepcopy(self.__dict__)
+ output["text_config"] = self.text_config.to_dict()
+ output["vision_config"] = self.vision_config.to_dict()
+ output["model_type"] = self.__class__.model_type
+ return output
+
+
+
+
+
+
+
+
+
+
diff --git a/videollava/model/multimodal_encoder/languagebind/depth/modeling_depth.py b/videollava/model/multimodal_encoder/languagebind/depth/modeling_depth.py
new file mode 100644
index 0000000000000000000000000000000000000000..849eade79b0f4bff345b73bcf6a71115a28d0a09
--- /dev/null
+++ b/videollava/model/multimodal_encoder/languagebind/depth/modeling_depth.py
@@ -0,0 +1,1030 @@
+import math
+from typing import Optional, Tuple, Union
+
+import torch
+from einops import rearrange
+from peft import LoraConfig, get_peft_model
+from torch import nn
+from torch.nn import functional as F
+from transformers import PreTrainedModel, add_start_docstrings
+from transformers.modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling
+from transformers.models.clip.modeling_clip import CLIPMLP, CLIPAttention, CLIPTextEmbeddings, CLIPVisionEmbeddings, \
+ CLIPVisionModelWithProjection, CLIPTextModelWithProjection, _expand_mask, CLIPOutput, clip_loss
+from transformers.utils import add_start_docstrings_to_model_forward, replace_return_docstrings
+
+from .configuration_depth import LanguageBindDepthConfig, CLIPVisionConfig, CLIPTextConfig
+
+
+
+class PatchDropout(nn.Module):
+ """
+ https://arxiv.org/abs/2212.00794
+ """
+
+ def __init__(self, prob, exclude_first_token=True):
+ super().__init__()
+ assert 0 <= prob < 1.
+ self.prob = prob
+ self.exclude_first_token = exclude_first_token # exclude CLS token
+
+ def forward(self, x, B, T):
+ if not self.training or self.prob == 0.:
+ return x
+
+ if self.exclude_first_token:
+ cls_tokens, x = x[:, :1], x[:, 1:]
+ else:
+ cls_tokens = torch.jit.annotate(torch.Tensor, x[:, :1])
+
+ batch = x.size()[0]
+ num_tokens = x.size()[1]
+
+ batch_indices = torch.arange(batch)
+ batch_indices = batch_indices[..., None]
+
+ keep_prob = 1 - self.prob
+ num_patches_keep = max(1, int(num_tokens * keep_prob))
+
+ if T == 1:
+ rand = torch.randn(batch, num_tokens)
+ patch_indices_keep = rand.topk(num_patches_keep, dim=-1).indices
+ else:
+ rand = torch.randn(B, num_tokens)
+ patch_indices_keep = rand.topk(num_patches_keep, dim=-1).indices
+ patch_indices_keep = patch_indices_keep.unsqueeze(1).repeat(1, T, 1)
+ patch_indices_keep = rearrange(patch_indices_keep, 'b t n -> (b t) n')
+
+
+ x = x[batch_indices, patch_indices_keep]
+
+ if self.exclude_first_token:
+ x = torch.cat((cls_tokens, x), dim=1)
+
+ return x
+
+class CLIPEncoderLayer(nn.Module):
+ def __init__(self, config: LanguageBindDepthConfig):
+ super().__init__()
+ self.embed_dim = config.hidden_size
+ self.self_attn = CLIPAttention(config)
+ self.layer_norm1 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
+ self.mlp = CLIPMLP(config)
+ self.layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
+
+ self.add_time_attn = config.add_time_attn
+ if self.add_time_attn:
+ self.t = config.num_frames
+ self.temporal_embedding = nn.Parameter(torch.zeros(1, config.num_frames, config.hidden_size))
+ nn.init.normal_(self.temporal_embedding, std=config.hidden_size ** -0.5)
+
+ self.embed_dim = config.hidden_size
+ self.temporal_attn = CLIPAttention(config)
+ self.temporal_layer_norm1 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
+ self.temporal_mlp = CLIPMLP(config)
+ self.temporal_layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
+
+ def forward(
+ self,
+ hidden_states: torch.Tensor,
+ attention_mask: torch.Tensor,
+ causal_attention_mask: torch.Tensor,
+ output_attentions: Optional[bool] = False,
+ ) -> Tuple[torch.FloatTensor]:
+ """
+ Args:
+ hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
+ attention_mask (`torch.FloatTensor`): attention mask of size
+ `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
+ `(config.encoder_attention_heads,)`.
+ output_attentions (`bool`, *optional*):
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under
+ returned tensors for more detail.
+ """
+
+
+ if self.add_time_attn:
+ bt, n, d = hidden_states.shape
+ t = self.t
+
+ # time embed
+ if t != 1:
+ n = hidden_states.shape[1]
+ hidden_states = rearrange(hidden_states, '(b t) n d -> (b n) t d', t=t)
+ hidden_states = hidden_states + self.temporal_embedding[:, :t, :]
+ hidden_states = rearrange(hidden_states, '(b n) t d -> (b t) n d', n=n)
+
+ # time attn
+ residual = hidden_states
+ hidden_states = rearrange(hidden_states, '(b t) n d -> (b n) t d', t=t)
+ # hidden_states = self.layer_norm1(hidden_states) # share layernorm
+ hidden_states = self.temporal_layer_norm1(hidden_states)
+ hidden_states, attn_weights = self.temporal_attn(
+ hidden_states=hidden_states,
+ attention_mask=attention_mask,
+ causal_attention_mask=causal_attention_mask,
+ output_attentions=output_attentions,
+ )
+ hidden_states = residual + rearrange(hidden_states, '(b n) t d -> (b t) n d', n=n)
+
+ residual = hidden_states
+ hidden_states = rearrange(hidden_states, '(b t) n d -> (b n) t d', t=t)
+ # hidden_states = self.layer_norm2(hidden_states) # share layernorm
+ hidden_states = self.temporal_layer_norm2(hidden_states)
+ hidden_states = self.temporal_mlp(hidden_states)
+ hidden_states = residual + rearrange(hidden_states, '(b n) t d -> (b t) n d', n=n)
+
+ # spatial attn
+ residual = hidden_states
+
+ hidden_states = self.layer_norm1(hidden_states)
+ hidden_states, attn_weights = self.self_attn(
+ hidden_states=hidden_states,
+ attention_mask=attention_mask,
+ causal_attention_mask=causal_attention_mask,
+ output_attentions=output_attentions,
+ )
+ hidden_states = residual + hidden_states
+
+ residual = hidden_states
+ hidden_states = self.layer_norm2(hidden_states)
+ hidden_states = self.mlp(hidden_states)
+ hidden_states = residual + hidden_states
+
+ outputs = (hidden_states,)
+
+ if output_attentions:
+ outputs += (attn_weights,)
+
+ return outputs
+
+
+
+
+
+
+
+
+
+class CLIPPreTrainedModel(PreTrainedModel):
+ """
+ An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
+ models.
+ """
+
+ config_class = LanguageBindDepthConfig
+ base_model_prefix = "clip"
+ supports_gradient_checkpointing = True
+ _keys_to_ignore_on_load_missing = [r"position_ids"]
+
+ def _init_weights(self, module):
+ """Initialize the weights"""
+ factor = self.config.initializer_factor
+ if isinstance(module, CLIPTextEmbeddings):
+ module.token_embedding.weight.data.normal_(mean=0.0, std=factor * 0.02)
+ module.position_embedding.weight.data.normal_(mean=0.0, std=factor * 0.02)
+ elif isinstance(module, CLIPVisionEmbeddings):
+ factor = self.config.initializer_factor
+ nn.init.normal_(module.class_embedding, mean=0.0, std=module.embed_dim**-0.5 * factor)
+ nn.init.normal_(module.patch_embedding.weight, std=module.config.initializer_range * factor)
+ nn.init.normal_(module.position_embedding.weight, std=module.config.initializer_range * factor)
+ elif isinstance(module, CLIPAttention):
+ factor = self.config.initializer_factor
+ in_proj_std = (module.embed_dim**-0.5) * ((2 * module.config.num_hidden_layers) ** -0.5) * factor
+ out_proj_std = (module.embed_dim**-0.5) * factor
+ nn.init.normal_(module.q_proj.weight, std=in_proj_std)
+ nn.init.normal_(module.k_proj.weight, std=in_proj_std)
+ nn.init.normal_(module.v_proj.weight, std=in_proj_std)
+ nn.init.normal_(module.out_proj.weight, std=out_proj_std)
+ elif isinstance(module, CLIPMLP):
+ factor = self.config.initializer_factor
+ in_proj_std = (
+ (module.config.hidden_size**-0.5) * ((2 * module.config.num_hidden_layers) ** -0.5) * factor
+ )
+ fc_std = (2 * module.config.hidden_size) ** -0.5 * factor
+ nn.init.normal_(module.fc1.weight, std=fc_std)
+ nn.init.normal_(module.fc2.weight, std=in_proj_std)
+ elif isinstance(module, LanguageBindDepth):
+ nn.init.normal_(
+ module.text_projection.weight,
+ std=module.text_embed_dim**-0.5 * self.config.initializer_factor,
+ )
+ nn.init.normal_(
+ module.visual_projection.weight,
+ std=module.vision_embed_dim**-0.5 * self.config.initializer_factor,
+ )
+ elif isinstance(module, CLIPVisionModelWithProjection):
+ nn.init.normal_(
+ module.visual_projection.weight,
+ std=self.config.hidden_size**-0.5 * self.config.initializer_factor,
+ )
+ elif isinstance(module, CLIPTextModelWithProjection):
+ nn.init.normal_(
+ module.text_projection.weight,
+ std=self.config.hidden_size**-0.5 * self.config.initializer_factor,
+ )
+
+ if isinstance(module, nn.LayerNorm):
+ module.bias.data.zero_()
+ module.weight.data.fill_(1.0)
+ if isinstance(module, nn.Linear) and module.bias is not None:
+ module.bias.data.zero_()
+
+ def _set_gradient_checkpointing(self, module, value=False):
+ if isinstance(module, CLIPEncoder):
+ module.gradient_checkpointing = value
+
+
+CLIP_START_DOCSTRING = r"""
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
+
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
+
+ Parameters:
+ config ([`CLIPConfig`]): Model configuration class with all the parameters of the model.
+ Initializing with a config file does not load the weights associated with the model, only the
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
+"""
+
+CLIP_TEXT_INPUTS_DOCSTRING = r"""
+ Args:
+ input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
+ Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
+ it.
+
+ Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
+
+ [What are input IDs?](../glossary#input-ids)
+ attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
+ Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
+
+ - 1 for tokens that are **not masked**,
+ - 0 for tokens that are **masked**.
+
+ [What are attention masks?](../glossary#attention-mask)
+ position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
+
+ [What are position IDs?](../glossary#position-ids)
+ output_attentions (`bool`, *optional*):
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
+ tensors for more detail.
+ output_hidden_states (`bool`, *optional*):
+ Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
+ more detail.
+ return_dict (`bool`, *optional*):
+ Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+"""
+
+CLIP_VISION_INPUTS_DOCSTRING = r"""
+ Args:
+ pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
+ Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using
+ [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details.
+ output_attentions (`bool`, *optional*):
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
+ tensors for more detail.
+ output_hidden_states (`bool`, *optional*):
+ Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
+ more detail.
+ return_dict (`bool`, *optional*):
+ Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+"""
+
+CLIP_INPUTS_DOCSTRING = r"""
+ Args:
+ input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
+ Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
+ it.
+
+ Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
+
+ [What are input IDs?](../glossary#input-ids)
+ attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
+ Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
+
+ - 1 for tokens that are **not masked**,
+ - 0 for tokens that are **masked**.
+
+ [What are attention masks?](../glossary#attention-mask)
+ position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
+
+ [What are position IDs?](../glossary#position-ids)
+ pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
+ Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using
+ [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details.
+ return_loss (`bool`, *optional*):
+ Whether or not to return the contrastive loss.
+ output_attentions (`bool`, *optional*):
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
+ tensors for more detail.
+ output_hidden_states (`bool`, *optional*):
+ Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
+ more detail.
+ return_dict (`bool`, *optional*):
+ Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+"""
+
+
+class CLIPEncoder(nn.Module):
+ """
+ Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
+ [`CLIPEncoderLayer`].
+
+ Args:
+ config: CLIPConfig
+ """
+
+ def __init__(self, config: LanguageBindDepthConfig):
+ super().__init__()
+ self.config = config
+ self.layers = nn.ModuleList([CLIPEncoderLayer(config) for _ in range(config.num_hidden_layers)])
+ self.gradient_checkpointing = False
+
+ def forward(
+ self,
+ inputs_embeds,
+ attention_mask: Optional[torch.Tensor] = None,
+ causal_attention_mask: Optional[torch.Tensor] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> Union[Tuple, BaseModelOutput]:
+ r"""
+ Args:
+ inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
+ This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+ than the model's internal embedding lookup matrix.
+ attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
+ Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
+
+ - 1 for tokens that are **not masked**,
+ - 0 for tokens that are **masked**.
+
+ [What are attention masks?](../glossary#attention-mask)
+ causal_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
+ Causal mask for the text model. Mask values selected in `[0, 1]`:
+
+ - 1 for tokens that are **not masked**,
+ - 0 for tokens that are **masked**.
+
+ [What are attention masks?](../glossary#attention-mask)
+ output_attentions (`bool`, *optional*):
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under
+ returned tensors for more detail.
+ output_hidden_states (`bool`, *optional*):
+ Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
+ for more detail.
+ return_dict (`bool`, *optional*):
+ Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+ """
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+ output_hidden_states = (
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+ )
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+ encoder_states = () if output_hidden_states else None
+ all_attentions = () if output_attentions else None
+
+ hidden_states = inputs_embeds
+ for idx, encoder_layer in enumerate(self.layers):
+ if output_hidden_states:
+ encoder_states = encoder_states + (hidden_states,)
+ if self.gradient_checkpointing and self.training:
+
+ def create_custom_forward(module):
+ def custom_forward(*inputs):
+ return module(*inputs, output_attentions)
+
+ return custom_forward
+
+ layer_outputs = torch.utils.checkpoint.checkpoint(
+ create_custom_forward(encoder_layer),
+ hidden_states,
+ attention_mask,
+ causal_attention_mask,
+ )
+ else:
+ layer_outputs = encoder_layer(
+ hidden_states,
+ attention_mask,
+ causal_attention_mask,
+ output_attentions=output_attentions,
+ )
+
+ hidden_states = layer_outputs[0]
+
+ if output_attentions:
+ all_attentions = all_attentions + (layer_outputs[1],)
+
+ if output_hidden_states:
+ encoder_states = encoder_states + (hidden_states,)
+
+ if not return_dict:
+ return tuple(v for v in [hidden_states, encoder_states, all_attentions] if v is not None)
+ return BaseModelOutput(
+ last_hidden_state=hidden_states, hidden_states=encoder_states, attentions=all_attentions
+ )
+
+
+# Copied from transformers.models.bart.modeling_bart._make_causal_mask
+def _make_causal_mask(
+ input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device, past_key_values_length: int = 0
+):
+ """
+ Make causal mask used for bi-directional self-attention.
+ """
+ bsz, tgt_len = input_ids_shape
+ mask = torch.full((tgt_len, tgt_len), torch.finfo(dtype).min, device=device)
+ mask_cond = torch.arange(mask.size(-1), device=device)
+ mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0)
+ mask = mask.to(dtype)
+
+ if past_key_values_length > 0:
+ mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), mask], dim=-1)
+ return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length)
+
+
+class CLIPTextTransformer(nn.Module):
+ def __init__(self, config: CLIPTextConfig):
+ super().__init__()
+ self.config = config
+ embed_dim = config.hidden_size
+ self.embeddings = CLIPTextEmbeddings(config)
+ self.encoder = CLIPEncoder(config)
+ self.final_layer_norm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
+
+ @add_start_docstrings_to_model_forward(CLIP_TEXT_INPUTS_DOCSTRING)
+ @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=CLIPTextConfig)
+ def forward(
+ self,
+ input_ids: Optional[torch.Tensor] = None,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.Tensor] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> Union[Tuple, BaseModelOutputWithPooling]:
+ r"""
+ Returns:
+
+ """
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+ output_hidden_states = (
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+ )
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+ if input_ids is None:
+ raise ValueError("You have to specify input_ids")
+
+ input_shape = input_ids.size()
+ input_ids = input_ids.view(-1, input_shape[-1])
+
+ hidden_states = self.embeddings(input_ids=input_ids, position_ids=position_ids)
+
+ # CLIP's text model uses causal mask, prepare it here.
+ # https://github.com/openai/CLIP/blob/cfcffb90e69f37bf2ff1e988237a0fbe41f33c04/clip/model.py#L324
+ causal_attention_mask = _make_causal_mask(input_shape, hidden_states.dtype, device=hidden_states.device)
+ # expand attention_mask
+ if attention_mask is not None:
+ # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]
+ attention_mask = _expand_mask(attention_mask, hidden_states.dtype)
+
+ encoder_outputs = self.encoder(
+ inputs_embeds=hidden_states,
+ attention_mask=attention_mask,
+ causal_attention_mask=causal_attention_mask,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+
+ last_hidden_state = encoder_outputs[0]
+ last_hidden_state = self.final_layer_norm(last_hidden_state)
+
+ # text_embeds.shape = [batch_size, sequence_length, transformer.width]
+ # take features from the eot embedding (eot_token is the highest number in each sequence)
+ # casting to torch.int for onnx compatibility: argmax doesn't support int64 inputs with opset 14
+ pooled_output = last_hidden_state[
+ torch.arange(last_hidden_state.shape[0], device=last_hidden_state.device),
+ input_ids.to(dtype=torch.int, device=last_hidden_state.device).argmax(dim=-1),
+ ]
+
+ if not return_dict:
+ return (last_hidden_state, pooled_output) + encoder_outputs[1:]
+
+ return BaseModelOutputWithPooling(
+ last_hidden_state=last_hidden_state,
+ pooler_output=pooled_output,
+ hidden_states=encoder_outputs.hidden_states,
+ attentions=encoder_outputs.attentions,
+ )
+
+
+@add_start_docstrings(
+ """The text model from CLIP without any head or projection on top.""",
+ CLIP_START_DOCSTRING,
+)
+class CLIPTextModel(CLIPPreTrainedModel):
+ config_class = CLIPTextConfig
+
+ _no_split_modules = ["CLIPEncoderLayer"]
+
+ def __init__(self, config: CLIPTextConfig):
+ super().__init__(config)
+ self.text_model = CLIPTextTransformer(config)
+ # Initialize weights and apply final processing
+ self.post_init()
+
+ def get_input_embeddings(self) -> nn.Module:
+ return self.text_model.embeddings.token_embedding
+
+ def set_input_embeddings(self, value):
+ self.text_model.embeddings.token_embedding = value
+
+ @add_start_docstrings_to_model_forward(CLIP_TEXT_INPUTS_DOCSTRING)
+ @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=CLIPTextConfig)
+ def forward(
+ self,
+ input_ids: Optional[torch.Tensor] = None,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.Tensor] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> Union[Tuple, BaseModelOutputWithPooling]:
+ r"""
+ Returns:
+
+ Examples:
+
+ ```python
+ >>> from transformers import AutoTokenizer, CLIPTextModel
+
+ >>> model = CLIPTextModel.from_pretrained("openai/clip-vit-base-patch32")
+ >>> tokenizer = AutoTokenizer.from_pretrained("openai/clip-vit-base-patch32")
+
+ >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt")
+
+ >>> outputs = model(**inputs)
+ >>> last_hidden_state = outputs.last_hidden_state
+ >>> pooled_output = outputs.pooler_output # pooled (EOS token) states
+ ```"""
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+ return self.text_model(
+ input_ids=input_ids,
+ attention_mask=attention_mask,
+ position_ids=position_ids,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+
+
+class CLIPVisionTransformer(nn.Module):
+ def __init__(self, config: CLIPVisionConfig):
+ super().__init__()
+ self.config = config
+ embed_dim = config.hidden_size
+
+ self.embeddings = CLIPVisionEmbeddings(config)
+ self.patch_dropout = PatchDropout(config.force_patch_dropout)
+ self.pre_layrnorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
+ self.encoder = CLIPEncoder(config)
+ self.post_layernorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
+
+ @add_start_docstrings_to_model_forward(CLIP_VISION_INPUTS_DOCSTRING)
+ @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=CLIPVisionConfig)
+ def forward(
+ self,
+ pixel_values: Optional[torch.FloatTensor] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> Union[Tuple, BaseModelOutputWithPooling]:
+ r"""
+ Returns:
+
+ """
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+ output_hidden_states = (
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+ )
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+ if pixel_values is None:
+ raise ValueError("You have to specify pixel_values")
+ ######################################
+ if len(pixel_values.shape) == 7:
+ b_new, pair_new, T, bs_new, channel_new, h_new, w_new = pixel_values.shape
+ # print(pixel_values.shape)
+ B = b_new * pair_new * bs_new
+ pixel_values = pixel_values.reshape(B*T, channel_new, h_new, w_new)
+
+ elif len(pixel_values.shape) == 5:
+ B, _, T, _, _ = pixel_values.shape
+ # print(pixel_values.shape)
+ pixel_values = rearrange(pixel_values, 'b c t h w -> (b t) c h w')
+ else:
+ # print(pixel_values.shape)
+ B, _, _, _ = pixel_values.shape
+ T = 1
+ ###########################
+ hidden_states = self.embeddings(pixel_values)
+
+ hidden_states = self.patch_dropout(hidden_states, B, T) ##############################################
+
+ hidden_states = self.pre_layrnorm(hidden_states)
+
+ encoder_outputs = self.encoder(
+ inputs_embeds=hidden_states,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+
+ last_hidden_state = encoder_outputs[0]
+ pooled_output = last_hidden_state[:, 0, :]
+ pooled_output = self.post_layernorm(pooled_output)
+
+ pooled_output = pooled_output.reshape(B, T, -1).mean(1) ################################
+
+ if not return_dict:
+ return (last_hidden_state, pooled_output) + encoder_outputs[1:]
+
+ return BaseModelOutputWithPooling(
+ last_hidden_state=last_hidden_state,
+ pooler_output=pooled_output,
+ hidden_states=encoder_outputs.hidden_states,
+ attentions=encoder_outputs.attentions,
+ )
+
+
+@add_start_docstrings(
+ """The vision model from CLIP without any head or projection on top.""",
+ CLIP_START_DOCSTRING,
+)
+class CLIPVisionModel(CLIPPreTrainedModel):
+ config_class = CLIPVisionConfig
+ main_input_name = "pixel_values"
+
+ def __init__(self, config: CLIPVisionConfig):
+ super().__init__(config)
+ self.vision_model = CLIPVisionTransformer(config)
+ # Initialize weights and apply final processing
+ self.post_init()
+
+ def get_input_embeddings(self) -> nn.Module:
+ return self.vision_model.embeddings.patch_embedding
+
+ @add_start_docstrings_to_model_forward(CLIP_VISION_INPUTS_DOCSTRING)
+ @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=CLIPVisionConfig)
+ def forward(
+ self,
+ pixel_values: Optional[torch.FloatTensor] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> Union[Tuple, BaseModelOutputWithPooling]:
+ r"""
+ Returns:
+
+ Examples:
+
+ ```python
+ >>> from PIL import Image
+ >>> import requests
+ >>> from transformers import AutoProcessor, CLIPVisionModel
+
+ >>> model = CLIPVisionModel.from_pretrained("openai/clip-vit-base-patch32")
+ >>> processor = AutoProcessor.from_pretrained("openai/clip-vit-base-patch32")
+
+ >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
+ >>> image = Image.open(requests.get(url, stream=True).raw)
+
+ >>> inputs = processor(images=image, return_tensors="pt")
+
+ >>> outputs = model(**inputs)
+ >>> last_hidden_state = outputs.last_hidden_state
+ >>> pooled_output = outputs.pooler_output # pooled CLS states
+ ```"""
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+ return self.vision_model(
+ pixel_values=pixel_values,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+
+
+@add_start_docstrings(CLIP_START_DOCSTRING)
+class LanguageBindDepth(CLIPPreTrainedModel):
+ config_class = LanguageBindDepthConfig
+
+ def __init__(self, config: LanguageBindDepthConfig):
+ super().__init__(config)
+
+ if not isinstance(config.text_config, CLIPTextConfig):
+ raise ValueError(
+ "config.text_config is expected to be of type CLIPTextConfig but is of type"
+ f" {type(config.text_config)}."
+ )
+
+ if not isinstance(config.vision_config, CLIPVisionConfig):
+ raise ValueError(
+ "config.vision_config is expected to be of type CLIPVisionConfig but is of type"
+ f" {type(config.vision_config)}."
+ )
+
+ text_config = config.text_config
+ vision_config = config.vision_config
+ self.add_time_attn = vision_config.add_time_attn
+ self.lora_r = vision_config.lora_r
+ self.lora_alpha = vision_config.lora_alpha
+ self.lora_dropout = vision_config.lora_dropout
+
+ self.projection_dim = config.projection_dim
+ self.text_embed_dim = text_config.hidden_size
+ self.vision_embed_dim = vision_config.hidden_size
+
+ self.text_model = CLIPTextTransformer(text_config)
+ self.vision_model = CLIPVisionTransformer(vision_config)
+
+ self.visual_projection = nn.Linear(self.vision_embed_dim, self.projection_dim, bias=False)
+ self.text_projection = nn.Linear(self.text_embed_dim, self.projection_dim, bias=False)
+ self.logit_scale = nn.Parameter(torch.tensor(self.config.logit_scale_init_value))
+
+ # Initialize weights and apply final processing
+ self.post_init()
+ self.convert_to_lora()
+ self.resize_pos(self.vision_model.embeddings, vision_config)
+
+ def convert_to_lora(self):
+ if self.lora_r == 0:
+ return
+ if self.add_time_attn:
+ target_modules = ["temporal_attn.k_proj", "temporal_attn.v_proj",
+ "temporal_attn.q_proj", "temporal_attn.out_proj",
+ "temporal_mlp.fc1", "temporal_mlp.fc2"]
+ else:
+ target_modules = ["k_proj", "v_proj", "q_proj", "out_proj"]
+ config = LoraConfig(
+ r=self.lora_r, # 16
+ lora_alpha=self.lora_alpha, # 16
+ target_modules=target_modules, # self_attn.out_proj
+ lora_dropout=self.lora_dropout, # 0.1
+ bias="none",
+ modules_to_save=[],
+ )
+ self.vision_model.encoder.is_gradient_checkpointing = False
+ self.vision_model.encoder = get_peft_model(self.vision_model.encoder, config)
+
+ def resize_pos(self, m, vision_config):
+ # convert embedding
+ if vision_config.num_mel_bins!=0 and vision_config.target_length!=0:
+ m.image_size = [vision_config.num_mel_bins, vision_config.target_length]
+ m.config.image_size = [m.image_size, m.image_size] if isinstance(m.image_size, int) else m.image_size
+ # pos resize
+ old_pos_embed_state_dict = m.position_embedding.state_dict()
+ old_pos_embed = old_pos_embed_state_dict['weight']
+ dtype = old_pos_embed.dtype
+ grid_size = [m.config.image_size[0] // m.patch_size, m.config.image_size[1] // m.patch_size]
+ extra_tokens = 1 # FIXME detect different token configs (ie no class token, or more)
+ new_seq_len = grid_size[0] * grid_size[1] + extra_tokens
+ if new_seq_len == old_pos_embed.shape[0]:
+ # m.to(args.device)
+ return
+
+ m.num_patches = grid_size[0] * grid_size[1]
+ m.num_positions = m.num_patches + 1
+ m.register_buffer("position_ids", torch.arange(m.num_positions).expand((1, -1)))
+ new_position_embedding = nn.Embedding(m.num_positions, m.embed_dim)
+
+ if extra_tokens:
+ pos_emb_tok, pos_emb_img = old_pos_embed[:extra_tokens], old_pos_embed[extra_tokens:]
+ else:
+ pos_emb_tok, pos_emb_img = None, old_pos_embed
+ old_grid_size = [int(math.sqrt(len(pos_emb_img)))] * 2
+
+ # if is_master(args):
+ # logging.info('Resizing position embedding grid-size from %s to %s', old_grid_size, grid_size)
+ pos_emb_img = pos_emb_img.reshape(1, old_grid_size[0], old_grid_size[1], -1).permute(0, 3, 1, 2)
+ pos_emb_img = F.interpolate(
+ pos_emb_img,
+ size=grid_size,
+ mode='bicubic',
+ antialias=True,
+ align_corners=False,
+ )
+ pos_emb_img = pos_emb_img.permute(0, 2, 3, 1).reshape(1, grid_size[0] * grid_size[1], -1)[0]
+ if pos_emb_tok is not None:
+ new_pos_embed = torch.cat([pos_emb_tok, pos_emb_img], dim=0)
+ else:
+ new_pos_embed = pos_emb_img
+ old_pos_embed_state_dict['weight'] = new_pos_embed.to(dtype)
+ m.position_embedding = new_position_embedding
+ m.position_embedding.load_state_dict(old_pos_embed_state_dict)
+
+ # m.to(args.device)
+
+ @add_start_docstrings_to_model_forward(CLIP_TEXT_INPUTS_DOCSTRING)
+ def get_text_features(
+ self,
+ input_ids: Optional[torch.Tensor] = None,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.Tensor] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> torch.FloatTensor:
+ r"""
+ Returns:
+ text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by
+ applying the projection layer to the pooled output of [`CLIPTextModel`].
+
+ Examples:
+
+ ```python
+ >>> from transformers import AutoTokenizer, CLIPModel
+
+ >>> model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")
+ >>> tokenizer = AutoTokenizer.from_pretrained("openai/clip-vit-base-patch32")
+
+ >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt")
+ >>> text_features = model.get_text_features(**inputs)
+ ```"""
+ # Use CLIP model's config for some fields (if specified) instead of those of vision & text components.
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+ output_hidden_states = (
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+ )
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+ text_outputs = self.text_model(
+ input_ids=input_ids,
+ attention_mask=attention_mask,
+ position_ids=position_ids,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+
+ pooled_output = text_outputs[1]
+ text_features = self.text_projection(pooled_output)
+
+ return text_features
+
+ @add_start_docstrings_to_model_forward(CLIP_VISION_INPUTS_DOCSTRING)
+ def get_image_features(
+ self,
+ pixel_values: Optional[torch.FloatTensor] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> torch.FloatTensor:
+ r"""
+ Returns:
+ image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by
+ applying the projection layer to the pooled output of [`CLIPVisionModel`].
+
+ Examples:
+
+ ```python
+ >>> from PIL import Image
+ >>> import requests
+ >>> from transformers import AutoProcessor, CLIPModel
+
+ >>> model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")
+ >>> processor = AutoProcessor.from_pretrained("openai/clip-vit-base-patch32")
+
+ >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
+ >>> image = Image.open(requests.get(url, stream=True).raw)
+
+ >>> inputs = processor(images=image, return_tensors="pt")
+
+ >>> image_features = model.get_image_features(**inputs)
+ ```"""
+ # Use CLIP model's config for some fields (if specified) instead of those of vision & text components.
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+ output_hidden_states = (
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+ )
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+ vision_outputs = self.vision_model(
+ pixel_values=pixel_values,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+
+ pooled_output = vision_outputs[1] # pooled_output
+ image_features = self.visual_projection(pooled_output)
+
+ return image_features
+
+ @add_start_docstrings_to_model_forward(CLIP_INPUTS_DOCSTRING)
+ @replace_return_docstrings(output_type=CLIPOutput, config_class=LanguageBindDepthConfig)
+ def forward(
+ self,
+ input_ids: Optional[torch.LongTensor] = None,
+ pixel_values: Optional[torch.FloatTensor] = None,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.LongTensor] = None,
+ return_loss: Optional[bool] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> Union[Tuple, CLIPOutput]:
+ r"""
+ Returns:
+
+ Examples:
+
+ ```python
+ >>> from PIL import Image
+ >>> import requests
+ >>> from transformers import AutoProcessor, CLIPModel
+
+ >>> model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")
+ >>> processor = AutoProcessor.from_pretrained("openai/clip-vit-base-patch32")
+
+ >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
+ >>> image = Image.open(requests.get(url, stream=True).raw)
+
+ >>> inputs = processor(
+ ... text=["a photo of a cat", "a photo of a dog"], images=image, return_tensors="pt", padding=True
+ ... )
+
+ >>> outputs = model(**inputs)
+ >>> logits_per_image = outputs.logits_per_image # this is the image-text similarity score
+ >>> probs = logits_per_image.softmax(dim=1) # we can take the softmax to get the label probabilities
+ ```"""
+ # Use CLIP model's config for some fields (if specified) instead of those of vision & text components.
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+ output_hidden_states = (
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+ )
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+ vision_outputs = self.vision_model(
+ pixel_values=pixel_values,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+
+ text_outputs = self.text_model(
+ input_ids=input_ids,
+ attention_mask=attention_mask,
+ position_ids=position_ids,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+
+ image_embeds = vision_outputs[1]
+ image_embeds = self.visual_projection(image_embeds)
+
+ text_embeds = text_outputs[1]
+ text_embeds = self.text_projection(text_embeds)
+
+ # normalized features
+ image_embeds = image_embeds / image_embeds.norm(p=2, dim=-1, keepdim=True)
+ text_embeds = text_embeds / text_embeds.norm(p=2, dim=-1, keepdim=True)
+
+ # cosine similarity as logits
+ logit_scale = self.logit_scale.exp()
+ logits_per_text = torch.matmul(text_embeds, image_embeds.t()) * logit_scale
+ logits_per_image = logits_per_text.t()
+
+ loss = None
+ if return_loss:
+ loss = clip_loss(logits_per_text)
+
+ if not return_dict:
+ output = (logits_per_image, logits_per_text, text_embeds, image_embeds, text_outputs, vision_outputs)
+ return ((loss,) + output) if loss is not None else output
+
+ return CLIPOutput(
+ loss=loss,
+ logits_per_image=logits_per_image,
+ logits_per_text=logits_per_text,
+ text_embeds=text_embeds,
+ image_embeds=image_embeds,
+ text_model_output=text_outputs,
+ vision_model_output=vision_outputs,
+ )
\ No newline at end of file
diff --git a/videollava/model/multimodal_encoder/languagebind/depth/processing_depth.py b/videollava/model/multimodal_encoder/languagebind/depth/processing_depth.py
new file mode 100644
index 0000000000000000000000000000000000000000..1019e0cb45c8be4bc7424c4d8f9d091dac5dab0b
--- /dev/null
+++ b/videollava/model/multimodal_encoder/languagebind/depth/processing_depth.py
@@ -0,0 +1,108 @@
+import cv2
+import torch
+from PIL import Image
+from torch import nn
+from torchvision import transforms
+from transformers import ProcessorMixin, BatchEncoding
+from transformers.image_processing_utils import BatchFeature
+
+OPENAI_DATASET_MEAN = (0.48145466, 0.4578275, 0.40821073)
+OPENAI_DATASET_STD = (0.26862954, 0.26130258, 0.27577711)
+
+def make_list_of_images(x):
+ if not isinstance(x, list):
+ return [x]
+ return x
+
+def opencv_loader(path):
+ return cv2.imread(path, cv2.IMREAD_UNCHANGED).astype('float32')
+
+
+class DepthNorm(nn.Module):
+ def __init__(
+ self,
+ max_depth=0,
+ min_depth=0.01,
+ ):
+ super().__init__()
+ self.max_depth = max_depth
+ self.min_depth = min_depth
+ self.scale = 1000.0 # nyuv2 abs.depth
+
+ def forward(self, image):
+ # image = np.array(image)
+ depth_img = image / self.scale # (H, W) in meters
+ depth_img = depth_img.clip(min=self.min_depth)
+ if self.max_depth != 0:
+ depth_img = depth_img.clip(max=self.max_depth)
+ depth_img /= self.max_depth # 0-1
+ else:
+ depth_img /= depth_img.max()
+ depth_img = torch.from_numpy(depth_img).unsqueeze(0).repeat(3, 1, 1) # assume image
+ return depth_img.to(torch.get_default_dtype())
+
+def get_depth_transform(config):
+ config = config.vision_config
+ transform = transforms.Compose(
+ [
+ DepthNorm(max_depth=config.max_depth),
+ transforms.Resize(224, interpolation=transforms.InterpolationMode.BICUBIC),
+ transforms.CenterCrop(224),
+ transforms.Normalize(OPENAI_DATASET_MEAN, OPENAI_DATASET_STD), # assume image
+ # transforms.Normalize((0.5, ), (0.5, )) # 0-1 to norm distribution
+ # transforms.Normalize((0.0418, ), (0.0295, )) # sun rgb-d imagebind
+ # transforms.Normalize((0.02, ), (0.00295, )) # nyuv2
+ ]
+ )
+ return transform
+
+def load_and_transform_depth(depth_path, transform):
+ depth = opencv_loader(depth_path)
+ depth_outputs = transform(depth)
+ return depth_outputs
+
+class LanguageBindDepthProcessor(ProcessorMixin):
+ attributes = []
+ tokenizer_class = ("LanguageBindDepthTokenizer")
+
+ def __init__(self, config, tokenizer=None, **kwargs):
+ super().__init__(**kwargs)
+ self.config = config
+ self.transform = get_depth_transform(config)
+ self.image_processor = load_and_transform_depth
+ self.tokenizer = tokenizer
+
+ def __call__(self, images=None, text=None, context_length=77, return_tensors=None, **kwargs):
+ if text is None and images is None:
+ raise ValueError("You have to specify either text or images. Both cannot be none.")
+
+ if text is not None:
+ encoding = self.tokenizer(text, max_length=context_length, padding='max_length',
+ truncation=True, return_tensors=return_tensors, **kwargs)
+
+ if images is not None:
+ images = make_list_of_images(images)
+ image_features = [self.image_processor(image, self.transform) for image in images]
+ image_features = torch.stack(image_features)
+
+ if text is not None and images is not None:
+ encoding["pixel_values"] = image_features
+ return encoding
+ elif text is not None:
+ return encoding
+ else:
+ return {"pixel_values": image_features}
+
+ def batch_decode(self, skip_special_tokens=True, *args, **kwargs):
+ """
+ This method forwards all its arguments to CLIPTokenizerFast's [`~PreTrainedTokenizer.batch_decode`]. Please
+ refer to the docstring of this method for more information.
+ """
+ return self.tokenizer.batch_decode(*args, skip_special_tokens=skip_special_tokens, **kwargs)
+
+ def decode(self, skip_special_tokens=True, *args, **kwargs):
+ """
+ This method forwards all its arguments to CLIPTokenizerFast's [`~PreTrainedTokenizer.decode`]. Please refer to
+ the docstring of this method for more information.
+ """
+ return self.tokenizer.decode(*args, skip_special_tokens=skip_special_tokens, **kwargs)
diff --git a/videollava/model/multimodal_encoder/languagebind/depth/tokenization_depth.py b/videollava/model/multimodal_encoder/languagebind/depth/tokenization_depth.py
new file mode 100644
index 0000000000000000000000000000000000000000..eda9905131c2240cddf982b2937fe96cb33b4053
--- /dev/null
+++ b/videollava/model/multimodal_encoder/languagebind/depth/tokenization_depth.py
@@ -0,0 +1,77 @@
+from transformers import CLIPTokenizer
+from transformers.utils import logging
+
+logger = logging.get_logger(__name__)
+
+VOCAB_FILES_NAMES = {
+ "vocab_file": "vocab.json",
+ "merges_file": "merges.txt",
+}
+
+PRETRAINED_VOCAB_FILES_MAP = {
+ "vocab_file": {
+ "lb203/LanguageBind-Depth": "https://huggingface.co/lb203/LanguageBind-Depth/resolve/main/vocab.json",
+ },
+ "merges_file": {
+ "lb203/LanguageBind-Depth": "https://huggingface.co/lb203/LanguageBind-Depth/resolve/main/merges.txt",
+ },
+}
+
+PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
+ "lb203/LanguageBind-Depth": 77,
+}
+
+
+PRETRAINED_INIT_CONFIGURATION = {
+ "lb203/LanguageBind-Thermal": {},
+}
+
+class LanguageBindDepthTokenizer(CLIPTokenizer):
+ """
+ Construct a CLIP tokenizer. Based on byte-level Byte-Pair-Encoding.
+
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+ this superclass for more information regarding those methods.
+
+ Args:
+ vocab_file (`str`):
+ Path to the vocabulary file.
+ merges_file (`str`):
+ Path to the merges file.
+ errors (`str`, *optional*, defaults to `"replace"`):
+ Paradigm to follow when decoding bytes to UTF-8. See
+ [bytes.decode](https://docs.python.org/3/library/stdtypes.html#bytes.decode) for more information.
+ unk_token (`str`, *optional*, defaults to `<|endoftext|>`):
+ The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
+ token instead.
+ bos_token (`str`, *optional*, defaults to `<|startoftext|>`):
+ The beginning of sequence token.
+ eos_token (`str`, *optional*, defaults to `<|endoftext|>`):
+ The end of sequence token.
+ """
+
+ vocab_files_names = VOCAB_FILES_NAMES
+ pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+ max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
+ model_input_names = ["input_ids", "attention_mask"]
+
+ def __init__(
+ self,
+ vocab_file,
+ merges_file,
+ errors="replace",
+ unk_token="<|endoftext|>",
+ bos_token="<|startoftext|>",
+ eos_token="<|endoftext|>",
+ pad_token="<|endoftext|>", # hack to enable padding
+ **kwargs,
+ ):
+ super(LanguageBindDepthTokenizer, self).__init__(
+ vocab_file,
+ merges_file,
+ errors,
+ unk_token,
+ bos_token,
+ eos_token,
+ pad_token, # hack to enable padding
+ **kwargs,)
\ No newline at end of file
diff --git a/videollava/model/multimodal_encoder/languagebind/image/configuration_image.py b/videollava/model/multimodal_encoder/languagebind/image/configuration_image.py
new file mode 100644
index 0000000000000000000000000000000000000000..c1c7b0f7aad10f791c89b2f89aa4161defb990ae
--- /dev/null
+++ b/videollava/model/multimodal_encoder/languagebind/image/configuration_image.py
@@ -0,0 +1,423 @@
+import copy
+import os
+from typing import Union
+
+from transformers import PretrainedConfig
+from transformers.utils import logging
+
+logger = logging.get_logger(__name__)
+
+
+
+
+
+
+
+class CLIPTextConfig(PretrainedConfig):
+ r"""
+ This is the configuration class to store the configuration of a [`CLIPTextModel`]. It is used to instantiate a CLIP
+ text encoder according to the specified arguments, defining the model architecture. Instantiating a configuration
+ with the defaults will yield a similar configuration to that of the text encoder of the CLIP
+ [openai/clip-vit-base-patch32](https://huggingface.co/openai/clip-vit-base-patch32) architecture.
+
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
+
+ Args:
+ vocab_size (`int`, *optional*, defaults to 49408):
+ Vocabulary size of the CLIP text model. Defines the number of different tokens that can be represented by
+ the `inputs_ids` passed when calling [`CLIPModel`].
+ hidden_size (`int`, *optional*, defaults to 512):
+ Dimensionality of the encoder layers and the pooler layer.
+ intermediate_size (`int`, *optional*, defaults to 2048):
+ Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
+ num_hidden_layers (`int`, *optional*, defaults to 12):
+ Number of hidden layers in the Transformer encoder.
+ num_attention_heads (`int`, *optional*, defaults to 8):
+ Number of attention heads for each attention layer in the Transformer encoder.
+ max_position_embeddings (`int`, *optional*, defaults to 77):
+ The maximum sequence length that this model might ever be used with. Typically set this to something large
+ just in case (e.g., 512 or 1024 or 2048).
+ hidden_act (`str` or `function`, *optional*, defaults to `"quick_gelu"`):
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"selu"` and `"gelu_new"` `"quick_gelu"` are supported.
+ layer_norm_eps (`float`, *optional*, defaults to 1e-5):
+ The epsilon used by the layer normalization layers.
+ attention_dropout (`float`, *optional*, defaults to 0.0):
+ The dropout ratio for the attention probabilities.
+ initializer_range (`float`, *optional*, defaults to 0.02):
+ The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+ initializer_factor (`float`, *optional*, defaults to 1):
+ A factor for initializing all weight matrices (should be kept to 1, used internally for initialization
+ testing).
+
+ Example:
+
+ ```python
+ >>> from transformers import CLIPTextConfig, CLIPTextModel
+
+ >>> # Initializing a CLIPTextConfig with openai/clip-vit-base-patch32 style configuration
+ >>> configuration = CLIPTextConfig()
+
+ >>> # Initializing a CLIPTextModel (with random weights) from the openai/clip-vit-base-patch32 style configuration
+ >>> model = CLIPTextModel(configuration)
+
+ >>> # Accessing the model configuration
+ >>> configuration = model.config
+ ```"""
+ model_type = "clip_text_model"
+
+ def __init__(
+ self,
+ vocab_size=49408,
+ hidden_size=512,
+ intermediate_size=2048,
+ projection_dim=512,
+ num_hidden_layers=12,
+ num_attention_heads=8,
+ max_position_embeddings=77,
+ hidden_act="quick_gelu",
+ layer_norm_eps=1e-5,
+ attention_dropout=0.0,
+ initializer_range=0.02,
+ initializer_factor=1.0,
+ # This differs from `CLIPTokenizer`'s default and from openai/clip
+ # See https://github.com/huggingface/transformers/pull/24773#issuecomment-1632287538
+ pad_token_id=1,
+ bos_token_id=49406,
+ eos_token_id=49407,
+ **kwargs,
+ ):
+ super().__init__(pad_token_id=pad_token_id, bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs)
+
+ self.vocab_size = vocab_size
+ self.hidden_size = hidden_size
+ self.intermediate_size = intermediate_size
+ self.projection_dim = projection_dim
+ self.num_hidden_layers = num_hidden_layers
+ self.num_attention_heads = num_attention_heads
+ self.max_position_embeddings = max_position_embeddings
+ self.layer_norm_eps = layer_norm_eps
+ self.hidden_act = hidden_act
+ self.initializer_range = initializer_range
+ self.initializer_factor = initializer_factor
+ self.attention_dropout = attention_dropout
+ self.add_time_attn = False ######################################
+
+ @classmethod
+ def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":
+ cls._set_token_in_kwargs(kwargs)
+
+ config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)
+
+ # get the text config dict if we are loading from CLIPConfig
+ if config_dict.get("model_type") == "clip":
+ config_dict = config_dict["text_config"]
+
+ if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:
+ logger.warning(
+ f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
+ f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
+ )
+
+ return cls.from_dict(config_dict, **kwargs)
+
+
+
+
+class CLIPVisionConfig(PretrainedConfig):
+ r"""
+ This is the configuration class to store the configuration of a [`CLIPVisionModel`]. It is used to instantiate a
+ CLIP vision encoder according to the specified arguments, defining the model architecture. Instantiating a
+ configuration with the defaults will yield a similar configuration to that of the vision encoder of the CLIP
+ [openai/clip-vit-base-patch32](https://huggingface.co/openai/clip-vit-base-patch32) architecture.
+
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
+
+ Args:
+ hidden_size (`int`, *optional*, defaults to 768):
+ Dimensionality of the encoder layers and the pooler layer.
+ intermediate_size (`int`, *optional*, defaults to 3072):
+ Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
+ num_hidden_layers (`int`, *optional*, defaults to 12):
+ Number of hidden layers in the Transformer encoder.
+ num_attention_heads (`int`, *optional*, defaults to 12):
+ Number of attention heads for each attention layer in the Transformer encoder.
+ image_size (`int`, *optional*, defaults to 224):
+ The size (resolution) of each image.
+ patch_size (`int`, *optional*, defaults to 32):
+ The size (resolution) of each patch.
+ hidden_act (`str` or `function`, *optional*, defaults to `"quick_gelu"`):
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"selu"` and `"gelu_new"` ``"quick_gelu"` are supported.
+ layer_norm_eps (`float`, *optional*, defaults to 1e-5):
+ The epsilon used by the layer normalization layers.
+ attention_dropout (`float`, *optional*, defaults to 0.0):
+ The dropout ratio for the attention probabilities.
+ initializer_range (`float`, *optional*, defaults to 0.02):
+ The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+ initializer_factor (`float`, *optional*, defaults to 1):
+ A factor for initializing all weight matrices (should be kept to 1, used internally for initialization
+ testing).
+
+ Example:
+
+ ```python
+ >>> from transformers import CLIPVisionConfig, CLIPVisionModel
+
+ >>> # Initializing a CLIPVisionConfig with openai/clip-vit-base-patch32 style configuration
+ >>> configuration = CLIPVisionConfig()
+
+ >>> # Initializing a CLIPVisionModel (with random weights) from the openai/clip-vit-base-patch32 style configuration
+ >>> model = CLIPVisionModel(configuration)
+
+ >>> # Accessing the model configuration
+ >>> configuration = model.config
+ ```"""
+
+ model_type = "clip_vision_model"
+
+ def __init__(
+ self,
+ hidden_size=768,
+ intermediate_size=3072,
+ projection_dim=512,
+ num_hidden_layers=12,
+ num_attention_heads=12,
+ num_channels=3,
+ image_size=224,
+ patch_size=32,
+ hidden_act="quick_gelu",
+ layer_norm_eps=1e-5,
+ attention_dropout=0.0,
+ initializer_range=0.02,
+ initializer_factor=1.0,
+
+ add_time_attn=False, ################################
+ num_frames=1, ################################
+ force_patch_dropout=0.0, ################################
+ lora_r=2, ################################
+ lora_alpha=16, ################################
+ lora_dropout=0.0, ################################
+ num_mel_bins=0.0, ################################
+ target_length=0.0, ################################
+ video_decode_backend='decord', #########################
+ **kwargs,
+ ):
+ super().__init__(**kwargs)
+
+ self.hidden_size = hidden_size
+ self.intermediate_size = intermediate_size
+ self.projection_dim = projection_dim
+ self.num_hidden_layers = num_hidden_layers
+ self.num_attention_heads = num_attention_heads
+ self.num_channels = num_channels
+ self.patch_size = patch_size
+ self.image_size = image_size
+ self.initializer_range = initializer_range
+ self.initializer_factor = initializer_factor
+ self.attention_dropout = attention_dropout
+ self.layer_norm_eps = layer_norm_eps
+ self.hidden_act = hidden_act
+
+ self.add_time_attn = add_time_attn ################
+ self.num_frames = num_frames ################
+ self.force_patch_dropout = force_patch_dropout ################
+ self.lora_r = lora_r ################
+ self.lora_alpha = lora_alpha ################
+ self.lora_dropout = lora_dropout ################
+ self.num_mel_bins = num_mel_bins ################
+ self.target_length = target_length ################
+ self.video_decode_backend = video_decode_backend ################
+
+ @classmethod
+ def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":
+ cls._set_token_in_kwargs(kwargs)
+
+ config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)
+
+ # get the vision config dict if we are loading from CLIPConfig
+ if config_dict.get("model_type") == "clip":
+ config_dict = config_dict["vision_config"]
+
+ if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:
+ logger.warning(
+ f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
+ f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
+ )
+
+ return cls.from_dict(config_dict, **kwargs)
+
+
+class LanguageBindImageConfig(PretrainedConfig):
+ r"""
+ [`CLIPConfig`] is the configuration class to store the configuration of a [`CLIPModel`]. It is used to instantiate
+ a CLIP model according to the specified arguments, defining the text model and vision model configs. Instantiating
+ a configuration with the defaults will yield a similar configuration to that of the CLIP
+ [openai/clip-vit-base-patch32](https://huggingface.co/openai/clip-vit-base-patch32) architecture.
+
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
+
+ Args:
+ text_config (`dict`, *optional*):
+ Dictionary of configuration options used to initialize [`CLIPTextConfig`].
+ vision_config (`dict`, *optional*):
+ Dictionary of configuration options used to initialize [`CLIPVisionConfig`].
+ projection_dim (`int`, *optional*, defaults to 512):
+ Dimentionality of text and vision projection layers.
+ logit_scale_init_value (`float`, *optional*, defaults to 2.6592):
+ The inital value of the *logit_scale* paramter. Default is used as per the original CLIP implementation.
+ kwargs (*optional*):
+ Dictionary of keyword arguments.
+
+ Example:
+
+ ```python
+ >>> from transformers import CLIPConfig, CLIPModel
+
+ >>> # Initializing a CLIPConfig with openai/clip-vit-base-patch32 style configuration
+ >>> configuration = CLIPConfig()
+
+ >>> # Initializing a CLIPModel (with random weights) from the openai/clip-vit-base-patch32 style configuration
+ >>> model = CLIPModel(configuration)
+
+ >>> # Accessing the model configuration
+ >>> configuration = model.config
+
+ >>> # We can also initialize a CLIPConfig from a CLIPTextConfig and a CLIPVisionConfig
+ >>> from transformers import CLIPTextConfig, CLIPVisionConfig
+
+ >>> # Initializing a CLIPText and CLIPVision configuration
+ >>> config_text = CLIPTextConfig()
+ >>> config_vision = CLIPVisionConfig()
+
+ >>> config = CLIPConfig.from_text_vision_configs(config_text, config_vision)
+ ```"""
+
+ model_type = "LanguageBindImage"
+ is_composition = True
+
+ def __init__(
+ self, text_config=None, vision_config=None, projection_dim=512, logit_scale_init_value=2.6592, **kwargs
+ ):
+ # If `_config_dict` exist, we use them for the backward compatibility.
+ # We pop out these 2 attributes before calling `super().__init__` to avoid them being saved (which causes a lot
+ # of confusion!).
+ text_config_dict = kwargs.pop("text_config_dict", None)
+ vision_config_dict = kwargs.pop("vision_config_dict", None)
+
+ super().__init__(**kwargs)
+
+ # Instead of simply assigning `[text|vision]_config_dict` to `[text|vision]_config`, we use the values in
+ # `[text|vision]_config_dict` to update the values in `[text|vision]_config`. The values should be same in most
+ # cases, but we don't want to break anything regarding `_config_dict` that existed before commit `8827e1b2`.
+ if text_config_dict is not None:
+ if text_config is None:
+ text_config = {}
+
+ # This is the complete result when using `text_config_dict`.
+ _text_config_dict = CLIPTextConfig(**text_config_dict).to_dict()
+
+ # Give a warning if the values exist in both `_text_config_dict` and `text_config` but being different.
+ for key, value in _text_config_dict.items():
+ if key in text_config and value != text_config[key] and key not in ["transformers_version"]:
+ # If specified in `text_config_dict`
+ if key in text_config_dict:
+ message = (
+ f"`{key}` is found in both `text_config_dict` and `text_config` but with different values. "
+ f'The value `text_config_dict["{key}"]` will be used instead.'
+ )
+ # If inferred from default argument values (just to be super careful)
+ else:
+ message = (
+ f"`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The "
+ f'value `text_config["{key}"]` will be overriden.'
+ )
+ logger.warning(message)
+
+ # Update all values in `text_config` with the ones in `_text_config_dict`.
+ text_config.update(_text_config_dict)
+
+ if vision_config_dict is not None:
+ if vision_config is None:
+ vision_config = {}
+
+ # This is the complete result when using `vision_config_dict`.
+ _vision_config_dict = CLIPVisionConfig(**vision_config_dict).to_dict()
+ # convert keys to string instead of integer
+ if "id2label" in _vision_config_dict:
+ _vision_config_dict["id2label"] = {
+ str(key): value for key, value in _vision_config_dict["id2label"].items()
+ }
+
+ # Give a warning if the values exist in both `_vision_config_dict` and `vision_config` but being different.
+ for key, value in _vision_config_dict.items():
+ if key in vision_config and value != vision_config[key] and key not in ["transformers_version"]:
+ # If specified in `vision_config_dict`
+ if key in vision_config_dict:
+ message = (
+ f"`{key}` is found in both `vision_config_dict` and `vision_config` but with different "
+ f'values. The value `vision_config_dict["{key}"]` will be used instead.'
+ )
+ # If inferred from default argument values (just to be super careful)
+ else:
+ message = (
+ f"`vision_config_dict` is provided which will be used to initialize `CLIPVisionConfig`. "
+ f'The value `vision_config["{key}"]` will be overriden.'
+ )
+ logger.warning(message)
+
+ # Update all values in `vision_config` with the ones in `_vision_config_dict`.
+ vision_config.update(_vision_config_dict)
+
+ if text_config is None:
+ text_config = {}
+ logger.info("`text_config` is `None`. Initializing the `CLIPTextConfig` with default values.")
+
+ if vision_config is None:
+ vision_config = {}
+ logger.info("`vision_config` is `None`. initializing the `CLIPVisionConfig` with default values.")
+
+ self.text_config = CLIPTextConfig(**text_config)
+ self.vision_config = CLIPVisionConfig(**vision_config)
+
+ self.projection_dim = projection_dim
+ self.logit_scale_init_value = logit_scale_init_value
+ self.initializer_factor = 1.0
+
+ @classmethod
+ def from_text_vision_configs(cls, text_config: CLIPTextConfig, vision_config: CLIPVisionConfig, **kwargs):
+ r"""
+ Instantiate a [`CLIPConfig`] (or a derived class) from clip text model configuration and clip vision model
+ configuration.
+
+ Returns:
+ [`CLIPConfig`]: An instance of a configuration object
+ """
+
+ return cls(text_config=text_config.to_dict(), vision_config=vision_config.to_dict(), **kwargs)
+
+ def to_dict(self):
+ """
+ Serializes this instance to a Python dictionary. Override the default [`~PretrainedConfig.to_dict`].
+
+ Returns:
+ `Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance,
+ """
+ output = copy.deepcopy(self.__dict__)
+ output["text_config"] = self.text_config.to_dict()
+ output["vision_config"] = self.vision_config.to_dict()
+ output["model_type"] = self.__class__.model_type
+ return output
+
+
+
+
+
+
+
+
+
+
diff --git a/videollava/model/multimodal_encoder/languagebind/image/modeling_image.py b/videollava/model/multimodal_encoder/languagebind/image/modeling_image.py
new file mode 100644
index 0000000000000000000000000000000000000000..e95ac475e0f6ec9916543476832b34e7d6ec7c3a
--- /dev/null
+++ b/videollava/model/multimodal_encoder/languagebind/image/modeling_image.py
@@ -0,0 +1,1030 @@
+import math
+from typing import Optional, Tuple, Union
+
+import torch
+from einops import rearrange
+from peft import LoraConfig, get_peft_model
+from torch import nn
+from torch.nn import functional as F
+from transformers import PreTrainedModel, add_start_docstrings
+from transformers.modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling
+from transformers.models.clip.modeling_clip import CLIPMLP, CLIPAttention, CLIPTextEmbeddings, CLIPVisionEmbeddings, \
+ CLIPVisionModelWithProjection, CLIPTextModelWithProjection, _expand_mask, CLIPOutput, clip_loss
+from transformers.utils import add_start_docstrings_to_model_forward, replace_return_docstrings
+
+from .configuration_image import LanguageBindImageConfig, CLIPVisionConfig, CLIPTextConfig
+
+
+
+class PatchDropout(nn.Module):
+ """
+ https://arxiv.org/abs/2212.00794
+ """
+
+ def __init__(self, prob, exclude_first_token=True):
+ super().__init__()
+ assert 0 <= prob < 1.
+ self.prob = prob
+ self.exclude_first_token = exclude_first_token # exclude CLS token
+
+ def forward(self, x, B, T):
+ if not self.training or self.prob == 0.:
+ return x
+
+ if self.exclude_first_token:
+ cls_tokens, x = x[:, :1], x[:, 1:]
+ else:
+ cls_tokens = torch.jit.annotate(torch.Tensor, x[:, :1])
+
+ batch = x.size()[0]
+ num_tokens = x.size()[1]
+
+ batch_indices = torch.arange(batch)
+ batch_indices = batch_indices[..., None]
+
+ keep_prob = 1 - self.prob
+ num_patches_keep = max(1, int(num_tokens * keep_prob))
+
+ if T == 1:
+ rand = torch.randn(batch, num_tokens)
+ patch_indices_keep = rand.topk(num_patches_keep, dim=-1).indices
+ else:
+ rand = torch.randn(B, num_tokens)
+ patch_indices_keep = rand.topk(num_patches_keep, dim=-1).indices
+ patch_indices_keep = patch_indices_keep.unsqueeze(1).repeat(1, T, 1)
+ patch_indices_keep = rearrange(patch_indices_keep, 'b t n -> (b t) n')
+
+
+ x = x[batch_indices, patch_indices_keep]
+
+ if self.exclude_first_token:
+ x = torch.cat((cls_tokens, x), dim=1)
+
+ return x
+
+class CLIPEncoderLayer(nn.Module):
+ def __init__(self, config: LanguageBindImageConfig):
+ super().__init__()
+ self.embed_dim = config.hidden_size
+ self.self_attn = CLIPAttention(config)
+ self.layer_norm1 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
+ self.mlp = CLIPMLP(config)
+ self.layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
+
+ self.add_time_attn = config.add_time_attn
+ if self.add_time_attn:
+ self.t = config.num_frames
+ self.temporal_embedding = nn.Parameter(torch.zeros(1, config.num_frames, config.hidden_size))
+ nn.init.normal_(self.temporal_embedding, std=config.hidden_size ** -0.5)
+
+ self.embed_dim = config.hidden_size
+ self.temporal_attn = CLIPAttention(config)
+ self.temporal_layer_norm1 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
+ self.temporal_mlp = CLIPMLP(config)
+ self.temporal_layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
+
+ def forward(
+ self,
+ hidden_states: torch.Tensor,
+ attention_mask: torch.Tensor,
+ causal_attention_mask: torch.Tensor,
+ output_attentions: Optional[bool] = False,
+ ) -> Tuple[torch.FloatTensor]:
+ """
+ Args:
+ hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
+ attention_mask (`torch.FloatTensor`): attention mask of size
+ `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
+ `(config.encoder_attention_heads,)`.
+ output_attentions (`bool`, *optional*):
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under
+ returned tensors for more detail.
+ """
+
+
+ if self.add_time_attn:
+ bt, n, d = hidden_states.shape
+ t = self.t
+
+ # time embed
+ if t != 1:
+ n = hidden_states.shape[1]
+ hidden_states = rearrange(hidden_states, '(b t) n d -> (b n) t d', t=t)
+ hidden_states = hidden_states + self.temporal_embedding[:, :t, :]
+ hidden_states = rearrange(hidden_states, '(b n) t d -> (b t) n d', n=n)
+
+ # time attn
+ residual = hidden_states
+ hidden_states = rearrange(hidden_states, '(b t) n d -> (b n) t d', t=t)
+ # hidden_states = self.layer_norm1(hidden_states) # share layernorm
+ hidden_states = self.temporal_layer_norm1(hidden_states)
+ hidden_states, attn_weights = self.temporal_attn(
+ hidden_states=hidden_states,
+ attention_mask=attention_mask,
+ causal_attention_mask=causal_attention_mask,
+ output_attentions=output_attentions,
+ )
+ hidden_states = residual + rearrange(hidden_states, '(b n) t d -> (b t) n d', n=n)
+
+ residual = hidden_states
+ hidden_states = rearrange(hidden_states, '(b t) n d -> (b n) t d', t=t)
+ # hidden_states = self.layer_norm2(hidden_states) # share layernorm
+ hidden_states = self.temporal_layer_norm2(hidden_states)
+ hidden_states = self.temporal_mlp(hidden_states)
+ hidden_states = residual + rearrange(hidden_states, '(b n) t d -> (b t) n d', n=n)
+
+ # spatial attn
+ residual = hidden_states
+
+ hidden_states = self.layer_norm1(hidden_states)
+ hidden_states, attn_weights = self.self_attn(
+ hidden_states=hidden_states,
+ attention_mask=attention_mask,
+ causal_attention_mask=causal_attention_mask,
+ output_attentions=output_attentions,
+ )
+ hidden_states = residual + hidden_states
+
+ residual = hidden_states
+ hidden_states = self.layer_norm2(hidden_states)
+ hidden_states = self.mlp(hidden_states)
+ hidden_states = residual + hidden_states
+
+ outputs = (hidden_states,)
+
+ if output_attentions:
+ outputs += (attn_weights,)
+
+ return outputs
+
+
+
+
+
+
+
+
+
+class CLIPPreTrainedModel(PreTrainedModel):
+ """
+ An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
+ models.
+ """
+
+ config_class = LanguageBindImageConfig
+ base_model_prefix = "clip"
+ supports_gradient_checkpointing = True
+ _keys_to_ignore_on_load_missing = [r"position_ids"]
+
+ def _init_weights(self, module):
+ """Initialize the weights"""
+ factor = self.config.initializer_factor
+ if isinstance(module, CLIPTextEmbeddings):
+ module.token_embedding.weight.data.normal_(mean=0.0, std=factor * 0.02)
+ module.position_embedding.weight.data.normal_(mean=0.0, std=factor * 0.02)
+ elif isinstance(module, CLIPVisionEmbeddings):
+ factor = self.config.initializer_factor
+ nn.init.normal_(module.class_embedding, mean=0.0, std=module.embed_dim**-0.5 * factor)
+ nn.init.normal_(module.patch_embedding.weight, std=module.config.initializer_range * factor)
+ nn.init.normal_(module.position_embedding.weight, std=module.config.initializer_range * factor)
+ elif isinstance(module, CLIPAttention):
+ factor = self.config.initializer_factor
+ in_proj_std = (module.embed_dim**-0.5) * ((2 * module.config.num_hidden_layers) ** -0.5) * factor
+ out_proj_std = (module.embed_dim**-0.5) * factor
+ nn.init.normal_(module.q_proj.weight, std=in_proj_std)
+ nn.init.normal_(module.k_proj.weight, std=in_proj_std)
+ nn.init.normal_(module.v_proj.weight, std=in_proj_std)
+ nn.init.normal_(module.out_proj.weight, std=out_proj_std)
+ elif isinstance(module, CLIPMLP):
+ factor = self.config.initializer_factor
+ in_proj_std = (
+ (module.config.hidden_size**-0.5) * ((2 * module.config.num_hidden_layers) ** -0.5) * factor
+ )
+ fc_std = (2 * module.config.hidden_size) ** -0.5 * factor
+ nn.init.normal_(module.fc1.weight, std=fc_std)
+ nn.init.normal_(module.fc2.weight, std=in_proj_std)
+ elif isinstance(module, LanguageBindImage):
+ nn.init.normal_(
+ module.text_projection.weight,
+ std=module.text_embed_dim**-0.5 * self.config.initializer_factor,
+ )
+ nn.init.normal_(
+ module.visual_projection.weight,
+ std=module.vision_embed_dim**-0.5 * self.config.initializer_factor,
+ )
+ elif isinstance(module, CLIPVisionModelWithProjection):
+ nn.init.normal_(
+ module.visual_projection.weight,
+ std=self.config.hidden_size**-0.5 * self.config.initializer_factor,
+ )
+ elif isinstance(module, CLIPTextModelWithProjection):
+ nn.init.normal_(
+ module.text_projection.weight,
+ std=self.config.hidden_size**-0.5 * self.config.initializer_factor,
+ )
+
+ if isinstance(module, nn.LayerNorm):
+ module.bias.data.zero_()
+ module.weight.data.fill_(1.0)
+ if isinstance(module, nn.Linear) and module.bias is not None:
+ module.bias.data.zero_()
+
+ def _set_gradient_checkpointing(self, module, value=False):
+ if isinstance(module, CLIPEncoder):
+ module.gradient_checkpointing = value
+
+
+CLIP_START_DOCSTRING = r"""
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
+
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
+
+ Parameters:
+ config ([`CLIPConfig`]): Model configuration class with all the parameters of the model.
+ Initializing with a config file does not load the weights associated with the model, only the
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
+"""
+
+CLIP_TEXT_INPUTS_DOCSTRING = r"""
+ Args:
+ input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
+ Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
+ it.
+
+ Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
+
+ [What are input IDs?](../glossary#input-ids)
+ attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
+ Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
+
+ - 1 for tokens that are **not masked**,
+ - 0 for tokens that are **masked**.
+
+ [What are attention masks?](../glossary#attention-mask)
+ position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
+
+ [What are position IDs?](../glossary#position-ids)
+ output_attentions (`bool`, *optional*):
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
+ tensors for more detail.
+ output_hidden_states (`bool`, *optional*):
+ Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
+ more detail.
+ return_dict (`bool`, *optional*):
+ Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+"""
+
+CLIP_VISION_INPUTS_DOCSTRING = r"""
+ Args:
+ pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
+ Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using
+ [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details.
+ output_attentions (`bool`, *optional*):
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
+ tensors for more detail.
+ output_hidden_states (`bool`, *optional*):
+ Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
+ more detail.
+ return_dict (`bool`, *optional*):
+ Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+"""
+
+CLIP_INPUTS_DOCSTRING = r"""
+ Args:
+ input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
+ Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
+ it.
+
+ Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
+
+ [What are input IDs?](../glossary#input-ids)
+ attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
+ Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
+
+ - 1 for tokens that are **not masked**,
+ - 0 for tokens that are **masked**.
+
+ [What are attention masks?](../glossary#attention-mask)
+ position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
+
+ [What are position IDs?](../glossary#position-ids)
+ pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
+ Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using
+ [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details.
+ return_loss (`bool`, *optional*):
+ Whether or not to return the contrastive loss.
+ output_attentions (`bool`, *optional*):
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
+ tensors for more detail.
+ output_hidden_states (`bool`, *optional*):
+ Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
+ more detail.
+ return_dict (`bool`, *optional*):
+ Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+"""
+
+
+class CLIPEncoder(nn.Module):
+ """
+ Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
+ [`CLIPEncoderLayer`].
+
+ Args:
+ config: CLIPConfig
+ """
+
+ def __init__(self, config: LanguageBindImageConfig):
+ super().__init__()
+ self.config = config
+ self.layers = nn.ModuleList([CLIPEncoderLayer(config) for _ in range(config.num_hidden_layers)])
+ self.gradient_checkpointing = False
+
+ def forward(
+ self,
+ inputs_embeds,
+ attention_mask: Optional[torch.Tensor] = None,
+ causal_attention_mask: Optional[torch.Tensor] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> Union[Tuple, BaseModelOutput]:
+ r"""
+ Args:
+ inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
+ This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+ than the model's internal embedding lookup matrix.
+ attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
+ Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
+
+ - 1 for tokens that are **not masked**,
+ - 0 for tokens that are **masked**.
+
+ [What are attention masks?](../glossary#attention-mask)
+ causal_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
+ Causal mask for the text model. Mask values selected in `[0, 1]`:
+
+ - 1 for tokens that are **not masked**,
+ - 0 for tokens that are **masked**.
+
+ [What are attention masks?](../glossary#attention-mask)
+ output_attentions (`bool`, *optional*):
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under
+ returned tensors for more detail.
+ output_hidden_states (`bool`, *optional*):
+ Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
+ for more detail.
+ return_dict (`bool`, *optional*):
+ Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+ """
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+ output_hidden_states = (
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+ )
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+ encoder_states = () if output_hidden_states else None
+ all_attentions = () if output_attentions else None
+
+ hidden_states = inputs_embeds
+ for idx, encoder_layer in enumerate(self.layers):
+ if output_hidden_states:
+ encoder_states = encoder_states + (hidden_states,)
+ if self.gradient_checkpointing and self.training:
+
+ def create_custom_forward(module):
+ def custom_forward(*inputs):
+ return module(*inputs, output_attentions)
+
+ return custom_forward
+
+ layer_outputs = torch.utils.checkpoint.checkpoint(
+ create_custom_forward(encoder_layer),
+ hidden_states,
+ attention_mask,
+ causal_attention_mask,
+ )
+ else:
+ layer_outputs = encoder_layer(
+ hidden_states,
+ attention_mask,
+ causal_attention_mask,
+ output_attentions=output_attentions,
+ )
+
+ hidden_states = layer_outputs[0]
+
+ if output_attentions:
+ all_attentions = all_attentions + (layer_outputs[1],)
+
+ if output_hidden_states:
+ encoder_states = encoder_states + (hidden_states,)
+
+ if not return_dict:
+ return tuple(v for v in [hidden_states, encoder_states, all_attentions] if v is not None)
+ return BaseModelOutput(
+ last_hidden_state=hidden_states, hidden_states=encoder_states, attentions=all_attentions
+ )
+
+
+# Copied from transformers.models.bart.modeling_bart._make_causal_mask
+def _make_causal_mask(
+ input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device, past_key_values_length: int = 0
+):
+ """
+ Make causal mask used for bi-directional self-attention.
+ """
+ bsz, tgt_len = input_ids_shape
+ mask = torch.full((tgt_len, tgt_len), torch.finfo(dtype).min, device=device)
+ mask_cond = torch.arange(mask.size(-1), device=device)
+ mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0)
+ mask = mask.to(dtype)
+
+ if past_key_values_length > 0:
+ mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), mask], dim=-1)
+ return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length)
+
+
+class CLIPTextTransformer(nn.Module):
+ def __init__(self, config: CLIPTextConfig):
+ super().__init__()
+ self.config = config
+ embed_dim = config.hidden_size
+ self.embeddings = CLIPTextEmbeddings(config)
+ self.encoder = CLIPEncoder(config)
+ self.final_layer_norm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
+
+ @add_start_docstrings_to_model_forward(CLIP_TEXT_INPUTS_DOCSTRING)
+ @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=CLIPTextConfig)
+ def forward(
+ self,
+ input_ids: Optional[torch.Tensor] = None,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.Tensor] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> Union[Tuple, BaseModelOutputWithPooling]:
+ r"""
+ Returns:
+
+ """
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+ output_hidden_states = (
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+ )
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+ if input_ids is None:
+ raise ValueError("You have to specify input_ids")
+
+ input_shape = input_ids.size()
+ input_ids = input_ids.view(-1, input_shape[-1])
+
+ hidden_states = self.embeddings(input_ids=input_ids, position_ids=position_ids)
+
+ # CLIP's text model uses causal mask, prepare it here.
+ # https://github.com/openai/CLIP/blob/cfcffb90e69f37bf2ff1e988237a0fbe41f33c04/clip/model.py#L324
+ causal_attention_mask = _make_causal_mask(input_shape, hidden_states.dtype, device=hidden_states.device)
+ # expand attention_mask
+ if attention_mask is not None:
+ # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]
+ attention_mask = _expand_mask(attention_mask, hidden_states.dtype)
+
+ encoder_outputs = self.encoder(
+ inputs_embeds=hidden_states,
+ attention_mask=attention_mask,
+ causal_attention_mask=causal_attention_mask,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+
+ last_hidden_state = encoder_outputs[0]
+ last_hidden_state = self.final_layer_norm(last_hidden_state)
+
+ # text_embeds.shape = [batch_size, sequence_length, transformer.width]
+ # take features from the eot embedding (eot_token is the highest number in each sequence)
+ # casting to torch.int for onnx compatibility: argmax doesn't support int64 inputs with opset 14
+ pooled_output = last_hidden_state[
+ torch.arange(last_hidden_state.shape[0], device=last_hidden_state.device),
+ input_ids.to(dtype=torch.int, device=last_hidden_state.device).argmax(dim=-1),
+ ]
+
+ if not return_dict:
+ return (last_hidden_state, pooled_output) + encoder_outputs[1:]
+
+ return BaseModelOutputWithPooling(
+ last_hidden_state=last_hidden_state,
+ pooler_output=pooled_output,
+ hidden_states=encoder_outputs.hidden_states,
+ attentions=encoder_outputs.attentions,
+ )
+
+
+@add_start_docstrings(
+ """The text model from CLIP without any head or projection on top.""",
+ CLIP_START_DOCSTRING,
+)
+class CLIPTextModel(CLIPPreTrainedModel):
+ config_class = CLIPTextConfig
+
+ _no_split_modules = ["CLIPEncoderLayer"]
+
+ def __init__(self, config: CLIPTextConfig):
+ super().__init__(config)
+ self.text_model = CLIPTextTransformer(config)
+ # Initialize weights and apply final processing
+ self.post_init()
+
+ def get_input_embeddings(self) -> nn.Module:
+ return self.text_model.embeddings.token_embedding
+
+ def set_input_embeddings(self, value):
+ self.text_model.embeddings.token_embedding = value
+
+ @add_start_docstrings_to_model_forward(CLIP_TEXT_INPUTS_DOCSTRING)
+ @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=CLIPTextConfig)
+ def forward(
+ self,
+ input_ids: Optional[torch.Tensor] = None,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.Tensor] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> Union[Tuple, BaseModelOutputWithPooling]:
+ r"""
+ Returns:
+
+ Examples:
+
+ ```python
+ >>> from transformers import AutoTokenizer, CLIPTextModel
+
+ >>> model = CLIPTextModel.from_pretrained("openai/clip-vit-base-patch32")
+ >>> tokenizer = AutoTokenizer.from_pretrained("openai/clip-vit-base-patch32")
+
+ >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt")
+
+ >>> outputs = model(**inputs)
+ >>> last_hidden_state = outputs.last_hidden_state
+ >>> pooled_output = outputs.pooler_output # pooled (EOS token) states
+ ```"""
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+ return self.text_model(
+ input_ids=input_ids,
+ attention_mask=attention_mask,
+ position_ids=position_ids,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+
+
+class CLIPVisionTransformer(nn.Module):
+ def __init__(self, config: CLIPVisionConfig):
+ super().__init__()
+ self.config = config
+ embed_dim = config.hidden_size
+
+ self.embeddings = CLIPVisionEmbeddings(config)
+ self.patch_dropout = PatchDropout(config.force_patch_dropout)
+ self.pre_layrnorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
+ self.encoder = CLIPEncoder(config)
+ self.post_layernorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
+
+ @add_start_docstrings_to_model_forward(CLIP_VISION_INPUTS_DOCSTRING)
+ @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=CLIPVisionConfig)
+ def forward(
+ self,
+ pixel_values: Optional[torch.FloatTensor] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> Union[Tuple, BaseModelOutputWithPooling]:
+ r"""
+ Returns:
+
+ """
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+ output_hidden_states = (
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+ )
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+ if pixel_values is None:
+ raise ValueError("You have to specify pixel_values")
+ ######################################
+ if len(pixel_values.shape) == 7:
+ b_new, pair_new, T, bs_new, channel_new, h_new, w_new = pixel_values.shape
+ # print(pixel_values.shape)
+ B = b_new * pair_new * bs_new
+ pixel_values = pixel_values.reshape(B*T, channel_new, h_new, w_new)
+
+ elif len(pixel_values.shape) == 5:
+ B, _, T, _, _ = pixel_values.shape
+ # print(pixel_values.shape)
+ pixel_values = rearrange(pixel_values, 'b c t h w -> (b t) c h w')
+ else:
+ # print(pixel_values.shape)
+ B, _, _, _ = pixel_values.shape
+ T = 1
+ ###########################
+ hidden_states = self.embeddings(pixel_values)
+
+ hidden_states = self.patch_dropout(hidden_states, B, T) ##############################################
+
+ hidden_states = self.pre_layrnorm(hidden_states)
+
+ encoder_outputs = self.encoder(
+ inputs_embeds=hidden_states,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+
+ last_hidden_state = encoder_outputs[0]
+ pooled_output = last_hidden_state[:, 0, :]
+ pooled_output = self.post_layernorm(pooled_output)
+
+ pooled_output = pooled_output.reshape(B, T, -1).mean(1) ################################
+
+ if not return_dict:
+ return (last_hidden_state, pooled_output) + encoder_outputs[1:]
+
+ return BaseModelOutputWithPooling(
+ last_hidden_state=last_hidden_state,
+ pooler_output=pooled_output,
+ hidden_states=encoder_outputs.hidden_states,
+ attentions=encoder_outputs.attentions,
+ )
+
+
+@add_start_docstrings(
+ """The vision model from CLIP without any head or projection on top.""",
+ CLIP_START_DOCSTRING,
+)
+class CLIPVisionModel(CLIPPreTrainedModel):
+ config_class = CLIPVisionConfig
+ main_input_name = "pixel_values"
+
+ def __init__(self, config: CLIPVisionConfig):
+ super().__init__(config)
+ self.vision_model = CLIPVisionTransformer(config)
+ # Initialize weights and apply final processing
+ self.post_init()
+
+ def get_input_embeddings(self) -> nn.Module:
+ return self.vision_model.embeddings.patch_embedding
+
+ @add_start_docstrings_to_model_forward(CLIP_VISION_INPUTS_DOCSTRING)
+ @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=CLIPVisionConfig)
+ def forward(
+ self,
+ pixel_values: Optional[torch.FloatTensor] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> Union[Tuple, BaseModelOutputWithPooling]:
+ r"""
+ Returns:
+
+ Examples:
+
+ ```python
+ >>> from PIL import Image
+ >>> import requests
+ >>> from transformers import AutoProcessor, CLIPVisionModel
+
+ >>> model = CLIPVisionModel.from_pretrained("openai/clip-vit-base-patch32")
+ >>> processor = AutoProcessor.from_pretrained("openai/clip-vit-base-patch32")
+
+ >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
+ >>> image = Image.open(requests.get(url, stream=True).raw)
+
+ >>> inputs = processor(images=image, return_tensors="pt")
+
+ >>> outputs = model(**inputs)
+ >>> last_hidden_state = outputs.last_hidden_state
+ >>> pooled_output = outputs.pooler_output # pooled CLS states
+ ```"""
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+ return self.vision_model(
+ pixel_values=pixel_values,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+
+
+@add_start_docstrings(CLIP_START_DOCSTRING)
+class LanguageBindImage(CLIPPreTrainedModel):
+ config_class = LanguageBindImageConfig
+
+ def __init__(self, config: LanguageBindImageConfig):
+ super().__init__(config)
+
+ if not isinstance(config.text_config, CLIPTextConfig):
+ raise ValueError(
+ "config.text_config is expected to be of type CLIPTextConfig but is of type"
+ f" {type(config.text_config)}."
+ )
+
+ if not isinstance(config.vision_config, CLIPVisionConfig):
+ raise ValueError(
+ "config.vision_config is expected to be of type CLIPVisionConfig but is of type"
+ f" {type(config.vision_config)}."
+ )
+
+ text_config = config.text_config
+ vision_config = config.vision_config
+ self.add_time_attn = vision_config.add_time_attn
+ self.lora_r = vision_config.lora_r
+ self.lora_alpha = vision_config.lora_alpha
+ self.lora_dropout = vision_config.lora_dropout
+
+ self.projection_dim = config.projection_dim
+ self.text_embed_dim = text_config.hidden_size
+ self.vision_embed_dim = vision_config.hidden_size
+
+ self.text_model = CLIPTextTransformer(text_config)
+ self.vision_model = CLIPVisionTransformer(vision_config)
+
+ self.visual_projection = nn.Linear(self.vision_embed_dim, self.projection_dim, bias=False)
+ self.text_projection = nn.Linear(self.text_embed_dim, self.projection_dim, bias=False)
+ self.logit_scale = nn.Parameter(torch.tensor(self.config.logit_scale_init_value))
+
+ # Initialize weights and apply final processing
+ self.post_init()
+ self.convert_to_lora()
+ # self.resize_pos(self.vision_model.embeddings, vision_config)
+
+ def convert_to_lora(self):
+ if self.lora_r == 0:
+ return
+ if self.add_time_attn:
+ target_modules = ["temporal_attn.k_proj", "temporal_attn.v_proj",
+ "temporal_attn.q_proj", "temporal_attn.out_proj",
+ "temporal_mlp.fc1", "temporal_mlp.fc2"]
+ else:
+ target_modules = ["k_proj", "v_proj", "q_proj", "out_proj"]
+ config = LoraConfig(
+ r=self.lora_r, # 16
+ lora_alpha=self.lora_alpha, # 16
+ target_modules=target_modules, # self_attn.out_proj
+ lora_dropout=self.lora_dropout, # 0.1
+ bias="none",
+ modules_to_save=[],
+ )
+ self.vision_model.encoder.is_gradient_checkpointing = False
+ self.vision_model.encoder = get_peft_model(self.vision_model.encoder, config)
+
+ def resize_pos(self, m, vision_config):
+ # convert embedding
+ if vision_config.num_mel_bins!=0 and vision_config.target_length!=0:
+ m.image_size = [vision_config.num_mel_bins, vision_config.target_length]
+ m.config.image_size = [m.image_size, m.image_size] if isinstance(m.image_size, int) else m.image_size
+ # pos resize
+ old_pos_embed_state_dict = m.position_embedding.state_dict()
+ old_pos_embed = old_pos_embed_state_dict['weight']
+ dtype = old_pos_embed.dtype
+ grid_size = [m.config.image_size[0] // m.patch_size, m.config.image_size[1] // m.patch_size]
+ extra_tokens = 1 # FIXME detect different token configs (ie no class token, or more)
+ new_seq_len = grid_size[0] * grid_size[1] + extra_tokens
+ if new_seq_len == old_pos_embed.shape[0]:
+ # m.to(args.device)
+ return
+
+ m.num_patches = grid_size[0] * grid_size[1]
+ m.num_positions = m.num_patches + 1
+ m.register_buffer("position_ids", torch.arange(m.num_positions).expand((1, -1)))
+ new_position_embedding = nn.Embedding(m.num_positions, m.embed_dim)
+
+ if extra_tokens:
+ pos_emb_tok, pos_emb_img = old_pos_embed[:extra_tokens], old_pos_embed[extra_tokens:]
+ else:
+ pos_emb_tok, pos_emb_img = None, old_pos_embed
+ old_grid_size = [int(math.sqrt(len(pos_emb_img)))] * 2
+
+ # if is_master(args):
+ # logging.info('Resizing position embedding grid-size from %s to %s', old_grid_size, grid_size)
+ pos_emb_img = pos_emb_img.reshape(1, old_grid_size[0], old_grid_size[1], -1).permute(0, 3, 1, 2)
+ pos_emb_img = F.interpolate(
+ pos_emb_img,
+ size=grid_size,
+ mode='bicubic',
+ antialias=True,
+ align_corners=False,
+ )
+ pos_emb_img = pos_emb_img.permute(0, 2, 3, 1).reshape(1, grid_size[0] * grid_size[1], -1)[0]
+ if pos_emb_tok is not None:
+ new_pos_embed = torch.cat([pos_emb_tok, pos_emb_img], dim=0)
+ else:
+ new_pos_embed = pos_emb_img
+ old_pos_embed_state_dict['weight'] = new_pos_embed.to(dtype)
+ m.position_embedding = new_position_embedding
+ m.position_embedding.load_state_dict(old_pos_embed_state_dict)
+
+ # m.to(args.device)
+
+ @add_start_docstrings_to_model_forward(CLIP_TEXT_INPUTS_DOCSTRING)
+ def get_text_features(
+ self,
+ input_ids: Optional[torch.Tensor] = None,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.Tensor] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> torch.FloatTensor:
+ r"""
+ Returns:
+ text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by
+ applying the projection layer to the pooled output of [`CLIPTextModel`].
+
+ Examples:
+
+ ```python
+ >>> from transformers import AutoTokenizer, CLIPModel
+
+ >>> model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")
+ >>> tokenizer = AutoTokenizer.from_pretrained("openai/clip-vit-base-patch32")
+
+ >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt")
+ >>> text_features = model.get_text_features(**inputs)
+ ```"""
+ # Use CLIP model's config for some fields (if specified) instead of those of vision & text components.
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+ output_hidden_states = (
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+ )
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+ text_outputs = self.text_model(
+ input_ids=input_ids,
+ attention_mask=attention_mask,
+ position_ids=position_ids,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+
+ pooled_output = text_outputs[1]
+ text_features = self.text_projection(pooled_output)
+
+ return text_features
+
+ @add_start_docstrings_to_model_forward(CLIP_VISION_INPUTS_DOCSTRING)
+ def get_image_features(
+ self,
+ pixel_values: Optional[torch.FloatTensor] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> torch.FloatTensor:
+ r"""
+ Returns:
+ image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by
+ applying the projection layer to the pooled output of [`CLIPVisionModel`].
+
+ Examples:
+
+ ```python
+ >>> from PIL import Image
+ >>> import requests
+ >>> from transformers import AutoProcessor, CLIPModel
+
+ >>> model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")
+ >>> processor = AutoProcessor.from_pretrained("openai/clip-vit-base-patch32")
+
+ >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
+ >>> image = Image.open(requests.get(url, stream=True).raw)
+
+ >>> inputs = processor(images=image, return_tensors="pt")
+
+ >>> image_features = model.get_image_features(**inputs)
+ ```"""
+ # Use CLIP model's config for some fields (if specified) instead of those of vision & text components.
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+ output_hidden_states = (
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+ )
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+ vision_outputs = self.vision_model(
+ pixel_values=pixel_values,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+
+ pooled_output = vision_outputs[1] # pooled_output
+ image_features = self.visual_projection(pooled_output)
+
+ return image_features
+
+ @add_start_docstrings_to_model_forward(CLIP_INPUTS_DOCSTRING)
+ @replace_return_docstrings(output_type=CLIPOutput, config_class=LanguageBindImageConfig)
+ def forward(
+ self,
+ input_ids: Optional[torch.LongTensor] = None,
+ pixel_values: Optional[torch.FloatTensor] = None,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.LongTensor] = None,
+ return_loss: Optional[bool] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> Union[Tuple, CLIPOutput]:
+ r"""
+ Returns:
+
+ Examples:
+
+ ```python
+ >>> from PIL import Image
+ >>> import requests
+ >>> from transformers import AutoProcessor, CLIPModel
+
+ >>> model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")
+ >>> processor = AutoProcessor.from_pretrained("openai/clip-vit-base-patch32")
+
+ >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
+ >>> image = Image.open(requests.get(url, stream=True).raw)
+
+ >>> inputs = processor(
+ ... text=["a photo of a cat", "a photo of a dog"], images=image, return_tensors="pt", padding=True
+ ... )
+
+ >>> outputs = model(**inputs)
+ >>> logits_per_image = outputs.logits_per_image # this is the image-text similarity score
+ >>> probs = logits_per_image.softmax(dim=1) # we can take the softmax to get the label probabilities
+ ```"""
+ # Use CLIP model's config for some fields (if specified) instead of those of vision & text components.
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+ output_hidden_states = (
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+ )
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+ vision_outputs = self.vision_model(
+ pixel_values=pixel_values,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+
+ text_outputs = self.text_model(
+ input_ids=input_ids,
+ attention_mask=attention_mask,
+ position_ids=position_ids,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+
+ image_embeds = vision_outputs[1]
+ image_embeds = self.visual_projection(image_embeds)
+
+ text_embeds = text_outputs[1]
+ text_embeds = self.text_projection(text_embeds)
+
+ # normalized features
+ image_embeds = image_embeds / image_embeds.norm(p=2, dim=-1, keepdim=True)
+ text_embeds = text_embeds / text_embeds.norm(p=2, dim=-1, keepdim=True)
+
+ # cosine similarity as logits
+ logit_scale = self.logit_scale.exp()
+ logits_per_text = torch.matmul(text_embeds, image_embeds.t()) * logit_scale
+ logits_per_image = logits_per_text.t()
+
+ loss = None
+ if return_loss:
+ loss = clip_loss(logits_per_text)
+
+ if not return_dict:
+ output = (logits_per_image, logits_per_text, text_embeds, image_embeds, text_outputs, vision_outputs)
+ return ((loss,) + output) if loss is not None else output
+
+ return CLIPOutput(
+ loss=loss,
+ logits_per_image=logits_per_image,
+ logits_per_text=logits_per_text,
+ text_embeds=text_embeds,
+ image_embeds=image_embeds,
+ text_model_output=text_outputs,
+ vision_model_output=vision_outputs,
+ )
\ No newline at end of file
diff --git a/videollava/model/multimodal_encoder/languagebind/image/processing_image.py b/videollava/model/multimodal_encoder/languagebind/image/processing_image.py
new file mode 100644
index 0000000000000000000000000000000000000000..1aafc79d8b17f2b2a45dd0eeb0700673d2521080
--- /dev/null
+++ b/videollava/model/multimodal_encoder/languagebind/image/processing_image.py
@@ -0,0 +1,82 @@
+import torch
+from PIL import Image
+from torchvision import transforms
+from transformers import ProcessorMixin, BatchEncoding
+from transformers.image_processing_utils import BatchFeature
+
+OPENAI_DATASET_MEAN = (0.48145466, 0.4578275, 0.40821073)
+OPENAI_DATASET_STD = (0.26862954, 0.26130258, 0.27577711)
+
+def make_list_of_images(x):
+ if not isinstance(x, list):
+ return [x]
+ return x
+
+def get_image_transform(config):
+ config = config.vision_config
+ transform = transforms.Compose(
+ [
+ transforms.ToTensor(),
+ transforms.Resize(224, interpolation=transforms.InterpolationMode.BICUBIC),
+ transforms.CenterCrop(224),
+ transforms.Normalize(OPENAI_DATASET_MEAN, OPENAI_DATASET_STD) # assume image
+ ]
+ )
+ return transform
+
+
+def load_and_transform_image(image_path, transform):
+ image = Image.open(image_path).convert('RGB') if isinstance(image_path, str) else image_path
+ image_outputs = transform(image)
+ return image_outputs
+
+class LanguageBindImageProcessor(ProcessorMixin):
+ attributes = []
+ tokenizer_class = ("LanguageBindImageTokenizer")
+
+ def __init__(self, config, tokenizer=None, **kwargs):
+ super().__init__(**kwargs)
+ self.config = config
+ self.transform = get_image_transform(config)
+ self.image_processor = load_and_transform_image
+ self.tokenizer = tokenizer
+ self.image_mean = OPENAI_DATASET_MEAN
+ self.crop_size = {'height': 224, 'width': 224}
+
+ def __call__(self, images=None, text=None, context_length=77, return_tensors=None, **kwargs):
+ if text is None and images is None:
+ raise ValueError("You have to specify either text or images. Both cannot be none.")
+
+ if text is not None:
+ encoding = self.tokenizer(text, max_length=context_length, padding='max_length',
+ truncation=True, return_tensors=return_tensors, **kwargs)
+
+ if images is not None:
+ images = make_list_of_images(images)
+ image_features = [self.image_processor(image, self.transform) for image in images]
+ image_features = torch.stack(image_features)
+
+ if text is not None and images is not None:
+ encoding["pixel_values"] = image_features
+ return encoding
+ elif text is not None:
+ return encoding
+ else:
+ return {"pixel_values": image_features}
+
+ def preprocess(self, images, return_tensors):
+ return self.__call__(images=images, return_tensors=return_tensors)
+
+ def batch_decode(self, skip_special_tokens=True, *args, **kwargs):
+ """
+ This method forwards all its arguments to CLIPTokenizerFast's [`~PreTrainedTokenizer.batch_decode`]. Please
+ refer to the docstring of this method for more information.
+ """
+ return self.tokenizer.batch_decode(*args, skip_special_tokens=skip_special_tokens, **kwargs)
+
+ def decode(self, skip_special_tokens=True, *args, **kwargs):
+ """
+ This method forwards all its arguments to CLIPTokenizerFast's [`~PreTrainedTokenizer.decode`]. Please refer to
+ the docstring of this method for more information.
+ """
+ return self.tokenizer.decode(*args, skip_special_tokens=skip_special_tokens, **kwargs)
diff --git a/videollava/model/multimodal_encoder/languagebind/image/tokenization_image.py b/videollava/model/multimodal_encoder/languagebind/image/tokenization_image.py
new file mode 100644
index 0000000000000000000000000000000000000000..593423d089100b3d61957f658cca04b541336f65
--- /dev/null
+++ b/videollava/model/multimodal_encoder/languagebind/image/tokenization_image.py
@@ -0,0 +1,77 @@
+from transformers import CLIPTokenizer
+from transformers.utils import logging
+
+logger = logging.get_logger(__name__)
+
+VOCAB_FILES_NAMES = {
+ "vocab_file": "vocab.json",
+ "merges_file": "merges.txt",
+}
+
+PRETRAINED_VOCAB_FILES_MAP = {
+ "vocab_file": {
+ "lb203/LanguageBind-Image": "https://huggingface.co/lb203/LanguageBind-Image/resolve/main/vocab.json",
+ },
+ "merges_file": {
+ "lb203/LanguageBind-Image": "https://huggingface.co/lb203/LanguageBind-Image/resolve/main/merges.txt",
+ },
+}
+
+PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
+ "lb203/LanguageBind-Image": 77,
+}
+
+
+PRETRAINED_INIT_CONFIGURATION = {
+ "lb203/LanguageBind-Image": {},
+}
+
+class LanguageBindImageTokenizer(CLIPTokenizer):
+ """
+ Construct a CLIP tokenizer. Based on byte-level Byte-Pair-Encoding.
+
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+ this superclass for more information regarding those methods.
+
+ Args:
+ vocab_file (`str`):
+ Path to the vocabulary file.
+ merges_file (`str`):
+ Path to the merges file.
+ errors (`str`, *optional*, defaults to `"replace"`):
+ Paradigm to follow when decoding bytes to UTF-8. See
+ [bytes.decode](https://docs.python.org/3/library/stdtypes.html#bytes.decode) for more information.
+ unk_token (`str`, *optional*, defaults to `<|endoftext|>`):
+ The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
+ token instead.
+ bos_token (`str`, *optional*, defaults to `<|startoftext|>`):
+ The beginning of sequence token.
+ eos_token (`str`, *optional*, defaults to `<|endoftext|>`):
+ The end of sequence token.
+ """
+
+ vocab_files_names = VOCAB_FILES_NAMES
+ pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+ max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
+ model_input_names = ["input_ids", "attention_mask"]
+
+ def __init__(
+ self,
+ vocab_file,
+ merges_file,
+ errors="replace",
+ unk_token="<|endoftext|>",
+ bos_token="<|startoftext|>",
+ eos_token="<|endoftext|>",
+ pad_token="<|endoftext|>", # hack to enable padding
+ **kwargs,
+ ):
+ super(LanguageBindImageTokenizer, self).__init__(
+ vocab_file,
+ merges_file,
+ errors,
+ unk_token,
+ bos_token,
+ eos_token,
+ pad_token, # hack to enable padding
+ **kwargs,)
\ No newline at end of file
diff --git a/videollava/model/multimodal_encoder/languagebind/thermal/configuration_thermal.py b/videollava/model/multimodal_encoder/languagebind/thermal/configuration_thermal.py
new file mode 100644
index 0000000000000000000000000000000000000000..fd6cedd5d44c248b32e89f51d5c28595bffcbefc
--- /dev/null
+++ b/videollava/model/multimodal_encoder/languagebind/thermal/configuration_thermal.py
@@ -0,0 +1,423 @@
+import copy
+import os
+from typing import Union
+
+from transformers import PretrainedConfig
+from transformers.utils import logging
+
+logger = logging.get_logger(__name__)
+
+
+
+
+
+
+
+class CLIPTextConfig(PretrainedConfig):
+ r"""
+ This is the configuration class to store the configuration of a [`CLIPTextModel`]. It is used to instantiate a CLIP
+ text encoder according to the specified arguments, defining the model architecture. Instantiating a configuration
+ with the defaults will yield a similar configuration to that of the text encoder of the CLIP
+ [openai/clip-vit-base-patch32](https://huggingface.co/openai/clip-vit-base-patch32) architecture.
+
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
+
+ Args:
+ vocab_size (`int`, *optional*, defaults to 49408):
+ Vocabulary size of the CLIP text model. Defines the number of different tokens that can be represented by
+ the `inputs_ids` passed when calling [`CLIPModel`].
+ hidden_size (`int`, *optional*, defaults to 512):
+ Dimensionality of the encoder layers and the pooler layer.
+ intermediate_size (`int`, *optional*, defaults to 2048):
+ Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
+ num_hidden_layers (`int`, *optional*, defaults to 12):
+ Number of hidden layers in the Transformer encoder.
+ num_attention_heads (`int`, *optional*, defaults to 8):
+ Number of attention heads for each attention layer in the Transformer encoder.
+ max_position_embeddings (`int`, *optional*, defaults to 77):
+ The maximum sequence length that this model might ever be used with. Typically set this to something large
+ just in case (e.g., 512 or 1024 or 2048).
+ hidden_act (`str` or `function`, *optional*, defaults to `"quick_gelu"`):
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"selu"` and `"gelu_new"` `"quick_gelu"` are supported.
+ layer_norm_eps (`float`, *optional*, defaults to 1e-5):
+ The epsilon used by the layer normalization layers.
+ attention_dropout (`float`, *optional*, defaults to 0.0):
+ The dropout ratio for the attention probabilities.
+ initializer_range (`float`, *optional*, defaults to 0.02):
+ The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+ initializer_factor (`float`, *optional*, defaults to 1):
+ A factor for initializing all weight matrices (should be kept to 1, used internally for initialization
+ testing).
+
+ Example:
+
+ ```python
+ >>> from transformers import CLIPTextConfig, CLIPTextModel
+
+ >>> # Initializing a CLIPTextConfig with openai/clip-vit-base-patch32 style configuration
+ >>> configuration = CLIPTextConfig()
+
+ >>> # Initializing a CLIPTextModel (with random weights) from the openai/clip-vit-base-patch32 style configuration
+ >>> model = CLIPTextModel(configuration)
+
+ >>> # Accessing the model configuration
+ >>> configuration = model.config
+ ```"""
+ model_type = "clip_text_model"
+
+ def __init__(
+ self,
+ vocab_size=49408,
+ hidden_size=512,
+ intermediate_size=2048,
+ projection_dim=512,
+ num_hidden_layers=12,
+ num_attention_heads=8,
+ max_position_embeddings=77,
+ hidden_act="quick_gelu",
+ layer_norm_eps=1e-5,
+ attention_dropout=0.0,
+ initializer_range=0.02,
+ initializer_factor=1.0,
+ # This differs from `CLIPTokenizer`'s default and from openai/clip
+ # See https://github.com/huggingface/transformers/pull/24773#issuecomment-1632287538
+ pad_token_id=1,
+ bos_token_id=49406,
+ eos_token_id=49407,
+ **kwargs,
+ ):
+ super().__init__(pad_token_id=pad_token_id, bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs)
+
+ self.vocab_size = vocab_size
+ self.hidden_size = hidden_size
+ self.intermediate_size = intermediate_size
+ self.projection_dim = projection_dim
+ self.num_hidden_layers = num_hidden_layers
+ self.num_attention_heads = num_attention_heads
+ self.max_position_embeddings = max_position_embeddings
+ self.layer_norm_eps = layer_norm_eps
+ self.hidden_act = hidden_act
+ self.initializer_range = initializer_range
+ self.initializer_factor = initializer_factor
+ self.attention_dropout = attention_dropout
+ self.add_time_attn = False ######################################
+
+ @classmethod
+ def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":
+ cls._set_token_in_kwargs(kwargs)
+
+ config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)
+
+ # get the text config dict if we are loading from CLIPConfig
+ if config_dict.get("model_type") == "clip":
+ config_dict = config_dict["text_config"]
+
+ if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:
+ logger.warning(
+ f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
+ f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
+ )
+
+ return cls.from_dict(config_dict, **kwargs)
+
+
+
+
+class CLIPVisionConfig(PretrainedConfig):
+ r"""
+ This is the configuration class to store the configuration of a [`CLIPVisionModel`]. It is used to instantiate a
+ CLIP vision encoder according to the specified arguments, defining the model architecture. Instantiating a
+ configuration with the defaults will yield a similar configuration to that of the vision encoder of the CLIP
+ [openai/clip-vit-base-patch32](https://huggingface.co/openai/clip-vit-base-patch32) architecture.
+
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
+
+ Args:
+ hidden_size (`int`, *optional*, defaults to 768):
+ Dimensionality of the encoder layers and the pooler layer.
+ intermediate_size (`int`, *optional*, defaults to 3072):
+ Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
+ num_hidden_layers (`int`, *optional*, defaults to 12):
+ Number of hidden layers in the Transformer encoder.
+ num_attention_heads (`int`, *optional*, defaults to 12):
+ Number of attention heads for each attention layer in the Transformer encoder.
+ image_size (`int`, *optional*, defaults to 224):
+ The size (resolution) of each image.
+ patch_size (`int`, *optional*, defaults to 32):
+ The size (resolution) of each patch.
+ hidden_act (`str` or `function`, *optional*, defaults to `"quick_gelu"`):
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"selu"` and `"gelu_new"` ``"quick_gelu"` are supported.
+ layer_norm_eps (`float`, *optional*, defaults to 1e-5):
+ The epsilon used by the layer normalization layers.
+ attention_dropout (`float`, *optional*, defaults to 0.0):
+ The dropout ratio for the attention probabilities.
+ initializer_range (`float`, *optional*, defaults to 0.02):
+ The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+ initializer_factor (`float`, *optional*, defaults to 1):
+ A factor for initializing all weight matrices (should be kept to 1, used internally for initialization
+ testing).
+
+ Example:
+
+ ```python
+ >>> from transformers import CLIPVisionConfig, CLIPVisionModel
+
+ >>> # Initializing a CLIPVisionConfig with openai/clip-vit-base-patch32 style configuration
+ >>> configuration = CLIPVisionConfig()
+
+ >>> # Initializing a CLIPVisionModel (with random weights) from the openai/clip-vit-base-patch32 style configuration
+ >>> model = CLIPVisionModel(configuration)
+
+ >>> # Accessing the model configuration
+ >>> configuration = model.config
+ ```"""
+
+ model_type = "clip_vision_model"
+
+ def __init__(
+ self,
+ hidden_size=768,
+ intermediate_size=3072,
+ projection_dim=512,
+ num_hidden_layers=12,
+ num_attention_heads=12,
+ num_channels=3,
+ image_size=224,
+ patch_size=32,
+ hidden_act="quick_gelu",
+ layer_norm_eps=1e-5,
+ attention_dropout=0.0,
+ initializer_range=0.02,
+ initializer_factor=1.0,
+
+ add_time_attn=False, ################################
+ num_frames=1, ################################
+ force_patch_dropout=0.0, ################################
+ lora_r=2, ################################
+ lora_alpha=16, ################################
+ lora_dropout=0.0, ################################
+ num_mel_bins=0.0, ################################
+ target_length=0.0, ################################
+ video_decode_backend='decord', #########################
+ **kwargs,
+ ):
+ super().__init__(**kwargs)
+
+ self.hidden_size = hidden_size
+ self.intermediate_size = intermediate_size
+ self.projection_dim = projection_dim
+ self.num_hidden_layers = num_hidden_layers
+ self.num_attention_heads = num_attention_heads
+ self.num_channels = num_channels
+ self.patch_size = patch_size
+ self.image_size = image_size
+ self.initializer_range = initializer_range
+ self.initializer_factor = initializer_factor
+ self.attention_dropout = attention_dropout
+ self.layer_norm_eps = layer_norm_eps
+ self.hidden_act = hidden_act
+
+ self.add_time_attn = add_time_attn ################
+ self.num_frames = num_frames ################
+ self.force_patch_dropout = force_patch_dropout ################
+ self.lora_r = lora_r ################
+ self.lora_alpha = lora_alpha ################
+ self.lora_dropout = lora_dropout ################
+ self.num_mel_bins = num_mel_bins ################
+ self.target_length = target_length ################
+ self.video_decode_backend = video_decode_backend ################
+
+ @classmethod
+ def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":
+ cls._set_token_in_kwargs(kwargs)
+
+ config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)
+
+ # get the vision config dict if we are loading from CLIPConfig
+ if config_dict.get("model_type") == "clip":
+ config_dict = config_dict["vision_config"]
+
+ if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:
+ logger.warning(
+ f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
+ f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
+ )
+
+ return cls.from_dict(config_dict, **kwargs)
+
+
+class LanguageBindThermalConfig(PretrainedConfig):
+ r"""
+ [`CLIPConfig`] is the configuration class to store the configuration of a [`CLIPModel`]. It is used to instantiate
+ a CLIP model according to the specified arguments, defining the text model and vision model configs. Instantiating
+ a configuration with the defaults will yield a similar configuration to that of the CLIP
+ [openai/clip-vit-base-patch32](https://huggingface.co/openai/clip-vit-base-patch32) architecture.
+
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
+
+ Args:
+ text_config (`dict`, *optional*):
+ Dictionary of configuration options used to initialize [`CLIPTextConfig`].
+ vision_config (`dict`, *optional*):
+ Dictionary of configuration options used to initialize [`CLIPVisionConfig`].
+ projection_dim (`int`, *optional*, defaults to 512):
+ Dimentionality of text and vision projection layers.
+ logit_scale_init_value (`float`, *optional*, defaults to 2.6592):
+ The inital value of the *logit_scale* paramter. Default is used as per the original CLIP implementation.
+ kwargs (*optional*):
+ Dictionary of keyword arguments.
+
+ Example:
+
+ ```python
+ >>> from transformers import CLIPConfig, CLIPModel
+
+ >>> # Initializing a CLIPConfig with openai/clip-vit-base-patch32 style configuration
+ >>> configuration = CLIPConfig()
+
+ >>> # Initializing a CLIPModel (with random weights) from the openai/clip-vit-base-patch32 style configuration
+ >>> model = CLIPModel(configuration)
+
+ >>> # Accessing the model configuration
+ >>> configuration = model.config
+
+ >>> # We can also initialize a CLIPConfig from a CLIPTextConfig and a CLIPVisionConfig
+ >>> from transformers import CLIPTextConfig, CLIPVisionConfig
+
+ >>> # Initializing a CLIPText and CLIPVision configuration
+ >>> config_text = CLIPTextConfig()
+ >>> config_vision = CLIPVisionConfig()
+
+ >>> config = CLIPConfig.from_text_vision_configs(config_text, config_vision)
+ ```"""
+
+ model_type = "LanguageBindThermal"
+ is_composition = True
+
+ def __init__(
+ self, text_config=None, vision_config=None, projection_dim=512, logit_scale_init_value=2.6592, **kwargs
+ ):
+ # If `_config_dict` exist, we use them for the backward compatibility.
+ # We pop out these 2 attributes before calling `super().__init__` to avoid them being saved (which causes a lot
+ # of confusion!).
+ text_config_dict = kwargs.pop("text_config_dict", None)
+ vision_config_dict = kwargs.pop("vision_config_dict", None)
+
+ super().__init__(**kwargs)
+
+ # Instead of simply assigning `[text|vision]_config_dict` to `[text|vision]_config`, we use the values in
+ # `[text|vision]_config_dict` to update the values in `[text|vision]_config`. The values should be same in most
+ # cases, but we don't want to break anything regarding `_config_dict` that existed before commit `8827e1b2`.
+ if text_config_dict is not None:
+ if text_config is None:
+ text_config = {}
+
+ # This is the complete result when using `text_config_dict`.
+ _text_config_dict = CLIPTextConfig(**text_config_dict).to_dict()
+
+ # Give a warning if the values exist in both `_text_config_dict` and `text_config` but being different.
+ for key, value in _text_config_dict.items():
+ if key in text_config and value != text_config[key] and key not in ["transformers_version"]:
+ # If specified in `text_config_dict`
+ if key in text_config_dict:
+ message = (
+ f"`{key}` is found in both `text_config_dict` and `text_config` but with different values. "
+ f'The value `text_config_dict["{key}"]` will be used instead.'
+ )
+ # If inferred from default argument values (just to be super careful)
+ else:
+ message = (
+ f"`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The "
+ f'value `text_config["{key}"]` will be overriden.'
+ )
+ logger.warning(message)
+
+ # Update all values in `text_config` with the ones in `_text_config_dict`.
+ text_config.update(_text_config_dict)
+
+ if vision_config_dict is not None:
+ if vision_config is None:
+ vision_config = {}
+
+ # This is the complete result when using `vision_config_dict`.
+ _vision_config_dict = CLIPVisionConfig(**vision_config_dict).to_dict()
+ # convert keys to string instead of integer
+ if "id2label" in _vision_config_dict:
+ _vision_config_dict["id2label"] = {
+ str(key): value for key, value in _vision_config_dict["id2label"].items()
+ }
+
+ # Give a warning if the values exist in both `_vision_config_dict` and `vision_config` but being different.
+ for key, value in _vision_config_dict.items():
+ if key in vision_config and value != vision_config[key] and key not in ["transformers_version"]:
+ # If specified in `vision_config_dict`
+ if key in vision_config_dict:
+ message = (
+ f"`{key}` is found in both `vision_config_dict` and `vision_config` but with different "
+ f'values. The value `vision_config_dict["{key}"]` will be used instead.'
+ )
+ # If inferred from default argument values (just to be super careful)
+ else:
+ message = (
+ f"`vision_config_dict` is provided which will be used to initialize `CLIPVisionConfig`. "
+ f'The value `vision_config["{key}"]` will be overriden.'
+ )
+ logger.warning(message)
+
+ # Update all values in `vision_config` with the ones in `_vision_config_dict`.
+ vision_config.update(_vision_config_dict)
+
+ if text_config is None:
+ text_config = {}
+ logger.info("`text_config` is `None`. Initializing the `CLIPTextConfig` with default values.")
+
+ if vision_config is None:
+ vision_config = {}
+ logger.info("`vision_config` is `None`. initializing the `CLIPVisionConfig` with default values.")
+
+ self.text_config = CLIPTextConfig(**text_config)
+ self.vision_config = CLIPVisionConfig(**vision_config)
+
+ self.projection_dim = projection_dim
+ self.logit_scale_init_value = logit_scale_init_value
+ self.initializer_factor = 1.0
+
+ @classmethod
+ def from_text_vision_configs(cls, text_config: CLIPTextConfig, vision_config: CLIPVisionConfig, **kwargs):
+ r"""
+ Instantiate a [`CLIPConfig`] (or a derived class) from clip text model configuration and clip vision model
+ configuration.
+
+ Returns:
+ [`CLIPConfig`]: An instance of a configuration object
+ """
+
+ return cls(text_config=text_config.to_dict(), vision_config=vision_config.to_dict(), **kwargs)
+
+ def to_dict(self):
+ """
+ Serializes this instance to a Python dictionary. Override the default [`~PretrainedConfig.to_dict`].
+
+ Returns:
+ `Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance,
+ """
+ output = copy.deepcopy(self.__dict__)
+ output["text_config"] = self.text_config.to_dict()
+ output["vision_config"] = self.vision_config.to_dict()
+ output["model_type"] = self.__class__.model_type
+ return output
+
+
+
+
+
+
+
+
+
+
diff --git a/videollava/model/multimodal_encoder/languagebind/thermal/modeling_thermal.py b/videollava/model/multimodal_encoder/languagebind/thermal/modeling_thermal.py
new file mode 100644
index 0000000000000000000000000000000000000000..f0323b3351a4eed0165a8b7a1e8cc610ea0669ca
--- /dev/null
+++ b/videollava/model/multimodal_encoder/languagebind/thermal/modeling_thermal.py
@@ -0,0 +1,1030 @@
+import math
+from typing import Optional, Tuple, Union
+
+import torch
+from einops import rearrange
+from peft import LoraConfig, get_peft_model
+from torch import nn
+from torch.nn import functional as F
+from transformers import PreTrainedModel, add_start_docstrings
+from transformers.modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling
+from transformers.models.clip.modeling_clip import CLIPMLP, CLIPAttention, CLIPTextEmbeddings, CLIPVisionEmbeddings, \
+ CLIPVisionModelWithProjection, CLIPTextModelWithProjection, _expand_mask, CLIPOutput, clip_loss
+from transformers.utils import add_start_docstrings_to_model_forward, replace_return_docstrings
+
+from .configuration_thermal import LanguageBindThermalConfig, CLIPVisionConfig, CLIPTextConfig
+
+
+
+class PatchDropout(nn.Module):
+ """
+ https://arxiv.org/abs/2212.00794
+ """
+
+ def __init__(self, prob, exclude_first_token=True):
+ super().__init__()
+ assert 0 <= prob < 1.
+ self.prob = prob
+ self.exclude_first_token = exclude_first_token # exclude CLS token
+
+ def forward(self, x, B, T):
+ if not self.training or self.prob == 0.:
+ return x
+
+ if self.exclude_first_token:
+ cls_tokens, x = x[:, :1], x[:, 1:]
+ else:
+ cls_tokens = torch.jit.annotate(torch.Tensor, x[:, :1])
+
+ batch = x.size()[0]
+ num_tokens = x.size()[1]
+
+ batch_indices = torch.arange(batch)
+ batch_indices = batch_indices[..., None]
+
+ keep_prob = 1 - self.prob
+ num_patches_keep = max(1, int(num_tokens * keep_prob))
+
+ if T == 1:
+ rand = torch.randn(batch, num_tokens)
+ patch_indices_keep = rand.topk(num_patches_keep, dim=-1).indices
+ else:
+ rand = torch.randn(B, num_tokens)
+ patch_indices_keep = rand.topk(num_patches_keep, dim=-1).indices
+ patch_indices_keep = patch_indices_keep.unsqueeze(1).repeat(1, T, 1)
+ patch_indices_keep = rearrange(patch_indices_keep, 'b t n -> (b t) n')
+
+
+ x = x[batch_indices, patch_indices_keep]
+
+ if self.exclude_first_token:
+ x = torch.cat((cls_tokens, x), dim=1)
+
+ return x
+
+class CLIPEncoderLayer(nn.Module):
+ def __init__(self, config: LanguageBindThermalConfig):
+ super().__init__()
+ self.embed_dim = config.hidden_size
+ self.self_attn = CLIPAttention(config)
+ self.layer_norm1 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
+ self.mlp = CLIPMLP(config)
+ self.layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
+
+ self.add_time_attn = config.add_time_attn
+ if self.add_time_attn:
+ self.t = config.num_frames
+ self.temporal_embedding = nn.Parameter(torch.zeros(1, config.num_frames, config.hidden_size))
+ nn.init.normal_(self.temporal_embedding, std=config.hidden_size ** -0.5)
+
+ self.embed_dim = config.hidden_size
+ self.temporal_attn = CLIPAttention(config)
+ self.temporal_layer_norm1 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
+ self.temporal_mlp = CLIPMLP(config)
+ self.temporal_layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
+
+ def forward(
+ self,
+ hidden_states: torch.Tensor,
+ attention_mask: torch.Tensor,
+ causal_attention_mask: torch.Tensor,
+ output_attentions: Optional[bool] = False,
+ ) -> Tuple[torch.FloatTensor]:
+ """
+ Args:
+ hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
+ attention_mask (`torch.FloatTensor`): attention mask of size
+ `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
+ `(config.encoder_attention_heads,)`.
+ output_attentions (`bool`, *optional*):
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under
+ returned tensors for more detail.
+ """
+
+
+ if self.add_time_attn:
+ bt, n, d = hidden_states.shape
+ t = self.t
+
+ # time embed
+ if t != 1:
+ n = hidden_states.shape[1]
+ hidden_states = rearrange(hidden_states, '(b t) n d -> (b n) t d', t=t)
+ hidden_states = hidden_states + self.temporal_embedding[:, :t, :]
+ hidden_states = rearrange(hidden_states, '(b n) t d -> (b t) n d', n=n)
+
+ # time attn
+ residual = hidden_states
+ hidden_states = rearrange(hidden_states, '(b t) n d -> (b n) t d', t=t)
+ # hidden_states = self.layer_norm1(hidden_states) # share layernorm
+ hidden_states = self.temporal_layer_norm1(hidden_states)
+ hidden_states, attn_weights = self.temporal_attn(
+ hidden_states=hidden_states,
+ attention_mask=attention_mask,
+ causal_attention_mask=causal_attention_mask,
+ output_attentions=output_attentions,
+ )
+ hidden_states = residual + rearrange(hidden_states, '(b n) t d -> (b t) n d', n=n)
+
+ residual = hidden_states
+ hidden_states = rearrange(hidden_states, '(b t) n d -> (b n) t d', t=t)
+ # hidden_states = self.layer_norm2(hidden_states) # share layernorm
+ hidden_states = self.temporal_layer_norm2(hidden_states)
+ hidden_states = self.temporal_mlp(hidden_states)
+ hidden_states = residual + rearrange(hidden_states, '(b n) t d -> (b t) n d', n=n)
+
+ # spatial attn
+ residual = hidden_states
+
+ hidden_states = self.layer_norm1(hidden_states)
+ hidden_states, attn_weights = self.self_attn(
+ hidden_states=hidden_states,
+ attention_mask=attention_mask,
+ causal_attention_mask=causal_attention_mask,
+ output_attentions=output_attentions,
+ )
+ hidden_states = residual + hidden_states
+
+ residual = hidden_states
+ hidden_states = self.layer_norm2(hidden_states)
+ hidden_states = self.mlp(hidden_states)
+ hidden_states = residual + hidden_states
+
+ outputs = (hidden_states,)
+
+ if output_attentions:
+ outputs += (attn_weights,)
+
+ return outputs
+
+
+
+
+
+
+
+
+
+class CLIPPreTrainedModel(PreTrainedModel):
+ """
+ An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
+ models.
+ """
+
+ config_class = LanguageBindThermalConfig
+ base_model_prefix = "clip"
+ supports_gradient_checkpointing = True
+ _keys_to_ignore_on_load_missing = [r"position_ids"]
+
+ def _init_weights(self, module):
+ """Initialize the weights"""
+ factor = self.config.initializer_factor
+ if isinstance(module, CLIPTextEmbeddings):
+ module.token_embedding.weight.data.normal_(mean=0.0, std=factor * 0.02)
+ module.position_embedding.weight.data.normal_(mean=0.0, std=factor * 0.02)
+ elif isinstance(module, CLIPVisionEmbeddings):
+ factor = self.config.initializer_factor
+ nn.init.normal_(module.class_embedding, mean=0.0, std=module.embed_dim**-0.5 * factor)
+ nn.init.normal_(module.patch_embedding.weight, std=module.config.initializer_range * factor)
+ nn.init.normal_(module.position_embedding.weight, std=module.config.initializer_range * factor)
+ elif isinstance(module, CLIPAttention):
+ factor = self.config.initializer_factor
+ in_proj_std = (module.embed_dim**-0.5) * ((2 * module.config.num_hidden_layers) ** -0.5) * factor
+ out_proj_std = (module.embed_dim**-0.5) * factor
+ nn.init.normal_(module.q_proj.weight, std=in_proj_std)
+ nn.init.normal_(module.k_proj.weight, std=in_proj_std)
+ nn.init.normal_(module.v_proj.weight, std=in_proj_std)
+ nn.init.normal_(module.out_proj.weight, std=out_proj_std)
+ elif isinstance(module, CLIPMLP):
+ factor = self.config.initializer_factor
+ in_proj_std = (
+ (module.config.hidden_size**-0.5) * ((2 * module.config.num_hidden_layers) ** -0.5) * factor
+ )
+ fc_std = (2 * module.config.hidden_size) ** -0.5 * factor
+ nn.init.normal_(module.fc1.weight, std=fc_std)
+ nn.init.normal_(module.fc2.weight, std=in_proj_std)
+ elif isinstance(module, LanguageBindThermal):
+ nn.init.normal_(
+ module.text_projection.weight,
+ std=module.text_embed_dim**-0.5 * self.config.initializer_factor,
+ )
+ nn.init.normal_(
+ module.visual_projection.weight,
+ std=module.vision_embed_dim**-0.5 * self.config.initializer_factor,
+ )
+ elif isinstance(module, CLIPVisionModelWithProjection):
+ nn.init.normal_(
+ module.visual_projection.weight,
+ std=self.config.hidden_size**-0.5 * self.config.initializer_factor,
+ )
+ elif isinstance(module, CLIPTextModelWithProjection):
+ nn.init.normal_(
+ module.text_projection.weight,
+ std=self.config.hidden_size**-0.5 * self.config.initializer_factor,
+ )
+
+ if isinstance(module, nn.LayerNorm):
+ module.bias.data.zero_()
+ module.weight.data.fill_(1.0)
+ if isinstance(module, nn.Linear) and module.bias is not None:
+ module.bias.data.zero_()
+
+ def _set_gradient_checkpointing(self, module, value=False):
+ if isinstance(module, CLIPEncoder):
+ module.gradient_checkpointing = value
+
+
+CLIP_START_DOCSTRING = r"""
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
+
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
+
+ Parameters:
+ config ([`CLIPConfig`]): Model configuration class with all the parameters of the model.
+ Initializing with a config file does not load the weights associated with the model, only the
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
+"""
+
+CLIP_TEXT_INPUTS_DOCSTRING = r"""
+ Args:
+ input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
+ Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
+ it.
+
+ Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
+
+ [What are input IDs?](../glossary#input-ids)
+ attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
+ Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
+
+ - 1 for tokens that are **not masked**,
+ - 0 for tokens that are **masked**.
+
+ [What are attention masks?](../glossary#attention-mask)
+ position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
+
+ [What are position IDs?](../glossary#position-ids)
+ output_attentions (`bool`, *optional*):
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
+ tensors for more detail.
+ output_hidden_states (`bool`, *optional*):
+ Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
+ more detail.
+ return_dict (`bool`, *optional*):
+ Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+"""
+
+CLIP_VISION_INPUTS_DOCSTRING = r"""
+ Args:
+ pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
+ Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using
+ [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details.
+ output_attentions (`bool`, *optional*):
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
+ tensors for more detail.
+ output_hidden_states (`bool`, *optional*):
+ Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
+ more detail.
+ return_dict (`bool`, *optional*):
+ Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+"""
+
+CLIP_INPUTS_DOCSTRING = r"""
+ Args:
+ input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
+ Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
+ it.
+
+ Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
+
+ [What are input IDs?](../glossary#input-ids)
+ attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
+ Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
+
+ - 1 for tokens that are **not masked**,
+ - 0 for tokens that are **masked**.
+
+ [What are attention masks?](../glossary#attention-mask)
+ position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
+
+ [What are position IDs?](../glossary#position-ids)
+ pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
+ Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using
+ [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details.
+ return_loss (`bool`, *optional*):
+ Whether or not to return the contrastive loss.
+ output_attentions (`bool`, *optional*):
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
+ tensors for more detail.
+ output_hidden_states (`bool`, *optional*):
+ Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
+ more detail.
+ return_dict (`bool`, *optional*):
+ Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+"""
+
+
+class CLIPEncoder(nn.Module):
+ """
+ Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
+ [`CLIPEncoderLayer`].
+
+ Args:
+ config: CLIPConfig
+ """
+
+ def __init__(self, config: LanguageBindThermalConfig):
+ super().__init__()
+ self.config = config
+ self.layers = nn.ModuleList([CLIPEncoderLayer(config) for _ in range(config.num_hidden_layers)])
+ self.gradient_checkpointing = False
+
+ def forward(
+ self,
+ inputs_embeds,
+ attention_mask: Optional[torch.Tensor] = None,
+ causal_attention_mask: Optional[torch.Tensor] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> Union[Tuple, BaseModelOutput]:
+ r"""
+ Args:
+ inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
+ This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+ than the model's internal embedding lookup matrix.
+ attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
+ Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
+
+ - 1 for tokens that are **not masked**,
+ - 0 for tokens that are **masked**.
+
+ [What are attention masks?](../glossary#attention-mask)
+ causal_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
+ Causal mask for the text model. Mask values selected in `[0, 1]`:
+
+ - 1 for tokens that are **not masked**,
+ - 0 for tokens that are **masked**.
+
+ [What are attention masks?](../glossary#attention-mask)
+ output_attentions (`bool`, *optional*):
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under
+ returned tensors for more detail.
+ output_hidden_states (`bool`, *optional*):
+ Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
+ for more detail.
+ return_dict (`bool`, *optional*):
+ Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+ """
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+ output_hidden_states = (
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+ )
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+ encoder_states = () if output_hidden_states else None
+ all_attentions = () if output_attentions else None
+
+ hidden_states = inputs_embeds
+ for idx, encoder_layer in enumerate(self.layers):
+ if output_hidden_states:
+ encoder_states = encoder_states + (hidden_states,)
+ if self.gradient_checkpointing and self.training:
+
+ def create_custom_forward(module):
+ def custom_forward(*inputs):
+ return module(*inputs, output_attentions)
+
+ return custom_forward
+
+ layer_outputs = torch.utils.checkpoint.checkpoint(
+ create_custom_forward(encoder_layer),
+ hidden_states,
+ attention_mask,
+ causal_attention_mask,
+ )
+ else:
+ layer_outputs = encoder_layer(
+ hidden_states,
+ attention_mask,
+ causal_attention_mask,
+ output_attentions=output_attentions,
+ )
+
+ hidden_states = layer_outputs[0]
+
+ if output_attentions:
+ all_attentions = all_attentions + (layer_outputs[1],)
+
+ if output_hidden_states:
+ encoder_states = encoder_states + (hidden_states,)
+
+ if not return_dict:
+ return tuple(v for v in [hidden_states, encoder_states, all_attentions] if v is not None)
+ return BaseModelOutput(
+ last_hidden_state=hidden_states, hidden_states=encoder_states, attentions=all_attentions
+ )
+
+
+# Copied from transformers.models.bart.modeling_bart._make_causal_mask
+def _make_causal_mask(
+ input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device, past_key_values_length: int = 0
+):
+ """
+ Make causal mask used for bi-directional self-attention.
+ """
+ bsz, tgt_len = input_ids_shape
+ mask = torch.full((tgt_len, tgt_len), torch.finfo(dtype).min, device=device)
+ mask_cond = torch.arange(mask.size(-1), device=device)
+ mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0)
+ mask = mask.to(dtype)
+
+ if past_key_values_length > 0:
+ mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), mask], dim=-1)
+ return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length)
+
+
+class CLIPTextTransformer(nn.Module):
+ def __init__(self, config: CLIPTextConfig):
+ super().__init__()
+ self.config = config
+ embed_dim = config.hidden_size
+ self.embeddings = CLIPTextEmbeddings(config)
+ self.encoder = CLIPEncoder(config)
+ self.final_layer_norm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
+
+ @add_start_docstrings_to_model_forward(CLIP_TEXT_INPUTS_DOCSTRING)
+ @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=CLIPTextConfig)
+ def forward(
+ self,
+ input_ids: Optional[torch.Tensor] = None,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.Tensor] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> Union[Tuple, BaseModelOutputWithPooling]:
+ r"""
+ Returns:
+
+ """
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+ output_hidden_states = (
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+ )
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+ if input_ids is None:
+ raise ValueError("You have to specify input_ids")
+
+ input_shape = input_ids.size()
+ input_ids = input_ids.view(-1, input_shape[-1])
+
+ hidden_states = self.embeddings(input_ids=input_ids, position_ids=position_ids)
+
+ # CLIP's text model uses causal mask, prepare it here.
+ # https://github.com/openai/CLIP/blob/cfcffb90e69f37bf2ff1e988237a0fbe41f33c04/clip/model.py#L324
+ causal_attention_mask = _make_causal_mask(input_shape, hidden_states.dtype, device=hidden_states.device)
+ # expand attention_mask
+ if attention_mask is not None:
+ # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]
+ attention_mask = _expand_mask(attention_mask, hidden_states.dtype)
+
+ encoder_outputs = self.encoder(
+ inputs_embeds=hidden_states,
+ attention_mask=attention_mask,
+ causal_attention_mask=causal_attention_mask,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+
+ last_hidden_state = encoder_outputs[0]
+ last_hidden_state = self.final_layer_norm(last_hidden_state)
+
+ # text_embeds.shape = [batch_size, sequence_length, transformer.width]
+ # take features from the eot embedding (eot_token is the highest number in each sequence)
+ # casting to torch.int for onnx compatibility: argmax doesn't support int64 inputs with opset 14
+ pooled_output = last_hidden_state[
+ torch.arange(last_hidden_state.shape[0], device=last_hidden_state.device),
+ input_ids.to(dtype=torch.int, device=last_hidden_state.device).argmax(dim=-1),
+ ]
+
+ if not return_dict:
+ return (last_hidden_state, pooled_output) + encoder_outputs[1:]
+
+ return BaseModelOutputWithPooling(
+ last_hidden_state=last_hidden_state,
+ pooler_output=pooled_output,
+ hidden_states=encoder_outputs.hidden_states,
+ attentions=encoder_outputs.attentions,
+ )
+
+
+@add_start_docstrings(
+ """The text model from CLIP without any head or projection on top.""",
+ CLIP_START_DOCSTRING,
+)
+class CLIPTextModel(CLIPPreTrainedModel):
+ config_class = CLIPTextConfig
+
+ _no_split_modules = ["CLIPEncoderLayer"]
+
+ def __init__(self, config: CLIPTextConfig):
+ super().__init__(config)
+ self.text_model = CLIPTextTransformer(config)
+ # Initialize weights and apply final processing
+ self.post_init()
+
+ def get_input_embeddings(self) -> nn.Module:
+ return self.text_model.embeddings.token_embedding
+
+ def set_input_embeddings(self, value):
+ self.text_model.embeddings.token_embedding = value
+
+ @add_start_docstrings_to_model_forward(CLIP_TEXT_INPUTS_DOCSTRING)
+ @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=CLIPTextConfig)
+ def forward(
+ self,
+ input_ids: Optional[torch.Tensor] = None,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.Tensor] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> Union[Tuple, BaseModelOutputWithPooling]:
+ r"""
+ Returns:
+
+ Examples:
+
+ ```python
+ >>> from transformers import AutoTokenizer, CLIPTextModel
+
+ >>> model = CLIPTextModel.from_pretrained("openai/clip-vit-base-patch32")
+ >>> tokenizer = AutoTokenizer.from_pretrained("openai/clip-vit-base-patch32")
+
+ >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt")
+
+ >>> outputs = model(**inputs)
+ >>> last_hidden_state = outputs.last_hidden_state
+ >>> pooled_output = outputs.pooler_output # pooled (EOS token) states
+ ```"""
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+ return self.text_model(
+ input_ids=input_ids,
+ attention_mask=attention_mask,
+ position_ids=position_ids,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+
+
+class CLIPVisionTransformer(nn.Module):
+ def __init__(self, config: CLIPVisionConfig):
+ super().__init__()
+ self.config = config
+ embed_dim = config.hidden_size
+
+ self.embeddings = CLIPVisionEmbeddings(config)
+ self.patch_dropout = PatchDropout(config.force_patch_dropout)
+ self.pre_layrnorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
+ self.encoder = CLIPEncoder(config)
+ self.post_layernorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
+
+ @add_start_docstrings_to_model_forward(CLIP_VISION_INPUTS_DOCSTRING)
+ @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=CLIPVisionConfig)
+ def forward(
+ self,
+ pixel_values: Optional[torch.FloatTensor] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> Union[Tuple, BaseModelOutputWithPooling]:
+ r"""
+ Returns:
+
+ """
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+ output_hidden_states = (
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+ )
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+ if pixel_values is None:
+ raise ValueError("You have to specify pixel_values")
+ ######################################
+ if len(pixel_values.shape) == 7:
+ b_new, pair_new, T, bs_new, channel_new, h_new, w_new = pixel_values.shape
+ # print(pixel_values.shape)
+ B = b_new * pair_new * bs_new
+ pixel_values = pixel_values.reshape(B*T, channel_new, h_new, w_new)
+
+ elif len(pixel_values.shape) == 5:
+ B, _, T, _, _ = pixel_values.shape
+ # print(pixel_values.shape)
+ pixel_values = rearrange(pixel_values, 'b c t h w -> (b t) c h w')
+ else:
+ # print(pixel_values.shape)
+ B, _, _, _ = pixel_values.shape
+ T = 1
+ ###########################
+ hidden_states = self.embeddings(pixel_values)
+
+ hidden_states = self.patch_dropout(hidden_states, B, T) ##############################################
+
+ hidden_states = self.pre_layrnorm(hidden_states)
+
+ encoder_outputs = self.encoder(
+ inputs_embeds=hidden_states,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+
+ last_hidden_state = encoder_outputs[0]
+ pooled_output = last_hidden_state[:, 0, :]
+ pooled_output = self.post_layernorm(pooled_output)
+
+ pooled_output = pooled_output.reshape(B, T, -1).mean(1) ################################
+
+ if not return_dict:
+ return (last_hidden_state, pooled_output) + encoder_outputs[1:]
+
+ return BaseModelOutputWithPooling(
+ last_hidden_state=last_hidden_state,
+ pooler_output=pooled_output,
+ hidden_states=encoder_outputs.hidden_states,
+ attentions=encoder_outputs.attentions,
+ )
+
+
+@add_start_docstrings(
+ """The vision model from CLIP without any head or projection on top.""",
+ CLIP_START_DOCSTRING,
+)
+class CLIPVisionModel(CLIPPreTrainedModel):
+ config_class = CLIPVisionConfig
+ main_input_name = "pixel_values"
+
+ def __init__(self, config: CLIPVisionConfig):
+ super().__init__(config)
+ self.vision_model = CLIPVisionTransformer(config)
+ # Initialize weights and apply final processing
+ self.post_init()
+
+ def get_input_embeddings(self) -> nn.Module:
+ return self.vision_model.embeddings.patch_embedding
+
+ @add_start_docstrings_to_model_forward(CLIP_VISION_INPUTS_DOCSTRING)
+ @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=CLIPVisionConfig)
+ def forward(
+ self,
+ pixel_values: Optional[torch.FloatTensor] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> Union[Tuple, BaseModelOutputWithPooling]:
+ r"""
+ Returns:
+
+ Examples:
+
+ ```python
+ >>> from PIL import Image
+ >>> import requests
+ >>> from transformers import AutoProcessor, CLIPVisionModel
+
+ >>> model = CLIPVisionModel.from_pretrained("openai/clip-vit-base-patch32")
+ >>> processor = AutoProcessor.from_pretrained("openai/clip-vit-base-patch32")
+
+ >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
+ >>> image = Image.open(requests.get(url, stream=True).raw)
+
+ >>> inputs = processor(images=image, return_tensors="pt")
+
+ >>> outputs = model(**inputs)
+ >>> last_hidden_state = outputs.last_hidden_state
+ >>> pooled_output = outputs.pooler_output # pooled CLS states
+ ```"""
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+ return self.vision_model(
+ pixel_values=pixel_values,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+
+
+@add_start_docstrings(CLIP_START_DOCSTRING)
+class LanguageBindThermal(CLIPPreTrainedModel):
+ config_class = LanguageBindThermalConfig
+
+ def __init__(self, config: LanguageBindThermalConfig):
+ super().__init__(config)
+
+ if not isinstance(config.text_config, CLIPTextConfig):
+ raise ValueError(
+ "config.text_config is expected to be of type CLIPTextConfig but is of type"
+ f" {type(config.text_config)}."
+ )
+
+ if not isinstance(config.vision_config, CLIPVisionConfig):
+ raise ValueError(
+ "config.vision_config is expected to be of type CLIPVisionConfig but is of type"
+ f" {type(config.vision_config)}."
+ )
+
+ text_config = config.text_config
+ vision_config = config.vision_config
+ self.add_time_attn = vision_config.add_time_attn
+ self.lora_r = vision_config.lora_r
+ self.lora_alpha = vision_config.lora_alpha
+ self.lora_dropout = vision_config.lora_dropout
+
+ self.projection_dim = config.projection_dim
+ self.text_embed_dim = text_config.hidden_size
+ self.vision_embed_dim = vision_config.hidden_size
+
+ self.text_model = CLIPTextTransformer(text_config)
+ self.vision_model = CLIPVisionTransformer(vision_config)
+
+ self.visual_projection = nn.Linear(self.vision_embed_dim, self.projection_dim, bias=False)
+ self.text_projection = nn.Linear(self.text_embed_dim, self.projection_dim, bias=False)
+ self.logit_scale = nn.Parameter(torch.tensor(self.config.logit_scale_init_value))
+
+ # Initialize weights and apply final processing
+ self.post_init()
+ self.convert_to_lora()
+ self.resize_pos(self.vision_model.embeddings, vision_config)
+
+ def convert_to_lora(self):
+ if self.lora_r == 0:
+ return
+ if self.add_time_attn:
+ target_modules = ["temporal_attn.k_proj", "temporal_attn.v_proj",
+ "temporal_attn.q_proj", "temporal_attn.out_proj",
+ "temporal_mlp.fc1", "temporal_mlp.fc2"]
+ else:
+ target_modules = ["k_proj", "v_proj", "q_proj", "out_proj"]
+ config = LoraConfig(
+ r=self.lora_r, # 16
+ lora_alpha=self.lora_alpha, # 16
+ target_modules=target_modules, # self_attn.out_proj
+ lora_dropout=self.lora_dropout, # 0.1
+ bias="none",
+ modules_to_save=[],
+ )
+ self.vision_model.encoder.is_gradient_checkpointing = False
+ self.vision_model.encoder = get_peft_model(self.vision_model.encoder, config)
+
+ def resize_pos(self, m, vision_config):
+ # convert embedding
+ if vision_config.num_mel_bins!=0 and vision_config.target_length!=0:
+ m.image_size = [vision_config.num_mel_bins, vision_config.target_length]
+ m.config.image_size = [m.image_size, m.image_size] if isinstance(m.image_size, int) else m.image_size
+ # pos resize
+ old_pos_embed_state_dict = m.position_embedding.state_dict()
+ old_pos_embed = old_pos_embed_state_dict['weight']
+ dtype = old_pos_embed.dtype
+ grid_size = [m.config.image_size[0] // m.patch_size, m.config.image_size[1] // m.patch_size]
+ extra_tokens = 1 # FIXME detect different token configs (ie no class token, or more)
+ new_seq_len = grid_size[0] * grid_size[1] + extra_tokens
+ if new_seq_len == old_pos_embed.shape[0]:
+ # m.to(args.device)
+ return
+
+ m.num_patches = grid_size[0] * grid_size[1]
+ m.num_positions = m.num_patches + 1
+ m.register_buffer("position_ids", torch.arange(m.num_positions).expand((1, -1)))
+ new_position_embedding = nn.Embedding(m.num_positions, m.embed_dim)
+
+ if extra_tokens:
+ pos_emb_tok, pos_emb_img = old_pos_embed[:extra_tokens], old_pos_embed[extra_tokens:]
+ else:
+ pos_emb_tok, pos_emb_img = None, old_pos_embed
+ old_grid_size = [int(math.sqrt(len(pos_emb_img)))] * 2
+
+ # if is_master(args):
+ # logging.info('Resizing position embedding grid-size from %s to %s', old_grid_size, grid_size)
+ pos_emb_img = pos_emb_img.reshape(1, old_grid_size[0], old_grid_size[1], -1).permute(0, 3, 1, 2)
+ pos_emb_img = F.interpolate(
+ pos_emb_img,
+ size=grid_size,
+ mode='bicubic',
+ antialias=True,
+ align_corners=False,
+ )
+ pos_emb_img = pos_emb_img.permute(0, 2, 3, 1).reshape(1, grid_size[0] * grid_size[1], -1)[0]
+ if pos_emb_tok is not None:
+ new_pos_embed = torch.cat([pos_emb_tok, pos_emb_img], dim=0)
+ else:
+ new_pos_embed = pos_emb_img
+ old_pos_embed_state_dict['weight'] = new_pos_embed.to(dtype)
+ m.position_embedding = new_position_embedding
+ m.position_embedding.load_state_dict(old_pos_embed_state_dict)
+
+ # m.to(args.device)
+
+ @add_start_docstrings_to_model_forward(CLIP_TEXT_INPUTS_DOCSTRING)
+ def get_text_features(
+ self,
+ input_ids: Optional[torch.Tensor] = None,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.Tensor] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> torch.FloatTensor:
+ r"""
+ Returns:
+ text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by
+ applying the projection layer to the pooled output of [`CLIPTextModel`].
+
+ Examples:
+
+ ```python
+ >>> from transformers import AutoTokenizer, CLIPModel
+
+ >>> model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")
+ >>> tokenizer = AutoTokenizer.from_pretrained("openai/clip-vit-base-patch32")
+
+ >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt")
+ >>> text_features = model.get_text_features(**inputs)
+ ```"""
+ # Use CLIP model's config for some fields (if specified) instead of those of vision & text components.
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+ output_hidden_states = (
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+ )
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+ text_outputs = self.text_model(
+ input_ids=input_ids,
+ attention_mask=attention_mask,
+ position_ids=position_ids,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+
+ pooled_output = text_outputs[1]
+ text_features = self.text_projection(pooled_output)
+
+ return text_features
+
+ @add_start_docstrings_to_model_forward(CLIP_VISION_INPUTS_DOCSTRING)
+ def get_image_features(
+ self,
+ pixel_values: Optional[torch.FloatTensor] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> torch.FloatTensor:
+ r"""
+ Returns:
+ image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by
+ applying the projection layer to the pooled output of [`CLIPVisionModel`].
+
+ Examples:
+
+ ```python
+ >>> from PIL import Image
+ >>> import requests
+ >>> from transformers import AutoProcessor, CLIPModel
+
+ >>> model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")
+ >>> processor = AutoProcessor.from_pretrained("openai/clip-vit-base-patch32")
+
+ >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
+ >>> image = Image.open(requests.get(url, stream=True).raw)
+
+ >>> inputs = processor(images=image, return_tensors="pt")
+
+ >>> image_features = model.get_image_features(**inputs)
+ ```"""
+ # Use CLIP model's config for some fields (if specified) instead of those of vision & text components.
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+ output_hidden_states = (
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+ )
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+ vision_outputs = self.vision_model(
+ pixel_values=pixel_values,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+
+ pooled_output = vision_outputs[1] # pooled_output
+ image_features = self.visual_projection(pooled_output)
+
+ return image_features
+
+ @add_start_docstrings_to_model_forward(CLIP_INPUTS_DOCSTRING)
+ @replace_return_docstrings(output_type=CLIPOutput, config_class=LanguageBindThermalConfig)
+ def forward(
+ self,
+ input_ids: Optional[torch.LongTensor] = None,
+ pixel_values: Optional[torch.FloatTensor] = None,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.LongTensor] = None,
+ return_loss: Optional[bool] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> Union[Tuple, CLIPOutput]:
+ r"""
+ Returns:
+
+ Examples:
+
+ ```python
+ >>> from PIL import Image
+ >>> import requests
+ >>> from transformers import AutoProcessor, CLIPModel
+
+ >>> model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")
+ >>> processor = AutoProcessor.from_pretrained("openai/clip-vit-base-patch32")
+
+ >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
+ >>> image = Image.open(requests.get(url, stream=True).raw)
+
+ >>> inputs = processor(
+ ... text=["a photo of a cat", "a photo of a dog"], images=image, return_tensors="pt", padding=True
+ ... )
+
+ >>> outputs = model(**inputs)
+ >>> logits_per_image = outputs.logits_per_image # this is the image-text similarity score
+ >>> probs = logits_per_image.softmax(dim=1) # we can take the softmax to get the label probabilities
+ ```"""
+ # Use CLIP model's config for some fields (if specified) instead of those of vision & text components.
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+ output_hidden_states = (
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+ )
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+ vision_outputs = self.vision_model(
+ pixel_values=pixel_values,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+
+ text_outputs = self.text_model(
+ input_ids=input_ids,
+ attention_mask=attention_mask,
+ position_ids=position_ids,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+
+ image_embeds = vision_outputs[1]
+ image_embeds = self.visual_projection(image_embeds)
+
+ text_embeds = text_outputs[1]
+ text_embeds = self.text_projection(text_embeds)
+
+ # normalized features
+ image_embeds = image_embeds / image_embeds.norm(p=2, dim=-1, keepdim=True)
+ text_embeds = text_embeds / text_embeds.norm(p=2, dim=-1, keepdim=True)
+
+ # cosine similarity as logits
+ logit_scale = self.logit_scale.exp()
+ logits_per_text = torch.matmul(text_embeds, image_embeds.t()) * logit_scale
+ logits_per_image = logits_per_text.t()
+
+ loss = None
+ if return_loss:
+ loss = clip_loss(logits_per_text)
+
+ if not return_dict:
+ output = (logits_per_image, logits_per_text, text_embeds, image_embeds, text_outputs, vision_outputs)
+ return ((loss,) + output) if loss is not None else output
+
+ return CLIPOutput(
+ loss=loss,
+ logits_per_image=logits_per_image,
+ logits_per_text=logits_per_text,
+ text_embeds=text_embeds,
+ image_embeds=image_embeds,
+ text_model_output=text_outputs,
+ vision_model_output=vision_outputs,
+ )
\ No newline at end of file
diff --git a/videollava/model/multimodal_encoder/languagebind/thermal/processing_thermal.py b/videollava/model/multimodal_encoder/languagebind/thermal/processing_thermal.py
new file mode 100644
index 0000000000000000000000000000000000000000..36ed1f09d3bf23514baf4859e462d28bc49dfd53
--- /dev/null
+++ b/videollava/model/multimodal_encoder/languagebind/thermal/processing_thermal.py
@@ -0,0 +1,77 @@
+import torch
+from PIL import Image
+from torchvision import transforms
+from transformers import ProcessorMixin, BatchEncoding
+from transformers.image_processing_utils import BatchFeature
+
+OPENAI_DATASET_MEAN = (0.48145466, 0.4578275, 0.40821073)
+OPENAI_DATASET_STD = (0.26862954, 0.26130258, 0.27577711)
+
+def make_list_of_images(x):
+ if not isinstance(x, list):
+ return [x]
+ return x
+
+def get_thermal_transform(config):
+ config = config.vision_config
+ transform = transforms.Compose(
+ [
+ transforms.ToTensor(),
+ transforms.Resize(224, interpolation=transforms.InterpolationMode.BICUBIC),
+ transforms.CenterCrop(224),
+ transforms.Normalize(OPENAI_DATASET_MEAN, OPENAI_DATASET_STD) # assume image
+ ]
+ )
+ return transform
+
+
+def load_and_transform_thermal(thermal_path, transform):
+ thermal = Image.open(thermal_path)
+ thermal_outputs = transform(thermal)
+ return thermal_outputs
+
+class LanguageBindThermalProcessor(ProcessorMixin):
+ attributes = []
+ tokenizer_class = ("LanguageBindThermalTokenizer")
+
+ def __init__(self, config, tokenizer=None, **kwargs):
+ super().__init__(**kwargs)
+ self.config = config
+ self.transform = get_thermal_transform(config)
+ self.image_processor = load_and_transform_thermal
+ self.tokenizer = tokenizer
+
+ def __call__(self, images=None, text=None, context_length=77, return_tensors=None, **kwargs):
+ if text is None and images is None:
+ raise ValueError("You have to specify either text or images. Both cannot be none.")
+
+ if text is not None:
+ encoding = self.tokenizer(text, max_length=context_length, padding='max_length',
+ truncation=True, return_tensors=return_tensors, **kwargs)
+
+ if images is not None:
+ images = make_list_of_images(images)
+ image_features = [self.image_processor(image, self.transform) for image in images]
+ image_features = torch.stack(image_features)
+
+ if text is not None and images is not None:
+ encoding["pixel_values"] = image_features
+ return encoding
+ elif text is not None:
+ return encoding
+ else:
+ return {"pixel_values": image_features}
+
+ def batch_decode(self, skip_special_tokens=True, *args, **kwargs):
+ """
+ This method forwards all its arguments to CLIPTokenizerFast's [`~PreTrainedTokenizer.batch_decode`]. Please
+ refer to the docstring of this method for more information.
+ """
+ return self.tokenizer.batch_decode(*args, skip_special_tokens=skip_special_tokens, **kwargs)
+
+ def decode(self, skip_special_tokens=True, *args, **kwargs):
+ """
+ This method forwards all its arguments to CLIPTokenizerFast's [`~PreTrainedTokenizer.decode`]. Please refer to
+ the docstring of this method for more information.
+ """
+ return self.tokenizer.decode(*args, skip_special_tokens=skip_special_tokens, **kwargs)
diff --git a/videollava/model/multimodal_encoder/languagebind/thermal/tokenization_thermal.py b/videollava/model/multimodal_encoder/languagebind/thermal/tokenization_thermal.py
new file mode 100644
index 0000000000000000000000000000000000000000..a4ebb5607bc8f2a24341a7b11f22663e760012dd
--- /dev/null
+++ b/videollava/model/multimodal_encoder/languagebind/thermal/tokenization_thermal.py
@@ -0,0 +1,77 @@
+from transformers import CLIPTokenizer
+from transformers.utils import logging
+
+logger = logging.get_logger(__name__)
+
+VOCAB_FILES_NAMES = {
+ "vocab_file": "vocab.json",
+ "merges_file": "merges.txt",
+}
+
+PRETRAINED_VOCAB_FILES_MAP = {
+ "vocab_file": {
+ "lb203/LanguageBind-Thermal": "https://huggingface.co/lb203/LanguageBind-Thermal/resolve/main/vocab.json",
+ },
+ "merges_file": {
+ "lb203/LanguageBind-Thermal": "https://huggingface.co/lb203/LanguageBind-Thermal/resolve/main/merges.txt",
+ },
+}
+
+PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
+ "lb203/LanguageBind-Thermal": 77,
+}
+
+
+PRETRAINED_INIT_CONFIGURATION = {
+ "lb203/LanguageBind-Thermal": {},
+}
+
+class LanguageBindThermalTokenizer(CLIPTokenizer):
+ """
+ Construct a CLIP tokenizer. Based on byte-level Byte-Pair-Encoding.
+
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+ this superclass for more information regarding those methods.
+
+ Args:
+ vocab_file (`str`):
+ Path to the vocabulary file.
+ merges_file (`str`):
+ Path to the merges file.
+ errors (`str`, *optional*, defaults to `"replace"`):
+ Paradigm to follow when decoding bytes to UTF-8. See
+ [bytes.decode](https://docs.python.org/3/library/stdtypes.html#bytes.decode) for more information.
+ unk_token (`str`, *optional*, defaults to `<|endoftext|>`):
+ The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
+ token instead.
+ bos_token (`str`, *optional*, defaults to `<|startoftext|>`):
+ The beginning of sequence token.
+ eos_token (`str`, *optional*, defaults to `<|endoftext|>`):
+ The end of sequence token.
+ """
+
+ vocab_files_names = VOCAB_FILES_NAMES
+ pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+ max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
+ model_input_names = ["input_ids", "attention_mask"]
+
+ def __init__(
+ self,
+ vocab_file,
+ merges_file,
+ errors="replace",
+ unk_token="<|endoftext|>",
+ bos_token="<|startoftext|>",
+ eos_token="<|endoftext|>",
+ pad_token="<|endoftext|>", # hack to enable padding
+ **kwargs,
+ ):
+ super(LanguageBindThermalTokenizer, self).__init__(
+ vocab_file,
+ merges_file,
+ errors,
+ unk_token,
+ bos_token,
+ eos_token,
+ pad_token, # hack to enable padding
+ **kwargs,)
\ No newline at end of file
diff --git a/videollava/model/multimodal_encoder/languagebind/video/configuration_video.py b/videollava/model/multimodal_encoder/languagebind/video/configuration_video.py
new file mode 100644
index 0000000000000000000000000000000000000000..4b108ec51799ae0d77432ffa85690e1a1858e60c
--- /dev/null
+++ b/videollava/model/multimodal_encoder/languagebind/video/configuration_video.py
@@ -0,0 +1,423 @@
+import copy
+import os
+from typing import Union
+
+from transformers import PretrainedConfig
+from transformers.utils import logging
+
+logger = logging.get_logger(__name__)
+
+
+
+
+
+
+
+class CLIPTextConfig(PretrainedConfig):
+ r"""
+ This is the configuration class to store the configuration of a [`CLIPTextModel`]. It is used to instantiate a CLIP
+ text encoder according to the specified arguments, defining the model architecture. Instantiating a configuration
+ with the defaults will yield a similar configuration to that of the text encoder of the CLIP
+ [openai/clip-vit-base-patch32](https://huggingface.co/openai/clip-vit-base-patch32) architecture.
+
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
+
+ Args:
+ vocab_size (`int`, *optional*, defaults to 49408):
+ Vocabulary size of the CLIP text model. Defines the number of different tokens that can be represented by
+ the `inputs_ids` passed when calling [`CLIPModel`].
+ hidden_size (`int`, *optional*, defaults to 512):
+ Dimensionality of the encoder layers and the pooler layer.
+ intermediate_size (`int`, *optional*, defaults to 2048):
+ Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
+ num_hidden_layers (`int`, *optional*, defaults to 12):
+ Number of hidden layers in the Transformer encoder.
+ num_attention_heads (`int`, *optional*, defaults to 8):
+ Number of attention heads for each attention layer in the Transformer encoder.
+ max_position_embeddings (`int`, *optional*, defaults to 77):
+ The maximum sequence length that this model might ever be used with. Typically set this to something large
+ just in case (e.g., 512 or 1024 or 2048).
+ hidden_act (`str` or `function`, *optional*, defaults to `"quick_gelu"`):
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"selu"` and `"gelu_new"` `"quick_gelu"` are supported.
+ layer_norm_eps (`float`, *optional*, defaults to 1e-5):
+ The epsilon used by the layer normalization layers.
+ attention_dropout (`float`, *optional*, defaults to 0.0):
+ The dropout ratio for the attention probabilities.
+ initializer_range (`float`, *optional*, defaults to 0.02):
+ The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+ initializer_factor (`float`, *optional*, defaults to 1):
+ A factor for initializing all weight matrices (should be kept to 1, used internally for initialization
+ testing).
+
+ Example:
+
+ ```python
+ >>> from transformers import CLIPTextConfig, CLIPTextModel
+
+ >>> # Initializing a CLIPTextConfig with openai/clip-vit-base-patch32 style configuration
+ >>> configuration = CLIPTextConfig()
+
+ >>> # Initializing a CLIPTextModel (with random weights) from the openai/clip-vit-base-patch32 style configuration
+ >>> model = CLIPTextModel(configuration)
+
+ >>> # Accessing the model configuration
+ >>> configuration = model.config
+ ```"""
+ model_type = "clip_text_model"
+
+ def __init__(
+ self,
+ vocab_size=49408,
+ hidden_size=512,
+ intermediate_size=2048,
+ projection_dim=512,
+ num_hidden_layers=12,
+ num_attention_heads=8,
+ max_position_embeddings=77,
+ hidden_act="quick_gelu",
+ layer_norm_eps=1e-5,
+ attention_dropout=0.0,
+ initializer_range=0.02,
+ initializer_factor=1.0,
+ # This differs from `CLIPTokenizer`'s default and from openai/clip
+ # See https://github.com/huggingface/transformers/pull/24773#issuecomment-1632287538
+ pad_token_id=1,
+ bos_token_id=49406,
+ eos_token_id=49407,
+ **kwargs,
+ ):
+ super().__init__(pad_token_id=pad_token_id, bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs)
+
+ self.vocab_size = vocab_size
+ self.hidden_size = hidden_size
+ self.intermediate_size = intermediate_size
+ self.projection_dim = projection_dim
+ self.num_hidden_layers = num_hidden_layers
+ self.num_attention_heads = num_attention_heads
+ self.max_position_embeddings = max_position_embeddings
+ self.layer_norm_eps = layer_norm_eps
+ self.hidden_act = hidden_act
+ self.initializer_range = initializer_range
+ self.initializer_factor = initializer_factor
+ self.attention_dropout = attention_dropout
+ self.add_time_attn = False ######################################
+
+ @classmethod
+ def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":
+ cls._set_token_in_kwargs(kwargs)
+
+ config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)
+
+ # get the text config dict if we are loading from CLIPConfig
+ if config_dict.get("model_type") == "clip":
+ config_dict = config_dict["text_config"]
+
+ if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:
+ logger.warning(
+ f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
+ f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
+ )
+
+ return cls.from_dict(config_dict, **kwargs)
+
+
+
+
+class CLIPVisionConfig(PretrainedConfig):
+ r"""
+ This is the configuration class to store the configuration of a [`CLIPVisionModel`]. It is used to instantiate a
+ CLIP vision encoder according to the specified arguments, defining the model architecture. Instantiating a
+ configuration with the defaults will yield a similar configuration to that of the vision encoder of the CLIP
+ [openai/clip-vit-base-patch32](https://huggingface.co/openai/clip-vit-base-patch32) architecture.
+
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
+
+ Args:
+ hidden_size (`int`, *optional*, defaults to 768):
+ Dimensionality of the encoder layers and the pooler layer.
+ intermediate_size (`int`, *optional*, defaults to 3072):
+ Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
+ num_hidden_layers (`int`, *optional*, defaults to 12):
+ Number of hidden layers in the Transformer encoder.
+ num_attention_heads (`int`, *optional*, defaults to 12):
+ Number of attention heads for each attention layer in the Transformer encoder.
+ image_size (`int`, *optional*, defaults to 224):
+ The size (resolution) of each image.
+ patch_size (`int`, *optional*, defaults to 32):
+ The size (resolution) of each patch.
+ hidden_act (`str` or `function`, *optional*, defaults to `"quick_gelu"`):
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"selu"` and `"gelu_new"` ``"quick_gelu"` are supported.
+ layer_norm_eps (`float`, *optional*, defaults to 1e-5):
+ The epsilon used by the layer normalization layers.
+ attention_dropout (`float`, *optional*, defaults to 0.0):
+ The dropout ratio for the attention probabilities.
+ initializer_range (`float`, *optional*, defaults to 0.02):
+ The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+ initializer_factor (`float`, *optional*, defaults to 1):
+ A factor for initializing all weight matrices (should be kept to 1, used internally for initialization
+ testing).
+
+ Example:
+
+ ```python
+ >>> from transformers import CLIPVisionConfig, CLIPVisionModel
+
+ >>> # Initializing a CLIPVisionConfig with openai/clip-vit-base-patch32 style configuration
+ >>> configuration = CLIPVisionConfig()
+
+ >>> # Initializing a CLIPVisionModel (with random weights) from the openai/clip-vit-base-patch32 style configuration
+ >>> model = CLIPVisionModel(configuration)
+
+ >>> # Accessing the model configuration
+ >>> configuration = model.config
+ ```"""
+
+ model_type = "clip_vision_model"
+
+ def __init__(
+ self,
+ hidden_size=768,
+ intermediate_size=3072,
+ projection_dim=512,
+ num_hidden_layers=12,
+ num_attention_heads=12,
+ num_channels=3,
+ image_size=224,
+ patch_size=32,
+ hidden_act="quick_gelu",
+ layer_norm_eps=1e-5,
+ attention_dropout=0.0,
+ initializer_range=0.02,
+ initializer_factor=1.0,
+
+ add_time_attn=False, ################################
+ num_frames=1, ################################
+ force_patch_dropout=0.0, ################################
+ lora_r=2, ################################
+ lora_alpha=16, ################################
+ lora_dropout=0.0, ################################
+ num_mel_bins=0.0, ################################
+ target_length=0.0, ################################
+ video_decode_backend='decord', #########################
+ **kwargs,
+ ):
+ super().__init__(**kwargs)
+
+ self.hidden_size = hidden_size
+ self.intermediate_size = intermediate_size
+ self.projection_dim = projection_dim
+ self.num_hidden_layers = num_hidden_layers
+ self.num_attention_heads = num_attention_heads
+ self.num_channels = num_channels
+ self.patch_size = patch_size
+ self.image_size = image_size
+ self.initializer_range = initializer_range
+ self.initializer_factor = initializer_factor
+ self.attention_dropout = attention_dropout
+ self.layer_norm_eps = layer_norm_eps
+ self.hidden_act = hidden_act
+
+ self.add_time_attn = add_time_attn ################
+ self.num_frames = num_frames ################
+ self.force_patch_dropout = force_patch_dropout ################
+ self.lora_r = lora_r ################
+ self.lora_alpha = lora_alpha ################
+ self.lora_dropout = lora_dropout ################
+ self.num_mel_bins = num_mel_bins ################
+ self.target_length = target_length ################
+ self.video_decode_backend = video_decode_backend ################
+
+ @classmethod
+ def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":
+ cls._set_token_in_kwargs(kwargs)
+
+ config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)
+
+ # get the vision config dict if we are loading from CLIPConfig
+ if config_dict.get("model_type") == "clip":
+ config_dict = config_dict["vision_config"]
+
+ if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:
+ logger.warning(
+ f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
+ f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
+ )
+
+ return cls.from_dict(config_dict, **kwargs)
+
+
+class LanguageBindVideoConfig(PretrainedConfig):
+ r"""
+ [`CLIPConfig`] is the configuration class to store the configuration of a [`CLIPModel`]. It is used to instantiate
+ a CLIP model according to the specified arguments, defining the text model and vision model configs. Instantiating
+ a configuration with the defaults will yield a similar configuration to that of the CLIP
+ [openai/clip-vit-base-patch32](https://huggingface.co/openai/clip-vit-base-patch32) architecture.
+
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
+
+ Args:
+ text_config (`dict`, *optional*):
+ Dictionary of configuration options used to initialize [`CLIPTextConfig`].
+ vision_config (`dict`, *optional*):
+ Dictionary of configuration options used to initialize [`CLIPVisionConfig`].
+ projection_dim (`int`, *optional*, defaults to 512):
+ Dimentionality of text and vision projection layers.
+ logit_scale_init_value (`float`, *optional*, defaults to 2.6592):
+ The inital value of the *logit_scale* paramter. Default is used as per the original CLIP implementation.
+ kwargs (*optional*):
+ Dictionary of keyword arguments.
+
+ Example:
+
+ ```python
+ >>> from transformers import CLIPConfig, CLIPModel
+
+ >>> # Initializing a CLIPConfig with openai/clip-vit-base-patch32 style configuration
+ >>> configuration = CLIPConfig()
+
+ >>> # Initializing a CLIPModel (with random weights) from the openai/clip-vit-base-patch32 style configuration
+ >>> model = CLIPModel(configuration)
+
+ >>> # Accessing the model configuration
+ >>> configuration = model.config
+
+ >>> # We can also initialize a CLIPConfig from a CLIPTextConfig and a CLIPVisionConfig
+ >>> from transformers import CLIPTextConfig, CLIPVisionConfig
+
+ >>> # Initializing a CLIPText and CLIPVision configuration
+ >>> config_text = CLIPTextConfig()
+ >>> config_vision = CLIPVisionConfig()
+
+ >>> config = CLIPConfig.from_text_vision_configs(config_text, config_vision)
+ ```"""
+
+ model_type = "LanguageBindVideo"
+ is_composition = True
+
+ def __init__(
+ self, text_config=None, vision_config=None, projection_dim=512, logit_scale_init_value=2.6592, **kwargs
+ ):
+ # If `_config_dict` exist, we use them for the backward compatibility.
+ # We pop out these 2 attributes before calling `super().__init__` to avoid them being saved (which causes a lot
+ # of confusion!).
+ text_config_dict = kwargs.pop("text_config_dict", None)
+ vision_config_dict = kwargs.pop("vision_config_dict", None)
+
+ super().__init__(**kwargs)
+
+ # Instead of simply assigning `[text|vision]_config_dict` to `[text|vision]_config`, we use the values in
+ # `[text|vision]_config_dict` to update the values in `[text|vision]_config`. The values should be same in most
+ # cases, but we don't want to break anything regarding `_config_dict` that existed before commit `8827e1b2`.
+ if text_config_dict is not None:
+ if text_config is None:
+ text_config = {}
+
+ # This is the complete result when using `text_config_dict`.
+ _text_config_dict = CLIPTextConfig(**text_config_dict).to_dict()
+
+ # Give a warning if the values exist in both `_text_config_dict` and `text_config` but being different.
+ for key, value in _text_config_dict.items():
+ if key in text_config and value != text_config[key] and key not in ["transformers_version"]:
+ # If specified in `text_config_dict`
+ if key in text_config_dict:
+ message = (
+ f"`{key}` is found in both `text_config_dict` and `text_config` but with different values. "
+ f'The value `text_config_dict["{key}"]` will be used instead.'
+ )
+ # If inferred from default argument values (just to be super careful)
+ else:
+ message = (
+ f"`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The "
+ f'value `text_config["{key}"]` will be overriden.'
+ )
+ logger.warning(message)
+
+ # Update all values in `text_config` with the ones in `_text_config_dict`.
+ text_config.update(_text_config_dict)
+
+ if vision_config_dict is not None:
+ if vision_config is None:
+ vision_config = {}
+
+ # This is the complete result when using `vision_config_dict`.
+ _vision_config_dict = CLIPVisionConfig(**vision_config_dict).to_dict()
+ # convert keys to string instead of integer
+ if "id2label" in _vision_config_dict:
+ _vision_config_dict["id2label"] = {
+ str(key): value for key, value in _vision_config_dict["id2label"].items()
+ }
+
+ # Give a warning if the values exist in both `_vision_config_dict` and `vision_config` but being different.
+ for key, value in _vision_config_dict.items():
+ if key in vision_config and value != vision_config[key] and key not in ["transformers_version"]:
+ # If specified in `vision_config_dict`
+ if key in vision_config_dict:
+ message = (
+ f"`{key}` is found in both `vision_config_dict` and `vision_config` but with different "
+ f'values. The value `vision_config_dict["{key}"]` will be used instead.'
+ )
+ # If inferred from default argument values (just to be super careful)
+ else:
+ message = (
+ f"`vision_config_dict` is provided which will be used to initialize `CLIPVisionConfig`. "
+ f'The value `vision_config["{key}"]` will be overriden.'
+ )
+ logger.warning(message)
+
+ # Update all values in `vision_config` with the ones in `_vision_config_dict`.
+ vision_config.update(_vision_config_dict)
+
+ if text_config is None:
+ text_config = {}
+ logger.info("`text_config` is `None`. Initializing the `CLIPTextConfig` with default values.")
+
+ if vision_config is None:
+ vision_config = {}
+ logger.info("`vision_config` is `None`. initializing the `CLIPVisionConfig` with default values.")
+
+ self.text_config = CLIPTextConfig(**text_config)
+ self.vision_config = CLIPVisionConfig(**vision_config)
+
+ self.projection_dim = projection_dim
+ self.logit_scale_init_value = logit_scale_init_value
+ self.initializer_factor = 1.0
+
+ @classmethod
+ def from_text_vision_configs(cls, text_config: CLIPTextConfig, vision_config: CLIPVisionConfig, **kwargs):
+ r"""
+ Instantiate a [`CLIPConfig`] (or a derived class) from clip text model configuration and clip vision model
+ configuration.
+
+ Returns:
+ [`CLIPConfig`]: An instance of a configuration object
+ """
+
+ return cls(text_config=text_config.to_dict(), vision_config=vision_config.to_dict(), **kwargs)
+
+ def to_dict(self):
+ """
+ Serializes this instance to a Python dictionary. Override the default [`~PretrainedConfig.to_dict`].
+
+ Returns:
+ `Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance,
+ """
+ output = copy.deepcopy(self.__dict__)
+ output["text_config"] = self.text_config.to_dict()
+ output["vision_config"] = self.vision_config.to_dict()
+ output["model_type"] = self.__class__.model_type
+ return output
+
+
+
+
+
+
+
+
+
+
diff --git a/videollava/model/multimodal_encoder/languagebind/video/modeling_video.py b/videollava/model/multimodal_encoder/languagebind/video/modeling_video.py
new file mode 100644
index 0000000000000000000000000000000000000000..cb5c6218ae0b3158a2ffa3c4daa70d06a93fadfa
--- /dev/null
+++ b/videollava/model/multimodal_encoder/languagebind/video/modeling_video.py
@@ -0,0 +1,1033 @@
+import math
+from typing import Optional, Tuple, Union
+
+import torch
+from einops import rearrange
+from peft import LoraConfig, get_peft_model
+from torch import nn
+from torch.nn import functional as F
+from transformers import PreTrainedModel, add_start_docstrings
+from transformers.modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling
+from transformers.models.clip.modeling_clip import CLIPMLP, CLIPAttention, CLIPTextEmbeddings, CLIPVisionEmbeddings, \
+ CLIPVisionModelWithProjection, CLIPTextModelWithProjection, _expand_mask, CLIPOutput, clip_loss
+from transformers.utils import add_start_docstrings_to_model_forward, replace_return_docstrings
+
+from .configuration_video import LanguageBindVideoConfig, CLIPVisionConfig, CLIPTextConfig
+
+
+
+class PatchDropout(nn.Module):
+ """
+ https://arxiv.org/abs/2212.00794
+ """
+
+ def __init__(self, prob, exclude_first_token=True):
+ super().__init__()
+ assert 0 <= prob < 1.
+ self.prob = prob
+ self.exclude_first_token = exclude_first_token # exclude CLS token
+
+ def forward(self, x, B, T):
+ if not self.training or self.prob == 0.:
+ return x
+
+ if self.exclude_first_token:
+ cls_tokens, x = x[:, :1], x[:, 1:]
+ else:
+ cls_tokens = torch.jit.annotate(torch.Tensor, x[:, :1])
+
+ batch = x.size()[0]
+ num_tokens = x.size()[1]
+
+ batch_indices = torch.arange(batch)
+ batch_indices = batch_indices[..., None]
+
+ keep_prob = 1 - self.prob
+ num_patches_keep = max(1, int(num_tokens * keep_prob))
+
+ if T == 1:
+ rand = torch.randn(batch, num_tokens)
+ patch_indices_keep = rand.topk(num_patches_keep, dim=-1).indices
+ else:
+ rand = torch.randn(B, num_tokens)
+ patch_indices_keep = rand.topk(num_patches_keep, dim=-1).indices
+ patch_indices_keep = patch_indices_keep.unsqueeze(1).repeat(1, T, 1)
+ patch_indices_keep = rearrange(patch_indices_keep, 'b t n -> (b t) n')
+
+
+ x = x[batch_indices, patch_indices_keep]
+
+ if self.exclude_first_token:
+ x = torch.cat((cls_tokens, x), dim=1)
+
+ return x
+
+class CLIPEncoderLayer(nn.Module):
+ def __init__(self, config: LanguageBindVideoConfig):
+ super().__init__()
+ self.embed_dim = config.hidden_size
+ self.self_attn = CLIPAttention(config)
+ self.layer_norm1 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
+ self.mlp = CLIPMLP(config)
+ self.layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
+
+ self.add_time_attn = config.add_time_attn
+ if self.add_time_attn:
+ self.t = config.num_frames
+ self.temporal_embedding = nn.Parameter(torch.zeros(1, config.num_frames, config.hidden_size))
+ nn.init.normal_(self.temporal_embedding, std=config.hidden_size ** -0.5)
+
+ self.embed_dim = config.hidden_size
+ self.temporal_attn = CLIPAttention(config)
+ self.temporal_layer_norm1 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
+ # self.temporal_mlp = CLIPMLP(config)
+ # self.temporal_layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
+
+ def forward(
+ self,
+ hidden_states: torch.Tensor,
+ attention_mask: torch.Tensor,
+ causal_attention_mask: torch.Tensor,
+ output_attentions: Optional[bool] = False,
+ ) -> Tuple[torch.FloatTensor]:
+ """
+ Args:
+ hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
+ attention_mask (`torch.FloatTensor`): attention mask of size
+ `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
+ `(config.encoder_attention_heads,)`.
+ output_attentions (`bool`, *optional*):
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under
+ returned tensors for more detail.
+ """
+
+
+ if self.add_time_attn:
+ bt, n, d = hidden_states.shape
+ t = self.t
+
+ # time embed
+ if t != 1:
+ n = hidden_states.shape[1]
+ hidden_states = rearrange(hidden_states, '(b t) n d -> (b n) t d', t=t)
+ hidden_states = hidden_states + self.temporal_embedding[:, :t, :]
+ hidden_states = rearrange(hidden_states, '(b n) t d -> (b t) n d', n=n)
+
+ # time attn
+ residual = hidden_states
+ hidden_states = rearrange(hidden_states, '(b t) n d -> (b n) t d', t=t)
+ # hidden_states = self.layer_norm1(hidden_states) # share layernorm
+ hidden_states = self.temporal_layer_norm1(hidden_states)
+ hidden_states, attn_weights = self.temporal_attn(
+ hidden_states=hidden_states,
+ attention_mask=attention_mask,
+ causal_attention_mask=causal_attention_mask,
+ output_attentions=output_attentions,
+ )
+ hidden_states = residual + rearrange(hidden_states, '(b n) t d -> (b t) n d', n=n)
+
+ # residual = hidden_states
+ # hidden_states = rearrange(hidden_states, '(b t) n d -> (b n) t d', t=t)
+ # # hidden_states = self.layer_norm2(hidden_states) # share layernorm
+ # hidden_states = self.temporal_layer_norm2(hidden_states)
+ # hidden_states = self.temporal_mlp(hidden_states)
+ # hidden_states = residual + rearrange(hidden_states, '(b n) t d -> (b t) n d', n=n)
+
+ # spatial attn
+ residual = hidden_states
+
+ hidden_states = self.layer_norm1(hidden_states)
+ hidden_states, attn_weights = self.self_attn(
+ hidden_states=hidden_states,
+ attention_mask=attention_mask,
+ causal_attention_mask=causal_attention_mask,
+ output_attentions=output_attentions,
+ )
+ hidden_states = residual + hidden_states
+
+ residual = hidden_states
+ hidden_states = self.layer_norm2(hidden_states)
+ hidden_states = self.mlp(hidden_states)
+ hidden_states = residual + hidden_states
+
+ outputs = (hidden_states,)
+
+ if output_attentions:
+ outputs += (attn_weights,)
+
+ return outputs
+
+
+
+
+
+
+
+
+
+class CLIPPreTrainedModel(PreTrainedModel):
+ """
+ An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
+ models.
+ """
+
+ config_class = LanguageBindVideoConfig
+ base_model_prefix = "clip"
+ supports_gradient_checkpointing = True
+ _keys_to_ignore_on_load_missing = [r"position_ids"]
+
+ def _init_weights(self, module):
+ """Initialize the weights"""
+ factor = self.config.initializer_factor
+ if isinstance(module, CLIPTextEmbeddings):
+ module.token_embedding.weight.data.normal_(mean=0.0, std=factor * 0.02)
+ module.position_embedding.weight.data.normal_(mean=0.0, std=factor * 0.02)
+ elif isinstance(module, CLIPVisionEmbeddings):
+ factor = self.config.initializer_factor
+ nn.init.normal_(module.class_embedding, mean=0.0, std=module.embed_dim**-0.5 * factor)
+ nn.init.normal_(module.patch_embedding.weight, std=module.config.initializer_range * factor)
+ nn.init.normal_(module.position_embedding.weight, std=module.config.initializer_range * factor)
+ elif isinstance(module, CLIPAttention):
+ factor = self.config.initializer_factor
+ in_proj_std = (module.embed_dim**-0.5) * ((2 * module.config.num_hidden_layers) ** -0.5) * factor
+ out_proj_std = (module.embed_dim**-0.5) * factor
+ nn.init.normal_(module.q_proj.weight, std=in_proj_std)
+ nn.init.normal_(module.k_proj.weight, std=in_proj_std)
+ nn.init.normal_(module.v_proj.weight, std=in_proj_std)
+ nn.init.normal_(module.out_proj.weight, std=out_proj_std)
+ elif isinstance(module, CLIPMLP):
+ factor = self.config.initializer_factor
+ in_proj_std = (
+ (module.config.hidden_size**-0.5) * ((2 * module.config.num_hidden_layers) ** -0.5) * factor
+ )
+ fc_std = (2 * module.config.hidden_size) ** -0.5 * factor
+ nn.init.normal_(module.fc1.weight, std=fc_std)
+ nn.init.normal_(module.fc2.weight, std=in_proj_std)
+ elif isinstance(module, LanguageBindVideo):
+ nn.init.normal_(
+ module.text_projection.weight,
+ std=module.text_embed_dim**-0.5 * self.config.initializer_factor,
+ )
+ nn.init.normal_(
+ module.visual_projection.weight,
+ std=module.vision_embed_dim**-0.5 * self.config.initializer_factor,
+ )
+ elif isinstance(module, CLIPVisionModelWithProjection):
+ nn.init.normal_(
+ module.visual_projection.weight,
+ std=self.config.hidden_size**-0.5 * self.config.initializer_factor,
+ )
+ elif isinstance(module, CLIPTextModelWithProjection):
+ nn.init.normal_(
+ module.text_projection.weight,
+ std=self.config.hidden_size**-0.5 * self.config.initializer_factor,
+ )
+
+ if isinstance(module, nn.LayerNorm):
+ module.bias.data.zero_()
+ module.weight.data.fill_(1.0)
+ if isinstance(module, nn.Linear) and module.bias is not None:
+ module.bias.data.zero_()
+
+ def _set_gradient_checkpointing(self, module, value=False):
+ if isinstance(module, CLIPEncoder):
+ module.gradient_checkpointing = value
+
+
+CLIP_START_DOCSTRING = r"""
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
+
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
+
+ Parameters:
+ config ([`CLIPConfig`]): Model configuration class with all the parameters of the model.
+ Initializing with a config file does not load the weights associated with the model, only the
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
+"""
+
+CLIP_TEXT_INPUTS_DOCSTRING = r"""
+ Args:
+ input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
+ Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
+ it.
+
+ Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
+
+ [What are input IDs?](../glossary#input-ids)
+ attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
+ Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
+
+ - 1 for tokens that are **not masked**,
+ - 0 for tokens that are **masked**.
+
+ [What are attention masks?](../glossary#attention-mask)
+ position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
+
+ [What are position IDs?](../glossary#position-ids)
+ output_attentions (`bool`, *optional*):
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
+ tensors for more detail.
+ output_hidden_states (`bool`, *optional*):
+ Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
+ more detail.
+ return_dict (`bool`, *optional*):
+ Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+"""
+
+CLIP_VISION_INPUTS_DOCSTRING = r"""
+ Args:
+ pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
+ Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using
+ [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details.
+ output_attentions (`bool`, *optional*):
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
+ tensors for more detail.
+ output_hidden_states (`bool`, *optional*):
+ Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
+ more detail.
+ return_dict (`bool`, *optional*):
+ Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+"""
+
+CLIP_INPUTS_DOCSTRING = r"""
+ Args:
+ input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
+ Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
+ it.
+
+ Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
+
+ [What are input IDs?](../glossary#input-ids)
+ attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
+ Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
+
+ - 1 for tokens that are **not masked**,
+ - 0 for tokens that are **masked**.
+
+ [What are attention masks?](../glossary#attention-mask)
+ position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
+
+ [What are position IDs?](../glossary#position-ids)
+ pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
+ Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using
+ [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details.
+ return_loss (`bool`, *optional*):
+ Whether or not to return the contrastive loss.
+ output_attentions (`bool`, *optional*):
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
+ tensors for more detail.
+ output_hidden_states (`bool`, *optional*):
+ Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
+ more detail.
+ return_dict (`bool`, *optional*):
+ Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+"""
+
+
+class CLIPEncoder(nn.Module):
+ """
+ Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
+ [`CLIPEncoderLayer`].
+
+ Args:
+ config: CLIPConfig
+ """
+
+ def __init__(self, config: LanguageBindVideoConfig):
+ super().__init__()
+ self.config = config
+ self.layers = nn.ModuleList([CLIPEncoderLayer(config) for _ in range(config.num_hidden_layers)])
+ self.gradient_checkpointing = False
+
+ def forward(
+ self,
+ inputs_embeds,
+ attention_mask: Optional[torch.Tensor] = None,
+ causal_attention_mask: Optional[torch.Tensor] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> Union[Tuple, BaseModelOutput]:
+ r"""
+ Args:
+ inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
+ This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+ than the model's internal embedding lookup matrix.
+ attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
+ Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
+
+ - 1 for tokens that are **not masked**,
+ - 0 for tokens that are **masked**.
+
+ [What are attention masks?](../glossary#attention-mask)
+ causal_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
+ Causal mask for the text model. Mask values selected in `[0, 1]`:
+
+ - 1 for tokens that are **not masked**,
+ - 0 for tokens that are **masked**.
+
+ [What are attention masks?](../glossary#attention-mask)
+ output_attentions (`bool`, *optional*):
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under
+ returned tensors for more detail.
+ output_hidden_states (`bool`, *optional*):
+ Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
+ for more detail.
+ return_dict (`bool`, *optional*):
+ Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+ """
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+ output_hidden_states = (
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+ )
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+ encoder_states = () if output_hidden_states else None
+ all_attentions = () if output_attentions else None
+
+ hidden_states = inputs_embeds
+ for idx, encoder_layer in enumerate(self.layers):
+ if output_hidden_states:
+ encoder_states = encoder_states + (hidden_states,)
+ if self.gradient_checkpointing and self.training:
+
+ def create_custom_forward(module):
+ def custom_forward(*inputs):
+ return module(*inputs, output_attentions)
+
+ return custom_forward
+
+ layer_outputs = torch.utils.checkpoint.checkpoint(
+ create_custom_forward(encoder_layer),
+ hidden_states,
+ attention_mask,
+ causal_attention_mask,
+ )
+ else:
+ layer_outputs = encoder_layer(
+ hidden_states,
+ attention_mask,
+ causal_attention_mask,
+ output_attentions=output_attentions,
+ )
+
+ hidden_states = layer_outputs[0]
+
+ if output_attentions:
+ all_attentions = all_attentions + (layer_outputs[1],)
+
+ if output_hidden_states:
+ encoder_states = encoder_states + (hidden_states,)
+
+ if not return_dict:
+ return tuple(v for v in [hidden_states, encoder_states, all_attentions] if v is not None)
+ return BaseModelOutput(
+ last_hidden_state=hidden_states, hidden_states=encoder_states, attentions=all_attentions
+ )
+
+
+# Copied from transformers.models.bart.modeling_bart._make_causal_mask
+def _make_causal_mask(
+ input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device, past_key_values_length: int = 0
+):
+ """
+ Make causal mask used for bi-directional self-attention.
+ """
+ bsz, tgt_len = input_ids_shape
+ mask = torch.full((tgt_len, tgt_len), torch.finfo(dtype).min, device=device)
+ mask_cond = torch.arange(mask.size(-1), device=device)
+ mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0)
+ mask = mask.to(dtype)
+
+ if past_key_values_length > 0:
+ mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), mask], dim=-1)
+ return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length)
+
+
+class CLIPTextTransformer(nn.Module):
+ def __init__(self, config: CLIPTextConfig):
+ super().__init__()
+ self.config = config
+ embed_dim = config.hidden_size
+ self.embeddings = CLIPTextEmbeddings(config)
+ self.encoder = CLIPEncoder(config)
+ self.final_layer_norm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
+
+ @add_start_docstrings_to_model_forward(CLIP_TEXT_INPUTS_DOCSTRING)
+ @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=CLIPTextConfig)
+ def forward(
+ self,
+ input_ids: Optional[torch.Tensor] = None,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.Tensor] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> Union[Tuple, BaseModelOutputWithPooling]:
+ r"""
+ Returns:
+
+ """
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+ output_hidden_states = (
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+ )
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+ if input_ids is None:
+ raise ValueError("You have to specify input_ids")
+
+ input_shape = input_ids.size()
+ input_ids = input_ids.view(-1, input_shape[-1])
+
+ hidden_states = self.embeddings(input_ids=input_ids, position_ids=position_ids)
+
+ # CLIP's text model uses causal mask, prepare it here.
+ # https://github.com/openai/CLIP/blob/cfcffb90e69f37bf2ff1e988237a0fbe41f33c04/clip/model.py#L324
+ causal_attention_mask = _make_causal_mask(input_shape, hidden_states.dtype, device=hidden_states.device)
+ # expand attention_mask
+ if attention_mask is not None:
+ # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]
+ attention_mask = _expand_mask(attention_mask, hidden_states.dtype)
+
+ encoder_outputs = self.encoder(
+ inputs_embeds=hidden_states,
+ attention_mask=attention_mask,
+ causal_attention_mask=causal_attention_mask,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+
+ last_hidden_state = encoder_outputs[0]
+ last_hidden_state = self.final_layer_norm(last_hidden_state)
+
+ # text_embeds.shape = [batch_size, sequence_length, transformer.width]
+ # take features from the eot embedding (eot_token is the highest number in each sequence)
+ # casting to torch.int for onnx compatibility: argmax doesn't support int64 inputs with opset 14
+ pooled_output = last_hidden_state[
+ torch.arange(last_hidden_state.shape[0], device=last_hidden_state.device),
+ input_ids.to(dtype=torch.int, device=last_hidden_state.device).argmax(dim=-1),
+ ]
+
+ if not return_dict:
+ return (last_hidden_state, pooled_output) + encoder_outputs[1:]
+
+ return BaseModelOutputWithPooling(
+ last_hidden_state=last_hidden_state,
+ pooler_output=pooled_output,
+ hidden_states=encoder_outputs.hidden_states,
+ attentions=encoder_outputs.attentions,
+ )
+
+
+@add_start_docstrings(
+ """The text model from CLIP without any head or projection on top.""",
+ CLIP_START_DOCSTRING,
+)
+class CLIPTextModel(CLIPPreTrainedModel):
+ config_class = CLIPTextConfig
+
+ _no_split_modules = ["CLIPEncoderLayer"]
+
+ def __init__(self, config: CLIPTextConfig):
+ super().__init__(config)
+ self.text_model = CLIPTextTransformer(config)
+ # Initialize weights and apply final processing
+ self.post_init()
+
+ def get_input_embeddings(self) -> nn.Module:
+ return self.text_model.embeddings.token_embedding
+
+ def set_input_embeddings(self, value):
+ self.text_model.embeddings.token_embedding = value
+
+ @add_start_docstrings_to_model_forward(CLIP_TEXT_INPUTS_DOCSTRING)
+ @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=CLIPTextConfig)
+ def forward(
+ self,
+ input_ids: Optional[torch.Tensor] = None,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.Tensor] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> Union[Tuple, BaseModelOutputWithPooling]:
+ r"""
+ Returns:
+
+ Examples:
+
+ ```python
+ >>> from transformers import AutoTokenizer, CLIPTextModel
+
+ >>> model = CLIPTextModel.from_pretrained("openai/clip-vit-base-patch32")
+ >>> tokenizer = AutoTokenizer.from_pretrained("openai/clip-vit-base-patch32")
+
+ >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt")
+
+ >>> outputs = model(**inputs)
+ >>> last_hidden_state = outputs.last_hidden_state
+ >>> pooled_output = outputs.pooler_output # pooled (EOS token) states
+ ```"""
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+ return self.text_model(
+ input_ids=input_ids,
+ attention_mask=attention_mask,
+ position_ids=position_ids,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+
+
+class CLIPVisionTransformer(nn.Module):
+ def __init__(self, config: CLIPVisionConfig):
+ super().__init__()
+ self.config = config
+ embed_dim = config.hidden_size
+
+ self.embeddings = CLIPVisionEmbeddings(config)
+ self.patch_dropout = PatchDropout(config.force_patch_dropout)
+ self.pre_layrnorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
+ self.encoder = CLIPEncoder(config)
+ self.post_layernorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
+
+ @add_start_docstrings_to_model_forward(CLIP_VISION_INPUTS_DOCSTRING)
+ @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=CLIPVisionConfig)
+ def forward(
+ self,
+ pixel_values: Optional[torch.FloatTensor] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> Union[Tuple, BaseModelOutputWithPooling]:
+ r"""
+ Returns:
+
+ """
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+ output_hidden_states = (
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+ )
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+ # print('input video raw shape', pixel_values.shape)
+
+ if pixel_values is None:
+ raise ValueError("You have to specify pixel_values")
+ ######################################
+ if len(pixel_values.shape) == 7:
+ b_new, pair_new, T, bs_new, channel_new, h_new, w_new = pixel_values.shape
+ # print(pixel_values.shape)
+ B = b_new * pair_new * bs_new
+ pixel_values = pixel_values.reshape(B*T, channel_new, h_new, w_new)
+
+ elif len(pixel_values.shape) == 5:
+ B, _, T, _, _ = pixel_values.shape
+ # print(pixel_values.shape)
+ pixel_values = rearrange(pixel_values, 'b c t h w -> (b t) c h w')
+ else:
+ # print(pixel_values.shape)
+ B, _, _, _ = pixel_values.shape
+ T = 1
+ ###########################
+ hidden_states = self.embeddings(pixel_values)
+ # print(B, T)
+ hidden_states = self.patch_dropout(hidden_states, B, T) ##############################################
+
+ hidden_states = self.pre_layrnorm(hidden_states)
+
+ encoder_outputs = self.encoder(
+ inputs_embeds=hidden_states,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+
+ last_hidden_state = encoder_outputs[0]
+ # print('video encoder last_hidden_state', last_hidden_state.shape)
+ pooled_output = last_hidden_state[:, 0, :]
+ pooled_output = self.post_layernorm(pooled_output)
+
+ pooled_output = pooled_output.reshape(B, T, -1).mean(1) ################################
+ #################################
+ encoder_outputs.hidden_states = [rearrange(i, '(b t) n c -> b t n c', b=B) for i in encoder_outputs.hidden_states]
+ if not return_dict:
+ return (last_hidden_state, pooled_output) + encoder_outputs[1:]
+
+ return BaseModelOutputWithPooling(
+ last_hidden_state=last_hidden_state,
+ pooler_output=pooled_output,
+ hidden_states=encoder_outputs.hidden_states,
+ attentions=encoder_outputs.attentions,
+ )
+
+
+@add_start_docstrings(
+ """The vision model from CLIP without any head or projection on top.""",
+ CLIP_START_DOCSTRING,
+)
+class CLIPVisionModel(CLIPPreTrainedModel):
+ config_class = CLIPVisionConfig
+ main_input_name = "pixel_values"
+
+ def __init__(self, config: CLIPVisionConfig):
+ super().__init__(config)
+ self.vision_model = CLIPVisionTransformer(config)
+ # Initialize weights and apply final processing
+ self.post_init()
+
+ def get_input_embeddings(self) -> nn.Module:
+ return self.vision_model.embeddings.patch_embedding
+
+ @add_start_docstrings_to_model_forward(CLIP_VISION_INPUTS_DOCSTRING)
+ @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=CLIPVisionConfig)
+ def forward(
+ self,
+ pixel_values: Optional[torch.FloatTensor] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> Union[Tuple, BaseModelOutputWithPooling]:
+ r"""
+ Returns:
+
+ Examples:
+
+ ```python
+ >>> from PIL import Image
+ >>> import requests
+ >>> from transformers import AutoProcessor, CLIPVisionModel
+
+ >>> model = CLIPVisionModel.from_pretrained("openai/clip-vit-base-patch32")
+ >>> processor = AutoProcessor.from_pretrained("openai/clip-vit-base-patch32")
+
+ >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
+ >>> image = Image.open(requests.get(url, stream=True).raw)
+
+ >>> inputs = processor(images=image, return_tensors="pt")
+
+ >>> outputs = model(**inputs)
+ >>> last_hidden_state = outputs.last_hidden_state
+ >>> pooled_output = outputs.pooler_output # pooled CLS states
+ ```"""
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+ return self.vision_model(
+ pixel_values=pixel_values,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+
+
+@add_start_docstrings(CLIP_START_DOCSTRING)
+class LanguageBindVideo(CLIPPreTrainedModel):
+ config_class = LanguageBindVideoConfig
+
+ def __init__(self, config: LanguageBindVideoConfig):
+ super().__init__(config)
+
+ if not isinstance(config.text_config, CLIPTextConfig):
+ raise ValueError(
+ "config.text_config is expected to be of type CLIPTextConfig but is of type"
+ f" {type(config.text_config)}."
+ )
+
+ if not isinstance(config.vision_config, CLIPVisionConfig):
+ raise ValueError(
+ "config.vision_config is expected to be of type CLIPVisionConfig but is of type"
+ f" {type(config.vision_config)}."
+ )
+
+ text_config = config.text_config
+ vision_config = config.vision_config
+ self.add_time_attn = vision_config.add_time_attn
+ self.lora_r = vision_config.lora_r
+ self.lora_alpha = vision_config.lora_alpha
+ self.lora_dropout = vision_config.lora_dropout
+
+ self.projection_dim = config.projection_dim
+ self.text_embed_dim = text_config.hidden_size
+ self.vision_embed_dim = vision_config.hidden_size
+
+ self.text_model = CLIPTextTransformer(text_config)
+ self.vision_model = CLIPVisionTransformer(vision_config)
+
+ self.visual_projection = nn.Linear(self.vision_embed_dim, self.projection_dim, bias=False)
+ self.text_projection = nn.Linear(self.text_embed_dim, self.projection_dim, bias=False)
+ self.logit_scale = nn.Parameter(torch.tensor(self.config.logit_scale_init_value))
+
+ # Initialize weights and apply final processing
+ self.post_init()
+ # self.convert_to_lora() ############################################
+ # self.resize_pos(self.vision_model.embeddings, vision_config)
+
+ def convert_to_lora(self):
+ if self.lora_r == 0:
+ return
+ if self.add_time_attn:
+ target_modules = ["temporal_attn.k_proj", "temporal_attn.v_proj",
+ "temporal_attn.q_proj", "temporal_attn.out_proj",
+ "temporal_mlp.fc1", "temporal_mlp.fc2"]
+ else:
+ target_modules = ["k_proj", "v_proj", "q_proj", "out_proj"]
+ config = LoraConfig(
+ r=self.lora_r, # 16
+ lora_alpha=self.lora_alpha, # 16
+ target_modules=target_modules, # self_attn.out_proj
+ lora_dropout=self.lora_dropout, # 0.1
+ bias="none",
+ modules_to_save=[],
+ )
+ self.vision_model.encoder.is_gradient_checkpointing = False
+ self.vision_model.encoder = get_peft_model(self.vision_model.encoder, config)
+
+ def resize_pos(self, m, vision_config):
+ # convert embedding
+ if vision_config.num_mel_bins!=0 and vision_config.target_length!=0:
+ m.image_size = [vision_config.num_mel_bins, vision_config.target_length]
+ m.config.image_size = [m.image_size, m.image_size] if isinstance(m.image_size, int) else m.image_size
+ # pos resize
+ old_pos_embed_state_dict = m.position_embedding.state_dict()
+ old_pos_embed = old_pos_embed_state_dict['weight']
+ dtype = old_pos_embed.dtype
+ grid_size = [m.config.image_size[0] // m.patch_size, m.config.image_size[1] // m.patch_size]
+ extra_tokens = 1 # FIXME detect different token configs (ie no class token, or more)
+ new_seq_len = grid_size[0] * grid_size[1] + extra_tokens
+ if new_seq_len == old_pos_embed.shape[0]:
+ # m.to(args.device)
+ return
+
+ m.num_patches = grid_size[0] * grid_size[1]
+ m.num_positions = m.num_patches + 1
+ m.register_buffer("position_ids", torch.arange(m.num_positions).expand((1, -1)))
+ new_position_embedding = nn.Embedding(m.num_positions, m.embed_dim)
+
+ if extra_tokens:
+ pos_emb_tok, pos_emb_img = old_pos_embed[:extra_tokens], old_pos_embed[extra_tokens:]
+ else:
+ pos_emb_tok, pos_emb_img = None, old_pos_embed
+ old_grid_size = [int(math.sqrt(len(pos_emb_img)))] * 2
+
+ # if is_master(args):
+ # logging.info('Resizing position embedding grid-size from %s to %s', old_grid_size, grid_size)
+ pos_emb_img = pos_emb_img.reshape(1, old_grid_size[0], old_grid_size[1], -1).permute(0, 3, 1, 2)
+ pos_emb_img = F.interpolate(
+ pos_emb_img,
+ size=grid_size,
+ mode='bicubic',
+ antialias=True,
+ align_corners=False,
+ )
+ pos_emb_img = pos_emb_img.permute(0, 2, 3, 1).reshape(1, grid_size[0] * grid_size[1], -1)[0]
+ if pos_emb_tok is not None:
+ new_pos_embed = torch.cat([pos_emb_tok, pos_emb_img], dim=0)
+ else:
+ new_pos_embed = pos_emb_img
+ old_pos_embed_state_dict['weight'] = new_pos_embed.to(dtype)
+ m.position_embedding = new_position_embedding
+ m.position_embedding.load_state_dict(old_pos_embed_state_dict)
+
+ # m.to(args.device)
+
+ @add_start_docstrings_to_model_forward(CLIP_TEXT_INPUTS_DOCSTRING)
+ def get_text_features(
+ self,
+ input_ids: Optional[torch.Tensor] = None,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.Tensor] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> torch.FloatTensor:
+ r"""
+ Returns:
+ text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by
+ applying the projection layer to the pooled output of [`CLIPTextModel`].
+
+ Examples:
+
+ ```python
+ >>> from transformers import AutoTokenizer, CLIPModel
+
+ >>> model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")
+ >>> tokenizer = AutoTokenizer.from_pretrained("openai/clip-vit-base-patch32")
+
+ >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt")
+ >>> text_features = model.get_text_features(**inputs)
+ ```"""
+ # Use CLIP model's config for some fields (if specified) instead of those of vision & text components.
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+ output_hidden_states = (
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+ )
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+ text_outputs = self.text_model(
+ input_ids=input_ids,
+ attention_mask=attention_mask,
+ position_ids=position_ids,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+
+ pooled_output = text_outputs[1]
+ text_features = self.text_projection(pooled_output)
+
+ return text_features
+
+ @add_start_docstrings_to_model_forward(CLIP_VISION_INPUTS_DOCSTRING)
+ def get_image_features(
+ self,
+ pixel_values: Optional[torch.FloatTensor] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> torch.FloatTensor:
+ r"""
+ Returns:
+ image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by
+ applying the projection layer to the pooled output of [`CLIPVisionModel`].
+
+ Examples:
+
+ ```python
+ >>> from PIL import Image
+ >>> import requests
+ >>> from transformers import AutoProcessor, CLIPModel
+
+ >>> model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")
+ >>> processor = AutoProcessor.from_pretrained("openai/clip-vit-base-patch32")
+
+ >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
+ >>> image = Image.open(requests.get(url, stream=True).raw)
+
+ >>> inputs = processor(images=image, return_tensors="pt")
+
+ >>> image_features = model.get_image_features(**inputs)
+ ```"""
+ # Use CLIP model's config for some fields (if specified) instead of those of vision & text components.
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+ output_hidden_states = (
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+ )
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+ vision_outputs = self.vision_model(
+ pixel_values=pixel_values,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+
+ pooled_output = vision_outputs[1] # pooled_output
+ image_features = self.visual_projection(pooled_output)
+
+ return image_features
+
+ @add_start_docstrings_to_model_forward(CLIP_INPUTS_DOCSTRING)
+ @replace_return_docstrings(output_type=CLIPOutput, config_class=LanguageBindVideoConfig)
+ def forward(
+ self,
+ input_ids: Optional[torch.LongTensor] = None,
+ pixel_values: Optional[torch.FloatTensor] = None,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.LongTensor] = None,
+ return_loss: Optional[bool] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> Union[Tuple, CLIPOutput]:
+ r"""
+ Returns:
+
+ Examples:
+
+ ```python
+ >>> from PIL import Image
+ >>> import requests
+ >>> from transformers import AutoProcessor, CLIPModel
+
+ >>> model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")
+ >>> processor = AutoProcessor.from_pretrained("openai/clip-vit-base-patch32")
+
+ >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
+ >>> image = Image.open(requests.get(url, stream=True).raw)
+
+ >>> inputs = processor(
+ ... text=["a photo of a cat", "a photo of a dog"], images=image, return_tensors="pt", padding=True
+ ... )
+
+ >>> outputs = model(**inputs)
+ >>> logits_per_image = outputs.logits_per_image # this is the image-text similarity score
+ >>> probs = logits_per_image.softmax(dim=1) # we can take the softmax to get the label probabilities
+ ```"""
+ # Use CLIP model's config for some fields (if specified) instead of those of vision & text components.
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+ output_hidden_states = (
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+ )
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+ vision_outputs = self.vision_model(
+ pixel_values=pixel_values,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+
+ text_outputs = self.text_model(
+ input_ids=input_ids,
+ attention_mask=attention_mask,
+ position_ids=position_ids,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+
+ image_embeds = vision_outputs[1]
+ image_embeds = self.visual_projection(image_embeds)
+
+ text_embeds = text_outputs[1]
+ text_embeds = self.text_projection(text_embeds)
+
+ # normalized features
+ image_embeds = image_embeds / image_embeds.norm(p=2, dim=-1, keepdim=True)
+ text_embeds = text_embeds / text_embeds.norm(p=2, dim=-1, keepdim=True)
+
+ # cosine similarity as logits
+ logit_scale = self.logit_scale.exp()
+ logits_per_text = torch.matmul(text_embeds, image_embeds.t()) * logit_scale
+ logits_per_image = logits_per_text.t()
+
+ loss = None
+ if return_loss:
+ loss = clip_loss(logits_per_text)
+
+ if not return_dict:
+ output = (logits_per_image, logits_per_text, text_embeds, image_embeds, text_outputs, vision_outputs)
+ return ((loss,) + output) if loss is not None else output
+
+ return CLIPOutput(
+ loss=loss,
+ logits_per_image=logits_per_image,
+ logits_per_text=logits_per_text,
+ text_embeds=text_embeds,
+ image_embeds=image_embeds,
+ text_model_output=text_outputs,
+ vision_model_output=vision_outputs,
+ )
\ No newline at end of file
diff --git a/videollava/model/multimodal_encoder/languagebind/video/processing_video.py b/videollava/model/multimodal_encoder/languagebind/video/processing_video.py
new file mode 100644
index 0000000000000000000000000000000000000000..d2151505cbe1485e2b1f91cac493b12cbebaad0e
--- /dev/null
+++ b/videollava/model/multimodal_encoder/languagebind/video/processing_video.py
@@ -0,0 +1,165 @@
+
+import torch
+import cv2
+import decord
+import numpy as np
+from PIL import Image
+from decord import VideoReader, cpu
+from torchvision import transforms
+from transformers import ProcessorMixin, BatchEncoding
+from transformers.image_processing_utils import BatchFeature
+from pytorchvideo.data.encoded_video import EncodedVideo
+from torchvision.transforms import Compose, Lambda, ToTensor
+from torchvision.transforms._transforms_video import NormalizeVideo, RandomCropVideo, RandomHorizontalFlipVideo, CenterCropVideo
+# from pytorchvideo.transforms import ApplyTransformToKey, ShortSideScale, UniformTemporalSubsample
+
+decord.bridge.set_bridge('torch')
+
+OPENAI_DATASET_MEAN = (0.48145466, 0.4578275, 0.40821073)
+OPENAI_DATASET_STD = (0.26862954, 0.26130258, 0.27577711)
+
+def make_list_of_images(x):
+ if not isinstance(x, list):
+ return [x]
+ return x
+
+def get_video_transform(config):
+ config = config.vision_config
+ if config.video_decode_backend == 'pytorchvideo':
+ transform = ApplyTransformToKey(
+ key="video",
+ transform=Compose(
+ [
+ UniformTemporalSubsample(config.num_frames),
+ Lambda(lambda x: x / 255.0),
+ NormalizeVideo(mean=OPENAI_DATASET_MEAN, std=OPENAI_DATASET_STD),
+ ShortSideScale(size=224),
+ CenterCropVideo(224),
+ RandomHorizontalFlipVideo(p=0.5),
+ ]
+ ),
+ )
+
+ elif config.video_decode_backend == 'decord':
+
+ transform = Compose(
+ [
+ # UniformTemporalSubsample(num_frames),
+ Lambda(lambda x: x / 255.0),
+ NormalizeVideo(mean=OPENAI_DATASET_MEAN, std=OPENAI_DATASET_STD),
+ # ShortSideScale(size=224),
+ CenterCropVideo(224),
+ RandomHorizontalFlipVideo(p=0.5),
+ ]
+ )
+
+ elif config.video_decode_backend == 'opencv':
+ transform = Compose(
+ [
+ # UniformTemporalSubsample(num_frames),
+ Lambda(lambda x: x / 255.0),
+ NormalizeVideo(mean=OPENAI_DATASET_MEAN, std=OPENAI_DATASET_STD),
+ ShortSideScale(size=224),
+ CenterCropVideo(224),
+ RandomHorizontalFlipVideo(p=0.5),
+ ]
+ )
+ else:
+ raise NameError('video_decode_backend should specify in (pytorchvideo, decord, opencv)')
+ return transform
+
+
+def load_and_transform_video(
+ video_path,
+ transform,
+ video_decode_backend='opencv',
+ clip_start_sec=0.0,
+ clip_end_sec=None,
+ num_frames=8,
+):
+ if video_decode_backend == 'pytorchvideo':
+ # decord pyav
+ video = EncodedVideo.from_path(video_path, decoder="decord", decode_audio=False)
+ duration = video.duration
+ start_sec = clip_start_sec # secs
+ end_sec = clip_end_sec if clip_end_sec is not None else duration # secs
+ video_data = video.get_clip(start_sec=start_sec, end_sec=end_sec)
+ video_outputs = transform(video_data)
+
+ elif video_decode_backend == 'decord':
+ decord.bridge.set_bridge('torch')
+ decord_vr = VideoReader(video_path, ctx=cpu(0))
+ duration = len(decord_vr)
+ frame_id_list = np.linspace(0, duration-1, num_frames, dtype=int)
+ video_data = decord_vr.get_batch(frame_id_list)
+ video_data = video_data.permute(3, 0, 1, 2) # (T, H, W, C) -> (C, T, H, W)
+ video_outputs = transform(video_data)
+
+ elif video_decode_backend == 'opencv':
+ cv2_vr = cv2.VideoCapture(video_path)
+ duration = int(cv2_vr.get(cv2.CAP_PROP_FRAME_COUNT))
+ frame_id_list = np.linspace(0, duration-1, num_frames, dtype=int)
+
+ video_data = []
+ for frame_idx in frame_id_list:
+ cv2_vr.set(1, frame_idx)
+ _, frame = cv2_vr.read()
+ frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+ video_data.append(torch.from_numpy(frame).permute(2, 0, 1))
+ cv2_vr.release()
+ video_data = torch.stack(video_data, dim=1)
+ video_outputs = transform(video_data)
+ else:
+ raise NameError('video_decode_backend should specify in (pytorchvideo, decord, opencv)')
+ return video_outputs
+
+class LanguageBindVideoProcessor(ProcessorMixin):
+ attributes = []
+ tokenizer_class = ("LanguageBindVideoTokenizer")
+
+ def __init__(self, config, tokenizer=None, **kwargs):
+ super().__init__(**kwargs)
+ self.config = config
+ self.transform = get_video_transform(config)
+ self.image_processor = load_and_transform_video
+ self.tokenizer = tokenizer
+
+ def __call__(self, images=None, text=None, context_length=77, return_tensors=None, **kwargs):
+ if text is None and images is None:
+ raise ValueError("You have to specify either text or images. Both cannot be none.")
+
+ if text is not None:
+ encoding = self.tokenizer(text, max_length=context_length, padding='max_length',
+ truncation=True, return_tensors=return_tensors, **kwargs)
+
+ if images is not None:
+ images = make_list_of_images(images)
+ image_features = [self.image_processor(image, self.transform,
+ video_decode_backend=self.config.vision_config.video_decode_backend,
+ num_frames=self.config.vision_config.num_frames) for image in images]
+ image_features = torch.stack(image_features)
+
+ if text is not None and images is not None:
+ encoding["pixel_values"] = image_features
+ return encoding
+ elif text is not None:
+ return encoding
+ else:
+ return {"pixel_values": image_features}
+
+ def preprocess(self, images, return_tensors):
+ return self.__call__(images=images, return_tensors=return_tensors)
+
+ def batch_decode(self, skip_special_tokens=True, *args, **kwargs):
+ """
+ This method forwards all its arguments to CLIPTokenizerFast's [`~PreTrainedTokenizer.batch_decode`]. Please
+ refer to the docstring of this method for more information.
+ """
+ return self.tokenizer.batch_decode(*args, skip_special_tokens=skip_special_tokens, **kwargs)
+
+ def decode(self, skip_special_tokens=True, *args, **kwargs):
+ """
+ This method forwards all its arguments to CLIPTokenizerFast's [`~PreTrainedTokenizer.decode`]. Please refer to
+ the docstring of this method for more information.
+ """
+ return self.tokenizer.decode(*args, skip_special_tokens=skip_special_tokens, **kwargs)
diff --git a/videollava/model/multimodal_encoder/languagebind/video/tokenization_video.py b/videollava/model/multimodal_encoder/languagebind/video/tokenization_video.py
new file mode 100644
index 0000000000000000000000000000000000000000..2864429c098770fd37fd61e8a7b82d1fee5b12dd
--- /dev/null
+++ b/videollava/model/multimodal_encoder/languagebind/video/tokenization_video.py
@@ -0,0 +1,77 @@
+from transformers import CLIPTokenizer
+from transformers.utils import logging
+
+logger = logging.get_logger(__name__)
+
+VOCAB_FILES_NAMES = {
+ "vocab_file": "vocab.json",
+ "merges_file": "merges.txt",
+}
+
+PRETRAINED_VOCAB_FILES_MAP = {
+ "vocab_file": {
+ "lb203/LanguageBind-Video": "https://huggingface.co/lb203/LanguageBind-Video/resolve/main/vocab.json",
+ },
+ "merges_file": {
+ "lb203/LanguageBind-Video": "https://huggingface.co/lb203/LanguageBind-Video/resolve/main/merges.txt",
+ },
+}
+
+PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
+ "lb203/LanguageBind-Video": 77,
+}
+
+
+PRETRAINED_INIT_CONFIGURATION = {
+ "lb203/LanguageBind-Video": {},
+}
+
+class LanguageBindVideoTokenizer(CLIPTokenizer):
+ """
+ Construct a CLIP tokenizer. Based on byte-level Byte-Pair-Encoding.
+
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+ this superclass for more information regarding those methods.
+
+ Args:
+ vocab_file (`str`):
+ Path to the vocabulary file.
+ merges_file (`str`):
+ Path to the merges file.
+ errors (`str`, *optional*, defaults to `"replace"`):
+ Paradigm to follow when decoding bytes to UTF-8. See
+ [bytes.decode](https://docs.python.org/3/library/stdtypes.html#bytes.decode) for more information.
+ unk_token (`str`, *optional*, defaults to `<|endoftext|>`):
+ The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
+ token instead.
+ bos_token (`str`, *optional*, defaults to `<|startoftext|>`):
+ The beginning of sequence token.
+ eos_token (`str`, *optional*, defaults to `<|endoftext|>`):
+ The end of sequence token.
+ """
+
+ vocab_files_names = VOCAB_FILES_NAMES
+ pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+ max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
+ model_input_names = ["input_ids", "attention_mask"]
+
+ def __init__(
+ self,
+ vocab_file,
+ merges_file,
+ errors="replace",
+ unk_token="<|endoftext|>",
+ bos_token="<|startoftext|>",
+ eos_token="<|endoftext|>",
+ pad_token="<|endoftext|>", # hack to enable padding
+ **kwargs,
+ ):
+ super(LanguageBindVideoTokenizer, self).__init__(
+ vocab_file,
+ merges_file,
+ errors,
+ unk_token,
+ bos_token,
+ eos_token,
+ pad_token, # hack to enable padding
+ **kwargs,)
\ No newline at end of file
diff --git a/videollava/model/multimodal_projector/builder.py b/videollava/model/multimodal_projector/builder.py
new file mode 100644
index 0000000000000000000000000000000000000000..31cd4f48e6055cd6d00a162af30b1c8139e26b57
--- /dev/null
+++ b/videollava/model/multimodal_projector/builder.py
@@ -0,0 +1,51 @@
+import torch
+import torch.nn as nn
+import re
+
+
+class IdentityMap(nn.Module):
+ def __init__(self):
+ super().__init__()
+
+ def forward(self, x, *args, **kwargs):
+ return x
+
+ @property
+ def config(self):
+ return {"mm_projector_type": 'identity'}
+
+
+class SimpleResBlock(nn.Module):
+ def __init__(self, channels):
+ super().__init__()
+ self.pre_norm = nn.LayerNorm(channels)
+
+ self.proj = nn.Sequential(
+ nn.Linear(channels, channels),
+ nn.GELU(),
+ nn.Linear(channels, channels)
+ )
+ def forward(self, x):
+ x = self.pre_norm(x)
+ return x + self.proj(x)
+
+
+def build_vision_projector(config, delay_load=False, **kwargs):
+ projector_type = getattr(config, 'mm_projector_type', 'linear')
+
+ if projector_type == 'linear':
+ return nn.Linear(config.mm_hidden_size, config.hidden_size)
+
+ mlp_gelu_match = re.match(r'^mlp(\d+)x_gelu$', projector_type)
+ if mlp_gelu_match:
+ mlp_depth = int(mlp_gelu_match.group(1))
+ modules = [nn.Linear(config.mm_hidden_size, config.hidden_size)]
+ for _ in range(1, mlp_depth):
+ modules.append(nn.GELU())
+ modules.append(nn.Linear(config.hidden_size, config.hidden_size))
+ return nn.Sequential(*modules)
+
+ if projector_type == 'identity':
+ return IdentityMap()
+
+ raise ValueError(f'Unknown projector type: {projector_type}')
diff --git a/videollava/model/utils.py b/videollava/model/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..2563f89c6cedf5e73508afec8f9979105df9b745
--- /dev/null
+++ b/videollava/model/utils.py
@@ -0,0 +1,20 @@
+from transformers import AutoConfig
+
+
+def auto_upgrade(config):
+ cfg = AutoConfig.from_pretrained(config)
+ if 'llava' in config and 'llava' not in cfg.model_type:
+ assert cfg.model_type == 'llama'
+ print("You are using newer LLaVA code base, while the checkpoint of v0 is from older code base.")
+ print("You must upgrade the checkpoint to the new code base (this can be done automatically).")
+ confirm = input("Please confirm that you want to upgrade the checkpoint. [Y/N]")
+ if confirm.lower() in ["y", "yes"]:
+ print("Upgrading checkpoint...")
+ assert len(cfg.architectures) == 1
+ setattr(cfg.__class__, "model_type", "llava")
+ cfg.architectures[0] = 'LlavaLlamaForCausalLM'
+ cfg.save_pretrained(config)
+ print("Checkpoint upgraded.")
+ else:
+ print("Checkpoint upgrade aborted.")
+ exit(1)
diff --git a/videollava/serve/examples/AID_cls.png b/videollava/serve/examples/AID_cls.png
new file mode 100644
index 0000000000000000000000000000000000000000..74c265c7b9c8677bc70fc1993943a6c0cbe407f1
Binary files /dev/null and b/videollava/serve/examples/AID_cls.png differ
diff --git a/videollava/serve/examples/HRBEN_qa.png b/videollava/serve/examples/HRBEN_qa.png
new file mode 100644
index 0000000000000000000000000000000000000000..7b898df1a6f4c221fb4a8595969f068d8c9896a5
Binary files /dev/null and b/videollava/serve/examples/HRBEN_qa.png differ
diff --git a/videollava/serve/examples/QFabric_rtqa_1.png b/videollava/serve/examples/QFabric_rtqa_1.png
new file mode 100644
index 0000000000000000000000000000000000000000..d14cbcf6924d925d87aa8a6dbf66e21dabe07cac
Binary files /dev/null and b/videollava/serve/examples/QFabric_rtqa_1.png differ
diff --git a/videollava/serve/examples/QFabric_rtqa_2.png b/videollava/serve/examples/QFabric_rtqa_2.png
new file mode 100644
index 0000000000000000000000000000000000000000..d14cbcf6924d925d87aa8a6dbf66e21dabe07cac
Binary files /dev/null and b/videollava/serve/examples/QFabric_rtqa_2.png differ
diff --git a/videollava/serve/examples/QFabric_rtqa_3.png b/videollava/serve/examples/QFabric_rtqa_3.png
new file mode 100644
index 0000000000000000000000000000000000000000..bdc70ed92f4e900fb55a50e859c437b1d9133572
Binary files /dev/null and b/videollava/serve/examples/QFabric_rtqa_3.png differ
diff --git a/videollava/serve/examples/QFabric_rtqa_4.png b/videollava/serve/examples/QFabric_rtqa_4.png
new file mode 100644
index 0000000000000000000000000000000000000000..82bed7221b32af600779ca47141871961199ffd9
Binary files /dev/null and b/videollava/serve/examples/QFabric_rtqa_4.png differ
diff --git a/videollava/serve/examples/QFabric_rtqa_5.png b/videollava/serve/examples/QFabric_rtqa_5.png
new file mode 100644
index 0000000000000000000000000000000000000000..24a6f941a56664cb6be0cc0daa2e0ec17e5899aa
Binary files /dev/null and b/videollava/serve/examples/QFabric_rtqa_5.png differ
diff --git a/videollava/serve/examples/S2Looking_cd_1.png b/videollava/serve/examples/S2Looking_cd_1.png
new file mode 100644
index 0000000000000000000000000000000000000000..6aae02a3a01ad7fe107abe42b3ca63813bb2ac7b
Binary files /dev/null and b/videollava/serve/examples/S2Looking_cd_1.png differ
diff --git a/videollava/serve/examples/S2Looking_cd_2.png b/videollava/serve/examples/S2Looking_cd_2.png
new file mode 100644
index 0000000000000000000000000000000000000000..2ebddcaf2266583e42d6d3356b6c89ec4fc2a821
Binary files /dev/null and b/videollava/serve/examples/S2Looking_cd_2.png differ
diff --git a/videollava/serve/examples/fMoW_cls_1.png b/videollava/serve/examples/fMoW_cls_1.png
new file mode 100644
index 0000000000000000000000000000000000000000..f763b41843e4f85acb46ec1d35939f265e570516
Binary files /dev/null and b/videollava/serve/examples/fMoW_cls_1.png differ
diff --git a/videollava/serve/examples/fMoW_cls_2.png b/videollava/serve/examples/fMoW_cls_2.png
new file mode 100644
index 0000000000000000000000000000000000000000..faa293c60a47e40394d8a9c8ee8b86d7cea0aad1
Binary files /dev/null and b/videollava/serve/examples/fMoW_cls_2.png differ
diff --git a/videollava/serve/examples/fMoW_cls_3.png b/videollava/serve/examples/fMoW_cls_3.png
new file mode 100644
index 0000000000000000000000000000000000000000..73fb71eec68fc8099571c36d15c6ef2cde463c2d
Binary files /dev/null and b/videollava/serve/examples/fMoW_cls_3.png differ
diff --git a/videollava/serve/examples/fMoW_cls_4.png b/videollava/serve/examples/fMoW_cls_4.png
new file mode 100644
index 0000000000000000000000000000000000000000..0815dfa46144aaabfe898941f84348a3b92f77df
Binary files /dev/null and b/videollava/serve/examples/fMoW_cls_4.png differ
diff --git a/videollava/serve/examples/rqa.png b/videollava/serve/examples/rqa.png
new file mode 100644
index 0000000000000000000000000000000000000000..9ebe9632d189d700dc040b944d079a8ddba80d00
Binary files /dev/null and b/videollava/serve/examples/rqa.png differ
diff --git a/videollava/serve/examples/xBD_cls_1.png b/videollava/serve/examples/xBD_cls_1.png
new file mode 100644
index 0000000000000000000000000000000000000000..f4b07542039c7126929222f0eb9ab9271702fbaf
Binary files /dev/null and b/videollava/serve/examples/xBD_cls_1.png differ
diff --git a/videollava/serve/examples/xBD_cls_2.png b/videollava/serve/examples/xBD_cls_2.png
new file mode 100644
index 0000000000000000000000000000000000000000..fffafac9a503a420d388bc1a6b4fbee25c6f580e
Binary files /dev/null and b/videollava/serve/examples/xBD_cls_2.png differ
diff --git a/videollava/serve/examples/xBD_dis_1.png b/videollava/serve/examples/xBD_dis_1.png
new file mode 100644
index 0000000000000000000000000000000000000000..32fdd88e87ccc0b428ff677125b70e4e56511086
Binary files /dev/null and b/videollava/serve/examples/xBD_dis_1.png differ
diff --git a/videollava/serve/examples/xBD_dis_2.png b/videollava/serve/examples/xBD_dis_2.png
new file mode 100644
index 0000000000000000000000000000000000000000..eeb3e004efd520919536444806c4c05127dba5e6
Binary files /dev/null and b/videollava/serve/examples/xBD_dis_2.png differ
diff --git a/videollava/serve/examples/xBD_loc.png b/videollava/serve/examples/xBD_loc.png
new file mode 100644
index 0000000000000000000000000000000000000000..e4cdcd4830bae95603556ed3d1562826671dfe2d
Binary files /dev/null and b/videollava/serve/examples/xBD_loc.png differ
diff --git a/videollava/serve/teochat_demo.py b/videollava/serve/teochat_demo.py
new file mode 100644
index 0000000000000000000000000000000000000000..b77e688768f7a439008903776acb37782b9d2bd5
--- /dev/null
+++ b/videollava/serve/teochat_demo.py
@@ -0,0 +1,943 @@
+import os
+import re
+import io
+import cv2
+import json
+import torch
+import random
+import argparse
+import tempfile
+import numpy as np
+import gradio as gr
+import plotly.graph_objects as go
+import torchvision.transforms as T
+import torch.backends.cudnn as cudnn
+
+from PIL import Image
+from gradio import Brush
+from gradio.themes.utils import sizes
+from pathlib import Path
+from collections import defaultdict
+
+# Add the grandparent directory to the path
+# This is necessary to import the videollava package
+import sys
+sys.path.append(str(Path(__file__).resolve().parents[2]))
+
+from videollava.utils import disable_torch_init
+from videollava.model.builder import load_pretrained_model
+from videollava.eval.infer_utils import run_inference_single
+from videollava.constants import DEFAULT_VIDEO_TOKEN
+from videollava.conversation import conv_templates, Conversation, conv_templates
+from videollava.mm_utils import get_model_name_from_path
+
+
+def parse_args():
+ parser = argparse.ArgumentParser(description="Demo")
+ parser.add_argument("--model-path", type=str, default="jirvin16/TEOChat")
+ parser.add_argument("--model-base", type=str, default=None)
+ parser.add_argument("--device", type=str, default="cuda")
+ parser.add_argument("--conv-mode", type=str, default="v1")
+ parser.add_argument("--max-new-tokens", type=int, default=300)
+ parser.add_argument("--quantization", type=str, default="8-bit")
+ parser.add_argument("--image-aspect-ratio", type=str, default='pad')
+ parser.add_argument('--cache-dir', type=str, default=None)
+ parser.add_argument('--dont-use-fast-api', action='store_true')
+ parser.add_argument('--planet-api-key', type=str, default=None)
+ parser.add_argument('--port', type=int, default=7860)
+ parser.add_argument('--server_name', type=str, default="0.0.0.0")
+ args = parser.parse_args()
+ return args
+
+
+def get_bbox_in_polyline_format(x1, y1, x2, y2):
+ return np.array([
+ [x1, y1],
+ [x2, y1],
+ [x2, y2],
+ [x1, y2]
+ ])
+
+
+def extract_box_sequences(string):
+ # Split the input string into segments where sequences of lists are separated by punctuation other than commas or periods
+ segments = re.split(r'[^\[\],\d\s]+', string)
+
+ # Pattern to find substrings of the form [a,b,c,d] where a, b, c, d are integers
+ pattern = r'\[\s*(-?\d+)\s*,\s*(-?\d+)\s*,\s*(-?\d+)\s*,\s*(-?\d+)\s*\]'
+
+ result = []
+ for segment in segments:
+ # Find all matches of the pattern in each segment
+ matches = re.findall(pattern, segment)
+ if matches:
+ # Convert each tuple of strings into a list of integers and collect them into a list
+ sublist = [list(map(int, match)) for match in matches]
+ result.append(sublist)
+
+ return result
+
+
+def is_overlapping(rect1, rect2):
+ x1, y1, x2, y2 = rect1
+ x3, y3, x4, y4 = rect2
+ return not (x2 < x3 or x1 > x4 or y2 < y3 or y1 > y4)
+
+
+def computeIoU(bbox1, bbox2):
+ x1, y1, x2, y2 = bbox1
+ x3, y3, x4, y4 = bbox2
+ intersection_x1 = max(x1, x3)
+ intersection_y1 = max(y1, y3)
+ intersection_x2 = min(x2, x4)
+ intersection_y2 = min(y2, y4)
+ intersection_area = max(0, intersection_x2 - intersection_x1 + 1) * max(0, intersection_y2 - intersection_y1 + 1)
+ bbox1_area = (x2 - x1 + 1) * (y2 - y1 + 1)
+ bbox2_area = (x4 - x3 + 1) * (y4 - y3 + 1)
+ union_area = bbox1_area + bbox2_area - intersection_area
+ iou = intersection_area / union_area
+ return iou
+
+
+def mask2bbox(mask):
+ if mask is None:
+ return ''
+ mask = Image.open(mask)
+ mask = mask.resize([100, 100], resample=Image.NEAREST)
+ mask = np.array(mask)[:, :, 0]
+
+ rows = np.any(mask, axis=1)
+ cols = np.any(mask, axis=0)
+
+ if rows.sum():
+ x1, x2 = np.where(cols)[0][[0, -1]]
+ y1, y2 = np.where(rows)[0][[0, -1]]
+
+ bbox = '[{}, {}, {}, {}]'.format(x1, y1, x2, y2)
+ else:
+ bbox = ''
+
+ return bbox
+
+
+def visualize_all_bbox_together(image_path, generation, bbox_presence):
+ # Resize the image to a fixed width and a height that preserves the aspect ratio
+ # For visualization in gradio
+ image = Image.open(image_path).convert("RGB")
+ image_width, image_height = image.size
+ image = image.resize([500, int(500 / image_width * image_height)])
+ image_width, image_height = image.size
+
+ sequence_list = extract_box_sequences(generation)
+ if sequence_list: # it is grounding or detection
+ mode = 'all'
+ entities = defaultdict(list)
+ i = 0
+ j = 0
+ for sequence in sequence_list:
+ try:
+ # TODO: Get object name from the string
+ # obj, sequence = sequence.split('
')
+ obj = 'TODO'
+ except ValueError:
+ print('wrong string: ', sequence)
+ continue
+ if "][" in sequence:
+ sequence=sequence.replace("][","], [")
+ flag = False
+ for bbox in sequence:
+
+ if len(bbox) == 4:
+ x1, y1, x2, y2 = int(bbox[0]), int(bbox[1]), int(bbox[2]), int(bbox[3])
+ x1 = x1 / bounding_box_size * image_width
+ y1 = y1 / bounding_box_size * image_height
+ x2 = x2 / bounding_box_size * image_width
+ y2 = y2 / bounding_box_size * image_height
+
+ entities[obj].append([x1, y1, x2, y2])
+
+ j += 1
+ flag = True
+ if flag:
+ i += 1
+ else:
+ bbox = re.findall(r'-?\d+', generation)
+ if len(bbox) == 4: # it is refer
+ mode = 'single'
+
+ entities = list()
+ x1, y1, x2, y2 = int(bbox[0]), int(bbox[1]), int(bbox[2]), int(bbox[3])
+ x1 = x1 / bounding_box_size * image_width
+ y1 = y1 / bounding_box_size * image_height
+ x2 = x2 / bounding_box_size * image_width
+ y2 = y2 / bounding_box_size * image_height
+ entities.append([x1, y1, x2, y2])
+ else:
+ # don't detect any valid bbox to visualize
+ return image, ''
+
+ if len(entities) == 0:
+ return image, ''
+
+ if isinstance(image, Image.Image):
+ image_h = image.height
+ image_w = image.width
+ image = np.array(image)
+
+ elif isinstance(image, str):
+ if os.path.exists(image):
+ pil_img = Image.open(image).convert("RGB")
+ image = np.array(pil_img)[:, :, [2, 1, 0]]
+ image_h = pil_img.height
+ image_w = pil_img.width
+ else:
+ raise ValueError(f"invaild image path, {image}")
+ elif isinstance(image, torch.Tensor):
+
+ image_tensor = image.cpu()
+ reverse_norm_mean = torch.tensor([0.48145466, 0.4578275, 0.40821073])[:, None, None]
+ reverse_norm_std = torch.tensor([0.26862954, 0.26130258, 0.27577711])[:, None, None]
+ image_tensor = image_tensor * reverse_norm_std + reverse_norm_mean
+ pil_img = T.ToPILImage()(image_tensor)
+ image_h = pil_img.height
+ image_w = pil_img.width
+ image = np.array(pil_img)[:, :, [2, 1, 0]]
+ else:
+ raise ValueError(f"invalid image format, {type(image)} for {image}")
+
+ new_image = image.copy()
+
+ previous_bboxes = []
+ # size of text
+ text_size = 0.4
+ # thickness of text
+ text_line = 1 # int(max(1 * min(image_h, image_w) / 512, 1))
+ box_line = 2
+ (c_width, text_height), _ = cv2.getTextSize("F", cv2.FONT_HERSHEY_COMPLEX, text_size, text_line)
+ base_height = int(text_height * 0.675)
+ text_offset_original = text_height - base_height
+ text_spaces = 2
+
+ # used_colors = colors # random.sample(colors, k=num_bboxes)
+ if bbox_presence == 'input':
+ color = (255, 0, 0)
+ color_string = 'red'
+ elif bbox_presence == 'output':
+ color = (0, 255, 0)
+ color_string = 'green'
+ else:
+ # Doesn't matter, should never be used
+ color = None
+
+ # color_id = -1
+ for entity_idx, entity_name in enumerate(entities):
+ if mode == 'single' or mode == 'identify':
+ bboxes = entity_name
+ bboxes = [bboxes]
+ else:
+ bboxes = entities[entity_name]
+ # color_id += 1
+ for (x1_norm, y1_norm, x2_norm, y2_norm) in bboxes:
+ skip_flag = False
+ orig_x1, orig_y1, orig_x2, orig_y2 = int(x1_norm), int(y1_norm), int(x2_norm), int(y2_norm)
+
+ # color = used_colors[entity_idx % len(used_colors)] # tuple(np.random.randint(0, 255, size=3).tolist())
+ bbox = get_bbox_in_polyline_format(orig_x1, orig_y1, orig_x2, orig_y2)
+ new_image=cv2.polylines(new_image, [bbox.astype(np.int32)], isClosed=True,thickness=2, color=color)
+
+ # TODO: Add this after delimeter
+ if False: # mode == 'all':
+ l_o, r_o = box_line // 2 + box_line % 2, box_line // 2 + box_line % 2 + 1
+
+ x1 = orig_x1 - l_o
+ y1 = orig_y1 - l_o
+
+ if y1 < text_height + text_offset_original + 2 * text_spaces:
+ y1 = orig_y1 + r_o + text_height + text_offset_original + 2 * text_spaces
+ x1 = orig_x1 + r_o
+
+ # add text background
+ (text_width, text_height), _ = cv2.getTextSize(f" {entity_name}", cv2.FONT_HERSHEY_COMPLEX, text_size,
+ text_line)
+ text_bg_x1, text_bg_y1, text_bg_x2, text_bg_y2 = x1, y1 - (
+ text_height + text_offset_original + 2 * text_spaces), x1 + text_width, y1
+
+ for prev_bbox in previous_bboxes:
+ if computeIoU((text_bg_x1, text_bg_y1, text_bg_x2, text_bg_y2), prev_bbox['bbox']) > 0.95 and \
+ prev_bbox['phrase'] == entity_name:
+ skip_flag = True
+ break
+ while is_overlapping((text_bg_x1, text_bg_y1, text_bg_x2, text_bg_y2), prev_bbox['bbox']):
+ text_bg_y1 += (text_height + text_offset_original + 2 * text_spaces)
+ text_bg_y2 += (text_height + text_offset_original + 2 * text_spaces)
+ y1 += (text_height + text_offset_original + 2 * text_spaces)
+
+ if text_bg_y2 >= image_h:
+ text_bg_y1 = max(0, image_h - (text_height + text_offset_original + 2 * text_spaces))
+ text_bg_y2 = image_h
+ y1 = image_h
+ break
+ if not skip_flag:
+ alpha = 0.5
+ for i in range(text_bg_y1, text_bg_y2):
+ for j in range(text_bg_x1, text_bg_x2):
+ if i < image_h and j < image_w:
+ if j < text_bg_x1 + 1.35 * c_width:
+ # original color
+ bg_color = color
+ else:
+ # white
+ bg_color = [255, 255, 255]
+ new_image[i, j] = (alpha * new_image[i, j] + (1 - alpha) * np.array(bg_color)).astype(
+ np.uint8)
+
+ cv2.putText(
+ new_image, f" {entity_name}", (x1, y1 - text_offset_original - 1 * text_spaces),
+ cv2.FONT_HERSHEY_COMPLEX, text_size, (0, 0, 0), text_line, cv2.LINE_AA
+ )
+
+ previous_bboxes.append(
+ {'bbox': (text_bg_x1, text_bg_y1, text_bg_x2, text_bg_y2), 'phrase': entity_name})
+
+ # TODO: Add this after delimeter
+ if False: # mode == 'all':
+ def color_iterator(colors):
+ while True:
+ for color in colors:
+ yield color
+
+ color_gen = color_iterator(colors)
+
+ # Add colors to phrases and remove
+ def colored_phrases(match):
+ phrase = match.group(1)
+ color = next(color_gen)
+ return f'{phrase} '
+
+ generation = re.sub(r'{<\d+><\d+><\d+><\d+>}|', '', generation)
+ generation_colored = re.sub(r'(.*?)
', colored_phrases, generation)
+ else:
+ # For now, just color the bounding box text the same color as the input
+ def color_bounding_boxes(text):
+ # Regex pattern to find patterns of the form [xmin, xmax, ymin, ymax]
+ pattern = r'\[\s*\d+\s*,\s*\d+\s*,\s*\d+\s*,\s*\d+\s*\]'
+
+ # Function to apply HTML styling
+ def replace_with_color(match):
+ return f'{match.group()} '
+
+ # Replace all matching patterns with colored version
+ colored_text = re.sub(pattern, replace_with_color, text)
+ return colored_text
+
+ if bbox_presence is not None:
+ # Detect the bounding boxes and replace them with colored versions
+ generation_colored = color_bounding_boxes(generation)
+ else:
+ generation_colored = generation
+
+ pil_image = Image.fromarray(new_image)
+ return pil_image, generation_colored
+
+
+def regenerate(state, state_):
+ state.messages.pop(-1)
+ state_.messages.pop(-1)
+ if len(state.messages) > 0:
+ return state, state_, state.to_gradio_chatbot(), False
+ return (state, state_, state.to_gradio_chatbot(), True)
+
+
+def clear_history(state, state_):
+ state = conv_templates[CONV_MODE].copy()
+ state_ = conv_templates[CONV_MODE].copy()
+ return (
+ gr.update(value=None, interactive=True),
+ gr.update(value=None, interactive=True),
+ gr.update(value=None, interactive=True),
+ True,
+ state,
+ state_,
+ state.to_gradio_chatbot()
+ )
+
+
+def single_example_trigger(image1, textbox):
+ return gr.update(value=None, interactive=True), *example_trigger()
+
+
+def temporal_example_trigger(image1, image_list, textbox):
+ return image_list, *example_trigger()
+
+
+def example_trigger():
+ state = conv_templates[CONV_MODE].copy()
+ state_ = conv_templates[CONV_MODE].copy()
+ return True, state, state_, state.to_gradio_chatbot()
+
+
+def generate(image1, image_list, textbox_in, first_run, state, state_):
+ flag = 1
+ if not textbox_in:
+ return "Please enter an instruction."
+
+ mask = None
+ if image1 is None:
+ image1 = []
+ elif isinstance(image1, str):
+ image1 = [image1]
+ elif isinstance(image1, dict):
+ mask = image1['layers'][0]
+ image1 = [image1['background']]
+ if image_list is None:
+ image_list = []
+
+ all_image_paths = [path for path in image1 + image_list if os.path.exists(path)]
+
+ if type(state) is not Conversation:
+ state = conv_templates[CONV_MODE].copy()
+ state_ = conv_templates[CONV_MODE].copy()
+
+ first_run = False if len(state.messages) > 0 else True
+
+ text_en_in = textbox_in.replace("picture", "image")
+
+ # Check if user provided bbox in the text input
+ integers = re.findall(r'-?\d+', text_en_in)
+ bbox_in_input = False
+ if len(integers) != 4:
+ # No bbox provided in input text. Try to use the bbox from the image editor
+ bbox = mask2bbox(mask)
+ if bbox:
+ bbox_in_input = True
+ text_en_in += f" {bbox}"
+ else:
+ bbox_in_input = True
+
+ text_en_out, state_ = handler.generate(all_image_paths, text_en_in, first_run=first_run, state=state_)
+ state_.messages[-1] = (state_.roles[1], text_en_out)
+
+ text_en_out = text_en_out.split('#')[0]
+
+ # Check if bbox is in the text output
+ integers = re.findall(r'-?\d+', text_en_out)
+ bbox_in_output = False
+ if len(integers) == 4:
+ bbox_in_output = True
+
+ show_images = ""
+ for idx, image_path in enumerate(all_image_paths, start=1):
+ if bbox_in_input and bbox_in_output:
+ # If both are present, only display the output bbox in the image
+ bbox_presence = "output"
+ image, text_en_out = visualize_all_bbox_together(image_path, text_en_out, bbox_presence=bbox_presence)
+ elif bbox_in_input and not bbox_in_output:
+ bbox_presence = "input"
+ image, text_en_in = visualize_all_bbox_together(image_path, text_en_in, bbox_presence=bbox_presence)
+ elif bbox_in_output:
+ bbox_presence = "output"
+ image, text_en_out = visualize_all_bbox_together(image_path, text_en_out, bbox_presence=bbox_presence)
+ else:
+ # No bboxes, pass in output text
+ bbox_presence = None
+ image, _ = visualize_all_bbox_together(image_path, text_en_out, bbox_presence=bbox_presence)
+
+ if bbox_presence is not None or first_run:
+ new_image_path = os.path.join(os.path.dirname(image_path), next(tempfile._get_candidate_names()) + '.png')
+ image.save(new_image_path)
+ show_images += f'Image {idx}: '
+
+ textbox_out = text_en_out
+ textbox_in = text_en_in
+
+ if flag:
+ state.append_message(state.roles[0], textbox_in + "\n" + show_images)
+ state.append_message(state.roles[1], textbox_out)
+
+ return (
+ state,
+ state_,
+ state.to_gradio_chatbot(),
+ False,
+ gr.update(value=None, interactive=True)
+ )
+
+
+class Chat:
+ def __init__(self, model_path, conv_mode, model_base=None, quantization=None, device='cuda', cache_dir=None):
+ disable_torch_init()
+ model_name = get_model_name_from_path(model_path)
+ # Add cache_dir attribute to config.json at model_path
+ if cache_dir is not None and cache_dir != "./cache_dir":
+ # Model path is a full path
+ config_path = os.path.join(model_path, 'config.json')
+ if not os.path.exists(config_path):
+ # Model path is relative to cache dir
+ config_path = os.path.join(cache_dir, model_path, 'config.json')
+ if not os.path.exists(config_path):
+ # Model path is a hf repo
+ user, repo_id = model_path.split('/')
+ snapshot_dir = os.path.join(cache_dir, f"models--{user}--{repo_id}", 'snapshots')
+ # Get most recent snapshot
+ snapshots = os.listdir(snapshot_dir)
+ snapshot = max(snapshots, key=lambda x: os.path.getctime(os.path.join(snapshot_dir, x)))
+ snapshot_dir = os.path.join(snapshot_dir, snapshot)
+ config_path = os.path.join(snapshot_dir, 'config.json')
+ # Download the model
+ from huggingface_hub import snapshot_download
+ snapshot_download(repo_id=model_path, cache_dir=cache_dir, use_auth_token=os.getenv('HF_AUTH_TOKEN'))
+
+ with open(config_path, 'r') as f:
+ config = json.load(f)
+ config['cache_dir'] = cache_dir
+ with open(config_path, 'w') as f:
+ json.dump(config, f)
+
+ load_8bit = quantization == "8-bit"
+ load_4bit = quantization == "4-bit"
+
+ self.tokenizer, self.model, processor, context_len = load_pretrained_model(model_path, model_base, model_name,
+ load_8bit, load_4bit,
+ device=device, cache_dir=cache_dir,
+ use_auth_token=os.getenv('HF_AUTH_TOKEN'))
+ self.image_processor = processor['image']
+ self.conv_mode = conv_mode
+ self.conv = conv_templates[conv_mode].copy()
+ self.device = self.model.device
+
+ def get_prompt(self, qs, state):
+ state.append_message(state.roles[0], qs)
+ state.append_message(state.roles[1], None)
+ return state
+
+ @torch.inference_mode()
+ def generate(self, image_paths: list, prompt: str, first_run: bool, state):
+
+ if first_run:
+ if len(image_paths) == 1:
+ prefix = f"This is a satellite image: {DEFAULT_VIDEO_TOKEN}\n"
+ else:
+ prefix = f"This a sequence of satellite images capturing the same location at different times in chronological order: {DEFAULT_VIDEO_TOKEN}\n"
+ prompt = prefix + prompt
+
+ state = self.get_prompt(prompt, state)
+ prompt = state.get_prompt()
+
+ prompt, outputs = run_inference_single(
+ self.model,
+ self.image_processor,
+ self.tokenizer,
+ self.conv_mode,
+ inp=None,
+ image_paths=image_paths,
+ metadata=None, # Assume no metatdata
+ prompt_strategy="interleave",
+ chronological_prefix=True,
+ prompt=prompt,
+ print_prompt=True,
+ return_prompt=True,
+ )
+
+ print("prompt", prompt)
+
+ outputs = outputs.strip()
+
+ print('response', outputs)
+ return outputs, state
+
+
+def center_map(lat, lon, zoom, basemap):
+
+ fig = go.Figure(go.Scattermapbox())
+
+ basemap2source = {
+ "Google Maps": "https://mt0.google.com/vt/lyrs=s&hl=en&x={x}&y={y}&z={z}",
+ "PlanetScope Q2 2024": "https://tiles.planet.com/basemaps/v1/planet-tiles/global_quarterly_2024q2_mosaic/gmap/{z}/{x}/{y}.png?api_key=",
+ "PlanetScope Q1 2024": "https://tiles.planet.com/basemaps/v1/planet-tiles/global_quarterly_2024q1_mosaic/gmap/{z}/{x}/{y}.png?api_key=",
+ "PlanetScope Q4 2023": "https://tiles.planet.com/basemaps/v1/planet-tiles/global_quarterly_2023q4_mosaic/gmap/{z}/{x}/{y}.png?api_key=",
+ "PlanetScope Q3 2023": "https://tiles.planet.com/basemaps/v1/planet-tiles/global_quarterly_2023q3_mosaic/gmap/{z}/{x}/{y}.png?api_key=",
+ "United States Geological Survey": "https://basemap.nationalmap.gov/arcgis/rest/services/USGSImageryOnly/MapServer/tile/{z}/{y}/{x}"
+ }
+ source = basemap2source[basemap]
+ if "Planet" in basemap and PLANET_API_KEY is None:
+ raise ValueError("Please provide a Planet API key using --planet-api-key")
+ elif "Planet" in basemap:
+ source += PLANET_API_KEY
+
+ # Update the layout to include the map configuration
+ fig.update_layout(
+ # title="Select Image(s) using Map",
+ mapbox={
+ "style": "white-bg",
+ "layers": [{
+ "below": 'traces',
+ "sourcetype": "raster",
+ "sourceattribution": basemap,
+ "source": [source]
+ }],
+ "center": {"lat": lat, "lon": lon},
+ "zoom": zoom # Adjust zoom level based on your preference
+ },
+ mapbox_style="white-bg",
+ margin={"r": 0, "t": 0, "l": 0, "b": 0},
+ height=700
+ )
+
+ return fig
+
+
+def get_single_map_image(lat, lon, zoom, basemap):
+ fig = center_map(lat, lon, zoom, basemap)
+ buf = io.BytesIO()
+ fig.write_image(buf, format='png')
+ buf.seek(0)
+ # Convert to PIL image
+ img = Image.open(buf)
+ # Center crop to the shortest dimension
+ width, height = img.size
+ if width > height:
+ left = (width - height) / 2
+ right = (width + height) / 2
+ top = 0
+ bottom = height
+ else:
+ left = 0
+ right = width
+ top = (height - width) / 2
+ bottom = (height + width) / 2
+ img = img.crop((left, top, right, bottom))
+ return img
+
+
+def get_temporal_map_image_paths(lat, lon, zoom):
+ first_image = get_single_map_image(lat, lon, zoom, "PlanetScope Q3 2023")
+ other_images = []
+ for basemap in ["PlanetScope Q2 2024", "PlanetScope Q1 2024", "PlanetScope Q4 2023"]:
+ other_images.append(get_single_map_image(lat, lon, zoom, basemap))
+
+ # Save each image to temporary files
+ first_image_path = os.path.join(os.getenv('TMPDIR'), next(tempfile._get_candidate_names()) + '.png')
+ first_image.save(first_image_path)
+ other_image_paths = []
+ for image in other_images:
+ image_path = os.path.join(os.getenv('TMPDIR'), next(tempfile._get_candidate_names()) + '.png')
+ image.save(image_path)
+ other_image_paths.append(image_path)
+
+ return first_image_path, other_image_paths
+
+
+def update_map(lat, lon, zoom, basemap):
+ return gr.Plot(center_map(lat, lon, zoom, basemap))
+
+
+if __name__ == '__main__':
+
+ random.seed(42)
+ np.random.seed(42)
+ torch.manual_seed(42)
+
+ cudnn.benchmark = False
+ cudnn.deterministic = True
+
+ print('Initializing Chat...')
+ args = parse_args()
+
+ device = args.device
+
+ bounding_box_size = 100
+
+ dtype = torch.float16
+
+ colors = [
+ (255, 0, 0),
+ (0, 255, 0),
+ (0, 0, 255),
+ (210, 210, 0),
+ (255, 0, 255),
+ (0, 255, 255),
+ (114, 128, 250),
+ (0, 165, 255),
+ (0, 128, 0),
+ (144, 238, 144),
+ (238, 238, 175),
+ (255, 191, 0),
+ (0, 128, 0),
+ (226, 43, 138),
+ (255, 0, 255),
+ (0, 215, 255),
+ ]
+
+ color_map = {
+ f"{color_id}": f"#{hex(color[2])[2:].zfill(2)}{hex(color[1])[2:].zfill(2)}{hex(color[0])[2:].zfill(2)}" for
+ color_id, color in enumerate(colors)
+ }
+
+ used_colors = colors
+
+ CONV_MODE = args.conv_mode
+ PLANET_API_KEY = args.planet_api_key
+ if PLANET_API_KEY is None:
+ PLANET_API_KEY = os.getenv('PLANET_API_KEY')
+
+ handler = Chat(
+ model_path=args.model_path,
+ conv_mode=args.conv_mode,
+ model_base=args.model_base,
+ quantization=args.quantization,
+ device=args.device,
+ cache_dir=args.cache_dir
+ )
+
+ # TODO: Consider adding github stars later
+ #
+ title_markdown = ("""
+
+
+
+
+
+
TEOChat: Large Language and Vision Assistant for Temporal Earth Observation Data
+ If you like our project, please give us a star ✨ on Github for the latest update.
+
+
+
+
+
+
+
+
+
+
+ """)
+
+ introduction = '''
+ **Instructions:**
+
+ Select image(s) to input to TEOChat by doing one of the following:
+
+ (Below) Click the image icon in the First Image widget to upload a single image, then optionally upload additional temporal images by clicking the Optional Additional Image(s) widget.
+ (On the right) Enter the latitude, longitude, zoom, and select the basemap to view the map image, then:
+
+ Upload the map image based on the entered latitude, longitude, zoom, and basemap.
+ Upload a temporal map image (including 4 images from PlanetScope) based on the entered latitude, longitude, and zoom.
+ Pan around and download the current map image by clicking the 📷 icon at the top right, then uploading that image.
+
+
+ (On the bottom) Select prespecified example image(s) (and text input).
+
+
+ Optionally draw a bounding box using the First Image widget by clicking the pen icon on the bottom.
+ Enter a text prompt in the text input above.
+ Click Send to generate the output.
+
+ '''
+
+ block_css = """
+ #buttons button {
+ min-width: min(120px,100%);
+ }
+ """
+
+ tos_markdown = """
+ ### Terms of use
+ By using this service, users are required to agree to the following terms:
+ The service is a research preview intended for non-commercial use only. It only provides limited safety measures and may generate offensive content. It must not be used for any illegal, harmful, violent, racist, or sexual purposes.
+ For an optimal experience, please use desktop computers for this demo, as mobile devices may compromise its quality.
+ """
+
+ learn_more_markdown = """
+ ### License
+ The service is a research preview intended for non-commercial use only, subject to the model [License](https://github.com/facebookresearch/llama/blob/main/MODEL_CARD.md) of LLaMA, [Terms of Use](https://openai.com/policies/terms-of-use) of the data generated by OpenAI, and [Privacy Practices](https://chrome.google.com/webstore/detail/sharegpt-share-your-chatg/daiacboceoaocpibfodeljbdfacokfjb) of ShareGPT. Please contact us if you find any potential violation.
+ """
+
+ cur_dir = os.path.dirname(os.path.abspath(__file__))
+ example_dir = os.path.join(cur_dir, 'examples')
+
+ textbox = gr.Textbox(
+ show_label=False, placeholder="Upload an image or obtain one using the map viewer, then enter text here and press Send ->", container=False
+ )
+ with gr.Blocks(title='TEOChat', theme=gr.themes.Default(text_size=sizes.text_lg), css=block_css) as demo:
+ gr.Markdown(title_markdown)
+ state = gr.State()
+ state_ = gr.State()
+ first_run = gr.State()
+
+ with gr.Row():
+ chatbot = gr.Chatbot(label="TEOChat", bubble_full_width=True)
+ with gr.Row():
+ with gr.Column(scale=8):
+ textbox.render()
+ with gr.Column(scale=1, min_width=50):
+ submit_btn = gr.Button(
+ value="Send", variant="primary", interactive=True
+ )
+ with gr.Row(elem_id="buttons") as button_row:
+ regenerate_btn = gr.Button(value="🔄 Regenerate", interactive=True)
+ clear_btn = gr.Button(value="🗑️ Clear history", interactive=True)
+
+ with gr.Row():
+ with gr.Column(scale=1, elem_id="introduction"):
+ gr.Markdown(introduction)
+ image1 = gr.ImageEditor(
+ label="First Image",
+ type="filepath",
+ layers=False,
+ transforms=(),
+ sources=('upload', 'clipboard'),
+ brush=Brush(colors=["red"], color_mode="fixed", default_size=3)
+ )
+ image_list = gr.File(
+ label="Optional Additional Image(s)",
+ file_count="multiple"
+ )
+
+ with gr.Column(scale=1):
+ with gr.Row():
+ map_view = gr.Plot(label="Map Image(s)")
+
+ with gr.Row():
+ lat = gr.Number(value=37.43144514632126, label="Latitude")
+ lon = gr.Number(value=-122.16210856357836, label="Longitude")
+ zoom = gr.Number(value=18, label="Zoom")
+ basemap = gr.Dropdown(
+ value="Google Maps",
+ choices=[
+ "Google Maps",
+ "PlanetScope Q2 2024",
+ "PlanetScope Q1 2024",
+ "PlanetScope Q4 2023",
+ "PlanetScope Q3 2023",
+ "United States Geological Survey",
+ ],
+ label="Basemap"
+ )
+ with gr.Row():
+ single_map_upload_button = gr.Button("Upload Map based on Lat/Lon/Zoom/Basemap")
+ temporal_map_upload_button = gr.Button("Upload Temporal Map (PlanetScope Q3-Q4 2023, Q1-Q2 2024) based on Lat/Lon/Zoom")
+
+ demo.load(center_map, [lat, lon, zoom, basemap], map_view)
+
+ with gr.Row():
+ gr.Examples(
+ examples=[
+ [
+ f"{example_dir}/rqa.png",
+ "What is this? [21, 3, 47, 19]",
+ ],
+ [
+ f"{example_dir}/xBD_loc.png",
+ "Identify the location of the building on the right of the image using a bounding box of the form [x_min, y_min, x_max, y_max].",
+ ],
+ [
+ f"{example_dir}/AID_cls.png",
+ "Classify this image as one of: Oil Refinery, Compressor Station, Pipeline, Processing Plant, Well Pad.",
+ ],
+ [
+ f"{example_dir}/HRBEN_qa.png",
+ "Is there a road next to a body of water?",
+ ]
+ ],
+ inputs=[image1, textbox],
+ outputs=[image_list, first_run, state, state_, chatbot],
+ label="Single Image Examples",
+ fn=single_example_trigger,
+ run_on_click=True,
+ cache_examples=False
+ )
+ gr.Examples(
+ examples=[
+ [
+ f"{example_dir}/fMoW_cls_1.png",
+ [f"{example_dir}/fMoW_cls_2.png", f"{example_dir}/fMoW_cls_3.png", f"{example_dir}/fMoW_cls_4.png"],
+ "Classify the sequence of images as one of: flooded road, lake or pond, aquaculture, dam, mountain trail.",
+ ],
+ [
+ f"{example_dir}/xBD_dis_1.png",
+ [f"{example_dir}/xBD_dis_2.png"],
+ "What disaster has occurred in the area?",
+ ],
+ [
+ f"{example_dir}/xBD_cls_1.png",
+ [f"{example_dir}/xBD_cls_2.png"],
+ "Classify the level of damage experienced by the building at location [0, 8, 49, 53].",
+ ],
+ [
+ f"{example_dir}/S2Looking_cd_1.png",
+ [f"{example_dir}/S2Looking_cd_2.png"],
+ "Identify all changed buildings using bounding boxes of the form [x_min, y_min, x_max, y_max].",
+ ],
+ [
+ f"{example_dir}/QFabric_rtqa_1.png",
+ [f"{example_dir}/QFabric_rtqa_2.png", f"{example_dir}/QFabric_rtqa_3.png", f"{example_dir}/QFabric_rtqa_4.png", f"{example_dir}/QFabric_rtqa_5.png"],
+ "In which image was construction finished?",
+ ],
+ ],
+ inputs=[image1, image_list, textbox],
+ outputs=[image_list, first_run, state, state_, chatbot],
+ label="Temporal Image Examples",
+ fn=temporal_example_trigger,
+ run_on_click=True,
+ cache_examples=False
+ )
+ gr.Markdown(tos_markdown)
+ gr.Markdown(learn_more_markdown)
+
+ lat.change(fn=update_map, inputs=[lat, lon, zoom, basemap], outputs=[map_view])
+ lon.change(fn=update_map, inputs=[lat, lon, zoom, basemap], outputs=[map_view])
+ zoom.change(fn=update_map, inputs=[lat, lon, zoom, basemap], outputs=[map_view])
+ basemap.change(fn=update_map, inputs=[lat, lon, zoom, basemap], outputs=[map_view])
+ single_map_upload_button.click(fn=get_single_map_image, inputs=[lat, lon, zoom, basemap], outputs=[image1])
+ temporal_map_upload_button.click(fn=get_temporal_map_image_paths, inputs=[lat, lon, zoom], outputs=[image1, image_list])
+
+ submit_btn.click(
+ generate,
+ [image1, image_list, textbox, first_run, state, state_],
+ [state, state_, chatbot, first_run, textbox]
+ )
+
+ regenerate_btn.click(
+ regenerate,
+ [state, state_], [state, state_, chatbot, first_run]
+ ).then(
+ generate,
+ [image1, image_list, textbox, first_run, state, state_],
+ [state, state_, chatbot, first_run, textbox]
+ )
+
+ clear_btn.click(
+ clear_history,
+ [state, state_],
+ [image1, image_list, textbox, first_run, state, state_, chatbot]
+ )
+
+ demo.queue()
+
+ if args.dont_use_fast_api:
+ demo.launch(
+ share=False,
+ server_name=args.server_name,
+ favicon_path='static/logo.svg',
+ server_port=args.port,
+ allowed_paths=['static/logo.png'],
+ )
+
+ else:
+
+ import uvicorn
+ from fastapi import FastAPI
+ from fastapi.staticfiles import StaticFiles
+ # create a FastAPI app
+ app = FastAPI()
+
+ # create a static directory to store the static files
+ static_dir = Path('./static')
+ static_dir.mkdir(parents=True, exist_ok=True)
+
+ # mount FastAPI StaticFiles server
+ app.mount("/static", StaticFiles(directory=static_dir), name="static")
+
+ # mount Gradio app to FastAPI app
+ app = gr.mount_gradio_app(app, demo, path="/", favicon_path='static/logo.svg')
+
+ uvicorn.run(app, host=args.server_name, port=args.port)
diff --git a/videollava/train/llava_trainer.py b/videollava/train/llava_trainer.py
new file mode 100644
index 0000000000000000000000000000000000000000..fa40d7273a34a7d557dfdf3aa001af3c18541cd0
--- /dev/null
+++ b/videollava/train/llava_trainer.py
@@ -0,0 +1,264 @@
+import os
+import torch
+
+from torch.utils.data import Sampler
+
+from transformers import Trainer
+from transformers.trainer import (
+ is_sagemaker_mp_enabled,
+ get_parameter_names,
+ has_length,
+ ALL_LAYERNORM_LAYERS,
+ ShardedDDPOption,
+ logger,
+)
+from typing import List, Optional
+
+
+def maybe_zero_3(param, ignore_status=False, name=None):
+ from deepspeed import zero
+ from deepspeed.runtime.zero.partition_parameters import ZeroParamStatus
+ if hasattr(param, "ds_id"):
+ if param.ds_status == ZeroParamStatus.NOT_AVAILABLE:
+ if not ignore_status:
+ print(name, 'no ignore status')
+ with zero.GatheredParameters([param]):
+ param = param.data.detach().cpu().clone()
+ else:
+ param = param.detach().cpu().clone()
+ return param
+
+
+def get_mm_adapter_state_maybe_zero_3(named_params, keys_to_match):
+ to_return = {k: t for k, t in named_params if any(key_match in k for key_match in keys_to_match)}
+ to_return = {k: maybe_zero_3(v, ignore_status=True, name=k).cpu() for k, v in to_return.items()}
+ return to_return
+
+
+def split_to_even_chunks(indices, lengths, num_chunks):
+ """
+ Split a list of indices into `chunks` chunks of roughly equal lengths.
+ """
+
+ if len(indices) % num_chunks != 0:
+ return [indices[i::num_chunks] for i in range(num_chunks)]
+
+ num_indices_per_chunk = len(indices) // num_chunks
+
+ chunks = [[] for _ in range(num_chunks)]
+ chunks_lengths = [0 for _ in range(num_chunks)]
+ for index in indices:
+ shortest_chunk = chunks_lengths.index(min(chunks_lengths))
+ chunks[shortest_chunk].append(index)
+ chunks_lengths[shortest_chunk] += lengths[index]
+ if len(chunks[shortest_chunk]) == num_indices_per_chunk:
+ chunks_lengths[shortest_chunk] = float("inf")
+
+ return chunks
+
+
+def get_modality_length_grouped_indices(lengths, batch_size, world_size, generator=None):
+ # We need to use torch for the random part as a distributed sampler will set the random seed for torch.
+ assert all(l != 0 for l in lengths), "Should not have zero length."
+ if all(l > 0 for l in lengths) or all(l < 0 for l in lengths):
+ # all samples are in the same modality
+ return get_length_grouped_indices(lengths, batch_size, world_size, generator=generator)
+ mm_indices, mm_lengths = zip(*[(i, l) for i, l in enumerate(lengths) if l > 0])
+ lang_indices, lang_lengths = zip(*[(i, -l) for i, l in enumerate(lengths) if l < 0])
+
+ mm_shuffle = [mm_indices[i] for i in get_length_grouped_indices(mm_lengths, batch_size, world_size, generator=None)]
+ lang_shuffle = [lang_indices[i] for i in get_length_grouped_indices(lang_lengths, batch_size, world_size, generator=None)]
+ megabatch_size = world_size * batch_size
+ mm_megabatches = [mm_shuffle[i : i + megabatch_size] for i in range(0, len(mm_shuffle), megabatch_size)]
+ lang_megabatches = [lang_shuffle[i : i + megabatch_size] for i in range(0, len(lang_shuffle), megabatch_size)]
+
+ last_mm = mm_megabatches[-1]
+ last_lang = lang_megabatches[-1]
+ additional_batch = last_mm + last_lang
+ megabatches = mm_megabatches[:-1] + lang_megabatches[:-1]
+ megabatch_indices = torch.randperm(len(megabatches), generator=generator)
+ megabatches = [megabatches[i] for i in megabatch_indices]
+
+ if len(additional_batch) > 0:
+ megabatches.append(sorted(additional_batch))
+
+ return [i for megabatch in megabatches for i in megabatch]
+
+
+def get_length_grouped_indices(lengths, batch_size, world_size, generator=None, merge=True):
+ # We need to use torch for the random part as a distributed sampler will set the random seed for torch.
+ indices = torch.randperm(len(lengths), generator=generator)
+ megabatch_size = world_size * batch_size
+ megabatches = [indices[i : i + megabatch_size].tolist() for i in range(0, len(lengths), megabatch_size)]
+ megabatches = [sorted(megabatch, key=lambda i: lengths[i], reverse=True) for megabatch in megabatches]
+ megabatches = [split_to_even_chunks(megabatch, lengths, world_size) for megabatch in megabatches]
+
+ return [i for megabatch in megabatches for batch in megabatch for i in batch]
+
+
+class LengthGroupedSampler(Sampler):
+ r"""
+ Sampler that samples indices in a way that groups together features of the dataset of roughly the same length while
+ keeping a bit of randomness.
+ """
+
+ def __init__(
+ self,
+ batch_size: int,
+ world_size: int,
+ lengths: Optional[List[int]] = None,
+ generator=None,
+ group_by_modality: bool = False,
+ ):
+ if lengths is None:
+ raise ValueError("Lengths must be provided.")
+
+ self.batch_size = batch_size
+ self.world_size = world_size
+ self.lengths = lengths
+ self.generator = generator
+ self.group_by_modality = group_by_modality
+
+ def __len__(self):
+ return len(self.lengths)
+
+ def __iter__(self):
+ if self.group_by_modality:
+ indices = get_modality_length_grouped_indices(self.lengths, self.batch_size, self.world_size, generator=self.generator)
+ else:
+ indices = get_length_grouped_indices(self.lengths, self.batch_size, self.world_size, generator=self.generator)
+ return iter(indices)
+
+
+class LLaVATrainer(Trainer):
+
+ def _get_train_sampler(self) -> Optional[torch.utils.data.Sampler]:
+ if self.train_dataset is None or not has_length(self.train_dataset):
+ return None
+
+ if self.args.group_by_modality_length:
+ lengths = self.train_dataset.modality_lengths
+ return LengthGroupedSampler(
+ self.args.train_batch_size,
+ world_size=self.args.world_size * self.args.gradient_accumulation_steps,
+ lengths=lengths,
+ group_by_modality=True,
+ )
+ else:
+ return super()._get_train_sampler()
+
+ def create_optimizer(self):
+ """
+ Setup the optimizer.
+
+ We provide a reasonable default that works well. If you want to use something else, you can pass a tuple in the
+ Trainer's init through `optimizers`, or subclass and override this method in a subclass.
+ """
+ if is_sagemaker_mp_enabled():
+ return super().create_optimizer()
+ if self.sharded_ddp == ShardedDDPOption.SIMPLE:
+ return super().create_optimizer()
+
+ opt_model = self.model
+
+ if self.optimizer is None:
+ decay_parameters = get_parameter_names(opt_model, ALL_LAYERNORM_LAYERS)
+ decay_parameters = [name for name in decay_parameters if "bias" not in name]
+ if self.args.mm_projector_lr is not None:
+ projector_parameters = [name for name, _ in opt_model.named_parameters() if "mm_projector" in name]
+ optimizer_grouped_parameters = [
+ {
+ "params": [
+ p for n, p in opt_model.named_parameters() if (n in decay_parameters and n not in projector_parameters and p.requires_grad)
+ ],
+ "weight_decay": self.args.weight_decay,
+ },
+ {
+ "params": [
+ p for n, p in opt_model.named_parameters() if (n not in decay_parameters and n not in projector_parameters and p.requires_grad)
+ ],
+ "weight_decay": 0.0,
+ },
+ {
+ "params": [
+ p for n, p in opt_model.named_parameters() if (n in decay_parameters and n in projector_parameters and p.requires_grad)
+ ],
+ "weight_decay": self.args.weight_decay,
+ "lr": self.args.mm_projector_lr,
+ },
+ {
+ "params": [
+ p for n, p in opt_model.named_parameters() if (n not in decay_parameters and n in projector_parameters and p.requires_grad)
+ ],
+ "weight_decay": 0.0,
+ "lr": self.args.mm_projector_lr,
+ },
+ ]
+ else:
+ optimizer_grouped_parameters = [
+ {
+ "params": [
+ p for n, p in opt_model.named_parameters() if (n in decay_parameters and p.requires_grad)
+ ],
+ "weight_decay": self.args.weight_decay,
+ },
+ {
+ "params": [
+ p for n, p in opt_model.named_parameters() if (n not in decay_parameters and p.requires_grad)
+ ],
+ "weight_decay": 0.0,
+ },
+ ]
+
+ optimizer_cls, optimizer_kwargs = Trainer.get_optimizer_cls_and_kwargs(self.args)
+
+ if self.sharded_ddp == ShardedDDPOption.SIMPLE:
+ self.optimizer = OSS(
+ params=optimizer_grouped_parameters,
+ optim=optimizer_cls,
+ **optimizer_kwargs,
+ )
+ else:
+ self.optimizer = optimizer_cls(optimizer_grouped_parameters, **optimizer_kwargs)
+ if optimizer_cls.__name__ == "Adam8bit":
+ import bitsandbytes
+
+ manager = bitsandbytes.optim.GlobalOptimManager.get_instance()
+
+ skipped = 0
+ for module in opt_model.modules():
+ if isinstance(module, nn.Embedding):
+ skipped += sum({p.data_ptr(): p.numel() for p in module.parameters()}.values())
+ logger.info(f"skipped {module}: {skipped/2**20}M params")
+ manager.register_module_override(module, "weight", {"optim_bits": 32})
+ logger.debug(f"bitsandbytes: will optimize {module} in fp32")
+ logger.info(f"skipped: {skipped/2**20}M params")
+
+ return self.optimizer
+
+ def _save_checkpoint(self, model, trial, metrics=None):
+ if getattr(self.args, 'tune_mm_mlp_adapter', False):
+ from transformers.trainer_utils import PREFIX_CHECKPOINT_DIR
+ checkpoint_folder = f"{PREFIX_CHECKPOINT_DIR}-{self.state.global_step}"
+
+ run_dir = self._get_output_dir(trial=trial)
+ output_dir = os.path.join(run_dir, checkpoint_folder)
+
+ # Only save Adapter
+ keys_to_match = ['mm_projector', 'vision_resampler']
+ if getattr(self.args, "use_im_start_end", False):
+ keys_to_match.extend(['embed_tokens', 'embed_in'])
+
+ weight_to_save = get_mm_adapter_state_maybe_zero_3(self.model.named_parameters(), keys_to_match)
+
+ if self.args.local_rank == 0 or self.args.local_rank == -1:
+ self.model.config.save_pretrained(output_dir)
+ torch.save(weight_to_save, os.path.join(output_dir, f'mm_projector.bin'))
+ else:
+ super(LLaVATrainer, self)._save_checkpoint(model, trial, metrics)
+
+ def _save(self, output_dir: Optional[str] = None, state_dict=None):
+ if getattr(self.args, 'tune_mm_mlp_adapter', False):
+ pass
+ else:
+ super(LLaVATrainer, self)._save(output_dir, state_dict)
diff --git a/videollava/train/train.py b/videollava/train/train.py
new file mode 100644
index 0000000000000000000000000000000000000000..06bafdad4a91c38478045d5d22a4b54682ad5046
--- /dev/null
+++ b/videollava/train/train.py
@@ -0,0 +1,1177 @@
+# Adopted from https://github.com/lm-sys/FastChat. Below is the original copyright:
+# Adopted from tatsu-lab@stanford_alpaca. Below is the original copyright:
+# Copyright 2023 Rohan Taori, Ishaan Gulrajani, Tianyi Zhang, Yann Dubois, Xuechen Li
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import json
+import copy
+import torch
+import random
+import logging
+import pathlib
+import transformers
+from PIL import Image
+from datetime import datetime
+from torch.utils.data import Dataset
+from dataclasses import dataclass, field
+from typing import Dict, Optional, Sequence, List
+
+from videollava.model import *
+from videollava.utils import order_pick_k
+from videollava.mm_utils import tokenizer_image_token
+from videollava import conversation as conversation_lib
+from videollava.train.llava_trainer import LLaVATrainer
+from videollava.constants import IGNORE_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_VIDEO_TOKEN,\
+ MAX_IMAGE_LENGTH, MAX_VIDEO_LENGTH
+
+
+os.environ["WANDB_PROJECT"] = "geovlm"
+
+
+local_rank = None
+
+
+def rank0_print(*args):
+ if local_rank == 0:
+ print(*args)
+
+
+@dataclass
+class ModelArguments:
+ model_name_or_path: Optional[str] = field(default="facebook/opt-125m")
+ version: Optional[str] = field(default="v0")
+ freeze_backbone: bool = field(default=False)
+ tune_mm_mlp_adapter: bool = field(default=False)
+ vision_tower: Optional[str] = field(default=None)
+ mm_vision_select_layer: Optional[int] = field(default=-1) # default to the last layer
+ pretrain_mm_mlp_adapter: Optional[str] = field(default=None)
+ mm_projector_type: Optional[str] = field(default='linear')
+ mm_use_im_start_end: bool = field(default=False)
+ mm_use_im_patch_token: bool = field(default=True)
+ mm_vision_select_feature: Optional[str] = field(default="patch")
+
+ # ===================================================================
+ image_tower: Optional[str] = field(default=None)
+ video_tower: Optional[str] = field(default=None)
+ # ===================================================================
+
+@dataclass
+class DataArguments:
+ lazy_preprocess: bool = False
+ is_multimodal: bool = False
+ image_aspect_ratio: str = 'square'
+ # ===================================================================
+ data_path: Optional[List[str]] = field(default=None, metadata={"help": "Path to the training data."})
+ image_folder: Optional[str] = field(default=None)
+ video_folder: Optional[str] = field(default=None)
+ num_frames: int = 8
+ video_as_image_list: bool = False
+ prompt_strategy: Optional[str] = field(default=None)
+ chronological_prefix: Optional[bool] = field(default=False)
+ # ===================================================================
+
+
+@dataclass
+class TrainingArguments(transformers.TrainingArguments):
+ cache_dir: Optional[str] = field(default=None)
+ optim: str = field(default="adamw_torch")
+ remove_unused_columns: bool = field(default=False)
+ freeze_mm_mlp_adapter: bool = field(default=False)
+ mpt_attn_impl: Optional[str] = field(default="triton")
+ model_max_length: int = field(
+ default=512,
+ metadata={
+ "help":
+ "Maximum sequence length. Sequences will be right padded (and possibly truncated)."
+ },
+ )
+ double_quant: bool = field(
+ default=True,
+ metadata={"help": "Compress the quantization statistics through double quantization."}
+ )
+ quant_type: str = field(
+ default="nf4",
+ metadata={"help": "Quantization data type to use. Should be one of `fp4` or `nf4`."}
+ )
+ bits: int = field(
+ default=16,
+ metadata={"help": "How many bits to use."}
+ )
+ lora_enable: bool = False
+ lora_r: int = 64
+ lora_alpha: int = 16
+ lora_dropout: float = 0.05
+ lora_weight_path: str = ""
+ lora_bias: str = "none"
+ mm_projector_lr: Optional[float] = None
+ group_by_modality_length: bool = field(default=False)
+
+ # ================================================
+ tokenizer_model_max_length: Optional[int] = None
+ # ================================================
+
+def maybe_zero_3(param, ignore_status=False, name=None):
+ from deepspeed import zero
+ from deepspeed.runtime.zero.partition_parameters import ZeroParamStatus
+ if hasattr(param, "ds_id"):
+ if param.ds_status == ZeroParamStatus.NOT_AVAILABLE:
+ if not ignore_status:
+ logging.warning(f"{name}: param.ds_status != ZeroParamStatus.NOT_AVAILABLE: {param.ds_status}")
+ with zero.GatheredParameters([param]):
+ param = param.data.detach().cpu().clone()
+ else:
+ param = param.detach().cpu().clone()
+ return param
+
+
+# Borrowed from peft.utils.get_peft_model_state_dict
+def get_peft_state_maybe_zero_3(named_params, bias):
+ if bias == "none":
+ to_return = {k: t for k, t in named_params if "lora_" in k}
+ elif bias == "all":
+ to_return = {k: t for k, t in named_params if "lora_" in k or "bias" in k}
+ elif bias == "lora_only":
+ to_return = {}
+ maybe_lora_bias = {}
+ lora_bias_names = set()
+ for k, t in named_params:
+ if "lora_" in k:
+ to_return[k] = t
+ bias_name = k.split("lora_")[0] + "bias"
+ lora_bias_names.add(bias_name)
+ elif "bias" in k:
+ maybe_lora_bias[k] = t
+ for k, t in maybe_lora_bias:
+ if bias_name in lora_bias_names:
+ to_return[bias_name] = t
+ else:
+ raise NotImplementedError
+ to_return = {k: maybe_zero_3(v, ignore_status=True) for k, v in to_return.items()}
+ return to_return
+
+
+def get_peft_state_non_lora_maybe_zero_3(named_params, require_grad_only=True):
+ to_return = {k: t for k, t in named_params if "lora_" not in k}
+ if require_grad_only:
+ to_return = {k: t for k, t in to_return.items() if t.requires_grad}
+ to_return = {k: maybe_zero_3(v, ignore_status=True).cpu() for k, v in to_return.items()}
+ return to_return
+
+
+def get_mm_adapter_state_maybe_zero_3(named_params, keys_to_match):
+ to_return = {k: t for k, t in named_params if any(key_match in k for key_match in keys_to_match)}
+ to_return = {k: maybe_zero_3(v, ignore_status=True).cpu() for k, v in to_return.items()}
+ return to_return
+
+
+def find_all_linear_names(model):
+ cls = torch.nn.Linear
+ lora_module_names = set()
+ multimodal_keywords = ['mm_projector', 'vision_tower', 'vision_resampler']
+ for name, module in model.named_modules():
+ if any(mm_keyword in name for mm_keyword in multimodal_keywords):
+ continue
+ if isinstance(module, cls):
+ names = name.split('.')
+ lora_module_names.add(names[0] if len(names) == 1 else names[-1])
+
+ if 'lm_head' in lora_module_names: # needed for 16-bit
+ lora_module_names.remove('lm_head')
+ return list(lora_module_names)
+
+
+def safe_save_model_for_hf_trainer(trainer: transformers.Trainer,
+ output_dir: str):
+ """Collects the state dict and dump to disk."""
+
+ if getattr(trainer.args, "tune_mm_mlp_adapter", False):
+ # Only save Adapter
+ keys_to_match = ['mm_projector']
+ if getattr(trainer.args, "use_im_start_end", False):
+ keys_to_match.extend(['embed_tokens', 'embed_in'])
+
+ weight_to_save = get_mm_adapter_state_maybe_zero_3(trainer.model.named_parameters(), keys_to_match)
+ trainer.model.config.save_pretrained(output_dir)
+
+ current_folder = output_dir.split('/')[-1]
+ parent_folder = os.path.dirname(output_dir)
+ if trainer.args.local_rank == 0 or trainer.args.local_rank == -1:
+ if current_folder.startswith('checkpoint-'):
+ mm_projector_folder = os.path.join(parent_folder, "mm_projector")
+ os.makedirs(mm_projector_folder, exist_ok=True)
+ torch.save(weight_to_save, os.path.join(mm_projector_folder, f'{current_folder}.bin'))
+ else:
+ torch.save(weight_to_save, os.path.join(output_dir, f'mm_projector.bin'))
+ return
+
+ if trainer.deepspeed:
+ torch.cuda.synchronize()
+ trainer.save_model(output_dir)
+ return
+
+ state_dict = trainer.model.state_dict()
+ if trainer.args.should_save:
+ cpu_state_dict = {
+ key: value.cpu()
+ for key, value in state_dict.items()
+ }
+ del state_dict
+ trainer._save(output_dir, state_dict=cpu_state_dict) # noqa
+
+
+def smart_tokenizer_and_embedding_resize(
+ special_tokens_dict: Dict,
+ tokenizer: transformers.PreTrainedTokenizer,
+ model: transformers.PreTrainedModel,
+):
+ """Resize tokenizer and embedding.
+
+ Note: This is the unoptimized version that may make your embedding size not be divisible by 64.
+ """
+ num_new_tokens = tokenizer.add_special_tokens(special_tokens_dict)
+ model.resize_token_embeddings(len(tokenizer))
+
+ if num_new_tokens > 0:
+ input_embeddings = model.get_input_embeddings().weight.data
+ output_embeddings = model.get_output_embeddings().weight.data
+
+ input_embeddings_avg = input_embeddings[:-num_new_tokens].mean(
+ dim=0, keepdim=True)
+ output_embeddings_avg = output_embeddings[:-num_new_tokens].mean(
+ dim=0, keepdim=True)
+
+ input_embeddings[-num_new_tokens:] = input_embeddings_avg
+ output_embeddings[-num_new_tokens:] = output_embeddings_avg
+
+
+def _tokenize_fn(strings: Sequence[str],
+ tokenizer: transformers.PreTrainedTokenizer) -> Dict:
+ """Tokenize a list of strings."""
+ tokenized_list = [
+ tokenizer(
+ text,
+ return_tensors="pt",
+ padding="longest",
+ max_length=tokenizer.model_max_length,
+ truncation=True,
+ ) for text in strings
+ ]
+ input_ids = labels = [
+ tokenized.input_ids[0] for tokenized in tokenized_list
+ ]
+ input_ids_lens = labels_lens = [
+ tokenized.input_ids.ne(tokenizer.pad_token_id).sum().item()
+ for tokenized in tokenized_list
+ ]
+ return dict(
+ input_ids=input_ids,
+ labels=labels,
+ input_ids_lens=input_ids_lens,
+ labels_lens=labels_lens,
+ )
+
+
+def _mask_targets(target, tokenized_lens, speakers):
+ # cur_idx = 0
+ cur_idx = tokenized_lens[0]
+ tokenized_lens = tokenized_lens[1:]
+ target[:cur_idx] = IGNORE_INDEX
+ for tokenized_len, speaker in zip(tokenized_lens, speakers):
+ if speaker == "human":
+ target[cur_idx+2:cur_idx + tokenized_len] = IGNORE_INDEX
+ cur_idx += tokenized_len
+
+
+def _add_speaker_and_signal(header, source, get_conversation=True):
+ """Add speaker and start/end signal on each round."""
+ BEGIN_SIGNAL = "### "
+ END_SIGNAL = "\n"
+ conversation = header
+ for sentence in source:
+ from_str = sentence["from"]
+ if from_str.lower() == "human":
+ from_str = conversation_lib.default_conversation.roles[0]
+ elif from_str.lower() == "gpt":
+ from_str = conversation_lib.default_conversation.roles[1]
+ else:
+ from_str = 'unknown'
+ sentence["value"] = (BEGIN_SIGNAL + from_str + ": " +
+ sentence["value"] + END_SIGNAL)
+ if get_conversation:
+ conversation += sentence["value"]
+ conversation += BEGIN_SIGNAL
+ return conversation
+
+
+def preprocess_multimodal(
+ sources: Sequence[str],
+ data_args: DataArguments,
+ num_video_images: int = 0
+) -> Dict:
+ is_multimodal = data_args.is_multimodal
+ if not is_multimodal:
+ return sources
+
+ for source in sources:
+ for sentence in source:
+
+ # ======================================================================================================
+ if sentence['value'].startswith(DEFAULT_IMAGE_TOKEN) or sentence['value'].startswith(DEFAULT_VIDEO_TOKEN): # run with multi-im, multi-vid, multi-im & multi-vid
+ # \nxxxxxxxxxxxxx # must first
+ # \nxxxxxxxxxxxxx -> \nxxxxxxxxxxxxx
+ # \nxxxxxxxxxxxxx -> \nxxxxxxxxxxxxx
+
+ if "mmtag" in conversation_lib.default_conversation.version:
+ sentence['value'] = sentence['value'].replace(DEFAULT_IMAGE_TOKEN, '' + DEFAULT_IMAGE_TOKEN + ' ')
+
+ IMAGE_TOKEN_NUM = sentence['value'].count(DEFAULT_IMAGE_TOKEN)
+ if IMAGE_TOKEN_NUM > MAX_IMAGE_LENGTH:
+ sentence['value'] = sentence['value'].replace(DEFAULT_IMAGE_TOKEN * IMAGE_TOKEN_NUM, DEFAULT_IMAGE_TOKEN * MAX_IMAGE_LENGTH).strip()
+ VIDEO_TOKEN_NUM = sentence['value'].count(DEFAULT_VIDEO_TOKEN)
+ if VIDEO_TOKEN_NUM > MAX_VIDEO_LENGTH:
+ raise ValueError(f"{sentence['value']}")
+ sentence['value'] = sentence['value'].replace(DEFAULT_VIDEO_TOKEN * VIDEO_TOKEN_NUM, DEFAULT_VIDEO_TOKEN * MAX_VIDEO_LENGTH).strip()
+
+ if data_args.chronological_prefix:
+ sentence['value'] = sentence['value'].replace("times:", "times in chronological order:")
+
+ # a is treated as `num_video_images * `
+ if data_args.prompt_strategy is None:
+ replace_token, vid_replace_token = DEFAULT_IMAGE_TOKEN, DEFAULT_IMAGE_TOKEN * num_video_images
+ elif data_args.prompt_strategy == 'interleave':
+ replace_token = f"Image: {DEFAULT_IMAGE_TOKEN}"
+ vid_replace_token = ''.join(f"Image {i+1}: {DEFAULT_IMAGE_TOKEN}" for i in range(num_video_images))
+ else:
+ raise ValueError(f"Unknown prompt strategy: {data_args.prompt_strategy}")
+
+ # \nxxxxxxxxxxxxx -> `num_video_images*``num_video_images*`\nxxxxxxxxxxxxx
+ # \nxxxxxxxxxxxxx -> `num_video_images*`\nxxxxxxxxxxxxx
+ # print('before replace_token:', [sentence['value']])
+ sentence["value"] = sentence["value"].replace(DEFAULT_IMAGE_TOKEN, replace_token)
+ sentence['value'] = sentence['value'].replace(DEFAULT_VIDEO_TOKEN, vid_replace_token)
+ # print('after replace_token:', [sentence['value']])
+ # ======================================================================================================
+
+ return sources
+
+
+def preprocess_llama_2(
+ sources,
+ tokenizer: transformers.PreTrainedTokenizer,
+ has_image: bool = False
+) -> Dict:
+ conv = conversation_lib.default_conversation.copy()
+ roles = {"human": conv.roles[0], "gpt": conv.roles[1]}
+
+ # Apply prompt templates
+ conversations = []
+ for i, source in enumerate(sources):
+ if roles[source[0]["from"]] != conv.roles[0]:
+ # Skip the first one if it is not from human
+ source = source[1:]
+
+ conv.messages = []
+ for j, sentence in enumerate(source):
+ role = roles[sentence["from"]]
+ assert role == conv.roles[j % 2], f"{i}"
+ conv.append_message(role, sentence["value"])
+ conversations.append(conv.get_prompt())
+
+ # Tokenize conversations
+
+ if has_image:
+ input_ids = torch.stack([tokenizer_image_token(prompt, tokenizer, return_tensors='pt') for prompt in conversations], dim=0)
+ else:
+ input_ids = tokenizer(
+ conversations,
+ return_tensors="pt",
+ padding="longest",
+ max_length=tokenizer.model_max_length,
+ truncation=True,
+ ).input_ids
+
+ targets = input_ids.clone()
+
+ assert conv.sep_style == conversation_lib.SeparatorStyle.LLAMA_2
+
+ # Mask targets
+ sep = "[/INST] "
+ for conversation, target in zip(conversations, targets):
+ total_len = int(target.ne(tokenizer.pad_token_id).sum())
+
+ rounds = conversation.split(conv.sep2)
+ cur_len = 1
+ target[:cur_len] = IGNORE_INDEX
+ for i, rou in enumerate(rounds):
+ if rou == "":
+ break
+
+ parts = rou.split(sep)
+ if len(parts) != 2:
+ break
+ parts[0] += sep
+
+ if has_image:
+ round_len = len(tokenizer_image_token(rou, tokenizer))
+ instruction_len = len(tokenizer_image_token(parts[0], tokenizer)) - 2
+ else:
+ round_len = len(tokenizer(rou).input_ids)
+ instruction_len = len(tokenizer(parts[0]).input_ids) - 2
+
+ target[cur_len : cur_len + instruction_len] = IGNORE_INDEX
+
+ cur_len += round_len
+ target[cur_len:] = IGNORE_INDEX
+
+ if cur_len < tokenizer.model_max_length:
+ if cur_len != total_len:
+ target[:] = IGNORE_INDEX
+ print(
+ f"WARNING: tokenization mismatch: {cur_len} vs. {total_len}."
+ f" (ignored)"
+ )
+
+ return dict(
+ input_ids=input_ids,
+ labels=targets,
+ )
+
+
+def preprocess_v1(
+ sources,
+ tokenizer: transformers.PreTrainedTokenizer,
+ has_image: bool = False
+) -> Dict:
+ conv = conversation_lib.default_conversation.copy()
+ roles = {"human": conv.roles[0], "gpt": conv.roles[1]}
+
+ # Apply prompt templates
+ conversations = []
+ for i, source in enumerate(sources):
+ if roles[source[0]["from"]] != conv.roles[0]:
+ # Skip the first one if it is not from human
+ source = source[1:]
+
+ conv.messages = []
+ for j, sentence in enumerate(source):
+ role = roles[sentence["from"]]
+ assert role == conv.roles[j % 2], f"{i}"
+ conv.append_message(role, sentence["value"])
+ conversations.append(conv.get_prompt())
+
+ # Tokenize conversations
+
+ if has_image:
+ input_ids = torch.stack([tokenizer_image_token(prompt, tokenizer, return_tensors='pt') for prompt in conversations], dim=0)
+ else:
+ input_ids = tokenizer(
+ conversations,
+ return_tensors="pt",
+ padding="longest",
+ max_length=tokenizer.model_max_length,
+ truncation=True,
+ ).input_ids
+
+ targets = input_ids.clone()
+
+ assert conv.sep_style == conversation_lib.SeparatorStyle.TWO
+
+ # Mask targets
+ sep = conv.sep + conv.roles[1] + ": "
+ for conversation, target in zip(conversations, targets):
+ total_len = int(target.ne(tokenizer.pad_token_id).sum())
+
+ rounds = conversation.split(conv.sep2)
+ cur_len = 1
+ target[:cur_len] = IGNORE_INDEX
+ for i, rou in enumerate(rounds):
+ if rou == "":
+ break
+
+ parts = rou.split(sep)
+ if len(parts) != 2:
+ break
+ parts[0] += sep
+
+ if has_image:
+ round_len = len(tokenizer_image_token(rou, tokenizer))
+ instruction_len = len(tokenizer_image_token(parts[0], tokenizer)) - 2
+ else:
+ round_len = len(tokenizer(rou).input_ids)
+ instruction_len = len(tokenizer(parts[0]).input_ids) - 2
+
+ target[cur_len : cur_len + instruction_len] = IGNORE_INDEX
+
+ cur_len += round_len
+ target[cur_len:] = IGNORE_INDEX
+
+ if cur_len < tokenizer.model_max_length:
+ if cur_len != total_len:
+ target[:] = IGNORE_INDEX
+ print(
+ f"WARNING: tokenization mismatch: {cur_len} vs. {total_len}."
+ f" (ignored)"
+ )
+
+ return dict(
+ input_ids=input_ids,
+ labels=targets,
+ )
+
+
+def preprocess_mpt(
+ sources,
+ tokenizer: transformers.PreTrainedTokenizer,
+) -> Dict:
+ conv = conversation_lib.default_conversation.copy()
+ roles = {"human": conv.roles[0], "gpt": conv.roles[1]}
+
+ # Apply prompt templates
+ conversations = []
+ for i, source in enumerate(sources):
+ if roles[source[0]["from"]] != conv.roles[0]:
+ # Skip the first one if it is not from human
+ source = source[1:]
+
+ conv.messages = []
+ for j, sentence in enumerate(source):
+ role = roles[sentence["from"]]
+ assert role == conv.roles[j % 2], f"{i}"
+ conv.append_message(role, sentence["value"])
+ conversations.append(conv.get_prompt())
+
+ # Tokenize conversations
+ input_ids = torch.stack([tokenizer_image_token(prompt, tokenizer, return_tensors='pt') for prompt in conversations], dim=0)
+ targets = input_ids.clone()
+ assert conv.sep_style == conversation_lib.SeparatorStyle.MPT
+
+ # Mask targets
+ sep = conv.sep + conv.roles[1]
+ for conversation, target in zip(conversations, targets):
+ total_len = int(target.ne(tokenizer.pad_token_id).sum())
+
+ rounds = conversation.split(conv.sep)
+ re_rounds = [conv.sep.join(rounds[:3])] # system + user + gpt
+ for conv_idx in range(3, len(rounds), 2):
+ re_rounds.append(conv.sep.join(rounds[conv_idx:conv_idx+2])) # user + gpt
+ cur_len = 0
+ target[:cur_len] = IGNORE_INDEX
+ for i, rou in enumerate(re_rounds):
+ if rou == "":
+ break
+
+ parts = rou.split(sep)
+ if len(parts) != 2:
+ break
+ parts[0] += sep
+ round_len = len(tokenizer_image_token(rou, tokenizer)) + len(tokenizer_image_token(conv.sep, tokenizer))
+ instruction_len = len(tokenizer_image_token(parts[0], tokenizer))
+ target[cur_len : cur_len + instruction_len] = IGNORE_INDEX
+
+ cur_len += round_len
+ target[cur_len:] = IGNORE_INDEX
+
+ if cur_len < tokenizer.model_max_length:
+ if cur_len != total_len:
+ target[:] = IGNORE_INDEX
+ print(
+ f"WARNING: tokenization mismatch: {cur_len} vs. {total_len}."
+ f" (ignored)"
+ )
+
+ return dict(
+ input_ids=input_ids,
+ labels=targets,
+ )
+
+
+def preprocess_plain(
+ sources: Sequence[str],
+ tokenizer: transformers.PreTrainedTokenizer,
+) -> Dict:
+ # add end signal and concatenate together
+ conversations = []
+ for source in sources:
+ assert len(source) == 2
+ assert DEFAULT_IMAGE_TOKEN in source[0]['value']
+ source[0]['value'] = DEFAULT_IMAGE_TOKEN
+ conversation = source[0]['value'] + source[1]['value'] + conversation_lib.default_conversation.sep
+ conversations.append(conversation)
+ # tokenize conversations
+ input_ids = [tokenizer_image_token(prompt, tokenizer, return_tensors='pt') for prompt in conversations]
+ targets = copy.deepcopy(input_ids)
+ for target, source in zip(targets, sources):
+ tokenized_len = len(tokenizer_image_token(source[0]['value'], tokenizer))
+ target[:tokenized_len] = IGNORE_INDEX
+
+ return dict(input_ids=input_ids, labels=targets)
+
+
+def preprocess(
+ sources: Sequence[str],
+ tokenizer: transformers.PreTrainedTokenizer,
+ has_image: bool = False
+) -> Dict:
+ """
+ Given a list of sources, each is a conversation list. This transform:
+ 1. Add signal '### ' at the beginning each sentence, with end signal '\n';
+ 2. Concatenate conversations together;
+ 3. Tokenize the concatenated conversation;
+ 4. Make a deepcopy as the target. Mask human words with IGNORE_INDEX.
+ """
+ if conversation_lib.default_conversation.sep_style == conversation_lib.SeparatorStyle.PLAIN:
+ return preprocess_plain(sources, tokenizer)
+ if conversation_lib.default_conversation.sep_style == conversation_lib.SeparatorStyle.LLAMA_2:
+ return preprocess_llama_2(sources, tokenizer, has_image=has_image)
+ if conversation_lib.default_conversation.version.startswith("v1"):
+ return preprocess_v1(sources, tokenizer, has_image=has_image)
+ if conversation_lib.default_conversation.version == "mpt":
+ return preprocess_mpt(sources, tokenizer)
+ # add end signal and concatenate together
+ conversations = []
+ for source in sources:
+ header = f"{conversation_lib.default_conversation.system}\n\n"
+ conversation = _add_speaker_and_signal(header, source)
+ conversations.append(conversation)
+ # tokenize conversations
+ def get_tokenize_len(prompts):
+ return [len(tokenizer_image_token(prompt, tokenizer)) for prompt in prompts]
+
+ if has_image:
+ input_ids = [tokenizer_image_token(prompt, tokenizer, return_tensors='pt') for prompt in conversations]
+ else:
+ conversations_tokenized = _tokenize_fn(conversations, tokenizer)
+ input_ids = conversations_tokenized["input_ids"]
+
+ targets = copy.deepcopy(input_ids)
+ for target, source in zip(targets, sources):
+ if has_image:
+ tokenized_lens = get_tokenize_len([header] + [s["value"] for s in source])
+ else:
+ tokenized_lens = _tokenize_fn([header] + [s["value"] for s in source], tokenizer)["input_ids_lens"]
+ speakers = [sentence["from"] for sentence in source]
+ _mask_targets(target, tokenized_lens, speakers)
+
+ return dict(input_ids=input_ids, labels=targets)
+
+def expand2square(pil_img, background_color):
+ width, height = pil_img.size
+ if width == height:
+ return pil_img
+ elif width > height:
+ result = Image.new(pil_img.mode, (width, width), background_color)
+ result.paste(pil_img, (0, (width - height) // 2))
+ return result
+ else:
+ result = Image.new(pil_img.mode, (height, height), background_color)
+ result.paste(pil_img, ((height - width) // 2, 0))
+ return result
+
+class LazySupervisedDataset(Dataset):
+ """Dataset for supervised fine-tuning."""
+
+ def __init__(self, data_path: str,
+ tokenizer: transformers.PreTrainedTokenizer,
+ data_args: DataArguments):
+ super(LazySupervisedDataset, self).__init__()
+ # ================================================
+ list_data_dict = []
+ for data in data_path:
+ data = json.load(open(data, "r"))
+ for i in data:
+ i['id'] = len(list_data_dict)
+ list_data_dict.append(i)
+ # ================================================
+
+ rank0_print("Formatting inputs...Skip in lazy mode")
+ self.tokenizer = tokenizer
+ self.list_data_dict = list_data_dict
+ self.data_args = data_args
+ if self.data_args.image_folder is None:
+ print("Warning: 'image_folder' is None. Full paths to the images will be expected in the JSON.")
+ if self.data_args.video_folder is None:
+ print("Warning: 'video_folder' is None. Full paths to the videos will be expected in the JSON.")
+
+ def __len__(self):
+ return len(self.list_data_dict)
+
+ # @property
+ # def lengths(self):
+ # length_list = []
+ # for sample in self.list_data_dict:
+ # img_tokens = 128 if 'image' in sample else 0
+ # length_list.append(sum(len(conv['value'].split()) for conv in sample['conversations']) + img_tokens)
+ # return length_list
+
+ @property
+ def modality_lengths(self):
+ length_list = []
+ for sample in self.list_data_dict:
+ cur_len = sum(len(conv['value'].split()) for conv in sample['conversations'])
+ # ===========================================================================
+ cur_len = cur_len if ('image' in sample or 'video' in sample) else -cur_len
+ # ===========================================================================
+ length_list.append(cur_len)
+ return length_list
+ def __getitem__(self, i) -> Dict[str, torch.Tensor]:
+ try:
+ sources = self.list_data_dict[i]
+ if isinstance(i, int):
+ sources = [sources]
+ assert len(sources) == 1, "Don't know why it is wrapped to a list" # FIXME
+ # ======================================================================================================
+ if 'image' in sources[0] and 'video' not in sources[0]:
+ image_file = self.list_data_dict[i]['image']
+ image_processor = self.data_args.image_processor
+ image_file = image_file if isinstance(image_file, list) else [image_file]
+ image_file, _ = order_pick_k(image_file, MAX_IMAGE_LENGTH)
+ image_folder = self.data_args.image_folder
+ if image_folder is not None:
+ image = [Image.open(os.path.join(image_folder, file)).convert('RGB') for file in image_file]
+ else:
+ image = [Image.open(file).convert('RGB') for file in image_file]
+ if self.data_args.image_aspect_ratio == 'pad':
+ image = [expand2square(i, tuple(int(x * 255) for x in image_processor.image_mean)) for i in image]
+ image = [image_processor.preprocess(i, return_tensors='pt')['pixel_values'][0] for i in image]
+ else:
+ image = [image_processor.preprocess(i, return_tensors='pt')['pixel_values'][0] for i in image]
+ sources = preprocess_multimodal(copy.deepcopy([e["conversations"] for e in sources]), self.data_args, 1)
+ data_dict = preprocess(sources, self.tokenizer, has_image=True)
+
+ elif 'image' not in sources[0] and 'video' in sources[0]:
+ if not self.data_args.video_as_image_list:
+ video_file = self.list_data_dict[i]['video']
+ video_folder = self.data_args.video_folder
+ video_processor = self.data_args.video_processor
+ video_file = video_file if isinstance(video_file, list) else [video_file]
+ video_file = order_pick_k(video_file, MAX_VIDEO_LENGTH)
+ if video_folder is not None:
+ video = [os.path.join(video_folder, file) for file in video_file]
+ else:
+ video = video_file
+ image = [video_processor(i, return_tensors='pt')['pixel_values'][0] for i in video] # fake image
+ else:
+ image_files = self.list_data_dict[i]['video']
+ if not isinstance(image_files, list):
+ raise ValueError("Found single image but list of images expected")
+ image_files, indices = order_pick_k(image_files, MAX_IMAGE_LENGTH)
+ if 'metadata' in self.list_data_dict[i]:
+ metadata = self.list_data_dict[i]['metadata']
+ if indices is not None:
+ metadata = [metadata[i] for i in indices]
+ if 'timestamp' in metadata[0]:
+ # metadata has a 'timestamp' key for each element formatted "Y-M-d"
+ # sort the image files by the timestamp
+ image_files, metadata = zip(*sorted(
+ zip(image_files, metadata),
+ key=lambda t: datetime.strptime(t[1]["timestamp"], "%Y-%m-%d")
+ ))
+ image_folder = self.data_args.image_folder
+ if image_folder is not None:
+ image = [Image.open(os.path.join(image_folder, file)).convert('RGB')
+ for file in image_files]
+ else:
+ image = [Image.open(file).convert('RGB') for file in image_files]
+ image_processor = self.data_args.image_processor
+ if self.data_args.image_aspect_ratio == 'pad':
+ image = [expand2square(i, tuple(int(x * 255) for x in image_processor.image_mean)) for i in image]
+ image = [image_processor.preprocess(i, return_tensors='pt')['pixel_values'][0] for i in image]
+ else:
+ image = [image_processor.preprocess(i, return_tensors='pt')['pixel_values'][0] for i in image]
+
+ sources = preprocess_multimodal(copy.deepcopy([e["conversations"] for e in sources]), self.data_args)
+ num_video_images = len(image)
+ data_dict = preprocess(sources, self.tokenizer, has_image=True)
+
+ elif 'image' in sources[0] and 'video' in sources[0]:
+ # rank0_print('image & video')
+ # video must before image
+ video_file = self.list_data_dict[i]['video']
+ video_folder = self.data_args.video_folder
+ video_processor = self.data_args.video_processor
+
+ image_file = self.list_data_dict[i]['image']
+ image_folder = self.data_args.image_folder
+ image_processor = self.data_args.image_processor
+
+ image_file = image_file if isinstance(image_file, list) else [image_file]
+ image_file = order_pick_k(image_file, MAX_IMAGE_LENGTH)
+ if image_folder is not None:
+ image = [Image.open(os.path.join(image_folder, file)).convert('RGB') for file in image_file]
+ else:
+ image = [Image.open(file).convert('RGB') for file in image_file]
+ if self.data_args.image_aspect_ratio == 'pad':
+ image = [expand2square(i, tuple(int(x * 255) for x in image_processor.image_mean)) for i in image]
+ image = [image_processor.preprocess(i, return_tensors='pt')['pixel_values'][0] for i in image]
+ else:
+ image = [image_processor.preprocess(i, return_tensors='pt')['pixel_values'][0] for i in image]
+
+ if not self.data_args.video_as_image_list:
+ video_file = video_file if isinstance(video_file, list) else [video_file]
+ video_file = order_pick_k(video_file, MAX_VIDEO_LENGTH)
+ if video_folder is not None:
+ video = [os.path.join(video_folder, file) for file in video_file]
+ else:
+ video = video_file
+ video = [video_processor(i, return_tensors='pt')['pixel_values'][0] for i in video] # fake image
+ else:
+ image_files, indices = order_pick_k(image_files, MAX_IMAGE_LENGTH)
+ if 'metadata' in self.list_data_dict[i]:
+ metadata = self.list_data_dict[i]['metadata']
+ if indices is not None:
+ metadata = [metadata[i] for i in indices]
+ if 'timestamp' in metadata[0]:
+ # metadata has a 'timestamp' key for each element formatted "Y-M-d"
+ # sort the image files by the timestamp
+ image_files, metadata = zip(*sorted(
+ zip(image_files, metadata),
+ key=lambda t: datetime.strptime(t[1]["timestamp"], "%Y-%m-%d")
+ ))
+ if image_folder is not None:
+ video = [Image.open(os.path.join(image_folder, file)).convert('RGB')
+ for file in image_files]
+ else:
+ video = [Image.open(file).convert('RGB') for file in image_files]
+ if self.data_args.image_aspect_ratio == 'pad':
+ video = [expand2square(i, tuple(int(x * 255) for x in image_processor.image_mean)) for i in video]
+ video = [image_processor.preprocess(i, return_tensors='pt')['pixel_values'][0] for i in video]
+ else:
+ video = [image_processor.preprocess(i, return_tensors='pt')['pixel_values'][0] for i in video]
+
+ image = video + image # video must before image
+
+ sources = preprocess_multimodal(copy.deepcopy([e["conversations"] for e in sources]), self.data_args)
+ data_dict = preprocess(sources, self.tokenizer, has_image=True)
+ else:
+ sources = copy.deepcopy([e["conversations"] for e in sources])
+ data_dict = preprocess(sources, self.tokenizer, has_image=False)
+
+ # ==========================================================================================================
+
+ if isinstance(i, int):
+ data_dict = dict(input_ids=data_dict["input_ids"][0],
+ labels=data_dict["labels"][0])
+ # image exist in the data
+ if 'image' in self.list_data_dict[i] or 'video' in self.list_data_dict[i]:
+ data_dict['image'] = image
+ elif self.data_args.is_multimodal:
+ # image does not exist in the data, but the model is multimodal
+ # crop_size = self.data_args.image_processor.crop_size
+ crop_size = {'height': 224, 'width': 224} # dummy image
+ data_dict['image'] = [torch.zeros(3, crop_size['height'], crop_size['width'])]
+ return data_dict
+ except Exception as e:
+ print(f'Error with {e}')
+ return self.__getitem__(random.randint(0, self.__len__() - 1))
+
+
+@dataclass
+class DataCollatorForSupervisedDataset(object):
+ """Collate examples for supervised fine-tuning."""
+
+ tokenizer: transformers.PreTrainedTokenizer
+
+ def __call__(self, instances: Sequence[Dict]) -> Dict[str, torch.Tensor]:
+ input_ids, labels = tuple([instance[key] for instance in instances]
+ for key in ("input_ids", "labels"))
+ input_ids = torch.nn.utils.rnn.pad_sequence(
+ input_ids,
+ batch_first=True,
+ padding_value=self.tokenizer.pad_token_id)
+ labels = torch.nn.utils.rnn.pad_sequence(labels,
+ batch_first=True,
+ padding_value=IGNORE_INDEX)
+ input_ids = input_ids[:, :self.tokenizer.model_max_length]
+ labels = labels[:, :self.tokenizer.model_max_length]
+ batch = dict(
+ input_ids=input_ids,
+ labels=labels,
+ attention_mask=input_ids.ne(self.tokenizer.pad_token_id),
+ )
+
+ # ======================================================================================================
+ # origin image, if batch_size=6: [[image], [image], [video], [image, image], [video, video], [video, image]]
+ '''
+ will be converted to a sequence of list, if batch size=6:
+ [
+ image(3, 224, 224), # sample 1
+ image(3, 224, 224), # sample 2
+ video(8, 3, 224, 224), # sample 3
+ image(3, 224, 224), # sample 4
+ image(3, 224, 224), # sample 4
+ video(8, 3, 224, 224), # sample 5
+ video(8, 3, 224, 224), # sample 5
+ video(8, 3, 224, 224), # sample 6
+ image(3, 224, 224), # sample 6
+ ]
+ '''
+ if 'image' in instances[0]:
+ images = [instance['image'] for instance in instances]
+
+ # adapt to multi-video or multi-image or multi-image & video
+ new_images = []
+ for image in images:
+ if type(image) is list:
+ for i in image:
+ new_images.append(i)
+ else:
+ new_images.append(image)
+ images = new_images
+
+ # ==========Too many videos or images may lead to OOM, so we encode them one by one======================
+ batch['images'] = images
+ # if all(x is not None and x.shape == images[0].shape for x in images): # if all images or all videos
+ # batch['images'] = torch.stack(images)
+ # else:
+ # batch['images'] = images
+ else:
+ raise ValueError(f'pretrain, {instances}')
+ return batch
+
+
+def make_supervised_data_module(tokenizer: transformers.PreTrainedTokenizer,
+ data_args) -> Dict:
+ """Make dataset and collator for supervised fine-tuning."""
+ train_dataset = LazySupervisedDataset(tokenizer=tokenizer,
+ data_path=data_args.data_path,
+ data_args=data_args)
+ data_collator = DataCollatorForSupervisedDataset(tokenizer=tokenizer)
+ return dict(train_dataset=train_dataset,
+ eval_dataset=None,
+ data_collator=data_collator)
+
+
+def train():
+ global local_rank
+
+ parser = transformers.HfArgumentParser(
+ (ModelArguments, DataArguments, TrainingArguments))
+ model_args, data_args, training_args = parser.parse_args_into_dataclasses()
+ local_rank = training_args.local_rank
+ compute_dtype = (torch.float16 if training_args.fp16 else (torch.bfloat16 if training_args.bf16 else torch.float32))
+
+ # Set wandb run name to the output directory name and the date+time
+ if 'wandb' in training_args.report_to:
+ training_args.run_name = (
+ training_args.logging_dir.split('/')[-3] + '-' + training_args.logging_dir.split('/')[-1]
+ )
+
+ bnb_model_from_pretrained_args = {}
+ if training_args.bits in [4, 8]:
+ from transformers import BitsAndBytesConfig
+ bnb_model_from_pretrained_args.update(dict(
+ # device_map={"": training_args.device},
+ load_in_4bit=training_args.bits == 4,
+ load_in_8bit=training_args.bits == 8,
+ quantization_config=BitsAndBytesConfig(
+ load_in_4bit=training_args.bits == 4,
+ load_in_8bit=training_args.bits == 8,
+ llm_int8_skip_modules=["mm_projector"],
+ llm_int8_threshold=6.0,
+ llm_int8_has_fp16_weight=False,
+ bnb_4bit_compute_dtype=compute_dtype,
+ bnb_4bit_use_double_quant=training_args.double_quant,
+ bnb_4bit_quant_type=training_args.quant_type # {'fp4', 'nf4'}
+ )
+ ))
+ # ==========================================================================
+ if model_args.image_tower is not None or model_args.video_tower is not None:
+ # ==========================================================================
+ if 'mpt' in model_args.model_name_or_path:
+ config = transformers.AutoConfig.from_pretrained(model_args.model_name_or_path, trust_remote_code=True)
+ config.attn_config['attn_impl'] = training_args.mpt_attn_impl
+ model = LlavaMPTForCausalLM.from_pretrained(
+ model_args.model_name_or_path,
+ config=config,
+ cache_dir=training_args.cache_dir,
+ **bnb_model_from_pretrained_args
+ )
+ else:
+ model = LlavaLlamaForCausalLM.from_pretrained(
+ model_args.model_name_or_path,
+ cache_dir=training_args.cache_dir,
+ **bnb_model_from_pretrained_args
+ )
+ else:
+ model = transformers.LlamaForCausalLM.from_pretrained(
+ model_args.model_name_or_path,
+ cache_dir=training_args.cache_dir,
+ **bnb_model_from_pretrained_args
+ )
+ model.config.use_cache = False
+
+ if model_args.freeze_backbone:
+ model.model.requires_grad_(False)
+
+ if training_args.bits in [4, 8]:
+ from peft import prepare_model_for_kbit_training
+ model.config.torch_dtype=(torch.float32 if training_args.fp16 else (torch.bfloat16 if training_args.bf16 else torch.float32))
+ model = prepare_model_for_kbit_training(model, use_gradient_checkpointing=training_args.gradient_checkpointing)
+
+ if training_args.gradient_checkpointing:
+ if hasattr(model, "enable_input_require_grads"):
+ model.enable_input_require_grads()
+ else:
+ def make_inputs_require_grad(module, input, output):
+ output.requires_grad_(True)
+ model.get_input_embeddings().register_forward_hook(make_inputs_require_grad)
+
+ if training_args.lora_enable:
+ from peft import LoraConfig, get_peft_model
+ lora_config = LoraConfig(
+ r=training_args.lora_r,
+ lora_alpha=training_args.lora_alpha,
+ target_modules=find_all_linear_names(model),
+ lora_dropout=training_args.lora_dropout,
+ bias=training_args.lora_bias,
+ task_type="CAUSAL_LM",
+ )
+ if training_args.bits == 16:
+ if training_args.bf16:
+ model.to(torch.bfloat16)
+ if training_args.fp16:
+ model.to(torch.float16)
+ rank0_print("Adding LoRA adapters...")
+ model = get_peft_model(model, lora_config)
+
+ if 'mpt' in model_args.model_name_or_path:
+ tokenizer = transformers.AutoTokenizer.from_pretrained(
+ model_args.model_name_or_path,
+ cache_dir=training_args.cache_dir,
+ model_max_length=training_args.model_max_length,
+ padding_side="right"
+ )
+ else:
+ tokenizer = transformers.AutoTokenizer.from_pretrained(
+ model_args.model_name_or_path,
+ cache_dir=training_args.cache_dir,
+ model_max_length=training_args.model_max_length,
+ padding_side="right",
+ use_fast=False,
+ )
+
+ if model_args.version == "v0":
+ if tokenizer.pad_token is None:
+ smart_tokenizer_and_embedding_resize(
+ special_tokens_dict=dict(pad_token="[PAD]"),
+ tokenizer=tokenizer,
+ model=model,
+ )
+ elif model_args.version == "v0.5":
+ tokenizer.pad_token = tokenizer.unk_token
+ else:
+ tokenizer.pad_token = tokenizer.unk_token
+ if model_args.version in conversation_lib.conv_templates:
+ conversation_lib.default_conversation = conversation_lib.conv_templates[model_args.version]
+ else:
+ conversation_lib.default_conversation = conversation_lib.conv_templates["vicuna_v1"]
+
+ # =============================================================================================================
+ if model_args.image_tower is not None or model_args.video_tower is not None:
+ # print(model_args)
+ model.get_model().initialize_vision_modules(
+ model_args=model_args,
+ fsdp=training_args.fsdp
+ )
+ if model_args.image_tower is not None:
+ image_tower = model.get_image_tower()
+ image_tower.to(dtype=torch.bfloat16 if training_args.bf16 else torch.float16, device=training_args.device)
+
+ data_args.image_processor = image_tower.image_processor
+ data_args.is_multimodal = True
+ if model_args.video_tower is not None:
+ video_tower = model.get_video_tower()
+ video_tower.to(dtype=torch.bfloat16 if training_args.bf16 else torch.float16, device=training_args.device)
+
+ data_args.video_processor = video_tower.video_processor
+ data_args.is_multimodal = True
+ data_args.num_frames = video_tower.config.num_frames
+ # =============================================================================================================
+
+
+ model.config.image_aspect_ratio = data_args.image_aspect_ratio
+ model.config.tokenizer_padding_side = tokenizer.padding_side
+
+ # =============================================================================================================
+ tokenizer_model_max_length = training_args.tokenizer_model_max_length
+ model.config.tokenizer_model_max_length = tokenizer.model_max_length if tokenizer_model_max_length is None else tokenizer_model_max_length
+ # =============================================================================================================
+
+ model.config.tune_mm_mlp_adapter = training_args.tune_mm_mlp_adapter = model_args.tune_mm_mlp_adapter
+ if model_args.tune_mm_mlp_adapter:
+ model.requires_grad_(False)
+ for p in model.get_model().mm_projector.parameters():
+ p.requires_grad = True
+
+ model.config.freeze_mm_mlp_adapter = training_args.freeze_mm_mlp_adapter
+ if training_args.freeze_mm_mlp_adapter:
+ for p in model.get_model().mm_projector.parameters():
+ p.requires_grad = False
+
+ if training_args.bits in [4, 8]:
+ model.get_model().mm_projector.to(dtype=compute_dtype, device=training_args.device)
+
+ model.config.mm_use_im_start_end = data_args.mm_use_im_start_end = model_args.mm_use_im_start_end
+ model.config.mm_projector_lr = training_args.mm_projector_lr
+ training_args.use_im_start_end = model_args.mm_use_im_start_end
+ model.config.mm_use_im_patch_token = model_args.mm_use_im_patch_token
+ model.initialize_vision_tokenizer(model_args, tokenizer=tokenizer)
+
+ if training_args.bits in [4, 8]:
+ from peft.tuners.lora import LoraLayer
+ for name, module in model.named_modules():
+ if isinstance(module, LoraLayer):
+ if training_args.bf16:
+ module = module.to(torch.bfloat16)
+ if 'norm' in name:
+ module = module.to(torch.float32)
+ if 'lm_head' in name or 'embed_tokens' in name:
+ if hasattr(module, 'weight'):
+ if training_args.bf16 and module.weight.dtype == torch.float32:
+ module = module.to(torch.bfloat16)
+
+ data_module = make_supervised_data_module(tokenizer=tokenizer,
+ data_args=data_args)
+ trainer = LLaVATrainer(model=model,
+ tokenizer=tokenizer,
+ args=training_args,
+ **data_module)
+
+ if list(pathlib.Path(training_args.output_dir).glob("checkpoint-*")):
+ trainer.train(resume_from_checkpoint=True)
+ else:
+ trainer.train()
+ trainer.save_state()
+
+ model.config.use_cache = True
+
+ if training_args.lora_enable:
+ state_dict = get_peft_state_maybe_zero_3(
+ model.named_parameters(), training_args.lora_bias
+ )
+ non_lora_state_dict = get_peft_state_non_lora_maybe_zero_3(
+ model.named_parameters()
+ )
+ if training_args.local_rank == 0 or training_args.local_rank == -1:
+ model.config.save_pretrained(training_args.output_dir)
+ model.save_pretrained(training_args.output_dir, state_dict=state_dict)
+ torch.save(non_lora_state_dict, os.path.join(training_args.output_dir, 'non_lora_trainables.bin'))
+ else:
+ safe_save_model_for_hf_trainer(trainer=trainer,
+ output_dir=training_args.output_dir)
+
+
+if __name__ == "__main__":
+ train()
diff --git a/videollava/train/train_mem.py b/videollava/train/train_mem.py
new file mode 100644
index 0000000000000000000000000000000000000000..4c26cfaa83b541cf4df48fda57cd305e8afbaf63
--- /dev/null
+++ b/videollava/train/train_mem.py
@@ -0,0 +1,13 @@
+# Adopted from https://github.com/lm-sys/FastChat. Below is the original copyright:
+# Adopted from tatsu-lab@stanford_alpaca. Below is the original copyright:
+# Make it more memory efficient by monkey patching the LLaMA model with FlashAttn.
+
+# Need to call this before importing transformers.
+from videollava.train.llama_flash_attn_monkey_patch import replace_llama_attn_with_flash_attn
+
+replace_llama_attn_with_flash_attn()
+
+from videollava.train.train import train
+
+if __name__ == "__main__":
+ train()
diff --git a/videollava/utils.py b/videollava/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..c235111d35e9c9fcc68c83903ec6ecceb8b9396d
--- /dev/null
+++ b/videollava/utils.py
@@ -0,0 +1,141 @@
+import datetime
+import logging
+import logging.handlers
+import os
+import sys
+from torch import nn
+import numpy as np
+import requests
+
+from videollava.constants import LOGDIR
+
+
+server_error_msg = "**NETWORK ERROR DUE TO HIGH TRAFFIC. PLEASE REGENERATE OR REFRESH THIS PAGE.**"
+moderation_msg = "YOUR INPUT VIOLATES OUR CONTENT MODERATION GUIDELINES. PLEASE TRY AGAIN."
+
+handler = None
+
+def order_pick_k(lst, k):
+ if len(lst) <= k:
+ return lst, None
+ rng = np.random.random(len(lst))
+ index = np.argsort(rng)[:k]
+ index_sort = sorted(index)
+ new_lst = [lst[i] for i in index_sort]
+ print(
+ f"WARNING: total file: {len(lst)}, random pick: {k}."
+ f" (ignored)"
+ )
+ return new_lst, index_sort
+
+
+def build_logger(logger_name, logger_filename):
+ global handler
+
+ formatter = logging.Formatter(
+ fmt="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
+ datefmt="%Y-%m-%d %H:%M:%S",
+ )
+
+ # Set the format of root handlers
+ if not logging.getLogger().handlers:
+ logging.basicConfig(level=logging.INFO)
+ logging.getLogger().handlers[0].setFormatter(formatter)
+
+ # Redirect stdout and stderr to loggers
+ stdout_logger = logging.getLogger("stdout")
+ stdout_logger.setLevel(logging.INFO)
+ sl = StreamToLogger(stdout_logger, logging.INFO)
+ sys.stdout = sl
+
+ stderr_logger = logging.getLogger("stderr")
+ stderr_logger.setLevel(logging.ERROR)
+ sl = StreamToLogger(stderr_logger, logging.ERROR)
+ sys.stderr = sl
+
+ # Get logger
+ logger = logging.getLogger(logger_name)
+ logger.setLevel(logging.INFO)
+
+ # Add a file handler for all loggers
+ if handler is None:
+ os.makedirs(LOGDIR, exist_ok=True)
+ filename = os.path.join(LOGDIR, logger_filename)
+ handler = logging.handlers.TimedRotatingFileHandler(
+ filename, when='D', utc=True, encoding='UTF-8')
+ handler.setFormatter(formatter)
+
+ for name, item in logging.root.manager.loggerDict.items():
+ if isinstance(item, logging.Logger):
+ item.addHandler(handler)
+
+ return logger
+
+
+class StreamToLogger(object):
+ """
+ Fake file-like stream object that redirects writes to a logger instance.
+ """
+ def __init__(self, logger, log_level=logging.INFO):
+ self.terminal = sys.stdout
+ self.logger = logger
+ self.log_level = log_level
+ self.linebuf = ''
+
+ def __getattr__(self, attr):
+ return getattr(self.terminal, attr)
+
+ def write(self, buf):
+ temp_linebuf = self.linebuf + buf
+ self.linebuf = ''
+ for line in temp_linebuf.splitlines(True):
+ # From the io.TextIOWrapper docs:
+ # On output, if newline is None, any '\n' characters written
+ # are translated to the system default line separator.
+ # By default sys.stdout.write() expects '\n' newlines and then
+ # translates them so this is still cross platform.
+ if line[-1] == '\n':
+ self.logger.log(self.log_level, line.rstrip())
+ else:
+ self.linebuf += line
+
+ def flush(self):
+ if self.linebuf != '':
+ self.logger.log(self.log_level, self.linebuf.rstrip())
+ self.linebuf = ''
+
+
+def disable_torch_init():
+ """
+ Disable the redundant torch default initialization to accelerate model creation.
+ """
+ import torch
+ setattr(torch.nn.Linear, "reset_parameters", lambda self: None)
+ setattr(torch.nn.LayerNorm, "reset_parameters", lambda self: None)
+
+
+def violates_moderation(text):
+ """
+ Check whether the text violates OpenAI moderation API.
+ """
+ url = "https://api.openai.com/v1/moderations"
+ headers = {"Content-Type": "application/json",
+ "Authorization": "Bearer " + os.environ["OPENAI_API_KEY"]}
+ text = text.replace("\n", "")
+ data = "{" + '"input": ' + f'"{text}"' + "}"
+ data = data.encode("utf-8")
+ try:
+ ret = requests.post(url, headers=headers, data=data, timeout=5)
+ flagged = ret.json()["results"][0]["flagged"]
+ except requests.exceptions.RequestException as e:
+ flagged = False
+ except KeyError as e:
+ flagged = False
+
+ return flagged
+
+
+def pretty_print_semaphore(semaphore):
+ if semaphore is None:
+ return "None"
+ return f"Semaphore(value={semaphore._value}, locked={semaphore.locked()})"