Upload 161 files

.gitattributes

@@ -33,3 +33,49 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+data/epic_gif.gif filter=lfs diff=lfs merge=lfs -text
+data/test_i2v/masked_videos/000000000005.2.013.mp4 filter=lfs diff=lfs merge=lfs -text
+data/test_i2v/masked_videos/000000000793.0.003.mp4 filter=lfs diff=lfs merge=lfs -text
+data/test_i2v/masked_videos/000000000896.2.005.mp4 filter=lfs diff=lfs merge=lfs -text
+data/test_i2v/masked_videos/000000001059.6.004.mp4 filter=lfs diff=lfs merge=lfs -text
+data/test_i2v/masked_videos/000000049282.4.005.mp4 filter=lfs diff=lfs merge=lfs -text
+data/test_i2v/masked_videos/000000049768.11.002.mp4 filter=lfs diff=lfs merge=lfs -text
+data/test_i2v/masked_videos/000002012796.6.001.mp4 filter=lfs diff=lfs merge=lfs -text
+data/test_i2v/masked_videos/000003031893.11.003.mp4 filter=lfs diff=lfs merge=lfs -text
+data/test_i2v/videos/000000000005.2.013.mp4 filter=lfs diff=lfs merge=lfs -text
+data/test_i2v/videos/000000000793.0.003.mp4 filter=lfs diff=lfs merge=lfs -text
+data/test_i2v/videos/000000000896.2.005.mp4 filter=lfs diff=lfs merge=lfs -text
+data/test_i2v/videos/000000001059.6.004.mp4 filter=lfs diff=lfs merge=lfs -text
+data/test_i2v/videos/000000049282.4.005.mp4 filter=lfs diff=lfs merge=lfs -text
+data/test_i2v/videos/000000049768.11.002.mp4 filter=lfs diff=lfs merge=lfs -text
+data/test_i2v/videos/000002012796.6.001.mp4 filter=lfs diff=lfs merge=lfs -text
+data/test_i2v/videos/000003031893.11.003.mp4 filter=lfs diff=lfs merge=lfs -text
+data/test_v2v/masked_videos/amalfi-coast_traj_loop2.mp4 filter=lfs diff=lfs merge=lfs -text
+data/test_v2v/masked_videos/art-museum_gradual_0_-30_0.0_0_0.mp4 filter=lfs diff=lfs merge=lfs -text
+data/test_v2v/masked_videos/dog_gradual_0_-30_-0.6_0_0.mp4 filter=lfs diff=lfs merge=lfs -text
+data/test_v2v/masked_videos/fish_gradual_25_0_0.0_0_0.mp4 filter=lfs diff=lfs merge=lfs -text
+data/test_v2v/masked_videos/grandma-birthday_gradual_0_0_0.2_0_0.mp4 filter=lfs diff=lfs merge=lfs -text
+data/test_v2v/masked_videos/happy-cat_gradual_-20_0_0.0_0_0.mp4 filter=lfs diff=lfs merge=lfs -text
+data/test_v2v/masked_videos/suv-in-the-dust_gradual_0_30_0.0_0_0.mp4 filter=lfs diff=lfs merge=lfs -text
+data/test_v2v/masked_videos/vlogger-corgi_traj_loop2.mp4 filter=lfs diff=lfs merge=lfs -text
+data/test_v2v/videos/amalfi-coast_traj_loop2.mp4 filter=lfs diff=lfs merge=lfs -text
+data/test_v2v/videos/art-museum_gradual_0_-30_0.0_0_0.mp4 filter=lfs diff=lfs merge=lfs -text
+data/test_v2v/videos/dog_gradual_0_-30_-0.6_0_0.mp4 filter=lfs diff=lfs merge=lfs -text
+data/test_v2v/videos/fish_gradual_25_0_0.0_0_0.mp4 filter=lfs diff=lfs merge=lfs -text
+data/test_v2v/videos/grandma-birthday_gradual_0_0_0.2_0_0.mp4 filter=lfs diff=lfs merge=lfs -text
+data/test_v2v/videos/happy-cat_gradual_-20_0_0.0_0_0.mp4 filter=lfs diff=lfs merge=lfs -text
+data/test_v2v/videos/suv-in-the-dust_gradual_0_30_0.0_0_0.mp4 filter=lfs diff=lfs merge=lfs -text
+data/test_v2v/videos/vlogger-corgi_traj_loop2.mp4 filter=lfs diff=lfs merge=lfs -text
+inference/v2v_data/test/videos/0-NNvgaTcVzAG0-r.mp4 filter=lfs diff=lfs merge=lfs -text
+inference/v2v_data/test/videos/p7.mp4 filter=lfs diff=lfs merge=lfs -text
+preprocess/RAFT/demo-frames/frame_0016.png filter=lfs diff=lfs merge=lfs -text
+preprocess/RAFT/demo-frames/frame_0017.png filter=lfs diff=lfs merge=lfs -text
+preprocess/RAFT/demo-frames/frame_0018.png filter=lfs diff=lfs merge=lfs -text
+preprocess/RAFT/demo-frames/frame_0019.png filter=lfs diff=lfs merge=lfs -text
+preprocess/RAFT/demo-frames/frame_0020.png filter=lfs diff=lfs merge=lfs -text
+preprocess/RAFT/demo-frames/frame_0021.png filter=lfs diff=lfs merge=lfs -text
+preprocess/RAFT/demo-frames/frame_0022.png filter=lfs diff=lfs merge=lfs -text
+preprocess/RAFT/demo-frames/frame_0023.png filter=lfs diff=lfs merge=lfs -text
+preprocess/RAFT/demo-frames/frame_0024.png filter=lfs diff=lfs merge=lfs -text
+preprocess/RAFT/demo-frames/frame_0025.png filter=lfs diff=lfs merge=lfs -text
+preprocess/RAFT/RAFT.png filter=lfs diff=lfs merge=lfs -text

Dockerfile

@@ -0,0 +1,44 @@
+FROM pytorch/pytorch:2.2.2-cuda12.1-cudnn8-runtime
+
+SHELL ["/bin/bash", "-c"]
+
+# Environment variables for Hugging Face cache
+ENV HF_HOME=/app/hf_cache
+ENV TRANSFORMERS_CACHE=/app/hf_cache
+ENV HF_TOKEN=${HF_TOKEN}
+ENV PATH=/opt/conda/bin:$PATH
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    git wget curl unzip ffmpeg libgl1-mesa-glx libglib2.0-0 && \
+    apt-get clean
+
+# Set up working directory as /app
+WORKDIR /app
+
+# Copy project into /app
+COPY . /app
+
+# Fix permissions for all subdirectories
+RUN mkdir -p /app/pretrained /app/hf_cache /.cache/gdown && \
+    chmod -R 777 /app && \
+    chmod -R 777 /.cache && \
+    chmod -R 777 /root
+
+# Create conda environment and install dependencies
+COPY requirements.txt /app/requirements.txt
+RUN conda create -n epic python=3.10 -y && \
+    conda run -n epic pip install --upgrade pip && \
+    conda run -n epic pip install -r /app/requirements.txt
+
+RUN chmod -R 777 /app /workspace
+
+# # List contents (for debug)
+RUN ls -la /app
+RUN pip install gradio
+
+# Expose Gradio default port
+EXPOSE 7860
+
+# Start the Gradio app
+CMD ["conda", "run", "--no-capture-output", "-n", "epic", "python", "gradio_app.py"]

README.md

@@ -1,11 +1,10 @@
 ---
-title: EPiC
-emoji: 🐨
-colorFrom: red
-colorTo: red
+title: EPiC - Control
+emoji: 📚
+colorFrom: indigo
+colorTo: blue
 sdk: docker
+app_file: Dockerfile
 pinned: false
 license: mit
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+---

cogvideo_controlnet_pcd.py

@@ -0,0 +1,235 @@
+from typing import Any, Dict, Optional, Tuple, Union
+
+import torch
+from torch import nn
+from einops import rearrange
+import torch.nn.functional as F
+from diffusers.models.transformers.cogvideox_transformer_3d import Transformer2DModelOutput, CogVideoXBlock
+from diffusers.utils import is_torch_version
+from diffusers.loaders import  PeftAdapterMixin
+from diffusers.utils.torch_utils import maybe_allow_in_graph
+from diffusers.models.embeddings import CogVideoXPatchEmbed, TimestepEmbedding, Timesteps, get_3d_sincos_pos_embed
+from diffusers.models.modeling_utils import ModelMixin
+from diffusers.models.attention import Attention, FeedForward
+from diffusers.models.attention_processor import AttentionProcessor, AttnProcessor2_0
+from diffusers.models.normalization import AdaLayerNorm, CogVideoXLayerNormZero, AdaLayerNormZeroSingle
+from diffusers.configuration_utils import ConfigMixin, register_to_config
+
+
+class CogVideoXControlnetPCD(ModelMixin, ConfigMixin, PeftAdapterMixin):
+    _supports_gradient_checkpointing = True
+    
+    @register_to_config
+    def __init__(
+        self,
+        num_attention_heads: int = 30,
+        use_zero_conv: bool = False,
+        attention_head_dim: int = 64,
+        vae_channels: int = 16,
+        in_channels: int = 3,
+        downscale_coef: int = 8,
+        flip_sin_to_cos: bool = True,
+        freq_shift: int = 0,
+        time_embed_dim: int = 512,
+        num_layers: int = 8,
+        dropout: float = 0.0,
+        attention_bias: bool = True,
+        sample_width: int = 90,
+        sample_height: int = 60,
+        sample_frames: int = 49,
+        patch_size: int = 2,
+        temporal_compression_ratio: int = 4,
+        max_text_seq_length: int = 226,
+        activation_fn: str = "gelu-approximate",
+        timestep_activation_fn: str = "silu",
+        norm_elementwise_affine: bool = True,
+        norm_eps: float = 1e-5,
+        spatial_interpolation_scale: float = 1.875,
+        temporal_interpolation_scale: float = 1.0,
+        use_rotary_positional_embeddings: bool = False,
+        use_learned_positional_embeddings: bool = False,
+        out_proj_dim: int = None,
+        out_proj_dim_zero_init: bool = False,
+    ):
+        super().__init__()
+        inner_dim = num_attention_heads * attention_head_dim
+
+        if not use_rotary_positional_embeddings and use_learned_positional_embeddings:
+            raise ValueError(
+                "There are no CogVideoX checkpoints available with disable rotary embeddings and learned positional "
+                "embeddings. If you're using a custom model and/or believe this should be supported, please open an "
+                "issue at https://github.com/huggingface/diffusers/issues."
+            )
+        
+        self.vae_channels = vae_channels
+        start_channels = in_channels * (downscale_coef ** 2)
+        input_channels = [start_channels, start_channels // 2, start_channels // 4]
+        self.unshuffle = nn.PixelUnshuffle(downscale_coef)
+        self.use_zero_conv = use_zero_conv
+        
+        if use_zero_conv:
+            self.controlnet_encode_first = nn.Sequential(
+                nn.Conv2d(input_channels[0], input_channels[1], kernel_size=1, stride=1, padding=0),
+                nn.GroupNorm(2, input_channels[1]),
+                nn.ReLU(),
+            )
+
+            self.controlnet_encode_second = nn.Sequential(
+                nn.Conv2d(input_channels[1], input_channels[2], kernel_size=1, stride=1, padding=0),
+                nn.GroupNorm(2, input_channels[2]),
+                nn.ReLU(),
+            )
+            patch_embed_in_channels = vae_channels + input_channels[2]
+        
+        else:
+            patch_embed_in_channels = vae_channels*2
+            
+        # 1. Patch embedding
+        self.patch_embed = CogVideoXPatchEmbed(
+            patch_size=patch_size,
+            in_channels=patch_embed_in_channels,
+            embed_dim=inner_dim,
+            bias=True,
+            sample_width=sample_width,
+            sample_height=sample_height,
+            sample_frames=sample_frames,
+            temporal_compression_ratio=temporal_compression_ratio,
+            spatial_interpolation_scale=spatial_interpolation_scale,
+            temporal_interpolation_scale=temporal_interpolation_scale,
+            use_positional_embeddings=not use_rotary_positional_embeddings,
+            use_learned_positional_embeddings=use_learned_positional_embeddings,
+        )
+        
+        self.embedding_dropout = nn.Dropout(dropout)
+
+        # 2. Time embeddings
+        self.time_proj = Timesteps(inner_dim, flip_sin_to_cos, freq_shift)
+        self.time_embedding = TimestepEmbedding(inner_dim, time_embed_dim, timestep_activation_fn)
+
+        # 3. Define spatio-temporal transformers blocks
+        self.transformer_blocks = nn.ModuleList(
+            [
+                CogVideoXBlock(
+                    dim=inner_dim,
+                    num_attention_heads=num_attention_heads,
+                    attention_head_dim=attention_head_dim,
+                    time_embed_dim=time_embed_dim,
+                    dropout=dropout,
+                    activation_fn=activation_fn,
+                    attention_bias=attention_bias,
+                    norm_elementwise_affine=norm_elementwise_affine,
+                    norm_eps=norm_eps,
+                )
+                for _ in range(num_layers)
+            ]
+        )
+
+        self.out_projectors = None
+        if out_proj_dim is not None:
+            self.out_projectors = nn.ModuleList(
+                [nn.Linear(inner_dim, out_proj_dim) for _ in range(num_layers)]
+            )
+            if out_proj_dim_zero_init:
+                for out_projector in self.out_projectors:
+                    self.zeros_init_linear(out_projector)   
+            
+        self.gradient_checkpointing = False
+    
+    def zeros_init_linear(self, linear: nn.Module):
+        if isinstance(linear, (nn.Linear, nn.Conv1d)):
+            if hasattr(linear, "weight"):
+                nn.init.zeros_(linear.weight)
+            if hasattr(linear, "bias"):
+                nn.init.zeros_(linear.bias)
+        
+    def _set_gradient_checkpointing(self, module, value=False):
+        self.gradient_checkpointing = value
+
+    def compress_time(self, x, num_frames):
+        x = rearrange(x, '(b f) c h w -> b f c h w', f=num_frames)
+        batch_size, frames, channels, height, width = x.shape
+        x = rearrange(x, 'b f c h w -> (b h w) c f')
+        
+        if x.shape[-1] % 2 == 1:
+            x_first, x_rest = x[..., 0], x[..., 1:]
+            if x_rest.shape[-1] > 0:
+                x_rest = F.avg_pool1d(x_rest, kernel_size=2, stride=2)
+
+            x = torch.cat([x_first[..., None], x_rest], dim=-1)
+        else:
+            x = F.avg_pool1d(x, kernel_size=2, stride=2)
+        x = rearrange(x, '(b h w) c f -> (b f) c h w', b=batch_size, h=height, w=width)
+        return x
+        
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        encoder_hidden_states: torch.Tensor,
+        controlnet_states: Tuple[torch.Tensor, torch.Tensor],
+        timestep: Union[int, float, torch.LongTensor],
+        controlnet_output_mask: Optional[torch.Tensor] = None,
+        image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
+        timestep_cond: Optional[torch.Tensor] = None,
+        return_dict: bool = True,
+    ):
+        hidden_states = torch.cat([hidden_states, controlnet_states], dim=2)
+        
+        # controlnet_states = self.controlnext_encoder(controlnet_states, timestep=timestep)
+        # 1. Time embedding
+        timesteps = timestep
+        t_emb = self.time_proj(timesteps)
+
+        # timesteps does not contain any weights and will always return f32 tensors
+        # but time_embedding might actually be running in fp16. so we need to cast here.
+        # there might be better ways to encapsulate this.
+        t_emb = t_emb.to(dtype=hidden_states.dtype)
+        emb = self.time_embedding(t_emb, timestep_cond)
+        
+        hidden_states = self.patch_embed(encoder_hidden_states, hidden_states)
+        hidden_states = self.embedding_dropout(hidden_states)
+
+
+        text_seq_length = encoder_hidden_states.shape[1]
+        encoder_hidden_states = hidden_states[:, :text_seq_length]
+        hidden_states = hidden_states[:, text_seq_length:]
+
+        
+        controlnet_hidden_states = ()
+        # 3. Transformer blocks
+        for i, block in enumerate(self.transformer_blocks):
+            if self.training and self.gradient_checkpointing:
+
+                def create_custom_forward(module):
+                    def custom_forward(*inputs):
+                        return module(*inputs)
+
+                    return custom_forward
+
+                ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {}
+                hidden_states, encoder_hidden_states = torch.utils.checkpoint.checkpoint(
+                    create_custom_forward(block),
+                    hidden_states,
+                    encoder_hidden_states,
+                    emb,
+                    image_rotary_emb,
+                    **ckpt_kwargs,
+                )
+            else:
+                hidden_states, encoder_hidden_states = block(
+                    hidden_states=hidden_states,
+                    encoder_hidden_states=encoder_hidden_states,
+                    temb=emb,
+                    image_rotary_emb=image_rotary_emb,
+                )
+                
+            if self.out_projectors is not None:
+                if controlnet_output_mask is not None:
+                    controlnet_hidden_states += (self.out_projectors[i](hidden_states) * controlnet_output_mask,)
+                else:
+                    controlnet_hidden_states += (self.out_projectors[i](hidden_states),)
+            else:
+                controlnet_hidden_states += (hidden_states,)
+
+        if not return_dict:
+            return (controlnet_hidden_states,)
+        return Transformer2DModelOutput(sample=controlnet_hidden_states)

cogvideo_transformer.py

@@ -0,0 +1,127 @@
+from typing import Any, Dict, Optional, Tuple, Union
+
+import torch
+import numpy as np
+from diffusers.utils import is_torch_version
+from diffusers.models.transformers.cogvideox_transformer_3d import CogVideoXTransformer3DModel, Transformer2DModelOutput
+
+
+class CustomCogVideoXTransformer3DModel(CogVideoXTransformer3DModel):      
+    def set_learnable_parameters(self, unfrozen_layers: int = 16):
+        for param in self.patch_embed.parameters():
+            param.requires_grad = True
+                
+        for i in range(unfrozen_layers):
+            block = self.transformer_blocks[i]
+            attn = block.attn1
+            for module in  [block.norm2, block.ff]:
+                for param in self.patch_embed.parameters():
+                    param.requires_grad = True
+                    
+            for name in ['to_q', 'to_k', 'to_v', 'norm_q', 'norm_k']:
+                module = getattr(attn, name, None)
+                if module is not None:
+                    for param in module.parameters():
+                        param.requires_grad = True
+                else:
+                    print(f"[Warning] {name} not found in attn1 of block {i}")
+        
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        encoder_hidden_states: torch.Tensor,
+        timestep: Union[int, float, torch.LongTensor],
+        start_frame = None,
+        timestep_cond: Optional[torch.Tensor] = None,
+        image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
+        controlnet_states: torch.Tensor = None,
+        controlnet_weights: Optional[Union[float, int, list, np.ndarray, torch.FloatTensor]] = 1.0,
+        return_dict: bool = True,
+    ):
+        batch_size, num_frames, channels, height, width = hidden_states.shape
+
+        if start_frame is not None:
+            hidden_states = torch.cat([start_frame, hidden_states], dim=2)
+        # 1. Time embedding
+        timesteps = timestep
+        t_emb = self.time_proj(timesteps)
+
+        # timesteps does not contain any weights and will always return f32 tensors
+        # but time_embedding might actually be running in fp16. so we need to cast here.
+        # there might be better ways to encapsulate this.
+        t_emb = t_emb.to(dtype=hidden_states.dtype)
+        emb = self.time_embedding(t_emb, timestep_cond)
+
+        # 2. Patch embedding
+        hidden_states = self.patch_embed(encoder_hidden_states, hidden_states)
+        hidden_states = self.embedding_dropout(hidden_states)
+
+        text_seq_length = encoder_hidden_states.shape[1]
+        encoder_hidden_states = hidden_states[:, :text_seq_length]
+        hidden_states = hidden_states[:, text_seq_length:]
+
+        # 3. Transformer blocks
+        for i, block in enumerate(self.transformer_blocks):
+            if self.gradient_checkpointing:
+                def create_custom_forward(module):
+                    def custom_forward(*inputs):
+                        return module(*inputs)
+
+                    return custom_forward
+
+                ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {}
+                hidden_states, encoder_hidden_states = torch.utils.checkpoint.checkpoint(
+                    create_custom_forward(block),
+                    hidden_states,
+                    encoder_hidden_states,
+                    emb,
+                    image_rotary_emb,
+                    **ckpt_kwargs,
+                )
+            else:
+                hidden_states, encoder_hidden_states = block(
+                    hidden_states=hidden_states,
+                    encoder_hidden_states=encoder_hidden_states,
+                    temb=emb,
+                    image_rotary_emb=image_rotary_emb,
+                )
+
+            if (controlnet_states is not None) and (i < len(controlnet_states)):
+                controlnet_states_block = controlnet_states[i]
+                controlnet_block_weight = 1.0
+                if isinstance(controlnet_weights, (list, np.ndarray)) or torch.is_tensor(controlnet_weights):
+                    controlnet_block_weight = controlnet_weights[i]
+                elif isinstance(controlnet_weights, (float, int)):
+                    controlnet_block_weight = controlnet_weights
+                
+                hidden_states = hidden_states + controlnet_states_block * controlnet_block_weight
+
+        if not self.config.use_rotary_positional_embeddings:
+            # CogVideoX-2B
+            hidden_states = self.norm_final(hidden_states)
+        else:
+            # CogVideoX-5B
+            hidden_states = torch.cat([encoder_hidden_states, hidden_states], dim=1)
+            hidden_states = self.norm_final(hidden_states)
+            hidden_states = hidden_states[:, text_seq_length:]
+
+        # 4. Final block
+        hidden_states = self.norm_out(hidden_states, temb=emb)
+        hidden_states = self.proj_out(hidden_states)
+
+        # 5. Unpatchify
+        p = self.config.patch_size
+        p_t = self.config.patch_size_t
+
+        if p_t is None:
+            output = hidden_states.reshape(batch_size, num_frames, height // p, width // p, -1, p, p)
+            output = output.permute(0, 1, 4, 2, 5, 3, 6).flatten(5, 6).flatten(3, 4)
+        else:
+            output = hidden_states.reshape(
+                batch_size, (num_frames + p_t - 1) // p_t, height // p, width // p, -1, p_t, p, p
+            )
+            output = output.permute(0, 1, 5, 4, 2, 6, 3, 7).flatten(6, 7).flatten(4, 5).flatten(1, 2)
+
+        if not return_dict:
+            return (output,)
+        return Transformer2DModelOutput(sample=output)

controlnet_pipeline.py

@@ -0,0 +1,807 @@
+import inspect
+import math
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
+
+import torch
+import numpy as np
+import PIL
+from PIL import Image
+from torchvision import transforms
+from einops import rearrange, repeat
+from transformers import T5EncoderModel, T5Tokenizer
+from diffusers.video_processor import VideoProcessor
+from diffusers.utils.torch_utils import randn_tensor
+from diffusers.models.embeddings import get_3d_rotary_pos_embed
+from diffusers.pipelines.pipeline_utils import DiffusionPipeline
+from diffusers.models import AutoencoderKLCogVideoX, CogVideoXTransformer3DModel
+from diffusers import CogVideoXDDIMScheduler, CogVideoXDPMScheduler
+from diffusers.callbacks import MultiPipelineCallbacks, PipelineCallback
+from diffusers.pipelines.cogvideo.pipeline_cogvideox import CogVideoXPipelineOutput, CogVideoXLoraLoaderMixin
+from diffusers.image_processor import PipelineImageInput
+
+from cogvideo_controlnet_pcd import CogVideoXControlnetPCD as CogVideoXControlnet
+
+
+def resize_for_crop(image, crop_h, crop_w):
+    img_h, img_w = image.shape[-2:]
+    if img_h >= crop_h and img_w >= crop_w:
+        coef = max(crop_h / img_h, crop_w / img_w)
+    elif img_h <= crop_h and img_w <= crop_w:
+        coef = max(crop_h / img_h, crop_w / img_w)
+    else:
+        coef = crop_h / img_h if crop_h > img_h else crop_w / img_w 
+    out_h, out_w = int(img_h * coef), int(img_w * coef)
+    resized_image = transforms.functional.resize(image, (out_h, out_w), antialias=True)
+    return resized_image
+
+
+def prepare_frames(input_images, video_size, do_resize=True, do_crop=True):
+    input_images = np.stack([np.array(x) for x in input_images])
+    images_tensor = torch.from_numpy(input_images).permute(0, 3, 1, 2) / 127.5 - 1
+    if do_resize:
+        images_tensor = [resize_for_crop(x, crop_h=video_size[0], crop_w=video_size[1]) for x in images_tensor]
+    if do_crop:
+        images_tensor = [transforms.functional.center_crop(x, video_size) for x in images_tensor]
+    if isinstance(images_tensor, list):
+        images_tensor = torch.stack(images_tensor)
+    return images_tensor.unsqueeze(0) 
+
+
+def get_resize_crop_region_for_grid(src, tgt_width, tgt_height):
+    tw = tgt_width
+    th = tgt_height
+    h, w = src
+    r = h / w
+    if r > (th / tw):
+        resize_height = th
+        resize_width = int(round(th / h * w))
+    else:
+        resize_width = tw
+        resize_height = int(round(tw / w * h))
+
+    crop_top = int(round((th - resize_height) / 2.0))
+    crop_left = int(round((tw - resize_width) / 2.0))
+
+    return (crop_top, crop_left), (crop_top + resize_height, crop_left + resize_width)
+
+
+# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps
+def retrieve_timesteps(
+    scheduler,
+    num_inference_steps: Optional[int] = None,
+    device: Optional[Union[str, torch.device]] = None,
+    timesteps: Optional[List[int]] = None,
+    sigmas: Optional[List[float]] = None,
+    **kwargs,
+):
+    """
+    Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles
+    custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`.
+
+    Args:
+        scheduler (`SchedulerMixin`):
+            The scheduler to get timesteps from.
+        num_inference_steps (`int`):
+            The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps`
+            must be `None`.
+        device (`str` or `torch.device`, *optional*):
+            The device to which the timesteps should be moved to. If `None`, the timesteps are not moved.
+        timesteps (`List[int]`, *optional*):
+            Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed,
+            `num_inference_steps` and `sigmas` must be `None`.
+        sigmas (`List[float]`, *optional*):
+            Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed,
+            `num_inference_steps` and `timesteps` must be `None`.
+
+    Returns:
+        `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the
+        second element is the number of inference steps.
+    """
+    if timesteps is not None and sigmas is not None:
+        raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values")
+    if timesteps is not None:
+        accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())
+        if not accepts_timesteps:
+            raise ValueError(
+                f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
+                f" timestep schedules. Please check whether you are using the correct scheduler."
+            )
+        scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs)
+        timesteps = scheduler.timesteps
+        num_inference_steps = len(timesteps)
+    elif sigmas is not None:
+        accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())
+        if not accept_sigmas:
+            raise ValueError(
+                f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
+                f" sigmas schedules. Please check whether you are using the correct scheduler."
+            )
+        scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs)
+        timesteps = scheduler.timesteps
+        num_inference_steps = len(timesteps)
+    else:
+        scheduler.set_timesteps(num_inference_steps, device=device, **kwargs)
+        timesteps = scheduler.timesteps
+    return timesteps, num_inference_steps
+    
+# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.retrieve_latents
+def retrieve_latents(
+    encoder_output: torch.Tensor, generator: Optional[torch.Generator] = None, sample_mode: str = "sample"
+):
+    if hasattr(encoder_output, "latent_dist") and sample_mode == "sample":
+        return encoder_output.latent_dist.sample(generator)
+    elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax":
+        return encoder_output.latent_dist.mode()
+    elif hasattr(encoder_output, "latents"):
+        return encoder_output.latents
+    else:
+        raise AttributeError("Could not access latents of provided encoder_output")
+
+class ControlnetCogVideoXImageToVideoPCDPipeline(DiffusionPipeline, CogVideoXLoraLoaderMixin):
+    _optional_components = []
+    model_cpu_offload_seq = "text_encoder->transformer->vae"
+
+    _callback_tensor_inputs = [
+        "latents",
+        "prompt_embeds",
+        "negative_prompt_embeds",
+    ]
+
+    def __init__(
+        self,
+        tokenizer: T5Tokenizer,
+        text_encoder: T5EncoderModel,
+        vae: AutoencoderKLCogVideoX,
+        transformer: CogVideoXTransformer3DModel,
+        scheduler: Union[CogVideoXDDIMScheduler, CogVideoXDPMScheduler],
+        controlnet: CogVideoXControlnet,
+    ):
+        super().__init__()
+
+        self.register_modules(
+            tokenizer=tokenizer, text_encoder=text_encoder, vae=vae, transformer=transformer, controlnet=controlnet, scheduler=scheduler
+        )
+        self.vae_scale_factor_spatial = (
+            2 ** (len(self.vae.config.block_out_channels) - 1) if hasattr(self, "vae") and self.vae is not None else 8
+        )
+        self.vae_scale_factor_temporal = (
+            self.vae.config.temporal_compression_ratio if hasattr(self, "vae") and self.vae is not None else 4
+        )
+        self.vae_scaling_factor_image = (
+            self.vae.config.scaling_factor if hasattr(self, "vae") and self.vae is not None else 0.7
+        )
+
+        self.video_processor = VideoProcessor(vae_scale_factor=self.vae_scale_factor_spatial)
+
+    # Copied from diffusers.pipelines.cogvideo.pipeline_cogvideox.CogVideoXPipeline._get_t5_prompt_embeds
+    def _get_t5_prompt_embeds(
+        self,
+        prompt: Union[str, List[str]] = None,
+        num_videos_per_prompt: int = 1,
+        max_sequence_length: int = 226,
+        device: Optional[torch.device] = None,
+        dtype: Optional[torch.dtype] = None,
+    ):
+        device = device or self._execution_device
+        dtype = dtype or self.text_encoder.dtype
+
+        prompt = [prompt] if isinstance(prompt, str) else prompt
+        batch_size = len(prompt)
+
+        text_inputs = self.tokenizer(
+            prompt,
+            padding="max_length",
+            max_length=max_sequence_length,
+            truncation=True,
+            add_special_tokens=True,
+            return_tensors="pt",
+        )
+        text_input_ids = text_inputs.input_ids
+        untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids
+
+        if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids):
+            removed_text = self.tokenizer.batch_decode(untruncated_ids[:, max_sequence_length - 1 : -1])
+            logger.warning(
+                "The following part of your input was truncated because `max_sequence_length` is set to "
+                f" {max_sequence_length} tokens: {removed_text}"
+            )
+
+        prompt_embeds = self.text_encoder(text_input_ids.to(device))[0]
+        prompt_embeds = prompt_embeds.to(dtype=dtype, device=device)
+
+        # duplicate text embeddings for each generation per prompt, using mps friendly method
+        _, seq_len, _ = prompt_embeds.shape
+        prompt_embeds = prompt_embeds.repeat(1, num_videos_per_prompt, 1)
+        prompt_embeds = prompt_embeds.view(batch_size * num_videos_per_prompt, seq_len, -1)
+
+        return prompt_embeds
+
+    # Copied from diffusers.pipelines.cogvideo.pipeline_cogvideox.CogVideoXPipeline.encode_prompt
+    def encode_prompt(
+        self,
+        prompt: Union[str, List[str]],
+        negative_prompt: Optional[Union[str, List[str]]] = None,
+        do_classifier_free_guidance: bool = True,
+        num_videos_per_prompt: int = 1,
+        prompt_embeds: Optional[torch.Tensor] = None,
+        negative_prompt_embeds: Optional[torch.Tensor] = None,
+        max_sequence_length: int = 226,
+        device: Optional[torch.device] = None,
+        dtype: Optional[torch.dtype] = None,
+    ):
+        r"""
+        Encodes the prompt into text encoder hidden states.
+
+        Args:
+            prompt (`str` or `List[str]`, *optional*):
+                prompt to be encoded
+            negative_prompt (`str` or `List[str]`, *optional*):
+                The prompt or prompts not to guide the image generation. If not defined, one has to pass
+                `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
+                less than `1`).
+            do_classifier_free_guidance (`bool`, *optional*, defaults to `True`):
+                Whether to use classifier free guidance or not.
+            num_videos_per_prompt (`int`, *optional*, defaults to 1):
+                Number of videos that should be generated per prompt. torch device to place the resulting embeddings on
+            prompt_embeds (`torch.Tensor`, *optional*):
+                Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
+                provided, text embeddings will be generated from `prompt` input argument.
+            negative_prompt_embeds (`torch.Tensor`, *optional*):
+                Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
+                weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
+                argument.
+            device: (`torch.device`, *optional*):
+                torch device
+            dtype: (`torch.dtype`, *optional*):
+                torch dtype
+        """
+        device = device or self._execution_device
+
+        prompt = [prompt] if isinstance(prompt, str) else prompt
+        if prompt is not None:
+            batch_size = len(prompt)
+        else:
+            batch_size = prompt_embeds.shape[0]
+
+        if prompt_embeds is None:
+            prompt_embeds = self._get_t5_prompt_embeds(
+                prompt=prompt,
+                num_videos_per_prompt=num_videos_per_prompt,
+                max_sequence_length=max_sequence_length,
+                device=device,
+                dtype=dtype,
+            )
+
+        if do_classifier_free_guidance and negative_prompt_embeds is None:
+            negative_prompt = negative_prompt or ""
+            negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt
+
+            if prompt is not None and type(prompt) is not type(negative_prompt):
+                raise TypeError(
+                    f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !="
+                    f" {type(prompt)}."
+                )
+            elif batch_size != len(negative_prompt):
+                raise ValueError(
+                    f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"
+                    f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"
+                    " the batch size of `prompt`."
+                )
+
+            negative_prompt_embeds = self._get_t5_prompt_embeds(
+                prompt=negative_prompt,
+                num_videos_per_prompt=num_videos_per_prompt,
+                max_sequence_length=max_sequence_length,
+                device=device,
+                dtype=dtype,
+            )
+
+        return prompt_embeds, negative_prompt_embeds
+
+    def prepare_latents(
+        self,
+        image: torch.Tensor,
+        batch_size: int = 1,
+        num_channels_latents: int = 16,
+        num_frames: int = 13,
+        height: int = 60,
+        width: int = 90,
+        dtype: Optional[torch.dtype] = None,
+        device: Optional[torch.device] = None,
+        generator: Optional[torch.Generator] = None,
+        latents: Optional[torch.Tensor] = None,
+    ):
+        if isinstance(generator, list) and len(generator) != batch_size:
+            raise ValueError(
+                f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"
+                f" size of {batch_size}. Make sure the batch size matches the length of the generators."
+            )
+
+        num_frames = (num_frames - 1) // self.vae_scale_factor_temporal + 1
+        shape = (
+            batch_size,
+            num_frames,
+            num_channels_latents,
+            height // self.vae_scale_factor_spatial,
+            width // self.vae_scale_factor_spatial,
+        )
+
+        # For CogVideoX1.5, the latent should add 1 for padding (Not use)
+        if self.transformer.config.patch_size_t is not None:
+            shape = shape[:1] + (shape[1] + shape[1] % self.transformer.config.patch_size_t,) + shape[2:]
+
+        image = image.unsqueeze(2)  # [B, C, F, H, W]
+
+        if isinstance(generator, list):
+            image_latents = [
+                retrieve_latents(self.vae.encode(image[i].unsqueeze(0)), generator[i]) for i in range(batch_size)
+            ]
+        else:
+            image_latents = [retrieve_latents(self.vae.encode(img.unsqueeze(0)), generator) for img in image]
+
+        image_latents = torch.cat(image_latents, dim=0).to(dtype).permute(0, 2, 1, 3, 4)  # [B, F, C, H, W]
+
+        if not self.vae.config.invert_scale_latents:
+            image_latents = self.vae_scaling_factor_image * image_latents
+        else:
+            # This is awkward but required because the CogVideoX team forgot to multiply the
+            # scaling factor during training :)
+            image_latents = 1 / self.vae_scaling_factor_image * image_latents
+
+        padding_shape = (
+            batch_size,
+            num_frames - 1,
+            num_channels_latents,
+            height // self.vae_scale_factor_spatial,
+            width // self.vae_scale_factor_spatial,
+        )
+
+        latent_padding = torch.zeros(padding_shape, device=device, dtype=dtype)
+        image_latents = torch.cat([image_latents, latent_padding], dim=1)
+
+        # Select the first frame along the second dimension
+        if self.transformer.config.patch_size_t is not None:
+            first_frame = image_latents[:, : image_latents.size(1) % self.transformer.config.patch_size_t, ...]
+            image_latents = torch.cat([first_frame, image_latents], dim=1)
+
+        if latents is None:
+            latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
+        else:
+            latents = latents.to(device)
+
+        # scale the initial noise by the standard deviation required by the scheduler
+        latents = latents * self.scheduler.init_noise_sigma
+        return latents, image_latents
+    
+    # Copied from diffusers.pipelines.cogvideo.pipeline_cogvideox.CogVideoXPipeline.decode_latents
+    def decode_latents(self, latents: torch.Tensor) -> torch.Tensor:
+        latents = latents.permute(0, 2, 1, 3, 4)  # [batch_size, num_channels, num_frames, height, width]
+        latents = 1 / self.vae_scaling_factor_image * latents
+
+        frames = self.vae.decode(latents).sample
+        return frames
+
+    # Copied from diffusers.pipelines.animatediff.pipeline_animatediff_video2video.AnimateDiffVideoToVideoPipeline.get_timesteps
+    def get_timesteps(self, num_inference_steps, timesteps, strength, device):
+        # get the original timestep using init_timestep
+        init_timestep = min(int(num_inference_steps * strength), num_inference_steps)
+
+        t_start = max(num_inference_steps - init_timestep, 0)
+        timesteps = timesteps[t_start * self.scheduler.order :]
+
+        return timesteps, num_inference_steps - t_start
+
+    # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs
+    def prepare_extra_step_kwargs(self, generator, eta):
+        # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature
+        # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.
+        # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502
+        # and should be between [0, 1]
+
+        accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys())
+        extra_step_kwargs = {}
+        if accepts_eta:
+            extra_step_kwargs["eta"] = eta
+
+        # check if the scheduler accepts generator
+        accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys())
+        if accepts_generator:
+            extra_step_kwargs["generator"] = generator
+        return extra_step_kwargs
+
+    def check_inputs(
+        self,
+        image,
+        prompt,
+        height,
+        width,
+        negative_prompt,
+        callback_on_step_end_tensor_inputs,
+        prompt_embeds=None,
+        negative_prompt_embeds=None,
+    ):
+        if (
+            not isinstance(image, torch.Tensor)
+            and not isinstance(image, PIL.Image.Image)
+            and not isinstance(image, list)
+        ):
+            raise ValueError(
+                "`image` has to be of type `torch.Tensor` or `PIL.Image.Image` or `List[PIL.Image.Image]` but is"
+                f" {type(image)}"
+            )
+
+        if height % 8 != 0 or width % 8 != 0:
+            raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.")
+
+        if callback_on_step_end_tensor_inputs is not None and not all(
+            k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs
+        ):
+            raise ValueError(
+                f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}"
+            )
+        if prompt is not None and prompt_embeds is not None:
+            raise ValueError(
+                f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"
+                " only forward one of the two."
+            )
+        elif prompt is None and prompt_embeds is None:
+            raise ValueError(
+                "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."
+            )
+        elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):
+            raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")
+
+        if prompt is not None and negative_prompt_embeds is not None:
+            raise ValueError(
+                f"Cannot forward both `prompt`: {prompt} and `negative_prompt_embeds`:"
+                f" {negative_prompt_embeds}. Please make sure to only forward one of the two."
+            )
+
+        if negative_prompt is not None and negative_prompt_embeds is not None:
+            raise ValueError(
+                f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:"
+                f" {negative_prompt_embeds}. Please make sure to only forward one of the two."
+            )
+
+        if prompt_embeds is not None and negative_prompt_embeds is not None:
+            if prompt_embeds.shape != negative_prompt_embeds.shape:
+                raise ValueError(
+                    "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but"
+                    f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`"
+                    f" {negative_prompt_embeds.shape}."
+                )
+                
+    # Copied from diffusers.pipelines.cogvideo.pipeline_cogvideox.CogVideoXPipeline.fuse_qkv_projections
+    def fuse_qkv_projections(self) -> None:
+        r"""Enables fused QKV projections."""
+        self.fusing_transformer = True
+        self.transformer.fuse_qkv_projections()
+
+    # Copied from diffusers.pipelines.cogvideo.pipeline_cogvideox.CogVideoXPipeline.unfuse_qkv_projections
+    def unfuse_qkv_projections(self) -> None:
+        r"""Disable QKV projection fusion if enabled."""
+        if not self.fusing_transformer:
+            logger.warning("The Transformer was not initially fused for QKV projections. Doing nothing.")
+        else:
+            self.transformer.unfuse_qkv_projections()
+            self.fusing_transformer = False
+
+    # Copied from diffusers.pipelines.cogvideo.pipeline_cogvideox.CogVideoXPipeline._prepare_rotary_positional_embeddings
+    def _prepare_rotary_positional_embeddings(
+        self,
+        height: int,
+        width: int,
+        num_frames: int,
+        device: torch.device,
+    ) -> Tuple[torch.Tensor, torch.Tensor]:
+        grid_height = height // (self.vae_scale_factor_spatial * self.transformer.config.patch_size)
+        grid_width = width // (self.vae_scale_factor_spatial * self.transformer.config.patch_size)
+
+        p = self.transformer.config.patch_size
+        p_t = self.transformer.config.patch_size_t
+
+        base_size_width = self.transformer.config.sample_width // p
+        base_size_height = self.transformer.config.sample_height // p
+
+        if p_t is None:
+            # CogVideoX 1.0
+            grid_crops_coords = get_resize_crop_region_for_grid(
+                (grid_height, grid_width), base_size_width, base_size_height
+            )
+            freqs_cos, freqs_sin = get_3d_rotary_pos_embed(
+                embed_dim=self.transformer.config.attention_head_dim,
+                crops_coords=grid_crops_coords,
+                grid_size=(grid_height, grid_width),
+                temporal_size=num_frames,
+                device=device,
+            )
+        else:
+            # CogVideoX 1.5
+            base_num_frames = (num_frames + p_t - 1) // p_t
+
+            freqs_cos, freqs_sin = get_3d_rotary_pos_embed(
+                embed_dim=self.transformer.config.attention_head_dim,
+                crops_coords=None,
+                grid_size=(grid_height, grid_width),
+                temporal_size=base_num_frames,
+                grid_type="slice",
+                max_size=(base_size_height, base_size_width),
+                device=device,
+            )
+
+        return freqs_cos, freqs_sin
+
+    def encode_video(self, video):
+        video = video.to(self.device, dtype=self.vae.dtype)
+        video = video.permute(0, 2, 1, 3, 4)  # [B, C, F, H, W]
+        latent_dist = self.vae.encode(video).latent_dist.sample() * self.vae.config.scaling_factor
+        return latent_dist.permute(0, 2, 1, 3, 4).to(memory_format=torch.contiguous_format)
+    
+    @property
+    def guidance_scale(self):
+        return self._guidance_scale
+
+    @property
+    def num_timesteps(self):
+        return self._num_timesteps
+
+    @property
+    def attention_kwargs(self):
+        return self._attention_kwargs
+
+    @property
+    def interrupt(self):
+        return self._interrupt
+
+    @torch.no_grad()
+    def __call__(
+        self,
+        image: PipelineImageInput,
+        anchor_video: None,
+        controlnet_output_mask: None,
+        prompt: Optional[Union[str, List[str]]] = None,
+        negative_prompt: Optional[Union[str, List[str]]] = None,
+        height: Optional[int] = None,
+        width: Optional[int] = None,
+        num_frames: int = 49,
+        num_inference_steps: int = 50,
+        timesteps: Optional[List[int]] = None,
+        guidance_scale: float = 6,
+        use_dynamic_cfg: bool = False,
+        num_videos_per_prompt: int = 1,
+        eta: float = 0.0,
+        generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,
+        latents: Optional[torch.FloatTensor] = None,
+        prompt_embeds: Optional[torch.FloatTensor] = None,
+        negative_prompt_embeds: Optional[torch.FloatTensor] = None,
+        output_type: str = "pil",
+        return_dict: bool = True,
+        attention_kwargs: Optional[Dict[str, Any]] = None,
+        callback_on_step_end: Optional[
+            Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks]
+        ] = None,
+        callback_on_step_end_tensor_inputs: List[str] = ["latents"],
+        max_sequence_length: int = 226,
+        controlnet_weights: Optional[Union[float, list, np.ndarray, torch.FloatTensor]] = 1.0,
+        controlnet_guidance_start: float = 0.0,
+        controlnet_guidance_end: float = 1.0,
+    ) -> Union[CogVideoXPipelineOutput, Tuple]:
+        if isinstance(callback_on_step_end, (PipelineCallback, MultiPipelineCallbacks)):
+            callback_on_step_end_tensor_inputs = callback_on_step_end.tensor_inputs
+
+        height = height or self.transformer.config.sample_height * self.vae_scale_factor_spatial
+        width = width or self.transformer.config.sample_width * self.vae_scale_factor_spatial
+        num_frames = num_frames or self.transformer.config.sample_frames
+
+        num_videos_per_prompt = 1
+
+        # 1. Check inputs. Raise error if not correct
+        self.check_inputs(
+            image=image,
+            prompt=prompt,
+            height=height,
+            width=width,
+            negative_prompt=negative_prompt,
+            callback_on_step_end_tensor_inputs=callback_on_step_end_tensor_inputs,
+            prompt_embeds=prompt_embeds,
+            negative_prompt_embeds=negative_prompt_embeds,
+        )
+        self._guidance_scale = guidance_scale
+        self._attention_kwargs = attention_kwargs
+        self._interrupt = False
+
+        # 2. Default call parameters
+        if prompt is not None and isinstance(prompt, str):
+            batch_size = 1
+        elif prompt is not None and isinstance(prompt, list):
+            batch_size = len(prompt)
+        else:
+            batch_size = prompt_embeds.shape[0]
+
+        device = self._execution_device
+
+        # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)
+        # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`
+        # corresponds to doing no classifier free guidance.
+        do_classifier_free_guidance = guidance_scale > 1.0
+
+        # 3. Encode input prompt
+        prompt_embeds, negative_prompt_embeds = self.encode_prompt(
+            prompt=prompt,
+            negative_prompt=negative_prompt,
+            do_classifier_free_guidance=do_classifier_free_guidance,
+            num_videos_per_prompt=num_videos_per_prompt,
+            prompt_embeds=prompt_embeds,
+            negative_prompt_embeds=negative_prompt_embeds,
+            max_sequence_length=max_sequence_length,
+            device=device,
+        )
+        if do_classifier_free_guidance:
+            prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0)
+
+        # 4. Prepare timesteps
+        timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps)
+        self._num_timesteps = len(timesteps)
+
+        # 5. Prepare latents
+        latent_frames = (num_frames - 1) // self.vae_scale_factor_temporal + 1
+
+        # For CogVideoX 1.5, the latent frames should be padded to make it divisible by patch_size_t
+        patch_size_t = self.transformer.config.patch_size_t
+        additional_frames = 0
+        if patch_size_t is not None and latent_frames % patch_size_t != 0:
+            additional_frames = patch_size_t - latent_frames % patch_size_t
+            num_frames += additional_frames * self.vae_scale_factor_temporal
+
+        image = self.video_processor.preprocess(image, height=height, width=width).to(
+            device, dtype=prompt_embeds.dtype
+        )
+
+        latent_channels = self.transformer.config.in_channels // 2
+        latents, image_latents = self.prepare_latents(
+            image,
+            batch_size * num_videos_per_prompt,
+            latent_channels,
+            num_frames,
+            height,
+            width,
+            prompt_embeds.dtype,
+            device,
+            generator,
+            latents,
+        )
+        
+        # 6 encoding anchor videos
+        
+        anchor_states = self.encode_video(anchor_video[None]).to(device)
+  
+        # 7. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
+        extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
+
+        # 8. Create rotary embeds if required
+        image_rotary_emb = (
+            self._prepare_rotary_positional_embeddings(height, width, latents.size(1), device)
+            if self.transformer.config.use_rotary_positional_embeddings
+            else None
+        )
+
+        # 8. Create ofs embeds if required
+        ofs_emb = None if self.transformer.config.ofs_embed_dim is None else latents.new_full((1,), fill_value=2.0)
+
+        # 9. Denoising loop
+        num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0)
+            
+        if do_classifier_free_guidance:
+            anchor_states = torch.cat([anchor_states] * 2)
+        
+        with self.progress_bar(total=num_inference_steps) as progress_bar:
+            # for DPM-solver++
+            old_pred_original_sample = None
+            for i, t in enumerate(timesteps):
+                if self.interrupt:
+                    continue
+                
+                latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents
+                latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)
+
+                latent_image_input = torch.cat([image_latents] * 2) if do_classifier_free_guidance else image_latents
+                
+                latent_and_image_input = torch.cat([latent_model_input, latent_image_input], dim=2)
+                
+                
+                # latent_and_image_input = torch.cat([latent_model_input], dim=2)
+                                    
+                # broadcast to batch dimension in a way that's compatible with ONNX/Core ML
+                timestep = t.expand(latent_model_input.shape[0])
+
+                current_sampling_percent = i / len(timesteps)
+
+                latent_model_input = latent_model_input.to(dtype=self.transformer.dtype)
+                prompt_embeds = prompt_embeds.to(dtype=self.transformer.dtype)
+                
+                controlnet_states = None
+                
+                input_controlnet_states = anchor_states
+                if (controlnet_guidance_start <= current_sampling_percent <= controlnet_guidance_end):
+                    controlnet_states = self.controlnet(
+                        hidden_states=latent_model_input,
+                        encoder_hidden_states=prompt_embeds,
+                        image_rotary_emb=image_rotary_emb,
+                        controlnet_states=input_controlnet_states,
+                        controlnet_output_mask = controlnet_output_mask,
+                        timestep=timestep,
+                        return_dict=False,
+                    )[0]
+                    if isinstance(controlnet_states, (tuple, list)):
+                        controlnet_states = [x.to(dtype=self.transformer.dtype) for x in controlnet_states]
+                    else:
+                        controlnet_states = controlnet_states.to(dtype=self.transformer.dtype)
+
+                # predict noise model_output
+                noise_pred = self.transformer(
+                    hidden_states=latent_and_image_input,
+                    encoder_hidden_states=prompt_embeds,
+                    timestep=timestep,
+                    image_rotary_emb=image_rotary_emb,
+                    controlnet_states=controlnet_states,
+                    controlnet_weights=controlnet_weights,
+                    return_dict=False,
+                )[0]
+                
+                noise_pred = noise_pred.float()
+
+                # perform guidance
+                if use_dynamic_cfg:
+                    self._guidance_scale = 1 + guidance_scale * (
+                        (1 - math.cos(math.pi * ((num_inference_steps - t.item()) / num_inference_steps) ** 5.0)) / 2
+                    )
+                if do_classifier_free_guidance:
+                    noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
+                    noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond)
+
+                # compute the previous noisy sample x_t -> x_t-1
+                if not isinstance(self.scheduler, CogVideoXDPMScheduler):
+                    latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0]
+                else:
+                    latents, old_pred_original_sample = self.scheduler.step(
+                        noise_pred,
+                        old_pred_original_sample,
+                        t,
+                        timesteps[i - 1] if i > 0 else None,
+                        latents,
+                        **extra_step_kwargs,
+                        return_dict=False,
+                    )
+                latents = latents.to(prompt_embeds.dtype)
+
+                # call the callback, if provided
+                if callback_on_step_end is not None:
+                    callback_kwargs = {}
+                    for k in callback_on_step_end_tensor_inputs:
+                        callback_kwargs[k] = locals()[k]
+                    callback_outputs = callback_on_step_end(self, i, t, callback_kwargs)
+
+                    latents = callback_outputs.pop("latents", latents)
+                    prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds)
+                    negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds)
+
+                if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
+                    progress_bar.update()
+
+        if not output_type == "latent":
+            # Discard any padding frames that were added for CogVideoX 1.5
+            latents = latents[:, additional_frames:]
+            video = self.decode_latents(latents)
+            video = self.video_processor.postprocess_video(video=video, output_type=output_type)
+        else:
+            video = latents
+
+        # Offload all models
+        self.maybe_free_model_hooks()
+
+        if not return_dict:
+            return (video,)
+
+        return CogVideoXPipelineOutput(frames=video)
+
+  

data/test_i2v/captions/000000000005.2.013.txt

@@ -0,0 +1 @@
+Inside a dimly lit subway car, a first-person perspective shows a gloved hand holding a handgun aimed at the camera, with the barrel pointing directly at the viewer. the scene is tense, with the hand's movement suggesting an imminent threat. the background is blurred, indicating motion, and the interior is adorned with teal and blue seats, silver poles, and red and white advertisements. two seconds later, the perspective shifts to a gloved hand holding a black handgun aimed at the camera, with the barrel pointing directly at the viewer. the interior is quiet, with teal and blue seats, silver poles, and red and white advertisements, including one for 'the lion king'. the atmosphere is tense, with the hand's movement suggesting an imminent threat.

data/test_i2v/captions/000000000793.0.003.txt

@@ -0,0 +1 @@
+A purple porsche suv drives on a dirt road through a mountainous landscape, with a helicopter parked on a rocky outcrop in the distance, under a sky with scattered clouds. the scene is part of a video game, indicated by the 'gta 5' logo. two seconds later, the same purple suv, now identified with a 'cayenne' badge, continues its journey on the dirt road, kicking up dust. the background features a rugged mountainous terrain, a clear blue sky with scattered clouds, and a distant helicopter on a rocky outcrop, suggesting an adventurous setting.

data/test_i2v/captions/000000000896.2.005.txt

@@ -0,0 +1 @@
+A character dressed in dark, medieval attire with white hair and yellow eyes rides a reddish-brown horse through a lush, green forest. the scene is set in a video game environment, with a user interface visible in the upper left corner, indicating gameplay mechanics. as the character continues, they traverse a grassy path surrounded by dense coniferous trees, with the game's interface showing health and stamina bars, a mini-map, and inventory icons. the player's progress is tracked by level indicators and currency, suggesting an immersive role-playing experience. the character's yellow eyes and white hair are consistent throughout the journey.

data/test_i2v/captions/000000001059.6.004.txt

@@ -0,0 +1 @@
+A lone explorer in a detailed exoskeleton suit with a helmet and visor is seen traversing a desolate, rocky terrain under an overcast sky, suggesting a science fiction setting. the explorer's journey is marked by solitude and the stark beauty of an extraterrestrial environment, with no other life or human activity visible. the scene is set against a backdrop of reddish-brown soil and dark volcanic rocks, with the mood of isolation and the allure of discovery emphasized by the subdued lighting and the absence of any other life or human activity.

data/test_i2v/captions/000000049282.4.005.txt

@@ -0,0 +1 @@
+A bustling street scene in rome, italy, with pedestrians walking by shops like 'superman' and 'salvatore ferragamo'. the architecture is european, with buildings painted in warm yellows and oranges. a man in a white t-shirt and backpack looks over his shoulder, while a woman in a red coat and sunglasses walks forward. two seconds later, the scene shifts to a narrow street lined with yellow buildings, where people stroll under an awning labeled 'ristorante'. the atmosphere is vibrant, with a mix of casual and smart-casual attire, and a 'sale' sign on a storefront suggests commercial activity.

data/test_i2v/captions/000000049768.11.002.txt

@@ -0,0 +1 @@
+A serene coastal town with a pastel-colored building adorned with green shutters and a striped awning. a clothesline with laundry adds a domestic touch to the tranquil scene. the town is set against a backdrop of a rugged cliff and a calm blue sea, with a clear sky overhead. as the video continues, the same building is shown with a green door and a yellow and white striped awning, maintaining the peaceful atmosphere. contrasting with the detailed coastal setting, the video caption suggests a group of people sitting at a table with food and drinks, which does not align with the visual content described in the frames.

data/test_i2v/captions/000002012796.6.001.txt

@@ -0,0 +1 @@
+A whimsical, nautical-themed interior with wooden cabinetry and a central mural of a pirate scene. initially, the room is adorned with a ship's wheel, a compass rose, and a framed picture of a ship, all under a warm, inviting light. at the two-second mark, the scene shifts to a corridor with wooden paneling, round portholes, and a framed picture of a shipwreck, with a solitary figure standing in the distance. by the fourth second, the setting changes to a warmly lit bar area with wooden tables, red stools, and a maritime-themed mural, creating a cozy atmosphere.

data/test_i2v/captions/000003031893.11.003.txt

@@ -0,0 +1 @@
+A muscular, tattooed man with a black cap and red shorts is seen performing bicep curls with a white dumbbell at a gym, his reflection visible in a large mirror. the gym is equipped with various weights and exercise machines, set against wooden walls and flooring, creating a warm atmosphere. two seconds later, the same man, now in a black tank top, continues his workout with a barbell on a bench, surrounded by a mirror reflecting the gym's interior, including a treadmill and weight racks, all under warm lighting. contrary to the detailed scenes, the overall description inaccurately mentions a shirtless man standing in front of a refrigerator.

data/test_i2v/masked_videos/000000000005.2.013.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e26c4e1563e7e7963a180408468e377f48cf6a0c9639c60abe7d97d794798304
+size 494333

data/test_i2v/masked_videos/000000000793.0.003.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1010d8cd07b77aaf14ee10a86f75916d8cc501aaf8fad6d4cf0b196dfb0f083
+size 321633

data/test_i2v/masked_videos/000000000896.2.005.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ebe76f05ad108e42d9a668cf3f08f0965f10dbf5f7f755b9dc4b13b4ccc9cd03
+size 1168624

data/test_i2v/masked_videos/000000001059.6.004.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d37c98497b103552cde6355e186bf7be0d9fbd18029d512af76c6df3a54f2c9c
+size 304053

data/test_i2v/masked_videos/000000049282.4.005.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:966ba1cf67689feafc6e3964041c7282957ee3bb8032116b9e58e3d11a52804f
+size 402082

data/test_i2v/masked_videos/000000049768.11.002.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a7c8d5ae12acbe72000b14741a645df14b3309f4a16854061e1b68cb2f30ae50
+size 1076475

data/test_i2v/masked_videos/000002012796.6.001.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ab4d4bee407c6f86a5d4188c3679bee832f070388a251e2d4629fd6d7e6250d
+size 157999

data/test_i2v/masked_videos/000003031893.11.003.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb7d9beb34ff81afd97834b3322a90ec6a28ac351be4211b78636b7b74300f7e
+size 171900

data/test_i2v/masks/000000000005.2.013.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f088f95b36218a7cc33406b77783b7913afed9213b2ee73d80c23caab5d464a6
+size 441136

data/test_i2v/masks/000000000793.0.003.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b37b20f2b1e3fe3843bb145c3279aa414018bb167ba28fc54e1a9bc5eed975b
+size 205096

data/test_i2v/masks/000000000896.2.005.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf449ba298632f7d2a811e15e5dbabbbaec3c2301e7ba70021afd42ad90831e3
+size 691623

data/test_i2v/masks/000000001059.6.004.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d03d4c6877ffbc1a6fb49fb59d9f5f1098d96bc1f7d73829768760fac3a7bf99
+size 148122

data/test_i2v/masks/000000049282.4.005.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ded019f277286353c850f08c49a94b70887e0ea71607c6eb8959b79739ecb924
+size 214608

data/test_i2v/masks/000000049768.11.002.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1fe0878aee08cc545f6b717515f142cb9299813ce3d89b4e64c5b2c7a6a2a708
+size 590769

data/test_i2v/masks/000002012796.6.001.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1951d5c5fdec649a59e569e3309398fb311745ed72930e8e720a5699208c4c4
+size 169341

data/test_i2v/masks/000003031893.11.003.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d79e7b6e102b7adcc5ddaafafa0c66e67c4efe81a0f7f9f1270cd17b66a193f
+size 158881

data/test_i2v/videos/000000000005.2.013.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a385d5f2066dacaf38a6819cb8594e1b44ca25574c23cc5e67d7a039b9309ef2
+size 575154

data/test_i2v/videos/000000000793.0.003.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e51cb421efeb29dfc0ddf93a01a7e8dea822b6ed1ae3ee53e94120170f4fad0
+size 757391

data/test_i2v/videos/000000000896.2.005.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32bd466321a25326e6c01c22d4342fbc9e2180609092d5ddc5f6b91b07929dda
+size 1349710

data/test_i2v/videos/000000001059.6.004.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f660782c05b937e2ec08834e76778818049a04d30c0617e538018528ab49bc73
+size 797266

data/test_i2v/videos/000000049282.4.005.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8f7c03191429649a4a23a503db089c6a864b2e4ae0418c8431ea9159954e460
+size 1177012

data/test_i2v/videos/000000049768.11.002.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b66d0e0729d314bb275297d27a2b6e073962d70cd6cf1d5caac20bf10adc254
+size 909919

data/test_i2v/videos/000002012796.6.001.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7e8b1ab3ca836e70e3386ca26658dcd74df503952cace96ae1a9743b3af44f4
+size 446478

data/test_i2v/videos/000003031893.11.003.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:26e186da08e186ad578a68da4bb0b376049e51f0039a302b63e0b2f4c9a25685
+size 506332

data/test_v2v/captions/amalfi-coast_traj_loop2.txt

@@ -0,0 +1 @@
+The video showcases an aerial view of a historic coastal structure perched on a cliff overlooking the sea. The architecture features a mix of stone and brick, with arched doorways and windows, and a prominent dome on one side. The surrounding area includes a large open plaza where people are gathered, some walking and others sitting, enjoying the scenic view. The coastline is rugged, with rocky outcrops jutting into the deep blue water below. In the background, there are more buildings and structures, indicating a nearby town or village. The overall scene is bathed in warm sunlight, highlighting the textures of the stone and the vibrant colors of. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic.

data/test_v2v/captions/art-museum_gradual_0_-30_0.0_0_0.txt

@@ -0,0 +1 @@
+The video showcases an art gallery with a spacious, well-lit interior featuring polished wooden floors arranged in a herringbone pattern. The walls are adorned with a variety of framed paintings, each with distinct styles and subjects. On the left side, there are landscapes and abstract artworks, while the right side displays more traditional portraits and historical scenes. The ceiling is ornate, with intricate designs and recessed lighting that highlights the artwork. The gallery appears to be empty, allowing the viewer to focus solely on the art pieces and the elegant architecture of the space.. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic.

data/test_v2v/captions/dog_gradual_0_-30_-0.6_0_0.txt

@@ -0,0 +1 @@
+A small, light brown puppy with floppy ears sits on a speckled carpet in front of a window with sheer curtains. The puppy looks around curiously, its head tilting slightly as it takes in its surroundings. The soft, natural light from the window illuminates the scene, creating a warm and cozy atmosphere. The puppy's expression is one of innocent curiosity, adding to the charm of the moment.. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic.

data/test_v2v/captions/fish_gradual_25_0_0.0_0_0.txt

@@ -0,0 +1 @@
+A vibrant blue and yellow striped fish, resembling an angelfish, swims gracefully among rocky formations in what appears to be an aquarium setting. The fish's flowing fins and the intricate patterns on its body are highlighted by the underwater lighting, creating a serene and captivating scene. The background consists of large, rugged rocks that add texture and depth to the environment, enhancing the naturalistic feel of the habitat.. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic.

data/test_v2v/captions/grandma-birthday_gradual_0_0_0.2_0_0.txt

@@ -0,0 +1 @@
+A joyful celebration unfolds as an elderly woman stands at a table, her face lit up with excitement and anticipation. In front of her is a beautifully decorated birthday cake adorned with numerous candles in shades of pink and yellow. The woman leans forward, her eyes sparkling with delight, as she prepares to blow out the candles. Surrounding her are several people, including family members and friends, who are clapping and cheering enthusiastically. The atmosphere is warm and festive, filled with laughter and happiness as everyone shares in the moment of joy.. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic.

data/test_v2v/captions/happy-cat_gradual_-20_0_0.0_0_0.txt

@@ -0,0 +1 @@
+A fluffy orange and white cat with striking green eyes is seen walking through a lush garden. The path is lined with vibrant green foliage and dotted with small yellow flowers. The cat moves gracefully, its paws padding softly on the earthy ground. As it walks, the camera follows closely behind, capturing the serene beauty of the garden and the curious nature of the feline. The sunlight filters through the leaves, casting dappled shadows that dance across the scene, adding to the tranquil atmosphere.. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic.

data/test_v2v/captions/suv-in-the-dust_gradual_0_30_0.0_0_0.txt

@@ -0,0 +1 @@
+A white off-road vehicle is seen driving along a winding dirt road in a mountainous, forested area. The terrain is rugged and uneven, with the vehicle kicking up dust as it maneuvers through the curves. The surrounding landscape features dense trees and shrubs, with patches of exposed earth and rocks. The sky above is clear and blue, indicating a sunny day. The vehicle's shadow stretches long on the ground, suggesting the sun is high in the sky. The overall scene conveys a sense of adventure and exploration in a remote, natural setting.. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic.

data/test_v2v/captions/vlogger-corgi_traj_loop2.txt

@@ -0,0 +1 @@
+A corgi dog is sitting on a vibrant blue beach towel adorned with yellow floral patterns, positioned on a sandy beach. The dog is wearing stylish sunglasses and a colorful lei around its neck, giving it a festive and playful appearance. A GoPro camera mounted on a selfie stick is pointed towards the dog, suggesting that it might be recording a video or taking photos. In the background, tall palm trees sway gently in the breeze, and the ocean stretches out to meet the horizon under a clear blue sky. The scene exudes a relaxed and joyful atmosphere, perfect for a day at the beach.. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic.

data/test_v2v/masked_videos/amalfi-coast_traj_loop2.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a88a29ce9e47e9f5c78222721f0adc5c5f7098f90c7371f3859d8742513f8ed
+size 752999

data/test_v2v/masked_videos/art-museum_gradual_0_-30_0.0_0_0.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa32d57497c1ded042b4e7944f07b15828d2a3199db79f94c0c670f846dbde88
+size 442795

data/test_v2v/masked_videos/dog_gradual_0_-30_-0.6_0_0.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be94b77bd5367476198b3e0758614e8ef672b8d6ae8bcd451a005980d43e3ffe
+size 334957