T2V, Video Pix2Pix and Pose-Guided Gen

README.md

@@ -1,12 +1,10 @@
 ---
-title: Text2Video Zero
-emoji: 💻
-colorFrom: blue
-colorTo: pink
+title: Text2Video-Zero
+emoji: 🚀
+colorFrom: green
+colorTo: blue
 sdk: gradio
 sdk_version: 3.23.0
 app_file: app.py
 pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+---

app.py

@@ -0,0 +1,73 @@
+import gradio as gr
+import torch
+
+from model import Model, ModelType
+
+# from app_canny import create_demo as create_demo_canny
+from app_pose import create_demo as create_demo_pose
+from app_text_to_video import create_demo as create_demo_text_to_video
+from app_pix2pix_video import create_demo as create_demo_pix2pix_video
+# from app_canny_db import create_demo as create_demo_canny_db
+
+
+model = Model(device='cuda', dtype=torch.float16)
+
+with gr.Blocks(css='style.css') as demo:
+    gr.HTML(
+        """
+        <div style="text-align: center; max-width: 1200px; margin: 20px auto;">
+        <h1 style="font-weight: 900; font-size: 3rem; margin: 0rem">
+            Text2Video-Zero
+        </h1>
+        <h2 style="font-weight: 450; font-size: 1rem; margin-top: 0.5rem; margin-bottom: 0.5rem">
+        We propose <b>Text2Video-Zero, the first zero-shot text-to-video syntenes framework</b>, that also natively supports, Video Instruct Pix2Pix, Pose Conditional, Edge Conditional 
+        and, Edge Conditional and DreamBooth Specialized applications.
+        </h2>
+        <h3 style="font-weight: 450; font-size: 1rem; margin: 0rem">
+        Levon Khachatryan, Andranik Movsisyan, Vahram Tadevosyan, Roberto Henschel, Atlas Wang, Shant Navasardyan
+        and <a href="https://www.humphreyshi.com/home">Humphrey Shi</a> 
+        [<a href="" style="color:blue;">arXiv</a>] 
+        [<a href="" style="color:blue;">GitHub</a>]
+        </h3>
+        </div>
+        """)
+
+    with gr.Tab('Zero-Shot Text2Video'):
+        # pass
+        create_demo_text_to_video(model)
+    with gr.Tab('Video Instruct Pix2Pix'):
+        # pass
+        create_demo_pix2pix_video(model)
+    with gr.Tab('Pose Conditional'):
+        # pass
+        create_demo_pose(model)
+    with gr.Tab('Edge Conditional'):
+        pass
+        # create_demo_canny(model)
+    with gr.Tab('Edge Conditional and Dreambooth Specialized'):
+        pass
+        # create_demo_canny_db(model)
+
+    gr.HTML(
+        """
+        <div style="text-align: justify; max-width: 1200px; margin: 20px auto;">
+        <h3 style="font-weight: 450; font-size: 0.8rem; margin: 0rem">
+        <b>Version: v1.0</b>
+        </h3>
+        <h3 style="font-weight: 450; font-size: 0.8rem; margin: 0rem">
+        <b>Caution</b>: 
+        We would like the raise the awareness of users of this demo of its potential issues and concerns.
+        Like previous large foundation models, Text2Video-Zero could be problematic in some cases, partially we use pretrained Stable Diffusion, therefore Text2Video-Zero can Inherit Its Imperfections.
+        So far, we keep all features available for research testing both to show the great potential of the Text2Video-Zero framework and to collect important feedback to improve the model in the future.
+        We welcome researchers and users to report issues with the HuggingFace community discussion feature or email the authors.
+        </h3>
+        <h3 style="font-weight: 450; font-size: 0.8rem; margin: 0rem">
+        <b>Biases and content acknowledgement</b>:
+        Beware that Text2Video-Zero may output content that reinforces or exacerbates societal biases, as well as realistic faces, pornography, and violence. 
+        Text2Video-Zero in this demo is meant only for research purposes.
+        </h3>
+        </div>
+        """)
+
+demo.launch(debug=True)
+# demo.queue(api_open=False).launch(file_directories=['temporal'], share=True)

app_pix2pix_video.py

@@ -0,0 +1,70 @@
+import gradio as gr
+from model import Model
+
+
+def create_demo(model: Model):
+    examples = [
+        ['__assets__/pix2pix video/camel.mp4', 'make it Van Gogh Starry Night style'],
+        ['__assets__/pix2pix video/mini-cooper.mp4', 'make it Picasso style'],
+        ['__assets__/pix2pix video/snowboard.mp4', 'replace man with robot'],
+        ['__assets__/pix2pix video/white-swan.mp4', 'replace swan with mallard'],
+    ]
+    with gr.Blocks() as demo:
+        with gr.Row():
+            gr.Markdown('## Video Instruct Pix2Pix')
+        with gr.Row():
+            with gr.Column():
+                input_image = gr.Video(label="Input Video",source='upload', type='numpy', format="mp4", visible=True).style(height="auto")
+            with gr.Column():
+                prompt = gr.Textbox(label='Prompt')
+                run_button = gr.Button(label='Run')
+                with gr.Accordion('Advanced options', open=False):
+                    image_resolution = gr.Slider(label='Image Resolution',
+                                                 minimum=256,
+                                                 maximum=1024,
+                                                 value=512,
+                                                 step=64)
+                    seed = gr.Slider(label='Seed',
+                                     minimum=0,
+                                     maximum=65536,
+                                     value=0,
+                                     step=1)
+                    start_t = gr.Slider(label='Starting time in seconds',
+                                        minimum=0,
+                                        maximum=10,
+                                        value=0,
+                                        step=1)
+                    end_t = gr.Slider(label='End time in seconds (-1 corresponds to uploaded video duration)',
+                                      minimum=0,
+                                      maximum=10,
+                                      value=-1,
+                                      step=1)
+                    out_fps = gr.Slider(label='Output video fps (-1 corresponds to uploaded video fps)',
+                                        minimum=1,
+                                        maximum=30,
+                                        value=-1,
+                                        step=1)
+            with gr.Column():
+                result = gr.Video(label='Output',
+                                    show_label=True)
+        inputs = [
+            input_image,
+            prompt,
+            image_resolution,
+            seed,
+            start_t,
+            end_t,
+            out_fps
+        ]
+
+        gr.Examples(examples=examples,
+                    inputs=inputs,
+                    outputs=result,
+                    # cache_examples=os.getenv('SYSTEM') == 'spaces',
+                    run_on_click=False,
+                    )
+
+        run_button.click(fn=model.process_pix2pix,
+                         inputs=inputs,
+                         outputs=result)
+    return demo

app_pose.py

@@ -0,0 +1,62 @@
+import gradio as gr
+import os
+
+from model import Model
+
+examples = [
+    ['Motion 1', "A Robot is dancing in Sahara desert"],
+    ['Motion 2', "A Robot is dancing in Sahara desert"],
+    ['Motion 3', "A Robot is dancing in Sahara desert"],
+    ['Motion 4', "A Robot is dancing in Sahara desert"],
+    ['Motion 5', "A Robot is dancing in Sahara desert"],
+]
+
+def create_demo(model: Model):
+    with gr.Blocks() as demo:
+        with gr.Row():
+            gr.Markdown('## Text and Pose Conditional Video Generation')
+
+        with gr.Row():
+            gr.Markdown('### You must select one pose sequence shown below, or use the examples')
+            with gr.Column():
+                gallery_pose_sequence = gr.Gallery(label="Pose Sequence", value=[('__assets__/poses_skeleton_gifs/dance1.gif', "Motion 1"), ('__assets__/poses_skeleton_gifs/dance2.gif', "Motion 2"), ('__assets__/poses_skeleton_gifs/dance3.gif', "Motion 3"), ('__assets__/poses_skeleton_gifs/dance4.gif', "Motion 4"), ('__assets__/poses_skeleton_gifs/dance5.gif', "Motion 5")]).style(grid=[2], height="auto")
+                input_video_path = gr.Textbox(label="Pose Sequence",visible=False,value="Motion 1")
+                gr.Markdown("## Selection")
+                pose_sequence_selector = gr.Markdown('Pose Sequence: **Motion 1**')
+            with gr.Column():
+                prompt = gr.Textbox(label='Prompt')
+                run_button = gr.Button(label='Run')
+            with gr.Column():
+                result = gr.Image(label="Generated Video")
+
+        input_video_path.change(on_video_path_update, None, pose_sequence_selector)
+        gallery_pose_sequence.select(pose_gallery_callback, None, input_video_path)
+        inputs = [
+            input_video_path,
+            #pose_sequence, 
+            prompt,
+        ]
+
+        gr.Examples(examples=examples,
+                    inputs=inputs,
+                    outputs=result,
+                    # cache_examples=os.getenv('SYSTEM') == 'spaces',
+                    fn=model.process_controlnet_pose,
+                    run_on_click=False,
+                    )
+        #fn=process,
+        #)
+
+
+        run_button.click(fn=model.process_controlnet_pose,
+                         inputs=inputs,
+                         outputs=result,)
+
+    return demo
+
+
+def on_video_path_update(evt: gr.EventData):
+    return f'Pose Sequence: **{evt._data}**'
+
+def pose_gallery_callback(evt: gr.SelectData):
+    return f"Motion {evt.index+1}"

app_text_to_video.py

@@ -0,0 +1,44 @@
+import gradio as gr
+from model import Model
+
+examples = [
+    "an astronaut waving the arm on the moon",
+    "a sloth surfing on a wakeboard",
+            "an astronaut walking on a street",
+            "a cute cat walking on grass",
+    "a horse is galloping on a street",
+   "an astronaut is skiing down the hill",
+    "a gorilla walking alone down the street"
+    "a gorilla dancing on times square",
+    "A panda dancing dancing like crazy on Times Square",
+    ]
+
+
+def create_demo(model: Model):
+
+    with gr.Blocks() as demo:
+        with gr.Row():
+            gr.Markdown('## Text2Video-Zero: Video Generation')
+
+        with gr.Row():
+            with gr.Column():
+                prompt = gr.Textbox(label='Prompt')
+                run_button = gr.Button(label='Run')
+            with gr.Column():
+                result = gr.Video(label="Generated Video")
+        inputs = [
+            prompt, 
+        ]
+
+        gr.Examples(examples=examples,
+                inputs=inputs,
+                outputs=result,
+                cache_examples=False,
+                #cache_examples=os.getenv('SYSTEM') == 'spaces')
+                run_on_click=False,
+        )
+
+        run_button.click(fn=model.process_text2video,
+                         inputs=inputs,
+                         outputs=result,)
+    return demo

config.py

@@ -0,0 +1 @@
+save_memory = False

gradio_utils.py

@@ -0,0 +1,77 @@
+# App Canny utils
+def edge_path_to_video_path(edge_path):
+    video_path = edge_path
+
+    vid_name = edge_path.split("/")[-1]
+    if vid_name == "butterfly.mp4":
+        video_path = "__assets__/canny_videos_mp4/butterfly.mp4"
+    elif vid_name == "deer.mp4":
+        video_path = "__assets__/canny_videos_mp4/deer.mp4"
+    elif vid_name == "fox.mp4":
+        video_path = "__assets__/canny_videos_mp4/fox.mp4"
+    elif vid_name == "girl_dancing.mp4":
+        video_path = "__assets__/canny_videos_mp4/girl_dancing.mp4"
+    elif vid_name == "girl_turning.mp4":
+        video_path = "__assets__/canny_videos_mp4/girl_turning.mp4"
+    elif vid_name == "halloween.mp4":
+        video_path = "__assets__/canny_videos_mp4/halloween.mp4"
+    elif vid_name == "santa.mp4":
+        video_path = "__assets__/canny_videos_mp4/santa.mp4"
+    return video_path
+
+
+# App Pose utils
+def motion_to_video_path(motion):
+    videos = [
+        "__assets__/poses_skeleton_gifs/dance1_corr.mp4",
+        "__assets__/poses_skeleton_gifs/dance2_corr.mp4",
+        "__assets__/poses_skeleton_gifs/dance3_corr.mp4",
+        "__assets__/poses_skeleton_gifs/dance4_corr.mp4",
+        "__assets__/poses_skeleton_gifs/dance5_corr.mp4"
+    ]
+    id = int(motion.split(" ")[1]) - 1
+    return videos[id]
+
+
+# App Canny Dreambooth utils
+def get_video_from_canny_selection(canny_selection):
+    if canny_selection == "woman1":
+        input_video_path = "__assets__/db_files/woman1.mp4"
+
+    elif canny_selection == "woman2":
+        input_video_path = "__assets__/db_files/woman2.mp4"
+
+    elif canny_selection == "man1":
+        input_video_path = "__assets__/db_files/man1.mp4"
+
+    elif canny_selection == "woman3":
+        input_video_path = "__assets__/db_files/woman3.mp4"
+    else:
+        raise Exception
+
+    return input_video_path
+
+
+def get_model_from_db_selection(db_selection):
+    if db_selection == "Anime DB":
+        input_video_path = 'PAIR/controlnet-canny-anime'
+    elif db_selection == "Avatar DB":
+        input_video_path = 'PAIR/controlnet-canny-avatar'
+    elif db_selection == "GTA-5 DB":
+        input_video_path = 'PAIR/controlnet-canny-gta5'
+    elif db_selection == "Arcane DB":
+        input_video_path = 'PAIR/controlnet-canny-arcane'
+    else:
+        raise Exception
+    return input_video_path
+
+
+def get_db_name_from_id(id):
+    db_names = ["Anime DB", "Arcane DB", "GTA-5 DB", "Avatar DB"]
+    return db_names[id]
+
+
+def get_canny_name_from_id(id):
+    canny_names = ["woman1", "woman2", "man1", "woman3"]
+    return canny_names[id]
+

model.py

@@ -0,0 +1,296 @@
+from enum import Enum
+import gc
+import numpy as np
+
+import torch
+import decord
+from diffusers import StableDiffusionInstructPix2PixPipeline, StableDiffusionControlNetPipeline, ControlNetModel, UNet2DConditionModel
+from diffusers.schedulers import EulerAncestralDiscreteScheduler, DDIMScheduler
+from text_to_video.text_to_video_pipeline import TextToVideoPipeline
+
+import utils
+import gradio_utils
+
+decord.bridge.set_bridge('torch')
+
+
+class ModelType(Enum):
+    Pix2Pix_Video = 1,
+    Text2Video = 2,
+    ControlNetCanny = 3,
+    ControlNetCannyDB = 4,
+    ControlNetPose = 5,
+
+
+class Model:
+    def __init__(self, device, dtype, **kwargs):
+        self.device = device
+        self.dtype = dtype
+        self.generator = torch.Generator(device=device)
+        self.pipe_dict = {
+            ModelType.Pix2Pix_Video: StableDiffusionInstructPix2PixPipeline,
+            ModelType.Text2Video: TextToVideoPipeline,
+            ModelType.ControlNetCanny: StableDiffusionControlNetPipeline,
+            ModelType.ControlNetCannyDB: StableDiffusionControlNetPipeline,
+            ModelType.ControlNetPose: StableDiffusionControlNetPipeline,
+        }
+        self.controlnet_attn_proc = utils.CrossFrameAttnProcessor(unet_chunk_size=2)
+        self.pix2pix_attn_proc = utils.CrossFrameAttnProcessor(unet_chunk_size=3)
+        self.text2video_attn_proc = utils.CrossFrameAttnProcessor(unet_chunk_size=2)
+
+        self.pipe = None
+        self.model_type = None
+
+        self.states = {}
+
+    def set_model(self, model_type: ModelType, model_id: str, **kwargs):
+        if self.pipe is not None:
+            del self.pipe
+        torch.cuda.empty_cache()
+        gc.collect()
+        safety_checker = kwargs.pop('safety_checker', None)
+        self.pipe = self.pipe_dict[model_type].from_pretrained(model_id, safety_checker=safety_checker, **kwargs).to(self.device).to(self.dtype)
+        self.model_type = model_type
+
+    def inference_chunk(self, frame_ids, **kwargs):
+        if self.pipe is None:
+            return
+        image = kwargs.pop('image')
+        prompt = np.array(kwargs.pop('prompt'))
+        negative_prompt = np.array(kwargs.pop('negative_prompt', ''))
+        latents = None
+        if 'latents' in kwargs:
+            latents = kwargs.pop('latents')[frame_ids]
+        return self.pipe(image=image[frame_ids],
+                         prompt=prompt[frame_ids].tolist(),
+                         negative_prompt=negative_prompt[frame_ids].tolist(),
+                         latents=latents,
+                         generator=self.generator,
+                         **kwargs)
+
+    def inference(self, split_to_chunks=False, chunk_size=8, **kwargs):
+        if self.pipe is None:
+            return
+        seed = kwargs.pop('seed', 0)
+        kwargs.pop('generator', '')
+        # self.generator.manual_seed(seed)
+        if split_to_chunks:
+            assert 'image' in kwargs
+            assert 'prompt' in kwargs
+            image = kwargs.pop('image')
+            prompt = kwargs.pop('prompt')
+            negative_prompt = kwargs.pop('negative_prompt', '')
+            f = image.shape[0]
+            chunk_ids = np.arange(0, f, chunk_size - 1)
+            result = []
+            for i in range(len(chunk_ids)):
+                ch_start = chunk_ids[i]
+                ch_end = f if i == len(chunk_ids) - 1 else chunk_ids[i + 1]
+                frame_ids = [0] + list(range(ch_start, ch_end))
+                self.generator.manual_seed(seed)
+                print(f'Processing chunk {i + 1} / {len(chunk_ids)}')
+                result.append(self.inference_chunk(frame_ids=frame_ids,
+                                                   image=image,
+                                                   prompt=[prompt] * f,
+                                                   negative_prompt=[negative_prompt] * f,
+                                                   **kwargs).images[1:])
+            result = np.concatenate(result)
+            return result
+        else:
+            return self.pipe(generator=self.generator, **kwargs).videos[0]
+
+    def process_controlnet_canny(self,
+                                 video_path,
+                                 prompt,
+                                 num_inference_steps=20,
+                                 controlnet_conditioning_scale=1.0,
+                                 guidance_scale=9.0,
+                                 seed=42,
+                                 eta=0.0,
+                                 low_threshold=100,
+                                 high_threshold=200,
+                                 resolution=512):
+        video_path = gradio_utils.edge_path_to_video_path(video_path)
+        if self.model_type != ModelType.ControlNetCanny:
+            controlnet = ControlNetModel.from_pretrained("lllyasviel/sd-controlnet-canny")
+            self.set_model(ModelType.ControlNetCanny, model_id="runwayml/stable-diffusion-v1-5", controlnet=controlnet)
+            self.pipe.scheduler = DDIMScheduler.from_config(self.pipe.scheduler.config)
+            self.pipe.unet.set_attn_processor(processor=self.controlnet_attn_proc)
+            self.pipe.controlnet.set_attn_processor(processor=self.controlnet_attn_proc)
+
+        # TODO: Check scheduler
+        added_prompt = 'best quality, extremely detailed'
+        negative_prompts = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, fewer digits, cropped, worst quality, low quality'
+
+        video, fps = utils.prepare_video(video_path, resolution, self.device, self.dtype, False)
+        control = utils.pre_process_canny(video, low_threshold, high_threshold).to(self.device).to(self.dtype)
+        f, _, h, w = video.shape
+        self.generator.manual_seed(seed)
+        latents = torch.randn((1, 4, h//8, w//8), dtype=self.dtype, device=self.device, generator=self.generator)
+        latents = latents.repeat(f, 1, 1, 1)
+        result = self.inference(image=control,
+                                prompt=prompt + ', ' + added_prompt,
+                                height=h,
+                                width=w,
+                                negative_prompt=negative_prompts,
+                                num_inference_steps=num_inference_steps,
+                                guidance_scale=guidance_scale,
+                                controlnet_conditioning_scale=controlnet_conditioning_scale,
+                                eta=eta,
+                                latents=latents,
+                                seed=seed,
+                                output_type='numpy',
+                                split_to_chunks=True,
+                                chunk_size=8,
+                                )
+        return utils.create_video(result, fps)
+
+    def process_controlnet_pose(self,
+                                video_path,
+                                prompt,
+                                num_inference_steps=20,
+                                controlnet_conditioning_scale=1.0,
+                                guidance_scale=9.0,
+                                seed=42,
+                                eta=0.0,
+                                resolution=512):
+        video_path = gradio_utils.motion_to_video_path(video_path)
+        if self.model_type != ModelType.ControlNetPose:
+            controlnet = ControlNetModel.from_pretrained("fusing/stable-diffusion-v1-5-controlnet-openpose")
+            self.set_model(ModelType.ControlNetPose, model_id="runwayml/stable-diffusion-v1-5", controlnet=controlnet)
+            self.pipe.scheduler = DDIMScheduler.from_config(self.pipe.scheduler.config)
+            self.pipe.unet.set_attn_processor(processor=self.controlnet_attn_proc)
+            self.pipe.controlnet.set_attn_processor(processor=self.controlnet_attn_proc)
+
+        added_prompt = 'best quality, extremely detailed, HD, ultra-realistic, 8K, HQ, masterpiece, trending on artstation, art, smooth'
+        negative_prompts = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, fewer difits, cropped, worst quality, low quality, deformed body, bloated, ugly, unrealistic'
+
+        video, fps = utils.prepare_video(video_path, resolution, self.device, self.dtype, False, output_fps=4)
+        control = utils.pre_process_pose(video, apply_pose_detect=False).to(self.device).to(self.dtype)
+        f, _, h, w = video.shape
+        self.generator.manual_seed(seed)
+        latents = torch.randn((1, 4, h//8, w//8), dtype=self.dtype, device=self.device, generator=self.generator)
+        latents = latents.repeat(f, 1, 1, 1)
+        result = self.inference(image=control,
+                                prompt=prompt + ', ' + added_prompt,
+                                height=h,
+                                width=w,
+                                negative_prompt=negative_prompts,
+                                num_inference_steps=num_inference_steps,
+                                guidance_scale=guidance_scale,
+                                controlnet_conditioning_scale=controlnet_conditioning_scale,
+                                eta=eta,
+                                latents=latents,
+                                seed=seed,
+                                output_type='numpy',
+                                split_to_chunks=True,
+                                chunk_size=8,
+                                )
+        return utils.create_gif(result, fps)
+
+    def process_controlnet_canny_db(self,
+                                    db_path,
+                                    video_path,
+                                    prompt,
+                                    num_inference_steps=20,
+                                    controlnet_conditioning_scale=1.0,
+                                    guidance_scale=9.0,
+                                    seed=42,
+                                    eta=0.0,
+                                    low_threshold=100,
+                                    high_threshold=200,
+                                    resolution=512):
+        db_path = gradio_utils.get_model_from_db_selection(db_path)
+        video_path = gradio_utils.get_video_from_canny_selection(video_path)
+        # Load db and controlnet weights
+        if 'db_path' not in self.states or db_path != self.states['db_path']:
+            controlnet = ControlNetModel.from_pretrained("lllyasviel/sd-controlnet-canny")
+            self.set_model(ModelType.ControlNetCannyDB, model_id=db_path, controlnet=controlnet)
+            self.pipe.scheduler = DDIMScheduler.from_config(self.pipe.scheduler.config)
+            self.pipe.unet.set_attn_processor(processor=self.controlnet_attn_proc)
+            self.pipe.controlnet.set_attn_processor(processor=self.controlnet_attn_proc)
+            self.states['db_path'] = db_path
+
+        added_prompt = 'best quality, extremely detailed'
+        negative_prompts = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, fewer digits, cropped, worst quality, low quality'
+
+        video, fps = utils.prepare_video(video_path, resolution, self.device, self.dtype, False)
+        control = utils.pre_process_canny(video, low_threshold, high_threshold).to(self.device).to(self.dtype)
+        f, _, h, w = video.shape
+        self.generator.manual_seed(seed)
+        latents = torch.randn((1, 4, h//8, w//8), dtype=self.dtype, device=self.device, generator=self.generator)
+        latents = latents.repeat(f, 1, 1, 1)
+        result = self.inference(image=control,
+                                prompt=prompt + ', ' + added_prompt,
+                                height=h,
+                                width=w,
+                                negative_prompt=negative_prompts,
+                                num_inference_steps=num_inference_steps,
+                                guidance_scale=guidance_scale,
+                                controlnet_conditioning_scale=controlnet_conditioning_scale,
+                                eta=eta,
+                                latents=latents,
+                                seed=seed,
+                                output_type='numpy',
+                                split_to_chunks=True,
+                                chunk_size=8,
+                                )
+        return utils.create_gif(result, fps)
+
+    def process_pix2pix(self, video, prompt, resolution=512, seed=0, start_t=0, end_t=-1, out_fps=-1):
+        if self.model_type != ModelType.Pix2Pix_Video:
+            self.set_model(ModelType.Pix2Pix_Video, model_id="timbrooks/instruct-pix2pix")
+            self.pipe.scheduler = EulerAncestralDiscreteScheduler.from_config(self.pipe.scheduler.config)
+            self.pipe.unet.set_attn_processor(processor=self.pix2pix_attn_proc)
+        video, fps = utils.prepare_video(video, resolution, self.device, self.dtype, True, start_t, end_t, out_fps)
+        self.generator.manual_seed(seed)
+        result = self.inference(image=video,
+                                prompt=prompt,
+                                seed=seed,
+                                output_type='numpy',
+                                num_inference_steps=50,
+                                image_guidance_scale=1.5,
+                                split_to_chunks=True,
+                                chunk_size=8,
+                                )
+        return utils.create_video(result, fps)
+
+    def process_text2video(self, prompt, resolution=512, seed=24, num_frames=8, fps=4, t0=881, t1=941,
+                           use_cf_attn=True, use_motion_field=True, use_foreground_motion_field=False,
+                           smooth_bg=False, smooth_bg_strength=0.4, motion_field_strength=12):
+        
+        if self.model_type != ModelType.Text2Video:
+            unet = UNet2DConditionModel.from_pretrained("runwayml/stable-diffusion-v1-5", subfolder="unet")
+            self.set_model(ModelType.Text2Video, model_id="runwayml/stable-diffusion-v1-5", unet=unet)
+            self.pipe.scheduler = DDIMScheduler.from_config(self.pipe.scheduler.config)
+            self.pipe.unet.set_attn_processor(processor=self.text2video_attn_proc)
+            self.generator.manual_seed(seed)
+        
+
+        added_prompt = "high quality, HD, 8K, trending on artstation, high focus, dramatic lighting"
+        self.generator.manual_seed(seed)
+
+        prompt = prompt.rstrip()
+        if len(prompt) > 0 and (prompt[-1] == "," or prompt[-1]  == "."):
+            prompt = prompt.rstrip()[:-1]
+        prompt = prompt.rstrip()
+        prompt = prompt + ", "+added_prompt
+
+        result = self.inference(prompt=[prompt],
+                                video_length=num_frames,
+                                height=resolution,
+                                width=resolution,
+                                num_inference_steps=50,
+                                guidance_scale=7.5,
+                                guidance_stop_step=1.0,
+                                t0=t0,
+                                t1=t1,
+                                use_foreground_motion_field=use_foreground_motion_field,
+                                motion_field_strength=motion_field_strength,
+                                use_motion_field=use_motion_field,
+                                smooth_bg=smooth_bg,
+                                smooth_bg_strength=smooth_bg_strength,
+                                seed=seed,
+                                output_type='numpy',
+                                )
+        return utils.create_video(result, fps)

requirements.txt

@@ -0,0 +1,34 @@
+accelerate==0.16.0
+addict==2.4.0
+albumentations==1.3.0
+basicsr==1.4.2
+decord==0.6.0
+diffusers==0.14.0
+einops==0.6.0
+gradio==3.23.0
+kornia==0.6
+imageio==2.9.0
+imageio-ffmpeg==0.4.2
+invisible-watermark>=0.1.5
+moviepy==1.0.3
+numpy==1.24.1
+omegaconf==2.3.0
+open_clip_torch==2.16.0
+opencv_python==4.7.0.68
+opencv-contrib-python==4.3.0.36
+Pillow==9.4.0
+pytorch_lightning==1.5.0
+prettytable==3.6.0
+scikit_image==0.19.3
+scipy==1.10.1
+tensorboardX==2.6
+tqdm==4.64.1
+timm==0.6.12
+transformers==4.26.0
+test-tube>=0.7.5
+webdataset==0.2.5
+yapf==0.32.0
+safetensors==0.2.7
+huggingface-hub==0.13.0
+torch==1.13.1
+torchvision==0.14.1

share.py

@@ -0,0 +1,8 @@
+import config
+from cldm.hack import disable_verbosity, enable_sliced_attention
+
+
+disable_verbosity()
+
+if config.save_memory:
+    enable_sliced_attention()

style.css

@@ -0,0 +1,3 @@
+h1 {
+  text-align: center;
+}

utils.py

@@ -0,0 +1,187 @@
+import os
+import numpy as np
+import torch
+import torchvision
+from torchvision.transforms import Resize
+import imageio
+from einops import rearrange
+import cv2
+from annotator.util import resize_image, HWC3
+from annotator.canny import CannyDetector
+from annotator.openpose import OpenposeDetector
+import decord
+decord.bridge.set_bridge('torch')
+
+apply_canny = CannyDetector()
+apply_openpose = OpenposeDetector()
+
+
+def add_watermark(image, im_size, watermark_path="__assets__/pair_watermark.png",
+                  wmsize=16, bbuf=5, opacity=0.9):
+    '''
+    Creates a watermark on the saved inference image.
+    We request that you do not remove this to properly assign credit to
+    Shi-Lab's work.
+    '''
+    watermark = Image.open(watermark_path).resize((wmsize, wmsize))
+    loc = im_size - wmsize - bbuf
+    image[:,:,loc:-bbuf, loc:-bbuf] = watermark
+    return image
+
+
+def pre_process_canny(input_video, low_threshold=100, high_threshold=200):
+    detected_maps = []
+    for frame in input_video:
+        img = rearrange(frame, 'c h w -> h w c').cpu().numpy().astype(np.uint8)
+        detected_map = apply_canny(img, low_threshold, high_threshold)
+        detected_map = HWC3(detected_map)
+        detected_maps.append(detected_map[None])
+    detected_maps = np.concatenate(detected_maps)
+    control = torch.from_numpy(detected_maps.copy()).float() / 255.0
+    return rearrange(control, 'f h w c -> f c h w')
+
+
+def pre_process_pose(input_video, apply_pose_detect: bool = True):
+    detected_maps = []
+    for frame in input_video:
+        img = rearrange(frame, 'c h w -> h w c').cpu().numpy().astype(np.uint8)
+        img = HWC3(img)
+        if apply_pose_detect:
+            detected_map, _ = apply_openpose(img)
+        else:
+            detected_map = img
+        detected_map = HWC3(detected_map)
+        H, W, C = img.shape
+        detected_map = cv2.resize(detected_map, (W, H), interpolation=cv2.INTER_NEAREST)
+        detected_maps.append(detected_map[None])
+    detected_maps = np.concatenate(detected_maps)
+    control = torch.from_numpy(detected_maps.copy()).float() / 255.0
+    return rearrange(control, 'f h w c -> f c h w')
+
+
+def create_video(frames, fps, rescale=False, path=None):
+    if path is None:
+        dir = "temporal"
+        os.makedirs(dir, exist_ok=True)
+        path = os.path.join(dir, 'movie.mp4')
+
+    outputs = []
+    for i, x in enumerate(frames):
+        x = torchvision.utils.make_grid(torch.Tensor(x), nrow=4)
+        if rescale:
+            x = (x + 1.0) / 2.0  # -1,1 -> 0,1
+        x = (x * 255).numpy().astype(np.uint8)
+        x = add_watermark(x, im_size=512)
+        outputs.append(x)
+        # imageio.imsave(os.path.join(dir, os.path.splitext(name)[0] + f'_{i}.jpg'), x)
+
+    imageio.mimsave(path, outputs, fps=fps)
+    return path
+
+def create_gif(frames, fps, rescale=False):
+    dir = "temporal"
+    os.makedirs(dir, exist_ok=True)
+    path = os.path.join(dir, 'canny_db.gif')
+
+    outputs = []
+    for i, x in enumerate(frames):
+        x = torchvision.utils.make_grid(torch.Tensor(x), nrow=4)
+        if rescale:
+            x = (x + 1.0) / 2.0  # -1,1 -> 0,1
+        x = (x * 255).numpy().astype(np.uint8)
+        x = add_watermark(x, im_size=512)
+        outputs.append(x)
+        # imageio.imsave(os.path.join(dir, os.path.splitext(name)[0] + f'_{i}.jpg'), x)
+
+    imageio.mimsave(path, outputs, fps=fps)
+    return path
+
+def prepare_video(video_path:str, resolution:int, device, dtype, normalize=True, start_t:float=0, end_t:float=-1, output_fps:int=-1):
+    vr = decord.VideoReader(video_path)
+    video = vr.get_batch(range(0, len(vr))).asnumpy()
+    initial_fps = vr.get_avg_fps()
+    if output_fps == -1:
+        output_fps = int(initial_fps)
+    if end_t == -1:
+        end_t = len(vr) / initial_fps
+    else:
+        end_t = min(len(vr) / initial_fps, end_t)
+    assert 0 <= start_t < end_t
+    assert output_fps > 0
+    f, h, w, c = video.shape
+    start_f_ind = int(start_t * initial_fps)
+    end_f_ind = int(end_t * initial_fps)
+    num_f = int((end_t - start_t) * output_fps)
+    sample_idx = np.linspace(start_f_ind, end_f_ind, num_f, endpoint=False).astype(int)
+    video = video[sample_idx]
+    video = rearrange(video, "f h w c -> f c h w")
+    video = torch.Tensor(video).to(device).to(dtype)
+    if h > w:
+        w = int(w * resolution / h)
+        w = w - w % 8
+        h = resolution - resolution % 8
+        video = Resize((h, w))(video)
+    else:
+        h = int(h * resolution / w)
+        h = h - h % 8
+        w = resolution - resolution % 8
+        video = Resize((h, w))(video)
+    if normalize:
+        video = video / 127.5 - 1.0
+    return video, output_fps
+
+
+def post_process_gif(list_of_results, image_resolution):
+    output_file = "/tmp/ddxk.gif"
+    imageio.mimsave(output_file, list_of_results, fps=4)
+    return output_file
+
+
+class CrossFrameAttnProcessor:
+    def __init__(self, unet_chunk_size=2):
+        self.unet_chunk_size = unet_chunk_size
+
+    def __call__(
+            self,
+            attn,
+            hidden_states,
+            encoder_hidden_states=None,
+            attention_mask=None):
+        batch_size, sequence_length, _ = hidden_states.shape
+        attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size)
+        query = attn.to_q(hidden_states)
+
+        is_cross_attention = encoder_hidden_states is not None
+        if encoder_hidden_states is None:
+            encoder_hidden_states = hidden_states
+        elif attn.cross_attention_norm:
+            encoder_hidden_states = attn.norm_cross(encoder_hidden_states)
+        key = attn.to_k(encoder_hidden_states)
+        value = attn.to_v(encoder_hidden_states)
+        # Sparse Attention
+        if not is_cross_attention:
+            video_length = key.size()[0] // self.unet_chunk_size
+            # former_frame_index = torch.arange(video_length) - 1
+            # former_frame_index[0] = 0
+            former_frame_index = [0] * video_length
+            key = rearrange(key, "(b f) d c -> b f d c", f=video_length)
+            key = key[:, former_frame_index]
+            key = rearrange(key, "b f d c -> (b f) d c")
+            value = rearrange(value, "(b f) d c -> b f d c", f=video_length)
+            value = value[:, former_frame_index]
+            value = rearrange(value, "b f d c -> (b f) d c")
+
+        query = attn.head_to_batch_dim(query)
+        key = attn.head_to_batch_dim(key)
+        value = attn.head_to_batch_dim(value)
+
+        attention_probs = attn.get_attention_scores(query, key, attention_mask)
+        hidden_states = torch.bmm(attention_probs, value)
+        hidden_states = attn.batch_to_head_dim(hidden_states)
+
+        # linear proj
+        hidden_states = attn.to_out[0](hidden_states)
+        # dropout
+        hidden_states = attn.to_out[1](hidden_states)
+
+        return hidden_states