Update app.py (#3)

- Update app.py (12606cbae593d8678b7be580e3e22cfdacec0262)
- Update app.py (9c65fb8e014f207a2b68116b2eca5264445ca365)
- Update app.py (8320f45d7c62ab8c39b9c8819b2dd9e5138a13ee)

app.py

@@ -111,19 +111,20 @@ def load_control_lora(control_type, current_lora_state):
         print(f"Error loading {control_type} LoRA: {e}")
         raise
 
-def process_video_for_canny(video):
+def process_video_for_canny(video, width, height):
     """
     Process video for canny control.
     """
     print("Processing video for canny control...")
     canny_video = []
+    detect_resolution = video[0].size()
+    
     for frame in video: 
         # TODO: change resolution logic
-        canny_video.append(canny_processor(frame, low_threshold=50, high_threshold=200, detect_resolution=1024, image_resolution=1024))
+        canny_video.append(canny_processor(frame, low_threshold=50, high_threshold=200, detect_resolution=detect_resolution, image_resolution=(width, height)))
      
     return canny_video
 
-
 @spaces.GPU()
 def process_video_for_pose(video):
     """
@@ -169,31 +170,54 @@ def process_video_for_pose(video):
     
     return pose_video
 
-def process_video_for_control(reference_video, control_type):
+def process_input_video(reference_video, width, height):
+    """
+    Process the input video for canny edges and return both processed video and preview.
+    """
+    if reference_video is None:
+        return None
+    
+    try:
+        # Load video into a list of PIL images
+        video = load_video(reference_video)
+        
+        # Process video for canny edges
+        processed_video = process_video_for_canny(video, width, height)
+        
+        # Create a preview video file for display
+        fps = 24
+        with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as tmp_file:
+            preview_path = tmp_file.name
+            export_to_video(processed_video, preview_path, fps=fps)
+        
+        return preview_path
+        
+    except Exception as e:
+        print(f"Error processing input video: {e}")
+        return None
+
+def process_video_for_control(reference_video, control_type, width, height):
+    """Process video based on the selected control type - now only used for non-canny types"""
     video = load_video(reference_video)
-    """Process video based on the selected control type"""
+    
     if control_type == "canny":
-        processed_video = process_video_for_canny(video)
+        # This should not be called for canny since it's pre-processed
+        processed_video = process_video_for_canny(video, width, height)
     elif control_type == "depth":
         processed_video = process_video_for_depth(video)
     elif control_type == "pose":
         processed_video = process_video_for_pose(video)
     else:
-        processed_video = reference_video
-    # fps = 24
-    # with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as tmp2_file:
-    #     output2_path = tmp2_file.name
-    #     export_to_video(processed_video, output2_path, fps=fps)
-    # return output2_path
+        processed_video = video
+    
     return processed_video
 
-
 @spaces.GPU(duration=160)
 def generate_video(
     reference_video,
+    control_video,  # New parameter for pre-processed video
     prompt,
     control_type,
-    # current_lora_state,
     duration=3.0,
     negative_prompt="worst quality, inconsistent motion, blurry, jittery, distorted",
     height=768,
@@ -225,19 +249,18 @@ def generate_video(
         temporal_compression = pipeline.vae_temporal_compression_ratio
         num_frames = ((num_frames - 1) // temporal_compression) * temporal_compression + 1
         
-    
-        
-        # Load the appropriate control LoRA and update state
-        # updated_lora_state = load_control_lora(control_type, current_lora_state)
-        
-        # # Loads video into a list of pil images 
-        # video = load_video(reference_video)
-        # progress(0.1, desc="Processing video for control...")
+        progress(0.1, desc="Preparing processed video...")
         
-        # Process video based on control type
-        processed_video = process_video_for_control(reference_video, control_type)
+        # Use pre-processed video frames if available (for canny), otherwise process on-demand
+        if control_video is not None:
+            # Use the pre-processed canny frames
+            processed_video = load_video(control_video)
+        else:
+            # Fallback to processing on-demand for other control types
+            processed_video = process_video_for_control(reference_video, control_type, width, height)
         
-        processed_video = read_video(processed_video) # turns to tensor
+        # Convert to tensor
+        processed_video = read_video(processed_video)
         
         progress(0.2, desc="Preparing generation parameters...")
         
@@ -262,9 +285,7 @@ def generate_video(
             num_inference_steps=num_inference_steps,
             decode_timestep=0.05,
             decode_noise_scale=0.025,
-            # image_cond_noise_scale=image_cond_noise_scale,
             guidance_scale=guidance_scale,
-            # guidance_rescale=guidance_rescale,
             generator=torch.Generator().manual_seed(seed),
             output_type="latent",
         ).frames
@@ -294,7 +315,6 @@ def generate_video(
             guidance_scale=guidance_scale,
             decode_noise_scale = 0.025,
             image_cond_noise_scale=0.025,
-            #guidance_rescale=guidance_rescale,
             generator=torch.Generator(device="cuda").manual_seed(seed),
             output_type="pil",
         ).frames[0]
@@ -325,8 +345,8 @@ with gr.Blocks() as demo:
         """
     )
     
-    # State variable for tracking current LoRA
-    current_lora_state = gr.State(value=None)
+    # State variables
+    #current_lora_state = gr.State(value=None)
     
     with gr.Row():
         with gr.Column(scale=1):
@@ -402,8 +422,6 @@ with gr.Blocks() as demo:
                         value=1.0
                     )
                     
-                
-                
                 with gr.Row():
                     randomize_seed = gr.Checkbox(
                         label="Randomize Seed",
@@ -425,22 +443,21 @@ with gr.Blocks() as demo:
                 height=400
             )
             control_video = gr.Video(
-                label="Control Video",
+                label="Processed Control Video (Canny Edges)",
                 height=400,
-                visible=False
+                visible=True 
             )
         
     gr.Examples(
         examples=[ 
-            ["video_assets/vid_1.mp4", "A sleek cybernetic wolf sprinting through a neon-lit futuristic cityscape, its metallic form gleaming with electric blue circuits. The wolf's powerful stride carries it down rain-slicked streets between towering skyscrapers, while holographic advertisements cast colorful reflections on its chrome surface. Sparks of digital energy trail behind the creature as it moves with fluid mechanical precision through the urban maze, creating streaks of light in the misty night air.", "canny", 3, "worst quality, inconsistent motion, blurry, jittery, distorted", 768, 1152, 7, 1, 0, True],
-            ["video_assets/vid_2.mp4", "A translucent ghost floating in a moonlit cemetery, raising a glowing spectral lantern that casts eerie light through the darkness. The ethereal figure's wispy form shimmers as it lifts the phantom light above its head, illuminating weathered tombstones and gnarled trees. Pale mist swirls around the ghost as the lantern pulses with otherworldly energy, creating haunting shadows that dance across the graveyard in the dead of night.", "canny", 2.5, "worst quality, inconsistent motion, blurry, jittery, distorted", 768, 1152, 7, 1, 0, True],
-            ["video_assets/vid_3.mp4", "A sleek android assassin poised in a combat stance atop a futuristic skyscraper, arms positioned for perfect balance. The chrome-plated figure gleams under neon city lights as holographic data streams flow around its metallic form. Rain droplets bead on its polished surface while the sprawling cyberpunk metropolis stretches endlessly below. Electric circuits pulse beneath the android's transparent panels as it maintains its precise, calculated pose against the backdrop of flying vehicles and towering digital billboards.", "canny", 3, "worst quality, inconsistent motion, blurry, jittery, distorted", 768, 1152, 7, 1, 0, True],
-            ["video_assets/vid_4.mp4", "Luminescent video game characters with glowing outlines and neon-bright details wandering through a digital landscape. Their bodies emit soft, colorful light that pulses gently as they move, creating trails of radiance behind them. The characters have a futuristic, stylized appearance with smooth surfaces that reflect their inner glow. They navigate naturally through their environment, their movements fluid and purposeful, while their bioluminescent features cast dynamic shadows and illuminate the surrounding area. The scene has a cyberpunk aesthetic with the characters' radiant presence serving as the primary light source in an otherwise darkened digital world.", "canny", 2.5, "worst quality, inconsistent motion, blurry, jittery, distorted", 768, 1152, 7, 1, 0, True],
+            ["video_assets/vid_1.mp4", None, "A sleek cybernetic wolf sprinting through a neon-lit futuristic cityscape, its metallic form gleaming with electric blue circuits. The wolf's powerful stride carries it down rain-slicked streets between towering skyscrapers, while holographic advertisements cast colorful reflections on its chrome surface. Sparks of digital energy trail behind the creature as it moves with fluid mechanical precision through the urban maze, creating streaks of light in the misty night air.", "canny", 3, "worst quality, inconsistent motion, blurry, jittery, distorted", 768, 1152, 7, 1, 0, True],
+            ["video_assets/vid_2.mp4", None, "A translucent ghost floating in a moonlit cemetery, raising a glowing spectral lantern that casts eerie light through the darkness. The ethereal figure's wispy form shimmers as it lifts the phantom light above its head, illuminating weathered tombstones and gnarled trees. Pale mist swirls around the ghost as the lantern pulses with otherworldly energy, creating haunting shadows that dance across the graveyard in the dead of night.", "canny", 2.5, "worst quality, inconsistent motion, blurry, jittery, distorted", 768, 1152, 7, 1, 0, True],
+            ["video_assets/vid_3.mp4", None,"A sleek android assassin poised in a combat stance atop a futuristic skyscraper, arms positioned for perfect balance. The chrome-plated figure gleams under neon city lights as holographic data streams flow around its metallic form. Rain droplets bead on its polished surface while the sprawling cyberpunk metropolis stretches endlessly below. Electric circuits pulse beneath the android's transparent panels as it maintains its precise, calculated pose against the backdrop of flying vehicles and towering digital billboards.", "canny", 3, "worst quality, inconsistent motion, blurry, jittery, distorted", 768, 1152, 7, 1, 0, True],
+            ["video_assets/vid_4.mp4", None, "Luminescent video game characters with glowing outlines and neon-bright details wandering through a digital landscape. Their bodies emit soft, colorful light that pulses gently as they move, creating trails of radiance behind them. The characters have a futuristic, stylized appearance with smooth surfaces that reflect their inner glow. They navigate naturally through their environment, their movements fluid and purposeful, while their bioluminescent features cast dynamic shadows and illuminate the surrounding area. The scene has a cyberpunk aesthetic with the characters' radiant presence serving as the primary light source in an otherwise darkened digital world.", "canny", 2.5, "worst quality, inconsistent motion, blurry, jittery, distorted", 768, 1152, 7, 1, 0, True],
         ],
         inputs=[reference_video,
             prompt,
             control_type,
-            # current_lora_state,
             duration,
             negative_prompt,
             height,
@@ -454,13 +471,22 @@ with gr.Blocks() as demo:
     )       
     
     # Event handlers
+    
+    # Auto-process video when uploaded
+    reference_video.upload(
+        fn=process_input_video,
+        inputs=[reference_video, width, height],
+        outputs=[control_video],
+        show_progress=True
+    )
+    
     generate_btn.click(
         fn=generate_video,
         inputs=[
             reference_video,
+            control_video,  # Use pre-processed video
             prompt,
             control_type,
-            # current_lora_state,
             duration,
             negative_prompt,
             height,