diff --git "a/app.py" "b/app.py" --- "a/app.py" +++ "b/app.py" @@ -4,15 +4,14 @@ import os os.environ['HF_HOME'] = os.path.abspath(os.path.realpath(os.path.join(os.path.dirname(__file__), './hf_download'))) -import spaces import gradio as gr import torch import traceback import einops import safetensors.torch as sf import numpy as np +import argparse import random -import time import math # 20250506 pftq: Added for video input loading import decord @@ -27,7 +26,7 @@ import imageio_ffmpeg import tempfile import shutil import subprocess - +import spaces from PIL import Image from diffusers import AutoencoderKLHunyuanVideo from transformers import LlamaModel, CLIPTextModel, LlamaTokenizerFast, CLIPTokenizer @@ -35,87 +34,83 @@ from diffusers_helper.hunyuan import encode_prompt_conds, vae_decode, vae_encode from diffusers_helper.utils import save_bcthw_as_mp4, crop_or_pad_yield_mask, soft_append_bcthw, resize_and_center_crop, state_dict_weighted_merge, state_dict_offset_merge, generate_timestamp from diffusers_helper.models.hunyuan_video_packed import HunyuanVideoTransformer3DModelPacked from diffusers_helper.pipelines.k_diffusion_hunyuan import sample_hunyuan -if torch.cuda.device_count() > 0: - from diffusers_helper.memory import cpu, gpu, get_cuda_free_memory_gb, move_model_to_device_with_memory_preservation, offload_model_from_device_for_memory_preservation, fake_diffusers_current_device, DynamicSwapInstaller, unload_complete_models, load_model_as_complete +from diffusers_helper.memory import cpu, gpu, get_cuda_free_memory_gb, move_model_to_device_with_memory_preservation, offload_model_from_device_for_memory_preservation, fake_diffusers_current_device, DynamicSwapInstaller, unload_complete_models, load_model_as_complete from diffusers_helper.thread_utils import AsyncStream, async_run from diffusers_helper.gradio.progress_bar import make_progress_bar_css, make_progress_bar_html from transformers import SiglipImageProcessor, SiglipVisionModel from diffusers_helper.clip_vision import hf_clip_vision_encode from diffusers_helper.bucket_tools import find_nearest_bucket -from diffusers import BitsAndBytesConfig as DiffusersBitsAndBytesConfig, HunyuanVideoTransformer3DModel, HunyuanVideoPipeline -import pillow_heif - -pillow_heif.register_heif_opener() - -high_vram = False -free_mem_gb = 0 - -if torch.cuda.device_count() > 0: - free_mem_gb = get_cuda_free_memory_gb(gpu) - high_vram = free_mem_gb > 60 - - print(f'Free VRAM {free_mem_gb} GB') - print(f'High-VRAM Mode: {high_vram}') - - text_encoder = LlamaModel.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='text_encoder', torch_dtype=torch.float16).cpu() - text_encoder_2 = CLIPTextModel.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='text_encoder_2', torch_dtype=torch.float16).cpu() - tokenizer = LlamaTokenizerFast.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='tokenizer') - tokenizer_2 = CLIPTokenizer.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='tokenizer_2') - vae = AutoencoderKLHunyuanVideo.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='vae', torch_dtype=torch.float16).cpu() - - feature_extractor = SiglipImageProcessor.from_pretrained("lllyasviel/flux_redux_bfl", subfolder='feature_extractor') - image_encoder = SiglipVisionModel.from_pretrained("lllyasviel/flux_redux_bfl", subfolder='image_encoder', torch_dtype=torch.float16).cpu() - - transformer = HunyuanVideoTransformer3DModelPacked.from_pretrained('lllyasviel/FramePack_F1_I2V_HY_20250503', torch_dtype=torch.bfloat16).cpu() - - vae.eval() - text_encoder.eval() - text_encoder_2.eval() - image_encoder.eval() - transformer.eval() - - if not high_vram: - vae.enable_slicing() - vae.enable_tiling() - transformer.high_quality_fp32_output_for_inference = True - print('transformer.high_quality_fp32_output_for_inference = True') - - transformer.to(dtype=torch.bfloat16) - vae.to(dtype=torch.float16) - image_encoder.to(dtype=torch.float16) - text_encoder.to(dtype=torch.float16) - text_encoder_2.to(dtype=torch.float16) - - vae.requires_grad_(False) - text_encoder.requires_grad_(False) - text_encoder_2.requires_grad_(False) - image_encoder.requires_grad_(False) - transformer.requires_grad_(False) - - if not high_vram: - # DynamicSwapInstaller is same as huggingface's enable_sequential_offload but 3x faster - DynamicSwapInstaller.install_model(transformer, device=gpu) - DynamicSwapInstaller.install_model(text_encoder, device=gpu) - else: - text_encoder.to(gpu) - text_encoder_2.to(gpu) - image_encoder.to(gpu) - vae.to(gpu) - transformer.to(gpu) +parser = argparse.ArgumentParser() +parser.add_argument('--share', action='store_true') +parser.add_argument("--server", type=str, default='0.0.0.0') +parser.add_argument("--port", type=int, required=False) +parser.add_argument("--inbrowser", action='store_true') +args = parser.parse_args() + +print(args) + +free_mem_gb = get_cuda_free_memory_gb(gpu) +high_vram = free_mem_gb > 60 + +print(f'Free VRAM {free_mem_gb} GB') +print(f'High-VRAM Mode: {high_vram}') + +text_encoder = LlamaModel.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='text_encoder', torch_dtype=torch.float16).cpu() +text_encoder_2 = CLIPTextModel.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='text_encoder_2', torch_dtype=torch.float16).cpu() +tokenizer = LlamaTokenizerFast.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='tokenizer') +tokenizer_2 = CLIPTokenizer.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='tokenizer_2') +vae = AutoencoderKLHunyuanVideo.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='vae', torch_dtype=torch.float16).cpu() + +feature_extractor = SiglipImageProcessor.from_pretrained("lllyasviel/flux_redux_bfl", subfolder='feature_extractor') +image_encoder = SiglipVisionModel.from_pretrained("lllyasviel/flux_redux_bfl", subfolder='image_encoder', torch_dtype=torch.float16).cpu() + +transformer = HunyuanVideoTransformer3DModelPacked.from_pretrained('lllyasviel/FramePackI2V_HY', torch_dtype=torch.bfloat16).cpu() + +vae.eval() +text_encoder.eval() +text_encoder_2.eval() +image_encoder.eval() +transformer.eval() + +if not high_vram: + vae.enable_slicing() + vae.enable_tiling() + +transformer.high_quality_fp32_output_for_inference = True +print('transformer.high_quality_fp32_output_for_inference = True') + +transformer.to(dtype=torch.bfloat16) +vae.to(dtype=torch.float16) +image_encoder.to(dtype=torch.float16) +text_encoder.to(dtype=torch.float16) +text_encoder_2.to(dtype=torch.float16) + +vae.requires_grad_(False) +text_encoder.requires_grad_(False) +text_encoder_2.requires_grad_(False) +image_encoder.requires_grad_(False) +transformer.requires_grad_(False) + +if not high_vram: + # DynamicSwapInstaller is same as huggingface's enable_sequential_offload but 3x faster + DynamicSwapInstaller.install_model(transformer, device=gpu) + DynamicSwapInstaller.install_model(text_encoder, device=gpu) +else: + text_encoder.to(gpu) + text_encoder_2.to(gpu) + image_encoder.to(gpu) + vae.to(gpu) + transformer.to(gpu) stream = AsyncStream() outputs_folder = './outputs/' os.makedirs(outputs_folder, exist_ok=True) -input_image_debug_value = input_video_debug_value = prompt_debug_value = total_second_length_debug_value = None - -default_local_storage = { - "generation-mode": "image", - } +input_video_debug_value = prompt_debug_value = total_second_length_debug_value = None -@spaces.GPU() +# 20250506 pftq: Added function to encode input video frames into latents @torch.no_grad() def video_encode(video_path, resolution, no_resize, vae, vae_batch_size=16, device="cuda", width=None, height=None): """ @@ -189,6 +184,7 @@ def video_encode(video_path, resolution, no_resize, vae, vae_batch_size=16, devi # 20250506 pftq: Save first frame for CLIP vision encoding input_image_np = processed_frames[0] + end_of_input_video_image_np = processed_frames[-1] # 20250506 pftq: Convert to tensor and normalize to [-1, 1] print("Converting frames to tensor...") @@ -244,6 +240,7 @@ def video_encode(video_path, resolution, no_resize, vae, vae_batch_size=16, devi # 20250506 pftq: Get first frame's latent start_latent = history_latents[:, :, :1] # Shape: (1, channels, 1, height//8, width//8) + end_of_input_video_latent = history_latents[:, :, -1:] # Shape: (1, channels, 1, height//8, width//8) print(f"Start latent shape: {start_latent.shape}") # 20250506 pftq: Move VAE back to CPU to free GPU memory @@ -252,12 +249,72 @@ def video_encode(video_path, resolution, no_resize, vae, vae_batch_size=16, devi torch.cuda.empty_cache() print("VAE moved back to CPU, CUDA cache cleared") - return start_latent, input_image_np, history_latents, fps, target_height, target_width, input_video_pixels + return start_latent, input_image_np, history_latents, fps, target_height, target_width, input_video_pixels, end_of_input_video_latent, end_of_input_video_image_np except Exception as e: print(f"Error in video_encode: {str(e)}") raise + +# 20250507 pftq: New function to encode a single image (end frame) +@torch.no_grad() +def image_encode(image_np, target_width, target_height, vae, image_encoder, feature_extractor, device="cuda"): + """ + Encode a single image into a latent and compute its CLIP vision embedding. + + Args: + image_np: Input image as numpy array. + target_width, target_height: Exact resolution to resize the image to (matches start frame). + vae: AutoencoderKLHunyuanVideo model. + image_encoder: SiglipVisionModel for CLIP vision encoding. + feature_extractor: SiglipImageProcessor for preprocessing. + device: Device for computation (e.g., "cuda"). + + Returns: + latent: Latent representation of the image (shape: [1, channels, 1, height//8, width//8]). + clip_embedding: CLIP vision embedding of the image. + processed_image_np: Processed image as numpy array (after resizing). + """ + # 20250507 pftq: Process end frame with exact start frame dimensions + print("Processing end frame...") + try: + print(f"Using exact start frame resolution for end frame: {target_width}x{target_height}") + + # Resize and preprocess image to match start frame + processed_image_np = resize_and_center_crop(image_np, target_width=target_width, target_height=target_height) + + # Convert to tensor and normalize + image_pt = torch.from_numpy(processed_image_np).float() / 127.5 - 1 + image_pt = image_pt.permute(2, 0, 1).unsqueeze(0).unsqueeze(2) # Shape: [1, channels, 1, height, width] + image_pt = image_pt.to(device) + + # Move VAE to device + vae.to(device) + + # Encode to latent + latent = vae_encode(image_pt, vae) + print(f"image_encode vae output shape: {latent.shape}") + + # Move image encoder to device + image_encoder.to(device) + + # Compute CLIP vision embedding + clip_embedding = hf_clip_vision_encode(processed_image_np, feature_extractor, image_encoder).last_hidden_state + + # Move models back to CPU and clear cache + if device == "cuda": + vae.to(cpu) + image_encoder.to(cpu) + torch.cuda.empty_cache() + print("VAE and image encoder moved back to CPU, CUDA cache cleared") + + print(f"End latent shape: {latent.shape}") + return latent, clip_embedding, processed_image_np + + except Exception as e: + print(f"Error in image_encode: {str(e)}") + raise + # 20250508 pftq: for saving prompt to mp4 metadata comments def set_mp4_comments_imageio_ffmpeg(input_file, comments): try: @@ -305,29 +362,9 @@ def set_mp4_comments_imageio_ffmpeg(input_file, comments): print(f"Error saving prompt to video metadata, ffmpeg may be required: "+str(e)) return False +# 20250506 pftq: Modified worker to accept video input, and clean frame count @torch.no_grad() -def worker(input_image, prompts, n_prompt, seed, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, mp4_crf): - def encode_prompt(prompt, n_prompt): - llama_vec, clip_l_pooler = encode_prompt_conds(prompt, text_encoder, text_encoder_2, tokenizer, tokenizer_2) - - if cfg == 1: - llama_vec_n, clip_l_pooler_n = torch.zeros_like(llama_vec), torch.zeros_like(clip_l_pooler) - else: - llama_vec_n, clip_l_pooler_n = encode_prompt_conds(n_prompt, text_encoder, text_encoder_2, tokenizer, tokenizer_2) - - llama_vec, llama_attention_mask = crop_or_pad_yield_mask(llama_vec, length=512) - llama_vec_n, llama_attention_mask_n = crop_or_pad_yield_mask(llama_vec_n, length=512) - - llama_vec = llama_vec.to(transformer.dtype) - llama_vec_n = llama_vec_n.to(transformer.dtype) - clip_l_pooler = clip_l_pooler.to(transformer.dtype) - clip_l_pooler_n = clip_l_pooler_n.to(transformer.dtype) - return [llama_vec, clip_l_pooler, llama_vec_n, clip_l_pooler_n, llama_attention_mask, llama_attention_mask_n] - - total_latent_sections = (total_second_length * 30) / (latent_window_size * 4) - total_latent_sections = int(max(round(total_latent_sections), 1)) - - job_id = generate_timestamp() +def worker(input_video, end_frame, end_frame_weight, prompt, n_prompt, seed, batch, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, no_resize, mp4_crf, num_clean_frames, vae_batch): stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Starting ...')))) @@ -339,191 +376,12 @@ def worker(input_image, prompts, n_prompt, seed, resolution, total_second_length ) # Text encoding - stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Text encoding ...')))) if not high_vram: fake_diffusers_current_device(text_encoder, gpu) # since we only encode one text - that is one model move and one encode, offload is same time consumption since it is also one load and one encode. load_model_as_complete(text_encoder_2, target_device=gpu) - prompt_parameters = [] - - for prompt_part in prompts: - prompt_parameters.append(encode_prompt(prompt_part, n_prompt)) - - # Processing input image - - stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Image processing ...')))) - - H, W, C = input_image.shape - height, width = find_nearest_bucket(H, W, resolution=resolution) - input_image_np = resize_and_center_crop(input_image, target_width=width, target_height=height) - - Image.fromarray(input_image_np).save(os.path.join(outputs_folder, f'{job_id}.png')) - - input_image_pt = torch.from_numpy(input_image_np).float() / 127.5 - 1 - input_image_pt = input_image_pt.permute(2, 0, 1)[None, :, None] - - # VAE encoding - - stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'VAE encoding ...')))) - - if not high_vram: - load_model_as_complete(vae, target_device=gpu) - - start_latent = vae_encode(input_image_pt, vae) - - # CLIP Vision - - stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'CLIP Vision encoding ...')))) - - if not high_vram: - load_model_as_complete(image_encoder, target_device=gpu) - - image_encoder_output = hf_clip_vision_encode(input_image_np, feature_extractor, image_encoder) - image_encoder_last_hidden_state = image_encoder_output.last_hidden_state - - # Dtype - - image_encoder_last_hidden_state = image_encoder_last_hidden_state.to(transformer.dtype) - - # Sampling - - stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Start sampling ...')))) - - rnd = torch.Generator("cpu").manual_seed(seed) - - history_latents = torch.zeros(size=(1, 16, 16 + 2 + 1, height // 8, width // 8), dtype=torch.float32).cpu() - history_pixels = None - - history_latents = torch.cat([history_latents, start_latent.to(history_latents)], dim=2) - total_generated_latent_frames = 1 - - if enable_preview: - def callback(d): - preview = d['denoised'] - preview = vae_decode_fake(preview) - - preview = (preview * 255.0).detach().cpu().numpy().clip(0, 255).astype(np.uint8) - preview = einops.rearrange(preview, 'b c t h w -> (b h) (t w) c') - - if stream.input_queue.top() == 'end': - stream.output_queue.push(('end', None)) - raise KeyboardInterrupt('User ends the task.') - - current_step = d['i'] + 1 - percentage = int(100.0 * current_step / steps) - hint = f'Sampling {current_step}/{steps}' - desc = f'Total generated frames: {int(max(0, total_generated_latent_frames * 4 - 3))}, Video length: {max(0, (total_generated_latent_frames * 4 - 3) / 30) :.2f} seconds (FPS-30), Resolution: {height}px * {width}px. The video is being extended now ...' - stream.output_queue.push(('progress', (preview, desc, make_progress_bar_html(percentage, hint)))) - return - else: - def callback(d): - return - - indices = torch.arange(0, sum([1, 16, 2, 1, latent_window_size])).unsqueeze(0) - clean_latent_indices_start, clean_latent_4x_indices, clean_latent_2x_indices, clean_latent_1x_indices, latent_indices = indices.split([1, 16, 2, 1, latent_window_size], dim=1) - clean_latent_indices = torch.cat([clean_latent_indices_start, clean_latent_1x_indices], dim=1) - - def post_process(generated_latents, total_generated_latent_frames, history_latents, high_vram, transformer, gpu, vae, history_pixels, latent_window_size, enable_preview, section_index, total_latent_sections, outputs_folder, mp4_crf, stream): - total_generated_latent_frames += int(generated_latents.shape[2]) - history_latents = torch.cat([history_latents, generated_latents.to(history_latents)], dim=2) - - if not high_vram: - offload_model_from_device_for_memory_preservation(transformer, target_device=gpu, preserved_memory_gb=8) - load_model_as_complete(vae, target_device=gpu) - - if history_pixels is None: - real_history_latents = history_latents[:, :, -total_generated_latent_frames:, :, :] - history_pixels = vae_decode(real_history_latents, vae).cpu() - else: - section_latent_frames = latent_window_size * 2 - overlapped_frames = latent_window_size * 4 - 3 - - real_history_latents = history_latents[:, :, -min(section_latent_frames, total_generated_latent_frames):, :, :] - history_pixels = soft_append_bcthw(history_pixels, vae_decode(real_history_latents, vae).cpu(), overlapped_frames) - - if not high_vram: - unload_complete_models() - - if enable_preview or section_index == total_latent_sections - 1: - output_filename = os.path.join(outputs_folder, f'{job_id}_{total_generated_latent_frames}.mp4') - - save_bcthw_as_mp4(history_pixels, output_filename, fps=30, crf=mp4_crf) - - print(f'Decoded. Current latent shape pixel shape {history_pixels.shape}') - - stream.output_queue.push(('file', output_filename)) - return [total_generated_latent_frames, history_latents, history_pixels] - - for section_index in range(total_latent_sections): - if stream.input_queue.top() == 'end': - stream.output_queue.push(('end', None)) - return - - print(f'section_index = {section_index}, total_latent_sections = {total_latent_sections}') - - if len(prompt_parameters) > 0: - [llama_vec, clip_l_pooler, llama_vec_n, clip_l_pooler_n, llama_attention_mask, llama_attention_mask_n] = prompt_parameters.pop(0) - - if not high_vram: - unload_complete_models() - move_model_to_device_with_memory_preservation(transformer, target_device=gpu, preserved_memory_gb=gpu_memory_preservation) - - if use_teacache: - transformer.initialize_teacache(enable_teacache=True, num_steps=steps) - else: - transformer.initialize_teacache(enable_teacache=False) - - clean_latents_4x, clean_latents_2x, clean_latents_1x = history_latents[:, :, -sum([16, 2, 1]):, :, :].split([16, 2, 1], dim=2) - clean_latents = torch.cat([start_latent.to(history_latents), clean_latents_1x], dim=2) - - generated_latents = sample_hunyuan( - transformer=transformer, - sampler='unipc', - width=width, - height=height, - frames=latent_window_size * 4 - 3, - real_guidance_scale=cfg, - distilled_guidance_scale=gs, - guidance_rescale=rs, - # shift=3.0, - num_inference_steps=steps, - generator=rnd, - prompt_embeds=llama_vec, - prompt_embeds_mask=llama_attention_mask, - prompt_poolers=clip_l_pooler, - negative_prompt_embeds=llama_vec_n, - negative_prompt_embeds_mask=llama_attention_mask_n, - negative_prompt_poolers=clip_l_pooler_n, - device=gpu, - dtype=torch.bfloat16, - image_embeddings=image_encoder_last_hidden_state, - latent_indices=latent_indices, - clean_latents=clean_latents, - clean_latent_indices=clean_latent_indices, - clean_latents_2x=clean_latents_2x, - clean_latent_2x_indices=clean_latent_2x_indices, - clean_latents_4x=clean_latents_4x, - clean_latent_4x_indices=clean_latent_4x_indices, - callback=callback, - ) - - [total_generated_latent_frames, history_latents, history_pixels] = post_process(generated_latents, total_generated_latent_frames, history_latents, high_vram, transformer, gpu, vae, history_pixels, latent_window_size, enable_preview, section_index, total_latent_sections, outputs_folder, mp4_crf, stream) - except: - traceback.print_exc() - - if not high_vram: - unload_complete_models( - text_encoder, text_encoder_2, image_encoder, vae, transformer - ) - - stream.output_queue.push(('end', None)) - return - -@torch.no_grad() -def worker_last_frame(input_image, prompts, n_prompt, seed, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, mp4_crf): - def encode_prompt(prompt, n_prompt): llama_vec, clip_l_pooler = encode_prompt_conds(prompt, text_encoder, text_encoder_2, tokenizer, tokenizer_2) if cfg == 1: @@ -534,63 +392,15 @@ def worker_last_frame(input_image, prompts, n_prompt, seed, resolution, total_se llama_vec, llama_attention_mask = crop_or_pad_yield_mask(llama_vec, length=512) llama_vec_n, llama_attention_mask_n = crop_or_pad_yield_mask(llama_vec_n, length=512) - llama_vec = llama_vec.to(transformer.dtype) - llama_vec_n = llama_vec_n.to(transformer.dtype) - clip_l_pooler = clip_l_pooler.to(transformer.dtype) - clip_l_pooler_n = clip_l_pooler_n.to(transformer.dtype) - return [llama_vec, clip_l_pooler, llama_vec_n, clip_l_pooler_n, llama_attention_mask, llama_attention_mask_n] - - total_latent_sections = (total_second_length * 30) / (latent_window_size * 4) - total_latent_sections = int(max(round(total_latent_sections), 1)) - - job_id = generate_timestamp() - - stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Starting ...')))) - - try: - # Clean GPU - if not high_vram: - unload_complete_models( - text_encoder, text_encoder_2, image_encoder, vae, transformer - ) - - # Text encoding - - stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Text encoding ...')))) - - if not high_vram: - fake_diffusers_current_device(text_encoder, gpu) # since we only encode one text - that is one model move and one encode, offload is same time consumption since it is also one load and one encode. - load_model_as_complete(text_encoder_2, target_device=gpu) - - prompt_parameters = [] - - for prompt_part in prompts: - prompt_parameters.append(encode_prompt(prompt_part, n_prompt)) - - # Processing input image - - stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Image processing ...')))) - - H, W, C = input_image.shape - height, width = find_nearest_bucket(H, W, resolution=resolution) - input_image_np = resize_and_center_crop(input_image, target_width=width, target_height=height) - - Image.fromarray(input_image_np).save(os.path.join(outputs_folder, f'{job_id}.png')) - - input_image_pt = torch.from_numpy(input_image_np).float() / 127.5 - 1 - input_image_pt = input_image_pt.permute(2, 0, 1)[None, :, None] - - # VAE encoding - - stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'VAE encoding ...')))) + # 20250506 pftq: Processing input video instead of image + stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Video processing ...')))) - if not high_vram: - load_model_as_complete(vae, target_device=gpu) + # 20250506 pftq: Encode video + start_latent, input_image_np, video_latents, fps, height, width, input_video_pixels, end_of_input_video_latent, end_of_input_video_image_np = video_encode(input_video, resolution, no_resize, vae, vae_batch_size=vae_batch, device=gpu) - start_latent = vae_encode(input_image_pt, vae) + #Image.fromarray(input_image_np).save(os.path.join(outputs_folder, f'{job_id}.png')) # CLIP Vision - stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'CLIP Vision encoding ...')))) if not high_vram: @@ -598,404 +408,65 @@ def worker_last_frame(input_image, prompts, n_prompt, seed, resolution, total_se image_encoder_output = hf_clip_vision_encode(input_image_np, feature_extractor, image_encoder) image_encoder_last_hidden_state = image_encoder_output.last_hidden_state - - # Dtype - - image_encoder_last_hidden_state = image_encoder_last_hidden_state.to(transformer.dtype) - - # Sampling - - stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Start sampling ...')))) - - rnd = torch.Generator("cpu").manual_seed(seed) - - history_latents = torch.zeros(size=(1, 16, 16 + 2 + 1, height // 8, width // 8), dtype=torch.float32).cpu() - history_pixels = None - - history_latents = torch.cat([start_latent.to(history_latents), history_latents], dim=2) - total_generated_latent_frames = 1 - - if enable_preview: - def callback(d): - preview = d['denoised'] - preview = vae_decode_fake(preview) - - preview = (preview * 255.0).detach().cpu().numpy().clip(0, 255).astype(np.uint8) - preview = einops.rearrange(preview, 'b c t h w -> (b h) (t w) c') - - if stream.input_queue.top() == 'end': - stream.output_queue.push(('end', None)) - raise KeyboardInterrupt('User ends the task.') - - current_step = d['i'] + 1 - percentage = int(100.0 * current_step / steps) - hint = f'Sampling {current_step}/{steps}' - desc = f'Total generated frames: {int(max(0, total_generated_latent_frames * 4 - 3))}, Video length: {max(0, (total_generated_latent_frames * 4 - 3) / 30) :.2f} seconds (FPS-30), Resolution: {height}px * {width}px. The video is being extended now ...' - stream.output_queue.push(('progress', (preview, desc, make_progress_bar_html(percentage, hint)))) - return - else: - def callback(d): - return - - indices = torch.arange(0, sum([1, 16, 2, 1, latent_window_size])).unsqueeze(0) - latent_indices, clean_latent_1x_indices, clean_latent_2x_indices, clean_latent_4x_indices, clean_latent_indices_start = indices.split([latent_window_size, 1, 2, 16, 1], dim=1) - clean_latent_indices = torch.cat([clean_latent_indices_start, clean_latent_1x_indices], dim=1) - - def post_process(generated_latents, total_generated_latent_frames, history_latents, high_vram, transformer, gpu, vae, history_pixels, latent_window_size, enable_preview, section_index, total_latent_sections, outputs_folder, mp4_crf, stream): - total_generated_latent_frames += int(generated_latents.shape[2]) - history_latents = torch.cat([generated_latents.to(history_latents), history_latents], dim=2) - - if not high_vram: - offload_model_from_device_for_memory_preservation(transformer, target_device=gpu, preserved_memory_gb=8) - load_model_as_complete(vae, target_device=gpu) - - if history_pixels is None: - real_history_latents = history_latents[:, :, :total_generated_latent_frames, :, :] - history_pixels = vae_decode(real_history_latents, vae).cpu() - else: - section_latent_frames = latent_window_size * 2 - overlapped_frames = latent_window_size * 4 - 3 - - real_history_latents = history_latents[:, :, :min(section_latent_frames, total_generated_latent_frames), :, :] - history_pixels = soft_append_bcthw(vae_decode(real_history_latents, vae).cpu(), history_pixels, overlapped_frames) - - if not high_vram: - unload_complete_models() - - #if enable_preview or section_index == 0: - if True: - output_filename = os.path.join(outputs_folder, f'{job_id}_{total_generated_latent_frames}.mp4') - - save_bcthw_as_mp4(history_pixels, output_filename, fps=30, crf=mp4_crf) - - print(f'Decoded. Current latent shape pixel shape {history_pixels.shape}') - - stream.output_queue.push(('file', output_filename)) - return [total_generated_latent_frames, history_latents, history_pixels] - - for section_index in range(total_latent_sections - 1, -1, -1): - if stream.input_queue.top() == 'end': - stream.output_queue.push(('end', None)) - return - - print(f'section_index = {section_index}, total_latent_sections = {total_latent_sections}') - - if len(prompt_parameters) > 0: - [llama_vec, clip_l_pooler, llama_vec_n, clip_l_pooler_n, llama_attention_mask, llama_attention_mask_n] = prompt_parameters.pop(len(prompt_parameters) - 1) - - if not high_vram: - unload_complete_models() - move_model_to_device_with_memory_preservation(transformer, target_device=gpu, preserved_memory_gb=gpu_memory_preservation) - - if use_teacache: - transformer.initialize_teacache(enable_teacache=True, num_steps=steps) - else: - transformer.initialize_teacache(enable_teacache=False) - - clean_latents_1x, clean_latents_2x, clean_latents_4x = history_latents[:, :, :sum([1, 2, 16]), :, :].split([1, 2, 16], dim=2) - clean_latents = torch.cat([clean_latents_1x, start_latent.to(history_latents)], dim=2) - - generated_latents = sample_hunyuan( - transformer=transformer, - sampler='unipc', - width=width, - height=height, - frames=latent_window_size * 4 - 3, - real_guidance_scale=cfg, - distilled_guidance_scale=gs, - guidance_rescale=rs, - # shift=3.0, - num_inference_steps=steps, - generator=rnd, - prompt_embeds=llama_vec, - prompt_embeds_mask=llama_attention_mask, - prompt_poolers=clip_l_pooler, - negative_prompt_embeds=llama_vec_n, - negative_prompt_embeds_mask=llama_attention_mask_n, - negative_prompt_poolers=clip_l_pooler_n, - device=gpu, - dtype=torch.bfloat16, - image_embeddings=image_encoder_last_hidden_state, - latent_indices=latent_indices, - clean_latents=clean_latents, - clean_latent_indices=clean_latent_indices, - clean_latents_2x=clean_latents_2x, - clean_latent_2x_indices=clean_latent_2x_indices, - clean_latents_4x=clean_latents_4x, - clean_latent_4x_indices=clean_latent_4x_indices, - callback=callback, + start_embedding = image_encoder_last_hidden_state + + end_of_input_video_output = hf_clip_vision_encode(end_of_input_video_image_np, feature_extractor, image_encoder) + end_of_input_video_last_hidden_state = end_of_input_video_output.last_hidden_state + end_of_input_video_embedding = end_of_input_video_last_hidden_state + + # 20250507 pftq: Process end frame if provided + end_latent = None + end_clip_embedding = None + if end_frame is not None: + stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'End frame encoding ...')))) + end_latent, end_clip_embedding, _ = image_encode( + end_frame, target_width=width, target_height=height, vae=vae, + image_encoder=image_encoder, feature_extractor=feature_extractor, device=gpu ) - [total_generated_latent_frames, history_latents, history_pixels] = post_process(generated_latents, total_generated_latent_frames, history_latents, high_vram, transformer, gpu, vae, history_pixels, latent_window_size, enable_preview, section_index, total_latent_sections, outputs_folder, mp4_crf, stream) - if section_index < total_latent_sections - 1: - break - except: - traceback.print_exc() - - if not high_vram: - unload_complete_models( - text_encoder, text_encoder_2, image_encoder, vae, transformer - ) - - stream.output_queue.push(('end', None)) - return - -def get_duration(input_image, image_position, prompt, generation_mode, n_prompt, randomize_seed, seed, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, mp4_crf): - global total_second_length_debug_value - - if total_second_length_debug_value is not None: - return min(total_second_length_debug_value * 60 * 10, 600) - return total_second_length * 60 * (0.9 if use_teacache else 1.5) * (1 + ((steps - 25) / 100)) - -@spaces.GPU(duration=get_duration) -def process(input_image, - image_position=0, - prompt="", - generation_mode="image", - n_prompt="", - randomize_seed=True, - seed=31337, - resolution=640, - total_second_length=5, - latent_window_size=9, - steps=25, - cfg=1.0, - gs=10.0, - rs=0.0, - gpu_memory_preservation=6, - enable_preview=True, - use_teacache=False, - mp4_crf=16 - ): - start = time.time() - global stream, input_image_debug_value, prompt_debug_value, total_second_length_debug_value - - if input_image_debug_value is not None or prompt_debug_value is not None or total_second_length_debug_value is not None: - input_image = input_image_debug_value - prompt = prompt_debug_value - total_second_length = total_second_length_debug_value - input_image_debug_value = prompt_debug_value = total_second_length_debug_value = None - - if torch.cuda.device_count() == 0: - gr.Warning('Set this space to GPU config to make it work.') - yield gr.update(), gr.update(), gr.update(), gr.update(), gr.update(), gr.update() - return - - if randomize_seed: - seed = random.randint(0, np.iinfo(np.int32).max) - - prompts = prompt.split(";") - - # assert input_image is not None, 'No input image!' - if generation_mode == "text": - default_height, default_width = 640, 640 - input_image = np.ones((default_height, default_width, 3), dtype=np.uint8) * 255 - print("No input image provided. Using a blank white image.") - - yield None, None, '', '', gr.update(interactive=False), gr.update(interactive=True) - - stream = AsyncStream() - - async_run(worker_last_frame if image_position == 100 else worker, input_image, prompts, n_prompt, seed, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, mp4_crf) - - output_filename = None - - while True: - flag, data = stream.output_queue.next() - - if flag == 'file': - output_filename = data - yield output_filename, gr.update(), gr.update(), gr.update(), gr.update(interactive=False), gr.update(interactive=True) - - if flag == 'progress': - preview, desc, html = data - yield gr.update(), gr.update(visible=True, value=preview), desc, html, gr.update(interactive=False), gr.update(interactive=True) - - if flag == 'end': - end = time.time() - secondes = int(end - start) - minutes = math.floor(secondes / 60) - secondes = secondes - (minutes * 60) - hours = math.floor(minutes / 60) - minutes = minutes - (hours * 60) - yield output_filename, gr.update(visible=False), gr.update(), "The video has been generated in " + \ - ((str(hours) + " h, ") if hours != 0 else "") + \ - ((str(minutes) + " min, ") if hours != 0 or minutes != 0 else "") + \ - str(secondes) + " sec. " + \ - "You can upscale the result with RIFE. To make all your generated scenes consistent, you can then apply a face swap on the main character.", gr.update(interactive=True), gr.update(interactive=False) - break - -# 20250506 pftq: Modified worker to accept video input and clean frame count -@spaces.GPU() -@torch.no_grad() -def worker_video(input_video, prompts, n_prompt, seed, batch, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, no_resize, mp4_crf, num_clean_frames, vae_batch): - def encode_prompt(prompt, n_prompt): - llama_vec, clip_l_pooler = encode_prompt_conds(prompt, text_encoder, text_encoder_2, tokenizer, tokenizer_2) - - if cfg == 1: - llama_vec_n, clip_l_pooler_n = torch.zeros_like(llama_vec), torch.zeros_like(clip_l_pooler) - else: - llama_vec_n, clip_l_pooler_n = encode_prompt_conds(n_prompt, text_encoder, text_encoder_2, tokenizer, tokenizer_2) - - llama_vec, llama_attention_mask = crop_or_pad_yield_mask(llama_vec, length=512) - llama_vec_n, llama_attention_mask_n = crop_or_pad_yield_mask(llama_vec_n, length=512) - + # Dtype llama_vec = llama_vec.to(transformer.dtype) llama_vec_n = llama_vec_n.to(transformer.dtype) clip_l_pooler = clip_l_pooler.to(transformer.dtype) clip_l_pooler_n = clip_l_pooler_n.to(transformer.dtype) - return [llama_vec, clip_l_pooler, llama_vec_n, clip_l_pooler_n, llama_attention_mask, llama_attention_mask_n] - - stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Starting ...')))) - - try: - # Clean GPU - if not high_vram: - unload_complete_models( - text_encoder, text_encoder_2, image_encoder, vae, transformer - ) - - # Text encoding - stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Text encoding ...')))) - - if not high_vram: - fake_diffusers_current_device(text_encoder, gpu) # since we only encode one text - that is one model move and one encode, offload is same time consumption since it is also one load and one encode. - load_model_as_complete(text_encoder_2, target_device=gpu) - - prompt_parameters = [] - - for prompt_part in prompts: - prompt_parameters.append(encode_prompt(prompt_part, n_prompt)) - - # 20250506 pftq: Processing input video instead of image - stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Video processing ...')))) - - # 20250506 pftq: Encode video - start_latent, input_image_np, video_latents, fps, height, width, input_video_pixels = video_encode(input_video, resolution, no_resize, vae, vae_batch_size=vae_batch, device=gpu) - - # CLIP Vision - stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'CLIP Vision encoding ...')))) - - if not high_vram: - load_model_as_complete(image_encoder, target_device=gpu) - - image_encoder_output = hf_clip_vision_encode(input_image_np, feature_extractor, image_encoder) - image_encoder_last_hidden_state = image_encoder_output.last_hidden_state - - # Dtype image_encoder_last_hidden_state = image_encoder_last_hidden_state.to(transformer.dtype) + end_of_input_video_embedding = end_of_input_video_embedding.to(transformer.dtype) + # 20250509 pftq: Restored original placement of total_latent_sections after video_encode total_latent_sections = (total_second_length * fps) / (latent_window_size * 4) total_latent_sections = int(max(round(total_latent_sections), 1)) - if enable_preview: - def callback(d): - preview = d['denoised'] - preview = vae_decode_fake(preview) - - preview = (preview * 255.0).detach().cpu().numpy().clip(0, 255).astype(np.uint8) - preview = einops.rearrange(preview, 'b c t h w -> (b h) (t w) c') - - if stream.input_queue.top() == 'end': - stream.output_queue.push(('end', None)) - raise KeyboardInterrupt('User ends the task.') - - current_step = d['i'] + 1 - percentage = int(100.0 * current_step / steps) - hint = f'Sampling {current_step}/{steps}' - desc = f'Total frames: {int(max(0, total_generated_latent_frames * 4 - 3))}, Video length: {max(0, (total_generated_latent_frames * 4 - 3) / fps) :.2f} seconds (FPS-{fps}), Resolution: {height}px * {width}px, Seed: {seed}, Video {idx+1} of {batch}. The video is generating part {section_index+1} of {total_latent_sections}...' - stream.output_queue.push(('progress', (preview, desc, make_progress_bar_html(percentage, hint)))) - return - else: - def callback(d): - return - - def compute_latent(history_latents, latent_window_size, num_clean_frames, start_latent): - # 20250506 pftq: Use user-specified number of context frames, matching original allocation for num_clean_frames=2 - available_frames = history_latents.shape[2] # Number of latent frames - max_pixel_frames = min(latent_window_size * 4 - 3, available_frames * 4) # Cap at available pixel frames - adjusted_latent_frames = max(1, (max_pixel_frames + 3) // 4) # Convert back to latent frames - # Adjust num_clean_frames to match original behavior: num_clean_frames=2 means 1 frame for clean_latents_1x - effective_clean_frames = max(0, num_clean_frames - 1) - effective_clean_frames = min(effective_clean_frames, available_frames - 2) if available_frames > 2 else 0 # 20250507 pftq: changed 1 to 2 for edge case for <=1 sec videos - num_2x_frames = min(2, max(1, available_frames - effective_clean_frames - 1)) if available_frames > effective_clean_frames + 1 else 0 # 20250507 pftq: subtracted 1 for edge case for <=1 sec videos - num_4x_frames = min(16, max(1, available_frames - effective_clean_frames - num_2x_frames)) if available_frames > effective_clean_frames + num_2x_frames else 0 # 20250507 pftq: Edge case for <=1 sec - - total_context_frames = num_4x_frames + num_2x_frames + effective_clean_frames - total_context_frames = min(total_context_frames, available_frames) # 20250507 pftq: Edge case for <=1 sec videos - - indices = torch.arange(0, sum([1, num_4x_frames, num_2x_frames, effective_clean_frames, adjusted_latent_frames])).unsqueeze(0) # 20250507 pftq: latent_window_size to adjusted_latent_frames for edge case for <=1 sec videos - clean_latent_indices_start, clean_latent_4x_indices, clean_latent_2x_indices, clean_latent_1x_indices, latent_indices = indices.split( - [1, num_4x_frames, num_2x_frames, effective_clean_frames, adjusted_latent_frames], dim=1 # 20250507 pftq: latent_window_size to adjusted_latent_frames for edge case for <=1 sec videos - ) - clean_latent_indices = torch.cat([clean_latent_indices_start, clean_latent_1x_indices], dim=1) - - # 20250506 pftq: Split history_latents dynamically based on available frames - fallback_frame_count = 2 # 20250507 pftq: Changed 0 to 2 Edge case for <=1 sec videos - context_frames = clean_latents_4x = clean_latents_2x = clean_latents_1x = history_latents[:, :, :fallback_frame_count, :, :] - - if total_context_frames > 0: - context_frames = history_latents[:, :, -total_context_frames:, :, :] - split_sizes = [num_4x_frames, num_2x_frames, effective_clean_frames] - split_sizes = [s for s in split_sizes if s > 0] # Remove zero sizes - if split_sizes: - splits = context_frames.split(split_sizes, dim=2) - split_idx = 0 - - if num_4x_frames > 0: - clean_latents_4x = splits[split_idx] - split_idx = 1 - if clean_latents_4x.shape[2] < 2: # 20250507 pftq: edge case for <=1 sec videos - print("Edge case for <=1 sec videos 4x") - clean_latents_4x = clean_latents_4x.expand(-1, -1, 2, -1, -1) - - if num_2x_frames > 0 and split_idx < len(splits): - clean_latents_2x = splits[split_idx] - if clean_latents_2x.shape[2] < 2: # 20250507 pftq: edge case for <=1 sec videos - print("Edge case for <=1 sec videos 2x") - clean_latents_2x = clean_latents_2x.expand(-1, -1, 2, -1, -1) - split_idx += 1 - elif clean_latents_2x.shape[2] < 2: # 20250507 pftq: edge case for <=1 sec videos - clean_latents_2x = clean_latents_4x - - if effective_clean_frames > 0 and split_idx < len(splits): - clean_latents_1x = splits[split_idx] - - clean_latents = torch.cat([start_latent.to(history_latents), clean_latents_1x], dim=2) - - # 20250507 pftq: Fix for <=1 sec videos. - max_frames = min(latent_window_size * 4 - 3, history_latents.shape[2] * 4) - return [max_frames, clean_latents, clean_latents_2x, clean_latents_4x, latent_indices, clean_latents, clean_latent_indices, clean_latent_2x_indices, clean_latent_4x_indices] - for idx in range(batch): if batch > 1: print(f"Beginning video {idx+1} of {batch} with seed {seed} ") - #job_id = generate_timestamp() - job_id = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")+f"_framepackf1-videoinput_{width}-{total_second_length}sec_seed-{seed}_steps-{steps}_distilled-{gs}_cfg-{cfg}" # 20250506 pftq: easier to read timestamp and filename + job_id = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")+f"_framepack-videoinput-endframe_{width}-{total_second_length}sec_seed-{seed}_steps-{steps}_distilled-{gs}_cfg-{cfg}" - # Sampling stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Start sampling ...')))) rnd = torch.Generator("cpu").manual_seed(seed) - # 20250506 pftq: Initialize history_latents with video latents history_latents = video_latents.cpu() - total_generated_latent_frames = history_latents.shape[2] - # 20250506 pftq: Initialize history_pixels to fix UnboundLocalError history_pixels = None + total_generated_latent_frames = 0 previous_video = None - for section_index in range(total_latent_sections): + + # 20250509 Generate backwards with end frame for better end frame anchoring + if total_latent_sections > 4: + latent_paddings = [3] + [2] * (total_latent_sections - 3) + [1, 0] + else: + latent_paddings = list(reversed(range(total_latent_sections))) + + for section_index, latent_padding in enumerate(latent_paddings): + is_start_of_video = latent_padding == 0 + is_end_of_video = latent_padding == latent_paddings[0] + latent_padding_size = latent_padding * latent_window_size + if stream.input_queue.top() == 'end': stream.output_queue.push(('end', None)) return - print(f'section_index = {section_index}, total_latent_sections = {total_latent_sections}') - - if len(prompt_parameters) > 0: - [llama_vec, clip_l_pooler, llama_vec_n, clip_l_pooler_n, llama_attention_mask, llama_attention_mask_n] = prompt_parameters.pop(0) - if not high_vram: unload_complete_models() move_model_to_device_with_memory_preservation(transformer, target_device=gpu, preserved_memory_gb=gpu_memory_preservation) @@ -1005,8 +476,87 @@ def worker_video(input_video, prompts, n_prompt, seed, batch, resolution, total_ else: transformer.initialize_teacache(enable_teacache=False) - [max_frames, clean_latents, clean_latents_2x, clean_latents_4x, latent_indices, clean_latents, clean_latent_indices, clean_latent_2x_indices, clean_latent_4x_indices] = compute_latent(history_latents, latent_window_size, num_clean_frames, start_latent) + def callback(d): + try: + preview = d['denoised'] + preview = vae_decode_fake(preview) + preview = (preview * 255.0).detach().cpu().numpy().clip(0, 255).astype(np.uint8) + preview = einops.rearrange(preview, 'b c t h w -> (b h) (t w) c') + if stream.input_queue.top() == 'end': + stream.output_queue.push(('end', None)) + raise KeyboardInterrupt('User ends the task.') + current_step = d['i'] + 1 + percentage = int(100.0 * current_step / steps) + hint = f'Sampling {current_step}/{steps}' + desc = f'Total frames: {int(max(0, total_generated_latent_frames * 4 - 3))}, Video length: {max(0, (total_generated_latent_frames * 4 - 3) / fps) :.2f} seconds (FPS-{fps}), Seed: {seed}, Video {idx+1} of {batch}. Generating part {total_latent_sections - section_index} of {total_latent_sections} backward...' + stream.output_queue.push(('progress', (preview, desc, make_progress_bar_html(percentage, hint)))) + except ConnectionResetError as e: + print(f"Suppressed ConnectionResetError in callback: {e}") + return + # 20250509 pftq: Dynamic frame allocation like original num_clean_frames, fix split error + available_frames = video_latents.shape[2] if is_start_of_video else history_latents.shape[2] + if is_start_of_video: + effective_clean_frames = 1 # avoid jumpcuts from input video + else: + effective_clean_frames = max(0, num_clean_frames - 1) if num_clean_frames > 1 else 1 + clean_latent_pre_frames = effective_clean_frames + num_2x_frames = min(2, max(1, available_frames - clean_latent_pre_frames - 1)) if available_frames > clean_latent_pre_frames + 1 else 1 + num_4x_frames = min(16, max(1, available_frames - clean_latent_pre_frames - num_2x_frames)) if available_frames > clean_latent_pre_frames + num_2x_frames else 1 + total_context_frames = num_2x_frames + num_4x_frames + total_context_frames = min(total_context_frames, available_frames - clean_latent_pre_frames) + + # 20250511 pftq: Dynamically adjust post_frames based on clean_latents_post + post_frames = 1 if is_end_of_video and end_latent is not None else effective_clean_frames # 20250511 pftq: Single frame for end_latent, otherwise padding causes still image + indices = torch.arange(0, clean_latent_pre_frames + latent_padding_size + latent_window_size + post_frames + num_2x_frames + num_4x_frames).unsqueeze(0) + clean_latent_indices_pre, blank_indices, latent_indices, clean_latent_indices_post, clean_latent_2x_indices, clean_latent_4x_indices = indices.split( + [clean_latent_pre_frames, latent_padding_size, latent_window_size, post_frames, num_2x_frames, num_4x_frames], dim=1 + ) + clean_latent_indices = torch.cat([clean_latent_indices_pre, clean_latent_indices_post], dim=1) + + # 20250509 pftq: Split context frames dynamically for 2x and 4x only + context_frames = history_latents[:, :, -(total_context_frames + clean_latent_pre_frames):-clean_latent_pre_frames, :, :] if total_context_frames > 0 else history_latents[:, :, :1, :, :] + split_sizes = [num_4x_frames, num_2x_frames] + split_sizes = [s for s in split_sizes if s > 0] + if split_sizes and context_frames.shape[2] >= sum(split_sizes): + splits = context_frames.split(split_sizes, dim=2) + split_idx = 0 + clean_latents_4x = splits[split_idx] if num_4x_frames > 0 else history_latents[:, :, :1, :, :] + split_idx += 1 if num_4x_frames > 0 else 0 + clean_latents_2x = splits[split_idx] if num_2x_frames > 0 and split_idx < len(splits) else history_latents[:, :, :1, :, :] + else: + clean_latents_4x = clean_latents_2x = history_latents[:, :, :1, :, :] + + clean_latents_pre = video_latents[:, :, -min(effective_clean_frames, video_latents.shape[2]):].to(history_latents) # smoother motion but jumpcuts if end frame is too different, must change clean_latent_pre_frames to effective_clean_frames also + clean_latents_post = history_latents[:, :, :min(effective_clean_frames, history_latents.shape[2]), :, :] # smoother motion, must change post_frames to effective_clean_frames also + + if is_end_of_video: + clean_latents_post = torch.zeros_like(end_of_input_video_latent).to(history_latents) + + # 20250509 pftq: handle end frame if available + if end_latent is not None: + #current_end_frame_weight = end_frame_weight * (latent_padding / latent_paddings[0]) + #current_end_frame_weight = current_end_frame_weight * 0.5 + 0.5 + current_end_frame_weight = end_frame_weight # changing this over time introduces discontinuity + # 20250511 pftq: Removed end frame weight adjustment as it has no effect + image_encoder_last_hidden_state = (1 - current_end_frame_weight) * end_of_input_video_embedding + end_clip_embedding * current_end_frame_weight + image_encoder_last_hidden_state = image_encoder_last_hidden_state.to(transformer.dtype) + + # 20250511 pftq: Use end_latent only + if is_end_of_video: + clean_latents_post = end_latent.to(history_latents)[:, :, :1, :, :] # Ensure single frame + + # 20250511 pftq: Pad clean_latents_pre to match clean_latent_pre_frames if needed + if clean_latents_pre.shape[2] < clean_latent_pre_frames: + clean_latents_pre = clean_latents_pre.repeat(1, 1, clean_latent_pre_frames // clean_latents_pre.shape[2], 1, 1) + # 20250511 pftq: Pad clean_latents_post to match post_frames if needed + if clean_latents_post.shape[2] < post_frames: + clean_latents_post = clean_latents_post.repeat(1, 1, post_frames // clean_latents_post.shape[2], 1, 1) + + clean_latents = torch.cat([clean_latents_pre, clean_latents_post], dim=2) + + max_frames = min(latent_window_size * 4 - 3, history_latents.shape[2] * 4) + print(f"Generating video {idx+1} of {batch} with seed {seed}, part {total_latent_sections - section_index} of {total_latent_sections} backward") generated_latents = sample_hunyuan( transformer=transformer, sampler='unipc', @@ -1037,49 +587,69 @@ def worker_video(input_video, prompts, n_prompt, seed, batch, resolution, total_ callback=callback, ) + if is_start_of_video: + generated_latents = torch.cat([video_latents[:, :, -1:].to(generated_latents), generated_latents], dim=2) + total_generated_latent_frames += int(generated_latents.shape[2]) - history_latents = torch.cat([history_latents, generated_latents.to(history_latents)], dim=2) + history_latents = torch.cat([generated_latents.to(history_latents), history_latents], dim=2) if not high_vram: offload_model_from_device_for_memory_preservation(transformer, target_device=gpu, preserved_memory_gb=8) load_model_as_complete(vae, target_device=gpu) - real_history_latents = history_latents[:, :, -total_generated_latent_frames:, :, :] - + real_history_latents = history_latents[:, :, :total_generated_latent_frames, :, :] if history_pixels is None: history_pixels = vae_decode(real_history_latents, vae).cpu() else: - section_latent_frames = latent_window_size * 2 - overlapped_frames = min(latent_window_size * 4 - 3, history_pixels.shape[2]) - - history_pixels = soft_append_bcthw(history_pixels, vae_decode(real_history_latents[:, :, -section_latent_frames:], vae).cpu(), overlapped_frames) + section_latent_frames = (latent_window_size * 2 + 1) if is_start_of_video else (latent_window_size * 2) + overlapped_frames = latent_window_size * 4 - 3 + current_pixels = vae_decode(real_history_latents[:, :, :section_latent_frames], vae).cpu() + history_pixels = soft_append_bcthw(current_pixels, history_pixels, overlapped_frames) if not high_vram: unload_complete_models() - if enable_preview or section_index == total_latent_sections - 1: - output_filename = os.path.join(outputs_folder, f'{job_id}_{total_generated_latent_frames}.mp4') - - # 20250506 pftq: Use input video FPS for output - save_bcthw_as_mp4(history_pixels, output_filename, fps=fps, crf=mp4_crf) - print(f"Latest video saved: {output_filename}") - # 20250508 pftq: Save prompt to mp4 metadata comments - set_mp4_comments_imageio_ffmpeg(output_filename, f"Prompt: {prompts} | Negative Prompt: {n_prompt}"); - print(f"Prompt saved to mp4 metadata comments: {output_filename}") - - # 20250506 pftq: Clean up previous partial files - if previous_video is not None and os.path.exists(previous_video): - try: - os.remove(previous_video) - print(f"Previous partial video deleted: {previous_video}") - except Exception as e: - print(f"Error deleting previous partial video {previous_video}: {e}") - previous_video = output_filename - - print(f'Decoded. Current latent shape {real_history_latents.shape}; pixel shape {history_pixels.shape}') - - stream.output_queue.push(('file', output_filename)) + output_filename = os.path.join(outputs_folder, f'{job_id}_{total_generated_latent_frames}.mp4') + save_bcthw_as_mp4(history_pixels, output_filename, fps=fps, crf=mp4_crf) + print(f"Latest video saved: {output_filename}") + set_mp4_comments_imageio_ffmpeg(output_filename, f"Prompt: {prompt} | Negative Prompt: {n_prompt}") + print(f"Prompt saved to mp4 metadata comments: {output_filename}") + + if previous_video is not None and os.path.exists(previous_video): + try: + os.remove(previous_video) + print(f"Previous partial video deleted: {previous_video}") + except Exception as e: + print(f"Error deleting previous partial video {previous_video}: {e}") + previous_video = output_filename + + print(f'Decoded. Current latent shape {real_history_latents.shape}; pixel shape {history_pixels.shape}') + stream.output_queue.push(('file', output_filename)) + + if is_start_of_video: + break + + history_pixels = torch.cat([input_video_pixels, history_pixels], dim=2) + + output_filename = os.path.join(outputs_folder, f'{job_id}_final.mp4') + save_bcthw_as_mp4(history_pixels, output_filename, fps=fps, crf=mp4_crf) + print(f"Final video with input blend saved: {output_filename}") + set_mp4_comments_imageio_ffmpeg(output_filename, f"Prompt: {prompt} | Negative Prompt: {n_prompt}") + print(f"Prompt saved to mp4 metadata comments: {output_filename}") + stream.output_queue.push(('file', output_filename)) + + if previous_video is not None and os.path.exists(previous_video): + try: + os.remove(previous_video) + print(f"Previous partial video deleted: {previous_video}") + except Exception as e: + print(f"Error deleting previous partial video {previous_video}: {e}") + previous_video = output_filename + + print(f'Decoded. Current latent shape {real_history_latents.shape}; pixel shape {history_pixels.shape}') + stream.output_queue.push(('file', output_filename)) + seed = (seed + 1) % np.iinfo(np.int32).max except: @@ -1093,34 +663,38 @@ def worker_video(input_video, prompts, n_prompt, seed, batch, resolution, total_ stream.output_queue.push(('end', None)) return -def get_duration_video(input_video, prompt, n_prompt, randomize_seed, seed, batch, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, no_resize, mp4_crf, num_clean_frames, vae_batch): +# 20250506 pftq: Modified process to pass clean frame count, etc +def get_duration( + input_video, end_frame, end_frame_weight, prompt, n_prompt, + randomize_seed, + seed, batch, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, + no_resize, mp4_crf, num_clean_frames, vae_batch): global total_second_length_debug_value if total_second_length_debug_value is not None: - return min(total_second_length_debug_value * 60 * 10, 600) - return total_second_length * 60 * (0.9 if use_teacache else 2.3) * (1 + ((steps - 25) / 100)) + return min(total_second_length_debug_value * 60 * 2, 600) + return total_second_length * 60 * 2 -# 20250506 pftq: Modified process to pass clean frame count, etc from video_encode -@spaces.GPU(duration=get_duration_video) -def process_video(input_video, prompt, n_prompt, randomize_seed, seed, batch, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, no_resize, mp4_crf, num_clean_frames, vae_batch): - start = time.time() +@spaces.GPU(duration=get_duration) +def process( + input_video, end_frame, end_frame_weight, prompt, n_prompt, + randomize_seed, + seed, batch, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, + no_resize, mp4_crf, num_clean_frames, vae_batch): global stream, high_vram, input_video_debug_value, prompt_debug_value, total_second_length_debug_value + if torch.cuda.device_count() == 0: + gr.Warning('Set this space to GPU config to make it work.') + return None, None, None, None, None, None + if input_video_debug_value is not None or prompt_debug_value is not None or total_second_length_debug_value is not None: input_video = input_video_debug_value prompt = prompt_debug_value total_second_length = total_second_length_debug_value input_video_debug_value = prompt_debug_value = total_second_length_debug_value = None - if torch.cuda.device_count() == 0: - gr.Warning('Set this space to GPU config to make it work.') - yield gr.update(), gr.update(), gr.update(), gr.update(), gr.update(), gr.update() - return - if randomize_seed: seed = random.randint(0, np.iinfo(np.int32).max) - prompts = prompt.split(";") - # 20250506 pftq: Updated assertion for video input assert input_video is not None, 'No input video!' @@ -1142,7 +716,7 @@ def process_video(input_video, prompt, n_prompt, randomize_seed, seed, batch, re stream = AsyncStream() # 20250506 pftq: Pass num_clean_frames, vae_batch, etc - async_run(worker_video, input_video, prompts, n_prompt, seed, batch, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, no_resize, mp4_crf, num_clean_frames, vae_batch) + async_run(worker, input_video, end_frame, end_frame_weight, prompt, n_prompt, seed, batch, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, no_resize, mp4_crf, num_clean_frames, vae_batch) output_filename = None @@ -1159,141 +733,69 @@ def process_video(input_video, prompt, n_prompt, randomize_seed, seed, batch, re yield output_filename, gr.update(visible=True, value=preview), desc, html, gr.update(interactive=False), gr.update(interactive=True) # 20250506 pftq: Keep refreshing the video in case it got hidden when the tab was in the background if flag == 'end': - end = time.time() - secondes = int(end - start) - minutes = math.floor(secondes / 60) - secondes = secondes - (minutes * 60) - hours = math.floor(minutes / 60) - minutes = minutes - (hours * 60) - yield output_filename, gr.update(visible=False), desc + \ - " The video has been generated in " + \ - ((str(hours) + " h, ") if hours != 0 else "") + \ - ((str(minutes) + " min, ") if hours != 0 or minutes != 0 else "") + \ - str(secondes) + " sec. " + \ - " Video complete. You can upscale the result with RIFE. To make all your generated scenes consistent, you can then apply a face swap on the main character.", '', gr.update(interactive=True), gr.update(interactive=False) + yield output_filename, gr.update(visible=False), desc+' Video complete.', '', gr.update(interactive=True), gr.update(interactive=False) break def end_process(): stream.input_queue.push('end') -timeless_prompt_value = [""] -timed_prompts = {} - -def handle_prompt_number_change(): - timed_prompts.clear() - return [] - -def handle_timeless_prompt_change(timeless_prompt): - timeless_prompt_value[0] = timeless_prompt - return refresh_prompt() - -def handle_timed_prompt_change(timed_prompt_id, timed_prompt): - timed_prompts[timed_prompt_id] = timed_prompt - return refresh_prompt() - -def refresh_prompt(): - dict_values = {k: v for k, v in timed_prompts.items()} - sorted_dict_values = sorted(dict_values.items(), key=lambda x: x[0]) - array = [] - for sorted_dict_value in sorted_dict_values: - if timeless_prompt_value[0] is not None and len(timeless_prompt_value[0]) and sorted_dict_value[1] is not None and len(sorted_dict_value[1]): - array.append(timeless_prompt_value[0] + ". " + sorted_dict_value[1]) - else: - array.append(timeless_prompt_value[0] + sorted_dict_value[1]) - print(str(array)) - return ";".join(array) - -title_html = """ -

FramePack

-
Generate videos from text/image/video freely, without account, without watermark and download it
 -
- -

This space is ready to work on ZeroGPU and GPU and has been tested successfully on ZeroGPU. Please leave a message in discussion if you encounter issues.

- """ - -js = """ -function createGradioAnimation() { - window.addEventListener("beforeunload", function (e) { - if (document.getElementById('end-button') && !document.getElementById('end-button').disabled) { - var confirmationMessage = 'A process is still running. ' - + 'If you leave before saving, your changes will be lost.'; - - (e || window.event).returnValue = confirmationMessage; - } - return confirmationMessage; - }); - return 'Animation created'; -} -""" - css = make_progress_bar_css() -block = gr.Blocks(css=css, js=js).queue() +block = gr.Blocks(css=css).queue( + max_size=10 # 20250507 pftq: Limit queue size +) with block: if torch.cuda.device_count() == 0: with gr.Row(): gr.HTML(""" -

⚠️To use FramePack, duplicate this space and set a GPU with 30 GB VRAM. - - You can't use FramePack directly here because this space runs on a CPU, which is not enough for FramePack. Please provide feedback if you have issues. +

⚠️To use FramePack, duplicate this space and set a GPU with 30 GB VRAM. + + You can't use FramePack directly here because this space runs on a CPU, which is not enough for FramePack. Please provide feedback if you have issues.

""") - gr.HTML(title_html) - local_storage = gr.BrowserState(default_local_storage) + # 20250506 pftq: Updated title to reflect video input functionality + gr.Markdown('# Framepack with Video Input (Video Extension) + End Frame') with gr.Row(): with gr.Column(): - generation_mode = gr.Radio([["Text-to-Video", "text"], ["Image-to-Video", "image"], ["Video Extension", "video"]], elem_id="generation-mode", label="Generation mode", value = "image") - text_to_video_hint = gr.HTML("I discourage to use the Text-to-Video feature. You should rather generate an image with Flux and use Image-to-Video. You will save time.") - input_image = gr.Image(sources='upload', type="numpy", label="Image", height=320) - image_position = gr.Slider(label="Image position", minimum=0, maximum=100, value=0, step=100, info='0=Video start; 100=Video end') - input_video = gr.Video(sources='upload', label="Input Video", height=320) - timeless_prompt = gr.Textbox(label="Timeless prompt", info='Used on the whole duration of the generation', value='', placeholder="The creature starts to move, fast motion, fixed camera, focus motion, consistent arm, consistent position, mute colors, insanely detailed") - prompt_number = gr.Slider(label="Timed prompt number", minimum=0, maximum=1000, value=0, step=1, info='Prompts will automatically appear') - - @gr.render(inputs=prompt_number) - def show_split(prompt_number): - for digit in range(prompt_number): - timed_prompt_id = gr.Textbox(value="timed_prompt_" + str(digit), visible=False) - timed_prompt = gr.Textbox(label="Timed prompt #" + str(digit + 1), elem_id="timed_prompt_" + str(digit), value="") - timed_prompt.change(fn=handle_timed_prompt_change, inputs=[timed_prompt_id, timed_prompt], outputs=[final_prompt]) - - final_prompt = gr.Textbox(label="Final prompt", value='', info='Use ; to separate in time') - prompt_hint = gr.HTML("Video extension barely follows the prompt; to force to follow the prompt, you have to set the Distilled CFG Scale to 3.0 and the Context Frames to 2 but the video quality will be poor.") - total_second_length = gr.Slider(label="Video Length to Generate (seconds)", minimum=1, maximum=120, value=2, step=0.1) + + # 20250506 pftq: Changed to Video input from Image + with gr.Row(): + input_video = gr.Video(sources='upload', label="Input Video", height=320) + with gr.Column(): + # 20250507 pftq: Added end_frame + weight + end_frame = gr.Image(sources='upload', type="numpy", label="End Frame (Optional) - Reduce context frames if very different from input video or if it is jumpcutting/slowing to still image.", height=320) + end_frame_weight = gr.Slider(label="End Frame Weight", minimum=0.0, maximum=1.0, value=1.0, step=0.01, info='Reduce to treat more as a reference image; no effect') + + prompt = gr.Textbox(label="Prompt", value='') with gr.Row(): - start_button = gr.Button(value="🎥 Generate", variant="primary") - start_button_video = gr.Button(value="🎥 Generate", variant="primary") - end_button = gr.Button(elem_id="end-button", value="End Generation", variant="stop", interactive=False) + start_button = gr.Button(value="Start Generation", variant="primary") + end_button = gr.Button(value="End Generation", variant="stop", interactive=False) with gr.Accordion("Advanced settings", open=False): - enable_preview = gr.Checkbox(label='Enable preview', value=True, info='Display a preview around each second generated but it costs 2 sec. for each second generated.') - use_teacache = gr.Checkbox(label='Use TeaCache', value=False, info='Faster speed and no break in brightness, but often makes hands and fingers slightly worse.') + with gr.Row(): + use_teacache = gr.Checkbox(label='Use TeaCache', value=True, info='Faster speed, but often makes hands and fingers slightly worse.') + no_resize = gr.Checkbox(label='Force Original Video Resolution (No Resizing)', value=False, info='Might run out of VRAM (720p requires > 24GB VRAM).') - n_prompt = gr.Textbox(label="Negative Prompt", value="Missing arm, unrealistic position, impossible contortion, visible bone, muscle contraction, blurred, blurry", info='Requires using normal CFG (undistilled) instead of Distilled (set Distilled=1 and CFG > 1).') + randomize_seed = gr.Checkbox(label='Randomize seed', value=True, info='If checked, the seed is always different') + seed = gr.Slider(label="Seed", minimum=0, maximum=np.iinfo(np.int32).max, step=1, randomize=True) - latent_window_size = gr.Slider(label="Latent Window Size", minimum=1, maximum=33, value=9, step=1, info='Generate more frames at a time (larger chunks). Less degradation and better blending but higher VRAM cost. Should not change.') - steps = gr.Slider(label="Steps", minimum=1, maximum=100, value=25, step=1, info='Increase for more quality, especially if using high non-distilled CFG. If your animation has very few motion, you may have brutal brightness change; this can be fixed increasing the steps.') + batch = gr.Slider(label="Batch Size (Number of Videos)", minimum=1, maximum=1000, value=1, step=1, info='Generate multiple videos each with a different seed.') - with gr.Row(): - no_resize = gr.Checkbox(label='Force Original Video Resolution (no Resizing)', value=False, info='Might run out of VRAM (720p requires > 24GB VRAM).') - resolution = gr.Dropdown([ - ["409,600 px (working)", 640], - ["451,584 px (working)", 672], - ["495,616 px (VRAM pb on HF)", 704], - ["589,824 px (not tested)", 768], - ["692,224 px (not tested)", 832], - ["746,496 px (not tested)", 864], - ["921,600 px (not tested)", 960] - ], value=672, label="Resolution (width x height)", info="Do not affect the generation time") + resolution = gr.Number(label="Resolution (max width or height)", value=640, precision=0) + + total_second_length = gr.Slider(label="Additional Video Length to Generate (Seconds)", minimum=1, maximum=120, value=5, step=0.1) # 20250506 pftq: Reduced default distilled guidance scale to improve adherence to input video - cfg = gr.Slider(label="CFG Scale", minimum=1.0, maximum=32.0, value=1.0, step=0.01, info='Use this instead of Distilled for more detail/control + Negative Prompt (make sure Distilled set to 1). Doubles render time. Should not change.') - gs = gr.Slider(label="Distilled CFG Scale", minimum=1.0, maximum=32.0, value=10.0, step=0.01, info='Prompt adherence at the cost of less details from the input video, but to a lesser extent than Context Frames; 3=follow the prompt but blurred motions & unsharped, 10=focus motion; changing this value is not recommended') - rs = gr.Slider(label="CFG Re-Scale", minimum=0.0, maximum=1.0, value=0.0, step=0.01, info='Should not change') + gs = gr.Slider(label="Distilled CFG Scale", minimum=1.0, maximum=32.0, value=10.0, step=0.01, info='Prompt adherence at the cost of less details from the input video, but to a lesser extent than Context Frames.') + cfg = gr.Slider(label="CFG Scale", minimum=1.0, maximum=32.0, value=1.0, step=0.01, info='Use instead of Distilled for more detail/control + Negative Prompt (make sure Distilled=1). Doubles render time.') # Should not change + rs = gr.Slider(label="CFG Re-Scale", minimum=0.0, maximum=1.0, value=0.0, step=0.01) # Should not change + + n_prompt = gr.Textbox(label="Negative Prompt", value="Missing arm, unrealistic position, blurred, blurry", info='Requires using normal CFG (undistilled) instead of Distilled (set Distilled=1 and CFG > 1).') + steps = gr.Slider(label="Steps", minimum=1, maximum=100, value=25, step=1, info='Expensive. Increase for more quality, especially if using high non-distilled CFG.') # 20250506 pftq: Renamed slider to Number of Context Frames and updated description - num_clean_frames = gr.Slider(label="Number of Context Frames", minimum=2, maximum=10, value=5, step=1, info="Retain more video details but increase memory use. Reduce to 2 to avoid memory issues or to give more weight to the prompt.") + num_clean_frames = gr.Slider(label="Number of Context Frames (Adherence to Video)", minimum=2, maximum=10, value=5, step=1, info="Expensive. Retain more video details. Reduce if memory issues or motion too restricted (jumpcut, ignoring prompt, still).") default_vae = 32 if high_vram: @@ -1301,22 +803,18 @@ with block: elif free_mem_gb>=20: default_vae = 64 - vae_batch = gr.Slider(label="VAE Batch Size for Input Video", minimum=4, maximum=256, value=default_vae, step=4, info="Reduce if running out of memory. Increase for better quality frames during fast motion.") + vae_batch = gr.Slider(label="VAE Batch Size for Input Video", minimum=4, maximum=256, value=default_vae, step=4, info="Expensive. Increase for better quality frames during fast motion. Reduce if running out of memory") + latent_window_size = gr.Slider(label="Latent Window Size", minimum=9, maximum=49, value=9, step=1, info='Expensive. Generate more frames at a time (larger chunks). Less degradation but higher VRAM cost.') gpu_memory_preservation = gr.Slider(label="GPU Inference Preserved Memory (GB) (larger means slower)", minimum=6, maximum=128, value=6, step=0.1, info="Set this number to a larger value if you encounter OOM. Larger value causes slower speed.") mp4_crf = gr.Slider(label="MP4 Compression", minimum=0, maximum=100, value=16, step=1, info="Lower means better quality. 0 is uncompressed. Change to 16 if you get black outputs. ") - batch = gr.Slider(label="Batch Size (Number of Videos)", minimum=1, maximum=1000, value=1, step=1, info='Generate multiple videos each with a different seed.') - with gr.Row(): - randomize_seed = gr.Checkbox(label='Randomize seed', value=True, info='If checked, the seed is always different') - seed = gr.Slider(label="Seed", minimum=0, maximum=np.iinfo(np.int32).max, step=1, randomize=True) with gr.Accordion("Debug", open=False): - input_image_debug = gr.Image(type="numpy", label="Image Debug", height=320) input_video_debug = gr.Video(sources='upload', label="Input Video Debug", height=320) prompt_debug = gr.Textbox(label="Prompt Debug", value='') - total_second_length_debug = gr.Slider(label="Additional Video Length to Generate (seconds) Debug", minimum=1, maximum=120, value=1, step=0.1) + total_second_length_debug = gr.Slider(label="Additional Video Length to Generate (Seconds) Debug", minimum=1, maximum=120, value=5, step=0.1) with gr.Column(): preview_image = gr.Image(label="Next Latents", height=200, visible=False) @@ -1324,190 +822,19 @@ with block: progress_desc = gr.Markdown('', elem_classes='no-generating-animation') progress_bar = gr.HTML('', elem_classes='no-generating-animation') - # 20250506 pftq: Updated inputs to include num_clean_frames - ips = [input_image, image_position, final_prompt, generation_mode, n_prompt, randomize_seed, seed, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, mp4_crf] - ips_video = [input_video, final_prompt, n_prompt, randomize_seed, seed, batch, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, no_resize, mp4_crf, num_clean_frames, vae_batch] - - with gr.Row(elem_id="image_examples", visible=False): - gr.Examples( - label = "Examples from image", - examples = [ - [ - "./img_examples/Example2.webp", # input_image - 100, # image_position - "A black man on the left and an Asian woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The man talks and the woman listens; A black man on the left and an Asian woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The woman talks and the man listens", - "image", # generation_mode - "Missing arm, unrealistic position, impossible contortion, visible bone, muscle contraction, blurred, blurry", # n_prompt - False, # randomize_seed - 42, # seed - 672, # resolution - 1, # total_second_length - 9, # latent_window_size - 25, # steps - 1.0, # cfg - 10.0, # gs - 0.0, # rs - 6, # gpu_memory_preservation - False, # enable_preview - False, # use_teacache - 16 # mp4_crf - ], - [ - "./img_examples/Example1.png", # input_image - 0, # image_position - "A dolphin emerges from the water, photorealistic, realistic, intricate details, 8k, insanely detailed", - "image", # generation_mode - "Missing arm, unrealistic position, impossible contortion, visible bone, muscle contraction, blurred, blurry", # n_prompt - True, # randomize_seed - 42, # seed - 672, # resolution - 1, # total_second_length - 9, # latent_window_size - 25, # steps - 1.0, # cfg - 10.0, # gs - 0.0, # rs - 6, # gpu_memory_preservation - False, # enable_preview - False, # use_teacache - 16 # mp4_crf - ], - [ - "./img_examples/Example3.jpg", # input_image - 0, # image_position - "A boy is walking to the right, full view, full-length view, cartoon", - "image", # generation_mode - "Missing arm, unrealistic position, impossible contortion, visible bone, muscle contraction, blurred, blurry", # n_prompt - True, # randomize_seed - 42, # seed - 672, # resolution - 1, # total_second_length - 9, # latent_window_size - 25, # steps - 1.0, # cfg - 10.0, # gs - 0.0, # rs - 6, # gpu_memory_preservation - False, # enable_preview - True, # use_teacache - 16 # mp4_crf - ], - ], - run_on_click = True, - fn = process, - inputs = ips, - outputs = [result_video, preview_image, progress_desc, progress_bar, start_button, end_button], - cache_examples = torch.cuda.device_count() > 0, - ) - - with gr.Row(elem_id="video_examples", visible=False): + with gr.Row(visible=False): gr.Examples( - label = "Examples from video", - examples = [ - [ - "./img_examples/Example1.mp4", # input_video - "View of the sea as far as the eye can see, from the seaside, a piece of land is barely visible on the horizon at the middle, the sky is radiant, reflections of the sun in the water, photorealistic, realistic, intricate details, 8k, insanely detailed", - "Missing arm, unrealistic position, impossible contortion, visible bone, muscle contraction, blurred, blurry", # n_prompt - True, # randomize_seed - 42, # seed - 1, # batch - 672, # resolution - 1, # total_second_length - 9, # latent_window_size - 25, # steps - 1.0, # cfg - 10.0, # gs - 0.0, # rs - 6, # gpu_memory_preservation - False, # enable_preview - False, # use_teacache - False, # no_resize - 16, # mp4_crf - 5, # num_clean_frames - default_vae - ], - [ - "./img_examples/Example1.mp4", # input_video - "View of the sea as far as the eye can see, from the seaside, a piece of land is barely visible on the horizon at the middle, the sky is radiant, reflections of the sun in the water, photorealistic, realistic, intricate details, 8k, insanely detailed", - "Missing arm, unrealistic position, impossible contortion, visible bone, muscle contraction, blurred, blurry", # n_prompt - True, # randomize_seed - 42, # seed - 1, # batch - 672, # resolution - 1, # total_second_length - 9, # latent_window_size - 25, # steps - 1.0, # cfg - 10.0, # gs - 0.0, # rs - 6, # gpu_memory_preservation - False, # enable_preview - True, # use_teacache - False, # no_resize - 16, # mp4_crf - 5, # num_clean_frames - default_vae - ], - ], - run_on_click = True, - fn = process_video, - inputs = ips_video, - outputs = [result_video, preview_image, progress_desc, progress_bar, start_button_video, end_button], - cache_examples = torch.cuda.device_count() > 0, - ) - - gr.Examples( - label = "Examples from image", examples = [ [ - "./img_examples/Example1.png", # input_image - 0, # image_position - "A dolphin emerges from the water, photorealistic, realistic, intricate details, 8k, insanely detailed", - "image", # generation_mode - "Missing arm, unrealistic position, impossible contortion, visible bone, muscle contraction, blurred, blurry", # n_prompt - True, # randomize_seed - 42, # seed - 672, # resolution - 1, # total_second_length - 9, # latent_window_size - 25, # steps - 1.0, # cfg - 10.0, # gs - 0.0, # rs - 6, # gpu_memory_preservation - False, # enable_preview - True, # use_teacache - 16 # mp4_crf - ], - [ - "./img_examples/Example2.webp", # input_image - 0, # image_position - "A black man on the left and an Asian woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The man talks and the woman listens; A black man on the left and an Asian woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The woman talks and the man listens", - "image", # generation_mode - "Missing arm, unrealistic position, impossible contortion, visible bone, muscle contraction, blurred, blurry", # n_prompt - True, # randomize_seed - 42, # seed - 672, # resolution - 2, # total_second_length - 9, # latent_window_size - 25, # steps - 1.0, # cfg - 10.0, # gs - 0.0, # rs - 6, # gpu_memory_preservation - False, # enable_preview - True, # use_teacache - 16 # mp4_crf - ], - [ - "./img_examples/Example2.webp", # input_image - 0, # image_position - "A black man on the left and an Asian woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The woman talks and the man listens; A black man on the left and an Asian woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The man talks and the woman listens", - "image", # generation_mode - "Missing arm, unrealistic position, impossible contortion, visible bone, muscle contraction, blurred, blurry", # n_prompt + "./img_examples/Example1.mp4", # input_video + "./img_examples/Example1.png", # end_frame + 0.0, # end_frame_weight + "View of the sea as far as the eye can see, from the seaside, a piece of land is barely visible on the horizon at the middle, the sky is radiant, reflections of the sun in the water, photorealistic, realistic, intricate details, 8k, insanely detailed", + "Missing arm, unrealistic position, blurred, blurry", # n_prompt True, # randomize_seed 42, # seed - 672, # resolution + 1, # batch + 640, # resolution 2, # total_second_length 9, # latent_window_size 25, # steps @@ -1515,49 +842,22 @@ with block: 10.0, # gs 0.0, # rs 6, # gpu_memory_preservation - False, # enable_preview - True, # use_teacache - 16 # mp4_crf + False, # use_teacache + False, # no_resize + 16, # mp4_crf + 5, # num_clean_frames + default_vae ], - [ - "./img_examples/Example3.jpg", # input_image - 0, # image_position - "A boy is walking to the right, full view, full-length view, cartoon", - "image", # generation_mode - "Missing arm, unrealistic position, impossible contortion, visible bone, muscle contraction, blurred, blurry", # n_prompt - True, # randomize_seed - 42, # seed - 672, # resolution - 1, # total_second_length - 9, # latent_window_size - 25, # steps - 1.0, # cfg - 10.0, # gs - 0.0, # rs - 6, # gpu_memory_preservation - False, # enable_preview - True, # use_teacache - 16 # mp4_crf - ] - ], - run_on_click = True, - fn = process, - inputs = ips, - outputs = [result_video, preview_image, progress_desc, progress_bar, start_button, end_button], - cache_examples = False, - ) - - gr.Examples( - label = "Examples from video", - examples = [ [ "./img_examples/Example1.mp4", # input_video + "./img_examples/Example1.png", # end_frame + 0.0, # end_frame_weight "View of the sea as far as the eye can see, from the seaside, a piece of land is barely visible on the horizon at the middle, the sky is radiant, reflections of the sun in the water, photorealistic, realistic, intricate details, 8k, insanely detailed", - "Missing arm, unrealistic position, impossible contortion, visible bone, muscle contraction, blurred, blurry", # n_prompt + "Missing arm, unrealistic position, blurred, blurry", # n_prompt True, # randomize_seed 42, # seed 1, # batch - 672, # resolution + 640, # resolution 1, # total_second_length 9, # latent_window_size 25, # steps @@ -1565,122 +865,49 @@ with block: 10.0, # gs 0.0, # rs 6, # gpu_memory_preservation - False, # enable_preview True, # use_teacache False, # no_resize 16, # mp4_crf 5, # num_clean_frames default_vae - ] + ], ], run_on_click = True, - fn = process_video, - inputs = ips_video, - outputs = [result_video, preview_image, progress_desc, progress_bar, start_button_video, end_button], - cache_examples = False, + fn = process, + inputs = [input_video, end_frame, end_frame_weight, prompt, n_prompt, randomize_seed, seed, batch, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, no_resize, mp4_crf, num_clean_frames, vae_batch], + outputs = [result_video, preview_image, progress_desc, progress_bar, start_button, end_button], + cache_examples = True, ) - - def save_preferences(preferences, value): - preferences["generation-mode"] = value - return preferences - - def load_preferences(saved_prefs): - saved_prefs = init_preferences(saved_prefs) - return saved_prefs["generation-mode"] - - def init_preferences(saved_prefs): - if saved_prefs is None: - saved_prefs = default_local_storage - return saved_prefs - - def check_parameters(generation_mode, input_image, input_video): - if generation_mode == "image" and input_image is None: - raise gr.Error("Please provide an image to extend.") - if generation_mode == "video" and input_video is None: - raise gr.Error("Please provide a video to extend.") - return gr.update(interactive=True) - - def handle_generation_mode_change(generation_mode_data): - if generation_mode_data == "text": - return [gr.update(visible = True), gr.update(visible = False), gr.update(visible = False), gr.update(visible = False), gr.update(visible = True), gr.update(visible = False), gr.update(visible = False), gr.update(visible = False), gr.update(visible = False), gr.update(visible = False), gr.update(visible = False)] - elif generation_mode_data == "image": - return [gr.update(visible = False), gr.update(visible = True), gr.update(visible = True), gr.update(visible = False), gr.update(visible = True), gr.update(visible = False), gr.update(visible = False), gr.update(visible = False), gr.update(visible = False), gr.update(visible = False), gr.update(visible = False)] - elif generation_mode_data == "video": - return [gr.update(visible = False), gr.update(visible = False), gr.update(visible = False), gr.update(visible = True), gr.update(visible = False), gr.update(visible = True), gr.update(visible = True), gr.update(visible = True), gr.update(visible = True), gr.update(visible = True), gr.update(visible = True)] + + # 20250506 pftq: Updated inputs to include num_clean_frames + ips = [input_video, end_frame, end_frame_weight, prompt, n_prompt, randomize_seed, seed, batch, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, no_resize, mp4_crf, num_clean_frames, vae_batch] + start_button.click(fn=process, inputs=ips, outputs=[result_video, preview_image, progress_desc, progress_bar, start_button, end_button]) + end_button.click(fn=end_process) - def handle_field_debug_change(input_image_debug_data, input_video_debug_data, prompt_debug_data, total_second_length_debug_data): - print("handle_field_debug_change") - global input_image_debug_value, input_video_debug_value, prompt_debug_value, total_second_length_debug_value - input_image_debug_value = input_image_debug_data + def handle_field_debug_change(input_video_debug_data, prompt_debug_data, total_second_length_debug_data): + global input_video_debug_value, prompt_debug_value, total_second_length_debug_value input_video_debug_value = input_video_debug_data prompt_debug_value = prompt_debug_data total_second_length_debug_value = total_second_length_debug_data return [] - - input_image_debug.upload( - fn=handle_field_debug_change, - inputs=[input_image_debug, input_video_debug, prompt_debug, total_second_length_debug], - outputs=[] - ) input_video_debug.upload( fn=handle_field_debug_change, - inputs=[input_image_debug, input_video_debug, prompt_debug, total_second_length_debug], + inputs=[input_video_debug, prompt_debug, total_second_length_debug], outputs=[] ) prompt_debug.change( fn=handle_field_debug_change, - inputs=[input_image_debug, input_video_debug, prompt_debug, total_second_length_debug], + inputs=[input_video_debug, prompt_debug, total_second_length_debug], outputs=[] ) total_second_length_debug.change( fn=handle_field_debug_change, - inputs=[input_image_debug, input_video_debug, prompt_debug, total_second_length_debug], + inputs=[input_video_debug, prompt_debug, total_second_length_debug], outputs=[] ) - - prompt_number.change(fn=handle_prompt_number_change, inputs=[], outputs=[]) - timeless_prompt.change(fn=handle_timeless_prompt_change, inputs=[timeless_prompt], outputs=[final_prompt]) - start_button.click(fn = check_parameters, inputs = [ - generation_mode, input_image, input_video - ], outputs = [end_button], queue = False, show_progress = False).success(fn=process, inputs=ips, outputs=[result_video, preview_image, progress_desc, progress_bar, start_button, end_button]) - start_button_video.click(fn = check_parameters, inputs = [ - generation_mode, input_image, input_video - ], outputs = [end_button], queue = False, show_progress = False).success(fn=process_video, inputs=ips_video, outputs=[result_video, preview_image, progress_desc, progress_bar, start_button_video, end_button]) - end_button.click(fn=end_process) - - generation_mode.change(fn = save_preferences, inputs = [ - local_storage, - generation_mode, - ], outputs = [ - local_storage - ]) - - generation_mode.change( - fn=handle_generation_mode_change, - inputs=[generation_mode], - outputs=[text_to_video_hint, image_position, input_image, input_video, start_button, start_button_video, no_resize, batch, num_clean_frames, vae_batch, prompt_hint] - ) - - # Update display when the page loads - block.load( - fn=handle_generation_mode_change, inputs = [ - generation_mode - ], outputs = [ - text_to_video_hint, image_position, input_image, input_video, start_button, start_button_video, no_resize, batch, num_clean_frames, vae_batch, prompt_hint - ] - ) - - # Load saved preferences when the page loads - block.load( - fn=load_preferences, inputs = [ - local_storage - ], outputs = [ - generation_mode - ] - ) block.launch(mcp_server=True, ssr_mode=False) \ No newline at end of file