Spaces:
Running
on
Zero
Running
on
Zero
This Pull Request uses an end frame
Browse filesYou can generate or extend a video that ends with a given frame
app.py
CHANGED
@@ -7,7 +7,12 @@ os.environ['HF_HOME'] = os.path.abspath(os.path.realpath(os.path.join(os.path.di
|
|
7 |
try:
|
8 |
import spaces
|
9 |
except:
|
10 |
-
|
|
|
|
|
|
|
|
|
|
|
11 |
import gradio as gr
|
12 |
import torch
|
13 |
import traceback
|
@@ -17,6 +22,7 @@ import numpy as np
|
|
17 |
import random
|
18 |
import time
|
19 |
import math
|
|
|
20 |
# 20250506 pftq: Added for video input loading
|
21 |
import decord
|
22 |
# 20250506 pftq: Added for progress bars in video_encode
|
@@ -198,9 +204,6 @@ def video_encode(video_path, resolution, no_resize, vae, vae_batch_size=16, devi
|
|
198 |
frames_pt = frames_pt.permute(0, 2, 1, 3, 4) # Shape: (1, channels, num_real_frames, height, width)
|
199 |
#print(f"Tensor shape: {frames_pt.shape}")
|
200 |
|
201 |
-
# 20250507 pftq: Save pixel frames for use in worker
|
202 |
-
input_video_pixels = frames_pt.cpu()
|
203 |
-
|
204 |
# 20250506 pftq: Move to device
|
205 |
#print(f"Moving tensor to device: {device}")
|
206 |
frames_pt = frames_pt.to(device)
|
@@ -252,7 +255,7 @@ def video_encode(video_path, resolution, no_resize, vae, vae_batch_size=16, devi
|
|
252 |
torch.cuda.empty_cache()
|
253 |
#print("VAE moved back to CPU, CUDA cache cleared")
|
254 |
|
255 |
-
return start_latent, input_image_np, history_latents, fps, target_height, target_width
|
256 |
|
257 |
except Exception as e:
|
258 |
print(f"Error in video_encode: {str(e)}")
|
@@ -305,8 +308,67 @@ def set_mp4_comments_imageio_ffmpeg(input_file, comments):
|
|
305 |
print(f"Error saving prompt to video metadata, ffmpeg may be required: "+str(e))
|
306 |
return False
|
307 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
308 |
@torch.no_grad()
|
309 |
-
def worker(input_image, image_position, prompts, n_prompt, seed, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, mp4_crf, fps_number):
|
310 |
def encode_prompt(prompt, n_prompt):
|
311 |
llama_vec, clip_l_pooler = encode_prompt_conds(prompt, text_encoder, text_encoder_2, tokenizer, tokenizer_2)
|
312 |
|
@@ -401,6 +463,8 @@ def worker(input_image, image_position, prompts, n_prompt, seed, resolution, tot
|
|
401 |
return [start_latent, image_encoder_last_hidden_state]
|
402 |
|
403 |
[start_latent, image_encoder_last_hidden_state] = get_start_latent(input_image, height, width, vae, gpu, image_encoder, high_vram)
|
|
|
|
|
404 |
|
405 |
# Dtype
|
406 |
|
@@ -412,7 +476,7 @@ def worker(input_image, image_position, prompts, n_prompt, seed, resolution, tot
|
|
412 |
|
413 |
rnd = torch.Generator("cpu").manual_seed(seed)
|
414 |
|
415 |
-
history_latents = torch.zeros(size=(1, 16, 16 + 2 + 1, height // 8, width // 8), dtype=torch.float32
|
416 |
start_latent = start_latent.to(history_latents)
|
417 |
history_pixels = None
|
418 |
|
@@ -496,7 +560,7 @@ def worker(input_image, image_position, prompts, n_prompt, seed, resolution, tot
|
|
496 |
[llama_vec, clip_l_pooler, llama_vec_n, clip_l_pooler_n, llama_attention_mask, llama_attention_mask_n] = prompt_parameters[prompt_index]
|
497 |
|
498 |
if prompt_index < len(prompt_parameters) - 1 or (prompt_index == total_latent_sections - 1):
|
499 |
-
prompt_parameters[prompt_index]
|
500 |
|
501 |
if not high_vram:
|
502 |
unload_complete_models()
|
@@ -544,6 +608,13 @@ def worker(input_image, image_position, prompts, n_prompt, seed, resolution, tot
|
|
544 |
clean_latent_4x_indices=clean_latent_4x_indices,
|
545 |
callback=callback,
|
546 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
547 |
|
548 |
[total_generated_latent_frames, history_latents, history_pixels] = post_process(forward, generated_latents, total_generated_latent_frames, history_latents, high_vram, transformer, gpu, vae, history_pixels, latent_window_size, enable_preview, section_index, total_latent_sections, outputs_folder, mp4_crf, stream)
|
549 |
|
@@ -557,7 +628,8 @@ def worker(input_image, image_position, prompts, n_prompt, seed, resolution, tot
|
|
557 |
real_history_latents = history_latents[:, :, :total_generated_latent_frames, :, :]
|
558 |
zero_latents = history_latents[:, :, total_generated_latent_frames:, :, :]
|
559 |
history_latents = torch.cat([zero_latents, real_history_latents], dim=2)
|
560 |
-
real_history_latents
|
|
|
561 |
|
562 |
forward = True
|
563 |
section_index = first_section_index
|
@@ -575,9 +647,293 @@ def worker(input_image, image_position, prompts, n_prompt, seed, resolution, tot
|
|
575 |
stream.output_queue.push(('end', None))
|
576 |
return
|
577 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
578 |
# 20250506 pftq: Modified worker to accept video input and clean frame count
|
579 |
@torch.no_grad()
|
580 |
-
def worker_video(input_video, prompts, n_prompt, seed, batch, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, no_resize, mp4_crf, num_clean_frames, vae_batch):
|
581 |
def encode_prompt(prompt, n_prompt):
|
582 |
llama_vec, clip_l_pooler = encode_prompt_conds(prompt, text_encoder, text_encoder_2, tokenizer, tokenizer_2)
|
583 |
|
@@ -602,8 +958,9 @@ def worker_video(input_video, prompts, n_prompt, seed, batch, resolution, total_
|
|
602 |
stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Video processing ...'))))
|
603 |
|
604 |
# 20250506 pftq: Encode video
|
605 |
-
start_latent, input_image_np, video_latents, fps, height, width = video_encode(input_video, resolution, no_resize, vae, vae_batch_size=vae_batch, device=gpu)
|
606 |
-
|
|
|
607 |
video_latents = video_latents.cpu()
|
608 |
|
609 |
total_latent_sections = (total_second_length * fps) / (latent_window_size * 4)
|
@@ -640,12 +997,29 @@ def worker_video(input_video, prompts, n_prompt, seed, batch, resolution, total_
|
|
640 |
load_model_as_complete(image_encoder, target_device=gpu)
|
641 |
|
642 |
image_encoder_output = hf_clip_vision_encode(input_image_np, feature_extractor, image_encoder)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
643 |
|
644 |
# Clean GPU
|
645 |
if not high_vram:
|
646 |
-
unload_complete_models(image_encoder)
|
647 |
|
648 |
image_encoder_last_hidden_state = image_encoder_output.last_hidden_state
|
|
|
649 |
|
650 |
# Dtype
|
651 |
image_encoder_last_hidden_state = image_encoder_last_hidden_state.to(transformer.dtype)
|
@@ -672,7 +1046,13 @@ def worker_video(input_video, prompts, n_prompt, seed, batch, resolution, total_
|
|
672 |
def callback(d):
|
673 |
return
|
674 |
|
675 |
-
def compute_latent(history_latents, latent_window_size, num_clean_frames, start_latent):
|
|
|
|
|
|
|
|
|
|
|
|
|
676 |
# 20250506 pftq: Use user-specified number of context frames, matching original allocation for num_clean_frames=2
|
677 |
available_frames = history_latents.shape[2] # Number of latent frames
|
678 |
max_pixel_frames = min(latent_window_size * 4 - 3, available_frames * 4) # Cap at available pixel frames
|
@@ -686,11 +1066,11 @@ def worker_video(input_video, prompts, n_prompt, seed, batch, resolution, total_
|
|
686 |
total_context_frames = num_4x_frames + num_2x_frames + effective_clean_frames
|
687 |
total_context_frames = min(total_context_frames, available_frames) # 20250507 pftq: Edge case for <=1 sec videos
|
688 |
|
689 |
-
indices = torch.arange(0, 1 + num_4x_frames + num_2x_frames + effective_clean_frames + adjusted_latent_frames).unsqueeze(0) # 20250507 pftq: latent_window_size to adjusted_latent_frames for edge case for <=1 sec videos
|
690 |
-
clean_latent_indices_start, clean_latent_4x_indices, clean_latent_2x_indices, clean_latent_1x_indices, latent_indices = indices.split(
|
691 |
-
[1, num_4x_frames, num_2x_frames, effective_clean_frames, adjusted_latent_frames], dim=1 # 20250507 pftq: latent_window_size to adjusted_latent_frames for edge case for <=1 sec videos
|
692 |
)
|
693 |
-
clean_latent_indices = torch.cat([clean_latent_indices_start, clean_latent_1x_indices], dim=1)
|
694 |
|
695 |
# 20250506 pftq: Split history_latents dynamically based on available frames
|
696 |
fallback_frame_count = 2 # 20250507 pftq: Changed 0 to 2 Edge case for <=1 sec videos
|
@@ -723,7 +1103,10 @@ def worker_video(input_video, prompts, n_prompt, seed, batch, resolution, total_
|
|
723 |
if effective_clean_frames > 0 and split_idx < len(splits):
|
724 |
clean_latents_1x = splits[split_idx]
|
725 |
|
726 |
-
|
|
|
|
|
|
|
727 |
|
728 |
# 20250507 pftq: Fix for <=1 sec videos.
|
729 |
max_frames = min(latent_window_size * 4 - 3, history_latents.shape[2] * 4)
|
@@ -745,10 +1128,18 @@ def worker_video(input_video, prompts, n_prompt, seed, batch, resolution, total_
|
|
745 |
history_latents = video_latents
|
746 |
total_generated_latent_frames = history_latents.shape[2]
|
747 |
# 20250506 pftq: Initialize history_pixels to fix UnboundLocalError
|
748 |
-
history_pixels = None
|
749 |
-
previous_video = None
|
750 |
|
751 |
-
for
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
752 |
if stream.input_queue.top() == 'end':
|
753 |
stream.output_queue.push(('end', None))
|
754 |
return
|
@@ -767,7 +1158,7 @@ def worker_video(input_video, prompts, n_prompt, seed, batch, resolution, total_
|
|
767 |
else:
|
768 |
transformer.initialize_teacache(enable_teacache=False)
|
769 |
|
770 |
-
[max_frames, clean_latents, clean_latents_2x, clean_latents_4x, latent_indices, clean_latents, clean_latent_indices, clean_latent_2x_indices, clean_latent_4x_indices] = compute_latent(history_latents, latent_window_size, num_clean_frames, start_latent)
|
771 |
|
772 |
generated_latents = sample_hunyuan(
|
773 |
transformer=transformer,
|
@@ -798,6 +1189,13 @@ def worker_video(input_video, prompts, n_prompt, seed, batch, resolution, total_
|
|
798 |
clean_latent_4x_indices=clean_latent_4x_indices,
|
799 |
callback=callback,
|
800 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
801 |
|
802 |
total_generated_latent_frames += int(generated_latents.shape[2])
|
803 |
history_latents = torch.cat([history_latents, generated_latents.to(history_latents)], dim=2)
|
@@ -855,18 +1253,17 @@ def worker_video(input_video, prompts, n_prompt, seed, batch, resolution, total_
|
|
855 |
stream.output_queue.push(('end', None))
|
856 |
return
|
857 |
|
858 |
-
def get_duration(input_image, image_position, prompts, generation_mode, n_prompt, seed, resolution, total_second_length, allocation_time, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, mp4_crf, fps_number):
|
859 |
return allocation_time
|
860 |
|
861 |
-
# Remove this decorator if you run on local
|
862 |
@spaces.GPU(duration=get_duration)
|
863 |
-
def process_on_gpu(input_image, image_position, prompts, generation_mode, n_prompt, seed, resolution, total_second_length, allocation_time, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, mp4_crf, fps_number
|
864 |
):
|
865 |
start = time.time()
|
866 |
global stream
|
867 |
stream = AsyncStream()
|
868 |
|
869 |
-
async_run(worker, input_image, image_position, prompts, n_prompt, seed, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, mp4_crf, fps_number)
|
870 |
|
871 |
output_filename = None
|
872 |
|
@@ -892,11 +1289,13 @@ def process_on_gpu(input_image, image_position, prompts, generation_mode, n_prom
|
|
892 |
((str(hours) + " h, ") if hours != 0 else "") + \
|
893 |
((str(minutes) + " min, ") if hours != 0 or minutes != 0 else "") + \
|
894 |
str(secondes) + " sec. " + \
|
895 |
-
"You can upscale the result with
|
896 |
break
|
897 |
|
898 |
def process(input_image,
|
|
|
899 |
image_position=0,
|
|
|
900 |
prompt="",
|
901 |
generation_mode="image",
|
902 |
n_prompt="",
|
@@ -907,18 +1306,18 @@ def process(input_image,
|
|
907 |
resolution=640,
|
908 |
total_second_length=5,
|
909 |
latent_window_size=9,
|
910 |
-
steps=
|
911 |
cfg=1.0,
|
912 |
gs=10.0,
|
913 |
rs=0.0,
|
914 |
gpu_memory_preservation=6,
|
915 |
-
enable_preview=
|
916 |
use_teacache=False,
|
917 |
mp4_crf=16,
|
918 |
fps_number=30
|
919 |
):
|
920 |
if auto_allocation:
|
921 |
-
allocation_time = min(total_second_length * 60 * (1.5 if use_teacache else 3.0) * (1 + ((steps - 25) / 25)), 600)
|
922 |
|
923 |
if torch.cuda.device_count() == 0:
|
924 |
gr.Warning('Set this space to GPU config to make it work.')
|
@@ -930,16 +1329,20 @@ def process(input_image,
|
|
930 |
|
931 |
prompts = prompt.split(";")
|
932 |
|
933 |
-
# assert input_image is not None, 'No input image!'
|
934 |
if generation_mode == "text":
|
935 |
-
default_height, default_width =
|
936 |
input_image = np.ones((default_height, default_width, 3), dtype=np.uint8) * 255
|
937 |
print("No input image provided. Using a blank white image.")
|
|
|
|
|
938 |
|
939 |
yield gr.update(label="Previewed Frames"), None, '', '', gr.update(interactive=False), gr.update(interactive=True), gr.skip()
|
940 |
|
|
|
941 |
yield from process_on_gpu(input_image,
|
|
|
942 |
image_position,
|
|
|
943 |
prompts,
|
944 |
generation_mode,
|
945 |
n_prompt,
|
@@ -959,18 +1362,17 @@ def process(input_image,
|
|
959 |
fps_number
|
960 |
)
|
961 |
|
962 |
-
def get_duration_video(input_video, prompts, n_prompt, seed, batch, resolution, total_second_length, allocation_time, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, no_resize, mp4_crf, num_clean_frames, vae_batch):
|
963 |
return allocation_time
|
964 |
|
965 |
-
# Remove this decorator if you run on local
|
966 |
@spaces.GPU(duration=get_duration_video)
|
967 |
-
def process_video_on_gpu(input_video, prompts, n_prompt, seed, batch, resolution, total_second_length, allocation_time, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, no_resize, mp4_crf, num_clean_frames, vae_batch):
|
968 |
start = time.time()
|
969 |
global stream
|
970 |
stream = AsyncStream()
|
971 |
|
972 |
# 20250506 pftq: Pass num_clean_frames, vae_batch, etc
|
973 |
-
async_run(worker_video, input_video, prompts, n_prompt, seed, batch, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, no_resize, mp4_crf, num_clean_frames, vae_batch)
|
974 |
|
975 |
output_filename = None
|
976 |
|
@@ -997,13 +1399,13 @@ def process_video_on_gpu(input_video, prompts, n_prompt, seed, batch, resolution
|
|
997 |
((str(hours) + " h, ") if hours != 0 else "") + \
|
998 |
((str(minutes) + " min, ") if hours != 0 or minutes != 0 else "") + \
|
999 |
str(secondes) + " sec. " + \
|
1000 |
-
" You can upscale the result with
|
1001 |
break
|
1002 |
|
1003 |
-
def process_video(input_video, prompt, n_prompt, randomize_seed, seed, auto_allocation, allocation_time, batch, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, no_resize, mp4_crf, num_clean_frames, vae_batch):
|
1004 |
global high_vram
|
1005 |
if auto_allocation:
|
1006 |
-
allocation_time = min(total_second_length * 60 * (2.5 if use_teacache else 3.5) * (1 + ((steps - 25) / 25)), 600)
|
1007 |
|
1008 |
if torch.cuda.device_count() == 0:
|
1009 |
gr.Warning('Set this space to GPU config to make it work.')
|
@@ -1033,7 +1435,8 @@ def process_video(input_video, prompt, n_prompt, randomize_seed, seed, auto_allo
|
|
1033 |
if cfg > 1:
|
1034 |
gs = 1
|
1035 |
|
1036 |
-
|
|
|
1037 |
|
1038 |
def end_process():
|
1039 |
stream.input_queue.push('end')
|
@@ -1103,11 +1506,12 @@ with block:
|
|
1103 |
local_storage = gr.BrowserState(default_local_storage)
|
1104 |
with gr.Row():
|
1105 |
with gr.Column():
|
1106 |
-
generation_mode = gr.Radio([["Text-to-Video", "text"], ["Image-to-Video", "image"], ["Video Extension", "video"]], elem_id="generation-mode", label="
|
1107 |
text_to_video_hint = gr.HTML("Text-to-Video badly works with a flash effect at the start. I discourage to use the Text-to-Video feature. You should rather generate an image with Flux and use Image-to-Video. You will save time.")
|
1108 |
input_image = gr.Image(sources='upload', type="numpy", label="Image", height=320)
|
1109 |
image_position = gr.Slider(label="Image position", minimum=0, maximum=100, value=0, step=1, info='0=Video start; 100=Video end (lower quality)')
|
1110 |
input_video = gr.Video(sources='upload', label="Input Video", height=320)
|
|
|
1111 |
timeless_prompt = gr.Textbox(label="Timeless prompt", info='Used on the whole duration of the generation', value='', placeholder="The creature starts to move, fast motion, fixed camera, focus motion, consistent arm, consistent position, mute colors, insanely detailed")
|
1112 |
prompt_number = gr.Slider(label="Timed prompt number", minimum=0, maximum=1000, value=0, step=1, info='Prompts will automatically appear')
|
1113 |
|
@@ -1131,9 +1535,10 @@ with block:
|
|
1131 |
enable_preview = gr.Checkbox(label='Enable preview', value=True, info='Display a preview around each second generated but it costs 2 sec. for each second generated.')
|
1132 |
use_teacache = gr.Checkbox(label='Use TeaCache', value=False, info='Faster speed and no break in brightness, but often makes hands and fingers slightly worse.')
|
1133 |
|
1134 |
-
n_prompt = gr.Textbox(label="Negative Prompt", value="Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, blurred, blurry", info='Requires using normal CFG (undistilled) instead of Distilled (set Distilled=1 and CFG > 1).')
|
1135 |
|
1136 |
fps_number = gr.Slider(label="Frame per seconds", info="The model is trained for 30 fps so other fps may generate weird results", minimum=10, maximum=60, value=30, step=1)
|
|
|
1137 |
|
1138 |
latent_window_size = gr.Slider(label="Latent Window Size", minimum=1, maximum=33, value=9, step=1, info='Generate more frames at a time (larger chunks). Less degradation and better blending but higher VRAM cost. Should not change.')
|
1139 |
steps = gr.Slider(label="Steps", minimum=1, maximum=100, value=30, step=1, info='Increase for more quality, especially if using high non-distilled CFG. If your animation has very few motion, you may have brutal brightness change; this can be fixed increasing the steps.')
|
@@ -1186,19 +1591,20 @@ with block:
|
|
1186 |
progress_desc = gr.Markdown('', elem_classes='no-generating-animation')
|
1187 |
progress_bar = gr.HTML('', elem_classes='no-generating-animation')
|
1188 |
|
1189 |
-
|
1190 |
-
|
1191 |
-
ips_video = [input_video, final_prompt, n_prompt, randomize_seed, seed, auto_allocation, allocation_time, batch, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, no_resize, mp4_crf, num_clean_frames, vae_batch]
|
1192 |
|
1193 |
gr.Examples(
|
1194 |
label = "✍️ Examples from text",
|
1195 |
examples = [
|
1196 |
[
|
1197 |
None, # input_image
|
|
|
1198 |
0, # image_position
|
|
|
1199 |
"Overcrowed street in Japan, photorealistic, realistic, intricate details, 8k, insanely detailed",
|
1200 |
"text", # generation_mode
|
1201 |
-
"Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, blurred, blurry", # n_prompt
|
1202 |
True, # randomize_seed
|
1203 |
42, # seed
|
1204 |
True, # auto_allocation
|
@@ -1229,10 +1635,12 @@ with block:
|
|
1229 |
examples = [
|
1230 |
[
|
1231 |
"./img_examples/Example1.png", # input_image
|
|
|
1232 |
0, # image_position
|
|
|
1233 |
"A dolphin emerges from the water, photorealistic, realistic, intricate details, 8k, insanely detailed",
|
1234 |
"image", # generation_mode
|
1235 |
-
"Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, blurred, blurry", # n_prompt
|
1236 |
True, # randomize_seed
|
1237 |
42, # seed
|
1238 |
True, # auto_allocation
|
@@ -1246,16 +1654,18 @@ with block:
|
|
1246 |
0.0, # rs
|
1247 |
6, # gpu_memory_preservation
|
1248 |
False, # enable_preview
|
1249 |
-
|
1250 |
16, # mp4_crf
|
1251 |
30 # fps_number
|
1252 |
],
|
1253 |
[
|
1254 |
"./img_examples/Example2.webp", # input_image
|
|
|
1255 |
0, # image_position
|
|
|
1256 |
"A man on the left and a woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The man talks and the woman listens; A man on the left and a woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The woman talks, the man stops talking and the man listens; A man on the left and a woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The woman talks and the man listens",
|
1257 |
"image", # generation_mode
|
1258 |
-
"Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, blurred, blurry", # n_prompt
|
1259 |
True, # randomize_seed
|
1260 |
42, # seed
|
1261 |
True, # auto_allocation
|
@@ -1269,16 +1679,18 @@ with block:
|
|
1269 |
0.0, # rs
|
1270 |
6, # gpu_memory_preservation
|
1271 |
False, # enable_preview
|
1272 |
-
|
1273 |
16, # mp4_crf
|
1274 |
30 # fps_number
|
1275 |
],
|
1276 |
[
|
1277 |
"./img_examples/Example2.webp", # input_image
|
|
|
1278 |
0, # image_position
|
|
|
1279 |
"A man on the left and a woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The woman talks and the man listens; A man on the left and a woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The man talks, the woman stops talking and the woman listens A man on the left and a woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The man talks and the woman listens",
|
1280 |
"image", # generation_mode
|
1281 |
-
"Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, blurred, blurry", # n_prompt
|
1282 |
True, # randomize_seed
|
1283 |
42, # seed
|
1284 |
True, # auto_allocation
|
@@ -1292,16 +1704,18 @@ with block:
|
|
1292 |
0.0, # rs
|
1293 |
6, # gpu_memory_preservation
|
1294 |
False, # enable_preview
|
1295 |
-
|
1296 |
16, # mp4_crf
|
1297 |
30 # fps_number
|
1298 |
],
|
1299 |
[
|
1300 |
"./img_examples/Example3.jpg", # input_image
|
|
|
1301 |
0, # image_position
|
1302 |
-
|
|
|
1303 |
"image", # generation_mode
|
1304 |
-
"
|
1305 |
True, # randomize_seed
|
1306 |
42, # seed
|
1307 |
True, # auto_allocation
|
@@ -1315,16 +1729,18 @@ with block:
|
|
1315 |
0.0, # rs
|
1316 |
6, # gpu_memory_preservation
|
1317 |
False, # enable_preview
|
1318 |
-
|
1319 |
16, # mp4_crf
|
1320 |
30 # fps_number
|
1321 |
],
|
1322 |
[
|
1323 |
"./img_examples/Example4.webp", # input_image
|
|
|
1324 |
100, # image_position
|
|
|
1325 |
"A building starting to explode, photorealistic, realisitc, 8k, insanely detailed",
|
1326 |
"image", # generation_mode
|
1327 |
-
"Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, blurred, blurry", # n_prompt
|
1328 |
True, # randomize_seed
|
1329 |
42, # seed
|
1330 |
True, # auto_allocation
|
@@ -1350,13 +1766,51 @@ with block:
|
|
1350 |
cache_examples = False,
|
1351 |
)
|
1352 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1353 |
gr.Examples(
|
1354 |
label = "🎥 Examples from video",
|
1355 |
examples = [
|
1356 |
[
|
1357 |
"./img_examples/Example1.mp4", # input_video
|
|
|
|
|
1358 |
"View of the sea as far as the eye can see, from the seaside, a piece of land is barely visible on the horizon at the middle, the sky is radiant, reflections of the sun in the water, photorealistic, realistic, intricate details, 8k, insanely detailed",
|
1359 |
-
"Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, blurred, blurry", # n_prompt
|
1360 |
True, # randomize_seed
|
1361 |
42, # seed
|
1362 |
True, # auto_allocation
|
@@ -1371,7 +1825,33 @@ with block:
|
|
1371 |
0.0, # rs
|
1372 |
6, # gpu_memory_preservation
|
1373 |
False, # enable_preview
|
1374 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1375 |
False, # no_resize
|
1376 |
16, # mp4_crf
|
1377 |
5, # num_clean_frames
|
@@ -1401,17 +1881,81 @@ with block:
|
|
1401 |
def check_parameters(generation_mode, input_image, input_video):
|
1402 |
if generation_mode == "image" and input_image is None:
|
1403 |
raise gr.Error("Please provide an image to extend.")
|
|
|
|
|
1404 |
if generation_mode == "video" and input_video is None:
|
1405 |
raise gr.Error("Please provide a video to extend.")
|
1406 |
return [gr.update(interactive=True), gr.update(visible = True)]
|
1407 |
|
1408 |
def handle_generation_mode_change(generation_mode_data):
|
1409 |
if generation_mode_data == "text":
|
1410 |
-
return [
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1411 |
elif generation_mode_data == "image":
|
1412 |
-
return [
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1413 |
elif generation_mode_data == "video":
|
1414 |
-
return [
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1415 |
|
1416 |
prompt_number.change(fn=handle_prompt_number_change, inputs=[], outputs=[])
|
1417 |
timeless_prompt.change(fn=handle_timeless_prompt_change, inputs=[timeless_prompt], outputs=[final_prompt])
|
@@ -1433,7 +1977,7 @@ with block:
|
|
1433 |
generation_mode.change(
|
1434 |
fn=handle_generation_mode_change,
|
1435 |
inputs=[generation_mode],
|
1436 |
-
outputs=[text_to_video_hint, image_position, input_image, input_video, start_button, start_button_video, no_resize, batch, num_clean_frames, vae_batch, prompt_hint, fps_number]
|
1437 |
)
|
1438 |
|
1439 |
# Update display when the page loads
|
@@ -1441,7 +1985,7 @@ with block:
|
|
1441 |
fn=handle_generation_mode_change, inputs = [
|
1442 |
generation_mode
|
1443 |
], outputs = [
|
1444 |
-
text_to_video_hint, image_position, input_image, input_video, start_button, start_button_video, no_resize, batch, num_clean_frames, vae_batch, prompt_hint, fps_number
|
1445 |
]
|
1446 |
)
|
1447 |
|
|
|
7 |
try:
|
8 |
import spaces
|
9 |
except:
|
10 |
+
class spaces():
|
11 |
+
def GPU(*args, **kwargs):
|
12 |
+
def decorator(function):
|
13 |
+
return lambda *dummy_args, **dummy_kwargs: function(*dummy_args, **dummy_kwargs)
|
14 |
+
return decorator
|
15 |
+
|
16 |
import gradio as gr
|
17 |
import torch
|
18 |
import traceback
|
|
|
22 |
import random
|
23 |
import time
|
24 |
import math
|
25 |
+
import gc
|
26 |
# 20250506 pftq: Added for video input loading
|
27 |
import decord
|
28 |
# 20250506 pftq: Added for progress bars in video_encode
|
|
|
204 |
frames_pt = frames_pt.permute(0, 2, 1, 3, 4) # Shape: (1, channels, num_real_frames, height, width)
|
205 |
#print(f"Tensor shape: {frames_pt.shape}")
|
206 |
|
|
|
|
|
|
|
207 |
# 20250506 pftq: Move to device
|
208 |
#print(f"Moving tensor to device: {device}")
|
209 |
frames_pt = frames_pt.to(device)
|
|
|
255 |
torch.cuda.empty_cache()
|
256 |
#print("VAE moved back to CPU, CUDA cache cleared")
|
257 |
|
258 |
+
return start_latent, input_image_np, history_latents, fps, target_height, target_width
|
259 |
|
260 |
except Exception as e:
|
261 |
print(f"Error in video_encode: {str(e)}")
|
|
|
308 |
print(f"Error saving prompt to video metadata, ffmpeg may be required: "+str(e))
|
309 |
return False
|
310 |
|
311 |
+
# 20250507 pftq: New function to encode a single image (end frame)
|
312 |
+
@torch.no_grad()
|
313 |
+
def image_encode(image_np, target_width, target_height, vae, image_encoder, feature_extractor, device="cuda"):
|
314 |
+
"""
|
315 |
+
Encode a single image into a latent and compute its CLIP vision embedding.
|
316 |
+
|
317 |
+
Args:
|
318 |
+
image_np: Input image as numpy array.
|
319 |
+
target_width, target_height: Exact resolution to resize the image to (matches start frame).
|
320 |
+
vae: AutoencoderKLHunyuanVideo model.
|
321 |
+
image_encoder: SiglipVisionModel for CLIP vision encoding.
|
322 |
+
feature_extractor: SiglipImageProcessor for preprocessing.
|
323 |
+
device: Device for computation (e.g., "cuda").
|
324 |
+
|
325 |
+
Returns:
|
326 |
+
latent: Latent representation of the image (shape: [1, channels, 1, height//8, width//8]).
|
327 |
+
clip_embedding: CLIP vision embedding of the image.
|
328 |
+
processed_image_np: Processed image as numpy array (after resizing).
|
329 |
+
"""
|
330 |
+
# 20250507 pftq: Process end frame with exact start frame dimensions
|
331 |
+
print("Processing end frame...")
|
332 |
+
try:
|
333 |
+
print(f"Using exact start frame resolution for end frame: {target_width}x{target_height}")
|
334 |
+
|
335 |
+
# Resize and preprocess image to match start frame
|
336 |
+
processed_image_np = resize_and_center_crop(image_np, target_width=target_width, target_height=target_height)
|
337 |
+
|
338 |
+
# Convert to tensor and normalize
|
339 |
+
image_pt = torch.from_numpy(processed_image_np).float() / 127.5 - 1
|
340 |
+
image_pt = image_pt.permute(2, 0, 1).unsqueeze(0).unsqueeze(2) # Shape: [1, channels, 1, height, width]
|
341 |
+
image_pt = image_pt.to(device)
|
342 |
+
|
343 |
+
# Move VAE to device
|
344 |
+
vae.to(device)
|
345 |
+
|
346 |
+
# Encode to latent
|
347 |
+
latent = vae_encode(image_pt, vae)
|
348 |
+
print(f"image_encode vae output shape: {latent.shape}")
|
349 |
+
|
350 |
+
# Move image encoder to device
|
351 |
+
image_encoder.to(device)
|
352 |
+
|
353 |
+
# Compute CLIP vision embedding
|
354 |
+
clip_embedding = hf_clip_vision_encode(processed_image_np, feature_extractor, image_encoder).last_hidden_state
|
355 |
+
|
356 |
+
# Move models back to CPU and clear cache
|
357 |
+
if device == "cuda":
|
358 |
+
vae.to(cpu)
|
359 |
+
image_encoder.to(cpu)
|
360 |
+
torch.cuda.empty_cache()
|
361 |
+
print("VAE and image encoder moved back to CPU, CUDA cache cleared")
|
362 |
+
|
363 |
+
print(f"End latent shape: {latent.shape}")
|
364 |
+
return latent, clip_embedding, processed_image_np
|
365 |
+
|
366 |
+
except Exception as e:
|
367 |
+
print(f"Error in image_encode: {str(e)}")
|
368 |
+
raise
|
369 |
+
|
370 |
@torch.no_grad()
|
371 |
+
def worker(input_image, end_image, image_position, end_stillness, prompts, n_prompt, seed, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, mp4_crf, fps_number):
|
372 |
def encode_prompt(prompt, n_prompt):
|
373 |
llama_vec, clip_l_pooler = encode_prompt_conds(prompt, text_encoder, text_encoder_2, tokenizer, tokenizer_2)
|
374 |
|
|
|
463 |
return [start_latent, image_encoder_last_hidden_state]
|
464 |
|
465 |
[start_latent, image_encoder_last_hidden_state] = get_start_latent(input_image, height, width, vae, gpu, image_encoder, high_vram)
|
466 |
+
del input_image
|
467 |
+
del end_image
|
468 |
|
469 |
# Dtype
|
470 |
|
|
|
476 |
|
477 |
rnd = torch.Generator("cpu").manual_seed(seed)
|
478 |
|
479 |
+
history_latents = torch.zeros(size=(1, 16, 16 + 2 + 1, height // 8, width // 8), dtype=torch.float32, device=cpu)
|
480 |
start_latent = start_latent.to(history_latents)
|
481 |
history_pixels = None
|
482 |
|
|
|
560 |
[llama_vec, clip_l_pooler, llama_vec_n, clip_l_pooler_n, llama_attention_mask, llama_attention_mask_n] = prompt_parameters[prompt_index]
|
561 |
|
562 |
if prompt_index < len(prompt_parameters) - 1 or (prompt_index == total_latent_sections - 1):
|
563 |
+
del prompt_parameters[prompt_index]
|
564 |
|
565 |
if not high_vram:
|
566 |
unload_complete_models()
|
|
|
608 |
clean_latent_4x_indices=clean_latent_4x_indices,
|
609 |
callback=callback,
|
610 |
)
|
611 |
+
del clean_latents
|
612 |
+
del clean_latents_2x
|
613 |
+
del clean_latents_4x
|
614 |
+
del latent_indices
|
615 |
+
del clean_latent_indices
|
616 |
+
del clean_latent_2x_indices
|
617 |
+
del clean_latent_4x_indices
|
618 |
|
619 |
[total_generated_latent_frames, history_latents, history_pixels] = post_process(forward, generated_latents, total_generated_latent_frames, history_latents, high_vram, transformer, gpu, vae, history_pixels, latent_window_size, enable_preview, section_index, total_latent_sections, outputs_folder, mp4_crf, stream)
|
620 |
|
|
|
628 |
real_history_latents = history_latents[:, :, :total_generated_latent_frames, :, :]
|
629 |
zero_latents = history_latents[:, :, total_generated_latent_frames:, :, :]
|
630 |
history_latents = torch.cat([zero_latents, real_history_latents], dim=2)
|
631 |
+
del real_history_latents
|
632 |
+
del zero_latents
|
633 |
|
634 |
forward = True
|
635 |
section_index = first_section_index
|
|
|
647 |
stream.output_queue.push(('end', None))
|
648 |
return
|
649 |
|
650 |
+
@torch.no_grad()
|
651 |
+
def worker_start_end(input_image, end_image, image_position, end_stillness, prompts, n_prompt, seed, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, mp4_crf, fps_number):
|
652 |
+
def encode_prompt(prompt, n_prompt):
|
653 |
+
llama_vec, clip_l_pooler = encode_prompt_conds(prompt, text_encoder, text_encoder_2, tokenizer, tokenizer_2)
|
654 |
+
|
655 |
+
if cfg == 1:
|
656 |
+
llama_vec_n, clip_l_pooler_n = torch.zeros_like(llama_vec), torch.zeros_like(clip_l_pooler)
|
657 |
+
else:
|
658 |
+
llama_vec_n, clip_l_pooler_n = encode_prompt_conds(n_prompt, text_encoder, text_encoder_2, tokenizer, tokenizer_2)
|
659 |
+
|
660 |
+
llama_vec, llama_attention_mask = crop_or_pad_yield_mask(llama_vec, length=512)
|
661 |
+
llama_vec_n, llama_attention_mask_n = crop_or_pad_yield_mask(llama_vec_n, length=512)
|
662 |
+
|
663 |
+
llama_vec = llama_vec.to(transformer.dtype)
|
664 |
+
llama_vec_n = llama_vec_n.to(transformer.dtype)
|
665 |
+
clip_l_pooler = clip_l_pooler.to(transformer.dtype)
|
666 |
+
clip_l_pooler_n = clip_l_pooler_n.to(transformer.dtype)
|
667 |
+
return [llama_vec, clip_l_pooler, llama_vec_n, clip_l_pooler_n, llama_attention_mask, llama_attention_mask_n]
|
668 |
+
|
669 |
+
total_latent_sections = (total_second_length * fps_number) / (latent_window_size * 4)
|
670 |
+
total_latent_sections = int(max(round(total_latent_sections), 1))
|
671 |
+
|
672 |
+
job_id = generate_timestamp()
|
673 |
+
|
674 |
+
stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Starting ...'))))
|
675 |
+
|
676 |
+
try:
|
677 |
+
# Clean GPU
|
678 |
+
if not high_vram:
|
679 |
+
unload_complete_models(
|
680 |
+
text_encoder, text_encoder_2, image_encoder, vae, transformer
|
681 |
+
)
|
682 |
+
|
683 |
+
# Text encoding
|
684 |
+
|
685 |
+
stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Text encoding ...'))))
|
686 |
+
|
687 |
+
if not high_vram:
|
688 |
+
fake_diffusers_current_device(text_encoder, gpu) # since we only encode one text - that is one model move and one encode, offload is same time consumption since it is also one load and one encode.
|
689 |
+
load_model_as_complete(text_encoder_2, target_device=gpu)
|
690 |
+
|
691 |
+
|
692 |
+
prompt_parameters = []
|
693 |
+
|
694 |
+
for prompt_part in prompts[:total_latent_sections]:
|
695 |
+
prompt_parameters.append(encode_prompt(prompt_part, n_prompt))
|
696 |
+
|
697 |
+
# Clean GPU
|
698 |
+
if not high_vram:
|
699 |
+
unload_complete_models(
|
700 |
+
text_encoder, text_encoder_2
|
701 |
+
)
|
702 |
+
|
703 |
+
# Processing input image (start frame)
|
704 |
+
stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Processing start frame ...'))))
|
705 |
+
|
706 |
+
H, W, C = input_image.shape
|
707 |
+
height, width = find_nearest_bucket(H, W, resolution=resolution)
|
708 |
+
has_end_image = end_image is not None
|
709 |
+
|
710 |
+
def get_start_latent(input_image, has_end_image, end_image, height, width, vae, gpu, image_encoder, high_vram):
|
711 |
+
input_image_np = resize_and_center_crop(input_image, target_width=width, target_height=height)
|
712 |
+
|
713 |
+
input_image_pt = torch.from_numpy(input_image_np).float() / 127.5 - 1
|
714 |
+
input_image_pt = input_image_pt.permute(2, 0, 1)[None, :, None]
|
715 |
+
|
716 |
+
# Processing end image (if provided)
|
717 |
+
if has_end_image:
|
718 |
+
stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Processing end frame ...'))))
|
719 |
+
|
720 |
+
end_image_np = resize_and_center_crop(end_image, target_width=width, target_height=height)
|
721 |
+
|
722 |
+
end_image_pt = torch.from_numpy(end_image_np).float() / 127.5 - 1
|
723 |
+
end_image_pt = end_image_pt.permute(2, 0, 1)[None, :, None]
|
724 |
+
|
725 |
+
# VAE encoding
|
726 |
+
stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'VAE encoding ...'))))
|
727 |
+
|
728 |
+
if not high_vram:
|
729 |
+
load_model_as_complete(vae, target_device=gpu)
|
730 |
+
|
731 |
+
start_latent = vae_encode(input_image_pt, vae)
|
732 |
+
|
733 |
+
if has_end_image:
|
734 |
+
end_latent = vae_encode(end_image_pt, vae)
|
735 |
+
|
736 |
+
# CLIP Vision
|
737 |
+
stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'CLIP Vision encoding ...'))))
|
738 |
+
|
739 |
+
if not high_vram:
|
740 |
+
load_model_as_complete(image_encoder, target_device=gpu)
|
741 |
+
|
742 |
+
image_encoder_output = hf_clip_vision_encode(input_image_np, feature_extractor, image_encoder)
|
743 |
+
image_encoder_last_hidden_state = image_encoder_output.last_hidden_state
|
744 |
+
|
745 |
+
if has_end_image:
|
746 |
+
end_image_encoder_output = hf_clip_vision_encode(end_image_np, feature_extractor, image_encoder)
|
747 |
+
end_image_encoder_last_hidden_state = end_image_encoder_output.last_hidden_state
|
748 |
+
# Combine both image embeddings or use a weighted approach
|
749 |
+
image_encoder_last_hidden_state = (image_encoder_last_hidden_state + end_image_encoder_last_hidden_state) / 2
|
750 |
+
|
751 |
+
# Clean GPU
|
752 |
+
if not high_vram:
|
753 |
+
unload_complete_models(
|
754 |
+
image_encoder
|
755 |
+
)
|
756 |
+
|
757 |
+
return [start_latent, end_latent, image_encoder_last_hidden_state]
|
758 |
+
|
759 |
+
[start_latent, end_latent, image_encoder_last_hidden_state] = get_start_latent(input_image, has_end_image, end_image, height, width, vae, gpu, image_encoder, high_vram)
|
760 |
+
del input_image
|
761 |
+
del end_image
|
762 |
+
|
763 |
+
# Dtype
|
764 |
+
image_encoder_last_hidden_state = image_encoder_last_hidden_state.to(transformer.dtype)
|
765 |
+
|
766 |
+
# Sampling
|
767 |
+
stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Start sampling ...'))))
|
768 |
+
|
769 |
+
rnd = torch.Generator("cpu").manual_seed(seed)
|
770 |
+
num_frames = latent_window_size * 4 - 3
|
771 |
+
|
772 |
+
history_latents = torch.zeros(size=(1, 16, 1 + 2 + 16, height // 8, width // 8), dtype=torch.float32, device=cpu)
|
773 |
+
start_latent = start_latent.to(history_latents)
|
774 |
+
if has_end_image:
|
775 |
+
end_latent = end_latent.to(history_latents)
|
776 |
+
|
777 |
+
history_pixels = None
|
778 |
+
total_generated_latent_frames = 0
|
779 |
+
|
780 |
+
if total_latent_sections > 4:
|
781 |
+
# In theory the latent_paddings should follow the else sequence, but it seems that duplicating some
|
782 |
+
# items looks better than expanding it when total_latent_sections > 4
|
783 |
+
# One can try to remove below trick and just
|
784 |
+
# use `latent_paddings = list(reversed(range(total_latent_sections)))` to compare
|
785 |
+
latent_paddings = [3] + [2] * (total_latent_sections - 3) + [1, 0]
|
786 |
+
else:
|
787 |
+
# Convert an iterator to a list
|
788 |
+
latent_paddings = list(range(total_latent_sections - 1, -1, -1))
|
789 |
+
|
790 |
+
if enable_preview:
|
791 |
+
def callback(d):
|
792 |
+
preview = d['denoised']
|
793 |
+
preview = vae_decode_fake(preview)
|
794 |
+
|
795 |
+
preview = (preview * 255.0).detach().cpu().numpy().clip(0, 255).astype(np.uint8)
|
796 |
+
preview = einops.rearrange(preview, 'b c t h w -> (b h) (t w) c')
|
797 |
+
|
798 |
+
if stream.input_queue.top() == 'end':
|
799 |
+
stream.output_queue.push(('end', None))
|
800 |
+
raise KeyboardInterrupt('User ends the task.')
|
801 |
+
|
802 |
+
current_step = d['i'] + 1
|
803 |
+
percentage = int(100.0 * current_step / steps)
|
804 |
+
hint = f'Sampling {current_step}/{steps}'
|
805 |
+
desc = f'Total generated frames: {int(max(0, total_generated_latent_frames * 4 - 3))}, Video length: {max(0, (total_generated_latent_frames * 4 - 3) / fps_number) :.2f} seconds (FPS-30), Resolution: {height}px * {width}px. The video is being extended now ...'
|
806 |
+
stream.output_queue.push(('progress', (preview, desc, make_progress_bar_html(percentage, hint))))
|
807 |
+
return
|
808 |
+
else:
|
809 |
+
def callback(d):
|
810 |
+
return
|
811 |
+
|
812 |
+
def post_process(job_id, start_latent, generated_latents, total_generated_latent_frames, history_latents, high_vram, transformer, gpu, vae, history_pixels, latent_window_size, enable_preview, outputs_folder, mp4_crf, stream, is_last_section):
|
813 |
+
if is_last_section:
|
814 |
+
generated_latents = torch.cat([start_latent.to(generated_latents), generated_latents], dim=2)
|
815 |
+
|
816 |
+
total_generated_latent_frames += int(generated_latents.shape[2])
|
817 |
+
history_latents = torch.cat([generated_latents.to(history_latents), history_latents], dim=2)
|
818 |
+
|
819 |
+
if not high_vram:
|
820 |
+
offload_model_from_device_for_memory_preservation(transformer, target_device=gpu, preserved_memory_gb=8)
|
821 |
+
load_model_as_complete(vae, target_device=gpu)
|
822 |
+
|
823 |
+
if history_pixels is None:
|
824 |
+
history_pixels = vae_decode(history_latents[:, :, :total_generated_latent_frames, :, :], vae).cpu()
|
825 |
+
else:
|
826 |
+
section_latent_frames = (latent_window_size * 2 + 1) if is_last_section else (latent_window_size * 2)
|
827 |
+
overlapped_frames = latent_window_size * 4 - 3
|
828 |
+
|
829 |
+
current_pixels = vae_decode(history_latents[:, :, :min(total_generated_latent_frames, section_latent_frames)], vae).cpu()
|
830 |
+
history_pixels = soft_append_bcthw(current_pixels, history_pixels, overlapped_frames)
|
831 |
+
|
832 |
+
if not high_vram:
|
833 |
+
unload_complete_models(vae)
|
834 |
+
|
835 |
+
if enable_preview or is_last_section:
|
836 |
+
output_filename = os.path.join(outputs_folder, f'{job_id}_{total_generated_latent_frames}.mp4')
|
837 |
+
|
838 |
+
save_bcthw_as_mp4(history_pixels, output_filename, fps=fps_number, crf=mp4_crf)
|
839 |
+
|
840 |
+
print(f'Decoded. Pixel shape {history_pixels.shape}')
|
841 |
+
|
842 |
+
stream.output_queue.push(('file', output_filename))
|
843 |
+
|
844 |
+
return [total_generated_latent_frames, history_latents, history_pixels]
|
845 |
+
|
846 |
+
for latent_padding in latent_paddings:
|
847 |
+
is_last_section = latent_padding == 0
|
848 |
+
is_first_section = latent_padding == latent_paddings[0]
|
849 |
+
latent_padding_size = latent_padding * latent_window_size
|
850 |
+
|
851 |
+
if stream.input_queue.top() == 'end':
|
852 |
+
stream.output_queue.push(('end', None))
|
853 |
+
return
|
854 |
+
|
855 |
+
print(f'latent_padding_size = {latent_padding_size}, is_last_section = {is_last_section}, is_first_section = {is_first_section}')
|
856 |
+
|
857 |
+
if len(prompt_parameters) > 0:
|
858 |
+
[llama_vec, clip_l_pooler, llama_vec_n, clip_l_pooler_n, llama_attention_mask, llama_attention_mask_n] = prompt_parameters.pop(len(prompt_parameters) - 1)
|
859 |
+
|
860 |
+
indices = torch.arange(1 + latent_padding_size + latent_window_size + 1 + (end_stillness if is_first_section else 0) + 2 + 16).unsqueeze(0)
|
861 |
+
clean_latent_indices_pre, blank_indices, latent_indices, clean_latent_indices_post, clean_latent_2x_indices, clean_latent_4x_indices = indices.split([1, latent_padding_size, latent_window_size, 1 + (end_stillness if is_first_section else 0), 2, 16], dim=1)
|
862 |
+
clean_latent_indices = torch.cat([clean_latent_indices_pre, clean_latent_indices_post], dim=1)
|
863 |
+
|
864 |
+
clean_latents_post, clean_latents_2x, clean_latents_4x = history_latents[:, :, :1 + 2 + 16, :, :].split([1, 2, 16], dim=2)
|
865 |
+
|
866 |
+
# Use end image latent for the first section if provided
|
867 |
+
if has_end_image and is_first_section:
|
868 |
+
clean_latents_post = end_latent.expand(-1, -1, 1 + end_stillness, -1, -1)
|
869 |
+
|
870 |
+
clean_latents = torch.cat([start_latent, clean_latents_post], dim=2)
|
871 |
+
|
872 |
+
if not high_vram:
|
873 |
+
unload_complete_models()
|
874 |
+
move_model_to_device_with_memory_preservation(transformer, target_device=gpu, preserved_memory_gb=gpu_memory_preservation)
|
875 |
+
|
876 |
+
if use_teacache:
|
877 |
+
transformer.initialize_teacache(enable_teacache=True, num_steps=steps)
|
878 |
+
else:
|
879 |
+
transformer.initialize_teacache(enable_teacache=False)
|
880 |
+
|
881 |
+
generated_latents = sample_hunyuan(
|
882 |
+
transformer=transformer,
|
883 |
+
sampler='unipc',
|
884 |
+
width=width,
|
885 |
+
height=height,
|
886 |
+
frames=num_frames,
|
887 |
+
real_guidance_scale=cfg,
|
888 |
+
distilled_guidance_scale=gs,
|
889 |
+
guidance_rescale=rs,
|
890 |
+
# shift=3.0,
|
891 |
+
num_inference_steps=steps,
|
892 |
+
generator=rnd,
|
893 |
+
prompt_embeds=llama_vec,
|
894 |
+
prompt_embeds_mask=llama_attention_mask,
|
895 |
+
prompt_poolers=clip_l_pooler,
|
896 |
+
negative_prompt_embeds=llama_vec_n,
|
897 |
+
negative_prompt_embeds_mask=llama_attention_mask_n,
|
898 |
+
negative_prompt_poolers=clip_l_pooler_n,
|
899 |
+
device=gpu,
|
900 |
+
dtype=torch.bfloat16,
|
901 |
+
image_embeddings=image_encoder_last_hidden_state,
|
902 |
+
latent_indices=latent_indices,
|
903 |
+
clean_latents=clean_latents,
|
904 |
+
clean_latent_indices=clean_latent_indices,
|
905 |
+
clean_latents_2x=clean_latents_2x,
|
906 |
+
clean_latent_2x_indices=clean_latent_2x_indices,
|
907 |
+
clean_latents_4x=clean_latents_4x,
|
908 |
+
clean_latent_4x_indices=clean_latent_4x_indices,
|
909 |
+
callback=callback,
|
910 |
+
)
|
911 |
+
del clean_latents
|
912 |
+
del clean_latents_2x
|
913 |
+
del clean_latents_4x
|
914 |
+
del latent_indices
|
915 |
+
del clean_latent_indices
|
916 |
+
del clean_latent_2x_indices
|
917 |
+
del clean_latent_4x_indices
|
918 |
+
|
919 |
+
[total_generated_latent_frames, history_latents, history_pixels] = post_process(job_id, start_latent, generated_latents, total_generated_latent_frames, history_latents, high_vram, transformer, gpu, vae, history_pixels, latent_window_size, enable_preview, outputs_folder, mp4_crf, stream, is_last_section)
|
920 |
+
|
921 |
+
if is_last_section:
|
922 |
+
break
|
923 |
+
except:
|
924 |
+
traceback.print_exc()
|
925 |
+
|
926 |
+
if not high_vram:
|
927 |
+
unload_complete_models(
|
928 |
+
text_encoder, text_encoder_2, image_encoder, vae, transformer
|
929 |
+
)
|
930 |
+
|
931 |
+
stream.output_queue.push(('end', None))
|
932 |
+
return
|
933 |
+
|
934 |
# 20250506 pftq: Modified worker to accept video input and clean frame count
|
935 |
@torch.no_grad()
|
936 |
+
def worker_video(input_video, end_frame, end_stillness, prompts, n_prompt, seed, batch, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, no_resize, mp4_crf, num_clean_frames, vae_batch):
|
937 |
def encode_prompt(prompt, n_prompt):
|
938 |
llama_vec, clip_l_pooler = encode_prompt_conds(prompt, text_encoder, text_encoder_2, tokenizer, tokenizer_2)
|
939 |
|
|
|
958 |
stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Video processing ...'))))
|
959 |
|
960 |
# 20250506 pftq: Encode video
|
961 |
+
start_latent, input_image_np, video_latents, fps, height, width = video_encode(input_video, resolution, no_resize, vae, vae_batch_size=vae_batch, device=gpu)
|
962 |
+
del input_video
|
963 |
+
start_latent = start_latent.to(dtype=torch.float32, device=cpu)
|
964 |
video_latents = video_latents.cpu()
|
965 |
|
966 |
total_latent_sections = (total_second_length * fps) / (latent_window_size * 4)
|
|
|
997 |
load_model_as_complete(image_encoder, target_device=gpu)
|
998 |
|
999 |
image_encoder_output = hf_clip_vision_encode(input_image_np, feature_extractor, image_encoder)
|
1000 |
+
del input_image_np
|
1001 |
+
|
1002 |
+
# 20250507 pftq: Process end frame if provided
|
1003 |
+
if end_frame is not None:
|
1004 |
+
if not high_vram:
|
1005 |
+
load_model_as_complete(vae, target_device=gpu)
|
1006 |
+
|
1007 |
+
stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'End frame encoding ...'))))
|
1008 |
+
end_latent = image_encode(
|
1009 |
+
end_frame, target_width=width, target_height=height, vae=vae,
|
1010 |
+
image_encoder=image_encoder, feature_extractor=feature_extractor, device=gpu
|
1011 |
+
)[0]
|
1012 |
+
del end_frame
|
1013 |
+
end_latent = end_latent.to(dtype=torch.float32, device=cpu)
|
1014 |
+
else:
|
1015 |
+
end_latent = None
|
1016 |
|
1017 |
# Clean GPU
|
1018 |
if not high_vram:
|
1019 |
+
unload_complete_models(image_encoder, vae)
|
1020 |
|
1021 |
image_encoder_last_hidden_state = image_encoder_output.last_hidden_state
|
1022 |
+
del image_encoder_output
|
1023 |
|
1024 |
# Dtype
|
1025 |
image_encoder_last_hidden_state = image_encoder_last_hidden_state.to(transformer.dtype)
|
|
|
1046 |
def callback(d):
|
1047 |
return
|
1048 |
|
1049 |
+
def compute_latent(history_latents, latent_window_size, latent_padding_size, num_clean_frames, start_latent, end_latent, end_stillness, is_end_of_video):
|
1050 |
+
if end_latent is not None and is_end_of_video:
|
1051 |
+
local_end_stillness = end_stillness
|
1052 |
+
local_end_latent = end_latent.expand(-1, -1, 1 + local_end_stillness, -1, -1)
|
1053 |
+
else:
|
1054 |
+
local_end_stillness = 0
|
1055 |
+
local_end_latent = end_latent
|
1056 |
# 20250506 pftq: Use user-specified number of context frames, matching original allocation for num_clean_frames=2
|
1057 |
available_frames = history_latents.shape[2] # Number of latent frames
|
1058 |
max_pixel_frames = min(latent_window_size * 4 - 3, available_frames * 4) # Cap at available pixel frames
|
|
|
1066 |
total_context_frames = num_4x_frames + num_2x_frames + effective_clean_frames
|
1067 |
total_context_frames = min(total_context_frames, available_frames) # 20250507 pftq: Edge case for <=1 sec videos
|
1068 |
|
1069 |
+
indices = torch.arange(0, 1 + num_4x_frames + num_2x_frames + effective_clean_frames + adjusted_latent_frames + ((latent_padding_size + 1 + local_end_stillness) if end_latent is not None else 0)).unsqueeze(0) # 20250507 pftq: latent_window_size to adjusted_latent_frames for edge case for <=1 sec videos
|
1070 |
+
clean_latent_indices_start, clean_latent_4x_indices, clean_latent_2x_indices, clean_latent_1x_indices, latent_indices, blank_indices, clean_latent_indices_post = indices.split(
|
1071 |
+
[1, num_4x_frames, num_2x_frames, effective_clean_frames, adjusted_latent_frames, latent_padding_size if end_latent is not None else 0, (1 + local_end_stillness) if end_latent is not None else 0], dim=1 # 20250507 pftq: latent_window_size to adjusted_latent_frames for edge case for <=1 sec videos
|
1072 |
)
|
1073 |
+
clean_latent_indices = torch.cat([clean_latent_indices_start, clean_latent_1x_indices, clean_latent_indices_post], dim=1)
|
1074 |
|
1075 |
# 20250506 pftq: Split history_latents dynamically based on available frames
|
1076 |
fallback_frame_count = 2 # 20250507 pftq: Changed 0 to 2 Edge case for <=1 sec videos
|
|
|
1103 |
if effective_clean_frames > 0 and split_idx < len(splits):
|
1104 |
clean_latents_1x = splits[split_idx]
|
1105 |
|
1106 |
+
if end_latent is not None:
|
1107 |
+
clean_latents = torch.cat([start_latent, clean_latents_1x, local_end_latent], dim=2)
|
1108 |
+
else:
|
1109 |
+
clean_latents = torch.cat([start_latent, clean_latents_1x], dim=2)
|
1110 |
|
1111 |
# 20250507 pftq: Fix for <=1 sec videos.
|
1112 |
max_frames = min(latent_window_size * 4 - 3, history_latents.shape[2] * 4)
|
|
|
1128 |
history_latents = video_latents
|
1129 |
total_generated_latent_frames = history_latents.shape[2]
|
1130 |
# 20250506 pftq: Initialize history_pixels to fix UnboundLocalError
|
1131 |
+
history_pixels = previous_video = None
|
|
|
1132 |
|
1133 |
+
# 20250509 Generate backwards with end frame for better end frame anchoring
|
1134 |
+
if total_latent_sections > 4:
|
1135 |
+
latent_paddings = [3] + [2] * (total_latent_sections - 3) + [1, 0]
|
1136 |
+
else:
|
1137 |
+
latent_paddings = list(reversed(range(total_latent_sections)))
|
1138 |
+
|
1139 |
+
for section_index, latent_padding in enumerate(latent_paddings):
|
1140 |
+
is_start_of_video = latent_padding == 0
|
1141 |
+
is_end_of_video = latent_padding == latent_paddings[0]
|
1142 |
+
latent_padding_size = latent_padding * latent_window_size
|
1143 |
if stream.input_queue.top() == 'end':
|
1144 |
stream.output_queue.push(('end', None))
|
1145 |
return
|
|
|
1158 |
else:
|
1159 |
transformer.initialize_teacache(enable_teacache=False)
|
1160 |
|
1161 |
+
[max_frames, clean_latents, clean_latents_2x, clean_latents_4x, latent_indices, clean_latents, clean_latent_indices, clean_latent_2x_indices, clean_latent_4x_indices] = compute_latent(history_latents, latent_window_size, latent_padding_size, num_clean_frames, start_latent, end_latent, end_stillness, is_end_of_video)
|
1162 |
|
1163 |
generated_latents = sample_hunyuan(
|
1164 |
transformer=transformer,
|
|
|
1189 |
clean_latent_4x_indices=clean_latent_4x_indices,
|
1190 |
callback=callback,
|
1191 |
)
|
1192 |
+
del clean_latents
|
1193 |
+
del clean_latents_2x
|
1194 |
+
del clean_latents_4x
|
1195 |
+
del latent_indices
|
1196 |
+
del clean_latent_indices
|
1197 |
+
del clean_latent_2x_indices
|
1198 |
+
del clean_latent_4x_indices
|
1199 |
|
1200 |
total_generated_latent_frames += int(generated_latents.shape[2])
|
1201 |
history_latents = torch.cat([history_latents, generated_latents.to(history_latents)], dim=2)
|
|
|
1253 |
stream.output_queue.push(('end', None))
|
1254 |
return
|
1255 |
|
1256 |
+
def get_duration(input_image, end_image, image_position, end_stillness, prompts, generation_mode, n_prompt, seed, resolution, total_second_length, allocation_time, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, mp4_crf, fps_number):
|
1257 |
return allocation_time
|
1258 |
|
|
|
1259 |
@spaces.GPU(duration=get_duration)
|
1260 |
+
def process_on_gpu(input_image, end_image, image_position, end_stillness, prompts, generation_mode, n_prompt, seed, resolution, total_second_length, allocation_time, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, mp4_crf, fps_number
|
1261 |
):
|
1262 |
start = time.time()
|
1263 |
global stream
|
1264 |
stream = AsyncStream()
|
1265 |
|
1266 |
+
async_run(worker_start_end if generation_mode == "start_end" else worker, input_image, end_image, image_position, end_stillness, prompts, n_prompt, seed, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, mp4_crf, fps_number)
|
1267 |
|
1268 |
output_filename = None
|
1269 |
|
|
|
1289 |
((str(hours) + " h, ") if hours != 0 else "") + \
|
1290 |
((str(minutes) + " min, ") if hours != 0 or minutes != 0 else "") + \
|
1291 |
str(secondes) + " sec. " + \
|
1292 |
+
"You can upscale the result with https://huggingface.co/spaces/Nick088/Real-ESRGAN_Pytorch. To make all your generated scenes consistent, you can then apply a face swap on the main character. If you do not see the generated video above, the process may have failed. See the logs for more information. If you see an error like ''NVML_SUCCESS == r INTERNAL ASSERT FAILED'', you probably haven't enough VRAM. Test an example or other options to compare. You can share your inputs to the original space or set your space in public for a peer review.", gr.update(interactive=True), gr.update(interactive=False), gr.update(visible = False)
|
1293 |
break
|
1294 |
|
1295 |
def process(input_image,
|
1296 |
+
end_image,
|
1297 |
image_position=0,
|
1298 |
+
end_stillness=1,
|
1299 |
prompt="",
|
1300 |
generation_mode="image",
|
1301 |
n_prompt="",
|
|
|
1306 |
resolution=640,
|
1307 |
total_second_length=5,
|
1308 |
latent_window_size=9,
|
1309 |
+
steps=30,
|
1310 |
cfg=1.0,
|
1311 |
gs=10.0,
|
1312 |
rs=0.0,
|
1313 |
gpu_memory_preservation=6,
|
1314 |
+
enable_preview=False,
|
1315 |
use_teacache=False,
|
1316 |
mp4_crf=16,
|
1317 |
fps_number=30
|
1318 |
):
|
1319 |
if auto_allocation:
|
1320 |
+
allocation_time = min(total_second_length * 60 * (1.5 if use_teacache else 3.0) * (1 + ((steps - 25) / 25))**2, 600)
|
1321 |
|
1322 |
if torch.cuda.device_count() == 0:
|
1323 |
gr.Warning('Set this space to GPU config to make it work.')
|
|
|
1329 |
|
1330 |
prompts = prompt.split(";")
|
1331 |
|
|
|
1332 |
if generation_mode == "text":
|
1333 |
+
default_height, default_width = resolution, resolution
|
1334 |
input_image = np.ones((default_height, default_width, 3), dtype=np.uint8) * 255
|
1335 |
print("No input image provided. Using a blank white image.")
|
1336 |
+
assert input_image is not None, 'No input image!'
|
1337 |
+
assert (generation_mode != "start_end") or end_image is not None, 'No end image!'
|
1338 |
|
1339 |
yield gr.update(label="Previewed Frames"), None, '', '', gr.update(interactive=False), gr.update(interactive=True), gr.skip()
|
1340 |
|
1341 |
+
gc.collect()
|
1342 |
yield from process_on_gpu(input_image,
|
1343 |
+
end_image,
|
1344 |
image_position,
|
1345 |
+
end_stillness,
|
1346 |
prompts,
|
1347 |
generation_mode,
|
1348 |
n_prompt,
|
|
|
1362 |
fps_number
|
1363 |
)
|
1364 |
|
1365 |
+
def get_duration_video(input_video, end_frame, end_stillness, prompts, n_prompt, seed, batch, resolution, total_second_length, allocation_time, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, no_resize, mp4_crf, num_clean_frames, vae_batch):
|
1366 |
return allocation_time
|
1367 |
|
|
|
1368 |
@spaces.GPU(duration=get_duration_video)
|
1369 |
+
def process_video_on_gpu(input_video, end_frame, end_stillness, prompts, n_prompt, seed, batch, resolution, total_second_length, allocation_time, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, no_resize, mp4_crf, num_clean_frames, vae_batch):
|
1370 |
start = time.time()
|
1371 |
global stream
|
1372 |
stream = AsyncStream()
|
1373 |
|
1374 |
# 20250506 pftq: Pass num_clean_frames, vae_batch, etc
|
1375 |
+
async_run(worker_video, input_video, end_frame, end_stillness, prompts, n_prompt, seed, batch, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, no_resize, mp4_crf, num_clean_frames, vae_batch)
|
1376 |
|
1377 |
output_filename = None
|
1378 |
|
|
|
1399 |
((str(hours) + " h, ") if hours != 0 else "") + \
|
1400 |
((str(minutes) + " min, ") if hours != 0 or minutes != 0 else "") + \
|
1401 |
str(secondes) + " sec. " + \
|
1402 |
+
" You can upscale the result with https://huggingface.co/spaces/Nick088/Real-ESRGAN_Pytorch. To make all your generated scenes consistent, you can then apply a face swap on the main character. If you do not see the generated video above, the process may have failed. See the logs for more information. If you see an error like ''NVML_SUCCESS == r INTERNAL ASSERT FAILED'', you probably haven't enough VRAM. Test an example or other options to compare. You can share your inputs to the original space or set your space in public for a peer review.", '', gr.update(interactive=True), gr.update(interactive=False), gr.update(visible = False)
|
1403 |
break
|
1404 |
|
1405 |
+
def process_video(input_video, end_frame, end_stillness, prompt, n_prompt, randomize_seed, seed, auto_allocation, allocation_time, batch, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, no_resize, mp4_crf, num_clean_frames, vae_batch):
|
1406 |
global high_vram
|
1407 |
if auto_allocation:
|
1408 |
+
allocation_time = min(total_second_length * 60 * (2.5 if use_teacache else 3.5) * (1 + ((steps - 25) / 25))**2, 600)
|
1409 |
|
1410 |
if torch.cuda.device_count() == 0:
|
1411 |
gr.Warning('Set this space to GPU config to make it work.')
|
|
|
1435 |
if cfg > 1:
|
1436 |
gs = 1
|
1437 |
|
1438 |
+
gc.collect()
|
1439 |
+
yield from process_video_on_gpu(input_video, end_frame, end_stillness, prompt, n_prompt, seed, batch, resolution, total_second_length, allocation_time, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, no_resize, mp4_crf, num_clean_frames, vae_batch)
|
1440 |
|
1441 |
def end_process():
|
1442 |
stream.input_queue.push('end')
|
|
|
1506 |
local_storage = gr.BrowserState(default_local_storage)
|
1507 |
with gr.Row():
|
1508 |
with gr.Column():
|
1509 |
+
generation_mode = gr.Radio([["Text-to-Video", "text"], ["Image-to-Video", "image"], ["Start & end frames", "start_end"], ["Video Extension", "video"]], elem_id="generation-mode", label="Input mode", value = "image")
|
1510 |
text_to_video_hint = gr.HTML("Text-to-Video badly works with a flash effect at the start. I discourage to use the Text-to-Video feature. You should rather generate an image with Flux and use Image-to-Video. You will save time.")
|
1511 |
input_image = gr.Image(sources='upload', type="numpy", label="Image", height=320)
|
1512 |
image_position = gr.Slider(label="Image position", minimum=0, maximum=100, value=0, step=1, info='0=Video start; 100=Video end (lower quality)')
|
1513 |
input_video = gr.Video(sources='upload', label="Input Video", height=320)
|
1514 |
+
end_image = gr.Image(sources='upload', type="numpy", label="End Frame (optional)", height=320)
|
1515 |
timeless_prompt = gr.Textbox(label="Timeless prompt", info='Used on the whole duration of the generation', value='', placeholder="The creature starts to move, fast motion, fixed camera, focus motion, consistent arm, consistent position, mute colors, insanely detailed")
|
1516 |
prompt_number = gr.Slider(label="Timed prompt number", minimum=0, maximum=1000, value=0, step=1, info='Prompts will automatically appear')
|
1517 |
|
|
|
1535 |
enable_preview = gr.Checkbox(label='Enable preview', value=True, info='Display a preview around each second generated but it costs 2 sec. for each second generated.')
|
1536 |
use_teacache = gr.Checkbox(label='Use TeaCache', value=False, info='Faster speed and no break in brightness, but often makes hands and fingers slightly worse.')
|
1537 |
|
1538 |
+
n_prompt = gr.Textbox(label="Negative Prompt", value="Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, poorly framed, blurred, blurry, over-smooth", info='Requires using normal CFG (undistilled) instead of Distilled (set Distilled=1 and CFG > 1).')
|
1539 |
|
1540 |
fps_number = gr.Slider(label="Frame per seconds", info="The model is trained for 30 fps so other fps may generate weird results", minimum=10, maximum=60, value=30, step=1)
|
1541 |
+
end_stillness = gr.Slider(label="End stillness", minimum=0, maximum=100, value=0, step=1, info='0=Realistic end; >0=Matches exactly the end image (but the time seems to freeze)')
|
1542 |
|
1543 |
latent_window_size = gr.Slider(label="Latent Window Size", minimum=1, maximum=33, value=9, step=1, info='Generate more frames at a time (larger chunks). Less degradation and better blending but higher VRAM cost. Should not change.')
|
1544 |
steps = gr.Slider(label="Steps", minimum=1, maximum=100, value=30, step=1, info='Increase for more quality, especially if using high non-distilled CFG. If your animation has very few motion, you may have brutal brightness change; this can be fixed increasing the steps.')
|
|
|
1591 |
progress_desc = gr.Markdown('', elem_classes='no-generating-animation')
|
1592 |
progress_bar = gr.HTML('', elem_classes='no-generating-animation')
|
1593 |
|
1594 |
+
ips = [input_image, end_image, image_position, end_stillness, final_prompt, generation_mode, n_prompt, randomize_seed, seed, auto_allocation, allocation_time, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, mp4_crf, fps_number]
|
1595 |
+
ips_video = [input_video, end_image, end_stillness, final_prompt, n_prompt, randomize_seed, seed, auto_allocation, allocation_time, batch, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, no_resize, mp4_crf, num_clean_frames, vae_batch]
|
|
|
1596 |
|
1597 |
gr.Examples(
|
1598 |
label = "✍️ Examples from text",
|
1599 |
examples = [
|
1600 |
[
|
1601 |
None, # input_image
|
1602 |
+
None, # end_image
|
1603 |
0, # image_position
|
1604 |
+
1, # end_stillness
|
1605 |
"Overcrowed street in Japan, photorealistic, realistic, intricate details, 8k, insanely detailed",
|
1606 |
"text", # generation_mode
|
1607 |
+
"Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, poorly framed, blurred, blurry, over-smooth", # n_prompt
|
1608 |
True, # randomize_seed
|
1609 |
42, # seed
|
1610 |
True, # auto_allocation
|
|
|
1635 |
examples = [
|
1636 |
[
|
1637 |
"./img_examples/Example1.png", # input_image
|
1638 |
+
None, # end_image
|
1639 |
0, # image_position
|
1640 |
+
1, # end_stillness
|
1641 |
"A dolphin emerges from the water, photorealistic, realistic, intricate details, 8k, insanely detailed",
|
1642 |
"image", # generation_mode
|
1643 |
+
"Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, poorly framed, blurred, blurry, over-smooth", # n_prompt
|
1644 |
True, # randomize_seed
|
1645 |
42, # seed
|
1646 |
True, # auto_allocation
|
|
|
1654 |
0.0, # rs
|
1655 |
6, # gpu_memory_preservation
|
1656 |
False, # enable_preview
|
1657 |
+
False, # use_teacache
|
1658 |
16, # mp4_crf
|
1659 |
30 # fps_number
|
1660 |
],
|
1661 |
[
|
1662 |
"./img_examples/Example2.webp", # input_image
|
1663 |
+
None, # end_image
|
1664 |
0, # image_position
|
1665 |
+
1, # end_stillness
|
1666 |
"A man on the left and a woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The man talks and the woman listens; A man on the left and a woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The woman talks, the man stops talking and the man listens; A man on the left and a woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The woman talks and the man listens",
|
1667 |
"image", # generation_mode
|
1668 |
+
"Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, poorly framed, blurred, blurry, over-smooth", # n_prompt
|
1669 |
True, # randomize_seed
|
1670 |
42, # seed
|
1671 |
True, # auto_allocation
|
|
|
1679 |
0.0, # rs
|
1680 |
6, # gpu_memory_preservation
|
1681 |
False, # enable_preview
|
1682 |
+
False, # use_teacache
|
1683 |
16, # mp4_crf
|
1684 |
30 # fps_number
|
1685 |
],
|
1686 |
[
|
1687 |
"./img_examples/Example2.webp", # input_image
|
1688 |
+
None, # end_image
|
1689 |
0, # image_position
|
1690 |
+
1, # end_stillness
|
1691 |
"A man on the left and a woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The woman talks and the man listens; A man on the left and a woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The man talks, the woman stops talking and the woman listens A man on the left and a woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The man talks and the woman listens",
|
1692 |
"image", # generation_mode
|
1693 |
+
"Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, poorly framed, blurred, blurry, over-smooth", # n_prompt
|
1694 |
True, # randomize_seed
|
1695 |
42, # seed
|
1696 |
True, # auto_allocation
|
|
|
1704 |
0.0, # rs
|
1705 |
6, # gpu_memory_preservation
|
1706 |
False, # enable_preview
|
1707 |
+
False, # use_teacache
|
1708 |
16, # mp4_crf
|
1709 |
30 # fps_number
|
1710 |
],
|
1711 |
[
|
1712 |
"./img_examples/Example3.jpg", # input_image
|
1713 |
+
None, # end_image
|
1714 |
0, # image_position
|
1715 |
+
1, # end_stillness
|
1716 |
+
"एउटा केटा दायाँतिर हिँडिरहेको छ, पूर्ण दृश्य, पूर्ण-लम्बाइको दृश्य, कार्टुन",
|
1717 |
"image", # generation_mode
|
1718 |
+
"हात छुटेको, लामो हात, अवास्तविक स्थिति, असम्भव विकृति, देखिने हड्डी, मांसपेशी संकुचन, कमजोर फ्रेम, धमिलो, धमिलो, अत्यधिक चिल्लो", # n_prompt
|
1719 |
True, # randomize_seed
|
1720 |
42, # seed
|
1721 |
True, # auto_allocation
|
|
|
1729 |
0.0, # rs
|
1730 |
6, # gpu_memory_preservation
|
1731 |
False, # enable_preview
|
1732 |
+
False, # use_teacache
|
1733 |
16, # mp4_crf
|
1734 |
30 # fps_number
|
1735 |
],
|
1736 |
[
|
1737 |
"./img_examples/Example4.webp", # input_image
|
1738 |
+
None, # end_image
|
1739 |
100, # image_position
|
1740 |
+
1, # end_stillness
|
1741 |
"A building starting to explode, photorealistic, realisitc, 8k, insanely detailed",
|
1742 |
"image", # generation_mode
|
1743 |
+
"Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, poorly framed, blurred, blurry, over-smooth", # n_prompt
|
1744 |
True, # randomize_seed
|
1745 |
42, # seed
|
1746 |
True, # auto_allocation
|
|
|
1766 |
cache_examples = False,
|
1767 |
)
|
1768 |
|
1769 |
+
gr.Examples(
|
1770 |
+
label = "🖼️ Examples from start and end frames",
|
1771 |
+
examples = [
|
1772 |
+
[
|
1773 |
+
"./img_examples/Example5.png", # input_image
|
1774 |
+
"./img_examples/Example6.png", # end_image
|
1775 |
+
0, # image_position
|
1776 |
+
0, # end_stillness
|
1777 |
+
"A woman jumps out of the train and arrives on the ground, viewed from the outside, photorealistic, realistic, amateur photography, midday, insanely detailed, 8k", # prompt
|
1778 |
+
"start_end", # generation_mode
|
1779 |
+
"Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, poorly framed, blurred, blurry, over-smooth, jumpcut, crossfader, crossfading", # n_prompt
|
1780 |
+
True, # randomize_seed
|
1781 |
+
42, # seed
|
1782 |
+
True, # auto_allocation
|
1783 |
+
180, # allocation_time
|
1784 |
+
672, # resolution
|
1785 |
+
1, # total_second_length
|
1786 |
+
9, # latent_window_size
|
1787 |
+
30, # steps
|
1788 |
+
1.0, # cfg
|
1789 |
+
10.0, # gs
|
1790 |
+
0.0, # rs
|
1791 |
+
6, # gpu_memory_preservation
|
1792 |
+
False, # enable_preview
|
1793 |
+
False, # use_teacache
|
1794 |
+
16, # mp4_crf
|
1795 |
+
30 # fps_number
|
1796 |
+
],
|
1797 |
+
],
|
1798 |
+
run_on_click = True,
|
1799 |
+
fn = process,
|
1800 |
+
inputs = ips,
|
1801 |
+
outputs = [result_video, preview_image, progress_desc, progress_bar, start_button, end_button, warning],
|
1802 |
+
cache_examples = False,
|
1803 |
+
)
|
1804 |
+
|
1805 |
gr.Examples(
|
1806 |
label = "🎥 Examples from video",
|
1807 |
examples = [
|
1808 |
[
|
1809 |
"./img_examples/Example1.mp4", # input_video
|
1810 |
+
None, # end_image
|
1811 |
+
1, # end_stillness
|
1812 |
"View of the sea as far as the eye can see, from the seaside, a piece of land is barely visible on the horizon at the middle, the sky is radiant, reflections of the sun in the water, photorealistic, realistic, intricate details, 8k, insanely detailed",
|
1813 |
+
"Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, poorly framed, blurred, blurry, over-smooth, jumpcut, crossfader, crossfading", # n_prompt
|
1814 |
True, # randomize_seed
|
1815 |
42, # seed
|
1816 |
True, # auto_allocation
|
|
|
1825 |
0.0, # rs
|
1826 |
6, # gpu_memory_preservation
|
1827 |
False, # enable_preview
|
1828 |
+
False, # use_teacache
|
1829 |
+
False, # no_resize
|
1830 |
+
16, # mp4_crf
|
1831 |
+
5, # num_clean_frames
|
1832 |
+
default_vae
|
1833 |
+
],
|
1834 |
+
[
|
1835 |
+
"./img_examples/Example1.mp4", # input_video
|
1836 |
+
"./img_examples/Example1.png", # end_image
|
1837 |
+
1, # end_stillness
|
1838 |
+
"View of the sea as far as the eye can see, from the seaside, a piece of land is barely visible on the horizon at the middle, the sky is radiant, reflections of the sun in the water, photorealistic, realistic, intricate details, 8k, insanely detailed",
|
1839 |
+
"Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, poorly framed, blurred, blurry, over-smooth, jumpcut, crossfader, crossfading", # n_prompt
|
1840 |
+
True, # randomize_seed
|
1841 |
+
42, # seed
|
1842 |
+
True, # auto_allocation
|
1843 |
+
180, # allocation_time
|
1844 |
+
1, # batch
|
1845 |
+
672, # resolution
|
1846 |
+
1, # total_second_length
|
1847 |
+
9, # latent_window_size
|
1848 |
+
30, # steps
|
1849 |
+
1.0, # cfg
|
1850 |
+
10.0, # gs
|
1851 |
+
0.0, # rs
|
1852 |
+
6, # gpu_memory_preservation
|
1853 |
+
False, # enable_preview
|
1854 |
+
False, # use_teacache
|
1855 |
False, # no_resize
|
1856 |
16, # mp4_crf
|
1857 |
5, # num_clean_frames
|
|
|
1881 |
def check_parameters(generation_mode, input_image, input_video):
|
1882 |
if generation_mode == "image" and input_image is None:
|
1883 |
raise gr.Error("Please provide an image to extend.")
|
1884 |
+
if generation_mode == "start_end" and input_image is None:
|
1885 |
+
raise gr.Error("Please provide an image to extend.")
|
1886 |
if generation_mode == "video" and input_video is None:
|
1887 |
raise gr.Error("Please provide a video to extend.")
|
1888 |
return [gr.update(interactive=True), gr.update(visible = True)]
|
1889 |
|
1890 |
def handle_generation_mode_change(generation_mode_data):
|
1891 |
if generation_mode_data == "text":
|
1892 |
+
return [
|
1893 |
+
gr.update(visible = True), # text_to_video_hint
|
1894 |
+
gr.update(visible = False), # image_position
|
1895 |
+
gr.update(visible = False), # input_image
|
1896 |
+
gr.update(visible = False), # end_image
|
1897 |
+
gr.update(visible = False), # end_stillness
|
1898 |
+
gr.update(visible = False), # input_video
|
1899 |
+
gr.update(visible = True), # start_button
|
1900 |
+
gr.update(visible = False), # start_button_video
|
1901 |
+
gr.update(visible = False), # no_resize
|
1902 |
+
gr.update(visible = False), # batch
|
1903 |
+
gr.update(visible = False), # num_clean_frames
|
1904 |
+
gr.update(visible = False), # vae_batch
|
1905 |
+
gr.update(visible = False), # prompt_hint
|
1906 |
+
gr.update(visible = True) # fps_number
|
1907 |
+
]
|
1908 |
elif generation_mode_data == "image":
|
1909 |
+
return [
|
1910 |
+
gr.update(visible = False), # text_to_video_hint
|
1911 |
+
gr.update(visible = True), # image_position
|
1912 |
+
gr.update(visible = True), # input_image
|
1913 |
+
gr.update(visible = False), # end_image
|
1914 |
+
gr.update(visible = False), # end_stillness
|
1915 |
+
gr.update(visible = False), # input_video
|
1916 |
+
gr.update(visible = True), # start_button
|
1917 |
+
gr.update(visible = False), # start_button_video
|
1918 |
+
gr.update(visible = False), # no_resize
|
1919 |
+
gr.update(visible = False), # batch
|
1920 |
+
gr.update(visible = False), # num_clean_frames
|
1921 |
+
gr.update(visible = False), # vae_batch
|
1922 |
+
gr.update(visible = False), # prompt_hint
|
1923 |
+
gr.update(visible = True) # fps_number
|
1924 |
+
]
|
1925 |
+
elif generation_mode_data == "start_end":
|
1926 |
+
return [
|
1927 |
+
gr.update(visible = False), # text_to_video_hint
|
1928 |
+
gr.update(visible = False), # image_position
|
1929 |
+
gr.update(visible = True), # input_image
|
1930 |
+
gr.update(visible = True), # end_image
|
1931 |
+
gr.update(visible = True), # end_stillness
|
1932 |
+
gr.update(visible = False), # input_video
|
1933 |
+
gr.update(visible = True), # start_button
|
1934 |
+
gr.update(visible = False), # start_button_video
|
1935 |
+
gr.update(visible = False), # no_resize
|
1936 |
+
gr.update(visible = False), # batch
|
1937 |
+
gr.update(visible = False), # num_clean_frames
|
1938 |
+
gr.update(visible = False), # vae_batch
|
1939 |
+
gr.update(visible = False), # prompt_hint
|
1940 |
+
gr.update(visible = True) # fps_number
|
1941 |
+
]
|
1942 |
elif generation_mode_data == "video":
|
1943 |
+
return [
|
1944 |
+
gr.update(visible = False), # text_to_video_hint
|
1945 |
+
gr.update(visible = False), # image_position
|
1946 |
+
gr.update(visible = False), # input_image
|
1947 |
+
gr.update(visible = True), # end_image
|
1948 |
+
gr.update(visible = True), # end_stillness
|
1949 |
+
gr.update(visible = True), # input_video
|
1950 |
+
gr.update(visible = False), # start_button
|
1951 |
+
gr.update(visible = True), # start_button_video
|
1952 |
+
gr.update(visible = True), # no_resize
|
1953 |
+
gr.update(visible = True), # batch
|
1954 |
+
gr.update(visible = True), # num_clean_frames
|
1955 |
+
gr.update(visible = True), # vae_batch
|
1956 |
+
gr.update(visible = True), # prompt_hint
|
1957 |
+
gr.update(visible = False) # fps_number
|
1958 |
+
]
|
1959 |
|
1960 |
prompt_number.change(fn=handle_prompt_number_change, inputs=[], outputs=[])
|
1961 |
timeless_prompt.change(fn=handle_timeless_prompt_change, inputs=[timeless_prompt], outputs=[final_prompt])
|
|
|
1977 |
generation_mode.change(
|
1978 |
fn=handle_generation_mode_change,
|
1979 |
inputs=[generation_mode],
|
1980 |
+
outputs=[text_to_video_hint, image_position, input_image, end_image, end_stillness, input_video, start_button, start_button_video, no_resize, batch, num_clean_frames, vae_batch, prompt_hint, fps_number]
|
1981 |
)
|
1982 |
|
1983 |
# Update display when the page loads
|
|
|
1985 |
fn=handle_generation_mode_change, inputs = [
|
1986 |
generation_mode
|
1987 |
], outputs = [
|
1988 |
+
text_to_video_hint, image_position, input_image, end_image, end_stillness, input_video, start_button, start_button_video, no_resize, batch, num_clean_frames, vae_batch, prompt_hint, fps_number
|
1989 |
]
|
1990 |
)
|
1991 |
|