Track Point Editor
+ + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/wan/utils/__init__.py b/wan/utils/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..6e9a339e69fd55dd226d3ce242613c19bd690522
--- /dev/null
+++ b/wan/utils/__init__.py
@@ -0,0 +1,8 @@
+from .fm_solvers import (FlowDPMSolverMultistepScheduler, get_sampling_sigmas,
+ retrieve_timesteps)
+from .fm_solvers_unipc import FlowUniPCMultistepScheduler
+
+__all__ = [
+ 'HuggingfaceTokenizer', 'get_sampling_sigmas', 'retrieve_timesteps',
+ 'FlowDPMSolverMultistepScheduler', 'FlowUniPCMultistepScheduler'
+]
diff --git a/wan/utils/basic_flowmatch.py b/wan/utils/basic_flowmatch.py
new file mode 100644
index 0000000000000000000000000000000000000000..ceb4657b0691ae2c70b7d0e94603b03b48b28f65
--- /dev/null
+++ b/wan/utils/basic_flowmatch.py
@@ -0,0 +1,83 @@
+"""
+The following code is copied from https://github.com/modelscope/DiffSynth-Studio/blob/main/diffsynth/schedulers/flow_match.py
+"""
+import torch
+
+
+class FlowMatchScheduler():
+
+ def __init__(self, num_inference_steps=100, num_train_timesteps=1000, shift=3.0, sigma_max=1.0, sigma_min=0.003 / 1.002, inverse_timesteps=False, extra_one_step=False, reverse_sigmas=False):
+ self.num_train_timesteps = num_train_timesteps
+ self.shift = shift
+ self.sigma_max = sigma_max
+ self.sigma_min = sigma_min
+ self.inverse_timesteps = inverse_timesteps
+ self.extra_one_step = extra_one_step
+ self.reverse_sigmas = reverse_sigmas
+ self.set_timesteps(num_inference_steps)
+
+ def set_timesteps(self, num_inference_steps=100, denoising_strength=1.0, training=False):
+ sigma_start = self.sigma_min + \
+ (self.sigma_max - self.sigma_min) * denoising_strength
+ if self.extra_one_step:
+ self.sigmas = torch.linspace(
+ sigma_start, self.sigma_min, num_inference_steps + 1)[:-1]
+ else:
+ self.sigmas = torch.linspace(
+ sigma_start, self.sigma_min, num_inference_steps)
+ if self.inverse_timesteps:
+ self.sigmas = torch.flip(self.sigmas, dims=[0])
+ self.sigmas = self.shift * self.sigmas / \
+ (1 + (self.shift - 1) * self.sigmas)
+ if self.reverse_sigmas:
+ self.sigmas = 1 - self.sigmas
+ self.timesteps = self.sigmas * self.num_train_timesteps
+ if training:
+ x = self.timesteps
+ y = torch.exp(-2 * ((x - num_inference_steps / 2) /
+ num_inference_steps) ** 2)
+ y_shifted = y - y.min()
+ bsmntw_weighing = y_shifted * \
+ (num_inference_steps / y_shifted.sum())
+ self.linear_timesteps_weights = bsmntw_weighing
+
+ def step(self, model_output, timestep, sample, to_final=False):
+ self.sigmas = self.sigmas.to(model_output.device)
+ self.timesteps = self.timesteps.to(model_output.device)
+ timestep_id = torch.argmin(
+ (self.timesteps - timestep).abs(), dim=0)
+ sigma = self.sigmas[timestep_id].reshape(-1, 1, 1, 1)
+ if to_final or (timestep_id + 1 >= len(self.timesteps)).any():
+ sigma_ = 1 if (
+ self.inverse_timesteps or self.reverse_sigmas) else 0
+ else:
+ sigma_ = self.sigmas[timestep_id + 1].reshape(-1, 1, 1, 1)
+ prev_sample = sample + model_output * (sigma_ - sigma)
+ return [prev_sample]
+
+ def add_noise(self, original_samples, noise, timestep):
+ """
+ Diffusion forward corruption process.
+ Input:
+ - clean_latent: the clean latent with shape [B, C, H, W]
+ - noise: the noise with shape [B, C, H, W]
+ - timestep: the timestep with shape [B]
+ Output: the corrupted latent with shape [B, C, H, W]
+ """
+ self.sigmas = self.sigmas.to(noise.device)
+ self.timesteps = self.timesteps.to(noise.device)
+ timestep_id = torch.argmin(
+ (self.timesteps.unsqueeze(0) - timestep.unsqueeze(1)).abs(), dim=1)
+ sigma = self.sigmas[timestep_id].reshape(-1, 1, 1, 1)
+ sample = (1 - sigma) * original_samples + sigma * noise
+ return sample.type_as(noise)
+
+ def training_target(self, sample, noise, timestep):
+ target = noise - sample
+ return target
+
+ def training_weight(self, timestep):
+ timestep_id = torch.argmin(
+ (self.timesteps - timestep.to(self.timesteps.device)).abs())
+ weights = self.linear_timesteps_weights[timestep_id]
+ return weights
\ No newline at end of file
diff --git a/wan/utils/cammmaster_tools.py b/wan/utils/cammmaster_tools.py
new file mode 100644
index 0000000000000000000000000000000000000000..6e255a04bb0ad71865127078cbd332c603f35220
--- /dev/null
+++ b/wan/utils/cammmaster_tools.py
@@ -0,0 +1,63 @@
+import torch
+from einops import rearrange
+import numpy as np
+import json
+
+class Camera(object):
+ def __init__(self, c2w):
+ c2w_mat = np.array(c2w).reshape(4, 4)
+ self.c2w_mat = c2w_mat
+ self.w2c_mat = np.linalg.inv(c2w_mat)
+
+
+
+def parse_matrix(matrix_str):
+ rows = matrix_str.strip().split('] [')
+ matrix = []
+ for row in rows:
+ row = row.replace('[', '').replace(']', '')
+ matrix.append(list(map(float, row.split())))
+ return np.array(matrix)
+
+
+def get_relative_pose(cam_params):
+ abs_w2cs = [cam_param.w2c_mat for cam_param in cam_params]
+ abs_c2ws = [cam_param.c2w_mat for cam_param in cam_params]
+
+ cam_to_origin = 0
+ target_cam_c2w = np.array([
+ [1, 0, 0, 0],
+ [0, 1, 0, -cam_to_origin],
+ [0, 0, 1, 0],
+ [0, 0, 0, 1]
+ ])
+ abs2rel = target_cam_c2w @ abs_w2cs[0]
+ ret_poses = [target_cam_c2w, ] + [abs2rel @ abs_c2w for abs_c2w in abs_c2ws[1:]]
+ ret_poses = np.array(ret_poses, dtype=np.float32)
+ return ret_poses
+
+
+def get_camera_embedding(cam_type, num_frames=81):
+
+ # load camera
+ tgt_camera_path = "wan/camera_extrinsics.json"
+ with open(tgt_camera_path, 'r') as file:
+ cam_data = json.load(file)
+
+ cam_idx = list(range(num_frames))[::4]
+ traj = [parse_matrix(cam_data[f"frame{idx}"][f"cam{int(cam_type):02d}"]) for idx in cam_idx]
+ traj = np.stack(traj).transpose(0, 2, 1)
+ c2ws = []
+ for c2w in traj:
+ c2w = c2w[:, [1, 2, 0, 3]]
+ c2w[:3, 1] *= -1.
+ c2w[:3, 3] /= 100
+ c2ws.append(c2w)
+ tgt_cam_params = [Camera(cam_param) for cam_param in c2ws]
+ relative_poses = []
+ for i in range(len(tgt_cam_params)):
+ relative_pose = get_relative_pose([tgt_cam_params[0], tgt_cam_params[i]])
+ relative_poses.append(torch.as_tensor(relative_pose)[:,:3,:][1])
+ pose_embedding = torch.stack(relative_poses, dim=0) # 21x3x4
+ pose_embedding = rearrange(pose_embedding, 'b c d -> b (c d)')
+ return pose_embedding
diff --git a/wan/utils/fm_solvers.py b/wan/utils/fm_solvers.py
new file mode 100644
index 0000000000000000000000000000000000000000..c908969e24849ce1381a8df9d5eb401dccf66524
--- /dev/null
+++ b/wan/utils/fm_solvers.py
@@ -0,0 +1,857 @@
+# Copied from https://github.com/huggingface/diffusers/blob/main/src/diffusers/schedulers/scheduling_dpmsolver_multistep.py
+# Convert dpm solver for flow matching
+# Copyright 2024-2025 The Alibaba Wan Team Authors. All rights reserved.
+
+import inspect
+import math
+from typing import List, Optional, Tuple, Union
+
+import numpy as np
+import torch
+from diffusers.configuration_utils import ConfigMixin, register_to_config
+from diffusers.schedulers.scheduling_utils import (KarrasDiffusionSchedulers,
+ SchedulerMixin,
+ SchedulerOutput)
+from diffusers.utils import deprecate, is_scipy_available
+from diffusers.utils.torch_utils import randn_tensor
+
+if is_scipy_available():
+ pass
+
+
+def get_sampling_sigmas(sampling_steps, shift):
+ sigma = np.linspace(1, 0, sampling_steps + 1)[:sampling_steps]
+ sigma = (shift * sigma / (1 + (shift - 1) * sigma))
+
+ return sigma
+
+
+def retrieve_timesteps(
+ scheduler,
+ num_inference_steps=None,
+ device=None,
+ timesteps=None,
+ sigmas=None,
+ **kwargs,
+):
+ if timesteps is not None and sigmas is not None:
+ raise ValueError(
+ "Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values"
+ )
+ if timesteps is not None:
+ accepts_timesteps = "timesteps" in set(
+ inspect.signature(scheduler.set_timesteps).parameters.keys())
+ if not accepts_timesteps:
+ raise ValueError(
+ f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
+ f" timestep schedules. Please check whether you are using the correct scheduler."
+ )
+ scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs)
+ timesteps = scheduler.timesteps
+ num_inference_steps = len(timesteps)
+ elif sigmas is not None:
+ accept_sigmas = "sigmas" in set(
+ inspect.signature(scheduler.set_timesteps).parameters.keys())
+ if not accept_sigmas:
+ raise ValueError(
+ f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
+ f" sigmas schedules. Please check whether you are using the correct scheduler."
+ )
+ scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs)
+ timesteps = scheduler.timesteps
+ num_inference_steps = len(timesteps)
+ else:
+ scheduler.set_timesteps(num_inference_steps, device=device, **kwargs)
+ timesteps = scheduler.timesteps
+ return timesteps, num_inference_steps
+
+
+class FlowDPMSolverMultistepScheduler(SchedulerMixin, ConfigMixin):
+ """
+ `FlowDPMSolverMultistepScheduler` is a fast dedicated high-order solver for diffusion ODEs.
+ This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic
+ methods the library implements for all schedulers such as loading and saving.
+ Args:
+ num_train_timesteps (`int`, defaults to 1000):
+ The number of diffusion steps to train the model. This determines the resolution of the diffusion process.
+ solver_order (`int`, defaults to 2):
+ The DPMSolver order which can be `1`, `2`, or `3`. It is recommended to use `solver_order=2` for guided
+ sampling, and `solver_order=3` for unconditional sampling. This affects the number of model outputs stored
+ and used in multistep updates.
+ prediction_type (`str`, defaults to "flow_prediction"):
+ Prediction type of the scheduler function; must be `flow_prediction` for this scheduler, which predicts
+ the flow of the diffusion process.
+ shift (`float`, *optional*, defaults to 1.0):
+ A factor used to adjust the sigmas in the noise schedule. It modifies the step sizes during the sampling
+ process.
+ use_dynamic_shifting (`bool`, defaults to `False`):
+ Whether to apply dynamic shifting to the timesteps based on image resolution. If `True`, the shifting is
+ applied on the fly.
+ thresholding (`bool`, defaults to `False`):
+ Whether to use the "dynamic thresholding" method. This method adjusts the predicted sample to prevent
+ saturation and improve photorealism.
+ dynamic_thresholding_ratio (`float`, defaults to 0.995):
+ The ratio for the dynamic thresholding method. Valid only when `thresholding=True`.
+ sample_max_value (`float`, defaults to 1.0):
+ The threshold value for dynamic thresholding. Valid only when `thresholding=True` and
+ `algorithm_type="dpmsolver++"`.
+ algorithm_type (`str`, defaults to `dpmsolver++`):
+ Algorithm type for the solver; can be `dpmsolver`, `dpmsolver++`, `sde-dpmsolver` or `sde-dpmsolver++`. The
+ `dpmsolver` type implements the algorithms in the [DPMSolver](https://huggingface.co/papers/2206.00927)
+ paper, and the `dpmsolver++` type implements the algorithms in the
+ [DPMSolver++](https://huggingface.co/papers/2211.01095) paper. It is recommended to use `dpmsolver++` or
+ `sde-dpmsolver++` with `solver_order=2` for guided sampling like in Stable Diffusion.
+ solver_type (`str`, defaults to `midpoint`):
+ Solver type for the second-order solver; can be `midpoint` or `heun`. The solver type slightly affects the
+ sample quality, especially for a small number of steps. It is recommended to use `midpoint` solvers.
+ lower_order_final (`bool`, defaults to `True`):
+ Whether to use lower-order solvers in the final steps. Only valid for < 15 inference steps. This can
+ stabilize the sampling of DPMSolver for steps < 15, especially for steps <= 10.
+ euler_at_final (`bool`, defaults to `False`):
+ Whether to use Euler's method in the final step. It is a trade-off between numerical stability and detail
+ richness. This can stabilize the sampling of the SDE variant of DPMSolver for small number of inference
+ steps, but sometimes may result in blurring.
+ final_sigmas_type (`str`, *optional*, defaults to "zero"):
+ The final `sigma` value for the noise schedule during the sampling process. If `"sigma_min"`, the final
+ sigma is the same as the last sigma in the training schedule. If `zero`, the final sigma is set to 0.
+ lambda_min_clipped (`float`, defaults to `-inf`):
+ Clipping threshold for the minimum value of `lambda(t)` for numerical stability. This is critical for the
+ cosine (`squaredcos_cap_v2`) noise schedule.
+ variance_type (`str`, *optional*):
+ Set to "learned" or "learned_range" for diffusion models that predict variance. If set, the model's output
+ contains the predicted Gaussian variance.
+ """
+
+ _compatibles = [e.name for e in KarrasDiffusionSchedulers]
+ order = 1
+
+ @register_to_config
+ def __init__(
+ self,
+ num_train_timesteps: int = 1000,
+ solver_order: int = 2,
+ prediction_type: str = "flow_prediction",
+ shift: Optional[float] = 1.0,
+ use_dynamic_shifting=False,
+ thresholding: bool = False,
+ dynamic_thresholding_ratio: float = 0.995,
+ sample_max_value: float = 1.0,
+ algorithm_type: str = "dpmsolver++",
+ solver_type: str = "midpoint",
+ lower_order_final: bool = True,
+ euler_at_final: bool = False,
+ final_sigmas_type: Optional[str] = "zero", # "zero", "sigma_min"
+ lambda_min_clipped: float = -float("inf"),
+ variance_type: Optional[str] = None,
+ invert_sigmas: bool = False,
+ ):
+ if algorithm_type in ["dpmsolver", "sde-dpmsolver"]:
+ deprecation_message = f"algorithm_type {algorithm_type} is deprecated and will be removed in a future version. Choose from `dpmsolver++` or `sde-dpmsolver++` instead"
+ deprecate("algorithm_types dpmsolver and sde-dpmsolver", "1.0.0",
+ deprecation_message)
+
+ # settings for DPM-Solver
+ if algorithm_type not in [
+ "dpmsolver", "dpmsolver++", "sde-dpmsolver", "sde-dpmsolver++"
+ ]:
+ if algorithm_type == "deis":
+ self.register_to_config(algorithm_type="dpmsolver++")
+ else:
+ raise NotImplementedError(
+ f"{algorithm_type} is not implemented for {self.__class__}")
+
+ if solver_type not in ["midpoint", "heun"]:
+ if solver_type in ["logrho", "bh1", "bh2"]:
+ self.register_to_config(solver_type="midpoint")
+ else:
+ raise NotImplementedError(
+ f"{solver_type} is not implemented for {self.__class__}")
+
+ if algorithm_type not in ["dpmsolver++", "sde-dpmsolver++"
+ ] and final_sigmas_type == "zero":
+ raise ValueError(
+ f"`final_sigmas_type` {final_sigmas_type} is not supported for `algorithm_type` {algorithm_type}. Please choose `sigma_min` instead."
+ )
+
+ # setable values
+ self.num_inference_steps = None
+ alphas = np.linspace(1, 1 / num_train_timesteps,
+ num_train_timesteps)[::-1].copy()
+ sigmas = 1.0 - alphas
+ sigmas = torch.from_numpy(sigmas).to(dtype=torch.float32)
+
+ if not use_dynamic_shifting:
+ # when use_dynamic_shifting is True, we apply the timestep shifting on the fly based on the image resolution
+ sigmas = shift * sigmas / (1 +
+ (shift - 1) * sigmas) # pyright: ignore
+
+ self.sigmas = sigmas
+ self.timesteps = sigmas * num_train_timesteps
+
+ self.model_outputs = [None] * solver_order
+ self.lower_order_nums = 0
+ self._step_index = None
+ self._begin_index = None
+
+ # self.sigmas = self.sigmas.to(
+ # "cpu") # to avoid too much CPU/GPU communication
+ self.sigma_min = self.sigmas[-1].item()
+ self.sigma_max = self.sigmas[0].item()
+
+ @property
+ def step_index(self):
+ """
+ The index counter for current timestep. It will increase 1 after each scheduler step.
+ """
+ return self._step_index
+
+ @property
+ def begin_index(self):
+ """
+ The index for the first timestep. It should be set from pipeline with `set_begin_index` method.
+ """
+ return self._begin_index
+
+ # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.set_begin_index
+ def set_begin_index(self, begin_index: int = 0):
+ """
+ Sets the begin index for the scheduler. This function should be run from pipeline before the inference.
+ Args:
+ begin_index (`int`):
+ The begin index for the scheduler.
+ """
+ self._begin_index = begin_index
+
+ # Modified from diffusers.schedulers.scheduling_flow_match_euler_discrete.FlowMatchEulerDiscreteScheduler.set_timesteps
+ def set_timesteps(
+ self,
+ num_inference_steps: Union[int, None] = None,
+ device: Union[str, torch.device] = None,
+ sigmas: Optional[List[float]] = None,
+ mu: Optional[Union[float, None]] = None,
+ shift: Optional[Union[float, None]] = None,
+ ):
+ """
+ Sets the discrete timesteps used for the diffusion chain (to be run before inference).
+ Args:
+ num_inference_steps (`int`):
+ Total number of the spacing of the time steps.
+ device (`str` or `torch.device`, *optional*):
+ The device to which the timesteps should be moved to. If `None`, the timesteps are not moved.
+ """
+
+ if self.config.use_dynamic_shifting and mu is None:
+ raise ValueError(
+ " you have to pass a value for `mu` when `use_dynamic_shifting` is set to be `True`"
+ )
+
+ if sigmas is None:
+ sigmas = np.linspace(self.sigma_max, self.sigma_min,
+ num_inference_steps +
+ 1).copy()[:-1] # pyright: ignore
+
+ if self.config.use_dynamic_shifting:
+ sigmas = self.time_shift(mu, 1.0, sigmas) # pyright: ignore
+ else:
+ if shift is None:
+ shift = self.config.shift
+ sigmas = shift * sigmas / (1 +
+ (shift - 1) * sigmas) # pyright: ignore
+
+ if self.config.final_sigmas_type == "sigma_min":
+ sigma_last = ((1 - self.alphas_cumprod[0]) /
+ self.alphas_cumprod[0])**0.5
+ elif self.config.final_sigmas_type == "zero":
+ sigma_last = 0
+ else:
+ raise ValueError(
+ f"`final_sigmas_type` must be one of 'zero', or 'sigma_min', but got {self.config.final_sigmas_type}"
+ )
+
+ timesteps = sigmas * self.config.num_train_timesteps
+ sigmas = np.concatenate([sigmas, [sigma_last]
+ ]).astype(np.float32) # pyright: ignore
+
+ self.sigmas = torch.from_numpy(sigmas)
+ self.timesteps = torch.from_numpy(timesteps).to(
+ device=device, dtype=torch.int64)
+
+ self.num_inference_steps = len(timesteps)
+
+ self.model_outputs = [
+ None,
+ ] * self.config.solver_order
+ self.lower_order_nums = 0
+
+ self._step_index = None
+ self._begin_index = None
+ # self.sigmas = self.sigmas.to(
+ # "cpu") # to avoid too much CPU/GPU communication
+
+ # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler._threshold_sample
+ def _threshold_sample(self, sample: torch.Tensor) -> torch.Tensor:
+ """
+ "Dynamic thresholding: At each sampling step we set s to a certain percentile absolute pixel value in xt0 (the
+ prediction of x_0 at timestep t), and if s > 1, then we threshold xt0 to the range [-s, s] and then divide by
+ s. Dynamic thresholding pushes saturated pixels (those near -1 and 1) inwards, thereby actively preventing
+ pixels from saturation at each step. We find that dynamic thresholding results in significantly better
+ photorealism as well as better image-text alignment, especially when using very large guidance weights."
+ https://arxiv.org/abs/2205.11487
+ """
+ dtype = sample.dtype
+ batch_size, channels, *remaining_dims = sample.shape
+
+ if dtype not in (torch.float32, torch.float64):
+ sample = sample.float(
+ ) # upcast for quantile calculation, and clamp not implemented for cpu half
+
+ # Flatten sample for doing quantile calculation along each image
+ sample = sample.reshape(batch_size, channels * np.prod(remaining_dims))
+
+ abs_sample = sample.abs() # "a certain percentile absolute pixel value"
+
+ s = torch.quantile(
+ abs_sample, self.config.dynamic_thresholding_ratio, dim=1)
+ s = torch.clamp(
+ s, min=1, max=self.config.sample_max_value
+ ) # When clamped to min=1, equivalent to standard clipping to [-1, 1]
+ s = s.unsqueeze(
+ 1) # (batch_size, 1) because clamp will broadcast along dim=0
+ sample = torch.clamp(
+ sample, -s, s
+ ) / s # "we threshold xt0 to the range [-s, s] and then divide by s"
+
+ sample = sample.reshape(batch_size, channels, *remaining_dims)
+ sample = sample.to(dtype)
+
+ return sample
+
+ # Copied from diffusers.schedulers.scheduling_flow_match_euler_discrete.FlowMatchEulerDiscreteScheduler._sigma_to_t
+ def _sigma_to_t(self, sigma):
+ return sigma * self.config.num_train_timesteps
+
+ def _sigma_to_alpha_sigma_t(self, sigma):
+ return 1 - sigma, sigma
+
+ # Copied from diffusers.schedulers.scheduling_flow_match_euler_discrete.set_timesteps
+ def time_shift(self, mu: float, sigma: float, t: torch.Tensor):
+ return math.exp(mu) / (math.exp(mu) + (1 / t - 1)**sigma)
+
+ # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.convert_model_output
+ def convert_model_output(
+ self,
+ model_output: torch.Tensor,
+ *args,
+ sample: torch.Tensor = None,
+ **kwargs,
+ ) -> torch.Tensor:
+ """
+ Convert the model output to the corresponding type the DPMSolver/DPMSolver++ algorithm needs. DPM-Solver is
+ designed to discretize an integral of the noise prediction model, and DPM-Solver++ is designed to discretize an
+ integral of the data prediction model.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ """
+ return stats_html, last_disk_io
+
+ def streaming_html(self, state):
+ if "stats_running" in state:
+ return
+ state["stats_running"] = True
+
+ self.running = True
+ last_disk_io = psutil.disk_io_counters()
+ i = 0
+ import time
+ try:
+ while self.running:
+ i+= 1
+ # if i % 2 == 0:
+ # print(f"time:{time.time()}")
+ html_content, last_disk_io = self.get_system_stats(False, last_disk_io)
+ yield html_content
+ # time.sleep(1)
+
+ except GeneratorExit:
+ # print("Generator stopped gracefully")
+ return
+ except Exception as e:
+ print(f"Streaming error: {e}")
+ # finally:
+ # # Send final message indicating clean shutdown
+ final_html = """
+
+
+
+
+
+ CPU: {cpu_percent:.1f}%
+
+
+
+
+
+ RAM {ram_percent:.1f}%
+ {ram_used_gb:.1f} / {ram_total_gb:.1f} GB
+
+
+
+
+
+ SSD R/W
+ {read_mb_s:.1f} / {write_mb_s:.1f} MB/s
+
+
+
+
+
+ GPU: {gpu_percent:.1f}%
+
+
+
+ VRAM {vram_percent:.1f}%
+ {vram_used_gb:.1f} / {vram_total_gb:.1f} GB
+
+
+
+
+
+ """
+ try:
+ yield final_html
+ except:
+ pass
+
+
+ def get_gradio_element(self):
+ self.system_stats_display = gr.HTML(self.get_system_stats(True)[0])
+ self.restart_btn = gr.Button("restart stats",elem_id="restart_stats", visible= False) # False)
+ return self.system_stats_display
+
+ def setup_events(self, main, state):
+ gr.on([main.load, self.restart_btn.click],
+ fn=self.streaming_html,
+ inputs = state,
+ outputs=self.system_stats_display,
+ show_progress=False
+ )
diff --git a/wan/utils/thread_utils.py b/wan/utils/thread_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..37a24ea7312e2b89ed5533d7ed001e7b24d44528
--- /dev/null
+++ b/wan/utils/thread_utils.py
@@ -0,0 +1,82 @@
+# based on FramePack https://github.com/lllyasviel/FramePack
+
+import time
+import traceback
+
+from threading import Thread, Lock
+
+
+class Listener:
+ task_queue = []
+ lock = Lock()
+ thread = None
+
+ @classmethod
+ def _process_tasks(cls):
+ while True:
+ task = None
+ with cls.lock:
+ if cls.task_queue:
+ task = cls.task_queue.pop(0)
+
+ if task is None:
+ time.sleep(0.001)
+ continue
+
+ func, args, kwargs = task
+ try:
+ func(*args, **kwargs)
+ except Exception as e:
+ tb = traceback.format_exc().split('\n')[:-1]
+ print('\n'.join(tb))
+
+ # print(f"Error in listener thread: {e}")
+
+ @classmethod
+ def add_task(cls, func, *args, **kwargs):
+ with cls.lock:
+ cls.task_queue.append((func, args, kwargs))
+
+ if cls.thread is None:
+ cls.thread = Thread(target=cls._process_tasks, daemon=True)
+ cls.thread.start()
+
+
+def async_run(func, *args, **kwargs):
+ Listener.add_task(func, *args, **kwargs)
+
+
+class FIFOQueue:
+ def __init__(self):
+ self.queue = []
+ self.lock = Lock()
+
+ def push(self, cmd, data = None):
+ with self.lock:
+ self.queue.append( (cmd, data) )
+
+ def pop(self):
+ with self.lock:
+ if self.queue:
+ return self.queue.pop(0)
+ return None
+
+ def top(self):
+ with self.lock:
+ if self.queue:
+ return self.queue[0]
+ return None
+
+ def next(self):
+ while True:
+ with self.lock:
+ if self.queue:
+ return self.queue.pop(0)
+
+ time.sleep(0.001)
+
+
+class AsyncStream:
+ def __init__(self):
+ self.input_queue = FIFOQueue()
+ self.output_queue = FIFOQueue()
diff --git a/wan/utils/utils.py b/wan/utils/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..041fdea917fb64614d753c497ec6fc308598c175
--- /dev/null
+++ b/wan/utils/utils.py
@@ -0,0 +1,646 @@
+# Copyright 2024-2025 The Alibaba Wan Team Authors. All rights reserved.
+import argparse
+import binascii
+import os
+import os.path as osp
+import torchvision.transforms.functional as TF
+import torch.nn.functional as F
+import cv2
+import tempfile
+import imageio
+import torch
+import decord
+import torchvision
+from PIL import Image
+import numpy as np
+from rembg import remove, new_session
+import random
+import ffmpeg
+import os
+import tempfile
+import subprocess
+import json
+
+__all__ = ['cache_video', 'cache_image', 'str2bool']
+
+
+
+from PIL import Image
+
+def seed_everything(seed: int):
+ random.seed(seed)
+ np.random.seed(seed)
+ torch.manual_seed(seed)
+ if torch.cuda.is_available():
+ torch.cuda.manual_seed(seed)
+ if torch.backends.mps.is_available():
+ torch.mps.manual_seed(seed)
+
+def resample(video_fps, video_frames_count, max_target_frames_count, target_fps, start_target_frame ):
+ import math
+
+ video_frame_duration = 1 /video_fps
+ target_frame_duration = 1 / target_fps
+
+ target_time = start_target_frame * target_frame_duration
+ frame_no = math.ceil(target_time / video_frame_duration)
+ cur_time = frame_no * video_frame_duration
+ frame_ids =[]
+ while True:
+ if max_target_frames_count != 0 and len(frame_ids) >= max_target_frames_count :
+ break
+ diff = round( (target_time -cur_time) / video_frame_duration , 5)
+ add_frames_count = math.ceil( diff)
+ frame_no += add_frames_count
+ if frame_no >= video_frames_count:
+ break
+ frame_ids.append(frame_no)
+ cur_time += add_frames_count * video_frame_duration
+ target_time += target_frame_duration
+ frame_ids = frame_ids[:max_target_frames_count]
+ return frame_ids
+
+import os
+from datetime import datetime
+
+def get_file_creation_date(file_path):
+ # On Windows
+ if os.name == 'nt':
+ return datetime.fromtimestamp(os.path.getctime(file_path))
+ # On Unix/Linux/Mac (gets last status change, not creation)
+ else:
+ stat = os.stat(file_path)
+ return datetime.fromtimestamp(stat.st_birthtime if hasattr(stat, 'st_birthtime') else stat.st_mtime)
+
+def truncate_for_filesystem(s, max_bytes=255):
+ if len(s.encode('utf-8')) <= max_bytes: return s
+ l, r = 0, len(s)
+ while l < r:
+ m = (l + r + 1) // 2
+ if len(s[:m].encode('utf-8')) <= max_bytes: l = m
+ else: r = m - 1
+ return s[:l]
+
+def get_video_info(video_path):
+ import cv2
+ cap = cv2.VideoCapture(video_path)
+
+ # Get FPS
+ fps = round(cap.get(cv2.CAP_PROP_FPS))
+
+ # Get resolution
+ width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
+ height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
+ frame_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
+ cap.release()
+
+ return fps, width, height, frame_count
+
+def get_video_frame(file_name, frame_no):
+ decord.bridge.set_bridge('torch')
+ reader = decord.VideoReader(file_name)
+
+ frame = reader.get_batch([frame_no]).squeeze(0)
+ img = Image.fromarray(frame.numpy().astype(np.uint8))
+ return img
+
+def convert_image_to_video(image):
+ if image is None:
+ return None
+
+ # Convert PIL/numpy image to OpenCV format if needed
+ if isinstance(image, np.ndarray):
+ # Gradio images are typically RGB, OpenCV expects BGR
+ img_bgr = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
+ else:
+ # Handle PIL Image
+ img_array = np.array(image)
+ img_bgr = cv2.cvtColor(img_array, cv2.COLOR_RGB2BGR)
+
+ height, width = img_bgr.shape[:2]
+
+ # Create temporary video file (auto-cleaned by Gradio)
+ with tempfile.NamedTemporaryFile(suffix='.mp4', delete=False) as temp_video:
+ fourcc = cv2.VideoWriter_fourcc(*'mp4v')
+ out = cv2.VideoWriter(temp_video.name, fourcc, 30.0, (width, height))
+ out.write(img_bgr)
+ out.release()
+ return temp_video.name
+
+def resize_lanczos(img, h, w):
+ img = (img + 1).float().mul_(127.5)
+ img = Image.fromarray(np.clip(img.movedim(0, -1).cpu().numpy(), 0, 255).astype(np.uint8))
+ img = img.resize((w,h), resample=Image.Resampling.LANCZOS)
+ img = torch.from_numpy(np.array(img).astype(np.float32)).movedim(-1, 0)
+ img = img.div(127.5).sub_(1)
+ return img
+
+def remove_background(img, session=None):
+ if session ==None:
+ session = new_session()
+ img = Image.fromarray(np.clip(255. * img.movedim(0, -1).cpu().numpy(), 0, 255).astype(np.uint8))
+ img = remove(img, session=session, alpha_matting = True, bgcolor=[255, 255, 255, 0]).convert('RGB')
+ return torch.from_numpy(np.array(img).astype(np.float32) / 255.0).movedim(-1, 0)
+
+def convert_tensor_to_image(t, frame_no = -1):
+ t = t[:, frame_no] if frame_no >= 0 else t
+ return Image.fromarray(t.clone().add_(1.).mul_(127.5).permute(1,2,0).to(torch.uint8).cpu().numpy())
+
+def save_image(tensor_image, name, frame_no = -1):
+ convert_tensor_to_image(tensor_image, frame_no).save(name)
+
+def get_outpainting_full_area_dimensions(frame_height,frame_width, outpainting_dims):
+ outpainting_top, outpainting_bottom, outpainting_left, outpainting_right= outpainting_dims
+ frame_height = int(frame_height * (100 + outpainting_top + outpainting_bottom) / 100)
+ frame_width = int(frame_width * (100 + outpainting_left + outpainting_right) / 100)
+ return frame_height, frame_width
+
+def get_outpainting_frame_location(final_height, final_width, outpainting_dims, block_size = 8):
+ outpainting_top, outpainting_bottom, outpainting_left, outpainting_right= outpainting_dims
+ raw_height = int(final_height / ((100 + outpainting_top + outpainting_bottom) / 100))
+ height = int(raw_height / block_size) * block_size
+ extra_height = raw_height - height
+
+ raw_width = int(final_width / ((100 + outpainting_left + outpainting_right) / 100))
+ width = int(raw_width / block_size) * block_size
+ extra_width = raw_width - width
+ margin_top = int(outpainting_top/(100 + outpainting_top + outpainting_bottom) * final_height)
+ if extra_height != 0 and (outpainting_top + outpainting_bottom) != 0:
+ margin_top += int(outpainting_top / (outpainting_top + outpainting_bottom) * extra_height)
+ if (margin_top + height) > final_height or outpainting_bottom == 0: margin_top = final_height - height
+ margin_left = int(outpainting_left/(100 + outpainting_left + outpainting_right) * final_width)
+ if extra_width != 0 and (outpainting_left + outpainting_right) != 0:
+ margin_left += int(outpainting_left / (outpainting_left + outpainting_right) * extra_height)
+ if (margin_left + width) > final_width or outpainting_right == 0: margin_left = final_width - width
+ return height, width, margin_top, margin_left
+
+def calculate_new_dimensions(canvas_height, canvas_width, height, width, fit_into_canvas, block_size = 16):
+ if fit_into_canvas == None:
+ return height, width
+ if fit_into_canvas:
+ scale1 = min(canvas_height / height, canvas_width / width)
+ scale2 = min(canvas_width / height, canvas_height / width)
+ scale = max(scale1, scale2)
+ else:
+ scale = (canvas_height * canvas_width / (height * width))**(1/2)
+
+ new_height = round( height * scale / block_size) * block_size
+ new_width = round( width * scale / block_size) * block_size
+ return new_height, new_width
+
+def resize_and_remove_background(img_list, budget_width, budget_height, rm_background, ignore_first, fit_into_canvas = False ):
+ if rm_background:
+ session = new_session()
+
+ output_list =[]
+ for i, img in enumerate(img_list):
+ width, height = img.size
+
+ if fit_into_canvas:
+ white_canvas = np.ones((budget_height, budget_width, 3), dtype=np.uint8) * 255
+ scale = min(budget_height / height, budget_width / width)
+ new_height = int(height * scale)
+ new_width = int(width * scale)
+ resized_image= img.resize((new_width,new_height), resample=Image.Resampling.LANCZOS)
+ top = (budget_height - new_height) // 2
+ left = (budget_width - new_width) // 2
+ white_canvas[top:top + new_height, left:left + new_width] = np.array(resized_image)
+ resized_image = Image.fromarray(white_canvas)
+ else:
+ scale = (budget_height * budget_width / (height * width))**(1/2)
+ new_height = int( round(height * scale / 16) * 16)
+ new_width = int( round(width * scale / 16) * 16)
+ resized_image= img.resize((new_width,new_height), resample=Image.Resampling.LANCZOS)
+ if rm_background and not (ignore_first and i == 0) :
+ # resized_image = remove(resized_image, session=session, alpha_matting_erode_size = 1,alpha_matting_background_threshold = 70, alpha_foreground_background_threshold = 100, alpha_matting = True, bgcolor=[255, 255, 255, 0]).convert('RGB')
+ resized_image = remove(resized_image, session=session, alpha_matting_erode_size = 1, alpha_matting = True, bgcolor=[255, 255, 255, 0]).convert('RGB')
+ output_list.append(resized_image) #alpha_matting_background_threshold = 30, alpha_foreground_background_threshold = 200,
+ return output_list
+
+
+def rand_name(length=8, suffix=''):
+ name = binascii.b2a_hex(os.urandom(length)).decode('utf-8')
+ if suffix:
+ if not suffix.startswith('.'):
+ suffix = '.' + suffix
+ name += suffix
+ return name
+
+
+def cache_video(tensor,
+ save_file=None,
+ fps=30,
+ suffix='.mp4',
+ nrow=8,
+ normalize=True,
+ value_range=(-1, 1),
+ retry=5):
+ # cache file
+ cache_file = osp.join('/tmp', rand_name(
+ suffix=suffix)) if save_file is None else save_file
+
+ # save to cache
+ error = None
+ for _ in range(retry):
+ try:
+ # preprocess
+ tensor = tensor.clamp(min(value_range), max(value_range))
+ tensor = torch.stack([
+ torchvision.utils.make_grid(
+ u, nrow=nrow, normalize=normalize, value_range=value_range)
+ for u in tensor.unbind(2)
+ ],
+ dim=1).permute(1, 2, 3, 0)
+ tensor = (tensor * 255).type(torch.uint8).cpu()
+
+ # write video
+ writer = imageio.get_writer(
+ cache_file, fps=fps, codec='libx264', quality=8)
+ for frame in tensor.numpy():
+ writer.append_data(frame)
+ writer.close()
+ return cache_file
+ except Exception as e:
+ error = e
+ continue
+ else:
+ print(f'cache_video failed, error: {error}', flush=True)
+ return None
+
+
+def cache_image(tensor,
+ save_file,
+ nrow=8,
+ normalize=True,
+ value_range=(-1, 1),
+ retry=5):
+ # cache file
+ suffix = osp.splitext(save_file)[1]
+ if suffix.lower() not in [
+ '.jpg', '.jpeg', '.png', '.tiff', '.gif', '.webp'
+ ]:
+ suffix = '.png'
+
+ # save to cache
+ error = None
+ for _ in range(retry):
+ try:
+ tensor = tensor.clamp(min(value_range), max(value_range))
+ torchvision.utils.save_image(
+ tensor,
+ save_file,
+ nrow=nrow,
+ normalize=normalize,
+ value_range=value_range)
+ return save_file
+ except Exception as e:
+ error = e
+ continue
+
+
+def str2bool(v):
+ """
+ Convert a string to a boolean.
+
+ Supported true values: 'yes', 'true', 't', 'y', '1'
+ Supported false values: 'no', 'false', 'f', 'n', '0'
+
+ Args:
+ v (str): String to convert.
+
+ Returns:
+ bool: Converted boolean value.
+
+ Raises:
+ argparse.ArgumentTypeError: If the value cannot be converted to boolean.
+ """
+ if isinstance(v, bool):
+ return v
+ v_lower = v.lower()
+ if v_lower in ('yes', 'true', 't', 'y', '1'):
+ return True
+ elif v_lower in ('no', 'false', 'f', 'n', '0'):
+ return False
+ else:
+ raise argparse.ArgumentTypeError('Boolean value expected (True/False)')
+
+
+import sys, time
+
+# Global variables to track download progress
+_start_time = None
+_last_time = None
+_last_downloaded = 0
+_speed_history = []
+_update_interval = 0.5 # Update speed every 0.5 seconds
+
+def progress_hook(block_num, block_size, total_size, filename=None):
+ """
+ Simple progress bar hook for urlretrieve
+
+ Args:
+ block_num: Number of blocks downloaded so far
+ block_size: Size of each block in bytes
+ total_size: Total size of the file in bytes
+ filename: Name of the file being downloaded (optional)
+ """
+ global _start_time, _last_time, _last_downloaded, _speed_history, _update_interval
+
+ current_time = time.time()
+ downloaded = block_num * block_size
+
+ # Initialize timing on first call
+ if _start_time is None or block_num == 0:
+ _start_time = current_time
+ _last_time = current_time
+ _last_downloaded = 0
+ _speed_history = []
+
+ # Calculate download speed only at specified intervals
+ speed = 0
+ if current_time - _last_time >= _update_interval:
+ if _last_time > 0:
+ current_speed = (downloaded - _last_downloaded) / (current_time - _last_time)
+ _speed_history.append(current_speed)
+ # Keep only last 5 speed measurements for smoothing
+ if len(_speed_history) > 5:
+ _speed_history.pop(0)
+ # Average the recent speeds for smoother display
+ speed = sum(_speed_history) / len(_speed_history)
+
+ _last_time = current_time
+ _last_downloaded = downloaded
+ elif _speed_history:
+ # Use the last calculated average speed
+ speed = sum(_speed_history) / len(_speed_history)
+ # Format file sizes and speed
+ def format_bytes(bytes_val):
+ for unit in ['B', 'KB', 'MB', 'GB']:
+ if bytes_val < 1024:
+ return f"{bytes_val:.1f}{unit}"
+ bytes_val /= 1024
+ return f"{bytes_val:.1f}TB"
+
+ file_display = filename if filename else "Unknown file"
+
+ if total_size <= 0:
+ # If total size is unknown, show downloaded bytes
+ speed_str = f" @ {format_bytes(speed)}/s" if speed > 0 else ""
+ line = f"\r{file_display}: {format_bytes(downloaded)}{speed_str}"
+ # Clear any trailing characters by padding with spaces
+ sys.stdout.write(line.ljust(80))
+ sys.stdout.flush()
+ return
+
+ downloaded = block_num * block_size
+ percent = min(100, (downloaded / total_size) * 100)
+
+ # Create progress bar (40 characters wide to leave room for other info)
+ bar_length = 40
+ filled = int(bar_length * percent / 100)
+ bar = '█' * filled + '░' * (bar_length - filled)
+
+ # Format file sizes and speed
+ def format_bytes(bytes_val):
+ for unit in ['B', 'KB', 'MB', 'GB']:
+ if bytes_val < 1024:
+ return f"{bytes_val:.1f}{unit}"
+ bytes_val /= 1024
+ return f"{bytes_val:.1f}TB"
+
+ speed_str = f" @ {format_bytes(speed)}/s" if speed > 0 else ""
+
+ # Display progress with filename first
+ line = f"\r{file_display}: [{bar}] {percent:.1f}% ({format_bytes(downloaded)}/{format_bytes(total_size)}){speed_str}"
+ # Clear any trailing characters by padding with spaces
+ sys.stdout.write(line.ljust(100))
+ sys.stdout.flush()
+
+ # Print newline when complete
+ if percent >= 100:
+ print()
+
+# Wrapper function to include filename in progress hook
+def create_progress_hook(filename):
+ """Creates a progress hook with the filename included"""
+ global _start_time, _last_time, _last_downloaded, _speed_history
+ # Reset timing variables for new download
+ _start_time = None
+ _last_time = None
+ _last_downloaded = 0
+ _speed_history = []
+
+ def hook(block_num, block_size, total_size):
+ return progress_hook(block_num, block_size, total_size, filename)
+ return hook
+
+
+import tempfile, os
+import ffmpeg
+
+def extract_audio_tracks(source_video, verbose=False, query_only=False):
+ """
+ Extract all audio tracks from a source video into temporary AAC files.
+
+ Returns:
+ Tuple:
+ - List of temp file paths for extracted audio tracks
+ - List of corresponding metadata dicts:
+ {'codec', 'sample_rate', 'channels', 'duration', 'language'}
+ where 'duration' is set to container duration (for consistency).
+ """
+ probe = ffmpeg.probe(source_video)
+ audio_streams = [s for s in probe['streams'] if s['codec_type'] == 'audio']
+ container_duration = float(probe['format'].get('duration', 0.0))
+
+ if not audio_streams:
+ if query_only: return 0
+ if verbose: print(f"No audio track found in {source_video}")
+ return [], []
+
+ if query_only:
+ return len(audio_streams)
+
+ if verbose:
+ print(f"Found {len(audio_streams)} audio track(s), container duration = {container_duration:.3f}s")
+
+ file_paths = []
+ metadata = []
+
+ for i, stream in enumerate(audio_streams):
+ fd, temp_path = tempfile.mkstemp(suffix=f'_track{i}.aac', prefix='audio_')
+ os.close(fd)
+
+ file_paths.append(temp_path)
+ metadata.append({
+ 'codec': stream.get('codec_name'),
+ 'sample_rate': int(stream.get('sample_rate', 0)),
+ 'channels': int(stream.get('channels', 0)),
+ 'duration': container_duration,
+ 'language': stream.get('tags', {}).get('language', None)
+ })
+
+ ffmpeg.input(source_video).output(
+ temp_path,
+ **{f'map': f'0:a:{i}', 'acodec': 'aac', 'b:a': '128k'}
+ ).overwrite_output().run(quiet=not verbose)
+
+ return file_paths, metadata
+
+
+import subprocess
+
+import subprocess
+
+def combine_and_concatenate_video_with_audio_tracks(
+ save_path_tmp, video_path,
+ source_audio_tracks, new_audio_tracks,
+ source_audio_duration, audio_sampling_rate,
+ new_audio_from_start=False,
+ source_audio_metadata=None,
+ audio_bitrate='128k',
+ audio_codec='aac',
+ verbose = False
+):
+ inputs, filters, maps, idx = ['-i', video_path], [], ['-map', '0:v'], 1
+ metadata_args = []
+ sources = source_audio_tracks or []
+ news = new_audio_tracks or []
+
+ duplicate_source = len(sources) == 1 and len(news) > 1
+ N = len(news) if source_audio_duration == 0 else max(len(sources), len(news)) or 1
+
+ for i in range(N):
+ s = (sources[i] if i < len(sources)
+ else sources[0] if duplicate_source else None)
+ n = news[i] if len(news) == N else (news[0] if news else None)
+
+ if source_audio_duration == 0:
+ if n:
+ inputs += ['-i', n]
+ filters.append(f'[{idx}:a]apad=pad_dur=100[aout{i}]')
+ idx += 1
+ else:
+ filters.append(f'anullsrc=r={audio_sampling_rate}:cl=mono,apad=pad_dur=100[aout{i}]')
+ else:
+ if s:
+ inputs += ['-i', s]
+ meta = source_audio_metadata[i] if source_audio_metadata and i < len(source_audio_metadata) else {}
+ needs_filter = (
+ meta.get('codec') != audio_codec or
+ meta.get('sample_rate') != audio_sampling_rate or
+ meta.get('channels') != 1 or
+ meta.get('duration', 0) < source_audio_duration
+ )
+ if needs_filter:
+ filters.append(
+ f'[{idx}:a]aresample={audio_sampling_rate},aformat=channel_layouts=mono,'
+ f'apad=pad_dur={source_audio_duration},atrim=0:{source_audio_duration},asetpts=PTS-STARTPTS[s{i}]')
+ else:
+ filters.append(
+ f'[{idx}:a]apad=pad_dur={source_audio_duration},atrim=0:{source_audio_duration},asetpts=PTS-STARTPTS[s{i}]')
+ if lang := meta.get('language'):
+ metadata_args += ['-metadata:s:a:' + str(i), f'language={lang}']
+ idx += 1
+ else:
+ filters.append(
+ f'anullsrc=r={audio_sampling_rate}:cl=mono,atrim=0:{source_audio_duration},asetpts=PTS-STARTPTS[s{i}]')
+
+ if n:
+ inputs += ['-i', n]
+ start = '0' if new_audio_from_start else source_audio_duration
+ filters.append(
+ f'[{idx}:a]aresample={audio_sampling_rate},aformat=channel_layouts=mono,'
+ f'atrim=start={start},asetpts=PTS-STARTPTS[n{i}]')
+ filters.append(f'[s{i}][n{i}]concat=n=2:v=0:a=1[aout{i}]')
+ idx += 1
+ else:
+ filters.append(f'[s{i}]apad=pad_dur=100[aout{i}]')
+
+ maps += ['-map', f'[aout{i}]']
+
+ cmd = ['ffmpeg', '-y', *inputs,
+ '-filter_complex', ';'.join(filters), # ✅ Only change made
+ *maps, *metadata_args,
+ '-c:v', 'copy',
+ '-c:a', audio_codec,
+ '-b:a', audio_bitrate,
+ '-ar', str(audio_sampling_rate),
+ '-ac', '1',
+ '-shortest', save_path_tmp]
+
+ if verbose:
+ print(f"ffmpeg command: {cmd}")
+ try:
+ subprocess.run(cmd, check=True, capture_output=True, text=True)
+ except subprocess.CalledProcessError as e:
+ raise Exception(f"FFmpeg error: {e.stderr}")
+
+
+import ffmpeg
+
+
+import subprocess
+import ffmpeg
+
+def combine_video_with_audio_tracks(target_video, audio_tracks, output_video,
+ audio_metadata=None, verbose=False):
+ if not audio_tracks:
+ if verbose: print("No audio tracks to combine."); return False
+
+ dur = float(next(s for s in ffmpeg.probe(target_video)['streams']
+ if s['codec_type'] == 'video')['duration'])
+ if verbose: print(f"Video duration: {dur:.3f}s")
+
+ cmd = ['ffmpeg', '-y', '-i', target_video]
+ for path in audio_tracks:
+ cmd += ['-i', path]
+
+ cmd += ['-map', '0:v']
+ for i in range(len(audio_tracks)):
+ cmd += ['-map', f'{i+1}:a']
+
+ for i, meta in enumerate(audio_metadata or []):
+ if (lang := meta.get('language')):
+ cmd += ['-metadata:s:a:' + str(i), f'language={lang}']
+
+ cmd += ['-c:v', 'copy', '-c:a', 'copy', '-t', str(dur), output_video]
+
+ result = subprocess.run(cmd, capture_output=not verbose, text=True)
+ if result.returncode != 0:
+ raise Exception(f"FFmpeg error:\n{result.stderr}")
+ if verbose:
+ print(f"Created {output_video} with {len(audio_tracks)} audio track(s)")
+ return True
+
+
+def cleanup_temp_audio_files(audio_tracks, verbose=False):
+ """
+ Clean up temporary audio files.
+
+ Args:
+ audio_tracks: List of audio file paths to delete
+ verbose: Enable verbose output (default: False)
+
+ Returns:
+ Number of files successfully deleted
+ """
+ deleted_count = 0
+
+ for audio_path in audio_tracks:
+ try:
+ if os.path.exists(audio_path):
+ os.unlink(audio_path)
+ deleted_count += 1
+ if verbose:
+ print(f"Cleaned up {audio_path}")
+ except PermissionError:
+ print(f"Warning: Could not delete {audio_path} (file may be in use)")
+ except Exception as e:
+ print(f"Warning: Error deleting {audio_path}: {e}")
+
+ if verbose and deleted_count > 0:
+ print(f"Successfully deleted {deleted_count} temporary audio file(s)")
+
+ return deleted_count
+
diff --git a/wan/utils/vace_preprocessor.py b/wan/utils/vace_preprocessor.py
new file mode 100644
index 0000000000000000000000000000000000000000..7fdb8c951edae3f1faf0fee0051dd4e30b89faa7
--- /dev/null
+++ b/wan/utils/vace_preprocessor.py
@@ -0,0 +1,273 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import numpy as np
+from PIL import Image
+import torch
+import torch.nn.functional as F
+import torchvision.transforms.functional as TF
+from .utils import calculate_new_dimensions
+
+
+class VaceImageProcessor(object):
+ def __init__(self, downsample=None, seq_len=None):
+ self.downsample = downsample
+ self.seq_len = seq_len
+
+ def _pillow_convert(self, image, cvt_type='RGB'):
+ if image.mode != cvt_type:
+ if image.mode == 'P':
+ image = image.convert(f'{cvt_type}A')
+ if image.mode == f'{cvt_type}A':
+ bg = Image.new(cvt_type,
+ size=(image.width, image.height),
+ color=(255, 255, 255))
+ bg.paste(image, (0, 0), mask=image)
+ image = bg
+ else:
+ image = image.convert(cvt_type)
+ return image
+
+ def _load_image(self, img_path):
+ if img_path is None or img_path == '':
+ return None
+ img = Image.open(img_path)
+ img = self._pillow_convert(img)
+ return img
+
+ def _resize_crop(self, img, oh, ow, normalize=True):
+ """
+ Resize, center crop, convert to tensor, and normalize.
+ """
+ # resize and crop
+ iw, ih = img.size
+ if iw != ow or ih != oh:
+ # resize
+ scale = max(ow / iw, oh / ih)
+ img = img.resize(
+ (round(scale * iw), round(scale * ih)),
+ resample=Image.Resampling.LANCZOS
+ )
+ assert img.width >= ow and img.height >= oh
+
+ # center crop
+ x1 = (img.width - ow) // 2
+ y1 = (img.height - oh) // 2
+ img = img.crop((x1, y1, x1 + ow, y1 + oh))
+
+ # normalize
+ if normalize:
+ img = TF.to_tensor(img).sub_(0.5).div_(0.5).unsqueeze(1)
+ return img
+
+ def _image_preprocess(self, img, oh, ow, normalize=True, **kwargs):
+ return self._resize_crop(img, oh, ow, normalize)
+
+ def load_image(self, data_key, **kwargs):
+ return self.load_image_batch(data_key, **kwargs)
+
+ def load_image_pair(self, data_key, data_key2, **kwargs):
+ return self.load_image_batch(data_key, data_key2, **kwargs)
+
+ def load_image_batch(self, *data_key_batch, normalize=True, seq_len=None, **kwargs):
+ seq_len = self.seq_len if seq_len is None else seq_len
+ imgs = []
+ for data_key in data_key_batch:
+ img = self._load_image(data_key)
+ imgs.append(img)
+ w, h = imgs[0].size
+ dh, dw = self.downsample[1:]
+
+ # compute output size
+ scale = min(1., np.sqrt(seq_len / ((h / dh) * (w / dw))))
+ oh = int(h * scale) // dh * dh
+ ow = int(w * scale) // dw * dw
+ assert (oh // dh) * (ow // dw) <= seq_len
+ imgs = [self._image_preprocess(img, oh, ow, normalize) for img in imgs]
+ return *imgs, (oh, ow)
+
+
+class VaceVideoProcessor(object):
+ def __init__(self, downsample, min_area, max_area, min_fps, max_fps, zero_start, seq_len, keep_last, **kwargs):
+ self.downsample = downsample
+ self.min_area = min_area
+ self.max_area = max_area
+ self.min_fps = min_fps
+ self.max_fps = max_fps
+ self.zero_start = zero_start
+ self.keep_last = keep_last
+ self.seq_len = seq_len
+ assert seq_len >= min_area / (self.downsample[1] * self.downsample[2])
+
+ @staticmethod
+ def resize_crop(video: torch.Tensor, oh: int, ow: int):
+ """
+ Resize, center crop and normalize for decord loaded video (torch.Tensor type)
+
+ Parameters:
+ video - video to process (torch.Tensor): Tensor from `reader.get_batch(frame_ids)`, in shape of (T, H, W, C)
+ oh - target height (int)
+ ow - target width (int)
+
+ Returns:
+ The processed video (torch.Tensor): Normalized tensor range [-1, 1], in shape of (C, T, H, W)
+
+ Raises:
+ """
+ # permute ([t, h, w, c] -> [t, c, h, w])
+ video = video.permute(0, 3, 1, 2)
+
+ # resize and crop
+ ih, iw = video.shape[2:]
+ if ih != oh or iw != ow:
+ # resize
+ scale = max(ow / iw, oh / ih)
+ video = F.interpolate(
+ video,
+ size=(round(scale * ih), round(scale * iw)),
+ mode='bicubic',
+ antialias=True
+ )
+ assert video.size(3) >= ow and video.size(2) >= oh
+
+ # center crop
+ x1 = (video.size(3) - ow) // 2
+ y1 = (video.size(2) - oh) // 2
+ video = video[:, :, y1:y1 + oh, x1:x1 + ow]
+
+ # permute ([t, c, h, w] -> [c, t, h, w]) and normalize
+ video = video.transpose(0, 1).float().div_(127.5).sub_(1.)
+ return video
+
+ def _video_preprocess(self, video, oh, ow):
+ return self.resize_crop(video, oh, ow)
+
+ def _get_frameid_bbox_default(self, fps, frame_timestamps, h, w, crop_box, rng):
+ target_fps = min(fps, self.max_fps)
+ duration = frame_timestamps[-1].mean()
+ x1, x2, y1, y2 = [0, w, 0, h] if crop_box is None else crop_box
+ h, w = y2 - y1, x2 - x1
+ ratio = h / w
+ df, dh, dw = self.downsample
+
+ # min/max area of the [latent video]
+ min_area_z = self.min_area / (dh * dw)
+ max_area_z = min(self.seq_len, self.max_area / (dh * dw), (h // dh) * (w // dw))
+
+ # sample a frame number of the [latent video]
+ rand_area_z = np.square(np.power(2, rng.uniform(
+ np.log2(np.sqrt(min_area_z)),
+ np.log2(np.sqrt(max_area_z))
+ )))
+ of = min(
+ (int(duration * target_fps) - 1) // df + 1,
+ int(self.seq_len / rand_area_z)
+ )
+
+ # deduce target shape of the [latent video]
+ target_area_z = min(max_area_z, int(self.seq_len / of))
+ oh = round(np.sqrt(target_area_z * ratio))
+ ow = int(target_area_z / oh)
+ of = (of - 1) * df + 1
+ oh *= dh
+ ow *= dw
+
+ # sample frame ids
+ target_duration = of / target_fps
+ begin = 0. if self.zero_start else rng.uniform(0, duration - target_duration)
+ timestamps = np.linspace(begin, begin + target_duration, of)
+ frame_ids = np.argmax(np.logical_and(
+ timestamps[:, None] >= frame_timestamps[None, :, 0],
+ timestamps[:, None] < frame_timestamps[None, :, 1]
+ ), axis=1).tolist()
+ return frame_ids, (x1, x2, y1, y2), (oh, ow), target_fps
+
+
+
+ def _get_frameid_bbox_adjust_last(self, fps, video_frames_count, canvas_height, canvas_width, h, w, fit_into_canvas, crop_box, rng, max_frames= 0, start_frame =0):
+ from wan.utils.utils import resample
+
+ target_fps = self.max_fps
+
+ frame_ids= resample(fps, video_frames_count, max_frames, target_fps, start_frame )
+
+ x1, x2, y1, y2 = [0, w, 0, h] if crop_box is None else crop_box
+ h, w = y2 - y1, x2 - x1
+ oh, ow = calculate_new_dimensions(canvas_height, canvas_width, h, w, fit_into_canvas)
+
+ return frame_ids, (x1, x2, y1, y2), (oh, ow), target_fps
+
+ def _get_frameid_bbox(self, fps, video_frames_count, h, w, crop_box, rng, max_frames= 0, start_frame= 0, canvas_height = 0, canvas_width = 0, fit_into_canvas= None):
+ if self.keep_last:
+ return self._get_frameid_bbox_adjust_last(fps, video_frames_count, canvas_height, canvas_width, h, w, fit_into_canvas, crop_box, rng, max_frames= max_frames, start_frame= start_frame)
+ else:
+ return self._get_frameid_bbox_default(fps, video_frames_count, h, w, crop_box, rng, max_frames= max_frames)
+
+ def load_video(self, data_key, crop_box=None, seed=2024, **kwargs):
+ return self.load_video_batch(data_key, crop_box=crop_box, seed=seed, **kwargs)
+
+ def load_video_pair(self, data_key, data_key2, crop_box=None, seed=2024, **kwargs):
+ return self.load_video_batch(data_key, data_key2, crop_box=crop_box, seed=seed, **kwargs)
+
+ def load_video_batch(self, *data_key_batch, crop_box=None, seed=2024, max_frames= 0, trim_video =0, start_frame = 0, canvas_height = 0, canvas_width = 0, fit_into_canvas = None, **kwargs):
+ rng = np.random.default_rng(seed + hash(data_key_batch[0]) % 10000)
+ # read video
+ import decord
+ decord.bridge.set_bridge('torch')
+ readers = []
+ src_videos = []
+ for data_k in data_key_batch:
+ if torch.is_tensor(data_k):
+ src_videos.append(data_k)
+ else:
+ reader = decord.VideoReader(data_k)
+ readers.append(reader)
+
+ if len(src_videos) >0:
+ fps = 16
+ length = src_videos[0].shape[0] + start_frame
+ if len(readers) > 0:
+ min_readers = min([len(r) for r in readers])
+ length = min(length, min_readers )
+ else:
+ fps = readers[0].get_avg_fps()
+ length = min([len(r) for r in readers])
+ # frame_timestamps = [readers[0].get_frame_timestamp(i) for i in range(length)]
+ # frame_timestamps = np.array(frame_timestamps, dtype=np.float32)
+ max_frames = min(max_frames, trim_video) if trim_video > 0 else max_frames
+ if len(src_videos) >0:
+ src_videos = [ src_video[:max_frames] for src_video in src_videos]
+ h, w = src_videos[0].shape[1:3]
+ else:
+ h, w = readers[0].next().shape[:2]
+ frame_ids, (x1, x2, y1, y2), (oh, ow), fps = self._get_frameid_bbox(fps, length, h, w, crop_box, rng, canvas_height = canvas_height, canvas_width = canvas_width, fit_into_canvas = fit_into_canvas, max_frames=max_frames, start_frame = start_frame )
+
+ # preprocess video
+ videos = [reader.get_batch(frame_ids)[:, y1:y2, x1:x2, :] for reader in readers]
+ if len(src_videos) >0:
+ videos = src_videos + videos
+ videos = [self._video_preprocess(video, oh, ow) for video in videos]
+ return *videos, frame_ids, (oh, ow), fps
+ # return videos if len(videos) > 1 else videos[0]
+
+
+def prepare_source(src_video, src_mask, src_ref_images, num_frames, image_size, device):
+ for i, (sub_src_video, sub_src_mask) in enumerate(zip(src_video, src_mask)):
+ if sub_src_video is None and sub_src_mask is None:
+ src_video[i] = torch.zeros((3, num_frames, image_size[0], image_size[1]), device=device)
+ src_mask[i] = torch.ones((1, num_frames, image_size[0], image_size[1]), device=device)
+ for i, ref_images in enumerate(src_ref_images):
+ if ref_images is not None:
+ for j, ref_img in enumerate(ref_images):
+ if ref_img is not None and ref_img.shape[-2:] != image_size:
+ canvas_height, canvas_width = image_size
+ ref_height, ref_width = ref_img.shape[-2:]
+ white_canvas = torch.ones((3, 1, canvas_height, canvas_width), device=device) # [-1, 1]
+ scale = min(canvas_height / ref_height, canvas_width / ref_width)
+ new_height = int(ref_height * scale)
+ new_width = int(ref_width * scale)
+ resized_image = F.interpolate(ref_img.squeeze(1).unsqueeze(0), size=(new_height, new_width), mode='bilinear', align_corners=False).squeeze(0).unsqueeze(1)
+ top = (canvas_height - new_height) // 2
+ left = (canvas_width - new_width) // 2
+ white_canvas[:, :, top:top + new_height, left:left + new_width] = resized_image
+ src_ref_images[i][j] = white_canvas
+ return src_video, src_mask, src_ref_images
diff --git a/wgp.py b/wgp.py
new file mode 100644
index 0000000000000000000000000000000000000000..5a54b75883185d133cf85366d061e43da01a733d
--- /dev/null
+++ b/wgp.py
@@ -0,0 +1,9076 @@
+import os
+import time
+import sys
+import threading
+import argparse
+from mmgp import offload, safetensors2, profile_type
+try:
+ import triton
+except ImportError:
+ pass
+from pathlib import Path
+from datetime import datetime
+import gradio as gr
+import random
+import json
+import wan
+from wan.utils import notification_sound
+from wan.configs import MAX_AREA_CONFIGS, WAN_CONFIGS, SUPPORTED_SIZES, VACE_SIZE_CONFIGS
+from wan.utils.loras_mutipliers import preparse_loras_multipliers, parse_loras_multipliers
+from wan.utils.utils import cache_video, convert_tensor_to_image, save_image, get_video_info, get_file_creation_date, convert_image_to_video
+from wan.utils.utils import extract_audio_tracks, combine_video_with_audio_tracks, combine_and_concatenate_video_with_audio_tracks, cleanup_temp_audio_files, calculate_new_dimensions
+
+from wan.modules.attention import get_attention_modes, get_supported_attention_modes
+from huggingface_hub import hf_hub_download, snapshot_download
+import torch
+import gc
+import traceback
+import math
+import typing
+import asyncio
+import inspect
+from wan.utils import prompt_parser
+import base64
+import io
+from PIL import Image
+import zipfile
+import tempfile
+import atexit
+import shutil
+import glob
+import cv2
+from transformers.utils import logging
+logging.set_verbosity_error
+from preprocessing.matanyone import app as matanyone_app
+from tqdm import tqdm
+import requests
+
+
+global_queue_ref = []
+AUTOSAVE_FILENAME = "queue.zip"
+PROMPT_VARS_MAX = 10
+
+target_mmgp_version = "3.5.6"
+WanGP_version = "7.61"
+settings_version = 2.23
+max_source_video_frames = 3000
+prompt_enhancer_image_caption_model, prompt_enhancer_image_caption_processor, prompt_enhancer_llm_model, prompt_enhancer_llm_tokenizer = None, None, None, None
+
+from importlib.metadata import version
+mmgp_version = version("mmgp")
+if mmgp_version != target_mmgp_version:
+ print(f"Incorrect version of mmgp ({mmgp_version}), version {target_mmgp_version} is needed. Please upgrade with the command 'pip install -r requirements.txt'")
+ exit()
+lock = threading.Lock()
+current_task_id = None
+task_id = 0
+vmc_event_handler = matanyone_app.get_vmc_event_handler()
+unique_id = 0
+unique_id_lock = threading.Lock()
+offloadobj = None
+wan_model = None
+
+def get_unique_id():
+ global unique_id
+ with unique_id_lock:
+ unique_id += 1
+ return str(time.time()+unique_id)
+
+def download_ffmpeg():
+ if os.name != 'nt': return
+ exes = ['ffmpeg.exe', 'ffprobe.exe', 'ffplay.exe']
+ if all(os.path.exists(e) for e in exes): return
+ api_url = 'https://api.github.com/repos/GyanD/codexffmpeg/releases/latest'
+ r = requests.get(api_url, headers={'Accept': 'application/vnd.github+json'})
+ assets = r.json().get('assets', [])
+ zip_asset = next((a for a in assets if 'essentials_build.zip' in a['name']), None)
+ if not zip_asset: return
+ zip_url = zip_asset['browser_download_url']
+ zip_name = zip_asset['name']
+ with requests.get(zip_url, stream=True) as resp:
+ total = int(resp.headers.get('Content-Length', 0))
+ with open(zip_name, 'wb') as f, tqdm(total=total, unit='B', unit_scale=True) as pbar:
+ for chunk in resp.iter_content(chunk_size=8192):
+ f.write(chunk)
+ pbar.update(len(chunk))
+ with zipfile.ZipFile(zip_name) as z:
+ for f in z.namelist():
+ if f.endswith(tuple(exes)) and '/bin/' in f:
+ z.extract(f)
+ os.rename(f, os.path.basename(f))
+ os.remove(zip_name)
+
+
+def format_time(seconds):
+ hours = int(seconds // 3600)
+ minutes = int((seconds % 3600) // 60)
+ secs = int(seconds % 60)
+
+ if hours > 0:
+ return f"{hours}h {minutes:02d}m {secs:02d}s"
+ elif seconds >= 60:
+ return f"{minutes}m {secs:02d}s"
+ else:
+ return f"{seconds:.1f}s"
+
+def pil_to_base64_uri(pil_image, format="png", quality=75):
+ if pil_image is None:
+ return None
+
+ if isinstance(pil_image, str):
+ from wan.utils.utils import get_video_frame
+ pil_image = get_video_frame(pil_image, 0)
+
+ buffer = io.BytesIO()
+ try:
+ img_to_save = pil_image
+ if format.lower() == 'jpeg' and pil_image.mode == 'RGBA':
+ img_to_save = pil_image.convert('RGB')
+ elif format.lower() == 'png' and pil_image.mode not in ['RGB', 'RGBA', 'L', 'P']:
+ img_to_save = pil_image.convert('RGBA')
+ elif pil_image.mode == 'P':
+ img_to_save = pil_image.convert('RGBA' if 'transparency' in pil_image.info else 'RGB')
+ if format.lower() == 'jpeg':
+ img_to_save.save(buffer, format=format, quality=quality)
+ else:
+ img_to_save.save(buffer, format=format)
+ img_bytes = buffer.getvalue()
+ encoded_string = base64.b64encode(img_bytes).decode("utf-8")
+ return f"data:image/{format.lower()};base64,{encoded_string}"
+ except Exception as e:
+ print(f"Error converting PIL to base64: {e}")
+ return None
+
+def is_integer(n):
+ try:
+ float(n)
+ except ValueError:
+ return False
+ else:
+ return float(n).is_integer()
+
+def compute_sliding_window_no(current_video_length, sliding_window_size, discard_last_frames, reuse_frames):
+ left_after_first_window = current_video_length - sliding_window_size + discard_last_frames
+ return 1 + math.ceil(left_after_first_window / (sliding_window_size - discard_last_frames - reuse_frames))
+
+
+def process_prompt_and_add_tasks(state, model_choice):
+
+ if state.get("validate_success",0) != 1:
+ return
+
+ state["validate_success"] = 0
+
+ model_filename = state["model_filename"]
+ model_type = state["model_type"]
+ inputs = get_model_settings(state, model_type)
+
+ if model_choice != model_type or inputs ==None:
+ raise gr.Error("Webform can not be used as the App has been restarted since the form was displayed. Please refresh the page")
+
+ inputs["state"] = state
+ gen = get_gen_info(state)
+ inputs["model_type"] = model_type
+ inputs.pop("lset_name")
+ if inputs == None:
+ gr.Warning("Internal state error: Could not retrieve inputs for the model.")
+ queue = gen.get("queue", [])
+ return get_queue_table(queue)
+ model_def = get_model_def(model_type)
+ image_outputs = inputs["image_mode"] == 1
+ no_steps_skipping = model_def.get("no_steps_skipping", False)
+ model_type = get_base_model_type(model_type)
+ inputs["model_filename"] = model_filename
+
+ mode = inputs["mode"]
+ if mode.startswith("edit_"):
+ edit_video_source =gen.get("edit_video_source", None)
+ edit_overrides =gen.get("edit_overrides", None)
+ _ , _ , _, frames_count = get_video_info(edit_video_source)
+ if frames_count > max_source_video_frames:
+ gr.Info(f"Post processing is not supported on videos longer than {max_source_video_frames} frames. Output Video will be truncated")
+ # return
+ for k in ["image_start", "image_end", "image_refs", "video_guide", "audio_guide", "audio_guide2", "audio_source" , "video_mask", "image_mask"]:
+ inputs[k] = None
+ inputs.update(edit_overrides)
+ del gen["edit_video_source"], gen["edit_overrides"]
+ inputs["video_source"]= edit_video_source
+ prompt = []
+
+ spatial_upsampling = inputs.get("spatial_upsampling","")
+ if len(spatial_upsampling) >0: prompt += ["Spatial Upsampling"]
+ temporal_upsampling = inputs.get("temporal_upsampling","")
+ if len(temporal_upsampling) >0: prompt += ["Temporal Upsampling"]
+ if has_image_file_extension(edit_video_source) and len(temporal_upsampling) > 0:
+ gr.Info("Temporal Upsampling can not be used with an Image")
+ return
+ film_grain_intensity = inputs.get("film_grain_intensity",0)
+ film_grain_saturation = inputs.get("film_grain_saturation",0.5)
+ # if film_grain_intensity >0: prompt += [f"Film Grain: intensity={film_grain_intensity}, saturation={film_grain_saturation}"]
+ if film_grain_intensity >0: prompt += ["Film Grain"]
+ MMAudio_setting = inputs.get("MMAudio_setting",0)
+ repeat_generation= inputs.get("repeat_generation",1)
+ if mode =="edit_remux":
+ audio_source = inputs["audio_source"]
+ if MMAudio_setting== 1:
+ prompt += ["MMAudio"]
+ audio_source = None
+ inputs["audio_source"] = audio_source
+ else:
+ if audio_source is None:
+ gr.Info("You must provide a custom Audio")
+ return
+ prompt += ["Custom Audio"]
+ repeat_generation == 1
+
+ seed = inputs.get("seed",None)
+ if len(prompt) == 0:
+ if mode=="edit_remux":
+ gr.Info("You must choose at least one Remux Method")
+ else:
+ gr.Info("You must choose at least one Post Processing Method")
+ return
+ inputs["prompt"] = ", ".join(prompt)
+ add_video_task(**inputs)
+ gen["prompts_max"] = 1 + gen.get("prompts_max",0)
+ state["validate_success"] = 1
+ queue= gen.get("queue", [])
+ return update_queue_data(queue)
+
+ if inputs.get("cfg_star_switch", 0) != 0 and inputs.get("apg_switch", 0) != 0:
+ gr.Info("Adaptive Progressive Guidance and Classifier Free Guidance Star can not be set at the same time")
+ return
+ prompt = inputs["prompt"]
+ if len(prompt) ==0:
+ gr.Info("Prompt cannot be empty.")
+ gen = get_gen_info(state)
+ queue = gen.get("queue", [])
+ return get_queue_table(queue)
+ prompt, errors = prompt_parser.process_template(prompt)
+ if len(errors) > 0:
+ gr.Info("Error processing prompt template: " + errors)
+ return
+ model_filename = get_model_filename(model_type)
+ prompts = prompt.replace("\r", "").split("\n")
+ prompts = [prompt.strip() for prompt in prompts if len(prompt.strip())>0 and not prompt.startswith("#")]
+ if len(prompts) == 0:
+ gr.Info("Prompt cannot be empty.")
+ gen = get_gen_info(state)
+ queue = gen.get("queue", [])
+ return get_queue_table(queue)
+
+ resolution = inputs["resolution"]
+ width, height = resolution.split("x")
+ width, height = int(width), int(height)
+ image_start = inputs["image_start"]
+ image_end = inputs["image_end"]
+ image_refs = inputs["image_refs"]
+ image_prompt_type = inputs["image_prompt_type"]
+ audio_prompt_type = inputs["audio_prompt_type"]
+ if image_prompt_type == None: image_prompt_type = ""
+ video_prompt_type = inputs["video_prompt_type"]
+ if video_prompt_type == None: video_prompt_type = ""
+ force_fps = inputs["force_fps"]
+ audio_guide = inputs["audio_guide"]
+ audio_guide2 = inputs["audio_guide2"]
+ audio_source = inputs["audio_source"]
+ video_guide = inputs["video_guide"]
+ image_guide = inputs["image_guide"]
+ video_mask = inputs["video_mask"]
+ image_mask = inputs["image_mask"]
+ speakers_locations = inputs["speakers_locations"]
+ video_source = inputs["video_source"]
+ frames_positions = inputs["frames_positions"]
+ keep_frames_video_guide= inputs["keep_frames_video_guide"]
+ keep_frames_video_source = inputs["keep_frames_video_source"]
+ denoising_strength= inputs["denoising_strength"]
+ sliding_window_size = inputs["sliding_window_size"]
+ sliding_window_overlap = inputs["sliding_window_overlap"]
+ sliding_window_discard_last_frames = inputs["sliding_window_discard_last_frames"]
+ video_length = inputs["video_length"]
+ num_inference_steps= inputs["num_inference_steps"]
+ skip_steps_cache_type= inputs["skip_steps_cache_type"]
+ MMAudio_setting = inputs["MMAudio_setting"]
+ image_mode = inputs["image_mode"]
+ switch_threshold = inputs["switch_threshold"]
+ loras_multipliers = inputs["loras_multipliers"]
+ activated_loras = inputs["activated_loras"]
+
+ if len(loras_multipliers) > 0:
+ _, _, errors = parse_loras_multipliers(loras_multipliers, len(activated_loras), num_inference_steps, max_phases= 2 if get_model_family(model_type)=="wan" and model_type not in ["sky_df_1.3B", "sky_df_14B"] else 1)
+ if len(errors) > 0:
+ gr.Info(f"Error parsing Loras Multipliers: {errors}")
+ return
+
+ if no_steps_skipping: skip_steps_cache_type = ""
+ if switch_threshold is not None and switch_threshold != 0 and len(skip_steps_cache_type) > 0:
+ gr.Info("Steps skipping is not yet supported if Switch Threshold is not null")
+ return
+ if not model_def.get("lock_inference_steps", False) and model_type in ["ltxv_13B"] and num_inference_steps < 20:
+ gr.Info("The minimum number of steps should be 20")
+ return
+ if skip_steps_cache_type == "mag":
+ if model_type in ["sky_df_1.3B", "sky_df_14B"]:
+ gr.Info("Mag Cache is not supported with Diffusion Forcing")
+ return
+ if num_inference_steps > 50:
+ gr.Info("Mag Cache maximum number of steps is 50")
+ return
+
+ if image_mode == 1:
+ audio_prompt_type = ""
+
+ if "B" in audio_prompt_type or "X" in audio_prompt_type:
+ from wan.multitalk.multitalk import parse_speakers_locations
+ speakers_bboxes, error = parse_speakers_locations(speakers_locations)
+ if len(error) > 0:
+ gr.Info(error)
+ return
+
+ if MMAudio_setting != 0 and server_config.get("mmaudio_enabled", 0) != 0 and video_length <16: #should depend on the architecture
+ gr.Info("MMAudio can generate an Audio track only if the Video is at least 1s long")
+ if "F" in video_prompt_type:
+ if len(frames_positions.strip()) > 0:
+ positions = frames_positions.split(" ")
+ for pos_str in positions:
+ if not is_integer(pos_str):
+ gr.Info(f"Invalid Frame Position '{pos_str}'")
+ return
+ pos = int(pos_str)
+ if pos <1 or pos > max_source_video_frames:
+ gr.Info(f"Invalid Frame Position Value'{pos_str}'")
+ return
+ else:
+ frames_positions = None
+
+ if audio_source is not None and MMAudio_setting != 0:
+ gr.Info("MMAudio and Custom Audio Soundtrack can't not be used at the same time")
+ return
+ if len(filter_letters(image_prompt_type, "VLG")) > 0 and len(keep_frames_video_source) > 0:
+ if not is_integer(keep_frames_video_source) or int(keep_frames_video_source) == 0:
+ gr.Info("The number of frames to keep must be a non null integer")
+ return
+ else:
+ keep_frames_video_source = ""
+
+ if "V" in image_prompt_type:
+ if video_source == None:
+ gr.Info("You must provide a Source Video file to continue")
+ return
+ else:
+ video_source = None
+
+ if "A" in audio_prompt_type:
+ if audio_guide == None:
+ gr.Info("You must provide an Audio Source")
+ return
+ if "B" in audio_prompt_type:
+ if audio_guide2 == None:
+ gr.Info("You must provide a second Audio Source")
+ return
+ else:
+ audio_guide2 = None
+ else:
+ audio_guide = None
+ audio_guide2 = None
+
+ if model_type in ["vace_multitalk_14B"] and ("B" in audio_prompt_type or "X" in audio_prompt_type):
+ if not "I" in video_prompt_type and not not "V" in video_prompt_type:
+ gr.Info("To get good results with Multitalk and two people speaking, it is recommended to set a Reference Frame or a Control Video (potentially truncated) that contains the two people one on each side")
+
+ # if len(filter_letters(image_prompt_type, "VL")) > 0 :
+ # if "R" in audio_prompt_type:
+ # gr.Info("Remuxing is not yet supported if there is a video source")
+ # audio_prompt_type= audio_prompt_type.replace("R" ,"")
+ # if "A" in audio_prompt_type:
+ # gr.Info("Creating an Audio track is not yet supported if there is a video source")
+ # return
+
+ if model_type in ["hunyuan_custom", "hunyuan_custom_edit", "hunyuan_audio", "hunyuan_avatar"]:
+ if image_refs == None :
+ gr.Info("You must provide an Image Reference")
+ return
+ if len(image_refs) > 1:
+ gr.Info("Only one Image Reference (a person) is supported for the moment by Hunyuan Custom / Avatar")
+ return
+
+ if "I" in video_prompt_type:
+ if image_refs == None or len(image_refs) == 0:
+ gr.Info("You must provide at least one Refererence Image")
+ return
+ if any(isinstance(image[0], str) for image in image_refs) :
+ gr.Info("A Reference Image should be an Image")
+ return
+ if isinstance(image_refs, list):
+ image_refs = [ convert_image(tup[0]) for tup in image_refs ]
+ else:
+ image_refs = None
+
+ if "V" in video_prompt_type:
+ if image_outputs:
+ if image_guide is None:
+ gr.Info("You must provide a Control Image")
+ return
+ else:
+ if video_guide is None:
+ gr.Info("You must provide a Control Video")
+ return
+ if "A" in video_prompt_type and not "U" in video_prompt_type:
+ if image_outputs:
+ if image_mask is None:
+ gr.Info("You must provide a Image Mask")
+ return
+ else:
+ if video_mask is None:
+ gr.Info("You must provide a Video Mask")
+ return
+ else:
+ video_mask = None
+ image_mask = None
+
+ if "G" in video_prompt_type:
+ gr.Info(f"With Denoising Strength {denoising_strength:.1f}, denoising will start a Step no {int(num_inference_steps * (1. - denoising_strength))} ")
+ else:
+ denoising_strength = 1.0
+ if len(keep_frames_video_guide) > 0 and model_type in ["ltxv_13B"]:
+ gr.Info("Keep Frames for Control Video is not supported with LTX Video")
+ return
+ _, error = parse_keep_frames_video_guide(keep_frames_video_guide, video_length)
+ if len(error) > 0:
+ gr.Info(f"Invalid Keep Frames property: {error}")
+ return
+ else:
+ video_guide = None
+ image_guide = None
+ video_mask = None
+ image_mask = None
+ keep_frames_video_guide = ""
+ denoising_strength = 1.0
+
+ if image_outputs:
+ video_guide = None
+ video_mask = None
+ else:
+ image_guide = None
+ image_mask = None
+
+
+ if "S" in image_prompt_type:
+ if image_start == None or isinstance(image_start, list) and len(image_start) == 0:
+ gr.Info("You must provide a Start Image")
+ return
+ if not isinstance(image_start, list):
+ image_start = [image_start]
+ if not all( not isinstance(img[0], str) for img in image_start) :
+ gr.Info("Start Image should be an Image")
+ return
+ image_start = [ convert_image(tup[0]) for tup in image_start ]
+ else:
+ image_start = None
+
+ if "E" in image_prompt_type:
+ if image_end == None or isinstance(image_end, list) and len(image_end) == 0:
+ gr.Info("You must provide an End Image")
+ return
+ if not isinstance(image_end, list):
+ image_end = [image_end]
+ if not all( not isinstance(img[0], str) for img in image_end) :
+ gr.Info("End Image should be an Image")
+ return
+ if len(image_start) != len(image_end):
+ gr.Info("The number of Start and End Images should be the same ")
+ return
+ image_end = [ convert_image(tup[0]) for tup in image_end ]
+ else:
+ image_end = None
+
+
+ if test_any_sliding_window(model_type) and image_mode == 0:
+ if video_length > sliding_window_size:
+ full_video_length = video_length if video_source is None else video_length + sliding_window_overlap
+ extra = "" if full_video_length == video_length else f" including {sliding_window_overlap} added for Video Continuation"
+ no_windows = compute_sliding_window_no(full_video_length, sliding_window_size, sliding_window_discard_last_frames, sliding_window_overlap)
+ gr.Info(f"The Number of Frames to generate ({video_length}{extra}) is greater than the Sliding Window Size ({sliding_window_size}), {no_windows} Windows will be generated")
+
+ if "recam" in model_filename:
+ if video_source == None:
+ gr.Info("You must provide a Source Video")
+ return
+
+ frames = get_resampled_video(video_source, 0, 81, get_computed_fps(force_fps, model_type , video_guide, video_source ))
+ if len(frames)<81:
+ gr.Info("Recammaster source video should be at least 81 frames once the resampling at 16 fps has been done")
+ return
+
+
+
+ if "hunyuan_custom_custom_edit" in model_filename:
+ if len(keep_frames_video_guide) > 0:
+ gr.Info("Filtering Frames with this model is not supported")
+ return
+
+ if inputs["multi_prompts_gen_type"] != 0:
+ if image_start != None and len(image_start) > 1:
+ gr.Info("Only one Start Image must be provided if multiple prompts are used for different windows")
+ return
+
+ if image_end != None and len(image_end) > 1:
+ gr.Info("Only one End Image must be provided if multiple prompts are used for different windows")
+ return
+
+ override_inputs = {
+ "image_start": image_start[0] if image_start !=None and len(image_start) > 0 else None,
+ "image_end": image_end[0] if image_end !=None and len(image_end) > 0 else None,
+ "image_refs": image_refs,
+ "audio_guide": audio_guide,
+ "audio_guide2": audio_guide2,
+ "audio_source": audio_source,
+ "video_guide": video_guide,
+ "image_guide": image_guide,
+ "video_mask": video_mask,
+ "image_mask": image_mask,
+ "video_source": video_source,
+ "frames_positions": frames_positions,
+ "keep_frames_video_source": keep_frames_video_source,
+ "keep_frames_video_guide": keep_frames_video_guide,
+ "denoising_strength": denoising_strength,
+ "image_prompt_type": image_prompt_type,
+ "video_prompt_type": video_prompt_type,
+ "audio_prompt_type": audio_prompt_type,
+ "skip_steps_cache_type": skip_steps_cache_type
+ }
+
+ if inputs["multi_prompts_gen_type"] == 0:
+ if image_start != None and len(image_start) > 0:
+ if inputs["multi_images_gen_type"] == 0:
+ new_prompts = []
+ new_image_start = []
+ new_image_end = []
+ for i in range(len(prompts) * len(image_start) ):
+ new_prompts.append( prompts[ i % len(prompts)] )
+ new_image_start.append(image_start[i // len(prompts)] )
+ if image_end != None:
+ new_image_end.append(image_end[i // len(prompts)] )
+ prompts = new_prompts
+ image_start = new_image_start
+ if image_end != None:
+ image_end = new_image_end
+ else:
+ if len(prompts) >= len(image_start):
+ if len(prompts) % len(image_start) != 0:
+ gr.Info("If there are more text prompts than input images the number of text prompts should be dividable by the number of images")
+ return
+ rep = len(prompts) // len(image_start)
+ new_image_start = []
+ new_image_end = []
+ for i, _ in enumerate(prompts):
+ new_image_start.append(image_start[i//rep] )
+ if image_end != None:
+ new_image_end.append(image_end[i//rep] )
+ image_start = new_image_start
+ if image_end != None:
+ image_end = new_image_end
+ else:
+ if len(image_start) % len(prompts) !=0:
+ gr.Info("If there are more input images than text prompts the number of images should be dividable by the number of text prompts")
+ return
+ rep = len(image_start) // len(prompts)
+ new_prompts = []
+ for i, _ in enumerate(image_start):
+ new_prompts.append( prompts[ i//rep] )
+ prompts = new_prompts
+ if image_end == None or len(image_end) == 0:
+ image_end = [None] * len(prompts)
+
+ for single_prompt, start, end in zip(prompts, image_start, image_end) :
+ override_inputs.update({
+ "prompt" : single_prompt,
+ "image_start": start,
+ "image_end" : end,
+ })
+ inputs.update(override_inputs)
+ add_video_task(**inputs)
+ else:
+ for single_prompt in prompts :
+ override_inputs["prompt"] = single_prompt
+ inputs.update(override_inputs)
+ add_video_task(**inputs)
+ else:
+ override_inputs["prompt"] = "\n".join(prompts)
+ inputs.update(override_inputs)
+ add_video_task(**inputs)
+
+ gen["prompts_max"] = len(prompts) + gen.get("prompts_max",0)
+ state["validate_success"] = 1
+ queue= gen.get("queue", [])
+ return update_queue_data(queue)
+
+def get_preview_images(inputs):
+ inputs_to_query = ["image_start", "image_end", "video_source", "video_guide", "image_guide", "video_mask", "image_mask", "image_refs" ]
+ labels = ["Start Image", "End Image", "Video Source", "Video Guide", "Image Guide", "Video Mask", "Image Mask", "Image Reference"]
+ start_image_data = None
+ start_image_labels = []
+ end_image_data = None
+ end_image_labels = []
+ for label, name in zip(labels,inputs_to_query):
+ image= inputs.get(name, None)
+ if image is not None:
+ image= [image] if not isinstance(image, list) else image.copy()
+ if start_image_data == None:
+ start_image_data = image
+ start_image_labels += [label] * len(image)
+ else:
+ if end_image_data == None:
+ end_image_data = image
+ else:
+ end_image_data += image
+ end_image_labels += [label] * len(image)
+
+ if start_image_data != None and len(start_image_data) > 1 and end_image_data == None:
+ end_image_data = start_image_data [1:]
+ end_image_labels = start_image_labels [1:]
+ start_image_data = start_image_data [:1]
+ start_image_labels = start_image_labels [:1]
+ return start_image_data, end_image_data, start_image_labels, end_image_labels
+
+def add_video_task(**inputs):
+ global task_id
+ state = inputs["state"]
+ gen = get_gen_info(state)
+ queue = gen["queue"]
+ task_id += 1
+ current_task_id = task_id
+
+ start_image_data, end_image_data, start_image_labels, end_image_labels = get_preview_images(inputs)
+
+ queue.append({
+ "id": current_task_id,
+ "params": inputs.copy(),
+ "repeats": inputs["repeat_generation"],
+ "length": inputs["video_length"], # !!!
+ "steps": inputs["num_inference_steps"],
+ "prompt": inputs["prompt"],
+ "start_image_labels": start_image_labels,
+ "end_image_labels": end_image_labels,
+ "start_image_data": start_image_data,
+ "end_image_data": end_image_data,
+ "start_image_data_base64": [pil_to_base64_uri(img, format="jpeg", quality=70) for img in start_image_data] if start_image_data != None else None,
+ "end_image_data_base64": [pil_to_base64_uri(img, format="jpeg", quality=70) for img in end_image_data] if end_image_data != None else None
+ })
+ return update_queue_data(queue)
+
+def update_task_thumbnails(task, inputs):
+ start_image_data, end_image_data, start_labels, end_labels = get_preview_images(inputs)
+
+ task.update({
+ "start_image_labels": start_labels,
+ "end_image_labels": end_labels,
+ "start_image_data_base64": [pil_to_base64_uri(img, format="jpeg", quality=70) for img in start_image_data] if start_image_data != None else None,
+ "end_image_data_base64": [pil_to_base64_uri(img, format="jpeg", quality=70) for img in end_image_data] if end_image_data != None else None
+ })
+
+def move_up(queue, selected_indices):
+ if not selected_indices or len(selected_indices) == 0:
+ return update_queue_data(queue)
+ idx = selected_indices[0]
+ if isinstance(idx, list):
+ idx = idx[0]
+ idx = int(idx)
+ with lock:
+ idx += 1
+ if idx > 1:
+ queue[idx], queue[idx-1] = queue[idx-1], queue[idx]
+ elif idx == 1:
+ queue[:] = queue[0:1] + queue[2:] + queue[1:2]
+
+ return update_queue_data(queue)
+
+def move_down(queue, selected_indices):
+ if not selected_indices or len(selected_indices) == 0:
+ return update_queue_data(queue)
+ idx = selected_indices[0]
+ if isinstance(idx, list):
+ idx = idx[0]
+ idx = int(idx)
+ with lock:
+ idx += 1
+ if idx < len(queue)-1:
+ queue[idx], queue[idx+1] = queue[idx+1], queue[idx]
+ elif idx == len(queue)-1:
+ queue[:] = queue[0:1] + queue[-1:] + queue[1:-1]
+
+ return update_queue_data(queue)
+
+def remove_task(queue, selected_indices):
+ if not selected_indices or len(selected_indices) == 0:
+ return update_queue_data(queue)
+ idx = selected_indices[0]
+ if isinstance(idx, list):
+ idx = idx[0]
+ idx = int(idx) + 1
+ with lock:
+ if idx < len(queue):
+ if idx == 0:
+ wan_model._interrupt = True
+ del queue[idx]
+ return update_queue_data(queue)
+
+def update_global_queue_ref(queue):
+ global global_queue_ref
+ with lock:
+ global_queue_ref = queue[:]
+
+def save_queue_action(state):
+ gen = get_gen_info(state)
+ queue = gen.get("queue", [])
+
+ if not queue or len(queue) <=1 :
+ gr.Info("Queue is empty. Nothing to save.")
+ return ""
+
+ zip_buffer = io.BytesIO()
+
+ with tempfile.TemporaryDirectory() as tmpdir:
+ queue_manifest = []
+ file_paths_in_zip = {}
+
+ for task_index, task in enumerate(queue):
+ if task is None or not isinstance(task, dict) or task.get('id') is None: continue
+
+ params_copy = task.get('params', {}).copy()
+ task_id_s = task.get('id', f"task_{task_index}")
+
+ image_keys = ["image_start", "image_end", "image_refs", "image_guide", "image_mask"]
+ video_keys = ["video_guide", "video_mask", "video_source", "audio_guide", "audio_guide2", "audio_source"]
+
+ for key in image_keys:
+ images_pil = params_copy.get(key)
+ if images_pil is None:
+ continue
+
+ is_originally_list = isinstance(images_pil, list)
+ if not is_originally_list:
+ images_pil = [images_pil]
+
+ image_filenames_for_json = []
+ for img_index, pil_image in enumerate(images_pil):
+ if not isinstance(pil_image, Image.Image):
+ print(f"Warning: Expected PIL Image for key '{key}' in task {task_id_s}, got {type(pil_image)}. Skipping image.")
+ continue
+
+ img_id = id(pil_image)
+ if img_id in file_paths_in_zip:
+ image_filenames_for_json.append(file_paths_in_zip[img_id])
+ continue
+
+ img_filename_in_zip = f"task{task_id_s}_{key}_{img_index}.png"
+ img_save_path = os.path.join(tmpdir, img_filename_in_zip)
+
+ try:
+ pil_image.save(img_save_path, "PNG")
+ image_filenames_for_json.append(img_filename_in_zip)
+ file_paths_in_zip[img_id] = img_filename_in_zip
+ print(f"Saved image: {img_filename_in_zip}")
+ except Exception as e:
+ print(f"Error saving image {img_filename_in_zip} for task {task_id_s}: {e}")
+
+ if image_filenames_for_json:
+ params_copy[key] = image_filenames_for_json if is_originally_list else image_filenames_for_json[0]
+ else:
+ pass
+ # params_copy.pop(key, None) #cant pop otherwise crash during reload
+
+ for key in video_keys:
+ video_path_orig = params_copy.get(key)
+ if video_path_orig is None or not isinstance(video_path_orig, str):
+ continue
+
+ if video_path_orig in file_paths_in_zip:
+ params_copy[key] = file_paths_in_zip[video_path_orig]
+ continue
+
+ if not os.path.isfile(video_path_orig):
+ print(f"Warning: Video file not found for key '{key}' in task {task_id_s}: {video_path_orig}. Skipping video.")
+ params_copy.pop(key, None)
+ continue
+
+ _, extension = os.path.splitext(video_path_orig)
+ vid_filename_in_zip = f"task{task_id_s}_{key}{extension if extension else '.mp4'}"
+ vid_save_path = os.path.join(tmpdir, vid_filename_in_zip)
+
+ try:
+ shutil.copy2(video_path_orig, vid_save_path)
+ params_copy[key] = vid_filename_in_zip
+ file_paths_in_zip[video_path_orig] = vid_filename_in_zip
+ print(f"Copied video: {video_path_orig} -> {vid_filename_in_zip}")
+ except Exception as e:
+ print(f"Error copying video {video_path_orig} to {vid_filename_in_zip} for task {task_id_s}: {e}")
+ params_copy.pop(key, None)
+
+
+ params_copy.pop('state', None)
+ params_copy.pop('start_image_labels', None)
+ params_copy.pop('end_image_labels', None)
+ params_copy.pop('start_image_data_base64', None)
+ params_copy.pop('end_image_data_base64', None)
+ params_copy.pop('start_image_data', None)
+ params_copy.pop('end_image_data', None)
+ task.pop('start_image_data', None)
+ task.pop('end_image_data', None)
+
+ manifest_entry = {
+ "id": task.get('id'),
+ "params": params_copy,
+ }
+ manifest_entry = {k: v for k, v in manifest_entry.items() if v is not None}
+ queue_manifest.append(manifest_entry)
+
+ manifest_path = os.path.join(tmpdir, "queue.json")
+ try:
+ with open(manifest_path, 'w', encoding='utf-8') as f:
+ json.dump(queue_manifest, f, indent=4)
+ except Exception as e:
+ print(f"Error writing queue.json: {e}")
+ gr.Warning("Failed to create queue manifest.")
+ return None
+
+ try:
+ with zipfile.ZipFile(zip_buffer, 'w', zipfile.ZIP_DEFLATED) as zf:
+ zf.write(manifest_path, arcname="queue.json")
+
+ for file_id, saved_file_rel_path in file_paths_in_zip.items():
+ saved_file_abs_path = os.path.join(tmpdir, saved_file_rel_path)
+ if os.path.exists(saved_file_abs_path):
+ zf.write(saved_file_abs_path, arcname=saved_file_rel_path)
+ print(f"Adding to zip: {saved_file_rel_path}")
+ else:
+ print(f"Warning: File {saved_file_rel_path} (ID: {file_id}) not found during zipping.")
+
+ zip_buffer.seek(0)
+ zip_binary_content = zip_buffer.getvalue()
+ zip_base64 = base64.b64encode(zip_binary_content).decode('utf-8')
+ print(f"Queue successfully prepared as base64 string ({len(zip_base64)} chars).")
+ return zip_base64
+
+ except Exception as e:
+ print(f"Error creating zip file in memory: {e}")
+ gr.Warning("Failed to create zip data for download.")
+ return None
+ finally:
+ zip_buffer.close()
+
+def load_queue_action(filepath, state, evt:gr.EventData):
+ global task_id
+
+ gen = get_gen_info(state)
+ original_queue = gen.get("queue", [])
+ delete_autoqueue_file = False
+ if evt.target == None:
+
+ if original_queue or not Path(AUTOSAVE_FILENAME).is_file():
+ return
+ print(f"Autoloading queue from {AUTOSAVE_FILENAME}...")
+ filename = AUTOSAVE_FILENAME
+ delete_autoqueue_file = True
+ else:
+ if not filepath or not hasattr(filepath, 'name') or not Path(filepath.name).is_file():
+ print("[load_queue_action] Warning: No valid file selected or file not found.")
+ return update_queue_data(original_queue)
+ filename = filepath.name
+
+
+ save_path_base = server_config.get("save_path", "outputs")
+ loaded_cache_dir = os.path.join(save_path_base, "_loaded_queue_cache")
+
+
+ newly_loaded_queue = []
+ max_id_in_file = 0
+ error_message = ""
+ local_queue_copy_for_global_ref = None
+
+ try:
+ print(f"[load_queue_action] Attempting to load queue from: {filename}")
+ os.makedirs(loaded_cache_dir, exist_ok=True)
+ print(f"[load_queue_action] Using cache directory: {loaded_cache_dir}")
+
+ with tempfile.TemporaryDirectory() as tmpdir:
+ with zipfile.ZipFile(filename, 'r') as zf:
+ if "queue.json" not in zf.namelist(): raise ValueError("queue.json not found in zip file")
+ print(f"[load_queue_action] Extracting {filename} to {tmpdir}")
+ zf.extractall(tmpdir)
+ print(f"[load_queue_action] Extraction complete.")
+
+ manifest_path = os.path.join(tmpdir, "queue.json")
+ print(f"[load_queue_action] Reading manifest: {manifest_path}")
+ with open(manifest_path, 'r', encoding='utf-8') as f:
+ loaded_manifest = json.load(f)
+ print(f"[load_queue_action] Manifest loaded. Processing {len(loaded_manifest)} tasks.")
+
+ for task_index, task_data in enumerate(loaded_manifest):
+ if task_data is None or not isinstance(task_data, dict):
+ print(f"[load_queue_action] Skipping invalid task data at index {task_index}")
+ continue
+
+ params = task_data.get('params', {})
+ task_id_loaded = task_data.get('id', 0)
+ max_id_in_file = max(max_id_in_file, task_id_loaded)
+ params['state'] = state
+
+ image_keys = ["image_start", "image_end", "image_refs", "image_guide", "image_mask"]
+ video_keys = ["video_guide", "video_mask", "video_source", "audio_guide", "audio_guide2", "audio_source"]
+
+ loaded_pil_images = {}
+ loaded_video_paths = {}
+
+ for key in image_keys:
+ image_filenames = params.get(key)
+ if image_filenames is None: continue
+
+ is_list = isinstance(image_filenames, list)
+ if not is_list: image_filenames = [image_filenames]
+
+ loaded_pils = []
+ for img_filename_in_zip in image_filenames:
+ if not isinstance(img_filename_in_zip, str):
+ print(f"[load_queue_action] Warning: Non-string filename found for image key '{key}'. Skipping.")
+ continue
+ img_load_path = os.path.join(tmpdir, img_filename_in_zip)
+ if not os.path.exists(img_load_path):
+ print(f"[load_queue_action] Image file not found in extracted data: {img_load_path}. Skipping.")
+ continue
+ try:
+ pil_image = Image.open(img_load_path)
+ pil_image.load()
+ converted_image = convert_image(pil_image)
+ loaded_pils.append(converted_image)
+ pil_image.close()
+ print(f"Loaded image: {img_filename_in_zip} for key {key}")
+ except Exception as img_e:
+ print(f"[load_queue_action] Error loading image {img_filename_in_zip}: {img_e}")
+ if loaded_pils:
+ params[key] = loaded_pils if is_list else loaded_pils[0]
+ loaded_pil_images[key] = params[key]
+ else:
+ params.pop(key, None)
+
+ for key in video_keys:
+ video_filename_in_zip = params.get(key)
+ if video_filename_in_zip is None or not isinstance(video_filename_in_zip, str):
+ continue
+
+ video_load_path = os.path.join(tmpdir, video_filename_in_zip)
+ if not os.path.exists(video_load_path):
+ print(f"[load_queue_action] Video file not found in extracted data: {video_load_path}. Skipping.")
+ params.pop(key, None)
+ continue
+
+ persistent_video_path = os.path.join(loaded_cache_dir, video_filename_in_zip)
+ try:
+ shutil.copy2(video_load_path, persistent_video_path)
+ params[key] = persistent_video_path
+ loaded_video_paths[key] = persistent_video_path
+ print(f"Loaded video: {video_filename_in_zip} -> {persistent_video_path}")
+ except Exception as vid_e:
+ print(f"[load_queue_action] Error copying video {video_filename_in_zip} to cache: {vid_e}")
+ params.pop(key, None)
+
+ primary_preview_pil_list, secondary_preview_pil_list, primary_preview_pil_labels, secondary_preview_pil_labels = get_preview_images(params)
+
+ start_b64 = [pil_to_base64_uri(primary_preview_pil_list[0], format="jpeg", quality=70)] if isinstance(primary_preview_pil_list, list) and primary_preview_pil_list else None
+ end_b64 = [pil_to_base64_uri(secondary_preview_pil_list[0], format="jpeg", quality=70)] if isinstance(secondary_preview_pil_list, list) and secondary_preview_pil_list else None
+
+ top_level_start_image = params.get("image_start") or params.get("image_refs")
+ top_level_end_image = params.get("image_end")
+
+ runtime_task = {
+ "id": task_id_loaded,
+ "params": params.copy(),
+ "repeats": params.get('repeat_generation', 1),
+ "length": params.get('video_length'),
+ "steps": params.get('num_inference_steps'),
+ "prompt": params.get('prompt'),
+ "start_image_labels": primary_preview_pil_labels,
+ "end_image_labels": secondary_preview_pil_labels,
+ "start_image_data": top_level_start_image,
+ "end_image_data": top_level_end_image,
+ "start_image_data_base64": start_b64,
+ "end_image_data_base64": end_b64,
+ }
+ newly_loaded_queue.append(runtime_task)
+ print(f"[load_queue_action] Reconstructed task {task_index+1}/{len(loaded_manifest)}, ID: {task_id_loaded}")
+
+ with lock:
+ print("[load_queue_action] Acquiring lock to update state...")
+ gen["queue"] = newly_loaded_queue[:]
+ local_queue_copy_for_global_ref = gen["queue"][:]
+
+ current_max_id_in_new_queue = max([t['id'] for t in newly_loaded_queue if 'id' in t] + [0])
+ if current_max_id_in_new_queue >= task_id:
+ new_task_id = current_max_id_in_new_queue + 1
+ print(f"[load_queue_action] Updating global task_id from {task_id} to {new_task_id}")
+ task_id = new_task_id
+ else:
+ print(f"[load_queue_action] Global task_id ({task_id}) is > max in file ({current_max_id_in_new_queue}). Not changing task_id.")
+
+ gen["prompts_max"] = len(newly_loaded_queue)
+ print("[load_queue_action] State update complete. Releasing lock.")
+
+ if local_queue_copy_for_global_ref is not None:
+ print("[load_queue_action] Updating global queue reference...")
+ update_global_queue_ref(local_queue_copy_for_global_ref)
+ else:
+ print("[load_queue_action] Warning: Skipping global ref update as local copy is None.")
+
+ print(f"[load_queue_action] Queue load successful. Returning DataFrame update for {len(newly_loaded_queue)} tasks.")
+ return update_queue_data(newly_loaded_queue)
+
+ except (ValueError, zipfile.BadZipFile, FileNotFoundError, Exception) as e:
+ error_message = f"Error during queue load: {e}"
+ print(f"[load_queue_action] Caught error: {error_message}")
+ traceback.print_exc()
+ gr.Warning(f"Failed to load queue: {error_message[:200]}")
+
+ print("[load_queue_action] Load failed. Returning DataFrame update for original queue.")
+ return update_queue_data(original_queue)
+ finally:
+ if delete_autoqueue_file:
+ if os.path.isfile(filename):
+ os.remove(filename)
+ print(f"Clear Queue: Deleted autosave file '{filename}'.")
+
+ if filepath and hasattr(filepath, 'name') and filepath.name and os.path.exists(filepath.name):
+ if tempfile.gettempdir() in os.path.abspath(filepath.name):
+ try:
+ os.remove(filepath.name)
+ print(f"[load_queue_action] Removed temporary upload file: {filepath.name}")
+ except OSError as e:
+ print(f"[load_queue_action] Info: Could not remove temp file {filepath.name}: {e}")
+ else:
+ print(f"[load_queue_action] Info: Did not remove non-temporary file: {filepath.name}")
+
+def clear_queue_action(state):
+ gen = get_gen_info(state)
+ queue = gen.get("queue", [])
+ aborted_current = False
+ cleared_pending = False
+
+ with lock:
+ if "in_progress" in gen and gen["in_progress"]:
+ print("Clear Queue: Signalling abort for in-progress task.")
+ gen["abort"] = True
+ gen["extra_orders"] = 0
+ if wan_model is not None:
+ wan_model._interrupt = True
+ aborted_current = True
+
+ if queue:
+ if len(queue) > 1 or (len(queue) == 1 and queue[0] is not None and queue[0].get('id') is not None):
+ print(f"Clear Queue: Clearing {len(queue)} tasks from queue.")
+ queue.clear()
+ cleared_pending = True
+ else:
+ pass
+
+ if aborted_current or cleared_pending:
+ gen["prompts_max"] = 0
+
+ if cleared_pending:
+ try:
+ if os.path.isfile(AUTOSAVE_FILENAME):
+ os.remove(AUTOSAVE_FILENAME)
+ print(f"Clear Queue: Deleted autosave file '{AUTOSAVE_FILENAME}'.")
+ except OSError as e:
+ print(f"Clear Queue: Error deleting autosave file '{AUTOSAVE_FILENAME}': {e}")
+ gr.Warning(f"Could not delete the autosave file '{AUTOSAVE_FILENAME}'. You may need to remove it manually.")
+
+ if aborted_current and cleared_pending:
+ gr.Info("Queue cleared and current generation aborted.")
+ elif aborted_current:
+ gr.Info("Current generation aborted.")
+ elif cleared_pending:
+ gr.Info("Queue cleared.")
+ else:
+ gr.Info("Queue is already empty or only contains the active task (which wasn't aborted now).")
+
+ return update_queue_data([])
+
+def quit_application():
+ print("Save and Quit requested...")
+ autosave_queue()
+ import signal
+ os.kill(os.getpid(), signal.SIGINT)
+
+def start_quit_process():
+ return 5, gr.update(visible=False), gr.update(visible=True)
+
+def cancel_quit_process():
+ return -1, gr.update(visible=True), gr.update(visible=False)
+
+def show_countdown_info_from_state(current_value: int):
+ if current_value > 0:
+ gr.Info(f"Quitting in {current_value}...")
+ return current_value - 1
+ return current_value
+quitting_app = False
+def autosave_queue():
+ global quitting_app
+ quitting_app = True
+ global global_queue_ref
+ if not global_queue_ref:
+ print("Autosave: Queue is empty, nothing to save.")
+ return
+
+ print(f"Autosaving queue ({len(global_queue_ref)} items) to {AUTOSAVE_FILENAME}...")
+ temp_state_for_save = {"gen": {"queue": global_queue_ref}}
+ zip_file_path = None
+ try:
+
+ def _save_queue_to_file(queue_to_save, output_filename):
+ if not queue_to_save: return None
+
+ with tempfile.TemporaryDirectory() as tmpdir:
+ queue_manifest = []
+ file_paths_in_zip = {}
+
+ for task_index, task in enumerate(queue_to_save):
+ if task is None or not isinstance(task, dict) or task.get('id') is None: continue
+
+ params_copy = task.get('params', {}).copy()
+ task_id_s = task.get('id', f"task_{task_index}")
+
+ image_keys = ["image_start", "image_end", "image_refs", "image_guide", "image_mask"]
+ video_keys = ["video_guide", "video_mask", "video_source", "audio_guide", "audio_guide2", "audio_source" ]
+
+ for key in image_keys:
+ images_pil = params_copy.get(key)
+ if images_pil is None: continue
+ is_list = isinstance(images_pil, list)
+ if not is_list: images_pil = [images_pil]
+ image_filenames_for_json = []
+ for img_index, pil_image in enumerate(images_pil):
+ if not isinstance(pil_image, Image.Image): continue
+ img_id = id(pil_image)
+ if img_id in file_paths_in_zip:
+ image_filenames_for_json.append(file_paths_in_zip[img_id])
+ continue
+ img_filename_in_zip = f"task{task_id_s}_{key}_{img_index}.png"
+ img_save_path = os.path.join(tmpdir, img_filename_in_zip)
+ try:
+ pil_image.save(img_save_path, "PNG")
+ image_filenames_for_json.append(img_filename_in_zip)
+ file_paths_in_zip[img_id] = img_filename_in_zip
+ except Exception as e:
+ print(f"Autosave error saving image {img_filename_in_zip}: {e}")
+ if image_filenames_for_json:
+ params_copy[key] = image_filenames_for_json if is_list else image_filenames_for_json[0]
+ else:
+ params_copy.pop(key, None)
+
+ for key in video_keys:
+ video_path_orig = params_copy.get(key)
+ if video_path_orig is None or not isinstance(video_path_orig, str):
+ continue
+
+ if video_path_orig in file_paths_in_zip:
+ params_copy[key] = file_paths_in_zip[video_path_orig]
+ continue
+
+ if not os.path.isfile(video_path_orig):
+ print(f"Warning (Autosave): Video file not found for key '{key}' in task {task_id_s}: {video_path_orig}. Skipping.")
+ params_copy.pop(key, None)
+ continue
+
+ _, extension = os.path.splitext(video_path_orig)
+ vid_filename_in_zip = f"task{task_id_s}_{key}{extension if extension else '.mp4'}"
+ vid_save_path = os.path.join(tmpdir, vid_filename_in_zip)
+
+ try:
+ shutil.copy2(video_path_orig, vid_save_path)
+ params_copy[key] = vid_filename_in_zip
+ file_paths_in_zip[video_path_orig] = vid_filename_in_zip
+ except Exception as e:
+ print(f"Error (Autosave) copying video {video_path_orig} to {vid_filename_in_zip} for task {task_id_s}: {e}")
+ params_copy.pop(key, None)
+ params_copy.pop('state', None)
+ params_copy.pop('start_image_data_base64', None)
+ params_copy.pop('end_image_data_base64', None)
+ params_copy.pop('start_image_data', None)
+ params_copy.pop('end_image_data', None)
+
+ manifest_entry = {
+ "id": task.get('id'),
+ "params": params_copy,
+ }
+ manifest_entry = {k: v for k, v in manifest_entry.items() if v is not None}
+ queue_manifest.append(manifest_entry)
+
+ manifest_path = os.path.join(tmpdir, "queue.json")
+ with open(manifest_path, 'w', encoding='utf-8') as f: json.dump(queue_manifest, f, indent=4)
+ with zipfile.ZipFile(output_filename, 'w', zipfile.ZIP_DEFLATED) as zf:
+ zf.write(manifest_path, arcname="queue.json")
+ for saved_file_rel_path in file_paths_in_zip.values():
+ saved_file_abs_path = os.path.join(tmpdir, saved_file_rel_path)
+ if os.path.exists(saved_file_abs_path):
+ zf.write(saved_file_abs_path, arcname=saved_file_rel_path)
+ else:
+ print(f"Warning (Autosave): File {saved_file_rel_path} not found during zipping.")
+ return output_filename
+ return None
+
+ saved_path = _save_queue_to_file(global_queue_ref, AUTOSAVE_FILENAME)
+
+ if saved_path:
+ print(f"Queue autosaved successfully to {saved_path}")
+ else:
+ print("Autosave failed.")
+ except Exception as e:
+ print(f"Error during autosave: {e}")
+ traceback.print_exc()
+
+def finalize_generation_with_state(current_state):
+ if not isinstance(current_state, dict) or 'gen' not in current_state:
+ return gr.update(), gr.update(interactive=True), gr.update(visible=True), gr.update(visible=False), gr.update(visible=False), gr.update(visible=False, value=""), gr.update(), current_state
+
+ gallery_update, abort_btn_update, gen_btn_update, add_queue_btn_update, current_gen_col_update, gen_info_update = finalize_generation(current_state)
+ accordion_update = gr.Accordion(open=False) if len(get_gen_info(current_state).get("queue", [])) <= 1 else gr.update()
+ return gallery_update, abort_btn_update, gen_btn_update, add_queue_btn_update, current_gen_col_update, gen_info_update, accordion_update, current_state
+
+def get_queue_table(queue):
+ data = []
+ if len(queue) == 1:
+ return data
+
+ for i, item in enumerate(queue):
+ if i==0:
+ continue
+ truncated_prompt = (item['prompt'][:97] + '...') if len(item['prompt']) > 100 else item['prompt']
+ full_prompt = item['prompt'].replace('"', '"')
+ prompt_cell = f'{truncated_prompt}'
+ start_img_uri =item.get('start_image_data_base64')
+ start_img_uri = start_img_uri[0] if start_img_uri !=None else None
+ start_img_labels =item.get('start_image_labels')
+ end_img_uri = item.get('end_image_data_base64')
+ end_img_uri = end_img_uri[0] if end_img_uri !=None else None
+ end_img_labels =item.get('end_image_labels')
+ thumbnail_size = "50px"
+ num_steps = item.get('steps')
+ length = item.get('length')
+ start_img_md = ""
+ end_img_md = ""
+ if start_img_uri:
+ start_img_md = f'
+
+
+ """
+ return html
+
+def update_generation_status(html_content):
+ if(html_content):
+ return gr.update(value=html_content)
+
+def _parse_args():
+ parser = argparse.ArgumentParser(
+ description="Generate a video from a text prompt or image using Gradio")
+
+ parser.add_argument(
+ "--save-masks",
+ action="store_true",
+ help="save proprocessed masks for debugging or editing"
+ )
+
+ parser.add_argument(
+ "--save-speakers",
+ action="store_true",
+ help="save proprocessed audio track with extract speakers for debugging or editing"
+ )
+
+ parser.add_argument(
+ "--share",
+ action="store_true",
+ help="Create a shared URL to access webserver remotely"
+ )
+
+ parser.add_argument(
+ "--lock-config",
+ action="store_true",
+ help="Prevent modifying the configuration from the web interface"
+ )
+
+ parser.add_argument(
+ "--lock-model",
+ action="store_true",
+ help="Prevent switch models"
+ )
+
+ parser.add_argument(
+ "--save-quantized",
+ action="store_true",
+ help="Save a quantized version of the current model"
+ )
+
+ parser.add_argument(
+ "--preload",
+ type=str,
+ default="0",
+ help="Megabytes of the diffusion model to preload in VRAM"
+ )
+
+ parser.add_argument(
+ "--multiple-images",
+ action="store_true",
+ help="Allow inputting multiple images with image to video"
+ )
+
+
+ parser.add_argument(
+ "--lora-dir-i2v",
+ type=str,
+ default="",
+ help="Path to a directory that contains Wan i2v Loras "
+ )
+
+
+ parser.add_argument(
+ "--lora-dir",
+ type=str,
+ default="",
+ help="Path to a directory that contains Wan t2v Loras"
+ )
+
+ parser.add_argument(
+ "--lora-dir-hunyuan",
+ type=str,
+ default="loras_hunyuan",
+ help="Path to a directory that contains Hunyuan Video t2v Loras"
+ )
+
+ parser.add_argument(
+ "--lora-dir-hunyuan-i2v",
+ type=str,
+ default="loras_hunyuan_i2v",
+ help="Path to a directory that contains Hunyuan Video i2v Loras"
+ )
+
+
+ parser.add_argument(
+ "--lora-dir-ltxv",
+ type=str,
+ default="loras_ltxv",
+ help="Path to a directory that contains LTX Videos Loras"
+ )
+
+ parser.add_argument(
+ "--lora-dir-flux",
+ type=str,
+ default="loras_flux",
+ help="Path to a directory that contains flux images Loras"
+ )
+
+
+ parser.add_argument(
+ "--check-loras",
+ action="store_true",
+ help="Filter Loras that are not valid"
+ )
+
+
+ parser.add_argument(
+ "--lora-preset",
+ type=str,
+ default="",
+ help="Lora preset to preload"
+ )
+
+ parser.add_argument(
+ "--settings",
+ type=str,
+ default="settings",
+ help="Path to settings folder"
+ )
+
+
+ # parser.add_argument(
+ # "--lora-preset-i2v",
+ # type=str,
+ # default="",
+ # help="Lora preset to preload for i2v"
+ # )
+
+ parser.add_argument(
+ "--profile",
+ type=str,
+ default=-1,
+ help="Profile No"
+ )
+
+ parser.add_argument(
+ "--verbose",
+ type=str,
+ default=1,
+ help="Verbose level"
+ )
+
+ parser.add_argument(
+ "--steps",
+ type=int,
+ default=0,
+ help="default denoising steps"
+ )
+
+
+ # parser.add_argument(
+ # "--teacache",
+ # type=float,
+ # default=-1,
+ # help="teacache speed multiplier"
+ # )
+
+ parser.add_argument(
+ "--frames",
+ type=int,
+ default=0,
+ help="default number of frames"
+ )
+
+ parser.add_argument(
+ "--seed",
+ type=int,
+ default=-1,
+ help="default generation seed"
+ )
+
+ parser.add_argument(
+ "--advanced",
+ action="store_true",
+ help="Access advanced options by default"
+ )
+
+ parser.add_argument(
+ "--fp16",
+ action="store_true",
+ help="For using fp16 transformer model"
+ )
+
+ parser.add_argument(
+ "--bf16",
+ action="store_true",
+ help="For using bf16 transformer model"
+ )
+
+ parser.add_argument(
+ "--server-port",
+ type=str,
+ default=0,
+ help="Server port"
+ )
+
+ parser.add_argument(
+ "--theme",
+ type=str,
+ default="",
+ help="set UI Theme"
+ )
+
+ parser.add_argument(
+ "--perc-reserved-mem-max",
+ type=float,
+ default=0,
+ help="% of RAM allocated to Reserved RAM"
+ )
+
+
+
+ parser.add_argument(
+ "--server-name",
+ type=str,
+ default="",
+ help="Server name"
+ )
+ parser.add_argument(
+ "--gpu",
+ type=str,
+ default="",
+ help="Default GPU Device"
+ )
+
+ parser.add_argument(
+ "--open-browser",
+ action="store_true",
+ help="open browser"
+ )
+
+ parser.add_argument(
+ "--t2v",
+ action="store_true",
+ help="text to video mode"
+ )
+
+ parser.add_argument(
+ "--i2v",
+ action="store_true",
+ help="image to video mode"
+ )
+
+ parser.add_argument(
+ "--t2v-14B",
+ action="store_true",
+ help="text to video mode 14B model"
+ )
+
+ parser.add_argument(
+ "--t2v-1-3B",
+ action="store_true",
+ help="text to video mode 1.3B model"
+ )
+
+ parser.add_argument(
+ "--vace-1-3B",
+ action="store_true",
+ help="Vace ControlNet 1.3B model"
+ )
+ parser.add_argument(
+ "--i2v-1-3B",
+ action="store_true",
+ help="Fun InP image to video mode 1.3B model"
+ )
+
+ parser.add_argument(
+ "--i2v-14B",
+ action="store_true",
+ help="image to video mode 14B model"
+ )
+
+
+ parser.add_argument(
+ "--compile",
+ action="store_true",
+ help="Enable pytorch compilation"
+ )
+
+ parser.add_argument(
+ "--listen",
+ action="store_true",
+ help="Server accessible on local network"
+ )
+
+ # parser.add_argument(
+ # "--fast",
+ # action="store_true",
+ # help="use Fast model"
+ # )
+
+ # parser.add_argument(
+ # "--fastest",
+ # action="store_true",
+ # help="activate the best config"
+ # )
+
+ parser.add_argument(
+ "--attention",
+ type=str,
+ default="",
+ help="attention mode"
+ )
+
+ parser.add_argument(
+ "--vae-config",
+ type=str,
+ default="",
+ help="vae config mode"
+ )
+
+ args = parser.parse_args()
+
+ return args
+
+def get_lora_dir(model_type):
+ model_family = get_model_family(model_type)
+ i2v = test_class_i2v(model_type) and not get_base_model_type(model_type) == "i2v_2_2"
+ if model_family == "wan":
+ lora_dir =args.lora_dir
+ if i2v and len(lora_dir)==0:
+ lora_dir =args.lora_dir_i2v
+ if len(lora_dir) > 0:
+ return lora_dir
+ root_lora_dir = "loras_i2v" if i2v else "loras"
+
+ if "1.3B" in model_type :
+ lora_dir_1_3B = os.path.join(root_lora_dir, "1.3B")
+ if os.path.isdir(lora_dir_1_3B ):
+ return lora_dir_1_3B
+ else:
+ lora_dir_14B = os.path.join(root_lora_dir, "14B")
+ if os.path.isdir(lora_dir_14B ):
+ return lora_dir_14B
+ return root_lora_dir
+ elif model_family == "ltxv":
+ return args.lora_dir_ltxv
+ elif model_family == "flux":
+ return args.lora_dir_flux
+ elif model_family =="hunyuan":
+ if i2v:
+ return args.lora_dir_hunyuan_i2v
+ else:
+ return args.lora_dir_hunyuan
+ else:
+ raise Exception("loras unknown")
+
+attention_modes_installed = get_attention_modes()
+attention_modes_supported = get_supported_attention_modes()
+args = _parse_args()
+
+major, minor = torch.cuda.get_device_capability(args.gpu if len(args.gpu) > 0 else None)
+if major < 8:
+ print("Switching to FP16 models when possible as GPU architecture doesn't support optimed BF16 Kernels")
+ bfloat16_supported = False
+else:
+ bfloat16_supported = True
+
+args.flow_reverse = True
+processing_device = args.gpu
+if len(processing_device) == 0:
+ processing_device ="cuda"
+# torch.backends.cuda.matmul.allow_fp16_accumulation = True
+lock_ui_attention = False
+lock_ui_transformer = False
+lock_ui_compile = False
+
+force_profile_no = int(args.profile)
+verbose_level = int(args.verbose)
+check_loras = args.check_loras ==1
+
+server_config_filename = "wgp_config.json"
+if not os.path.isdir("settings"):
+ os.mkdir("settings")
+if os.path.isfile("t2v_settings.json"):
+ for f in glob.glob(os.path.join(".", "*_settings.json*")):
+ target_file = os.path.join("settings", Path(f).parts[-1] )
+ shutil.move(f, target_file)
+
+if not os.path.isfile(server_config_filename) and os.path.isfile("gradio_config.json"):
+ shutil.move("gradio_config.json", server_config_filename)
+
+if not os.path.isdir("ckpts/umt5-xxl/"):
+ os.makedirs("ckpts/umt5-xxl/")
+src_move = [ "ckpts/models_clip_open-clip-xlm-roberta-large-vit-huge-14-bf16.safetensors", "ckpts/models_t5_umt5-xxl-enc-bf16.safetensors", "ckpts/models_t5_umt5-xxl-enc-quanto_int8.safetensors" ]
+tgt_move = [ "ckpts/xlm-roberta-large/", "ckpts/umt5-xxl/", "ckpts/umt5-xxl/"]
+for src,tgt in zip(src_move,tgt_move):
+ if os.path.isfile(src):
+ try:
+ if os.path.isfile(tgt):
+ shutil.remove(src)
+ else:
+ shutil.move(src, tgt)
+ except:
+ pass
+
+
+if not Path(server_config_filename).is_file():
+ server_config = {
+ "attention_mode" : "auto",
+ "transformer_types": [],
+ "transformer_quantization": "int8",
+ "text_encoder_quantization" : "int8",
+ "save_path": "outputs", #os.path.join(os.getcwd(),
+ "compile" : "",
+ "metadata_type": "metadata",
+ "default_ui": "t2v",
+ "boost" : 1,
+ "clear_file_list" : 5,
+ "vae_config": 0,
+ "profile" : profile_type.LowRAM_LowVRAM,
+ "preload_model_policy": [],
+ "UI_theme": "default"
+ }
+
+ with open(server_config_filename, "w", encoding="utf-8") as writer:
+ writer.write(json.dumps(server_config))
+else:
+ with open(server_config_filename, "r", encoding="utf-8") as reader:
+ text = reader.read()
+ server_config = json.loads(text)
+
+# Deprecated models
+for path in ["wan2.1_Vace_1.3B_preview_bf16.safetensors", "sky_reels2_diffusion_forcing_1.3B_bf16.safetensors","sky_reels2_diffusion_forcing_720p_14B_bf16.safetensors",
+"sky_reels2_diffusion_forcing_720p_14B_quanto_int8.safetensors", "sky_reels2_diffusion_forcing_720p_14B_quanto_fp16_int8.safetensors", "wan2.1_image2video_480p_14B_bf16.safetensors", "wan2.1_image2video_480p_14B_quanto_int8.safetensors",
+"wan2.1_image2video_720p_14B_quanto_int8.safetensors", "wan2.1_image2video_720p_14B_quanto_fp16_int8.safetensors", "wan2.1_image2video_720p_14B_bf16.safetensors",
+"wan2.1_text2video_14B_bf16.safetensors", "wan2.1_text2video_14B_quanto_int8.safetensors",
+"wan2.1_Vace_14B_mbf16.safetensors", "wan2.1_Vace_14B_quanto_mbf16_int8.safetensors", "wan2.1_FLF2V_720p_14B_quanto_int8.safetensors", "wan2.1_FLF2V_720p_14B_bf16.safetensors", "wan2.1_FLF2V_720p_14B_fp16.safetensors", "wan2.1_Vace_1.3B_mbf16.safetensors", "wan2.1_text2video_1.3B_bf16.safetensors",
+"ltxv_0.9.7_13B_dev_bf16.safetensors"
+]:
+ if Path(os.path.join("ckpts" , path)).is_file():
+ print(f"Removing old version of model '{path}'. A new version of this model will be downloaded next time you use it.")
+ os.remove( os.path.join("ckpts" , path))
+
+families_infos = {"wan":(0, "Wan2.1"), "wan2_2":(1, "Wan2.2"), "ltxv":(10, "LTX Video"), "hunyuan":(20, "Hunyuan Video"), "flux":(30, "Flux 1"), "unknown": (100, "Unknown") }
+
+models_def = {}
+
+modules_files = {
+ "vace_14B" : ["ckpts/wan2.1_Vace_14B_module_mbf16.safetensors", "ckpts/wan2.1_Vace_14B_module_quanto_mbf16_int8.safetensors", "ckpts/wan2.1_Vace_14B_module_quanto_mfp16_int8.safetensors"],
+ "vace_1.3B" : ["ckpts/wan2.1_Vace_1_3B_module.safetensors"],
+ "fantasy": ["ckpts/wan2.1_fantasy_speaking_14B_bf16.safetensors"],
+ "multitalk": ["ckpts/wan2.1_multitalk_14B_mbf16.safetensors", "ckpts/wan2.1_multitalk_14B_quanto_mbf16_int8.safetensors", "ckpts/wan2.1_multitalk_14B_quanto_mfp16_int8.safetensors"]
+}
+
+# architectures supported
+base_types = ["multitalk", "fantasy", "vace_14B", "vace_multitalk_14B",
+ "t2v_1.3B", "t2v", "vace_1.3B", "phantom_1.3B", "phantom_14B",
+ "recam_1.3B", "sky_df_1.3B", "sky_df_14B",
+ "i2v", "i2v_2_2", "flf2v_720p", "fun_inp_1.3B", "fun_inp", "ltxv_13B",
+ "hunyuan", "hunyuan_i2v", "hunyuan_custom", "hunyuan_custom_audio", "hunyuan_custom_edit", "hunyuan_avatar", "flux"
+ ]
+
+# only needed for imported old settings files
+model_signatures = {"t2v": "text2video_14B", "t2v_1.3B" : "text2video_1.3B", "fun_inp_1.3B" : "Fun_InP_1.3B", "fun_inp" : "Fun_InP_14B",
+ "i2v" : "image2video_480p", "i2v_720p" : "image2video_720p" , "vace_1.3B" : "Vace_1.3B", "vace_14B": "Vace_14B", "recam_1.3B": "recammaster_1.3B",
+ "sky_df_1.3B" : "sky_reels2_diffusion_forcing_1.3B", "sky_df_14B" : "sky_reels2_diffusion_forcing_14B",
+ "sky_df_720p_14B" : "sky_reels2_diffusion_forcing_720p_14B",
+ "phantom_1.3B" : "phantom_1.3B", "phantom_14B" : "phantom_14B", "ltxv_13B" : "ltxv_0.9.7_13B_dev", "ltxv_13B_distilled" : "ltxv_0.9.7_13B_distilled",
+ "hunyuan" : "hunyuan_video_720", "hunyuan_i2v" : "hunyuan_video_i2v_720", "hunyuan_custom" : "hunyuan_video_custom_720", "hunyuan_custom_audio" : "hunyuan_video_custom_audio", "hunyuan_custom_edit" : "hunyuan_video_custom_edit",
+ "hunyuan_avatar" : "hunyuan_video_avatar" }
+
+def get_base_model_type(model_type):
+ model_def = get_model_def(model_type)
+ if model_def == None:
+ return model_type if model_type in base_types else None
+ # return model_type
+ else:
+ return model_def["architecture"]
+
+def are_model_types_compatible(imported_model_type, current_model_type):
+ imported_base_model_type = get_base_model_type(imported_model_type)
+ curent_base_model_type = get_base_model_type(current_model_type)
+ if imported_base_model_type == curent_base_model_type:
+ return True
+
+ eqv_map = {
+ "flf2v_720p" : "i2v",
+ "t2v_1.3B" : "t2v",
+ "sky_df_1.3B" : "sky_df_14B",
+ }
+ if imported_base_model_type in eqv_map:
+ imported_base_model_type = eqv_map[imported_base_model_type]
+ comp_map = {
+ "vace_14B" : [ "vace_multitalk_14B"],
+ "t2v" : [ "vace_14B", "vace_1.3B" "vace_multitalk_14B", "t2v_1.3B", "phantom_1.3B","phantom_14B"],
+ "i2v" : [ "fantasy", "multitalk", "flf2v_720p" ],
+ "fantasy": ["multitalk"],
+ "sky_df_14B": ["sky_df_1.3B"],
+ "hunyuan_custom": ["hunyuan_custom_edit", "hunyuan_custom_audio"],
+ }
+ comp_list= comp_map.get(imported_base_model_type, None)
+ if comp_list == None: return False
+ return curent_base_model_type in comp_list
+
+def get_model_def(model_type):
+ return models_def.get(model_type, None )
+
+
+
+def get_model_type(model_filename):
+ for model_type, signature in model_signatures.items():
+ if signature in model_filename:
+ return model_type
+ return None
+ # raise Exception("Unknown model:" + model_filename)
+
+def get_model_family(model_type, for_ui = False):
+ base_model_type = get_base_model_type(model_type)
+ if base_model_type is None:
+ return "unknown"
+
+ if for_ui :
+ model_def = get_model_def(model_type)
+ model_family = model_def.get("group", None)
+ if model_family is not None and model_family in families_infos:
+ return model_family
+
+ if "hunyuan" in base_model_type :
+ return "hunyuan"
+ elif "ltxv" in base_model_type:
+ return "ltxv"
+ elif "flux" in base_model_type:
+ return "flux"
+ else:
+ return "wan"
+
+def test_class_i2v(model_type):
+ model_type = get_base_model_type(model_type)
+ return model_type in ["i2v", "i2v_2_2", "fun_inp_1.3B", "fun_inp", "flf2v_720p", "fantasy", "multitalk" ] #"hunyuan_i2v",
+
+def test_vace_module(model_type):
+ model_type = get_base_model_type(model_type)
+ return model_type in ["vace_14B", "vace_1.3B", "vace_multitalk_14B"]
+
+def test_any_sliding_window(model_type):
+ model_type = get_base_model_type(model_type)
+ return test_vace_module(model_type) or model_type in ["sky_df_1.3B", "sky_df_14B", "ltxv_13B", "multitalk", "t2v", "fantasy"] or test_class_i2v(model_type)
+
+def get_model_min_frames_and_step(model_type):
+ model_type = get_base_model_type(model_type)
+ if model_type in ["sky_df_14B"]:
+ return 17, 20
+ elif model_type in ["ltxv_13B"]:
+ return 17, 8
+ elif test_vace_module(model_type):
+ return 17, 4
+ else:
+ return 5, 4
+
+def get_model_fps(model_type):
+ model_type = get_base_model_type(model_type)
+ if model_type in ["hunyuan_avatar", "hunyuan_custom_audio", "multitalk", "vace_multitalk_14B"]:
+ fps = 25
+ elif model_type in ["sky_df_14B", "hunyuan", "hunyuan_i2v", "hunyuan_custom_edit", "hunyuan_custom"]:
+ fps = 24
+ elif model_type in ["fantasy"]:
+ fps = 23
+ elif model_type in ["ltxv_13B"]:
+ fps = 30
+ else:
+ fps = 16
+ return fps
+
+def get_computed_fps(force_fps, base_model_type , video_guide, video_source ):
+ if force_fps == "auto":
+ if video_source != None:
+ fps, _, _, _ = get_video_info(video_source)
+ elif video_guide != None:
+ fps, _, _, _ = get_video_info(video_guide)
+ else:
+ fps = get_model_fps(base_model_type)
+ elif force_fps == "control" and video_guide != None:
+ fps, _, _, _ = get_video_info(video_guide)
+ elif force_fps == "source" and video_source != None:
+ fps, _, _, _ = get_video_info(video_source)
+ elif len(force_fps) > 0 and is_integer(force_fps) :
+ fps = int(force_fps)
+ else:
+ fps = get_model_fps(base_model_type)
+ return fps
+
+def get_model_name(model_type, description_container = [""]):
+ model_def = get_model_def(model_type)
+ if model_def == None:
+ return f"Unknown model {model_type}"
+ model_name = model_def["name"]
+ description = model_def["description"]
+ description_container[0] = description
+ return model_name
+
+def get_model_record(model_name):
+ return f"WanGP v{WanGP_version} by DeepBeepMeep - " + model_name
+
+def get_model_recursive_prop(model_type, prop = "URLs", return_list = True, stack= []):
+ model_def = models_def.get(model_type, None)
+ if model_def != None:
+ prop_value = model_def.get(prop, None)
+ if prop_value == None:
+ return []
+ if isinstance(prop_value, str):
+ if len(stack) > 10: raise Exception(f"Circular Reference in Model {prop} dependencies: {stack}")
+ return get_model_recursive_prop(prop_value, prop = prop, stack = stack + [prop_value] )
+ else:
+ return prop_value
+ else:
+ if model_type in model_types:
+ return [] if return_list else model_type
+ else:
+ raise Exception(f"Unknown model type '{model_type}'")
+
+
+def get_model_filename(model_type, quantization ="int8", dtype_policy = "", is_module = False, submodel_no = 1, stack=[]):
+ if is_module:
+ choices = modules_files.get(model_type, None)
+ if choices == None: raise Exception(f"Invalid Module Id '{model_type}'")
+ else:
+ key_name = "URLs" if submodel_no <= 1 else f"URLs{submodel_no}"
+
+ model_def = models_def.get(model_type, None)
+ if model_def == None: return ""
+ URLs = model_def[key_name]
+ if isinstance(URLs, str):
+ if len(stack) > 10: raise Exception(f"Circular Reference in Model {key_name} dependencies: {stack}")
+ return get_model_filename(URLs, quantization=quantization, dtype_policy=dtype_policy, submodel_no = submodel_no, stack = stack + [URLs])
+ else:
+ choices = [ ("ckpts/" + os.path.basename(path) if path.startswith("http") else path) for path in URLs ]
+ if len(quantization) == 0:
+ quantization = "bf16"
+
+ model_family = get_model_family(model_type)
+ dtype = get_transformer_dtype(model_family, dtype_policy)
+ if len(choices) <= 1:
+ raw_filename = choices[0]
+ else:
+ if quantization in ("int8", "fp8"):
+ sub_choices = [ name for name in choices if quantization in name or quantization.upper() in name]
+ else:
+ sub_choices = [ name for name in choices if "quanto" not in name]
+
+ if len(sub_choices) > 0:
+ dtype_str = "fp16" if dtype == torch.float16 else "bf16"
+ new_sub_choices = [ name for name in sub_choices if dtype_str in name or dtype_str.upper() in name]
+ sub_choices = new_sub_choices if len(new_sub_choices) > 0 else sub_choices
+ raw_filename = sub_choices[0]
+ else:
+ raw_filename = choices[0]
+
+ return raw_filename
+
+def get_transformer_dtype(model_family, transformer_dtype_policy):
+ if not isinstance(transformer_dtype_policy, str):
+ return transformer_dtype_policy
+ if len(transformer_dtype_policy) == 0:
+ if not bfloat16_supported:
+ return torch.float16
+ else:
+ if model_family == "wan"and False:
+ return torch.float16
+ else:
+ return torch.bfloat16
+ return transformer_dtype
+ elif transformer_dtype_policy =="fp16":
+ return torch.float16
+ else:
+ return torch.bfloat16
+
+def get_settings_file_name(model_type):
+ return os.path.join(args.settings, model_type + "_settings.json")
+
+def fix_settings(model_type, ui_defaults):
+ if model_type == None: return
+
+ video_settings_version = ui_defaults.get("settings_version", 0)
+ model_def = get_model_def(model_type)
+ model_type = get_base_model_type(model_type)
+
+ prompts = ui_defaults.get("prompts", "")
+ if len(prompts) > 0:
+ ui_defaults["prompt"] = prompts
+ image_prompt_type = ui_defaults.get("image_prompt_type", None)
+ if image_prompt_type != None :
+ if not isinstance(image_prompt_type, str):
+ image_prompt_type = "S" if image_prompt_type == 0 else "SE"
+ # if model_type == "flf2v_720p" and not "E" in image_prompt_type:
+ # image_prompt_type = "SE"
+ if video_settings_version <= 2:
+ image_prompt_type = image_prompt_type.replace("G","")
+ ui_defaults["image_prompt_type"] = image_prompt_type
+
+ if "lset_name" in ui_defaults: del ui_defaults["lset_name"]
+
+ audio_prompt_type = ui_defaults.get("audio_prompt_type", None)
+ if video_settings_version < 2.2:
+ if not model_type in ["vace_1.3B","vace_14B", "sky_df_1.3B", "sky_df_14B", "ltxv_13B"]:
+ for p in ["sliding_window_size", "sliding_window_overlap", "sliding_window_overlap_noise", "sliding_window_discard_last_frames"]:
+ if p in ui_defaults: del ui_defaults[p]
+
+ if audio_prompt_type == None :
+ if any_audio_track(model_type):
+ audio_prompt_type ="A"
+ ui_defaults["audio_prompt_type"] = audio_prompt_type
+
+
+ video_prompt_type = ui_defaults.get("video_prompt_type", "")
+ any_reference_image = model_def.get("reference_image", False)
+ if model_type in ["hunyuan_custom", "hunyuan_custom_edit", "hunyuan_custom_audio", "hunyuan_avatar", "phantom_14B", "phantom_1.3B"] or any_reference_image:
+ if not "I" in video_prompt_type: # workaround for settings corruption
+ video_prompt_type += "I"
+ if model_type in ["hunyuan"]:
+ video_prompt_type = video_prompt_type.replace("I", "")
+
+ if model_type in ["flux"] and video_settings_version < 2.23:
+ video_prompt_type = video_prompt_type.replace("K", "").replace("I", "KI")
+
+ remove_background_images_ref = ui_defaults.get("remove_background_images_ref", 1)
+ if video_settings_version < 2.22:
+ if "I" in video_prompt_type:
+ if remove_background_images_ref == 2:
+ video_prompt_type = video_prompt_type.replace("I", "KI")
+ if remove_background_images_ref != 0:
+ remove_background_images_ref = 1
+ if model_type in ["hunyuan_avatar"]: remove_background_images_ref = 0
+ ui_defaults["remove_background_images_ref"] = remove_background_images_ref
+
+ ui_defaults["video_prompt_type"] = video_prompt_type
+
+ tea_cache_setting = ui_defaults.get("tea_cache_setting", None)
+ tea_cache_start_step_perc = ui_defaults.get("tea_cache_start_step_perc", None)
+
+ if tea_cache_setting != None:
+ del ui_defaults["tea_cache_setting"]
+ if tea_cache_setting > 0:
+ ui_defaults["skip_steps_multiplier"] = tea_cache_setting
+ ui_defaults["skip_steps_cache_type"] = "tea"
+ else:
+ ui_defaults["skip_steps_multiplier"] = 1.75
+ ui_defaults["skip_steps_cache_type"] = ""
+
+ if tea_cache_start_step_perc != None:
+ del ui_defaults["tea_cache_start_step_perc"]
+ ui_defaults["skip_steps_start_step_perc"] = tea_cache_start_step_perc
+
+def get_default_settings(model_type):
+ def get_default_prompt(i2v):
+ if i2v:
+ return "Several giant wooly mammoths approach treading through a snowy meadow, their long wooly fur lightly blows in the wind as they walk, snow covered trees and dramatic snow capped mountains in the distance, mid afternoon light with wispy clouds and a sun high in the distance creates a warm glow, the low camera view is stunning capturing the large furry mammal with beautiful photography, depth of field."
+ else:
+ return "A large orange octopus is seen resting on the bottom of the ocean floor, blending in with the sandy and rocky terrain. Its tentacles are spread out around its body, and its eyes are closed. The octopus is unaware of a king crab that is crawling towards it from behind a rock, its claws raised and ready to attack. The crab is brown and spiny, with long legs and antennae. The scene is captured from a wide angle, showing the vastness and depth of the ocean. The water is clear and blue, with rays of sunlight filtering through. The shot is sharp and crisp, with a high dynamic range. The octopus and the crab are in focus, while the background is slightly blurred, creating a depth of field effect."
+ i2v = test_class_i2v(model_type)
+ defaults_filename = get_settings_file_name(model_type)
+ if not Path(defaults_filename).is_file():
+ model_def = get_model_def(model_type)
+ base_model_type = get_base_model_type(model_type)
+ ui_defaults = {
+ "prompt": get_default_prompt(i2v),
+ "resolution": "1280x720" if "720" in base_model_type else "832x480",
+ "video_length": 81,
+ "num_inference_steps": 30,
+ "seed": -1,
+ "repeat_generation": 1,
+ "multi_images_gen_type": 0,
+ "guidance_scale": 5.0,
+ "embedded_guidance_scale" : 6.0,
+ "flow_shift": 7.0 if not "720" in base_model_type and i2v else 5.0,
+ "negative_prompt": "",
+ "activated_loras": [],
+ "loras_multipliers": "",
+ "skip_steps_multiplier": 1.5,
+ "skip_steps_start_step_perc": 20,
+ "RIFLEx_setting": 0,
+ "slg_switch": 0,
+ "slg_layers": [9],
+ "slg_start_perc": 10,
+ "slg_end_perc": 90
+ }
+ if base_model_type in ["fantasy"]:
+ ui_defaults["audio_guidance_scale"] = 5.0
+ elif base_model_type in ["multitalk"]:
+ ui_defaults.update({
+ "guidance_scale": 5.0,
+ "flow_shift": 7, # 11 for 720p
+ "audio_guidance_scale": 4,
+ "sliding_window_discard_last_frames" : 4,
+ "sample_solver" : "euler",
+ "adaptive_switch" : 1,
+ })
+
+ elif base_model_type in ["hunyuan","hunyuan_i2v"]:
+ ui_defaults.update({
+ "guidance_scale": 7.0,
+ })
+
+ elif base_model_type in ["flux"]:
+ ui_defaults.update({
+ "embedded_guidance": 2.5,
+ })
+ if model_def.get("reference_image", False):
+ ui_defaults.update({
+ "video_prompt_type": "KI",
+ })
+ elif base_model_type in ["sky_df_1.3B", "sky_df_14B"]:
+ ui_defaults.update({
+ "guidance_scale": 6.0,
+ "flow_shift": 8,
+ "sliding_window_discard_last_frames" : 0,
+ "resolution": "1280x720" if "720" in base_model_type else "960x544",
+ "sliding_window_size" : 121 if "720" in base_model_type else 97,
+ "RIFLEx_setting": 2,
+ "guidance_scale": 6,
+ "flow_shift": 8,
+ })
+
+
+ elif base_model_type in ["phantom_1.3B", "phantom_14B"]:
+ ui_defaults.update({
+ "guidance_scale": 7.5,
+ "flow_shift": 5,
+ "remove_background_images_ref": 1,
+ "video_prompt_type": "I",
+ # "resolution": "1280x720"
+ })
+
+ elif base_model_type in ["hunyuan_custom"]:
+ ui_defaults.update({
+ "guidance_scale": 7.5,
+ "flow_shift": 13,
+ "resolution": "1280x720",
+ "video_prompt_type": "I",
+ })
+ elif base_model_type in ["hunyuan_custom_audio"]:
+ ui_defaults.update({
+ "guidance_scale": 7.5,
+ "flow_shift": 13,
+ "video_prompt_type": "I",
+ })
+ elif base_model_type in ["hunyuan_custom_edit"]:
+ ui_defaults.update({
+ "guidance_scale": 7.5,
+ "flow_shift": 13,
+ "video_prompt_type": "MVAI",
+ "sliding_window_size": 129,
+ })
+ elif base_model_type in ["hunyuan_avatar"]:
+ ui_defaults.update({
+ "guidance_scale": 7.5,
+ "flow_shift": 5,
+ "remove_background_images_ref": 0,
+ "skip_steps_start_step_perc": 25,
+ "video_length": 129,
+ "video_prompt_type": "I",
+ })
+ elif base_model_type in ["vace_14B", "vace_multitalk_14B"]:
+ ui_defaults.update({
+ "sliding_window_discard_last_frames": 0,
+ })
+
+
+ ui_defaults_update = model_def.get("settings", None)
+ if ui_defaults_update is not None: ui_defaults.update(ui_defaults_update)
+
+ if len(ui_defaults.get("prompt","")) == 0:
+ ui_defaults["prompt"]= get_default_prompt(i2v)
+
+ with open(defaults_filename, "w", encoding="utf-8") as f:
+ json.dump(ui_defaults, f, indent=4)
+ else:
+ with open(defaults_filename, "r", encoding="utf-8") as f:
+ ui_defaults = json.load(f)
+ fix_settings(model_type, ui_defaults)
+
+ default_seed = args.seed
+ if default_seed > -1:
+ ui_defaults["seed"] = default_seed
+ default_number_frames = args.frames
+ if default_number_frames > 0:
+ ui_defaults["video_length"] = default_number_frames
+ default_number_steps = args.steps
+ if default_number_steps > 0:
+ ui_defaults["num_inference_steps"] = default_number_steps
+ return ui_defaults
+
+def get_model_query_handler(model_type):
+ base_model_type = get_base_model_type(model_type)
+ model_family= get_model_family(base_model_type)
+ if model_family == "wan":
+ if base_model_type in ("sky_df_1.3B", "sky_df_14B"):
+ from wan.diffusion_forcing import query_model_def
+ else:
+ from wan.any2video import query_model_def
+ elif model_family == "hunyuan":
+ from hyvideo.hunyuan import query_model_def
+ elif model_family == "ltxv":
+ from ltx_video.ltxv import query_model_def
+ elif model_family == "flux":
+ from flux.flux_main import query_model_def
+ else:
+ raise Exception(f"Unknown / unsupported model type {model_type}")
+ return query_model_def
+
+def init_model_def(model_type, model_def):
+ query_handler = get_model_query_handler(model_type)
+ default_model_def = query_handler(model_type, model_def)
+ if default_model_def is None: return model_def
+ default_model_def.update(model_def)
+ return default_model_def
+
+
+models_def_paths = glob.glob( os.path.join("defaults", "*.json") ) + glob.glob( os.path.join("finetunes", "*.json") )
+models_def_paths.sort()
+for file_path in models_def_paths:
+ model_type = os.path.basename(file_path)[:-5]
+ with open(file_path, "r", encoding="utf-8") as f:
+ try:
+ json_def = json.load(f)
+ except Exception as e:
+ raise Exception(f"Error while parsing Model Definition File '{file_path}': {str(e)}")
+ model_def = json_def["model"]
+ model_def["path"] = file_path
+ del json_def["model"]
+ settings = json_def
+ existing_model_def = models_def.get(model_type, None)
+ if existing_model_def is not None:
+ existing_settings = models_def.get("settings", None)
+ if existing_settings != None:
+ existing_settings.update(settings)
+ existing_model_def.update(model_def)
+ else:
+ models_def[model_type] = model_def # partial def
+ model_def= init_model_def(model_type, model_def)
+ models_def[model_type] = model_def # replace with full def
+ model_def["settings"] = settings
+
+model_types = models_def.keys()
+displayed_model_types= []
+for model_type in model_types:
+ model_def = get_model_def(model_type)
+ if not model_def is None and model_def.get("visible", True):
+ displayed_model_types.append(model_type)
+
+
+transformer_types = server_config.get("transformer_types", [])
+new_transformer_types = []
+for model_type in transformer_types:
+ if get_model_def(model_type) == None:
+ print(f"Model '{model_type}' is missing. Either install it in the finetune folder or remove this model from ley 'transformer_types' in wgp_config.json")
+ else:
+ new_transformer_types.append(model_type)
+transformer_types = new_transformer_types
+transformer_type = server_config.get("last_model_type", None)
+advanced = server_config.get("last_advanced_choice", False)
+last_resolution = server_config.get("last_resolution_choice", None)
+if args.advanced: advanced = True
+
+if transformer_type != None and not transformer_type in model_types and not transformer_type in models_def: transformer_type = None
+if transformer_type == None:
+ transformer_type = transformer_types[0] if len(transformer_types) > 0 else "t2v"
+
+transformer_quantization =server_config.get("transformer_quantization", "int8")
+
+transformer_dtype_policy = server_config.get("transformer_dtype_policy", "")
+if args.fp16:
+ transformer_dtype_policy = "fp16"
+if args.bf16:
+ transformer_dtype_policy = "bf16"
+text_encoder_quantization =server_config.get("text_encoder_quantization", "int8")
+attention_mode = server_config["attention_mode"]
+if len(args.attention)> 0:
+ if args.attention in ["auto", "sdpa", "sage", "sage2", "flash", "xformers"]:
+ attention_mode = args.attention
+ lock_ui_attention = True
+ else:
+ raise Exception(f"Unknown attention mode '{args.attention}'")
+
+profile = force_profile_no if force_profile_no >=0 else server_config["profile"]
+compile = server_config.get("compile", "")
+boost = server_config.get("boost", 1)
+vae_config = server_config.get("vae_config", 0)
+if len(args.vae_config) > 0:
+ vae_config = int(args.vae_config)
+
+reload_needed = False
+default_ui = server_config.get("default_ui", "t2v")
+save_path = server_config.get("save_path", os.path.join(os.getcwd(), "gradio_outputs"))
+preload_model_policy = server_config.get("preload_model_policy", [])
+
+
+if args.t2v_14B or args.t2v:
+ transformer_type = "t2v"
+
+if args.i2v_14B or args.i2v:
+ transformer_type = "i2v"
+
+if args.t2v_1_3B:
+ transformer_type = "t2v_1.3B"
+
+if args.i2v_1_3B:
+ transformer_type = "fun_inp_1.3B"
+
+if args.vace_1_3B:
+ transformer_type = "vace_1.3B"
+
+only_allow_edit_in_advanced = False
+lora_preselected_preset = args.lora_preset
+lora_preset_model = transformer_type
+
+if args.compile: #args.fastest or
+ compile="transformer"
+ lock_ui_compile = True
+
+
+def save_model(model, model_type, dtype, config_file, submodel_no = 1):
+ model_def = get_model_def(model_type)
+ if model_def == None: return
+ url_key = "URLs" if submodel_no <=1 else "URLs" + str(submodel_no)
+ URLs= model_def.get(url_key, None)
+ if URLs is None: return
+ if isinstance(URLs, str):
+ print("Unable to save model for a finetune that references external files")
+ return
+ from mmgp import offload
+ if dtype == torch.bfloat16:
+ dtypestr= "bf16"
+ else:
+ dtypestr= "fp16"
+ model_filename = None
+ for url in URLs:
+ if "quanto" not in url and dtypestr in url:
+ model_filename = os.path.basename(url)
+ break
+ if model_filename is None:
+ print(f"No target filename mentioned in {url_key}")
+ return
+ if not os.path.isfile(model_filename):
+ offload.save_model(model, os.path.join("ckpts",model_filename), config_file_path=config_file)
+ print(f"New model file '{model_filename}' had been created for finetune Id '{model_type}'.")
+ finetune_file = os.path.join(os.path.dirname(model_def["path"]) , model_type + ".json")
+ with open(finetune_file, 'r', encoding='utf-8') as reader:
+ saved_finetune_def = json.load(reader)
+ del saved_finetune_def["model"]["source"]
+ del model_def["source"]
+ with open(finetune_file, "w", encoding="utf-8") as writer:
+ writer.write(json.dumps(saved_finetune_def, indent=4))
+ print(f"The 'source' entry has been removed in the '{finetune_file}' definition file.")
+
+def save_quantized_model(model, model_type, model_filename, dtype, config_file, submodel_no = 1):
+ if "quanto" in model_filename: return
+ model_def = get_model_def(model_type)
+ if model_def == None: return
+ url_key = "URLs" if submodel_no <=1 else "URLs" + str(submodel_no)
+ URLs= model_def.get(url_key, None)
+ if URLs is None: return
+ if isinstance(URLs, str):
+ print("Unable to create a quantized model for a finetune that references external files")
+ return
+ from mmgp import offload
+ if dtype == torch.bfloat16:
+ model_filename = model_filename.replace("fp16", "bf16").replace("FP16", "bf16")
+ elif dtype == torch.float16:
+ model_filename = model_filename.replace("bf16", "fp16").replace("BF16", "bf16")
+
+ for rep in ["mfp16", "fp16", "mbf16", "bf16"]:
+ if "_" + rep in model_filename:
+ model_filename = model_filename.replace("_" + rep, "_quanto_" + rep + "_int8")
+ break
+ if not "quanto" in model_filename:
+ pos = model_filename.rfind(".")
+ model_filename = model_filename[:pos] + "_quanto_int8" + model_filename[pos+1:]
+
+ if os.path.isfile(model_filename):
+ print(f"There isn't any model to quantize as quantized model '{model_filename}' aready exists")
+ else:
+ offload.save_model(model, model_filename, do_quantize= True, config_file_path=config_file)
+ print(f"New quantized file '{model_filename}' had been created for finetune Id '{model_type}'.")
+ if not model_filename in URLs:
+ URLs.append(model_filename)
+ finetune_file = os.path.join(os.path.dirname(model_def["path"]) , model_type + ".json")
+ with open(finetune_file, 'r', encoding='utf-8') as reader:
+ saved_finetune_def = json.load(reader)
+ saved_finetune_def["model"][url_key] = URLs
+ with open(finetune_file, "w", encoding="utf-8") as writer:
+ writer.write(json.dumps(saved_finetune_def, indent=4))
+ print(f"The '{finetune_file}' definition file has been automatically updated with the local path to the new quantized model.")
+
+def get_loras_preprocessor(transformer, model_type):
+ preprocessor = getattr(transformer, "preprocess_loras", None)
+ if preprocessor == None:
+ return None
+
+ def preprocessor_wrapper(sd):
+ return preprocessor(model_type, sd)
+
+ return preprocessor_wrapper
+
+
+def get_wan_text_encoder_filename(text_encoder_quantization):
+ text_encoder_filename = "ckpts/umt5-xxl/models_t5_umt5-xxl-enc-bf16.safetensors"
+ if text_encoder_quantization =="int8":
+ text_encoder_filename = text_encoder_filename.replace("bf16", "quanto_int8")
+ return text_encoder_filename
+
+def get_ltxv_text_encoder_filename(text_encoder_quantization):
+ text_encoder_filename = "ckpts/T5_xxl_1.1/T5_xxl_1.1_enc_bf16.safetensors"
+ if text_encoder_quantization =="int8":
+ text_encoder_filename = text_encoder_filename.replace("bf16", "quanto_bf16_int8")
+ return text_encoder_filename
+
+def get_hunyuan_text_encoder_filename(text_encoder_quantization):
+ if text_encoder_quantization =="int8":
+ text_encoder_filename = "ckpts/llava-llama-3-8b/llava-llama-3-8b-v1_1_vlm_quanto_int8.safetensors"
+ else:
+ text_encoder_filename = "ckpts/llava-llama-3-8b/llava-llama-3-8b-v1_1_vlm_fp16.safetensors"
+
+ return text_encoder_filename
+
+
+def process_files_def(repoId, sourceFolderList, fileList):
+ targetRoot = "ckpts/"
+ for sourceFolder, files in zip(sourceFolderList,fileList ):
+ if len(files)==0:
+ if not Path(targetRoot + sourceFolder).exists():
+ snapshot_download(repo_id=repoId, allow_patterns=sourceFolder +"/*", local_dir= targetRoot)
+ else:
+ for onefile in files:
+ if len(sourceFolder) > 0:
+ if not os.path.isfile(targetRoot + sourceFolder + "/" + onefile ):
+ hf_hub_download(repo_id=repoId, filename=onefile, local_dir = targetRoot, subfolder=sourceFolder)
+ else:
+ if not os.path.isfile(targetRoot + onefile ):
+ hf_hub_download(repo_id=repoId, filename=onefile, local_dir = targetRoot)
+
+def download_mmaudio():
+ if server_config.get("mmaudio_enabled", 0) != 0:
+ enhancer_def = {
+ "repoId" : "DeepBeepMeep/Wan2.1",
+ "sourceFolderList" : [ "mmaudio", "DFN5B-CLIP-ViT-H-14-378" ],
+ "fileList" : [ ["mmaudio_large_44k_v2.pth", "synchformer_state_dict.pth", "v1-44.pth"],["open_clip_config.json", "open_clip_pytorch_model.bin"]]
+ }
+ process_files_def(**enhancer_def)
+
+def download_models(model_filename, model_type, submodel_no = 1):
+ def computeList(filename):
+ if filename == None:
+ return []
+ pos = filename.rfind("/")
+ filename = filename[pos+1:]
+ return [filename]
+
+
+
+ from urllib.request import urlretrieve
+ from wan.utils.utils import create_progress_hook
+
+ shared_def = {
+ "repoId" : "DeepBeepMeep/Wan2.1",
+ "sourceFolderList" : [ "pose", "scribble", "flow", "depth", "mask", "wav2vec", "chinese-wav2vec2-base", "pyannote", "" ],
+ "fileList" : [ ["dw-ll_ucoco_384.onnx", "yolox_l.onnx"],["netG_A_latest.pth"], ["raft-things.pth"],
+ ["depth_anything_v2_vitl.pth","depth_anything_v2_vitb.pth"], ["sam_vit_h_4b8939_fp16.safetensors"],
+ ["config.json", "feature_extractor_config.json", "model.safetensors", "preprocessor_config.json", "special_tokens_map.json", "tokenizer_config.json", "vocab.json"],
+ ["config.json", "pytorch_model.bin", "preprocessor_config.json"],
+ ["pyannote_model_wespeaker-voxceleb-resnet34-LM.bin", "pytorch_model_segmentation-3.0.bin"], [ "flownet.pkl" ] ]
+ }
+ process_files_def(**shared_def)
+
+
+ if server_config.get("enhancer_enabled", 0) == 1:
+ enhancer_def = {
+ "repoId" : "DeepBeepMeep/LTX_Video",
+ "sourceFolderList" : [ "Florence2", "Llama3_2" ],
+ "fileList" : [ ["config.json", "configuration_florence2.py", "model.safetensors", "modeling_florence2.py", "preprocessor_config.json", "processing_florence2.py", "tokenizer.json", "tokenizer_config.json"],["config.json", "generation_config.json", "Llama3_2_quanto_bf16_int8.safetensors", "special_tokens_map.json", "tokenizer.json", "tokenizer_config.json"] ]
+ }
+ process_files_def(**enhancer_def)
+
+ download_mmaudio()
+
+ def download_file(url,filename):
+ if url.startswith("https://huggingface.co/") and "/resolve/main/" in url:
+ base_dir = os.path.dirname(filename)
+ url = url[len("https://huggingface.co/"):]
+ url_parts = url.split("/resolve/main/")
+ repoId = url_parts[0]
+ onefile = os.path.basename(url_parts[-1])
+ sourceFolder = os.path.dirname(url_parts[-1])
+ if len(sourceFolder) == 0:
+ hf_hub_download(repo_id=repoId, filename=onefile, local_dir = "ckpts/" if len(base_dir)==0 else base_dir)
+ else:
+ target_path = "ckpts/temp/" + sourceFolder
+ if not os.path.exists(target_path):
+ os.makedirs(target_path)
+ hf_hub_download(repo_id=repoId, filename=onefile, local_dir = "ckpts/temp/", subfolder=sourceFolder)
+ shutil.move(os.path.join( "ckpts", "temp" , sourceFolder , onefile), "ckpts/" if len(base_dir)==0 else base_dir)
+ shutil.rmtree("ckpts/temp")
+ else:
+ urlretrieve(url,filename, create_progress_hook(filename))
+
+ model_family = get_model_family(model_type)
+ model_def = get_model_def(model_type)
+
+ source = model_def.get("source", None)
+
+
+ key_name = "URLs" if submodel_no <= 1 else f"URLs{submodel_no}"
+ if source is not None:
+ model_filename = None
+ elif not model_type in modules_files:
+ if not os.path.isfile(model_filename ):
+ URLs = get_model_recursive_prop(model_type, key_name, return_list= False)
+ if isinstance(URLs, str):
+ raise Exception("Missing model " + URLs)
+ use_url = model_filename
+ for url in URLs:
+ if os.path.basename(model_filename) in url:
+ use_url = url
+ break
+ if not url.startswith("http"):
+ raise Exception(f"Model '{model_filename}' in field '{key_name}' was not found locally and no URL was provided to download it. Please add an URL in the model definition file.")
+ try:
+ download_file(use_url, model_filename)
+ except Exception as e:
+ if os.path.isfile(model_filename): os.remove(model_filename)
+ raise Exception(f"{key_name} '{use_url}' is invalid for Model '{model_filename}' : {str(e)}'")
+
+ model_filename = None
+
+ preload_URLs = get_model_recursive_prop(model_type, "preload_URLs", return_list= True)
+ for url in preload_URLs:
+ filename = "ckpts/" + url.split("/")[-1]
+ if not os.path.isfile(filename ):
+ if not url.startswith("http"):
+ raise Exception(f"File '{filename}' to preload was not found locally and no URL was provided to download it. Please add an URL in the model definition file.")
+ try:
+ download_file(url, filename)
+ except Exception as e:
+ if os.path.isfile(filename): os.remove(filename)
+ raise Exception(f"Preload URL '{url}' is invalid: {str(e)}'")
+
+ model_loras = get_model_recursive_prop(model_type, "loras", return_list= True)
+ for url in model_loras:
+ filename = os.path.join(get_lora_dir(model_type), url.split("/")[-1])
+ if not os.path.isfile(filename ):
+ if not url.startswith("http"):
+ raise Exception(f"Lora '{filename}' was not found in the Loras Folder and no URL was provided to download it. Please add an URL in the model definition file.")
+ try:
+ download_file(url, filename)
+ except Exception as e:
+ if os.path.isfile(filename): os.remove(filename)
+ raise Exception(f"Lora URL '{url}' is invalid: {str(e)}'")
+
+ if model_family == "wan":
+ text_encoder_filename = get_wan_text_encoder_filename(text_encoder_quantization)
+ model_files = {
+ "repoId" : "DeepBeepMeep/Wan2.1",
+ "sourceFolderList" : ["xlm-roberta-large", "umt5-xxl", "" ],
+ "fileList" : [ [ "models_clip_open-clip-xlm-roberta-large-vit-huge-14-bf16.safetensors", "sentencepiece.bpe.model", "special_tokens_map.json", "tokenizer.json", "tokenizer_config.json"], ["special_tokens_map.json", "spiece.model", "tokenizer.json", "tokenizer_config.json"] + computeList(text_encoder_filename) , ["Wan2.1_VAE.safetensors", "fantasy_proj_model.safetensors" ] + computeList(model_filename) ]
+ }
+ elif model_family == "ltxv":
+ text_encoder_filename = get_ltxv_text_encoder_filename(text_encoder_quantization)
+ model_files = {
+ "repoId" : "DeepBeepMeep/LTX_Video",
+ "sourceFolderList" : ["T5_xxl_1.1", "" ],
+ "fileList" : [ ["added_tokens.json", "special_tokens_map.json", "spiece.model", "tokenizer_config.json"] + computeList(text_encoder_filename), ["ltxv_0.9.7_VAE.safetensors", "ltxv_0.9.7_spatial_upscaler.safetensors", "ltxv_scheduler.json"] + computeList(model_filename) ]
+ }
+ elif model_family == "hunyuan":
+ text_encoder_filename = get_hunyuan_text_encoder_filename(text_encoder_quantization)
+ model_files = {
+ "repoId" : "DeepBeepMeep/HunyuanVideo",
+ "sourceFolderList" : [ "llava-llama-3-8b", "clip_vit_large_patch14", "whisper-tiny" , "det_align", "" ],
+ "fileList" :[ ["config.json", "special_tokens_map.json", "tokenizer.json", "tokenizer_config.json", "preprocessor_config.json"] + computeList(text_encoder_filename) ,
+ ["config.json", "merges.txt", "model.safetensors", "preprocessor_config.json", "special_tokens_map.json", "tokenizer.json", "tokenizer_config.json", "vocab.json"],
+ ["config.json", "model.safetensors", "preprocessor_config.json", "special_tokens_map.json", "tokenizer_config.json"],
+ ["detface.pt"],
+ [ "hunyuan_video_720_quanto_int8_map.json", "hunyuan_video_custom_VAE_fp32.safetensors", "hunyuan_video_custom_VAE_config.json", "hunyuan_video_VAE_fp32.safetensors", "hunyuan_video_VAE_config.json" , "hunyuan_video_720_quanto_int8_map.json" ] + computeList(model_filename)
+ ]
+ }
+ elif model_family == "flux":
+ text_encoder_filename = get_ltxv_text_encoder_filename(text_encoder_quantization)
+ model_files = [
+ {
+ "repoId" : "DeepBeepMeep/Flux",
+ "sourceFolderList" : [""],
+ "fileList" : [ ["flux_vae.safetensors"] ]
+ },
+ {
+ "repoId" : "DeepBeepMeep/LTX_Video",
+ "sourceFolderList" : ["T5_xxl_1.1"],
+ "fileList" : [ ["added_tokens.json", "special_tokens_map.json", "spiece.model", "tokenizer_config.json"] + computeList(text_encoder_filename) ]
+ },
+ {
+ "repoId" : "DeepBeepMeep/HunyuanVideo",
+ "sourceFolderList" : [ "clip_vit_large_patch14", ],
+ "fileList" :[
+ ["config.json", "merges.txt", "model.safetensors", "preprocessor_config.json", "special_tokens_map.json", "tokenizer.json", "tokenizer_config.json", "vocab.json"],
+ ]
+ }
+ ]
+
+ if not isinstance(model_files, list): model_files = [model_files]
+ for one_repo in model_files:
+ process_files_def(**one_repo)
+
+offload.default_verboseLevel = verbose_level
+
+
+def sanitize_file_name(file_name, rep =""):
+ return file_name.replace("/",rep).replace("\\",rep).replace(":",rep).replace("|",rep).replace("?",rep).replace("<",rep).replace(">",rep).replace("\"",rep).replace("\n",rep).replace("\r",rep)
+
+def extract_preset(model_type, lset_name, loras):
+ loras_choices = []
+ loras_choices_files = []
+ loras_mult_choices = ""
+ prompt =""
+ full_prompt =""
+ lset_name = sanitize_file_name(lset_name)
+ lora_dir = get_lora_dir(model_type)
+ if not lset_name.endswith(".lset"):
+ lset_name_filename = os.path.join(lora_dir, lset_name + ".lset" )
+ else:
+ lset_name_filename = os.path.join(lora_dir, lset_name )
+ error = ""
+ if not os.path.isfile(lset_name_filename):
+ error = f"Preset '{lset_name}' not found "
+ else:
+ missing_loras = []
+
+ with open(lset_name_filename, "r", encoding="utf-8") as reader:
+ text = reader.read()
+ lset = json.loads(text)
+
+ loras_choices_files = lset["loras"]
+ for lora_file in loras_choices_files:
+ choice = os.path.join(lora_dir, lora_file)
+ if choice not in loras:
+ missing_loras.append(lora_file)
+ else:
+ loras_choice_no = loras.index(choice)
+ loras_choices.append(str(loras_choice_no))
+
+ if len(missing_loras) > 0:
+ error = f"Unable to apply Lora preset '{lset_name} because the following Loras files are missing or invalid: {missing_loras}"
+
+ loras_mult_choices = lset["loras_mult"]
+ prompt = lset.get("prompt", "")
+ full_prompt = lset.get("full_prompt", False)
+ return loras_choices, loras_mult_choices, prompt, full_prompt, error
+
+
+def setup_loras(model_type, transformer, lora_dir, lora_preselected_preset, split_linear_modules_map = None):
+ loras =[]
+ loras_names = []
+ default_loras_choices = []
+ default_loras_multis_str = ""
+ loras_presets = []
+ default_lora_preset = ""
+ default_lora_preset_prompt = ""
+
+ from pathlib import Path
+
+ lora_dir = get_lora_dir(model_type)
+ if lora_dir != None :
+ if not os.path.isdir(lora_dir):
+ raise Exception("--lora-dir should be a path to a directory that contains Loras")
+
+
+ if lora_dir != None:
+ dir_loras = glob.glob( os.path.join(lora_dir , "*.sft") ) + glob.glob( os.path.join(lora_dir , "*.safetensors") )
+ dir_loras.sort()
+ loras += [element for element in dir_loras if element not in loras ]
+
+ dir_presets_settings = glob.glob( os.path.join(lora_dir , "*.json") )
+ dir_presets_settings.sort()
+ dir_presets = glob.glob( os.path.join(lora_dir , "*.lset") )
+ dir_presets.sort()
+ # loras_presets = [ Path(Path(file_path).parts[-1]).stem for file_path in dir_presets_settings + dir_presets]
+ loras_presets = [ Path(file_path).parts[-1] for file_path in dir_presets_settings + dir_presets]
+
+ if transformer !=None:
+ loras = offload.load_loras_into_model(transformer, loras, activate_all_loras=False, check_only= True, preprocess_sd=get_loras_preprocessor(transformer, model_type), split_linear_modules_map = split_linear_modules_map) #lora_multiplier,
+
+ if len(loras) > 0:
+ loras_names = [ Path(lora).stem for lora in loras ]
+
+ if len(lora_preselected_preset) > 0:
+ if not os.path.isfile(os.path.join(lora_dir, lora_preselected_preset + ".lset")):
+ raise Exception(f"Unknown preset '{lora_preselected_preset}'")
+ default_lora_preset = lora_preselected_preset
+ default_loras_choices, default_loras_multis_str, default_lora_preset_prompt, _ , error = extract_preset(model_type, default_lora_preset, loras)
+ if len(error) > 0:
+ print(error[:200])
+ return loras, loras_names, loras_presets, default_loras_choices, default_loras_multis_str, default_lora_preset_prompt, default_lora_preset
+
+
+def load_wan_model(model_filename, model_type, base_model_type, model_def, quantizeTransformer = False, dtype = torch.bfloat16, VAE_dtype = torch.float32, mixed_precision_transformer = False, save_quantized= False):
+ if test_class_i2v(base_model_type):
+ cfg = WAN_CONFIGS['i2v-14B']
+ else:
+ cfg = WAN_CONFIGS['t2v-14B']
+ # cfg = WAN_CONFIGS['t2v-1.3B']
+ if base_model_type in ("sky_df_1.3B", "sky_df_14B"):
+ model_factory = wan.DTT2V
+ else:
+ model_factory = wan.WanAny2V
+
+ wan_model = model_factory(
+ config=cfg,
+ checkpoint_dir="ckpts",
+ model_filename=model_filename,
+ model_type = model_type,
+ model_def = model_def,
+ base_model_type=base_model_type,
+ text_encoder_filename= get_wan_text_encoder_filename(text_encoder_quantization),
+ quantizeTransformer = quantizeTransformer,
+ dtype = dtype,
+ VAE_dtype = VAE_dtype,
+ mixed_precision_transformer = mixed_precision_transformer,
+ save_quantized = save_quantized
+ )
+
+ pipe = {"transformer": wan_model.model, "text_encoder" : wan_model.text_encoder.model, "vae": wan_model.vae.model }
+ if hasattr(wan_model,"model2") and wan_model.model2 is not None:
+ pipe["transformer2"] = wan_model.model2
+ if hasattr(wan_model, "clip"):
+ pipe["text_encoder_2"] = wan_model.clip.model
+ return wan_model, pipe
+
+def load_ltxv_model(model_filename, model_type, base_model_type, model_def, quantizeTransformer = False, dtype = torch.bfloat16, VAE_dtype = torch.float32, mixed_precision_transformer = False, save_quantized = False):
+ from ltx_video.ltxv import LTXV
+
+ ltxv_model = LTXV(
+ model_filepath = model_filename,
+ text_encoder_filepath = get_ltxv_text_encoder_filename(text_encoder_quantization),
+ model_type = model_type,
+ base_model_type = base_model_type,
+ model_def = model_def,
+ dtype = dtype,
+ # quantizeTransformer = quantizeTransformer,
+ VAE_dtype = VAE_dtype,
+ mixed_precision_transformer = mixed_precision_transformer
+ )
+
+ pipeline = ltxv_model.pipeline
+ pipe = {"transformer" : pipeline.video_pipeline.transformer, "vae" : pipeline.vae, "text_encoder" : pipeline.video_pipeline.text_encoder, "latent_upsampler" : pipeline.latent_upsampler}
+
+ return ltxv_model, pipe
+
+
+def load_flux_model(model_filename, model_type, base_model_type, model_def, quantizeTransformer = False, dtype = torch.bfloat16, VAE_dtype = torch.float32, mixed_precision_transformer = False, save_quantized = False):
+ from flux.flux_main import model_factory
+
+ flux_model = model_factory(
+ checkpoint_dir="ckpts",
+ model_filename=model_filename,
+ model_type = model_type,
+ model_def = model_def,
+ base_model_type=base_model_type,
+ text_encoder_filename= get_ltxv_text_encoder_filename(text_encoder_quantization),
+ quantizeTransformer = quantizeTransformer,
+ dtype = dtype,
+ VAE_dtype = VAE_dtype,
+ mixed_precision_transformer = mixed_precision_transformer,
+ save_quantized = save_quantized
+ )
+
+ pipe = { "transformer": flux_model.model, "vae" : flux_model.vae, "text_encoder" : flux_model.clip, "text_encoder_2" : flux_model.t5}
+
+ return flux_model, pipe
+
+def load_hunyuan_model(model_filename, model_type = None, base_model_type = None, model_def = None, quantizeTransformer = False, dtype = torch.bfloat16, VAE_dtype = torch.float32, mixed_precision_transformer = False, save_quantized = False):
+ from hyvideo.hunyuan import HunyuanVideoSampler
+
+ hunyuan_model = HunyuanVideoSampler.from_pretrained(
+ model_filepath = model_filename,
+ model_type = model_type,
+ base_model_type = base_model_type,
+ text_encoder_filepath = get_hunyuan_text_encoder_filename(text_encoder_quantization),
+ dtype = dtype,
+ quantizeTransformer = quantizeTransformer,
+ VAE_dtype = VAE_dtype,
+ mixed_precision_transformer = mixed_precision_transformer,
+ save_quantized = save_quantized
+ )
+
+ pipe = { "transformer" : hunyuan_model.model, "text_encoder" : hunyuan_model.text_encoder, "text_encoder_2" : hunyuan_model.text_encoder_2, "vae" : hunyuan_model.vae }
+
+ if hunyuan_model.wav2vec != None:
+ pipe["wav2vec"] = hunyuan_model.wav2vec
+
+
+ # if hunyuan_model.align_instance != None:
+ # pipe["align_instance"] = hunyuan_model.align_instance.facedet.model
+
+
+ from hyvideo.modules.models import get_linear_split_map
+
+ split_linear_modules_map = get_linear_split_map()
+ hunyuan_model.model.split_linear_modules_map = split_linear_modules_map
+ offload.split_linear_modules(hunyuan_model.model, split_linear_modules_map )
+
+
+ return hunyuan_model, pipe
+
+def get_transformer_model(model, submodel_no = 1):
+ if submodel_no > 1:
+ model_key = f"model{submodel_no}"
+ if not hasattr(model, model_key): return None
+
+ if hasattr(model, "model"):
+ if submodel_no > 1:
+ return getattr(model, f"model{submodel_no}")
+ else:
+ return model.model
+ elif hasattr(model, "transformer"):
+ return model.transformer
+ else:
+ raise Exception("no transformer found")
+
+
+def load_models(model_type):
+ global transformer_type
+ base_model_type = get_base_model_type(model_type)
+ model_def = get_model_def(model_type)
+ preload =int(args.preload)
+ save_quantized = args.save_quantized and model_def != None
+ model_filename = get_model_filename(model_type=model_type, quantization= "" if save_quantized else transformer_quantization, dtype_policy = transformer_dtype_policy)
+ if "URLs2" in model_def:
+ model_filename2 = get_model_filename(model_type=model_type, quantization= "" if save_quantized else transformer_quantization, dtype_policy = transformer_dtype_policy, submodel_no=2) # !!!!
+ else:
+ model_filename2 = None
+ modules = get_model_recursive_prop(model_type, "modules", return_list= True)
+ if save_quantized and "quanto" in model_filename:
+ save_quantized = False
+ print("Need to provide a non quantized model to create a quantized model to be saved")
+ if save_quantized and len(modules) > 0:
+ print(f"Unable to create a finetune quantized model as some modules are declared in the finetune definition. If your finetune includes already the module weights you can remove the 'modules' entry and try again. If not you will need also to change temporarly the model 'architecture' to an architecture that wont require the modules part ({modules}) to quantize and then add back the original 'modules' and 'architecture' entries.")
+ save_quantized = False
+ quantizeTransformer = not save_quantized and model_def !=None and transformer_quantization in ("int8", "fp8") and model_def.get("auto_quantize", False) and not "quanto" in model_filename
+ if quantizeTransformer and len(modules) > 0:
+ print(f"Autoquantize is not yet supported if some modules are declared")
+ quantizeTransformer = False
+ model_family = get_model_family(model_type)
+ transformer_dtype = get_transformer_dtype(model_family, transformer_dtype_policy)
+ if quantizeTransformer or "quanto" in model_filename:
+ transformer_dtype = torch.bfloat16 if "bf16" in model_filename or "BF16" in model_filename else transformer_dtype
+ transformer_dtype = torch.float16 if "fp16" in model_filename or"FP16" in model_filename else transformer_dtype
+ perc_reserved_mem_max = args.perc_reserved_mem_max
+ if preload == 0:
+ preload = server_config.get("preload_in_VRAM", 0)
+ model_file_list = [model_filename]
+ model_type_list = [model_type]
+ model_submodel_no_list = [1]
+ if model_filename2 != None:
+ model_file_list += [model_filename2]
+ model_type_list += [model_type]
+ model_submodel_no_list += [2]
+ for module_type in modules:
+ model_file_list.append(get_model_filename(module_type, transformer_quantization, transformer_dtype, is_module= True))
+ model_type_list.append(module_type)
+ model_submodel_no_list.append(0)
+ for filename, file_model_type, submodel_no in zip(model_file_list, model_type_list, model_submodel_no_list):
+ download_models(filename, file_model_type, submodel_no)
+ VAE_dtype = torch.float16 if server_config.get("vae_precision","16") == "16" else torch.float
+ mixed_precision_transformer = server_config.get("mixed_precision","0") == "1"
+ transformer_type = None
+ for submodel_no, filename in zip(model_submodel_no_list, model_file_list):
+ if submodel_no>=1:
+ print(f"Loading Model '{filename}' ...")
+ else:
+ print(f"Loading Module '{filename}' ...")
+
+ if model_family == "wan" :
+ wan_model, pipe = load_wan_model(model_file_list, model_type, base_model_type, model_def, quantizeTransformer = quantizeTransformer, dtype = transformer_dtype, VAE_dtype = VAE_dtype, mixed_precision_transformer = mixed_precision_transformer, save_quantized = save_quantized)
+ elif model_family == "ltxv":
+ wan_model, pipe = load_ltxv_model(model_file_list, model_type, base_model_type, model_def, quantizeTransformer = quantizeTransformer, dtype = transformer_dtype, VAE_dtype = VAE_dtype, mixed_precision_transformer = mixed_precision_transformer, save_quantized = save_quantized)
+ elif model_family == "flux":
+ wan_model, pipe = load_flux_model(model_file_list, model_type, base_model_type, model_def, quantizeTransformer = quantizeTransformer, dtype = transformer_dtype, VAE_dtype = VAE_dtype, mixed_precision_transformer = mixed_precision_transformer, save_quantized = save_quantized)
+ elif model_family == "hunyuan":
+ wan_model, pipe = load_hunyuan_model(model_file_list, model_type, base_model_type, model_def, quantizeTransformer = quantizeTransformer, dtype = transformer_dtype, VAE_dtype = VAE_dtype, mixed_precision_transformer = mixed_precision_transformer, save_quantized = save_quantized)
+ else:
+ raise Exception(f"Model '{model_filename}' not supported.")
+ kwargs = { "extraModelsToQuantize": None }
+ loras_transformer = ["transformer"]
+ if profile in (2, 4, 5):
+ budgets = { "transformer" : 100 if preload == 0 else preload, "text_encoder" : 100 if preload == 0 else preload, "*" : max(1000 if profile==5 else 3000 , preload) }
+ if "transformer2" in pipe:
+ budgets["transformer2"] = 100 if preload == 0 else preload
+ kwargs["budgets"] = budgets
+ elif profile == 3:
+ kwargs["budgets"] = { "*" : "70%" }
+
+ if "transformer2" in pipe:
+ loras_transformer += ["transformer2"]
+ if profile in [3,4]:
+ kwargs["pinnedMemory"] = ["transformer", "transformer2"]
+
+ global prompt_enhancer_image_caption_model, prompt_enhancer_image_caption_processor, prompt_enhancer_llm_model, prompt_enhancer_llm_tokenizer
+ if server_config.get("enhancer_enabled", 0) == 1:
+ from transformers import ( AutoModelForCausalLM, AutoProcessor, AutoTokenizer, LlamaForCausalLM )
+ prompt_enhancer_image_caption_model = AutoModelForCausalLM.from_pretrained( "ckpts/Florence2", trust_remote_code=True)
+ prompt_enhancer_image_caption_processor = AutoProcessor.from_pretrained( "ckpts/Florence2", trust_remote_code=True)
+ prompt_enhancer_llm_model = offload.fast_load_transformers_model("ckpts/Llama3_2/Llama3_2_quanto_bf16_int8.safetensors") #, configKwargs= {"_attn_implementation" :"XXXsdpa"}
+ prompt_enhancer_llm_tokenizer = AutoTokenizer.from_pretrained("ckpts/Llama3_2")
+ pipe["prompt_enhancer_image_caption_model"] = prompt_enhancer_image_caption_model
+ pipe["prompt_enhancer_llm_model"] = prompt_enhancer_llm_model
+ prompt_enhancer_image_caption_model._model_dtype = torch.float
+ if "budgets" in kwargs:
+ kwargs["budgets"]["prompt_enhancer_llm_model"] = 5000
+ else:
+ prompt_enhancer_image_caption_model = None
+ prompt_enhancer_image_caption_processor = None
+ prompt_enhancer_llm_model = None
+ prompt_enhancer_llm_tokenizer = None
+
+
+ offloadobj = offload.profile(pipe, profile_no= profile, compile = compile, quantizeTransformer = False, loras = loras_transformer, coTenantsMap= {}, perc_reserved_mem_max = perc_reserved_mem_max , convertWeightsFloatTo = transformer_dtype, **kwargs)
+ if len(args.gpu) > 0:
+ torch.set_default_device(args.gpu)
+ transformer_type = model_type
+ return wan_model, offloadobj
+
+if not "P" in preload_model_policy:
+ wan_model, offloadobj, transformer = None, None, None
+ reload_needed = True
+else:
+ wan_model, offloadobj = load_models(transformer_type)
+ if check_loras:
+ transformer = get_transformer_model(wan_model)
+ setup_loras(transformer_type, transformer, get_lora_dir(transformer_type), "", None)
+ exit()
+
+gen_in_progress = False
+
+def get_auto_attention():
+ for attn in ["sage2","sage","sdpa"]:
+ if attn in attention_modes_supported:
+ return attn
+ return "sdpa"
+
+def generate_header(model_type, compile, attention_mode):
+
+ description_container = [""]
+ get_model_name(model_type, description_container)
+ model_filename = get_model_filename(model_type, transformer_quantization, transformer_dtype_policy) or ""
+ description = description_container[0]
+ header = f"{description}
"
+
+ header += "Attention mode " + (attention_mode if attention_mode!="auto" else "auto/" + get_auto_attention() )
+ if attention_mode not in attention_modes_installed:
+ header += " -NOT INSTALLED-"
+ elif attention_mode not in attention_modes_supported:
+ header += " -NOT SUPPORTED-"
+ header += ""
+
+ if compile:
+ header += ", Pytorch compilation ON"
+ if "fp16" in model_filename:
+ header += ", Data Type FP16"
+ else:
+ header += ", Data Type BF16"
+
+ if "int8" in model_filename:
+ header += ", Quantization Scaled Int8"
+ header += "
"
+
+ return header
+
+def apply_changes( state,
+ transformer_types_choices,
+ transformer_dtype_policy_choice,
+ text_encoder_quantization_choice,
+ VAE_precision_choice,
+ mixed_precision_choice,
+ save_path_choice,
+ attention_choice,
+ compile_choice,
+ profile_choice,
+ vae_config_choice,
+ metadata_choice,
+ quantization_choice,
+ boost_choice = 1,
+ clear_file_list = 0,
+ preload_model_policy_choice = 1,
+ UI_theme_choice = "default",
+ enhancer_enabled_choice = 0,
+ mmaudio_enabled_choice = 0,
+ fit_canvas_choice = 0,
+ preload_in_VRAM_choice = 0,
+ depth_anything_v2_variant_choice = "vitl",
+ notification_sound_enabled_choice = 1,
+ notification_sound_volume_choice = 50,
+ max_frames_multiplier_choice = 1,
+ display_stats_choice = 0,
+ last_resolution_choice = None,
+):
+ if args.lock_config:
+ return
+ if gen_in_progress:
+ return "Unable to change config when a generation is in progress
",*[gr.update()]*6
+ global offloadobj, wan_model, server_config, loras, loras_names, default_loras_choices, default_loras_multis_str, default_lora_preset_prompt, default_lora_preset, loras_presets
+ server_config = {
+ "attention_mode" : attention_choice,
+ "transformer_types": transformer_types_choices,
+ "text_encoder_quantization" : text_encoder_quantization_choice,
+ "save_path" : save_path_choice,
+ "compile" : compile_choice,
+ "profile" : profile_choice,
+ "vae_config" : vae_config_choice,
+ "vae_precision" : VAE_precision_choice,
+ "mixed_precision" : mixed_precision_choice,
+ "metadata_type": metadata_choice,
+ "transformer_quantization" : quantization_choice,
+ "transformer_dtype_policy" : transformer_dtype_policy_choice,
+ "boost" : boost_choice,
+ "clear_file_list" : clear_file_list,
+ "preload_model_policy" : preload_model_policy_choice,
+ "UI_theme" : UI_theme_choice,
+ "fit_canvas": fit_canvas_choice,
+ "enhancer_enabled" : enhancer_enabled_choice,
+ "mmaudio_enabled" : mmaudio_enabled_choice,
+ "preload_in_VRAM" : preload_in_VRAM_choice,
+ "depth_anything_v2_variant": depth_anything_v2_variant_choice,
+ "notification_sound_enabled" : notification_sound_enabled_choice,
+ "notification_sound_volume" : notification_sound_volume_choice,
+ "max_frames_multiplier" : max_frames_multiplier_choice,
+ "display_stats" : display_stats_choice,
+ "last_model_type" : state["model_type"],
+ "last_model_per_family": state["last_model_per_family"],
+ "last_advanced_choice": state["advanced"],
+ "last_resolution_choice": last_resolution_choice,
+ "last_resolution_per_group": state["last_resolution_per_group"],
+ }
+
+ if Path(server_config_filename).is_file():
+ with open(server_config_filename, "r", encoding="utf-8") as reader:
+ text = reader.read()
+ old_server_config = json.loads(text)
+ if lock_ui_attention:
+ server_config["attention_mode"] = old_server_config["attention_mode"]
+ if lock_ui_compile:
+ server_config["compile"] = old_server_config["compile"]
+
+ with open(server_config_filename, "w", encoding="utf-8") as writer:
+ writer.write(json.dumps(server_config, indent=4))
+
+ changes = []
+ for k, v in server_config.items():
+ v_old = old_server_config.get(k, None)
+ if v != v_old:
+ changes.append(k)
+
+ global attention_mode, profile, compile, vae_config, boost, lora_dir, reload_needed, preload_model_policy, transformer_quantization, transformer_dtype_policy, transformer_types, text_encoder_quantization, save_path
+ attention_mode = server_config["attention_mode"]
+ profile = server_config["profile"]
+ compile = server_config["compile"]
+ text_encoder_quantization = server_config["text_encoder_quantization"]
+ vae_config = server_config["vae_config"]
+ boost = server_config["boost"]
+ save_path = server_config["save_path"]
+ preload_model_policy = server_config["preload_model_policy"]
+ transformer_quantization = server_config["transformer_quantization"]
+ transformer_dtype_policy = server_config["transformer_dtype_policy"]
+ text_encoder_quantization = server_config["text_encoder_quantization"]
+ transformer_types = server_config["transformer_types"]
+ model_filename = get_model_filename(transformer_type, transformer_quantization, transformer_dtype_policy)
+ state["model_filename"] = model_filename
+ if all(change in ["attention_mode", "vae_config", "boost", "save_path", "metadata_type", "clear_file_list", "fit_canvas", "depth_anything_v2_variant", "notification_sound_enabled", "notification_sound_volume", "mmaudio_enabled", "max_frames_multiplier", "display_stats"] for change in changes ):
+ model_family = gr.Dropdown()
+ model_choice = gr.Dropdown()
+ else:
+ reload_needed = True
+ model_family, model_choice = generate_dropdown_model_list(transformer_type)
+
+ header = generate_header(state["model_type"], compile=compile, attention_mode= attention_mode)
+ mmaudio_enabled = server_config["mmaudio_enabled"] > 0
+ return "The new configuration has been succesfully applied
", header, model_family, model_choice, gr.Row(visible= server_config["enhancer_enabled"] == 1), gr.Row(visible= mmaudio_enabled), gr.Column(visible= mmaudio_enabled)
+
+
+
+from moviepy.editor import ImageSequenceClip
+import numpy as np
+
+def save_video(final_frames, output_path, fps=24):
+ assert final_frames.ndim == 4 and final_frames.shape[3] == 3, f"invalid shape: {final_frames} (need t h w c)"
+ if final_frames.dtype != np.uint8:
+ final_frames = (final_frames * 255).astype(np.uint8)
+ ImageSequenceClip(list(final_frames), fps=fps).write_videofile(output_path, verbose= False)
+
+
+def get_gen_info(state):
+ cache = state.get("gen", None)
+ if cache == None:
+ cache = dict()
+ state["gen"] = cache
+ return cache
+
+def build_callback(state, pipe, send_cmd, status, num_inference_steps):
+ gen = get_gen_info(state)
+ gen["num_inference_steps"] = num_inference_steps
+ start_time = time.time()
+ def callback(step_idx, latent, force_refresh, read_state = False, override_num_inference_steps = -1, pass_no = -1):
+ refresh_id = gen.get("refresh", -1)
+ if force_refresh or step_idx >= 0:
+ pass
+ else:
+ refresh_id = gen.get("refresh", -1)
+ if refresh_id < 0:
+ return
+ UI_refresh = state.get("refresh", 0)
+ if UI_refresh >= refresh_id:
+ return
+ if override_num_inference_steps > 0:
+ gen["num_inference_steps"] = override_num_inference_steps
+
+ num_inference_steps = gen.get("num_inference_steps", 0)
+ status = gen["progress_status"]
+ state["refresh"] = refresh_id
+ if read_state:
+ phase, step_idx = gen["progress_phase"]
+ else:
+ step_idx += 1
+ if gen.get("abort", False):
+ # pipe._interrupt = True
+ phase = "Aborting"
+ elif step_idx == num_inference_steps:
+ phase = "VAE Decoding"
+ else:
+ if pass_no <=0:
+ phase = "Denoising"
+ elif pass_no == 1:
+ phase = "Denoising First Pass"
+ elif pass_no == 2:
+ phase = "Denoising Second Pass"
+ elif pass_no == 3:
+ phase = "Denoising Third Pass"
+ else:
+ phase = f"Denoising {pass_no}th Pass"
+
+ gen["progress_phase"] = (phase, step_idx)
+ status_msg = merge_status_context(status, phase)
+
+ elapsed_time = time.time() - start_time
+ status_msg = merge_status_context(status, f"{phase} | {format_time(elapsed_time)}")
+ if step_idx >= 0:
+ progress_args = [(step_idx , num_inference_steps) , status_msg , num_inference_steps]
+ else:
+ progress_args = [0, status_msg]
+
+ # progress(*progress_args)
+ send_cmd("progress", progress_args)
+ if latent != None:
+ latent = latent.to("cpu", non_blocking=True)
+ send_cmd("preview", latent)
+
+ # gen["progress_args"] = progress_args
+
+ return callback
+def abort_generation(state):
+ gen = get_gen_info(state)
+ if "in_progress" in gen: # and wan_model != None:
+ if wan_model != None:
+ wan_model._interrupt= True
+ gen["abort"] = True
+ msg = "Processing Request to abort Current Generation"
+ gen["status"] = msg
+ gr.Info(msg)
+ return gr.Button(interactive= False)
+ else:
+ return gr.Button(interactive= True)
+
+
+
+def refresh_gallery(state): #, msg
+ gen = get_gen_info(state)
+
+ # gen["last_msg"] = msg
+ file_list = gen.get("file_list", None)
+ choice = gen.get("selected",0)
+ in_progress = "in_progress" in gen
+ if in_progress:
+ if gen.get("last_selected", True):
+ choice = max(len(file_list) - 1,0)
+
+ queue = gen.get("queue", [])
+ abort_interactive = not gen.get("abort", False)
+ if not in_progress or len(queue) == 0:
+ return gr.Gallery(selected_index=choice, value = file_list), gr.HTML("", visible= False), gr.Button(visible=True), gr.Button(visible=False), gr.Row(visible=False), gr.Row(visible=False), update_queue_data(queue), gr.Button(interactive= abort_interactive), gr.Button(visible= False)
+ else:
+ task = queue[0]
+ start_img_md = ""
+ end_img_md = ""
+ prompt = task["prompt"]
+ params = task["params"]
+ model_type = params["model_type"]
+ base_model_type = get_base_model_type(model_type)
+ model_def = get_model_def(model_type)
+ is_image = model_def.get("image_outputs", False)
+ onemorewindow_visible = test_any_sliding_window(base_model_type) and params.get("image_mode",0) == 0 and not params.get("mode","").startswith("edit_")
+ enhanced = False
+ if prompt.startswith("!enhanced!\n"):
+ enhanced = True
+ prompt = prompt[len("!enhanced!\n"):]
+ if "\n" in prompt :
+ prompts = prompt.split("\n")
+ window_no= gen.get("window_no",1)
+ if window_no > len(prompts):
+ window_no = len(prompts)
+ window_no -= 1
+ prompts[window_no]="" + prompts[window_no] + ""
+ prompt = "".join(prompts) + if enhanced: + prompt = "Enhanced:
" + prompt + list_uri = [] + list_labels = [] + start_img_uri = task.get('start_image_data_base64') + if start_img_uri != None: + list_uri += start_img_uri + list_labels += task.get('start_image_labels') + end_img_uri = task.get('end_image_data_base64') + if end_img_uri != None: + list_uri += end_img_uri + list_labels += task.get('end_image_labels') + + thumbnail_size = "100px" + thumbnails = "" + for i, (img_label, img_uri) in enumerate(zip(list_labels,list_uri)): + thumbnails += f'
| " + prompt + " | " + thumbnails + "
" + label + "
WanGP - Wan 2.1 model for the GPU Poor by DeepBeepMeep (GitHub)
") + gr.Markdown("Original Wan 2.1 Model by Alibaba (GitHub)") + gr.Markdown("Many thanks to:") + gr.Markdown("- Alibaba Wan team for the best open source video generator") + gr.Markdown("- Alibaba Vace, Multitalk and Fun Teams for their incredible control net models") + gr.Markdown("- Tencent for the impressive Hunyuan Video models") + gr.Markdown("- Blackforest Labs for the innovative Flux image generators") + gr.Markdown("- Lightricks for their super fast LTX Video models") + gr.Markdown("Huge acknowlegments to these great open source projects used in WanGP:") + gr.Markdown("- Rife: temporal upsampler (https://github.com/hzwer/ECCV2022-RIFE)") + gr.Markdown("- DwPose: Open Pose extractor (https://github.com/IDEA-Research/DWPose)") + gr.Markdown("- DepthAnything & Midas: Depth extractors (https://github.com/DepthAnything/Depth-Anything-V2) and (https://github.com/isl-org/MiDaS") + gr.Markdown("- Matanyone and SAM2: Mask Generation (https://github.com/pq-yang/MatAnyone) and (https://github.com/facebookresearch/sam2)") + gr.Markdown("- Pyannote: speaker diarization (https://github.com/pyannote/pyannote-audio)") + + gr.Markdown("
Special thanks to the following people for their support:") + gr.Markdown("- Cocktail Peanuts : QA and simple installation via Pinokio.computer") + gr.Markdown("- Tophness : created (former) multi tabs and queuing frameworks") + gr.Markdown("- AmericanPresidentJimmyCarter : added original support for Skip Layer Guidance") + gr.Markdown("- Remade_AI : for their awesome Loras collection") + gr.Markdown("- Reevoy24 : for his repackaging / completing the documentation") + gr.Markdown("- Redtash1 : for designing the protype of the RAM /VRAM stats viewer") + +def generate_info_tab(): + + + with open("docs/VACE.md", "r", encoding="utf-8") as reader: + vace= reader.read() + + with open("docs/MODELS.md", "r", encoding="utf-8") as reader: + models = reader.read() + + with open("docs/LORAS.md", "r", encoding="utf-8") as reader: + loras = reader.read() + + with gr.Tabs() : + with gr.Tab("Models", id="models"): + gr.Markdown(models) + with gr.Tab("Loras", id="loras"): + gr.Markdown(loras) + with gr.Tab("Vace", id="vace"): + gr.Markdown(vace) + +def compact_name(family_name, model_name): + if model_name.startswith(family_name): + return model_name[len(family_name):].strip() + return model_name + +def get_sorted_dropdown(dropdown_types, current_model_family): + models_families = [get_model_family(type, for_ui= True) for type in dropdown_types] + families = {} + for family in models_families: + if family not in families: families[family] = 1 + + families_orders = [ families_infos[family][0] for family in families ] + families_labels = [ families_infos[family][1] for family in families ] + sorted_familes = [ info[1:] for info in sorted(zip(families_orders, families_labels, families), key=lambda c: c[0])] + if current_model_family is None: + dropdown_choices = [ (families_infos[family][0], get_model_name(model_type), model_type) for model_type, family in zip(dropdown_types, models_families)] + else: + dropdown_choices = [ (families_infos[family][0], compact_name(families_infos[family][1], get_model_name(model_type)), model_type) for model_type, family in zip( dropdown_types, models_families) if family == current_model_family] + dropdown_choices = sorted(dropdown_choices, key=lambda c: (c[0], c[1])) + dropdown_choices = [model[1:] for model in dropdown_choices] + return sorted_familes, dropdown_choices + +def generate_dropdown_model_list(current_model_type): + dropdown_types= transformer_types if len(transformer_types) > 0 else displayed_model_types + if current_model_type not in dropdown_types: + dropdown_types.append(current_model_type) + current_model_family = get_model_family(current_model_type, for_ui= True) + sorted_familes, dropdown_choices = get_sorted_dropdown(dropdown_types, current_model_family) + + dropdown_families = gr.Dropdown( + choices= sorted_familes, + value= current_model_family, + show_label= False, + scale= 1, + elem_id="family_list", + min_width=50 + ) + + return dropdown_families, gr.Dropdown( + choices= dropdown_choices, + value= current_model_type, + show_label= False, + scale= 4, + elem_id="model_list", + ) + +def change_model_family(state, current_model_family): + dropdown_types= transformer_types if len(transformer_types) > 0 else displayed_model_types + current_family_name = families_infos[current_model_family][1] + models_families = [get_model_family(type, for_ui= True) for type in dropdown_types] + dropdown_choices = [ (compact_name(current_family_name, get_model_name(model_type)), model_type) for model_type, family in zip(dropdown_types, models_families) if family == current_model_family ] + dropdown_choices = sorted(dropdown_choices, key=lambda c: c[0]) + last_model_per_family = state.get("last_model_per_family", {}) + model_type = last_model_per_family.get(current_model_family, "") + if len(model_type) == "" or model_type not in [choice[1] for choice in dropdown_choices] : model_type = dropdown_choices[0][1] + return gr.Dropdown(choices= dropdown_choices, value = model_type ) + +def set_new_tab(tab_state, new_tab_no): + global vmc_event_handler + + tab_video_mask_creator = 2 + + old_tab_no = tab_state.get("tab_no",0) + # print(f"old tab {old_tab_no}, new tab {new_tab_no}") + if old_tab_no == tab_video_mask_creator: + vmc_event_handler(False) + elif new_tab_no == tab_video_mask_creator: + if gen_in_progress: + gr.Info("Unable to access this Tab while a Generation is in Progress. Please come back later") + tab_state["tab_no"] = 0 + return gr.Tabs(selected="video_gen") + else: + vmc_event_handler(True) + tab_state["tab_no"] = new_tab_no + return gr.Tabs() + +def select_tab(tab_state, evt:gr.SelectData): + return set_new_tab(tab_state, evt.index) + +def get_js(): + start_quit_timer_js = """ + () => { + function findAndClickGradioButton(elemId) { + const gradioApp = document.querySelector('gradio-app') || document; + const button = gradioApp.querySelector(`#${elemId}`); + if (button) { button.click(); } + } + + if (window.quitCountdownTimeoutId) clearTimeout(window.quitCountdownTimeoutId); + + let js_click_count = 0; + const max_clicks = 5; + + function countdownStep() { + if (js_click_count < max_clicks) { + findAndClickGradioButton('trigger_info_single_btn'); + js_click_count++; + window.quitCountdownTimeoutId = setTimeout(countdownStep, 1000); + } else { + findAndClickGradioButton('force_quit_btn_hidden'); + } + } + + countdownStep(); + } + """ + + cancel_quit_timer_js = """ + () => { + if (window.quitCountdownTimeoutId) { + clearTimeout(window.quitCountdownTimeoutId); + window.quitCountdownTimeoutId = null; + console.log("Quit countdown cancelled (single trigger)."); + } + } + """ + + trigger_zip_download_js = """ + (base64String) => { + if (!base64String) { + console.log("No base64 zip data received, skipping download."); + return; + } + try { + const byteCharacters = atob(base64String); + const byteNumbers = new Array(byteCharacters.length); + for (let i = 0; i < byteCharacters.length; i++) { + byteNumbers[i] = byteCharacters.charCodeAt(i); + } + const byteArray = new Uint8Array(byteNumbers); + const blob = new Blob([byteArray], { type: 'application/zip' }); + + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.style.display = 'none'; + a.href = url; + a.download = 'queue.zip'; + document.body.appendChild(a); + a.click(); + + window.URL.revokeObjectURL(url); + document.body.removeChild(a); + console.log("Zip download triggered."); + } catch (e) { + console.error("Error processing base64 data or triggering download:", e); + } + } + """ + + trigger_settings_download_js = """ + (base64String, filename) => { + if (!base64String) { + console.log("No base64 settings data received, skipping download."); + return; + } + try { + const byteCharacters = atob(base64String); + const byteNumbers = new Array(byteCharacters.length); + for (let i = 0; i < byteCharacters.length; i++) { + byteNumbers[i] = byteCharacters.charCodeAt(i); + } + const byteArray = new Uint8Array(byteNumbers); + const blob = new Blob([byteArray], { type: 'application/text' }); + + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.style.display = 'none'; + a.href = url; + a.download = filename; + document.body.appendChild(a); + a.click(); + + window.URL.revokeObjectURL(url); + document.body.removeChild(a); + console.log("settings download triggered."); + } catch (e) { + console.error("Error processing base64 data or triggering download:", e); + } + } + """ + return start_quit_timer_js, cancel_quit_timer_js, trigger_zip_download_js, trigger_settings_download_js + +def create_ui(): + global vmc_event_handler + css = """ + .postprocess div, + .postprocess span, + .postprocess label, + .postprocess input, + .postprocess select, + .postprocess textarea { + font-size: 12px !important; + padding: 0px !important; + border: 5px !important; + border-radius: 0px !important; + --form-gap-width: 0px !important; + box-shadow: none !important; + --layout-gap: 0px !important; + } + .postprocess span {margin-top:4px;margin-bottom:4px} + #model_list, #family_list{ + background-color:black; + padding:1px} + + #model_list input, #family_list input { + font-size:25px} + + #family_list div div { + border-radius: 4px 0px 0px 4px; + } + + #model_list div div { + border-radius: 0px 4px 4px 0px; + } + + .title-with-lines { + display: flex; + align-items: center; + margin: 25px 0; + } + .line { + flex-grow: 1; + height: 1px; + background-color: #333; + } + h2 { + margin: 0 20px; + white-space: nowrap; + } + .queue-item { + border: 1px solid #ccc; + padding: 10px; + margin: 5px 0; + border-radius: 5px; + } + .current { + background: #f8f9fa; + border-left: 4px solid #007bff; + } + .task-header { + display: flex; + justify-content: space-between; + margin-bottom: 5px; + } + .progress-container { + height: 10px; + background: #e9ecef; + border-radius: 5px; + overflow: hidden; + } + .progress-bar { + height: 100%; + background: #007bff; + transition: width 0.3s ease; + } + .task-details { + display: flex; + justify-content: space-between; + font-size: 0.9em; + color: #6c757d; + margin-top: 5px; + } + .task-prompt { + font-size: 0.8em; + color: #868e96; + margin-top: 5px; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + } + #queue_df th { + pointer-events: none; + text-align: center; + vertical-align: middle; + font-size:11px; + } + #xqueue_df table { + width: 100%; + overflow: hidden !important; + } + #xqueue_df::-webkit-scrollbar { + display: none !important; + } + #xqueue_df { + scrollbar-width: none !important; + -ms-overflow-style: none !important; + } + .selection-button { + display: none; + } + .cell-selected { + --ring-color: none; + } + #queue_df th:nth-child(1), + #queue_df td:nth-child(1) { + width: 60px; + text-align: center; + vertical-align: middle; + cursor: default !important; + pointer-events: none; + } + #xqueue_df th:nth-child(2), + #queue_df td:nth-child(2) { + text-align: center; + vertical-align: middle; + white-space: normal; + } + #queue_df td:nth-child(2) { + cursor: default !important; + } + #queue_df th:nth-child(3), + #queue_df td:nth-child(3) { + width: 60px; + text-align: center; + vertical-align: middle; + cursor: default !important; + pointer-events: none; + } + #queue_df th:nth-child(4), + #queue_df td:nth-child(4) { + width: 60px; + text-align: center; + white-space: nowrap; + cursor: default !important; + pointer-events: none; + } + #queue_df th:nth-child(5), #queue_df td:nth-child(7), + #queue_df th:nth-child(6), #queue_df td:nth-child(8) { + width: 60px; + text-align: center; + vertical-align: middle; + } + #queue_df td:nth-child(5) img, + #queue_df td:nth-child(6) img { + max-width: 50px; + max-height: 50px; + object-fit: contain; + display: block; + margin: auto; + cursor: pointer; + } + #queue_df th:nth-child(7), #queue_df td:nth-child(9), + #queue_df th:nth-child(8), #queue_df td:nth-child(10), + #queue_df th:nth-child(9), #queue_df td:nth-child(11) { + width: 20px; + padding: 2px !important; + cursor: pointer; + text-align: center; + font-weight: bold; + vertical-align: middle; + } + #queue_df td:nth-child(5):hover, + #queue_df td:nth-child(6):hover, + #queue_df td:nth-child(7):hover, + #queue_df td:nth-child(8):hover, + #queue_df td:nth-child(9):hover { + background-color: #e0e0e0; + } + #image-modal-container { + position: fixed; + top: 0; + left: 0; + width: 100%; + height: 100%; + background-color: rgba(0, 0, 0, 0.7); + justify-content: center; + align-items: center; + z-index: 1000; + padding: 20px; + box-sizing: border-box; + } + #image-modal-container > div { + background-color: white; + padding: 15px; + border-radius: 8px; + max-width: 90%; + max-height: 90%; + overflow: auto; + position: relative; + display: flex; + flex-direction: column; + } + #image-modal-container img { + max-width: 100%; + max-height: 80vh; + object-fit: contain; + margin-top: 10px; + } + #image-modal-close-button-row { + display: flex; + justify-content: flex-end; + } + #image-modal-close-button-row button { + cursor: pointer; + } + .progress-container-custom { + width: 100%; + background-color: #e9ecef; + border-radius: 0.375rem; + overflow: hidden; + height: 25px; + position: relative; + margin-top: 5px; + margin-bottom: 5px; + } + .progress-bar-custom { + height: 100%; + background-color: #0d6efd; + transition: width 0.3s ease-in-out; + display: flex; + align-items: center; + justify-content: center; + color: white; + font-size: 0.9em; + font-weight: bold; + white-space: nowrap; + overflow: hidden; + } + .progress-bar-custom.idle { + background-color: #6c757d; + } + .progress-bar-text { + position: absolute; + top: 0; + left: 0; + width: 100%; + height: 100%; + display: flex; + align-items: center; + justify-content: center; + color: white; + mix-blend-mode: difference; + font-size: 0.9em; + font-weight: bold; + white-space: nowrap; + z-index: 2; + pointer-events: none; + } + + .hover-image { + cursor: pointer; + position: relative; + display: inline-block; /* Important for positioning */ + } + + .hover-image .tooltip { + visibility: hidden; + opacity: 0; + position: absolute; + top: 100%; + left: 50%; + transform: translateX(-50%); + background-color: rgba(0, 0, 0, 0.8); + color: white; + padding: 4px 6px; + border-radius: 2px; + font-size: 14px; + white-space: nowrap; + pointer-events: none; + z-index: 9999; + transition: visibility 0s linear 1s, opacity 0.3s linear 1s; /* Delay both properties */ + } + div.compact_tab , span.compact_tab + { padding: 0px !important; + } + .hover-image .tooltip2 { + visibility: hidden; + opacity: 0; + position: absolute; + top: 50%; /* Center vertically with the image */ + left: 0; /* Position to the left of the image */ + transform: translateY(-50%); /* Center vertically */ + margin-left: -10px; /* Small gap to the left of image */ + background-color: rgba(0, 0, 0, 0.8); + color: white; + padding: 8px 12px; + border-radius: 4px; + font-size: 14px; + white-space: nowrap; + pointer-events: none; + z-index: 9999; + transition: visibility 0s linear 1s, opacity 0.3s linear 1s; + } + + .hover-image:hover .tooltip, .hover-image:hover .tooltip2 { + visibility: visible; + opacity: 1; + transition: visibility 0s linear 1s, opacity 0.3s linear 1s; /* 1s delay before showing */ + } + """ + UI_theme = server_config.get("UI_theme", "default") + UI_theme = args.theme if len(args.theme) > 0 else UI_theme + if UI_theme == "gradio": + theme = None + else: + theme = gr.themes.Soft(font=["Verdana"], primary_hue="sky", neutral_hue="slate", text_size="md") + + js = """ + function() { + // Attach function to window object to make it globally accessible + window.sendColIndex = function(index) { + const input= document.querySelector('#preview_column_no textarea'); + if (input) { + input.value = index; + input.dispatchEvent(new Event("input", { bubbles: true })); + input.focus(); + input.blur(); + console.log('Events dispatched for column:', index); + } + }; + + console.log('sendColIndex function attached to window'); + } + """ + if server_config.get("display_stats", 0) == 1: + from wan.utils.stats import SystemStatsApp + stats_app = SystemStatsApp() + else: + stats_app = None + + with gr.Blocks(css=css, js=js, theme=theme, title= "WanGP") as main: + gr.Markdown(f"
WanGP v{WanGP_version} by DeepBeepMeep ") # (Updates)
" + get_model_name(transformer_type) + "
") + model_family = gr.Dropdown(visible=False, value= "") + model_choice = gr.Dropdown(visible=False, value= transformer_type, choices= [transformer_type]) + else: + gr.Markdown("")
+ model_family, model_choice = generate_dropdown_model_list(transformer_type)
+ gr.Markdown("
")
+ with gr.Row():
+ header = gr.Markdown(generate_header(transformer_type, compile, attention_mode), visible= True)
+ if stats_app is not None:
+ stats_element = stats_app.get_gradio_element()
+
+ with gr.Row():
+ ( state, loras_choices, lset_name, resolution,
+ video_guide, image_guide, video_mask, image_mask, image_refs, prompt_enhancer_row, mmaudio_tab, PP_MMAudio_col
+ ) = generate_video_tab(model_family=model_family, model_choice=model_choice, header=header, main = main)
+ with gr.Tab("Guides", id="info") as info_tab:
+ generate_info_tab()
+ with gr.Tab("Video Mask Creator", id="video_mask_creator") as video_mask_creator:
+ matanyone_app.display(main_tabs, tab_state, video_guide, image_guide, video_mask, image_mask, image_refs)
+ if not args.lock_config:
+ with gr.Tab("Downloads", id="downloads") as downloads_tab:
+ generate_download_tab(lset_name, loras_choices, state)
+ with gr.Tab("Configuration", id="configuration") as configuration_tab:
+ generate_configuration_tab(state, main, header, model_family, model_choice, resolution, prompt_enhancer_row, mmaudio_tab, PP_MMAudio_col)
+ with gr.Tab("About"):
+ generate_about_tab()
+ if stats_app is not None:
+ stats_app.setup_events(main, state)
+ main_tabs.select(fn=select_tab, inputs= [tab_state], outputs= main_tabs, trigger_mode="multiple")
+ return main
+
+if __name__ == "__main__":
+ atexit.register(autosave_queue)
+ download_ffmpeg()
+ # threading.Thread(target=runner, daemon=True).start()
+ os.environ["GRADIO_ANALYTICS_ENABLED"] = "False"
+ server_port = int(args.server_port)
+ if os.name == "nt":
+ asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
+ if server_port == 0:
+ server_port = int(os.getenv("SERVER_PORT", "7860"))
+ server_name = args.server_name
+ if args.listen:
+ server_name = "0.0.0.0"
+ if len(server_name) == 0:
+ server_name = os.getenv("SERVER_NAME", "localhost")
+ demo = create_ui()
+ if args.open_browser:
+ import webbrowser
+ if server_name.startswith("http"):
+ url = server_name
+ else:
+ url = "http://" + server_name
+ webbrowser.open(url + ":" + str(server_port), new = 0, autoraise = True)
+ demo.launch(favicon_path="favicon.png", server_name=server_name, server_port=server_port, share=args.share, allowed_paths=[save_path])
+