Spaces:
Sleeping
Sleeping
| """Modified from https://github.com/guoyww/AnimateDiff/blob/main/app.py | |
| """ | |
| import os | |
| import random | |
| import cv2 | |
| import gradio as gr | |
| import numpy as np | |
| import torch | |
| from omegaconf import OmegaConf | |
| from PIL import Image | |
| from safetensors import safe_open | |
| from ..data.bucket_sampler import ASPECT_RATIO_512, get_closest_ratio | |
| from ..models import (AutoencoderKLWan, AutoTokenizer, CLIPModel, | |
| WanT5EncoderModel, WanTransformer3DModel) | |
| from ..pipeline import WanFunInpaintPipeline, WanFunPipeline | |
| from ..utils.fp8_optimization import (convert_model_weight_to_float8, | |
| convert_weight_dtype_wrapper, | |
| replace_parameters_by_name) | |
| from ..utils.lora_utils import merge_lora, unmerge_lora | |
| from ..utils.utils import (filter_kwargs, get_image_to_video_latent, | |
| get_video_to_video_latent, save_videos_grid) | |
| from .controller import (Fun_Controller, Fun_Controller_EAS, all_cheduler_dict, | |
| css, ddpm_scheduler_dict, flow_scheduler_dict, | |
| gradio_version, gradio_version_is_above_4) | |
| from .ui import (create_cfg_and_seedbox, | |
| create_fake_finetune_models_checkpoints, | |
| create_fake_height_width, create_fake_model_checkpoints, | |
| create_fake_model_type, create_finetune_models_checkpoints, | |
| create_generation_method, | |
| create_generation_methods_and_video_length, | |
| create_height_width, create_model_checkpoints, | |
| create_model_type, create_prompts, create_samplers, | |
| create_ui_outputs) | |
| class Wan_Fun_Controller(Fun_Controller): | |
| def update_diffusion_transformer(self, diffusion_transformer_dropdown): | |
| print("Update diffusion transformer") | |
| self.diffusion_transformer_dropdown = diffusion_transformer_dropdown | |
| if diffusion_transformer_dropdown == "none": | |
| return gr.update() | |
| self.vae = AutoencoderKLWan.from_pretrained( | |
| os.path.join(diffusion_transformer_dropdown, self.config['vae_kwargs'].get('vae_subpath', 'vae')), | |
| additional_kwargs=OmegaConf.to_container(self.config['vae_kwargs']), | |
| ).to(self.weight_dtype) | |
| # Get Transformer | |
| self.transformer = WanTransformer3DModel.from_pretrained( | |
| os.path.join(diffusion_transformer_dropdown, self.config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), | |
| transformer_additional_kwargs=OmegaConf.to_container(self.config['transformer_additional_kwargs']), | |
| low_cpu_mem_usage=True, | |
| torch_dtype=self.weight_dtype, | |
| ) | |
| # Get Tokenizer | |
| self.tokenizer = AutoTokenizer.from_pretrained( | |
| os.path.join(diffusion_transformer_dropdown, self.config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), | |
| ) | |
| # Get Text encoder | |
| self.text_encoder = WanT5EncoderModel.from_pretrained( | |
| os.path.join(diffusion_transformer_dropdown, self.config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), | |
| additional_kwargs=OmegaConf.to_container(self.config['text_encoder_kwargs']), | |
| ).to(self.weight_dtype) | |
| self.text_encoder = self.text_encoder.eval() | |
| if self.transformer.config.in_channels != self.vae.config.latent_channels: | |
| # Get Clip Image Encoder | |
| self.clip_image_encoder = CLIPModel.from_pretrained( | |
| os.path.join(diffusion_transformer_dropdown, self.config['image_encoder_kwargs'].get('image_encoder_subpath', 'image_encoder')), | |
| ).to(self.weight_dtype) | |
| self.clip_image_encoder = self.clip_image_encoder.eval() | |
| else: | |
| self.clip_image_encoder = None | |
| Choosen_Scheduler = self.scheduler_dict[list(self.scheduler_dict.keys())[0]] | |
| self.scheduler = Choosen_Scheduler( | |
| **filter_kwargs(Choosen_Scheduler, OmegaConf.to_container(self.config['scheduler_kwargs'])) | |
| ) | |
| # Get pipeline | |
| if self.model_type == "Inpaint": | |
| if self.transformer.config.in_channels != self.vae.config.latent_channels: | |
| self.pipeline = WanFunInpaintPipeline( | |
| vae=self.vae, | |
| tokenizer=self.tokenizer, | |
| text_encoder=self.text_encoder, | |
| transformer=self.transformer, | |
| scheduler=self.scheduler, | |
| clip_image_encoder=self.clip_image_encoder, | |
| ) | |
| else: | |
| self.pipeline = WanFunPipeline( | |
| vae=self.vae, | |
| tokenizer=self.tokenizer, | |
| text_encoder=self.text_encoder, | |
| transformer=self.transformer, | |
| scheduler=self.scheduler, | |
| ) | |
| else: | |
| raise ValueError("Not support now") | |
| if self.GPU_memory_mode == "sequential_cpu_offload": | |
| replace_parameters_by_name(self.transformer, ["modulation",], device="cuda") | |
| self.transformer.freqs = self.transformer.freqs.to(device="cuda") | |
| self.pipeline.enable_sequential_cpu_offload() | |
| elif self.GPU_memory_mode == "model_cpu_offload_and_qfloat8": | |
| convert_model_weight_to_float8(self.transformer, exclude_module_name=["modulation",]) | |
| convert_weight_dtype_wrapper(self.transformer, self.weight_dtype) | |
| self.pipeline.enable_model_cpu_offload() | |
| else: | |
| self.pipeline.enable_model_cpu_offload() | |
| print("Update diffusion transformer done") | |
| return gr.update() | |
| def generate( | |
| self, | |
| diffusion_transformer_dropdown, | |
| base_model_dropdown, | |
| lora_model_dropdown, | |
| lora_alpha_slider, | |
| prompt_textbox, | |
| negative_prompt_textbox, | |
| sampler_dropdown, | |
| sample_step_slider, | |
| resize_method, | |
| width_slider, | |
| height_slider, | |
| base_resolution, | |
| generation_method, | |
| length_slider, | |
| overlap_video_length, | |
| partial_video_length, | |
| cfg_scale_slider, | |
| start_image, | |
| end_image, | |
| validation_video, | |
| validation_video_mask, | |
| control_video, | |
| denoise_strength, | |
| seed_textbox, | |
| is_api = False, | |
| ): | |
| self.clear_cache() | |
| self.input_check( | |
| resize_method, generation_method, start_image, end_image, validation_video,control_video, is_api | |
| ) | |
| is_image = True if generation_method == "Image Generation" else False | |
| if self.base_model_path != base_model_dropdown: | |
| self.update_base_model(base_model_dropdown) | |
| if self.lora_model_path != lora_model_dropdown: | |
| self.update_lora_model(lora_model_dropdown) | |
| self.pipeline.scheduler = self.scheduler_dict[sampler_dropdown].from_config(self.pipeline.scheduler.config) | |
| if resize_method == "Resize according to Reference": | |
| height_slider, width_slider = self.get_height_width_from_reference( | |
| base_resolution, start_image, validation_video, control_video, | |
| ) | |
| if self.lora_model_path != "none": | |
| # lora part | |
| self.pipeline = merge_lora(self.pipeline, self.lora_model_path, multiplier=lora_alpha_slider) | |
| if int(seed_textbox) != -1 and seed_textbox != "": torch.manual_seed(int(seed_textbox)) | |
| else: seed_textbox = np.random.randint(0, 1e10) | |
| generator = torch.Generator(device="cuda").manual_seed(int(seed_textbox)) | |
| try: | |
| if self.model_type == "Inpaint": | |
| if self.transformer.config.in_channels != self.vae.config.latent_channels: | |
| if generation_method == "Long Video Generation": | |
| if validation_video is not None: | |
| raise gr.Error(f"Video to Video is not Support Long Video Generation now.") | |
| init_frames = 0 | |
| last_frames = init_frames + partial_video_length | |
| while init_frames < length_slider: | |
| if last_frames >= length_slider: | |
| _partial_video_length = length_slider - init_frames | |
| _partial_video_length = int((_partial_video_length - 1) // self.vae.config.temporal_compression_ratio * self.vae.config.temporal_compression_ratio) + 1 | |
| if _partial_video_length <= 0: | |
| break | |
| else: | |
| _partial_video_length = partial_video_length | |
| if last_frames >= length_slider: | |
| input_video, input_video_mask, clip_image = get_image_to_video_latent(start_image, end_image, video_length=_partial_video_length, sample_size=(height_slider, width_slider)) | |
| else: | |
| input_video, input_video_mask, clip_image = get_image_to_video_latent(start_image, None, video_length=_partial_video_length, sample_size=(height_slider, width_slider)) | |
| with torch.no_grad(): | |
| sample = self.pipeline( | |
| prompt_textbox, | |
| negative_prompt = negative_prompt_textbox, | |
| num_inference_steps = sample_step_slider, | |
| guidance_scale = cfg_scale_slider, | |
| width = width_slider, | |
| height = height_slider, | |
| num_frames = _partial_video_length, | |
| generator = generator, | |
| video = input_video, | |
| mask_video = input_video_mask, | |
| clip_image = clip_image | |
| ).videos | |
| if init_frames != 0: | |
| mix_ratio = torch.from_numpy( | |
| np.array([float(_index) / float(overlap_video_length) for _index in range(overlap_video_length)], np.float32) | |
| ).unsqueeze(0).unsqueeze(0).unsqueeze(-1).unsqueeze(-1) | |
| new_sample[:, :, -overlap_video_length:] = new_sample[:, :, -overlap_video_length:] * (1 - mix_ratio) + \ | |
| sample[:, :, :overlap_video_length] * mix_ratio | |
| new_sample = torch.cat([new_sample, sample[:, :, overlap_video_length:]], dim = 2) | |
| sample = new_sample | |
| else: | |
| new_sample = sample | |
| if last_frames >= length_slider: | |
| break | |
| start_image = [ | |
| Image.fromarray( | |
| (sample[0, :, _index].transpose(0, 1).transpose(1, 2) * 255).numpy().astype(np.uint8) | |
| ) for _index in range(-overlap_video_length, 0) | |
| ] | |
| init_frames = init_frames + _partial_video_length - overlap_video_length | |
| last_frames = init_frames + _partial_video_length | |
| else: | |
| if validation_video is not None: | |
| input_video, input_video_mask, clip_image = get_video_to_video_latent(validation_video, length_slider if not is_image else 1, sample_size=(height_slider, width_slider), validation_video_mask=validation_video_mask, fps=16) | |
| strength = denoise_strength | |
| else: | |
| input_video, input_video_mask, clip_image = get_image_to_video_latent(start_image, end_image, length_slider if not is_image else 1, sample_size=(height_slider, width_slider)) | |
| strength = 1 | |
| sample = self.pipeline( | |
| prompt_textbox, | |
| negative_prompt = negative_prompt_textbox, | |
| num_inference_steps = sample_step_slider, | |
| guidance_scale = cfg_scale_slider, | |
| width = width_slider, | |
| height = height_slider, | |
| num_frames = length_slider if not is_image else 1, | |
| generator = generator, | |
| video = input_video, | |
| mask_video = input_video_mask, | |
| clip_image = clip_image | |
| ).videos | |
| else: | |
| sample = self.pipeline( | |
| prompt_textbox, | |
| negative_prompt = negative_prompt_textbox, | |
| num_inference_steps = sample_step_slider, | |
| guidance_scale = cfg_scale_slider, | |
| width = width_slider, | |
| height = height_slider, | |
| num_frames = length_slider if not is_image else 1, | |
| generator = generator | |
| ).videos | |
| else: | |
| input_video, input_video_mask, clip_image = get_video_to_video_latent(control_video, length_slider if not is_image else 1, sample_size=(height_slider, width_slider), fps=16) | |
| sample = self.pipeline( | |
| prompt_textbox, | |
| negative_prompt = negative_prompt_textbox, | |
| num_inference_steps = sample_step_slider, | |
| guidance_scale = cfg_scale_slider, | |
| width = width_slider, | |
| height = height_slider, | |
| num_frames = length_slider if not is_image else 1, | |
| generator = generator, | |
| control_video = input_video, | |
| ).videos | |
| except Exception as e: | |
| self.clear_cache() | |
| if self.lora_model_path != "none": | |
| self.pipeline = unmerge_lora(self.pipeline, self.lora_model_path, multiplier=lora_alpha_slider) | |
| if is_api: | |
| return "", f"Error. error information is {str(e)}" | |
| else: | |
| return gr.update(), gr.update(), f"Error. error information is {str(e)}" | |
| self.clear_cache() | |
| # lora part | |
| if self.lora_model_path != "none": | |
| self.pipeline = unmerge_lora(self.pipeline, self.lora_model_path, multiplier=lora_alpha_slider) | |
| save_sample_path = self.save_outputs( | |
| is_image, length_slider, sample, fps=16 | |
| ) | |
| if is_image or length_slider == 1: | |
| if is_api: | |
| return save_sample_path, "Success" | |
| else: | |
| if gradio_version_is_above_4: | |
| return gr.Image(value=save_sample_path, visible=True), gr.Video(value=None, visible=False), "Success" | |
| else: | |
| return gr.Image.update(value=save_sample_path, visible=True), gr.Video.update(value=None, visible=False), "Success" | |
| else: | |
| if is_api: | |
| return save_sample_path, "Success" | |
| else: | |
| if gradio_version_is_above_4: | |
| return gr.Image(visible=False, value=None), gr.Video(value=save_sample_path, visible=True), "Success" | |
| else: | |
| return gr.Image.update(visible=False, value=None), gr.Video.update(value=save_sample_path, visible=True), "Success" | |
| class Wan_Fun_Controller_Modelscope(Wan_Fun_Controller): | |
| def __init__(self, model_name, model_type, savedir_sample, GPU_memory_mode, scheduler_dict, weight_dtype, config_path): | |
| # Basic dir | |
| self.basedir = os.getcwd() | |
| self.personalized_model_dir = os.path.join(self.basedir, "models", "Personalized_Model") | |
| self.lora_model_path = "none" | |
| self.base_model_path = "none" | |
| self.savedir_sample = savedir_sample | |
| self.scheduler_dict = scheduler_dict | |
| self.config = OmegaConf.load(config_path) | |
| self.refresh_personalized_model() | |
| os.makedirs(self.savedir_sample, exist_ok=True) | |
| # model path | |
| self.model_type = model_type | |
| self.weight_dtype = weight_dtype | |
| self.vae = AutoencoderKLWan.from_pretrained( | |
| os.path.join(model_name, self.config['vae_kwargs'].get('vae_subpath', 'vae')), | |
| additional_kwargs=OmegaConf.to_container(self.config['vae_kwargs']), | |
| ).to(self.weight_dtype) | |
| # Get Transformer | |
| self.transformer = WanTransformer3DModel.from_pretrained( | |
| os.path.join(model_name, self.config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), | |
| transformer_additional_kwargs=OmegaConf.to_container(self.config['transformer_additional_kwargs']), | |
| low_cpu_mem_usage=True, | |
| torch_dtype=self.weight_dtype, | |
| ) | |
| # Get Tokenizer | |
| self.tokenizer = AutoTokenizer.from_pretrained( | |
| os.path.join(model_name, self.config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), | |
| ) | |
| # Get Text encoder | |
| self.text_encoder = WanT5EncoderModel.from_pretrained( | |
| os.path.join(model_name, self.config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), | |
| additional_kwargs=OmegaConf.to_container(self.config['text_encoder_kwargs']), | |
| ).to(self.weight_dtype) | |
| self.text_encoder = self.text_encoder.eval() | |
| if self.transformer.config.in_channels != self.vae.config.latent_channels: | |
| # Get Clip Image Encoder | |
| self.clip_image_encoder = CLIPModel.from_pretrained( | |
| os.path.join(model_name, self.config['image_encoder_kwargs'].get('image_encoder_subpath', 'image_encoder')), | |
| ).to(self.weight_dtype) | |
| self.clip_image_encoder = self.clip_image_encoder.eval() | |
| else: | |
| self.clip_image_encoder = None | |
| Choosen_Scheduler = self.scheduler_dict[list(self.scheduler_dict.keys())[0]] | |
| self.scheduler = Choosen_Scheduler( | |
| **filter_kwargs(Choosen_Scheduler, OmegaConf.to_container(self.config['scheduler_kwargs'])) | |
| ) | |
| # Get pipeline | |
| if self.model_type == "Inpaint": | |
| if self.transformer.config.in_channels != self.vae.config.latent_channels: | |
| self.pipeline = WanFunInpaintPipeline( | |
| vae=self.vae, | |
| tokenizer=self.tokenizer, | |
| text_encoder=self.text_encoder, | |
| transformer=self.transformer, | |
| scheduler=self.scheduler, | |
| clip_image_encoder=self.clip_image_encoder, | |
| ) | |
| else: | |
| self.pipeline = WanFunPipeline( | |
| vae=self.vae, | |
| tokenizer=self.tokenizer, | |
| text_encoder=self.text_encoder, | |
| transformer=self.transformer, | |
| scheduler=self.scheduler, | |
| ) | |
| else: | |
| raise ValueError("Not support now") | |
| if GPU_memory_mode == "sequential_cpu_offload": | |
| replace_parameters_by_name(self.transformer, ["modulation",], device="cuda") | |
| self.transformer.freqs = self.transformer.freqs.to(device="cuda") | |
| self.pipeline.enable_sequential_cpu_offload() | |
| elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": | |
| convert_model_weight_to_float8(self.transformer, exclude_module_name=["modulation",]) | |
| convert_weight_dtype_wrapper(self.transformer, self.weight_dtype) | |
| self.pipeline.enable_model_cpu_offload() | |
| else: | |
| self.pipeline.enable_model_cpu_offload() | |
| print("Update diffusion transformer done") | |
| Wan_Fun_Controller_EAS = Fun_Controller_EAS | |
| def ui(GPU_memory_mode, scheduler_dict, weight_dtype, config_path): | |
| controller = Wan_Fun_Controller(GPU_memory_mode, scheduler_dict, weight_dtype, config_path) | |
| with gr.Blocks(css=css) as demo: | |
| gr.Markdown( | |
| """ | |
| # Wan-Fun: | |
| A Wan with more flexible generation conditions, capable of producing videos of different resolutions, around 6 seconds, and fps 8 (frames 1 to 81), as well as image generated videos. | |
| [Github](https://github.com/aigc-apps/CogVideoX-Fun/) | |
| """ | |
| ) | |
| with gr.Column(variant="panel"): | |
| model_type = create_model_type(visible=True) | |
| diffusion_transformer_dropdown, diffusion_transformer_refresh_button = \ | |
| create_model_checkpoints(controller, visible=True) | |
| base_model_dropdown, lora_model_dropdown, lora_alpha_slider, personalized_refresh_button = \ | |
| create_finetune_models_checkpoints(controller, visible=True) | |
| with gr.Column(variant="panel"): | |
| prompt_textbox, negative_prompt_textbox = create_prompts(negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走") | |
| with gr.Row(): | |
| with gr.Column(): | |
| sampler_dropdown, sample_step_slider = create_samplers(controller) | |
| resize_method, width_slider, height_slider, base_resolution = create_height_width( | |
| default_height = 480, default_width = 832, maximum_height = 1344, | |
| maximum_width = 1344, | |
| ) | |
| generation_method, length_slider, overlap_video_length, partial_video_length = \ | |
| create_generation_methods_and_video_length( | |
| ["Video Generation", "Image Generation"], | |
| default_video_length=81, | |
| maximum_video_length=81, | |
| ) | |
| image_to_video_col, video_to_video_col, control_video_col, source_method, start_image, template_gallery, end_image, validation_video, validation_video_mask, denoise_strength, control_video = create_generation_method( | |
| ["Text to Video (文本到视频)", "Image to Video (图片到视频)"], prompt_textbox | |
| ) | |
| cfg_scale_slider, seed_textbox, seed_button = create_cfg_and_seedbox(gradio_version_is_above_4) | |
| generate_button = gr.Button(value="Generate (生成)", variant='primary') | |
| result_image, result_video, infer_progress = create_ui_outputs() | |
| model_type.change( | |
| fn=controller.update_model_type, | |
| inputs=[model_type], | |
| outputs=[] | |
| ) | |
| def upload_generation_method(generation_method): | |
| if generation_method == "Video Generation": | |
| return [gr.update(visible=True, maximum=81, value=81, interactive=True), gr.update(visible=False), gr.update(visible=False)] | |
| elif generation_method == "Image Generation": | |
| return [gr.update(minimum=1, maximum=1, value=1, interactive=False), gr.update(visible=False), gr.update(visible=False)] | |
| else: | |
| return [gr.update(visible=True, maximum=1344), gr.update(visible=True), gr.update(visible=True)] | |
| generation_method.change( | |
| upload_generation_method, generation_method, [length_slider, overlap_video_length, partial_video_length] | |
| ) | |
| def upload_source_method(source_method): | |
| if source_method == "Text to Video (文本到视频)": | |
| return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=False), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None)] | |
| elif source_method == "Image to Video (图片到视频)": | |
| return [gr.update(visible=True), gr.update(visible=False), gr.update(visible=False), gr.update(), gr.update(), gr.update(value=None), gr.update(value=None), gr.update(value=None)] | |
| elif source_method == "Video to Video (视频到视频)": | |
| return [gr.update(visible=False), gr.update(visible=True), gr.update(visible=False), gr.update(value=None), gr.update(value=None), gr.update(), gr.update(), gr.update(value=None)] | |
| else: | |
| return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=True), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update()] | |
| source_method.change( | |
| upload_source_method, source_method, [ | |
| image_to_video_col, video_to_video_col, control_video_col, start_image, end_image, | |
| validation_video, validation_video_mask, control_video | |
| ] | |
| ) | |
| def upload_resize_method(resize_method): | |
| if resize_method == "Generate by": | |
| return [gr.update(visible=True), gr.update(visible=True), gr.update(visible=False)] | |
| else: | |
| return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=True)] | |
| resize_method.change( | |
| upload_resize_method, resize_method, [width_slider, height_slider, base_resolution] | |
| ) | |
| generate_button.click( | |
| fn=controller.generate, | |
| inputs=[ | |
| diffusion_transformer_dropdown, | |
| base_model_dropdown, | |
| lora_model_dropdown, | |
| lora_alpha_slider, | |
| prompt_textbox, | |
| negative_prompt_textbox, | |
| sampler_dropdown, | |
| sample_step_slider, | |
| resize_method, | |
| width_slider, | |
| height_slider, | |
| base_resolution, | |
| generation_method, | |
| length_slider, | |
| overlap_video_length, | |
| partial_video_length, | |
| cfg_scale_slider, | |
| start_image, | |
| end_image, | |
| validation_video, | |
| validation_video_mask, | |
| control_video, | |
| denoise_strength, | |
| seed_textbox, | |
| ], | |
| outputs=[result_image, result_video, infer_progress] | |
| ) | |
| return demo, controller | |
| def ui_modelscope(model_name, model_type, savedir_sample, GPU_memory_mode, scheduler_dict, weight_dtype, config_path): | |
| controller = Wan_Fun_Controller_Modelscope(model_name, model_type, savedir_sample, GPU_memory_mode, scheduler_dict, weight_dtype, config_path) | |
| with gr.Blocks(css=css) as demo: | |
| gr.Markdown( | |
| """ | |
| # Wan-Fun: | |
| A Wan with more flexible generation conditions, capable of producing videos of different resolutions, around 6 seconds, and fps 8 (frames 1 to 81), as well as image generated videos. | |
| [Github](https://github.com/aigc-apps/CogVideoX-Fun/) | |
| """ | |
| ) | |
| with gr.Column(variant="panel"): | |
| model_type = create_fake_model_type(visible=True) | |
| diffusion_transformer_dropdown = create_fake_model_checkpoints(model_name, visible=True) | |
| base_model_dropdown, lora_model_dropdown, lora_alpha_slider = create_fake_finetune_models_checkpoints(visible=True) | |
| with gr.Column(variant="panel"): | |
| prompt_textbox, negative_prompt_textbox = create_prompts(negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走") | |
| with gr.Row(): | |
| with gr.Column(): | |
| sampler_dropdown, sample_step_slider = create_samplers(controller) | |
| resize_method, width_slider, height_slider, base_resolution = create_height_width( | |
| default_height = 480, default_width = 832, maximum_height = 1344, | |
| maximum_width = 1344, | |
| ) | |
| generation_method, length_slider, overlap_video_length, partial_video_length = \ | |
| create_generation_methods_and_video_length( | |
| ["Video Generation", "Image Generation"], | |
| default_video_length=81, | |
| maximum_video_length=81, | |
| ) | |
| image_to_video_col, video_to_video_col, control_video_col, source_method, start_image, template_gallery, end_image, validation_video, validation_video_mask, denoise_strength, control_video = create_generation_method( | |
| ["Text to Video (文本到视频)", "Image to Video (图片到视频)"], prompt_textbox | |
| ) | |
| cfg_scale_slider, seed_textbox, seed_button = create_cfg_and_seedbox(gradio_version_is_above_4) | |
| generate_button = gr.Button(value="Generate (生成)", variant='primary') | |
| result_image, result_video, infer_progress = create_ui_outputs() | |
| def upload_generation_method(generation_method): | |
| if generation_method == "Video Generation": | |
| return gr.update(visible=True, minimum=1, maximum=81, value=81, interactive=True) | |
| elif generation_method == "Image Generation": | |
| return gr.update(minimum=1, maximum=1, value=1, interactive=False) | |
| generation_method.change( | |
| upload_generation_method, generation_method, [length_slider] | |
| ) | |
| def upload_source_method(source_method): | |
| if source_method == "Text to Video (文本到视频)": | |
| return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=False), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None)] | |
| elif source_method == "Image to Video (图片到视频)": | |
| return [gr.update(visible=True), gr.update(visible=False), gr.update(visible=False), gr.update(), gr.update(), gr.update(value=None), gr.update(value=None), gr.update(value=None)] | |
| elif source_method == "Video to Video (视频到视频)": | |
| return [gr.update(visible=False), gr.update(visible=True), gr.update(visible=False), gr.update(value=None), gr.update(value=None), gr.update(), gr.update(), gr.update(value=None)] | |
| else: | |
| return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=True), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update()] | |
| source_method.change( | |
| upload_source_method, source_method, [ | |
| image_to_video_col, video_to_video_col, control_video_col, start_image, end_image, | |
| validation_video, validation_video_mask, control_video | |
| ] | |
| ) | |
| def upload_resize_method(resize_method): | |
| if resize_method == "Generate by": | |
| return [gr.update(visible=True), gr.update(visible=True), gr.update(visible=False)] | |
| else: | |
| return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=True)] | |
| resize_method.change( | |
| upload_resize_method, resize_method, [width_slider, height_slider, base_resolution] | |
| ) | |
| generate_button.click( | |
| fn=controller.generate, | |
| inputs=[ | |
| diffusion_transformer_dropdown, | |
| base_model_dropdown, | |
| lora_model_dropdown, | |
| lora_alpha_slider, | |
| prompt_textbox, | |
| negative_prompt_textbox, | |
| sampler_dropdown, | |
| sample_step_slider, | |
| resize_method, | |
| width_slider, | |
| height_slider, | |
| base_resolution, | |
| generation_method, | |
| length_slider, | |
| overlap_video_length, | |
| partial_video_length, | |
| cfg_scale_slider, | |
| start_image, | |
| end_image, | |
| validation_video, | |
| validation_video_mask, | |
| control_video, | |
| denoise_strength, | |
| seed_textbox, | |
| ], | |
| outputs=[result_image, result_video, infer_progress] | |
| ) | |
| return demo, controller | |
| def ui_eas(model_name, scheduler_dict, savedir_sample, config_path): | |
| controller = Wan_Fun_Controller_EAS(model_name, scheduler_dict, savedir_sample) | |
| with gr.Blocks(css=css) as demo: | |
| gr.Markdown( | |
| """ | |
| # Wan-Fun: | |
| A Wan with more flexible generation conditions, capable of producing videos of different resolutions, around 6 seconds, and fps 8 (frames 1 to 81), as well as image generated videos. | |
| [Github](https://github.com/aigc-apps/CogVideoX-Fun/) | |
| """ | |
| ) | |
| with gr.Column(variant="panel"): | |
| diffusion_transformer_dropdown = create_fake_model_checkpoints(model_name, visible=True) | |
| base_model_dropdown, lora_model_dropdown, lora_alpha_slider = create_fake_finetune_models_checkpoints(visible=True) | |
| with gr.Column(variant="panel"): | |
| prompt_textbox, negative_prompt_textbox = create_prompts(negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走") | |
| with gr.Row(): | |
| with gr.Column(): | |
| sampler_dropdown, sample_step_slider = create_samplers(controller, maximum_step=40) | |
| resize_method, width_slider, height_slider, base_resolution = create_fake_height_width( | |
| default_height = 480, default_width = 832, maximum_height = 1344, | |
| maximum_width = 1344, | |
| ) | |
| generation_method, length_slider, overlap_video_length, partial_video_length = \ | |
| create_generation_methods_and_video_length( | |
| ["Video Generation", "Image Generation"], | |
| default_video_length=29, | |
| maximum_video_length=29, | |
| ) | |
| image_to_video_col, video_to_video_col, control_video_col, source_method, start_image, template_gallery, end_image, validation_video, validation_video_mask, denoise_strength, control_video = create_generation_method( | |
| ["Text to Video (文本到视频)", "Image to Video (图片到视频)"], prompt_textbox | |
| ) | |
| cfg_scale_slider, seed_textbox, seed_button = create_cfg_and_seedbox(gradio_version_is_above_4) | |
| generate_button = gr.Button(value="Generate (生成)", variant='primary') | |
| result_image, result_video, infer_progress = create_ui_outputs() | |
| def upload_generation_method(generation_method): | |
| if generation_method == "Video Generation": | |
| return gr.update(visible=True, minimum=5, maximum=29, value=29, interactive=True) | |
| elif generation_method == "Image Generation": | |
| return gr.update(minimum=1, maximum=1, value=1, interactive=False) | |
| generation_method.change( | |
| upload_generation_method, generation_method, [length_slider] | |
| ) | |
| def upload_source_method(source_method): | |
| if source_method == "Text to Video (文本到视频)": | |
| return [gr.update(visible=False), gr.update(visible=False), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None)] | |
| elif source_method == "Image to Video (图片到视频)": | |
| return [gr.update(visible=True), gr.update(visible=False), gr.update(), gr.update(), gr.update(value=None), gr.update(value=None)] | |
| else: | |
| return [gr.update(visible=False), gr.update(visible=True), gr.update(value=None), gr.update(value=None), gr.update(), gr.update()] | |
| source_method.change( | |
| upload_source_method, source_method, [image_to_video_col, video_to_video_col, start_image, end_image, validation_video, validation_video_mask] | |
| ) | |
| def upload_resize_method(resize_method): | |
| if resize_method == "Generate by": | |
| return [gr.update(visible=True), gr.update(visible=True), gr.update(visible=False)] | |
| else: | |
| return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=True)] | |
| resize_method.change( | |
| upload_resize_method, resize_method, [width_slider, height_slider, base_resolution] | |
| ) | |
| generate_button.click( | |
| fn=controller.generate, | |
| inputs=[ | |
| diffusion_transformer_dropdown, | |
| base_model_dropdown, | |
| lora_model_dropdown, | |
| lora_alpha_slider, | |
| prompt_textbox, | |
| negative_prompt_textbox, | |
| sampler_dropdown, | |
| sample_step_slider, | |
| resize_method, | |
| width_slider, | |
| height_slider, | |
| base_resolution, | |
| generation_method, | |
| length_slider, | |
| cfg_scale_slider, | |
| start_image, | |
| end_image, | |
| validation_video, | |
| validation_video_mask, | |
| denoise_strength, | |
| seed_textbox, | |
| ], | |
| outputs=[result_image, result_video, infer_progress] | |
| ) | |
| return demo, controller |