Update app.py

app.py

@@ -24,7 +24,10 @@ from lxml.html import fromstring
 from diffusers.utils import export_to_gif, load_image
 from huggingface_hub import hf_hub_download
 from safetensors.torch import load_file, save_file
-from diffusers import AnimateDiffPipeline, MotionAdapter, DDIMScheduler, StableDiffusionPipeline
+from diffusers import StableDiffusionPipeline
+from diffusers import AnimateDiffSparseControlNetPipeline
+from diffusers.models import AutoencoderKL, MotionAdapter, SparseControlNetModel
+from diffusers.schedulers import DPMSolverMultistepScheduler
 from PIL import Image, ImageDraw, ImageFont
 
 # logging
@@ -49,6 +52,10 @@ else:
 
 #base = "emilianJR/epiCRealism"
 base = "SG161222/Realistic_Vision_V5.1_noVAE"
+controlnet_id = "guoyww/animatediff-sparsectrl-scribble"
+lora_adapter_id = "guoyww/animatediff-motion-lora-v1-5-3"
+vae_id = "stabilityai/sd-vae-ft-mse"
+motion_adapter = "guoyww/animatediff-motion-adapter-v1-5-3"
 
 # variable data
 
@@ -108,9 +115,27 @@ function custom(){
 
 # torch pipes
 
-adapter = MotionAdapter.from_pretrained("guoyww/animatediff-motion-adapter-v1-5-2", torch_dtype=dtype, device=device, safety_checker=None)
 image_pipe = StableDiffusionPipeline.from_pretrained(base, torch_dtype=dtype, safety_checker=None).to(device)
-pipe = AnimateDiffPipeline.from_pretrained(base, torch_dtype=dtype, motion_adapter=adapter).to(device)
+
+adapter = MotionAdapter.from_pretrained(motion_adapter, torch_dtype=dtype, safety_checker=None).to(device)
+controlnet = SparseControlNetModel.from_pretrained(controlnet_id, torch_dtype=torch.float16).to(device)
+vae = AutoencoderKL.from_pretrained(vae_id, torch_dtype=torch.float16).to(device)
+scheduler = DPMSolverMultistepScheduler.from_pretrained(
+    model_id,
+    subfolder="scheduler",
+    beta_schedule="linear",
+    algorithm_type="dpmsolver++",
+    use_karras_sigmas=True,
+)
+pipe = AnimateDiffSparseControlNetPipeline.from_pretrained(
+    model_id,
+    motion_adapter=motion_adapter,
+    controlnet=controlnet,
+    vae=vae,
+    scheduler=scheduler,
+    torch_dtype=torch.float16,
+).to(device)
+
 pipe.scheduler = DDIMScheduler.from_pretrained(
     base,
     subfolder="scheduler",
@@ -303,14 +328,14 @@ def ui():
                             container=False,
                             choices=[
                                 ("(No Effect)", ""),
-                                ("Zoom in", "guoyww/#animatediff-motion-lora-zoom-in"),
-                                ("Zoom out", "guoyww/#animatediff-motion-lora-zoom-out"),
-                                ("Tilt up", "guoyww/#animatediff-motion-lora-tilt-up"),
-                                ("Tilt down", "guoyww/#animatediff-motion-lora-tilt-down"),
-                                ("Pan left", "guoyww/#animatediff-motion-lora-pan-left"),
-                                ("Pan right", "guoyww/#animatediff-motion-lora-pan-right"),
-                                ("Roll left", "guoyww/#animatediff-motion-lora-rolling-anticlockwise"),
-                                ("Roll right", "guoyww/#animatediff-motion-lora-rolling-clockwise"),
+                                ("Zoom in", "guoyww/animatediff-motion-lora-zoom-in"),
+                                ("Zoom out", "guoyww/animatediff-motion-lora-zoom-out"),
+                                ("Tilt up", "guoyww/animatediff-motion-lora-tilt-up"),
+                                ("Tilt down", "guoyww/animatediff-motion-lora-tilt-down"),
+                                ("Pan left", "guoyww/animatediff-motion-lora-pan-left"),
+                                ("Pan right", "guoyww/animatediff-motion-lora-pan-right"),
+                                ("Roll left", "guoyww/animatediff-motion-lora-rolling-anticlockwise"),
+                                ("Roll right", "guoyww/animatediff-motion-lora-rolling-clockwise"),
                             ],
                             value="",
                             interactive=True