Update app.py

app.py

@@ -29,13 +29,13 @@ device = "cuda"
 dtype = torch.float16
 result=[]
 step = 2
-repo = "ByteDance/AnimateDiff-Lightning"
-ckpt = f"animatediff_lightning_{step}step_diffusers.safetensors"
+#repo = "ByteDance/AnimateDiff-Lightning"
+#ckpt = f"animatediff_lightning_{step}step_diffusers.safetensors"
 #base = "emilianJR/epiCRealism"
 base = "SG161222/Realistic_Vision_V6.0_B1_noVAE"
-#adapter = MotionAdapter.from_pretrained("guoyww/animatediff-motion-adapter-v1-5-3")
 vae = AutoencoderKL.from_single_file("https://huggingface.co/stabilityai/sd-vae-ft-mse-original/vae-ft-mse-840000-ema-pruned.safetensors")
 #unet = UNet2DConditionModel.from_config("emilianJR/epiCRealism",subfolder="unet").to(device, dtype).load_state_dict(load_file(hf_hub_download("emilianJR/epiCRealism", "unet/diffusion_pytorch_model.safetensors"), device=device), strict=False)
+adapter = MotionAdapter.from_pretrained("guoyww/animatediff-motion-adapter-v1-5-3", torch_dtype=dtype)
 
 css="""
 input, input::placeholder {
@@ -128,10 +128,10 @@ def generate_random_string(length):
     return ''.join(random.choice(characters) for _ in range(length))
 
 @spaces.GPU(duration=65)
-def Piper(positive,negative,motion):
+def Piper(image,positive,negative,motion):
     global last_motion
 
-    pipe.unet.load_state_dict(load_file(hf_hub_download(repo, ckpt), device=device), strict=False)
+    #pipe.unet.load_state_dict(load_file(hf_hub_download(repo, ckpt), device=device), strict=False)
 
     if last_motion != motion:
         pipe.unload_lora_weights()
@@ -145,6 +145,7 @@ def Piper(positive,negative,motion):
         negative_prompt=negative,
         height=896,
         width=896,
+        ip_adapter_image=image,
         num_inference_steps=step,
         guidance_scale=1,
         num_frames=(fps*time)
@@ -162,15 +163,15 @@ def infer(pm):
             _do.append(f'{p1}')
         posi = " ".join(_do)
 
-        out = Piper(posi,neg,pm["m"])
+        out = Piper(pm["i"],posi,neg,pm["m"])
         export_to_gif(out.frames[0],name,fps=fps)
         return name
 
-def run(m,p1,p2,*result):
+def run(i,m,p1,p2,*result):
     
         p1_en = translate(p1,"english")
         p2_en = translate(p2,"english")
-        pm = {"p":p1_en,"n":p2_en,"m":m}
+        pm = {"p":p1_en,"n":p2_en,"m":m,"i":i}
         ln = len(result)
         print("images: "+str(ln))
         rng = list(range(ln))
@@ -184,8 +185,17 @@ def run(m,p1,p2,*result):
 
         return out
 
-pipe = AnimateDiffPipeline.from_pretrained(base, vae=vae, torch_dtype=dtype).to(device)
-pipe.scheduler = EulerDiscreteScheduler.from_config(pipe.scheduler.config, timestep_spacing="trailing", beta_schedule="linear")
+pipe = AnimateDiffPipeline.from_pretrained(base, vae=vae, motion_adapter=adapter, torch_dtype=dtype).to(device)
+pipe.scheduler = EulerDiscreteScheduler(
+    clip_sample=False,
+    beta_start=0.00085,
+    beta_end=0.012,
+    beta_schedule="linear",
+    timestep_spacing="trailing",
+    steps_offset=1
+)
+pipe.load_ip_adapter("h94/IP-Adapter", subfolder="models", weight_name="ip-adapter_sd15.bin")
+pipe.enable_vae_slicing()
 pipe.enable_free_init(method="butterworth", use_fast_sampling=False)
 
 mp.set_start_method("spawn", force=True)
@@ -195,6 +205,8 @@ with gr.Blocks(theme=gr.themes.Soft(),css=css,js=js) as demo:
         gr.Markdown(f"""
             # MULTI-LANGUAGE IMAGE GENERATOR
         """)
+        with gr.Row():
+            img = gr.Image(type="filepath")
         with gr.Row():
             prompt = gr.Textbox(
                 elem_id="prompt",
@@ -235,6 +247,6 @@ with gr.Blocks(theme=gr.themes.Soft(),css=css,js=js) as demo:
                 
     gr.on(
             triggers=[run_button.click, prompt.submit, prompt2.submit],
-            fn=run,inputs=[motion,prompt,prompt2,*result],outputs=result
+            fn=run,inputs=[img,motion,prompt,prompt2,*result],outputs=result
     )
     demo.queue().launch()