adapt controlnet script

1

@@ -1,66 +0,0 @@
-#!/usr/bin/env python3
-from diffusers import DiffusionPipeline, EulerDiscreteScheduler, StableDiffusionPipeline, KDPM2DiscreteScheduler, StableDiffusionImg2ImgPipeline, HeunDiscreteScheduler, KDPM2AncestralDiscreteScheduler, DDIMScheduler
-from diffusers import StableDiffusionXLPipeline, StableDiffusionXLImg2ImgPipeline
-import time
-from pytorch_lightning import seed_everything
-import os
-from huggingface_hub import HfApi
-# from compel import Compel
-import torch
-import sys
-from pathlib import Path
-import requests
-from PIL import Image
-from io import BytesIO
-
-api = HfApi()
-start_time = time.time()
-
-use_refiner = bool(int(sys.argv[1]))
-use_diffusers = True
-
-if use_diffusers:
-    start_time = time.time()
-    pipe = StableDiffusionXLPipeline.from_pretrained("stabilityai/stable-diffusion-xl-base-0.9", torch_dtype=torch.float16, variant="fp16", use_safetensors=True, local_files_only=True)
-    pipe.to("cuda")
-
-    if use_refiner:
-        refiner = StableDiffusionXLImg2ImgPipeline.from_pretrained("stabilityai/stable-diffusion-xl-refiner-0.9", torch_dtype=torch.float16, use_safetensors=True, variant="fp16")
-        refiner.to("cuda")
-        # refiner.enable_sequential_cpu_offload()
-else:
-    pipe = StableDiffusionXLPipeline.from_single_file("https://huggingface.co/stabilityai/stable-diffusion-xl-base-0.9/blob/main/sd_xl_base_0.9.safetensors", torch_dtype=torch.float16, use_safetensors=True)
-    pipe.to("cuda")
-
-    if use_refiner:
-        refiner = StableDiffusionXLImg2ImgPipeline.from_single_file("https://huggingface.co/stabilityai/stable-diffusion-xl-refiner-0.9/blob/main/sd_xl_refiner_0.9.safetensors", torch_dtype=torch.float16, use_safetensors=True)
-        refiner.to("cuda")
-
-
-prompt = "An astronaut riding a green horse on Mars"
-for steps in [24, 27, 31]:
-    for denoising_end in [0.63, 0.66, 0.67, 0.71]:
-        seed = 0
-        seed_everything(seed)
-        image = pipe(prompt=prompt, num_inference_steps=40, denoising_end=0.675, output_type="latent" if use_refiner else "pil").images[0]
-# image = pipe(prompt=prompt, output_type="latent" if use_refiner else "pil").images[0]
-
-        if use_refiner:
-            image = refiner(prompt=prompt, num_inference_steps=40, denoising_start=0.675, image=image[None, :]).images[0]
-
-# pipe.unet.to(memory_format=torch.channels_last)
-# pipe(prompt=prompt, num_inference_steps=2).images[0]
-
-# image = pipe(prompt=prompt, num_images_per_prompt=1, num_inference_steps=40, output_type="latent").images
-
-        file_name = f"aaa_{seed}"
-        path = os.path.join(Path.home(), "images", "ediffi_sdxl", f"{file_name}.png")
-        image.save(path)
-
-        api.upload_file(
-            path_or_fileobj=path,
-            path_in_repo=path.split("/")[-1],
-            repo_id="patrickvonplaten/images",
-            repo_type="dataset",
-        )
-        print(f"https://huggingface.co/datasets/patrickvonplaten/images/blob/main/ediffi_sdxl/{file_name}.png")

control_net_canny.py

@@ -33,14 +33,12 @@ canny_image = Image.fromarray(image)
 
 controlnet = ControlNetModel.from_pretrained(checkpoint, torch_dtype=torch.float16)
 pipe = StableDiffusionControlNetPipeline.from_pretrained(
-    "runwayml/stable-diffusion-v1-5", controlnet=[controlnet, controlnet], torch_dtype=torch.float16
+    "stabilityai/stable-diffusion-xl-base-0.9", controlnet=[controlnet, controlnet], torch_dtype=torch.float16
 )
-
-pipe.scheduler = EulerDiscreteScheduler.from_config(pipe.scheduler.config)
 pipe.enable_model_cpu_offload()
 
 generator = torch.manual_seed(33)
-out_image = pipe("a blue paradise bird in the jungle", control_guidance_start=[0.2, 0.2], num_inference_steps=20, generator=generator, image=[canny_image, canny_image]).images[0]
+out_image = pipe("a blue paradise bird in the jungle", generator=generator, image=[canny_image, canny_image]).images[0]
 
 path = os.path.join(Path.home(), "images", "aa.png")
 out_image.save(path)