up

combined_pipe.py

@@ -0,0 +1,4 @@
+#!/usr/bin/env python3
+from diffusers import KandinskyV22CombinedPipeline
+
+pipe = KandinskyV22CombinedPipeline.from_pretrained("kandinsky-community/kandinsky-2-2-decoder")

control_net_canny.py

@@ -12,43 +12,75 @@ from diffusers import (
     ControlNetModel,
     EulerDiscreteScheduler,
     StableDiffusionControlNetPipeline,
+    StableDiffusionXLControlNetPipeline,
     UniPCMultistepScheduler,
 )
 import sys
 
 checkpoint = sys.argv[1]
 
-image = load_image(
-    "https://huggingface.co/lllyasviel/sd-controlnet-canny/resolve/main/images/bird.png"
-)
-image = np.array(image)
+prompts = [
+    "beautiful room",
+    "a photo-realistic image of two paradise birds",
+    "a snowy house behind a forest",
+    "a couple watching a romantic sunset",
+    "boats in the Amazonas",
+    "a photo of a beautiful face of a woman",
+    "a skater in Brooklyn",
+    "a tornado in Iowa"
+]
 
-low_threshold = 100
-high_threshold = 200
+sd_xl = "control_v11p" not in checkpoint
 
-image = cv2.Canny(image, low_threshold, high_threshold)
-image = image[:, :, None]
-image = np.concatenate([image, image, image], axis=2)
-canny_image = Image.fromarray(image)
+if sd_xl:
+    base_ckpt = "stabilityai/stable-diffusion-xl-base-0.9"
+    controlnet = ControlNetModel.from_pretrained(checkpoint, torch_dtype=torch.float16)
+    pipe = StableDiffusionXLControlNetPipeline.from_pretrained(
+        base_ckpt, controlnet=controlnet, torch_dtype=torch.float16
+    )
+    size = 1024
+else:
+    base_ckpt = "runwayml/stable-diffusion-v1-5"
+    controlnet = ControlNetModel.from_pretrained(checkpoint, torch_dtype=torch.float16)
+    pipe = StableDiffusionControlNetPipeline.from_pretrained(
+        base_ckpt, controlnet=controlnet, torch_dtype=torch.float16
+    )
+    size = 512
 
-controlnet = ControlNetModel.from_pretrained(checkpoint, torch_dtype=torch.float16)
-pipe = StableDiffusionControlNetPipeline.from_pretrained(
-    "stabilityai/stable-diffusion-xl-base-0.9", controlnet=[controlnet, controlnet], torch_dtype=torch.float16
-)
-pipe.enable_model_cpu_offload()
+# pipe.enable_model_cpu_offload()
+pipe.to("cuda")
 
-generator = torch.manual_seed(33)
-out_image = pipe("a blue paradise bird in the jungle", generator=generator, image=[canny_image, canny_image]).images[0]
 
-path = os.path.join(Path.home(), "images", "aa.png")
-out_image.save(path)
+for i in range(8):
+    for seed in range(4):
+        image = load_image(
+            f"https://huggingface.co/datasets/patrickvonplaten/webdatasets_images/resolve/main/image_{i}.png"
+        )
+        image = image.resize((size, size))
+        prompt = prompts[i]
+        image = np.array(image)
 
-api = HfApi()
+        low_threshold = 100
+        high_threshold = 200
 
-api.upload_file(
-    path_or_fileobj=path,
-    path_in_repo=path.split("/")[-1],
-    repo_id="patrickvonplaten/images",
-    repo_type="dataset",
-)
-print("https://huggingface.co/datasets/patrickvonplaten/images/blob/main/aa.png")
+        image = cv2.Canny(image, low_threshold, high_threshold)
+        image = image[:, :, None]
+        image = np.concatenate([image, image, image], axis=2)
+        canny_image = Image.fromarray(image)
+
+        generator = torch.manual_seed(seed)
+        out_image = pipe(prompt, generator=generator, num_inference_steps=20, image=canny_image, controlnet_conditioning_scale=1.0).images[0]
+
+        path = os.path.join(Path.home(), "images", "control_sdxl", f"{i}_{seed}.png")
+        path_in_repo = "/".join(path.split("/")[-2:])
+        out_image.save(path)
+
+        api = HfApi()
+
+        api.upload_file(
+            path_or_fileobj=path,
+            path_in_repo=path_in_repo,
+            repo_id="patrickvonplaten/images",
+            repo_type="dataset",
+        )
+        print(f"https://huggingface.co/datasets/patrickvonplaten/images/blob/main/control_sdxl/{i}_{seed}.png")

run_xl_ediffi.py

@@ -18,40 +18,31 @@ from torch.nn.functional import fractional_max_pool2d_with_indices
 api = HfApi()
 start_time = time.time()
 
-scheduler = DPMSolverMultistepScheduler(
-    beta_start=0.00085,
-    beta_end=0.012,
-    beta_schedule="scaled_linear",
-    prediction_type="epsilon",
-    num_train_timesteps=1000,
-    trained_betas=None,
-    thresholding=False,
-    algorithm_type="dpmsolver++",
-    solver_type="midpoint",
-    lower_order_final=True,
-    use_karras_sigmas=True,
-)
+model_id = "stabilityai/stable-diffusion-xl-base-0.9"
+scheduler = DPMSolverMultistepScheduler.from_pretrained(model_id, subfolder="scheduler")
 
 model_id = "stabilityai/stable-diffusion-xl-base-0.9"
 pipe_high_noise = DiffusionPipeline.from_pretrained(model_id, torch_dtype=torch.float16, variant="fp16", use_safetensors=True, local_files_only=True)
-pipe_high_noise.scheduler = scheduler
+# pipe_high_noise.scheduler = scheduler
 pipe_high_noise.to("cuda")
 
 pipe_low_noise = DiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-xl-refiner-0.9", torch_dtype=torch.float16, use_safetensors=True, variant="fp16")
-pipe_low_noise.scheduler = scheduler
+# pipe_low_noise.scheduler = scheduler
 pipe_low_noise.to("cuda")
 
 prompt = "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k"
 
-
 random_generator = torch.Generator()
 random_generator.manual_seed(0)
 
-num_inference_steps = 100
+num_inference_steps = 40
 high_noise_frac = 0.8
 
-image = pipe_high_noise(prompt=prompt, num_inference_steps=num_inference_steps, denoising_end=high_noise_frac, output_type="latent").images
-image = pipe_low_noise(prompt=prompt, num_inference_steps=num_inference_steps, denoising_start=high_noise_frac, image=image).images[0]
+image = pipe_high_noise(prompt=prompt, num_inference_steps=num_inference_steps, num_images_per_prompt=2, denoising_end=high_noise_frac, output_type="latent").images
+images = pipe_low_noise(prompt=prompt, num_inference_steps=num_inference_steps, num_images_per_prompt=2, denoising_start=high_noise_frac, image=image).images
+
+print(len(images))
+image = images[1]
 
 file_name = f"aaa_1"
 path = os.path.join(Path.home(), "images", "ediffi_sdxl", f"{file_name}.png")