init

app.py

@@ -0,0 +1,51 @@
+import gradio as gr
+import torch
+from diffusers import StableDiffusionControlNetPipeline, ControlNetModel, UniPCMultistepScheduler
+
+controlnet = ControlNetModel.from_pretrained(
+    "williamberman/controlnet-model-3-12-learning-rates", 
+    torch_dtype=torch.float16
+)
+
+pipe = StableDiffusionControlNetPipeline.from_pretrained(
+    "runwayml/stable-diffusion-v1-5", 
+    controlnet=controlnet,
+    safety_checker=None,
+    torch_dtype=torch.float16, 
+)
+pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)
+pipe.enable_xformers_memory_efficient_attention()
+pipe.enable_model_cpu_offload()
+
+def inference(prompt, image, seed=-1):
+    if seed == -1:
+        generator = None
+    else:
+        generator = torch.Generator().manual_seed(seed)
+
+    image = pipe(prompt, image, num_inference_steps=20, generator=generator).images[0]
+
+    return image
+
+io = gr.Interface(
+  inference,
+  inputs = [
+    gr.Textbox(lines=3, label="Prompt"),
+    gr.Image(label="Controlnet conditioning", type="pil"),
+    gr.Number(-1, label="Seed", precision=0),
+  ],
+  outputs=[
+    gr.Image(type="pil"),
+  ],
+  examples=[
+    ["red circle with blue background", "images/0.png", 0],
+    ["cyan circle with brown floral background", "images/1.png", 0],
+    ["light coral circle with white background", "images/2.png", 0],
+    ["cornflower blue circle with light golden rod yellow background", "images/3.png", 0],
+    ["light slate gray circle with blue background", "images/4.png", 0],
+    ["light golden rod yellow circle with turquoise background", "images/5.png", 0],
+  ],
+  title="fill50k controlnet",
+  cache_examples=True,
+)
+io.launch(share=True)

requirements.txt

@@ -0,0 +1,6 @@
+--extra-index-url https://download.pytorch.org/whl/cu113
+torch
+transformers
+accelerate
+git+https://github.com/huggingface/diffusers.git
+xformers