Initial commit

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,12 @@
+---
+title: Image Transformation
+emoji: ⚡
+colorFrom: blue
+colorTo: purple
+sdk: gradio
+sdk_version: 3.27.0
+app_file: app.py
+pinned: false
+tags:
+- tool
+---

app.py

@@ -0,0 +1,4 @@
+from transformers.tools.base import launch_gradio_demo
+from image_transformation import ImageTransformationTool
+
+launch_gradio_demo(ImageTransformationTool)

image_transformation.py

@@ -0,0 +1,96 @@
+
+import numpy as np
+import torch
+
+from transformers.tools.base import Tool, get_default_device
+from transformers.utils import (
+    is_accelerate_available,
+    is_diffusers_available,
+    is_opencv_available,
+    is_vision_available,
+)
+
+
+if is_vision_available():
+    from PIL import Image
+
+if is_diffusers_available():
+    from diffusers import ControlNetModel, StableDiffusionControlNetPipeline, UniPCMultistepScheduler
+
+if is_opencv_available():
+    import cv2
+
+
+IMAGE_TRANSFORMATION_DESCRIPTION = (
+    "This is a tool that transforms an image according to a prompt. It takes two inputs: `image`, which should be "
+    "the image to transform, and `prompt`, which should be the prompt to use to change it. It returns the "
+    "modified image."
+)
+
+
+class ImageTransformationTool(Tool):
+    default_stable_diffusion_checkpoint = "runwayml/stable-diffusion-v1-5"
+    default_controlnet_checkpoint = "lllyasviel/sd-controlnet-canny"
+    description = IMAGE_TRANSFORMATION_DESCRIPTION
+
+    def __init__(self, device=None, controlnet=None, stable_diffusion=None, **hub_kwargs) -> None:
+        if not is_accelerate_available():
+            raise ImportError("Accelerate should be installed in order to use tools.")
+        if not is_diffusers_available():
+            raise ImportError("Diffusers should be installed in order to use the StableDiffusionTool.")
+        if not is_vision_available():
+            raise ImportError("Pillow should be installed in order to use the StableDiffusionTool.")
+        if not is_opencv_available():
+            raise ImportError("opencv should be installed in order to use the StableDiffusionTool.")
+
+        super().__init__()
+
+        if controlnet is None:
+            controlnet = self.default_controlnet_checkpoint
+        self.controlnet_checkpoint = controlnet
+
+        if stable_diffusion is None:
+            stable_diffusion = self.default_stable_diffusion_checkpoint
+        self.stable_diffusion_checkpoint = stable_diffusion
+
+        self.device = device
+        self.hub_kwargs = hub_kwargs
+
+    def setup(self):
+        if self.device is None:
+            self.device = get_default_device()
+
+        self.controlnet = ControlNetModel.from_pretrained(self.controlnet_checkpoint, torch_dtype=torch.float16)
+        self.pipeline = StableDiffusionControlNetPipeline.from_pretrained(
+            self.stable_diffusion_checkpoint, controlnet=self.controlnet, torch_dtype=torch.float16
+        )
+        self.pipeline.scheduler = UniPCMultistepScheduler.from_config(self.pipeline.scheduler.config)
+        self.pipeline.enable_model_cpu_offload()
+
+        self.is_initialized = True
+
+    def __call__(self, image, prompt):
+        if not self.is_initialized:
+            self.setup()
+
+        initial_prompt = "super-hero character, best quality, extremely detailed"
+        prompt = initial_prompt + prompt
+
+        low_threshold = 100
+        high_threshold = 200
+
+        image = np.array(image)
+        image = cv2.Canny(image, low_threshold, high_threshold)
+        image = image[:, :, None]
+        image = np.concatenate([image, image, image], axis=2)
+        canny_image = Image.fromarray(image)
+
+        generator = torch.Generator(device="cpu").manual_seed(2)
+
+        return self.pipeline(
+            prompt,
+            canny_image,
+            negative_prompt="monochrome, lowres, bad anatomy, worst quality, low quality",
+            num_inference_steps=20,
+            generator=generator,
+        ).images[0]

requirements.txt

@@ -0,0 +1,5 @@
+transformers @ git+https://github.com/huggingface/transformers@test_composition
+diffusers
+accelerate
+opencv-python
+torch

tool_config.json

@@ -0,0 +1,3 @@
+{
+  "custom_tools": {"image-transformation": "image_transformation.ImageTransformationTool"}
+}