Duplicate from ikechan8370/cp-extra

Co-authored-by: Yinchi Ge <[email protected]>

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.idea/.gitignore

@@ -0,0 +1,8 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Editor-based HTTP Client requests
+/httpRequests/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml

Dockerfile

@@ -0,0 +1,29 @@
+FROM python:3.9
+ENV HOME=/home/user \
+	PATH=/home/user/.local/bin:$PATH
+
+RUN apt-get update && apt-get install  ffmpeg libsm6 libxext6 -y
+
+RUN mkdir -p $HOME/app/.cache
+RUN useradd -m -u 1000 user
+RUN chown -R user $HOME/
+RUN chmod 777 $HOME/app/.cache
+
+USER user
+WORKDIR $HOME/app
+
+ENV TRANSFORMERS_CACHE=$HOME/app/.cache \
+    HUGGINGFACE_HUB_CACHE=$HOME/app/.cache
+
+COPY --chown=user . $HOME/app
+
+
+
+
+
+
+RUN pip install --no-cache-dir --upgrade -r $HOME/app/requirements.txt
+
+
+
+CMD ["python", "main.py"]

README.md

@@ -0,0 +1,43 @@
+---
+title: Yunzai-chatgpt-plugin-extras
+emoji: 🥑
+colorFrom: green
+colorTo: yellow
+sdk: docker
+sdk_version: 3.9
+app_file: main.py
+pinned: true
+app_port: 5000
+duplicated_from: ikechan8370/cp-extra
+---
+
+# chatgpt-plugin-extras
+
+chatgpt-plugin一些花活额外工具
+
+## 安装
+
+`pip install -r requirements.txt`
+
+`mkdir -p data/upload`
+
+`python main.py`
+
+运行在5000端口。目前默认使用CPU
+
+## 使用
+
+### ImageCaption
+
+POST http://127.0.0.1:5000/image-captioning
+
+Form-Data \
+file: 图片文件
+
+### Visual QA
+
+POST http://127.0.0.1:5000/visual-qa
+
+Form-Data \
+file: 图片文件 \
+q: 问题

main.py

@@ -0,0 +1,177 @@
+import os
+from uuid import uuid4
+
+import torch
+from PIL import Image
+from controlnet_aux import HEDdetector
+from diffusers import StableDiffusionInstructPix2PixPipeline, EulerAncestralDiscreteScheduler
+from flask import Flask, request, send_file
+from transformers import BlipProcessor, BlipForConditionalGeneration, BlipForQuestionAnswering
+from transformers import pipeline
+
+app = Flask('chatgpt-plugin-extras')
+
+
+class VitGPT2:
+    def __init__(self, device):
+        print(f"Initializing VitGPT2 ImageCaptioning to {device}")
+        self.pipeline = pipeline("image-to-text", model="nlpconnect/vit-gpt2-image-captioning")
+
+    def inference(self, image_path):
+        captions = self.pipeline(image_path)[0]['generated_text']
+        print(f"\nProcessed ImageCaptioning, Input Image: {image_path}, Output Text: {captions}")
+        return captions
+
+
+class ImageCaptioning:
+    def __init__(self, device):
+        print(f"Initializing ImageCaptioning to {device}")
+        self.device = device
+        self.torch_dtype = torch.float16 if 'cuda' in device else torch.float32
+        self.processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-large")
+        self.model = BlipForConditionalGeneration.from_pretrained(
+            "Salesforce/blip-image-captioning-large", torch_dtype=self.torch_dtype).to(self.device)
+
+    def inference(self, image_path):
+        inputs = self.processor(Image.open(image_path), return_tensors="pt").to(self.device, self.torch_dtype)
+        out = self.model.generate(**inputs)
+        captions = self.processor.decode(out[0], skip_special_tokens=True)
+        print(f"\nProcessed ImageCaptioning, Input Image: {image_path}, Output Text: {captions}")
+        return captions
+
+
+class VQA:
+    def __init__(self, device):
+        print(f"Initializing Visual QA to {device}")
+        self.device = device
+        self.torch_dtype = torch.float16 if 'cuda' in device else torch.float32
+        self.processor = BlipProcessor.from_pretrained("Salesforce/blip-vqa-base")
+        self.model = BlipForQuestionAnswering.from_pretrained("Salesforce/blip-vqa-base",
+                                                              torch_dtype=self.torch_dtype).to(self.device)
+
+    def inference(self, image_path, question):
+        inputs = self.processor(Image.open(image_path), question, return_tensors="pt").to(self.device, self.torch_dtype)
+        out = self.model.generate(**inputs)
+        answers = self.processor.decode(out[0], skip_special_tokens=True)
+        print(f"\nProcessed Visual QA, Input Image: {image_path}, Output Text: {answers}")
+        return answers
+
+
+class Image2Hed:
+    def __init__(self, device):
+        print("Initializing Image2Hed")
+        self.detector = HEDdetector.from_pretrained('lllyasviel/ControlNet')
+
+    def inference(self, inputs, output_filename):
+        output_path = os.path.join('data', output_filename)
+        image = Image.open(inputs)
+        hed = self.detector(image)
+        hed.save(output_path)
+        print(f"\nProcessed Image2Hed, Input Image: {inputs}, Output Hed: {output_path}")
+        return '/result/' + output_filename
+
+
+class Image2Scribble:
+    def __init__(self, device):
+        print("Initializing Image2Scribble")
+        self.detector = HEDdetector.from_pretrained('lllyasviel/ControlNet')
+
+    def inference(self, inputs, output_filename):
+        output_path = os.path.join('data', output_filename)
+        image = Image.open(inputs)
+        hed = self.detector(image, scribble=True)
+        hed.save(output_path)
+        print(f"\nProcessed Image2Hed, Input Image: {inputs}, Output Hed: {output_path}")
+        return '/result/' + output_filename
+
+class InstructPix2Pix:
+    def __init__(self, device):
+        print(f"Initializing InstructPix2Pix to {device}")
+        self.device = device
+        self.torch_dtype = torch.float16 if 'cuda' in device else torch.float32
+        self.pipe = StableDiffusionInstructPix2PixPipeline.from_pretrained("timbrooks/instruct-pix2pix",
+                                                                           safety_checker=None,
+                                                                           torch_dtype=self.torch_dtype).to(device)
+        self.pipe.scheduler = EulerAncestralDiscreteScheduler.from_config(self.pipe.scheduler.config)
+
+    def inference(self, image_path, text, output_filename):
+        """Change style of image."""
+        print("===>Starting InstructPix2Pix Inference")
+        original_image = Image.open(image_path)
+        image = self.pipe(text, image=original_image, num_inference_steps=40, image_guidance_scale=1.2).images[0]
+        output_path = os.path.join('data', output_filename)
+        image.save(output_path)
+
+        print(f"\nProcessed InstructPix2Pix, Input Image: {image_path}, Instruct Text: {text}, "
+              f"Output Image: {output_path}")
+        return '/result/' + output_path
+
+@app.route('/result/<filename>')
+def get_result(filename):
+    file_path = os.path.join('data', filename)
+    return send_file(file_path, mimetype='image/png')
+
+
+ic = ImageCaptioning("cpu")
+vqa = VQA("cpu")
+i2h = Image2Hed("cpu")
+i2s = Image2Scribble("cpu")
+# vgic = VitGPT2("cpu")
+# ip2p = InstructPix2Pix("cpu")
+
+@app.route('/image2hed', methods=['POST'])
+def imag2hed():
+    file = request.files['file']  # 获取上传的文件
+    filename = str(uuid4()) + '.png'
+    filepath = os.path.join('data', 'upload', filename)
+    file.save(filepath)
+    output_filename = str(uuid4()) + '.png'
+    result = i2h.inference(filepath, output_filename)
+    return result
+
+
+@app.route('/image2Scribble', methods=['POST'])
+def image2Scribble():
+    file = request.files['file']  # 获取上传的文件
+    filename = str(uuid4()) + '.png'
+    filepath = os.path.join('data', 'upload', filename)
+    file.save(filepath)
+    output_filename = str(uuid4()) + '.png'
+    result = i2s.inference(filepath, output_filename)
+    return result
+
+
+@app.route('/image-captioning', methods=['POST'])
+def image_caption():
+    file = request.files['file']  # 获取上传的文件
+    filename = str(uuid4()) + '.png'
+    filepath = os.path.join('data', 'upload', filename)
+    file.save(filepath)
+    # result1 = vgic.inference(filepath)
+    result2 = ic.inference(filepath)
+    return result2
+
+
+@app.route('/visual-qa', methods=['POST'])
+def visual_qa():
+    file = request.files['file']  # 获取上传的文件
+    filename = str(uuid4()) + '.png'
+    filepath = os.path.join('data', 'upload', filename)
+    file.save(filepath)
+    question = request.args.get('q')
+    result = vqa.inference(filepath, question=question)
+    return result
+
+@app.route('/instruct-pix2pix', methods=['POST'])
+def InstructPix2Pix():
+    file = request.files['file']  # 获取上传的文件
+    filename = str(uuid4()) + '.png'
+    filepath = os.path.join('data', 'upload', filename)
+    file.save(filepath)
+    output_filename = str(uuid4()) + '.png'
+    question = request.args.get('t')
+    result = ip2p.inference(filepath, question, output_filename)
+    return result
+
+if __name__ == '__main__':
+    app.run(host='0.0.0.0')

requirements.txt

@@ -0,0 +1,7 @@
+torch~=2.0.1
+transformers~=4.30.2
+Pillow~=9.5.0
+Flask~=2.3.2
+controlnet_aux==0.0.5
+matplotlib
+diffusers==0.17.1