Initial

.gitignore

@@ -0,0 +1,124 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/en/_build/
+docs/zh_cn/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+data/
+data
+.vscode
+.idea
+.DS_Store
+
+# custom
+*.pkl
+*.pkl.json
+*.log.json
+docs/modelzoo_statistics.md
+mmdet/.mim
+work_dirs/
+
+# Pytorch
+*.pth
+*.py~
+*.sh~

LICENSE

@@ -0,0 +1,201 @@
+                                Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2023 Rohan Taori, Ishaan Gulrajani, Tianyi Zhang, Yann Dubois, Xuechen Li
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -1,12 +1,140 @@
----
-title: GPT4Tools
-emoji: 🐠
-colorFrom: red
-colorTo: gray
-sdk: gradio
-sdk_version: 3.28.2
-app_file: app.py
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# GPT4Tools: Teaching LLM to Use Tools via Self-instruction
+
+[Lin Song](http://linsong.info/), [Yanwei Li](https://yanwei-li.com/), [Rui Yang](https://github.com/Yangr116), Sijie Zhao, [Yixiao Ge](https://geyixiao.com/), [Ying Shan](https://scholar.google.com/citations?user=4oXBp9UAAAAJ&hl=en)
+
+GPT4Tools is a centralized system that can control multiple visual foundation models. 
+It is based on Vicuna (LLaMA), and 71K self-built instruction data.
+By analyzing the language content, GPT4Tools is capable of automatically deciding, controlling, and utilizing different visual foundation models, allowing the user to interact with images during a conversation.
+With this approach, GPT4Tools provides a seamless and efficient solution to fulfill various image-related requirements in a conversation.
+Different from previous work, we support users teach their own LLM to use tools with simple refinement via self-instruction and LoRA.
+
+<a href='https://gpt4tools.github.io'><img src='https://img.shields.io/badge/Project-Page-Green'></a>  <a href='https://huggingface.co/stevengrove/gpt4tools-vicuna-13b-lora'><img src='https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Model-blue'></a>  [![YouTube](https://badges.aleen42.com/src/youtube.svg)](https://youtu.be/Qrj94ibQIT8) [![arXiv](https://img.shields.io/badge/arXiv-Paper-<COLOR>.svg)]()
+
+## Updates
+
+* 🔥 We now release pretrained **GPT4Tools** models with <strong><font color="#008AD7">Vicuna-13B</font></strong> and released the dataset for <strong><font color="#008AD7">self-instruction</font></strong>. Check out the [project page](https://gpt4tools.github.io/) and [demo]( https://7db8018df6b6879722.gradio.live).
+
+## Demo
+We provide some selected examples using GPT4Tools in this section. More examples can be found in our [project page](https://gpt4tools.github.io). Feel free to try our onlin [demo](https://7db8018df6b6879722.gradio.live)!
+
+<div align=center>
+<img width="80%" src="demos/demo.gif"/>
+</div>
+
+  |   |   |
+:-------------------------:|:-------------------------:
+![segment](demos/demo_seg.png) |  ![detect kps](demos/demo_kps.png)
+![solve problem](demos/demo_explain.png)  |  ![style transfer](demos/demo_style.png)
+
+## Dataset
+| **Data file name** | **Size** | OneDrive| Google Driver|
+|:------------------:|:--------:| :--------: | :---------:|
+| gpt4tools_71k.json    | 229 MB   | [link](https://1drv.ms/u/s!AqPQkBZ4aeVnhRdryHC9b1NtWJpZ?e=ZHBCqd) | [link](https://drive.google.com/file/d/1JKIT-Or1of7TJuWvmrJpPoOx0cLdcWry/view?usp=share_link)|
+
+```gpt4tools_71k.json``` contains 71K instruction-following data we used for fine-tuning the GPT4Tools model. 
+
+The data collection process is illustrated below:
+
+We fed GPT-3.5 with captions from 3K images and descriptions of 22 visual tasks. This produced 66K instructions, each corresponding to a specific visual task and a visual foundation model (tool). Subsequently, we eliminated duplicate instructions and retained 41K sound instructions. To teach the model to utilize tools in a predefined manner, we followed the prompt format used in Visual ChatGPT and converted these instructions into a conversational format. Concurrently, we generated negative data without tool usage by randomly sampling 3K instructions from [`alpaca_gpt4_data`](https://github.com/Instruction-Tuning-with-GPT-4/GPT-4-LLM/blob/main/data/alpaca_gpt4_data.json) and converting them to the defined format. Using the generated 71K instructions, we finetuned the Vicuna using LoRA and got our GPT4Tools, which can automatically decide, control, and utilize distinct tools in a conversation.
+
+
+## Models
+GTP4Tools mainly contains three parts: LLM for instruction, LoRA for adaptation, and Visual Agent for provided functions.
+It is a flexible and extensible system that can be easily extended to support more tools and functions.
+For example, users can replace the existing LLM or tools with their own models, or add new tools to the system.
+The only things needed are finetuned the LoRA with the provided instruction, which teaches LLM to use the provided tools.
+
+![image](images/overview.png)
+
+### Preparation
+
+
+```
+git clone https://github.com/stevengrove/GPT4Tools
+cd GPT4Tools
+pip install -r requirements.txt
+```
+* If bitsandbytes doesn't work, [install it from source.](https://github.com/TimDettmers/bitsandbytes/blob/main/compile_from_source.md) Windows users can follow [these instructions](https://github.com/tloen/alpaca-lora/issues/17).
+
+
+### Weights
+GPT4Tools is based on the Vicuna, we release the LoRA weights of GPT4Tools to comply with the LLaMA model license. You can merge our LoRA weights with the Vicuna weights to obtain the GPT4Tools weights.
+
+Steps:
+1. Get the original LLaMA weights in the Hugging Face format from [here](https://huggingface.co/docs/transformers/main/model_doc/llama).
+2. Using the [FastChat](https://github.com/lm-sys/FastChat/blob/main/README.md) to get Vicuna weigths by applying [the delta weights](https://huggingface.co/lmsys), more details please check [here](https://github.com/lm-sys/FastChat#vicuna-weights).
+3. Get the LoRA weights of GPT4Tools ([Hugging Face](https://huggingface.co/stevengrove/gpt4tools-vicuna-13b-lora), [OneDrive](https://1drv.ms/f/s!AqPQkBZ4aeVnhRzM69NOXLyG8cTY?e=apmpyQ), or [Google Driver](https://drive.google.com/drive/folders/1ebUINGR0QzNL-4hoKl19-6D_5rfeWTPD?usp=share_link)).
+
+### Tools
+GPT4Tools can support 22 tools, more details please check [tools.md](docs/tools.md).
+When using tools for the first time, the weights of tools need to be downloaded. If you don't like stored them on default cache, please revise the shell environment varibles: 
+```
+export TRANSFORMERS_CACHE=${your_transformers_cache}
+export HUGGINGFACE_HUB_CACHE=${your_diffusers_cache}
+```
+For SAM (Segmenting tools) and GrundingDINO (Text2Box tools):
+```
+export checkpoints=${your_checkpoints_cache} 
+# or
+ln -s ${your_checkpoints_path} checkpoints
+```
+
+### Serving with Web GUI 
+Making a gradio interface on your own devices:
+```
+# Advice for 1 GPU
+python gpt4tools.py \
+	--base_model <path_to_vicuna_with_tokenizer> \
+	--lora_model <path_to_lora_weights> \
+	--llm_device "cpu" \
+	--load "Text2Box_cuda:0,Segmenting_cuda:0,Inpainting_cuda:0,ImageCaptioning_cuda:0"
+```
+
+```
+# Advice for 4 GPUs
+python gpt4tools.py \
+	--base_model <path_to_vicuna_with_tokenizer> \
+	--lora_model <path_to_lora_weights> \
+	--llm_device "cuda:3" \
+	--load "Text2Box_cuda:0,Segmenting_cuda:0,Inpainting_cuda:0,ImageCaptioning_cuda:0,
+		   Text2Image_cuda:1,VisualQuestionAnswering_cuda:1,InstructPix2Pix_cuda:2,
+		   SegText2Image_cuda:2,Image2Pose_cpu,PoseText2Image_cuda:2"
+```
+You can customize the used tools by specifying ```{tools_name}_{devices}``` after args ```--load``` of ```gpt4tools.py```. ```tools_name``` is illustrated in [tools.md](./docs/tools.md).
+
+### Finetuning with LoRA
+
+```
+# Training with 8 GPUs
+torchrun --nproc_per_node=8 --master_port=29005 lora_finetune.py \
+	--base_model <path_to_vicuna_with_tokenizer> \
+	--data_path <path_to_gpt4tools_71k.json> \
+	--output_dir output/gpt4tools \
+	--prompt_template_name gpt4tools \
+	--num_epochs 6 \
+	--batch_size 512 \
+	--cutoff_len 2048 \
+	--group_by_length \
+	--lora_target_modules '[q_proj,k_proj,v_proj,o_proj]' \
+	--lora_r 16 \
+	--micro_batch_size=8
+```
+
+| Hyperparameter | Global Batch Size | Learning rate | Max length | Weight decay | LoRA attention dimension (lora_r) | LoRA scaling alpha(lora_alpha) | LoRA dropout (lora_dropout) | Modules to apply LoRA (lora_target_modules)      |
+|:--------------:|:-----------------:|:-------------:|:----------:|:------------:|:---------------------------------:|:----------:|:------------:|:-----------------------------:|
+|    GPT4Tools & Vicuna-13B   |        512        |      3e-4     |    2048    |      0.0     |                 16                |     16     |     0.05     | [q_proj,k_proj,v_proj,o_proj] |
+
+
+## Acknowledgement
+* [VisualChatGPT](https://github.com/microsoft/TaskMatrix): It connects ChatGPT and a series of Visual Foundation Models to enable sending and receiving images during chatting.
+* [Vicuna](https://github.com/lm-sys/FastChat): The language ability of Vicuna is fantastic and amazing. And it is open-source!
+* [Alpaca-LoRA](https://github.com/tloen/alpaca-lora): Instruct-tune LLaMA on consumer hardware.
+
+If you're using our GPT4Tools in your research or applications, please cite using this BibTeX:
+```
+@misc{gpt4tools,
+  title = {GPT4Tools: Teaching LLM to Use Tools via Self-instruction},
+  author = {Lin Song and Yanwei Li and Rui Yang and Sijie Zhao and Yixiao Ge and Ying Shan},
+  year = {2023},
+}
+```

docs/tools.md

@@ -0,0 +1,36 @@
+GPT4Tools can support 22 tools, including:
+
+| No. |        Tools Name       |                                    Function                                    |
+|-----|:-----------------------:|:------------------------------------------------------------------------------:|
+|  1  |     InstructPix2Pix     | Style the image to be like the text.                                           |
+|  2  |        Text2Image       | Generate an image from an input text.                                          |
+|  3  |     ImageCaptioning     | Describe the input image.                                                      |
+|  4  |       Image2Canny       | Detect the edge of the image                                                   |
+|  5  |     CannyText2Image     | Generate a new real image from both the user description and a canny image.    |
+|  6  |        Image2Line       | Detect the straight line of the image.                                         |
+|  7  |        Image2Hed        | Detect the soft hed boundary of the image.                                     |
+|  8  |      HedText2Image      | Generate a new real image from both the user description.                      |
+|  9  |      Image2Scribble     | Generate a scribble of the image.                                              |
+|  10 |    ScribbleText2Image   | Generate a new real image from both the user description and a scribble image. |
+|  11 |        Image2Pose       | Detect the human pose of the image.                                            |
+|  12 |      PoseText2Image     | Generate a new real image from both the user description.                      |
+|  13 |      SegText2Image      | Generate a new real image from both the user description and segmentations.    |
+|  14 |       Image2Depth       | Detect depth of the image.                                                     |
+|  15 |     DepthText2Image     | Generate a new real image from both the user description and depth image.      |
+|  16 |       Image2Normal      | Detect norm map of the image.                                                  |
+|  17 |     NormalText2Image    | Generate a new real image from both the user description and normal map.       |
+|  18 | VisualQuestionAnswering | Answer for a question based on an image.                                       |
+|  19 |        Segmenting       | Segment all the part of the image.                                             |
+|  20 |         Text2Box        | Detect or find out given objects in the picture.                               |
+|  21 |     ObjectSegmenting    | Segment the certain objects in the picture.                                    |
+|  22 |       ImageEditing      | Remove and object or something from the photo.                                 |
+
+You can customize the used tools by specifying ```{tools_name}_{devices}``` after args ```--load``` of ```gpt4tools.py```. For example, enabling ```Text2Box```, ```Segmenting```, and ```ImageCaptioning```:
+```
+python gpt4tools.py \
+	--base_model <path_to_vicuna_with_tokenizer> \
+	--lora_model <path_to_lora_weights> \
+	--llm_device "cpu" \
+	--load "Text2Box_cuda:0,Segmenting_cuda:0,ImageCaptioning_cuda:0"
+```
+More tools will be supported in the future!

gpt4tools.py

@@ -0,0 +1,1224 @@
+# coding: utf-8
+import os
+import gradio as gr
+import random
+import torch
+import cv2
+import re
+import uuid
+from PIL import Image, ImageDraw, ImageOps, ImageFont
+import math
+import numpy as np
+import argparse
+import inspect
+import tempfile
+from transformers import CLIPSegProcessor, CLIPSegForImageSegmentation
+from transformers import pipeline, BlipProcessor, BlipForConditionalGeneration, BlipForQuestionAnswering
+from transformers import AutoImageProcessor, UperNetForSemanticSegmentation
+
+from diffusers import StableDiffusionPipeline, StableDiffusionInpaintPipeline, StableDiffusionInstructPix2PixPipeline
+from diffusers import EulerAncestralDiscreteScheduler
+from diffusers import StableDiffusionControlNetPipeline, ControlNetModel, UniPCMultistepScheduler
+from controlnet_aux import OpenposeDetector, MLSDdetector, HEDdetector
+
+from langchain.agents.initialize import initialize_agent
+from langchain.agents.tools import Tool
+from langchain.chains.conversation.memory import ConversationBufferMemory
+
+# Grounding DINO
+import groundingdino.datasets.transforms as T
+from groundingdino.models import build_model
+from groundingdino.util import box_ops
+from groundingdino.util.slconfig import SLConfig
+from groundingdino.util.utils import clean_state_dict, get_phrases_from_posmap
+
+# segment anything
+from segment_anything import build_sam, SamPredictor, SamAutomaticMaskGenerator
+import cv2
+import numpy as np
+import matplotlib.pyplot as plt
+import wget
+
+from llama import Llama
+
+
+GPT4TOOLS_PREFIX = """GPT4Tools can handle various text and visual tasks, such as answering questions and providing in-depth explanations and discussions. It generates human-like text and uses tools to indirectly understand images. When referring to images, GPT4Tools follows strict file name rules. To complete visual tasks, GPT4Tools uses tools and stays loyal to observation outputs. Users can provide new images to GPT4Tools with a description, but tools must be used for subsequent tasks.
+TOOLS:
+------
+
+GPT4Tools has access to the following tools:"""
+
+
+GPT4TOOLS_FORMAT_INSTRUCTIONS = """To use a tool, please use the following format:
+
+```
+Thought: Do I need to use a tool? Yes
+Action: the action to take, should be one of [{tool_names}]
+Action Input: the input to the action
+Observation: the result of the action
+```
+
+When you have a response to say to the Human, or if you do not need to use a tool, you MUST use the format:
+
+```
+Thought: Do I need to use a tool? No
+{ai_prefix}: [your response here]
+```
+"""
+
+GPT4TOOLS_SUFFIX = """Follow file name rules and do not fake non-existent file names. Remember to provide the image file name loyally from the last tool observation.
+
+Previous conversation:
+{chat_history}
+
+New input: {input}
+GPT4Tools needs to use tools to observe images, not directly imagine them. Thoughts and observations in the conversation are only visible to GPT4Tools. When answering human questions, repeat important information. Let's think step by step.
+{agent_scratchpad}"""
+
+
+os.makedirs('image', exist_ok=True)
+
+
+def seed_everything(seed):
+    random.seed(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed_all(seed)
+    return seed
+
+
+def prompts(name, description):
+    def decorator(func):
+        func.name = name
+        func.description = description
+        return func
+
+    return decorator
+
+
+def blend_gt2pt(old_image, new_image, sigma=0.15, steps=100):
+    new_size = new_image.size
+    old_size = old_image.size
+    easy_img = np.array(new_image)
+    gt_img_array = np.array(old_image)
+    pos_w = (new_size[0] - old_size[0]) // 2
+    pos_h = (new_size[1] - old_size[1]) // 2
+
+    kernel_h = cv2.getGaussianKernel(old_size[1], old_size[1] * sigma)
+    kernel_w = cv2.getGaussianKernel(old_size[0], old_size[0] * sigma)
+    kernel = np.multiply(kernel_h, np.transpose(kernel_w))
+
+    kernel[steps:-steps, steps:-steps] = 1
+    kernel[:steps, :steps] = kernel[:steps, :steps] / kernel[steps - 1, steps - 1]
+    kernel[:steps, -steps:] = kernel[:steps, -steps:] / kernel[steps - 1, -(steps)]
+    kernel[-steps:, :steps] = kernel[-steps:, :steps] / kernel[-steps, steps - 1]
+    kernel[-steps:, -steps:] = kernel[-steps:, -steps:] / kernel[-steps, -steps]
+    kernel = np.expand_dims(kernel, 2)
+    kernel = np.repeat(kernel, 3, 2)
+
+    weight = np.linspace(0, 1, steps)
+    top = np.expand_dims(weight, 1)
+    top = np.repeat(top, old_size[0] - 2 * steps, 1)
+    top = np.expand_dims(top, 2)
+    top = np.repeat(top, 3, 2)
+
+    weight = np.linspace(1, 0, steps)
+    down = np.expand_dims(weight, 1)
+    down = np.repeat(down, old_size[0] - 2 * steps, 1)
+    down = np.expand_dims(down, 2)
+    down = np.repeat(down, 3, 2)
+
+    weight = np.linspace(0, 1, steps)
+    left = np.expand_dims(weight, 0)
+    left = np.repeat(left, old_size[1] - 2 * steps, 0)
+    left = np.expand_dims(left, 2)
+    left = np.repeat(left, 3, 2)
+
+    weight = np.linspace(1, 0, steps)
+    right = np.expand_dims(weight, 0)
+    right = np.repeat(right, old_size[1] - 2 * steps, 0)
+    right = np.expand_dims(right, 2)
+    right = np.repeat(right, 3, 2)
+
+    kernel[:steps, steps:-steps] = top
+    kernel[-steps:, steps:-steps] = down
+    kernel[steps:-steps, :steps] = left
+    kernel[steps:-steps, -steps:] = right
+
+    pt_gt_img = easy_img[pos_h:pos_h + old_size[1], pos_w:pos_w + old_size[0]]
+    gaussian_gt_img = kernel * gt_img_array + (1 - kernel) * pt_gt_img  # gt img with blur img
+    gaussian_gt_img = gaussian_gt_img.astype(np.int64)
+    easy_img[pos_h:pos_h + old_size[1], pos_w:pos_w + old_size[0]] = gaussian_gt_img
+    gaussian_img = Image.fromarray(easy_img)
+    return gaussian_img
+
+
+def cut_dialogue_history(history_memory, keep_last_n_paragraphs=1):
+    if history_memory is None or len(history_memory) == 0:
+        return history_memory
+    paragraphs = history_memory.split('Human:')
+    if len(paragraphs) <= keep_last_n_paragraphs:
+        return history_memory
+    return 'Human:' + 'Human:'.join(paragraphs[-1:])
+
+
+def get_new_image_name(org_img_name, func_name="update"):
+    head_tail = os.path.split(org_img_name)
+    head = head_tail[0]
+    tail = head_tail[1]
+    new_file_name = f'{str(uuid.uuid4())[:8]}.png'
+    return os.path.join(head, new_file_name)
+
+
+class InstructPix2Pix:
+    def __init__(self, device):
+        print(f"Initializing InstructPix2Pix to {device}")
+        self.device = device
+        self.torch_dtype = torch.float16 if 'cuda' in device else torch.float32
+        self.pipe = StableDiffusionInstructPix2PixPipeline.from_pretrained("timbrooks/instruct-pix2pix",
+                                                                           safety_checker=None,
+                                                                           torch_dtype=self.torch_dtype).to(device)
+        self.pipe.scheduler = EulerAncestralDiscreteScheduler.from_config(self.pipe.scheduler.config)
+
+    @prompts(name="Instruct Image Using Text",
+             description="useful when you want to the style of the image to be like the text. "
+                         "like: make it look like a painting. or make it like a robot. "
+                         "The input to this tool should be a comma separated string of two, "
+                         "representing the image_path and the text. ")
+    def inference(self, inputs):
+        """Change style of image."""
+        print("===>Starting InstructPix2Pix Inference")
+        image_path, text = inputs.split(",")[0], ','.join(inputs.split(',')[1:])
+        original_image = Image.open(image_path)
+        image = self.pipe(text, image=original_image, num_inference_steps=40, image_guidance_scale=1.2).images[0]
+        updated_image_path = get_new_image_name(image_path, func_name="pix2pix")
+        image.save(updated_image_path)
+        print(f"\nProcessed InstructPix2Pix, Input Image: {image_path}, Instruct Text: {text}, "
+              f"Output Image: {updated_image_path}")
+        return updated_image_path
+
+
+class Text2Image:
+    def __init__(self, device):
+        print(f"Initializing Text2Image to {device}")
+        self.device = device
+        self.torch_dtype = torch.float16 if 'cuda' in device else torch.float32
+        self.pipe = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5",
+                                                            torch_dtype=self.torch_dtype)
+        self.pipe.to(device)
+        self.a_prompt = 'best quality, extremely detailed'
+        self.n_prompt = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, ' \
+                        'fewer digits, cropped, worst quality, low quality'
+
+    @prompts(name="Generate Image From User Input Text",
+             description="useful when you want to generate an image from a user input text and save it to a file. "
+                         "like: generate an image of an object or something, or generate an image that includes some objects. "
+                         "The input to this tool should be a string, representing the text used to generate image. ")
+    def inference(self, text):
+        image_filename = os.path.join('image', f"{str(uuid.uuid4())[:8]}.png")
+        prompt = text + ', ' + self.a_prompt
+        image = self.pipe(prompt, negative_prompt=self.n_prompt).images[0]
+        image.save(image_filename)
+        print(
+            f"\nProcessed Text2Image, Input Text: {text}, Output Image: {image_filename}")
+        return image_filename
+
+
+class ImageCaptioning:
+    def __init__(self, device):
+        print(f"Initializing ImageCaptioning to {device}")
+        self.device = device
+        self.torch_dtype = torch.float16 if 'cuda' in device else torch.float32
+        self.processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base")
+        self.model = BlipForConditionalGeneration.from_pretrained(
+            "Salesforce/blip-image-captioning-base", torch_dtype=self.torch_dtype).to(self.device)
+
+    @prompts(name="Get Photo Description",
+             description="useful when you want to know what is inside the photo. receives image_path as input. "
+                         "The input to this tool should be a string, representing the image_path. ")
+    def inference(self, image_path):
+        inputs = self.processor(Image.open(image_path), return_tensors="pt").to(self.device, self.torch_dtype)
+        out = self.model.generate(**inputs)
+        captions = self.processor.decode(out[0], skip_special_tokens=True)
+        print(f"\nProcessed ImageCaptioning, Input Image: {image_path}, Output Text: {captions}")
+        return captions
+
+
+class Image2Canny:
+    def __init__(self, device):
+        print("Initializing Image2Canny")
+        self.low_threshold = 100
+        self.high_threshold = 200
+
+    @prompts(name="Edge Detection On Image",
+             description="useful when you want to detect the edge of the image. "
+                         "like: detect the edges of this image, or canny detection on image, "
+                         "or perform edge detection on this image, or detect the canny image of this image. "
+                         "The input to this tool should be a string, representing the image_path")
+    def inference(self, inputs):
+        image = Image.open(inputs)
+        image = np.array(image)
+        canny = cv2.Canny(image, self.low_threshold, self.high_threshold)
+        canny = canny[:, :, None]
+        canny = np.concatenate([canny, canny, canny], axis=2)
+        canny = Image.fromarray(canny)
+        updated_image_path = get_new_image_name(inputs, func_name="edge")
+        canny.save(updated_image_path)
+        print(f"\nProcessed Image2Canny, Input Image: {inputs}, Output Text: {updated_image_path}")
+        return updated_image_path
+
+
+class CannyText2Image:
+    def __init__(self, device):
+        print(f"Initializing CannyText2Image to {device}")
+        self.torch_dtype = torch.float16 if 'cuda' in device else torch.float32
+        self.controlnet = ControlNetModel.from_pretrained("fusing/stable-diffusion-v1-5-controlnet-canny",
+                                                          torch_dtype=self.torch_dtype)
+        self.pipe = StableDiffusionControlNetPipeline.from_pretrained(
+            "runwayml/stable-diffusion-v1-5", controlnet=self.controlnet, safety_checker=None,
+            torch_dtype=self.torch_dtype)
+        self.pipe.scheduler = UniPCMultistepScheduler.from_config(self.pipe.scheduler.config)
+        self.pipe.to(device)
+        self.seed = -1
+        self.a_prompt = 'best quality, extremely detailed'
+        self.n_prompt = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, ' \
+                            'fewer digits, cropped, worst quality, low quality'
+
+    @prompts(name="Generate Image Condition On Canny Image",
+             description="useful when you want to generate a new real image from both the user description and a canny image."
+                         " like: generate a real image of a object or something from this canny image,"
+                         " or generate a new real image of a object or something from this edge image. "
+                         "The input to this tool should be a comma separated string of two, "
+                         "representing the image_path and the user description. ")
+    def inference(self, inputs):
+        image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:])
+        image = Image.open(image_path)
+        self.seed = random.randint(0, 65535)
+        seed_everything(self.seed)
+        prompt = f'{instruct_text}, {self.a_prompt}'
+        image = self.pipe(prompt, image, num_inference_steps=20, eta=0.0, negative_prompt=self.n_prompt,
+                          guidance_scale=9.0).images[0]
+        updated_image_path = get_new_image_name(image_path, func_name="canny2image")
+        image.save(updated_image_path)
+        print(f"\nProcessed CannyText2Image, Input Canny: {image_path}, Input Text: {instruct_text}, "
+              f"Output Text: {updated_image_path}")
+        return updated_image_path
+
+
+class Image2Line:
+    def __init__(self, device):
+        print("Initializing Image2Line")
+        self.detector = MLSDdetector.from_pretrained('lllyasviel/ControlNet')
+
+    @prompts(name="Line Detection On Image",
+             description="useful when you want to detect the straight line of the image. "
+                         "like: detect the straight lines of this image, or straight line detection on image, "
+                         "or perform straight line detection on this image, or detect the straight line image of this image. "
+                         "The input to this tool should be a string, representing the image_path")
+    def inference(self, inputs):
+        image = Image.open(inputs)
+        mlsd = self.detector(image)
+        updated_image_path = get_new_image_name(inputs, func_name="line-of")
+        mlsd.save(updated_image_path)
+        print(f"\nProcessed Image2Line, Input Image: {inputs}, Output Line: {updated_image_path}")
+        return updated_image_path
+
+
+class LineText2Image:
+    def __init__(self, device):
+        print(f"Initializing LineText2Image to {device}")
+        self.torch_dtype = torch.float16 if 'cuda' in device else torch.float32
+        self.controlnet = ControlNetModel.from_pretrained("fusing/stable-diffusion-v1-5-controlnet-mlsd",
+                                                          torch_dtype=self.torch_dtype)
+        self.pipe = StableDiffusionControlNetPipeline.from_pretrained(
+            "runwayml/stable-diffusion-v1-5", controlnet=self.controlnet, safety_checker=None,
+            torch_dtype=self.torch_dtype
+        )
+        self.pipe.scheduler = UniPCMultistepScheduler.from_config(self.pipe.scheduler.config)
+        self.pipe.to(device)
+        self.seed = -1
+        self.a_prompt = 'best quality, extremely detailed'
+        self.n_prompt = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, ' \
+                            'fewer digits, cropped, worst quality, low quality'
+
+    @prompts(name="Generate Image Condition On Line Image",
+             description="useful when you want to generate a new real image from both the user description "
+                         "and a straight line image. "
+                         "like: generate a real image of a object or something from this straight line image, "
+                         "or generate a new real image of a object or something from this straight lines. "
+                         "The input to this tool should be a comma separated string of two, "
+                         "representing the image_path and the user description. ")
+    def inference(self, inputs):
+        image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:])
+        image = Image.open(image_path)
+        self.seed = random.randint(0, 65535)
+        seed_everything(self.seed)
+        prompt = f'{instruct_text}, {self.a_prompt}'
+        image = self.pipe(prompt, image, num_inference_steps=20, eta=0.0, negative_prompt=self.n_prompt,
+                          guidance_scale=9.0).images[0]
+        updated_image_path = get_new_image_name(image_path, func_name="line2image")
+        image.save(updated_image_path)
+        print(f"\nProcessed LineText2Image, Input Line: {image_path}, Input Text: {instruct_text}, "
+              f"Output Text: {updated_image_path}")
+        return updated_image_path
+
+
+class Image2Hed:
+    def __init__(self, device):
+        print("Initializing Image2Hed")
+        self.detector = HEDdetector.from_pretrained('lllyasviel/ControlNet')
+
+    @prompts(name="Hed Detection On Image",
+             description="useful when you want to detect the soft hed boundary of the image. "
+                         "like: detect the soft hed boundary of this image, or hed boundary detection on image, "
+                         "or perform hed boundary detection on this image, or detect soft hed boundary image of this image. "
+                         "The input to this tool should be a string, representing the image_path")
+    def inference(self, inputs):
+        image = Image.open(inputs)
+        hed = self.detector(image)
+        updated_image_path = get_new_image_name(inputs, func_name="hed-boundary")
+        hed.save(updated_image_path)
+        print(f"\nProcessed Image2Hed, Input Image: {inputs}, Output Hed: {updated_image_path}")
+        return updated_image_path
+
+
+class HedText2Image:
+    def __init__(self, device):
+        print(f"Initializing HedText2Image to {device}")
+        self.torch_dtype = torch.float16 if 'cuda' in device else torch.float32
+        self.controlnet = ControlNetModel.from_pretrained("fusing/stable-diffusion-v1-5-controlnet-hed",
+                                                          torch_dtype=self.torch_dtype)
+        self.pipe = StableDiffusionControlNetPipeline.from_pretrained(
+            "runwayml/stable-diffusion-v1-5", controlnet=self.controlnet, safety_checker=None,
+            torch_dtype=self.torch_dtype
+        )
+        self.pipe.scheduler = UniPCMultistepScheduler.from_config(self.pipe.scheduler.config)
+        self.pipe.to(device)
+        self.seed = -1
+        self.a_prompt = 'best quality, extremely detailed'
+        self.n_prompt = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, ' \
+                            'fewer digits, cropped, worst quality, low quality'
+
+    @prompts(name="Generate Image Condition On Soft Hed Boundary Image",
+             description="useful when you want to generate a new real image from both the user description "
+                         "and a soft hed boundary image. "
+                         "like: generate a real image of a object or something from this soft hed boundary image, "
+                         "or generate a new real image of a object or something from this hed boundary. "
+                         "The input to this tool should be a comma separated string of two, "
+                         "representing the image_path and the user description")
+    def inference(self, inputs):
+        image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:])
+        image = Image.open(image_path)
+        self.seed = random.randint(0, 65535)
+        seed_everything(self.seed)
+        prompt = f'{instruct_text}, {self.a_prompt}'
+        image = self.pipe(prompt, image, num_inference_steps=20, eta=0.0, negative_prompt=self.n_prompt,
+                          guidance_scale=9.0).images[0]
+        updated_image_path = get_new_image_name(image_path, func_name="hed2image")
+        image.save(updated_image_path)
+        print(f"\nProcessed HedText2Image, Input Hed: {image_path}, Input Text: {instruct_text}, "
+              f"Output Image: {updated_image_path}")
+        return updated_image_path
+
+
+class Image2Scribble:
+    def __init__(self, device):
+        print("Initializing Image2Scribble")
+        self.detector = HEDdetector.from_pretrained('lllyasviel/ControlNet')
+
+    @prompts(name="Sketch Detection On Image",
+             description="useful when you want to generate a scribble of the image. "
+                         "like: generate a scribble of this image, or generate a sketch from this image, "
+                         "detect the sketch from this image. "
+                         "The input to this tool should be a string, representing the image_path")
+    def inference(self, inputs):
+        image = Image.open(inputs)
+        scribble = self.detector(image, scribble=True)
+        updated_image_path = get_new_image_name(inputs, func_name="scribble")
+        scribble.save(updated_image_path)
+        print(f"\nProcessed Image2Scribble, Input Image: {inputs}, Output Scribble: {updated_image_path}")
+        return updated_image_path
+
+
+class ScribbleText2Image:
+    def __init__(self, device):
+        print(f"Initializing ScribbleText2Image to {device}")
+        self.torch_dtype = torch.float16 if 'cuda' in device else torch.float32
+        self.controlnet = ControlNetModel.from_pretrained("fusing/stable-diffusion-v1-5-controlnet-scribble",
+                                                          torch_dtype=self.torch_dtype)
+        self.pipe = StableDiffusionControlNetPipeline.from_pretrained(
+            "runwayml/stable-diffusion-v1-5", controlnet=self.controlnet, safety_checker=None,
+            torch_dtype=self.torch_dtype
+        )
+        self.pipe.scheduler = UniPCMultistepScheduler.from_config(self.pipe.scheduler.config)
+        self.pipe.to(device)
+        self.seed = -1
+        self.a_prompt = 'best quality, extremely detailed'
+        self.n_prompt = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, ' \
+                            'fewer digits, cropped, worst quality, low quality'
+
+    @prompts(name="Generate Image Condition On Sketch Image",
+             description="useful when you want to generate a new real image from both the user description and "
+                         "a scribble image or a sketch image. "
+                         "The input to this tool should be a comma separated string of two, "
+                         "representing the image_path and the user description")
+    def inference(self, inputs):
+        image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:])
+        image = Image.open(image_path)
+        self.seed = random.randint(0, 65535)
+        seed_everything(self.seed)
+        prompt = f'{instruct_text}, {self.a_prompt}'
+        image = self.pipe(prompt, image, num_inference_steps=20, eta=0.0, negative_prompt=self.n_prompt,
+                          guidance_scale=9.0).images[0]
+        updated_image_path = get_new_image_name(image_path, func_name="scribble2image")
+        image.save(updated_image_path)
+        print(f"\nProcessed ScribbleText2Image, Input Scribble: {image_path}, Input Text: {instruct_text}, "
+              f"Output Image: {updated_image_path}")
+        return updated_image_path
+
+
+class Image2Pose:
+    def __init__(self, device):
+        print("Initializing Image2Pose")
+        self.detector = OpenposeDetector.from_pretrained('lllyasviel/ControlNet')
+
+    @prompts(name="Pose Detection On Image",
+             description="useful when you want to detect the human pose of the image. "
+                         "like: generate human poses of this image, or generate a pose image from this image. "
+                         "The input to this tool should be a string, representing the image_path")
+    def inference(self, inputs):
+        image = Image.open(inputs)
+        pose = self.detector(image)
+        updated_image_path = get_new_image_name(inputs, func_name="human-pose")
+        pose.save(updated_image_path)
+        print(f"\nProcessed Image2Pose, Input Image: {inputs}, Output Pose: {updated_image_path}")
+        return updated_image_path
+
+
+class PoseText2Image:
+    def __init__(self, device):
+        print(f"Initializing PoseText2Image to {device}")
+        self.torch_dtype = torch.float16 if 'cuda' in device else torch.float32
+        self.controlnet = ControlNetModel.from_pretrained("fusing/stable-diffusion-v1-5-controlnet-openpose",
+                                                          torch_dtype=self.torch_dtype)
+        self.pipe = StableDiffusionControlNetPipeline.from_pretrained(
+            "runwayml/stable-diffusion-v1-5", controlnet=self.controlnet, safety_checker=None,
+            torch_dtype=self.torch_dtype)
+        self.pipe.scheduler = UniPCMultistepScheduler.from_config(self.pipe.scheduler.config)
+        self.pipe.to(device)
+        self.num_inference_steps = 20
+        self.seed = -1
+        self.unconditional_guidance_scale = 9.0
+        self.a_prompt = 'best quality, extremely detailed'
+        self.n_prompt = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit,' \
+                            ' fewer digits, cropped, worst quality, low quality'
+
+    @prompts(name="Generate Image Condition On Pose Image",
+             description="useful when you want to generate a new real image from both the user description "
+                         "and a human pose image. "
+                         "like: generate a real image of a human from this human pose image, "
+                         "or generate a new real image of a human from this pose. "
+                         "The input to this tool should be a comma separated string of two, "
+                         "representing the image_path and the user description")
+    def inference(self, inputs):
+        image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:])
+        image = Image.open(image_path)
+        self.seed = random.randint(0, 65535)
+        seed_everything(self.seed)
+        prompt = f'{instruct_text}, {self.a_prompt}'
+        image = self.pipe(prompt, image, num_inference_steps=20, eta=0.0, negative_prompt=self.n_prompt,
+                          guidance_scale=9.0).images[0]
+        updated_image_path = get_new_image_name(image_path, func_name="pose2image")
+        image.save(updated_image_path)
+        print(f"\nProcessed PoseText2Image, Input Pose: {image_path}, Input Text: {instruct_text}, "
+              f"Output Image: {updated_image_path}")
+        return updated_image_path
+
+
+class SegText2Image:
+    def __init__(self, device):
+        print(f"Initializing SegText2Image to {device}")
+        self.torch_dtype = torch.float16 if 'cuda' in device else torch.float32
+        self.controlnet = ControlNetModel.from_pretrained("fusing/stable-diffusion-v1-5-controlnet-seg",
+                                                          torch_dtype=self.torch_dtype)
+        self.pipe = StableDiffusionControlNetPipeline.from_pretrained(
+            "runwayml/stable-diffusion-v1-5", controlnet=self.controlnet, safety_checker=None,
+            torch_dtype=self.torch_dtype)
+        self.pipe.scheduler = UniPCMultistepScheduler.from_config(self.pipe.scheduler.config)
+        self.pipe.to(device)
+        self.seed = -1
+        self.a_prompt = 'best quality, extremely detailed'
+        self.n_prompt = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit,' \
+                            ' fewer digits, cropped, worst quality, low quality'
+
+    @prompts(name="Generate Image Condition On Segmentations",
+             description="useful when you want to generate a new real image from both the user description and segmentations. "
+                         "like: generate a real image of a object or something from this segmentation image, "
+                         "or generate a new real image of a object or something from these segmentations. "
+                         "The input to this tool should be a comma separated string of two, "
+                         "representing the image_path and the user description")
+    def inference(self, inputs):
+        image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:])
+        image = Image.open(image_path)
+        self.seed = random.randint(0, 65535)
+        seed_everything(self.seed)
+        prompt = f'{instruct_text}, {self.a_prompt}'
+        image = self.pipe(prompt, image, num_inference_steps=20, eta=0.0, negative_prompt=self.n_prompt,
+                          guidance_scale=9.0).images[0]
+        updated_image_path = get_new_image_name(image_path, func_name="segment2image")
+        image.save(updated_image_path)
+        print(f"\nProcessed SegText2Image, Input Seg: {image_path}, Input Text: {instruct_text}, "
+              f"Output Image: {updated_image_path}")
+        return updated_image_path
+
+
+class Image2Depth:
+    def __init__(self, device):
+        print("Initializing Image2Depth")
+        self.depth_estimator = pipeline('depth-estimation')
+
+    @prompts(name="Predict Depth On Image",
+             description="useful when you want to detect depth of the image. like: generate the depth from this image, "
+                         "or detect the depth map on this image, or predict the depth for this image. "
+                         "The input to this tool should be a string, representing the image_path")
+    def inference(self, inputs):
+        image = Image.open(inputs)
+        depth = self.depth_estimator(image)['depth']
+        depth = np.array(depth)
+        depth = depth[:, :, None]
+        depth = np.concatenate([depth, depth, depth], axis=2)
+        depth = Image.fromarray(depth)
+        updated_image_path = get_new_image_name(inputs, func_name="depth")
+        depth.save(updated_image_path)
+        print(f"\nProcessed Image2Depth, Input Image: {inputs}, Output Depth: {updated_image_path}")
+        return updated_image_path
+
+
+class DepthText2Image:
+    def __init__(self, device):
+        print(f"Initializing DepthText2Image to {device}")
+        self.torch_dtype = torch.float16 if 'cuda' in device else torch.float32
+        self.controlnet = ControlNetModel.from_pretrained(
+            "fusing/stable-diffusion-v1-5-controlnet-depth", torch_dtype=self.torch_dtype)
+        self.pipe = StableDiffusionControlNetPipeline.from_pretrained(
+            "runwayml/stable-diffusion-v1-5", controlnet=self.controlnet, safety_checker=None,
+            torch_dtype=self.torch_dtype)
+        self.pipe.scheduler = UniPCMultistepScheduler.from_config(self.pipe.scheduler.config)
+        self.pipe.to(device)
+        self.seed = -1
+        self.a_prompt = 'best quality, extremely detailed'
+        self.n_prompt = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit,' \
+                            ' fewer digits, cropped, worst quality, low quality'
+
+    @prompts(name="Generate Image Condition On Depth",
+             description="useful when you want to generate a new real image from both the user description and depth image. "
+                         "like: generate a real image of a object or something from this depth image, "
+                         "or generate a new real image of a object or something from the depth map. "
+                         "The input to this tool should be a comma separated string of two, "
+                         "representing the image_path and the user description")
+    def inference(self, inputs):
+        image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:])
+        image = Image.open(image_path)
+        self.seed = random.randint(0, 65535)
+        seed_everything(self.seed)
+        prompt = f'{instruct_text}, {self.a_prompt}'
+        image = self.pipe(prompt, image, num_inference_steps=20, eta=0.0, negative_prompt=self.n_prompt,
+                          guidance_scale=9.0).images[0]
+        updated_image_path = get_new_image_name(image_path, func_name="depth2image")
+        image.save(updated_image_path)
+        print(f"\nProcessed DepthText2Image, Input Depth: {image_path}, Input Text: {instruct_text}, "
+              f"Output Image: {updated_image_path}")
+        return updated_image_path
+
+
+class Image2Normal:
+    def __init__(self, device):
+        print("Initializing Image2Normal")
+        self.depth_estimator = pipeline("depth-estimation", model="Intel/dpt-hybrid-midas")
+        self.bg_threhold = 0.4
+
+    @prompts(name="Predict Normal Map On Image",
+             description="useful when you want to detect norm map of the image. "
+                         "like: generate normal map from this image, or predict normal map of this image. "
+                         "The input to this tool should be a string, representing the image_path")
+    def inference(self, inputs):
+        image = Image.open(inputs)
+        original_size = image.size
+        image = self.depth_estimator(image)['predicted_depth'][0]
+        image = image.numpy()
+        image_depth = image.copy()
+        image_depth -= np.min(image_depth)
+        image_depth /= np.max(image_depth)
+        x = cv2.Sobel(image, cv2.CV_32F, 1, 0, ksize=3)
+        x[image_depth < self.bg_threhold] = 0
+        y = cv2.Sobel(image, cv2.CV_32F, 0, 1, ksize=3)
+        y[image_depth < self.bg_threhold] = 0
+        z = np.ones_like(x) * np.pi * 2.0
+        image = np.stack([x, y, z], axis=2)
+        image /= np.sum(image ** 2.0, axis=2, keepdims=True) ** 0.5
+        image = (image * 127.5 + 127.5).clip(0, 255).astype(np.uint8)
+        image = Image.fromarray(image)
+        image = image.resize(original_size)
+        updated_image_path = get_new_image_name(inputs, func_name="normal-map")
+        image.save(updated_image_path)
+        print(f"\nProcessed Image2Normal, Input Image: {inputs}, Output Depth: {updated_image_path}")
+        return updated_image_path
+
+
+class NormalText2Image:
+    def __init__(self, device):
+        print(f"Initializing NormalText2Image to {device}")
+        self.torch_dtype = torch.float16 if 'cuda' in device else torch.float32
+        self.controlnet = ControlNetModel.from_pretrained(
+            "fusing/stable-diffusion-v1-5-controlnet-normal", torch_dtype=self.torch_dtype)
+        self.pipe = StableDiffusionControlNetPipeline.from_pretrained(
+            "runwayml/stable-diffusion-v1-5", controlnet=self.controlnet, safety_checker=None,
+            torch_dtype=self.torch_dtype)
+        self.pipe.scheduler = UniPCMultistepScheduler.from_config(self.pipe.scheduler.config)
+        self.pipe.to(device)
+        self.seed = -1
+        self.a_prompt = 'best quality, extremely detailed'
+        self.n_prompt = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit,' \
+                            ' fewer digits, cropped, worst quality, low quality'
+
+    @prompts(name="Generate Image Condition On Normal Map",
+             description="useful when you want to generate a new real image from both the user description and normal map. "
+                         "like: generate a real image of a object or something from this normal map, "
+                         "or generate a new real image of a object or something from the normal map. "
+                         "The input to this tool should be a comma separated string of two, "
+                         "representing the image_path and the user description")
+    def inference(self, inputs):
+        image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:])
+        image = Image.open(image_path)
+        self.seed = random.randint(0, 65535)
+        seed_everything(self.seed)
+        prompt = f'{instruct_text}, {self.a_prompt}'
+        image = self.pipe(prompt, image, num_inference_steps=20, eta=0.0, negative_prompt=self.n_prompt,
+                          guidance_scale=9.0).images[0]
+        updated_image_path = get_new_image_name(image_path, func_name="normal2image")
+        image.save(updated_image_path)
+        print(f"\nProcessed NormalText2Image, Input Normal: {image_path}, Input Text: {instruct_text}, "
+              f"Output Image: {updated_image_path}")
+        return updated_image_path
+
+
+class VisualQuestionAnswering:
+    def __init__(self, device):
+        print(f"Initializing VisualQuestionAnswering to {device}")
+        self.torch_dtype = torch.float16 if 'cuda' in device else torch.float32
+        self.device = device
+        self.processor = BlipProcessor.from_pretrained("Salesforce/blip-vqa-base")
+        self.model = BlipForQuestionAnswering.from_pretrained(
+            "Salesforce/blip-vqa-base", torch_dtype=self.torch_dtype).to(self.device)
+
+    @prompts(name="Answer Question About The Image",
+             description="useful when you need an answer for a question based on an image. "
+                         "like: what is the background color of the last image, how many cats in this figure, what is in this figure. "
+                         "The input to this tool should be a comma separated string of two, representing the image_path and the question")
+    def inference(self, inputs):
+        image_path, question = inputs.split(",")[0], ','.join(inputs.split(',')[1:])
+        raw_image = Image.open(image_path).convert('RGB')
+        inputs = self.processor(raw_image, question, return_tensors="pt").to(self.device, self.torch_dtype)
+        out = self.model.generate(**inputs)
+        answer = self.processor.decode(out[0], skip_special_tokens=True)
+        print(f"\nProcessed VisualQuestionAnswering, Input Image: {image_path}, Input Question: {question}, "
+              f"Output Answer: {answer}")
+        return answer
+
+
+class Segmenting:
+    def __init__(self, device):
+        print(f"Inintializing Segmentation to {device}")
+        self.device = device
+        self.torch_dtype = torch.float16 if 'cuda' in device else torch.float32
+        self.model_checkpoint_path = os.path.join("checkpoints","sam")
+
+        self.download_parameters()
+        self.sam = build_sam(checkpoint=self.model_checkpoint_path).to(device)
+        self.sam_predictor = SamPredictor(self.sam)
+        self.mask_generator = SamAutomaticMaskGenerator(self.sam)
+
+    def download_parameters(self):
+        url = "https://dl.fbaipublicfiles.com/segment_anything/sam_vit_h_4b8939.pth"
+        if not os.path.exists(self.model_checkpoint_path):
+            wget.download(url,out=self.model_checkpoint_path)
+
+    def show_mask(self, mask, ax, random_color=False):
+        if random_color:
+            color = np.concatenate([np.random.random(3), np.array([1])], axis=0)
+        else:
+            color = np.array([30/255, 144/255, 255/255, 1])
+        h, w = mask.shape[-2:]
+        mask_image = mask.reshape(h, w, 1) * color.reshape(1, 1, -1)
+        ax.imshow(mask_image)
+        
+    def show_box(self, box, ax, label):
+        x0, y0 = box[0], box[1]
+        w, h = box[2] - box[0], box[3] - box[1]
+        ax.add_patch(plt.Rectangle((x0, y0), w, h, edgecolor='green', facecolor=(0,0,0,0), lw=2)) 
+        ax.text(x0, y0, label)
+
+    
+    def get_mask_with_boxes(self, image_pil, image, boxes_filt):
+
+        size = image_pil.size
+        H, W = size[1], size[0]
+        for i in range(boxes_filt.size(0)):
+            boxes_filt[i] = boxes_filt[i] * torch.Tensor([W, H, W, H])
+            boxes_filt[i][:2] -= boxes_filt[i][2:] / 2
+            boxes_filt[i][2:] += boxes_filt[i][:2]
+
+        boxes_filt = boxes_filt.cpu()
+        transformed_boxes = self.sam_predictor.transform.apply_boxes_torch(boxes_filt, image.shape[:2]).to(self.device)
+
+        masks, _, _ = self.sam_predictor.predict_torch(
+            point_coords = None,
+            point_labels = None,
+            boxes = transformed_boxes.to(self.device),
+            multimask_output = False,
+        )
+        return masks
+    
+    def segment_image_with_boxes(self, image_pil, image_path, boxes_filt, pred_phrases):
+
+        image = cv2.imread(image_path)
+        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+        self.sam_predictor.set_image(image)
+
+        masks = self.get_mask_with_boxes(image_pil, image, boxes_filt)
+        
+        # draw output image
+        plt.figure(figsize=(10, 10))
+        plt.imshow(image)
+        for mask in masks:
+            self.show_mask(mask.cpu().numpy(), plt.gca(), random_color=True)
+
+        updated_image_path = get_new_image_name(image_path, func_name="segmentation")
+        plt.axis('off')
+        plt.savefig(
+            updated_image_path, 
+            bbox_inches="tight", dpi=300, pad_inches=0.0
+        )
+        return updated_image_path
+
+
+    @prompts(name="Segment the Image",
+             description="useful when you want to segment all the part of the image, but not segment a certain object."
+                         "like: segment all the object in this image, or generate segmentations on this image, "
+                         "or segment the image,"
+                         "or perform segmentation on this image, "
+                         "or segment all the object in this image."
+                         "The input to this tool should be a string, representing the image_path")
+    def inference_all(self,image_path):
+        image = cv2.imread(image_path)
+        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+        masks = self.mask_generator.generate(image)
+        plt.figure(figsize=(20,20))
+        plt.imshow(image)
+        if len(masks) == 0:
+            return
+        sorted_anns = sorted(masks, key=(lambda x: x['area']), reverse=True)
+        ax = plt.gca()
+        ax.set_autoscale_on(False)
+        polygons = []
+        color = []
+        for ann in sorted_anns:
+            m = ann['segmentation']
+            img = np.ones((m.shape[0], m.shape[1], 3))
+            color_mask = np.random.random((1, 3)).tolist()[0]
+            for i in range(3):
+                img[:,:,i] = color_mask[i]
+            ax.imshow(np.dstack((img, m)))
+
+        updated_image_path = get_new_image_name(image_path, func_name="segment-image")
+        plt.axis('off')
+        plt.savefig(
+            updated_image_path, 
+            bbox_inches="tight", dpi=300, pad_inches=0.0
+        )
+        return updated_image_path
+
+    
+class Text2Box:
+    def __init__(self, device):
+        print(f"Initializing ObjectDetection to {device}")
+        self.device = device
+        self.torch_dtype = torch.float16 if 'cuda' in device else torch.float32
+        self.model_checkpoint_path = os.path.join("checkpoints","groundingdino")
+        self.model_config_path = os.path.join("checkpoints","grounding_config.py")
+        self.download_parameters()
+        self.box_threshold = 0.3
+        self.text_threshold = 0.25
+        self.grounding = (self.load_model()).to(self.device)
+
+    def download_parameters(self):
+        url = "https://github.com/IDEA-Research/GroundingDINO/releases/download/v0.1.0-alpha/groundingdino_swint_ogc.pth"
+        if not os.path.exists(self.model_checkpoint_path):
+            wget.download(url,out=self.model_checkpoint_path)
+        config_url = "https://raw.githubusercontent.com/IDEA-Research/GroundingDINO/main/groundingdino/config/GroundingDINO_SwinT_OGC.py"
+        if not os.path.exists(self.model_config_path):
+            wget.download(config_url,out=self.model_config_path)
+    def load_image(self,image_path):
+         # load image
+        image_pil = Image.open(image_path).convert("RGB")  # load image
+
+        transform = T.Compose(
+            [
+                T.RandomResize([512], max_size=1333),
+                T.ToTensor(),
+                T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
+            ]
+        )
+        image, _ = transform(image_pil, None)  # 3, h, w
+        return image_pil, image
+
+    def load_model(self):
+        args = SLConfig.fromfile(self.model_config_path)
+        args.device = self.device
+        model = build_model(args)
+        checkpoint = torch.load(self.model_checkpoint_path, map_location="cpu")
+        load_res = model.load_state_dict(clean_state_dict(checkpoint["model"]), strict=False)
+        print(load_res)
+        _ = model.eval()
+        return model
+
+    def get_grounding_boxes(self, image, caption, with_logits=True):
+        caption = caption.lower()
+        caption = caption.strip()
+        if not caption.endswith("."):
+            caption = caption + "."
+        image = image.to(self.device)
+        with torch.no_grad():
+            outputs = self.grounding(image[None], captions=[caption])
+        logits = outputs["pred_logits"].cpu().sigmoid()[0]  # (nq, 256)
+        boxes = outputs["pred_boxes"].cpu()[0]  # (nq, 4)
+        logits.shape[0]
+
+        # filter output
+        logits_filt = logits.clone()
+        boxes_filt = boxes.clone()
+        filt_mask = logits_filt.max(dim=1)[0] > self.box_threshold
+        logits_filt = logits_filt[filt_mask]  # num_filt, 256
+        boxes_filt = boxes_filt[filt_mask]  # num_filt, 4
+        logits_filt.shape[0]
+
+        # get phrase
+        tokenlizer = self.grounding.tokenizer
+        tokenized = tokenlizer(caption)
+        # build pred
+        pred_phrases = []
+        for logit, box in zip(logits_filt, boxes_filt):
+            pred_phrase = get_phrases_from_posmap(logit > self.text_threshold, tokenized, tokenlizer)
+            if with_logits:
+                pred_phrases.append(pred_phrase + f"({str(logit.max().item())[:4]})")
+            else:
+                pred_phrases.append(pred_phrase)
+
+        return boxes_filt, pred_phrases
+    
+    def plot_boxes_to_image(self, image_pil, tgt):
+        H, W = tgt["size"]
+        boxes = tgt["boxes"]
+        labels = tgt["labels"]
+        assert len(boxes) == len(labels), "boxes and labels must have same length"
+
+        draw = ImageDraw.Draw(image_pil)
+        mask = Image.new("L", image_pil.size, 0)
+        mask_draw = ImageDraw.Draw(mask)
+
+        # draw boxes and masks
+        for box, label in zip(boxes, labels):
+            # from 0..1 to 0..W, 0..H
+            box = box * torch.Tensor([W, H, W, H])
+            # from xywh to xyxy
+            box[:2] -= box[2:] / 2
+            box[2:] += box[:2]
+            # random color
+            color = tuple(np.random.randint(0, 255, size=3).tolist())
+            # draw
+            x0, y0, x1, y1 = box
+            x0, y0, x1, y1 = int(x0), int(y0), int(x1), int(y1)
+
+            draw.rectangle([x0, y0, x1, y1], outline=color, width=6)
+            # draw.text((x0, y0), str(label), fill=color)
+
+            font = ImageFont.load_default()
+            if hasattr(font, "getbbox"):
+                bbox = draw.textbbox((x0, y0), str(label), font)
+            else:
+                w, h = draw.textsize(str(label), font)
+                bbox = (x0, y0, w + x0, y0 + h)
+            # bbox = draw.textbbox((x0, y0), str(label))
+            draw.rectangle(bbox, fill=color)
+            draw.text((x0, y0), str(label), fill="white")
+
+            mask_draw.rectangle([x0, y0, x1, y1], fill=255, width=2)
+
+        return image_pil, mask
+    
+    @prompts(name="Detect the Give Object",
+             description="useful when you only want to detect or find out given objects in the picture"  
+                         "The input to this tool should be a comma separated string of two, "
+                         "representing the image_path, the text description of the object to be found")
+    def inference(self, inputs):
+        image_path, det_prompt = inputs.split(",")
+        print(f"image_path={image_path}, text_prompt={det_prompt}")
+        image_pil, image = self.load_image(image_path)
+
+        boxes_filt, pred_phrases = self.get_grounding_boxes(image, det_prompt)
+
+        size = image_pil.size
+        pred_dict = {
+        "boxes": boxes_filt,
+        "size": [size[1], size[0]],  # H,W
+        "labels": pred_phrases,}
+
+        image_with_box = self.plot_boxes_to_image(image_pil, pred_dict)[0]
+
+        updated_image_path = get_new_image_name(image_path, func_name="detect-something")
+        updated_image = image_with_box.resize(size)
+        updated_image.save(updated_image_path)
+        print(
+            f"\nProcessed ObejectDetecting, Input Image: {image_path}, Object to be Detect {det_prompt}, "
+            f"Output Image: {updated_image_path}")
+        return updated_image_path
+
+
+class Inpainting:
+    def __init__(self, device):
+        self.device = device
+        self.revision = 'fp16' if 'cuda' in self.device else None
+        self.torch_dtype = torch.float16 if 'cuda' in self.device else torch.float32
+
+        self.inpaint = StableDiffusionInpaintPipeline.from_pretrained(
+            "runwayml/stable-diffusion-inpainting", revision=self.revision, torch_dtype=self.torch_dtype).to(device)
+    def __call__(self, prompt, original_image, mask_image):
+        update_image = self.inpaint(prompt=prompt, image=original_image.resize((512, 512)),
+                                     mask_image=mask_image.resize((512, 512))).images[0]
+        return update_image
+
+
+class ObjectSegmenting:
+    template_model = True # Add this line to show this is a template model.
+    def __init__(self,  Text2Box:Text2Box, Segmenting:Segmenting):
+        self.grounding = Text2Box
+        self.sam = Segmenting
+
+    @prompts(name="Segment the given object",
+            description="useful when you only want to segment the certain objects in the picture"
+                        "according to the given text"  
+                        "like: segment the cat,"
+                        "or can you segment an obeject for me"
+                        "The input to this tool should be a comma separated string of two, "
+                        "representing the image_path, the text description of the object to be found")
+    def inference(self, inputs):
+        image_path, det_prompt = inputs.split(",")
+        print(f"image_path={image_path}, text_prompt={det_prompt}")
+        image_pil, image = self.grounding.load_image(image_path)
+        boxes_filt, pred_phrases = self.grounding.get_grounding_boxes(image, det_prompt)
+        updated_image_path = self.sam.segment_image_with_boxes(image_pil,image_path,boxes_filt,pred_phrases)
+        print(
+            f"\nProcessed ObejectSegmenting, Input Image: {image_path}, Object to be Segment {det_prompt}, "
+            f"Output Image: {updated_image_path}")
+        return updated_image_path
+
+
+class ImageEditing:
+    template_model = True
+    def __init__(self, Text2Box:Text2Box, Segmenting:Segmenting, Inpainting:Inpainting):
+        print(f"Initializing ImageEditing")
+        self.sam = Segmenting
+        self.grounding = Text2Box
+        self.inpaint = Inpainting
+
+    def pad_edge(self,mask,padding):
+        #mask Tensor [H,W]
+        mask = mask.numpy()
+        true_indices = np.argwhere(mask)
+        mask_array = np.zeros_like(mask, dtype=bool)
+        for idx in true_indices:
+            padded_slice = tuple(slice(max(0, i - padding), i + padding + 1) for i in idx)
+            mask_array[padded_slice] = True
+        new_mask = (mask_array * 255).astype(np.uint8)
+        #new_mask
+        return new_mask
+
+    @prompts(name="Remove Something From The Photo",
+             description="useful when you want to remove and object or something from the photo "
+                         "from its description or location. "
+                         "The input to this tool should be a comma separated string of two, "
+                         "representing the image_path and the object need to be removed. ")    
+    def inference_remove(self, inputs):
+        image_path, to_be_removed_txt = inputs.split(",")[0], ','.join(inputs.split(',')[1:])
+        return self.inference_replace_sam(f"{image_path},{to_be_removed_txt},background")
+
+    @prompts(name="Replace Something From The Photo",
+            description="useful when you want to replace an object from the object description or "
+                        "location with another object from its description. "
+                        "The input to this tool should be a comma separated string of three, "
+                        "representing the image_path, the object to be replaced, the object to be replaced with ")
+    def inference_replace_sam(self,inputs):
+        image_path, to_be_replaced_txt, replace_with_txt = inputs.split(",")
+        
+        print(f"image_path={image_path}, to_be_replaced_txt={to_be_replaced_txt}")
+        image_pil, image = self.grounding.load_image(image_path)
+        boxes_filt, pred_phrases = self.grounding.get_grounding_boxes(image, to_be_replaced_txt)
+        image = cv2.imread(image_path)
+        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+        self.sam.sam_predictor.set_image(image)
+        masks = self.sam.get_mask_with_boxes(image_pil, image, boxes_filt)
+        mask = torch.sum(masks, dim=0).unsqueeze(0)
+        mask = torch.where(mask > 0, True, False)
+        mask = mask.squeeze(0).squeeze(0).cpu() #tensor
+
+        mask = self.pad_edge(mask,padding=20) #numpy
+        mask_image = Image.fromarray(mask)
+
+        updated_image = self.inpaint(prompt=replace_with_txt, original_image=image_pil,
+                                     mask_image=mask_image)
+        updated_image_path = get_new_image_name(image_path, func_name="replace-something")
+        updated_image = updated_image.resize(image_pil.size)
+        updated_image.save(updated_image_path)
+        print(
+            f"\nProcessed ImageEditing, Input Image: {image_path}, Replace {to_be_replaced_txt} to {replace_with_txt}, "
+            f"Output Image: {updated_image_path}")
+        return updated_image_path
+
+
+class ConversationBot:
+    def __init__(self, load_dict, llm_kwargs):
+        # load_dict = {'VisualQuestionAnswering':'cuda:0', 'ImageCaptioning':'cuda:1',...}
+        print(f"Initializing GPT4Tools, load_dict={load_dict}")
+        if 'ImageCaptioning' not in load_dict:
+            raise ValueError("You have to load ImageCaptioning as a basic function for GPT4Tools")
+
+        self.models = {}
+        # Load Basic Foundation Models
+        for class_name, device in load_dict.items():
+            self.models[class_name] = globals()[class_name](device=device)
+
+        # Load Template Foundation Models
+        for class_name, module in globals().items():
+            if getattr(module, 'template_model', False):
+                template_required_names = {k for k in inspect.signature(module.__init__).parameters.keys() if k!='self'}
+                loaded_names = set([type(e).__name__ for e in self.models.values()])
+                if template_required_names.issubset(loaded_names):
+                    self.models[class_name] = globals()[class_name](
+                        **{name: self.models[name] for name in template_required_names})
+        
+        print(f"All the Available Functions: {self.models}")
+
+        self.tools = []
+        for instance in self.models.values():
+            for e in dir(instance):
+                if e.startswith('inference'):
+                    func = getattr(instance, e)
+                    self.tools.append(Tool(name=func.name, description=func.description, func=func))
+        self.llm = Llama(model_kwargs=llm_kwargs) 
+        self.memory = ConversationBufferMemory(memory_key="chat_history", output_key='output')
+
+    def init_agent(self, lang):
+        self.memory.clear() #clear previous history
+        if lang=='English':
+            PREFIX, FORMAT_INSTRUCTIONS, SUFFIX = GPT4TOOLS_PREFIX, GPT4TOOLS_FORMAT_INSTRUCTIONS, GPT4TOOLS_SUFFIX
+            place = "Enter text and press enter, or upload an image"
+            label_clear = "Clear"
+        else:
+            raise NotImplementedError(f'{lang} is not supported yet')
+        self.agent = initialize_agent(
+            self.tools,
+            self.llm,
+            agent="conversational-react-description",
+            verbose=True,
+            memory=self.memory,
+            return_intermediate_steps=True,
+            agent_kwargs={'prefix': PREFIX, 'format_instructions': FORMAT_INSTRUCTIONS,
+                          'suffix': SUFFIX}, )
+        return gr.update(visible = True), gr.update(visible = False), gr.update(placeholder=place), gr.update(value=label_clear)
+
+    def run_text(self, text, state):
+        self.agent.memory.buffer = cut_dialogue_history(self.agent.memory.buffer)
+        res = self.agent({"input": text.strip()})
+        res['output'] = res['output'].replace("\\", "/")
+        response = re.sub('(image/[-\w]*.png)', lambda m: f'![](file={m.group(0)})*{m.group(0)}*', res['output'])
+        state = state + [(text, response)]
+        print(f"\nProcessed run_text, Input text: {text}\nCurrent state: {state}\n"
+              f"Current Memory: {self.agent.memory.buffer}")
+        return state, state
+
+    def run_image(self, image, state, txt, lang='English'):
+        image_filename = os.path.join('image', f"{str(uuid.uuid4())[:8]}.png")
+        print("======>Auto Resize Image...")
+        img = Image.open(image.name)
+        width, height = img.size
+        ratio = min(512 / width, 512 / height)
+        width_new, height_new = (round(width * ratio), round(height * ratio))
+        width_new = int(np.round(width_new / 64.0)) * 64
+        height_new = int(np.round(height_new / 64.0)) * 64
+        img = img.resize((width_new, height_new))
+        img = img.convert('RGB')
+        img.save(image_filename, "PNG")
+        print(f"Resize image form {width}x{height} to {width_new}x{height_new}")
+        description = self.models['ImageCaptioning'].inference(image_filename)
+        if lang == 'English':
+            Human_prompt = f'\nHuman: Provide an image named {image_filename}. The description is: {description}. Understand the image using tools.\n'
+            AI_prompt = "Received."
+        else:
+            raise NotImplementedError(f'{lang} is not supported yet')
+        self.agent.memory.buffer = self.agent.memory.buffer + Human_prompt + 'AI: ' + AI_prompt
+        state = state + [(f"![](file={image_filename})*{image_filename}*", AI_prompt)]
+        print(f"\nProcessed run_image, Input image: {image_filename}\nCurrent state: {state}\n"
+              f"Current Memory: {self.agent.memory.buffer}")
+        return state, state, f'{txt} {image_filename} '
+
+
+if __name__ == '__main__':
+    if not os.path.exists("checkpoints"):
+        os.mkdir("checkpoints")
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--base_model', type=str, required=True, help='folder path to the vicuna with tokenizer')
+    parser.add_argument('--lora_model', type=str, required=True, help='folder path to the lora model')
+    parser.add_argument('--load', type=str, default='ImageCaptioning_cuda:0,Text2Image_cuda:0')
+    parser.add_argument('--llm_device', type=str, default='cpu', help='device to run the llm model')
+    parser.add_argument('--temperature', type=float, default=0.1, help='temperature for the llm model')
+    parser.add_argument('--max_new_tokens', type=int, default=512, help='max number of new tokens to generate')
+    parser.add_argument('--top_p', type=float, default=0.75, help='top_p for the llm model')
+    parser.add_argument('--top_k', type=int, default=40, help='top_k for the llm model')
+    parser.add_argument('--num_beams', type=int, default=1, help='num_beams for the llm model')
+    args = parser.parse_args()
+    load_dict = {e.split('_')[0].strip(): e.split('_')[1].strip() for e in args.load.split(',')}
+    llm_kwargs = {'base_model': args.base_model,
+                  'lora_model': args.lora_model,
+                  'device': args.llm_device,
+                  'temperature': args.temperature,
+                  'max_new_tokens': args.max_new_tokens,
+                  'top_p': args.top_p,
+                  'top_k': args.top_k,
+                  'num_beams': args.num_beams}
+    bot = ConversationBot(load_dict=load_dict, llm_kwargs=llm_kwargs)
+    with gr.Blocks() as demo:
+        chatbot = gr.Chatbot(elem_id="chatbot", label="🦙 GPT4Tools").style(height=700)
+        state = gr.State([])
+        with gr.Row(visible=True) as input_raws:
+            with gr.Column(scale=0.7):
+                txt = gr.Textbox(show_label=False, placeholder="Enter text and press enter, or upload an image").style(
+                    container=False)
+            with gr.Column(scale=0.15, min_width=0):
+                clear = gr.Button("Clear")
+            with gr.Column(scale=0.15, min_width=0):
+                btn = gr.UploadButton(label="🖼️",file_types=["image"])
+
+        # TODO: support more language 
+        bot.init_agent('English')
+        txt.submit(bot.run_text, [txt, state], [chatbot, state])
+        txt.submit(lambda: "", None, txt)
+        btn.upload(bot.run_image, [btn, state, txt], [chatbot, state, txt])
+        clear.click(bot.memory.clear)
+        clear.click(lambda: [], None, chatbot)
+        clear.click(lambda: [], None, state)
+        gr.Examples(
+            examples=["Generate an image of a happy vicuna running in the grass",
+                      "Tell me a funny story about dog"],
+            inputs=txt
+        )
+    demo.launch(server_name="0.0.0.0", server_port=80)

llama.py

@@ -0,0 +1,175 @@
+"""Wrapper around HuggingFace APIs."""
+import torch
+from typing import Any, Dict, List, Mapping, Optional
+
+from pydantic import BaseModel, Extra, root_validator
+
+from langchain.llms.base import LLM
+from langchain.llms.utils import enforce_stop_tokens
+from langchain.utils import get_from_dict_or_env
+
+from peft import PeftModel
+from transformers import LlamaForCausalLM, LlamaTokenizer, GenerationConfig
+
+DEFAULT_REPO_ID = "gpt2"
+VALID_TASKS = ("text2text-generation", "text-generation")
+
+
+class LlamaHuggingFace:
+
+    def __init__(self, 
+                 base_model,
+                 lora_model,
+                 task='text-generation',
+                 device='cpu',
+                 max_new_tokens=512,
+                 temperature=0.1,
+                 top_p=0.75,
+                 top_k=40,
+                 num_beams=1):
+        self.task = task
+        self.device = device
+        self.temperature = temperature
+        self.max_new_tokens = max_new_tokens
+        self.top_p = top_p
+        self.top_k = top_k
+        self.num_beams = num_beams
+        self.tokenizer = LlamaTokenizer.from_pretrained(
+            base_model, use_fast=False)
+        model = LlamaForCausalLM.from_pretrained(
+            base_model,
+            torch_dtype=torch.float16)
+        self.model = PeftModel.from_pretrained(
+            model,
+            lora_model,
+            torch_dtype=torch.float16)
+        self.model.to(device)
+
+        self.tokenizer.pad_token_id = 0
+        self.model.config.pad_token_id = 0
+        self.model.config.bos_token_id = 1
+        self.model.config.eos_token_id = 2
+
+        if device == "cpu":
+            self.model.float()
+        else:
+            self.model.half()
+        self.model.eval()
+    
+    @torch.no_grad()
+    def __call__(self, inputs, params):
+        if inputs.endswith('Thought:'):
+            inputs = inputs[:-len('Thought:')]
+        inputs = inputs.replace('Observation:\n\nObservation:', 'Observation:')
+        inputs = inputs + '### ASSISTANT:\n'
+        input_ids = self.tokenizer(inputs, return_tensors="pt").to(self.device).input_ids
+
+        generation_config = GenerationConfig(
+            temperature=self.temperature,
+            top_p=self.top_p,
+            top_k=self.top_k,
+            num_beams=self.num_beams)
+
+        generate_ids = self.model.generate(
+            input_ids=input_ids,
+            generation_config=generation_config,
+            max_new_tokens=self.max_new_tokens)
+        response = self.tokenizer.batch_decode(
+            generate_ids,
+            skip_special_tokens=True,
+            clean_up_tokenization_spaces=False)
+
+        response = [res.replace('### ASSISTANT:\n', '') for res in response]
+        response = [{'generated_text': res} for res in response]
+        return response
+
+
+class Llama(LLM, BaseModel):
+    """Wrapper around LLAMA  models.
+    """
+
+    client: Any  #: :meta private:
+    repo_id: str = DEFAULT_REPO_ID
+    """Model name to use."""
+    task: Optional[str] = "text-generation" 
+    """Task to call the model with. Should be a task that returns `generated_text`."""
+    model_kwargs: Optional[dict] = None
+    """Key word arguments to pass to the model."""
+
+    class Config:
+        """Configuration for this pydantic object."""
+
+        extra = Extra.forbid
+
+    @root_validator()
+    def validate_environment(cls, values: Dict) -> Dict:
+        """Validate that api key and python package exists in environment."""
+        repo_id = values["repo_id"]
+        model_kwargs = values.get("model_kwargs")
+        client = LlamaHuggingFace(
+            base_model=model_kwargs.get("base_model"),
+            lora_model=model_kwargs.get("lora_model"),
+            task=values.get("task"),
+            device=model_kwargs.get("device"),
+            max_new_tokens=model_kwargs.get("max_new_tokens"),
+            temperature=model_kwargs.get("temperature"),
+            top_p=model_kwargs.get("top_p"),
+            top_k=model_kwargs.get("top_k"),
+            num_beams=model_kwargs.get("num_beams")
+        )
+        if client.task not in VALID_TASKS:
+            raise ValueError(
+                f"Got invalid task {client.task}, "
+                f"currently only {VALID_TASKS} are supported"
+            )
+        values["client"] = client
+        return values
+
+    @property
+    def _identifying_params(self) -> Mapping[str, Any]:
+        """Get the identifying parameters."""
+        _model_kwargs = self.model_kwargs or {}
+        return {
+            **{"repo_id": self.repo_id, "task": self.task},
+            **{"model_kwargs": _model_kwargs},
+        }
+
+    @property
+    def _llm_type(self) -> str:
+        """Return type of llm."""
+        return "huggingface_hub"
+
+    def _call(self, prompt: str, stop: Optional[List[str]] = None) -> str:
+        """Call out to HuggingFace Hub's inference endpoint.
+
+        Args:
+            prompt: The prompt to pass into the model.
+            stop: Optional list of stop words to use when generating.
+
+        Returns:
+            The string generated by the model.
+
+        Example:
+            .. code-block:: python
+
+                response = hf("Tell me a joke.")
+        """
+        _model_kwargs = self.model_kwargs or {}
+        response = self.client(inputs=prompt, params=_model_kwargs)
+        if "error" in response:
+            raise ValueError(f"Error raised by inference API: {response['error']}")
+        if self.client.task == "text-generation":
+            # Text generation return includes the starter text.
+            text = response[0]["generated_text"][len(prompt) :]
+        elif self.client.task == "text2text-generation":
+            text = response[0]["generated_text"]
+        else:
+            raise ValueError(
+                f"Got invalid task {self.client.task}, "
+                f"currently only {VALID_TASKS} are supported"
+            )
+        if stop is not None:
+            # This is a bit hacky, but I can't figure out a better way to enforce
+            # stop tokens when making calls to huggingface_hub.
+            text = enforce_stop_tokens(text, stop)
+        return text

lora_finetune.py

@@ -0,0 +1,283 @@
+import os
+import sys
+from typing import List
+
+import fire
+import torch
+import transformers
+from datasets import load_dataset
+
+"""
+Unused imports:
+import torch.nn as nn
+import bitsandbytes as bnb
+"""
+
+from peft import (
+    LoraConfig,
+    get_peft_model,
+    get_peft_model_state_dict,
+    prepare_model_for_int8_training,
+    set_peft_model_state_dict,
+)
+from transformers import LlamaForCausalLM, LlamaTokenizer
+
+from utils.prompter import Prompter
+
+
+def train(
+    # model/data params
+    base_model: str = "",  # the only required argument
+    data_path: str = "yahma/alpaca-cleaned",
+    output_dir: str = "./lora-alpaca",
+    # training hyperparams
+    batch_size: int = 128,
+    micro_batch_size: int = 4,
+    num_epochs: int = 3,
+    learning_rate: float = 3e-4,
+    cutoff_len: int = 256,
+    val_set_size: int = 2000,
+    # lora hyperparams
+    lora_r: int = 8,
+    lora_alpha: int = 16,
+    lora_dropout: float = 0.05,
+    lora_target_modules: List[str] = [
+        "q_proj",
+        "v_proj",
+    ],
+    # llm hyperparams
+    train_on_inputs: bool = True,  # if False, masks out inputs in loss
+    add_eos_token: bool = False,
+    group_by_length: bool = False,  # faster, but produces an odd training loss curve
+    # wandb params
+    wandb_project: str = "",
+    wandb_run_name: str = "",
+    wandb_watch: str = "",  # options: false | gradients | all
+    wandb_log_model: str = "",  # options: false | true
+    resume_from_checkpoint: str = None,  # either training checkpoint or final adapter
+    prompt_template_name: str = "alpaca",  # The prompt template to use, will default to alpaca.
+):
+    if int(os.environ.get("LOCAL_RANK", 0)) == 0:
+        print(
+            f"Training Alpaca-LoRA model with params:\n"
+            f"base_model: {base_model}\n"
+            f"data_path: {data_path}\n"
+            f"output_dir: {output_dir}\n"
+            f"batch_size: {batch_size}\n"
+            f"micro_batch_size: {micro_batch_size}\n"
+            f"num_epochs: {num_epochs}\n"
+            f"learning_rate: {learning_rate}\n"
+            f"cutoff_len: {cutoff_len}\n"
+            f"val_set_size: {val_set_size}\n"
+            f"lora_r: {lora_r}\n"
+            f"lora_alpha: {lora_alpha}\n"
+            f"lora_dropout: {lora_dropout}\n"
+            f"lora_target_modules: {lora_target_modules}\n"
+            f"train_on_inputs: {train_on_inputs}\n"
+            f"add_eos_token: {add_eos_token}\n"
+            f"group_by_length: {group_by_length}\n"
+            f"wandb_project: {wandb_project}\n"
+            f"wandb_run_name: {wandb_run_name}\n"
+            f"wandb_watch: {wandb_watch}\n"
+            f"wandb_log_model: {wandb_log_model}\n"
+            f"resume_from_checkpoint: {resume_from_checkpoint or False}\n"
+            f"prompt template: {prompt_template_name}\n"
+        )
+    assert (
+        base_model
+    ), "Please specify a --base_model, e.g. --base_model='huggyllama/llama-7b'"
+    gradient_accumulation_steps = batch_size // micro_batch_size
+
+    prompter = Prompter(prompt_template_name)
+
+    device_map = "auto"
+    world_size = int(os.environ.get("WORLD_SIZE", 1))
+    ddp = world_size != 1
+    if ddp:
+        device_map = {"": int(os.environ.get("LOCAL_RANK") or 0)}
+        gradient_accumulation_steps = gradient_accumulation_steps // world_size
+
+    # Check if parameter passed or if set within environ
+    use_wandb = len(wandb_project) > 0 or (
+        "WANDB_PROJECT" in os.environ and len(os.environ["WANDB_PROJECT"]) > 0
+    )
+    # Only overwrite environ if wandb param passed
+    if len(wandb_project) > 0:
+        os.environ["WANDB_PROJECT"] = wandb_project
+    if len(wandb_watch) > 0:
+        os.environ["WANDB_WATCH"] = wandb_watch
+    if len(wandb_log_model) > 0:
+        os.environ["WANDB_LOG_MODEL"] = wandb_log_model
+
+    model = LlamaForCausalLM.from_pretrained(
+        base_model,
+        load_in_8bit=True,
+        torch_dtype=torch.float16,
+        device_map=device_map,
+    )
+
+    tokenizer = LlamaTokenizer.from_pretrained(base_model)
+
+    tokenizer.pad_token_id = (
+        0  # unk. we want this to be different from the eos token
+    )
+    tokenizer.padding_side = "left"  # Allow batched inference
+
+    def tokenize(prompt, add_eos_token=True):
+        # there's probably a way to do this with the tokenizer settings
+        # but again, gotta move fast
+        result = tokenizer(
+            prompt,
+            truncation=True,
+            max_length=cutoff_len,
+            padding=False,
+            return_tensors=None,
+        )
+        if (
+            result["input_ids"][-1] != tokenizer.eos_token_id
+            and len(result["input_ids"]) < cutoff_len
+            and add_eos_token
+        ):
+            result["input_ids"].append(tokenizer.eos_token_id)
+            result["attention_mask"].append(1)
+
+        result["labels"] = result["input_ids"].copy()
+
+        return result
+
+    def generate_and_tokenize_prompt(data_point):
+        full_prompt = prompter.generate_prompt(
+            data_point["instruction"],
+            data_point["input"],
+            data_point["output"],
+        )
+        tokenized_full_prompt = tokenize(full_prompt)
+        if not train_on_inputs:
+            user_prompt = prompter.generate_prompt(
+                data_point["instruction"], data_point["input"]
+            )
+            tokenized_user_prompt = tokenize(
+                user_prompt, add_eos_token=add_eos_token
+            )
+            user_prompt_len = len(tokenized_user_prompt["input_ids"])
+
+            if add_eos_token:
+                user_prompt_len -= 1
+
+            tokenized_full_prompt["labels"] = [
+                -100
+            ] * user_prompt_len + tokenized_full_prompt["labels"][
+                user_prompt_len:
+            ]  # could be sped up, probably
+        return tokenized_full_prompt
+
+    model = prepare_model_for_int8_training(model)
+
+    config = LoraConfig(
+        r=lora_r,
+        lora_alpha=lora_alpha,
+        target_modules=lora_target_modules,
+        lora_dropout=lora_dropout,
+        bias="none",
+        task_type="CAUSAL_LM",
+    )
+    model = get_peft_model(model, config)
+
+    if data_path.endswith(".json") or data_path.endswith(".jsonl"):
+        data = load_dataset("json", data_files=data_path)
+    else:
+        data = load_dataset(data_path)
+
+    if resume_from_checkpoint:
+        # Check the available weights and load them
+        checkpoint_name = os.path.join(
+            resume_from_checkpoint, "pytorch_model.bin"
+        )  # Full checkpoint
+        if not os.path.exists(checkpoint_name):
+            checkpoint_name = os.path.join(
+                resume_from_checkpoint, "adapter_model.bin"
+            )  # only LoRA model - LoRA config above has to fit
+            resume_from_checkpoint = (
+                False  # So the trainer won't try loading its state
+            )
+        # The two files above have a different name depending on how they were saved, but are actually the same.
+        if os.path.exists(checkpoint_name):
+            print(f"Restarting from {checkpoint_name}")
+            adapters_weights = torch.load(checkpoint_name)
+            set_peft_model_state_dict(model, adapters_weights)
+        else:
+            print(f"Checkpoint {checkpoint_name} not found")
+
+    model.print_trainable_parameters()  # Be more transparent about the % of trainable params.
+
+    if val_set_size > 0:
+        train_val = data["train"].train_test_split(
+            test_size=val_set_size, shuffle=True, seed=42
+        )
+        train_data = (
+            train_val["train"].shuffle().map(generate_and_tokenize_prompt)
+        )
+        val_data = (
+            train_val["test"].shuffle().map(generate_and_tokenize_prompt)
+        )
+    else:
+        train_data = data["train"].shuffle().map(generate_and_tokenize_prompt)
+        val_data = None
+
+    if not ddp and torch.cuda.device_count() > 1:
+        # keeps Trainer from trying its own DataParallelism when more than 1 gpu is available
+        model.is_parallelizable = True
+        model.model_parallel = True
+
+    trainer = transformers.Trainer(
+        model=model,
+        train_dataset=train_data,
+        eval_dataset=val_data,
+        args=transformers.TrainingArguments(
+            per_device_train_batch_size=micro_batch_size,
+            gradient_accumulation_steps=gradient_accumulation_steps,
+            warmup_steps=100,
+            num_train_epochs=num_epochs,
+            learning_rate=learning_rate,
+            fp16=True,
+            logging_steps=10,
+            optim="adamw_torch",
+            evaluation_strategy="steps" if val_set_size > 0 else "no",
+            save_strategy="steps",
+            eval_steps=200 if val_set_size > 0 else None,
+            save_steps=200,
+            output_dir=output_dir,
+            save_total_limit=3,
+            load_best_model_at_end=True if val_set_size > 0 else False,
+            ddp_find_unused_parameters=False if ddp else None,
+            group_by_length=group_by_length,
+            report_to="wandb" if use_wandb else None,
+            run_name=wandb_run_name if use_wandb else None,
+        ),
+        data_collator=transformers.DataCollatorForSeq2Seq(
+            tokenizer, pad_to_multiple_of=8, return_tensors="pt", padding=True
+        ),
+    )
+    model.config.use_cache = False
+
+    old_state_dict = model.state_dict
+    model.state_dict = (
+        lambda self, *_, **__: get_peft_model_state_dict(
+            self, old_state_dict()
+        )
+    ).__get__(model, type(model))
+
+    if torch.__version__ >= "2" and sys.platform != "win32":
+        model = torch.compile(model)
+
+    trainer.train(resume_from_checkpoint=resume_from_checkpoint)
+
+    model.save_pretrained(output_dir)
+
+    print(
+        "\n If there's a warning about missing keys above, please disregard :)"
+    )
+
+
+if __name__ == "__main__":
+    fire.Fire(train)

requirements.txt

@@ -0,0 +1,41 @@
+langchain==0.0.101
+torch==1.13.1
+torchvision==0.14.1
+wget==3.2
+accelerate
+addict
+albumentations
+basicsr
+controlnet-aux
+diffusers
+einops
+gradio
+imageio
+imageio-ffmpeg
+invisible-watermark
+kornia
+numpy
+omegaconf
+open_clip_torch
+openai
+opencv-python
+prettytable
+safetensors
+streamlit
+test-tube
+timm
+torchmetrics
+webdataset
+yapf
+appdirs
+loralib
+bitsandbytes==0.37.2
+black
+datasets
+fire
+git+https://github.com/huggingface/peft.git@e536616888d51b453ed354a6f1e243fecb02ea08
+transformers>=4.28.0
+sentencepiece
+fire
+git+https://github.com/IDEA-Research/GroundingDINO.git
+git+https://github.com/facebookresearch/segment-anything.git

templates/gpt4tools.json

@@ -0,0 +1,6 @@
+{
+    "description": "A short template for gpt4tools.",
+    "prompt_input": "{instruction}\n### USER: {input}\n### ASSISTANT:\n",
+    "prompt_no_input": "{instruction}\n### ASSISTANT:\n",
+    "response_split": "### ASSISTANT:"    
+}

utils/prompter.py

@@ -0,0 +1,51 @@
+"""
+A dedicated helper to manage templates and prompt building.
+"""
+
+import json
+import os.path as osp
+from typing import Union
+
+
+class Prompter(object):
+    __slots__ = ("template", "_verbose")
+
+    def __init__(self, template_name: str = "", verbose: bool = False):
+        self._verbose = verbose
+        if not template_name:
+            # Enforce the default here, so the constructor can be called with '' and will not break.
+            template_name = "alpaca"
+        file_name = osp.join("templates", f"{template_name}.json")
+        if not osp.exists(file_name):
+            raise ValueError(f"Can't read {file_name}")
+        with open(file_name) as fp:
+            self.template = json.load(fp)
+        if self._verbose:
+            print(
+                f"Using prompt template {template_name}: {self.template['description']}"
+            )
+
+    def generate_prompt(
+        self,
+        instruction: str,
+        input: Union[None, str] = None,
+        label: Union[None, str] = None,
+    ) -> str:
+        # returns the full prompt from instruction and optional input
+        # if a label (=response, =output) is provided, it's also appended.
+        if input:
+            res = self.template["prompt_input"].format(
+                instruction=instruction, input=input
+            )
+        else:
+            res = self.template["prompt_no_input"].format(
+                instruction=instruction
+            )
+        if label:
+            res = f"{res}{label}"
+        if self._verbose:
+            print(res)
+        return res
+
+    def get_response(self, output: str) -> str:
+        return output.split(self.template["response_split"])[1].strip()