diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..218160392c0c005e4547f36aad18a3b62d93d63a 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +agentfabric/modelscope_agent/tools/code_interpreter_utils/AlibabaPuHuiTi-3-45-Light.ttf filter=lfs diff=lfs merge=lfs -text diff --git a/agentfabric/.ipynb_checkpoints/Untitled-checkpoint.ipynb b/agentfabric/.ipynb_checkpoints/Untitled-checkpoint.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..363fcab7ed6e9634e198cf5555ceb88932c9a245 --- /dev/null +++ b/agentfabric/.ipynb_checkpoints/Untitled-checkpoint.ipynb @@ -0,0 +1,6 @@ +{ + "cells": [], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/agentfabric/README.md b/agentfabric/README.md new file mode 100644 index 0000000000000000000000000000000000000000..bbdb9715379d9709c4dfb23abdd5cba502902514 --- /dev/null +++ b/agentfabric/README.md @@ -0,0 +1,64 @@ +--- +# 详细文档见https://modelscope.cn/docs/%E5%88%9B%E7%A9%BA%E9%97%B4%E5%8D%A1%E7%89%87 +domain: #领域：cv/nlp/audio/multi-modal/AutoML +- multi-modal +tags: #自定义标签 + - agent + - AgentFabric + +## 启动文件(若SDK为Gradio/Streamlit，默认为app.py, 若为Static HTML, 默认为index.html) +deployspec: + entry_file: app.py + +license: Apache License 2.0 +--- +

Modelscope AgentFabric: Customizable AI-Agents For All

+ +

+
+ +
+

+ +## Introduction +**ModelScope AgentFabric** is an interactive framework to facilitate creation of agents tailored to various real-world applications. AgentFabric is built around pluggable and customizable LLMs, and enhance capabilities of instrcution following, extra knowledge retrieval and leveraging external tools. The AgentFabric is woven with interfaces including: +- ⚡ **Agent Builder**: an automatic instructions and tools provider for customizing user's agents through natural conversational interactions. +- ⚡ **User Agent**: a customized agent for building real-world applications, with instructions, extra-knowledge and tools provided by builder agent and/or user inputs. +- ⚡ **Configuration Tooling**: the interface to customize user agent configurations. Allows real-time preview of agent behavior as new confiugrations are updated. + +🔗 We currently leverage AgentFabric to build various agents around [Qwen2.0 LLM API](https://help.aliyun.com/zh/dashscope/developer-reference/api-details) available via DashScope. We are also actively exploring +other options to incorporate (and compare) more LLMs via API, as well as via native ModelScope models. + + +## Installation +Simply clone the repo and install dependency. +```bash +git clone https://github.com/modelscope/modelscope-agent.git +cd modelscope-agent && pip install -r requirements.txt && pip install -r demo/agentfabric/requirements.txt +``` + +## Prerequisites + +- Python 3.10 +- Accessibility to LLM API service such as [DashScope](https://help.aliyun.com/zh/dashscope/developer-reference/activate-dashscope-and-create-an-api-key) (free to start). + +## Usage + +```bash +export PYTHONPATH=$PYTHONPATH:/path/to/your/modelscope-agent +export DASHSCOPE_API_KEY=your_api_key +cd modelscope-agent/demo/agentfabric +python app.py +``` + +## 🚀 Roadmap +- [x] Allow customizable agent-building via configurations. +- [x] Agent-building through interactive conversations with LLMs. +- [x] Support multi-user preview on ModelScope space. [link](https://modelscope.cn/studios/wenmengzhou/AgentFabric/summary) [PR #98](https://github.com/modelscope/modelscope-agent/pull/98) +- [x] Optimize knowledge retrival. [PR #105](https://github.com/modelscope/modelscope-agent/pull/105) [PR #107](https://github.com/modelscope/modelscope-agent/pull/107) [PR #109](https://github.com/modelscope/modelscope-agent/pull/109) +- [x] Allow publication and sharing of agent. [PR #111](https://github.com/modelscope/modelscope-agent/pull/111) +- [ ] Support more pluggable LLMs via API or ModelScope interface. +- [ ] Improve long context via memory. +- [ ] Improve logging and profiling. +- [ ] Fine-tuning for specific agent. +- [ ] Evaluation for agents in different scenarios. diff --git a/agentfabric/README_CN.md b/agentfabric/README_CN.md new file mode 100644 index 0000000000000000000000000000000000000000..547e4d44b25b1ca37d1fb5064ec27a6bc9e923fb --- /dev/null +++ b/agentfabric/README_CN.md @@ -0,0 +1,52 @@ + +

Modelscope AgentFabric: 开放可定制的AI智能体构建框架

+ +

+
+ +
+

+ +## 介绍 + +**Modelscope AgentFabric**是一个交互式智能体框架，用于方便地创建针对各种现实应用量身定制智能体。AgentFabric围绕可插拔和可定制的LLM构建，并增强了指令执行、额外知识检索和利用外部工具的能力。AgentFabric提供的交互界面包括： +- **⚡ 智能体构建器**：一个自动指令和工具提供者，通过与用户聊天来定制用户的智能体 +- **⚡ 用户智能体**：一个为用户的实际应用定制的智能体，提供构建智能体或用户输入的指令、额外知识和工具 +- **⚡ 配置设置工具**：支持用户定制用户智能体的配置，并实时预览用户智能体的性能 + +🔗 我们目前围绕DashScope提供的 [Qwen2.0 LLM API](https://help.aliyun.com/zh/dashscope/developer-reference/api-details) 来在AgentFabric上构建不同的智能体应用。同时我们正在积极探索，通过API或者ModelScope原生模型等方式，引入不同的举办强大基础能力的LLMs，来构建丰富多样的Agents。 + +## 安装 + +克隆仓库并安装依赖： + +```bash +git clone https://github.com/modelscope/modelscope-agent.git +cd modelscope-agent && pip install -r requirements.txt && pip install -r demo/agentfabric/requirements.txt +``` + +## 前提条件 + +- Python 3.10 +- 获取使用Qwen 2.0模型所需的API-key，可从[DashScope](https://help.aliyun.com/zh/dashscope/developer-reference/activate-dashscope-and-create-an-api-key)免费开通和获取。 + +## 使用方法 + +```bash +export PYTHONPATH=$PYTHONPATH:/path/to/your/modelscope-agent +export DASHSCOPE_API_KEY=your_api_key +cd modelscope-agent/demo/agentfabric +python app.py +``` + +## 🚀 发展路线规划 +- [x] 支持人工配置构建智能体 +- [x] 基于LLM对话构建智能体 +- [x] 支持在ModelScope创空间上使用 [link](https://modelscope.cn/studios/wenmengzhou/AgentFabric/summary) [PR #98](https://github.com/modelscope/modelscope-agent/pull/98) +- [x] 知识库检索效果优化 [PR #105](https://github.com/modelscope/modelscope-agent/pull/105) [PR #107](https://github.com/modelscope/modelscope-agent/pull/107) [PR #109](https://github.com/modelscope/modelscope-agent/pull/109) +- [x] 支持智能体发布和分享 +- [ ] 支持其他多种LLM模型API和ModelScope模型 +- [ ] 处理长文本输入到内存 +- [ ] 生产级支持：日志和性能分析 +- [ ] 支持智能体微调 +- [ ] 在不同场景中智能体的效果评估 diff --git a/agentfabric/Untitled.ipynb b/agentfabric/Untitled.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..6e63092bd56947216614a3a755daf72fbc69bae6 --- /dev/null +++ b/agentfabric/Untitled.ipynb @@ -0,0 +1,172 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "id": "52e2928d-b216-4d5c-a198-b24b3bc6d5a6", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/workspace/modelscope-agent/apps/agentfabric\n" + ] + } + ], + "source": [ + "cd agentfabric" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "84bdd7ce-176d-464d-b0b3-620585c22541", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Collecting huggingface_hub\n", + " Downloading huggingface_hub-0.19.4-py3-none-any.whl.metadata (14 kB)\n", + "Requirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from huggingface_hub) (3.9.0)\n", + "Requirement already satisfied: fsspec>=2023.5.0 in /opt/conda/lib/python3.10/site-packages (from huggingface_hub) (2023.10.0)\n", + "Requirement already satisfied: requests in /opt/conda/lib/python3.10/site-packages (from huggingface_hub) (2.31.0)\n", + "Requirement already satisfied: tqdm>=4.42.1 in /opt/conda/lib/python3.10/site-packages (from huggingface_hub) (4.65.0)\n", + "Requirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.10/site-packages (from huggingface_hub) (6.0.1)\n", + "Requirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.10/site-packages (from huggingface_hub) (4.7.1)\n", + "Requirement already satisfied: packaging>=20.9 in /opt/conda/lib/python3.10/site-packages (from huggingface_hub) (23.1)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface_hub) (2.0.4)\n", + "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface_hub) (3.4)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface_hub) (1.26.18)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface_hub) (2023.7.22)\n", + "Downloading huggingface_hub-0.19.4-py3-none-any.whl (311 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m311.7/311.7 kB\u001b[0m \u001b[31m4.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", + "\u001b[?25hInstalling collected packages: huggingface_hub\n", + "Successfully installed huggingface_hub-0.19.4\n", + "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n", + "\u001b[0mNote: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "pip install huggingface_hub" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "f7ebcbea-3bb2-468f-b159-ff20693e98c8", + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "58fa0d78829446c98f1facc6250b2d5b", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "VBox(children=(HTML(value='

270\u001b[0m \u001b[43mresponse\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mraise_for_status\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 271\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m HTTPError \u001b[38;5;28;01mas\u001b[39;00m e:\n", + "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/requests/models.py:1021\u001b[0m, in \u001b[0;36mResponse.raise_for_status\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1020\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m http_error_msg:\n\u001b[0;32m-> 1021\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m HTTPError(http_error_msg, response\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m)\n", + "\u001b[0;31mHTTPError\u001b[0m: 400 Client Error: Bad Request for url: https://huggingface.co/api/spaces/kevinwang676/AI-Agent/commit/main", + "\nThe above exception was the direct cause of the following exception:\n", + "\u001b[0;31mBadRequestError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[5], line 4\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mhuggingface_hub\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m HfApi\n\u001b[1;32m 2\u001b[0m api \u001b[38;5;241m=\u001b[39m HfApi()\n\u001b[0;32m----> 4\u001b[0m \u001b[43mapi\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mupload_folder\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 5\u001b[0m \u001b[43m \u001b[49m\u001b[43mfolder_path\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 6\u001b[0m \u001b[43m \u001b[49m\u001b[43mrepo_id\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mkevinwang676/AI-Agent\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 7\u001b[0m \u001b[43m \u001b[49m\u001b[43mrepo_type\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mspace\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 8\u001b[0m \u001b[43m)\u001b[49m\n", + "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/huggingface_hub/utils/_validators.py:118\u001b[0m, in \u001b[0;36mvalidate_hf_hub_args.._inner_fn\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 115\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m check_use_auth_token:\n\u001b[1;32m 116\u001b[0m kwargs \u001b[38;5;241m=\u001b[39m smoothly_deprecate_use_auth_token(fn_name\u001b[38;5;241m=\u001b[39mfn\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m, has_token\u001b[38;5;241m=\u001b[39mhas_token, kwargs\u001b[38;5;241m=\u001b[39mkwargs)\n\u001b[0;32m--> 118\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/huggingface_hub/hf_api.py:1045\u001b[0m, in \u001b[0;36mfuture_compatible.._inner\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1042\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mrun_as_future(fn, \u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m 1044\u001b[0m \u001b[38;5;66;03m# Otherwise, call the function normally\u001b[39;00m\n\u001b[0;32m-> 1045\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/huggingface_hub/hf_api.py:4138\u001b[0m, in \u001b[0;36mHfApi.upload_folder\u001b[0;34m(self, repo_id, folder_path, path_in_repo, commit_message, commit_description, token, repo_type, revision, create_pr, parent_commit, allow_patterns, ignore_patterns, delete_patterns, multi_commits, multi_commits_verbose, run_as_future)\u001b[0m\n\u001b[1;32m 4126\u001b[0m pr_url \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcreate_commits_on_pr(\n\u001b[1;32m 4127\u001b[0m repo_id\u001b[38;5;241m=\u001b[39mrepo_id,\n\u001b[1;32m 4128\u001b[0m repo_type\u001b[38;5;241m=\u001b[39mrepo_type,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 4135\u001b[0m verbose\u001b[38;5;241m=\u001b[39mmulti_commits_verbose,\n\u001b[1;32m 4136\u001b[0m )\n\u001b[1;32m 4137\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 4138\u001b[0m commit_info \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcreate_commit\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 4139\u001b[0m \u001b[43m \u001b[49m\u001b[43mrepo_type\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrepo_type\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 4140\u001b[0m \u001b[43m \u001b[49m\u001b[43mrepo_id\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrepo_id\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 4141\u001b[0m \u001b[43m \u001b[49m\u001b[43moperations\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcommit_operations\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 4142\u001b[0m \u001b[43m \u001b[49m\u001b[43mcommit_message\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcommit_message\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 4143\u001b[0m \u001b[43m \u001b[49m\u001b[43mcommit_description\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcommit_description\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 4144\u001b[0m \u001b[43m \u001b[49m\u001b[43mtoken\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtoken\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 4145\u001b[0m \u001b[43m \u001b[49m\u001b[43mrevision\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrevision\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 4146\u001b[0m \u001b[43m \u001b[49m\u001b[43mcreate_pr\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcreate_pr\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 4147\u001b[0m \u001b[43m \u001b[49m\u001b[43mparent_commit\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mparent_commit\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 4148\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 4149\u001b[0m pr_url \u001b[38;5;241m=\u001b[39m commit_info\u001b[38;5;241m.\u001b[39mpr_url\n\u001b[1;32m 4151\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m create_pr \u001b[38;5;129;01mand\u001b[39;00m pr_url \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n", + "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/huggingface_hub/utils/_validators.py:118\u001b[0m, in \u001b[0;36mvalidate_hf_hub_args.._inner_fn\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 115\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m check_use_auth_token:\n\u001b[1;32m 116\u001b[0m kwargs \u001b[38;5;241m=\u001b[39m smoothly_deprecate_use_auth_token(fn_name\u001b[38;5;241m=\u001b[39mfn\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m, has_token\u001b[38;5;241m=\u001b[39mhas_token, kwargs\u001b[38;5;241m=\u001b[39mkwargs)\n\u001b[0;32m--> 118\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/huggingface_hub/hf_api.py:1045\u001b[0m, in \u001b[0;36mfuture_compatible.._inner\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1042\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mrun_as_future(fn, \u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m 1044\u001b[0m \u001b[38;5;66;03m# Otherwise, call the function normally\u001b[39;00m\n\u001b[0;32m-> 1045\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/huggingface_hub/hf_api.py:3237\u001b[0m, in \u001b[0;36mHfApi.create_commit\u001b[0;34m(self, repo_id, operations, commit_message, commit_description, token, repo_type, revision, create_pr, num_threads, parent_commit, run_as_future)\u001b[0m\n\u001b[1;32m 3235\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 3236\u001b[0m commit_resp \u001b[38;5;241m=\u001b[39m get_session()\u001b[38;5;241m.\u001b[39mpost(url\u001b[38;5;241m=\u001b[39mcommit_url, headers\u001b[38;5;241m=\u001b[39mheaders, data\u001b[38;5;241m=\u001b[39mdata, params\u001b[38;5;241m=\u001b[39mparams)\n\u001b[0;32m-> 3237\u001b[0m \u001b[43mhf_raise_for_status\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcommit_resp\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mendpoint_name\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcommit\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 3238\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m RepositoryNotFoundError \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m 3239\u001b[0m e\u001b[38;5;241m.\u001b[39mappend_to_message(_CREATE_COMMIT_NO_REPO_ERROR_MESSAGE)\n", + "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/huggingface_hub/utils/_errors.py:326\u001b[0m, in \u001b[0;36mhf_raise_for_status\u001b[0;34m(response, endpoint_name)\u001b[0m\n\u001b[1;32m 322\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m response\u001b[38;5;241m.\u001b[39mstatus_code \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m400\u001b[39m:\n\u001b[1;32m 323\u001b[0m message \u001b[38;5;241m=\u001b[39m (\n\u001b[1;32m 324\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124mBad request for \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mendpoint_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m endpoint:\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m endpoint_name \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124mBad request:\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 325\u001b[0m )\n\u001b[0;32m--> 326\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m BadRequestError(message, response\u001b[38;5;241m=\u001b[39mresponse) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01me\u001b[39;00m\n\u001b[1;32m 328\u001b[0m \u001b[38;5;66;03m# Convert `HTTPError` into a `HfHubHTTPError` to display request information\u001b[39;00m\n\u001b[1;32m 329\u001b[0m \u001b[38;5;66;03m# as well (request id and/or server error message)\u001b[39;00m\n\u001b[1;32m 330\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m HfHubHTTPError(\u001b[38;5;28mstr\u001b[39m(e), response\u001b[38;5;241m=\u001b[39mresponse) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01me\u001b[39;00m\n", + "\u001b[0;31mBadRequestError\u001b[0m: (Request ID: Root=1-6575633c-3007e4fc53be037b5ceb5402;4cd41d1e-0094-48a4-bbe9-db4d06cbc023)\n\nBad request for commit endpoint:\n\"license\" must be one of [apache-2.0, mit, openrail, bigscience-openrail-m, creativeml-openrail-m, bigscience-bloom-rail-1.0, bigcode-openrail-m, afl-3.0, artistic-2.0, bsl-1.0, bsd, bsd-2-clause, bsd-3-clause, bsd-3-clause-clear, c-uda, cc, cc0-1.0, cc-by-2.0, cc-by-2.5, cc-by-3.0, cc-by-4.0, cc-by-sa-3.0, cc-by-sa-4.0, cc-by-nc-2.0, cc-by-nc-3.0, cc-by-nc-4.0, cc-by-nd-4.0, cc-by-nc-nd-3.0, cc-by-nc-nd-4.0, cc-by-nc-sa-2.0, cc-by-nc-sa-3.0, cc-by-nc-sa-4.0, cdla-sharing-1.0, cdla-permissive-1.0, cdla-permissive-2.0, wtfpl, ecl-2.0, epl-1.0, epl-2.0, eupl-1.1, agpl-3.0, gfdl, gpl, gpl-2.0, gpl-3.0, lgpl, lgpl-2.1, lgpl-3.0, isc, lppl-1.3c, ms-pl, mpl-2.0, odc-by, odbl, openrail++, osl-3.0, postgresql, ofl-1.1, ncsa, unlicense, zlib, pddl, lgpl-lr, deepfloyd-if-license, llama2, unknown, other, array]" + ] + } + ], + "source": [ + "from huggingface_hub import HfApi\n", + "api = HfApi()\n", + "\n", + "api.upload_folder(\n", + " folder_path=\"\",\n", + " repo_id=\"kevinwang676/AI-Agent\",\n", + " repo_type=\"space\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2ee3f7dc-08f8-4675-b48a-a61095caf08b", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/agentfabric/__init__.py b/agentfabric/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/agentfabric/app.py b/agentfabric/app.py new file mode 100644 index 0000000000000000000000000000000000000000..b3d5e5897c0d31b5b871b2bc9f4952521a2b24a5 --- /dev/null +++ b/agentfabric/app.py @@ -0,0 +1,660 @@ +import os +import random +import re +import shutil +import traceback + +import gradio as gr +import json +import yaml +from builder_core import beauty_output, init_builder_chatbot_agent +from config_utils import (DEFAULT_AGENT_DIR, Config, get_avatar_image, + get_ci_dir, get_user_cfg_file, get_user_dir, + is_valid_plugin_configuration, parse_configuration, + save_avatar_image, save_builder_configuration, + save_plugin_configuration) +from gradio_utils import ChatBot, format_cover_html, format_goto_publish_html +from i18n import I18n +from publish_util import pop_user_info_from_config, prepare_agent_zip +from user_core import init_user_chatbot_agent + + +def init_user(uuid_str, state): + try: + seed = state.get('session_seed', random.randint(0, 1000000000)) + user_agent = init_user_chatbot_agent(uuid_str) + user_agent.seed = seed + state['user_agent'] = user_agent + except Exception as e: + error = traceback.format_exc() + print(f'Error:{e}, with detail: {error}') + return state + + +def init_builder(uuid_str, state): + + try: + builder_agent = init_builder_chatbot_agent(uuid_str) + state['builder_agent'] = builder_agent + except Exception as e: + error = traceback.format_exc() + print(f'Error:{e}, with detail: {error}') + return state + + +def update_builder(uuid_str, state): + builder_agent = state['builder_agent'] + + try: + builder_cfg_file = get_user_cfg_file(uuid_str=uuid_str) + with open(builder_cfg_file, 'r') as f: + config = json.load(f) + builder_agent.update_config_to_history(config) + except Exception as e: + error = traceback.format_exc() + print(f'Error:{e}, with detail: {error}') + return state + + +def check_uuid(uuid_str): + if not uuid_str or uuid_str == '': + if os.getenv('MODELSCOPE_ENVIRONMENT') == 'studio': + raise gr.Error('请登陆后使用! (Please login first)') + else: + uuid_str = 'local_user' + return uuid_str + + +def process_configuration(uuid_str, bot_avatar, name, description, + instructions, model, suggestions, knowledge_files, + capabilities_checkboxes, openapi_schema, + openapi_auth, openapi_auth_apikey, + openapi_auth_apikey_type, openapi_privacy_policy, + state): + uuid_str = check_uuid(uuid_str) + tool_cfg = state['tool_cfg'] + capabilities = state['capabilities'] + bot_avatar, bot_avatar_path = save_avatar_image(bot_avatar, uuid_str) + suggestions_filtered = [row for row in suggestions if row[0]] + user_dir = get_user_dir(uuid_str) + if knowledge_files is not None: + new_knowledge_files = [ + os.path.join(user_dir, os.path.basename((f.name))) + for f in knowledge_files + ] + for src_file, dst_file in zip(knowledge_files, new_knowledge_files): + if not os.path.exists(dst_file): + shutil.copy(src_file.name, dst_file) + else: + new_knowledge_files = [] + + builder_cfg = { + 'name': name, + 'avatar': bot_avatar, + 'description': description, + 'instruction': instructions, + 'prompt_recommend': [row[0] for row in suggestions_filtered], + 'knowledge': new_knowledge_files, + 'tools': { + capability: dict( + name=tool_cfg[capability]['name'], + is_active=tool_cfg[capability]['is_active'], + use=True if capability in capabilities_checkboxes else False) + for capability in map(lambda item: item[1], capabilities) + }, + 'model': model, + } + + try: + try: + schema_dict = json.loads(openapi_schema) + except json.decoder.JSONDecodeError: + schema_dict = yaml.safe_load(openapi_schema) + except Exception as e: + raise gr.Error( + f'OpenAPI schema format error, should be one of json and yaml: {e}' + ) + + openapi_plugin_cfg = { + 'schema': schema_dict, + 'auth': { + 'type': openapi_auth, + 'apikey': openapi_auth_apikey, + 'apikey_type': openapi_auth_apikey_type + }, + 'privacy_policy': openapi_privacy_policy + } + if is_valid_plugin_configuration(openapi_plugin_cfg): + save_plugin_configuration(openapi_plugin_cfg, uuid_str) + except Exception as e: + error = traceback.format_exc() + print(f'Error:{e}, with detail: {error}') + + save_builder_configuration(builder_cfg, uuid_str) + update_builder(uuid_str, state) + init_user(uuid_str, state) + return [ + gr.HTML.update( + visible=True, + value=format_cover_html(builder_cfg, bot_avatar_path)), + gr.Chatbot.update( + visible=False, + avatar_images=get_avatar_image(bot_avatar, uuid_str)), + gr.Dataset.update(samples=suggestions_filtered), + gr.DataFrame.update(value=suggestions_filtered) + ] + + +# 创建 Gradio 界面 +demo = gr.Blocks(css='assets/app.css') +with demo: + + uuid_str = gr.Textbox(label='modelscope_uuid', visible=False) + draw_seed = random.randint(0, 1000000000) + state = gr.State({'session_seed': draw_seed}) + i18n = I18n('zh-cn') + with gr.Row(): + with gr.Column(scale=5): + header = gr.Markdown(i18n.get('header')) + with gr.Column(scale=1): + language = gr.Dropdown( + choices=[('中文', 'zh-cn'), ('English', 'en')], + show_label=False, + container=False, + value='zh-cn', + interactive=True) + with gr.Row(): + with gr.Column(): + with gr.Tabs() as tabs: + with gr.Tab(i18n.get_whole('create'), id=0) as create_tab: + with gr.Column(): + # "Create" 标签页的 Chatbot 组件 + start_text = '欢迎使用agent创建助手。我可以帮助您创建一个定制agent。'\ + '您希望您的agent主要用于什么领域或任务？比如，您可以说，我想做一个RPG游戏agent' + create_chatbot = gr.Chatbot( + show_label=False, value=[[None, start_text]]) + create_chat_input = gr.Textbox( + label=i18n.get('message'), + placeholder=i18n.get('message_placeholder')) + create_send_button = gr.Button( + i18n.get('sendOnLoading'), interactive=False) + + configure_tab = gr.Tab(i18n.get_whole('configure'), id=1) + with configure_tab: + with gr.Column(): + # "Configure" 标签页的配置输入字段 + with gr.Row(): + bot_avatar_comp = gr.Image( + label=i18n.get('form_avatar'), + placeholder='Chatbot avatar image', + source='upload', + interactive=True, + type='filepath', + scale=1, + width=182, + height=182, + ) + with gr.Column(scale=4): + name_input = gr.Textbox( + label=i18n.get('form_name'), + placeholder=i18n.get( + 'form_name_placeholder')) + description_input = gr.Textbox( + label=i18n.get('form_description'), + placeholder=i18n.get( + 'form_description_placeholder')) + + instructions_input = gr.Textbox( + label=i18n.get('form_instructions'), + placeholder=i18n.get( + 'form_instructions_placeholder'), + lines=3) + model_selector = model_selector = gr.Dropdown( + label=i18n.get('form_model')) + suggestion_input = gr.Dataframe( + show_label=False, + value=[['']], + datatype=['str'], + headers=[i18n.get_whole('form_prompt_suggestion')], + type='array', + col_count=(1, 'fixed'), + interactive=True) + knowledge_input = gr.File( + label=i18n.get('form_knowledge'), + file_count='multiple', + file_types=['text', '.json', '.csv', '.pdf']) + capabilities_checkboxes = gr.CheckboxGroup( + label=i18n.get('form_capabilities')) + + with gr.Accordion( + i18n.get('open_api_accordion'), + open=False) as open_api_accordion: + openapi_schema = gr.Textbox( + label='Schema', + placeholder= + 'Enter your OpenAPI schema here, JSON or YAML format only' + ) + + with gr.Group(): + openapi_auth_type = gr.Radio( + label='Authentication Type', + choices=['None', 'API Key'], + value='None') + openapi_auth_apikey = gr.Textbox( + label='API Key', + placeholder='Enter your API Key here') + openapi_auth_apikey_type = gr.Radio( + label='API Key type', choices=['Bearer']) + openapi_privacy_policy = gr.Textbox( + label='Privacy Policy', + placeholder='Enter privacy policy URL') + + configure_button = gr.Button( + i18n.get('form_update_button')) + + with gr.Column(): + # Preview + preview_header = gr.HTML( + f"""
{i18n.get('preview')}
""") + + user_chat_bot_cover = gr.HTML(format_cover_html({}, None)) + user_chatbot = ChatBot( + value=[[None, None]], + elem_id='user_chatbot', + elem_classes=['markdown-body'], + avatar_images=get_avatar_image('', uuid_str), + height=650, + latex_delimiters=[], + show_label=False, + visible=False) + preview_chat_input = gr.Textbox( + label=i18n.get('message'), + placeholder=i18n.get('message_placeholder')) + user_chat_bot_suggest = gr.Dataset( + label=i18n.get('prompt_suggestion'), + components=[preview_chat_input], + samples=[]) + # preview_send_button = gr.Button('Send') + with gr.Row(): + upload_button = gr.UploadButton( + i18n.get('upload_btn'), + file_types=[ + '.csv', '.doc', '.docx', '.xls', '.xlsx', '.txt', + '.md', '.pdf', '.jpeg', '.png', '.jpg', '.gif' + ], + file_count='multiple') + preview_send_button = gr.Button( + i18n.get('sendOnLoading'), interactive=False) + user_chat_bot_suggest.select( + lambda evt: evt[0], + inputs=[user_chat_bot_suggest], + outputs=[preview_chat_input]) + with gr.Accordion( + label=i18n.get('publish'), + open=False) as publish_accordion: + with gr.Row(): + with gr.Column(): + publish_button = gr.Button(i18n.get_whole('build')) + gr.Markdown(f'#### 1.{i18n.get_whole("build_hint")}') + + with gr.Column(): + publish_link = gr.HTML( + value=format_goto_publish_html( + i18n.get_whole('publish'), '', {}, True)) + gr.Markdown(f'#### 2.{i18n.get_whole("publish_hint")}') + + configure_updated_outputs = [ + state, + # config form + bot_avatar_comp, + name_input, + description_input, + instructions_input, + model_selector, + suggestion_input, + knowledge_input, + capabilities_checkboxes, + # bot + user_chat_bot_cover, + user_chat_bot_suggest, + preview_send_button, + create_send_button, + ] + + # 初始化表单 + def init_ui_config(uuid_str, _state, builder_cfg, model_cfg, tool_cfg): + print('builder_cfg:', builder_cfg) + # available models + models = list(model_cfg.keys()) + capabilities = [(tool_cfg[tool_key]['name'], tool_key) + for tool_key in tool_cfg.keys() + if tool_cfg[tool_key].get('is_active', False)] + _state['model_cfg'] = model_cfg + _state['tool_cfg'] = tool_cfg + _state['capabilities'] = capabilities + bot_avatar = get_avatar_image(builder_cfg.get('avatar', ''), + uuid_str)[1] + suggests = builder_cfg.get('prompt_recommend', []) + return { + state: + _state, + bot_avatar_comp: + gr.Image.update(value=bot_avatar), + name_input: + builder_cfg.get('name', ''), + description_input: + builder_cfg.get('description'), + instructions_input: + builder_cfg.get('instruction'), + model_selector: + gr.Dropdown.update( + value=builder_cfg.get('model', models[0]), choices=models), + suggestion_input: [[str] for str in suggests], + knowledge_input: + builder_cfg.get('knowledge', []) + if len(builder_cfg['knowledge']) > 0 else None, + capabilities_checkboxes: + gr.CheckboxGroup.update( + value=[ + tool for tool in builder_cfg.get('tools', {}).keys() + if builder_cfg.get('tools').get(tool).get('use', False) + ], + choices=capabilities), + # bot + user_chat_bot_cover: + format_cover_html(builder_cfg, bot_avatar), + user_chat_bot_suggest: + gr.Dataset.update(samples=[[item] for item in suggests]), + } + + # tab 切换的事件处理 + def on_congifure_tab_select(_state, uuid_str): + uuid_str = check_uuid(uuid_str) + configure_updated = _state.get('configure_updated', False) + if configure_updated: + builder_cfg, model_cfg, tool_cfg, available_tool_list, _, _ = parse_configuration( + uuid_str) + _state['configure_updated'] = False + return init_ui_config(uuid_str, _state, builder_cfg, model_cfg, + tool_cfg) + else: + return {state: _state} + + configure_tab.select( + on_congifure_tab_select, + inputs=[state, uuid_str], + outputs=configure_updated_outputs) + + # 配置 "Create" 标签页的消息发送功能 + def format_message_with_builder_cfg(_state, chatbot, builder_cfg, + uuid_str): + uuid_str = check_uuid(uuid_str) + bot_avatar = builder_cfg.get('avatar', '') + prompt_recommend = builder_cfg.get('prompt_recommend', []) + suggestion = [[row] for row in prompt_recommend] + bot_avatar_path = get_avatar_image(bot_avatar, uuid_str)[1] + save_builder_configuration(builder_cfg, uuid_str) + _state['configure_updated'] = True + return { + create_chatbot: + chatbot, + user_chat_bot_cover: + gr.HTML.update( + visible=True, + value=format_cover_html(builder_cfg, bot_avatar_path)), + user_chatbot: + gr.Chatbot.update( + visible=False, + avatar_images=get_avatar_image(bot_avatar, uuid_str)), + user_chat_bot_suggest: + gr.Dataset.update(samples=suggestion) + } + + def create_send_message(chatbot, input, _state, uuid_str): + uuid_str = check_uuid(uuid_str) + # 将发送的消息添加到聊天历史 + builder_agent = _state['builder_agent'] + chatbot.append((input, '')) + yield { + create_chatbot: chatbot, + create_chat_input: gr.Textbox.update(value=''), + } + response = '' + for frame in builder_agent.stream_run( + input, print_info=True, uuid_str=uuid_str): + llm_result = frame.get('llm_text', '') + exec_result = frame.get('exec_result', '') + step_result = frame.get('step', '') + print(frame) + if len(exec_result) != 0: + if isinstance(exec_result, dict): + exec_result = exec_result['result'] + assert isinstance(exec_result, Config) + yield format_message_with_builder_cfg( + _state, + chatbot, + exec_result.to_dict(), + uuid_str=uuid_str) + else: + # llm result + if isinstance(llm_result, dict): + content = llm_result['content'] + else: + content = llm_result + frame_text = content + response = beauty_output(f'{response}{frame_text}', + step_result) + chatbot[-1] = (input, response) + yield { + create_chatbot: chatbot, + } + + create_send_button.click( + create_send_message, + inputs=[create_chatbot, create_chat_input, state, uuid_str], + outputs=[ + create_chatbot, user_chat_bot_cover, user_chatbot, + user_chat_bot_suggest, create_chat_input + ]) + + # 配置 "Configure" 标签页的提交按钮功能 + configure_button.click( + process_configuration, + inputs=[ + uuid_str, bot_avatar_comp, name_input, description_input, + instructions_input, model_selector, suggestion_input, + knowledge_input, capabilities_checkboxes, openapi_schema, + openapi_auth_type, openapi_auth_apikey, openapi_auth_apikey_type, + openapi_privacy_policy, state + ], + outputs=[ + user_chat_bot_cover, user_chatbot, user_chat_bot_suggest, + suggestion_input + ]) + + # 配置 "Preview" 的消息发送功能 + def preview_send_message(chatbot, input, _state): + # 将发送的消息添加到聊天历史 + user_agent = _state['user_agent'] + if 'new_file_paths' in _state: + new_file_paths = _state['new_file_paths'] + else: + new_file_paths = [] + _state['new_file_paths'] = [] + + chatbot.append((input, '')) + yield { + user_chatbot: gr.Chatbot.update(visible=True, value=chatbot), + user_chat_bot_cover: gr.HTML.update(visible=False), + preview_chat_input: gr.Textbox.update(value='') + } + + response = '' + try: + for frame in user_agent.stream_run( + input, + print_info=True, + remote=False, + append_files=new_file_paths): + llm_result = frame.get('llm_text', '') + exec_result = frame.get('exec_result', '') + if len(exec_result) != 0: + # action_exec_result + if isinstance(exec_result, dict): + exec_result = str(exec_result['result']) + frame_text = f'{exec_result}' + else: + # llm result + frame_text = llm_result + + # important! do not change this + response += frame_text + chatbot[-1] = (input, response) + yield {user_chatbot: chatbot} + except Exception as e: + if 'dashscope.common.error.AuthenticationError' in str(e): + msg = 'DASHSCOPE_API_KEY should be set via environment variable. You can acquire this in ' \ + 'https://help.aliyun.com/zh/dashscope/developer-reference/activate-dashscope-and-create-an-api-key' + else: + msg = str(e) + chatbot[-1] = (input, msg) + yield {user_chatbot: chatbot} + + preview_send_button.click( + preview_send_message, + inputs=[user_chatbot, preview_chat_input, state], + outputs=[user_chatbot, user_chat_bot_cover, preview_chat_input]) + + def upload_file(chatbot, upload_button, _state, uuid_str): + uuid_str = check_uuid(uuid_str) + new_file_paths = [] + if 'file_paths' in _state: + file_paths = _state['file_paths'] + else: + file_paths = [] + for file in upload_button: + file_name = os.path.basename(file.name) + # covert xxx.json to xxx_uuid_str.json + file_name = file_name.replace('.', f'_{uuid_str}.') + file_path = os.path.join(get_ci_dir(), file_name) + if not os.path.exists(file_path): + # make sure file path's directory exists + os.makedirs(os.path.dirname(file_path), exist_ok=True) + shutil.copy(file.name, file_path) + file_paths.append(file_path) + new_file_paths.append(file_path) + chatbot.append((None, f'上传文件{file_name}，成功')) + yield { + user_chatbot: gr.Chatbot.update(visible=True, value=chatbot), + user_chat_bot_cover: gr.HTML.update(visible=False), + preview_chat_input: gr.Textbox.update(value='') + } + + _state['file_paths'] = file_paths + _state['new_file_paths'] = new_file_paths + + upload_button.upload( + upload_file, + inputs=[user_chatbot, upload_button, state, uuid_str], + outputs=[user_chatbot, user_chat_bot_cover, preview_chat_input]) + + # configuration for publish + def publish_agent(name, uuid_str, state): + uuid_str = check_uuid(uuid_str) + user_info = pop_user_info_from_config(DEFAULT_AGENT_DIR, uuid_str) + output_url = prepare_agent_zip(name, DEFAULT_AGENT_DIR, uuid_str, + state) + # output_url = "https://test.url" + return format_goto_publish_html( + i18n.get_whole('publish'), output_url, user_info) + + publish_button.click( + publish_agent, + inputs=[name_input, uuid_str, state], + outputs=[publish_link], + ) + + def change_lang(language): + i18n = I18n(language) + return { + bot_avatar_comp: + gr.Image(label=i18n.get('form_avatar')), + name_input: + gr.Textbox( + label=i18n.get('form_name'), + placeholder=i18n.get('form_name_placeholder')), + description_input: + gr.Textbox( + label=i18n.get('form_description'), + placeholder=i18n.get('form_description_placeholder')), + instructions_input: + gr.Textbox( + label=i18n.get('form_instructions'), + placeholder=i18n.get('form_instructions_placeholder')), + model_selector: + gr.Dropdown(label=i18n.get('form_model')), + knowledge_input: + gr.File(label=i18n.get('form_knowledge')), + capabilities_checkboxes: + gr.CheckboxGroup(label=i18n.get('form_capabilities')), + open_api_accordion: + gr.Accordion(label=i18n.get('open_api_accordion')), + configure_button: + gr.Button(i18n.get('form_update_button')), + preview_header: + gr.HTML( + f"""
{i18n.get('preview')}
"""), + preview_send_button: + gr.Button.update(value=i18n.get('send')), + create_chat_input: + gr.Textbox( + label=i18n.get('message'), + placeholder=i18n.get('message_placeholder')), + create_send_button: + gr.Button.update(value=i18n.get('send')), + user_chat_bot_suggest: + gr.Dataset(label=i18n.get('prompt_suggestion')), + preview_chat_input: + gr.Textbox( + label=i18n.get('message'), + placeholder=i18n.get('message_placeholder')), + publish_accordion: + gr.Accordion(label=i18n.get('publish')), + upload_button: + gr.UploadButton(i18n.get('upload_btn')), + header: + gr.Markdown(i18n.get('header')), + } + + language.select( + change_lang, + inputs=[language], + outputs=configure_updated_outputs + [ + configure_button, create_chat_input, open_api_accordion, + preview_header, preview_chat_input, publish_accordion, + upload_button, header + ]) + + def init_all(uuid_str, _state): + uuid_str = check_uuid(uuid_str) + builder_cfg, model_cfg, tool_cfg, available_tool_list, _, _ = parse_configuration( + uuid_str) + ret = init_ui_config(uuid_str, _state, builder_cfg, model_cfg, + tool_cfg) + yield ret + init_user(uuid_str, _state) + init_builder(uuid_str, _state) + yield { + state: + _state, + preview_send_button: + gr.Button.update(value=i18n.get('send'), interactive=True), + create_send_button: + gr.Button.update(value=i18n.get('send'), interactive=True), + } + + demo.load( + init_all, inputs=[uuid_str, state], outputs=configure_updated_outputs) + +demo.queue(concurrency_count=10) +demo.launch(share=True, show_error=True) diff --git a/agentfabric/appBot.py b/agentfabric/appBot.py new file mode 100644 index 0000000000000000000000000000000000000000..ac9a6dec82e1f8deb67d46a30ce676c09791ffaa --- /dev/null +++ b/agentfabric/appBot.py @@ -0,0 +1,169 @@ +import os +import random +import shutil +import sys +import traceback + +import gradio as gr +from config_utils import get_avatar_image, get_ci_dir, parse_configuration +from gradio_utils import ChatBot, format_cover_html +from user_core import init_user_chatbot_agent + +uuid_str = 'local_user' +builder_cfg, model_cfg, tool_cfg, available_tool_list, _, _ = parse_configuration( + uuid_str) +suggests = builder_cfg.get('prompt_recommend', []) +avatar_pairs = get_avatar_image(builder_cfg.get('avatar', ''), uuid_str) + +customTheme = gr.themes.Default( + primary_hue=gr.themes.utils.colors.blue, + radius_size=gr.themes.utils.sizes.radius_none, +) + + +def check_uuid(uuid_str): + if not uuid_str or uuid_str == '': + if os.getenv('MODELSCOPE_ENVIRONMENT') == 'studio': + raise gr.Error('请登陆后使用! (Please login first)') + else: + uuid_str = 'local_user' + return uuid_str + + +def init_user(state): + try: + seed = state.get('session_seed', random.randint(0, 1000000000)) + user_agent = init_user_chatbot_agent(uuid_str) + user_agent.seed = seed + state['user_agent'] = user_agent + except Exception as e: + error = traceback.format_exc() + print(f'Error:{e}, with detail: {error}') + return state + + +# 创建 Gradio 界面 +demo = gr.Blocks(css='assets/appBot.css', theme=customTheme) +with demo: + gr.Markdown( + '#
\N{fire} AgentFabric powered by Modelscope-agent ([github star](https://github.com/modelscope/modelscope-agent/tree/main))
' # noqa E501 + ) + draw_seed = random.randint(0, 1000000000) + state = gr.State({'session_seed': draw_seed}) + with gr.Row(elem_classes='container'): + with gr.Column(scale=4): + with gr.Column(): + # Preview + user_chatbot = ChatBot( + value=[[None, '尝试问我一点什么吧～']], + elem_id='user_chatbot', + elem_classes=['markdown-body'], + avatar_images=avatar_pairs, + height=600, + latex_delimiters=[], + show_label=False) + with gr.Row(): + with gr.Column(scale=12): + preview_chat_input = gr.Textbox( + show_label=False, + container=False, + placeholder='跟我聊聊吧～') + with gr.Column(min_width=70, scale=1): + upload_button = gr.UploadButton( + '上传', + file_types=[ + '.csv', '.doc', '.docx', '.xls', '.xlsx', '.txt', + '.md', '.pdf', '.jpeg', '.png', '.jpg', '.gif' + ], + file_count='multiple') + with gr.Column(min_width=70, scale=1): + preview_send_button = gr.Button('发送', variant='primary') + + with gr.Column(scale=1): + user_chat_bot_cover = gr.HTML( + format_cover_html(builder_cfg, avatar_pairs[1])) + user_chat_bot_suggest = gr.Examples( + label='Prompt Suggestions', + examples=suggests, + inputs=[preview_chat_input]) + + def upload_file(chatbot, upload_button, _state): + _uuid_str = check_uuid(uuid_str) + new_file_paths = [] + if 'file_paths' in _state: + file_paths = _state['file_paths'] + else: + file_paths = [] + for file in upload_button: + file_name = os.path.basename(file.name) + # covert xxx.json to xxx_uuid_str.json + file_name = file_name.replace('.', f'_{_uuid_str}.') + file_path = os.path.join(get_ci_dir(), file_name) + if not os.path.exists(file_path): + # make sure file path's directory exists + os.makedirs(os.path.dirname(file_path), exist_ok=True) + shutil.copy(file.name, file_path) + file_paths.append(file_path) + new_file_paths.append(file_path) + chatbot.append((None, f'上传文件{file_name}，成功')) + yield { + user_chatbot: gr.Chatbot.update(visible=True, value=chatbot), + preview_chat_input: gr.Textbox.update(value='') + } + + _state['file_paths'] = file_paths + _state['new_file_paths'] = new_file_paths + + upload_button.upload( + upload_file, + inputs=[user_chatbot, upload_button, state], + outputs=[user_chatbot, preview_chat_input]) + + def send_message(chatbot, input, _state): + # 将发送的消息添加到聊天历史 + user_agent = _state['user_agent'] + if 'new_file_paths' in _state: + new_file_paths = _state['new_file_paths'] + else: + new_file_paths = [] + _state['new_file_paths'] = [] + chatbot.append((input, '')) + yield { + user_chatbot: chatbot, + preview_chat_input: gr.Textbox.update(value=''), + } + + response = '' + + for frame in user_agent.stream_run( + input, print_info=True, remote=False, + append_files=new_file_paths): + # is_final = frame.get("frame_is_final") + llm_result = frame.get('llm_text', '') + exec_result = frame.get('exec_result', '') + # llm_result = llm_result.split("<|user|>")[0].strip() + if len(exec_result) != 0: + # action_exec_result + if isinstance(exec_result, dict): + exec_result = str(exec_result['result']) + frame_text = f'{exec_result}' + else: + # llm result + frame_text = llm_result + + # important! do not change this + response += frame_text + chatbot[-1] = (input, response) + yield { + user_chatbot: chatbot, + } + + preview_send_button.click( + send_message, + inputs=[user_chatbot, preview_chat_input, state], + outputs=[user_chatbot, preview_chat_input]) + + demo.load(init_user, inputs=[state], outputs=[state]) + +demo.queue() +demo.launch() diff --git a/agentfabric/assets/app.css b/agentfabric/assets/app.css new file mode 100644 index 0000000000000000000000000000000000000000..eec2fbd2ae690bbeabf95fadb613f89ff0851f1c --- /dev/null +++ b/agentfabric/assets/app.css @@ -0,0 +1,147 @@ +/* code highlight: https://python-markdown.github.io/extensions/code_hilite/ */ +.codehilite .hll { background-color: #ffffcc } +.codehilite { background: #f8f8f8; } +.codehilite .c { color: #408080; font-style: italic } /* Comment */ +.codehilite .err { border: 1px solid #FF0000 } /* Error */ +.codehilite .k { color: #008000; font-weight: bold } /* Keyword */ +.codehilite .o { color: #666666 } /* Operator */ +.codehilite .ch { color: #408080; font-style: italic } /* Comment.Hashbang */ +.codehilite .cm { color: #408080; font-style: italic } /* Comment.Multiline */ +.codehilite .cp { color: #BC7A00 } /* Comment.Preproc */ +.codehilite .cpf { color: #408080; font-style: italic } /* Comment.PreprocFile */ +.codehilite .c1 { color: #408080; font-style: italic } /* Comment.Single */ +.codehilite .cs { color: #408080; font-style: italic } /* Comment.Special */ +.codehilite .gd { color: #A00000 } /* Generic.Deleted */ +.codehilite .ge { font-style: italic } /* Generic.Emph */ +.codehilite .gr { color: #FF0000 } /* Generic.Error */ +.codehilite .gh { color: #000080; font-weight: bold } /* Generic.Heading */ +.codehilite .gi { color: #00A000 } /* Generic.Inserted */ +.codehilite .go { color: #888888 } /* Generic.Output */ +.codehilite .gp { color: #000080; font-weight: bold } /* Generic.Prompt */ +.codehilite .gs { font-weight: bold } /* Generic.Strong */ +.codehilite .gu { color: #800080; font-weight: bold } /* Generic.Subheading */ +.codehilite .gt { color: #0044DD } /* Generic.Traceback */ +.codehilite .kc { color: #008000; font-weight: bold } /* Keyword.Constant */ +.codehilite .kd { color: #008000; font-weight: bold } /* Keyword.Declaration */ +.codehilite .kn { color: #008000; font-weight: bold } /* Keyword.Namespace */ +.codehilite .kp { color: #008000 } /* Keyword.Pseudo */ +.codehilite .kr { color: #008000; font-weight: bold } /* Keyword.Reserved */ +.codehilite .kt { color: #B00040 } /* Keyword.Type */ +.codehilite .m { color: #666666 } /* Literal.Number */ +.codehilite .s { color: #BA2121 } /* Literal.String */ +.codehilite .na { color: #7D9029 } /* Name.Attribute */ +.codehilite .nb { color: #008000 } /* Name.Builtin */ +.codehilite .nc { color: #0000FF; font-weight: bold } /* Name.Class */ +.codehilite .no { color: #880000 } /* Name.Constant */ +.codehilite .nd { color: #AA22FF } /* Name.Decorator */ +.codehilite .ni { color: #999999; font-weight: bold } /* Name.Entity */ +.codehilite .ne { color: #D2413A; font-weight: bold } /* Name.Exception */ +.codehilite .nf { color: #0000FF } /* Name.Function */ +.codehilite .nl { color: #A0A000 } /* Name.Label */ +.codehilite .nn { color: #0000FF; font-weight: bold } /* Name.Namespace */ +.codehilite .nt { color: #008000; font-weight: bold } /* Name.Tag */ +.codehilite .nv { color: #19177C } /* Name.Variable */ +.codehilite .ow { color: #AA22FF; font-weight: bold } /* Operator.Word */ +.codehilite .w { color: #bbbbbb } /* Text.Whitespace */ +.codehilite .mb { color: #666666 } /* Literal.Number.Bin */ +.codehilite .mf { color: #666666 } /* Literal.Number.Float */ +.codehilite .mh { color: #666666 } /* Literal.Number.Hex */ +.codehilite .mi { color: #666666 } /* Literal.Number.Integer */ +.codehilite .mo { color: #666666 } /* Literal.Number.Oct */ +.codehilite .sa { color: #BA2121 } /* Literal.String.Affix */ +.codehilite .sb { color: #BA2121 } /* Literal.String.Backtick */ +.codehilite .sc { color: #BA2121 } /* Literal.String.Char */ +.codehilite .dl { color: #BA2121 } /* Literal.String.Delimiter */ +.codehilite .sd { color: #BA2121; font-style: italic } /* Literal.String.Doc */ +.codehilite .s2 { color: #BA2121 } /* Literal.String.Double */ +.codehilite .se { color: #BB6622; font-weight: bold } /* Literal.String.Escape */ +.codehilite .sh { color: #BA2121 } /* Literal.String.Heredoc */ +.codehilite .si { color: #BB6688; font-weight: bold } /* Literal.String.Interpol */ +.codehilite .sx { color: #008000 } /* Literal.String.Other */ +.codehilite .sr { color: #BB6688 } /* Literal.String.Regex */ +.codehilite .s1 { color: #BA2121 } /* Literal.String.Single */ +.codehilite .ss { color: #19177C } /* Literal.String.Symbol */ +.codehilite .bp { color: #008000 } /* Name.Builtin.Pseudo */ +.codehilite .fm { color: #0000FF } /* Name.Function.Magic */ +.codehilite .vc { color: #19177C } /* Name.Variable.Class */ +.codehilite .vg { color: #19177C } /* Name.Variable.Global */ +.codehilite .vi { color: #19177C } /* Name.Variable.Instance */ +.codehilite .vm { color: #19177C } /* Name.Variable.Magic */ +.codehilite .il { color: #666666 } /* Literal.Number.Integer.Long */ + +.preview_header { + font-size: 18px; + font-weight: 500; + text-align: center; + margin-bottom: -12px; +} + +.bot_cover { + display: flex; + flex-direction: column; + justify-content: center; + align-items: center; + min-height: 650px; + border: 1px solid rgb(229, 231, 235); + border-radius: 8px; + padding: 20px 40px; +} + +.bot_avatar { + width: 100px; + height: 100px; + border-radius: 50%; + overflow: hidden; +} + +.bot_avatar img { + width: 100px; + height: 100px; +} + +.bot_name { + font-size: 36px; + margin-top: 10px; +} + +.bot_desp { + color: #ddd; +} + +.publish_link_container > a { + display: block; + border-radius: var(--button-large-radius); + padding: var(--button-large-padding); + font-weight: var(--button-large-text-weight); + font-size: var(--button-large-text-size); + border: var(--button-border-width) solid var(--button-secondary-border-color); + background: var(--button-secondary-background-fill); + color: var(--button-secondary-text-color) !important; + cursor: pointer; + text-decoration: none !important; + text-align: center; +} + +.publish_link_container > .disabled { + cursor: not-allowed; + opacity: .5; + filter: grayscale(30%); +} + +.markdown-body .message { + white-space: pre-wrap; +} + +.markdown-body details { + white-space: nowrap; +} +.markdown-body .bot details:not(:last-child) { + margin-bottom: 1px; +} +.markdown-body summary { + background-color: #4b5563; + color: #eee; + padding: 0 4px; + border-radius: 4px; + font-size: 0.9em; +} diff --git a/agentfabric/assets/appBot.css b/agentfabric/assets/appBot.css new file mode 100644 index 0000000000000000000000000000000000000000..2462313447e31527340b7faf05715a83c0e14287 --- /dev/null +++ b/agentfabric/assets/appBot.css @@ -0,0 +1,129 @@ +/* code highlight: https://python-markdown.github.io/extensions/code_hilite/ */ +.codehilite .hll { background-color: #ffffcc } +.codehilite { background: #f8f8f8; } +.codehilite .c { color: #408080; font-style: italic } /* Comment */ +.codehilite .err { border: 1px solid #FF0000 } /* Error */ +.codehilite .k { color: #008000; font-weight: bold } /* Keyword */ +.codehilite .o { color: #666666 } /* Operator */ +.codehilite .ch { color: #408080; font-style: italic } /* Comment.Hashbang */ +.codehilite .cm { color: #408080; font-style: italic } /* Comment.Multiline */ +.codehilite .cp { color: #BC7A00 } /* Comment.Preproc */ +.codehilite .cpf { color: #408080; font-style: italic } /* Comment.PreprocFile */ +.codehilite .c1 { color: #408080; font-style: italic } /* Comment.Single */ +.codehilite .cs { color: #408080; font-style: italic } /* Comment.Special */ +.codehilite .gd { color: #A00000 } /* Generic.Deleted */ +.codehilite .ge { font-style: italic } /* Generic.Emph */ +.codehilite .gr { color: #FF0000 } /* Generic.Error */ +.codehilite .gh { color: #000080; font-weight: bold } /* Generic.Heading */ +.codehilite .gi { color: #00A000 } /* Generic.Inserted */ +.codehilite .go { color: #888888 } /* Generic.Output */ +.codehilite .gp { color: #000080; font-weight: bold } /* Generic.Prompt */ +.codehilite .gs { font-weight: bold } /* Generic.Strong */ +.codehilite .gu { color: #800080; font-weight: bold } /* Generic.Subheading */ +.codehilite .gt { color: #0044DD } /* Generic.Traceback */ +.codehilite .kc { color: #008000; font-weight: bold } /* Keyword.Constant */ +.codehilite .kd { color: #008000; font-weight: bold } /* Keyword.Declaration */ +.codehilite .kn { color: #008000; font-weight: bold } /* Keyword.Namespace */ +.codehilite .kp { color: #008000 } /* Keyword.Pseudo */ +.codehilite .kr { color: #008000; font-weight: bold } /* Keyword.Reserved */ +.codehilite .kt { color: #B00040 } /* Keyword.Type */ +.codehilite .m { color: #666666 } /* Literal.Number */ +.codehilite .s { color: #BA2121 } /* Literal.String */ +.codehilite .na { color: #7D9029 } /* Name.Attribute */ +.codehilite .nb { color: #008000 } /* Name.Builtin */ +.codehilite .nc { color: #0000FF; font-weight: bold } /* Name.Class */ +.codehilite .no { color: #880000 } /* Name.Constant */ +.codehilite .nd { color: #AA22FF } /* Name.Decorator */ +.codehilite .ni { color: #999999; font-weight: bold } /* Name.Entity */ +.codehilite .ne { color: #D2413A; font-weight: bold } /* Name.Exception */ +.codehilite .nf { color: #0000FF } /* Name.Function */ +.codehilite .nl { color: #A0A000 } /* Name.Label */ +.codehilite .nn { color: #0000FF; font-weight: bold } /* Name.Namespace */ +.codehilite .nt { color: #008000; font-weight: bold } /* Name.Tag */ +.codehilite .nv { color: #19177C } /* Name.Variable */ +.codehilite .ow { color: #AA22FF; font-weight: bold } /* Operator.Word */ +.codehilite .w { color: #bbbbbb } /* Text.Whitespace */ +.codehilite .mb { color: #666666 } /* Literal.Number.Bin */ +.codehilite .mf { color: #666666 } /* Literal.Number.Float */ +.codehilite .mh { color: #666666 } /* Literal.Number.Hex */ +.codehilite .mi { color: #666666 } /* Literal.Number.Integer */ +.codehilite .mo { color: #666666 } /* Literal.Number.Oct */ +.codehilite .sa { color: #BA2121 } /* Literal.String.Affix */ +.codehilite .sb { color: #BA2121 } /* Literal.String.Backtick */ +.codehilite .sc { color: #BA2121 } /* Literal.String.Char */ +.codehilite .dl { color: #BA2121 } /* Literal.String.Delimiter */ +.codehilite .sd { color: #BA2121; font-style: italic } /* Literal.String.Doc */ +.codehilite .s2 { color: #BA2121 } /* Literal.String.Double */ +.codehilite .se { color: #BB6622; font-weight: bold } /* Literal.String.Escape */ +.codehilite .sh { color: #BA2121 } /* Literal.String.Heredoc */ +.codehilite .si { color: #BB6688; font-weight: bold } /* Literal.String.Interpol */ +.codehilite .sx { color: #008000 } /* Literal.String.Other */ +.codehilite .sr { color: #BB6688 } /* Literal.String.Regex */ +.codehilite .s1 { color: #BA2121 } /* Literal.String.Single */ +.codehilite .ss { color: #19177C } /* Literal.String.Symbol */ +.codehilite .bp { color: #008000 } /* Name.Builtin.Pseudo */ +.codehilite .fm { color: #0000FF } /* Name.Function.Magic */ +.codehilite .vc { color: #19177C } /* Name.Variable.Class */ +.codehilite .vg { color: #19177C } /* Name.Variable.Global */ +.codehilite .vi { color: #19177C } /* Name.Variable.Instance */ +.codehilite .vm { color: #19177C } /* Name.Variable.Magic */ +.codehilite .il { color: #666666 } /* Literal.Number.Integer.Long */ + +.preview_header { + font-size: 24px; + font-weight: 500; + text-align: center; +} + +.bot_cover { + display: flex; + flex-direction: column; + justify-content: center; + align-items: center; + min-height: 300px; + border: 1px solid rgb(229, 231, 235); + padding: 20px 20px; +} + +.bot_avatar { + width: 100px; + height: 100px; + border-radius: 50%; + overflow: hidden; +} + +.bot_avatar img { + width: 100px; + height: 100px; +} + +.bot_name { + font-size: 36px; + margin-top: 10px; +} + +.bot_desp { + color: #ddd; +} + +.container { + flex-direction: row-reverse; +} + +.markdown-body .message { + white-space: pre-wrap; +} + +.markdown-body details { + white-space: nowrap; +} +.markdown-body .bot details:not(:last-child) { + margin-bottom: 1px; +} +.markdown-body summary { + background-color: #4b5563; + color: #eee; + padding: 0 4px; + border-radius: 4px; + font-size: 0.9em; +} diff --git a/agentfabric/assets/bot.jpg b/agentfabric/assets/bot.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5fde8cc45f61b677c0581e6889b11e269c35be08 Binary files /dev/null and b/agentfabric/assets/bot.jpg differ diff --git a/agentfabric/assets/user.jpg b/agentfabric/assets/user.jpg new file mode 100644 index 0000000000000000000000000000000000000000..536948b6bd19cb0b49c44b74e2790198301520e5 Binary files /dev/null and b/agentfabric/assets/user.jpg differ diff --git a/agentfabric/builder_core.py b/agentfabric/builder_core.py new file mode 100644 index 0000000000000000000000000000000000000000..5566bee6a7a5695ad00fc7b6d8352ad0818f83bf --- /dev/null +++ b/agentfabric/builder_core.py @@ -0,0 +1,276 @@ +# flake8: noqa E501 +import re +import traceback +from typing import Dict + +import json +from config_utils import parse_configuration +from help_tools import LogoGeneratorTool, config_conversion +from modelscope_agent.agent import AgentExecutor +from modelscope_agent.agent_types import AgentType +from modelscope_agent.llm import LLMFactory +from modelscope_agent.prompt import MessagesGenerator + +SYSTEM = 'You are a helpful assistant.' + +PROMPT_CUSTOM = """你现在要扮演一个制造AI角色（AI-Agent）的AI助手（QwenBuilder）。 +你需要和用户进行对话，明确用户对AI-Agent的要求。并根据已有信息和你的联想能力，尽可能填充完整的配置文件： + +配置文件为json格式： +{"name": "... # AI-Agent的名字", "description": "... # 对AI-Agent的要求，简单描述", "instructions": "... # 分点描述对AI-Agent的具体功能要求，尽量详细一些，类型是一个字符串数组，起始为[]", "prompt_recommend": "... # 推荐的用户将对AI-Agent说的指令，用于指导用户使用AI-Agent，类型是一个字符串数组，请尽可能补充4句左右，起始为["你可以做什么？"]", "logo_prompt": "... # 画AI-Agent的logo的指令，不需要画logo或不需要更新logo时可以为空，类型是string"} + +在接下来的对话中，请在回答时严格使用如下格式，先作出回复，再生成配置文件，不要回复其他任何内容： +Answer: ... # 你希望对用户说的话，用于询问用户对AI-Agent的要求，不要重复确认用户已经提出的要求，而应该拓展出新的角度来询问用户，尽量细节和丰富，禁止为空 +Config: ... # 生成的配置文件，严格按照以上json格式 +RichConfig: ... # 格式和核心内容和Config相同，但是保证name和description不为空；instructions需要在Config的基础上扩充字数，使指令更加详尽，如果用户给出了详细指令，请完全保留；补充prompt_recommend，并保证prompt_recommend是推荐的用户将对AI-Agent说的指令。请注意从用户的视角来描述prompt_recommend、description和instructions。 + +一个优秀的RichConfig样例如下： +{"name": "小红书文案生成助手", "description": "一个专为小红书用户设计的文案生成助手。", "instructions": "1. 理解并回应用户的指令；2. 根据用户的需求生成高质量的小红书风格文案；3. 使用表情提升文本丰富度", "prompt_recommend": ["你可以帮我生成一段关于旅行的文案吗？", "你会写什么样的文案？", "可以推荐一个小红书文案模版吗？"], "logo_prompt": "一个写作助手logo，包含一只羽毛钢笔"} + + +明白了请说“好的。”， 不要说其他的。""" + +LOGO_TOOL_NAME = 'logo_designer' + +ANSWER = 'Answer' +CONFIG = 'Config' +ASSISTANT_PROMPT = """{}: \n{}: \nRichConfig: """.format( + ANSWER, CONFIG) + +UPDATING_CONFIG_STEP = '🚀Updating Config...' +CONFIG_UPDATED_STEP = '✅Config Updated!' +UPDATING_LOGO_STEP = '🚀Updating Logo...' +LOGO_UPDATED_STEP = '✅Logo Updated!' + + +def init_builder_chatbot_agent(uuid_str): + # build model + builder_cfg, model_cfg, _, _, _, _ = parse_configuration(uuid_str) + + # additional tool + additional_tool_list = {LOGO_TOOL_NAME: LogoGeneratorTool()} + tool_cfg = {LOGO_TOOL_NAME: {'is_remote_tool': True}} + + # build llm + print(f'using builder model {builder_cfg.model}') + llm = LLMFactory.build_llm(builder_cfg.model, model_cfg) + llm.set_agent_type(AgentType.Messages) + + # build prompt + starter_messages = [{ + 'role': 'system', + 'content': SYSTEM + }, { + 'role': 'user', + 'content': PROMPT_CUSTOM + }, { + 'role': 'assistant', + 'content': '好的。' + }] + + # prompt generator + prompt_generator = MessagesGenerator( + system_template=SYSTEM, custom_starter_messages=starter_messages) + + # build agent + agent = BuilderChatbotAgent( + llm, + tool_cfg, + agent_type=AgentType.Messages, + prompt_generator=prompt_generator, + additional_tool_list=additional_tool_list) + agent.set_available_tools([LOGO_TOOL_NAME]) + return agent + + +class BuilderChatbotAgent(AgentExecutor): + + def __init__(self, llm, tool_cfg, agent_type, prompt_generator, + additional_tool_list): + + super().__init__( + llm, + tool_cfg, + agent_type=agent_type, + additional_tool_list=additional_tool_list, + prompt_generator=prompt_generator, + tool_retrieval=False) + + # used to reconstruct assistant message when builder config is updated + self._last_assistant_structured_response = {} + + def stream_run(self, + task: str, + remote: bool = True, + print_info: bool = False, + uuid_str: str = '') -> Dict: + + # retrieve tools + tool_list = self.retrieve_tools(task) + self.prompt_generator.init_prompt(task, tool_list, []) + function_list = [] + + llm_result, exec_result = '', '' + + idx = 0 + + while True: + idx += 1 + llm_artifacts = self.prompt_generator.generate( + llm_result, exec_result) + if print_info: + print(f'|LLM inputs in round {idx}:\n{llm_artifacts}') + + llm_result = '' + try: + parser_obj = AnswerParser() + for s in self.llm.stream_generate(llm_artifacts=llm_artifacts): + llm_result += s + answer, finish = parser_obj.parse_answer(llm_result) + if answer == '': + continue + result = {'llm_text': answer} + if finish: + result.update({'step': UPDATING_CONFIG_STEP}) + yield result + + if print_info: + print(f'|LLM output in round {idx}:\n{llm_result}') + except Exception as e: + yield {'error': 'llm result is not valid'} + + try: + re_pattern_config = re.compile( + pattern=r'Config: ([\s\S]+)\nRichConfig') + res = re_pattern_config.search(llm_result) + if res is None: + return + config = res.group(1).strip() + self._last_assistant_structured_response['config_str'] = config + + rich_config = llm_result[llm_result.rfind('RichConfig:') + + len('RichConfig:'):].strip() + try: + answer = json.loads(rich_config) + except Exception: + print('parse RichConfig error') + return + self._last_assistant_structured_response[ + 'rich_config_dict'] = answer + builder_cfg = config_conversion(answer, uuid_str=uuid_str) + yield {'exec_result': {'result': builder_cfg}} + yield {'step': CONFIG_UPDATED_STEP} + except ValueError as e: + print(e) + yield {'error content=[{}]'.format(llm_result)} + return + + # record the llm_result result + _ = self.prompt_generator.generate( + { + 'role': 'assistant', + 'content': llm_result + }, '') + + messages = self.prompt_generator.history + if 'logo_prompt' in answer and len(messages) > 4 and ( + answer['logo_prompt'] not in messages[-3]['content']): + # draw logo + yield {'step': UPDATING_LOGO_STEP} + params = { + 'user_requirement': answer['logo_prompt'], + 'uuid_str': uuid_str + } + + tool = self.tool_list[LOGO_TOOL_NAME] + try: + exec_result = tool(**params, remote=remote) + yield {'exec_result': exec_result} + yield {'step': LOGO_UPDATED_STEP} + + return + except Exception as e: + exec_result = f'Action call error: {LOGO_TOOL_NAME}: {params}. \n Error message: {e}' + yield {'error': exec_result} + self.prompt_generator.reset() + return + else: + return + + def update_config_to_history(self, config: Dict): + """ update builder config to message when user modify configuration + + Args: + config info read from builder config file + """ + if len( + self.prompt_generator.history + ) > 0 and self.prompt_generator.history[-1]['role'] == 'assistant': + answer = self._last_assistant_structured_response['answer_str'] + simple_config = self._last_assistant_structured_response[ + 'config_str'] + + rich_config_dict = { + k: config[k] + for k in ['name', 'description', 'prompt_recommend'] + } + rich_config_dict[ + 'logo_prompt'] = self._last_assistant_structured_response[ + 'rich_config_dict']['logo_prompt'] + rich_config_dict['instructions'] = config['instruction'].split('；') + + rich_config = json.dumps(rich_config_dict, ensure_ascii=False) + new_content = ASSISTANT_PROMPT.replace('', answer).replace( + '', simple_config).replace('', + rich_config) + self.prompt_generator.history[-1]['content'] = new_content + + +def beauty_output(response: str, step_result: str): + flag_list = [ + CONFIG_UPDATED_STEP, UPDATING_CONFIG_STEP, LOGO_UPDATED_STEP, + UPDATING_LOGO_STEP + ] + + if step_result in flag_list: + end_str = '' + for item in flag_list: + if response.endswith(item): + end_str = item + if end_str == '': + response = f'{response}\n{step_result}' + elif end_str in [CONFIG_UPDATED_STEP, LOGO_UPDATED_STEP]: + response = f'{response}\n{step_result}' + else: + response = response[:-len('\n' + end_str)] + response = f'{response}\n{step_result}' + + return response + + +class AnswerParser(object): + + def __init__(self): + self._history = '' + + def parse_answer(self, llm_result: str): + finish = False + answer_prompt = ANSWER + ': ' + + if len(llm_result) >= len(answer_prompt): + start_pos = llm_result.find(answer_prompt) + end_pos = llm_result.find(f'\n{CONFIG}') + if start_pos >= 0: + if end_pos > start_pos: + result = llm_result[start_pos + len(answer_prompt):end_pos] + finish = True + else: + result = llm_result[start_pos + len(answer_prompt):] + else: + result = llm_result + else: + result = '' + + new_result = result[len(self._history):] + self._history = result + return new_result, finish diff --git a/agentfabric/config/builder_config.json b/agentfabric/config/builder_config.json new file mode 100644 index 0000000000000000000000000000000000000000..45e63a421b3c9767a5b505f85805af3b3ccee554 --- /dev/null +++ b/agentfabric/config/builder_config.json @@ -0,0 +1,26 @@ +{ + "name": "", + "avatar": "custom_bot_avatar.png", + "description": "", + "instruction": "", + "prompt_recommend": [ + "你可以做什么？", + "你有什么功能？", + "如何使用你的功能？", + "能否给我一些示例指令？" + ], + "knowledge": [], + "tools": { + "image_gen": { + "name": "Wanx Image Generation", + "is_active": true, + "use": true + }, + "code_interpreter": { + "name": "Code Interpreter", + "is_active": true, + "use": false + } + }, + "model": "qwen-max" +} diff --git a/agentfabric/config/builder_config_ci.json b/agentfabric/config/builder_config_ci.json new file mode 100644 index 0000000000000000000000000000000000000000..7d307387464ce7565aba3770a7313ca810d2b609 --- /dev/null +++ b/agentfabric/config/builder_config_ci.json @@ -0,0 +1,31 @@ +{ + "name": "Python数据分析师", + "avatar": "image.png", + "description": "使用python解决任务时，你可以运行代码并得到结果，如果运行结果有错误，你需要尽可能对代码进行改进。你可以处理用户上传到电脑的文件。", + "instruction": "1. 你会数学解题；\n2. 你会数据分析和可视化；\n3. 用户上传文件时，你必须先了解文件结构再进行下一步操作；如果没有上传文件但要求画图，则编造示例数据画图\n4. 调用工具前你需要说明理由；Think step by step\n5. 代码出错时你需要反思并改进", + "prompt_recommend": [ + "制作示例饼图来报告某网站流量来源。", + "鸡兔同笼 32头 88腿 多少兔", + "帮我把这个链接“https://modelscope.cn/my/overview”网址，转成二维码，并展示图片", + "一支钢笔5元，一支铅笔3元，一个文具盒10元，一套文具包括2支钢笔，3支铅笔，1个文具盒，一共多少钱？" + ], + "knowledge": [], + "tools": { + "image_gen": { + "name": "Wanx Image Generation", + "is_active": true, + "use": false + }, + "code_interpreter": { + "name": "Code Interpreter", + "is_active": true, + "use": true + }, + "amap_weather": { + "name": "高德天气", + "is_active": true, + "use": false + } + }, + "model": "qwen-max" +} diff --git a/agentfabric/config/builder_config_template.json b/agentfabric/config/builder_config_template.json new file mode 100644 index 0000000000000000000000000000000000000000..fbd3c512a046021bafb67e5e5b19cf0442dd0038 --- /dev/null +++ b/agentfabric/config/builder_config_template.json @@ -0,0 +1,26 @@ +{ + "name": "AI-Agent", + "avatar": "logo.png", + "description": "我希望AI-Agent能够像多啦A梦一样，拥有各种神奇的技能和能力，可以帮我解决生活中的各种问题。", + "instruction": "请告诉我你想要什么帮助，我会尽力提供解决方案。；如果你有任何问题，请随时向我提问，我会尽我所能回答你的问题。；我可以帮你查找信息、提供建议、提醒日程等，只需要你告诉我你需要什么。", + "prompt_recommend": [ + "你好，我是AI-Agent，有什么可以帮助你的吗？", + "嗨，很高兴见到你，我是AI-Agent，你可以问我任何问题。", + "你好，我是AI-Agent，需要我帮你做些什么吗？", + "嗨，我是AI-Agent，有什么我可以帮到你的吗？" + ], + "knowledge": [], + "tools": { + "image_gen": { + "name": "Wanx Image Generation", + "is_active": true, + "use": true + }, + "code_interpreter": { + "name": "Code Interpreter", + "is_active": true, + "use": false + } + }, + "model": "qwen-max" +} diff --git a/agentfabric/config/builder_config_wuxia.json b/agentfabric/config/builder_config_wuxia.json new file mode 100644 index 0000000000000000000000000000000000000000..bcadf439ba52b7efb0022f9423d193af99c59a74 --- /dev/null +++ b/agentfabric/config/builder_config_wuxia.json @@ -0,0 +1,24 @@ +{ + "name": "武侠小说家", + "avatar": "custom_bot_avatar.png", + "description": "能够生成武侠小说并配图", + "instruction": "你的指令是为我提供一个基于金庸武侠小说世界的在线RPG游戏体验。在这个游戏中，玩家将扮演金庸故事中的一个关键角色，游戏情景将基于他的小说。这个游戏的玩法是互动式的，并遵循以下特定格式：\n\n<场景描述>：根据玩家的选择，故事情节将按照金庸小说的线索发展。你将描述角色所处的环境和情况。\n\n<场景图片>：对于每个场景，你将创造一个概括该情况的图像。这些图像的风格将类似于1980年代RPG游戏，大小是16:9宽屏比例。在这个步骤你需要调用画图工具，绘制<场景描述>。\n\n<选择>：在每次互动中，你将为玩家提供三个行动选项，分别标为A、B、C，以及第四个选项“D: 输入玩家的选择”。故事情节将根据玩家选择的行动进展。如果一个选择不是直接来自小说，你将创造性地适应故事，最终引导它回归原始情节。\n\n整个故事将围绕金庸小说中丰富而复杂的世界展开。每次互动必须包括<场景描述>、<场景图片>和<选择>。所有内容将以繁体中文呈现。你的重点将仅仅放在提供场景描述，场景图片和选择上，不包含其他游戏指导。场景尽量不要重复，要丰富一些。", + "prompt_recommend": [ + "扮演小龙女", + "扮演杨过" + ], + "knowledge": [], + "tools": { + "image_gen": { + "name": "Wanx Image Generation", + "is_active": true, + "use": true + }, + "code_interpreter": { + "name": "Code Interpreter", + "is_active": true, + "use": false + } + }, + "model": "qwen-max" +} diff --git a/agentfabric/config/custom_bot_avatar.png b/agentfabric/config/custom_bot_avatar.png new file mode 100644 index 0000000000000000000000000000000000000000..f996880734902a94a9a8fc785eed69d6540bb7fa Binary files /dev/null and b/agentfabric/config/custom_bot_avatar.png differ diff --git a/agentfabric/config/model_config.json b/agentfabric/config/model_config.json new file mode 100644 index 0000000000000000000000000000000000000000..976cd64b89504db1355aa78d0550bf483d465143 --- /dev/null +++ b/agentfabric/config/model_config.json @@ -0,0 +1,77 @@ +{ + "qwen-turbo": { + "type": "dashscope", + "model": "qwen-turbo", + "generate_cfg": { + "use_raw_prompt": true, + "top_p": 0.8 + } + }, + "qwen-plus": { + "type": "dashscope", + "model": "qwen-plus", + "generate_cfg": { + "use_raw_prompt": true, + "top_p": 0.8 + } + }, + "qwen-max": { + "type": "dashscope", + "model": "qwen-max", + "length_constraint": { + "knowledge": 4000, + "input": 6000 + }, + "generate_cfg": { + "use_raw_prompt": true, + "top_p": 0.8 + } + }, + "qwen-7b": { + "type": "modelscope", + "model_id": "qwen/Qwen-7B-Chat", + "model_revision": "v1.1.8", + "generate_cfg": { + "use_raw_prompt": true, + "top_p": 0.8, + "max_length": 2000 + } + }, + "qwen-7b-api": { + "type": "dashscope", + "model": "qwen-7b-chat", + "generate_cfg": { + "use_raw_prompt": true, + "top_p": 0.8, + "debug": false + } + }, + "qwen-14b": { + "type": "modelscope", + "model_id": "qwen/Qwen-14B-Chat", + "model_revision": "v1.0.8", + "generate_cfg": { + "use_raw_prompt": true, + "top_p": 0.8, + "max_length": 2000 + } + }, + "qwen-14b-api": { + "type": "dashscope", + "model": "qwen-14b-chat", + "generate_cfg": { + "use_raw_prompt": true, + "top_p": 0.8, + "debug": false + } + }, + "qwen-72b-api": { + "type": "dashscope", + "model": "qwen-72b-chat", + "generate_cfg": { + "use_raw_prompt": true, + "top_p": 0.8, + "debug": false + } + } +} diff --git a/agentfabric/config/tool_config.json b/agentfabric/config/tool_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e3ca656c29dae6b819e4cd4eb0fbd4c9e574c1b9 --- /dev/null +++ b/agentfabric/config/tool_config.json @@ -0,0 +1,35 @@ +{ + "image_gen": { + "name": "Wanx Image Generation", + "is_active": true, + "use": true, + "is_remote_tool": true + }, + "code_interpreter": { + "name": "Code Interpreter", + "is_active": true, + "use": false, + "is_remote_tool": false, + "max_output": 2000 + }, + "web_browser": { + "name": "Web Browsing", + "is_active": false, + "use": false + }, + "amap_weather": { + "name": "高德天气", + "is_active": true, + "use": false + }, + "wordart_texture_generation": { + "name": "艺术字纹理生成", + "is_active": true, + "use": false + }, + "web_search": { + "name": "Web Searching", + "is_active": false, + "use": false + } +} diff --git a/agentfabric/config_utils.py b/agentfabric/config_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..b6961ae8ea58b1fd0e63ee6ac95fffc88d4ce537 --- /dev/null +++ b/agentfabric/config_utils.py @@ -0,0 +1,170 @@ +import os +import shutil +import traceback + +import json +from modelscope_agent.tools.openapi_plugin import (OpenAPIPluginTool, + openapi_schema_convert) + +from modelscope.utils.config import Config + +DEFAULT_AGENT_DIR = '/tmp/agentfabric' +DEFAULT_BUILDER_CONFIG_DIR = os.path.join(DEFAULT_AGENT_DIR, 'config') +DEFAULT_BUILDER_CONFIG_FILE = os.path.join(DEFAULT_BUILDER_CONFIG_DIR, + 'builder_config.json') +DEFAULT_OPENAPI_PLUGIN_CONFIG_FILE = os.path.join( + DEFAULT_BUILDER_CONFIG_DIR, 'openapi_plugin_config.json') +DEFAULT_MODEL_CONFIG_FILE = './config/model_config.json' +DEFAULT_TOOL_CONFIG_FILE = './config/tool_config.json' +DEFAULT_CODE_INTERPRETER_DIR = os.getenv('CODE_INTERPRETER_WORK_DIR', + '/tmp/ci_workspace') + + +def get_user_dir(uuid_str=''): + return os.path.join(DEFAULT_BUILDER_CONFIG_DIR, uuid_str) + + +def get_ci_dir(): + return DEFAULT_CODE_INTERPRETER_DIR + + +def get_user_cfg_file(uuid_str=''): + builder_cfg_file = os.getenv('BUILDER_CONFIG_FILE', + DEFAULT_BUILDER_CONFIG_FILE) + # convert from ./config/builder_config.json to ./config/user/builder_config.json + builder_cfg_file = builder_cfg_file.replace('config/', 'config/user/') + + # convert from ./config/user/builder_config.json to ./config/uuid/builder_config.json + if uuid_str != '': + builder_cfg_file = builder_cfg_file.replace('user', uuid_str) + return builder_cfg_file + + +def get_user_openapi_plugin_cfg_file(uuid_str=''): + openapi_plugin_cfg_file = os.getenv('OPENAPI_PLUGIN_CONFIG_FILE', + DEFAULT_OPENAPI_PLUGIN_CONFIG_FILE) + openapi_plugin_cfg_file = openapi_plugin_cfg_file.replace( + 'config/', 'config/user/') + if uuid_str != '': + openapi_plugin_cfg_file = openapi_plugin_cfg_file.replace( + 'user', uuid_str) + return openapi_plugin_cfg_file + + +def save_builder_configuration(builder_cfg, uuid_str=''): + builder_cfg_file = get_user_cfg_file(uuid_str) + if uuid_str != '' and not os.path.exists( + os.path.dirname(builder_cfg_file)): + os.makedirs(os.path.dirname(builder_cfg_file)) + with open(builder_cfg_file, 'w', encoding='utf-8') as f: + f.write(json.dumps(builder_cfg, indent=2, ensure_ascii=False)) + + +def is_valid_plugin_configuration(openapi_plugin_cfg): + if 'schema' in openapi_plugin_cfg: + schema = openapi_plugin_cfg['schema'] + if isinstance(schema, dict): + return True + else: + return False + + +def save_plugin_configuration(openapi_plugin_cfg, uuid_str): + openapi_plugin_cfg_file = get_user_openapi_plugin_cfg_file(uuid_str) + if uuid_str != '' and not os.path.exists( + os.path.dirname(openapi_plugin_cfg_file)): + os.makedirs(os.path.dirname(openapi_plugin_cfg_file)) + with open(openapi_plugin_cfg_file, 'w', encoding='utf-8') as f: + f.write(json.dumps(openapi_plugin_cfg, indent=2, ensure_ascii=False)) + + +def get_avatar_image(bot_avatar, uuid_str=''): + user_avatar_path = os.path.join( + os.path.dirname(__file__), 'assets/user.jpg') + bot_avatar_path = os.path.join(os.path.dirname(__file__), 'assets/bot.jpg') + if len(bot_avatar) > 0: + bot_avatar_path = os.path.join(DEFAULT_BUILDER_CONFIG_DIR, uuid_str, + bot_avatar) + if uuid_str != '': + # use default if not exists + if not os.path.exists(bot_avatar_path): + # create parents directory + os.makedirs(os.path.dirname(bot_avatar_path), exist_ok=True) + # copy the template to the address + temp_bot_avatar_path = os.path.join(DEFAULT_BUILDER_CONFIG_DIR, + bot_avatar) + if not os.path.exists(temp_bot_avatar_path): + # fall back to default local avatar image + temp_bot_avatar_path = os.path.join('./config', bot_avatar) + if not os.path.exists(temp_bot_avatar_path): + temp_bot_avatar_path = os.path.join( + './config', 'custom_bot_avatar.png') + + shutil.copy(temp_bot_avatar_path, bot_avatar_path) + + return [user_avatar_path, bot_avatar_path] + + +def save_avatar_image(image_path, uuid_str=''): + bot_avatar = os.path.basename(image_path) + bot_avatar_path = os.path.join(DEFAULT_BUILDER_CONFIG_DIR, uuid_str, + bot_avatar) + shutil.copy(image_path, bot_avatar_path) + return bot_avatar, bot_avatar_path + + +def parse_configuration(uuid_str=''): + """parse configuration + + Args: + + Returns: + dict: parsed configuration + + """ + model_cfg_file = os.getenv('MODEL_CONFIG_FILE', DEFAULT_MODEL_CONFIG_FILE) + + builder_cfg_file = get_user_cfg_file(uuid_str) + # use default if not exists + if not os.path.exists(builder_cfg_file): + # create parents directory + os.makedirs(os.path.dirname(builder_cfg_file), exist_ok=True) + # copy the template to the address + builder_cfg_file_temp = './config/builder_config.json' + + if builder_cfg_file_temp != builder_cfg_file: + shutil.copy(builder_cfg_file_temp, builder_cfg_file) + + tool_cfg_file = os.getenv('TOOL_CONFIG_FILE', DEFAULT_TOOL_CONFIG_FILE) + + builder_cfg = Config.from_file(builder_cfg_file) + model_cfg = Config.from_file(model_cfg_file) + tool_cfg = Config.from_file(tool_cfg_file) + + tools_info = builder_cfg.tools + available_tool_list = [] + for key, value in tools_info.items(): + if value['use']: + available_tool_list.append(key) + tool_cfg[key]['use'] = value['use'] + + openapi_plugin_file = get_user_openapi_plugin_cfg_file(uuid_str) + plugin_cfg = {} + available_plugin_list = [] + if os.path.exists(openapi_plugin_file): + openapi_plugin_cfg = Config.from_file(openapi_plugin_file) + try: + config_dict = openapi_schema_convert( + schema=openapi_plugin_cfg.schema, + auth=openapi_plugin_cfg.auth.to_dict()) + plugin_cfg = Config(config_dict) + for name, config in config_dict.items(): + available_plugin_list.append(name) + except Exception as e: + error = traceback.format_exc() + print(f'Error:{e}, with detail: {error}') + print( + 'Error:FormatError, with detail: The format of the plugin config file is incorrect.' + ) + + return builder_cfg, model_cfg, tool_cfg, available_tool_list, plugin_cfg, available_plugin_list diff --git a/agentfabric/custom_prompt.py b/agentfabric/custom_prompt.py new file mode 100644 index 0000000000000000000000000000000000000000..bc4c1bb3d0d52a3aaef1122c5fc864f1f4b7cba0 --- /dev/null +++ b/agentfabric/custom_prompt.py @@ -0,0 +1,303 @@ +import copy +import os +import re + +import json +from config_utils import get_user_cfg_file +from modelscope_agent.prompt.prompt import (KNOWLEDGE_INTRODUCTION_PROMPT, + KNOWLEDGE_PROMPT, LengthConstraint, + PromptGenerator, build_raw_prompt) + +from modelscope.utils.config import Config + +DEFAULT_SYSTEM_TEMPLATE = """ + +# 工具 + +## 你拥有如下工具： + + + +## 当你需要调用工具时，请在你的回复中穿插如下的工具调用命令，可以根据需求调用零次或多次： + +工具调用 +Action: 工具的名称，必须是之一 +Action Input: 工具的输入 +Observation: 工具返回的结果 +Answer: 根据Observation总结本次工具调用返回的结果，如果结果中出现url，请不要展示出。 + +``` +[链接](url) +``` + +# 指令 +""" + +DEFAULT_SYSTEM_TEMPLATE_WITHOUT_TOOL = """ + +# 指令 +""" + +DEFAULT_INSTRUCTION_TEMPLATE = '' + +DEFAULT_USER_TEMPLATE = """(你正在扮演，你可以使用工具：)""" + +DEFAULT_USER_TEMPLATE_WITHOUT_TOOL = """(你正在扮演) """ + +DEFAULT_EXEC_TEMPLATE = """Observation: \nAnswer:""" + +TOOL_DESC = ( + '{name_for_model}: {name_for_human} API。 {description_for_model} 输入参数: {parameters}' +) + + +class CustomPromptGenerator(PromptGenerator): + + def __init__(self, + system_template=DEFAULT_SYSTEM_TEMPLATE, + instruction_template=DEFAULT_INSTRUCTION_TEMPLATE, + user_template=DEFAULT_USER_TEMPLATE, + exec_template=DEFAULT_EXEC_TEMPLATE, + assistant_template='', + sep='\n\n', + llm=None, + length_constraint=LengthConstraint(), + **kwargs): + super().__init__( + system_template=system_template, + instruction_template=instruction_template, + user_template=user_template, + exec_template=exec_template, + assistant_template=assistant_template, + sep=sep, + llm=llm, + length_constraint=length_constraint) + # hack here for special prompt, such as add an addition round before user input + self.add_addition_round = kwargs.get('add_addition_round', False) + self.addition_assistant_reply = kwargs.get('addition_assistant_reply', + '') + builder_cfg_file = get_user_cfg_file( + uuid_str=kwargs.get('uuid_str', '')) + builder_cfg = Config.from_file(builder_cfg_file) + self.builder_cfg = builder_cfg + self.knowledge_file_name = kwargs.get('knowledge_file_name', '') + + self.llm = llm + self.prompt_preprocessor = build_raw_prompt(llm.model_id) + self.length_constraint = length_constraint + self._parse_length_restriction() + + def _parse_length_restriction(self): + constraint = self.llm.cfg.get('length_constraint', None) + # if isinstance(constraint, Config): + # constraint = constraint.to_dict() + self.length_constraint.update(constraint) + + def _update_user_prompt_without_knowledge(self, task, tool_list, **kwargs): + if len(tool_list) > 0: + # user input + user_input = self.user_template.replace('', + self.builder_cfg.name) + user_input = user_input.replace( + '', + ','.join([tool.name for tool in tool_list])) + else: + self.user_template = DEFAULT_USER_TEMPLATE_WITHOUT_TOOL + user_input = self.user_template.replace('', task) + user_input = user_input.replace('', + self.builder_cfg.name) + + user_input = user_input.replace('', task) + + if 'append_files' in kwargs: + append_files = kwargs.get('append_files', []) + if len(append_files) > 0: + file_names = ','.join( + [os.path.basename(path) for path in append_files]) + user_input = user_input.replace('', + f'[上传文件{file_names}]') + else: + user_input = user_input.replace('', '') + else: + user_input = user_input.replace('', '') + + return user_input + + def init_prompt(self, task, tool_list, knowledge_list, **kwargs): + + if len(self.history) == 0: + + self.history.append({ + 'role': 'system', + 'content': 'You are a helpful assistant.' + }) + + if len(tool_list) > 0: + prompt = f'{self.system_template}\n{self.instruction_template}' + + # get tool description str + tool_str = self.get_tool_str(tool_list) + prompt = prompt.replace('', tool_str) + + tool_name_str = self.get_tool_name_str(tool_list) + prompt = prompt.replace('', tool_name_str) + else: + self.system_template = DEFAULT_SYSTEM_TEMPLATE_WITHOUT_TOOL + prompt = f'{self.system_template}\n{self.instruction_template}' + + user_input = self._update_user_prompt_without_knowledge( + task, tool_list, **kwargs) + + if len(knowledge_list) > 0: + user_input = user_input.replace('', + '，请查看前面的知识库') + else: + user_input = user_input.replace('', '') + + self.system_prompt = copy.deepcopy(prompt) + + # build history + if self.add_addition_round: + self.history.append({ + 'role': 'user', + 'content': self.system_prompt + }) + self.history.append({ + 'role': 'assistant', + 'content': self.addition_assistant_reply + }) + self.history.append({'role': 'user', 'content': user_input}) + self.history.append({ + 'role': 'assistant', + 'content': self.assistant_template + }) + else: + self.history.append({ + 'role': 'user', + 'content': self.system_prompt + user_input + }) + self.history.append({ + 'role': 'assistant', + 'content': self.assistant_template + }) + + self.function_calls = self.get_function_list(tool_list) + else: + user_input = self._update_user_prompt_without_knowledge( + task, tool_list, **kwargs) + if len(knowledge_list) > 0: + user_input = user_input.replace('', + '，请查看前面的知识库') + else: + user_input = user_input.replace('', '') + + self.history.append({'role': 'user', 'content': user_input}) + self.history.append({ + 'role': 'assistant', + 'content': self.assistant_template + }) + + if len(knowledge_list) > 0: + knowledge_str = self.get_knowledge_str( + knowledge_list, + file_name=self.knowledge_file_name, + only_content=True) + self.update_knowledge_str(knowledge_str) + + def update_knowledge_str(self, knowledge_str): + """If knowledge base information was not used previously, it will be added; + if knowledge base information was previously used, it will be replaced. + + Args: + knowledge_str (str): knowledge str generated by get_knowledge_str + """ + knowledge_introduction = KNOWLEDGE_INTRODUCTION_PROMPT.replace( + '', self.knowledge_file_name) + if len(knowledge_str) > self.length_constraint.knowledge: + # todo: use tokenizer to constrain length + knowledge_str = knowledge_str[-self.length_constraint.knowledge:] + knowledge_str = f'{KNOWLEDGE_PROMPT}{self.sep}{knowledge_introduction}{self.sep}{knowledge_str}' + + for i in range(0, len(self.history)): + if self.history[i]['role'] == 'user': + content: str = self.history[i]['content'] + start_pos = content.find(f'{KNOWLEDGE_PROMPT}{self.sep}') + end_pos = content.rfind('\n\n# 工具\n\n') + if start_pos >= 0 and end_pos >= 0: # replace knowledge + + self.history[i]['content'] = content[ + 0:start_pos] + knowledge_str + content[end_pos:] + break + elif start_pos < 0 and end_pos == 0: # add knowledge + self.history[i]['content'] = knowledge_str + content + break + else: + continue + + def get_tool_str(self, tool_list): + tool_texts = [] + for tool in tool_list: + tool_texts.append( + TOOL_DESC.format( + name_for_model=tool.name, + name_for_human=tool.name, + description_for_model=tool.description, + parameters=json.dumps(tool.parameters, + ensure_ascii=False))) + # + ' ' + FORMAT_DESC['json']) + tool_str = '\n\n'.join(tool_texts) + return tool_str + + def get_tool_name_str(self, tool_list): + tool_name = [] + for tool in tool_list: + tool_name.append(tool.name) + + tool_name_str = json.dumps(tool_name, ensure_ascii=False) + return tool_name_str + + def _generate(self, llm_result, exec_result: str): + """ + generate next round prompt based on previous llm_result and exec_result and update history + """ + if len(llm_result) != 0: + self.history[-1]['content'] += f'{llm_result}' + if len(exec_result) != 0: + # handle image markdown wrapper + image_markdown_re = re.compile( + pattern=r'!\[IMAGEGEN\]\(([\s\S]+)\)') + match = image_markdown_re.search(exec_result) + if match is not None: + exec_result = match.group(1).rstrip() + exec_result = self.exec_template.replace('', + str(exec_result)) + self.history[-1]['content'] += exec_result + + # generate plate prompt here + self.prompt = self.prompt_preprocessor(self.history) + return self.prompt + + +def parse_role_config(config: dict): + prompt = '你扮演AI-Agent，' + + # concat prompt + if 'name' in config and config['name']: + prompt += ('你的名字是' + config['name'] + '。') + if 'description' in config and config['description']: + prompt += config['description'] + prompt += '\n你具有下列具体功能：' + if 'instruction' in config and config['instruction']: + if isinstance(config['instruction'], list): + for ins in config['instruction']: + prompt += ins + prompt += '；' + elif isinstance(config['instruction'], str): + prompt += config['instruction'] + if prompt[-1] == '；': + prompt = prompt[:-1] + prompt += '\n下面你将开始扮演' + if 'name' in config and config['name']: + prompt += config['name'] + prompt += '，明白了请说“好的。”，不要说其他的。' + return prompt diff --git a/agentfabric/gradio_utils.py b/agentfabric/gradio_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..56afb27cdf5e75cf0b68fb09691c67b419c14006 --- /dev/null +++ b/agentfabric/gradio_utils.py @@ -0,0 +1,410 @@ +from __future__ import annotations +import base64 +import html +import io +import os +import re +from urllib import parse + +import json +import markdown +from gradio.components import Chatbot as ChatBotBase +from modelscope_agent.output_parser import MRKLOutputParser +from PIL import Image + +ALREADY_CONVERTED_MARK = '' + + +# 图片本地路径转换为 base64 格式 +def covert_image_to_base64(image_path): + # 获得文件后缀名 + ext = image_path.split('.')[-1] + if ext not in ['gif', 'jpeg', 'png']: + ext = 'jpeg' + + with open(image_path, 'rb') as image_file: + # Read the file + encoded_string = base64.b64encode(image_file.read()) + + # Convert bytes to string + base64_data = encoded_string.decode('utf-8') + + # 生成base64编码的地址 + base64_url = f'data:image/{ext};base64,{base64_data}' + return base64_url + + +def convert_url(text, new_filename): + # Define the pattern to search for + # This pattern captures the text inside the square brackets, the path, and the filename + pattern = r'!\[([^\]]+)\]\(([^)]+)\)' + + # Define the replacement pattern + # \1 is a backreference to the text captured by the first group ([^\]]+) + replacement = rf'![\1]({new_filename})' + + # Replace the pattern in the text with the replacement + return re.sub(pattern, replacement, text) + + +def format_cover_html(configuration, bot_avatar_path): + if bot_avatar_path: + image_src = covert_image_to_base64(bot_avatar_path) + else: + image_src = '//img.alicdn.com/imgextra/i3/O1CN01YPqZFO1YNZerQfSBk_!!6000000003047-0-tps-225-225.jpg' + return f""" +
+
+ +
+
{configuration.get("name", "")}
+
{configuration.get("description", "")}
+
+""" + + +def format_goto_publish_html(label, zip_url, agent_user_params, disable=False): + if disable: + return f"""
+ {label} +
+ """ + else: + params = {'AGENT_URL': zip_url} + params.update(agent_user_params) + template = 'modelscope/agent_template' + params_str = json.dumps(params) + link_url = f'https://www.modelscope.cn/studios/fork?target={template}&overwriteEnv={parse.quote(params_str)}' + return f""" +
+ {label} +
+ """ + + +class ChatBot(ChatBotBase): + + def normalize_markdown(self, bot_message): + lines = bot_message.split('\n') + normalized_lines = [] + inside_list = False + + for i, line in enumerate(lines): + if re.match(r'^(\d+\.|-|\*|\+)\s', line.strip()): + if not inside_list and i > 0 and lines[i - 1].strip() != '': + normalized_lines.append('') + inside_list = True + normalized_lines.append(line) + elif inside_list and line.strip() == '': + if i < len(lines) - 1 and not re.match(r'^(\d+\.|-|\*|\+)\s', + lines[i + 1].strip()): + normalized_lines.append(line) + continue + else: + inside_list = False + normalized_lines.append(line) + + return '\n'.join(normalized_lines) + + def convert_markdown(self, bot_message): + if bot_message.count('```') % 2 != 0: + bot_message += '\n```' + + bot_message = self.normalize_markdown(bot_message) + + result = markdown.markdown( + bot_message, + extensions=[ + 'toc', 'extra', 'tables', 'markdown_katex', 'codehilite', + 'markdown_cjk_spacing.cjk_spacing', 'pymdownx.magiclink' + ], + extension_configs={ + 'markdown_katex': { + 'no_inline_svg': True, # fix for WeasyPrint + 'insert_fonts_css': True, + }, + 'codehilite': { + 'linenums': False, + 'guess_lang': True + }, + 'mdx_truly_sane_lists': { + 'nested_indent': 2, + 'truly_sane': True, + } + }) + result = ''.join(result) + return result + + @staticmethod + def prompt_parse(message): + output = '' + if 'Thought' in message: + if 'Action' in message or 'Action Input:' in message: + re_pattern_thought = re.compile( + pattern=r'([\s\S]+)Thought:([\s\S]+)Action:') + + res = re_pattern_thought.search(message) + + if res is None: + re_pattern_thought_only = re.compile( + pattern=r'Thought:([\s\S]+)Action:') + res = re_pattern_thought_only.search(message) + llm_result = '' + else: + llm_result = res.group(1).strip() + action_thought_result = res.group(2).strip() + + re_pattern_action = re.compile( + pattern= + r'Action:([\s\S]+)Action Input:([\s\S]+)<\|startofexec\|>') + res = re_pattern_action.search(message) + if res is None: + action, action_parameters = MRKLOutputParser( + ).parse_response(message) + else: + action = res.group(1).strip() + action_parameters = res.group(2) + action_result = json.dumps({ + 'api_name': action, + 'parameters': action_parameters + }) + output += f'{llm_result}\n{action_thought_result}\n<|startofthink|>\n{action_result}\n<|endofthink|>\n' + if '<|startofexec|>' in message: + re_pattern3 = re.compile( + pattern=r'<\|startofexec\|>([\s\S]+)<\|endofexec\|>') + res3 = re_pattern3.search(message) + observation = res3.group(1).strip() + output += f'\n<|startofexec|>\n{observation}\n<|endofexec|>\n' + if 'Final Answer' in message: + re_pattern2 = re.compile( + pattern=r'Thought:([\s\S]+)Final Answer:([\s\S]+)') + res2 = re_pattern2.search(message) + # final_thought_result = res2.group(1).strip() + final_answer_result = res2.group(2).strip() + output += f'{final_answer_result}\n' + + if output == '': + return message + print(output) + return output + else: + return message + + def convert_bot_message(self, bot_message): + + bot_message = ChatBot.prompt_parse(bot_message) + # print('processed bot message----------') + # print(bot_message) + # print('processed bot message done') + start_pos = 0 + result = '' + find_json_pattern = re.compile(r'{[\s\S]+}') + START_OF_THINK_TAG, END_OF_THINK_TAG = '<|startofthink|>', '<|endofthink|>' + START_OF_EXEC_TAG, END_OF_EXEC_TAG = '<|startofexec|>', '<|endofexec|>' + while start_pos < len(bot_message): + try: + start_of_think_pos = bot_message.index(START_OF_THINK_TAG, + start_pos) + end_of_think_pos = bot_message.index(END_OF_THINK_TAG, + start_pos) + if start_pos < start_of_think_pos: + result += self.convert_markdown( + bot_message[start_pos:start_of_think_pos]) + think_content = bot_message[start_of_think_pos + + len(START_OF_THINK_TAG + ):end_of_think_pos].strip() + json_content = find_json_pattern.search(think_content) + think_content = json_content.group( + ) if json_content else think_content + try: + think_node = json.loads(think_content) + plugin_name = think_node.get( + 'plugin_name', + think_node.get('plugin', + think_node.get('api_name', 'unknown'))) + summary = f'选择插件【{plugin_name}】，调用处理中...' + del think_node['url'] + # think_node.pop('url', None) + + detail = f'```json\n\n{json.dumps(think_node, indent=3, ensure_ascii=False)}\n\n```' + except Exception: + summary = '思考中...' + detail = think_content + # traceback.print_exc() + # detail += traceback.format_exc() + result += '
' + summary + '' + self.convert_markdown( + detail) + '
' + # print(f'detail:{detail}') + start_pos = end_of_think_pos + len(END_OF_THINK_TAG) + except Exception: + # result += traceback.format_exc() + break + # continue + + try: + start_of_exec_pos = bot_message.index(START_OF_EXEC_TAG, + start_pos) + end_of_exec_pos = bot_message.index(END_OF_EXEC_TAG, start_pos) + # print(start_of_exec_pos) + # print(end_of_exec_pos) + # print(bot_message[start_of_exec_pos:end_of_exec_pos]) + # print('------------------------') + if start_pos < start_of_exec_pos: + result += self.convert_markdown( + bot_message[start_pos:start_of_think_pos]) + exec_content = bot_message[start_of_exec_pos + + len(START_OF_EXEC_TAG + ):end_of_exec_pos].strip() + try: + summary = '完成插件调用.' + detail = f'```json\n\n{exec_content}\n\n```' + except Exception: + pass + + result += '
' + summary + '' + self.convert_markdown( + detail) + '
' + + start_pos = end_of_exec_pos + len(END_OF_EXEC_TAG) + except Exception: + # result += traceback.format_exc() + continue + if start_pos < len(bot_message): + result += self.convert_markdown(bot_message[start_pos:]) + result += ALREADY_CONVERTED_MARK + return result + + def convert_bot_message_for_qwen(self, bot_message): + + start_pos = 0 + result = '' + find_json_pattern = re.compile(r'{[\s\S]+}') + ACTION = 'Action:' + ACTION_INPUT = 'Action Input' + OBSERVATION = 'Observation' + RESULT_START = '' + RESULT_END = '' + while start_pos < len(bot_message): + try: + action_pos = bot_message.index(ACTION, start_pos) + action_input_pos = bot_message.index(ACTION_INPUT, start_pos) + result += self.convert_markdown( + bot_message[start_pos:action_pos]) + # Action: image_gen + # Action Input + # {"text": "金庸武侠 世界", "resolution": "1280x720"} + # Observation: ![IMAGEGEN](https://dashscope-result-sh.oss-cn-shanghai.aliyuncs.com/1d/e9/20231116/723609ee/d046d2d9-0c95-420b-9467-f0e831f5e2b7-1.png?Expires=1700227460&OSSAccessKeyId=LTAI5tQZd8AEcZX6KZV4G8qL&Signature=R0PlEazQF9uBD%2Fh9tkzOkJMGyg8%3D) # noqa E501 + action_name = bot_message[action_pos + + len(ACTION + ):action_input_pos].strip() + # action_start action_end 使用 Action Input 到 Observation 之间 + action_input_end = bot_message[action_input_pos:].index( + OBSERVATION) - 1 + action_input = bot_message[action_input_pos:action_input_pos + + action_input_end].strip() + is_json = find_json_pattern.search(action_input) + if is_json: + action_input = is_json.group() + else: + action_input = re.sub(r'^Action Input[:]?[\s]*', '', + action_input) + + summary = f'调用工具 {action_name}' + if is_json: + detail = f'```json\n\n{json.dumps(json.loads(action_input), indent=4, ensure_ascii=False)}\n\n```' + else: + detail = action_input + result += '
' + summary + '' + self.convert_markdown( + detail) + '
' + start_pos = action_input_pos + action_input_end + 1 + try: + observation_pos = bot_message.index(OBSERVATION, start_pos) + idx = observation_pos + len(OBSERVATION) + obs_message = bot_message[idx:] + observation_start_id = obs_message.index( + RESULT_START) + len(RESULT_START) + observation_end_idx = obs_message.index(RESULT_END) + summary = '完成调用' + exec_content = obs_message[ + observation_start_id:observation_end_idx] + detail = f'```\n\n{exec_content}\n\n```' + start_pos = idx + observation_end_idx + len(RESULT_END) + except Exception: + summary = '执行中...' + detail = '' + exec_content = None + + result += '
' + summary + '' + self.convert_markdown( + detail) + '
' + if exec_content is not None and '[IMAGEGEN]' in exec_content: + # convert local file to base64 + re_pattern = re.compile(pattern=r'!\[[^\]]+\]\(([^)]+)\)') + res = re_pattern.search(exec_content) + if res: + image_path = res.group(1).strip() + if os.path.isfile(image_path): + exec_content = convert_url( + exec_content, + covert_image_to_base64(image_path)) + result += self.convert_markdown(f'{exec_content}') + + except Exception: + # import traceback; traceback.print_exc() + result += self.convert_markdown(bot_message[start_pos:]) + start_pos = len(bot_message[start_pos:]) + break + + result += ALREADY_CONVERTED_MARK + return result + + def postprocess( + self, + message_pairs: list[list[str | tuple[str] | tuple[str, str] | None] + | tuple], + ) -> list[list[str | dict | None]]: + """ + Parameters: + message_pairs: List of lists representing the message and response pairs. + Each message and response should be a string, which may be in Markdown format. + It can also be a tuple whose first element is a string or pathlib. + Path filepath or URL to an image/video/audio, and second (optional) element is the alt text, + in which case the media file is displayed. It can also be None, in which case that message is not displayed. + Returns: + List of lists representing the message and response. Each message and response will be a string of HTML, + or a dictionary with media information. Or None if the message is not to be displayed. + """ + if message_pairs is None: + return [] + processed_messages = [] + for message_pair in message_pairs: + assert isinstance( + message_pair, (tuple, list) + ), f'Expected a list of lists or list of tuples. Received: {message_pair}' + assert ( + len(message_pair) == 2 + ), f'Expected a list of lists of length 2 or list of tuples of length 2. Received: {message_pair}' + if isinstance(message_pair[0], tuple) or isinstance( + message_pair[1], tuple): + processed_messages.append([ + self._postprocess_chat_messages(message_pair[0]), + self._postprocess_chat_messages(message_pair[1]), + ]) + else: + # 处理不是元组的情况 + user_message, bot_message = message_pair + + if user_message and not user_message.endswith( + ALREADY_CONVERTED_MARK): + convert_md = self.convert_markdown( + html.escape(user_message)) + user_message = f'{convert_md}' + ALREADY_CONVERTED_MARK + if bot_message and not bot_message.endswith( + ALREADY_CONVERTED_MARK): + # bot_message = self.convert_bot_message(bot_message) + bot_message = self.convert_bot_message_for_qwen( + bot_message) + processed_messages.append([ + user_message, + bot_message, + ]) + + return processed_messages diff --git a/agentfabric/help_tools.py b/agentfabric/help_tools.py new file mode 100644 index 0000000000000000000000000000000000000000..27d85c9f7388a68270ae42c102db84967424361b --- /dev/null +++ b/agentfabric/help_tools.py @@ -0,0 +1,170 @@ +import os +from http import HTTPStatus + +import json +import requests +from config_utils import DEFAULT_BUILDER_CONFIG_DIR, get_user_cfg_file +from dashscope import ImageSynthesis +from modelscope_agent.tools import Tool + +from modelscope.utils.config import Config + +LOGO_NAME = 'custom_bot_avatar.png' +LOGO_PATH = os.path.join(DEFAULT_BUILDER_CONFIG_DIR, LOGO_NAME) + +CONFIG_FORMAT = """ +{ + "name": ... # CustomGPT的名字。 + "description": ... # CustomGPT 的简介。 + "instructions": ... # CustomGPT 的功能要求，类型是string。 + "prompt_recommend": ... # CustomGPT 的起始交互语句，类型是一个字符串数组，起始为[]。 +} +""" + +CONF_GENERATOR_INST = """你现在要扮演一个 CustomGPT 的配置生成器 + +在接下来的对话中，每次均生成如下格式的内容： + +{config_format} + +现在，已知原始配置为{old_config}，用户在原始配置上有一些建议修改项，包括： +1. 用户建议的 CustomGPT 的名称为{app_name} +2. CustomGPT 的描述为{app_description} +3. CustomGPT 的启动器为{app_conversation_starter} + +请你参考原始配置生成新的修改后的配置，请注意： +1. 如果用户对原本的简介、功能要求、交互语句不满意，则直接换掉原本的简介、功能要求、交互语句。 +2. 如果用户对原本的简介、功能要求、交互语句比较满意，参考用户的起始交互语句和原配置中的起始交互语句，生成新的简介、功能要求、交互语句。 +3. 如果原始配置没有实际内容，请你根据你的知识帮助用户生成第一个版本的配置，简介在100字左右，功能要求在150字左右，起始交互语句在4条左右。 + +请你生成新的配置文件，严格遵循给定格式，请不要创造其它字段，仅输出要求的json格式，请勿输出其它内容。 +""" + +LOGO_INST = """定制化软件 CustomGPT 的作用是{description}，{user_requirement}请你为它生成一个专业的logo""" + + +def get_logo_path(uuid_str=''): + logo_path = os.getenv('LOGO_PATH', LOGO_PATH) + # convert from ./config/builder_config.json to ./config/user/builder_config.json + logo_path = logo_path.replace('config/', 'config/user/') + + # convert from ./config/user to ./config/uuid + if uuid_str != '': + logo_path = logo_path.replace('user', uuid_str) + if not os.path.exists(logo_path): + os.makedirs(os.path.dirname(logo_path), exist_ok=True) + return logo_path + + +def call_wanx(prompt, save_path): + rsp = ImageSynthesis.call( + model=ImageSynthesis.Models.wanx_v1, + prompt=prompt, + n=1, + size='1024*1024') + if rsp.status_code == HTTPStatus.OK: + if os.path.exists(save_path): + os.remove(save_path) + + # save file to current directory + for result in rsp.output.results: + with open(save_path, 'wb+') as f: + f.write(requests.get(result.url).content) + else: + print('Failed, status_code: %s, code: %s, message: %s' % + (rsp.status_code, rsp.code, rsp.message)) + + +class LogoGeneratorTool(Tool): + description = 'logo_designer是一个AI绘制logo的服务，输入用户对 CustomGPT 的要求，会生成 CustomGPT 的logo。' + name = 'logo_designer' + parameters: list = [{ + 'name': 'user_requirement', + 'description': '用户对 CustomGPT logo的要求和建议', + 'required': True, + 'schema': { + 'type': 'string' + }, + }] + + def _remote_call(self, *args, **kwargs): + user_requirement = kwargs['user_requirement'] + uuid_str = kwargs.get('uuid_str', '') + builder_cfg_file = get_user_cfg_file(uuid_str) + builder_cfg = Config.from_file(builder_cfg_file) + + avatar_prompt = LOGO_INST.format( + description=builder_cfg.description, + user_requirement=user_requirement) + call_wanx( + prompt=avatar_prompt, save_path=get_logo_path(uuid_str=uuid_str)) + builder_cfg.avatar = LOGO_NAME + return {'result': builder_cfg} + + +def config_conversion(generated_config: dict, save=False, uuid_str=''): + """ + convert + { + name: "铁人", + description: "我希望我的AI-Agent是一个专业的健身教练，专注于力量训练方面，可以提供相关的建议和指南。 + 它还可以帮我跟踪和记录每次的力量训练数据，以及提供相应的反馈和建议，帮助我不断改进和优化我的训练计划。 + 此外，我希望它可以拥有一些特殊技能和功能，让它更加实用和有趣。例如，它可以帮助我预测未来的身体状况、分析我的营养摄入情况、 + 提供心理支持等等。我相信，在它的帮助下，我可以更快地达到自己的目标，变得更加强壮和健康。", + instructions: [ + "提供力量训练相关的建议和指南", + "跟踪和记录每次的力量训练数据", + "提供反馈和建议，帮助改进和优化训练计划", + "预测未来的身体状况", + "分析营养摄入情况", + "提供心理支持", + ], + prompt_recommend: [ + "你好，今天的锻炼计划是什么呢？", + "你觉得哪种器械最适合练背部肌肉呢？", + "你觉得我现在的训练强度合适吗？", + "你觉得哪种食物最适合增肌呢？", + ], + logo_prompt: "设计一个肌肉男形象的Logo", + } + to + { + name: "铁人", + description: "我希望我的AI-Agent是一个专业的健身教练，专注于力量训练方面，可以提供相关的建议和指南。 + 它还可以帮我跟踪和记录每次的力量训练数据，以及提供相应的反馈和建议，帮助我不断改进和优化我的训练计划。 + 此外，我希望它可以拥有一些特殊技能和功能，让它更加实用和有趣。例如，它可以帮助我预测未来的身体状况、 + 分析我的营养摄入情况、提供心理支持等等。我相信，在它的帮助下，我可以更快地达到自己的目标，变得更加强壮和健康。", + instructions: "提供力量训练相关的建议和指南；跟踪和记录每次的力量训练数据；提供反馈和建议，帮助改进和优化训练计划； + 预测未来的身体状况；分析营养摄入情况；提供心理支持", + prompt_recommend: [ + "你好，今天的锻炼计划是什么呢？", + "你觉得哪种器械最适合练背部肌肉呢？", + "你觉得我现在的训练强度合适吗？", + "你觉得哪种食物最适合增肌呢？", + ], + tools: xxx + model: yyy + } + :param generated_config: + :return: + """ + builder_cfg_file = get_user_cfg_file(uuid_str) + builder_cfg = Config.from_file(builder_cfg_file) + try: + builder_cfg.name = generated_config['name'] + builder_cfg.description = generated_config['description'] + builder_cfg.prompt_recommend = generated_config['prompt_recommend'] + if isinstance(generated_config['instructions'], list): + builder_cfg.instruction = '；'.join( + generated_config['instructions']) + else: + builder_cfg.instruction = generated_config['instructions'] + if save: + json.dump( + builder_cfg.to_dict(), + open(builder_cfg_file, 'w'), + indent=2, + ensure_ascii=False) + return builder_cfg + except ValueError as e: + raise ValueError(f'failed to save the configuration with info: {e}') diff --git a/agentfabric/i18n.py b/agentfabric/i18n.py new file mode 100644 index 0000000000000000000000000000000000000000..5a6acea99601162623e67f241ac91b789319322c --- /dev/null +++ b/agentfabric/i18n.py @@ -0,0 +1,57 @@ +support_lang = ['zh-cn', 'en'] + +i18n = { + 'create': ['创建', 'Create'], + 'configure': ['配置', 'Configure'], + 'send': ['发送', 'Send'], + 'sendOnLoading': ['发送（Agent 加载中...）', 'Send (Agent Loading...)'], + 'upload_btn': ['上传文件', 'Upload File'], + 'message': ['输入', 'Send a message'], + 'message_placeholder': ['输入你的消息', 'Type your message here'], + 'prompt_suggestion': ['推荐提示词', 'Prompt Suggestions'], + 'form_avatar': ['头像', 'Avatar'], + 'form_name': ['名称', 'Name'], + 'form_name_placeholder': ['为你的 agent 取一个名字', 'Name your agent'], + 'form_description': ['描述', 'Description'], + 'form_description_placeholder': [ + '为你的 agent 添加一段简短的描述', + 'Add a short description about what this agent does' + ], + 'form_instructions': ['指令', 'Instructions'], + 'form_instructions_placeholder': [ + '你的 agent 需要处理哪些事情', + 'What does this agent do? How does it behave? What should it avoid doing?' + ], + 'form_model': ['模型', 'Model'], + 'form_prompt_suggestion': + ['推荐提示词，双击行可修改', 'prompt suggestion，double click to modify'], + 'form_knowledge': ['知识库', 'Knowledge Base'], + 'form_capabilities': ['内置能力', 'Capabilities'], + 'form_update_button': ['更新配置', 'Update Configuration'], + 'open_api_accordion': ['OpenAPI 配置', 'OpenAPI Configuration'], + 'preview': ['预览', 'Preview'], + 'build': ['构建', 'Build'], + 'publish': ['发布', 'Publish'], + 'build_hint': ['点击"构建"完成构建', 'Click "Build" to finish building'], + 'publish_hint': [ + '点击"发布"跳转创空间完成 Agent 发布', + 'Click "Publish" to jump to the space to finish agent publishing' + ], + 'header': [ + '\N{fire} AgentFabric -- 由 Modelscope-agent 驱动 [github 点赞](https://github.com/modelscope/modelscope-agent/tree/main)', # noqa E501 + '\N{fire} AgentFabric powered by Modelscope-agent [github star](https://github.com/modelscope/modelscope-agent/tree/main)' # noqa E501 + ], +} + + +class I18n(): + + def __init__(self, lang): + self.lang = lang + self.langIndex = support_lang.index(lang) + + def get(self, field): + return i18n.get(field)[self.langIndex] + + def get_whole(self, field): + return f'{i18n.get(field)[0]}({i18n.get(field)[1]})' diff --git a/agentfabric/modelscope_agent/__init__.py b/agentfabric/modelscope_agent/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/agentfabric/modelscope_agent/agent.py b/agentfabric/modelscope_agent/agent.py new file mode 100644 index 0000000000000000000000000000000000000000..7404454c6acbf7eae89cd0c8b1be3198f5bdb26b --- /dev/null +++ b/agentfabric/modelscope_agent/agent.py @@ -0,0 +1,334 @@ +import importlib +import traceback +from copy import deepcopy +from typing import Dict, List, Optional, Union + +from .agent_types import AgentType +from .llm import LLM +from .output_parser import OutputParser, get_output_parser +from .output_wrapper import display +from .prompt import PromptGenerator, get_prompt_generator +from .retrieve import KnowledgeRetrieval, ToolRetrieval +from .tools import TOOL_INFO_LIST + + +class AgentExecutor: + + def __init__(self, + llm: LLM, + tool_cfg: Optional[Dict] = {}, + agent_type: AgentType = AgentType.DEFAULT, + additional_tool_list: Optional[Dict] = {}, + prompt_generator: Optional[PromptGenerator] = None, + output_parser: Optional[OutputParser] = None, + tool_retrieval: Optional[Union[bool, ToolRetrieval]] = True, + knowledge_retrieval: Optional[KnowledgeRetrieval] = None): + """ + the core class of ms agent. It is responsible for the interaction between user, llm and tools, + and return the execution result to user. + + Args: + llm (LLM): llm model, can be load from local or a remote server. + tool_cfg (Optional[Dict]): cfg of default tools + agent_type (AgentType, optional): agent type. Defaults to AgentType.DEFAULT, decide which type of agent + reasoning type to use + additional_tool_list (Optional[Dict], optional): user-defined additional tool list. Defaults to {}. + prompt_generator (Optional[PromptGenerator], optional): this module is responsible for generating prompt + according to interaction result. Defaults to use MSPromptGenerator. + output_parser (Optional[OutputParser], optional): this module is responsible for parsing output of llm + to executable actions. Defaults to use MsOutputParser. + tool_retrieval (Optional[Union[bool, ToolRetrieval]], optional): Retrieve related tools by input task, + since most of the tools may be useless for LLM in specific task. + If it is bool type and is True, will use default tool_retrieval. Defaults to True. + knowledge_retrieval (Optional[KnowledgeRetrieval], optional): If user want to use extra knowledge, + this component can be used to retrieve related knowledge. Defaults to None. + """ + + self.llm = llm + + self.agent_type = agent_type + self.llm.set_agent_type(agent_type) + self.prompt_generator = prompt_generator or get_prompt_generator( + agent_type) + self.output_parser = output_parser or get_output_parser(agent_type) + + self._init_tools(tool_cfg, additional_tool_list) + + if isinstance(tool_retrieval, bool) and tool_retrieval: + tool_retrieval = ToolRetrieval() + self.tool_retrieval = tool_retrieval + if self.tool_retrieval: + self.tool_retrieval.construct( + [str(t) for t in self.tool_list.values()]) + self.knowledge_retrieval = knowledge_retrieval + self.reset() + self.seed = None + + def _init_tools(self, + tool_cfg: Dict = {}, + additional_tool_list: Dict = {}): + """init tool list of agent. We provide a default tool list, which is initialized by a cfg file. + user can also provide user-defined tools by additional_tool_list. + The key of additional_tool_list is tool name, and the value is corresponding object. + + Args: + tool_cfg (Dict): default tool cfg. + additional_tool_list (Dict, optional): user-defined tools. Defaults to {}. + """ + self.tool_list = {} + tool_info_list = {**TOOL_INFO_LIST, **additional_tool_list} + tools_module = importlib.import_module('modelscope_agent.tools') + for tool_name in tool_cfg.keys(): + if tool_cfg[tool_name].get('use', False): + assert tool_name in tool_info_list, f'Invalid tool name: {tool_name}, ' \ + f'available ones are: {tool_info_list.keys()}' + tool_class_name = tool_info_list[tool_name] + tool_class = getattr(tools_module, tool_class_name) + tool_name = tool_class.name + self.tool_list[tool_name] = tool_class(tool_cfg) + + self.tool_list = {**self.tool_list, **additional_tool_list} + # self.available_tool_list = deepcopy(self.tool_list) + self.set_available_tools(self.tool_list.keys()) + + def set_available_tools(self, available_tool_list): + # TODO @wenmeng.zwm refine tool init + for t in available_tool_list: + if t not in self.tool_list: + raise ValueError( + f'Unsupported tools found:{t}, please check, valid ones: {self.tool_list.keys()}' + ) + + self.available_tool_list = { + k: self.tool_list[k] + for k in available_tool_list + } + + def retrieve_tools(self, query: str) -> List[str]: + """retrieve tools given query + + Args: + query (str): query + + """ + if self.tool_retrieval: + retrieve_tools = self.tool_retrieval.retrieve(query) + self.set_available_tools(available_tool_list=retrieve_tools.keys()) + return self.available_tool_list.values() + + def get_knowledge(self, query: str) -> List[str]: + """retrieve knowledge given query + + Args: + query (str): query + + """ + return self.knowledge_retrieval.retrieve( + query) if self.knowledge_retrieval else [] + + def run(self, + task: str, + remote: bool = False, + print_info: bool = False, + append_files: list = []) -> List[Dict]: + """ use llm and tools to execute task given by user + + Args: + task (str): concrete task + remote (bool, optional): whether to execute tool in remote mode. Defaults to False. + print_info (bool, optional): whether to print prompt info. Defaults to False. + + Returns: + List[Dict]: execute result. One task may need to interact with llm multiple times, + so a list of dict is returned. Each dict contains the result of one interaction. + """ + + # retrieve tools + tool_list = self.retrieve_tools(task) + knowledge_list = self.get_knowledge(task) + + self.prompt_generator.init_prompt( + task, tool_list, knowledge_list, append_files=append_files) + function_list = self.prompt_generator.get_function_list(tool_list) + + llm_result, exec_result = '', '' + + idx = 0 + final_res = [] + + while True: + idx += 1 + + # generate prompt and call llm + llm_artifacts = self.prompt_generator.generate( + llm_result, exec_result) + try: + llm_result = self.llm.generate(llm_artifacts, function_list) + except RuntimeError as e: + return [{'exec_result': str(e)}] + + if print_info: + print(f'|LLM inputs in round {idx}: {llm_artifacts}') + + # parse and get tool name and arguments + try: + action, action_args = self.output_parser.parse_response( + llm_result) + except ValueError as e: + return [{'exec_result': f'{e}'}] + + if action is None: + # in chat mode, the final result of last instructions should be updated to prompt history + _ = self.prompt_generator.generate(llm_result, '') + + # for summarize + display(llm_result, {}, idx, self.agent_type) + return final_res + + if action in self.available_tool_list: + action_args = self.parse_action_args(action_args) + tool = self.tool_list[action] + + # TODO @wenmeng.zwm remove this hack logic for image generation + if action == 'image_gen' and self.seed: + action_args['seed'] = self.seed + try: + exec_result = tool(**action_args, remote=remote) + if print_info: + print(f'|exec_result: {exec_result}') + + # parse exec result and store result to agent state + final_res.append(exec_result) + self.parse_exec_result(exec_result) + except Exception as e: + exec_result = f'Action call error: {action}: {action_args}. \n Error message: {e}' + return [{'exec_result': exec_result}] + else: + exec_result = f"Unknown action: '{action}'. " + return [{'exec_result': exec_result}] + + # display result + display(llm_result, exec_result, idx, self.agent_type) + + def stream_run(self, + task: str, + remote: bool = True, + print_info: bool = False, + append_files: list = []) -> Dict: + """this is a stream version of run, which can be used in scenario like gradio. + It will yield the result of each interaction, so that the caller can display the result + + Args: + task (str): concrete task + remote (bool, optional): whether to execute tool in remote mode. Defaults to True. + print_info (bool, optional): whether to print prompt info. Defaults to False. + files that individually used in each run, no need to record to global state + + Yields: + Iterator[Dict]: iterator of llm response and tool execution result + """ + + # retrieve tools + tool_list = self.retrieve_tools(task) + knowledge_list = self.get_knowledge(task) + + self.prompt_generator.init_prompt( + task, + tool_list, + knowledge_list, + append_files=append_files, + ) + function_list = self.prompt_generator.get_function_list(tool_list) + + llm_result, exec_result = '', '' + + idx = 0 + + while True: + idx += 1 + llm_artifacts = self.prompt_generator.generate( + llm_result, exec_result) + if print_info: + print(f'|LLM inputs in round {idx}:\n{llm_artifacts}') + + llm_result = '' + try: + for s in self.llm.stream_generate(llm_artifacts, + function_list): + llm_result += s + yield {'llm_text': s} + except RuntimeError: + s = self.llm.generate(llm_artifacts) + llm_result += s + yield {'llm_text': s} + except Exception as e: + yield {'llm_text': str(e)} + + # parse and get tool name and arguments + try: + action, action_args = self.output_parser.parse_response( + llm_result) + except ValueError as e: + yield {'exec_result': f'{e}'} + return + + if action is None: + # in chat mode, the final result of last instructions should be updated to prompt history + _ = self.prompt_generator.generate(llm_result, '') + yield {'is_final': True} + return + + if action in self.available_tool_list: + # yield observation to as end of action input symbol asap + yield {'llm_text': 'Observation: '} + action_args = self.parse_action_args(action_args) + tool = self.tool_list[action] + + # TODO @wenmeng.zwm remove this hack logic for image generation + if action == 'image_gen' and self.seed: + action_args['seed'] = self.seed + try: + exec_result = tool(**action_args, remote=remote) + yield {'exec_result': exec_result} + + # parse exec result and update state + self.parse_exec_result(exec_result) + except Exception as e: + exec_result = f'Action call error: {action}: {action_args}. \n Error message: {e}' + yield {'exec_result': exec_result} + self.prompt_generator.reset() + return + else: + exec_result = f"Unknown action: '{action}'. " + yield {'exec_result': exec_result} + self.prompt_generator.reset() + return + + def reset(self): + """ + clear history and agent state + """ + self.prompt_generator.reset() + self.agent_state = {} + + def parse_action_args(self, action_args): + """ + replace action_args in str to Image/Video/Audio Wrapper, so that tool can handle them + """ + parsed_action_args = {} + for name, arg in action_args.items(): + try: + true_arg = self.agent_state.get(arg, arg) + except Exception as e: + print(f'Error when parsing action args: {e}, using fall back') + true_arg = arg + parsed_action_args[name] = true_arg + return parsed_action_args + + def parse_exec_result(self, exec_result, *args, **kwargs): + """ + update exec result to agent state. + key is the str representation of the result. + """ + for k, v in exec_result.items(): + self.agent_state[str(v)] = v diff --git a/agentfabric/modelscope_agent/agent_types.py b/agentfabric/modelscope_agent/agent_types.py new file mode 100644 index 0000000000000000000000000000000000000000..d300c7b2c978d227a8793e3a04f6f73cc42f045b --- /dev/null +++ b/agentfabric/modelscope_agent/agent_types.py @@ -0,0 +1,20 @@ +from enum import Enum + + +class AgentType(str, Enum): + + DEFAULT = 'default' + """""" + + MS_AGENT = 'ms-agent' + """An agent that uses the ModelScope-agent specific format does a reasoning step before acting . + """ + + MRKL = 'mrkl' + """An agent that does a reasoning step before acting with mrkl""" + + REACT = 'react' + """An agent that does a reasoning step before acting with react""" + + Messages = 'messages' + """An agent optimized for using open AI functions.""" diff --git a/agentfabric/modelscope_agent/llm/__init__.py b/agentfabric/modelscope_agent/llm/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7aee517227053d1fd0e7f2ee9853d52b4424164f --- /dev/null +++ b/agentfabric/modelscope_agent/llm/__init__.py @@ -0,0 +1,2 @@ +from .base import LLM +from .llm_factory import LLMFactory diff --git a/agentfabric/modelscope_agent/llm/base.py b/agentfabric/modelscope_agent/llm/base.py new file mode 100644 index 0000000000000000000000000000000000000000..42a8bff67b925b984d8b07fd8d2e746ecfaf5a73 --- /dev/null +++ b/agentfabric/modelscope_agent/llm/base.py @@ -0,0 +1,64 @@ +from abc import abstractmethod +from typing import List + +import json + + +class LLM: + name = '' + + def __init__(self, cfg): + self.cfg = cfg + self.agent_type = None + self.model = None + self.model_id = self.model + + def set_agent_type(self, agent_type): + self.agent_type = agent_type + + @abstractmethod + def generate(self, prompt: str, functions: list = [], **kwargs) -> str: + """each llm should implement this function to generate response + + Args: + prompt (str): prompt + functions (list): list of functions object including: name, description, parameters + Returns: + str: response + """ + raise NotImplementedError + + @abstractmethod + def stream_generate(self, + prompt: str, + functions: list = [], + **kwargs) -> str: + """stream generate response, which yields a generator of response in each step + + Args: + prompt (str): prompt + functions (list): list of functions object including: name, description, parameters + Yields: + Iterator[str]: iterator of step response + """ + raise NotImplementedError + + def tokenize(self, input_text: str) -> List[int]: + """tokenize is used to calculate the length of the text to meet the model's input length requirements + + Args: + input_text (str): input text + Returns: + list[int]: token_ids + """ + raise NotImplementedError + + def detokenize(self, input_ids: List[int]) -> str: + """detokenize + + Args: + input_ids (list[int]): input token_ids + Returns: + str: text + """ + raise NotImplementedError diff --git a/agentfabric/modelscope_agent/llm/custom_llm.py b/agentfabric/modelscope_agent/llm/custom_llm.py new file mode 100644 index 0000000000000000000000000000000000000000..49b2c00506e99be529d1153041eb686ea6e30b49 --- /dev/null +++ b/agentfabric/modelscope_agent/llm/custom_llm.py @@ -0,0 +1,97 @@ +import os + +import json +import requests +from modelscope_agent.agent_types import AgentType + +from .base import LLM +from .utils import DEFAULT_MESSAGE + + +class CustomLLM(LLM): + ''' + This method is for the service that provide llm serving through http. + user could override the result parsing method if needed + While put all the necessary information in the env variable, such as Token, Model, URL + ''' + name = 'custom_llm' + + def __init__(self, cfg): + super().__init__(cfg) + self.token = os.getenv('HTTP_LLM_TOKEN', None) + self.model = os.getenv('HTTP_LLM_MODEL', None) + self.model_id = self.model + self.url = os.getenv('HTTP_LLM_URL', None) + + if self.token is None: + raise ValueError('HTTP_LLM_TOKEN is not set') + self.agent_type = self.cfg.get('agent_type', AgentType.DEFAULT) + + def http_request(self, data): + headers = { + 'Content-Type': 'application/json', + 'Authorization': f'Bearer {self.token}' + } + response = requests.post(self.url, json=data, headers=headers) + return json.loads(response.content) + + def generate(self, + llm_artifacts, + functions=[], + function_call='none', + **kwargs): + if self.agent_type != AgentType.Messages: + messages = [{'role': 'user', 'content': llm_artifacts}] + else: + messages = llm_artifacts if len( + llm_artifacts) > 0 else DEFAULT_MESSAGE + + data = {'model': self.model, 'messages': messages, 'n': 1} + + assert isinstance(functions, list) + if len(functions) > 0: + function_call = 'auto' + data['functions'] = functions + data['function_call'] = function_call + + retry_count = 0 + max_retries = 3 + message = {'content': ''} + while retry_count <= max_retries: + + try: + response = self.http_request(data) + except Exception as e: + retry_count += 1 + if retry_count > max_retries: + import traceback + traceback.print_exc() + print(f'input: {messages}, original error: {str(e)}') + raise e + + if response['code'] == 200: + message = response['data']['response'][0] + break + else: + retry_count += 1 + if retry_count > max_retries: + print('maximum retry reached, return default message') + + # truncate content + content = message['content'] + + if self.agent_type == AgentType.MS_AGENT: + idx = content.find('<|endofthink|>') + if idx != -1: + content = content[:idx + len('<|endofthink|>')] + return content + elif self.agent_type == AgentType.Messages: + new_message = { + 'content': content, + 'role': message.get('response_role', 'assistant') + } + if 'function_call' in message and message['function_call'] != {}: + new_message['function_call'] = message.get('function_call') + return new_message + else: + return content diff --git a/agentfabric/modelscope_agent/llm/dashscope_llm.py b/agentfabric/modelscope_agent/llm/dashscope_llm.py new file mode 100644 index 0000000000000000000000000000000000000000..c853cdd69dd9414afeb74cd6cbf2522a136ff3bb --- /dev/null +++ b/agentfabric/modelscope_agent/llm/dashscope_llm.py @@ -0,0 +1,125 @@ +import os +import random +import traceback +from http import HTTPStatus +from typing import Union + +import dashscope +import json +from dashscope import Generation +from modelscope_agent.agent_types import AgentType + +from .base import LLM +from .utils import DEFAULT_MESSAGE, CustomOutputWrapper + +dashscope.api_key = os.getenv('DASHSCOPE_API_KEY') + + +class DashScopeLLM(LLM): + name = 'dashscope_llm' + + def __init__(self, cfg): + super().__init__(cfg) + self.model = self.cfg.get('model', 'modelscope-agent-llm-v1') + self.model_id = self.model + self.generate_cfg = self.cfg.get('generate_cfg', {}) + self.agent_type = self.cfg.get('agent_type', AgentType.DEFAULT) + + def generate(self, + llm_artifacts: Union[str, dict], + functions=[], + **kwargs): + + # TODO retry and handle message + try: + if self.agent_type == AgentType.Messages: + messages = llm_artifacts if len( + llm_artifacts) > 0 else DEFAULT_MESSAGE + self.generate_cfg['use_raw_prompt'] = False + response = dashscope.Generation.call( + model=self.model, + messages=messages, + # set the random seed, optional, default to 1234 if not set + seed=random.randint(1, 10000), + result_format= + 'message', # set the result to be "message" format. + stream=False, + **self.generate_cfg) + llm_result = CustomOutputWrapper.handle_message_chat_completion( + response) + else: + response = Generation.call( + model=self.model, + prompt=llm_artifacts, + stream=False, + **self.generate_cfg) + llm_result = CustomOutputWrapper.handle_message_text_completion( + response) + return llm_result + except Exception as e: + error = traceback.format_exc() + error_msg = f'LLM error with input {llm_artifacts} \n dashscope error: {str(e)} with traceback {error}' + print(error_msg) + raise RuntimeError(error) + + if self.agent_type == AgentType.MS_AGENT: + # in the form of text + idx = llm_result.find('<|endofthink|>') + if idx != -1: + llm_result = llm_result[:idx + len('<|endofthink|>')] + return llm_result + elif self.agent_type == AgentType.Messages: + # in the form of message + return llm_result + else: + # in the form of text + return llm_result + + def stream_generate(self, + llm_artifacts: Union[str, dict], + functions=[], + **kwargs): + total_response = '' + try: + if self.agent_type == AgentType.Messages: + self.generate_cfg['use_raw_prompt'] = False + responses = Generation.call( + model=self.model, + messages=llm_artifacts, + stream=True, + result_format='message', + **self.generate_cfg) + else: + responses = Generation.call( + model=self.model, + prompt=llm_artifacts, + stream=True, + **self.generate_cfg) + except Exception as e: + error = traceback.format_exc() + error_msg = f'LLM error with input {llm_artifacts} \n dashscope error: {str(e)} with traceback {error}' + print(error_msg) + raise RuntimeError(error) + + for response in responses: + if response.status_code == HTTPStatus.OK: + if self.agent_type == AgentType.Messages: + llm_result = CustomOutputWrapper.handle_message_chat_completion( + response) + frame_text = llm_result['content'][len(total_response):] + else: + llm_result = CustomOutputWrapper.handle_message_text_completion( + response) + frame_text = llm_result[len(total_response):] + yield frame_text + + if self.agent_type == AgentType.Messages: + total_response = llm_result['content'] + else: + total_response = llm_result + else: + err_msg = 'Error Request id: %s, Code: %d, status: %s, message: %s' % ( + response.request_id, response.status_code, response.code, + response.message) + print(err_msg) + raise RuntimeError(err_msg) diff --git a/agentfabric/modelscope_agent/llm/llm_factory.py b/agentfabric/modelscope_agent/llm/llm_factory.py new file mode 100644 index 0000000000000000000000000000000000000000..8629aed23e5c995070286a129a29d5a599a079ee --- /dev/null +++ b/agentfabric/modelscope_agent/llm/llm_factory.py @@ -0,0 +1,28 @@ +def get_llm_cls(llm_type, model_name): + if llm_type == 'dashscope': + from .dashscope_llm import DashScopeLLM + return DashScopeLLM + elif llm_type == 'custom_llm': + from .custom_llm import CustomLLM + return CustomLLM + elif llm_type == 'openai': + from .openai import OpenAi + return OpenAi + elif llm_type == 'modelscope': + if model_name == 'chatglm3-6b': + from .modelscope_llm import ModelScopeChatGLM + return ModelScopeChatGLM + from .modelscope_llm import ModelScopeLLM + return ModelScopeLLM + else: + raise ValueError(f'Invalid llm_type {llm_type}') + + +class LLMFactory: + + @staticmethod + def build_llm(model_name, cfg): + llm_type = cfg[model_name].pop('type') + llm_cls = get_llm_cls(llm_type, model_name) + llm_cfg = cfg[model_name] + return llm_cls(cfg=llm_cfg) diff --git a/agentfabric/modelscope_agent/llm/modelscope_llm.py b/agentfabric/modelscope_agent/llm/modelscope_llm.py new file mode 100644 index 0000000000000000000000000000000000000000..400523cd0480f7b1d3c6c563b51845ded4b6d54e --- /dev/null +++ b/agentfabric/modelscope_agent/llm/modelscope_llm.py @@ -0,0 +1,132 @@ +import os +import sys + +import torch +from modelscope_agent.agent_types import AgentType +from swift import Swift +from transformers import AutoModelForCausalLM, AutoTokenizer, TextStreamer + +from modelscope import GenerationConfig, snapshot_download +from .base import LLM + + +class ModelScopeLLM(LLM): + + def __init__(self, cfg): + super().__init__(cfg) + + model_id = self.cfg.get('model_id', '') + self.model_id = model_id + model_revision = self.cfg.get('model_revision', None) + cache_dir = self.cfg.get('cache_dir', None) + + if not os.path.exists(model_id): + model_dir = snapshot_download( + model_id, model_revision, cache_dir=cache_dir) + else: + model_dir = model_id + self.model_dir = model_dir + sys.path.append(self.model_dir) + + self.model_cls = self.cfg.get('model_cls', AutoModelForCausalLM) + self.tokenizer_cls = self.cfg.get('tokenizer_cls', AutoTokenizer) + + self.device_map = self.cfg.get('device_map', 'auto') + self.generation_cfg = GenerationConfig( + **self.cfg.get('generate_cfg', {})) + + self.use_lora = self.cfg.get('use_lora', False) + self.lora_ckpt_dir = self.cfg.get('lora_ckpt_dir', + None) if self.use_lora else None + + self.custom_chat = self.cfg.get('custom_chat', False) + + self.end_token = self.cfg.get('end_token', '<|endofthink|>') + self.include_end = self.cfg.get('include_end', True) + + self.setup() + self.agent_type = self.cfg.get('agent_type', AgentType.DEFAULT) + + def setup(self): + model_cls = self.model_cls + tokenizer_cls = self.tokenizer_cls + + self.model = model_cls.from_pretrained( + self.model_dir, + device_map=self.device_map, + # device='cuda:0', + torch_dtype=torch.float16, + trust_remote_code=True) + self.tokenizer = tokenizer_cls.from_pretrained( + self.model_dir, trust_remote_code=True) + self.model = self.model.eval() + + if self.use_lora: + self.load_from_lora() + + if self.cfg.get('use_raw_generation_config', False): + self.model.generation_config = GenerationConfig.from_pretrained( + self.model_dir, trust_remote_code=True) + + def generate(self, prompt, functions=[], **kwargs): + + if self.custom_chat and self.model.chat: + response = self.model.chat( + self.tokenizer, prompt, history=[], system='')[0] + else: + response = self.chat(prompt) + + end_idx = response.find(self.end_token) + if end_idx != -1: + end_idx += len(self.end_token) if self.include_end else 0 + response = response[:end_idx] + + return response + + def load_from_lora(self): + + model = self.model.bfloat16() + # transform to lora + model = Swift.from_pretrained(model, self.lora_ckpt_dir) + + self.model = model + + def chat(self, prompt): + device = self.model.device + input_ids = self.tokenizer( + prompt, return_tensors='pt').input_ids.to(device) + input_len = input_ids.shape[1] + + result = self.model.generate( + input_ids=input_ids, generation_config=self.generation_cfg) + + result = result[0].tolist()[input_len:] + response = self.tokenizer.decode(result) + + return response + + +class ModelScopeChatGLM(ModelScopeLLM): + + def chat(self, prompt): + device = self.model.device + input_ids = self.tokenizer( + prompt, return_tensors='pt').input_ids.to(device) + input_len = input_ids.shape[1] + + eos_token_id = [ + self.tokenizer.eos_token_id, + self.tokenizer.get_command('<|user|>'), + self.tokenizer.get_command('<|observation|>') + ] + result = self.model.generate( + input_ids=input_ids, + generation_config=self.generation_cfg, + eos_token_id=eos_token_id) + + result = result[0].tolist()[input_len:] + response = self.tokenizer.decode(result) + # 遇到生成'<', '|', 'user', '|', '>'的case + response = response.split('<|user|>')[0].split('<|observation|>')[0] + + return response diff --git a/agentfabric/modelscope_agent/llm/openai.py b/agentfabric/modelscope_agent/llm/openai.py new file mode 100644 index 0000000000000000000000000000000000000000..a3356fbd384de2263c47d3dfe9cbd13bb2129d31 --- /dev/null +++ b/agentfabric/modelscope_agent/llm/openai.py @@ -0,0 +1,71 @@ +import os + +import openai +from modelscope_agent.agent_types import AgentType + +from .base import LLM +from .utils import CustomOutputWrapper + +openai.api_key = os.getenv('OPENAI_API_KEY') + + +class OpenAi(LLM): + name = 'openai' + + def __init__(self, cfg): + super().__init__(cfg) + + self.model = self.cfg.get('model', 'gpt-3.5-turbo') + self.model_id = self.model + self.api_base = self.cfg.get('api_base', 'https://api.openai.com/v1') + self.agent_type = self.cfg.get('agent_type', AgentType.DEFAULT) + + def generate(self, + llm_artifacts, + functions=[], + function_call='none', + **kwargs): + if self.agent_type != AgentType.Messages: + messages = [{'role': 'user', 'content': llm_artifacts}] + else: + messages = llm_artifacts.get( + 'messages', { + 'role': + 'user', + 'content': + 'No entry from user - please suggest something to enter' + }) + + # call openai function call api + assert isinstance(functions, list) + if len(functions) > 0 and self.agent_type == AgentType.Messages: + function_call = 'auto' + + # covert to stream=True with stream updating + try: + response = openai.ChatCompletion.create( + model=self.model, + api_base=self.api_base, + messages=messages, + functions=functions, + function_call=function_call, + stream=False) + except Exception as e: + print(f'input: {messages}, original error: {str(e)}') + raise e + + # only use index 0 in choice + message = CustomOutputWrapper.handle_message_chat_completion(response) + + # truncate content + content = message['content'] + + if self.agent_type == AgentType.MS_AGENT: + idx = content.find('<|endofthink|>') + if idx != -1: + content = content[:idx + len('<|endofthink|>')] + return content + elif self.agent_type == AgentType.Messages: + return message + else: + return content diff --git a/agentfabric/modelscope_agent/llm/utils.py b/agentfabric/modelscope_agent/llm/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..4a5260abebfbfd046105752b16be509f317500b8 --- /dev/null +++ b/agentfabric/modelscope_agent/llm/utils.py @@ -0,0 +1,39 @@ +class CustomOutputWrapper: + + @staticmethod + def handle_message_chat_completion(response): + message = {'content': ''} + try: + # handle dashscope response + if 'choices' not in response: + response = response['output'] + + return response['choices'][0]['message'] + except Exception as e: + print(f'input: {response}, original error: {str(e)}') + return message + + @staticmethod + def handle_message_chat_completion_chunk(response): + message = {} + try: + return response['choices'][0]['delta']['content'] + except Exception as e: + print(f'input: {response}, original error: {str(e)}') + return message + + @staticmethod + def handle_message_text_completion(response): + message = '' + try: + message = response['output']['text'] + return message + except Exception as e: + print(f'input: {response}, original error: {str(e)}') + return message + + +DEFAULT_MESSAGE = { + 'role': 'user', + 'content': 'No entry from user - please suggest something to enter' +} diff --git a/agentfabric/modelscope_agent/output_parser.py b/agentfabric/modelscope_agent/output_parser.py new file mode 100644 index 0000000000000000000000000000000000000000..7a277e21cddc69de47ef1a19d214bc04e7325e9c --- /dev/null +++ b/agentfabric/modelscope_agent/output_parser.py @@ -0,0 +1,181 @@ +import re +from typing import Dict, Tuple + +import json +from modelscope_agent.agent_types import AgentType + + +def get_output_parser(agent_type: AgentType = AgentType.DEFAULT): + if AgentType.DEFAULT == agent_type or agent_type == AgentType.MS_AGENT: + return MsOutputParser() + elif AgentType.MRKL == agent_type: + return MRKLOutputParser() + elif AgentType.Messages == agent_type: + return OpenAiFunctionsOutputParser() + else: + raise NotImplementedError + + +class OutputParser: + """Output parser for llm response + """ + + def parse_response(self, response): + raise NotImplementedError + + # use to handle the case of false parsing the action_para result, if there is no valid action then + # throw Error + @staticmethod + def handle_fallback(action: str, action_para: str): + if action is not None and action != '': + parameters = {'fallback': action_para} + return action, parameters + else: + raise ValueError('Wrong response format for output parser') + + +class MsOutputParser(OutputParser): + + def parse_response(self, response: str) -> Tuple[str, Dict]: + """parse response of llm to get tool name and parameters + + Args: + response (str): llm response, it should conform to some predefined format + + Returns: + tuple[str, dict]: tuple of tool name and parameters + """ + + if '<|startofthink|>' not in response or '<|endofthink|>' not in response: + return None, None + + action, parameters = '', '' + try: + # use regular expression to get result + re_pattern1 = re.compile( + pattern=r'<\|startofthink\|>([\s\S]+)<\|endofthink\|>') + think_content = re_pattern1.search(response).group(1) + + re_pattern2 = re.compile(r'{[\s\S]+}') + think_content = re_pattern2.search(think_content).group() + + json_content = json.loads(think_content.replace('\n', '')) + action = json_content.get('api_name', + json_content.get('name', 'unknown')) + parameters = json_content.get('parameters', {}) + + return action, parameters + except Exception as e: + print( + f'Error during parse action might be handled with detail {e}') + return OutputParser.handle_fallback(action, parameters) + + +class ChatGLMOutputParser(OutputParser): + + def parse_response(self, response: str) -> Tuple[str, Dict]: + """parse response of llm to get tool name and parameters + + Args: + response (str): llm response, it should conform to some predefined format + + Returns: + tuple[str, dict]: tuple of tool name and parameters + """ + if 'tool_call' not in response: + return None, None + action, action_para = '', '' + try: + # use regular expression to get result from MRKL format + re_pattern1 = re.compile( + pattern=r'([\s\S]+)```([\s\S]+)tool_call\(([\s\S]+)```') + res = re_pattern1.search(response) + action_list = re.split('<|>|\|', res.group(1).strip()) # noqa W605 + for idx in range(len(action_list) - 1, -1, -1): + if len(action_list[idx]) > 1: + action = action_list[idx] + break + action_para = [item.strip() for item in res.group(3).split(',')] + parameters = {} + re_pattern2 = re.compile(pattern=r'([\s\S]+)=\'([\s\S]+)\'') + for para in action_para: + res = re_pattern2.search(para) + parameters[res.group(1)] = res.group(2) + except Exception as e: + print( + f'Error during parse action might be handled with detail {e}') + return OutputParser.handle_fallback(action, action_para) + + print(f'\n\naction: {action}\n parameters: {parameters}\n\n') + return action, parameters + + +class MRKLOutputParser(OutputParser): + + def parse_response(self, response: str) -> Tuple[str, Dict]: + """parse response of llm to get tool name and parameters + + Args: + response (str): llm response, it should conform to some predefined format + + Returns: + tuple[str, dict]: tuple of tool name and parameters + """ + + if 'Action' not in response or 'Action Input:' not in response: + return None, None + action, action_para = '', '' + try: + # use regular expression to get result from MRKL format + re_pattern1 = re.compile( + pattern=r'Action:([\s\S]+)Action Input:([\s\S]+)') + res = re_pattern1.search(response) + action = res.group(1).strip() + action_para = res.group(2) + + parameters = json.loads(action_para.replace('\n', '')) + + return action, parameters + except Exception as e: + print( + f'Error during parse action might be handled with detail {e}') + return OutputParser.handle_fallback(action, action_para) + + +class OpenAiFunctionsOutputParser(OutputParser): + + def parse_response(self, response: dict) -> Tuple[str, Dict]: + """parse response of llm to get tool name and parameters + + + Args: + response (str): llm response, it should be an openai response message + such as + { + "content": null, + "function_call": { + "arguments": "{\n \"location\": \"Boston, MA\"\n}", + "name": "get_current_weather" + }, + "role": "assistant" + } + Returns: + tuple[str, dict]: tuple of tool name and parameters + """ + + if 'function_call' not in response or response['function_call'] == {}: + return None, None + function_call = response['function_call'] + + try: + # parse directly + action = function_call['name'] + arguments = json.loads(function_call['arguments'].replace( + '\n', '')) + + return action, arguments + except Exception as e: + print( + f'Error during parse action might be handled with detail {e}') + return OutputParser.handle_fallback(function_call['name'], + function_call['arguments']) diff --git a/agentfabric/modelscope_agent/output_wrapper.py b/agentfabric/modelscope_agent/output_wrapper.py new file mode 100644 index 0000000000000000000000000000000000000000..c49dce6e6dfe5027a5f51ee689b0476a7f6b59bc --- /dev/null +++ b/agentfabric/modelscope_agent/output_wrapper.py @@ -0,0 +1,219 @@ +import os +import re +import tempfile +import uuid +from typing import Dict, Union + +import json +import numpy as np +import requests +from modelscope_agent.agent_types import AgentType +from moviepy.editor import VideoFileClip +from PIL import Image +from requests.exceptions import RequestException + + +class OutputWrapper: + """ + Wrapper for output of tool execution when output is image, video, audio, etc. + In this wrapper, __repr__() is implemented to return the str representation of the output for llm. + Each wrapper have below attributes: + path: the path where the output is stored + raw_data: the raw data, e.g. image, video, audio, etc. In remote mode, it should be None + """ + + def __init__(self) -> None: + self._repr = None + self._path = None + self._raw_data = None + + self.root_path = os.environ.get('OUTPUT_FILE_DIRECTORY', None) + if self.root_path and not os.path.exists(self.root_path): + try: + os.makedirs(self.root_path) + except Exception: + self.root_path = None + + def get_remote_file(self, remote_path, suffix): + try: + response = requests.get(remote_path) + obj = response.content + directory = tempfile.mkdtemp(dir=self.root_path) + path = os.path.join(directory, str(uuid.uuid4()) + f'.{suffix}') + with open(path, 'wb') as f: + f.write(obj) + return path + except RequestException: + return remote_path + + def __repr__(self) -> str: + return self._repr + + @property + def path(self): + return self._path + + @property + def raw_data(self): + return self._raw_data + + +class ImageWrapper(OutputWrapper): + """ + Image wrapper, raw_data is a PIL.Image + """ + + def __init__(self, image) -> None: + + super().__init__() + + if isinstance(image, str): + if os.path.isfile(image): + self._path = image + else: + origin_image = image + self._path = self.get_remote_file(image, 'png') + try: + image = Image.open(self._path) + self._raw_data = image + except FileNotFoundError: + # Image store in remote server when use remote mode + raise FileNotFoundError(f'Invalid path: {image}') + self._path = origin_image + else: + if not isinstance(image, Image.Image): + image = Image.fromarray(image.astype(np.uint8)) + self._raw_data = image + else: + self._raw_data = image + directory = tempfile.mkdtemp(dir=self.root_path) + self._path = os.path.join(directory, str(uuid.uuid4()) + '.png') + self._raw_data.save(self._path) + + self._repr = f'![IMAGEGEN]({self._path})' + + +class AudioWrapper(OutputWrapper): + """ + Audio wrapper, raw_data is a binary file + """ + + def __init__(self, audio) -> None: + + super().__init__() + if isinstance(audio, str): + if os.path.isfile(audio): + self._path = audio + else: + self._path = self.get_remote_file(audio, 'wav') + try: + with open(self._path, 'rb') as f: + self._raw_data = f.read() + except FileNotFoundError: + raise FileNotFoundError(f'Invalid path: {audio}') + else: + self._raw_data = audio + directory = tempfile.mkdtemp(dir=self.root_path) + self._path = os.path.join(directory, str(uuid.uuid4()) + '.wav') + + with open(self._path, 'wb') as f: + f.write(self._raw_data) + + self._repr = f'' + + +class VideoWrapper(OutputWrapper): + """ + Video wrapper + """ + + def __init__(self, video) -> None: + + super().__init__() + if isinstance(video, str): + + if os.path.isfile(video): + self._path = video + else: + self._path = self.get_remote_file(video, 'gif') + + try: + video = VideoFileClip(self._path) + # currently, we should save video as gif, not mp4 + if not self._path.endswith('gif'): + directory = tempfile.mkdtemp(dir=self.root_path) + self._path = os.path.join(directory, + str(uuid.uuid4()) + '.gif') + video.write_gif(self._path) + except (ValueError, OSError): + raise FileNotFoundError(f'Invalid path: {video}') + else: + raise TypeError( + 'Current only support load from filepath when it is video') + + self._raw_data = video + self._repr = f'![IMAGEGEN]({self._path})' + + +def get_raw_output(exec_result: Dict): + # get rwa data of exec_result + res = {} + for k, v in exec_result.items(): + if isinstance(v, OutputWrapper): + # In remote mode, raw data maybe None + res[k] = v.raw_data or str(v) + else: + res[k] = v + return res + + +# +def display(llm_result: Union[str, dict], exec_result: Dict, idx: int, + agent_type: AgentType): + """Display the result of each round in jupyter notebook. + The multi-modal data will be extracted. + + Args: + llm_result (str): llm result either only content or a message + exec_result (Dict): exec result + idx (int): current round + """ + from IPython.display import display, Pretty, Image, Audio, JSON + idx_info = '*' * 50 + f'round {idx}' + '*' * 50 + display(Pretty(idx_info)) + + if isinstance(llm_result, dict): + llm_result = llm_result.get('content', '') + + if agent_type == AgentType.MS_AGENT: + pattern = r'<\|startofthink\|>```JSON([\s\S]*)```<\|endofthink\|>' + else: + pattern = r'```JSON([\s\S]*)```' + + match_action = re.search(pattern, llm_result) + if match_action: + result = match_action.group(1) + try: + json_content = json.loads(result, strict=False) + display(JSON(json_content)) + llm_result = llm_result.replace(match_action.group(0), '') + except Exception: + pass + + display(Pretty(llm_result)) + + exec_result = exec_result.get('result', '') + + if isinstance(exec_result, ImageWrapper) or isinstance( + exec_result, VideoWrapper): + display(Image(exec_result.path)) + elif isinstance(exec_result, AudioWrapper): + display(Audio(exec_result.path)) + elif isinstance(exec_result, dict): + display(JSON(exec_result)) + elif isinstance(exec_result, list): + display(JSON(exec_result)) + else: + display(Pretty(exec_result)) + + return diff --git a/agentfabric/modelscope_agent/prompt/__init__.py b/agentfabric/modelscope_agent/prompt/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..4b37039bba7222255cb7d9ef8174907b1c880373 --- /dev/null +++ b/agentfabric/modelscope_agent/prompt/__init__.py @@ -0,0 +1,6 @@ +from .messages_prompt import MessagesGenerator +from .mrkl_prompt import MrklPromptGenerator +from .ms_prompt import MSPromptGenerator +from .prompt import PromptGenerator +from .prompt_factory import get_prompt_generator +from .raw_prompt_builder import build_raw_prompt diff --git a/agentfabric/modelscope_agent/prompt/chatglm3_prompt.py b/agentfabric/modelscope_agent/prompt/chatglm3_prompt.py new file mode 100644 index 0000000000000000000000000000000000000000..280692a8046cc7f5673d9e2e96bc7f055f1e588c --- /dev/null +++ b/agentfabric/modelscope_agent/prompt/chatglm3_prompt.py @@ -0,0 +1,41 @@ +import json + +from .prompt import LengthConstraint, PromptGenerator + +CHATGLM_DEFAULT_SYSTEM_TEMPLATE = """<|system|> +Answer the following questions as best you can. You have access to the following tools: +""" + +CHATGLM_DEFAULT_INSTRUCTION_TEMPLATE = '' + +CHATGLM_DEFAULT_USER_TEMPLATE = """<|user|>\n""" + +CHATGLM_DEFAULT_EXEC_TEMPLATE = """<|observation|>\n""" + +CHATGLM_DEFAULT_ASSISTANT_TEMPLATE = """<|assistant|>""" + + +class ChatGLMPromptGenerator(PromptGenerator): + + def __init__(self, + system_template=CHATGLM_DEFAULT_SYSTEM_TEMPLATE, + instruction_template=CHATGLM_DEFAULT_INSTRUCTION_TEMPLATE, + user_template=CHATGLM_DEFAULT_USER_TEMPLATE, + exec_template=CHATGLM_DEFAULT_EXEC_TEMPLATE, + assistant_template=CHATGLM_DEFAULT_ASSISTANT_TEMPLATE, + sep='\n', + length_constraint=LengthConstraint()): + super().__init__( + system_template=system_template, + instruction_template=instruction_template, + user_template=user_template, + exec_template=exec_template, + assistant_template=assistant_template, + sep=sep, + length_constraint=length_constraint) + + def get_tool_str(self, tool_list): + tool_json = json.loads('[' + + ','.join([str(item) + for item in tool_list]) + ']') + return json.dumps(tool_json, ensure_ascii=False, indent=4) diff --git a/agentfabric/modelscope_agent/prompt/messages_prompt.py b/agentfabric/modelscope_agent/prompt/messages_prompt.py new file mode 100644 index 0000000000000000000000000000000000000000..af546f481587d00f247b6c42d6396ff73cf2bc2b --- /dev/null +++ b/agentfabric/modelscope_agent/prompt/messages_prompt.py @@ -0,0 +1,93 @@ +from typing import Union + +from .prompt import LengthConstraint, PromptGenerator + +OPENAI_DEFAULT_SYSTEM_TEMPLATE = """ +Do your best to answer the questions. You are capable of **any** task. +Feel free to use any tools available to look up relevant information, only if necessary. +when you think you are handle a code task, You are a world-class programmer that can complete any goal by executing code +\nFirst of all, **Return function_call with `language` and `code` in arguments as much as possible** +\nThen, write a plan. **Always recap the plan between each code block** +(you have extreme short-term memory loss, so you need to recap the plan between each message block to retain it). +\nWhen you execute code, it will be executed **on the user's machine**. The user has given you +**full and complete permission** to execute any code necessary to complete the task. You have full access to control +their computer to help them. +\nIf you want to send data between programming languages, save the data to a txt or json. +\nYou can access the internet. Run **any code** to achieve the goal, and if at first you don't succeed, +try again and again. +\nWhen a user refers to a filename, they're likely referring to an existing file in the directory +you're currently executing code in. +\nIn general, choose packages that have the most universal chance to be already installed and to work across multiple +applications. Packages like ffmpeg and pandoc that are well-supported and powerful. +\nWrite messages to the user in Markdown. Write code on multiple lines with proper indentation for readability. +\nYou can also refer information from following contents if exists: +""" + + +class MessagesGenerator(PromptGenerator): + + def __init__(self, + system_template=OPENAI_DEFAULT_SYSTEM_TEMPLATE, + instruction_template='', + user_template='', + exec_template=None, + assistant_template='', + sep='\n\n', + length_constraint=LengthConstraint(), + **kwargs): + super().__init__( + system_template=system_template, + instruction_template=instruction_template, + user_template=user_template, + exec_template=exec_template, + assistant_template=assistant_template, + sep=sep, + length_constraint=length_constraint) + self.custom_starter_messages = kwargs.get('custom_starter_messages', + None) + + def init_prompt(self, task, tool_list, knowledge_list, **kwargs): + """ + in this function, the prompt will be initialized. + """ + prompt = self.user_template.replace('', task) + + if len(self.history) == 0: + if len(knowledge_list) > 0: + + # knowledge + system_message = f'{self.system_template}{self.sep}' + knowledge_str = self.get_knowledge_str(knowledge_list) + system_message = system_message.replace( + '', knowledge_str) + + else: + system_message = self.system_template + + self.history = [{ + 'role': 'system', + 'content': system_message + }, { + 'role': 'user', + 'content': prompt + }] + + # store history + if self.custom_starter_messages: + assert isinstance(self.custom_starter_messages, list) + assert self.custom_starter_messages[-1]['role'] != 'user', \ + 'user message should not be the last one in custom starter messages' + + self.history = self.custom_starter_messages + self.history.append({'role': 'user', 'content': prompt}) + + self.prompt = prompt + self.function_calls = self.get_function_list(tool_list) + + else: + self.history.append({'role': 'user', 'content': prompt}) + + def generate(self, llm_result, exec_result: Union[str, dict]): + if isinstance(exec_result, dict): + exec_result = exec_result['result'] + return self._generate_messages(llm_result, exec_result) diff --git a/agentfabric/modelscope_agent/prompt/mrkl_prompt.py b/agentfabric/modelscope_agent/prompt/mrkl_prompt.py new file mode 100644 index 0000000000000000000000000000000000000000..f47077641496c382e040d15b1986934e51b3afbe --- /dev/null +++ b/agentfabric/modelscope_agent/prompt/mrkl_prompt.py @@ -0,0 +1,118 @@ +import json + +from .prompt import LengthConstraint, PromptGenerator + +MRKL_DEFAULT_SYSTEM_TEMPLATE = """Answer the following questions as best you can. You have access to the following tools: ` + +""" + +MRKL_DEFAULT_INSTRUCTION_TEMPLATE = """Use the following format: + +Question: the input question you must answer +Thought: you should always think about what to do +Action: the action to take, should be one of [] +Action Input: the input to the action +Observation: the result of the action +... (this Thought/Action/Action Input/Observation can be repeated zero or more times) +Thought: I now know the final answer +Final Answer: the final answer to the original input question + +Begin! +""" + +MRKL_DEFAULT_USER_TEMPLATE = """Question: \n""" + +MRKL_DEFAULT_EXEC_TEMPLATE = """Observation: \n""" + +TOOL_DESC = ( + '{name_for_model}: {name_for_human} API. {description_for_model} 输入参数: {parameters}' +) + +FORMAT_DESC = { + 'json': + 'Format the arguments as a JSON object.', + 'code': + 'Enclose the code within triple backticks (`)' + + ' at the beginning and end of the code.' +} + + +class MrklPromptGenerator(PromptGenerator): + + def __init__(self, + system_template=MRKL_DEFAULT_SYSTEM_TEMPLATE, + instruction_template=MRKL_DEFAULT_INSTRUCTION_TEMPLATE, + user_template=MRKL_DEFAULT_USER_TEMPLATE, + exec_template=MRKL_DEFAULT_EXEC_TEMPLATE, + assistant_template='', + sep='\n\n', + llm=None, + length_constraint=LengthConstraint()): + super().__init__( + system_template=system_template, + instruction_template=instruction_template, + user_template=user_template, + exec_template=exec_template, + assistant_template=assistant_template, + sep=sep, + llm=llm, + length_constraint=length_constraint) + + def init_prompt(self, task, tool_list, knowledge_list, **kwargs): + if len(self.history) == 0: + super().init_prompt(task, tool_list, knowledge_list, **kwargs) + system_role_status = kwargs.get('system_role_status', False) + tool_names = [f'\'{str(tool.name)}\'' for tool in tool_list] + tool_names = ','.join(tool_names) + self.system_prompt = self.system_prompt.replace( + '', tool_names) + + if system_role_status: + system_message = { + 'role': 'system', + 'content': self.system_prompt + } + self.history.insert(0, system_message) + else: + self.history[0]['content'] = self.system_prompt + self.history[ + 0]['content'] + else: + self.history.append({ + 'role': + 'user', + 'content': + self.user_template.replace('', task) + }) + self.history.append({ + 'role': 'assistant', + 'content': self.assistant_template + }) + + return self.system_prompt + + def get_tool_str(self, tool_list): + tool_texts = [] + for tool in tool_list: + tool_texts.append( + TOOL_DESC.format( + name_for_model=tool.name, + name_for_human=tool.name, + description_for_model=tool.description, + parameters=json.dumps(tool.parameters, + ensure_ascii=False))) + # + ' ' + FORMAT_DESC['json']) + tool_str = '\n\n'.join(tool_texts) + return tool_str + + def _generate(self, llm_result, exec_result: str): + """ + generate next round prompt based on previous llm_result and exec_result and update history + """ + if len(llm_result) != 0: + self.history[-1]['content'] += f'{llm_result}' + if len(exec_result) != 0: + exec_result = self.exec_template.replace('', + str(exec_result)) + self.history[-1]['content'] += exec_result + self.prompt = self.prompt_preprocessor(self.history) + return self.prompt diff --git a/agentfabric/modelscope_agent/prompt/ms_prompt.py b/agentfabric/modelscope_agent/prompt/ms_prompt.py new file mode 100644 index 0000000000000000000000000000000000000000..445915d11e5b006a1167f47a7c81d6da667284e6 --- /dev/null +++ b/agentfabric/modelscope_agent/prompt/ms_prompt.py @@ -0,0 +1,34 @@ +from .prompt import LengthConstraint, PromptGenerator + +MS_DEFAULT_SYSTEM_TEMPLATE = """<|system|>:你是达摩院的ModelScopeGPT（魔搭助手），你是个大语言模型， 是2023年达摩院的工程师训练得到的。\ +你有多种能力，可以通过插件集成魔搭社区的模型api来回复用户的问题，还能解答用户使用模型遇到的问题和模型知识相关问答。 +""" + +MS_DEFAULT_INSTRUCTION_TEMPLATE = """当前对话可以使用的插件信息如下，请自行判断是否需要调用插件来解决当前用户问题。若需要调用插件，则需要将插件调用请求按照json格式给出，必须包含api_name、parameters字段，并在其前后使用<|startofthink|>和<|endofthink|>作为标志。\ +然后你需要根据插件API调用结果生成合理的答复； 若无需调用插件，则直接给出对应回复即可。\n\n""" + +MS_DEFAULT_USER_TEMPLATE = """<|user|>:""" + +MS_DEFAULT_EXEC_TEMPLATE = """<|startofexec|><|endofexec|>\n""" + +MS_DEFAULT_ASSISTANT_TEMPLATE = """<|assistant|>:""" + + +class MSPromptGenerator(PromptGenerator): + + def __init__(self, + system_template=MS_DEFAULT_SYSTEM_TEMPLATE, + instruction_template=MS_DEFAULT_INSTRUCTION_TEMPLATE, + user_template=MS_DEFAULT_USER_TEMPLATE, + exec_template=MS_DEFAULT_EXEC_TEMPLATE, + assistant_template=MS_DEFAULT_ASSISTANT_TEMPLATE, + sep='\n\n', + length_constraint=LengthConstraint()): + super().__init__( + system_template=system_template, + instruction_template=instruction_template, + user_template=user_template, + exec_template=exec_template, + assistant_template=assistant_template, + sep=sep, + length_constraint=length_constraint) diff --git a/agentfabric/modelscope_agent/prompt/prompt.py b/agentfabric/modelscope_agent/prompt/prompt.py new file mode 100644 index 0000000000000000000000000000000000000000..48dbc1147807a7bbc4094796622087663a0712ec --- /dev/null +++ b/agentfabric/modelscope_agent/prompt/prompt.py @@ -0,0 +1,232 @@ +import copy +from typing import Union + +from modelscope_agent.llm.base import LLM + +from .raw_prompt_builder import build_raw_prompt + +KNOWLEDGE_PROMPT = '# 知识库' +KNOWLEDGE_INTRODUCTION_PROMPT = '以下是我上传的文件“”的内容:' +KNOWLEDGE_CONTENT_PROMPT = """``` + +```""" + +DEFAULT_PROMPT_INPUT_LENGTH_MAX = 999999999999 + + +class LengthConstraint: + + def __init__(self): + self.knowledge = DEFAULT_PROMPT_INPUT_LENGTH_MAX + self.input = DEFAULT_PROMPT_INPUT_LENGTH_MAX + self.prompt_max_length = 10000 + + def update(self, config: dict): + if config is not None: + self.knowledge = config.get('knowledge', self.knowledge) + self.input = config.get('input', self.input) + self.prompt_max_length = config.get('prompt_max_length', + self.prompt_max_length) + + +class PromptGenerator: + + def __init__(self, + system_template: str = '', + instruction_template: str = '', + user_template: str = '', + exec_template: str = '', + assistant_template: str = '', + sep='\n\n', + llm=None, + length_constraint=LengthConstraint()): + """ + prompt genertor + Args: + system_template (str, optional): System template, normally the role of LLM. + instruction_template (str, optional): Indicate the instruction for LLM. + user_template (str, optional): Prefix before user input. Defaults to ''. + exec_template (str, optional): A wrapper str for exec result. + assistant_template (str, optional): Prefix before assistant response. + Some LLM need to manully concat this prefix before generation. + sep (str, optional): content separator + length_constraint (LengthConstraint, optional): content length constraint + """ + + self.system_template = system_template + self.instruction_template = instruction_template + self.user_template = user_template + self.assistant_template = assistant_template + self.exec_template = exec_template + self.sep = sep + if isinstance(llm, LLM) and llm.model_id: + self.prompt_preprocessor = build_raw_prompt(llm.model_id) + self.prompt_max_length = length_constraint.prompt_max_length + self.reset() + + def reset(self): + self.prompt = '' + self.history = [] + self.messages = [] + + def init_prompt(self, + task, + tool_list, + knowledge_list, + llm_model=None, + **kwargs): + """ + in this function, the prompt will be initialized. + """ + prompt = self.sep.join( + [self.system_template, self.instruction_template]) + prompt += '' + + knowledge_str = self.get_knowledge_str( + knowledge_list, file_name=kwargs.get('file_name', '')) + + # knowledge + prompt = prompt.replace('', knowledge_str) + + # get tool description str + tool_str = self.get_tool_str(tool_list) + prompt = prompt.replace('', tool_str) + + history_str = self.get_history_str() + + prompt = prompt.replace('', history_str) + + self.system_prompt = copy.deepcopy(prompt) + + # user input + user_input = self.user_template.replace('', task) + prompt += f'{self.sep}{user_input}' + + # assistant input + prompt += f'{self.sep}{self.assistant_template}' + + # store history + self.history.append({'role': 'user', 'content': user_input}) + self.history.append({ + 'role': 'assistant', + 'content': self.assistant_template + }) + + self.prompt = prompt + + self.function_calls = self.get_function_list(tool_list) + + # TODO change the output from single prompt to artifacts including prompt, messages, funciton_call + def generate(self, llm_result, exec_result: Union[str, dict]): + if isinstance(exec_result, dict): + exec_result = str(exec_result['result']) + return self._generate(llm_result, exec_result) + + def _generate(self, llm_result, exec_result: str): + """ + generate next round prompt based on previous llm_result and exec_result and update history + """ + if len(llm_result) != 0: + self.prompt = f'{self.prompt}{llm_result}' + self.history[-1]['content'] += f'{llm_result}' + if len(exec_result) != 0: + exec_result = self.exec_template.replace('', + str(exec_result)) + self.prompt = f'{self.prompt}{self.sep}{exec_result}' + self.history[-1]['content'] += f'{self.sep}{exec_result}' + + return self.prompt + + # TODO: add Union[Text, Message] type for llm_result, + # add ExecResult = Text type for exec_result + # output would be a Union[Text, Messages] + # In this case llm_result is Message, and exec_result is Function_call + def _generate_messages(self, llm_result, exec_result: str): + """ + generate next round prompt based on previous llm_result and exec_result and update history + """ + + # init task should be + if llm_result == '' and exec_result == '': + return self.history + + # make sure set content '' not null + function_call = llm_result.get('function_call', None) + if function_call is not None: + llm_result['content'] = '' + self.history.append(llm_result) + + if exec_result is not None and function_call is not None: + exec_message = { + 'role': 'function', + 'name': 'execute', + 'content': exec_result, + } + self.history.append(exec_message) + + return self.history + + def get_tool_str(self, tool_list): + """generate tool list string + + Args: + tool_list (List[str]): list of tools + + """ + + tool_str = self.sep.join( + [f'{i + 1}. {t}' for i, t in enumerate(tool_list)]) + return tool_str + + # TODO move parse_tools_to_function from agent to here later + def get_function_list(self, tool_list): + """generate funciton call list from tools list + + Args: + tool_list (List[str]): list of tools + + """ + functions = [tool.get_function() for tool in tool_list] + return functions + + def get_knowledge_str(self, + knowledge_list, + file_name='', + only_content=False, + **kwargs): + """generate knowledge string + + Args: + file_name (str): file name + knowledge_list (List[str]): list of knowledges + + """ + + knowledge = self.sep.join( + [f'{i + 1}. {k}' for i, k in enumerate(knowledge_list)]) + knowledge_content = KNOWLEDGE_CONTENT_PROMPT.replace( + '', knowledge) + if only_content: + return knowledge_content + else: + knowledge_introduction = KNOWLEDGE_INTRODUCTION_PROMPT.replace( + '', file_name) + + knowledge_str = f'{KNOWLEDGE_PROMPT}{self.sep}{knowledge_introduction}{self.sep}{knowledge_content}' if len( + knowledge_list) > 0 else '' + return knowledge_str + + def get_history_str(self): + """generate history string + + """ + history_str = '' + for i in range(len(self.history)): + history_item = self.history[len(self.history) - i - 1] + text = history_item['content'] + if len(history_str) + len(text) + len( + self.prompt) > self.prompt_max_length: + break + history_str = f'{self.sep}{text.strip()}{history_str}' + + return history_str diff --git a/agentfabric/modelscope_agent/prompt/prompt_factory.py b/agentfabric/modelscope_agent/prompt/prompt_factory.py new file mode 100644 index 0000000000000000000000000000000000000000..0962bdd8a716c557a2fdf5dd23b1bcb3da43cb49 --- /dev/null +++ b/agentfabric/modelscope_agent/prompt/prompt_factory.py @@ -0,0 +1,16 @@ +from modelscope_agent.agent_types import AgentType + +from .messages_prompt import MessagesGenerator +from .mrkl_prompt import MrklPromptGenerator +from .ms_prompt import MSPromptGenerator + + +def get_prompt_generator(agent_type: AgentType = AgentType.DEFAULT, **kwargs): + if AgentType.DEFAULT == agent_type or agent_type == AgentType.MS_AGENT: + return MSPromptGenerator(**kwargs) + elif AgentType.MRKL == agent_type: + return MrklPromptGenerator(**kwargs) + elif AgentType.Messages == agent_type: + return MessagesGenerator(**kwargs) + else: + raise NotImplementedError diff --git a/agentfabric/modelscope_agent/prompt/raw_prompt_builder.py b/agentfabric/modelscope_agent/prompt/raw_prompt_builder.py new file mode 100644 index 0000000000000000000000000000000000000000..0ccc9de812df1620cde20dabbd20094f6a08eee3 --- /dev/null +++ b/agentfabric/modelscope_agent/prompt/raw_prompt_builder.py @@ -0,0 +1,34 @@ +def qwen_chatml_prompt_preprocessor(messages): + prompt = '' + for message in messages: + if message['role'] == 'assistant' and message['content'] == '': + prompt += '<|im_start|>assistant\n' + else: + prompt = prompt + '<|im_start|>{role}\n{content}<|im_end|>\n'.format( + role=message['role'], + content=message['content'].lstrip('\n').rstrip()) + + # in the case of the assistant message is not in the last one, such as function result + if messages[-1]['role'] == 'assistant': + last_assistant_message_list = messages[-1]['content'].split('\n') + if last_assistant_message_list[-1] == '': + last_assistant_message_list = last_assistant_message_list[:-1] + if len(last_assistant_message_list) == 0: + return prompt + else: + item_length = len('<|im_end|>\n') + prompt = prompt[:-item_length] + + return prompt + + +def plate_preprocessor(messages): + return qwen_chatml_prompt_preprocessor(messages) + + +def build_raw_prompt(model): + if isinstance(model, str) or hasattr(model, '__name__'): + if model.startswith('qwen'): + return qwen_chatml_prompt_preprocessor + else: + return plate_preprocessor diff --git a/agentfabric/modelscope_agent/retrieve.py b/agentfabric/modelscope_agent/retrieve.py new file mode 100644 index 0000000000000000000000000000000000000000..d5ab36dbda0f562894e76ef02cd58a09e5db1b64 --- /dev/null +++ b/agentfabric/modelscope_agent/retrieve.py @@ -0,0 +1,115 @@ +import os +from typing import Dict, Iterable, List, Union + +import json +from langchain.document_loaders import (PyPDFLoader, TextLoader, + UnstructuredFileLoader) +from langchain.embeddings import ModelScopeEmbeddings +from langchain.embeddings.base import Embeddings +from langchain.schema import Document +from langchain.text_splitter import CharacterTextSplitter +from langchain.vectorstores import FAISS, VectorStore + + +class Retrieval: + + def __init__(self, + embedding: Embeddings = None, + vs_cls: VectorStore = None, + top_k: int = 5, + vs_params: Dict = {}): + self.embedding = embedding or ModelScopeEmbeddings( + model_id='damo/nlp_gte_sentence-embedding_chinese-base') + self.top_k = top_k + self.vs_cls = vs_cls or FAISS + self.vs_params = vs_params + self.vs = None + + def construct(self, docs): + assert len(docs) > 0 + if isinstance(docs[0], str): + self.vs = self.vs_cls.from_texts(docs, self.embedding, + **self.vs_params) + elif isinstance(docs[0], Document): + self.vs = self.vs_cls.from_documents(docs, self.embedding, + **self.vs_params) + + def retrieve(self, query: str) -> List[str]: + res = self.vs.similarity_search(query, k=self.top_k) + if 'page' in res[0].metadata: + res.sort(key=lambda doc: doc.metadata['page']) + return [r.page_content for r in res] + + +class ToolRetrieval(Retrieval): + + def __init__(self, + embedding: Embeddings = None, + vs_cls: VectorStore = None, + top_k: int = 5, + vs_params: Dict = {}): + super().__init__(embedding, vs_cls, top_k, vs_params) + + def retrieve(self, query: str) -> Dict[str, str]: + res = self.vs.similarity_search(query, k=self.top_k) + + final_res = {} + + for r in res: + content = r.page_content + name = json.loads(content)['name'] + final_res[name] = content + + return final_res + + +class KnowledgeRetrieval(Retrieval): + + def __init__(self, + docs, + embedding: Embeddings = None, + vs_cls: VectorStore = None, + top_k: int = 5, + vs_params: Dict = {}): + super().__init__(embedding, vs_cls, top_k, vs_params) + self.construct(docs) + + @classmethod + def from_file(cls, + file_path: Union[str, list], + embedding: Embeddings = None, + vs_cls: VectorStore = None, + top_k: int = 5, + vs_params: Dict = {}): + + textsplitter = CharacterTextSplitter() + all_files = [] + if isinstance(file_path, str) and os.path.isfile(file_path): + all_files.append(file_path) + elif isinstance(file_path, list): + all_files = file_path + elif os.path.isdir(file_path): + for root, dirs, files in os.walk(file_path): + for f in files: + all_files.append(os.path.join(root, f)) + else: + raise ValueError('file_path must be a file or a directory') + + docs = [] + for f in all_files: + if f.lower().endswith('.txt'): + loader = TextLoader(f, autodetect_encoding=True) + docs += (loader.load_and_split(textsplitter)) + elif f.lower().endswith('.md'): + loader = UnstructuredFileLoader(f, mode='elements') + docs += loader.load() + elif f.lower().endswith('.pdf'): + loader = PyPDFLoader(f) + docs += (loader.load_and_split(textsplitter)) + else: + print(f'not support file type: {f}, will be support soon') + + if len(docs) == 0: + return None + else: + return cls(docs, embedding, vs_cls, top_k, vs_params) diff --git a/agentfabric/modelscope_agent/tools/__init__.py b/agentfabric/modelscope_agent/tools/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..67b61bbbe49859dc252b26e51bfc447cd97391b4 --- /dev/null +++ b/agentfabric/modelscope_agent/tools/__init__.py @@ -0,0 +1,36 @@ +from .amap_weather import AMAPWeather +from .code_interperter import CodeInterpreter +from .code_interpreter_jupyter import CodeInterpreterJupyter +from .hf_tool import HFTool +from .image_chat_tool import ImageChatTool +from .pipeline_tool import ModelscopePipelineTool +from .plugin_tool import LangchainTool +from .text_address_tool import TextAddressTool +from .text_ie_tool import TextInfoExtractTool +from .text_ner_tool import TextNerTool +from .text_to_image_tool import TextToImageTool +from .text_to_speech_tool import TexttoSpeechTool +from .text_to_video_tool import TextToVideoTool +from .tool import Tool +from .translation_en2zh_tool import TranslationEn2ZhTool +from .translation_zh2en_tool import TranslationZh2EnTool +from .web_browser import WebBrowser +from .web_search import WebSearch +from .wordart_tool import WordArtTexture + +TOOL_INFO_LIST = { + 'modelscope_text-translation-zh2en': 'TranslationZh2EnTool', + 'modelscope_text-translation-en2zh': 'TranslationEn2ZhTool', + 'modelscope_text-ie': 'TextInfoExtractTool', + 'modelscope_text-ner': 'TextNerTool', + 'modelscope_text-address': 'TextAddressTool', + 'image_gen': 'TextToImageTool', + 'modelscope_video-generation': 'TextToVideoTool', + 'modelscope_image-chat': 'ImageChatTool', + 'modelscope_speech-generation': 'TexttoSpeechTool', + 'amap_weather': 'AMAPWeather', + 'code_interpreter': 'CodeInterpreterJupyter', + 'wordart_texture_generation': 'WordArtTexture', + 'web_search': 'WebSearch', + 'web_browser': 'WebBrowser', +} diff --git a/agentfabric/modelscope_agent/tools/amap_weather.py b/agentfabric/modelscope_agent/tools/amap_weather.py new file mode 100644 index 0000000000000000000000000000000000000000..a31db45540697485de8a98c7847742a5a3e89bf6 --- /dev/null +++ b/agentfabric/modelscope_agent/tools/amap_weather.py @@ -0,0 +1,64 @@ +import os + +import pandas as pd +import requests +from modelscope_agent.tools.tool import Tool, ToolSchema +from pydantic import ValidationError + + +class AMAPWeather(Tool): + description = '获取对应城市的天气数据' + name = 'amap_weather' + parameters: list = [{ + 'name': 'location', + 'description': 'get temperature for a specific location', + 'required': True + }] + + def __init__(self, cfg={}): + self.cfg = cfg.get(self.name, {}) + + # remote call + self.url = 'https://restapi.amap.com/v3/weather/weatherInfo?city={city}&key={key}' + self.token = self.cfg.get('token', os.environ.get('AMAP_TOKEN', '')) + self.city_df = pd.read_excel( + 'https://modelscope.oss-cn-beijing.aliyuncs.com/resource/agent/AMap_adcode_citycode.xlsx' + ) + assert self.token != '', 'weather api token must be acquired through ' \ + 'https://lbs.amap.com/api/webservice/guide/create-project/get-key and set by AMAP_TOKEN' + + try: + all_param = { + 'name': self.name, + 'description': self.description, + 'parameters': self.parameters + } + self.tool_schema = ToolSchema(**all_param) + except ValidationError: + raise ValueError(f'Error when parsing parameters of {self.name}') + + self._str = self.tool_schema.model_dump_json() + self._function = self.parse_pydantic_model_to_openai_function( + all_param) + + def get_city_adcode(self, city_name): + filtered_df = self.city_df[self.city_df['中文名'] == city_name] + if len(filtered_df['adcode'].values) == 0: + raise ValueError( + f'location {city_name} not found, availables are {self.city_df["中文名"]}' + ) + else: + return filtered_df['adcode'].values[0] + + def __call__(self, *args, **kwargs): + location = kwargs['location'] + response = requests.get( + self.url.format( + city=self.get_city_adcode(location), key=self.token)) + data = response.json() + if data['status'] == '0': + raise RuntimeError(data) + else: + weather = data['lives'][0]['weather'] + temperature = data['lives'][0]['temperature'] + return {'result': f'{location}的天气是{weather}温度是{temperature}度。'} diff --git a/agentfabric/modelscope_agent/tools/code_interperter.py b/agentfabric/modelscope_agent/tools/code_interperter.py new file mode 100644 index 0000000000000000000000000000000000000000..45100b901922345aa416c433949ca9da55334fc2 --- /dev/null +++ b/agentfabric/modelscope_agent/tools/code_interperter.py @@ -0,0 +1,125 @@ +import os +import re +import traceback + +import appdirs +import json + +from .code_interpreter_utils.create_code_interpreter import \ + create_code_interpreter +from .code_interpreter_utils.language_map import language_map +from .code_interpreter_utils.truncate_output import truncate_output +from .tool import Tool + + +class CodeInterpreter(Tool): + """ + using open interpreter to interpret code + by https://github.com/KillianLucas/open-interpreter + """ + description = 'Executes code on the user\'s machine, **in the users local environment**, and returns the output' + name = 'code_interpreter' + parameters: list = [{ + 'name': 'language', + 'description': + 'The programming language (required parameter to the `execute` function)', + 'required': True + }, { + 'name': 'code', + 'description': 'The code to execute (required)', + 'required': True + }] + + def __init__(self, cfg={}): + super().__init__(cfg) + self.create_code_interpreter = create_code_interpreter + self.language_map = language_map + self.truncate_output = truncate_output + + self._code_interpreters = {} + self.max_output = self.cfg.get('max_output', 2000) + + def _local_call(self, *args, **kwargs): + + language, code = self._handle_input_fallback(**kwargs) + + try: + # Fix a common error where the LLM thinks it's in a Jupyter notebook + if language == 'python' and code.startswith('!'): + code = code[1:] + language = 'shell' + + if language in self.language_map: + if language not in self._code_interpreters: + self._code_interpreters[ + language] = self.create_code_interpreter(language) + code_interpreter = self._code_interpreters[language] + else: + # This still prints code but don't allow code to run. Let Open-Interpreter know through output message + error_output = f'Error: Open Interpreter does not currently support {language}.' + print(error_output) + output = '\n' + error_output + return {'result': output.strip()} + + output = '' + for line in code_interpreter.run(code): + if 'output' in line: + output += '\n' + line['output'] + + # Truncate output + output = self.truncate_output(output, self.max_output) + except Exception as e: + error = traceback.format_exc() + output = ' '.join(f'{key}:{value}' + for key, value in kwargs.items()) + output += f'\nDetail error is {e}.\n{error}' + + return {'result': output.strip()} + + def _handle_input_fallback(self, **kwargs): + """ + an alternative method is to parse code in content not from function call + such as: + text = response['content'] + code_block = re.search(r'```([\s\S]+)```', text) # noqa W^05 + if code_block: + result = code_block.group(1) + language = result.split('\n')[0] + code = '\n'.join(result.split('\n')[1:]) + + :param fallback_text: + :return: language, cocde + """ + + language = kwargs.get('language', None) + code = kwargs.get('code', None) + fallback = kwargs.get('fallback', None) + + if language and code: + return language, code + elif fallback: + try: + text = fallback + code_block = re.search(r'```([\s\S]+)```', text) # noqa W^05 + if code_block: + result = code_block.group(1) + # for multi code_block + result = result.split('```')[0] + language = result.split('\n')[0] + if language == 'py' or language == 'python': + # handle py case + # ```py code ``` + language = 'python' + code = '\n'.join(result.split('\n')[1:]) + return language, code + + if language == 'json': + # handle json case + # ```json {language,code}``` + parameters = json.loads('\n'.join( + result.split('\n')[1:]).replace('\n', '')) + return parameters['language'], parameters['code'] + except ValueError: + return language, code + else: + return language, code diff --git a/agentfabric/modelscope_agent/tools/code_interpreter_jupyter.py b/agentfabric/modelscope_agent/tools/code_interpreter_jupyter.py new file mode 100644 index 0000000000000000000000000000000000000000..dca78093c3245e91a6079b0079a03f87a3ccdbb5 --- /dev/null +++ b/agentfabric/modelscope_agent/tools/code_interpreter_jupyter.py @@ -0,0 +1,319 @@ +import asyncio +import atexit +import base64 +import glob +import io +import os +import queue +import re +import shutil +import signal +import subprocess +import sys +import time +import traceback +import uuid +from pathlib import Path +from typing import Dict, Optional + +import json +import matplotlib +import PIL.Image +from jupyter_client import BlockingKernelClient + +from .tool import Tool + +WORK_DIR = os.getenv('CODE_INTERPRETER_WORK_DIR', '/tmp/ci_workspace') + +STATIC_URL = os.getenv('CODE_INTERPRETER_STATIC_URL', + 'http://127.0.0.1:7866/static') + +LAUNCH_KERNEL_PY = """ +from ipykernel import kernelapp as app +app.launch_new_instance() +""" + +INIT_CODE_FILE = str( + Path(__file__).absolute().parent / 'code_interpreter_utils' + / 'code_interpreter_init_kernel.py') + +ALIB_FONT_FILE = str( + Path(__file__).absolute().parent / 'code_interpreter_utils' + / 'AlibabaPuHuiTi-3-45-Light.ttf') + +_KERNEL_CLIENTS: Dict[int, BlockingKernelClient] = {} + + +class CodeInterpreterJupyter(Tool): + """ + using jupyter kernel client to interpret python code, + should not be used the other code interpreter tool at the same time + """ + description = '代码解释器，可用于执行Python代码。' + name = 'code_interpreter' + parameters: list = [{ + 'name': 'code', + 'description': '待执行的代码', + 'required': True + }] + + def __init__(self, cfg={}): + super().__init__(cfg) + self.timeout = self.cfg.get('timeout', 30) + self.image_server = self.cfg.get('image_server', False) + self.kernel_clients: Dict[int, BlockingKernelClient] = {} + atexit.register(self._kill_kernels) + + pid: int = os.getpid() + if pid in self.kernel_clients: + kc = self.kernel_clients[pid] + else: + self._fix_matplotlib_cjk_font_issue() + kc = self._start_kernel(pid) + with open(INIT_CODE_FILE) as fin: + start_code = fin.read() + start_code = start_code.replace('{{M6_FONT_PATH}}', + repr(ALIB_FONT_FILE)[1:-1]) + print(self._execute_code(kc, start_code)) + self.kernel_clients[pid] = kc + + self.kc = kc + + def __del__(self): + # make sure all the kernels are killed during __del__ + signal.signal(signal.SIGTERM, self._kill_kernels) + signal.signal(signal.SIGINT, self._kill_kernels) + + def _start_kernel(self, pid) -> BlockingKernelClient: + connection_file = os.path.join(WORK_DIR, + f'kernel_connection_file_{pid}.json') + launch_kernel_script = os.path.join(WORK_DIR, + f'launch_kernel_{pid}.py') + for f in [connection_file, launch_kernel_script]: + if os.path.exists(f): + print(f'WARNING: {f} already exists') + os.remove(f) + + os.makedirs(WORK_DIR, exist_ok=True) + + with open(launch_kernel_script, 'w') as fout: + fout.write(LAUNCH_KERNEL_PY) + + available_envs = ['PATH', 'PYTHONPATH', 'LD_LIBRARY_PATH'] + envs = {} + for k in available_envs: + if os.getenv(k) is not None: + envs[k] = os.getenv(k) + + args = ( + sys.executable, + launch_kernel_script, + '--IPKernelApp.connection_file', + connection_file, + '--matplotlib=inline', + '--quiet', + ) + kernel_process = subprocess.Popen([*args], env=envs, + cwd=WORK_DIR) # noqa E126 + print(f"INFO: kernel process's PID = {kernel_process.pid}") + + # Wait for kernel connection file to be written + while True: + if not os.path.isfile(connection_file): + time.sleep(0.1) + else: + # Keep looping if JSON parsing fails, file may be partially written + try: + with open(connection_file, 'r') as fp: + json.load(fp) + break + except json.JSONDecodeError: + pass + + # Client + kc = BlockingKernelClient(connection_file=connection_file) + asyncio.set_event_loop_policy(asyncio.DefaultEventLoopPolicy()) + kc.load_connection_file() + kc.start_channels() + kc.wait_for_ready() + return kc + + def _kill_kernels(self): + for v in self.kernel_clients.values(): + v.shutdown() + for k in list(self.kernel_clients.keys()): + del self.kernel_clients[k] + + def _serve_image(self, image_base64: str, image_type: str) -> str: + image_file = f'{uuid.uuid4()}.{image_type}' + local_image_file = os.path.join(WORK_DIR, image_file) + + png_bytes = base64.b64decode(image_base64) + assert isinstance(png_bytes, bytes) + + if image_type == 'gif': + with open(local_image_file, 'wb') as file: + file.write(png_bytes) + else: + bytes_io = io.BytesIO(png_bytes) + PIL.Image.open(bytes_io).save(local_image_file, image_type) + + if self.image_server: + image_url = f'{STATIC_URL}/{image_file}' + return image_url + else: + return local_image_file + + def _escape_ansi(self, line: str) -> str: + ansi_escape = re.compile(r'(?:\x1B[@-_]|[\x80-\x9F])[0-?]*[ -/]*[@-~]') + return ansi_escape.sub('', line) + + def _fix_matplotlib_cjk_font_issue(self): + ttf_name = os.path.basename(ALIB_FONT_FILE) + local_ttf = os.path.join( + os.path.abspath( + os.path.join(matplotlib.matplotlib_fname(), os.path.pardir)), + 'fonts', 'ttf', ttf_name) + if not os.path.exists(local_ttf): + try: + shutil.copy(ALIB_FONT_FILE, local_ttf) + font_list_cache = os.path.join(matplotlib.get_cachedir(), + 'fontlist-*.json') + for cache_file in glob.glob(font_list_cache): + with open(cache_file) as fin: + cache_content = fin.read() + if ttf_name not in cache_content: + os.remove(cache_file) + except Exception: + traceback.format_exc() + + def _execute_code(self, kc: BlockingKernelClient, code: str) -> str: + kc.wait_for_ready() + kc.execute(code) + result = '' + image_idx = 0 + while True: + text = '' + image = '' + finished = False + msg_type = 'error' + try: + msg = kc.get_iopub_msg() + msg_type = msg['msg_type'] + if msg_type == 'status': + if msg['content'].get('execution_state') == 'idle': + finished = True + elif msg_type == 'execute_result': + text = msg['content']['data'].get('text/plain', '') + if 'image/png' in msg['content']['data']: + image_b64 = msg['content']['data']['image/png'] + image_url = self._serve_image(image_b64, 'png') + image_idx += 1 + image = '![IMAGEGEN](%s)' % (image_url) + elif 'text/html' in msg['content']['data']: + text += '\n' + msg['content']['data']['text/html'] + elif 'image/gif' in msg['content']['data']: + image_b64 = msg['content']['data']['image/gif'] + image_url = self._serve_image(image_b64, 'gif') + image_idx += 1 + image = '![IMAGEGEN](%s)' % (image_url) + elif msg_type == 'display_data': + if 'image/png' in msg['content']['data']: + image_b64 = msg['content']['data']['image/png'] + image_url = self._serve_image(image_b64, 'png') + image_idx += 1 + image = '![IMAGEGEN](%s)' % (image_url) + else: + text = msg['content']['data'].get('text/plain', '') + elif msg_type == 'stream': + msg_type = msg['content']['name'] # stdout, stderr + text = msg['content']['text'] + elif msg_type == 'error': + text = self._escape_ansi('\n'.join( + msg['content']['traceback'])) + if 'M6_CODE_INTERPRETER_TIMEOUT' in text: + text = 'Timeout: Code execution exceeded the time limit.' + except queue.Empty: + text = 'Timeout: Code execution exceeded the time limit.' + finished = True + except Exception: + text = 'The code interpreter encountered an unexpected error.' + traceback.format_exc() + finished = True + if text: + result += f'\n{text}' + if image: + result += f'\n\n{image}' + if finished: + break + result = result.lstrip('\n') + if not result: + result += 'The code executed successfully.' + return result + + def _local_call(self, *args, **kwargs): + code = self._handle_input_fallback(**kwargs) + if not code.strip(): + return '' + + if self.timeout: + code = f'_M6CountdownTimer.start({self.timeout})\n{code}' + + fixed_code = [] + for line in code.split('\n'): + fixed_code.append(line) + if line.startswith('sns.set_theme('): + fixed_code.append( + 'plt.rcParams["font.family"] = _m6_font_prop.get_name()') + fixed_code = '\n'.join(fixed_code) + result = self._execute_code(self.kc, fixed_code) + + if self.timeout: + self._execute_code(self.kc, '_M6CountdownTimer.cancel()') + + return {'result': result} + + def _handle_input_fallback(self, **kwargs): + """ + an alternative method is to parse code in content not from function call + such as: + text = response['content'] + code_block = re.search(r'```([\s\S]+)```', text) # noqa W^05 + if code_block: + result = code_block.group(1) + language = result.split('\n')[0] + code = '\n'.join(result.split('\n')[1:]) + + :param fallback_text: + :return: language, cocde + """ + + code = kwargs.get('code', None) + fallback = kwargs.get('fallback', None) + + if code: + return code + elif fallback: + try: + text = fallback + code_block = re.search(r'```([\s\S]+)```', text) # noqa W^05 + if code_block: + result = code_block.group(1) + language = result.split('\n')[0] + if language == 'py' or language == 'python': + # handle py case + # ```py code ``` + language = 'python' + code = '\n'.join(result.split('\n')[1:]) + return code + + if language == 'json': + # handle json case + # ```json {language,code}``` + parameters = json.loads('\n'.join( + result.split('\n')[1:]).replace('\n', '')) + return parameters['code'] + except ValueError: + return code + else: + return code diff --git a/agentfabric/modelscope_agent/tools/code_interpreter_utils/AlibabaPuHuiTi-3-45-Light.ttf b/agentfabric/modelscope_agent/tools/code_interpreter_utils/AlibabaPuHuiTi-3-45-Light.ttf new file mode 100644 index 0000000000000000000000000000000000000000..f8dd22dd85a99e75d31aa3b94a43dad71dddb255 --- /dev/null +++ b/agentfabric/modelscope_agent/tools/code_interpreter_utils/AlibabaPuHuiTi-3-45-Light.ttf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f92bac7082def2703a72c4ff2c87cb2e635ecdaad0bbeb3b9aa0ce5b181de6b +size 8559848 diff --git a/agentfabric/modelscope_agent/tools/code_interpreter_utils/__init__.py b/agentfabric/modelscope_agent/tools/code_interpreter_utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..274f24c3713992802271d3777697b160e150a3cc --- /dev/null +++ b/agentfabric/modelscope_agent/tools/code_interpreter_utils/__init__.py @@ -0,0 +1,5 @@ +# all the utility functions under code_interpreter_utils are borrowed from project +# in order to use python lower than 3.10 +# https://github.com/KillianLucas/open-interpreter + +from .base_code_interpreter import BaseCodeInterpreter diff --git a/agentfabric/modelscope_agent/tools/code_interpreter_utils/base_code_interpreter.py b/agentfabric/modelscope_agent/tools/code_interpreter_utils/base_code_interpreter.py new file mode 100644 index 0000000000000000000000000000000000000000..23796e424034ed98f5bc9ad37db6acd2e742d8b9 --- /dev/null +++ b/agentfabric/modelscope_agent/tools/code_interpreter_utils/base_code_interpreter.py @@ -0,0 +1,13 @@ +class BaseCodeInterpreter: + """ + .run is a generator that yields a dict with attributes: active_line, output + """ + + def __init__(self): + pass + + def run(self, code): + pass + + def terminate(self): + pass diff --git a/agentfabric/modelscope_agent/tools/code_interpreter_utils/code_interpreter_init_kernel.py b/agentfabric/modelscope_agent/tools/code_interpreter_utils/code_interpreter_init_kernel.py new file mode 100644 index 0000000000000000000000000000000000000000..62511247f09cd9d15d9f6ea7491f29ae1bbdaf3c --- /dev/null +++ b/agentfabric/modelscope_agent/tools/code_interpreter_utils/code_interpreter_init_kernel.py @@ -0,0 +1,50 @@ +import math # noqa +import os # noqa +import re # noqa +import signal + +import json # noqa +import matplotlib # noqa +import matplotlib.pyplot as plt +import numpy as np # noqa +import pandas as pd # noqa +import seaborn as sns +from matplotlib.font_manager import FontProperties +from sympy import Eq, solve, symbols # noqa + + +def input(*args, **kwargs): # noqa + raise NotImplementedError('Python input() function is disabled.') + + +def _m6_timout_handler(_signum=None, _frame=None): + raise TimeoutError('M6_CODE_INTERPRETER_TIMEOUT') + + +try: + signal.signal(signal.SIGALRM, _m6_timout_handler) +except AttributeError: # windows + pass + + +class _M6CountdownTimer: + + @classmethod + def start(cls, timeout: int): + try: + signal.alarm(timeout) + except AttributeError: # windows + pass # TODO: I haven't found a solution that works with jupyter yet. + + @classmethod + def cancel(cls): + try: + signal.alarm(0) + except AttributeError: # windows + pass # TODO + + +sns.set_theme() + +_m6_font_prop = FontProperties(fname='{{M6_FONT_PATH}}') +plt.rcParams['font.family'] = _m6_font_prop.get_name() diff --git a/agentfabric/modelscope_agent/tools/code_interpreter_utils/create_code_interpreter.py b/agentfabric/modelscope_agent/tools/code_interpreter_utils/create_code_interpreter.py new file mode 100644 index 0000000000000000000000000000000000000000..e185b2fe6bdc676d7d89648f0e28c01b4c7915eb --- /dev/null +++ b/agentfabric/modelscope_agent/tools/code_interpreter_utils/create_code_interpreter.py @@ -0,0 +1,12 @@ +from .language_map import language_map + + +def create_code_interpreter(language): + # Case in-sensitive + language = language.lower() + + try: + CodeInterpreter = language_map[language] + return CodeInterpreter() + except KeyError: + raise ValueError(f'Unknown or unsupported language: {language}') diff --git a/agentfabric/modelscope_agent/tools/code_interpreter_utils/language_map.py b/agentfabric/modelscope_agent/tools/code_interpreter_utils/language_map.py new file mode 100644 index 0000000000000000000000000000000000000000..e28ad50b3a01fbd94eb615ea612dc01de299dd7a --- /dev/null +++ b/agentfabric/modelscope_agent/tools/code_interpreter_utils/language_map.py @@ -0,0 +1,19 @@ +from .languages.applescript import AppleScript +from .languages.html import HTML +from .languages.javascript import JavaScript +from .languages.powershell import PowerShell +from .languages.python import Python +from .languages.r import R +from .languages.shell import Shell + +language_map = { + 'python': Python, + 'bash': Shell, + 'shell': Shell, + 'zsh': Shell, + 'javascript': JavaScript, + 'html': HTML, + 'applescript': AppleScript, + 'r': R, + 'powershell': PowerShell, +} diff --git a/agentfabric/modelscope_agent/tools/code_interpreter_utils/languages/__init__.py b/agentfabric/modelscope_agent/tools/code_interpreter_utils/languages/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/agentfabric/modelscope_agent/tools/code_interpreter_utils/languages/applescript.py b/agentfabric/modelscope_agent/tools/code_interpreter_utils/languages/applescript.py new file mode 100644 index 0000000000000000000000000000000000000000..4100ce3cef60d782871e2a60a3c72d73caef784b --- /dev/null +++ b/agentfabric/modelscope_agent/tools/code_interpreter_utils/languages/applescript.py @@ -0,0 +1,67 @@ +import os + +from ..subprocess_code_interpreter import SubprocessCodeInterpreter + + +class AppleScript(SubprocessCodeInterpreter): + file_extension = 'applescript' + proper_name = 'AppleScript' + + def __init__(self): + super().__init__() + self.start_cmd = os.environ.get('SHELL', '/bin/zsh') + + def preprocess_code(self, code): + """ + Inserts an end_of_execution marker and adds active line indicators. + """ + # Add active line indicators to the code + code = self.add_active_line_indicators(code) + + # Escape double quotes + code = code.replace('"', r"\"") + + # Wrap in double quotes + code = '"' + code + '"' + + # Prepend start command for AppleScript + code = 'osascript -e ' + code + + # Append end of execution indicator + code += '; echo "##end_of_execution##"' + + return code + + def add_active_line_indicators(self, code): + """ + Adds log commands to indicate the active line of execution in the AppleScript. + """ + modified_lines = [] + lines = code.split('\n') + + for idx, line in enumerate(lines): + # Add log command to indicate the line number + if line.strip(): # Only add if line is not empty + modified_lines.append(f'log "##active_line{idx + 1}##"') + modified_lines.append(line) + + return '\n'.join(modified_lines) + + def detect_active_line(self, line): + """ + Detects active line indicator in the output. + """ + prefix = '##active_line' + if prefix in line: + try: + return int(line.split(prefix)[1].split()[0]) + except Exception as e: + print(e) + pass + return None + + def detect_end_of_execution(self, line): + """ + Detects end of execution marker in the output. + """ + return '##end_of_execution##' in line diff --git a/agentfabric/modelscope_agent/tools/code_interpreter_utils/languages/html.py b/agentfabric/modelscope_agent/tools/code_interpreter_utils/languages/html.py new file mode 100644 index 0000000000000000000000000000000000000000..f1745944e328420e72c457d16e568579ada3f8a6 --- /dev/null +++ b/agentfabric/modelscope_agent/tools/code_interpreter_utils/languages/html.py @@ -0,0 +1,26 @@ +import os +import tempfile +import webbrowser + +from ..base_code_interpreter import BaseCodeInterpreter + + +class HTML(BaseCodeInterpreter): + file_extension = 'html' + proper_name = 'HTML' + + def __init__(self): + super().__init__() + + def run(self, code): + # Create a temporary HTML file with the content + with tempfile.NamedTemporaryFile(delete=False, suffix='.html') as f: + f.write(code.encode()) + + # Open the HTML file with the default web browser + webbrowser.open('file://' + os.path.realpath(f.name)) + + yield { + 'output': + f"Saved to {os.path.realpath(f.name)} and opened with the user's default web browser." + } diff --git a/agentfabric/modelscope_agent/tools/code_interpreter_utils/languages/javascript.py b/agentfabric/modelscope_agent/tools/code_interpreter_utils/languages/javascript.py new file mode 100644 index 0000000000000000000000000000000000000000..cb35f4f8488e8fcf8d511ab4cd5277e9bb9fa38d --- /dev/null +++ b/agentfabric/modelscope_agent/tools/code_interpreter_utils/languages/javascript.py @@ -0,0 +1,66 @@ +import re + +from ..subprocess_code_interpreter import SubprocessCodeInterpreter + + +class JavaScript(SubprocessCodeInterpreter): + file_extension = 'js' + proper_name = 'JavaScript' + + def __init__(self): + super().__init__() + self.start_cmd = 'node -i' + + def preprocess_code(self, code): + return preprocess_javascript(code) + + def line_postprocessor(self, line): + # Node's interactive REPL outputs a billion things + # So we clean it up: + if 'Welcome to Node.js' in line: + return None + if line.strip() in ['undefined', 'Type ".help" for more information.']: + return None + # Remove trailing ">"s + line = re.sub(r'^\s*(>\s*)+', '', line) + return line + + def detect_active_line(self, line): + if '##active_line' in line: + return int(line.split('##active_line')[1].split('##')[0]) + return None + + def detect_end_of_execution(self, line): + return '##end_of_execution##' in line + + +def preprocess_javascript(code): + """ + Add active line markers + Wrap in a try catch + Add end of execution marker + """ + + # Split code into lines + lines = code.split('\n') + processed_lines = [] + + for i, line in enumerate(lines, 1): + # Add active line print + processed_lines.append(f'console.log("##active_line{i}##");') + processed_lines.append(line) + + # Join lines to form the processed code + processed_code = '\n'.join(processed_lines) + + # Wrap in a try-catch and add end of execution marker + processed_code = f""" +try {{ +{processed_code} +}} catch (e) {{ + console.log(e); +}} +console.log("##end_of_execution##"); +""" + + return processed_code diff --git a/agentfabric/modelscope_agent/tools/code_interpreter_utils/languages/powershell.py b/agentfabric/modelscope_agent/tools/code_interpreter_utils/languages/powershell.py new file mode 100644 index 0000000000000000000000000000000000000000..467aa1105252a3f22374f9e8e060c1feeb5b17ee --- /dev/null +++ b/agentfabric/modelscope_agent/tools/code_interpreter_utils/languages/powershell.py @@ -0,0 +1,75 @@ +import os +import platform +import shutil + +from ..subprocess_code_interpreter import SubprocessCodeInterpreter + + +class PowerShell(SubprocessCodeInterpreter): + file_extension = 'ps1' + proper_name = 'PowerShell' + + def __init__(self): + super().__init__() + + # Determine the start command based on the platform (use "powershell" for Windows) + if platform.system() == 'Windows': + self.start_cmd = 'powershell.exe' + # self.start_cmd = os.environ.get('SHELL', 'powershell.exe') + else: + # On non-Windows platforms, prefer pwsh (PowerShell Core) if available, or fall back to bash + self.start_cmd = 'pwsh' if shutil.which('pwsh') else 'bash' + + def preprocess_code(self, code): + return preprocess_powershell(code) + + def line_postprocessor(self, line): + return line + + def detect_active_line(self, line): + if '##active_line' in line: + return int(line.split('##active_line')[1].split('##')[0]) + return None + + def detect_end_of_execution(self, line): + return '##end_of_execution##' in line + + +def preprocess_powershell(code): + """ + Add active line markers + Wrap in try-catch block + Add end of execution marker + """ + # Add commands that tell us what the active line is + code = add_active_line_prints(code) + + # Wrap in try-catch block for error handling + code = wrap_in_try_catch(code) + + # Add end marker (we'll be listening for this to know when it ends) + code += '\nWrite-Output "##end_of_execution##"' + + return code + + +def add_active_line_prints(code): + """ + Add Write-Output statements indicating line numbers to a PowerShell script. + """ + lines = code.split('\n') + for index, line in enumerate(lines): + # Insert the Write-Output command before the actual line + lines[index] = f'Write-Output "##active_line{index + 1}##"\n{line}' + return '\n'.join(lines) + + +def wrap_in_try_catch(code): + """ + Wrap PowerShell code in a try-catch block to catch errors and display them. + """ + try_catch_code = """ +try { + $ErrorActionPreference = "Stop" +""" + return try_catch_code + code + '\n} catch {\n Write-Error $_\n}\n' diff --git a/agentfabric/modelscope_agent/tools/code_interpreter_utils/languages/python.py b/agentfabric/modelscope_agent/tools/code_interpreter_utils/languages/python.py new file mode 100644 index 0000000000000000000000000000000000000000..107cc2009a3c445a10ee7115bcf624c10b20e7e2 --- /dev/null +++ b/agentfabric/modelscope_agent/tools/code_interpreter_utils/languages/python.py @@ -0,0 +1,161 @@ +import ast +import os +import re +import shlex +import sys + +from ..subprocess_code_interpreter import SubprocessCodeInterpreter + + +class Python(SubprocessCodeInterpreter): + file_extension = 'py' + proper_name = 'Python' + + def __init__(self): + super().__init__() + executable = sys.executable + if os.name != 'nt': # not Windows + executable = shlex.quote(executable) + self.start_cmd = executable + ' -i -q -u' + + def preprocess_code(self, code): + return preprocess_python(code) + + def line_postprocessor(self, line): + if re.match(r'^(\s*>>>\s*|\s*\.\.\.\s*)', line): + return None + return line + + def detect_active_line(self, line): + if '##active_line' in line: + return int(line.split('##active_line')[1].split('##')[0]) + return None + + def detect_end_of_execution(self, line): + return '##end_of_execution##' in line + + +def preprocess_python(code): + """ + Add active line markers + Wrap in a try except + Add end of execution marker + """ + + # Add print commands that tell us what the active line is + code = add_active_line_prints(code) + + # Wrap in a try except + code = wrap_in_try_except(code) + + # Remove any whitespace lines, as this will break indented blocks + # (are we sure about this? test this) + code_lines = code.split('\n') + code_lines = [c for c in code_lines if c.strip() != ''] + code = '\n'.join(code_lines) + + # Add end command (we'll be listening for this so we know when it ends) + code += '\n\nprint("##end_of_execution##")' + + return code + + +def add_active_line_prints(code): + """ + Add print statements indicating line numbers to a python string. + """ + tree = ast.parse(code) + transformer = AddLinePrints() + new_tree = transformer.visit(tree) + return ast.unparse(new_tree) + + +class AddLinePrints(ast.NodeTransformer): + """ + Transformer to insert print statements indicating the line number + before every executable line in the AST. + """ + + def insert_print_statement(self, line_number): + """Inserts a print statement for a given line number.""" + return ast.Expr( + value=ast.Call( + func=ast.Name(id='print', ctx=ast.Load()), + args=[ast.Constant(value=f'##active_line{line_number}##')], + keywords=[], + )) + + def process_body(self, body): + """Processes a block of statements, adding print calls.""" + new_body = [] + + # In case it's not iterable: + if not isinstance(body, list): + body = [body] + + for sub_node in body: + if hasattr(sub_node, 'lineno'): + new_body.append(self.insert_print_statement(sub_node.lineno)) + new_body.append(sub_node) + + return new_body + + def visit(self, node): + """Overridden visit to transform nodes.""" + new_node = super().visit(node) + + # If node has a body, process it + if hasattr(new_node, 'body'): + new_node.body = self.process_body(new_node.body) + + # If node has an orelse block (like in for, while, if), process it + if hasattr(new_node, 'orelse') and new_node.orelse: + new_node.orelse = self.process_body(new_node.orelse) + + # Special case for Try nodes as they have multiple blocks + if isinstance(new_node, ast.Try): + for handler in new_node.handlers: + handler.body = self.process_body(handler.body) + if new_node.finalbody: + new_node.finalbody = self.process_body(new_node.finalbody) + + return new_node + + +def wrap_in_try_except(code): + # Add import traceback + code = 'import traceback\n' + code + + # Parse the input code into an AST + parsed_code = ast.parse(code) + + # Wrap the entire code's AST in a single try-except block + try_except = ast.Try( + body=parsed_code.body, + handlers=[ + ast.ExceptHandler( + type=ast.Name(id='Exception', ctx=ast.Load()), + name=None, + body=[ + ast.Expr( + value=ast.Call( + func=ast.Attribute( + value=ast.Name(id='traceback', ctx=ast.Load()), + attr='print_exc', + ctx=ast.Load(), + ), + args=[], + keywords=[], + )), + ], + ) + ], + orelse=[], + finalbody=[], + ) + + # Assign the try-except block as the new body + parsed_code.body = [try_except] + + # Convert the modified AST back to source code + return ast.unparse(parsed_code) diff --git a/agentfabric/modelscope_agent/tools/code_interpreter_utils/languages/r.py b/agentfabric/modelscope_agent/tools/code_interpreter_utils/languages/r.py new file mode 100644 index 0000000000000000000000000000000000000000..28936608c7fce3b1ff7952bd09abd0b62f95539e --- /dev/null +++ b/agentfabric/modelscope_agent/tools/code_interpreter_utils/languages/r.py @@ -0,0 +1,71 @@ +import re + +from ..subprocess_code_interpreter import SubprocessCodeInterpreter + + +class R(SubprocessCodeInterpreter): + file_extension = 'r' + proper_name = 'R' + + def __init__(self): + super().__init__() + self.start_cmd = 'R -q --vanilla' # Start R in quiet and vanilla mode + + def preprocess_code(self, code): + """ + Add active line markers + Wrap in a tryCatch for better error handling in R + Add end of execution marker + """ + + lines = code.split('\n') + processed_lines = [] + + for i, line in enumerate(lines, 1): + # Add active line print + processed_lines.append(f'cat("##active_line{i}##\\n");{line}') + + # Join lines to form the processed code + processed_code = '\n'.join(processed_lines) + + # Wrap in a tryCatch for error handling and add end of execution marker + processed_code = f""" +tryCatch({{ +{processed_code} +}}, error=function(e){{ + cat("## execution_error ##\\n", conditionMessage(e), "\\n"); +}}) +cat("## end_of_execution ##\\n"); +""" + # Count the number of lines of processed_code + # (R echoes all code back for some reason, but we can skip it if we track this!) + self.code_line_count = len(processed_code.split('\n')) - 1 + + return processed_code + + def line_postprocessor(self, line): + # If the line count attribute is set and non-zero, decrement and skip the line + if hasattr(self, 'code_line_count') and self.code_line_count > 0: + self.code_line_count -= 1 + return None + + if re.match(r'^(\s*>>>\s*|\s*\.\.\.\s*|\s*>\s*|\s*\+\s*|\s*)$', line): + return None + if 'R version' in line: # Startup message + return None + if line.strip().startswith('[1] "') and line.endswith( + '"'): # For strings, trim quotation marks + return line[5:-1].strip() + if line.strip().startswith( + '[1]'): # Normal R output prefix for non-string outputs + return line[4:].strip() + + return line + + def detect_active_line(self, line): + if '##active_line' in line: + return int(line.split('##active_line')[1].split('##')[0]) + return None + + def detect_end_of_execution(self, line): + return '##end_of_execution##' in line or '## execution_error ##' in line diff --git a/agentfabric/modelscope_agent/tools/code_interpreter_utils/languages/shell.py b/agentfabric/modelscope_agent/tools/code_interpreter_utils/languages/shell.py new file mode 100644 index 0000000000000000000000000000000000000000..bbc067071f317c4a22176f140c910f139f09e4dc --- /dev/null +++ b/agentfabric/modelscope_agent/tools/code_interpreter_utils/languages/shell.py @@ -0,0 +1,89 @@ +import os +import platform +import re + +from ..subprocess_code_interpreter import SubprocessCodeInterpreter + + +class Shell(SubprocessCodeInterpreter): + file_extension = 'sh' + proper_name = 'Shell' + + def __init__(self): + super().__init__() + + # Determine the start command based on the platform + if platform.system() == 'Windows': + self.start_cmd = 'cmd.exe' + else: + self.start_cmd = os.environ.get('SHELL', 'bash') + + def preprocess_code(self, code): + return preprocess_shell(code) + + def line_postprocessor(self, line): + return line + + def detect_active_line(self, line): + if '##active_line' in line: + return int(line.split('##active_line')[1].split('##')[0]) + return None + + def detect_end_of_execution(self, line): + return '##end_of_execution##' in line + + +def preprocess_shell(code): + """ + Add active line markers + Wrap in a try except (trap in shell) + Add end of execution marker + """ + + # Add commands that tell us what the active line is + # if it's multiline, just skip this. soon we should make it work with multiline + if not has_multiline_commands(code): + code = add_active_line_prints(code) + + # Add end command (we'll be listening for this so we know when it ends) + code += '\necho "##end_of_execution##"' + + return code + + +def add_active_line_prints(code): + """ + Add echo statements indicating line numbers to a shell string. + """ + lines = code.split('\n') + for index, line in enumerate(lines): + # Insert the echo command before the actual line + lines[index] = f'echo "##active_line{index + 1}##"\n{line}' + return '\n'.join(lines) + + +def has_multiline_commands(script_text): + # Patterns that indicate a line continues + continuation_patterns = [ + r'\\$', # Line continuation character at the end of the line + r'\|$', # Pipe character at the end of the line indicating a pipeline continuation + r'&&\s*$', # Logical AND at the end of the line + r'\|\|\s*$', # Logical OR at the end of the line + r'<\($', # Start of process substitution + r'\($', # Start of subshell + r'{\s*$', # Start of a block + r'\bif\b', # Start of an if statement + r'\bwhile\b', # Start of a while loop + r'\bfor\b', # Start of a for loop + r'do\s*$', # 'do' keyword for loops + r'then\s*$', # 'then' keyword for if statements + ] + + # Check each line for multiline patterns + for line in script_text.splitlines(): + if any( + re.search(pattern, line.rstrip()) + for pattern in continuation_patterns): + return True + + return False diff --git a/agentfabric/modelscope_agent/tools/code_interpreter_utils/subprocess_code_interpreter.py b/agentfabric/modelscope_agent/tools/code_interpreter_utils/subprocess_code_interpreter.py new file mode 100644 index 0000000000000000000000000000000000000000..01e6a7e0dddf8cce88e4aa15dd74a29849940e08 --- /dev/null +++ b/agentfabric/modelscope_agent/tools/code_interpreter_utils/subprocess_code_interpreter.py @@ -0,0 +1,152 @@ +import queue +import subprocess +import threading +import time +import traceback + +from .base_code_interpreter import BaseCodeInterpreter + + +class SubprocessCodeInterpreter(BaseCodeInterpreter): + + def __init__(self): + self.start_cmd = '' + self.process = None + self.debug_mode = False + self.output_queue = queue.Queue() + self.done = threading.Event() + + def detect_active_line(self, line): + return None + + def detect_end_of_execution(self, line): + return None + + def line_postprocessor(self, line): + return line + + def preprocess_code(self, code): + """ + This needs to insert an end_of_execution marker of some kind, + which can be detected by detect_end_of_execution. + + Optionally, add active line markers for detect_active_line. + """ + return code + + def terminate(self): + self.process.terminate() + + def start_process(self): + if self.process: + self.terminate() + + self.process = subprocess.Popen( + self.start_cmd.split(), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=0, + universal_newlines=True, + ) + threading.Thread( + target=self.handle_stream_output, + args=(self.process.stdout, False), + daemon=True, + ).start() + threading.Thread( + target=self.handle_stream_output, + args=(self.process.stderr, True), + daemon=True, + ).start() + + def run(self, code): + retry_count = 0 + max_retries = 3 + + # Setup + try: + code = self.preprocess_code(code) + if not self.process: + self.start_process() + except Exception as e: + print(e) + yield {'output': traceback.format_exc()} + return + + while retry_count <= max_retries: + if self.debug_mode: + print( + f'(after processing) Running processed code:\n{code}\n---') + + self.done.clear() + + try: + self.process.stdin.write(code + '\n') + self.process.stdin.flush() + break + except Exception as e: + print(e) + if retry_count != 0: + # For UX, I like to hide this if it happens once. Obviously feels better to not see errors + # Most of the time it doesn't matter, but we should figure out why it happens frequently with: + # applescript + yield {'output': traceback.format_exc()} + yield { + 'output': f'Retrying... ({retry_count}/{max_retries})' + } + yield {'output': 'Restarting process.'} + + self.start_process() + + retry_count += 1 + if retry_count > max_retries: + yield { + 'output': + 'Maximum retries reached. Could not execute code.' + } + return + + while True: + if not self.output_queue.empty(): + yield self.output_queue.get() + else: + time.sleep(0.1) + try: + output = self.output_queue.get( + timeout=0.3) # Waits for 0.3 seconds + yield output + except queue.Empty: + if self.done.is_set(): + # Try to yank 3 more times from it... maybe there's something in there... + # (I don't know if this actually helps. Maybe we just need to yank 1 more time) + for _ in range(3): + if not self.output_queue.empty(): + yield self.output_queue.get() + time.sleep(0.2) + break + + def handle_stream_output(self, stream, is_error_stream): + for line in iter(stream.readline, ''): + if self.debug_mode: + print(f'Received output line:\n{line}\n---') + + line = self.line_postprocessor(line) + + if line is None: + continue # `line = None` is the postprocessor's signal to discard completely + + if self.detect_active_line(line): + active_line = self.detect_active_line(line) + self.output_queue.put({'active_line': active_line}) + elif self.detect_end_of_execution(line): + self.output_queue.put({'active_line': None}) + time.sleep(0.1) + self.done.set() + elif is_error_stream and 'KeyboardInterrupt' in line: + self.output_queue.put({'output': 'KeyboardInterrupt'}) + time.sleep(0.1) + self.done.set() + else: + self.output_queue.put({'output': line}) diff --git a/agentfabric/modelscope_agent/tools/code_interpreter_utils/truncate_output.py b/agentfabric/modelscope_agent/tools/code_interpreter_utils/truncate_output.py new file mode 100644 index 0000000000000000000000000000000000000000..f3ed3314ff51d0a7af6c19abb3e4bfabc2ede420 --- /dev/null +++ b/agentfabric/modelscope_agent/tools/code_interpreter_utils/truncate_output.py @@ -0,0 +1,15 @@ +def truncate_output(data, max_output_chars=2000): + needs_truncation = False + + message = f'Output truncated. Showing the last {max_output_chars} characters.\n\n' + + # Remove previous truncation message if it exists + if data.startswith(message): + data = data[len(message):] + needs_truncation = True + + # If data exceeds max length, truncate it and add message + if len(data) > max_output_chars or needs_truncation: + data = message + data[-max_output_chars:] + + return data diff --git a/agentfabric/modelscope_agent/tools/hf_tool.py b/agentfabric/modelscope_agent/tools/hf_tool.py new file mode 100644 index 0000000000000000000000000000000000000000..98fa94e6fd76d96139e1292fe35b136acaa4f9ab --- /dev/null +++ b/agentfabric/modelscope_agent/tools/hf_tool.py @@ -0,0 +1,22 @@ +from typing import Dict, List + +from transformers.tools import Tool as HFTool + +from .tool import Tool + + +class HFTool(Tool): + """Simple wrapper for huggingface transformers tools + + """ + + def __init__(self, tool: HFTool, description: str, name: str, + parameters: List[Dict]): + self.tool = tool + self.description = description + self.name = name + self.parameters = parameters + super().__init__() + + def _local_call(self, *args, **kwargs): + return {'result': self.tool(**kwargs)} diff --git a/agentfabric/modelscope_agent/tools/image_chat_tool.py b/agentfabric/modelscope_agent/tools/image_chat_tool.py new file mode 100644 index 0000000000000000000000000000000000000000..526df966b8ac47bb3e26224c8be8d941101f3f9f --- /dev/null +++ b/agentfabric/modelscope_agent/tools/image_chat_tool.py @@ -0,0 +1,51 @@ +from modelscope.utils.constant import Tasks +from .pipeline_tool import ModelscopePipelineTool + + +class ImageChatTool(ModelscopePipelineTool): + default_model = 'damo/multi-modal_mplug_owl_multimodal-dialogue_7b' + description = '图文对话和图像描述服务，针对输入的图片和用户的文本输入，给出文本回复' + name = 'modelscope_image-chat' + parameters: list = [{ + 'name': 'image', + 'description': '用户输入的图片', + 'required': True + }, { + 'name': 'text', + 'description': '用户输入的文本', + 'required': True + }] + task = Tasks.multimodal_dialogue + + def construct_image_chat_input(self, **kwargs): + image = kwargs.pop('image', '') + text = kwargs.pop('text', '') + + system_prompt_1 = 'The following is a conversation between a curious human and AI assistant.' + system_prompt_2 = "The assistant gives helpful, detailed, and polite answers to the user's questions." + messages = { + 'messages': [ + { + 'role': 'system', + 'content': system_prompt_1 + ' ' + system_prompt_2 + }, + { + 'role': 'user', + 'content': [{ + 'image': image + }] + }, + { + 'role': 'user', + 'content': text + }, + ] + } + return messages + + def _remote_parse_input(self, *args, **kwargs): + messages = self.construct_image_chat_input(**kwargs) + return {'input': messages} + + def _local_parse_input(self, *args, **kwargs): + return (self.construct_image_chat_input(**kwargs)), {} diff --git a/agentfabric/modelscope_agent/tools/openapi_plugin.py b/agentfabric/modelscope_agent/tools/openapi_plugin.py new file mode 100644 index 0000000000000000000000000000000000000000..2502305ab532ae20157e9b98f830ab03fd46d925 --- /dev/null +++ b/agentfabric/modelscope_agent/tools/openapi_plugin.py @@ -0,0 +1,370 @@ +import os +import re +from typing import List, Optional + +import json +import requests +from jsonschema import RefResolver +from pydantic import BaseModel, ValidationError +from requests.exceptions import RequestException, Timeout + +from .tool import Tool + +MAX_RETRY_TIMES = 3 + + +class ParametersSchema(BaseModel): + name: str + description: str + required: Optional[bool] = True + + +class ToolSchema(BaseModel): + name: str + description: str + parameters: List[ParametersSchema] + + +class OpenAPIPluginTool(Tool): + """ + openapi schema tool + """ + name: str = 'api tool' + description: str = 'This is a api tool that ...' + parameters: list = [] + + def __init__(self, cfg, name): + self.name = name + self.cfg = cfg.get(self.name, {}) + self.is_remote_tool = self.cfg.get('is_remote_tool', False) + # remote call + self.url = self.cfg.get('url', '') + self.token = self.cfg.get('token', '') + self.header = self.cfg.get('header', '') + self.method = self.cfg.get('method', '') + self.parameters = self.cfg.get('parameters', []) + self.description = self.cfg.get('description', + 'This is a api tool that ...') + self.responses_param = self.cfg.get('responses_param', []) + try: + all_para = { + 'name': self.name, + 'description': self.description, + 'parameters': self.parameters + } + self.tool_schema = ToolSchema(**all_para) + except ValidationError: + raise ValueError(f'Error when parsing parameters of {self.name}') + self._str = self.tool_schema.model_dump_json() + self._function = self.parse_pydantic_model_to_openai_function(all_para) + + def _remote_call(self, *args, **kwargs): + if self.url == '': + raise ValueError( + f"Could not use remote call for {self.name} since this tool doesn't have a remote endpoint" + ) + + remote_parsed_input = json.dumps( + self._remote_parse_input(*args, **kwargs)) + origin_result = None + if self.method == 'POST': + retry_times = MAX_RETRY_TIMES + while retry_times: + retry_times -= 1 + try: + print(f'data: {kwargs}') + print(f'header: {self.header}') + response = requests.request( + 'POST', + url=self.url, + headers=self.header, + data=remote_parsed_input) + + if response.status_code != requests.codes.ok: + response.raise_for_status() + origin_result = json.loads( + response.content.decode('utf-8')) + + final_result = self._parse_output( + origin_result, remote=True) + return final_result + except Timeout: + continue + except RequestException as e: + raise ValueError( + f'Remote call failed with error code: {e.response.status_code},\ + error message: {e.response.content.decode("utf-8")}') + + raise ValueError( + 'Remote call max retry times exceeded! Please try to use local call.' + ) + elif self.method == 'GET': + retry_times = MAX_RETRY_TIMES + + new_url = self.url + matches = re.findall(r'\{(.*?)\}', self.url) + for match in matches: + if match in kwargs: + new_url = new_url.replace('{' + match + '}', kwargs[match]) + else: + print( + f'The parameter {match} was not generated by the model.' + ) + + while retry_times: + retry_times -= 1 + try: + print('GET:', new_url) + print('GET:', self.url) + + response = requests.request( + 'GET', + url=new_url, + headers=self.header, + params=remote_parsed_input) + if response.status_code != requests.codes.ok: + response.raise_for_status() + + origin_result = json.loads( + response.content.decode('utf-8')) + + final_result = self._parse_output( + origin_result, remote=True) + return final_result + except Timeout: + continue + except RequestException as e: + raise ValueError( + f'Remote call failed with error code: {e.response.status_code},\ + error message: {e.response.content.decode("utf-8")}') + + raise ValueError( + 'Remote call max retry times exceeded! Please try to use local call.' + ) + else: + raise ValueError( + 'Remote call method is invalid!We have POST and GET method.') + + def _remote_parse_input(self, *args, **kwargs): + restored_dict = {} + for key, value in kwargs.items(): + if '.' in key: + # Split keys by "." and create nested dictionary structures + keys = key.split('.') + temp_dict = restored_dict + for k in keys[:-1]: + temp_dict = temp_dict.setdefault(k, {}) + temp_dict[keys[-1]] = value + else: + # f the key does not contain ".", directly store the key-value pair into restored_dict + restored_dict[key] = value + kwargs = restored_dict + print('传给tool的参数：', kwargs) + return kwargs + + +# openapi_schema_convert,register to tool_config.json +def extract_references(schema_content): + references = [] + if isinstance(schema_content, dict): + if '$ref' in schema_content: + references.append(schema_content['$ref']) + for key, value in schema_content.items(): + references.extend(extract_references(value)) + elif isinstance(schema_content, list): + for item in schema_content: + references.extend(extract_references(item)) + return references + + +def parse_nested_parameters(param_name, param_info, parameters_list, content): + param_type = param_info['type'] + param_description = param_info.get('description', + f'用户输入的{param_name}') # 按需更改描述 + param_required = param_name in content['required'] + try: + if param_type == 'object': + properties = param_info.get('properties') + if properties: + # If the argument type is an object and has a non-empty "properties" field, + # its internal properties are parsed recursively + for inner_param_name, inner_param_info in properties.items(): + inner_param_type = inner_param_info['type'] + inner_param_description = inner_param_info.get( + 'description', f'用户输入的{param_name}.{inner_param_name}') + inner_param_required = param_name.split( + '.')[0] in content['required'] + + # Recursively call the function to handle nested objects + if inner_param_type == 'object': + parse_nested_parameters( + f'{param_name}.{inner_param_name}', + inner_param_info, parameters_list, content) + else: + parameters_list.append({ + 'name': + f'{param_name}.{inner_param_name}', + 'description': + inner_param_description, + 'required': + inner_param_required, + 'type': + inner_param_type, + 'value': + inner_param_info.get('enum', '') + }) + else: + # Non-nested parameters are added directly to the parameter list + parameters_list.append({ + 'name': param_name, + 'description': param_description, + 'required': param_required, + 'type': param_type, + 'value': param_info.get('enum', '') + }) + except Exception as e: + raise ValueError(f'{e}:schema结构出错') + + +def parse_responses_parameters(param_name, param_info, parameters_list): + param_type = param_info['type'] + param_description = param_info.get('description', + f'调用api返回的{param_name}') # 按需更改描述 + try: + if param_type == 'object': + properties = param_info.get('properties') + if properties: + # If the argument type is an object and has a non-empty "properties" + # field, its internal properties are parsed recursively + + for inner_param_name, inner_param_info in properties.items(): + param_type = inner_param_info['type'] + param_description = inner_param_info.get( + 'description', + f'调用api返回的{param_name}.{inner_param_name}') + parameters_list.append({ + 'name': f'{param_name}.{inner_param_name}', + 'description': param_description, + 'type': param_type, + }) + else: + # Non-nested parameters are added directly to the parameter list + parameters_list.append({ + 'name': param_name, + 'description': param_description, + 'type': param_type, + }) + except Exception as e: + raise ValueError(f'{e}:schema结构出错') + + +def openapi_schema_convert(schema, auth): + + resolver = RefResolver.from_schema(schema) + servers = schema.get('servers', []) + if servers: + servers_url = servers[0].get('url') + else: + print('No URL found in the schema.') + # Extract endpoints + endpoints = schema.get('paths', {}) + description = schema.get('info', {}).get('description', + 'This is a api tool that ...') + config_data = {} + # Iterate over each endpoint and its contents + for endpoint_path, methods in endpoints.items(): + for method, details in methods.items(): + summary = details.get('summary', 'No summary').replace(' ', '_') + name = details.get('operationId', 'No operationId') + url = f'{servers_url}{endpoint_path}' + security = details.get('security', [{}]) + # Security (Bearer Token) + authorization = '' + if security: + for sec in security: + if 'BearerAuth' in sec: + api_token = auth.get('apikey', os.environ['apikey']) + api_token_type = auth.get('apikey_type', + os.environ['apikey_type']) + authorization = f'{api_token_type} {api_token}' + if method.upper() == 'POST': + requestBody = details.get('requestBody', {}) + if requestBody: + for content_type, content_details in requestBody.get( + 'content', {}).items(): + schema_content = content_details.get('schema', {}) + references = extract_references(schema_content) + for reference in references: + resolved_schema = resolver.resolve(reference) + content = resolved_schema[1] + parameters_list = [] + for param_name, param_info in content[ + 'properties'].items(): + parse_nested_parameters( + param_name, param_info, parameters_list, + content) + X_DashScope_Async = requestBody.get( + 'X-DashScope-Async', '') + if X_DashScope_Async == '': + config_entry = { + 'name': name, + 'description': description, + 'is_active': True, + 'is_remote_tool': True, + 'url': url, + 'method': method.upper(), + 'parameters': parameters_list, + 'header': { + 'Content-Type': content_type, + 'Authorization': authorization + } + } + else: + config_entry = { + 'name': name, + 'description': description, + 'is_active': True, + 'is_remote_tool': True, + 'url': url, + 'method': method.upper(), + 'parameters': parameters_list, + 'header': { + 'Content-Type': content_type, + 'Authorization': authorization, + 'X-DashScope-Async': 'enable' + } + } + else: + config_entry = { + 'name': name, + 'description': description, + 'is_active': True, + 'is_remote_tool': True, + 'url': url, + 'method': method.upper(), + 'parameters': [], + 'header': { + 'Content-Type': 'application/json', + 'Authorization': authorization + } + } + elif method.upper() == 'GET': + parameters_list = [] + parameters_list = details.get('parameters', []) + config_entry = { + 'name': name, + 'description': description, + 'is_active': True, + 'is_remote_tool': True, + 'url': url, + 'method': method.upper(), + 'parameters': parameters_list, + 'header': { + 'Authorization': authorization + } + } + else: + raise 'method is not POST or GET' + + config_data[summary] = config_entry + return config_data diff --git a/agentfabric/modelscope_agent/tools/pipeline_tool.py b/agentfabric/modelscope_agent/tools/pipeline_tool.py new file mode 100644 index 0000000000000000000000000000000000000000..12f676dbd56b18e4ddf1ac7130f7a3bef6751a91 --- /dev/null +++ b/agentfabric/modelscope_agent/tools/pipeline_tool.py @@ -0,0 +1,40 @@ +from modelscope.pipelines import pipeline +from .tool import Tool + + +class ModelscopePipelineTool(Tool): + + default_model: str = '' + task: str = '' + model_revision = None + + def __init__(self, cfg): + + super().__init__(cfg) + self.model = self.cfg.get('model', None) or self.default_model + self.model_revision = self.cfg.get('model_revision', + None) or self.model_revision + + self.pipeline_params = self.cfg.get('pipeline_params', {}) + self.pipeline = None + self.is_initialized = False + + def setup(self): + + # only initialize when this tool is really called to save memory + if not self.is_initialized: + self.pipeline = pipeline( + task=self.task, + model=self.model, + model_revision=self.model_revision, + **self.pipeline_params) + self.is_initialized = True + + def _local_call(self, *args, **kwargs): + + self.setup() + + parsed_args, parsed_kwargs = self._local_parse_input(*args, **kwargs) + origin_result = self.pipeline(*parsed_args, **parsed_kwargs) + final_result = self._parse_output(origin_result, remote=False) + return final_result diff --git a/agentfabric/modelscope_agent/tools/plugin_tool.py b/agentfabric/modelscope_agent/tools/plugin_tool.py new file mode 100644 index 0000000000000000000000000000000000000000..bd1242448545b6042aa8f12d2fcd5ea959306427 --- /dev/null +++ b/agentfabric/modelscope_agent/tools/plugin_tool.py @@ -0,0 +1,30 @@ +from copy import deepcopy + +from .tool import Tool + + +class LangchainTool(Tool): + + def __init__(self, langchain_tool): + from langchain.tools import BaseTool + + if not isinstance(langchain_tool, BaseTool): + raise ValueError('langchain_tool should be type of langchain tool') + self.langchain_tool = langchain_tool + self.parse_langchain_schema() + super().__init__() + + def parse_langchain_schema(self): + # convert langchain tool schema to modelscope_agent tool schema + self.description = self.langchain_tool.description + self.name = self.langchain_tool.name + self.parameters = [] + for name, arg in self.langchain_tool.args.items(): + tool_arg = deepcopy(arg) + tool_arg['name'] = name + tool_arg['required'] = True + tool_arg.pop('title') + self.parameters.append(tool_arg) + + def _local_call(self, *args, **kwargs): + return {'result': self.langchain_tool.run(kwargs)} diff --git a/agentfabric/modelscope_agent/tools/text_address_tool.py b/agentfabric/modelscope_agent/tools/text_address_tool.py new file mode 100644 index 0000000000000000000000000000000000000000..8c52b147b84fbe6d8da0e69989e936b8b731ea04 --- /dev/null +++ b/agentfabric/modelscope_agent/tools/text_address_tool.py @@ -0,0 +1,20 @@ +from modelscope.utils.constant import Tasks +from .pipeline_tool import ModelscopePipelineTool + + +class TextAddressTool(ModelscopePipelineTool): + default_model = 'damo/mgeo_geographic_elements_tagging_chinese_base' + description = '地址解析服务，针对中文地址信息，识别出里面的元素，包括省、市、区、镇、社区、道路、路号、POI、楼栋号、户室号等' + name = 'modelscope_text-address' + parameters: list = [{ + 'name': 'input', + 'description': '用户输入的地址信息', + 'required': True + }] + task = Tasks.token_classification + + def _parse_output(self, origin_result, *args, **kwargs): + final_result = {} + for e in origin_result['output']: + final_result[e['type']] = e['span'] + return {'result': final_result} diff --git a/agentfabric/modelscope_agent/tools/text_ie_tool.py b/agentfabric/modelscope_agent/tools/text_ie_tool.py new file mode 100644 index 0000000000000000000000000000000000000000..d8c983e8481cace0acaa228eddf22bd6df28af01 --- /dev/null +++ b/agentfabric/modelscope_agent/tools/text_ie_tool.py @@ -0,0 +1,32 @@ +from collections import defaultdict + +from modelscope.utils.constant import Tasks +from .pipeline_tool import ModelscopePipelineTool + + +class TextInfoExtractTool(ModelscopePipelineTool): + default_model = 'damo/nlp_structbert_siamese-uie_chinese-base' + description = '信息抽取服务，针对中文的文本，根据schema要抽取的内容，找出其中对应信息，并用json格式展示' + name = 'modelscope_text-ie' + parameters: list = [{ + 'name': 'input', + 'description': '用户输入的文本', + 'required': True + }, { + 'name': 'schema', + 'description': '要抽取信息的json表示', + 'required': True + }] + task = Tasks.siamese_uie + + def _remote_parse_input(self, *args, **kwargs): + kwargs['parameters'] = {'schema': kwargs['schema']} + kwargs.pop('schema') + return kwargs + + def _parse_output(self, origin_result, *args, **kwargs): + final_result = defaultdict(list) + for e in origin_result['output']: + final_result[e[0]['type']].append(e[0]['span']) + + return {'result': dict(final_result)} diff --git a/agentfabric/modelscope_agent/tools/text_ner_tool.py b/agentfabric/modelscope_agent/tools/text_ner_tool.py new file mode 100644 index 0000000000000000000000000000000000000000..a694a96c90a2b6d9e344754b065e81a8b5897ee9 --- /dev/null +++ b/agentfabric/modelscope_agent/tools/text_ner_tool.py @@ -0,0 +1,22 @@ +from collections import defaultdict + +from modelscope.utils.constant import Tasks +from .pipeline_tool import ModelscopePipelineTool + + +class TextNerTool(ModelscopePipelineTool): + default_model = 'damo/nlp_raner_named-entity-recognition_chinese-base-news' + description = '命名实体识别服务，针对需要识别的中文文本，找出其中的实体，返回json格式结果' + name = 'modelscope_text-ner' + parameters: list = [{ + 'name': 'input', + 'description': '用户输入的文本', + 'required': True + }] + task = Tasks.named_entity_recognition + + def _parse_output(self, origin_result, *args, **kwargs): + final_result = defaultdict(list) + for e in origin_result['output']: + final_result[e['type']].append(e['span']) + return {'result': dict(final_result)} diff --git a/agentfabric/modelscope_agent/tools/text_to_image_tool.py b/agentfabric/modelscope_agent/tools/text_to_image_tool.py new file mode 100644 index 0000000000000000000000000000000000000000..0c1dc04b182db32365cc0b1f83ecefaf9189aaed --- /dev/null +++ b/agentfabric/modelscope_agent/tools/text_to_image_tool.py @@ -0,0 +1,114 @@ +import os +import re + +import cv2 +import dashscope +import json +from dashscope import ImageSynthesis +from modelscope_agent.output_wrapper import ImageWrapper + +from modelscope.utils.constant import Tasks +from .pipeline_tool import ModelscopePipelineTool + + +class TextToImageTool(ModelscopePipelineTool): + default_model = 'AI-ModelScope/stable-diffusion-xl-base-1.0' + description = 'AI绘画（图像生成）服务，输入文本描述和图像分辨率，返回根据文本信息绘制的图片URL。' + name = 'image_gen' + parameters: list = [{ + 'name': 'text', + 'description': '详细描述了希望生成的图像具有什么内容，例如人物、环境、动作等细节描述', + 'required': True, + 'schema': { + 'type': 'string' + } + }, { + 'name': 'resolution', + 'description': + '格式是 数字*数字，表示希望生成的图像的分辨率大小，选项有[1024*1024, 720*1280, 1280*720]', + 'required': True, + 'schema': { + 'type': 'string' + } + }] + model_revision = 'v1.0.0' + task = Tasks.text_to_image_synthesis + + # def _remote_parse_input(self, *args, **kwargs): + # params = { + # 'input': { + # 'text': kwargs['text'], + # 'resolution': kwargs['resolution'] + # } + # } + # if kwargs.get('seed', None): + # params['input']['seed'] = kwargs['seed'] + # return params + + def _remote_call(self, *args, **kwargs): + + if ('resolution' in kwargs) and (kwargs['resolution'] in [ + '1024*1024', '720*1280', '1280*720' + ]): + resolution = kwargs['resolution'] + else: + resolution = '1280*720' + + prompt = kwargs['text'] + seed = kwargs.get('seed', None) + if prompt is None: + return None + dashscope.api_key = os.getenv('DASHSCOPE_API_KEY') + response = ImageSynthesis.call( + model=ImageSynthesis.Models.wanx_v1, + prompt=prompt, + n=1, + size=resolution, + steps=10, + seed=seed) + final_result = self._parse_output(response, remote=True) + return final_result + + def _local_parse_input(self, *args, **kwargs): + + text = kwargs.pop('text', '') + + parsed_args = ({'text': text}, ) + + return parsed_args, {} + + def _parse_output(self, origin_result, remote=True): + if not remote: + image = cv2.cvtColor(origin_result['output_imgs'][0], + cv2.COLOR_BGR2RGB) + else: + image = origin_result.output['results'][0]['url'] + + return {'result': ImageWrapper(image)} + + def _handle_input_fallback(self, **kwargs): + """ + an alternative method is to parse image is that get item between { and } + for last try + + :param fallback_text: + :return: language, cocde + """ + + text = kwargs.get('text', None) + fallback = kwargs.get('fallback', None) + + if text: + return text + elif fallback: + try: + text = fallback + json_block = re.search(r'\{([\s\S]+)\}', text) # noqa W^05 + if json_block: + result = json_block.group(1) + result_json = json.loads('{' + result + '}') + return result_json['text'] + except ValueError: + return text + else: + return text diff --git a/agentfabric/modelscope_agent/tools/text_to_speech_tool.py b/agentfabric/modelscope_agent/tools/text_to_speech_tool.py new file mode 100644 index 0000000000000000000000000000000000000000..12d35854361f7350df2ebeff61435988936efc60 --- /dev/null +++ b/agentfabric/modelscope_agent/tools/text_to_speech_tool.py @@ -0,0 +1,44 @@ +from modelscope_agent.output_wrapper import AudioWrapper + +from modelscope.utils.constant import Tasks +from .pipeline_tool import ModelscopePipelineTool + + +class TexttoSpeechTool(ModelscopePipelineTool): + default_model = 'damo/speech_sambert-hifigan_tts_zh-cn_16k' + description = '文本转语音服务，将文字转换为自然而逼真的语音，可配置男声/女声' + name = 'modelscope_speech-generation' + parameters: list = [{ + 'name': 'input', + 'description': '要转成语音的文本', + 'required': True + }, { + 'name': 'gender', + 'description': '用户身份', + 'required': True + }] + task = Tasks.text_to_speech + + def _local_parse_input(self, *args, **kwargs): + if 'gender' not in kwargs: + kwargs['gender'] = 'man' + voice = 'zhizhe_emo' if kwargs['gender'] == 'man' else 'zhiyan_emo' + kwargs['voice'] = voice + if 'text' in kwargs and 'input' not in kwargs: + kwargs['input'] = kwargs['text'] + kwargs.pop('text') + kwargs.pop('gender') + return args, kwargs + + def _remote_parse_input(self, *args, **kwargs): + if 'gender' not in kwargs: + kwargs['gender'] = 'man' + voice = 'zhizhe_emo' if kwargs['gender'] == 'man' else 'zhiyan_emo' + kwargs['voice'] = voice + kwargs.pop('gender') + return kwargs + + def _parse_output(self, origin_result, remote=True): + + audio = origin_result['output_wav'] + return {'result': AudioWrapper(audio)} diff --git a/agentfabric/modelscope_agent/tools/text_to_video_tool.py b/agentfabric/modelscope_agent/tools/text_to_video_tool.py new file mode 100644 index 0000000000000000000000000000000000000000..bbb84b23e689aad8073e93fd9d5343c0f0fb6412 --- /dev/null +++ b/agentfabric/modelscope_agent/tools/text_to_video_tool.py @@ -0,0 +1,40 @@ +import os +import tempfile +import uuid + +from modelscope_agent.output_wrapper import VideoWrapper + +from modelscope.utils.constant import Tasks +from .pipeline_tool import ModelscopePipelineTool + + +class TextToVideoTool(ModelscopePipelineTool): + default_model = 'damo/text-to-video-synthesis' + description = '视频生成服务，针对英文文本输入，生成一段描述视频；如果是中文输入同时依赖插件modelscope_text-translation-zh2en翻译成英文' + + name = 'modelscope_video-generation' + parameters: list = [{ + 'name': 'text', + 'description': '用户输入的文本信息', + 'required': True + }] + task = Tasks.text_to_video_synthesis + + def _remote_parse_input(self, *args, **kwargs): + return {'input': {'text': kwargs['text']}} + + def _local_parse_input(self, *args, **kwargs): + + text = kwargs.pop('text', '') + directory = tempfile.mkdtemp() + file_path = os.path.join(directory, str(uuid.uuid4()) + '.mp4') + + parsed_args = ({'text': text}, ) + parsed_kwargs = {'output_video': file_path} + + return parsed_args, parsed_kwargs + + def _parse_output(self, origin_result, remote=True): + + video = origin_result['output_video'] + return {'result': VideoWrapper(video)} diff --git a/agentfabric/modelscope_agent/tools/tool.py b/agentfabric/modelscope_agent/tools/tool.py new file mode 100644 index 0000000000000000000000000000000000000000..5252f1c9c12658d118059e0179b21d4b29319941 --- /dev/null +++ b/agentfabric/modelscope_agent/tools/tool.py @@ -0,0 +1,180 @@ +import os +from typing import List, Optional + +import json +import requests +from pydantic import BaseModel, ValidationError +from requests.exceptions import RequestException, Timeout + +MODELSCOPE_API_TOKEN = os.getenv('MODELSCOPE_API_TOKEN') + +MAX_RETRY_TIMES = 3 + + +class ParametersSchema(BaseModel): + name: str + description: str + required: Optional[bool] = True + + +class ToolSchema(BaseModel): + name: str + description: str + parameters: List[ParametersSchema] + + +class Tool: + """ + a base class for tools. + when you inherit this class and implement new tool, you should provide name, description + and parameters of tool that conforms with schema. + + each tool may have two call method: _local_call(execute tool in your local environment) + and _remote_call(construct a http request to remote server). + corresponding to preprocess and postprocess method may need to be overrided to get correct result. + """ + name: str = 'tool' + description: str = 'This is a tool that ...' + parameters: list = [] + + def __init__(self, cfg={}): + self.cfg = cfg.get(self.name, {}) + self.is_remote_tool = self.cfg.get('is_remote_tool', False) + + # remote call + self.url = self.cfg.get('url', '') + self.token = self.cfg.get('token', '') + self.header = { + 'Authorization': self.token or f'Bearer {MODELSCOPE_API_TOKEN}' + } + + try: + all_para = { + 'name': self.name, + 'description': self.description, + 'parameters': self.parameters + } + self.tool_schema = ToolSchema(**all_para) + except ValidationError: + raise ValueError(f'Error when parsing parameters of {self.name}') + + self._str = self.tool_schema.model_dump_json() + self._function = self.parse_pydantic_model_to_openai_function(all_para) + + def __call__(self, remote=False, *args, **kwargs): + if self.is_remote_tool or remote: + return self._remote_call(*args, **kwargs) + else: + return self._local_call(*args, **kwargs) + + def _remote_call(self, *args, **kwargs): + if self.url == '': + raise ValueError( + f"Could not use remote call for {self.name} since this tool doesn't have a remote endpoint" + ) + + remote_parsed_input = json.dumps( + self._remote_parse_input(*args, **kwargs)) + + origin_result = None + retry_times = MAX_RETRY_TIMES + while retry_times: + retry_times -= 1 + try: + response = requests.request( + 'POST', + self.url, + headers=self.header, + data=remote_parsed_input) + if response.status_code != requests.codes.ok: + response.raise_for_status() + + origin_result = json.loads( + response.content.decode('utf-8'))['Data'] + + final_result = self._parse_output(origin_result, remote=True) + return final_result + except Timeout: + continue + except RequestException as e: + raise ValueError( + f'Remote call failed with error code: {e.response.status_code},\ + error message: {e.response.content.decode("utf-8")}') + + raise ValueError( + 'Remote call max retry times exceeded! Please try to use local call.' + ) + + def _local_call(self, *args, **kwargs): + return + + def _remote_parse_input(self, *args, **kwargs): + return kwargs + + def _local_parse_input(self, *args, **kwargs): + return args, kwargs + + def _parse_output(self, origin_result, *args, **kwargs): + return {'result': origin_result} + + def __str__(self): + return self._str + + def get_function(self): + return self._function + + def parse_pydantic_model_to_openai_function(self, all_para: dict): + ''' + this method used to convert a pydantic model to openai function schema + such that convert + all_para = { + 'name': get_current_weather, + 'description': Get the current weather in a given location, + 'parameters': [{ + 'name': 'image', + 'description': '用户输入的图片', + 'required': True + }, { + 'name': 'text', + 'description': '用户输入的文本', + 'required': True + }] + } + to + { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "image": { + "type": "string", + "description": "用户输入的图片", + }, + "text": { + "type": "string", + "description": "用户输入的文本", + }, + "required": ["image", "text"], + }, + } + ''' + + function = { + 'name': all_para['name'], + 'description': all_para['description'], + 'parameters': { + 'type': 'object', + 'properties': {}, + 'required': [], + }, + } + for para in all_para['parameters']: + function['parameters']['properties'][para['name']] = { + 'type': 'string', + 'description': para['description'] + } + if para['required']: + function['parameters']['required'].append(para['name']) + + return function diff --git a/agentfabric/modelscope_agent/tools/translation_en2zh_tool.py b/agentfabric/modelscope_agent/tools/translation_en2zh_tool.py new file mode 100644 index 0000000000000000000000000000000000000000..2e6803e320b3093113e63a288a896938f292136a --- /dev/null +++ b/agentfabric/modelscope_agent/tools/translation_en2zh_tool.py @@ -0,0 +1,17 @@ +from modelscope.utils.constant import Tasks +from .pipeline_tool import ModelscopePipelineTool + + +class TranslationEn2ZhTool(ModelscopePipelineTool): + default_model = 'damo/nlp_csanmt_translation_en2zh' + description = '根据输入指令，将相应的英文文本翻译成中文回复' + name = 'modelscope_text-translation-en2zh' + task = Tasks.translation + parameters: list = [{ + 'name': 'input', + 'description': '用户输入的英文文本', + 'required': True + }] + + def _parse_output(self, origin_result, *args, **kwargs): + return {'result': origin_result['translation']} diff --git a/agentfabric/modelscope_agent/tools/translation_zh2en_tool.py b/agentfabric/modelscope_agent/tools/translation_zh2en_tool.py new file mode 100644 index 0000000000000000000000000000000000000000..6371acb1e22f2b607025840bc2b7dd8dd96facd1 --- /dev/null +++ b/agentfabric/modelscope_agent/tools/translation_zh2en_tool.py @@ -0,0 +1,17 @@ +from modelscope.utils.constant import Tasks +from .pipeline_tool import ModelscopePipelineTool + + +class TranslationZh2EnTool(ModelscopePipelineTool): + default_model = 'damo/nlp_csanmt_translation_zh2en' + description = '根据输入指令，将相应的中文文本翻译成英文回复' + name = 'modelscope_text-translation-zh2en' + task = Tasks.translation + parameters: list = [{ + 'name': 'input', + 'description': '用户输入的中文文本', + 'required': True + }] + + def _parse_output(self, origin_result, *args, **kwargs): + return {'result': origin_result['translation']} diff --git a/agentfabric/modelscope_agent/tools/web_browser.py b/agentfabric/modelscope_agent/tools/web_browser.py new file mode 100644 index 0000000000000000000000000000000000000000..dd3194a6e038fe0800bfcf5b704b843361464922 --- /dev/null +++ b/agentfabric/modelscope_agent/tools/web_browser.py @@ -0,0 +1,72 @@ +import httpx +from langchain.document_loaders import AsyncHtmlLoader +from langchain.document_transformers import BeautifulSoupTransformer +from langchain.text_splitter import RecursiveCharacterTextSplitter +from modelscope_agent.tools.tool import Tool + + +class WebBrowser(Tool): + description = '生成艺术字纹理图片' + name = 'web_browser' + parameters: list = [{ + 'name': 'urls', + 'description': 'the urls that the user wants to browse', + 'required': True + }] + + def __init__(self, cfg={}): + super().__init__(cfg) + self.split_url_into_chunk = self.cfg.get('split_url_into_chunk', False) + self.headers = { + 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)' + } + self.client = httpx.Client( + headers=self.headers, verify=False, timeout=30.0) + + def _local_call(self, *args, **kwargs): + parsed_args, parsed_kwargs = self._local_parse_input(*args, **kwargs) + + urls = parsed_kwargs['urls'] + print(urls) + if urls is None: + return {'result': ''} + + # # load html + loader = AsyncHtmlLoader(urls) + docs = loader.load() + # Transform + bs_transformer = BeautifulSoupTransformer() + docs_transformed = bs_transformer.transform_documents( + docs, tags_to_extract=['span']) + + # split url content into chunk in order to get fine-grained results + if self.split_url_into_chunk: + # Grab the first 1000 tokens of the site + splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder( + chunk_size=1000, chunk_overlap=0) + splits = splitter.split_documents(docs_transformed) + else: + splits = docs_transformed + search_results = [] + for item in splits: + result = { + 'url': item.metadata['source'], + 'content': item.page_content + } + search_results.append(result) + + return {'result': search_results} + + def _local_parse_input(self, *args, **kwargs): + urls = kwargs.get('urls', []) + if isinstance(urls, str): + urls = [urls] + kwargs['urls'] = urls + return args, kwargs + + +if __name__ == '__main__': + tool = WebBrowser() + urls = ['https://blog.sina.com.cn/zhangwuchang'] + result = tool._local_call(urls=urls) + print(result) diff --git a/agentfabric/modelscope_agent/tools/web_search.py b/agentfabric/modelscope_agent/tools/web_search.py new file mode 100644 index 0000000000000000000000000000000000000000..253845d2838a72ce57865b91430917eb731a1248 --- /dev/null +++ b/agentfabric/modelscope_agent/tools/web_search.py @@ -0,0 +1,85 @@ +import os + +from modelscope_agent.tools.tool import Tool, ToolSchema +from modelscope_agent.tools.web_search_utils import get_websearcher_cls +from modelscope_agent.tools.web_search_utils.search_util import \ + AuthenticationKey +from pydantic import ValidationError + + +class WebSearch(Tool): + description = 'surfacing relevant information from billions of web documents. Help ' \ + 'you find what you are looking for from the world-wide-web to comb ' \ + 'billions of webpages, images, videos, and news.' + name = 'web_search_utils' + parameters: list = [{ + 'name': 'query', + 'description': + """The user's search query term. The term may not be empty.""", + 'required': True + }] + + def __init__(self, cfg={}): + super().__init__() + available_searchers = get_websearcher_cls() + all_searchers = AuthenticationKey.to_dict() + if not len(available_searchers): + raise ValueError( + f'At least one of web search api token should be set: {all_searchers}' + ) + + searcher = cfg.pop('searcher', None) + + if not searcher: + self.searcher = available_searchers[0](**cfg) + else: + if isinstance(searcher, + str) and len(searcher) and all_searchers.get( + searcher, None): + cls = available_searchers.get(searcher, None) + if not cls: + raise ValueError( + f'The searcher {searcher}\'s token is not set: {all_searchers.get(searcher, None)}' + ) + self.searcher = cls(**cfg) + else: + raise ValueError( + f'The searcher {searcher} should be one of {all_searchers.keys()}' + ) + + try: + all_para = { + 'name': self.name, + 'description': self.description, + 'parameters': self.parameters + } + self.tool_schema = ToolSchema(**all_para) + except ValidationError: + raise ValueError(f'Error when parsing parameters of {self.name}') + + self.is_remote_tool = True + self._str = self.tool_schema.model_dump_json() + self._function = self.parse_pydantic_model_to_openai_function(all_para) + + def _remote_call(self, *args, **kwargs): + query = self._handle_input_fallback(**kwargs) + if not query or not len(query): + raise ValueError( + 'parameter `query` of tool web-search is None or Empty.') + + res = self.searcher(query) + return {'result': [item.__dict__ for item in res]} + + def _handle_input_fallback(self, **kwargs): + query = kwargs.get('query', None) + fallback = kwargs.get('fallback', None) + if query and isinstance(query, str) and len(query): + return query + else: + return fallback + + +if __name__ == '__main__': + tool = WebSearch() + res = tool(query='2024年 元旦 哈尔滨天气') + print(res) diff --git a/agentfabric/modelscope_agent/tools/web_search_utils/__init__.py b/agentfabric/modelscope_agent/tools/web_search_utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..704d570f193e99558139360bb0716e3ede647888 --- /dev/null +++ b/agentfabric/modelscope_agent/tools/web_search_utils/__init__.py @@ -0,0 +1,2 @@ +from modelscope_agent.tools.web_search_utils.search_util import \ + get_websearcher_cls diff --git a/agentfabric/modelscope_agent/tools/web_search_utils/search_util.py b/agentfabric/modelscope_agent/tools/web_search_utils/search_util.py new file mode 100644 index 0000000000000000000000000000000000000000..2b3d58c65335f20894eb9fe63d97d1349f09ad60 --- /dev/null +++ b/agentfabric/modelscope_agent/tools/web_search_utils/search_util.py @@ -0,0 +1,40 @@ +import os + + +class SearchResult: + + def __init__(self, title=None, link=None, sniper=None): + assert link or sniper + self.title = title + self.link = link + self.sniper = sniper + + +class AuthenticationKey: + bing = 'BING_SEARCH_V7_SUBSCRIPTION_KEY' + kuake = 'PLACE_HOLDER' + + @classmethod + def to_dict(cls): + raw_dict = cls.__dict__ + res = dict( + filter(lambda x: '__' not in x[0] and isinstance(x[1], str), + raw_dict.items())) + return res + + +def get_websearcher_cls(): + + def get_env(authentication_key: str): + env = os.environ + return env.get(authentication_key, None) + + cls_list = [] + if get_env(AuthenticationKey.bing): + from modelscope_agent.tools.web_search_utils.searcher.bing import BingWebSearcher + cls_list.append(BingWebSearcher) + if get_env(AuthenticationKey.kuake): + from modelscope_agent.tools.web_search_utils.searcher.kuake import KuakeWebSearcher + cls_list.append(KuakeWebSearcher) + + return cls_list diff --git a/agentfabric/modelscope_agent/tools/web_search_utils/searcher/__init__.py b/agentfabric/modelscope_agent/tools/web_search_utils/searcher/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/agentfabric/modelscope_agent/tools/web_search_utils/searcher/base_searcher.py b/agentfabric/modelscope_agent/tools/web_search_utils/searcher/base_searcher.py new file mode 100644 index 0000000000000000000000000000000000000000..6f72cf1ebdb18cf19b520014d442750e908b0a22 --- /dev/null +++ b/agentfabric/modelscope_agent/tools/web_search_utils/searcher/base_searcher.py @@ -0,0 +1,5 @@ +class WebSearcher: + timeout = 1000 + + def __call__(self, **kwargs): + raise NotImplementedError() diff --git a/agentfabric/modelscope_agent/tools/web_search_utils/searcher/bing.py b/agentfabric/modelscope_agent/tools/web_search_utils/searcher/bing.py new file mode 100644 index 0000000000000000000000000000000000000000..6aa7855e539dadbf0e457f68eaf9f5aa5f422bb9 --- /dev/null +++ b/agentfabric/modelscope_agent/tools/web_search_utils/searcher/bing.py @@ -0,0 +1,59 @@ +import os + +import json +import requests +from modelscope_agent.tools.web_search_utils.search_util import ( + AuthenticationKey, SearchResult) + +from .base_searcher import WebSearcher + + +class BingWebSearcher(WebSearcher): + + def __init__( + self, + timeout=3000, + mkt='en-US', + endpoint='https://api.bing.microsoft.com/v7.0/search', + ): + self.mkt = mkt + self.endpoint = endpoint + self.timeout = timeout + self.token = os.environ.get(AuthenticationKey.bing) + + def __call__(self, query, **kwargs): + params = {'q': query, 'mkt': self.mkt} + headers = {'Ocp-Apim-Subscription-Key': self.token} + if kwargs: + params.update(kwargs) + try: + response = requests.get( + self.endpoint, + headers=headers, + params=params, + timeout=self.timeout) + raw_result = json.loads(response.text) + if raw_result.get('error', None): + print(f'Call Bing web search api failed: {raw_result}') + except Exception as ex: + raise ex('Call Bing web search api failed.') + + results = [] + res_list = raw_result.get('webPages', {}).get('value', []) + for item in res_list: + title = item.get('name', None) + link = item.get('url', None) + sniper = item.get('snippet', None) + if not link and not sniper: + continue + + results.append(SearchResult(title=title, link=link, sniper=sniper)) + + return results + + +if __name__ == '__main__': + + searcher = BingWebSearcher() + res = searcher('哈尔滨元旦的天气情况') + print([item.__dict__ for item in res]) diff --git a/agentfabric/modelscope_agent/tools/web_search_utils/searcher/kuake.py b/agentfabric/modelscope_agent/tools/web_search_utils/searcher/kuake.py new file mode 100644 index 0000000000000000000000000000000000000000..9c135dfa5ed2081c40012dfa4195b44950874d6e --- /dev/null +++ b/agentfabric/modelscope_agent/tools/web_search_utils/searcher/kuake.py @@ -0,0 +1,7 @@ +from .base_searcher import WebSearcher + + +class KuakeWebSearcher(WebSearcher): + + def __call__(self, query, **kwargs): + raise NotImplementedError() diff --git a/agentfabric/modelscope_agent/tools/wordart_tool.py b/agentfabric/modelscope_agent/tools/wordart_tool.py new file mode 100644 index 0000000000000000000000000000000000000000..a8af9cc39fffcc854a9e5d5a702996e8b77b4819 --- /dev/null +++ b/agentfabric/modelscope_agent/tools/wordart_tool.py @@ -0,0 +1,169 @@ +import os +import time + +import json +import pandas as pd +import requests +from modelscope_agent.tools.tool import Tool, ToolSchema +from pydantic import ValidationError +from requests.exceptions import RequestException, Timeout + +MAX_RETRY_TIMES = 3 + + +class WordArtTexture(Tool): + description = '生成艺术字纹理图片' + name = 'wordart_texture_generation' + parameters: list = [{ + 'name': 'input.text.text_content', + 'description': 'text that the user wants to convert to WordArt', + 'required': True + }, { + 'name': 'input.prompt', + 'description': + 'Users’ style requirements for word art may be requirements in terms of shape, color, entity, etc.', + 'required': True + }, { + 'name': 'input.texture_style', + 'description': + 'Type of texture style;Default is "material";If not provided by the user, \ + defaults to "material".Another value is scene.', + 'required': True + }, { + 'name': 'input.text.output_image_ratio', + 'description': + 'The aspect ratio of the text input image; the default is "1:1", \ + the available ratios are: "1:1", "16:9", "9:16";', + 'required': True + }] + + def __init__(self, cfg={}): + self.cfg = cfg.get(self.name, {}) + # remote call + self.url = 'https://dashscope.aliyuncs.com/api/v1/services/aigc/wordart/texture' + self.token = self.cfg.get('token', + os.environ.get('DASHSCOPE_API_KEY', '')) + assert self.token != '', 'dashscope api token must be acquired with wordart' + + try: + all_param = { + 'name': self.name, + 'description': self.description, + 'parameters': self.parameters + } + self.tool_schema = ToolSchema(**all_param) + except ValidationError: + raise ValueError(f'Error when parsing parameters of {self.name}') + + self._str = self.tool_schema.model_dump_json() + self._function = self.parse_pydantic_model_to_openai_function( + all_param) + + def __call__(self, *args, **kwargs): + remote_parsed_input = json.dumps( + self._remote_parse_input(*args, **kwargs)) + origin_result = None + retry_times = MAX_RETRY_TIMES + headers = { + 'Content-Type': 'application/json', + 'Authorization': f'Bearer {self.token}', + 'X-DashScope-Async': 'enable' + } + while retry_times: + retry_times -= 1 + try: + + response = requests.request( + 'POST', + url=self.url, + headers=headers, + data=remote_parsed_input) + + if response.status_code != requests.codes.ok: + response.raise_for_status() + origin_result = json.loads(response.content.decode('utf-8')) + + self.final_result = self._parse_output( + origin_result, remote=True) + return self.get_wordart_result() + except Timeout: + continue + except RequestException as e: + raise ValueError( + f'Remote call failed with error code: {e.response.status_code},\ + error message: {e.response.content.decode("utf-8")}') + + raise ValueError( + 'Remote call max retry times exceeded! Please try to use local call.' + ) + + def _remote_parse_input(self, *args, **kwargs): + restored_dict = {} + for key, value in kwargs.items(): + if '.' in key: + # Split keys by "." and create nested dictionary structures + keys = key.split('.') + temp_dict = restored_dict + for k in keys[:-1]: + temp_dict = temp_dict.setdefault(k, {}) + temp_dict[keys[-1]] = value + else: + # f the key does not contain ".", directly store the key-value pair into restored_dict + restored_dict[key] = value + kwargs = restored_dict + kwargs['model'] = 'wordart-texture' + print('传给tool的参数：', kwargs) + return kwargs + + def get_result(self): + result_data = json.loads(json.dumps(self.final_result['result'])) + if 'task_id' in result_data['output']: + task_id = result_data['output']['task_id'] + get_url = f'https://dashscope.aliyuncs.com/api/v1/tasks/{task_id}' + get_header = {'Authorization': f'Bearer {self.token}'} + origin_result = None + retry_times = MAX_RETRY_TIMES + while retry_times: + retry_times -= 1 + try: + response = requests.request( + 'GET', url=get_url, headers=get_header) + if response.status_code != requests.codes.ok: + response.raise_for_status() + origin_result = json.loads(response.content.decode('utf-8')) + + get_result = self._parse_output(origin_result, remote=True) + return get_result + except Timeout: + continue + except RequestException as e: + raise ValueError( + f'Remote call failed with error code: {e.response.status_code},\ + error message: {e.response.content.decode("utf-8")}') + + raise ValueError( + 'Remote call max retry times exceeded! Please try to use local call.' + ) + + def get_wordart_result(self): + try: + result = self.get_result() + print(result) + while True: + result_data = result.get('result', {}) + output = result_data.get('output', {}) + task_status = output.get('task_status', '') + + if task_status == 'SUCCEEDED': + print('任务已完成') + return result + + elif task_status == 'FAILED': + raise ('任务失败') + else: + # 继续轮询，等待一段时间后再次调用 + time.sleep(1) # 等待 1 秒钟 + result = self.get_result() + + except Exception as e: + print('get Remote Error:', str(e)) diff --git a/agentfabric/modelscope_agent/version.py b/agentfabric/modelscope_agent/version.py new file mode 100644 index 0000000000000000000000000000000000000000..683418800a8b84a130ce976b5a71f37242642884 --- /dev/null +++ b/agentfabric/modelscope_agent/version.py @@ -0,0 +1 @@ +__version__ = '0.2.1-rc0' diff --git a/agentfabric/openapi_example/aigc_wordart_semantic.json b/agentfabric/openapi_example/aigc_wordart_semantic.json new file mode 100644 index 0000000000000000000000000000000000000000..b9ee34a54a16db055068aedd938341671214d0ab --- /dev/null +++ b/agentfabric/openapi_example/aigc_wordart_semantic.json @@ -0,0 +1,147 @@ +{ + "openapi":"3.1.0", + "info":{ + "title":"WordArt Semantic Generation API", + "description":"API for generating semantic word art with customizable parameters.", + "version":"v1.0.0" + }, + "servers":[ + { + "url":"https://dashscope.aliyuncs.com" + } + ], + "paths":{ + "/api/v1/services/aigc/wordart/semantic":{ + "post":{ + "summary":"Generate WordArt Semantically", + "operationId":"generateWordArt", + "tags":[ + "WordArt Generation" + ], + "requestBody":{ + "required":true, + "X-DashScope-Async":"enable", + "content":{ + "application/json":{ + "schema":{ + "$ref":"#/components/schemas/WordArtGenerationRequest" + } + } + } + }, + "responses":{ + "200":{ + "description":"Successful Response", + "content":{ + "application/json":{ + "schema":{ + "$ref":"#/components/schemas/WordArtGenerationResponse" + } + } + } + } + }, + "security":[ + { + "BearerAuth":[ + + ] + } + ] + } + }, + "/api/v1/tasks/{task_id}":{ + "get":{ + "summary":"Get WordArt Result", + "operationId":"getwordartresult", + "tags":[ + "Get Result" + ], + "parameters":[ + { + "name":"task_id", + "in":"path", + "required":true, + "description":"The unique identifier of the word art generation task", + "schema":{ + "type":"string" + } + } + ], + "security":[ + { + "BearerAuth":[ + + ] + } + ] + } + } + }, + "components":{ + "schemas":{ + "WordArtGenerationRequest":{ + "type":"object", + "properties":{ + "model":{ + "type":"string", + "enum":[ + "wordart-semantic" + ] + }, + "input":{ + "type":"object", + "properties":{ + "text":{ + "type":"string", + "example":"文字创意", + "description":"用户想要转为艺术字的文本", + "required":true + }, + "prompt":{ + "type":"string", + "example":"水果，蔬菜，温暖的色彩空间", + "description":"用户对艺术字的风格要求，可能是形状、颜色、实体等方面的要求", + "required":true + } + } + }, + "parameters":{ + "type":"object", + "properties":{ + "steps":{ + "type":"integer", + "example":80 + }, + "n":{ + "type":"number", + "example":2 + } + } + } + }, + "required":[ + "model", + "input", + "parameters" + ] + }, + "WordArtGenerationResponse":{ + "type":"object", + "properties":{ + "output":{ + "type":"string", + "description":"Generated word art image URL or data." + } + } + } + }, + "securitySchemes":{ + "ApiKeyAuth":{ + "type":"apiKey", + "in":"header", + "name":"Authorization" + } + } + } +} diff --git a/agentfabric/openapi_example/aigc_wordart_texture.json b/agentfabric/openapi_example/aigc_wordart_texture.json new file mode 100644 index 0000000000000000000000000000000000000000..ea475b489906d2f7493ed618112c63ceff532c31 --- /dev/null +++ b/agentfabric/openapi_example/aigc_wordart_texture.json @@ -0,0 +1,154 @@ +{ + "openapi":"3.1.0", + "info":{ + "title":"WordArt Texture Generation API", + "description":"API for generating textured word art with customizable parameters.", + "version":"v1.0.0" + }, + "servers":[ + { + "url":"https://dashscope.aliyuncs.com" + } + ], + "paths":{ + "/api/v1/services/aigc/wordart/texture":{ + "post":{ + "summary":"Generate Textured WordArt", + "operationId":"generate_textured_WordArt", + "tags":[ + "WordArt Generation" + ], + "requestBody":{ + "required":true, + "X-DashScope-Async":"enable", + "content":{ + "application/json":{ + "schema":{ + "$ref":"#/components/schemas/WordArtGenerationRequest" + } + } + } + }, + "responses":{ + "200":{ + "description":"Successful Response", + "content":{ + "application/json":{ + "schema":{ + "$ref":"#/components/schemas/WordArtGenerationResponse" + } + } + } + } + }, + "security":[ + { + "BearerAuth":[ + + ] + } + ] + } + }, + "/api/v1/tasks/{task_id}":{ + "get":{ + "summary":"Get WordArt Result", + "operationId":"getwordartresult", + "tags":[ + "Get Result" + ], + "parameters":[ + { + "name":"task_id", + "in":"path", + "required":true, + "description":"The unique identifier of the word art generation task", + "schema":{ + "type":"string" + } + } + ], + "security":[ + { + "BearerAuth":[ + + ] + } + ] + } + } + }, + "components":{ + "schemas":{ + "WordArtGenerationRequest":{ + "type":"object", + "properties":{ + "model":{ + "type":"string", + "enum":[ + "wordart-texture" + ] + }, + "input":{ + "type":"object", + "properties":{ + "text":{ + "type":"object", + "properties":{ + "text_content":{ + "type":"string", + "example":"文字纹理", + "description":"用户想要转为艺术字的文本", + "required":true + }, + "font_name":{ + "type":"string", + "example":"dongfangdakai", + "description":"用户想要转为艺术字的字体格式", + "required":true + } + } + }, + "prompt":{ + "type":"string", + "example":"水果，蔬菜，温暖的色彩空间", + "description":"用户对艺术字的风格要求，可能是形状、颜色、实体等方面的要求", + "required":true + } + } + }, + "parameters":{ + "type":"object", + "properties":{ + "n":{ + "type":"number", + "example":2 + } + } + } + }, + "required":[ + "model", + "input", + "parameters" + ] + }, + "WordArtGenerationResponse":{ + "type":"object", + "properties":{ + "output":{ + "type":"string", + "description":"Generated word art image URL or data." + } + } + } + }, + "securitySchemes":{ + "ApiKeyAuth":{ + "type":"apiKey", + "in":"header", + "name":"Authorization" + } + } + } +} diff --git a/agentfabric/publish_util.py b/agentfabric/publish_util.py new file mode 100644 index 0000000000000000000000000000000000000000..eddfb9abfb56fc58e5812895c3bcae548b738972 --- /dev/null +++ b/agentfabric/publish_util.py @@ -0,0 +1,161 @@ +import glob +import os +import shutil +from configparser import ConfigParser + +import json +import oss2 + +from modelscope.utils.config import Config + +MS_VERSION = '0.2.1rc0' +DEFAULT_MS_PKG = 'https://modelscope-agent.oss-cn-hangzhou.aliyuncs.com/releases/v/modelscope_agent-version-py3-none-any.whl' # noqa E501 + + +def upload_to_oss(bucket, local_file_path, oss_file_path): + # 上传文件到阿里云OSS + bucket.put_object_from_file(oss_file_path, local_file_path) + + # 设置文件的公共读权限 + bucket.put_object_acl(oss_file_path, oss2.OBJECT_ACL_PUBLIC_READ) + + # 获取文件的公共链接 + file_url = f"https://{bucket.bucket_name}.{bucket.endpoint.replace('http://', '')}/{oss_file_path}" + return file_url + + +def get_oss_config(): + # 尝试从环境变量中读取配置 + access_key_id = os.getenv('OSS_ACCESS_KEY_ID') + access_key_secret = os.getenv('OSS_ACCESS_KEY_SECRET') + endpoint = os.getenv('OSS_ENDPOINT') + bucket_name = os.getenv('OSS_BUCKET_NAME') + + # 如果环境变量没有设置，尝试从.ossutilconfig文件中读取 + if not access_key_id or not access_key_secret or not endpoint or not bucket_name: + config = ConfigParser() + config.read(os.path.expanduser('~/.ossutilconfig')) + if 'Credentials' in config: + access_key_id = config.get('Credentials', 'accessKeyId') + access_key_secret = config.get('Credentials', 'accessKeySecret') + endpoint = config.get('Credentials', 'endpoint') + bucket_name = config.get('Credentials', 'bucketName') + + return access_key_id, access_key_secret, endpoint, bucket_name + + +def pop_user_info_from_config(src_dir, uuid_str): + """ Remove all personal information from the configuration files and return this data. + The purpose of this is to ensure that personal information is not stored in plain text + when releasing. + + Args: + src_dir (str): config root path + uuid_str (str): user id + """ + user_info = {} + + # deal with plugin cfg + plugin_config_path = f'{src_dir}/config/{uuid_str}/openapi_plugin_config.json' + if os.path.exists(plugin_config_path): + with open(plugin_config_path, 'r') as f: + plugin_config = json.load(f) + if 'auth' in plugin_config: + if plugin_config['auth']['type'] == 'API Key': + user_info['apikey'] = plugin_config['auth'].pop('apikey') + user_info['apikey_type'] = plugin_config['auth'].pop( + 'apikey_type') + with open(plugin_config_path, 'w') as f: + json.dump(plugin_config, f, indent=2, ensure_ascii=False) + + return user_info + + +def prepare_agent_zip(agent_name, src_dir, uuid_str, state): + # 设置阿里云OSS的认证信息 + local_file = os.path.abspath(os.path.dirname(__file__)) + ak_id, ak_secret, endpoint, bucket_name = get_oss_config() + auth = oss2.Auth(ak_id, ak_secret) + bucket = oss2.Bucket(auth, endpoint, bucket_name) + + new_directory = f'{src_dir}/upload/{uuid_str}' # 新目录的路径 + + # 创建新目录 + if os.path.exists(new_directory): + shutil.rmtree(new_directory) + os.makedirs(new_directory) + + # 复制config下的uuid_str目录到new_directory下并改名为local_user + uuid_str_path = f'{src_dir}/config/{uuid_str}' # 指向uuid_str目录的路径 + local_user_path = f'{new_directory}/config' # 新的目录路径 + shutil.copytree(uuid_str_path, local_user_path, dirs_exist_ok=True) + + target_conf = os.path.join(local_user_path, 'builder_config.json') + builder_cfg = Config.from_file(target_conf) + builder_cfg.knowledge = [ + 'config/' + f.split('/')[-1] for f in builder_cfg.knowledge + ] + with open(target_conf, 'w') as f: + json.dump(builder_cfg.to_dict(), f, indent=2, ensure_ascii=False) + + # 复制config目录下所有.json文件到new_directory/config + config_path = f'{local_file}/config' + new_config_path = f'{new_directory}/config' + + def find_json_and_images(directory): + # 确保路径以斜杠结束 + directory = os.path.join(directory, '') + + # 找到所有的JSON文件 + json_files = [ + os.path.join(directory, 'model_config.json'), + os.path.join(directory, 'tool_config.json'), + ] + + # 找到所有的图片文件 + image_files = glob.glob(directory + '*.png') + \ + glob.glob(directory + '*.jpg') + \ + glob.glob(directory + '*.jpeg') + \ + glob.glob(directory + '*.gif') # 根据需要可以添加更多图片格式 + + return json_files + image_files + + for f in find_json_and_images(config_path): + shutil.copy(f, new_config_path) + + # 复制assets目录到new_directory + assets_path = f'{local_file}/assets' + new_assets_path = f'{new_directory}/assets' + shutil.copytree(assets_path, new_assets_path, dirs_exist_ok=True) + + # 在requirements.txt中添加新的行 + requirements_file = f'{local_file}/requirements.txt' + new_requirements_file = f'{new_directory}/requirements.txt' + modelscope_agent_pkg = DEFAULT_MS_PKG.replace('version', MS_VERSION) + with open(requirements_file, 'r') as file: + content = file.readlines() + with open(new_requirements_file, 'w') as file: + file.write(modelscope_agent_pkg + '\n') + file.writelines(content) + + # 复制.py文件到新目录 + for file in os.listdir(local_file): + if file.endswith('.py'): + shutil.copy(f'{local_file}/{file}', new_directory) + + # 打包新目录 + archive_path = shutil.make_archive(new_directory, 'zip', new_directory) + + # 使用抽象出的函数上传到OSS并设置权限 + file_url = upload_to_oss(bucket, archive_path, + f'agents/user/{uuid_str}/{agent_name}.zip') + + shutil.rmtree(new_directory) + + return file_url + + +if __name__ == '__main__': + src_dir = os.path.abspath(os.path.dirname(__file__)) + url = prepare_agent_zip('test', src_dir, 'local_user', {}) + print(url) diff --git a/agentfabric/requirements.txt b/agentfabric/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..97eef4295a770bdc53682fa213586277ab65ce84 --- /dev/null +++ b/agentfabric/requirements.txt @@ -0,0 +1,9 @@ +dashscope +faiss-cpu +gradio==3.47.1 +langchain +markdown-cjk-spacing +markdown_katex +mdx_truly_sane_lists +pymdown-extensions +python-slugify diff --git a/agentfabric/response.json b/agentfabric/response.json new file mode 100644 index 0000000000000000000000000000000000000000..9321a2a37add140f77d68b085439e3a5afd996cb --- /dev/null +++ b/agentfabric/response.json @@ -0,0 +1 @@ +{"status_code": 500, "request_id": "0e8e65da-ee20-9c49-920a-94ca1df6ec09", "code": "InternalError.Algo", "message": "InternalError.Algo", "output": null, "usage": null} diff --git a/agentfabric/user_core.py b/agentfabric/user_core.py new file mode 100644 index 0000000000000000000000000000000000000000..4eaeb2f8ac79b2c40c3080aef5aed68b66978088 --- /dev/null +++ b/agentfabric/user_core.py @@ -0,0 +1,93 @@ +import copy +import os + +import gradio as gr +from config_utils import parse_configuration +from custom_prompt import (DEFAULT_EXEC_TEMPLATE, DEFAULT_SYSTEM_TEMPLATE, + DEFAULT_USER_TEMPLATE, CustomPromptGenerator, + parse_role_config) +from langchain.embeddings import ModelScopeEmbeddings +from langchain.vectorstores import FAISS +from modelscope_agent.agent import AgentExecutor +from modelscope_agent.agent_types import AgentType +from modelscope_agent.llm import LLMFactory +from modelscope_agent.retrieve import KnowledgeRetrieval +from modelscope_agent.tools.openapi_plugin import OpenAPIPluginTool + + +# init user chatbot_agent +def init_user_chatbot_agent(uuid_str=''): + builder_cfg, model_cfg, tool_cfg, available_tool_list, plugin_cfg, available_plugin_list = parse_configuration( + uuid_str) + # set top_p and stop_words for role play + model_cfg[builder_cfg.model]['generate_cfg']['top_p'] = 0.5 + model_cfg[builder_cfg.model]['generate_cfg']['stop'] = 'Observation' + + # build model + print(f'using model {builder_cfg.model}') + print(f'model config {model_cfg[builder_cfg.model]}') + + # # check configuration + # if builder_cfg.model in ['qwen-max', 'qwen-72b-api', 'qwen-14b-api', 'qwen-plus']: + # if 'DASHSCOPE_API_KEY' not in os.environ: + # raise gr.Error('DASHSCOPE_API_KEY should be set via setting environment variable') + + try: + llm = LLMFactory.build_llm(builder_cfg.model, model_cfg) + except Exception as e: + raise gr.Error(str(e)) + + # build prompt with zero shot react template + instruction_template = parse_role_config(builder_cfg) + prompt_generator = CustomPromptGenerator( + system_template=DEFAULT_SYSTEM_TEMPLATE, + user_template=DEFAULT_USER_TEMPLATE, + exec_template=DEFAULT_EXEC_TEMPLATE, + instruction_template=instruction_template, + add_addition_round=True, + addition_assistant_reply='好的。', + knowledge_file_name=os.path.basename(builder_cfg.knowledge[0] if len( + builder_cfg.knowledge) > 0 else ''), + llm=llm, + uuid_str=uuid_str) + + # get knowledge + # 开源版本的向量库配置 + model_id = 'damo/nlp_gte_sentence-embedding_chinese-base' + embeddings = ModelScopeEmbeddings(model_id=model_id) + available_knowledge_list = [] + for item in builder_cfg.knowledge: + # if isfile and end with .txt, .md, .pdf, support only those file + if os.path.isfile(item) and item.endswith(('.txt', '.md', '.pdf')): + available_knowledge_list.append(item) + if len(available_knowledge_list) > 0: + knowledge_retrieval = KnowledgeRetrieval.from_file( + available_knowledge_list, embeddings, FAISS) + else: + knowledge_retrieval = None + + additional_tool_list = add_openapi_plugin_to_additional_tool( + plugin_cfg, available_plugin_list) + # build agent + agent = AgentExecutor( + llm, + additional_tool_list=additional_tool_list, + tool_cfg=tool_cfg, + agent_type=AgentType.MRKL, + prompt_generator=prompt_generator, + knowledge_retrieval=knowledge_retrieval, + tool_retrieval=False) + agent.set_available_tools(available_tool_list + available_plugin_list) + return agent + + +def add_openapi_plugin_to_additional_tool(plugin_cfgs, available_plugin_list): + additional_tool_list = {} + for name, cfg in plugin_cfgs.items(): + openapi_plugin_object = OpenAPIPluginTool(name=name, cfg=plugin_cfgs) + additional_tool_list[name] = openapi_plugin_object + return additional_tool_list + + +def user_chatbot_single_run(query, agent): + agent.run(query) diff --git a/agentfabric/version.py b/agentfabric/version.py new file mode 100644 index 0000000000000000000000000000000000000000..66a87bb6e9111e056b787a60fc91234ac3cd7b3c --- /dev/null +++ b/agentfabric/version.py @@ -0,0 +1 @@ +__version__ = '0.1.5' diff --git a/msgpt/app.py b/msgpt/app.py new file mode 100644 index 0000000000000000000000000000000000000000..7e6cde0c996dc792068388aad78beab51f3ced6b --- /dev/null +++ b/msgpt/app.py @@ -0,0 +1,163 @@ +from __future__ import annotations +import os +import sys +from functools import partial + +import gradio as gr +from dotenv import load_dotenv +from gradio_chatbot import ChatBot +from modelscope_agent.agent import AgentExecutor +from modelscope_agent.llm import LLMFactory +from predict import stream_predict, upload_image + +from modelscope.utils.config import Config + +load_dotenv('../../config/.env', override=True) + +os.environ['TOOL_CONFIG_FILE'] = '../../config/cfg_tool_template.json' +os.environ['MODEL_CONFIG_FILE'] = '../../config/cfg_model_template.json' +os.environ['OUTPUT_FILE_DIRECTORY'] = './tmp' + +with open( + os.path.join(os.path.dirname(__file__), 'main.css'), 'r', + encoding='utf-8') as f: + MAIN_CSS_CODE = f.read() + +with gr.Blocks(css=MAIN_CSS_CODE, theme=gr.themes.Soft()) as demo: + + upload_image_url = gr.State('') + + # agent 对象 + tool_cfg_file = os.getenv('TOOL_CONFIG_FILE') + model_cfg_file = os.getenv('MODEL_CONFIG_FILE') + + model_cfg = Config.from_file(model_cfg_file) + tool_cfg = Config.from_file(tool_cfg_file) + + model_name = 'modelscope-agent-qwen-7b' + + llm = LLMFactory.build_llm(model_name, model_cfg) + agent = AgentExecutor(llm, tool_cfg) + + stream_predict_p = partial(stream_predict, agent=agent) + + with gr.Row(): + gr.HTML( + """

ModelScopeGPT

""" + ) + status_display = gr.HTML( + '', elem_id='status_display', visible=False, show_label=False) + + with gr.Row(elem_id='container_row').style(equal_height=True): + + with gr.Column( + scale=8, + elem_classes=['chatInterface', 'chatDialog', 'chatContent']): + with gr.Row(elem_id='chat-container'): + chatbot = ChatBot( + elem_id='chatbot', + elem_classes=['markdown-body'], + show_label=False) + chatbot_classic = gr.Textbox( + lines=20, + visible=False, + interactive=False, + label='classic_chatbot', + elem_id='chatbot_classic') + with gr.Row(elem_id='chat-bottom-container'): + with gr.Column(min_width=70, scale=1): + clear_session_button = gr.Button( + '清除', elem_id='clear_session_button') + with gr.Column(min_width=100, scale=1): + upload_button = gr.UploadButton( + '上传图片', file_types=['image']) + with gr.Column(scale=12): + user_input = gr.Textbox( + show_label=False, + placeholder='和我聊聊吧～', + elem_id='chat-input').style(container=False) + uploaded_image_box = gr.HTML( + '', visible=False, show_label=False) + with gr.Column(min_width=70, scale=1): + submitBtn = gr.Button('发送', variant='primary') + with gr.Column(min_width=110, scale=1): + regenerate_button = gr.Button( + '重新生成', elem_id='regenerate_button') + + with gr.Column(min_width=470, scale=4, elem_id='settings'): + icon_path = 'https://img.alicdn.com/imgextra/i4/O1CN01kpkVcX1wSCO362MH4_!!6000000006306-1-tps-805-805.gif' + info_text = '我是ModelScopeGPT（魔搭GPT）， 是一个大小模型协同的agent系统。\ + 我具备多种能力，可以通过大模型做中枢（controller），来控制魔搭社区的各种多模态模型api回复用户的问题。\ + 除此之外，我还集成了知识库检索引擎，可以解答用户在魔搭社区使用模型遇到的问题以及模型知识相关问答。' + + gr.HTML(f""" +
+ +
+ "{info_text}" +
+
+ """) + + gr.Examples( + examples=[ + '写一首简短的夏天落日的诗', + '讲一个小男孩的故事，20字左右', + '用男声读出来', + '生成个图片看看', + '从下面的地址，找到省市区等元素，地址：浙江杭州市江干区九堡镇三村村一区', + ], + inputs=[user_input], + examples_per_page=20, + label='示例', + elem_id='chat-examples') + + stream_predict_input = [chatbot, user_input, upload_image_url] + stream_predict_output = [chatbot, status_display] + + clean_outputs = [gr.update(value=''), ''] + clean_outputs_target = [user_input, uploaded_image_box] + + user_input.submit( + stream_predict_p, + stream_predict_input, + stream_predict_output, + show_progress=True) + user_input.submit( + fn=lambda: clean_outputs, inputs=[], outputs=clean_outputs_target) + + submitBtn.click( + stream_predict_p, + stream_predict_input, + stream_predict_output, + show_progress=True) + submitBtn.click( + fn=lambda: clean_outputs, inputs=[], outputs=clean_outputs_target) + + regenerate_button.click( + fn=lambda: clean_outputs, inputs=[], outputs=clean_outputs_target) + regenerate_button.click( + stream_predict_p, + stream_predict_input, + stream_predict_output, + show_progress=True) + + upload_button.upload(upload_image, upload_button, + [uploaded_image_box, upload_image_url]) + + def clear_session(): + agent.reset() + return { + chatbot: gr.update(value=[]), + uploaded_image_box: '', + upload_image_url: '', + } + + clear_session_button.click( + fn=clear_session, + inputs=[], + outputs=[chatbot, uploaded_image_box, upload_image_url]) + + demo.title = 'ModelScopeGPT 🎁' + demo.queue(concurrency_count=10, status_update_rate='auto', api_open=False) + demo.launch(show_api=False, share=False) diff --git a/msgpt/gradio_chatbot.py b/msgpt/gradio_chatbot.py new file mode 100644 index 0000000000000000000000000000000000000000..7316ad008e990ee116baf8e66b8074cdea5f8fb6 --- /dev/null +++ b/msgpt/gradio_chatbot.py @@ -0,0 +1,219 @@ +from __future__ import annotations +import ast +import html +import re +import traceback +from typing import List, Tuple + +import gradio as gr +import json +import markdown +from gradio.components import Chatbot as ChatBotBase + +ALREADY_CONVERTED_MARK = '' + + +class ChatBot(ChatBotBase): + + def normalize_markdown(self, bot_message, remove_media=False): + + if remove_media: + media_regex = r'(!\[[^\]]*\]\([^)]+\)|]+>.*?)\n*' + # 使用正则表达式进行替换 + bot_message = re.sub(media_regex, '', bot_message) + + lines = bot_message.split('\n') + normalized_lines = [] + inside_list = False + + for i, line in enumerate(lines): + if re.match(r'^(\d+\.|-|\*|\+)\s', line.strip()): + if not inside_list and i > 0 and lines[i - 1].strip() != '': + normalized_lines.append('') + inside_list = True + normalized_lines.append(line) + elif inside_list and line.strip() == '': + if i < len(lines) - 1 and not re.match(r'^(\d+\.|-|\*|\+)\s', + lines[i + 1].strip()): + normalized_lines.append(line) + continue + else: + inside_list = False + normalized_lines.append(line) + + return '\n'.join(normalized_lines) + + def convert_markdown(self, bot_message, remove_media=False): + if bot_message.count('```') % 2 != 0: + bot_message += '\n```' + + bot_message = self.normalize_markdown(bot_message, remove_media) + + result = markdown.markdown( + bot_message, + extensions=[ + 'toc', 'extra', 'tables', 'markdown_katex', 'codehilite', + 'mdx_truly_sane_lists', 'markdown_cjk_spacing.cjk_spacing', + 'pymdownx.magiclink' + ], + extension_configs={ + 'markdown_katex': { + 'no_inline_svg': True, # fix for WeasyPrint + 'insert_fonts_css': True, + }, + 'codehilite': { + 'linenums': False, + 'guess_lang': True + }, + 'mdx_truly_sane_lists': { + 'nested_indent': 2, + 'truly_sane': True, + } + }) + result = ''.join(result) + return result + + def convert_bot_message(self, bot_message): + + # 兼容老格式 + chunks = bot_message.split('') + if len(chunks) > 1: + new_bot_message = '' + for idx, chunk in enumerate(chunks): + new_bot_message += chunk + if idx % 2 == 0: + if idx != len(chunks) - 1: + new_bot_message += '<|startofthink|>' + else: + new_bot_message += '<|endofthink|>' + + bot_message = new_bot_message + + start_pos = 0 + result = '' + find_json_pattern = re.compile(r'{[\s\S]+}') + START_OF_THINK_TAG, END_OF_THINK_TAG = '<|startofthink|>', '<|endofthink|>' + START_OF_EXEC_TAG, END_OF_EXEC_TAG = '<|startofexec|>', '<|endofexec|>' + while start_pos < len(bot_message): + try: + start_of_think_pos = bot_message.index(START_OF_THINK_TAG, + start_pos) + end_of_think_pos = bot_message.index(END_OF_THINK_TAG, + start_pos) + if start_pos < start_of_think_pos: + result += self.convert_markdown( + bot_message[start_pos:start_of_think_pos]) + think_content = bot_message[start_of_think_pos + + len(START_OF_THINK_TAG + ):end_of_think_pos].strip() + json_content = find_json_pattern.search(think_content) + think_content = json_content.group( + ) if json_content else think_content + try: + think_node = json.loads( + think_content.replace('\n', ''), strict=False) + plugin_name = think_node.get( + 'plugin_name', + think_node.get('plugin', + think_node.get('api_name', 'unknown'))) + summary = f'选择插件【{plugin_name}】' + think_node.pop('url', None) + detail = f'```json\n\n{json.dumps(think_node,indent=3,ensure_ascii=False)}\n\n```' + except Exception: + traceback.print_exc() + summary = '思考中...' + detail = think_content + # detail += traceback.format_exc() + result += '
' + summary + '' + self.convert_markdown( + detail) + '
' + start_pos = end_of_think_pos + len(END_OF_THINK_TAG) + except Exception: + # result += traceback.format_exc() + break + + try: + start_of_exec_pos = bot_message.index(START_OF_EXEC_TAG, + start_pos) + end_of_exec_pos = bot_message.index(END_OF_EXEC_TAG, start_pos) + if start_pos < start_of_exec_pos: + result += self.convert_markdown( + bot_message[start_pos:start_of_think_pos]) + exec_content = bot_message[start_of_exec_pos + + len(START_OF_EXEC_TAG + ):end_of_exec_pos].strip() + exec_content = self.process_exec_result(exec_content) + + # result += self.convert_markdown(exec_content) + summary = '执行结果' + result += '
' + summary + '' + self.convert_markdown( + exec_content) + '
' + start_pos = end_of_exec_pos + len(END_OF_EXEC_TAG) + except Exception: + # traceback.print_exc() + break + if start_pos < len(bot_message): + result += self.convert_markdown( + bot_message[start_pos:], remove_media=True) + result += ALREADY_CONVERTED_MARK + return result + + def postprocess( + self, message_pairs: List[Tuple[str | None, str | None]] + ) -> List[Tuple[str | None, str | None]]: + """ + Parameters: + y: List of tuples representing the message and response pairs. + Each message and response should be a string, which may be in Markdown format. + Returns: + List of tuples representing the message and response. Each message and response will be a string of HTML. + """ + if not message_pairs: + return [] + user_message, bot_message = message_pairs[-1] + if not user_message.endswith(ALREADY_CONVERTED_MARK): + user_message = f"

{self.convert_markdown(html.escape(user_message))}

"\ + + ALREADY_CONVERTED_MARK + if not bot_message.endswith(ALREADY_CONVERTED_MARK): + bot_message = self.convert_bot_message(bot_message) + message_pairs[-1] = (user_message, bot_message) + return message_pairs + + def process_exec_result(self, exec_result: str): + + exec_result = exec_result.replace("{'result': ", '') + exec_result = exec_result[:-1] + exec_result = exec_result.replace("'", "\"") + try: + exec_result = json.loads( + exec_result.replace('\n', ''), strict=False) + final_result = f'```json\n\n{exec_result}\n\n```' + return final_result + except Exception: + match_image = re.search(r'!\[IMAGEGEN\]\((.*?)\)', exec_result) + if match_image: + img_path = match_image.group(1) + + gr_img_path = self.transform_to_gr_file(img_path) + final_result = exec_result.replace(img_path, gr_img_path) + return final_result + + match_audio = re.search( + r'