Upload 2 files

app.py

@@ -1,133 +1,181 @@
-# Copyright: DAMO Academy, Alibaba Group
-# By Xuan Phi Nguyen at DAMO Academy, Alibaba Group
-
-# Description:
-"""
-Demo script to launch Language chat model 
-"""
-
-import spaces
-import os
-from gradio.themes import ThemeClass as Theme
-import numpy as np
-import argparse
-# import torch
 import gradio as gr
-from typing import Any, Iterator
-from typing import Iterator, List, Optional, Tuple
-import filelock
-import glob
-import json
 import time
-from gradio.routes import Request
-from gradio.utils import SyncToAsyncIterator, async_iteration
-from gradio.helpers import special_args
-import anyio
-from typing import AsyncGenerator, Callable, Literal, Union, cast
-
-from gradio_client.documentation import document, set_documentation_group
-
-from typing import List, Optional, Union, Dict, Tuple
-from tqdm.auto import tqdm
-from huggingface_hub import snapshot_download
-from langchain_community.embeddings import HuggingFaceEmbeddings, HuggingFaceBgeEmbeddings
-from gradio.components import Button, Component
-from gradio.events import Dependency, EventListenerMethod
-
-from multipurpose_chatbot.demos.base_demo import CustomTabbedInterface
-
-from multipurpose_chatbot.configs import (
-    MODEL_TITLE,
-    MODEL_DESC,
-    MODEL_INFO,
-    CITE_MARKDOWN,
-    ALLOWED_PATHS,
-    PROXY,
-    PORT,
-    MODEL_PATH,
-    MODEL_NAME,
-    BACKEND,
-    DEMOS,
-    DELETE_FOLDER,
-)
-
-
-demo = None
-
-
-
-if DELETE_FOLDER is not None and os.path.exists(DELETE_FOLDER):
-    print(F'WARNING deleting folder: {DELETE_FOLDER}')
-    import shutil
-    print(f'DELETE ALL FILES IN {DELETE_FOLDER}')
-    for filename in os.listdir(DELETE_FOLDER):
-        file_path = os.path.join(DELETE_FOLDER, filename)
-        try:
-            if os.path.isfile(file_path) or os.path.islink(file_path):
-                os.unlink(file_path)
-            elif os.path.isdir(file_path):
-                shutil.rmtree(file_path)
-            print(f'deleted: {file_path}')
-        except Exception as e:
-            print('Failed to delete %s. Reason: %s' % (file_path, e))
-
-
-def launch_demo():
-    global demo, MODEL_ENGINE
-    model_desc = MODEL_DESC
-    model_path = MODEL_PATH
-
-    print(f'Begin importing models')
-    from multipurpose_chatbot.demos import get_demo_class
-
-    # demos = {
-    #     k: get_demo_class(k)().create_demo()
-    #     for k in demo_and_tab_names.keys()
-    # }
-    print(f'{DEMOS=}')
-    demo_class_objects = {
-        k: get_demo_class(k)()
-        for k in DEMOS
-    }
-    demos = {
-        k: get_demo_class(k)().create_demo()
-        for k in DEMOS
-    }
-    demos_names = [x.tab_name for x in demo_class_objects.values()]
-
-    descriptions = model_desc
-    if MODEL_INFO is not None and MODEL_INFO != "":
-        descriptions += (
-            f"<br>" + 
-            MODEL_INFO.format(model_path=model_path)
-        )
-    if len(demos) == 1:
-        demo = demos[DEMOS[0]]
+from transformers import Qwen2AudioForConditionalGeneration, AutoProcessor
+from io import BytesIO
+from urllib.request import urlopen
+import librosa
+import os, json
+from sys import argv
+from vllm import LLM, SamplingParams
+
+def load_model_processor(model_path):
+    processor = AutoProcessor.from_pretrained(model_path)
+    llm = LLM(
+        model=model_path, trust_remote_code=True, gpu_memory_utilization=0.8,  
+        enforce_eager=True,  device = "cuda",
+        limit_mm_per_prompt={"audio": 5},
+    )
+    return llm, processor
+
+model_path1 = "Qwen/Qwen2-Audio-7B-Instruct" #argv[1]
+model1, processor1 = load_model_processor(model_path1)
+
+def response_to_audio_conv(conversation, model=None, processor=None, temperature = 0.1,repetition_penalty=1.1, top_p = 0.9,
+                    max_new_tokens = 2048):
+    text = processor.apply_chat_template(conversation, add_generation_prompt=True, tokenize=False)
+    audios = []
+    for message in conversation:
+        if isinstance(message["content"], list):
+            for ele in message["content"]:
+                if ele["type"] == "audio":
+                    if ele['audio_url'] != None:
+                        audios.append(librosa.load(
+                            ele['audio_url'], 
+                            sr=processor.feature_extractor.sampling_rate)[0]
+                        )
+
+    sampling_params = SamplingParams(
+        temperature=temperature, max_tokens=max_new_tokens, repetition_penalty=repetition_penalty, top_p=top_p, top_k=20,
+        stop_token_ids=[],
+    )
+
+    input = {
+            'prompt': text,
+            'multi_modal_data': {
+                'audio': [(audio, 16000) for audio in audios]
+            }
+            }
+
+    output = model.generate([input], sampling_params=sampling_params)[0]
+    response = output.outputs[0].text
+    return response
+
+def print_like_dislike(x: gr.LikeData):
+    print(x.index, x.value, x.liked)
+
+def add_message(history, message):
+    paths = []
+    for turn in history: 
+        if turn['role'] == "user" and type(turn['content']) != str:
+            paths.append(turn['content'][0])
+    for x in message["files"]:
+        if x not in paths:
+            history.append({"role": "user", "content": {"path": x}})
+    if message["text"] is not None:
+        history.append({"role": "user", "content": message["text"]})
+    return history, gr.MultimodalTextbox(value=None, interactive=False)
+
+def format_user_messgae(message):
+    if type(message['content']) == str:
+        return {"role": "user", "content": [{"type": "text", "text": message['content']}]}
     else:
-        demo = CustomTabbedInterface(
-            interface_list=list(demos.values()),
-            tab_names=demos_names,
-            title=f"{MODEL_TITLE}",
-            description=descriptions,
-        )
-
-    demo.title = MODEL_NAME
+        return {"role": "user", "content": [{"type": "audio", "audio_url": message['content'][0]}]}
+
+def history_to_conversation(history):
+    conversation = []
+    audio_paths = []
+    for turn in history:
+        if turn['role'] == "user":
+            if not turn['content']: 
+                continue
+            turn = format_user_messgae(turn)
+            if turn['content'][0]['type'] == 'audio':
+                if turn['content'][0]['audio_url'] in audio_paths:
+                    continue
+                else: 
+                    audio_paths.append(turn['content'][0]['audio_url'])
+
+            if len(conversation) > 0 and conversation[-1]["role"] == "user":
+                conversation[-1]['content'].append(turn['content'][0])
+            else:
+                conversation.append(turn)
+        else:
+            conversation.append(turn)
     
-    # with demo:
-    #     gr.Markdown(CITE_MARKDOWN)
+    print(json.dumps(conversation, indent=4, ensure_ascii=False))
+    return conversation
+
+def bot(history: list, temperature = 0.1,repetition_penalty=1.1, top_p = 0.9,
+                    max_new_tokens = 2048):
+    conversation = history_to_conversation(history)
+    response = response_to_audio_conv(conversation, model=model1, processor=processor1, temperature = temperature,repetition_penalty=repetition_penalty, top_p = top_p, max_new_tokens = max_new_tokens)
+    # response = "Nice to meet you!"
+    print("Bot:",response)
+
+    history.append({"role": "assistant", "content": ""})
+    for character in response:
+        history[-1]["content"] += character
+        time.sleep(0.01)
+        yield history
+
+insturctions = """**Instruction**: there are three input format: 
+    1. text: input text message only 
+    2. audio: upload audio file or record a voice message
+    3. audio + text: record a voice message and input text message"""
+
+with gr.Blocks() as demo:
+    # gr.Markdown("""<p align="center"><img src="images/seal_logo.png" style="height: 80px"/><p>""")
+    # gr.Image("images/seal_logo.png", elem_id="seal_logo", show_label=False,height=80,show_fullscreen_button=False)
+    gr.Markdown(
+        """<div style="text-align: center; font-size: 32px; font-weight: bold;">SeaLLMs-Audio ChatBot</div>""",
+    )
+
+    # Description text
+    gr.Markdown(
+        """<div style="text-align: center; font-size: 16px;">
+    This WebUI is based on SeaLLMs-Audio-7B-Chat, developed by Alibaba DAMO Academy.<br>
+    You can interact with the chatbot in <b>English, Chinese, Indonesian, Thai, or Vietnamese</b>.<br>
+    For each round, you can input <b>audio and/or text</b>.
+    </div>""",
+    )
+
+    # Links with proper formatting
+    gr.Markdown(
+        """<div style="text-align: center; font-size: 16px;">
+        <a href="https://huggingface.co/SeaLLMs/SeaLLMs-v3-7B-Chat">[Website]</a> &nbsp; 
+        <a href="https://huggingface.co/SeaLLMs/SeaLLMs-v3-7B-Chat">[Model🤗]</a> &nbsp; 
+        <a href="https://github.com/liuchaoqun/SeaLLMs-Audio">[Github]</a>
+        </div>""",
+    )
+
+    # gr.Markdown(insturctions)
+    # with gr.Row():
+    #     with gr.Column():
+    #         temperature = gr.Slider(minimum=0, maximum=1, value=0.3, step=0.1, label="Temperature")
+    #     with gr.Column():
+    #         top_p = gr.Slider(minimum=0.1, maximum=1, value=0.5, step=0.1, label="Top P")
+    #     with gr.Column():
+    #         repetition_penalty = gr.Slider(minimum=0, maximum=2, value=1.1, step=0.1, label="Repetition Penalty")
+    chatbot = gr.Chatbot(elem_id="chatbot", bubble_full_width=False, type="messages")
+
+    chat_input = gr.MultimodalTextbox(
+        interactive=True,
+        file_count="single",
+        file_types=['.wav'],
+        placeholder="Enter message (optional) ...",
+        show_label=False,
+        sources=["microphone", "upload"],
+    )
+
+    chat_msg = chat_input.submit(
+        add_message, [chatbot, chat_input], [chatbot, chat_input]
+    )
+    bot_msg = chat_msg.then(bot, chatbot, chatbot, api_name="bot_response")
+    # bot_msg = chat_msg.then(bot, [chatbot, temperature, repetition_penalty, top_p], chatbot, api_name="bot_response")
+    bot_msg.then(lambda: gr.MultimodalTextbox(interactive=True), None, [chat_input])
+
+    # chatbot.like(print_like_dislike, None, None, like_user_message=True)
+
+    clear_button = gr.ClearButton([chatbot, chat_input])
         
-    demo.queue(api_open=False)
-    return demo
-
-
-
-if __name__ == "__main__":
-    demo = launch_demo()
-    if PROXY is not None and PROXY != "":
-        print(f'{PROXY=} {PORT=}')
-        print(f"{ALLOWED_PATHS=}")
-        demo.launch(server_port=PORT, root_path=PROXY, show_api=False, allowed_paths=ALLOWED_PATHS)
-    else:
-        demo.launch(server_port=PORT, show_api=False, allowed_paths=ALLOWED_PATHS)
-
-
+# PORT = 7950
+# demo.launch(server_port=PORT, show_api = True, allowed_paths = [], 
+#     root_path = f"https://dsw-gateway.alibaba-inc.com/dsw81322/proxy/{PORT}/")
+
+demo.launch(
+    share=False,
+    inbrowser=True,
+    server_port=7950,
+    server_name="0.0.0.0",
+    max_threads=40
+)

requirements.txt

@@ -1,31 +1,178 @@
-spaces
-torch
-gradio
-tiktoken
-openai
-transformers==4.38
-langchain
-langchain-community
-langchain-core
-chromadb
-pypdf
-docx2txt
-sentencepiece
-accelerate
-evaluate
-datasets
-sacrebleu
-websockets
-omegaconf
-scikit-learn
-jiwer
-tenacity
-pynvml
-ninja
-fastapi
-geomloss
-einops
-langdetect
-plotly
-faiss-cpu
-sentence-transformers
+accelerate==1.2.1
+aiofiles==23.2.1
+aiohappyeyeballs==2.4.4
+aiohttp==3.11.11
+aiohttp-cors==0.7.0
+aiosignal==1.3.2
+airportsdata==20241001
+annotated-types==0.7.0
+anyio==4.7.0
+astor==0.8.1
+async-timeout==5.0.1
+attrs==24.3.0
+audioread==3.0.1
+blake3==1.0.4
+cachetools==5.5.1
+certifi==2024.12.14
+cffi==1.17.1
+charset-normalizer==3.4.0
+click==8.1.7
+cloudpickle==3.1.1
+colorful==0.5.6
+compressed-tensors==0.9.1
+cupy-cuda12x==13.3.0
+dashscope==1.20.14
+datasets==3.3.2
+depyf==0.18.0
+dill==0.3.8
+diskcache==5.6.3
+distlib==0.3.9
+distro==1.9.0
+dnspython==2.7.0
+einops==0.8.1
+email_validator==2.2.0
+fastapi==0.115.6
+fastapi-cli==0.0.7
+fastrlock==0.8.3
+ffmpy==0.4.0
+filelock==3.16.1
+frozenlist==1.5.0
+fsspec==2024.10.0
+gguf==0.10.0
+google-api-core==2.24.1
+google-auth==2.38.0
+googleapis-common-protos==1.67.0
+gradio==5.10.0
+gradio_client==1.5.3
+gradio_leaderboard==0.0.13
+grpcio==1.70.0
+h11==0.14.0
+httpcore==1.0.7
+httptools==0.6.4
+httpx==0.28.1
+huggingface-hub==0.27.0
+idna==3.10
+importlib_metadata==8.6.1
+iniconfig==2.0.0
+interegular==0.3.3
+Jinja2==3.1.4
+jiter==0.8.2
+joblib==1.4.2
+jsonschema==4.23.0
+jsonschema-specifications==2024.10.1
+lark==1.2.2
+lazy_loader==0.4
+librosa==0.10.2.post1
+llvmlite==0.43.0
+lm-format-enforcer==0.10.10
+markdown-it-py==3.0.0
+MarkupSafe==2.1.5
+mdurl==0.1.2
+mistral_common==1.5.3
+modelscope_studio==1.0.2
+mpmath==1.3.0
+msgpack==1.1.0
+msgspec==0.19.0
+multidict==6.1.0
+multiprocess==0.70.16
+networkx==3.4.2
+numba==0.60.0
+numpy==1.26.4
+nvidia-cublas-cu12==12.4.5.8
+nvidia-cuda-cupti-cu12==12.4.127
+nvidia-cuda-nvrtc-cu12==12.4.127
+nvidia-cuda-runtime-cu12==12.4.127
+nvidia-cudnn-cu12==9.1.0.70
+nvidia-cufft-cu12==11.2.1.3
+nvidia-curand-cu12==10.3.5.147
+nvidia-cusolver-cu12==11.6.1.9
+nvidia-cusparse-cu12==12.3.1.170
+nvidia-ml-py==12.570.86
+nvidia-nccl-cu12==2.21.5
+nvidia-nvjitlink-cu12==12.4.127
+nvidia-nvtx-cu12==12.4.127
+openai==1.63.0
+opencensus==0.11.4
+opencensus-context==0.1.3
+opencv-python-headless==4.11.0.86
+orjson==3.10.12
+outlines==0.1.11
+outlines_core==0.1.26
+pandas==2.2.3
+partial-json-parser==0.2.1.1.post5
+peft==0.14.0
+pillow==11.0.0
+pluggy==1.5.0
+pooch==1.8.2
+prometheus-fastapi-instrumentator==7.0.2
+prometheus_client==0.21.1
+propcache==0.2.1
+proto-plus==1.26.0
+protobuf==5.29.3
+py-cpuinfo==9.0.0
+py-spy==0.4.0
+pyarrow==19.0.1
+pyasn1==0.6.1
+pyasn1_modules==0.4.1
+pybind11==2.13.6
+pycountry==24.6.1
+pycparser==2.22
+pydantic==2.10.3
+pydantic_core==2.27.1
+pydub==0.25.1
+Pygments==2.18.0
+pytest==8.3.4
+python-dotenv==1.0.1
+python-multipart==0.0.20
+pytz==2024.2
+PyYAML==6.0.2
+ray==2.40.0
+referencing==0.36.2
+regex==2024.11.6
+requests==2.32.3
+rich==13.9.4
+rich-toolkit==0.13.2
+rpds-py==0.22.3
+rsa==4.9
+ruff==0.8.4
+safehttpx==0.1.6
+safetensors==0.4.5
+scikit-learn==1.6.0
+scipy==1.14.1
+semantic-version==2.10.0
+sentencepiece==0.2.0
+shellingham==1.5.4
+smart-open==7.1.0
+sniffio==1.3.1
+soundfile==0.12.1
+soxr==0.5.0.post1
+starlette==0.41.3
+sympy==1.13.1
+tenacity==9.0.0
+threadpoolctl==3.5.0
+tiktoken==0.9.0
+tokenizers==0.21.0
+tomli==2.2.1
+tomlkit==0.13.2
+torch==2.5.1
+torchaudio==2.5.1
+torchvision==0.20.1
+tqdm==4.67.1
+transformers==4.48.3
+triton==3.1.0
+typer==0.15.1
+tzdata==2024.2
+urllib3==2.2.3
+uvicorn==0.34.0
+uvloop==0.21.0
+virtualenv==20.29.2
+vllm==0.7.3
+watchfiles==1.0.4
+websocket-client==1.8.0
+websockets==14.1
+wrapt==1.17.2
+xformers==0.0.28.post3
+xgrammar==0.1.11
+xxhash==3.5.0
+yarl==1.18.3
+zipp==3.21.0