Instructions to use MERaLiON/MERaLiON-2-10B with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use MERaLiON/MERaLiON-2-10B with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("automatic-speech-recognition", model="MERaLiON/MERaLiON-2-10B", trust_remote_code=True)# pip install -U transformers accelerate # Load model directly from transformers import AutoModelForSpeechSeq2Seq model = AutoModelForSpeechSeq2Seq.from_pretrained("MERaLiON/MERaLiON-2-10B", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
init vllm
Browse files- vllm_plugin_meralion2/offline_example.py +52 -0
- vllm_plugin_meralion2/openai_client_curl.sh +21 -0
- vllm_plugin_meralion2/openai_client_example.py +69 -0
- vllm_plugin_meralion2/openai_serve_example.sh +6 -0
- vllm_plugin_meralion2/readme.md +35 -0
- vllm_plugin_meralion2/setup.py +10 -0
- vllm_plugin_meralion2/vllm_plugin_meralion2/__init__.py +46 -0
- vllm_plugin_meralion2/vllm_plugin_meralion2/transformers_utils/__init__.py +0 -0
- vllm_plugin_meralion2/vllm_plugin_meralion2/transformers_utils/configuration_meralion2.py +76 -0
- vllm_plugin_meralion2/vllm_plugin_meralion2/transformers_utils/modules.py +72 -0
- vllm_plugin_meralion2/vllm_plugin_meralion2/transformers_utils/no_repeat_logits_processor.py +98 -0
- vllm_plugin_meralion2/vllm_plugin_meralion2/transformers_utils/processing_meralion2.py +193 -0
- vllm_plugin_meralion2/vllm_plugin_meralion2/vllm064_post1.py +453 -0
- vllm_plugin_meralion2/vllm_plugin_meralion2/vllm085.py +333 -0
vllm_plugin_meralion2/offline_example.py
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import torch
|
| 3 |
+
|
| 4 |
+
from vllm import ModelRegistry, LLM, SamplingParams
|
| 5 |
+
from vllm.assets.audio import AudioAsset
|
| 6 |
+
|
| 7 |
+
from vllm_plugin_meralion2 import NoRepeatNGramLogitsProcessor
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
model_name = "MERaLiON/MERaLiON-2-10B"
|
| 11 |
+
# model_name = "MERaLiON/MERaLiON-2-10B-ASR"
|
| 12 |
+
# model_name = "MERaLiON/MERaLiON-2-3B"
|
| 13 |
+
|
| 14 |
+
llm = LLM(model=model_name,
|
| 15 |
+
tokenizer=model_name,
|
| 16 |
+
limit_mm_per_prompt={"audio": 1},
|
| 17 |
+
trust_remote_code=True,
|
| 18 |
+
dtype=torch.bfloat16
|
| 19 |
+
)
|
| 20 |
+
|
| 21 |
+
audio_asset = AudioAsset("mary_had_lamb")
|
| 22 |
+
|
| 23 |
+
question= "Please trancribe this speech."
|
| 24 |
+
|
| 25 |
+
prompt = (
|
| 26 |
+
"<start_of_turn>user\n"
|
| 27 |
+
f"Instruction: {question} \nFollow the text instruction based on the following audio: <SpeechHere><end_of_turn>\n"
|
| 28 |
+
"<start_of_turn>model\n")
|
| 29 |
+
|
| 30 |
+
sampling_params = SamplingParams(
|
| 31 |
+
temperature=0.0,
|
| 32 |
+
top_p=0.9,
|
| 33 |
+
top_k=50,
|
| 34 |
+
repetition_penalty=1.1,
|
| 35 |
+
seed=42,
|
| 36 |
+
max_tokens=1024,
|
| 37 |
+
stop_token_ids=None,
|
| 38 |
+
logits_processors=[NoRepeatNGramLogitsProcessor(2)]
|
| 39 |
+
)
|
| 40 |
+
|
| 41 |
+
mm_data = {"audio": [audio_asset.audio_and_sample_rate]}
|
| 42 |
+
inputs = {"prompt": prompt, "multi_modal_data": mm_data}
|
| 43 |
+
|
| 44 |
+
# batch inference
|
| 45 |
+
inputs = [inputs] * 2
|
| 46 |
+
|
| 47 |
+
outputs = llm.generate(inputs, sampling_params=sampling_params)
|
| 48 |
+
|
| 49 |
+
for o in outputs:
|
| 50 |
+
generated_text = o.outputs[0].text
|
| 51 |
+
print(generated_text)
|
| 52 |
+
|
vllm_plugin_meralion2/openai_client_curl.sh
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
curl http://localhost:8000/v1/chat/completions \
|
| 2 |
+
-H "Content-Type: application/json" \
|
| 3 |
+
-d '{
|
| 4 |
+
"model": "MERaLiON/MERaLiON-2-10B",
|
| 5 |
+
"messages": [
|
| 6 |
+
{"role": "user",
|
| 7 |
+
"content": [
|
| 8 |
+
{"type": "text", "text": "Instruction: {YOUR-INSTRUCTION-HERE} \nFollow the text instruction based on the following audio: <SpeechHere>"},
|
| 9 |
+
{"type": "audio_url", "audio_url": {"url": "data:audio/ogg;base64,{YOUR-AUDIO-BASE64-STRING}"}}
|
| 10 |
+
]
|
| 11 |
+
}
|
| 12 |
+
],
|
| 13 |
+
"max_completion_tokens": 1024,
|
| 14 |
+
"temperature": 0.0,
|
| 15 |
+
"top_p": 0.9,
|
| 16 |
+
"repetition_penalty": 1.1,
|
| 17 |
+
"top_k": 50,
|
| 18 |
+
"length_penalty": 1.0,
|
| 19 |
+
"logits_processors": [{"qualname": "vllm_plugin_meralion2.NoRepeatNGramLogitsProcessor", "args": []}],
|
| 20 |
+
"seed": 42
|
| 21 |
+
}'
|
vllm_plugin_meralion2/openai_client_example.py
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import base64
|
| 2 |
+
|
| 3 |
+
from openai import OpenAI
|
| 4 |
+
|
| 5 |
+
prompt_template = "Instruction: {text_input} \nFollow the text instruction based on the following audio: <SpeechHere>"
|
| 6 |
+
|
| 7 |
+
def get_client(api_key="EMPTY", base_url="http://localhost:8000/v1"):
|
| 8 |
+
client = OpenAI(
|
| 9 |
+
api_key=api_key,
|
| 10 |
+
base_url=base_url,
|
| 11 |
+
)
|
| 12 |
+
|
| 13 |
+
models = client.models.list()
|
| 14 |
+
model_name = models.data[0].id
|
| 15 |
+
return client, model_name
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def get_response(text_input, base64_audio_input=None, **params):
|
| 19 |
+
if base64_audio_input:
|
| 20 |
+
content = [
|
| 21 |
+
{
|
| 22 |
+
"type": "text",
|
| 23 |
+
"text": prompt_template.format(text_input=text_input)
|
| 24 |
+
},
|
| 25 |
+
{
|
| 26 |
+
"type": "audio_url",
|
| 27 |
+
"audio_url": {
|
| 28 |
+
"url": f"data:audio/ogg;base64,{base64_audio_input}"
|
| 29 |
+
},
|
| 30 |
+
},
|
| 31 |
+
]
|
| 32 |
+
else:
|
| 33 |
+
content = text_input
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
response_obj = client.chat.completions.create(
|
| 37 |
+
messages=[{
|
| 38 |
+
"role": "user",
|
| 39 |
+
"content": content,
|
| 40 |
+
}],
|
| 41 |
+
**params
|
| 42 |
+
)
|
| 43 |
+
return response_obj
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
audio_bytes = open("/path/to/wav/or/mp3/file", "rb").read()
|
| 47 |
+
audio_base64 = base64.b64encode(audio_bytes).decode('utf-8')
|
| 48 |
+
|
| 49 |
+
# use the port number of your vllm service.
|
| 50 |
+
client, model_name = get_client(base_url="http://localhost:8000/v1")
|
| 51 |
+
|
| 52 |
+
generation_parameters = dict(
|
| 53 |
+
model=model_name,
|
| 54 |
+
max_completion_tokens=1024,
|
| 55 |
+
temperature=0.0,
|
| 56 |
+
top_p=0.9,
|
| 57 |
+
extra_body={
|
| 58 |
+
"repetition_penalty": 1.1,
|
| 59 |
+
"top_k": 50,
|
| 60 |
+
"length_penalty": 1.0,
|
| 61 |
+
"logits_processors": [
|
| 62 |
+
{"qualname": "vllm_plugin_meralion2.NoRepeatNGramLogitsProcessor", "args": []}
|
| 63 |
+
]
|
| 64 |
+
},
|
| 65 |
+
seed=42
|
| 66 |
+
)
|
| 67 |
+
|
| 68 |
+
response_obj = get_response("Please transcribe this speech.", audio_base64, **generation_parameters)
|
| 69 |
+
print(response_obj.choices[0].message.content)
|
vllm_plugin_meralion2/openai_serve_example.sh
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
vllm serve MERaLiON/MERaLiON-2-10B \
|
| 2 |
+
--trust-remote-code \
|
| 3 |
+
--dtype bfloat16 \
|
| 4 |
+
--gpu-memory-utilization 0.78 \
|
| 5 |
+
--logits-processor-pattern vllm_plugin_meralion2.NoRepeatNGramLogitsProcessor \
|
| 6 |
+
--port 8004
|
vllm_plugin_meralion2/readme.md
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
## MERaLiON2 vLLM Plugin
|
| 2 |
+
|
| 3 |
+
### Set up Environment
|
| 4 |
+
|
| 5 |
+
This vLLM plugin for MERaLiON2 requires transformers version `4.50.1`. It supports vLLM version `0.6.5` ~ `0.7.3` (V0 engine), and `0.8.5` ~ `0.8.5.post1` (V1 engine).
|
| 6 |
+
|
| 7 |
+
```bash
|
| 8 |
+
pip install transformers==4.50.1
|
| 9 |
+
pip install vllm==0.6.5
|
| 10 |
+
```
|
| 11 |
+
|
| 12 |
+
Install the MERaLiON2 vLLM plugin.
|
| 13 |
+
|
| 14 |
+
```bash
|
| 15 |
+
python install .
|
| 16 |
+
```
|
| 17 |
+
|
| 18 |
+
It's strongly recommended to install flash-attn for better memory and gpu utilization.
|
| 19 |
+
|
| 20 |
+
```bash
|
| 21 |
+
pip install flash-attn --no-build-isolation
|
| 22 |
+
```
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
### Offline Inference
|
| 26 |
+
|
| 27 |
+
Refer to [offline_example.py](offline_example.py) for offline inference example.
|
| 28 |
+
|
| 29 |
+
### OpenAI-compatible Serving
|
| 30 |
+
|
| 31 |
+
Refer to [openai_serve_example.sh](openai_serve_example.sh) for openAI-compatible serving example.
|
| 32 |
+
|
| 33 |
+
To call the server, you can refer to [openai_client_example.py](openai_client_example.py).
|
| 34 |
+
|
| 35 |
+
Alternatively, you can try calling the server with curl, refer to [openai_client_curl.sh](openai_client_curl.sh).
|
vllm_plugin_meralion2/setup.py
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from setuptools import setup, find_packages
|
| 2 |
+
|
| 3 |
+
setup(name='vllm_plugin_meralion2',
|
| 4 |
+
version='0.1.0',
|
| 5 |
+
package_dir={"": "src"},
|
| 6 |
+
packages=find_packages(where="src"),
|
| 7 |
+
entry_points={
|
| 8 |
+
'vllm.general_plugins':
|
| 9 |
+
["register_dummy_model = vllm_plugin_meralion2:register"]
|
| 10 |
+
})
|
vllm_plugin_meralion2/vllm_plugin_meralion2/__init__.py
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Optional
|
| 2 |
+
|
| 3 |
+
from vllm.entrypoints.chat_utils import BaseMultiModalItemTracker
|
| 4 |
+
|
| 5 |
+
from .transformers_utils.no_repeat_logits_processor import NoRepeatNGramLogitsProcessor
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
_original_placeholder_str = BaseMultiModalItemTracker._placeholder_str
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def custom_placeholder_str(self, modality,
|
| 12 |
+
current_count: int) -> Optional[str]:
|
| 13 |
+
hf_config = self._model_config.hf_config
|
| 14 |
+
model_type = hf_config.model_type
|
| 15 |
+
|
| 16 |
+
if modality == "audio" and model_type == "meralion2":
|
| 17 |
+
return "<SpeechHere>"
|
| 18 |
+
|
| 19 |
+
return _original_placeholder_str(self, modality=modality, current_count=current_count)
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def register():
|
| 23 |
+
import vllm
|
| 24 |
+
from vllm import ModelRegistry
|
| 25 |
+
|
| 26 |
+
v064_compatible_versions = ['0.6.5', '0.6.6', '0.6.6.post1', '0.7.0', '0.7.1', '0.7.2', '0.7.3']
|
| 27 |
+
v085_compatible_versions = ['0.8.5', '0.8.5.post1']
|
| 28 |
+
sorted_compatible_versions = sorted(v064_compatible_versions + v085_compatible_versions)
|
| 29 |
+
|
| 30 |
+
if vllm.__version__ in v064_compatible_versions:
|
| 31 |
+
from .vllm064_post1 import MERaLiON2ForConditionalGeneration
|
| 32 |
+
elif vllm.__version__ in v085_compatible_versions:
|
| 33 |
+
from .vllm085 import MERaLiON2ForConditionalGeneration
|
| 34 |
+
else:
|
| 35 |
+
raise Exception((
|
| 36 |
+
f"MERaLiON2 doesn't support vLLM version {vllm.__version__}."
|
| 37 |
+
f" Supported vLLM versions: {', '.join((sorted_compatible_versions))}"
|
| 38 |
+
))
|
| 39 |
+
|
| 40 |
+
if "MERaLiON2ForConditionalGeneration" not in ModelRegistry.get_supported_archs():
|
| 41 |
+
ModelRegistry.register_model(
|
| 42 |
+
"MERaLiON2ForConditionalGeneration",
|
| 43 |
+
MERaLiON2ForConditionalGeneration
|
| 44 |
+
)
|
| 45 |
+
|
| 46 |
+
vllm.entrypoints.chat_utils.BaseMultiModalItemTracker._placeholder_str = custom_placeholder_str
|
vllm_plugin_meralion2/vllm_plugin_meralion2/transformers_utils/__init__.py
ADDED
|
File without changes
|
vllm_plugin_meralion2/vllm_plugin_meralion2/transformers_utils/configuration_meralion2.py
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""MERaLiON2 model configuration"""
|
| 2 |
+
|
| 3 |
+
from transformers import Gemma2Config, WhisperConfig
|
| 4 |
+
from transformers.configuration_utils import PretrainedConfig
|
| 5 |
+
from transformers.utils import logging
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
logger = logging.get_logger(__name__)
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
class MERaLiON2Config(PretrainedConfig):
|
| 12 |
+
r"""
|
| 13 |
+
This is the configuration class to store the configuration of a [`MERaLiON2ForConditionalGeneration`]. It is used to instantiate an
|
| 14 |
+
MERaLiON2 model according to the specified arguments, defining the model architecture. Instantiating a configuration
|
| 15 |
+
with the defaults will yield a similar configuration to that of the MERaLiON2.
|
| 16 |
+
|
| 17 |
+
Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
|
| 18 |
+
documentation from [`PretrainedConfig`] for more information.
|
| 19 |
+
|
| 20 |
+
Args:
|
| 21 |
+
audio_config (`Union[AutoConfig, dict]`, *optional*, defaults to `CLIPVisionConfig`):
|
| 22 |
+
The config object or dictionary of the audio backbone.
|
| 23 |
+
text_config (`Union[AutoConfig, dict]`, *optional*, defaults to `LlamaConfig`):
|
| 24 |
+
The config object or dictionary of the text backbone.
|
| 25 |
+
audio_token_index (`int`, *optional*, defaults to 151646):
|
| 26 |
+
The image token index to encode the image prompt.
|
| 27 |
+
"""
|
| 28 |
+
|
| 29 |
+
model_type = "meralion2"
|
| 30 |
+
is_composition = False
|
| 31 |
+
|
| 32 |
+
def __init__(
|
| 33 |
+
self,
|
| 34 |
+
speech_config=None,
|
| 35 |
+
text_config=None,
|
| 36 |
+
speech_mlp_scale_factor=15,
|
| 37 |
+
speech_token_index=255999,
|
| 38 |
+
**kwargs,
|
| 39 |
+
):
|
| 40 |
+
|
| 41 |
+
if isinstance(speech_config, dict):
|
| 42 |
+
speech_config = WhisperConfig(**speech_config)
|
| 43 |
+
elif speech_config is None:
|
| 44 |
+
speech_config = WhisperConfig(
|
| 45 |
+
d_model=1280,
|
| 46 |
+
encoder_attention_heads=20,
|
| 47 |
+
encoder_ffn_dim=5120,
|
| 48 |
+
encoder_layerdrop=0.0,
|
| 49 |
+
encoder_layers=32,
|
| 50 |
+
num_mel_bins=128,
|
| 51 |
+
max_source_positions=1500,
|
| 52 |
+
scale_embedding=False,
|
| 53 |
+
activation_function="gelu",
|
| 54 |
+
)
|
| 55 |
+
|
| 56 |
+
self.speech_config = speech_config
|
| 57 |
+
|
| 58 |
+
if isinstance(text_config, dict):
|
| 59 |
+
text_config = Gemma2Config(**text_config)
|
| 60 |
+
elif text_config is None:
|
| 61 |
+
text_config = Gemma2Config()
|
| 62 |
+
|
| 63 |
+
self.text_config = text_config
|
| 64 |
+
|
| 65 |
+
self.speech_mlp_scale_factor = speech_mlp_scale_factor
|
| 66 |
+
self.speech_token_index = speech_token_index
|
| 67 |
+
|
| 68 |
+
self.sliding_window = self.text_config.sliding_window
|
| 69 |
+
self.hidden_size = self.text_config.hidden_size
|
| 70 |
+
self.num_attention_heads = self.text_config.num_attention_heads
|
| 71 |
+
self.num_hidden_layers = self.text_config.num_hidden_layers
|
| 72 |
+
self.num_key_value_heads = self.text_config.num_key_value_heads
|
| 73 |
+
self.head_dim = self.text_config.head_dim
|
| 74 |
+
self.intermediate_size = self.text_config.intermediate_size
|
| 75 |
+
|
| 76 |
+
super().__init__(**kwargs)
|
vllm_plugin_meralion2/vllm_plugin_meralion2/transformers_utils/modules.py
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Inference-only MERaLiON AudioLLM model compatible with HuggingFace weights."""
|
| 2 |
+
from typing import Any, Optional, Set, Tuple, TypedDict, Union, List
|
| 3 |
+
|
| 4 |
+
import torch
|
| 5 |
+
import torch.nn as nn
|
| 6 |
+
from transformers.utils.import_utils import is_torch_sdpa_available, is_flash_attn_2_available
|
| 7 |
+
|
| 8 |
+
# === Audio Inputs === #
|
| 9 |
+
class MERaLiON2Inputs(TypedDict):
|
| 10 |
+
input_features: torch.Tensor
|
| 11 |
+
"""Shape:
|
| 12 |
+
`(num_audios, num_mel_bins, 3000)`
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
+
feature_attention_mask: torch.Tensor
|
| 16 |
+
"""Shape: `(num_audios, 3000)`
|
| 17 |
+
"""
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
# === Audio Encoder === #
|
| 21 |
+
class MERaLiON2SpeechAudioAdaper(nn.Module):
|
| 22 |
+
def __init__(self, audio_hidden_size: int, text_hidden_size: int):
|
| 23 |
+
super(MERaLiON2SpeechAudioAdaper, self).__init__()
|
| 24 |
+
speech_mlp_scale_factor = 15
|
| 25 |
+
|
| 26 |
+
self.speech_mlp_scale_factor = speech_mlp_scale_factor
|
| 27 |
+
self.mlp_adapter = nn.Sequential(
|
| 28 |
+
nn.Linear(
|
| 29 |
+
in_features=audio_hidden_size * speech_mlp_scale_factor,
|
| 30 |
+
out_features=audio_hidden_size * 5,
|
| 31 |
+
),
|
| 32 |
+
nn.SiLU(),
|
| 33 |
+
)
|
| 34 |
+
|
| 35 |
+
self.gate_proj = nn.Linear(
|
| 36 |
+
in_features=audio_hidden_size * 5,
|
| 37 |
+
out_features=audio_hidden_size * 5,
|
| 38 |
+
)
|
| 39 |
+
|
| 40 |
+
self.pool_proj = nn.Linear(
|
| 41 |
+
in_features=audio_hidden_size * 5,
|
| 42 |
+
out_features=audio_hidden_size * 5,
|
| 43 |
+
)
|
| 44 |
+
self.act_fn = nn.SiLU()
|
| 45 |
+
self.out_proj = nn.Linear(
|
| 46 |
+
audio_hidden_size * 5,
|
| 47 |
+
text_hidden_size,
|
| 48 |
+
)
|
| 49 |
+
|
| 50 |
+
def forward(self, speech_embeds, **kwargs):
|
| 51 |
+
B, T, C = speech_embeds.shape
|
| 52 |
+
speech_embeds = self.mlp_adapter(
|
| 53 |
+
speech_embeds.reshape(
|
| 54 |
+
B,
|
| 55 |
+
T // self.speech_mlp_scale_factor,
|
| 56 |
+
C * self.speech_mlp_scale_factor,
|
| 57 |
+
)
|
| 58 |
+
)
|
| 59 |
+
speech_embeds = self.act_fn(self.gate_proj(speech_embeds)) * self.pool_proj(speech_embeds)
|
| 60 |
+
speech_embeds = self.out_proj(speech_embeds)
|
| 61 |
+
return speech_embeds
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def autoset_attn_implementation_for_whisper(config):
|
| 65 |
+
_implementation = "eager"
|
| 66 |
+
if is_torch_sdpa_available():
|
| 67 |
+
_implementation = "sdpa"
|
| 68 |
+
if is_flash_attn_2_available():
|
| 69 |
+
_implementation = "flash_attention_2"
|
| 70 |
+
|
| 71 |
+
config._attn_implementation = _implementation
|
| 72 |
+
return config
|
vllm_plugin_meralion2/vllm_plugin_meralion2/transformers_utils/no_repeat_logits_processor.py
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Iterable, List
|
| 2 |
+
|
| 3 |
+
import torch
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
def _get_ngrams(ngram_size: int, prev_input_ids: torch.Tensor, num_hypos: int):
|
| 7 |
+
"""
|
| 8 |
+
Assume ngram_size=2 and prev_input_ids=tensor([[40, 2883, 2712, 4346]]). The output of generated ngrams look like
|
| 9 |
+
this {(40,): [2883], (2883,): [2712], (2712,): [4346]}.
|
| 10 |
+
|
| 11 |
+
Args:
|
| 12 |
+
ngram_size (`int`):
|
| 13 |
+
The number sequential tokens taken as a group which may only occur once before being banned.
|
| 14 |
+
prev_input_ids (`torch.Tensor`):
|
| 15 |
+
Generated token ids for the current hypothesis.
|
| 16 |
+
num_hypos (`int`):
|
| 17 |
+
The number of hypotheses for which n-grams need to be generated.
|
| 18 |
+
|
| 19 |
+
Returns:
|
| 20 |
+
generated_ngrams (`dict`):
|
| 21 |
+
Dictionary of generated ngrams.
|
| 22 |
+
"""
|
| 23 |
+
# Initialize an empty list of dictionaries, one for each hypothesis (index) in the range of num_hypos
|
| 24 |
+
generated_ngrams = [{} for _ in range(num_hypos)]
|
| 25 |
+
for idx in range(num_hypos):
|
| 26 |
+
gen_tokens = prev_input_ids[idx].tolist()
|
| 27 |
+
generated_ngram = generated_ngrams[idx]
|
| 28 |
+
# Loop through each n-gram of size ngram_size in the list of tokens (gen_tokens)
|
| 29 |
+
for ngram in zip(*[gen_tokens[i:] for i in range(ngram_size)]):
|
| 30 |
+
prev_ngram_tuple = tuple(ngram[:-1])
|
| 31 |
+
generated_ngram[prev_ngram_tuple] = generated_ngram.get(prev_ngram_tuple, []) + [ngram[-1]]
|
| 32 |
+
return generated_ngrams
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def _get_generated_ngrams(banned_ngrams, prev_input_ids, ngram_size, cur_len):
|
| 36 |
+
"""
|
| 37 |
+
Determines the banned tokens for the current hypothesis based on previously generated n-grams.
|
| 38 |
+
|
| 39 |
+
Args:
|
| 40 |
+
banned_ngrams (`dict`):
|
| 41 |
+
A dictionary containing previously generated n-grams for each hypothesis.
|
| 42 |
+
prev_input_ids (`torch.Tensor`):
|
| 43 |
+
Generated token ids for the current hypothesis.
|
| 44 |
+
ngram_size (`int`):
|
| 45 |
+
The number sequential tokens taken as a group which may only occur once before being banned.
|
| 46 |
+
cur_len (`int`):
|
| 47 |
+
The current length of the token sequences for which the n-grams are being checked.
|
| 48 |
+
|
| 49 |
+
Returns:
|
| 50 |
+
List of tokens that are banned.
|
| 51 |
+
"""
|
| 52 |
+
# Before decoding the next token, prevent decoding of ngrams that have already appeared
|
| 53 |
+
start_idx = cur_len + 1 - ngram_size
|
| 54 |
+
ngram_idx = tuple(prev_input_ids[start_idx:cur_len].tolist())
|
| 55 |
+
return banned_ngrams.get(ngram_idx, [])
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def _calc_banned_ngram_tokens(
|
| 59 |
+
ngram_size: int, prev_input_ids: torch.Tensor, num_hypos: int, cur_len: int
|
| 60 |
+
) -> List[Iterable[int]]:
|
| 61 |
+
"""Copied from fairseq for no_repeat_ngram in beam_search"""
|
| 62 |
+
if cur_len + 1 < ngram_size:
|
| 63 |
+
# return no banned tokens if we haven't generated no_repeat_ngram_size tokens yet
|
| 64 |
+
return [[] for _ in range(num_hypos)]
|
| 65 |
+
generated_ngrams = _get_ngrams(ngram_size, prev_input_ids, num_hypos)
|
| 66 |
+
banned_tokens = [
|
| 67 |
+
_get_generated_ngrams(generated_ngrams[hypo_idx], prev_input_ids[hypo_idx], ngram_size, cur_len)
|
| 68 |
+
for hypo_idx in range(num_hypos)
|
| 69 |
+
]
|
| 70 |
+
return banned_tokens
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
class NoRepeatNGramLogitsProcessor:
|
| 74 |
+
|
| 75 |
+
def __init__(self, ngram_size: int=4):
|
| 76 |
+
if not isinstance(ngram_size, int) or ngram_size <= 0:
|
| 77 |
+
raise ValueError(f"`ngram_size` has to be a strictly positive integer, but is {ngram_size}")
|
| 78 |
+
self.ngram_size = ngram_size
|
| 79 |
+
|
| 80 |
+
def __call__(self, prompt_tokens_ids: tuple, past_tokens_ids: tuple, scores: torch.FloatTensor) -> torch.FloatTensor:
|
| 81 |
+
# score: [B, vocab_size]
|
| 82 |
+
# input_ids: [B, cur_len]
|
| 83 |
+
input_ids = prompt_tokens_ids + past_tokens_ids
|
| 84 |
+
if len(input_ids) < self.ngram_size:
|
| 85 |
+
return scores
|
| 86 |
+
|
| 87 |
+
if len(scores.shape) == 1:
|
| 88 |
+
scores = scores.reshape(1, -1)
|
| 89 |
+
|
| 90 |
+
num_batch_hypotheses = scores.shape[0]
|
| 91 |
+
input_ids = torch.LongTensor(input_ids).reshape(num_batch_hypotheses, -1)
|
| 92 |
+
cur_len = input_ids.shape[-1]
|
| 93 |
+
scores_processed = scores.clone()
|
| 94 |
+
banned_batch_tokens = _calc_banned_ngram_tokens(self.ngram_size, input_ids, num_batch_hypotheses, cur_len)
|
| 95 |
+
for i, banned_tokens in enumerate(banned_batch_tokens):
|
| 96 |
+
scores_processed[i, banned_tokens] = -float("inf")
|
| 97 |
+
|
| 98 |
+
return scores_processed
|
vllm_plugin_meralion2/vllm_plugin_meralion2/transformers_utils/processing_meralion2.py
ADDED
|
@@ -0,0 +1,193 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Processor class for MERaLiON2."""
|
| 2 |
+
|
| 3 |
+
from typing import List, Optional, Union
|
| 4 |
+
|
| 5 |
+
import numpy as np
|
| 6 |
+
|
| 7 |
+
from transformers.feature_extraction_utils import BatchFeature
|
| 8 |
+
from transformers.processing_utils import ProcessorMixin
|
| 9 |
+
from transformers.tokenization_utils_base import PaddingStrategy, PreTokenizedInput, TextInput
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
# copied from transformers.models.qwen2_audio.processing_qwen2_audio.Qwen2AudioProcessor
|
| 13 |
+
class MERaLiON2Processor(ProcessorMixin):
|
| 14 |
+
r"""
|
| 15 |
+
Constructs a MERaLiON2 processor which wraps a whisper feature extractor and a gemma tokenizer into a single processor.
|
| 16 |
+
|
| 17 |
+
[`MERaLiON2Processor`] offers all the functionalities of [`WhisperFeatureExtractor`] and [`GemmaTokenizer`]. See the
|
| 18 |
+
[`~MERaLiON2Processor.__call__`] and [`~MERaLiON2Processor.decode`] for more information.
|
| 19 |
+
|
| 20 |
+
Args:
|
| 21 |
+
feature_extractor ([`WhisperFeatureExtractor`], *optional*):
|
| 22 |
+
The feature extractor is a required input.
|
| 23 |
+
tokenizer ([`GemmaTokenizer`], *optional*):
|
| 24 |
+
The tokenizer is a required input.
|
| 25 |
+
chat_template (`Optional[str]`, *optional*):
|
| 26 |
+
The Jinja template to use for formatting the conversation. If not provided, the default chat template
|
| 27 |
+
is used.
|
| 28 |
+
"""
|
| 29 |
+
|
| 30 |
+
attributes = ["feature_extractor", "tokenizer"]
|
| 31 |
+
feature_extractor_class = "WhisperFeatureExtractor"
|
| 32 |
+
tokenizer_class = "AutoTokenizer"
|
| 33 |
+
valid_kwargs = [
|
| 34 |
+
"fixed_speech_embeds_length",
|
| 35 |
+
"speech_token_index",
|
| 36 |
+
"time_duration_limit",
|
| 37 |
+
"whisper_chunk_size",
|
| 38 |
+
"do_normalize"
|
| 39 |
+
]
|
| 40 |
+
|
| 41 |
+
def __init__(
|
| 42 |
+
self,
|
| 43 |
+
feature_extractor=None,
|
| 44 |
+
tokenizer=None,
|
| 45 |
+
fixed_speech_embeds_length=100,
|
| 46 |
+
speech_token_index=255999,
|
| 47 |
+
time_duration_limit=300,
|
| 48 |
+
whisper_chunk_size=30,
|
| 49 |
+
do_normalize=True
|
| 50 |
+
):
|
| 51 |
+
self.fixed_speech_embeds_length = fixed_speech_embeds_length
|
| 52 |
+
self.speech_token_index = speech_token_index
|
| 53 |
+
self.time_duration_limit = time_duration_limit
|
| 54 |
+
self.whisper_chunk_size = whisper_chunk_size
|
| 55 |
+
self.number_chunk_limit = self.time_duration_limit // self.whisper_chunk_size
|
| 56 |
+
self.do_normalize = do_normalize
|
| 57 |
+
|
| 58 |
+
super().__init__(feature_extractor, tokenizer)
|
| 59 |
+
|
| 60 |
+
self.speech_token = self.tokenizer.added_tokens_decoder[self.speech_token_index].content
|
| 61 |
+
self.feature_chunk_size = self.whisper_chunk_size * self.feature_extractor.sampling_rate
|
| 62 |
+
|
| 63 |
+
def _process_text(self, text: List[str], audio_number_chunks: np.ndarray):
|
| 64 |
+
pieces = []
|
| 65 |
+
for i, item in enumerate(text):
|
| 66 |
+
target_string = self.speech_token * self.fixed_speech_embeds_length * audio_number_chunks[i]
|
| 67 |
+
pieces.append(item.replace(self.speech_token, target_string))
|
| 68 |
+
return pieces
|
| 69 |
+
|
| 70 |
+
def _get_number_chunks(self, audios: List[np.ndarray]):
|
| 71 |
+
audio_lengths = np.array([_.shape[0] for _ in audios])
|
| 72 |
+
number_chunks = (audio_lengths // self.feature_chunk_size) + 1
|
| 73 |
+
return np.clip(number_chunks, a_min=None, a_max=self.number_chunk_limit)
|
| 74 |
+
|
| 75 |
+
def _get_chunked_audios(self, audios: Union[np.ndarray, List[np.ndarray]]):
|
| 76 |
+
if isinstance(audios, np.ndarray):
|
| 77 |
+
audios = [audios]
|
| 78 |
+
|
| 79 |
+
audio_number_chunks = self._get_number_chunks(audios)
|
| 80 |
+
chunked_audios = []
|
| 81 |
+
|
| 82 |
+
for audio_idx, audio in enumerate(audios):
|
| 83 |
+
for cid in range(audio_number_chunks[audio_idx]):
|
| 84 |
+
chunked_audios.append(
|
| 85 |
+
audio[cid * self.feature_chunk_size: (cid + 1) * self.feature_chunk_size]
|
| 86 |
+
)
|
| 87 |
+
return audio_number_chunks, chunked_audios
|
| 88 |
+
|
| 89 |
+
def __call__(
|
| 90 |
+
self,
|
| 91 |
+
text: Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None,
|
| 92 |
+
audios: Union[np.ndarray, List[np.ndarray]] = None,
|
| 93 |
+
padding: Union[bool, str, PaddingStrategy] = True,
|
| 94 |
+
sampling_rate: Optional[int] = None,
|
| 95 |
+
do_normalize: Optional[bool] = None,
|
| 96 |
+
**kwargs,
|
| 97 |
+
) -> BatchFeature:
|
| 98 |
+
"""
|
| 99 |
+
Main method to prepare for the model one or several sequences(s) and audio(s). This method forwards the `text`
|
| 100 |
+
and `kwargs` arguments to GemmaTokenizer's [`~GemmaTokenizer.__call__`] if `text` is not `None` to encode
|
| 101 |
+
the text. To prepare the audio(s), this method forwards the `audios` and `kwrags` arguments to
|
| 102 |
+
WhisperFeatureExtractor's [`~WhisperFeatureExtractor.__call__`] if `audios` is not `None`. Please refer to the doctsring
|
| 103 |
+
of the above two methods for more information.
|
| 104 |
+
|
| 105 |
+
Args:
|
| 106 |
+
text (`str`, `List[str]`):
|
| 107 |
+
The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings
|
| 108 |
+
(pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set
|
| 109 |
+
`is_split_into_words=True` (to lift the ambiguity with a batch of sequences).
|
| 110 |
+
audios (`np.ndarray`, `List[np.ndarray]`):
|
| 111 |
+
The audio or batch of audios to be prepared. Each audio can be a NumPy array.
|
| 112 |
+
padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `True`):
|
| 113 |
+
Select a strategy to pad the returned sequences (according to the model's padding side and padding
|
| 114 |
+
index) among:
|
| 115 |
+
- `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
|
| 116 |
+
sequence if provided).
|
| 117 |
+
- `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
|
| 118 |
+
acceptable input length for the model if that argument is not provided.
|
| 119 |
+
- `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
|
| 120 |
+
lengths).
|
| 121 |
+
sampling_rate (`int`, defaults to 16000):
|
| 122 |
+
The sampling rate at which the audio files should be digitalized expressed in hertz (Hz).
|
| 123 |
+
do_normalize (`bool`, defaults to `True`):
|
| 124 |
+
Whether or not to zero-mean unit-variance normalize the input.
|
| 125 |
+
Normalizing can help to significantly improve the performance of the model.
|
| 126 |
+
"""
|
| 127 |
+
|
| 128 |
+
if text is None:
|
| 129 |
+
raise ValueError("You need to specify either a `text` input to process.")
|
| 130 |
+
if not isinstance(text, list):
|
| 131 |
+
text = [text]
|
| 132 |
+
if not isinstance(audios, list):
|
| 133 |
+
audios = [audios]
|
| 134 |
+
if sampling_rate is None:
|
| 135 |
+
sampling_rate = self.feature_extractor.sampling_rate
|
| 136 |
+
if do_normalize is None:
|
| 137 |
+
do_normalize = self.do_normalize
|
| 138 |
+
|
| 139 |
+
for i, audio in enumerate(audios):
|
| 140 |
+
if audio.ndim > 1:
|
| 141 |
+
raise Exception(f"MERaLiON2 only accepts mono channel audio, {i+1}th audio have {audios[0].ndim} channels")
|
| 142 |
+
|
| 143 |
+
inputs_dict = {}
|
| 144 |
+
|
| 145 |
+
if audios is not None:
|
| 146 |
+
audio_number_chunks, chunked_audios = self._get_chunked_audios(audios)
|
| 147 |
+
text = self._process_text(text, audio_number_chunks)
|
| 148 |
+
|
| 149 |
+
audio_inputs = self.feature_extractor(
|
| 150 |
+
chunked_audios,
|
| 151 |
+
sampling_rate=sampling_rate,
|
| 152 |
+
return_tensors="pt",
|
| 153 |
+
return_attention_mask=True,
|
| 154 |
+
padding="max_length",
|
| 155 |
+
do_normalize=self.do_normalize,
|
| 156 |
+
)
|
| 157 |
+
audio_inputs["feature_attention_mask"] = audio_inputs.pop(
|
| 158 |
+
"attention_mask"
|
| 159 |
+
) # rename attention_mask to prevent conflicts later on
|
| 160 |
+
inputs_dict.update(audio_inputs)
|
| 161 |
+
|
| 162 |
+
text_input = self.tokenizer(
|
| 163 |
+
text=text,
|
| 164 |
+
return_tensors="pt",
|
| 165 |
+
add_special_tokens=False,
|
| 166 |
+
return_attention_mask=True,
|
| 167 |
+
padding=padding,
|
| 168 |
+
)
|
| 169 |
+
|
| 170 |
+
inputs_dict["input_ids"] = text_input.input_ids
|
| 171 |
+
inputs_dict["attention_mask"] = text_input.attention_mask
|
| 172 |
+
|
| 173 |
+
return BatchFeature(data={**inputs_dict})
|
| 174 |
+
|
| 175 |
+
def batch_decode(self, *args, **kwargs):
|
| 176 |
+
"""
|
| 177 |
+
This method forwards all its arguments to GemmaTokenizer's [`~PreTrainedTokenizer.batch_decode`]. Please
|
| 178 |
+
refer to the docstring of this method for more information.
|
| 179 |
+
"""
|
| 180 |
+
return self.tokenizer.batch_decode(*args, **kwargs)
|
| 181 |
+
|
| 182 |
+
def decode(self, *args, **kwargs):
|
| 183 |
+
"""
|
| 184 |
+
This method forwards all its arguments to GemmaTokenizer's [`~PreTrainedTokenizer.decode`]. Please refer to
|
| 185 |
+
the docstring of this method for more information.
|
| 186 |
+
"""
|
| 187 |
+
return self.tokenizer.decode(*args, **kwargs)
|
| 188 |
+
|
| 189 |
+
@property
|
| 190 |
+
def model_input_names(self):
|
| 191 |
+
tokenizer_input_names = self.tokenizer.model_input_names
|
| 192 |
+
feature_extractor_input_names = self.feature_extractor.model_input_names
|
| 193 |
+
return list(dict.fromkeys(tokenizer_input_names + feature_extractor_input_names + ["feature_attention_mask"]))
|
vllm_plugin_meralion2/vllm_plugin_meralion2/vllm064_post1.py
ADDED
|
@@ -0,0 +1,453 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Inference-only MERaLiON AudioLLM model compatible with HuggingFace weights."""
|
| 2 |
+
from functools import lru_cache
|
| 3 |
+
from typing import Iterable, List, Mapping, Optional, Tuple, Union
|
| 4 |
+
|
| 5 |
+
import librosa
|
| 6 |
+
import numpy as np
|
| 7 |
+
import torch
|
| 8 |
+
import torch.nn as nn
|
| 9 |
+
|
| 10 |
+
from vllm.attention import AttentionMetadata
|
| 11 |
+
from vllm.config import VllmConfig
|
| 12 |
+
from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
|
| 13 |
+
InputContext, token_inputs)
|
| 14 |
+
from vllm.logger import init_logger
|
| 15 |
+
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
| 16 |
+
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
|
| 17 |
+
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
| 18 |
+
from vllm.model_executor.model_loader.weight_utils import (
|
| 19 |
+
default_weight_loader, maybe_remap_kv_scale_name)
|
| 20 |
+
from vllm.model_executor.models.gemma2 import Gemma2Model
|
| 21 |
+
from vllm.model_executor.sampling_metadata import SamplingMetadata
|
| 22 |
+
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
|
| 23 |
+
from vllm.multimodal.utils import consecutive_placeholder_ranges
|
| 24 |
+
from vllm.sequence import IntermediateTensors, SequenceData
|
| 25 |
+
from transformers.models.whisper.modeling_whisper import WhisperEncoder
|
| 26 |
+
|
| 27 |
+
from vllm.model_executor.models.interfaces import SupportsMultiModal, SupportsLoRA, SupportsPP
|
| 28 |
+
from vllm.model_executor.models.utils import maybe_prefix
|
| 29 |
+
|
| 30 |
+
from .transformers_utils.modules import (autoset_attn_implementation_for_whisper,
|
| 31 |
+
MERaLiON2Inputs, MERaLiON2SpeechAudioAdaper)
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
logger = init_logger(__name__)
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
# gemma2 ties word embedding by default
|
| 38 |
+
_KEYS_TO_MODIFY_MAPPING = {
|
| 39 |
+
"text_decoder.model": "model",
|
| 40 |
+
}
|
| 41 |
+
|
| 42 |
+
# === Constants === #
|
| 43 |
+
DEFAULT_SAMPLE_RATE = 16000
|
| 44 |
+
FEATURE_CHUNK_SIZE = DEFAULT_SAMPLE_RATE * 30
|
| 45 |
+
OUTPUT_CHUNK_SIZE = 100
|
| 46 |
+
MAX_NUMBER_CHUNKS = 8
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def dummy_data_for_meralion(ctx: InputContext, seq_len: int,
|
| 50 |
+
mm_counts: Mapping[str, int]):
|
| 51 |
+
num_audios = mm_counts["audio"]
|
| 52 |
+
max_tokens_per_audio = get_max_meralion_audio_tokens(ctx)
|
| 53 |
+
max_llm_audio_tokens = max_tokens_per_audio * num_audios
|
| 54 |
+
if seq_len - max_llm_audio_tokens - 2 < 0:
|
| 55 |
+
raise RuntimeError(
|
| 56 |
+
f"MERaLiON-AudioLLM cannot process {num_audios} audios in a prompt, "
|
| 57 |
+
"please increase max_model_len or reduce audio limit by "
|
| 58 |
+
"--limit-mm-per-prompt.")
|
| 59 |
+
|
| 60 |
+
speech_token_index = ctx.model_config.hf_config.speech_token_index
|
| 61 |
+
|
| 62 |
+
dummy_seqdata = SequenceData.from_prompt_token_counts(
|
| 63 |
+
(speech_token_index, max_llm_audio_tokens),
|
| 64 |
+
(0, seq_len - max_llm_audio_tokens),
|
| 65 |
+
)
|
| 66 |
+
dummy_audio = np.full((max_llm_audio_tokens * 15 * 2 * 160, ), 0.)
|
| 67 |
+
return DummyData(
|
| 68 |
+
dummy_seqdata, {"audio": [(dummy_audio, DEFAULT_SAMPLE_RATE)] * num_audios}, {
|
| 69 |
+
"audio":
|
| 70 |
+
consecutive_placeholder_ranges(num_items=num_audios,
|
| 71 |
+
item_size=max_tokens_per_audio)
|
| 72 |
+
})
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def get_processor(
|
| 76 |
+
processor_name: str,
|
| 77 |
+
*args,
|
| 78 |
+
trust_remote_code: bool = True,
|
| 79 |
+
**kwargs,
|
| 80 |
+
):
|
| 81 |
+
"""Gets a processor for the given model name via HuggingFace.
|
| 82 |
+
|
| 83 |
+
Derived from `vllm.transformers_utils.image_processor.get_image_processor`.
|
| 84 |
+
"""
|
| 85 |
+
# don't put this import at the top level
|
| 86 |
+
# it will call torch.cuda.device_count()
|
| 87 |
+
from transformers import AutoProcessor
|
| 88 |
+
|
| 89 |
+
try:
|
| 90 |
+
processor = AutoProcessor.from_pretrained(
|
| 91 |
+
processor_name,
|
| 92 |
+
*args,
|
| 93 |
+
trust_remote_code=trust_remote_code,
|
| 94 |
+
**kwargs)
|
| 95 |
+
except ValueError as e:
|
| 96 |
+
# If the error pertains to the processor class not existing or not
|
| 97 |
+
# currently being imported, suggest using the --trust-remote-code flag.
|
| 98 |
+
# Unlike AutoTokenizer, AutoProcessor does not separate such errors
|
| 99 |
+
if not trust_remote_code:
|
| 100 |
+
err_msg = (
|
| 101 |
+
"Failed to load the processor. If the processor is "
|
| 102 |
+
"a custom processor not yet available in the HuggingFace "
|
| 103 |
+
"transformers library, consider setting "
|
| 104 |
+
"`trust_remote_code=True` in LLM or using the "
|
| 105 |
+
"`--trust-remote-code` flag in the CLI.")
|
| 106 |
+
raise RuntimeError(err_msg) from e
|
| 107 |
+
else:
|
| 108 |
+
raise e
|
| 109 |
+
|
| 110 |
+
return processor
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
cached_get_processor = lru_cache(get_processor)
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
def _get_number_chunks(audios: List[np.ndarray]):
|
| 117 |
+
audio_lengths = np.array([_.shape[0] for _ in audios])
|
| 118 |
+
number_chunks = (audio_lengths // FEATURE_CHUNK_SIZE) + 1
|
| 119 |
+
return np.clip(number_chunks, a_min=None, a_max=MAX_NUMBER_CHUNKS)
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def _get_feat_extract_output_lengths(audios: List[np.ndarray]):
|
| 123 |
+
return _get_number_chunks(audios) * OUTPUT_CHUNK_SIZE
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
def _get_chunked_audios(audios: List[np.ndarray]):
|
| 127 |
+
audio_number_chunks = _get_number_chunks(audios)
|
| 128 |
+
chunked_resampled_audios = []
|
| 129 |
+
|
| 130 |
+
for audio_idx, audio in enumerate(audios):
|
| 131 |
+
for cid in range(audio_number_chunks[audio_idx]):
|
| 132 |
+
chunked_resampled_audios.append(
|
| 133 |
+
audio[cid * FEATURE_CHUNK_SIZE: (cid + 1) * FEATURE_CHUNK_SIZE]
|
| 134 |
+
)
|
| 135 |
+
return chunked_resampled_audios
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
def _maybe_resample_audio(audio, orig_sample_rate, target_sample_rate):
|
| 139 |
+
if orig_sample_rate != target_sample_rate:
|
| 140 |
+
return librosa.resample(
|
| 141 |
+
audio,
|
| 142 |
+
orig_sr=orig_sample_rate,
|
| 143 |
+
target_sr=target_sample_rate
|
| 144 |
+
)
|
| 145 |
+
return audio
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
def get_max_meralion_audio_tokens(ctx: InputContext) -> int:
|
| 149 |
+
"""
|
| 150 |
+
The max number of tokens after speech audio adapter.
|
| 151 |
+
"""
|
| 152 |
+
return MAX_NUMBER_CHUNKS * OUTPUT_CHUNK_SIZE
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
def input_processor_for_meralion(
|
| 156 |
+
ctx: InputContext, inputs: DecoderOnlyInputs) -> DecoderOnlyInputs:
|
| 157 |
+
multi_modal_data = inputs.get("multi_modal_data")
|
| 158 |
+
if multi_modal_data is None or "audio" not in multi_modal_data:
|
| 159 |
+
return inputs
|
| 160 |
+
|
| 161 |
+
audios = multi_modal_data["audio"]
|
| 162 |
+
if not isinstance(audios, list):
|
| 163 |
+
audios = [audios]
|
| 164 |
+
|
| 165 |
+
if len(audios) == 0:
|
| 166 |
+
return inputs
|
| 167 |
+
|
| 168 |
+
processor = cached_get_processor(ctx.model_config.model)
|
| 169 |
+
|
| 170 |
+
resampled_audios = [
|
| 171 |
+
librosa.resample(audio,
|
| 172 |
+
orig_sr=sampling_rate,
|
| 173 |
+
target_sr=processor.feature_extractor.sampling_rate)
|
| 174 |
+
for audio, sampling_rate in audios
|
| 175 |
+
]
|
| 176 |
+
|
| 177 |
+
audio_output_lengths = _get_feat_extract_output_lengths(resampled_audios)
|
| 178 |
+
speech_token_index = ctx.model_config.hf_config.speech_token_index
|
| 179 |
+
|
| 180 |
+
input_ids = inputs['prompt_token_ids']
|
| 181 |
+
|
| 182 |
+
new_input_ids = []
|
| 183 |
+
audio_num = input_ids.count(speech_token_index)
|
| 184 |
+
assert len(audio_output_lengths) == audio_num, \
|
| 185 |
+
(f'The text input contains {audio_num} audio tokens, '
|
| 186 |
+
f'but {len(audio_output_lengths)} audios provided')
|
| 187 |
+
start = 0
|
| 188 |
+
for audio_idx in range(audio_num):
|
| 189 |
+
end = input_ids.index(speech_token_index, start)
|
| 190 |
+
new_input_ids.extend(input_ids[start:end]) # text part
|
| 191 |
+
|
| 192 |
+
new_input_ids.extend([speech_token_index] *
|
| 193 |
+
audio_output_lengths[audio_idx])
|
| 194 |
+
start = end + 1
|
| 195 |
+
new_input_ids.extend(input_ids[start:])
|
| 196 |
+
|
| 197 |
+
return token_inputs(
|
| 198 |
+
prompt_token_ids=new_input_ids,
|
| 199 |
+
prompt=inputs.get('prompt'),
|
| 200 |
+
multi_modal_data=multi_modal_data,
|
| 201 |
+
)
|
| 202 |
+
|
| 203 |
+
|
| 204 |
+
def input_mapper_for_meralion(
|
| 205 |
+
ctx: InputContext,
|
| 206 |
+
multi_modal_data: Union[np.ndarray, List[np.ndarray]],
|
| 207 |
+
) -> MultiModalKwargs:
|
| 208 |
+
"""Input mapper for MERaLiON-AudioLLM."""
|
| 209 |
+
if not isinstance(multi_modal_data, list):
|
| 210 |
+
multi_modal_data = [multi_modal_data]
|
| 211 |
+
|
| 212 |
+
if len(multi_modal_data) == 0:
|
| 213 |
+
return MultiModalKwargs()
|
| 214 |
+
|
| 215 |
+
processor = cached_get_processor(ctx.model_config.model)
|
| 216 |
+
audio_feature_extractor = processor.feature_extractor
|
| 217 |
+
if audio_feature_extractor is None:
|
| 218 |
+
raise RuntimeError(
|
| 219 |
+
"No HuggingFace audio_feature_extractor is available "
|
| 220 |
+
"to process the audio object")
|
| 221 |
+
|
| 222 |
+
try:
|
| 223 |
+
target_sample_rate = processor.feature_extractor.sampling_rate
|
| 224 |
+
|
| 225 |
+
resampled_audios = [
|
| 226 |
+
_maybe_resample_audio(
|
| 227 |
+
audio=audio,
|
| 228 |
+
orig_sample_rate=sampling_rate,
|
| 229 |
+
target_sample_rate=target_sample_rate,
|
| 230 |
+
)
|
| 231 |
+
for audio, sampling_rate in multi_modal_data
|
| 232 |
+
]
|
| 233 |
+
|
| 234 |
+
resampled_audios = _get_chunked_audios(resampled_audios)
|
| 235 |
+
|
| 236 |
+
batch_data = audio_feature_extractor(resampled_audios,
|
| 237 |
+
sampling_rate=target_sample_rate,
|
| 238 |
+
return_attention_mask=True,
|
| 239 |
+
padding="max_length",
|
| 240 |
+
return_tensors="pt").data
|
| 241 |
+
batch_data["feature_attention_mask"] = batch_data.pop("attention_mask")
|
| 242 |
+
except Exception:
|
| 243 |
+
logger.error("Failed to process audio (%s)", multi_modal_data)
|
| 244 |
+
raise
|
| 245 |
+
|
| 246 |
+
return MultiModalKwargs(batch_data)
|
| 247 |
+
|
| 248 |
+
|
| 249 |
+
@INPUT_REGISTRY.register_dummy_data(dummy_data_for_meralion)
|
| 250 |
+
@INPUT_REGISTRY.register_input_processor(input_processor_for_meralion)
|
| 251 |
+
@MULTIMODAL_REGISTRY.register_input_mapper("audio",
|
| 252 |
+
input_mapper_for_meralion)
|
| 253 |
+
@MULTIMODAL_REGISTRY.register_max_multimodal_tokens(
|
| 254 |
+
"audio", get_max_meralion_audio_tokens)
|
| 255 |
+
class MERaLiON2ForConditionalGeneration(nn.Module, SupportsMultiModal,
|
| 256 |
+
SupportsLoRA, SupportsPP):
|
| 257 |
+
packed_modules_mapping = {
|
| 258 |
+
"qkv_proj": [
|
| 259 |
+
"q_proj",
|
| 260 |
+
"k_proj",
|
| 261 |
+
"v_proj",
|
| 262 |
+
],
|
| 263 |
+
"gate_up_proj": [
|
| 264 |
+
"gate_proj",
|
| 265 |
+
"up_proj",
|
| 266 |
+
],
|
| 267 |
+
}
|
| 268 |
+
|
| 269 |
+
# LoRA specific attributes
|
| 270 |
+
supported_lora_modules = [
|
| 271 |
+
"qkv_proj",
|
| 272 |
+
"o_proj",
|
| 273 |
+
"gate_up_proj",
|
| 274 |
+
"down_proj",
|
| 275 |
+
]
|
| 276 |
+
|
| 277 |
+
embedding_modules = {}
|
| 278 |
+
embedding_padding_modules = []
|
| 279 |
+
|
| 280 |
+
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
| 281 |
+
super().__init__()
|
| 282 |
+
config = vllm_config.model_config.hf_config
|
| 283 |
+
quant_config = vllm_config.quant_config
|
| 284 |
+
multimodal_config = vllm_config.model_config.multimodal_config
|
| 285 |
+
|
| 286 |
+
self.config = config
|
| 287 |
+
self.multimodal_config = multimodal_config
|
| 288 |
+
|
| 289 |
+
config.speech_config = \
|
| 290 |
+
autoset_attn_implementation_for_whisper(config.speech_config)
|
| 291 |
+
self.speech_encoder = WhisperEncoder(config.speech_config)
|
| 292 |
+
self.ln_speech = nn.LayerNorm(config.speech_config.d_model)
|
| 293 |
+
self.speech_audio_adapter = MERaLiON2SpeechAudioAdaper(
|
| 294 |
+
config.speech_config.d_model, config.text_config.hidden_size)
|
| 295 |
+
|
| 296 |
+
self.quant_config = quant_config
|
| 297 |
+
|
| 298 |
+
self.model = Gemma2Model(
|
| 299 |
+
vllm_config=vllm_config.with_hf_config(config.text_config),
|
| 300 |
+
prefix=maybe_prefix(prefix, "model"))
|
| 301 |
+
self.unpadded_vocab_size = config.text_config.vocab_size
|
| 302 |
+
if config.text_config.tie_word_embeddings:
|
| 303 |
+
self.lm_head = self.model.embed_tokens
|
| 304 |
+
else:
|
| 305 |
+
self.lm_head = ParallelLMHead(config.text_config.vocab_size,
|
| 306 |
+
config.text_config.hidden_size,
|
| 307 |
+
quant_config=quant_config)
|
| 308 |
+
logit_scale = getattr(config, "logit_scale", 1.0)
|
| 309 |
+
self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
|
| 310 |
+
config.text_config.vocab_size,
|
| 311 |
+
logit_scale)
|
| 312 |
+
|
| 313 |
+
self.sampler = get_sampler()
|
| 314 |
+
self.make_empty_intermediate_tensors = (
|
| 315 |
+
self.model.make_empty_intermediate_tensors)
|
| 316 |
+
|
| 317 |
+
def _validate_and_reshape_mm_tensor(self,
|
| 318 |
+
mm_input: Union[torch.Tensor,
|
| 319 |
+
List[torch.Tensor]],
|
| 320 |
+
name: str) -> torch.Tensor:
|
| 321 |
+
if not isinstance(mm_input, (torch.Tensor, list)):
|
| 322 |
+
raise ValueError(f"Incorrect type of {name}. "
|
| 323 |
+
f"Got type: {type(mm_input)}")
|
| 324 |
+
if isinstance(mm_input, torch.Tensor):
|
| 325 |
+
return torch.concat(list(mm_input))
|
| 326 |
+
else:
|
| 327 |
+
return torch.concat(mm_input)
|
| 328 |
+
|
| 329 |
+
def _parse_and_validate_audio_input(
|
| 330 |
+
self, **kwargs: object) -> Optional[MERaLiON2Inputs]:
|
| 331 |
+
input_features = kwargs.pop('input_features', None)
|
| 332 |
+
feature_attention_mask = kwargs.pop('feature_attention_mask', None)
|
| 333 |
+
if input_features is None:
|
| 334 |
+
return None
|
| 335 |
+
input_features = self._validate_and_reshape_mm_tensor(
|
| 336 |
+
input_features, 'input_features')
|
| 337 |
+
feature_attention_mask = self._validate_and_reshape_mm_tensor(
|
| 338 |
+
feature_attention_mask, 'feature_attention_mask')
|
| 339 |
+
if not isinstance(input_features, (torch.Tensor, list)):
|
| 340 |
+
raise ValueError("Incorrect type of audio input features. "
|
| 341 |
+
f"Got type: {type(input_features)}")
|
| 342 |
+
return MERaLiON2Inputs(input_features=input_features,
|
| 343 |
+
feature_attention_mask=feature_attention_mask)
|
| 344 |
+
|
| 345 |
+
def _process_audio_input(self,
|
| 346 |
+
audio_input: MERaLiON2Inputs) -> torch.Tensor:
|
| 347 |
+
|
| 348 |
+
input_features = audio_input["input_features"].to(self.speech_encoder.dtype)
|
| 349 |
+
feature_attention_mask = audio_input["feature_attention_mask"]
|
| 350 |
+
|
| 351 |
+
audio_outputs = self.speech_encoder(input_features,
|
| 352 |
+
attention_mask=feature_attention_mask)
|
| 353 |
+
audio_features = audio_outputs.last_hidden_state
|
| 354 |
+
audio_features = self.ln_speech(audio_features)
|
| 355 |
+
audio_features = self.speech_audio_adapter(audio_features)
|
| 356 |
+
audio_features = audio_features.view(-1, audio_features.size(-1))
|
| 357 |
+
|
| 358 |
+
return audio_features
|
| 359 |
+
|
| 360 |
+
def forward(
|
| 361 |
+
self,
|
| 362 |
+
input_ids: torch.Tensor,
|
| 363 |
+
positions: torch.Tensor,
|
| 364 |
+
kv_caches: List[torch.Tensor],
|
| 365 |
+
attn_metadata: AttentionMetadata,
|
| 366 |
+
intermediate_tensors: Optional[IntermediateTensors] = None,
|
| 367 |
+
**kwargs: object,
|
| 368 |
+
) -> Union[torch.Tensor, IntermediateTensors]:
|
| 369 |
+
if intermediate_tensors is not None:
|
| 370 |
+
input_ids = None
|
| 371 |
+
inputs_embeds = None
|
| 372 |
+
else:
|
| 373 |
+
audio_input = self._parse_and_validate_audio_input(**kwargs)
|
| 374 |
+
|
| 375 |
+
if audio_input is None:
|
| 376 |
+
inputs_embeds = None
|
| 377 |
+
else:
|
| 378 |
+
inputs_embeds = self.model.embed_tokens(input_ids)
|
| 379 |
+
processed_audio_features = self._process_audio_input(audio_input)
|
| 380 |
+
# merge llm embeddings and audio features
|
| 381 |
+
mask = (input_ids == self.config.speech_token_index)
|
| 382 |
+
inputs_embeds[mask, :] = processed_audio_features
|
| 383 |
+
|
| 384 |
+
input_ids = None
|
| 385 |
+
|
| 386 |
+
hidden_states = self.model(
|
| 387 |
+
input_ids=input_ids,
|
| 388 |
+
positions=positions,
|
| 389 |
+
kv_caches=kv_caches,
|
| 390 |
+
attn_metadata=attn_metadata,
|
| 391 |
+
intermediate_tensors=intermediate_tensors,
|
| 392 |
+
inputs_embeds=inputs_embeds,
|
| 393 |
+
)
|
| 394 |
+
return hidden_states
|
| 395 |
+
|
| 396 |
+
def compute_logits(self, hidden_states: torch.Tensor,
|
| 397 |
+
sampling_metadata: SamplingMetadata) -> torch.Tensor:
|
| 398 |
+
logits = self.logits_processor(self.lm_head, hidden_states,
|
| 399 |
+
sampling_metadata)
|
| 400 |
+
return logits
|
| 401 |
+
|
| 402 |
+
def sample(
|
| 403 |
+
self,
|
| 404 |
+
logits: torch.Tensor,
|
| 405 |
+
sampling_metadata: SamplingMetadata,
|
| 406 |
+
) -> Optional[SamplerOutput]:
|
| 407 |
+
next_tokens = self.sampler(logits, sampling_metadata)
|
| 408 |
+
return next_tokens
|
| 409 |
+
|
| 410 |
+
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
|
| 411 |
+
stacked_params_mapping = [
|
| 412 |
+
# (param_name, shard_name, shard_id)
|
| 413 |
+
("qkv_proj", "q_proj", "q"),
|
| 414 |
+
("qkv_proj", "k_proj", "k"),
|
| 415 |
+
("qkv_proj", "v_proj", "v"),
|
| 416 |
+
("gate_up_proj", "gate_proj", 0),
|
| 417 |
+
("gate_up_proj", "up_proj", 1),
|
| 418 |
+
]
|
| 419 |
+
params_dict = dict(self.named_parameters(remove_duplicate=False))
|
| 420 |
+
|
| 421 |
+
for name, loaded_weight in weights:
|
| 422 |
+
if "rotary_emb.inv_freq" in name:
|
| 423 |
+
continue
|
| 424 |
+
if (self.config.text_config.tie_word_embeddings
|
| 425 |
+
and "lm_head.weight" in name):
|
| 426 |
+
continue
|
| 427 |
+
for key_to_modify, new_key in _KEYS_TO_MODIFY_MAPPING.items():
|
| 428 |
+
if key_to_modify in name:
|
| 429 |
+
name = name.replace(key_to_modify, new_key)
|
| 430 |
+
for (param_name, weight_name, shard_id) in stacked_params_mapping:
|
| 431 |
+
if weight_name not in name or 'speech_' in name:
|
| 432 |
+
continue
|
| 433 |
+
name = name.replace(weight_name, param_name)
|
| 434 |
+
# Skip loading extra bias for GPTQ models.
|
| 435 |
+
if name.endswith(".bias") and name not in params_dict:
|
| 436 |
+
continue
|
| 437 |
+
param = params_dict[name]
|
| 438 |
+
weight_loader = param.weight_loader
|
| 439 |
+
weight_loader(param, loaded_weight, shard_id)
|
| 440 |
+
break
|
| 441 |
+
else:
|
| 442 |
+
# Skip loading extra bias for GPTQ models.
|
| 443 |
+
if name.endswith(".bias") and name not in params_dict:
|
| 444 |
+
continue
|
| 445 |
+
# Remapping the name of FP8 kv-scale.
|
| 446 |
+
name = maybe_remap_kv_scale_name(name, params_dict)
|
| 447 |
+
if name is None:
|
| 448 |
+
continue
|
| 449 |
+
|
| 450 |
+
param = params_dict[name]
|
| 451 |
+
weight_loader = getattr(param, "weight_loader",
|
| 452 |
+
default_weight_loader)
|
| 453 |
+
weight_loader(param, loaded_weight)
|
vllm_plugin_meralion2/vllm_plugin_meralion2/vllm085.py
ADDED
|
@@ -0,0 +1,333 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Inference-only MERaLiON AudioLLM model compatible with HuggingFace weights."""
|
| 2 |
+
from collections.abc import Iterable, Mapping, Sequence
|
| 3 |
+
from typing import Any, Optional, Set, Tuple, TypedDict, Union, List
|
| 4 |
+
|
| 5 |
+
import math
|
| 6 |
+
import torch
|
| 7 |
+
import torch.nn as nn
|
| 8 |
+
from transformers import BatchFeature
|
| 9 |
+
from transformers.models.whisper.modeling_whisper import WhisperEncoder
|
| 10 |
+
from transformers.models.whisper.feature_extraction_whisper import WhisperFeatureExtractor
|
| 11 |
+
|
| 12 |
+
from vllm.config import VllmConfig
|
| 13 |
+
from vllm.model_executor.sampling_metadata import SamplingMetadata
|
| 14 |
+
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
|
| 15 |
+
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
|
| 16 |
+
MultiModalKwargs)
|
| 17 |
+
from vllm.multimodal.parse import (AudioProcessorItems, MultiModalDataItems,
|
| 18 |
+
MultiModalDataParser)
|
| 19 |
+
from vllm.multimodal.processing import (BaseMultiModalProcessor,
|
| 20 |
+
BaseProcessingInfo, PromptReplacement,
|
| 21 |
+
PromptUpdate, PromptUpdateDetails)
|
| 22 |
+
from vllm.multimodal.profiling import BaseDummyInputsBuilder
|
| 23 |
+
from vllm.sequence import IntermediateTensors
|
| 24 |
+
from vllm.model_executor.models.interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
|
| 25 |
+
from vllm.model_executor.models.utils import (AutoWeightsLoader, init_vllm_registered_model,
|
| 26 |
+
maybe_prefix, merge_multimodal_embeddings)
|
| 27 |
+
|
| 28 |
+
from .transformers_utils.processing_meralion2 import MERaLiON2Processor
|
| 29 |
+
from .transformers_utils.configuration_meralion2 import MERaLiON2Config
|
| 30 |
+
from .transformers_utils.modules import (autoset_attn_implementation_for_whisper,
|
| 31 |
+
MERaLiON2Inputs, MERaLiON2SpeechAudioAdaper)
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
class MERaLiON2ProcessingInfo(BaseProcessingInfo):
|
| 35 |
+
|
| 36 |
+
def get_hf_config(self):
|
| 37 |
+
return self.ctx.get_hf_config(MERaLiON2Config)
|
| 38 |
+
|
| 39 |
+
def get_hf_processor(
|
| 40 |
+
self,
|
| 41 |
+
*,
|
| 42 |
+
# Ignored in initialization
|
| 43 |
+
sampling_rate: Optional[int] = None,
|
| 44 |
+
**kwargs: object,
|
| 45 |
+
) -> MERaLiON2Processor:
|
| 46 |
+
return self.ctx.get_hf_processor(MERaLiON2Processor, **kwargs)
|
| 47 |
+
|
| 48 |
+
def get_feature_extractor(
|
| 49 |
+
self,
|
| 50 |
+
*,
|
| 51 |
+
# Ignored in initialization
|
| 52 |
+
sampling_rate: Optional[int] = None,
|
| 53 |
+
) -> WhisperFeatureExtractor:
|
| 54 |
+
hf_processor = self.get_hf_processor(sampling_rate=sampling_rate)
|
| 55 |
+
feature_extractor = hf_processor.feature_extractor # type: ignore
|
| 56 |
+
assert isinstance(feature_extractor, WhisperFeatureExtractor)
|
| 57 |
+
return feature_extractor
|
| 58 |
+
|
| 59 |
+
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
|
| 60 |
+
return {"audio": None}
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
class MERaLiON2DummyInputsBuilder(
|
| 64 |
+
BaseDummyInputsBuilder[MERaLiON2ProcessingInfo]):
|
| 65 |
+
|
| 66 |
+
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
|
| 67 |
+
num_audios = mm_counts.get("audio", 0)
|
| 68 |
+
|
| 69 |
+
hf_processor = self.info.get_hf_processor()
|
| 70 |
+
audio_token = hf_processor.speech_token
|
| 71 |
+
|
| 72 |
+
return audio_token * num_audios
|
| 73 |
+
|
| 74 |
+
def get_dummy_mm_data(
|
| 75 |
+
self,
|
| 76 |
+
seq_len: int,
|
| 77 |
+
mm_counts: Mapping[str, int],
|
| 78 |
+
) -> MultiModalDataDict:
|
| 79 |
+
processor = self.info.get_hf_processor()
|
| 80 |
+
feature_extractor = self.info.get_feature_extractor()
|
| 81 |
+
|
| 82 |
+
# This is to specify the audio length
|
| 83 |
+
num_chunk_limit = processor.number_chunk_limit
|
| 84 |
+
sampling_rate = feature_extractor.sampling_rate
|
| 85 |
+
audio_len = num_chunk_limit * feature_extractor.chunk_length * sampling_rate
|
| 86 |
+
num_audios = mm_counts.get("audio", 0)
|
| 87 |
+
|
| 88 |
+
return {
|
| 89 |
+
"audio":
|
| 90 |
+
self._get_dummy_audios(length=audio_len, num_audios=num_audios)
|
| 91 |
+
}
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
class MERaLiON2MultiModalProcessor(
|
| 95 |
+
BaseMultiModalProcessor[MERaLiON2ProcessingInfo]):
|
| 96 |
+
|
| 97 |
+
def _get_data_parser(self) -> MultiModalDataParser:
|
| 98 |
+
feature_extractor = self.info.get_feature_extractor()
|
| 99 |
+
return MultiModalDataParser(target_sr=feature_extractor.sampling_rate)
|
| 100 |
+
|
| 101 |
+
def _call_hf_processor(
|
| 102 |
+
self,
|
| 103 |
+
prompt: str,
|
| 104 |
+
mm_data: Mapping[str, object],
|
| 105 |
+
mm_kwargs: Mapping[str, Any],
|
| 106 |
+
) -> BatchFeature:
|
| 107 |
+
# Text-only input not supported in composite processor
|
| 108 |
+
if not mm_data.get("audios", []):
|
| 109 |
+
prompt_ids = self.info.get_tokenizer().encode(prompt)
|
| 110 |
+
prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
|
| 111 |
+
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
|
| 112 |
+
|
| 113 |
+
processor = self.info.get_hf_processor()
|
| 114 |
+
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
|
| 115 |
+
speech_token_id = getattr(processor, "speech_token_index", 255999)
|
| 116 |
+
output_chunk_size = getattr(processor, "fixed_speech_embeds_length", 100)
|
| 117 |
+
|
| 118 |
+
mm_kwargs = dict(
|
| 119 |
+
**mm_kwargs,
|
| 120 |
+
sampling_rate=feature_extractor.sampling_rate,
|
| 121 |
+
)
|
| 122 |
+
|
| 123 |
+
results = super()._call_hf_processor(
|
| 124 |
+
prompt=prompt,
|
| 125 |
+
mm_data=mm_data,
|
| 126 |
+
mm_kwargs=mm_kwargs,
|
| 127 |
+
)
|
| 128 |
+
|
| 129 |
+
chunk_sizes = (results["input_ids"] == speech_token_id).sum(axis=1) // output_chunk_size
|
| 130 |
+
splitted_input_features = torch.split(results["input_features"], chunk_sizes.tolist())
|
| 131 |
+
splitted_feature_attention_mask = torch.split(results["feature_attention_mask"], chunk_sizes.tolist())
|
| 132 |
+
|
| 133 |
+
results["input_features"] = splitted_input_features
|
| 134 |
+
results["feature_attention_mask"] = splitted_feature_attention_mask
|
| 135 |
+
return results
|
| 136 |
+
|
| 137 |
+
def _get_mm_fields_config(
|
| 138 |
+
self,
|
| 139 |
+
hf_inputs: BatchFeature,
|
| 140 |
+
hf_processor_mm_kwargs: Mapping[str, object],
|
| 141 |
+
) -> Mapping[str, MultiModalFieldConfig]:
|
| 142 |
+
return dict(
|
| 143 |
+
input_features=MultiModalFieldConfig.batched("audio"),
|
| 144 |
+
feature_attention_mask=MultiModalFieldConfig.batched("audio"),
|
| 145 |
+
)
|
| 146 |
+
|
| 147 |
+
def _get_prompt_updates(
|
| 148 |
+
self,
|
| 149 |
+
mm_items: MultiModalDataItems,
|
| 150 |
+
hf_processor_mm_kwargs: Mapping[str, object],
|
| 151 |
+
out_mm_kwargs: MultiModalKwargs,
|
| 152 |
+
) -> Sequence[PromptUpdate]:
|
| 153 |
+
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
| 154 |
+
|
| 155 |
+
speech_token = getattr(processor, "audio_token", "<SpeechHere>")
|
| 156 |
+
speech_token_id = getattr(processor, "speech_token_index", 255999)
|
| 157 |
+
output_chunk_size = getattr(processor, "fixed_speech_embeds_length", 100)
|
| 158 |
+
feature_chunk_size = getattr(processor, "feature_chunk_size", 30 * 16000)
|
| 159 |
+
|
| 160 |
+
def get_replacement_meralion2_audio(item_idx: int):
|
| 161 |
+
audios = mm_items.get_items("audio", AudioProcessorItems)
|
| 162 |
+
audio_length = audios.get_audio_length(item_idx)
|
| 163 |
+
number_chunks = ((audio_length - 1) // feature_chunk_size) + 1
|
| 164 |
+
speech_tokens = [speech_token_id] * number_chunks * output_chunk_size
|
| 165 |
+
|
| 166 |
+
return PromptUpdateDetails.select_token_id(
|
| 167 |
+
speech_tokens,
|
| 168 |
+
embed_token_id=speech_token_id,
|
| 169 |
+
)
|
| 170 |
+
|
| 171 |
+
return [
|
| 172 |
+
PromptReplacement(
|
| 173 |
+
modality="audio",
|
| 174 |
+
target=speech_token,
|
| 175 |
+
replacement=get_replacement_meralion2_audio,
|
| 176 |
+
)
|
| 177 |
+
]
|
| 178 |
+
|
| 179 |
+
|
| 180 |
+
@MULTIMODAL_REGISTRY.register_processor(
|
| 181 |
+
MERaLiON2MultiModalProcessor,
|
| 182 |
+
info=MERaLiON2ProcessingInfo,
|
| 183 |
+
dummy_inputs=MERaLiON2DummyInputsBuilder)
|
| 184 |
+
class MERaLiON2ForConditionalGeneration(nn.Module, SupportsMultiModal,
|
| 185 |
+
SupportsPP):
|
| 186 |
+
|
| 187 |
+
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
| 188 |
+
super().__init__()
|
| 189 |
+
config = vllm_config.model_config.hf_config
|
| 190 |
+
quant_config = vllm_config.quant_config
|
| 191 |
+
multimodal_config = vllm_config.model_config.multimodal_config
|
| 192 |
+
self.config = config
|
| 193 |
+
self.multimodal_config = multimodal_config
|
| 194 |
+
|
| 195 |
+
config.speech_config = \
|
| 196 |
+
autoset_attn_implementation_for_whisper(config.speech_config)
|
| 197 |
+
self.speech_encoder = WhisperEncoder(config.speech_config)
|
| 198 |
+
self.ln_speech = nn.LayerNorm(config.speech_config.d_model)
|
| 199 |
+
self.speech_audio_adapter = MERaLiON2SpeechAudioAdaper(
|
| 200 |
+
config.speech_config.d_model, config.text_config.hidden_size)
|
| 201 |
+
|
| 202 |
+
self.quant_config = quant_config
|
| 203 |
+
|
| 204 |
+
self.text_decoder = init_vllm_registered_model(
|
| 205 |
+
vllm_config=vllm_config,
|
| 206 |
+
hf_config=config.text_config,
|
| 207 |
+
prefix=maybe_prefix(prefix, "text_decoder"),
|
| 208 |
+
architectures=["Gemma2ForCausalLM"],
|
| 209 |
+
)
|
| 210 |
+
|
| 211 |
+
self.make_empty_intermediate_tensors = (
|
| 212 |
+
self.text_decoder.make_empty_intermediate_tensors)
|
| 213 |
+
|
| 214 |
+
def _validate_and_reshape_mm_tensor(self,
|
| 215 |
+
mm_input: Union[torch.Tensor,
|
| 216 |
+
List[torch.Tensor]],
|
| 217 |
+
name: str) -> torch.Tensor:
|
| 218 |
+
if not isinstance(mm_input, (torch.Tensor, list)):
|
| 219 |
+
raise ValueError(f"Incorrect type of {name}. "
|
| 220 |
+
f"Got type: {type(mm_input)}")
|
| 221 |
+
|
| 222 |
+
input_ids_lst = mm_input.size()
|
| 223 |
+
with open("log.txt", "a") as f:
|
| 224 |
+
f.write(",".join([str(e) for e in input_ids_lst]) + "\n")
|
| 225 |
+
|
| 226 |
+
if isinstance(mm_input, torch.Tensor):
|
| 227 |
+
result = torch.concat(list(mm_input))
|
| 228 |
+
else:
|
| 229 |
+
result = torch.concat(mm_input)
|
| 230 |
+
|
| 231 |
+
flattened_result = result.view(-1, result.size(-2), result.size(-1))
|
| 232 |
+
return flattened_result
|
| 233 |
+
|
| 234 |
+
def _parse_and_validate_audio_input(
|
| 235 |
+
self, **kwargs: object) -> Optional[MERaLiON2Inputs]:
|
| 236 |
+
input_features = kwargs.pop('input_features', None)
|
| 237 |
+
feature_attention_mask = kwargs.pop('feature_attention_mask', None)
|
| 238 |
+
|
| 239 |
+
if input_features is None:
|
| 240 |
+
return None
|
| 241 |
+
input_features = self._validate_and_reshape_mm_tensor(
|
| 242 |
+
input_features, 'input_features')
|
| 243 |
+
feature_attention_mask = self._validate_and_reshape_mm_tensor(
|
| 244 |
+
feature_attention_mask, 'feature_attention_mask')
|
| 245 |
+
if not isinstance(input_features, (torch.Tensor, list)):
|
| 246 |
+
raise ValueError("Incorrect type of audio input features. "
|
| 247 |
+
f"Got type: {type(input_features)}")
|
| 248 |
+
return MERaLiON2Inputs(input_features=input_features,
|
| 249 |
+
feature_attention_mask=feature_attention_mask)
|
| 250 |
+
|
| 251 |
+
def _process_audio_input(self,
|
| 252 |
+
audio_input: MERaLiON2Inputs) -> torch.Tensor:
|
| 253 |
+
|
| 254 |
+
input_features = audio_input["input_features"].to(self.speech_encoder.dtype)
|
| 255 |
+
feature_attention_mask = audio_input["feature_attention_mask"]
|
| 256 |
+
|
| 257 |
+
audio_outputs = self.speech_encoder(input_features,
|
| 258 |
+
attention_mask=feature_attention_mask)
|
| 259 |
+
audio_features = audio_outputs.last_hidden_state
|
| 260 |
+
audio_features = self.ln_speech(audio_features)
|
| 261 |
+
audio_features = self.speech_audio_adapter(audio_features)
|
| 262 |
+
audio_features = audio_features.view(-1, audio_features.size(-1))
|
| 263 |
+
|
| 264 |
+
return audio_features
|
| 265 |
+
|
| 266 |
+
def get_language_model(self) -> torch.nn.Module:
|
| 267 |
+
return self.text_decoder
|
| 268 |
+
|
| 269 |
+
def get_multimodal_embeddings(
|
| 270 |
+
self, **kwargs: object) -> Optional[MultiModalEmbeddings]:
|
| 271 |
+
|
| 272 |
+
if "input_features" not in kwargs:
|
| 273 |
+
return None
|
| 274 |
+
|
| 275 |
+
input_features = kwargs["input_features"]
|
| 276 |
+
audio_input = self._parse_and_validate_audio_input(**kwargs)
|
| 277 |
+
if isinstance(input_features, torch.Tensor):
|
| 278 |
+
input_features = list(input_features)
|
| 279 |
+
|
| 280 |
+
audio_lengths = [math.prod(audio.shape[:-2]) * 100 for audio in input_features]
|
| 281 |
+
masked_audio_features = self._process_audio_input(audio_input)
|
| 282 |
+
masked_audio_features = torch.split(masked_audio_features, audio_lengths)
|
| 283 |
+
return masked_audio_features
|
| 284 |
+
|
| 285 |
+
def get_input_embeddings(
|
| 286 |
+
self,
|
| 287 |
+
input_ids: torch.Tensor,
|
| 288 |
+
multimodal_embeddings: Optional[MultiModalEmbeddings] = None,
|
| 289 |
+
) -> torch.Tensor:
|
| 290 |
+
inputs_embeds = self.text_decoder.get_input_embeddings(input_ids)
|
| 291 |
+
if multimodal_embeddings is not None:
|
| 292 |
+
inputs_embeds = merge_multimodal_embeddings(
|
| 293 |
+
input_ids, inputs_embeds, multimodal_embeddings,
|
| 294 |
+
self.config.speech_token_index)
|
| 295 |
+
return inputs_embeds
|
| 296 |
+
|
| 297 |
+
def forward(
|
| 298 |
+
self,
|
| 299 |
+
input_ids: torch.Tensor,
|
| 300 |
+
positions: torch.Tensor,
|
| 301 |
+
intermediate_tensors: Optional[IntermediateTensors] = None,
|
| 302 |
+
inputs_embeds: Optional[torch.Tensor] = None,
|
| 303 |
+
**kwargs: object,
|
| 304 |
+
) -> Union[torch.Tensor, IntermediateTensors]:
|
| 305 |
+
if intermediate_tensors is not None:
|
| 306 |
+
inputs_embeds = None
|
| 307 |
+
|
| 308 |
+
# NOTE: In v1, inputs_embeds is always generated at model runner, this
|
| 309 |
+
# condition is for v0 compatibility.
|
| 310 |
+
elif inputs_embeds is None:
|
| 311 |
+
multimodal_embeddings = self.get_multimodal_embeddings(**kwargs)
|
| 312 |
+
inputs_embeds = self.get_input_embeddings(input_ids,
|
| 313 |
+
multimodal_embeddings)
|
| 314 |
+
input_ids = None
|
| 315 |
+
|
| 316 |
+
hidden_states = self.text_decoder.model(input_ids,
|
| 317 |
+
positions,
|
| 318 |
+
intermediate_tensors,
|
| 319 |
+
inputs_embeds=inputs_embeds)
|
| 320 |
+
return hidden_states
|
| 321 |
+
|
| 322 |
+
def compute_logits(
|
| 323 |
+
self,
|
| 324 |
+
hidden_states: torch.Tensor,
|
| 325 |
+
sampling_metadata: SamplingMetadata,
|
| 326 |
+
) -> Optional[torch.Tensor]:
|
| 327 |
+
return self.text_decoder.compute_logits(hidden_states,
|
| 328 |
+
sampling_metadata)
|
| 329 |
+
|
| 330 |
+
def load_weights(self, weights: Iterable[Tuple[str,
|
| 331 |
+
torch.Tensor]]) -> Set[str]:
|
| 332 |
+
loader = AutoWeightsLoader(self)
|
| 333 |
+
return loader.load_weights(weights)
|