Automatic Speech Recognition
Transformers
Safetensors
meralion2
meralion
meralion-2
custom_code
YingxuHe commited on
Commit
95618df
·
1 Parent(s): f36243b

init vllm

Browse files
vllm_plugin_meralion2/offline_example.py ADDED
@@ -0,0 +1,52 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import torch
3
+
4
+ from vllm import ModelRegistry, LLM, SamplingParams
5
+ from vllm.assets.audio import AudioAsset
6
+
7
+ from vllm_plugin_meralion2 import NoRepeatNGramLogitsProcessor
8
+
9
+
10
+ model_name = "MERaLiON/MERaLiON-2-10B"
11
+ # model_name = "MERaLiON/MERaLiON-2-10B-ASR"
12
+ # model_name = "MERaLiON/MERaLiON-2-3B"
13
+
14
+ llm = LLM(model=model_name,
15
+ tokenizer=model_name,
16
+ limit_mm_per_prompt={"audio": 1},
17
+ trust_remote_code=True,
18
+ dtype=torch.bfloat16
19
+ )
20
+
21
+ audio_asset = AudioAsset("mary_had_lamb")
22
+
23
+ question= "Please trancribe this speech."
24
+
25
+ prompt = (
26
+ "<start_of_turn>user\n"
27
+ f"Instruction: {question} \nFollow the text instruction based on the following audio: <SpeechHere><end_of_turn>\n"
28
+ "<start_of_turn>model\n")
29
+
30
+ sampling_params = SamplingParams(
31
+ temperature=0.0,
32
+ top_p=0.9,
33
+ top_k=50,
34
+ repetition_penalty=1.1,
35
+ seed=42,
36
+ max_tokens=1024,
37
+ stop_token_ids=None,
38
+ logits_processors=[NoRepeatNGramLogitsProcessor(2)]
39
+ )
40
+
41
+ mm_data = {"audio": [audio_asset.audio_and_sample_rate]}
42
+ inputs = {"prompt": prompt, "multi_modal_data": mm_data}
43
+
44
+ # batch inference
45
+ inputs = [inputs] * 2
46
+
47
+ outputs = llm.generate(inputs, sampling_params=sampling_params)
48
+
49
+ for o in outputs:
50
+ generated_text = o.outputs[0].text
51
+ print(generated_text)
52
+
vllm_plugin_meralion2/openai_client_curl.sh ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ curl http://localhost:8000/v1/chat/completions \
2
+ -H "Content-Type: application/json" \
3
+ -d '{
4
+ "model": "MERaLiON/MERaLiON-2-10B",
5
+ "messages": [
6
+ {"role": "user",
7
+ "content": [
8
+ {"type": "text", "text": "Instruction: {YOUR-INSTRUCTION-HERE} \nFollow the text instruction based on the following audio: <SpeechHere>"},
9
+ {"type": "audio_url", "audio_url": {"url": "data:audio/ogg;base64,{YOUR-AUDIO-BASE64-STRING}"}}
10
+ ]
11
+ }
12
+ ],
13
+ "max_completion_tokens": 1024,
14
+ "temperature": 0.0,
15
+ "top_p": 0.9,
16
+ "repetition_penalty": 1.1,
17
+ "top_k": 50,
18
+ "length_penalty": 1.0,
19
+ "logits_processors": [{"qualname": "vllm_plugin_meralion2.NoRepeatNGramLogitsProcessor", "args": []}],
20
+ "seed": 42
21
+ }'
vllm_plugin_meralion2/openai_client_example.py ADDED
@@ -0,0 +1,69 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import base64
2
+
3
+ from openai import OpenAI
4
+
5
+ prompt_template = "Instruction: {text_input} \nFollow the text instruction based on the following audio: <SpeechHere>"
6
+
7
+ def get_client(api_key="EMPTY", base_url="http://localhost:8000/v1"):
8
+ client = OpenAI(
9
+ api_key=api_key,
10
+ base_url=base_url,
11
+ )
12
+
13
+ models = client.models.list()
14
+ model_name = models.data[0].id
15
+ return client, model_name
16
+
17
+
18
+ def get_response(text_input, base64_audio_input=None, **params):
19
+ if base64_audio_input:
20
+ content = [
21
+ {
22
+ "type": "text",
23
+ "text": prompt_template.format(text_input=text_input)
24
+ },
25
+ {
26
+ "type": "audio_url",
27
+ "audio_url": {
28
+ "url": f"data:audio/ogg;base64,{base64_audio_input}"
29
+ },
30
+ },
31
+ ]
32
+ else:
33
+ content = text_input
34
+
35
+
36
+ response_obj = client.chat.completions.create(
37
+ messages=[{
38
+ "role": "user",
39
+ "content": content,
40
+ }],
41
+ **params
42
+ )
43
+ return response_obj
44
+
45
+
46
+ audio_bytes = open("/path/to/wav/or/mp3/file", "rb").read()
47
+ audio_base64 = base64.b64encode(audio_bytes).decode('utf-8')
48
+
49
+ # use the port number of your vllm service.
50
+ client, model_name = get_client(base_url="http://localhost:8000/v1")
51
+
52
+ generation_parameters = dict(
53
+ model=model_name,
54
+ max_completion_tokens=1024,
55
+ temperature=0.0,
56
+ top_p=0.9,
57
+ extra_body={
58
+ "repetition_penalty": 1.1,
59
+ "top_k": 50,
60
+ "length_penalty": 1.0,
61
+ "logits_processors": [
62
+ {"qualname": "vllm_plugin_meralion2.NoRepeatNGramLogitsProcessor", "args": []}
63
+ ]
64
+ },
65
+ seed=42
66
+ )
67
+
68
+ response_obj = get_response("Please transcribe this speech.", audio_base64, **generation_parameters)
69
+ print(response_obj.choices[0].message.content)
vllm_plugin_meralion2/openai_serve_example.sh ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ vllm serve MERaLiON/MERaLiON-2-10B \
2
+ --trust-remote-code \
3
+ --dtype bfloat16 \
4
+ --gpu-memory-utilization 0.78 \
5
+ --logits-processor-pattern vllm_plugin_meralion2.NoRepeatNGramLogitsProcessor \
6
+ --port 8004
vllm_plugin_meralion2/readme.md ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ## MERaLiON2 vLLM Plugin
2
+
3
+ ### Set up Environment
4
+
5
+ This vLLM plugin for MERaLiON2 requires transformers version `4.50.1`. It supports vLLM version `0.6.5` ~ `0.7.3` (V0 engine), and `0.8.5` ~ `0.8.5.post1` (V1 engine).
6
+
7
+ ```bash
8
+ pip install transformers==4.50.1
9
+ pip install vllm==0.6.5
10
+ ```
11
+
12
+ Install the MERaLiON2 vLLM plugin.
13
+
14
+ ```bash
15
+ python install .
16
+ ```
17
+
18
+ It's strongly recommended to install flash-attn for better memory and gpu utilization.
19
+
20
+ ```bash
21
+ pip install flash-attn --no-build-isolation
22
+ ```
23
+
24
+
25
+ ### Offline Inference
26
+
27
+ Refer to [offline_example.py](offline_example.py) for offline inference example.
28
+
29
+ ### OpenAI-compatible Serving
30
+
31
+ Refer to [openai_serve_example.sh](openai_serve_example.sh) for openAI-compatible serving example.
32
+
33
+ To call the server, you can refer to [openai_client_example.py](openai_client_example.py).
34
+
35
+ Alternatively, you can try calling the server with curl, refer to [openai_client_curl.sh](openai_client_curl.sh).
vllm_plugin_meralion2/setup.py ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ from setuptools import setup, find_packages
2
+
3
+ setup(name='vllm_plugin_meralion2',
4
+ version='0.1.0',
5
+ package_dir={"": "src"},
6
+ packages=find_packages(where="src"),
7
+ entry_points={
8
+ 'vllm.general_plugins':
9
+ ["register_dummy_model = vllm_plugin_meralion2:register"]
10
+ })
vllm_plugin_meralion2/vllm_plugin_meralion2/__init__.py ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Optional
2
+
3
+ from vllm.entrypoints.chat_utils import BaseMultiModalItemTracker
4
+
5
+ from .transformers_utils.no_repeat_logits_processor import NoRepeatNGramLogitsProcessor
6
+
7
+
8
+ _original_placeholder_str = BaseMultiModalItemTracker._placeholder_str
9
+
10
+
11
+ def custom_placeholder_str(self, modality,
12
+ current_count: int) -> Optional[str]:
13
+ hf_config = self._model_config.hf_config
14
+ model_type = hf_config.model_type
15
+
16
+ if modality == "audio" and model_type == "meralion2":
17
+ return "<SpeechHere>"
18
+
19
+ return _original_placeholder_str(self, modality=modality, current_count=current_count)
20
+
21
+
22
+ def register():
23
+ import vllm
24
+ from vllm import ModelRegistry
25
+
26
+ v064_compatible_versions = ['0.6.5', '0.6.6', '0.6.6.post1', '0.7.0', '0.7.1', '0.7.2', '0.7.3']
27
+ v085_compatible_versions = ['0.8.5', '0.8.5.post1']
28
+ sorted_compatible_versions = sorted(v064_compatible_versions + v085_compatible_versions)
29
+
30
+ if vllm.__version__ in v064_compatible_versions:
31
+ from .vllm064_post1 import MERaLiON2ForConditionalGeneration
32
+ elif vllm.__version__ in v085_compatible_versions:
33
+ from .vllm085 import MERaLiON2ForConditionalGeneration
34
+ else:
35
+ raise Exception((
36
+ f"MERaLiON2 doesn't support vLLM version {vllm.__version__}."
37
+ f" Supported vLLM versions: {', '.join((sorted_compatible_versions))}"
38
+ ))
39
+
40
+ if "MERaLiON2ForConditionalGeneration" not in ModelRegistry.get_supported_archs():
41
+ ModelRegistry.register_model(
42
+ "MERaLiON2ForConditionalGeneration",
43
+ MERaLiON2ForConditionalGeneration
44
+ )
45
+
46
+ vllm.entrypoints.chat_utils.BaseMultiModalItemTracker._placeholder_str = custom_placeholder_str
vllm_plugin_meralion2/vllm_plugin_meralion2/transformers_utils/__init__.py ADDED
File without changes
vllm_plugin_meralion2/vllm_plugin_meralion2/transformers_utils/configuration_meralion2.py ADDED
@@ -0,0 +1,76 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """MERaLiON2 model configuration"""
2
+
3
+ from transformers import Gemma2Config, WhisperConfig
4
+ from transformers.configuration_utils import PretrainedConfig
5
+ from transformers.utils import logging
6
+
7
+
8
+ logger = logging.get_logger(__name__)
9
+
10
+
11
+ class MERaLiON2Config(PretrainedConfig):
12
+ r"""
13
+ This is the configuration class to store the configuration of a [`MERaLiON2ForConditionalGeneration`]. It is used to instantiate an
14
+ MERaLiON2 model according to the specified arguments, defining the model architecture. Instantiating a configuration
15
+ with the defaults will yield a similar configuration to that of the MERaLiON2.
16
+
17
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
18
+ documentation from [`PretrainedConfig`] for more information.
19
+
20
+ Args:
21
+ audio_config (`Union[AutoConfig, dict]`, *optional*, defaults to `CLIPVisionConfig`):
22
+ The config object or dictionary of the audio backbone.
23
+ text_config (`Union[AutoConfig, dict]`, *optional*, defaults to `LlamaConfig`):
24
+ The config object or dictionary of the text backbone.
25
+ audio_token_index (`int`, *optional*, defaults to 151646):
26
+ The image token index to encode the image prompt.
27
+ """
28
+
29
+ model_type = "meralion2"
30
+ is_composition = False
31
+
32
+ def __init__(
33
+ self,
34
+ speech_config=None,
35
+ text_config=None,
36
+ speech_mlp_scale_factor=15,
37
+ speech_token_index=255999,
38
+ **kwargs,
39
+ ):
40
+
41
+ if isinstance(speech_config, dict):
42
+ speech_config = WhisperConfig(**speech_config)
43
+ elif speech_config is None:
44
+ speech_config = WhisperConfig(
45
+ d_model=1280,
46
+ encoder_attention_heads=20,
47
+ encoder_ffn_dim=5120,
48
+ encoder_layerdrop=0.0,
49
+ encoder_layers=32,
50
+ num_mel_bins=128,
51
+ max_source_positions=1500,
52
+ scale_embedding=False,
53
+ activation_function="gelu",
54
+ )
55
+
56
+ self.speech_config = speech_config
57
+
58
+ if isinstance(text_config, dict):
59
+ text_config = Gemma2Config(**text_config)
60
+ elif text_config is None:
61
+ text_config = Gemma2Config()
62
+
63
+ self.text_config = text_config
64
+
65
+ self.speech_mlp_scale_factor = speech_mlp_scale_factor
66
+ self.speech_token_index = speech_token_index
67
+
68
+ self.sliding_window = self.text_config.sliding_window
69
+ self.hidden_size = self.text_config.hidden_size
70
+ self.num_attention_heads = self.text_config.num_attention_heads
71
+ self.num_hidden_layers = self.text_config.num_hidden_layers
72
+ self.num_key_value_heads = self.text_config.num_key_value_heads
73
+ self.head_dim = self.text_config.head_dim
74
+ self.intermediate_size = self.text_config.intermediate_size
75
+
76
+ super().__init__(**kwargs)
vllm_plugin_meralion2/vllm_plugin_meralion2/transformers_utils/modules.py ADDED
@@ -0,0 +1,72 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Inference-only MERaLiON AudioLLM model compatible with HuggingFace weights."""
2
+ from typing import Any, Optional, Set, Tuple, TypedDict, Union, List
3
+
4
+ import torch
5
+ import torch.nn as nn
6
+ from transformers.utils.import_utils import is_torch_sdpa_available, is_flash_attn_2_available
7
+
8
+ # === Audio Inputs === #
9
+ class MERaLiON2Inputs(TypedDict):
10
+ input_features: torch.Tensor
11
+ """Shape:
12
+ `(num_audios, num_mel_bins, 3000)`
13
+ """
14
+
15
+ feature_attention_mask: torch.Tensor
16
+ """Shape: `(num_audios, 3000)`
17
+ """
18
+
19
+
20
+ # === Audio Encoder === #
21
+ class MERaLiON2SpeechAudioAdaper(nn.Module):
22
+ def __init__(self, audio_hidden_size: int, text_hidden_size: int):
23
+ super(MERaLiON2SpeechAudioAdaper, self).__init__()
24
+ speech_mlp_scale_factor = 15
25
+
26
+ self.speech_mlp_scale_factor = speech_mlp_scale_factor
27
+ self.mlp_adapter = nn.Sequential(
28
+ nn.Linear(
29
+ in_features=audio_hidden_size * speech_mlp_scale_factor,
30
+ out_features=audio_hidden_size * 5,
31
+ ),
32
+ nn.SiLU(),
33
+ )
34
+
35
+ self.gate_proj = nn.Linear(
36
+ in_features=audio_hidden_size * 5,
37
+ out_features=audio_hidden_size * 5,
38
+ )
39
+
40
+ self.pool_proj = nn.Linear(
41
+ in_features=audio_hidden_size * 5,
42
+ out_features=audio_hidden_size * 5,
43
+ )
44
+ self.act_fn = nn.SiLU()
45
+ self.out_proj = nn.Linear(
46
+ audio_hidden_size * 5,
47
+ text_hidden_size,
48
+ )
49
+
50
+ def forward(self, speech_embeds, **kwargs):
51
+ B, T, C = speech_embeds.shape
52
+ speech_embeds = self.mlp_adapter(
53
+ speech_embeds.reshape(
54
+ B,
55
+ T // self.speech_mlp_scale_factor,
56
+ C * self.speech_mlp_scale_factor,
57
+ )
58
+ )
59
+ speech_embeds = self.act_fn(self.gate_proj(speech_embeds)) * self.pool_proj(speech_embeds)
60
+ speech_embeds = self.out_proj(speech_embeds)
61
+ return speech_embeds
62
+
63
+
64
+ def autoset_attn_implementation_for_whisper(config):
65
+ _implementation = "eager"
66
+ if is_torch_sdpa_available():
67
+ _implementation = "sdpa"
68
+ if is_flash_attn_2_available():
69
+ _implementation = "flash_attention_2"
70
+
71
+ config._attn_implementation = _implementation
72
+ return config
vllm_plugin_meralion2/vllm_plugin_meralion2/transformers_utils/no_repeat_logits_processor.py ADDED
@@ -0,0 +1,98 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Iterable, List
2
+
3
+ import torch
4
+
5
+
6
+ def _get_ngrams(ngram_size: int, prev_input_ids: torch.Tensor, num_hypos: int):
7
+ """
8
+ Assume ngram_size=2 and prev_input_ids=tensor([[40, 2883, 2712, 4346]]). The output of generated ngrams look like
9
+ this {(40,): [2883], (2883,): [2712], (2712,): [4346]}.
10
+
11
+ Args:
12
+ ngram_size (`int`):
13
+ The number sequential tokens taken as a group which may only occur once before being banned.
14
+ prev_input_ids (`torch.Tensor`):
15
+ Generated token ids for the current hypothesis.
16
+ num_hypos (`int`):
17
+ The number of hypotheses for which n-grams need to be generated.
18
+
19
+ Returns:
20
+ generated_ngrams (`dict`):
21
+ Dictionary of generated ngrams.
22
+ """
23
+ # Initialize an empty list of dictionaries, one for each hypothesis (index) in the range of num_hypos
24
+ generated_ngrams = [{} for _ in range(num_hypos)]
25
+ for idx in range(num_hypos):
26
+ gen_tokens = prev_input_ids[idx].tolist()
27
+ generated_ngram = generated_ngrams[idx]
28
+ # Loop through each n-gram of size ngram_size in the list of tokens (gen_tokens)
29
+ for ngram in zip(*[gen_tokens[i:] for i in range(ngram_size)]):
30
+ prev_ngram_tuple = tuple(ngram[:-1])
31
+ generated_ngram[prev_ngram_tuple] = generated_ngram.get(prev_ngram_tuple, []) + [ngram[-1]]
32
+ return generated_ngrams
33
+
34
+
35
+ def _get_generated_ngrams(banned_ngrams, prev_input_ids, ngram_size, cur_len):
36
+ """
37
+ Determines the banned tokens for the current hypothesis based on previously generated n-grams.
38
+
39
+ Args:
40
+ banned_ngrams (`dict`):
41
+ A dictionary containing previously generated n-grams for each hypothesis.
42
+ prev_input_ids (`torch.Tensor`):
43
+ Generated token ids for the current hypothesis.
44
+ ngram_size (`int`):
45
+ The number sequential tokens taken as a group which may only occur once before being banned.
46
+ cur_len (`int`):
47
+ The current length of the token sequences for which the n-grams are being checked.
48
+
49
+ Returns:
50
+ List of tokens that are banned.
51
+ """
52
+ # Before decoding the next token, prevent decoding of ngrams that have already appeared
53
+ start_idx = cur_len + 1 - ngram_size
54
+ ngram_idx = tuple(prev_input_ids[start_idx:cur_len].tolist())
55
+ return banned_ngrams.get(ngram_idx, [])
56
+
57
+
58
+ def _calc_banned_ngram_tokens(
59
+ ngram_size: int, prev_input_ids: torch.Tensor, num_hypos: int, cur_len: int
60
+ ) -> List[Iterable[int]]:
61
+ """Copied from fairseq for no_repeat_ngram in beam_search"""
62
+ if cur_len + 1 < ngram_size:
63
+ # return no banned tokens if we haven't generated no_repeat_ngram_size tokens yet
64
+ return [[] for _ in range(num_hypos)]
65
+ generated_ngrams = _get_ngrams(ngram_size, prev_input_ids, num_hypos)
66
+ banned_tokens = [
67
+ _get_generated_ngrams(generated_ngrams[hypo_idx], prev_input_ids[hypo_idx], ngram_size, cur_len)
68
+ for hypo_idx in range(num_hypos)
69
+ ]
70
+ return banned_tokens
71
+
72
+
73
+ class NoRepeatNGramLogitsProcessor:
74
+
75
+ def __init__(self, ngram_size: int=4):
76
+ if not isinstance(ngram_size, int) or ngram_size <= 0:
77
+ raise ValueError(f"`ngram_size` has to be a strictly positive integer, but is {ngram_size}")
78
+ self.ngram_size = ngram_size
79
+
80
+ def __call__(self, prompt_tokens_ids: tuple, past_tokens_ids: tuple, scores: torch.FloatTensor) -> torch.FloatTensor:
81
+ # score: [B, vocab_size]
82
+ # input_ids: [B, cur_len]
83
+ input_ids = prompt_tokens_ids + past_tokens_ids
84
+ if len(input_ids) < self.ngram_size:
85
+ return scores
86
+
87
+ if len(scores.shape) == 1:
88
+ scores = scores.reshape(1, -1)
89
+
90
+ num_batch_hypotheses = scores.shape[0]
91
+ input_ids = torch.LongTensor(input_ids).reshape(num_batch_hypotheses, -1)
92
+ cur_len = input_ids.shape[-1]
93
+ scores_processed = scores.clone()
94
+ banned_batch_tokens = _calc_banned_ngram_tokens(self.ngram_size, input_ids, num_batch_hypotheses, cur_len)
95
+ for i, banned_tokens in enumerate(banned_batch_tokens):
96
+ scores_processed[i, banned_tokens] = -float("inf")
97
+
98
+ return scores_processed
vllm_plugin_meralion2/vllm_plugin_meralion2/transformers_utils/processing_meralion2.py ADDED
@@ -0,0 +1,193 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Processor class for MERaLiON2."""
2
+
3
+ from typing import List, Optional, Union
4
+
5
+ import numpy as np
6
+
7
+ from transformers.feature_extraction_utils import BatchFeature
8
+ from transformers.processing_utils import ProcessorMixin
9
+ from transformers.tokenization_utils_base import PaddingStrategy, PreTokenizedInput, TextInput
10
+
11
+
12
+ # copied from transformers.models.qwen2_audio.processing_qwen2_audio.Qwen2AudioProcessor
13
+ class MERaLiON2Processor(ProcessorMixin):
14
+ r"""
15
+ Constructs a MERaLiON2 processor which wraps a whisper feature extractor and a gemma tokenizer into a single processor.
16
+
17
+ [`MERaLiON2Processor`] offers all the functionalities of [`WhisperFeatureExtractor`] and [`GemmaTokenizer`]. See the
18
+ [`~MERaLiON2Processor.__call__`] and [`~MERaLiON2Processor.decode`] for more information.
19
+
20
+ Args:
21
+ feature_extractor ([`WhisperFeatureExtractor`], *optional*):
22
+ The feature extractor is a required input.
23
+ tokenizer ([`GemmaTokenizer`], *optional*):
24
+ The tokenizer is a required input.
25
+ chat_template (`Optional[str]`, *optional*):
26
+ The Jinja template to use for formatting the conversation. If not provided, the default chat template
27
+ is used.
28
+ """
29
+
30
+ attributes = ["feature_extractor", "tokenizer"]
31
+ feature_extractor_class = "WhisperFeatureExtractor"
32
+ tokenizer_class = "AutoTokenizer"
33
+ valid_kwargs = [
34
+ "fixed_speech_embeds_length",
35
+ "speech_token_index",
36
+ "time_duration_limit",
37
+ "whisper_chunk_size",
38
+ "do_normalize"
39
+ ]
40
+
41
+ def __init__(
42
+ self,
43
+ feature_extractor=None,
44
+ tokenizer=None,
45
+ fixed_speech_embeds_length=100,
46
+ speech_token_index=255999,
47
+ time_duration_limit=300,
48
+ whisper_chunk_size=30,
49
+ do_normalize=True
50
+ ):
51
+ self.fixed_speech_embeds_length = fixed_speech_embeds_length
52
+ self.speech_token_index = speech_token_index
53
+ self.time_duration_limit = time_duration_limit
54
+ self.whisper_chunk_size = whisper_chunk_size
55
+ self.number_chunk_limit = self.time_duration_limit // self.whisper_chunk_size
56
+ self.do_normalize = do_normalize
57
+
58
+ super().__init__(feature_extractor, tokenizer)
59
+
60
+ self.speech_token = self.tokenizer.added_tokens_decoder[self.speech_token_index].content
61
+ self.feature_chunk_size = self.whisper_chunk_size * self.feature_extractor.sampling_rate
62
+
63
+ def _process_text(self, text: List[str], audio_number_chunks: np.ndarray):
64
+ pieces = []
65
+ for i, item in enumerate(text):
66
+ target_string = self.speech_token * self.fixed_speech_embeds_length * audio_number_chunks[i]
67
+ pieces.append(item.replace(self.speech_token, target_string))
68
+ return pieces
69
+
70
+ def _get_number_chunks(self, audios: List[np.ndarray]):
71
+ audio_lengths = np.array([_.shape[0] for _ in audios])
72
+ number_chunks = (audio_lengths // self.feature_chunk_size) + 1
73
+ return np.clip(number_chunks, a_min=None, a_max=self.number_chunk_limit)
74
+
75
+ def _get_chunked_audios(self, audios: Union[np.ndarray, List[np.ndarray]]):
76
+ if isinstance(audios, np.ndarray):
77
+ audios = [audios]
78
+
79
+ audio_number_chunks = self._get_number_chunks(audios)
80
+ chunked_audios = []
81
+
82
+ for audio_idx, audio in enumerate(audios):
83
+ for cid in range(audio_number_chunks[audio_idx]):
84
+ chunked_audios.append(
85
+ audio[cid * self.feature_chunk_size: (cid + 1) * self.feature_chunk_size]
86
+ )
87
+ return audio_number_chunks, chunked_audios
88
+
89
+ def __call__(
90
+ self,
91
+ text: Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None,
92
+ audios: Union[np.ndarray, List[np.ndarray]] = None,
93
+ padding: Union[bool, str, PaddingStrategy] = True,
94
+ sampling_rate: Optional[int] = None,
95
+ do_normalize: Optional[bool] = None,
96
+ **kwargs,
97
+ ) -> BatchFeature:
98
+ """
99
+ Main method to prepare for the model one or several sequences(s) and audio(s). This method forwards the `text`
100
+ and `kwargs` arguments to GemmaTokenizer's [`~GemmaTokenizer.__call__`] if `text` is not `None` to encode
101
+ the text. To prepare the audio(s), this method forwards the `audios` and `kwrags` arguments to
102
+ WhisperFeatureExtractor's [`~WhisperFeatureExtractor.__call__`] if `audios` is not `None`. Please refer to the doctsring
103
+ of the above two methods for more information.
104
+
105
+ Args:
106
+ text (`str`, `List[str]`):
107
+ The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings
108
+ (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set
109
+ `is_split_into_words=True` (to lift the ambiguity with a batch of sequences).
110
+ audios (`np.ndarray`, `List[np.ndarray]`):
111
+ The audio or batch of audios to be prepared. Each audio can be a NumPy array.
112
+ padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `True`):
113
+ Select a strategy to pad the returned sequences (according to the model's padding side and padding
114
+ index) among:
115
+ - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
116
+ sequence if provided).
117
+ - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
118
+ acceptable input length for the model if that argument is not provided.
119
+ - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
120
+ lengths).
121
+ sampling_rate (`int`, defaults to 16000):
122
+ The sampling rate at which the audio files should be digitalized expressed in hertz (Hz).
123
+ do_normalize (`bool`, defaults to `True`):
124
+ Whether or not to zero-mean unit-variance normalize the input.
125
+ Normalizing can help to significantly improve the performance of the model.
126
+ """
127
+
128
+ if text is None:
129
+ raise ValueError("You need to specify either a `text` input to process.")
130
+ if not isinstance(text, list):
131
+ text = [text]
132
+ if not isinstance(audios, list):
133
+ audios = [audios]
134
+ if sampling_rate is None:
135
+ sampling_rate = self.feature_extractor.sampling_rate
136
+ if do_normalize is None:
137
+ do_normalize = self.do_normalize
138
+
139
+ for i, audio in enumerate(audios):
140
+ if audio.ndim > 1:
141
+ raise Exception(f"MERaLiON2 only accepts mono channel audio, {i+1}th audio have {audios[0].ndim} channels")
142
+
143
+ inputs_dict = {}
144
+
145
+ if audios is not None:
146
+ audio_number_chunks, chunked_audios = self._get_chunked_audios(audios)
147
+ text = self._process_text(text, audio_number_chunks)
148
+
149
+ audio_inputs = self.feature_extractor(
150
+ chunked_audios,
151
+ sampling_rate=sampling_rate,
152
+ return_tensors="pt",
153
+ return_attention_mask=True,
154
+ padding="max_length",
155
+ do_normalize=self.do_normalize,
156
+ )
157
+ audio_inputs["feature_attention_mask"] = audio_inputs.pop(
158
+ "attention_mask"
159
+ ) # rename attention_mask to prevent conflicts later on
160
+ inputs_dict.update(audio_inputs)
161
+
162
+ text_input = self.tokenizer(
163
+ text=text,
164
+ return_tensors="pt",
165
+ add_special_tokens=False,
166
+ return_attention_mask=True,
167
+ padding=padding,
168
+ )
169
+
170
+ inputs_dict["input_ids"] = text_input.input_ids
171
+ inputs_dict["attention_mask"] = text_input.attention_mask
172
+
173
+ return BatchFeature(data={**inputs_dict})
174
+
175
+ def batch_decode(self, *args, **kwargs):
176
+ """
177
+ This method forwards all its arguments to GemmaTokenizer's [`~PreTrainedTokenizer.batch_decode`]. Please
178
+ refer to the docstring of this method for more information.
179
+ """
180
+ return self.tokenizer.batch_decode(*args, **kwargs)
181
+
182
+ def decode(self, *args, **kwargs):
183
+ """
184
+ This method forwards all its arguments to GemmaTokenizer's [`~PreTrainedTokenizer.decode`]. Please refer to
185
+ the docstring of this method for more information.
186
+ """
187
+ return self.tokenizer.decode(*args, **kwargs)
188
+
189
+ @property
190
+ def model_input_names(self):
191
+ tokenizer_input_names = self.tokenizer.model_input_names
192
+ feature_extractor_input_names = self.feature_extractor.model_input_names
193
+ return list(dict.fromkeys(tokenizer_input_names + feature_extractor_input_names + ["feature_attention_mask"]))
vllm_plugin_meralion2/vllm_plugin_meralion2/vllm064_post1.py ADDED
@@ -0,0 +1,453 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Inference-only MERaLiON AudioLLM model compatible with HuggingFace weights."""
2
+ from functools import lru_cache
3
+ from typing import Iterable, List, Mapping, Optional, Tuple, Union
4
+
5
+ import librosa
6
+ import numpy as np
7
+ import torch
8
+ import torch.nn as nn
9
+
10
+ from vllm.attention import AttentionMetadata
11
+ from vllm.config import VllmConfig
12
+ from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
13
+ InputContext, token_inputs)
14
+ from vllm.logger import init_logger
15
+ from vllm.model_executor.layers.logits_processor import LogitsProcessor
16
+ from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
17
+ from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
18
+ from vllm.model_executor.model_loader.weight_utils import (
19
+ default_weight_loader, maybe_remap_kv_scale_name)
20
+ from vllm.model_executor.models.gemma2 import Gemma2Model
21
+ from vllm.model_executor.sampling_metadata import SamplingMetadata
22
+ from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
23
+ from vllm.multimodal.utils import consecutive_placeholder_ranges
24
+ from vllm.sequence import IntermediateTensors, SequenceData
25
+ from transformers.models.whisper.modeling_whisper import WhisperEncoder
26
+
27
+ from vllm.model_executor.models.interfaces import SupportsMultiModal, SupportsLoRA, SupportsPP
28
+ from vllm.model_executor.models.utils import maybe_prefix
29
+
30
+ from .transformers_utils.modules import (autoset_attn_implementation_for_whisper,
31
+ MERaLiON2Inputs, MERaLiON2SpeechAudioAdaper)
32
+
33
+
34
+ logger = init_logger(__name__)
35
+
36
+
37
+ # gemma2 ties word embedding by default
38
+ _KEYS_TO_MODIFY_MAPPING = {
39
+ "text_decoder.model": "model",
40
+ }
41
+
42
+ # === Constants === #
43
+ DEFAULT_SAMPLE_RATE = 16000
44
+ FEATURE_CHUNK_SIZE = DEFAULT_SAMPLE_RATE * 30
45
+ OUTPUT_CHUNK_SIZE = 100
46
+ MAX_NUMBER_CHUNKS = 8
47
+
48
+
49
+ def dummy_data_for_meralion(ctx: InputContext, seq_len: int,
50
+ mm_counts: Mapping[str, int]):
51
+ num_audios = mm_counts["audio"]
52
+ max_tokens_per_audio = get_max_meralion_audio_tokens(ctx)
53
+ max_llm_audio_tokens = max_tokens_per_audio * num_audios
54
+ if seq_len - max_llm_audio_tokens - 2 < 0:
55
+ raise RuntimeError(
56
+ f"MERaLiON-AudioLLM cannot process {num_audios} audios in a prompt, "
57
+ "please increase max_model_len or reduce audio limit by "
58
+ "--limit-mm-per-prompt.")
59
+
60
+ speech_token_index = ctx.model_config.hf_config.speech_token_index
61
+
62
+ dummy_seqdata = SequenceData.from_prompt_token_counts(
63
+ (speech_token_index, max_llm_audio_tokens),
64
+ (0, seq_len - max_llm_audio_tokens),
65
+ )
66
+ dummy_audio = np.full((max_llm_audio_tokens * 15 * 2 * 160, ), 0.)
67
+ return DummyData(
68
+ dummy_seqdata, {"audio": [(dummy_audio, DEFAULT_SAMPLE_RATE)] * num_audios}, {
69
+ "audio":
70
+ consecutive_placeholder_ranges(num_items=num_audios,
71
+ item_size=max_tokens_per_audio)
72
+ })
73
+
74
+
75
+ def get_processor(
76
+ processor_name: str,
77
+ *args,
78
+ trust_remote_code: bool = True,
79
+ **kwargs,
80
+ ):
81
+ """Gets a processor for the given model name via HuggingFace.
82
+
83
+ Derived from `vllm.transformers_utils.image_processor.get_image_processor`.
84
+ """
85
+ # don't put this import at the top level
86
+ # it will call torch.cuda.device_count()
87
+ from transformers import AutoProcessor
88
+
89
+ try:
90
+ processor = AutoProcessor.from_pretrained(
91
+ processor_name,
92
+ *args,
93
+ trust_remote_code=trust_remote_code,
94
+ **kwargs)
95
+ except ValueError as e:
96
+ # If the error pertains to the processor class not existing or not
97
+ # currently being imported, suggest using the --trust-remote-code flag.
98
+ # Unlike AutoTokenizer, AutoProcessor does not separate such errors
99
+ if not trust_remote_code:
100
+ err_msg = (
101
+ "Failed to load the processor. If the processor is "
102
+ "a custom processor not yet available in the HuggingFace "
103
+ "transformers library, consider setting "
104
+ "`trust_remote_code=True` in LLM or using the "
105
+ "`--trust-remote-code` flag in the CLI.")
106
+ raise RuntimeError(err_msg) from e
107
+ else:
108
+ raise e
109
+
110
+ return processor
111
+
112
+
113
+ cached_get_processor = lru_cache(get_processor)
114
+
115
+
116
+ def _get_number_chunks(audios: List[np.ndarray]):
117
+ audio_lengths = np.array([_.shape[0] for _ in audios])
118
+ number_chunks = (audio_lengths // FEATURE_CHUNK_SIZE) + 1
119
+ return np.clip(number_chunks, a_min=None, a_max=MAX_NUMBER_CHUNKS)
120
+
121
+
122
+ def _get_feat_extract_output_lengths(audios: List[np.ndarray]):
123
+ return _get_number_chunks(audios) * OUTPUT_CHUNK_SIZE
124
+
125
+
126
+ def _get_chunked_audios(audios: List[np.ndarray]):
127
+ audio_number_chunks = _get_number_chunks(audios)
128
+ chunked_resampled_audios = []
129
+
130
+ for audio_idx, audio in enumerate(audios):
131
+ for cid in range(audio_number_chunks[audio_idx]):
132
+ chunked_resampled_audios.append(
133
+ audio[cid * FEATURE_CHUNK_SIZE: (cid + 1) * FEATURE_CHUNK_SIZE]
134
+ )
135
+ return chunked_resampled_audios
136
+
137
+
138
+ def _maybe_resample_audio(audio, orig_sample_rate, target_sample_rate):
139
+ if orig_sample_rate != target_sample_rate:
140
+ return librosa.resample(
141
+ audio,
142
+ orig_sr=orig_sample_rate,
143
+ target_sr=target_sample_rate
144
+ )
145
+ return audio
146
+
147
+
148
+ def get_max_meralion_audio_tokens(ctx: InputContext) -> int:
149
+ """
150
+ The max number of tokens after speech audio adapter.
151
+ """
152
+ return MAX_NUMBER_CHUNKS * OUTPUT_CHUNK_SIZE
153
+
154
+
155
+ def input_processor_for_meralion(
156
+ ctx: InputContext, inputs: DecoderOnlyInputs) -> DecoderOnlyInputs:
157
+ multi_modal_data = inputs.get("multi_modal_data")
158
+ if multi_modal_data is None or "audio" not in multi_modal_data:
159
+ return inputs
160
+
161
+ audios = multi_modal_data["audio"]
162
+ if not isinstance(audios, list):
163
+ audios = [audios]
164
+
165
+ if len(audios) == 0:
166
+ return inputs
167
+
168
+ processor = cached_get_processor(ctx.model_config.model)
169
+
170
+ resampled_audios = [
171
+ librosa.resample(audio,
172
+ orig_sr=sampling_rate,
173
+ target_sr=processor.feature_extractor.sampling_rate)
174
+ for audio, sampling_rate in audios
175
+ ]
176
+
177
+ audio_output_lengths = _get_feat_extract_output_lengths(resampled_audios)
178
+ speech_token_index = ctx.model_config.hf_config.speech_token_index
179
+
180
+ input_ids = inputs['prompt_token_ids']
181
+
182
+ new_input_ids = []
183
+ audio_num = input_ids.count(speech_token_index)
184
+ assert len(audio_output_lengths) == audio_num, \
185
+ (f'The text input contains {audio_num} audio tokens, '
186
+ f'but {len(audio_output_lengths)} audios provided')
187
+ start = 0
188
+ for audio_idx in range(audio_num):
189
+ end = input_ids.index(speech_token_index, start)
190
+ new_input_ids.extend(input_ids[start:end]) # text part
191
+
192
+ new_input_ids.extend([speech_token_index] *
193
+ audio_output_lengths[audio_idx])
194
+ start = end + 1
195
+ new_input_ids.extend(input_ids[start:])
196
+
197
+ return token_inputs(
198
+ prompt_token_ids=new_input_ids,
199
+ prompt=inputs.get('prompt'),
200
+ multi_modal_data=multi_modal_data,
201
+ )
202
+
203
+
204
+ def input_mapper_for_meralion(
205
+ ctx: InputContext,
206
+ multi_modal_data: Union[np.ndarray, List[np.ndarray]],
207
+ ) -> MultiModalKwargs:
208
+ """Input mapper for MERaLiON-AudioLLM."""
209
+ if not isinstance(multi_modal_data, list):
210
+ multi_modal_data = [multi_modal_data]
211
+
212
+ if len(multi_modal_data) == 0:
213
+ return MultiModalKwargs()
214
+
215
+ processor = cached_get_processor(ctx.model_config.model)
216
+ audio_feature_extractor = processor.feature_extractor
217
+ if audio_feature_extractor is None:
218
+ raise RuntimeError(
219
+ "No HuggingFace audio_feature_extractor is available "
220
+ "to process the audio object")
221
+
222
+ try:
223
+ target_sample_rate = processor.feature_extractor.sampling_rate
224
+
225
+ resampled_audios = [
226
+ _maybe_resample_audio(
227
+ audio=audio,
228
+ orig_sample_rate=sampling_rate,
229
+ target_sample_rate=target_sample_rate,
230
+ )
231
+ for audio, sampling_rate in multi_modal_data
232
+ ]
233
+
234
+ resampled_audios = _get_chunked_audios(resampled_audios)
235
+
236
+ batch_data = audio_feature_extractor(resampled_audios,
237
+ sampling_rate=target_sample_rate,
238
+ return_attention_mask=True,
239
+ padding="max_length",
240
+ return_tensors="pt").data
241
+ batch_data["feature_attention_mask"] = batch_data.pop("attention_mask")
242
+ except Exception:
243
+ logger.error("Failed to process audio (%s)", multi_modal_data)
244
+ raise
245
+
246
+ return MultiModalKwargs(batch_data)
247
+
248
+
249
+ @INPUT_REGISTRY.register_dummy_data(dummy_data_for_meralion)
250
+ @INPUT_REGISTRY.register_input_processor(input_processor_for_meralion)
251
+ @MULTIMODAL_REGISTRY.register_input_mapper("audio",
252
+ input_mapper_for_meralion)
253
+ @MULTIMODAL_REGISTRY.register_max_multimodal_tokens(
254
+ "audio", get_max_meralion_audio_tokens)
255
+ class MERaLiON2ForConditionalGeneration(nn.Module, SupportsMultiModal,
256
+ SupportsLoRA, SupportsPP):
257
+ packed_modules_mapping = {
258
+ "qkv_proj": [
259
+ "q_proj",
260
+ "k_proj",
261
+ "v_proj",
262
+ ],
263
+ "gate_up_proj": [
264
+ "gate_proj",
265
+ "up_proj",
266
+ ],
267
+ }
268
+
269
+ # LoRA specific attributes
270
+ supported_lora_modules = [
271
+ "qkv_proj",
272
+ "o_proj",
273
+ "gate_up_proj",
274
+ "down_proj",
275
+ ]
276
+
277
+ embedding_modules = {}
278
+ embedding_padding_modules = []
279
+
280
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
281
+ super().__init__()
282
+ config = vllm_config.model_config.hf_config
283
+ quant_config = vllm_config.quant_config
284
+ multimodal_config = vllm_config.model_config.multimodal_config
285
+
286
+ self.config = config
287
+ self.multimodal_config = multimodal_config
288
+
289
+ config.speech_config = \
290
+ autoset_attn_implementation_for_whisper(config.speech_config)
291
+ self.speech_encoder = WhisperEncoder(config.speech_config)
292
+ self.ln_speech = nn.LayerNorm(config.speech_config.d_model)
293
+ self.speech_audio_adapter = MERaLiON2SpeechAudioAdaper(
294
+ config.speech_config.d_model, config.text_config.hidden_size)
295
+
296
+ self.quant_config = quant_config
297
+
298
+ self.model = Gemma2Model(
299
+ vllm_config=vllm_config.with_hf_config(config.text_config),
300
+ prefix=maybe_prefix(prefix, "model"))
301
+ self.unpadded_vocab_size = config.text_config.vocab_size
302
+ if config.text_config.tie_word_embeddings:
303
+ self.lm_head = self.model.embed_tokens
304
+ else:
305
+ self.lm_head = ParallelLMHead(config.text_config.vocab_size,
306
+ config.text_config.hidden_size,
307
+ quant_config=quant_config)
308
+ logit_scale = getattr(config, "logit_scale", 1.0)
309
+ self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
310
+ config.text_config.vocab_size,
311
+ logit_scale)
312
+
313
+ self.sampler = get_sampler()
314
+ self.make_empty_intermediate_tensors = (
315
+ self.model.make_empty_intermediate_tensors)
316
+
317
+ def _validate_and_reshape_mm_tensor(self,
318
+ mm_input: Union[torch.Tensor,
319
+ List[torch.Tensor]],
320
+ name: str) -> torch.Tensor:
321
+ if not isinstance(mm_input, (torch.Tensor, list)):
322
+ raise ValueError(f"Incorrect type of {name}. "
323
+ f"Got type: {type(mm_input)}")
324
+ if isinstance(mm_input, torch.Tensor):
325
+ return torch.concat(list(mm_input))
326
+ else:
327
+ return torch.concat(mm_input)
328
+
329
+ def _parse_and_validate_audio_input(
330
+ self, **kwargs: object) -> Optional[MERaLiON2Inputs]:
331
+ input_features = kwargs.pop('input_features', None)
332
+ feature_attention_mask = kwargs.pop('feature_attention_mask', None)
333
+ if input_features is None:
334
+ return None
335
+ input_features = self._validate_and_reshape_mm_tensor(
336
+ input_features, 'input_features')
337
+ feature_attention_mask = self._validate_and_reshape_mm_tensor(
338
+ feature_attention_mask, 'feature_attention_mask')
339
+ if not isinstance(input_features, (torch.Tensor, list)):
340
+ raise ValueError("Incorrect type of audio input features. "
341
+ f"Got type: {type(input_features)}")
342
+ return MERaLiON2Inputs(input_features=input_features,
343
+ feature_attention_mask=feature_attention_mask)
344
+
345
+ def _process_audio_input(self,
346
+ audio_input: MERaLiON2Inputs) -> torch.Tensor:
347
+
348
+ input_features = audio_input["input_features"].to(self.speech_encoder.dtype)
349
+ feature_attention_mask = audio_input["feature_attention_mask"]
350
+
351
+ audio_outputs = self.speech_encoder(input_features,
352
+ attention_mask=feature_attention_mask)
353
+ audio_features = audio_outputs.last_hidden_state
354
+ audio_features = self.ln_speech(audio_features)
355
+ audio_features = self.speech_audio_adapter(audio_features)
356
+ audio_features = audio_features.view(-1, audio_features.size(-1))
357
+
358
+ return audio_features
359
+
360
+ def forward(
361
+ self,
362
+ input_ids: torch.Tensor,
363
+ positions: torch.Tensor,
364
+ kv_caches: List[torch.Tensor],
365
+ attn_metadata: AttentionMetadata,
366
+ intermediate_tensors: Optional[IntermediateTensors] = None,
367
+ **kwargs: object,
368
+ ) -> Union[torch.Tensor, IntermediateTensors]:
369
+ if intermediate_tensors is not None:
370
+ input_ids = None
371
+ inputs_embeds = None
372
+ else:
373
+ audio_input = self._parse_and_validate_audio_input(**kwargs)
374
+
375
+ if audio_input is None:
376
+ inputs_embeds = None
377
+ else:
378
+ inputs_embeds = self.model.embed_tokens(input_ids)
379
+ processed_audio_features = self._process_audio_input(audio_input)
380
+ # merge llm embeddings and audio features
381
+ mask = (input_ids == self.config.speech_token_index)
382
+ inputs_embeds[mask, :] = processed_audio_features
383
+
384
+ input_ids = None
385
+
386
+ hidden_states = self.model(
387
+ input_ids=input_ids,
388
+ positions=positions,
389
+ kv_caches=kv_caches,
390
+ attn_metadata=attn_metadata,
391
+ intermediate_tensors=intermediate_tensors,
392
+ inputs_embeds=inputs_embeds,
393
+ )
394
+ return hidden_states
395
+
396
+ def compute_logits(self, hidden_states: torch.Tensor,
397
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
398
+ logits = self.logits_processor(self.lm_head, hidden_states,
399
+ sampling_metadata)
400
+ return logits
401
+
402
+ def sample(
403
+ self,
404
+ logits: torch.Tensor,
405
+ sampling_metadata: SamplingMetadata,
406
+ ) -> Optional[SamplerOutput]:
407
+ next_tokens = self.sampler(logits, sampling_metadata)
408
+ return next_tokens
409
+
410
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
411
+ stacked_params_mapping = [
412
+ # (param_name, shard_name, shard_id)
413
+ ("qkv_proj", "q_proj", "q"),
414
+ ("qkv_proj", "k_proj", "k"),
415
+ ("qkv_proj", "v_proj", "v"),
416
+ ("gate_up_proj", "gate_proj", 0),
417
+ ("gate_up_proj", "up_proj", 1),
418
+ ]
419
+ params_dict = dict(self.named_parameters(remove_duplicate=False))
420
+
421
+ for name, loaded_weight in weights:
422
+ if "rotary_emb.inv_freq" in name:
423
+ continue
424
+ if (self.config.text_config.tie_word_embeddings
425
+ and "lm_head.weight" in name):
426
+ continue
427
+ for key_to_modify, new_key in _KEYS_TO_MODIFY_MAPPING.items():
428
+ if key_to_modify in name:
429
+ name = name.replace(key_to_modify, new_key)
430
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
431
+ if weight_name not in name or 'speech_' in name:
432
+ continue
433
+ name = name.replace(weight_name, param_name)
434
+ # Skip loading extra bias for GPTQ models.
435
+ if name.endswith(".bias") and name not in params_dict:
436
+ continue
437
+ param = params_dict[name]
438
+ weight_loader = param.weight_loader
439
+ weight_loader(param, loaded_weight, shard_id)
440
+ break
441
+ else:
442
+ # Skip loading extra bias for GPTQ models.
443
+ if name.endswith(".bias") and name not in params_dict:
444
+ continue
445
+ # Remapping the name of FP8 kv-scale.
446
+ name = maybe_remap_kv_scale_name(name, params_dict)
447
+ if name is None:
448
+ continue
449
+
450
+ param = params_dict[name]
451
+ weight_loader = getattr(param, "weight_loader",
452
+ default_weight_loader)
453
+ weight_loader(param, loaded_weight)
vllm_plugin_meralion2/vllm_plugin_meralion2/vllm085.py ADDED
@@ -0,0 +1,333 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Inference-only MERaLiON AudioLLM model compatible with HuggingFace weights."""
2
+ from collections.abc import Iterable, Mapping, Sequence
3
+ from typing import Any, Optional, Set, Tuple, TypedDict, Union, List
4
+
5
+ import math
6
+ import torch
7
+ import torch.nn as nn
8
+ from transformers import BatchFeature
9
+ from transformers.models.whisper.modeling_whisper import WhisperEncoder
10
+ from transformers.models.whisper.feature_extraction_whisper import WhisperFeatureExtractor
11
+
12
+ from vllm.config import VllmConfig
13
+ from vllm.model_executor.sampling_metadata import SamplingMetadata
14
+ from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
15
+ from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
16
+ MultiModalKwargs)
17
+ from vllm.multimodal.parse import (AudioProcessorItems, MultiModalDataItems,
18
+ MultiModalDataParser)
19
+ from vllm.multimodal.processing import (BaseMultiModalProcessor,
20
+ BaseProcessingInfo, PromptReplacement,
21
+ PromptUpdate, PromptUpdateDetails)
22
+ from vllm.multimodal.profiling import BaseDummyInputsBuilder
23
+ from vllm.sequence import IntermediateTensors
24
+ from vllm.model_executor.models.interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
25
+ from vllm.model_executor.models.utils import (AutoWeightsLoader, init_vllm_registered_model,
26
+ maybe_prefix, merge_multimodal_embeddings)
27
+
28
+ from .transformers_utils.processing_meralion2 import MERaLiON2Processor
29
+ from .transformers_utils.configuration_meralion2 import MERaLiON2Config
30
+ from .transformers_utils.modules import (autoset_attn_implementation_for_whisper,
31
+ MERaLiON2Inputs, MERaLiON2SpeechAudioAdaper)
32
+
33
+
34
+ class MERaLiON2ProcessingInfo(BaseProcessingInfo):
35
+
36
+ def get_hf_config(self):
37
+ return self.ctx.get_hf_config(MERaLiON2Config)
38
+
39
+ def get_hf_processor(
40
+ self,
41
+ *,
42
+ # Ignored in initialization
43
+ sampling_rate: Optional[int] = None,
44
+ **kwargs: object,
45
+ ) -> MERaLiON2Processor:
46
+ return self.ctx.get_hf_processor(MERaLiON2Processor, **kwargs)
47
+
48
+ def get_feature_extractor(
49
+ self,
50
+ *,
51
+ # Ignored in initialization
52
+ sampling_rate: Optional[int] = None,
53
+ ) -> WhisperFeatureExtractor:
54
+ hf_processor = self.get_hf_processor(sampling_rate=sampling_rate)
55
+ feature_extractor = hf_processor.feature_extractor # type: ignore
56
+ assert isinstance(feature_extractor, WhisperFeatureExtractor)
57
+ return feature_extractor
58
+
59
+ def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
60
+ return {"audio": None}
61
+
62
+
63
+ class MERaLiON2DummyInputsBuilder(
64
+ BaseDummyInputsBuilder[MERaLiON2ProcessingInfo]):
65
+
66
+ def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
67
+ num_audios = mm_counts.get("audio", 0)
68
+
69
+ hf_processor = self.info.get_hf_processor()
70
+ audio_token = hf_processor.speech_token
71
+
72
+ return audio_token * num_audios
73
+
74
+ def get_dummy_mm_data(
75
+ self,
76
+ seq_len: int,
77
+ mm_counts: Mapping[str, int],
78
+ ) -> MultiModalDataDict:
79
+ processor = self.info.get_hf_processor()
80
+ feature_extractor = self.info.get_feature_extractor()
81
+
82
+ # This is to specify the audio length
83
+ num_chunk_limit = processor.number_chunk_limit
84
+ sampling_rate = feature_extractor.sampling_rate
85
+ audio_len = num_chunk_limit * feature_extractor.chunk_length * sampling_rate
86
+ num_audios = mm_counts.get("audio", 0)
87
+
88
+ return {
89
+ "audio":
90
+ self._get_dummy_audios(length=audio_len, num_audios=num_audios)
91
+ }
92
+
93
+
94
+ class MERaLiON2MultiModalProcessor(
95
+ BaseMultiModalProcessor[MERaLiON2ProcessingInfo]):
96
+
97
+ def _get_data_parser(self) -> MultiModalDataParser:
98
+ feature_extractor = self.info.get_feature_extractor()
99
+ return MultiModalDataParser(target_sr=feature_extractor.sampling_rate)
100
+
101
+ def _call_hf_processor(
102
+ self,
103
+ prompt: str,
104
+ mm_data: Mapping[str, object],
105
+ mm_kwargs: Mapping[str, Any],
106
+ ) -> BatchFeature:
107
+ # Text-only input not supported in composite processor
108
+ if not mm_data.get("audios", []):
109
+ prompt_ids = self.info.get_tokenizer().encode(prompt)
110
+ prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
111
+ return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
112
+
113
+ processor = self.info.get_hf_processor()
114
+ feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
115
+ speech_token_id = getattr(processor, "speech_token_index", 255999)
116
+ output_chunk_size = getattr(processor, "fixed_speech_embeds_length", 100)
117
+
118
+ mm_kwargs = dict(
119
+ **mm_kwargs,
120
+ sampling_rate=feature_extractor.sampling_rate,
121
+ )
122
+
123
+ results = super()._call_hf_processor(
124
+ prompt=prompt,
125
+ mm_data=mm_data,
126
+ mm_kwargs=mm_kwargs,
127
+ )
128
+
129
+ chunk_sizes = (results["input_ids"] == speech_token_id).sum(axis=1) // output_chunk_size
130
+ splitted_input_features = torch.split(results["input_features"], chunk_sizes.tolist())
131
+ splitted_feature_attention_mask = torch.split(results["feature_attention_mask"], chunk_sizes.tolist())
132
+
133
+ results["input_features"] = splitted_input_features
134
+ results["feature_attention_mask"] = splitted_feature_attention_mask
135
+ return results
136
+
137
+ def _get_mm_fields_config(
138
+ self,
139
+ hf_inputs: BatchFeature,
140
+ hf_processor_mm_kwargs: Mapping[str, object],
141
+ ) -> Mapping[str, MultiModalFieldConfig]:
142
+ return dict(
143
+ input_features=MultiModalFieldConfig.batched("audio"),
144
+ feature_attention_mask=MultiModalFieldConfig.batched("audio"),
145
+ )
146
+
147
+ def _get_prompt_updates(
148
+ self,
149
+ mm_items: MultiModalDataItems,
150
+ hf_processor_mm_kwargs: Mapping[str, object],
151
+ out_mm_kwargs: MultiModalKwargs,
152
+ ) -> Sequence[PromptUpdate]:
153
+ processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
154
+
155
+ speech_token = getattr(processor, "audio_token", "<SpeechHere>")
156
+ speech_token_id = getattr(processor, "speech_token_index", 255999)
157
+ output_chunk_size = getattr(processor, "fixed_speech_embeds_length", 100)
158
+ feature_chunk_size = getattr(processor, "feature_chunk_size", 30 * 16000)
159
+
160
+ def get_replacement_meralion2_audio(item_idx: int):
161
+ audios = mm_items.get_items("audio", AudioProcessorItems)
162
+ audio_length = audios.get_audio_length(item_idx)
163
+ number_chunks = ((audio_length - 1) // feature_chunk_size) + 1
164
+ speech_tokens = [speech_token_id] * number_chunks * output_chunk_size
165
+
166
+ return PromptUpdateDetails.select_token_id(
167
+ speech_tokens,
168
+ embed_token_id=speech_token_id,
169
+ )
170
+
171
+ return [
172
+ PromptReplacement(
173
+ modality="audio",
174
+ target=speech_token,
175
+ replacement=get_replacement_meralion2_audio,
176
+ )
177
+ ]
178
+
179
+
180
+ @MULTIMODAL_REGISTRY.register_processor(
181
+ MERaLiON2MultiModalProcessor,
182
+ info=MERaLiON2ProcessingInfo,
183
+ dummy_inputs=MERaLiON2DummyInputsBuilder)
184
+ class MERaLiON2ForConditionalGeneration(nn.Module, SupportsMultiModal,
185
+ SupportsPP):
186
+
187
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
188
+ super().__init__()
189
+ config = vllm_config.model_config.hf_config
190
+ quant_config = vllm_config.quant_config
191
+ multimodal_config = vllm_config.model_config.multimodal_config
192
+ self.config = config
193
+ self.multimodal_config = multimodal_config
194
+
195
+ config.speech_config = \
196
+ autoset_attn_implementation_for_whisper(config.speech_config)
197
+ self.speech_encoder = WhisperEncoder(config.speech_config)
198
+ self.ln_speech = nn.LayerNorm(config.speech_config.d_model)
199
+ self.speech_audio_adapter = MERaLiON2SpeechAudioAdaper(
200
+ config.speech_config.d_model, config.text_config.hidden_size)
201
+
202
+ self.quant_config = quant_config
203
+
204
+ self.text_decoder = init_vllm_registered_model(
205
+ vllm_config=vllm_config,
206
+ hf_config=config.text_config,
207
+ prefix=maybe_prefix(prefix, "text_decoder"),
208
+ architectures=["Gemma2ForCausalLM"],
209
+ )
210
+
211
+ self.make_empty_intermediate_tensors = (
212
+ self.text_decoder.make_empty_intermediate_tensors)
213
+
214
+ def _validate_and_reshape_mm_tensor(self,
215
+ mm_input: Union[torch.Tensor,
216
+ List[torch.Tensor]],
217
+ name: str) -> torch.Tensor:
218
+ if not isinstance(mm_input, (torch.Tensor, list)):
219
+ raise ValueError(f"Incorrect type of {name}. "
220
+ f"Got type: {type(mm_input)}")
221
+
222
+ input_ids_lst = mm_input.size()
223
+ with open("log.txt", "a") as f:
224
+ f.write(",".join([str(e) for e in input_ids_lst]) + "\n")
225
+
226
+ if isinstance(mm_input, torch.Tensor):
227
+ result = torch.concat(list(mm_input))
228
+ else:
229
+ result = torch.concat(mm_input)
230
+
231
+ flattened_result = result.view(-1, result.size(-2), result.size(-1))
232
+ return flattened_result
233
+
234
+ def _parse_and_validate_audio_input(
235
+ self, **kwargs: object) -> Optional[MERaLiON2Inputs]:
236
+ input_features = kwargs.pop('input_features', None)
237
+ feature_attention_mask = kwargs.pop('feature_attention_mask', None)
238
+
239
+ if input_features is None:
240
+ return None
241
+ input_features = self._validate_and_reshape_mm_tensor(
242
+ input_features, 'input_features')
243
+ feature_attention_mask = self._validate_and_reshape_mm_tensor(
244
+ feature_attention_mask, 'feature_attention_mask')
245
+ if not isinstance(input_features, (torch.Tensor, list)):
246
+ raise ValueError("Incorrect type of audio input features. "
247
+ f"Got type: {type(input_features)}")
248
+ return MERaLiON2Inputs(input_features=input_features,
249
+ feature_attention_mask=feature_attention_mask)
250
+
251
+ def _process_audio_input(self,
252
+ audio_input: MERaLiON2Inputs) -> torch.Tensor:
253
+
254
+ input_features = audio_input["input_features"].to(self.speech_encoder.dtype)
255
+ feature_attention_mask = audio_input["feature_attention_mask"]
256
+
257
+ audio_outputs = self.speech_encoder(input_features,
258
+ attention_mask=feature_attention_mask)
259
+ audio_features = audio_outputs.last_hidden_state
260
+ audio_features = self.ln_speech(audio_features)
261
+ audio_features = self.speech_audio_adapter(audio_features)
262
+ audio_features = audio_features.view(-1, audio_features.size(-1))
263
+
264
+ return audio_features
265
+
266
+ def get_language_model(self) -> torch.nn.Module:
267
+ return self.text_decoder
268
+
269
+ def get_multimodal_embeddings(
270
+ self, **kwargs: object) -> Optional[MultiModalEmbeddings]:
271
+
272
+ if "input_features" not in kwargs:
273
+ return None
274
+
275
+ input_features = kwargs["input_features"]
276
+ audio_input = self._parse_and_validate_audio_input(**kwargs)
277
+ if isinstance(input_features, torch.Tensor):
278
+ input_features = list(input_features)
279
+
280
+ audio_lengths = [math.prod(audio.shape[:-2]) * 100 for audio in input_features]
281
+ masked_audio_features = self._process_audio_input(audio_input)
282
+ masked_audio_features = torch.split(masked_audio_features, audio_lengths)
283
+ return masked_audio_features
284
+
285
+ def get_input_embeddings(
286
+ self,
287
+ input_ids: torch.Tensor,
288
+ multimodal_embeddings: Optional[MultiModalEmbeddings] = None,
289
+ ) -> torch.Tensor:
290
+ inputs_embeds = self.text_decoder.get_input_embeddings(input_ids)
291
+ if multimodal_embeddings is not None:
292
+ inputs_embeds = merge_multimodal_embeddings(
293
+ input_ids, inputs_embeds, multimodal_embeddings,
294
+ self.config.speech_token_index)
295
+ return inputs_embeds
296
+
297
+ def forward(
298
+ self,
299
+ input_ids: torch.Tensor,
300
+ positions: torch.Tensor,
301
+ intermediate_tensors: Optional[IntermediateTensors] = None,
302
+ inputs_embeds: Optional[torch.Tensor] = None,
303
+ **kwargs: object,
304
+ ) -> Union[torch.Tensor, IntermediateTensors]:
305
+ if intermediate_tensors is not None:
306
+ inputs_embeds = None
307
+
308
+ # NOTE: In v1, inputs_embeds is always generated at model runner, this
309
+ # condition is for v0 compatibility.
310
+ elif inputs_embeds is None:
311
+ multimodal_embeddings = self.get_multimodal_embeddings(**kwargs)
312
+ inputs_embeds = self.get_input_embeddings(input_ids,
313
+ multimodal_embeddings)
314
+ input_ids = None
315
+
316
+ hidden_states = self.text_decoder.model(input_ids,
317
+ positions,
318
+ intermediate_tensors,
319
+ inputs_embeds=inputs_embeds)
320
+ return hidden_states
321
+
322
+ def compute_logits(
323
+ self,
324
+ hidden_states: torch.Tensor,
325
+ sampling_metadata: SamplingMetadata,
326
+ ) -> Optional[torch.Tensor]:
327
+ return self.text_decoder.compute_logits(hidden_states,
328
+ sampling_metadata)
329
+
330
+ def load_weights(self, weights: Iterable[Tuple[str,
331
+ torch.Tensor]]) -> Set[str]:
332
+ loader = AutoWeightsLoader(self)
333
+ return loader.load_weights(weights)