pinecoresystems commited on
Commit
678981c
·
verified ·
1 Parent(s): bd72cd0

Mirror processing_moss_tts.py

Browse files
Files changed (1) hide show
  1. processing_moss_tts.py +1013 -0
processing_moss_tts.py ADDED
@@ -0,0 +1,1013 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # coding=utf-8
2
+ # Copyright 2026 OpenMOSS and the HuggingFace Inc. team. All rights reserved.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ import os
17
+ from typing import Any, Dict, List, Optional, Tuple, Type, Union, Literal, Final, cast
18
+ from dataclasses import dataclass
19
+ from pathlib import Path
20
+ import re
21
+ import torchaudio
22
+
23
+ from transformers import processing_utils
24
+
25
+ processing_utils.MODALITY_TO_BASE_CLASS_MAPPING["audio_tokenizer"] = "PreTrainedModel"
26
+
27
+ import torch
28
+ from transformers import (
29
+ PreTrainedTokenizerBase,
30
+ BatchFeature,
31
+ ProcessorMixin,
32
+ logging,
33
+ AutoConfig,
34
+ AutoModel,
35
+ AutoTokenizer,
36
+ )
37
+
38
+ from .configuration_moss_tts import MossTTSDelayConfig
39
+
40
+
41
+ def normalize_instruction(instruction: str) -> str:
42
+ """
43
+ Normalize instruction:
44
+ 1. Remove [] and {} tags
45
+ 2. Replace decorative symbols with comma
46
+ 3. Remove consecutive duplicate punctuation
47
+ 4. Remove line breaks
48
+ 5. If contains Chinese, replace English comma with Chinese comma
49
+ 6. Keep quotes
50
+ """
51
+ if not instruction:
52
+ return instruction
53
+
54
+ # Remove line breaks
55
+ instruction = instruction.replace("\n", " ")
56
+
57
+ # Remove [] and {} tags
58
+ instruction = re.sub(r"\[.*?\]", "", instruction)
59
+ instruction = re.sub(r"\{.*?\}", "", instruction)
60
+
61
+ # Replace decorative symbols with comma
62
+ decorative_chars = "【】《》()『』「」~-_"
63
+ for char in decorative_chars:
64
+ instruction = instruction.replace(char, ",")
65
+
66
+ # Remove consecutive punctuation (keep only first one)
67
+ instruction = re.sub(r'([,。!?,.!?;;])+', r'\1', instruction)
68
+
69
+ # Check if contains Chinese characters
70
+ has_chinese = bool(re.search(r'[\u4e00-\u9fff]', instruction))
71
+
72
+ if has_chinese:
73
+ # Replace English comma with Chinese comma
74
+ instruction = instruction.replace(',', ',')
75
+
76
+ return instruction.strip()
77
+
78
+ def normalize_text(text: str) -> str:
79
+ """
80
+ Normalize text:
81
+ 1. Remove [] and {} tags
82
+ 2. Replace decorative symbols with comma
83
+ 3. Remove consecutive duplicate punctuation
84
+ 4. Remove line breaks
85
+ 5. Remove quotes (only double quotes)
86
+ """
87
+ if not text:
88
+ return text
89
+
90
+ # Remove line breaks
91
+ text = text.replace("\n", " ")
92
+
93
+ # Remove [] and {} tags
94
+ text = re.sub(r"\[.*?\]", "", text)
95
+ text = re.sub(r"\{.*?\}", "", text)
96
+
97
+ # Replace decorative symbols with comma
98
+ decorative_chars = "【】《》()『』「」~"
99
+ for char in decorative_chars:
100
+ text = text.replace(char, ",")
101
+
102
+ # Remove double quotes only (保留单引号)
103
+ quotes = ['"', '"', '"']
104
+ for q in quotes:
105
+ text = text.replace(q, "")
106
+
107
+ # Remove consecutive punctuation (keep only first one)
108
+ text = re.sub(r'([,。!?,.!?;;])+', r'\1', text)
109
+
110
+ return text.strip()
111
+
112
+ logger = logging.get_logger(__name__)
113
+
114
+
115
+ AUDIO_PLACEHOLDER = "<|audio|>"
116
+
117
+
118
+ @dataclass
119
+ class Message:
120
+ def to_dict(self) -> Dict[str, Any]:
121
+ raise NotImplementedError
122
+
123
+
124
+ @dataclass
125
+ class UserMessage(Message):
126
+ text: Optional[str] = None
127
+ reference: Optional[List[Optional[Union[str, torch.Tensor]]]] = None
128
+ instruction: Optional[str] = None
129
+ tokens: Optional[int] = None
130
+ quality: Optional[str] = None
131
+ sound_event: Optional[str] = None
132
+ ambient_sound: Optional[str] = None
133
+ language: Optional[str] = None
134
+
135
+ def __post_init__(self):
136
+ template = """<user_inst>
137
+ - Reference(s):
138
+ {reference}
139
+ - Instruction:
140
+ {instruction}
141
+ - Tokens:
142
+ {tokens}
143
+ - Quality:
144
+ {quality}
145
+ - Sound Event:
146
+ {sound_event}
147
+ - Ambient Sound:
148
+ {ambient_sound}
149
+ - Language:
150
+ {language}
151
+ - Text:
152
+ {text}
153
+ </user_inst>"""
154
+
155
+ audio_codes_list = []
156
+ if self.reference is None:
157
+ reference = "None"
158
+ elif isinstance(self.reference, List):
159
+ reference = []
160
+ for speaker_idx, speaker_reference in enumerate(self.reference):
161
+ if speaker_reference is not None:
162
+ reference.append(f"[S{speaker_idx+1}]:\n{AUDIO_PLACEHOLDER}")
163
+ reference = "\n".join(reference)
164
+ audio_codes_list = [
165
+ speaker_reference
166
+ for speaker_reference in self.reference
167
+ if speaker_reference is not None
168
+ ]
169
+ else:
170
+ raise TypeError("`reference` should be exactly a list when it is not None.")
171
+
172
+ content = (
173
+ template.replace("{reference}", str(reference))
174
+ .replace("{instruction}", str(self.instruction))
175
+ .replace("{tokens}", str(self.tokens))
176
+ .replace("{quality}", str(self.quality))
177
+ .replace("{sound_event}", str(self.sound_event))
178
+ .replace("{ambient_sound}", str(self.ambient_sound))
179
+ .replace("{language}", str(self.language))
180
+ .replace("{text}", str(self.text))
181
+ )
182
+
183
+ self._content = content
184
+ self._audio_codes_list = audio_codes_list
185
+
186
+ def to_dict(self):
187
+ return {
188
+ "role": "user",
189
+ "content": self._content,
190
+ "audio_codes_list": self._audio_codes_list,
191
+ }
192
+
193
+
194
+ @dataclass
195
+ class AssistantMessage(Message):
196
+ audio_codes_list: List[Union[str, torch.Tensor]]
197
+ content: str = AUDIO_PLACEHOLDER
198
+
199
+ def to_dict(self):
200
+ return {
201
+ "role": "assistant",
202
+ "content": self.content,
203
+ "audio_codes_list": self.audio_codes_list,
204
+ }
205
+
206
+
207
+ USER_MESSAGE_FIELDS = (
208
+ "text",
209
+ "reference",
210
+ "instruction",
211
+ "tokens",
212
+ "quality",
213
+ "sound_event",
214
+ "ambient_sound",
215
+ "language",
216
+ )
217
+
218
+
219
+ class MossTTSDelayProcessor(ProcessorMixin):
220
+ tokenizer_class = "AutoTokenizer"
221
+ audio_tokenizer_class = "AutoModel"
222
+
223
+ tokenizer: PreTrainedTokenizerBase
224
+ audio_tokenizer: Any
225
+
226
+ def __init__(
227
+ self,
228
+ tokenizer: PreTrainedTokenizerBase,
229
+ audio_tokenizer: Any = None,
230
+ model_config: Optional[MossTTSDelayConfig] = None,
231
+ normalize_inputs: bool = False,
232
+ **kwargs,
233
+ ):
234
+ super().__init__(tokenizer=tokenizer, audio_tokenizer=audio_tokenizer, **kwargs)
235
+
236
+ # Explicit assignments for type-checkers; ProcessorMixin sets these too.
237
+ self.tokenizer = tokenizer
238
+ self.audio_tokenizer = audio_tokenizer
239
+ if model_config is None:
240
+ model_config = MossTTSDelayConfig()
241
+ self.model_config = model_config
242
+ self.normalize_inputs = normalize_inputs
243
+
244
+ self.imstart_token_id = tokenizer.convert_tokens_to_ids("<|im_start|>")
245
+ self.imend_token_id = tokenizer.convert_tokens_to_ids("<|im_end|>")
246
+ self.newline_token_id = 198
247
+
248
+ def _id_to_token(token_id: int) -> str:
249
+ tok = tokenizer.convert_ids_to_tokens(int(token_id))
250
+ if isinstance(tok, list):
251
+ return tok[0] if len(tok) > 0 else ""
252
+ return cast(str, tok)
253
+
254
+ self.audio_user_slot_token = _id_to_token(
255
+ self.model_config.audio_user_slot_token_id
256
+ )
257
+ self.audio_assistant_gen_slot_token = _id_to_token(
258
+ self.model_config.audio_assistant_gen_slot_token_id
259
+ )
260
+ self.audio_assistant_delay_slot_token = _id_to_token(
261
+ self.model_config.audio_assistant_delay_slot_token_id
262
+ )
263
+ self.audio_start_token = _id_to_token(self.model_config.audio_start_token_id)
264
+ self.audio_end_token = _id_to_token(self.model_config.audio_end_token_id)
265
+
266
+ @classmethod
267
+ def from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs):
268
+ trust_remote_code = kwargs.pop("trust_remote_code", True)
269
+ kwargs.pop("_from_auto", None)
270
+
271
+ audio_tokenizer_name_or_path = kwargs.pop(
272
+ "codec_path", "OpenMOSS-Team/MOSS-Audio-Tokenizer"
273
+ )
274
+ normalize_inputs = kwargs.pop("normalize_inputs", False)
275
+
276
+ pretrained_model_name_or_path = Path(pretrained_model_name_or_path)
277
+ model_config = cast(
278
+ MossTTSDelayConfig,
279
+ AutoConfig.from_pretrained(
280
+ pretrained_model_name_or_path,
281
+ *args,
282
+ trust_remote_code=trust_remote_code,
283
+ **kwargs,
284
+ ),
285
+ )
286
+ tokenizer = AutoTokenizer.from_pretrained(
287
+ pretrained_model_name_or_path,
288
+ *args,
289
+ trust_remote_code=trust_remote_code,
290
+ **kwargs,
291
+ )
292
+ audio_tokenizer = AutoModel.from_pretrained(
293
+ audio_tokenizer_name_or_path,
294
+ trust_remote_code=trust_remote_code,
295
+ **kwargs,
296
+ )
297
+
298
+ return cls(
299
+ tokenizer=tokenizer,
300
+ audio_tokenizer=audio_tokenizer,
301
+ model_config=model_config,
302
+ normalize_inputs=normalize_inputs,
303
+ **kwargs,
304
+ )
305
+
306
+ def __call__(self, *args, **kwargs) -> BatchFeature:
307
+ conversations = args[0] if len(args) > 0 else kwargs.pop("conversations")
308
+ mode: str = kwargs.pop("mode", "generation")
309
+ apply_chat_template: bool = kwargs.pop("apply_chat_template", True)
310
+ n_vq: Optional[int] = kwargs.pop("n_vq", None)
311
+
312
+ # Common ProcessorMixin kwargs that we ignore because we always return torch tensors.
313
+ kwargs.pop("return_tensors", None)
314
+ kwargs.pop("padding", None)
315
+ kwargs.pop("truncation", None)
316
+
317
+ """
318
+ mode only works when a Message is converted to a dict.
319
+ """
320
+
321
+ if mode not in {"generation", "continuation"}:
322
+ raise RuntimeError
323
+
324
+ if isinstance(conversations, (Message, Dict)):
325
+ conversations = [conversations]
326
+
327
+ truncation = False
328
+ if mode == "continuation":
329
+ truncation = True
330
+
331
+ input_ids_list = []
332
+ for conversation in conversations:
333
+ if isinstance(conversation, (Message, Dict)):
334
+ conversation = [conversation]
335
+
336
+ # Normalize early so downstream logic always deals with dict messages.
337
+ conversation = [self._normalize_message(m) for m in conversation]
338
+
339
+ if (mode == "generation") ^ (len(conversation) % 2 != 0):
340
+ raise ValueError
341
+
342
+ if (mode == "generation") ^ (conversation[-1]["role"] == "user"):
343
+ raise ValueError
344
+
345
+ unified_codes = []
346
+ for message_idx, message in enumerate(conversation):
347
+ if apply_chat_template:
348
+ add_generation_prompt = (
349
+ mode == "generation" and message_idx == len(conversation) - 1
350
+ )
351
+ try:
352
+ content = self.tokenizer.apply_chat_template(
353
+ [{"role": message["role"], "content": message["content"]}],
354
+ add_generation_prompt=add_generation_prompt,
355
+ tokenize=False,
356
+ )
357
+ except TypeError:
358
+ try:
359
+ content = self.tokenizer.apply_chat_template(
360
+ [
361
+ {
362
+ "role": message["role"],
363
+ "content": message["content"],
364
+ }
365
+ ],
366
+ add_generation_prompt=add_generation_prompt,
367
+ )
368
+ except Exception:
369
+ logger.warning(
370
+ "apply_chat_template failed; fallback to raw content."
371
+ )
372
+ content = message["content"]
373
+ else:
374
+ content = message["content"]
375
+
376
+ if not isinstance(content, str):
377
+ content = str(content)
378
+
379
+ # Batch-encode all path-based references in one call when possible.
380
+ # This ensures we actually exercise audio_tokenizer.batch_encode for multi-reference prompts,
381
+ # instead of repeatedly calling it with batch=1.
382
+ raw_audio_items = message.get("audio_codes_list", [])
383
+
384
+ audio_codes_list: List[torch.Tensor] = []
385
+ if len(raw_audio_items) > 0:
386
+ encoded_items: List[Optional[torch.Tensor]] = [None] * len(
387
+ raw_audio_items
388
+ )
389
+ paths: List[str] = []
390
+ path_positions: List[int] = []
391
+
392
+ for idx, item in enumerate(raw_audio_items):
393
+ if isinstance(item, torch.Tensor):
394
+ if n_vq is not None and item.shape[1] != n_vq:
395
+ raise RuntimeError(
396
+ "audio_codes's n_vq is not equal to the parameter `n_vq`. Your can set the parameter `n_vq` as None if you have already tokenzied the wavs."
397
+ )
398
+ encoded_items[idx] = item
399
+ continue
400
+
401
+ if isinstance(item, (str, os.PathLike)):
402
+ paths.append(str(item))
403
+ path_positions.append(idx)
404
+ continue
405
+
406
+ raise TypeError(
407
+ "Each audio item must be a torch.Tensor of codes or a path-like string."
408
+ )
409
+
410
+ if len(paths) > 0:
411
+ encoded_from_paths = self.encode_audios_from_path(paths, n_vq)
412
+ if len(encoded_from_paths) != len(paths):
413
+ raise RuntimeError(
414
+ "encode_audios_from_path returned an unexpected number of items."
415
+ )
416
+ for pos, codes in zip(path_positions, encoded_from_paths):
417
+ encoded_items[pos] = codes
418
+
419
+ audio_codes_list = [cast(torch.Tensor, t) for t in encoded_items]
420
+ unified_codes.append(
421
+ self._get_unified_codes(
422
+ message["role"], content, audio_codes_list, truncation
423
+ )
424
+ )
425
+
426
+ unified_codes = torch.cat(unified_codes)
427
+ input_ids_list.append(unified_codes)
428
+
429
+ return BatchFeature(data=self._pad(input_ids_list))
430
+
431
+ @staticmethod
432
+ def build_user_message(
433
+ text: Optional[str] = None,
434
+ reference: Optional[List[Optional[Union[str, torch.Tensor]]]] = None,
435
+ instruction: Optional[str] = None,
436
+ tokens: Optional[int] = None,
437
+ quality: Optional[str] = None,
438
+ sound_event: Optional[str] = None,
439
+ ambient_sound: Optional[str] = None,
440
+ language: Optional[str] = None,
441
+ normalize: bool = False,
442
+ ) -> Dict:
443
+ if normalize:
444
+ if text is not None:
445
+ text = normalize_text(text)
446
+ if instruction is not None:
447
+ instruction = normalize_instruction(instruction)
448
+ if reference is not None and not isinstance(reference, list):
449
+ reference = [reference]
450
+ return UserMessage(
451
+ text=text,
452
+ reference=reference,
453
+ instruction=instruction,
454
+ tokens=tokens,
455
+ quality=quality,
456
+ sound_event=sound_event,
457
+ ambient_sound=ambient_sound,
458
+ language=language,
459
+ ).to_dict()
460
+
461
+ @staticmethod
462
+ def build_assistant_message(
463
+ audio_codes_list: List[Union[str, torch.Tensor]],
464
+ content: str = AUDIO_PLACEHOLDER,
465
+ ) -> Dict:
466
+ return AssistantMessage(
467
+ audio_codes_list=audio_codes_list,
468
+ content=content,
469
+ ).to_dict()
470
+
471
+ def _normalize_message(self, message: Union[Message, Dict]) -> Dict:
472
+ if isinstance(message, Message):
473
+ return message.to_dict()
474
+ if not isinstance(message, dict):
475
+ raise TypeError("Each message must be a Message or dict.")
476
+ if "role" not in message:
477
+ raise ValueError("Message dict must include a 'role' field.")
478
+ if "content" in message and "audio_codes_list" in message:
479
+ return message
480
+ role = message["role"]
481
+ if role == "user":
482
+ kwargs = {key: message.get(key) for key in USER_MESSAGE_FIELDS}
483
+ # 应用processor的全局normalize设置
484
+ kwargs['normalize'] = self.normalize_inputs
485
+ return self.build_user_message(**kwargs)
486
+ if role == "assistant":
487
+ return self.build_assistant_message(
488
+ audio_codes_list=message.get("audio_codes_list", []),
489
+ content=message.get("content", AUDIO_PLACEHOLDER),
490
+ )
491
+ raise ValueError(f"Unsupported role: {role}")
492
+
493
+ def _pad(self, input_ids_list: List[torch.Tensor]):
494
+ device = input_ids_list[0].device
495
+ lengths = torch.tensor([w.shape[0] for w in input_ids_list], device=device)
496
+ pad_input_ids = torch.nn.utils.rnn.pad_sequence(
497
+ input_ids_list,
498
+ batch_first=True,
499
+ padding_value=self.model_config.audio_pad_code,
500
+ padding_side="left",
501
+ )
502
+ other_channel_mask = (pad_input_ids.shape[1] - lengths).unsqueeze(
503
+ 1
504
+ ) > torch.arange(pad_input_ids.shape[1], device=device).unsqueeze(0)
505
+ pad_input_ids[..., 0][other_channel_mask] = self.model_config.pad_token_id
506
+ attention_mask = torch.zeros(
507
+ pad_input_ids.shape[0], pad_input_ids.shape[1], device=device
508
+ )
509
+ attention_mask[~other_channel_mask] = 1
510
+ attention_mask = attention_mask.bool()
511
+ return {
512
+ "input_ids": pad_input_ids, # [batch_size, seqlen, n_vq]
513
+ "attention_mask": attention_mask,
514
+ }
515
+
516
+ @staticmethod
517
+ def _replace_audio_placeholders(
518
+ content: str,
519
+ lengths: List[int],
520
+ n_vq: int,
521
+ gen_slot_token: str,
522
+ delay_slot_token: str,
523
+ audio_start_token: str,
524
+ audio_end_token: str,
525
+ ) -> str:
526
+ if n_vq < 1:
527
+ raise ValueError(f"n_vq must be >= 1, got {n_vq}")
528
+
529
+ num_placeholders = content.count(AUDIO_PLACEHOLDER)
530
+ if num_placeholders != len(lengths):
531
+ raise ValueError(
532
+ f"Number of {AUDIO_PLACEHOLDER} ({num_placeholders}) "
533
+ f"does not match lengths ({len(lengths)})"
534
+ )
535
+
536
+ def build_audio_block(length: int) -> str:
537
+ if length < 0:
538
+ raise ValueError(f"length must be >= 0, got {length}")
539
+
540
+ if length == 0:
541
+ return f"{audio_start_token}{audio_end_token}"
542
+
543
+ step_tokens = gen_slot_token * length + (delay_slot_token * (n_vq - 1))
544
+ return f"{audio_start_token}{step_tokens}{audio_end_token}"
545
+
546
+ lengths_iter = iter(lengths)
547
+
548
+ def replacer(match: re.Match) -> str:
549
+ length = next(lengths_iter)
550
+ return build_audio_block(length)
551
+
552
+ result = re.sub(re.escape(AUDIO_PLACEHOLDER), replacer, content)
553
+
554
+ return result
555
+
556
+ @staticmethod
557
+ def _merge_consecutive_audio_placeholders(
558
+ content: str,
559
+ audio_codes_list: List[torch.Tensor],
560
+ ) -> Tuple[str, List[torch.Tensor]]:
561
+ matches = list(re.finditer(re.escape(AUDIO_PLACEHOLDER), content))
562
+ if len(matches) <= 1:
563
+ return content, audio_codes_list
564
+
565
+ if len(matches) != len(audio_codes_list):
566
+ raise ValueError(
567
+ "Audio placeholders do not match the provided audio codes list."
568
+ )
569
+
570
+ new_audio_codes_list = []
571
+ new_parts = []
572
+ last_pos = 0
573
+ i = 0
574
+ while i < len(matches):
575
+ j = i
576
+ while (
577
+ j + 1 < len(matches)
578
+ and content[matches[j].end() : matches[j + 1].start()].strip() == ""
579
+ ):
580
+ j += 1
581
+
582
+ new_parts.append(content[last_pos : matches[i].start()])
583
+ new_parts.append(AUDIO_PLACEHOLDER)
584
+ last_pos = matches[j].end()
585
+
586
+ if j == i:
587
+ new_audio_codes_list.append(audio_codes_list[i])
588
+ else:
589
+ new_audio_codes_list.append(
590
+ torch.cat(audio_codes_list[i : j + 1], dim=0)
591
+ )
592
+
593
+ i = j + 1
594
+
595
+ new_parts.append(content[last_pos:])
596
+ return "".join(new_parts), new_audio_codes_list
597
+
598
+ @staticmethod
599
+ def apply_delay_pattern(codes: torch.Tensor, pad_code: int) -> torch.Tensor:
600
+ delayed_tokens = torch.full(
601
+ (codes.shape[0] + codes.shape[1] - 1, codes.shape[1]),
602
+ pad_code,
603
+ device=codes.device,
604
+ dtype=codes.dtype,
605
+ )
606
+ for i in range(codes.shape[1]):
607
+ delayed_tokens[i : i + codes.shape[0], i] = codes[:, i]
608
+ return delayed_tokens
609
+
610
+ @staticmethod
611
+ def apply_de_delay_pattern(delay_codes: torch.Tensor) -> torch.Tensor:
612
+ tokens = torch.full(
613
+ (delay_codes.shape[0] - delay_codes.shape[1] + 1, delay_codes.shape[1]),
614
+ 0,
615
+ device=delay_codes.device,
616
+ dtype=delay_codes.dtype,
617
+ )
618
+ for i in range(delay_codes.shape[1]):
619
+ tokens[:, i] = delay_codes[i : i + tokens.shape[0], i]
620
+ return tokens
621
+
622
+ def _get_unified_codes(
623
+ self,
624
+ role: str,
625
+ content: str,
626
+ audio_codes_list: List[torch.Tensor],
627
+ truncation: bool,
628
+ ) -> torch.Tensor:
629
+ """
630
+ 此时的 content 已经是带上了对话格式
631
+ """
632
+ if role == "user":
633
+ audio_gen_slot_token = audio_delay_slot_token = self.audio_user_slot_token
634
+ truncation = False
635
+ else:
636
+ audio_gen_slot_token = self.audio_assistant_gen_slot_token
637
+ audio_delay_slot_token = self.audio_assistant_delay_slot_token
638
+
639
+ if len(audio_codes_list):
640
+ n_vq = audio_codes_list[0].shape[1]
641
+ else:
642
+ n_vq = self.model_config.n_vq
643
+
644
+ if len(audio_codes_list) > 1 and AUDIO_PLACEHOLDER in content:
645
+ content, audio_codes_list = self._merge_consecutive_audio_placeholders(
646
+ content, audio_codes_list
647
+ )
648
+ content = self._replace_audio_placeholders(
649
+ content=content,
650
+ lengths=[len(audio_codes) for audio_codes in audio_codes_list],
651
+ n_vq=n_vq,
652
+ gen_slot_token=audio_gen_slot_token,
653
+ delay_slot_token=audio_delay_slot_token,
654
+ audio_start_token=self.audio_start_token,
655
+ audio_end_token=self.audio_end_token,
656
+ )
657
+ text_codes = torch.tensor(
658
+ self.tokenizer.encode(content),
659
+ device=audio_codes_list[0].device if audio_codes_list else None,
660
+ )
661
+
662
+ audio_start_indices = torch.where(
663
+ text_codes == self.model_config.audio_start_token_id
664
+ )[0]
665
+ audio_end_indices = torch.where(
666
+ text_codes == self.model_config.audio_end_token_id
667
+ )[0]
668
+ if len(audio_start_indices) != len(audio_codes_list) or len(
669
+ audio_end_indices
670
+ ) != len(audio_codes_list):
671
+ raise ValueError(
672
+ "Audio placeholders do not match the provided audio codes list."
673
+ )
674
+
675
+ delay_audio_codes_list = []
676
+ if len(audio_codes_list) == 0:
677
+ delay_audio_codes_list = torch.full(
678
+ (len(text_codes), n_vq),
679
+ self.model_config.audio_pad_code,
680
+ device=text_codes.device,
681
+ dtype=text_codes.dtype,
682
+ )
683
+ else:
684
+ prefix_idx = 0
685
+ for audio_start_idx_t, audio_end_idx_t, audio_codes in zip(
686
+ audio_start_indices, audio_end_indices, audio_codes_list
687
+ ):
688
+ audio_start_idx = int(audio_start_idx_t.item())
689
+ audio_end_idx = int(audio_end_idx_t.item())
690
+ delay_audio_codes = self.apply_delay_pattern(
691
+ audio_codes, self.model_config.audio_pad_code
692
+ )
693
+ pad_codes = torch.full(
694
+ (audio_start_idx - prefix_idx + 1, n_vq),
695
+ self.model_config.audio_pad_code,
696
+ device=audio_codes.device,
697
+ dtype=audio_codes.dtype,
698
+ )
699
+ delay_audio_codes_list.extend([pad_codes, delay_audio_codes])
700
+ prefix_idx = audio_end_idx
701
+
702
+ if truncation:
703
+ delay_audio_codes_list[-1] = delay_audio_codes_list[-1][
704
+ : -(n_vq - 1), :
705
+ ]
706
+ else:
707
+ last_audio_end_idx = int(audio_end_indices[-1].item())
708
+ pad_codes = torch.full(
709
+ (len(text_codes) - last_audio_end_idx, n_vq),
710
+ self.model_config.audio_pad_code,
711
+ device=audio_codes_list[0].device,
712
+ dtype=audio_codes_list[0].dtype,
713
+ )
714
+ delay_audio_codes_list.append(pad_codes)
715
+
716
+ delay_audio_codes_list = torch.cat(delay_audio_codes_list)
717
+
718
+ if text_codes.shape[0] != delay_audio_codes_list.shape[0]:
719
+ text_codes = text_codes[: delay_audio_codes_list.shape[0]]
720
+
721
+ unified_codes = torch.cat(
722
+ [text_codes.unsqueeze(1), delay_audio_codes_list], dim=1
723
+ )
724
+ return unified_codes
725
+
726
+ def _parse_text_codes(self, start_length, text_codes):
727
+ text = cast(str, self.tokenizer.decode(text_codes))
728
+ prefix = cast(str, self.tokenizer.decode(text_codes[:start_length]))
729
+ text = text[len(prefix) :]
730
+
731
+ AUDIO_PATTERN = re.compile(
732
+ rf"(?:{self.audio_start_token})?"
733
+ rf"(?:{self.audio_assistant_gen_slot_token})*"
734
+ rf"(?:{self.audio_assistant_delay_slot_token})*"
735
+ rf"{self.audio_end_token}"
736
+ )
737
+
738
+ def normalize_audio_segments(text: str) -> str:
739
+ def repl(match: re.Match) -> str:
740
+ seg = match.group(0)
741
+ # Replace with <|audio|> if gen_slot is present in the segment;
742
+ if self.audio_assistant_gen_slot_token in seg:
743
+ return AUDIO_PLACEHOLDER
744
+ # Otherwise, remove it.
745
+ return ""
746
+
747
+ return AUDIO_PATTERN.sub(repl, text)
748
+
749
+ return normalize_audio_segments(text)
750
+
751
+ def _parse_audio_codes(self, start_length, audio_codes):
752
+ # De-delay back to [T', n_vq]
753
+ audio_codes = self.apply_de_delay_pattern(audio_codes)
754
+
755
+ # Rows that are all pad are separators between real audio segments.
756
+ is_pad = (audio_codes == self.model_config.audio_pad_code).all(dim=1)
757
+ non_pad = ~is_pad
758
+ if not non_pad.any():
759
+ return []
760
+
761
+ idx = torch.nonzero(non_pad).squeeze(1)
762
+ breaks = torch.where(idx[1:] != idx[:-1] + 1)[0] + 1
763
+ if breaks.numel() == 0:
764
+ segments_idx = [idx]
765
+ else:
766
+ segments_idx = torch.split(idx, breaks.tolist())
767
+
768
+ audio_codes_list = [audio_codes[s] for s in segments_idx]
769
+
770
+ # Batch-decode all audio segments together.
771
+ decoded_audio_list = self.decode_audio_codes(audio_codes_list)
772
+
773
+ # Keep codec causal context by decoding the whole first segment first,
774
+ # then trim at waveform level according to start_length ratio.
775
+ if (
776
+ start_length > 0
777
+ and len(audio_codes_list) > 0
778
+ and len(decoded_audio_list) > 0
779
+ ):
780
+ first_codes_length = audio_codes_list[0].shape[0]
781
+ if first_codes_length > 0:
782
+ trim_ratio = max(
783
+ 0.0, min(float(start_length) / float(first_codes_length), 1.0)
784
+ )
785
+ first_audio = decoded_audio_list[0]
786
+ if trim_ratio >= 1.0:
787
+ decoded_audio_list = decoded_audio_list[1:]
788
+ elif trim_ratio > 0.0:
789
+ trim_samples = int(first_audio.shape[-1] * trim_ratio)
790
+ decoded_audio_list[0] = first_audio[..., trim_samples:]
791
+
792
+ return decoded_audio_list
793
+
794
+ def decode(self, output: List[Tuple[int, torch.Tensor]]):
795
+ """
796
+ 1. 这里不管怎样,都需要一个完整的 assistant generation ids;
797
+ 2. 支持从任意位置进行截断;
798
+ """
799
+
800
+ genearted_messages = []
801
+ for start_length, generation_ids in output:
802
+ content = self._parse_text_codes(start_length, generation_ids[:, 0])
803
+ audio_codes_list = self._parse_audio_codes(
804
+ start_length, generation_ids[:, 1:]
805
+ )
806
+ if content == "":
807
+ message = None
808
+ else:
809
+ message = AssistantMessage(
810
+ content=content,
811
+ audio_codes_list=cast(
812
+ List[Union[str, torch.Tensor]], audio_codes_list
813
+ ),
814
+ )
815
+ genearted_messages.append(message)
816
+ return genearted_messages
817
+
818
+ @staticmethod
819
+ def loudness_normalize(
820
+ wav: torch.Tensor,
821
+ target_dbfs: float = -20,
822
+ gain_range: tuple[float, float] = (-3.0, 3.0),
823
+ ) -> torch.Tensor:
824
+ wav = wav.to(torch.float32)
825
+ if wav.numel() == 0:
826
+ return wav
827
+ current_dbfs = 10.0 * torch.log10(torch.mean(wav**2) + 1e-9)
828
+ gain = float(target_dbfs - current_dbfs)
829
+ gain = max(gain_range[0], min(gain, gain_range[1]))
830
+ factor = 10.0 ** (gain / 20.0)
831
+ return wav * factor
832
+
833
+ def _get_audio_tokenizer_device(self) -> torch.device:
834
+ """Best-effort device inference for `self.audio_tokenizer`.
835
+
836
+ Notes:
837
+ - Old TAC wrapper exposed `.device`, but standard `torch.nn.Module` does not.
838
+ - New MossAudioTokenizerModel is a `PreTrainedModel`; parameters define its device.
839
+ """
840
+
841
+ audio_tokenizer = getattr(self, "audio_tokenizer", None)
842
+ if audio_tokenizer is None:
843
+ logger.warning(
844
+ "audio_tokenizer is not set on processor. Using CPU as default."
845
+ )
846
+ return torch.device("cpu")
847
+
848
+ device_attr = getattr(audio_tokenizer, "device", None)
849
+ if isinstance(device_attr, torch.device):
850
+ return device_attr
851
+
852
+ try:
853
+ return next(audio_tokenizer.parameters()).device
854
+ except StopIteration:
855
+ # No parameters (shouldn't happen for real models); default to CPU.
856
+ logger.warning(
857
+ "No parameters found on audio_tokenizer. Using CPU as default."
858
+ )
859
+ return torch.device("cpu")
860
+
861
+ def encode_audios_from_wav(
862
+ self,
863
+ wav_list: List[torch.Tensor],
864
+ sampling_rate: int,
865
+ n_vq: Optional[int] = None,
866
+ ):
867
+ if self.audio_tokenizer is None:
868
+ raise RuntimeError("audio_tokenizer is not set on processor.")
869
+ audio_tokenizer = self.audio_tokenizer
870
+
871
+ if isinstance(wav_list, torch.Tensor):
872
+ wav_list = [wav_list]
873
+ wav_list_ = []
874
+ resample = False
875
+ if sampling_rate != self.model_config.sampling_rate:
876
+ resample = True
877
+ device = self._get_audio_tokenizer_device()
878
+ for wav in wav_list:
879
+ if wav.shape[0] > 1:
880
+ wav = torch.mean(wav, dim=0, keepdim=True)
881
+ if resample:
882
+ wav = torchaudio.functional.resample(
883
+ waveform=wav,
884
+ orig_freq=sampling_rate,
885
+ new_freq=self.model_config.sampling_rate,
886
+ )
887
+ wav = wav.to(device)
888
+ wav_list_.append(self.loudness_normalize(wav.squeeze(0)))
889
+
890
+ # New MossAudioTokenizerModel API: prefer batch_encode(list[wav])
891
+ if hasattr(audio_tokenizer, "batch_encode"):
892
+ enc = audio_tokenizer.batch_encode(wav_list_, num_quantizers=n_vq)
893
+ audio_codes = enc.audio_codes # (NQ, B, T)
894
+ audio_codes_lengths = enc.audio_codes_lengths # (B,)
895
+ else:
896
+ # Fallback: use encode() with explicit padding.
897
+ max_len = max(int(wav.shape[-1]) for wav in wav_list_)
898
+ input_values = torch.zeros(
899
+ len(wav_list_), 1, max_len, device=device, dtype=torch.float32
900
+ )
901
+ padding_mask = torch.zeros(
902
+ len(wav_list_), max_len, device=device, dtype=torch.bool
903
+ )
904
+ for i, wav in enumerate(wav_list_):
905
+ this_len = int(wav.shape[-1])
906
+ input_values[i, 0, :this_len] = wav
907
+ padding_mask[i, :this_len] = True
908
+ enc = audio_tokenizer.encode(
909
+ input_values,
910
+ padding_mask=padding_mask,
911
+ num_quantizers=n_vq,
912
+ return_dict=True,
913
+ )
914
+ audio_codes = enc.audio_codes
915
+ audio_codes_lengths = enc.audio_codes_lengths
916
+
917
+ if audio_codes is None or audio_codes_lengths is None:
918
+ raise RuntimeError(
919
+ "audio_tokenizer.encode() returned empty outputs (audio_codes/audio_codes_lengths)."
920
+ )
921
+
922
+ # Keep processor's historical contract: list[Tensor] with shape (T, NQ)
923
+ # and on CPU (so downstream text/audio packing remains device-agnostic).
924
+ codes_list: List[torch.Tensor] = []
925
+ for i in range(int(audio_codes.shape[1])):
926
+ length_i = int(audio_codes_lengths[i].item())
927
+ codes_i = (
928
+ audio_codes[:, i, :length_i]
929
+ .transpose(0, 1)
930
+ .contiguous()
931
+ .to(torch.long)
932
+ .cpu()
933
+ )
934
+ codes_list.append(codes_i)
935
+ return codes_list
936
+
937
+ def encode_audios_from_path(
938
+ self, wav_path_list: Union[str, List[str]], n_vq: Optional[int] = None
939
+ ):
940
+ if isinstance(wav_path_list, str):
941
+ wav_path_list = [wav_path_list]
942
+
943
+ if len(wav_path_list) == 0:
944
+ raise ValueError("Empty wav_path_list")
945
+
946
+ # Load + (if needed) resample each wav independently, so callers can
947
+ # pass a heterogeneous batch of files while still benefiting from
948
+ # audio_tokenizer.batch_encode.
949
+ target_sr = int(self.model_config.sampling_rate)
950
+ wav_list: List[torch.Tensor] = []
951
+ for wav_path in wav_path_list:
952
+ wav, sr = torchaudio.load(wav_path)
953
+ if int(sr) != target_sr:
954
+ wav = torchaudio.functional.resample(
955
+ waveform=wav,
956
+ orig_freq=int(sr),
957
+ new_freq=target_sr,
958
+ )
959
+ wav_list.append(wav)
960
+
961
+ return self.encode_audios_from_wav(wav_list, target_sr, n_vq)
962
+
963
+ def decode_audio_codes(
964
+ self, audio_tokens_list: Union[torch.Tensor, List[torch.Tensor]]
965
+ ):
966
+ if self.audio_tokenizer is None:
967
+ raise RuntimeError("audio_tokenizer is not set on processor.")
968
+ audio_tokenizer = self.audio_tokenizer
969
+
970
+ if isinstance(audio_tokens_list, torch.Tensor):
971
+ audio_tokens_list = [audio_tokens_list]
972
+ if len(audio_tokens_list) == 0:
973
+ return []
974
+
975
+ device = self._get_audio_tokenizer_device()
976
+
977
+ # Processor uses (T, NQ); MossAudioTokenizer expects (NQ, T) (or (NQ, B, T)).
978
+ codes_list = [
979
+ codes.transpose(0, 1).contiguous().to(device=device, dtype=torch.long)
980
+ for codes in audio_tokens_list
981
+ ]
982
+
983
+ # Fallback: pad to (NQ, B, T) + mask, then decode.
984
+ nq = int(codes_list[0].shape[0])
985
+ max_t = max(int(c.shape[1]) for c in codes_list)
986
+ audio_codes = torch.zeros(
987
+ nq, len(codes_list), max_t, device=device, dtype=torch.long
988
+ )
989
+ padding_mask = torch.zeros(
990
+ len(codes_list), max_t, device=device, dtype=torch.bool
991
+ )
992
+ for i, c in enumerate(codes_list):
993
+ t = int(c.shape[1])
994
+ audio_codes[:, i, :t] = c
995
+ padding_mask[i, :t] = True
996
+ dec = audio_tokenizer.decode(
997
+ audio_codes, padding_mask=padding_mask, return_dict=True, chunk_duration=8
998
+ )
999
+ audio = dec.audio
1000
+ audio_lengths = dec.audio_lengths
1001
+
1002
+ if audio is None or audio_lengths is None:
1003
+ raise RuntimeError(
1004
+ "audio_tokenizer.decode() returned empty outputs (audio/audio_lengths)."
1005
+ )
1006
+
1007
+ # Return historical contract: list of 1D waveforms (T,)
1008
+ wav_list: List[torch.Tensor] = []
1009
+ for i in range(int(audio.shape[0])):
1010
+ length_i = int(audio_lengths[i].item())
1011
+ wav = audio[i, 0, :length_i].contiguous().to(torch.float32).cpu()
1012
+ wav_list.append(wav)
1013
+ return wav_list