Mirror processing_moss_tts.py
Browse files- processing_moss_tts.py +1013 -0
processing_moss_tts.py
ADDED
|
@@ -0,0 +1,1013 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# coding=utf-8
|
| 2 |
+
# Copyright 2026 OpenMOSS and the HuggingFace Inc. team. All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 5 |
+
# you may not use this file except in compliance with the License.
|
| 6 |
+
# You may obtain a copy of the License at
|
| 7 |
+
#
|
| 8 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 9 |
+
#
|
| 10 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 11 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 12 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 13 |
+
# See the License for the specific language governing permissions and
|
| 14 |
+
# limitations under the License.
|
| 15 |
+
|
| 16 |
+
import os
|
| 17 |
+
from typing import Any, Dict, List, Optional, Tuple, Type, Union, Literal, Final, cast
|
| 18 |
+
from dataclasses import dataclass
|
| 19 |
+
from pathlib import Path
|
| 20 |
+
import re
|
| 21 |
+
import torchaudio
|
| 22 |
+
|
| 23 |
+
from transformers import processing_utils
|
| 24 |
+
|
| 25 |
+
processing_utils.MODALITY_TO_BASE_CLASS_MAPPING["audio_tokenizer"] = "PreTrainedModel"
|
| 26 |
+
|
| 27 |
+
import torch
|
| 28 |
+
from transformers import (
|
| 29 |
+
PreTrainedTokenizerBase,
|
| 30 |
+
BatchFeature,
|
| 31 |
+
ProcessorMixin,
|
| 32 |
+
logging,
|
| 33 |
+
AutoConfig,
|
| 34 |
+
AutoModel,
|
| 35 |
+
AutoTokenizer,
|
| 36 |
+
)
|
| 37 |
+
|
| 38 |
+
from .configuration_moss_tts import MossTTSDelayConfig
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def normalize_instruction(instruction: str) -> str:
|
| 42 |
+
"""
|
| 43 |
+
Normalize instruction:
|
| 44 |
+
1. Remove [] and {} tags
|
| 45 |
+
2. Replace decorative symbols with comma
|
| 46 |
+
3. Remove consecutive duplicate punctuation
|
| 47 |
+
4. Remove line breaks
|
| 48 |
+
5. If contains Chinese, replace English comma with Chinese comma
|
| 49 |
+
6. Keep quotes
|
| 50 |
+
"""
|
| 51 |
+
if not instruction:
|
| 52 |
+
return instruction
|
| 53 |
+
|
| 54 |
+
# Remove line breaks
|
| 55 |
+
instruction = instruction.replace("\n", " ")
|
| 56 |
+
|
| 57 |
+
# Remove [] and {} tags
|
| 58 |
+
instruction = re.sub(r"\[.*?\]", "", instruction)
|
| 59 |
+
instruction = re.sub(r"\{.*?\}", "", instruction)
|
| 60 |
+
|
| 61 |
+
# Replace decorative symbols with comma
|
| 62 |
+
decorative_chars = "【】《》()『』「」~-_"
|
| 63 |
+
for char in decorative_chars:
|
| 64 |
+
instruction = instruction.replace(char, ",")
|
| 65 |
+
|
| 66 |
+
# Remove consecutive punctuation (keep only first one)
|
| 67 |
+
instruction = re.sub(r'([,。!?,.!?;;])+', r'\1', instruction)
|
| 68 |
+
|
| 69 |
+
# Check if contains Chinese characters
|
| 70 |
+
has_chinese = bool(re.search(r'[\u4e00-\u9fff]', instruction))
|
| 71 |
+
|
| 72 |
+
if has_chinese:
|
| 73 |
+
# Replace English comma with Chinese comma
|
| 74 |
+
instruction = instruction.replace(',', ',')
|
| 75 |
+
|
| 76 |
+
return instruction.strip()
|
| 77 |
+
|
| 78 |
+
def normalize_text(text: str) -> str:
|
| 79 |
+
"""
|
| 80 |
+
Normalize text:
|
| 81 |
+
1. Remove [] and {} tags
|
| 82 |
+
2. Replace decorative symbols with comma
|
| 83 |
+
3. Remove consecutive duplicate punctuation
|
| 84 |
+
4. Remove line breaks
|
| 85 |
+
5. Remove quotes (only double quotes)
|
| 86 |
+
"""
|
| 87 |
+
if not text:
|
| 88 |
+
return text
|
| 89 |
+
|
| 90 |
+
# Remove line breaks
|
| 91 |
+
text = text.replace("\n", " ")
|
| 92 |
+
|
| 93 |
+
# Remove [] and {} tags
|
| 94 |
+
text = re.sub(r"\[.*?\]", "", text)
|
| 95 |
+
text = re.sub(r"\{.*?\}", "", text)
|
| 96 |
+
|
| 97 |
+
# Replace decorative symbols with comma
|
| 98 |
+
decorative_chars = "【】《》()『』「」~"
|
| 99 |
+
for char in decorative_chars:
|
| 100 |
+
text = text.replace(char, ",")
|
| 101 |
+
|
| 102 |
+
# Remove double quotes only (保留单引号)
|
| 103 |
+
quotes = ['"', '"', '"']
|
| 104 |
+
for q in quotes:
|
| 105 |
+
text = text.replace(q, "")
|
| 106 |
+
|
| 107 |
+
# Remove consecutive punctuation (keep only first one)
|
| 108 |
+
text = re.sub(r'([,。!?,.!?;;])+', r'\1', text)
|
| 109 |
+
|
| 110 |
+
return text.strip()
|
| 111 |
+
|
| 112 |
+
logger = logging.get_logger(__name__)
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
AUDIO_PLACEHOLDER = "<|audio|>"
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
@dataclass
|
| 119 |
+
class Message:
|
| 120 |
+
def to_dict(self) -> Dict[str, Any]:
|
| 121 |
+
raise NotImplementedError
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
@dataclass
|
| 125 |
+
class UserMessage(Message):
|
| 126 |
+
text: Optional[str] = None
|
| 127 |
+
reference: Optional[List[Optional[Union[str, torch.Tensor]]]] = None
|
| 128 |
+
instruction: Optional[str] = None
|
| 129 |
+
tokens: Optional[int] = None
|
| 130 |
+
quality: Optional[str] = None
|
| 131 |
+
sound_event: Optional[str] = None
|
| 132 |
+
ambient_sound: Optional[str] = None
|
| 133 |
+
language: Optional[str] = None
|
| 134 |
+
|
| 135 |
+
def __post_init__(self):
|
| 136 |
+
template = """<user_inst>
|
| 137 |
+
- Reference(s):
|
| 138 |
+
{reference}
|
| 139 |
+
- Instruction:
|
| 140 |
+
{instruction}
|
| 141 |
+
- Tokens:
|
| 142 |
+
{tokens}
|
| 143 |
+
- Quality:
|
| 144 |
+
{quality}
|
| 145 |
+
- Sound Event:
|
| 146 |
+
{sound_event}
|
| 147 |
+
- Ambient Sound:
|
| 148 |
+
{ambient_sound}
|
| 149 |
+
- Language:
|
| 150 |
+
{language}
|
| 151 |
+
- Text:
|
| 152 |
+
{text}
|
| 153 |
+
</user_inst>"""
|
| 154 |
+
|
| 155 |
+
audio_codes_list = []
|
| 156 |
+
if self.reference is None:
|
| 157 |
+
reference = "None"
|
| 158 |
+
elif isinstance(self.reference, List):
|
| 159 |
+
reference = []
|
| 160 |
+
for speaker_idx, speaker_reference in enumerate(self.reference):
|
| 161 |
+
if speaker_reference is not None:
|
| 162 |
+
reference.append(f"[S{speaker_idx+1}]:\n{AUDIO_PLACEHOLDER}")
|
| 163 |
+
reference = "\n".join(reference)
|
| 164 |
+
audio_codes_list = [
|
| 165 |
+
speaker_reference
|
| 166 |
+
for speaker_reference in self.reference
|
| 167 |
+
if speaker_reference is not None
|
| 168 |
+
]
|
| 169 |
+
else:
|
| 170 |
+
raise TypeError("`reference` should be exactly a list when it is not None.")
|
| 171 |
+
|
| 172 |
+
content = (
|
| 173 |
+
template.replace("{reference}", str(reference))
|
| 174 |
+
.replace("{instruction}", str(self.instruction))
|
| 175 |
+
.replace("{tokens}", str(self.tokens))
|
| 176 |
+
.replace("{quality}", str(self.quality))
|
| 177 |
+
.replace("{sound_event}", str(self.sound_event))
|
| 178 |
+
.replace("{ambient_sound}", str(self.ambient_sound))
|
| 179 |
+
.replace("{language}", str(self.language))
|
| 180 |
+
.replace("{text}", str(self.text))
|
| 181 |
+
)
|
| 182 |
+
|
| 183 |
+
self._content = content
|
| 184 |
+
self._audio_codes_list = audio_codes_list
|
| 185 |
+
|
| 186 |
+
def to_dict(self):
|
| 187 |
+
return {
|
| 188 |
+
"role": "user",
|
| 189 |
+
"content": self._content,
|
| 190 |
+
"audio_codes_list": self._audio_codes_list,
|
| 191 |
+
}
|
| 192 |
+
|
| 193 |
+
|
| 194 |
+
@dataclass
|
| 195 |
+
class AssistantMessage(Message):
|
| 196 |
+
audio_codes_list: List[Union[str, torch.Tensor]]
|
| 197 |
+
content: str = AUDIO_PLACEHOLDER
|
| 198 |
+
|
| 199 |
+
def to_dict(self):
|
| 200 |
+
return {
|
| 201 |
+
"role": "assistant",
|
| 202 |
+
"content": self.content,
|
| 203 |
+
"audio_codes_list": self.audio_codes_list,
|
| 204 |
+
}
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
USER_MESSAGE_FIELDS = (
|
| 208 |
+
"text",
|
| 209 |
+
"reference",
|
| 210 |
+
"instruction",
|
| 211 |
+
"tokens",
|
| 212 |
+
"quality",
|
| 213 |
+
"sound_event",
|
| 214 |
+
"ambient_sound",
|
| 215 |
+
"language",
|
| 216 |
+
)
|
| 217 |
+
|
| 218 |
+
|
| 219 |
+
class MossTTSDelayProcessor(ProcessorMixin):
|
| 220 |
+
tokenizer_class = "AutoTokenizer"
|
| 221 |
+
audio_tokenizer_class = "AutoModel"
|
| 222 |
+
|
| 223 |
+
tokenizer: PreTrainedTokenizerBase
|
| 224 |
+
audio_tokenizer: Any
|
| 225 |
+
|
| 226 |
+
def __init__(
|
| 227 |
+
self,
|
| 228 |
+
tokenizer: PreTrainedTokenizerBase,
|
| 229 |
+
audio_tokenizer: Any = None,
|
| 230 |
+
model_config: Optional[MossTTSDelayConfig] = None,
|
| 231 |
+
normalize_inputs: bool = False,
|
| 232 |
+
**kwargs,
|
| 233 |
+
):
|
| 234 |
+
super().__init__(tokenizer=tokenizer, audio_tokenizer=audio_tokenizer, **kwargs)
|
| 235 |
+
|
| 236 |
+
# Explicit assignments for type-checkers; ProcessorMixin sets these too.
|
| 237 |
+
self.tokenizer = tokenizer
|
| 238 |
+
self.audio_tokenizer = audio_tokenizer
|
| 239 |
+
if model_config is None:
|
| 240 |
+
model_config = MossTTSDelayConfig()
|
| 241 |
+
self.model_config = model_config
|
| 242 |
+
self.normalize_inputs = normalize_inputs
|
| 243 |
+
|
| 244 |
+
self.imstart_token_id = tokenizer.convert_tokens_to_ids("<|im_start|>")
|
| 245 |
+
self.imend_token_id = tokenizer.convert_tokens_to_ids("<|im_end|>")
|
| 246 |
+
self.newline_token_id = 198
|
| 247 |
+
|
| 248 |
+
def _id_to_token(token_id: int) -> str:
|
| 249 |
+
tok = tokenizer.convert_ids_to_tokens(int(token_id))
|
| 250 |
+
if isinstance(tok, list):
|
| 251 |
+
return tok[0] if len(tok) > 0 else ""
|
| 252 |
+
return cast(str, tok)
|
| 253 |
+
|
| 254 |
+
self.audio_user_slot_token = _id_to_token(
|
| 255 |
+
self.model_config.audio_user_slot_token_id
|
| 256 |
+
)
|
| 257 |
+
self.audio_assistant_gen_slot_token = _id_to_token(
|
| 258 |
+
self.model_config.audio_assistant_gen_slot_token_id
|
| 259 |
+
)
|
| 260 |
+
self.audio_assistant_delay_slot_token = _id_to_token(
|
| 261 |
+
self.model_config.audio_assistant_delay_slot_token_id
|
| 262 |
+
)
|
| 263 |
+
self.audio_start_token = _id_to_token(self.model_config.audio_start_token_id)
|
| 264 |
+
self.audio_end_token = _id_to_token(self.model_config.audio_end_token_id)
|
| 265 |
+
|
| 266 |
+
@classmethod
|
| 267 |
+
def from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs):
|
| 268 |
+
trust_remote_code = kwargs.pop("trust_remote_code", True)
|
| 269 |
+
kwargs.pop("_from_auto", None)
|
| 270 |
+
|
| 271 |
+
audio_tokenizer_name_or_path = kwargs.pop(
|
| 272 |
+
"codec_path", "OpenMOSS-Team/MOSS-Audio-Tokenizer"
|
| 273 |
+
)
|
| 274 |
+
normalize_inputs = kwargs.pop("normalize_inputs", False)
|
| 275 |
+
|
| 276 |
+
pretrained_model_name_or_path = Path(pretrained_model_name_or_path)
|
| 277 |
+
model_config = cast(
|
| 278 |
+
MossTTSDelayConfig,
|
| 279 |
+
AutoConfig.from_pretrained(
|
| 280 |
+
pretrained_model_name_or_path,
|
| 281 |
+
*args,
|
| 282 |
+
trust_remote_code=trust_remote_code,
|
| 283 |
+
**kwargs,
|
| 284 |
+
),
|
| 285 |
+
)
|
| 286 |
+
tokenizer = AutoTokenizer.from_pretrained(
|
| 287 |
+
pretrained_model_name_or_path,
|
| 288 |
+
*args,
|
| 289 |
+
trust_remote_code=trust_remote_code,
|
| 290 |
+
**kwargs,
|
| 291 |
+
)
|
| 292 |
+
audio_tokenizer = AutoModel.from_pretrained(
|
| 293 |
+
audio_tokenizer_name_or_path,
|
| 294 |
+
trust_remote_code=trust_remote_code,
|
| 295 |
+
**kwargs,
|
| 296 |
+
)
|
| 297 |
+
|
| 298 |
+
return cls(
|
| 299 |
+
tokenizer=tokenizer,
|
| 300 |
+
audio_tokenizer=audio_tokenizer,
|
| 301 |
+
model_config=model_config,
|
| 302 |
+
normalize_inputs=normalize_inputs,
|
| 303 |
+
**kwargs,
|
| 304 |
+
)
|
| 305 |
+
|
| 306 |
+
def __call__(self, *args, **kwargs) -> BatchFeature:
|
| 307 |
+
conversations = args[0] if len(args) > 0 else kwargs.pop("conversations")
|
| 308 |
+
mode: str = kwargs.pop("mode", "generation")
|
| 309 |
+
apply_chat_template: bool = kwargs.pop("apply_chat_template", True)
|
| 310 |
+
n_vq: Optional[int] = kwargs.pop("n_vq", None)
|
| 311 |
+
|
| 312 |
+
# Common ProcessorMixin kwargs that we ignore because we always return torch tensors.
|
| 313 |
+
kwargs.pop("return_tensors", None)
|
| 314 |
+
kwargs.pop("padding", None)
|
| 315 |
+
kwargs.pop("truncation", None)
|
| 316 |
+
|
| 317 |
+
"""
|
| 318 |
+
mode only works when a Message is converted to a dict.
|
| 319 |
+
"""
|
| 320 |
+
|
| 321 |
+
if mode not in {"generation", "continuation"}:
|
| 322 |
+
raise RuntimeError
|
| 323 |
+
|
| 324 |
+
if isinstance(conversations, (Message, Dict)):
|
| 325 |
+
conversations = [conversations]
|
| 326 |
+
|
| 327 |
+
truncation = False
|
| 328 |
+
if mode == "continuation":
|
| 329 |
+
truncation = True
|
| 330 |
+
|
| 331 |
+
input_ids_list = []
|
| 332 |
+
for conversation in conversations:
|
| 333 |
+
if isinstance(conversation, (Message, Dict)):
|
| 334 |
+
conversation = [conversation]
|
| 335 |
+
|
| 336 |
+
# Normalize early so downstream logic always deals with dict messages.
|
| 337 |
+
conversation = [self._normalize_message(m) for m in conversation]
|
| 338 |
+
|
| 339 |
+
if (mode == "generation") ^ (len(conversation) % 2 != 0):
|
| 340 |
+
raise ValueError
|
| 341 |
+
|
| 342 |
+
if (mode == "generation") ^ (conversation[-1]["role"] == "user"):
|
| 343 |
+
raise ValueError
|
| 344 |
+
|
| 345 |
+
unified_codes = []
|
| 346 |
+
for message_idx, message in enumerate(conversation):
|
| 347 |
+
if apply_chat_template:
|
| 348 |
+
add_generation_prompt = (
|
| 349 |
+
mode == "generation" and message_idx == len(conversation) - 1
|
| 350 |
+
)
|
| 351 |
+
try:
|
| 352 |
+
content = self.tokenizer.apply_chat_template(
|
| 353 |
+
[{"role": message["role"], "content": message["content"]}],
|
| 354 |
+
add_generation_prompt=add_generation_prompt,
|
| 355 |
+
tokenize=False,
|
| 356 |
+
)
|
| 357 |
+
except TypeError:
|
| 358 |
+
try:
|
| 359 |
+
content = self.tokenizer.apply_chat_template(
|
| 360 |
+
[
|
| 361 |
+
{
|
| 362 |
+
"role": message["role"],
|
| 363 |
+
"content": message["content"],
|
| 364 |
+
}
|
| 365 |
+
],
|
| 366 |
+
add_generation_prompt=add_generation_prompt,
|
| 367 |
+
)
|
| 368 |
+
except Exception:
|
| 369 |
+
logger.warning(
|
| 370 |
+
"apply_chat_template failed; fallback to raw content."
|
| 371 |
+
)
|
| 372 |
+
content = message["content"]
|
| 373 |
+
else:
|
| 374 |
+
content = message["content"]
|
| 375 |
+
|
| 376 |
+
if not isinstance(content, str):
|
| 377 |
+
content = str(content)
|
| 378 |
+
|
| 379 |
+
# Batch-encode all path-based references in one call when possible.
|
| 380 |
+
# This ensures we actually exercise audio_tokenizer.batch_encode for multi-reference prompts,
|
| 381 |
+
# instead of repeatedly calling it with batch=1.
|
| 382 |
+
raw_audio_items = message.get("audio_codes_list", [])
|
| 383 |
+
|
| 384 |
+
audio_codes_list: List[torch.Tensor] = []
|
| 385 |
+
if len(raw_audio_items) > 0:
|
| 386 |
+
encoded_items: List[Optional[torch.Tensor]] = [None] * len(
|
| 387 |
+
raw_audio_items
|
| 388 |
+
)
|
| 389 |
+
paths: List[str] = []
|
| 390 |
+
path_positions: List[int] = []
|
| 391 |
+
|
| 392 |
+
for idx, item in enumerate(raw_audio_items):
|
| 393 |
+
if isinstance(item, torch.Tensor):
|
| 394 |
+
if n_vq is not None and item.shape[1] != n_vq:
|
| 395 |
+
raise RuntimeError(
|
| 396 |
+
"audio_codes's n_vq is not equal to the parameter `n_vq`. Your can set the parameter `n_vq` as None if you have already tokenzied the wavs."
|
| 397 |
+
)
|
| 398 |
+
encoded_items[idx] = item
|
| 399 |
+
continue
|
| 400 |
+
|
| 401 |
+
if isinstance(item, (str, os.PathLike)):
|
| 402 |
+
paths.append(str(item))
|
| 403 |
+
path_positions.append(idx)
|
| 404 |
+
continue
|
| 405 |
+
|
| 406 |
+
raise TypeError(
|
| 407 |
+
"Each audio item must be a torch.Tensor of codes or a path-like string."
|
| 408 |
+
)
|
| 409 |
+
|
| 410 |
+
if len(paths) > 0:
|
| 411 |
+
encoded_from_paths = self.encode_audios_from_path(paths, n_vq)
|
| 412 |
+
if len(encoded_from_paths) != len(paths):
|
| 413 |
+
raise RuntimeError(
|
| 414 |
+
"encode_audios_from_path returned an unexpected number of items."
|
| 415 |
+
)
|
| 416 |
+
for pos, codes in zip(path_positions, encoded_from_paths):
|
| 417 |
+
encoded_items[pos] = codes
|
| 418 |
+
|
| 419 |
+
audio_codes_list = [cast(torch.Tensor, t) for t in encoded_items]
|
| 420 |
+
unified_codes.append(
|
| 421 |
+
self._get_unified_codes(
|
| 422 |
+
message["role"], content, audio_codes_list, truncation
|
| 423 |
+
)
|
| 424 |
+
)
|
| 425 |
+
|
| 426 |
+
unified_codes = torch.cat(unified_codes)
|
| 427 |
+
input_ids_list.append(unified_codes)
|
| 428 |
+
|
| 429 |
+
return BatchFeature(data=self._pad(input_ids_list))
|
| 430 |
+
|
| 431 |
+
@staticmethod
|
| 432 |
+
def build_user_message(
|
| 433 |
+
text: Optional[str] = None,
|
| 434 |
+
reference: Optional[List[Optional[Union[str, torch.Tensor]]]] = None,
|
| 435 |
+
instruction: Optional[str] = None,
|
| 436 |
+
tokens: Optional[int] = None,
|
| 437 |
+
quality: Optional[str] = None,
|
| 438 |
+
sound_event: Optional[str] = None,
|
| 439 |
+
ambient_sound: Optional[str] = None,
|
| 440 |
+
language: Optional[str] = None,
|
| 441 |
+
normalize: bool = False,
|
| 442 |
+
) -> Dict:
|
| 443 |
+
if normalize:
|
| 444 |
+
if text is not None:
|
| 445 |
+
text = normalize_text(text)
|
| 446 |
+
if instruction is not None:
|
| 447 |
+
instruction = normalize_instruction(instruction)
|
| 448 |
+
if reference is not None and not isinstance(reference, list):
|
| 449 |
+
reference = [reference]
|
| 450 |
+
return UserMessage(
|
| 451 |
+
text=text,
|
| 452 |
+
reference=reference,
|
| 453 |
+
instruction=instruction,
|
| 454 |
+
tokens=tokens,
|
| 455 |
+
quality=quality,
|
| 456 |
+
sound_event=sound_event,
|
| 457 |
+
ambient_sound=ambient_sound,
|
| 458 |
+
language=language,
|
| 459 |
+
).to_dict()
|
| 460 |
+
|
| 461 |
+
@staticmethod
|
| 462 |
+
def build_assistant_message(
|
| 463 |
+
audio_codes_list: List[Union[str, torch.Tensor]],
|
| 464 |
+
content: str = AUDIO_PLACEHOLDER,
|
| 465 |
+
) -> Dict:
|
| 466 |
+
return AssistantMessage(
|
| 467 |
+
audio_codes_list=audio_codes_list,
|
| 468 |
+
content=content,
|
| 469 |
+
).to_dict()
|
| 470 |
+
|
| 471 |
+
def _normalize_message(self, message: Union[Message, Dict]) -> Dict:
|
| 472 |
+
if isinstance(message, Message):
|
| 473 |
+
return message.to_dict()
|
| 474 |
+
if not isinstance(message, dict):
|
| 475 |
+
raise TypeError("Each message must be a Message or dict.")
|
| 476 |
+
if "role" not in message:
|
| 477 |
+
raise ValueError("Message dict must include a 'role' field.")
|
| 478 |
+
if "content" in message and "audio_codes_list" in message:
|
| 479 |
+
return message
|
| 480 |
+
role = message["role"]
|
| 481 |
+
if role == "user":
|
| 482 |
+
kwargs = {key: message.get(key) for key in USER_MESSAGE_FIELDS}
|
| 483 |
+
# 应用processor的全局normalize设置
|
| 484 |
+
kwargs['normalize'] = self.normalize_inputs
|
| 485 |
+
return self.build_user_message(**kwargs)
|
| 486 |
+
if role == "assistant":
|
| 487 |
+
return self.build_assistant_message(
|
| 488 |
+
audio_codes_list=message.get("audio_codes_list", []),
|
| 489 |
+
content=message.get("content", AUDIO_PLACEHOLDER),
|
| 490 |
+
)
|
| 491 |
+
raise ValueError(f"Unsupported role: {role}")
|
| 492 |
+
|
| 493 |
+
def _pad(self, input_ids_list: List[torch.Tensor]):
|
| 494 |
+
device = input_ids_list[0].device
|
| 495 |
+
lengths = torch.tensor([w.shape[0] for w in input_ids_list], device=device)
|
| 496 |
+
pad_input_ids = torch.nn.utils.rnn.pad_sequence(
|
| 497 |
+
input_ids_list,
|
| 498 |
+
batch_first=True,
|
| 499 |
+
padding_value=self.model_config.audio_pad_code,
|
| 500 |
+
padding_side="left",
|
| 501 |
+
)
|
| 502 |
+
other_channel_mask = (pad_input_ids.shape[1] - lengths).unsqueeze(
|
| 503 |
+
1
|
| 504 |
+
) > torch.arange(pad_input_ids.shape[1], device=device).unsqueeze(0)
|
| 505 |
+
pad_input_ids[..., 0][other_channel_mask] = self.model_config.pad_token_id
|
| 506 |
+
attention_mask = torch.zeros(
|
| 507 |
+
pad_input_ids.shape[0], pad_input_ids.shape[1], device=device
|
| 508 |
+
)
|
| 509 |
+
attention_mask[~other_channel_mask] = 1
|
| 510 |
+
attention_mask = attention_mask.bool()
|
| 511 |
+
return {
|
| 512 |
+
"input_ids": pad_input_ids, # [batch_size, seqlen, n_vq]
|
| 513 |
+
"attention_mask": attention_mask,
|
| 514 |
+
}
|
| 515 |
+
|
| 516 |
+
@staticmethod
|
| 517 |
+
def _replace_audio_placeholders(
|
| 518 |
+
content: str,
|
| 519 |
+
lengths: List[int],
|
| 520 |
+
n_vq: int,
|
| 521 |
+
gen_slot_token: str,
|
| 522 |
+
delay_slot_token: str,
|
| 523 |
+
audio_start_token: str,
|
| 524 |
+
audio_end_token: str,
|
| 525 |
+
) -> str:
|
| 526 |
+
if n_vq < 1:
|
| 527 |
+
raise ValueError(f"n_vq must be >= 1, got {n_vq}")
|
| 528 |
+
|
| 529 |
+
num_placeholders = content.count(AUDIO_PLACEHOLDER)
|
| 530 |
+
if num_placeholders != len(lengths):
|
| 531 |
+
raise ValueError(
|
| 532 |
+
f"Number of {AUDIO_PLACEHOLDER} ({num_placeholders}) "
|
| 533 |
+
f"does not match lengths ({len(lengths)})"
|
| 534 |
+
)
|
| 535 |
+
|
| 536 |
+
def build_audio_block(length: int) -> str:
|
| 537 |
+
if length < 0:
|
| 538 |
+
raise ValueError(f"length must be >= 0, got {length}")
|
| 539 |
+
|
| 540 |
+
if length == 0:
|
| 541 |
+
return f"{audio_start_token}{audio_end_token}"
|
| 542 |
+
|
| 543 |
+
step_tokens = gen_slot_token * length + (delay_slot_token * (n_vq - 1))
|
| 544 |
+
return f"{audio_start_token}{step_tokens}{audio_end_token}"
|
| 545 |
+
|
| 546 |
+
lengths_iter = iter(lengths)
|
| 547 |
+
|
| 548 |
+
def replacer(match: re.Match) -> str:
|
| 549 |
+
length = next(lengths_iter)
|
| 550 |
+
return build_audio_block(length)
|
| 551 |
+
|
| 552 |
+
result = re.sub(re.escape(AUDIO_PLACEHOLDER), replacer, content)
|
| 553 |
+
|
| 554 |
+
return result
|
| 555 |
+
|
| 556 |
+
@staticmethod
|
| 557 |
+
def _merge_consecutive_audio_placeholders(
|
| 558 |
+
content: str,
|
| 559 |
+
audio_codes_list: List[torch.Tensor],
|
| 560 |
+
) -> Tuple[str, List[torch.Tensor]]:
|
| 561 |
+
matches = list(re.finditer(re.escape(AUDIO_PLACEHOLDER), content))
|
| 562 |
+
if len(matches) <= 1:
|
| 563 |
+
return content, audio_codes_list
|
| 564 |
+
|
| 565 |
+
if len(matches) != len(audio_codes_list):
|
| 566 |
+
raise ValueError(
|
| 567 |
+
"Audio placeholders do not match the provided audio codes list."
|
| 568 |
+
)
|
| 569 |
+
|
| 570 |
+
new_audio_codes_list = []
|
| 571 |
+
new_parts = []
|
| 572 |
+
last_pos = 0
|
| 573 |
+
i = 0
|
| 574 |
+
while i < len(matches):
|
| 575 |
+
j = i
|
| 576 |
+
while (
|
| 577 |
+
j + 1 < len(matches)
|
| 578 |
+
and content[matches[j].end() : matches[j + 1].start()].strip() == ""
|
| 579 |
+
):
|
| 580 |
+
j += 1
|
| 581 |
+
|
| 582 |
+
new_parts.append(content[last_pos : matches[i].start()])
|
| 583 |
+
new_parts.append(AUDIO_PLACEHOLDER)
|
| 584 |
+
last_pos = matches[j].end()
|
| 585 |
+
|
| 586 |
+
if j == i:
|
| 587 |
+
new_audio_codes_list.append(audio_codes_list[i])
|
| 588 |
+
else:
|
| 589 |
+
new_audio_codes_list.append(
|
| 590 |
+
torch.cat(audio_codes_list[i : j + 1], dim=0)
|
| 591 |
+
)
|
| 592 |
+
|
| 593 |
+
i = j + 1
|
| 594 |
+
|
| 595 |
+
new_parts.append(content[last_pos:])
|
| 596 |
+
return "".join(new_parts), new_audio_codes_list
|
| 597 |
+
|
| 598 |
+
@staticmethod
|
| 599 |
+
def apply_delay_pattern(codes: torch.Tensor, pad_code: int) -> torch.Tensor:
|
| 600 |
+
delayed_tokens = torch.full(
|
| 601 |
+
(codes.shape[0] + codes.shape[1] - 1, codes.shape[1]),
|
| 602 |
+
pad_code,
|
| 603 |
+
device=codes.device,
|
| 604 |
+
dtype=codes.dtype,
|
| 605 |
+
)
|
| 606 |
+
for i in range(codes.shape[1]):
|
| 607 |
+
delayed_tokens[i : i + codes.shape[0], i] = codes[:, i]
|
| 608 |
+
return delayed_tokens
|
| 609 |
+
|
| 610 |
+
@staticmethod
|
| 611 |
+
def apply_de_delay_pattern(delay_codes: torch.Tensor) -> torch.Tensor:
|
| 612 |
+
tokens = torch.full(
|
| 613 |
+
(delay_codes.shape[0] - delay_codes.shape[1] + 1, delay_codes.shape[1]),
|
| 614 |
+
0,
|
| 615 |
+
device=delay_codes.device,
|
| 616 |
+
dtype=delay_codes.dtype,
|
| 617 |
+
)
|
| 618 |
+
for i in range(delay_codes.shape[1]):
|
| 619 |
+
tokens[:, i] = delay_codes[i : i + tokens.shape[0], i]
|
| 620 |
+
return tokens
|
| 621 |
+
|
| 622 |
+
def _get_unified_codes(
|
| 623 |
+
self,
|
| 624 |
+
role: str,
|
| 625 |
+
content: str,
|
| 626 |
+
audio_codes_list: List[torch.Tensor],
|
| 627 |
+
truncation: bool,
|
| 628 |
+
) -> torch.Tensor:
|
| 629 |
+
"""
|
| 630 |
+
此时的 content 已经是带上了对话格式
|
| 631 |
+
"""
|
| 632 |
+
if role == "user":
|
| 633 |
+
audio_gen_slot_token = audio_delay_slot_token = self.audio_user_slot_token
|
| 634 |
+
truncation = False
|
| 635 |
+
else:
|
| 636 |
+
audio_gen_slot_token = self.audio_assistant_gen_slot_token
|
| 637 |
+
audio_delay_slot_token = self.audio_assistant_delay_slot_token
|
| 638 |
+
|
| 639 |
+
if len(audio_codes_list):
|
| 640 |
+
n_vq = audio_codes_list[0].shape[1]
|
| 641 |
+
else:
|
| 642 |
+
n_vq = self.model_config.n_vq
|
| 643 |
+
|
| 644 |
+
if len(audio_codes_list) > 1 and AUDIO_PLACEHOLDER in content:
|
| 645 |
+
content, audio_codes_list = self._merge_consecutive_audio_placeholders(
|
| 646 |
+
content, audio_codes_list
|
| 647 |
+
)
|
| 648 |
+
content = self._replace_audio_placeholders(
|
| 649 |
+
content=content,
|
| 650 |
+
lengths=[len(audio_codes) for audio_codes in audio_codes_list],
|
| 651 |
+
n_vq=n_vq,
|
| 652 |
+
gen_slot_token=audio_gen_slot_token,
|
| 653 |
+
delay_slot_token=audio_delay_slot_token,
|
| 654 |
+
audio_start_token=self.audio_start_token,
|
| 655 |
+
audio_end_token=self.audio_end_token,
|
| 656 |
+
)
|
| 657 |
+
text_codes = torch.tensor(
|
| 658 |
+
self.tokenizer.encode(content),
|
| 659 |
+
device=audio_codes_list[0].device if audio_codes_list else None,
|
| 660 |
+
)
|
| 661 |
+
|
| 662 |
+
audio_start_indices = torch.where(
|
| 663 |
+
text_codes == self.model_config.audio_start_token_id
|
| 664 |
+
)[0]
|
| 665 |
+
audio_end_indices = torch.where(
|
| 666 |
+
text_codes == self.model_config.audio_end_token_id
|
| 667 |
+
)[0]
|
| 668 |
+
if len(audio_start_indices) != len(audio_codes_list) or len(
|
| 669 |
+
audio_end_indices
|
| 670 |
+
) != len(audio_codes_list):
|
| 671 |
+
raise ValueError(
|
| 672 |
+
"Audio placeholders do not match the provided audio codes list."
|
| 673 |
+
)
|
| 674 |
+
|
| 675 |
+
delay_audio_codes_list = []
|
| 676 |
+
if len(audio_codes_list) == 0:
|
| 677 |
+
delay_audio_codes_list = torch.full(
|
| 678 |
+
(len(text_codes), n_vq),
|
| 679 |
+
self.model_config.audio_pad_code,
|
| 680 |
+
device=text_codes.device,
|
| 681 |
+
dtype=text_codes.dtype,
|
| 682 |
+
)
|
| 683 |
+
else:
|
| 684 |
+
prefix_idx = 0
|
| 685 |
+
for audio_start_idx_t, audio_end_idx_t, audio_codes in zip(
|
| 686 |
+
audio_start_indices, audio_end_indices, audio_codes_list
|
| 687 |
+
):
|
| 688 |
+
audio_start_idx = int(audio_start_idx_t.item())
|
| 689 |
+
audio_end_idx = int(audio_end_idx_t.item())
|
| 690 |
+
delay_audio_codes = self.apply_delay_pattern(
|
| 691 |
+
audio_codes, self.model_config.audio_pad_code
|
| 692 |
+
)
|
| 693 |
+
pad_codes = torch.full(
|
| 694 |
+
(audio_start_idx - prefix_idx + 1, n_vq),
|
| 695 |
+
self.model_config.audio_pad_code,
|
| 696 |
+
device=audio_codes.device,
|
| 697 |
+
dtype=audio_codes.dtype,
|
| 698 |
+
)
|
| 699 |
+
delay_audio_codes_list.extend([pad_codes, delay_audio_codes])
|
| 700 |
+
prefix_idx = audio_end_idx
|
| 701 |
+
|
| 702 |
+
if truncation:
|
| 703 |
+
delay_audio_codes_list[-1] = delay_audio_codes_list[-1][
|
| 704 |
+
: -(n_vq - 1), :
|
| 705 |
+
]
|
| 706 |
+
else:
|
| 707 |
+
last_audio_end_idx = int(audio_end_indices[-1].item())
|
| 708 |
+
pad_codes = torch.full(
|
| 709 |
+
(len(text_codes) - last_audio_end_idx, n_vq),
|
| 710 |
+
self.model_config.audio_pad_code,
|
| 711 |
+
device=audio_codes_list[0].device,
|
| 712 |
+
dtype=audio_codes_list[0].dtype,
|
| 713 |
+
)
|
| 714 |
+
delay_audio_codes_list.append(pad_codes)
|
| 715 |
+
|
| 716 |
+
delay_audio_codes_list = torch.cat(delay_audio_codes_list)
|
| 717 |
+
|
| 718 |
+
if text_codes.shape[0] != delay_audio_codes_list.shape[0]:
|
| 719 |
+
text_codes = text_codes[: delay_audio_codes_list.shape[0]]
|
| 720 |
+
|
| 721 |
+
unified_codes = torch.cat(
|
| 722 |
+
[text_codes.unsqueeze(1), delay_audio_codes_list], dim=1
|
| 723 |
+
)
|
| 724 |
+
return unified_codes
|
| 725 |
+
|
| 726 |
+
def _parse_text_codes(self, start_length, text_codes):
|
| 727 |
+
text = cast(str, self.tokenizer.decode(text_codes))
|
| 728 |
+
prefix = cast(str, self.tokenizer.decode(text_codes[:start_length]))
|
| 729 |
+
text = text[len(prefix) :]
|
| 730 |
+
|
| 731 |
+
AUDIO_PATTERN = re.compile(
|
| 732 |
+
rf"(?:{self.audio_start_token})?"
|
| 733 |
+
rf"(?:{self.audio_assistant_gen_slot_token})*"
|
| 734 |
+
rf"(?:{self.audio_assistant_delay_slot_token})*"
|
| 735 |
+
rf"{self.audio_end_token}"
|
| 736 |
+
)
|
| 737 |
+
|
| 738 |
+
def normalize_audio_segments(text: str) -> str:
|
| 739 |
+
def repl(match: re.Match) -> str:
|
| 740 |
+
seg = match.group(0)
|
| 741 |
+
# Replace with <|audio|> if gen_slot is present in the segment;
|
| 742 |
+
if self.audio_assistant_gen_slot_token in seg:
|
| 743 |
+
return AUDIO_PLACEHOLDER
|
| 744 |
+
# Otherwise, remove it.
|
| 745 |
+
return ""
|
| 746 |
+
|
| 747 |
+
return AUDIO_PATTERN.sub(repl, text)
|
| 748 |
+
|
| 749 |
+
return normalize_audio_segments(text)
|
| 750 |
+
|
| 751 |
+
def _parse_audio_codes(self, start_length, audio_codes):
|
| 752 |
+
# De-delay back to [T', n_vq]
|
| 753 |
+
audio_codes = self.apply_de_delay_pattern(audio_codes)
|
| 754 |
+
|
| 755 |
+
# Rows that are all pad are separators between real audio segments.
|
| 756 |
+
is_pad = (audio_codes == self.model_config.audio_pad_code).all(dim=1)
|
| 757 |
+
non_pad = ~is_pad
|
| 758 |
+
if not non_pad.any():
|
| 759 |
+
return []
|
| 760 |
+
|
| 761 |
+
idx = torch.nonzero(non_pad).squeeze(1)
|
| 762 |
+
breaks = torch.where(idx[1:] != idx[:-1] + 1)[0] + 1
|
| 763 |
+
if breaks.numel() == 0:
|
| 764 |
+
segments_idx = [idx]
|
| 765 |
+
else:
|
| 766 |
+
segments_idx = torch.split(idx, breaks.tolist())
|
| 767 |
+
|
| 768 |
+
audio_codes_list = [audio_codes[s] for s in segments_idx]
|
| 769 |
+
|
| 770 |
+
# Batch-decode all audio segments together.
|
| 771 |
+
decoded_audio_list = self.decode_audio_codes(audio_codes_list)
|
| 772 |
+
|
| 773 |
+
# Keep codec causal context by decoding the whole first segment first,
|
| 774 |
+
# then trim at waveform level according to start_length ratio.
|
| 775 |
+
if (
|
| 776 |
+
start_length > 0
|
| 777 |
+
and len(audio_codes_list) > 0
|
| 778 |
+
and len(decoded_audio_list) > 0
|
| 779 |
+
):
|
| 780 |
+
first_codes_length = audio_codes_list[0].shape[0]
|
| 781 |
+
if first_codes_length > 0:
|
| 782 |
+
trim_ratio = max(
|
| 783 |
+
0.0, min(float(start_length) / float(first_codes_length), 1.0)
|
| 784 |
+
)
|
| 785 |
+
first_audio = decoded_audio_list[0]
|
| 786 |
+
if trim_ratio >= 1.0:
|
| 787 |
+
decoded_audio_list = decoded_audio_list[1:]
|
| 788 |
+
elif trim_ratio > 0.0:
|
| 789 |
+
trim_samples = int(first_audio.shape[-1] * trim_ratio)
|
| 790 |
+
decoded_audio_list[0] = first_audio[..., trim_samples:]
|
| 791 |
+
|
| 792 |
+
return decoded_audio_list
|
| 793 |
+
|
| 794 |
+
def decode(self, output: List[Tuple[int, torch.Tensor]]):
|
| 795 |
+
"""
|
| 796 |
+
1. 这里不管怎样,都需要一个完整的 assistant generation ids;
|
| 797 |
+
2. 支持从任意位置进行截断;
|
| 798 |
+
"""
|
| 799 |
+
|
| 800 |
+
genearted_messages = []
|
| 801 |
+
for start_length, generation_ids in output:
|
| 802 |
+
content = self._parse_text_codes(start_length, generation_ids[:, 0])
|
| 803 |
+
audio_codes_list = self._parse_audio_codes(
|
| 804 |
+
start_length, generation_ids[:, 1:]
|
| 805 |
+
)
|
| 806 |
+
if content == "":
|
| 807 |
+
message = None
|
| 808 |
+
else:
|
| 809 |
+
message = AssistantMessage(
|
| 810 |
+
content=content,
|
| 811 |
+
audio_codes_list=cast(
|
| 812 |
+
List[Union[str, torch.Tensor]], audio_codes_list
|
| 813 |
+
),
|
| 814 |
+
)
|
| 815 |
+
genearted_messages.append(message)
|
| 816 |
+
return genearted_messages
|
| 817 |
+
|
| 818 |
+
@staticmethod
|
| 819 |
+
def loudness_normalize(
|
| 820 |
+
wav: torch.Tensor,
|
| 821 |
+
target_dbfs: float = -20,
|
| 822 |
+
gain_range: tuple[float, float] = (-3.0, 3.0),
|
| 823 |
+
) -> torch.Tensor:
|
| 824 |
+
wav = wav.to(torch.float32)
|
| 825 |
+
if wav.numel() == 0:
|
| 826 |
+
return wav
|
| 827 |
+
current_dbfs = 10.0 * torch.log10(torch.mean(wav**2) + 1e-9)
|
| 828 |
+
gain = float(target_dbfs - current_dbfs)
|
| 829 |
+
gain = max(gain_range[0], min(gain, gain_range[1]))
|
| 830 |
+
factor = 10.0 ** (gain / 20.0)
|
| 831 |
+
return wav * factor
|
| 832 |
+
|
| 833 |
+
def _get_audio_tokenizer_device(self) -> torch.device:
|
| 834 |
+
"""Best-effort device inference for `self.audio_tokenizer`.
|
| 835 |
+
|
| 836 |
+
Notes:
|
| 837 |
+
- Old TAC wrapper exposed `.device`, but standard `torch.nn.Module` does not.
|
| 838 |
+
- New MossAudioTokenizerModel is a `PreTrainedModel`; parameters define its device.
|
| 839 |
+
"""
|
| 840 |
+
|
| 841 |
+
audio_tokenizer = getattr(self, "audio_tokenizer", None)
|
| 842 |
+
if audio_tokenizer is None:
|
| 843 |
+
logger.warning(
|
| 844 |
+
"audio_tokenizer is not set on processor. Using CPU as default."
|
| 845 |
+
)
|
| 846 |
+
return torch.device("cpu")
|
| 847 |
+
|
| 848 |
+
device_attr = getattr(audio_tokenizer, "device", None)
|
| 849 |
+
if isinstance(device_attr, torch.device):
|
| 850 |
+
return device_attr
|
| 851 |
+
|
| 852 |
+
try:
|
| 853 |
+
return next(audio_tokenizer.parameters()).device
|
| 854 |
+
except StopIteration:
|
| 855 |
+
# No parameters (shouldn't happen for real models); default to CPU.
|
| 856 |
+
logger.warning(
|
| 857 |
+
"No parameters found on audio_tokenizer. Using CPU as default."
|
| 858 |
+
)
|
| 859 |
+
return torch.device("cpu")
|
| 860 |
+
|
| 861 |
+
def encode_audios_from_wav(
|
| 862 |
+
self,
|
| 863 |
+
wav_list: List[torch.Tensor],
|
| 864 |
+
sampling_rate: int,
|
| 865 |
+
n_vq: Optional[int] = None,
|
| 866 |
+
):
|
| 867 |
+
if self.audio_tokenizer is None:
|
| 868 |
+
raise RuntimeError("audio_tokenizer is not set on processor.")
|
| 869 |
+
audio_tokenizer = self.audio_tokenizer
|
| 870 |
+
|
| 871 |
+
if isinstance(wav_list, torch.Tensor):
|
| 872 |
+
wav_list = [wav_list]
|
| 873 |
+
wav_list_ = []
|
| 874 |
+
resample = False
|
| 875 |
+
if sampling_rate != self.model_config.sampling_rate:
|
| 876 |
+
resample = True
|
| 877 |
+
device = self._get_audio_tokenizer_device()
|
| 878 |
+
for wav in wav_list:
|
| 879 |
+
if wav.shape[0] > 1:
|
| 880 |
+
wav = torch.mean(wav, dim=0, keepdim=True)
|
| 881 |
+
if resample:
|
| 882 |
+
wav = torchaudio.functional.resample(
|
| 883 |
+
waveform=wav,
|
| 884 |
+
orig_freq=sampling_rate,
|
| 885 |
+
new_freq=self.model_config.sampling_rate,
|
| 886 |
+
)
|
| 887 |
+
wav = wav.to(device)
|
| 888 |
+
wav_list_.append(self.loudness_normalize(wav.squeeze(0)))
|
| 889 |
+
|
| 890 |
+
# New MossAudioTokenizerModel API: prefer batch_encode(list[wav])
|
| 891 |
+
if hasattr(audio_tokenizer, "batch_encode"):
|
| 892 |
+
enc = audio_tokenizer.batch_encode(wav_list_, num_quantizers=n_vq)
|
| 893 |
+
audio_codes = enc.audio_codes # (NQ, B, T)
|
| 894 |
+
audio_codes_lengths = enc.audio_codes_lengths # (B,)
|
| 895 |
+
else:
|
| 896 |
+
# Fallback: use encode() with explicit padding.
|
| 897 |
+
max_len = max(int(wav.shape[-1]) for wav in wav_list_)
|
| 898 |
+
input_values = torch.zeros(
|
| 899 |
+
len(wav_list_), 1, max_len, device=device, dtype=torch.float32
|
| 900 |
+
)
|
| 901 |
+
padding_mask = torch.zeros(
|
| 902 |
+
len(wav_list_), max_len, device=device, dtype=torch.bool
|
| 903 |
+
)
|
| 904 |
+
for i, wav in enumerate(wav_list_):
|
| 905 |
+
this_len = int(wav.shape[-1])
|
| 906 |
+
input_values[i, 0, :this_len] = wav
|
| 907 |
+
padding_mask[i, :this_len] = True
|
| 908 |
+
enc = audio_tokenizer.encode(
|
| 909 |
+
input_values,
|
| 910 |
+
padding_mask=padding_mask,
|
| 911 |
+
num_quantizers=n_vq,
|
| 912 |
+
return_dict=True,
|
| 913 |
+
)
|
| 914 |
+
audio_codes = enc.audio_codes
|
| 915 |
+
audio_codes_lengths = enc.audio_codes_lengths
|
| 916 |
+
|
| 917 |
+
if audio_codes is None or audio_codes_lengths is None:
|
| 918 |
+
raise RuntimeError(
|
| 919 |
+
"audio_tokenizer.encode() returned empty outputs (audio_codes/audio_codes_lengths)."
|
| 920 |
+
)
|
| 921 |
+
|
| 922 |
+
# Keep processor's historical contract: list[Tensor] with shape (T, NQ)
|
| 923 |
+
# and on CPU (so downstream text/audio packing remains device-agnostic).
|
| 924 |
+
codes_list: List[torch.Tensor] = []
|
| 925 |
+
for i in range(int(audio_codes.shape[1])):
|
| 926 |
+
length_i = int(audio_codes_lengths[i].item())
|
| 927 |
+
codes_i = (
|
| 928 |
+
audio_codes[:, i, :length_i]
|
| 929 |
+
.transpose(0, 1)
|
| 930 |
+
.contiguous()
|
| 931 |
+
.to(torch.long)
|
| 932 |
+
.cpu()
|
| 933 |
+
)
|
| 934 |
+
codes_list.append(codes_i)
|
| 935 |
+
return codes_list
|
| 936 |
+
|
| 937 |
+
def encode_audios_from_path(
|
| 938 |
+
self, wav_path_list: Union[str, List[str]], n_vq: Optional[int] = None
|
| 939 |
+
):
|
| 940 |
+
if isinstance(wav_path_list, str):
|
| 941 |
+
wav_path_list = [wav_path_list]
|
| 942 |
+
|
| 943 |
+
if len(wav_path_list) == 0:
|
| 944 |
+
raise ValueError("Empty wav_path_list")
|
| 945 |
+
|
| 946 |
+
# Load + (if needed) resample each wav independently, so callers can
|
| 947 |
+
# pass a heterogeneous batch of files while still benefiting from
|
| 948 |
+
# audio_tokenizer.batch_encode.
|
| 949 |
+
target_sr = int(self.model_config.sampling_rate)
|
| 950 |
+
wav_list: List[torch.Tensor] = []
|
| 951 |
+
for wav_path in wav_path_list:
|
| 952 |
+
wav, sr = torchaudio.load(wav_path)
|
| 953 |
+
if int(sr) != target_sr:
|
| 954 |
+
wav = torchaudio.functional.resample(
|
| 955 |
+
waveform=wav,
|
| 956 |
+
orig_freq=int(sr),
|
| 957 |
+
new_freq=target_sr,
|
| 958 |
+
)
|
| 959 |
+
wav_list.append(wav)
|
| 960 |
+
|
| 961 |
+
return self.encode_audios_from_wav(wav_list, target_sr, n_vq)
|
| 962 |
+
|
| 963 |
+
def decode_audio_codes(
|
| 964 |
+
self, audio_tokens_list: Union[torch.Tensor, List[torch.Tensor]]
|
| 965 |
+
):
|
| 966 |
+
if self.audio_tokenizer is None:
|
| 967 |
+
raise RuntimeError("audio_tokenizer is not set on processor.")
|
| 968 |
+
audio_tokenizer = self.audio_tokenizer
|
| 969 |
+
|
| 970 |
+
if isinstance(audio_tokens_list, torch.Tensor):
|
| 971 |
+
audio_tokens_list = [audio_tokens_list]
|
| 972 |
+
if len(audio_tokens_list) == 0:
|
| 973 |
+
return []
|
| 974 |
+
|
| 975 |
+
device = self._get_audio_tokenizer_device()
|
| 976 |
+
|
| 977 |
+
# Processor uses (T, NQ); MossAudioTokenizer expects (NQ, T) (or (NQ, B, T)).
|
| 978 |
+
codes_list = [
|
| 979 |
+
codes.transpose(0, 1).contiguous().to(device=device, dtype=torch.long)
|
| 980 |
+
for codes in audio_tokens_list
|
| 981 |
+
]
|
| 982 |
+
|
| 983 |
+
# Fallback: pad to (NQ, B, T) + mask, then decode.
|
| 984 |
+
nq = int(codes_list[0].shape[0])
|
| 985 |
+
max_t = max(int(c.shape[1]) for c in codes_list)
|
| 986 |
+
audio_codes = torch.zeros(
|
| 987 |
+
nq, len(codes_list), max_t, device=device, dtype=torch.long
|
| 988 |
+
)
|
| 989 |
+
padding_mask = torch.zeros(
|
| 990 |
+
len(codes_list), max_t, device=device, dtype=torch.bool
|
| 991 |
+
)
|
| 992 |
+
for i, c in enumerate(codes_list):
|
| 993 |
+
t = int(c.shape[1])
|
| 994 |
+
audio_codes[:, i, :t] = c
|
| 995 |
+
padding_mask[i, :t] = True
|
| 996 |
+
dec = audio_tokenizer.decode(
|
| 997 |
+
audio_codes, padding_mask=padding_mask, return_dict=True, chunk_duration=8
|
| 998 |
+
)
|
| 999 |
+
audio = dec.audio
|
| 1000 |
+
audio_lengths = dec.audio_lengths
|
| 1001 |
+
|
| 1002 |
+
if audio is None or audio_lengths is None:
|
| 1003 |
+
raise RuntimeError(
|
| 1004 |
+
"audio_tokenizer.decode() returned empty outputs (audio/audio_lengths)."
|
| 1005 |
+
)
|
| 1006 |
+
|
| 1007 |
+
# Return historical contract: list of 1D waveforms (T,)
|
| 1008 |
+
wav_list: List[torch.Tensor] = []
|
| 1009 |
+
for i in range(int(audio.shape[0])):
|
| 1010 |
+
length_i = int(audio_lengths[i].item())
|
| 1011 |
+
wav = audio[i, 0, :length_i].contiguous().to(torch.float32).cpu()
|
| 1012 |
+
wav_list.append(wav)
|
| 1013 |
+
return wav_list
|