Vansh Chugh commited on
Commit
ad73d02
·
1 Parent(s): b06789a

initial deploy

Browse files
Files changed (49) hide show
  1. .gitignore +5 -0
  2. README.md +5 -4
  3. SOURCES.md +4 -0
  4. app.py +214 -0
  5. model.json +20 -0
  6. requirements.txt +19 -0
  7. songgen/__init__.py +31 -0
  8. songgen/configuration_songgen.py +250 -0
  9. songgen/custom_modeling_outputs.py +114 -0
  10. songgen/logits_processors.py +85 -0
  11. songgen/lyrics_utils/__init__.py +0 -0
  12. songgen/lyrics_utils/lyrics_encoder.py +1071 -0
  13. songgen/lyrics_utils/lyrics_tokenizer.py +874 -0
  14. songgen/lyrics_utils/zh_num2words.py +1209 -0
  15. songgen/modeling_songgen_dual_track.py +623 -0
  16. songgen/modeling_songgen_mixed.py +0 -0
  17. songgen/processing_songgen.py +141 -0
  18. songgen/xcodec_wrapper/__init__.py +2 -0
  19. songgen/xcodec_wrapper/configuration_xcodec.py +24 -0
  20. songgen/xcodec_wrapper/modeling_xcodec.py +167 -0
  21. songgen/xcodec_wrapper/xcodec_infer/ckpts/general_more/config_hubert_general.yaml +137 -0
  22. songgen/xcodec_wrapper/xcodec_infer/ckpts/general_more/xcodec_hubert_general_audio_v2.pth +3 -0
  23. songgen/xcodec_wrapper/xcodec_infer/descriptaudiocodec/__init__.py +0 -0
  24. songgen/xcodec_wrapper/xcodec_infer/descriptaudiocodec/dac/__init__.py +16 -0
  25. songgen/xcodec_wrapper/xcodec_infer/descriptaudiocodec/dac/model/__init__.py +4 -0
  26. songgen/xcodec_wrapper/xcodec_infer/descriptaudiocodec/dac/model/base.py +294 -0
  27. songgen/xcodec_wrapper/xcodec_infer/descriptaudiocodec/dac/model/dac.py +369 -0
  28. songgen/xcodec_wrapper/xcodec_infer/descriptaudiocodec/dac/model/discriminator.py +228 -0
  29. songgen/xcodec_wrapper/xcodec_infer/descriptaudiocodec/dac/nn/__init__.py +3 -0
  30. songgen/xcodec_wrapper/xcodec_infer/descriptaudiocodec/dac/nn/layers.py +33 -0
  31. songgen/xcodec_wrapper/xcodec_infer/descriptaudiocodec/dac/nn/loss.py +368 -0
  32. songgen/xcodec_wrapper/xcodec_infer/descriptaudiocodec/dac/nn/quantize.py +262 -0
  33. songgen/xcodec_wrapper/xcodec_infer/descriptaudiocodec/dac/utils/__init__.py +123 -0
  34. songgen/xcodec_wrapper/xcodec_infer/models/__init__.py +0 -0
  35. songgen/xcodec_wrapper/xcodec_infer/models/soundstream_semantic.py +157 -0
  36. songgen/xcodec_wrapper/xcodec_infer/modules/__init__.py +22 -0
  37. songgen/xcodec_wrapper/xcodec_infer/modules/conv.py +252 -0
  38. songgen/xcodec_wrapper/xcodec_infer/modules/lstm.py +28 -0
  39. songgen/xcodec_wrapper/xcodec_infer/modules/norm.py +28 -0
  40. songgen/xcodec_wrapper/xcodec_infer/modules/seanet.py +255 -0
  41. songgen/xcodec_wrapper/xcodec_infer/modules/semantic_module.py +294 -0
  42. songgen/xcodec_wrapper/xcodec_infer/modules/transformer.py +119 -0
  43. songgen/xcodec_wrapper/xcodec_infer/quantization/__init__.py +8 -0
  44. songgen/xcodec_wrapper/xcodec_infer/quantization/core_vq_lsx_version.py +433 -0
  45. songgen/xcodec_wrapper/xcodec_infer/quantization/distrib.py +125 -0
  46. songgen/xcodec_wrapper/xcodec_infer/quantization/vq.py +115 -0
  47. songgen/xcodec_wrapper/xcodec_infer/xutils/__init__.py +0 -0
  48. songgen/xcodec_wrapper/xcodec_infer/xutils/ddp_utils.py +200 -0
  49. songgen/xcodec_wrapper/xcodec_infer/xutils/utils.py +259 -0
.gitignore ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ __pycache__/
2
+ *.pyc
3
+ .DS_Store
4
+ .venv/
5
+ SongGen-repo/
README.md CHANGED
@@ -1,13 +1,14 @@
1
  ---
2
  title: SongGen
3
- emoji: 🦀
4
  colorFrom: purple
5
- colorTo: green
6
  sdk: gradio
7
- sdk_version: 6.20.0
8
- python_version: '3.12'
9
  app_file: app.py
10
  pinned: false
 
11
  ---
12
 
13
  Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
1
  ---
2
  title: SongGen
3
+ emoji: 🎤
4
  colorFrom: purple
5
+ colorTo: pink
6
  sdk: gradio
7
+ sdk_version: 5.28.0
8
+ python_version: '3.11'
9
  app_file: app.py
10
  pinned: false
11
+ license: mit
12
  ---
13
 
14
  Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
SOURCES.md ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ # Sources — SongGen
2
+
3
+ - Source repo: https://github.com/LiuZH-19/SongGen
4
+ - Paper: https://arxiv.org/html/2502.13128v2
app.py ADDED
@@ -0,0 +1,214 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import sys
2
+
3
+ sys.stdout.reconfigure(line_buffering=True)
4
+
5
+ try:
6
+ import spaces
7
+ except ImportError:
8
+ # keep @spaces.GPU usable as a no-op; ZeroGPU requires this exact name.
9
+ class spaces:
10
+ class GPU:
11
+ def __init__(self, func=None, duration=60):
12
+ self.func = func
13
+
14
+ def __call__(self, *args, **kwargs):
15
+ if self.func is not None:
16
+ return self.func(*args, **kwargs)
17
+ func = args[0]
18
+ return func
19
+
20
+ import os
21
+ import shutil
22
+ import tempfile
23
+ import threading
24
+
25
+ import gradio as gr
26
+ import numpy as np
27
+ import soundfile as sf
28
+ import torch
29
+ from huggingface_hub import hf_hub_download
30
+ from pyharp import ModelCard, build_endpoint
31
+
32
+ from songgen import (
33
+ SongGenDualTrackForConditionalGeneration,
34
+ SongGenMixedForConditionalGeneration,
35
+ SongGenProcessor,
36
+ )
37
+
38
+ DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
39
+
40
+ MODES = {
41
+ "Mixed": SongGenMixedForConditionalGeneration,
42
+ "Dual-track (vocals + accompaniment)": SongGenDualTrackForConditionalGeneration,
43
+ }
44
+ CHECKPOINTS = {
45
+ "Mixed": "LiuZH-19/SongGen_mixed_pro",
46
+ "Dual-track (vocals + accompaniment)": "LiuZH-19/SongGen_interleaving_A_V",
47
+ }
48
+
49
+ state = {name: {"model": None, "processor": None} for name in MODES}
50
+ loading_done = False
51
+ loading_error = None
52
+
53
+
54
+ def fetch_xcodec_assets():
55
+ """Grab the X-Codec checkpoint and its config. SongGen's XCodecModel
56
+ hardcodes this exact folder rather than taking a path argument, so the
57
+ files have to be here before any SongGen model is constructed."""
58
+ ckpt_dir = os.path.join(
59
+ os.path.dirname(__file__), "songgen", "xcodec_wrapper", "xcodec_infer", "ckpts", "general_more"
60
+ )
61
+ os.makedirs(ckpt_dir, exist_ok=True)
62
+ for filename in ("xcodec_hubert_general_audio_v2.pth", "config_hubert_general.yaml"):
63
+ dest = os.path.join(ckpt_dir, filename)
64
+ if not os.path.exists(dest):
65
+ src = hf_hub_download(repo_id="ZhenYe234/xcodec", filename=filename)
66
+ shutil.copy(src, dest)
67
+
68
+
69
+ def load_all():
70
+ """Download and build both modes on CPU only. Do NOT call .to(DEVICE) or
71
+ otherwise touch CUDA here — ZeroGPU only intercepts CUDA calls made
72
+ inside an @spaces.GPU-decorated call, not from a background thread. This
73
+ split is a no-op on CPU-only hardware (DEVICE == "cpu"), so keep it even
74
+ while testing on a personal CPU-tier Space, before GPU code is added later."""
75
+ global loading_done, loading_error
76
+ try:
77
+ fetch_xcodec_assets()
78
+ for mode_name, model_cls in MODES.items():
79
+ repo_id = CHECKPOINTS[mode_name]
80
+ print(f"Loading {mode_name} ({repo_id})...")
81
+ state[mode_name]["model"] = model_cls.from_pretrained(repo_id, attn_implementation="sdpa")
82
+ state[mode_name]["processor"] = SongGenProcessor(repo_id, "cpu")
83
+ print(f"{mode_name} loaded (CPU).")
84
+ except Exception as e:
85
+ loading_error = str(e)
86
+ print(f"Load error: {e}")
87
+ finally:
88
+ loading_done = True
89
+
90
+
91
+ threading.Thread(target=load_all, daemon=True).start()
92
+
93
+
94
+ model_card = ModelCard(
95
+ name="SongGen",
96
+ description=(
97
+ "Single-stage auto-regressive transformer for text-to-song generation. "
98
+ "Give it lyrics and a description of the music, plus an optional reference "
99
+ "voice, and it generates a full song."
100
+ ),
101
+ author="Zihan Liu, Shuangrui Ding, Zhixiong Zhang, Xiaoyi Dong, Pan Zhang, Yuhang Zang, Yuhang Cao, Dahua Lin, Jiaqi Wang",
102
+ tags=["text-to-music", "song-generation"],
103
+ )
104
+
105
+
106
+ def move_mode_to_device(mode_name):
107
+ """Move one mode's model and helper submodules onto DEVICE the first time
108
+ it's used. Rebuilding isn't needed here since each mode's model/processor
109
+ pair is only ever referenced from this one spot."""
110
+ entry = state[mode_name]
111
+ model = entry["model"]
112
+ processor = entry["processor"]
113
+ if next(model.parameters()).device.type != DEVICE:
114
+ model.to(DEVICE)
115
+ processor.device = DEVICE
116
+ processor.demucs = processor.demucs.to(DEVICE)
117
+ return model, processor
118
+
119
+
120
+ def write_wav(audio_arr, sample_rate):
121
+ """Save a numpy audio array to a temp wav file and return its path."""
122
+ path = tempfile.NamedTemporaryFile(suffix=".wav", delete=False).name
123
+ sf.write(path, audio_arr, sample_rate)
124
+ return path
125
+
126
+
127
+ def silent_wav(sample_rate, seconds=0.1):
128
+ """A short silent placeholder for the vocals-stem output when the
129
+ selected mode doesn't produce a separate vocal track."""
130
+ return write_wav(np.zeros(int(seconds * sample_rate), dtype=np.float32), sample_rate)
131
+
132
+
133
+ @spaces.GPU(duration=120)
134
+ @torch.inference_mode()
135
+ def process_fn(mode_name, description, lyrics, ref_voice_path, extract_vocals, temperature):
136
+ """Generate a song from a description and lyrics, optionally guided by a reference voice."""
137
+ if not loading_done:
138
+ raise gr.Error("Model is still loading, please wait a moment and try again.")
139
+ if loading_error:
140
+ raise gr.Error(f"Model failed to load: {loading_error}")
141
+
142
+ model, processor = move_mode_to_device(mode_name)
143
+
144
+ model_inputs = processor(
145
+ text=description,
146
+ lyrics=lyrics,
147
+ ref_voice_path=ref_voice_path if ref_voice_path else None,
148
+ separate=extract_vocals,
149
+ )
150
+ generation = model.generate(**model_inputs, do_sample=True, temperature=temperature)
151
+ sample_rate = model.config.sampling_rate
152
+
153
+ if mode_name == "Mixed":
154
+ song_path = write_wav(generation.cpu().numpy().squeeze(), sample_rate)
155
+ vocals_path = silent_wav(sample_rate)
156
+ else:
157
+ acc_array = generation[0].cpu().numpy().squeeze()
158
+ vocal_array = generation[1].cpu().numpy().squeeze()
159
+ min_len = min(vocal_array.shape[0], acc_array.shape[0])
160
+ acc_array = acc_array[:min_len]
161
+ vocal_array = vocal_array[:min_len]
162
+
163
+ song_path = write_wav(vocal_array + acc_array, sample_rate)
164
+ vocals_path = write_wav(vocal_array, sample_rate)
165
+
166
+ return song_path, vocals_path
167
+
168
+
169
+ with gr.Blocks() as demo:
170
+ input_components = [
171
+ gr.Dropdown(
172
+ choices=list(MODES.keys()),
173
+ value="Mixed",
174
+ label="Mode",
175
+ info="Mixed outputs one finished track; Dual-track also gives you an isolated vocal stem",
176
+ ),
177
+ gr.Textbox(
178
+ label="Music Description",
179
+ info="Genre, mood, instruments (e.g. \"upbeat pop song with piano and drums\")",
180
+ ),
181
+ gr.Textbox(
182
+ label="Lyrics",
183
+ info="English lyrics for the song (required)",
184
+ ),
185
+ gr.Audio(type="filepath", label="Reference Voice (optional)").harp_required(False),
186
+ gr.Checkbox(
187
+ value=True,
188
+ label="Extract vocals from reference",
189
+ info="(default: True, per repo example) turn off only if the reference audio is already an isolated vocal, not a full mix",
190
+ ),
191
+ gr.Slider(
192
+ minimum=0.1, maximum=2.0, step=0.1, value=1.0,
193
+ label="Creativity",
194
+ info="(default: 1.0, per repo config) sampling temperature — higher is more varied/unpredictable, lower is safer/more repetitive",
195
+ ),
196
+ ]
197
+ output_components = [
198
+ gr.Audio(type="filepath", label="Song").set_info(
199
+ "The generated song. In Dual-track mode this is vocals + accompaniment combined."
200
+ ),
201
+ gr.Audio(type="filepath", label="Isolated Vocals").set_info(
202
+ "Vocal-only stem. Only populated in Dual-track mode; silent in Mixed mode."
203
+ ),
204
+ ]
205
+
206
+ build_endpoint(
207
+ model_card=model_card,
208
+ input_components=input_components,
209
+ output_components=output_components,
210
+ process_fn=process_fn,
211
+ )
212
+
213
+ if __name__ == "__main__":
214
+ demo.queue().launch(pwa=True)
model.json ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "SongGen",
3
+ "local_smoke_threshold_gb": 20.0,
4
+ "package_dir": "songgen",
5
+ "entry_point": [
6
+ "songgen.modeling_songgen_mixed.SongGenMixedForConditionalGeneration",
7
+ "songgen.modeling_songgen_dual_track.SongGenDualTrackForConditionalGeneration",
8
+ "songgen.processing_songgen.SongGenProcessor"
9
+ ],
10
+ "checkpoint": {
11
+ "size_mb": 14114,
12
+ "note": "loaded via from_pretrained/hf_hub_download at runtime, not committed to Space repo. Both mode checkpoints held in memory simultaneously (dropdown selects mode at inference time): LiuZH-19/SongGen_mixed_pro (~5457MB) + LiuZH-19/SongGen_interleaving_A_V (~5422MB), each of which internally constructs its own full m-a-p/MERT-v1-330M model via AutoModel.from_pretrained(trust_remote_code=True) (~1262MB each, loaded twice total) for reference-voice conditioning. Plus xcodec ZhenYe234/xcodec xcodec_hubert_general_audio_v2.pth + config_hubert_general.yaml (~710MB, shared single copy) and htdemucs separation weights (~150MB) auto-downloaded by SongGenProcessor (also loaded once per mode).",
13
+ "repos": {
14
+ "mixed_pro": "LiuZH-19/SongGen_mixed_pro",
15
+ "dual_track": "LiuZH-19/SongGen_interleaving_A_V",
16
+ "xcodec": {"repo": "ZhenYe234/xcodec", "filename": "xcodec_hubert_general_audio_v2.pth", "config_filename": "config_hubert_general.yaml"},
17
+ "mert": "m-a-p/MERT-v1-330M"
18
+ }
19
+ }
20
+ }
requirements.txt ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ git+https://github.com/TEAMuP-dev/pyharp.git@develop
2
+ # model-specific deps below:
3
+ torch==2.8.0
4
+ torchaudio==2.8.0
5
+ transformers>=4.43.0,<=4.43.3
6
+ sentencepiece
7
+ descript-audio-codec
8
+ descript-audiotools>=0.7.2
9
+ omegaconf
10
+ einops
11
+ demucs
12
+ pypinyin
13
+ hangul_romanize
14
+ num2words
15
+ spacy
16
+ nnAudio
17
+ soundfile>=0.12.1
18
+ librosa
19
+ huggingface_hub
songgen/__init__.py ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ __version__ = "0.1"
2
+
3
+
4
+ from transformers import AutoConfig, AutoModel
5
+
6
+ from .configuration_songgen import SongGenConfig, SongGenDecoderConfig
7
+ from .xcodec_wrapper import XCodecConfig, XCodecModel
8
+ from .modeling_songgen_mixed import (
9
+ SongGenForCausalLM,
10
+ SongGenMixedForConditionalGeneration,
11
+ apply_delay_pattern_mask,
12
+ build_delay_pattern_mask,
13
+ )
14
+
15
+ from .modeling_songgen_dual_track import (
16
+ SongGenDualTrackForConditionalGeneration,
17
+ split_combined_track_input_ids,
18
+ build_combined_delay_pattern_mask,
19
+ combine_track_input_ids
20
+ )
21
+
22
+ from .processing_songgen import SongGenProcessor
23
+
24
+ # from .streamer import SongGenStreamer
25
+
26
+ from .lyrics_utils.lyrics_tokenizer import VoiceBpeTokenizer
27
+
28
+ AutoConfig.register("xcodec", XCodecConfig)
29
+ AutoModel.register(XCodecConfig, XCodecModel)
30
+
31
+
songgen/configuration_songgen.py ADDED
@@ -0,0 +1,250 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # coding=utf-8
2
+ # Copyright 2024 and The HuggingFace Inc. team. All rights reserved.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ """ SongGen model configuration"""
17
+
18
+ from transformers import AutoConfig, logging
19
+ from transformers.configuration_utils import PretrainedConfig
20
+
21
+
22
+ logger = logging.get_logger(__name__)
23
+
24
+
25
+ class SongGenDecoderConfig(PretrainedConfig):
26
+ r"""
27
+ This is the configuration class to store the configuration of an [`SongGenDecoder`]. It is used to instantiate a
28
+ SongGen decoder according to the specified arguments, defining the model architecture.
29
+
30
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
31
+ documentation from [`PretrainedConfig`] for more information.
32
+
33
+
34
+ Args:
35
+ vocab_size (`int`, *optional*, defaults to 2049):
36
+ Vocabulary size of the SongGenDecoder model. Defines the number of different tokens that can be
37
+ represented by the `inputs_ids` passed when calling [`SongGenDecoder`].
38
+ hidden_size (`int`, *optional*, defaults to 1024):
39
+ Dimensionality of the layers and the pooler layer.
40
+ num_hidden_layers (`int`, *optional*, defaults to 24):
41
+ Number of decoder layers.
42
+ num_attention_heads (`int`, *optional*, defaults to 16):
43
+ Number of attention heads for each attention layer in the Transformer block.
44
+ num_key_value_heads (`int`, *optional*):
45
+ This is the number of key_value heads that should be used to implement Grouped Query Attention. If
46
+ `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
47
+ `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
48
+ converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
49
+ by meanpooling all the original heads within that group. For more details checkout [this
50
+ paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to
51
+ `num_attention_heads`.
52
+ num_cross_attention_key_value_heads (`int`, *optional*):
53
+ This is the number of key_value heads that should be used to implement Grouped Query Attention in the cross-attention layers.
54
+ If it is not specified, will default to `num_key_value_heads`.
55
+ ffn_dim (`int`, *optional*, defaults to 4096):
56
+ Dimensionality of the "intermediate" (often named feed-forward) layer in the Transformer block.
57
+ activation_function (`str` or `function`, *optional*, defaults to `"gelu"`):
58
+ The non-linear activation function (function or string) in the decoder and pooler. If string, `"gelu"`,
59
+ `"relu"`, `"silu"` and `"gelu_new"` are supported.
60
+ dropout (`float`, *optional*, defaults to 0.1):
61
+ The dropout probability for all fully connected layers in the embeddings, text_encoder, and pooler.
62
+ attention_dropout (`float`, *optional*, defaults to 0.0):
63
+ The dropout ratio for the attention probabilities.
64
+ activation_dropout (`float`, *optional*, defaults to 0.0):
65
+ The dropout ratio for activations inside the fully connected layer.
66
+ max_position_embeddings (`int`, *optional*, defaults to 2048):
67
+ The maximum sequence length that this model might ever be used with. Typically, set this to something large
68
+ just in case (e.g., 512 or 1024 or 2048).
69
+ initializer_factor (`float`, *optional*, defaults to 0.02):
70
+ The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
71
+ layerdrop (`float`, *optional*, defaults to 0.0):
72
+ The LayerDrop probability for the decoder. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556)
73
+ for more details.
74
+ scale_embedding (`bool`, *optional*, defaults to `False`):
75
+ Scale embeddings by diving by sqrt(hidden_size).
76
+ use_cache (`bool`, *optional*, defaults to `True`):
77
+ Whether the model should return the last key/values attentions (not used by all models)
78
+ num_codebooks (`int`, *optional*, defaults to 4):
79
+ The number of parallel codebooks forwarded to the model.
80
+ tie_word_embeddings(`bool`, *optional*, defaults to `False`):
81
+ Whether input and output word embeddings should be tied.
82
+ rope_embeddings (`bool`, *optional*, defaults to `False`):
83
+ Whether to use ROPE or absolute positional embeddings.
84
+ rope_theta (`float`, *optional*, defaults to 100000.0):
85
+ The base period of the RoPE embeddings.
86
+ cross_attention_implementation_strategy (`str`, *optional*):
87
+ If not specified, the cross-attention implementation will be the same as `_attn_implementation`. If `always_eager`, it will always be the eager implementation. If `always_sdpa`, it will always be the sdpa implementation.
88
+ """
89
+
90
+ model_type = "songgen_decoder"
91
+ keys_to_ignore_at_inference = ["past_key_values"]
92
+
93
+ def __init__(
94
+ self,
95
+ vocab_size=1088, # vocab size = 1024 (codec vocab size) + 64
96
+ max_position_embeddings=6000,
97
+ num_hidden_layers=24,
98
+ ffn_dim=4096,
99
+ num_attention_heads=16,
100
+ num_key_value_heads=None,
101
+ num_cross_attention_key_value_heads=None,
102
+ layerdrop=0.0,
103
+ use_cache=True,
104
+ activation_function="gelu",
105
+ hidden_size=1024,
106
+ dropout=0.1,
107
+ attention_dropout=0.0,
108
+ activation_dropout=0.0,
109
+ initializer_factor=0.02,
110
+ scale_embedding=False,
111
+ num_codebooks=4,
112
+ pad_token_id=1024,
113
+ bos_token_id=1025,
114
+ eos_token_id=1024,
115
+ tie_word_embeddings=False,
116
+ rope_embeddings=False,
117
+ rope_theta=10_000.0,
118
+ cross_attention_implementation_strategy=None,
119
+ **kwargs,
120
+ ):
121
+ self.vocab_size = vocab_size
122
+ self.max_position_embeddings = max_position_embeddings
123
+ self.hidden_size = hidden_size
124
+ self.ffn_dim = ffn_dim
125
+ self.num_hidden_layers = num_hidden_layers
126
+ self.num_attention_heads = num_attention_heads
127
+ if num_key_value_heads is None:
128
+ num_key_value_heads = num_attention_heads
129
+ self.num_key_value_heads = num_key_value_heads
130
+ if num_cross_attention_key_value_heads is None:
131
+ num_cross_attention_key_value_heads = num_key_value_heads
132
+ self.num_cross_attention_key_value_heads = num_cross_attention_key_value_heads
133
+ self.dropout = dropout
134
+ self.attention_dropout = attention_dropout
135
+ self.activation_dropout = activation_dropout
136
+ self.activation_function = activation_function
137
+ self.initializer_factor = initializer_factor
138
+ self.layerdrop = layerdrop
139
+ self.use_cache = use_cache
140
+ self.scale_embedding = scale_embedding # scale factor will be sqrt(d_model) if True
141
+ self.num_codebooks = num_codebooks
142
+ self.rope_embeddings = rope_embeddings
143
+ self.rope_theta = rope_theta
144
+ self.cross_attention_implementation_strategy = cross_attention_implementation_strategy
145
+
146
+ super().__init__(
147
+ pad_token_id=pad_token_id,
148
+ bos_token_id=bos_token_id,
149
+ eos_token_id=eos_token_id,
150
+ tie_word_embeddings=tie_word_embeddings,
151
+ **kwargs,
152
+ )
153
+
154
+
155
+ class SongGenConfig(PretrainedConfig):
156
+ r"""
157
+ This is the configuration class to store the configuration of a [`SongGenModel`]. It is used to instantiate a
158
+ SongGen model according to the specified arguments, defining the text encoder, audio encoder and SongGen decoder
159
+ configs.
160
+
161
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
162
+ documentation from [`PretrainedConfig`] for more information.
163
+
164
+ Args:
165
+ vocab_size (`int`, *optional*, defaults to 1024):
166
+ Vocabulary size of the prompt token ids. Defines the number of different tokens that can be
167
+ represented by the `prompt_inputs_ids`.
168
+ prompt_cross_attention (`bool`, *optional*, defaults to `False`):
169
+ Whether to use cross-attention conditioning for the prompt (as well as the description).
170
+ kwargs (*optional*):
171
+ Dictionary of keyword arguments. Notably:
172
+
173
+ - **text_encoder** ([`PretrainedConfig`], *optional*) -- An instance of a configuration object that
174
+ defines the text encoder config.
175
+ - **audio_encoder** ([`PretrainedConfig`], *optional*) -- An instance of a configuration object that
176
+ defines the audio encoder config.
177
+ - **decoder** ([`PretrainedConfig`], *optional*) -- An instance of a configuration object that defines
178
+ the decoder config.
179
+ """
180
+
181
+ model_type = "songgen"
182
+ is_composition = True
183
+
184
+ def __init__(self, vocab_size=1024, prompt_cross_attention=True, add_prenet=True, **kwargs):
185
+ super().__init__(**kwargs)
186
+ if "text_encoder" not in kwargs or "decoder" not in kwargs: #or "audio_encoder" not in kwargs
187
+ raise ValueError("Config has to be initialized with text_encoder and decoder config") #, audio_encoder
188
+
189
+ text_encoder_config = kwargs.pop("text_encoder")
190
+ text_encoder_model_type = text_encoder_config.pop("model_type")
191
+
192
+ # audio_encoder_config = kwargs.pop("audio_encoder")
193
+ # audio_encoder_model_type = audio_encoder_config.pop("model_type")
194
+
195
+ decoder_config = kwargs.pop("decoder")
196
+
197
+ self.vocab_size = vocab_size
198
+ self.prompt_cross_attention = prompt_cross_attention
199
+ self.add_prenet = add_prenet
200
+ self.text_encoder = AutoConfig.for_model(text_encoder_model_type, **text_encoder_config)
201
+ # self.audio_encoder = AutoConfig.for_model(audio_encoder_model_type, **audio_encoder_config)
202
+ self.decoder = SongGenDecoderConfig(**decoder_config)
203
+ self.is_encoder_decoder = True
204
+
205
+
206
+ @classmethod
207
+ def from_sub_models_config(
208
+ cls,
209
+ text_encoder_config: PretrainedConfig,
210
+ # audio_encoder_config: PretrainedConfig,
211
+ decoder_config: SongGenDecoderConfig,
212
+ **kwargs,
213
+ ):
214
+ r"""
215
+ Instantiate a [`SongGenConfig`] (or a derived class) from text encoder, audio encoder and decoder
216
+ configurations.
217
+
218
+ Returns:
219
+ [`SongGenConfig`]: An instance of a configuration object
220
+ """
221
+
222
+ return cls(
223
+ text_encoder=text_encoder_config.to_dict(),
224
+ # audio_encoder=audio_encoder_config.to_dict(),
225
+ decoder=decoder_config.to_dict(),
226
+ **kwargs,
227
+ )
228
+
229
+ @property
230
+ # This is a property because you might want to change the codec model on the fly
231
+ def sampling_rate(self):
232
+ return 16000 #self.audio_encoder.sampling_rate xcodec sampling_rate 16khz
233
+
234
+ # Copy from musicgen
235
+ @property
236
+ def _attn_implementation(self):
237
+ # This property is made private for now (as it cannot be changed and a PreTrainedModel.use_attn_implementation method needs to be implemented.)
238
+ if hasattr(self, "_attn_implementation_internal"):
239
+ if self._attn_implementation_internal is None:
240
+ # `config.attn_implementation` should never be None, for backward compatibility.
241
+ return "eager"
242
+ else:
243
+ return self._attn_implementation_internal
244
+ else:
245
+ return "eager"
246
+
247
+ @_attn_implementation.setter
248
+ def _attn_implementation(self, value):
249
+ self._attn_implementation_internal = value
250
+ self.decoder._attn_implementation = value
songgen/custom_modeling_outputs.py ADDED
@@ -0,0 +1,114 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import warnings
2
+ from dataclasses import dataclass
3
+ from typing import Optional, Tuple, List
4
+ import torch
5
+ from transformers.utils import ModelOutput
6
+
7
+ @dataclass
8
+ class Seq2SeqLMOutput(ModelOutput):
9
+ """
10
+ Base class for sequence-to-sequence language models outputs.
11
+
12
+ Args:
13
+ loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
14
+ Language modeling loss.
15
+ logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
16
+ Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
17
+ past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
18
+ Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
19
+ `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape
20
+ `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
21
+
22
+ Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
23
+ blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
24
+ decoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
25
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
26
+ one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.
27
+
28
+ Hidden-states of the decoder at the output of each layer plus the initial embedding outputs.
29
+ decoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
30
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
31
+ sequence_length)`.
32
+
33
+ Attentions weights of the decoder, after the attention softmax, used to compute the weighted average in the
34
+ self-attention heads.
35
+ cross_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
36
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
37
+ sequence_length)`.
38
+
39
+ Attentions weights of the decoder's cross-attention layer, after the attention softmax, used to compute the
40
+ weighted average in the cross-attention heads.
41
+ encoder_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
42
+ Sequence of hidden-states at the output of the last layer of the encoder of the model.
43
+ encoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
44
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
45
+ one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.
46
+
47
+ Hidden-states of the encoder at the output of each layer plus the initial embedding outputs.
48
+ encoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
49
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
50
+ sequence_length)`.
51
+
52
+ Attentions weights of the encoder, after the attention softmax, used to compute the weighted average in the
53
+ self-attention heads.
54
+ """
55
+
56
+ loss: Optional[torch.FloatTensor] = None
57
+ vocal_loss: Optional[torch.FloatTensor] = None
58
+ logits: torch.FloatTensor = None
59
+ past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None
60
+ decoder_hidden_states: Optional[Tuple[torch.FloatTensor, ...]] = None
61
+ decoder_attentions: Optional[Tuple[torch.FloatTensor, ...]] = None
62
+ cross_attentions: Optional[Tuple[torch.FloatTensor, ...]] = None
63
+ encoder_last_hidden_state: Optional[torch.FloatTensor] = None
64
+ encoder_hidden_states: Optional[Tuple[torch.FloatTensor, ...]] = None
65
+ encoder_attentions: Optional[Tuple[torch.FloatTensor, ...]] = None
66
+ codebook_losses: Optional[List[torch.FloatTensor]]=None
67
+ vocal_codebook_losses: Optional[List[torch.FloatTensor]]=None
68
+
69
+
70
+ @dataclass
71
+ class CausalLMOutputWithCrossAttentions(ModelOutput):
72
+ """
73
+ Base class for causal language model (or autoregressive) outputs.
74
+
75
+ Args:
76
+ loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
77
+ Language modeling loss (for next-token prediction).
78
+ logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
79
+ Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
80
+ hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
81
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
82
+ one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.
83
+
84
+ Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
85
+ attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
86
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
87
+ sequence_length)`.
88
+
89
+ Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
90
+ heads.
91
+ cross_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
92
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
93
+ sequence_length)`.
94
+
95
+ Cross attentions weights after the attention softmax, used to compute the weighted average in the
96
+ cross-attention heads.
97
+ past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
98
+ Tuple of `torch.FloatTensor` tuples of length `config.n_layers`, with each tuple containing the cached key,
99
+ value states of the self-attention and the cross-attention layers if model is used in encoder-decoder
100
+ setting. Only relevant if `config.is_decoder = True`.
101
+
102
+ Contains pre-computed hidden-states (key and values in the attention blocks) that can be used (see
103
+ `past_key_values` input) to speed up sequential decoding.
104
+ """
105
+
106
+ loss: Optional[torch.FloatTensor] = None
107
+ vocal_loss: Optional[torch.FloatTensor] = None
108
+ logits: torch.FloatTensor = None
109
+ past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None
110
+ hidden_states: Optional[Tuple[torch.FloatTensor, ...]] = None
111
+ attentions: Optional[Tuple[torch.FloatTensor, ...]] = None
112
+ cross_attentions: Optional[Tuple[torch.FloatTensor, ...]] = None
113
+ codebook_losses: Optional[List[torch.FloatTensor]]=None
114
+ vocal_codebook_losses: Optional[List[torch.FloatTensor]]=None
songgen/logits_processors.py ADDED
@@ -0,0 +1,85 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from transformers import LogitsProcessor, LogitsProcessorList
2
+ import math
3
+ import torch
4
+ from typing import Union
5
+
6
+
7
+ def isin_mps_friendly(elements: torch.Tensor, test_elements: Union[torch.Tensor, int]) -> torch.Tensor:
8
+ """
9
+ Same as `torch.isin` without flags, but MPS-friendly. We can remove this function when we stop supporting
10
+ torch <= 2.3. See https://github.com/pytorch/pytorch/issues/77764#issuecomment-2067838075
11
+
12
+ Args:
13
+ elements (`torch.Tensor`): Input elements
14
+ test_elements (`torch.Tensor` or `int`): The elements to check against.
15
+
16
+ Returns:
17
+ `torch.Tensor`: A boolean tensor of the same shape as `elements` that is True for `elements` in `test_elements`
18
+ and False otherwise
19
+ """
20
+
21
+ if elements.device.type == "mps" and not is_torch_greater_or_equal_than_2_4:
22
+ test_elements = torch.tensor(test_elements)
23
+ if test_elements.ndim == 0:
24
+ test_elements = test_elements.unsqueeze(0)
25
+ return elements.tile(test_elements.shape[0], 1).eq(test_elements.unsqueeze(1)).sum(dim=0).bool().squeeze()
26
+ else:
27
+ # Note: don't use named arguments in `torch.isin`, see https://github.com/pytorch/pytorch/issues/126045
28
+ return torch.isin(elements, test_elements)
29
+
30
+ class SongGenLogitsProcessor(LogitsProcessor):
31
+ r"""This processor ensures that the delayed pattern mask constraints are respected.
32
+
33
+ <Tip warning={true}>
34
+
35
+ This logits processor is exclusively compatible with SongGen.
36
+ See the model documentation for examples.
37
+
38
+ </Tip>
39
+
40
+ Args:
41
+ eos_token_id (`Union[int, List[int], torch.Tensor]`):
42
+ The id(s) of the *end-of-sequence* token.
43
+ min_eos_p (`float`, *optional*):
44
+ Minimum end of speech threshold.
45
+ """
46
+
47
+ def __init__(self, eos_token_id, num_codebooks: int, batch_size: int, device: str = "cpu", track_pattern='mix'):
48
+ if not isinstance(eos_token_id, torch.Tensor):
49
+ if isinstance(eos_token_id, int):
50
+ eos_token_id = [eos_token_id]
51
+ eos_token_id = torch.tensor(eos_token_id, device=device)
52
+ self.eos_token_id = eos_token_id
53
+ self.batch_size = batch_size
54
+ self.track_pattern = track_pattern
55
+
56
+ if torch.is_floating_point(eos_token_id) or (eos_token_id < 0).any():
57
+ raise ValueError(f"`eos_token_id` has to be a list of positive integers, but is {eos_token_id}")
58
+
59
+ self.num_codebooks = num_codebooks
60
+ self.device = device
61
+
62
+
63
+ self.codebook_idx = torch.arange(self.batch_size*self.num_codebooks, device=self.device)
64
+
65
+ if self.track_pattern.startswith('parallel'):
66
+ self.first_codebooks_unfinished = torch.arange(batch_size*2, device=device)*(num_codebooks//2)
67
+ max_codebooks = torch.arange(self.batch_size*2, device=self.device)*(self.num_codebooks//2) + (self.num_codebooks//2) -1
68
+ self.track_num_codebooks = self.num_codebooks//2
69
+ else:
70
+ self.first_codebooks_unfinished = torch.arange(batch_size, device=device)*num_codebooks
71
+ max_codebooks = torch.arange(self.batch_size, device=self.device)*self.num_codebooks + self.num_codebooks -1
72
+ self.track_num_codebooks = self.num_codebooks
73
+
74
+ self.max_codebooks = max_codebooks
75
+
76
+ def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
77
+ is_eos = isin_mps_friendly(input_ids, self.eos_token_id).sum(1)
78
+
79
+ self.first_codebooks_unfinished = torch.where((is_eos[self.first_codebooks_unfinished]>0) & (self.first_codebooks_unfinished<self.max_codebooks), self.first_codebooks_unfinished+1, self.first_codebooks_unfinished)
80
+
81
+ # every codebook higher than the first one unfinished will never be eos
82
+ eos_token_mask = self.codebook_idx > self.first_codebooks_unfinished.repeat_interleave(self.track_num_codebooks)
83
+ scores[eos_token_mask, self.eos_token_id] = -math.inf
84
+
85
+ return scores
songgen/lyrics_utils/__init__.py ADDED
File without changes
songgen/lyrics_utils/lyrics_encoder.py ADDED
@@ -0,0 +1,1071 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Optional, Tuple, Union
2
+ import math
3
+ import torch
4
+ from torch import nn
5
+
6
+ class ConvolutionModule(nn.Module):
7
+ """ConvolutionModule in Conformer model."""
8
+
9
+ def __init__(self,
10
+ channels: int,
11
+ kernel_size: int = 15,
12
+ activation: nn.Module = nn.ReLU(),
13
+ norm: str = "batch_norm",
14
+ causal: bool = False,
15
+ bias: bool = True):
16
+ """Construct an ConvolutionModule object.
17
+ Args:
18
+ channels (int): The number of channels of conv layers.
19
+ kernel_size (int): Kernel size of conv layers.
20
+ causal (int): Whether use causal convolution or not
21
+ """
22
+ super().__init__()
23
+
24
+ self.pointwise_conv1 = nn.Conv1d(
25
+ channels,
26
+ 2 * channels,
27
+ kernel_size=1,
28
+ stride=1,
29
+ padding=0,
30
+ bias=bias,
31
+ )
32
+ # self.lorder is used to distinguish if it's a causal convolution,
33
+ # if self.lorder > 0: it's a causal convolution, the input will be
34
+ # padded with self.lorder frames on the left in forward.
35
+ # else: it's a symmetrical convolution
36
+ if causal:
37
+ padding = 0
38
+ self.lorder = kernel_size - 1
39
+ else:
40
+ # kernel_size should be an odd number for none causal convolution
41
+ assert (kernel_size - 1) % 2 == 0
42
+ padding = (kernel_size - 1) // 2
43
+ self.lorder = 0
44
+ self.depthwise_conv = nn.Conv1d(
45
+ channels,
46
+ channels,
47
+ kernel_size,
48
+ stride=1,
49
+ padding=padding,
50
+ groups=channels,
51
+ bias=bias,
52
+ )
53
+
54
+ assert norm in ['batch_norm', 'layer_norm']
55
+ if norm == "batch_norm":
56
+ self.use_layer_norm = False
57
+ self.norm = nn.BatchNorm1d(channels)
58
+ else:
59
+ self.use_layer_norm = True
60
+ self.norm = nn.LayerNorm(channels)
61
+
62
+ self.pointwise_conv2 = nn.Conv1d(
63
+ channels,
64
+ channels,
65
+ kernel_size=1,
66
+ stride=1,
67
+ padding=0,
68
+ bias=bias,
69
+ )
70
+ self.activation = activation
71
+
72
+ def forward(
73
+ self,
74
+ x: torch.Tensor,
75
+ mask_pad: torch.Tensor = torch.ones((0, 0, 0), dtype=torch.bool),
76
+ cache: torch.Tensor = torch.zeros((0, 0, 0)),
77
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
78
+ """Compute convolution module.
79
+ Args:
80
+ x (torch.Tensor): Input tensor (#batch, time, channels).
81
+ mask_pad (torch.Tensor): used for batch padding (#batch, 1, time),
82
+ (0, 0, 0) means fake mask.
83
+ cache (torch.Tensor): left context cache, it is only
84
+ used in causal convolution (#batch, channels, cache_t),
85
+ (0, 0, 0) meas fake cache.
86
+ Returns:
87
+ torch.Tensor: Output tensor (#batch, time, channels).
88
+ """
89
+ # exchange the temporal dimension and the feature dimension
90
+ x = x.transpose(1, 2) # (#batch, channels, time)
91
+
92
+ # mask batch padding
93
+ if mask_pad.size(2) > 0: # time > 0
94
+ x.masked_fill_(~mask_pad, 0.0)
95
+
96
+ if self.lorder > 0:
97
+ if cache.size(2) == 0: # cache_t == 0
98
+ x = nn.functional.pad(x, (self.lorder, 0), 'constant', 0.0)
99
+ else:
100
+ assert cache.size(0) == x.size(0) # equal batch
101
+ assert cache.size(1) == x.size(1) # equal channel
102
+ x = torch.cat((cache, x), dim=2)
103
+ assert (x.size(2) > self.lorder)
104
+ new_cache = x[:, :, -self.lorder:]
105
+ else:
106
+ # It's better we just return None if no cache is required,
107
+ # However, for JIT export, here we just fake one tensor instead of
108
+ # None.
109
+ new_cache = torch.zeros((0, 0, 0), dtype=x.dtype, device=x.device)
110
+
111
+ # GLU mechanism
112
+ x = self.pointwise_conv1(x) # (batch, 2*channel, dim)
113
+ x = nn.functional.glu(x, dim=1) # (batch, channel, dim)
114
+
115
+ # 1D Depthwise Conv
116
+ x = self.depthwise_conv(x)
117
+ if self.use_layer_norm:
118
+ x = x.transpose(1, 2)
119
+ x = self.activation(self.norm(x))
120
+ if self.use_layer_norm:
121
+ x = x.transpose(1, 2)
122
+ x = self.pointwise_conv2(x)
123
+ # mask batch padding
124
+ if mask_pad.size(2) > 0: # time > 0
125
+ x.masked_fill_(~mask_pad, 0.0)
126
+
127
+ return x.transpose(1, 2), new_cache
128
+
129
+ class PositionwiseFeedForward(torch.nn.Module):
130
+ """Positionwise feed forward layer.
131
+
132
+ FeedForward are appied on each position of the sequence.
133
+ The output dim is same with the input dim.
134
+
135
+ Args:
136
+ idim (int): Input dimenstion.
137
+ hidden_units (int): The number of hidden units.
138
+ dropout_rate (float): Dropout rate.
139
+ activation (torch.nn.Module): Activation function
140
+ """
141
+
142
+ def __init__(
143
+ self,
144
+ idim: int,
145
+ hidden_units: int,
146
+ dropout_rate: float,
147
+ activation: torch.nn.Module = torch.nn.ReLU(),
148
+ ):
149
+ """Construct a PositionwiseFeedForward object."""
150
+ super(PositionwiseFeedForward, self).__init__()
151
+ self.w_1 = torch.nn.Linear(idim, hidden_units)
152
+ self.activation = activation
153
+ self.dropout = torch.nn.Dropout(dropout_rate)
154
+ self.w_2 = torch.nn.Linear(hidden_units, idim)
155
+
156
+ def forward(self, xs: torch.Tensor) -> torch.Tensor:
157
+ """Forward function.
158
+
159
+ Args:
160
+ xs: input tensor (B, L, D)
161
+ Returns:
162
+ output tensor, (B, L, D)
163
+ """
164
+ return self.w_2(self.dropout(self.activation(self.w_1(xs))))
165
+
166
+ class Swish(torch.nn.Module):
167
+ """Construct an Swish object."""
168
+
169
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
170
+ """Return Swish activation function."""
171
+ return x * torch.sigmoid(x)
172
+
173
+ class MultiHeadedAttention(nn.Module):
174
+ """Multi-Head Attention layer.
175
+
176
+ Args:
177
+ n_head (int): The number of heads.
178
+ n_feat (int): The number of features.
179
+ dropout_rate (float): Dropout rate.
180
+
181
+ """
182
+
183
+ def __init__(self,
184
+ n_head: int,
185
+ n_feat: int,
186
+ dropout_rate: float,
187
+ key_bias: bool = True):
188
+ """Construct an MultiHeadedAttention object."""
189
+ super().__init__()
190
+ assert n_feat % n_head == 0
191
+ # We assume d_v always equals d_k
192
+ self.d_k = n_feat // n_head
193
+ self.h = n_head
194
+ self.linear_q = nn.Linear(n_feat, n_feat)
195
+ self.linear_k = nn.Linear(n_feat, n_feat, bias=key_bias)
196
+ self.linear_v = nn.Linear(n_feat, n_feat)
197
+ self.linear_out = nn.Linear(n_feat, n_feat)
198
+ self.dropout = nn.Dropout(p=dropout_rate)
199
+
200
+ def forward_qkv(
201
+ self, query: torch.Tensor, key: torch.Tensor, value: torch.Tensor
202
+ ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
203
+ """Transform query, key and value.
204
+
205
+ Args:
206
+ query (torch.Tensor): Query tensor (#batch, time1, size).
207
+ key (torch.Tensor): Key tensor (#batch, time2, size).
208
+ value (torch.Tensor): Value tensor (#batch, time2, size).
209
+
210
+ Returns:
211
+ torch.Tensor: Transformed query tensor, size
212
+ (#batch, n_head, time1, d_k).
213
+ torch.Tensor: Transformed key tensor, size
214
+ (#batch, n_head, time2, d_k).
215
+ torch.Tensor: Transformed value tensor, size
216
+ (#batch, n_head, time2, d_k).
217
+
218
+ """
219
+ n_batch = query.size(0)
220
+ q = self.linear_q(query).view(n_batch, -1, self.h, self.d_k)
221
+ k = self.linear_k(key).view(n_batch, -1, self.h, self.d_k)
222
+ v = self.linear_v(value).view(n_batch, -1, self.h, self.d_k)
223
+ q = q.transpose(1, 2) # (batch, head, time1, d_k)
224
+ k = k.transpose(1, 2) # (batch, head, time2, d_k)
225
+ v = v.transpose(1, 2) # (batch, head, time2, d_k)
226
+ return q, k, v
227
+
228
+ def forward_attention(
229
+ self,
230
+ value: torch.Tensor,
231
+ scores: torch.Tensor,
232
+ mask: torch.Tensor = torch.ones((0, 0, 0), dtype=torch.bool)
233
+ ) -> torch.Tensor:
234
+ """Compute attention context vector.
235
+
236
+ Args:
237
+ value (torch.Tensor): Transformed value, size
238
+ (#batch, n_head, time2, d_k).
239
+ scores (torch.Tensor): Attention score, size
240
+ (#batch, n_head, time1, time2).
241
+ mask (torch.Tensor): Mask, size (#batch, 1, time2) or
242
+ (#batch, time1, time2), (0, 0, 0) means fake mask.
243
+
244
+ Returns:
245
+ torch.Tensor: Transformed value (#batch, time1, d_model)
246
+ weighted by the attention score (#batch, time1, time2).
247
+
248
+ """
249
+ n_batch = value.size(0)
250
+
251
+ if mask.size(2) > 0: # time2 > 0
252
+ mask = mask.unsqueeze(1).eq(0) # (batch, 1, *, time2)
253
+ # For last chunk, time2 might be larger than scores.size(-1)
254
+ mask = mask[:, :, :, :scores.size(-1)] # (batch, 1, *, time2)
255
+ scores = scores.masked_fill(mask, -float('inf'))
256
+ attn = torch.softmax(scores, dim=-1).masked_fill(
257
+ mask, 0.0) # (batch, head, time1, time2)
258
+
259
+ else:
260
+ attn = torch.softmax(scores, dim=-1) # (batch, head, time1, time2)
261
+
262
+ p_attn = self.dropout(attn)
263
+ x = torch.matmul(p_attn, value) # (batch, head, time1, d_k)
264
+ x = (x.transpose(1, 2).contiguous().view(n_batch, -1,
265
+ self.h * self.d_k)
266
+ ) # (batch, time1, d_model)
267
+
268
+ return self.linear_out(x) # (batch, time1, d_model)
269
+
270
+ def forward(
271
+ self,
272
+ query: torch.Tensor,
273
+ key: torch.Tensor,
274
+ value: torch.Tensor,
275
+ mask: torch.Tensor = torch.ones((0, 0, 0), dtype=torch.bool),
276
+ pos_emb: torch.Tensor = torch.empty(0),
277
+ cache: torch.Tensor = torch.zeros((0, 0, 0, 0))
278
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
279
+ """Compute scaled dot product attention.
280
+
281
+ Args:
282
+ query (torch.Tensor): Query tensor (#batch, time1, size).
283
+ key (torch.Tensor): Key tensor (#batch, time2, size).
284
+ value (torch.Tensor): Value tensor (#batch, time2, size).
285
+ mask (torch.Tensor): Mask tensor (#batch, 1, time2) or
286
+ (#batch, time1, time2).
287
+ 1.When applying cross attention between decoder and encoder,
288
+ the batch padding mask for input is in (#batch, 1, T) shape.
289
+ 2.When applying self attention of encoder,
290
+ the mask is in (#batch, T, T) shape.
291
+ 3.When applying self attention of decoder,
292
+ the mask is in (#batch, L, L) shape.
293
+ 4.If the different position in decoder see different block
294
+ of the encoder, such as Mocha, the passed in mask could be
295
+ in (#batch, L, T) shape. But there is no such case in current
296
+ CosyVoice.
297
+ cache (torch.Tensor): Cache tensor (1, head, cache_t, d_k * 2),
298
+ where `cache_t == chunk_size * num_decoding_left_chunks`
299
+ and `head * d_k == size`
300
+
301
+
302
+ Returns:
303
+ torch.Tensor: Output tensor (#batch, time1, d_model).
304
+ torch.Tensor: Cache tensor (1, head, cache_t + time1, d_k * 2)
305
+ where `cache_t == chunk_size * num_decoding_left_chunks`
306
+ and `head * d_k == size`
307
+
308
+ """
309
+ q, k, v = self.forward_qkv(query, key, value)
310
+ if cache.size(0) > 0:
311
+ key_cache, value_cache = torch.split(cache,
312
+ cache.size(-1) // 2,
313
+ dim=-1)
314
+ k = torch.cat([key_cache, k], dim=2)
315
+ v = torch.cat([value_cache, v], dim=2)
316
+ new_cache = torch.cat((k, v), dim=-1)
317
+
318
+ scores = torch.matmul(q, k.transpose(-2, -1)) / math.sqrt(self.d_k)
319
+ return self.forward_attention(v, scores, mask), new_cache
320
+
321
+
322
+ class RelPositionMultiHeadedAttention(MultiHeadedAttention):
323
+ """Multi-Head Attention layer with relative position encoding.
324
+ Paper: https://arxiv.org/abs/1901.02860
325
+ Args:
326
+ n_head (int): The number of heads.
327
+ n_feat (int): The number of features.
328
+ dropout_rate (float): Dropout rate.
329
+ """
330
+
331
+ def __init__(self,
332
+ n_head: int,
333
+ n_feat: int,
334
+ dropout_rate: float,
335
+ key_bias: bool = True):
336
+ """Construct an RelPositionMultiHeadedAttention object."""
337
+ super().__init__(n_head, n_feat, dropout_rate, key_bias)
338
+ # linear transformation for positional encoding
339
+ self.linear_pos = nn.Linear(n_feat, n_feat, bias=False)
340
+ # these two learnable bias are used in matrix c and matrix d
341
+ # as described in https://arxiv.org/abs/1901.02860 Section 3.3
342
+ self.pos_bias_u = nn.Parameter(torch.Tensor(self.h, self.d_k))
343
+ self.pos_bias_v = nn.Parameter(torch.Tensor(self.h, self.d_k))
344
+ torch.nn.init.xavier_uniform_(self.pos_bias_u)
345
+ torch.nn.init.xavier_uniform_(self.pos_bias_v)
346
+
347
+ def rel_shift(self, x: torch.Tensor) -> torch.Tensor:
348
+ """Compute relative positional encoding.
349
+
350
+ Args:
351
+ x (torch.Tensor): Input tensor (batch, head, time1, 2*time1-1).
352
+ time1 means the length of query vector.
353
+
354
+ Returns:
355
+ torch.Tensor: Output tensor.
356
+
357
+ """
358
+ zero_pad = torch.zeros((x.size()[0], x.size()[1], x.size()[2], 1),
359
+ device=x.device,
360
+ dtype=x.dtype)
361
+ x_padded = torch.cat([zero_pad, x], dim=-1)
362
+
363
+ x_padded = x_padded.view(x.size()[0],
364
+ x.size()[1],
365
+ x.size(3) + 1, x.size(2))
366
+ x = x_padded[:, :, 1:].view_as(x)[
367
+ :, :, :, : x.size(-1) // 2 + 1
368
+ ] # only keep the positions from 0 to time2
369
+ return x
370
+
371
+ def forward(
372
+ self,
373
+ query: torch.Tensor,
374
+ key: torch.Tensor,
375
+ value: torch.Tensor,
376
+ mask: torch.Tensor = torch.ones((0, 0, 0), dtype=torch.bool),
377
+ pos_emb: torch.Tensor = torch.empty(0),
378
+ cache: torch.Tensor = torch.zeros((0, 0, 0, 0))
379
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
380
+ """Compute 'Scaled Dot Product Attention' with rel. positional encoding.
381
+ Args:
382
+ query (torch.Tensor): Query tensor (#batch, time1, size).
383
+ key (torch.Tensor): Key tensor (#batch, time2, size).
384
+ value (torch.Tensor): Value tensor (#batch, time2, size).
385
+ mask (torch.Tensor): Mask tensor (#batch, 1, time2) or
386
+ (#batch, time1, time2), (0, 0, 0) means fake mask.
387
+ pos_emb (torch.Tensor): Positional embedding tensor
388
+ (#batch, time2, size).
389
+ cache (torch.Tensor): Cache tensor (1, head, cache_t, d_k * 2),
390
+ where `cache_t == chunk_size * num_decoding_left_chunks`
391
+ and `head * d_k == size`
392
+ Returns:
393
+ torch.Tensor: Output tensor (#batch, time1, d_model).
394
+ torch.Tensor: Cache tensor (1, head, cache_t + time1, d_k * 2)
395
+ where `cache_t == chunk_size * num_decoding_left_chunks`
396
+ and `head * d_k == size`
397
+ """
398
+ q, k, v = self.forward_qkv(query, key, value)
399
+ q = q.transpose(1, 2) # (batch, time1, head, d_k)
400
+
401
+ if cache.size(0) > 0:
402
+ key_cache, value_cache = torch.split(cache,
403
+ cache.size(-1) // 2,
404
+ dim=-1)
405
+ k = torch.cat([key_cache, k], dim=2)
406
+ v = torch.cat([value_cache, v], dim=2)
407
+ # NOTE(xcsong): We do cache slicing in encoder.forward_chunk, since it's
408
+ # non-trivial to calculate `next_cache_start` here.
409
+ new_cache = torch.cat((k, v), dim=-1)
410
+
411
+ n_batch_pos = pos_emb.size(0)
412
+ p = self.linear_pos(pos_emb).view(n_batch_pos, -1, self.h, self.d_k)
413
+ p = p.transpose(1, 2) # (batch, head, time1, d_k)
414
+
415
+ # (batch, head, time1, d_k)
416
+ q_with_bias_u = (q + self.pos_bias_u).transpose(1, 2)
417
+ # (batch, head, time1, d_k)
418
+ q_with_bias_v = (q + self.pos_bias_v).transpose(1, 2)
419
+
420
+ # compute attention score
421
+ # first compute matrix a and matrix c
422
+ # as described in https://arxiv.org/abs/1901.02860 Section 3.3
423
+ # (batch, head, time1, time2)
424
+ matrix_ac = torch.matmul(q_with_bias_u, k.transpose(-2, -1))
425
+
426
+ # compute matrix b and matrix d
427
+ # (batch, head, time1, time2)
428
+ matrix_bd = torch.matmul(q_with_bias_v, p.transpose(-2, -1))
429
+ # NOTE(Xiang Lyu): Keep rel_shift since espnet rel_pos_emb is used
430
+ if matrix_ac.shape != matrix_bd.shape:
431
+ matrix_bd = self.rel_shift(matrix_bd)
432
+
433
+ scores = (matrix_ac + matrix_bd) / math.sqrt(
434
+ self.d_k) # (batch, head, time1, time2)
435
+
436
+ return self.forward_attention(v, scores, mask), new_cache
437
+
438
+
439
+
440
+ def subsequent_mask(
441
+ size: int,
442
+ device: torch.device = torch.device("cpu"),
443
+ ) -> torch.Tensor:
444
+ """Create mask for subsequent steps (size, size).
445
+
446
+ This mask is used only in decoder which works in an auto-regressive mode.
447
+ This means the current step could only do attention with its left steps.
448
+
449
+ In encoder, fully attention is used when streaming is not necessary and
450
+ the sequence is not long. In this case, no attention mask is needed.
451
+
452
+ When streaming is need, chunk-based attention is used in encoder. See
453
+ subsequent_chunk_mask for the chunk-based attention mask.
454
+
455
+ Args:
456
+ size (int): size of mask
457
+ str device (str): "cpu" or "cuda" or torch.Tensor.device
458
+ dtype (torch.device): result dtype
459
+
460
+ Returns:
461
+ torch.Tensor: mask
462
+
463
+ Examples:
464
+ >>> subsequent_mask(3)
465
+ [[1, 0, 0],
466
+ [1, 1, 0],
467
+ [1, 1, 1]]
468
+ """
469
+ arange = torch.arange(size, device=device)
470
+ mask = arange.expand(size, size)
471
+ arange = arange.unsqueeze(-1)
472
+ mask = mask <= arange
473
+ return mask
474
+
475
+
476
+ def subsequent_chunk_mask(
477
+ size: int,
478
+ chunk_size: int,
479
+ num_left_chunks: int = -1,
480
+ device: torch.device = torch.device("cpu"),
481
+ ) -> torch.Tensor:
482
+ """Create mask for subsequent steps (size, size) with chunk size,
483
+ this is for streaming encoder
484
+
485
+ Args:
486
+ size (int): size of mask
487
+ chunk_size (int): size of chunk
488
+ num_left_chunks (int): number of left chunks
489
+ <0: use full chunk
490
+ >=0: use num_left_chunks
491
+ device (torch.device): "cpu" or "cuda" or torch.Tensor.device
492
+
493
+ Returns:
494
+ torch.Tensor: mask
495
+
496
+ Examples:
497
+ >>> subsequent_chunk_mask(4, 2)
498
+ [[1, 1, 0, 0],
499
+ [1, 1, 0, 0],
500
+ [1, 1, 1, 1],
501
+ [1, 1, 1, 1]]
502
+ """
503
+ ret = torch.zeros(size, size, device=device, dtype=torch.bool)
504
+ for i in range(size):
505
+ if num_left_chunks < 0:
506
+ start = 0
507
+ else:
508
+ start = max((i // chunk_size - num_left_chunks) * chunk_size, 0)
509
+ ending = min((i // chunk_size + 1) * chunk_size, size)
510
+ ret[i, start:ending] = True
511
+ return ret
512
+
513
+ def add_optional_chunk_mask(xs: torch.Tensor,
514
+ masks: torch.Tensor,
515
+ use_dynamic_chunk: bool,
516
+ use_dynamic_left_chunk: bool,
517
+ decoding_chunk_size: int,
518
+ static_chunk_size: int,
519
+ num_decoding_left_chunks: int,
520
+ enable_full_context: bool = True):
521
+ """ Apply optional mask for encoder.
522
+
523
+ Args:
524
+ xs (torch.Tensor): padded input, (B, L, D), L for max length
525
+ mask (torch.Tensor): mask for xs, (B, 1, L)
526
+ use_dynamic_chunk (bool): whether to use dynamic chunk or not
527
+ use_dynamic_left_chunk (bool): whether to use dynamic left chunk for
528
+ training.
529
+ decoding_chunk_size (int): decoding chunk size for dynamic chunk, it's
530
+ 0: default for training, use random dynamic chunk.
531
+ <0: for decoding, use full chunk.
532
+ >0: for decoding, use fixed chunk size as set.
533
+ static_chunk_size (int): chunk size for static chunk training/decoding
534
+ if it's greater than 0, if use_dynamic_chunk is true,
535
+ this parameter will be ignored
536
+ num_decoding_left_chunks: number of left chunks, this is for decoding,
537
+ the chunk size is decoding_chunk_size.
538
+ >=0: use num_decoding_left_chunks
539
+ <0: use all left chunks
540
+ enable_full_context (bool):
541
+ True: chunk size is either [1, 25] or full context(max_len)
542
+ False: chunk size ~ U[1, 25]
543
+
544
+ Returns:
545
+ torch.Tensor: chunk mask of the input xs.
546
+ """
547
+ # Whether to use chunk mask or not
548
+ if use_dynamic_chunk:
549
+ max_len = xs.size(1)
550
+ if decoding_chunk_size < 0:
551
+ chunk_size = max_len
552
+ num_left_chunks = -1
553
+ elif decoding_chunk_size > 0:
554
+ chunk_size = decoding_chunk_size
555
+ num_left_chunks = num_decoding_left_chunks
556
+ else:
557
+ # chunk size is either [1, 25] or full context(max_len).
558
+ # Since we use 4 times subsampling and allow up to 1s(100 frames)
559
+ # delay, the maximum frame is 100 / 4 = 25.
560
+ chunk_size = torch.randint(1, max_len, (1, )).item()
561
+ num_left_chunks = -1
562
+ if chunk_size > max_len // 2 and enable_full_context:
563
+ chunk_size = max_len
564
+ else:
565
+ chunk_size = chunk_size % 25 + 1
566
+ if use_dynamic_left_chunk:
567
+ max_left_chunks = (max_len - 1) // chunk_size
568
+ num_left_chunks = torch.randint(0, max_left_chunks,
569
+ (1, )).item()
570
+ chunk_masks = subsequent_chunk_mask(xs.size(1), chunk_size,
571
+ num_left_chunks,
572
+ xs.device) # (L, L)
573
+ chunk_masks = chunk_masks.unsqueeze(0) # (1, L, L)
574
+ chunk_masks = masks & chunk_masks # (B, L, L)
575
+ elif static_chunk_size > 0:
576
+ num_left_chunks = num_decoding_left_chunks
577
+ chunk_masks = subsequent_chunk_mask(xs.size(1), static_chunk_size,
578
+ num_left_chunks,
579
+ xs.device) # (L, L)
580
+ chunk_masks = chunk_masks.unsqueeze(0) # (1, L, L)
581
+ chunk_masks = masks & chunk_masks # (B, L, L)
582
+ else:
583
+ chunk_masks = masks
584
+ return chunk_masks
585
+
586
+
587
+ class ConformerEncoderLayer(nn.Module):
588
+ """Encoder layer module.
589
+ Args:
590
+ size (int): Input dimension.
591
+ self_attn (torch.nn.Module): Self-attention module instance.
592
+ `MultiHeadedAttention` or `RelPositionMultiHeadedAttention`
593
+ instance can be used as the argument.
594
+ feed_forward (torch.nn.Module): Feed-forward module instance.
595
+ `PositionwiseFeedForward` instance can be used as the argument.
596
+ feed_forward_macaron (torch.nn.Module): Additional feed-forward module
597
+ instance.
598
+ `PositionwiseFeedForward` instance can be used as the argument.
599
+ conv_module (torch.nn.Module): Convolution module instance.
600
+ `ConvlutionModule` instance can be used as the argument.
601
+ dropout_rate (float): Dropout rate.
602
+ normalize_before (bool):
603
+ True: use layer_norm before each sub-block.
604
+ False: use layer_norm after each sub-block.
605
+ """
606
+
607
+ def __init__(
608
+ self,
609
+ size: int,
610
+ self_attn: torch.nn.Module,
611
+ feed_forward: Optional[nn.Module] = None,
612
+ feed_forward_macaron: Optional[nn.Module] = None,
613
+ conv_module: Optional[nn.Module] = None,
614
+ dropout_rate: float = 0.1,
615
+ normalize_before: bool = True,
616
+ ):
617
+ """Construct an EncoderLayer object."""
618
+ super().__init__()
619
+ self.self_attn = self_attn
620
+ self.feed_forward = feed_forward
621
+ self.feed_forward_macaron = feed_forward_macaron
622
+ self.conv_module = conv_module
623
+ self.norm_ff = nn.LayerNorm(size, eps=1e-5) # for the FNN module
624
+ self.norm_mha = nn.LayerNorm(size, eps=1e-5) # for the MHA module
625
+ if feed_forward_macaron is not None:
626
+ self.norm_ff_macaron = nn.LayerNorm(size, eps=1e-5)
627
+ self.ff_scale = 0.5
628
+ else:
629
+ self.ff_scale = 1.0
630
+ if self.conv_module is not None:
631
+ self.norm_conv = nn.LayerNorm(size, eps=1e-5) # for the CNN module
632
+ self.norm_final = nn.LayerNorm(
633
+ size, eps=1e-5) # for the final output of the block
634
+ self.dropout = nn.Dropout(dropout_rate)
635
+ self.size = size
636
+ self.normalize_before = normalize_before
637
+
638
+ def forward(
639
+ self,
640
+ x: torch.Tensor,
641
+ mask: torch.Tensor,
642
+ pos_emb: torch.Tensor,
643
+ mask_pad: torch.Tensor = torch.ones((0, 0, 0), dtype=torch.bool),
644
+ att_cache: torch.Tensor = torch.zeros((0, 0, 0, 0)),
645
+ cnn_cache: torch.Tensor = torch.zeros((0, 0, 0, 0)),
646
+ ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
647
+ """Compute encoded features.
648
+
649
+ Args:
650
+ x (torch.Tensor): (#batch, time, size)
651
+ mask (torch.Tensor): Mask tensor for the input (#batch, time,time),
652
+ (0, 0, 0) means fake mask.
653
+ pos_emb (torch.Tensor): positional encoding, must not be None
654
+ for ConformerEncoderLayer.
655
+ mask_pad (torch.Tensor): batch padding mask used for conv module.
656
+ (#batch, 1,time), (0, 0, 0) means fake mask.
657
+ att_cache (torch.Tensor): Cache tensor of the KEY & VALUE
658
+ (#batch=1, head, cache_t1, d_k * 2), head * d_k == size.
659
+ cnn_cache (torch.Tensor): Convolution cache in conformer layer
660
+ (#batch=1, size, cache_t2)
661
+ Returns:
662
+ torch.Tensor: Output tensor (#batch, time, size).
663
+ torch.Tensor: Mask tensor (#batch, time, time).
664
+ torch.Tensor: att_cache tensor,
665
+ (#batch=1, head, cache_t1 + time, d_k * 2).
666
+ torch.Tensor: cnn_cahce tensor (#batch, size, cache_t2).
667
+ """
668
+
669
+ # whether to use macaron style
670
+ if self.feed_forward_macaron is not None:
671
+ residual = x
672
+ if self.normalize_before:
673
+ x = self.norm_ff_macaron(x)
674
+ x = residual + self.ff_scale * self.dropout(
675
+ self.feed_forward_macaron(x))
676
+ if not self.normalize_before:
677
+ x = self.norm_ff_macaron(x)
678
+
679
+ # multi-headed self-attention module
680
+ residual = x
681
+ if self.normalize_before:
682
+ x = self.norm_mha(x)
683
+ x_att, new_att_cache = self.self_attn(x, x, x, mask, pos_emb,
684
+ att_cache)
685
+ x = residual + self.dropout(x_att)
686
+ if not self.normalize_before:
687
+ x = self.norm_mha(x)
688
+
689
+ # convolution module
690
+ # Fake new cnn cache here, and then change it in conv_module
691
+ new_cnn_cache = torch.zeros((0, 0, 0), dtype=x.dtype, device=x.device)
692
+ if self.conv_module is not None:
693
+ residual = x
694
+ if self.normalize_before:
695
+ x = self.norm_conv(x)
696
+ x, new_cnn_cache = self.conv_module(x, mask_pad, cnn_cache)
697
+ x = residual + self.dropout(x)
698
+
699
+ if not self.normalize_before:
700
+ x = self.norm_conv(x)
701
+
702
+ # feed forward module
703
+ residual = x
704
+ if self.normalize_before:
705
+ x = self.norm_ff(x)
706
+
707
+ x = residual + self.ff_scale * self.dropout(self.feed_forward(x))
708
+ if not self.normalize_before:
709
+ x = self.norm_ff(x)
710
+
711
+ if self.conv_module is not None:
712
+ x = self.norm_final(x)
713
+
714
+ return x, mask, new_att_cache, new_cnn_cache
715
+
716
+
717
+
718
+ class EspnetRelPositionalEncoding(torch.nn.Module):
719
+ """Relative positional encoding module (new implementation).
720
+
721
+ Details can be found in https://github.com/espnet/espnet/pull/2816.
722
+
723
+ See : Appendix B in https://arxiv.org/abs/1901.02860
724
+
725
+ Args:
726
+ d_model (int): Embedding dimension.
727
+ dropout_rate (float): Dropout rate.
728
+ max_len (int): Maximum input length.
729
+
730
+ """
731
+
732
+ def __init__(self, d_model: int, dropout_rate: float, max_len: int = 5000):
733
+ """Construct an PositionalEncoding object."""
734
+ super(EspnetRelPositionalEncoding, self).__init__()
735
+ self.d_model = d_model
736
+ self.xscale = math.sqrt(self.d_model)
737
+ self.dropout = torch.nn.Dropout(p=dropout_rate)
738
+ self.pe = None
739
+ self.extend_pe(torch.tensor(0.0).expand(1, max_len))
740
+
741
+ def extend_pe(self, x: torch.Tensor):
742
+ """Reset the positional encodings."""
743
+ if self.pe is not None:
744
+ # self.pe contains both positive and negative parts
745
+ # the length of self.pe is 2 * input_len - 1
746
+ if self.pe.size(1) >= x.size(1) * 2 - 1:
747
+ if self.pe.dtype != x.dtype or self.pe.device != x.device:
748
+ self.pe = self.pe.to(dtype=x.dtype, device=x.device)
749
+ return
750
+ # Suppose `i` means to the position of query vecotr and `j` means the
751
+ # position of key vector. We use position relative positions when keys
752
+ # are to the left (i>j) and negative relative positions otherwise (i<j).
753
+ pe_positive = torch.zeros(x.size(1), self.d_model)
754
+ pe_negative = torch.zeros(x.size(1), self.d_model)
755
+ position = torch.arange(0, x.size(1), dtype=torch.float32).unsqueeze(1)
756
+ div_term = torch.exp(
757
+ torch.arange(0, self.d_model, 2, dtype=torch.float32)
758
+ * -(math.log(10000.0) / self.d_model)
759
+ )
760
+ pe_positive[:, 0::2] = torch.sin(position * div_term)
761
+ pe_positive[:, 1::2] = torch.cos(position * div_term)
762
+ pe_negative[:, 0::2] = torch.sin(-1 * position * div_term)
763
+ pe_negative[:, 1::2] = torch.cos(-1 * position * div_term)
764
+
765
+ # Reserve the order of positive indices and concat both positive and
766
+ # negative indices. This is used to support the shifting trick
767
+ # as in https://arxiv.org/abs/1901.02860
768
+ pe_positive = torch.flip(pe_positive, [0]).unsqueeze(0)
769
+ pe_negative = pe_negative[1:].unsqueeze(0)
770
+ pe = torch.cat([pe_positive, pe_negative], dim=1)
771
+ self.pe = pe.to(device=x.device, dtype=x.dtype)
772
+
773
+ def forward(self, x: torch.Tensor, offset: Union[int, torch.Tensor] = 0) \
774
+ -> Tuple[torch.Tensor, torch.Tensor]:
775
+ """Add positional encoding.
776
+
777
+ Args:
778
+ x (torch.Tensor): Input tensor (batch, time, `*`).
779
+
780
+ Returns:
781
+ torch.Tensor: Encoded tensor (batch, time, `*`).
782
+
783
+ """
784
+ self.extend_pe(x)
785
+ x = x * self.xscale
786
+ pos_emb = self.position_encoding(size=x.size(1), offset=offset)
787
+ return self.dropout(x), self.dropout(pos_emb)
788
+
789
+ def position_encoding(self,
790
+ offset: Union[int, torch.Tensor],
791
+ size: int) -> torch.Tensor:
792
+ """ For getting encoding in a streaming fashion
793
+
794
+ Attention!!!!!
795
+ we apply dropout only once at the whole utterance level in a none
796
+ streaming way, but will call this function several times with
797
+ increasing input size in a streaming scenario, so the dropout will
798
+ be applied several times.
799
+
800
+ Args:
801
+ offset (int or torch.tensor): start offset
802
+ size (int): required size of position encoding
803
+
804
+ Returns:
805
+ torch.Tensor: Corresponding encoding
806
+ """
807
+ pos_emb = self.pe[
808
+ :,
809
+ self.pe.size(1) // 2 - size + 1: self.pe.size(1) // 2 + size,
810
+ ]
811
+ return pos_emb
812
+
813
+
814
+
815
+ class LinearEmbed(torch.nn.Module):
816
+ """Linear transform the input without subsampling
817
+
818
+ Args:
819
+ idim (int): Input dimension.
820
+ odim (int): Output dimension.
821
+ dropout_rate (float): Dropout rate.
822
+
823
+ """
824
+
825
+ def __init__(self, idim: int, odim: int, dropout_rate: float,
826
+ pos_enc_class: torch.nn.Module):
827
+ """Construct an linear object."""
828
+ super().__init__()
829
+ self.out = torch.nn.Sequential(
830
+ torch.nn.Linear(idim, odim),
831
+ torch.nn.LayerNorm(odim, eps=1e-5),
832
+ torch.nn.Dropout(dropout_rate),
833
+ )
834
+ self.pos_enc = pos_enc_class #rel_pos_espnet
835
+
836
+ def position_encoding(self, offset: Union[int, torch.Tensor],
837
+ size: int) -> torch.Tensor:
838
+ return self.pos_enc.position_encoding(offset, size)
839
+
840
+ def forward(
841
+ self,
842
+ x: torch.Tensor,
843
+ offset: Union[int, torch.Tensor] = 0
844
+ ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
845
+ """Input x.
846
+
847
+ Args:
848
+ x (torch.Tensor): Input tensor (#batch, time, idim).
849
+ x_mask (torch.Tensor): Input mask (#batch, 1, time).
850
+
851
+ Returns:
852
+ torch.Tensor: linear input tensor (#batch, time', odim),
853
+ where time' = time .
854
+ torch.Tensor: linear input mask (#batch, 1, time'),
855
+ where time' = time .
856
+
857
+ """
858
+ x = self.out(x)
859
+ x, pos_emb = self.pos_enc(x, offset)
860
+ return x, pos_emb
861
+
862
+
863
+ ATTENTION_CLASSES = {
864
+ "selfattn": MultiHeadedAttention,
865
+ "rel_selfattn": RelPositionMultiHeadedAttention,
866
+ }
867
+
868
+ ACTIVATION_CLASSES = {
869
+ "hardtanh": torch.nn.Hardtanh,
870
+ "tanh": torch.nn.Tanh,
871
+ "relu": torch.nn.ReLU,
872
+ "selu": torch.nn.SELU,
873
+ "swish": getattr(torch.nn, "SiLU", Swish),
874
+ "gelu": torch.nn.GELU,
875
+ }
876
+
877
+
878
+ def make_pad_mask(lengths: torch.Tensor, max_len: int = 0) -> torch.Tensor:
879
+ """Make mask tensor containing indices of padded part.
880
+
881
+ See description of make_non_pad_mask.
882
+
883
+ Args:
884
+ lengths (torch.Tensor): Batch of lengths (B,).
885
+ Returns:
886
+ torch.Tensor: Mask tensor containing indices of padded part.
887
+
888
+ Examples:
889
+ >>> lengths = [5, 3, 2]
890
+ >>> make_pad_mask(lengths)
891
+ masks = [[0, 0, 0, 0 ,0],
892
+ [0, 0, 0, 1, 1],
893
+ [0, 0, 1, 1, 1]]
894
+ """
895
+ batch_size = lengths.size(0)
896
+ max_len = max_len if max_len > 0 else lengths.max().item()
897
+ seq_range = torch.arange(0,
898
+ max_len,
899
+ dtype=torch.int64,
900
+ device=lengths.device)
901
+ seq_range_expand = seq_range.unsqueeze(0).expand(batch_size, max_len)
902
+ seq_length_expand = lengths.unsqueeze(-1)
903
+ mask = seq_range_expand >= seq_length_expand
904
+ return mask
905
+
906
+ #https://github.com/FunAudioLLM/CosyVoice/blob/main/examples/magicdata-read/cosyvoice/conf/cosyvoice.yaml
907
+ class ConformerEncoder(torch.nn.Module):
908
+ """Conformer encoder module."""
909
+
910
+ def __init__(
911
+ self,
912
+ input_size: int,
913
+ output_size: int = 1024,
914
+ attention_heads: int = 16,
915
+ linear_units: int = 4096,
916
+ num_blocks: int = 6,
917
+ dropout_rate: float = 0.1,
918
+ positional_dropout_rate: float = 0.1,
919
+ attention_dropout_rate: float = 0.0,
920
+ input_layer: str = 'linear',
921
+ pos_enc_layer_type: str = 'rel_pos_espnet',
922
+ normalize_before: bool = True,
923
+ static_chunk_size: int = 1, # 1: causal_mask; 0: full_mask
924
+ use_dynamic_chunk: bool = False,
925
+ use_dynamic_left_chunk: bool = False,
926
+ positionwise_conv_kernel_size: int = 1,
927
+ macaron_style: bool =False,
928
+ selfattention_layer_type: str = "rel_selfattn",
929
+ activation_type: str = "swish",
930
+ use_cnn_module: bool = False,
931
+ cnn_module_kernel: int = 15,
932
+ causal: bool = False,
933
+ cnn_module_norm: str = "batch_norm",
934
+ key_bias: bool = True,
935
+ gradient_checkpointing: bool = False,
936
+ ):
937
+ """Construct ConformerEncoder
938
+
939
+ Args:
940
+ input_size to use_dynamic_chunk, see in BaseEncoder
941
+ positionwise_conv_kernel_size (int): Kernel size of positionwise
942
+ conv1d layer.
943
+ macaron_style (bool): Whether to use macaron style for
944
+ positionwise layer.
945
+ selfattention_layer_type (str): Encoder attention layer type,
946
+ the parameter has no effect now, it's just for configure
947
+ compatibility. #'rel_selfattn'
948
+ activation_type (str): Encoder activation function type.
949
+ use_cnn_module (bool): Whether to use convolution module.
950
+ cnn_module_kernel (int): Kernel size of convolution module.
951
+ causal (bool): whether to use causal convolution or not.
952
+ key_bias: whether use bias in attention.linear_k, False for whisper models.
953
+ """
954
+ super().__init__()
955
+ self.output_size = output_size
956
+ self.embed = LinearEmbed(input_size, output_size, dropout_rate,
957
+ EspnetRelPositionalEncoding(output_size, positional_dropout_rate))
958
+ self.normalize_before = normalize_before
959
+ self.after_norm = torch.nn.LayerNorm(output_size, eps=1e-5)
960
+ self.gradient_checkpointing = gradient_checkpointing
961
+ self.use_dynamic_chunk = use_dynamic_chunk
962
+
963
+ self.static_chunk_size = static_chunk_size
964
+ self.use_dynamic_chunk = use_dynamic_chunk
965
+ self.use_dynamic_left_chunk = use_dynamic_left_chunk
966
+ activation = ACTIVATION_CLASSES[activation_type]()
967
+
968
+ # self-attention module definition
969
+ encoder_selfattn_layer_args = (
970
+ attention_heads,
971
+ output_size,
972
+ attention_dropout_rate,
973
+ key_bias,
974
+ )
975
+ # feed-forward module definition
976
+ positionwise_layer_args = (
977
+ output_size,
978
+ linear_units,
979
+ dropout_rate,
980
+ activation,
981
+ )
982
+ # convolution module definition
983
+ convolution_layer_args = (output_size, cnn_module_kernel, activation,
984
+ cnn_module_norm, causal)
985
+
986
+ self.encoders = torch.nn.ModuleList([
987
+ ConformerEncoderLayer(
988
+ output_size,
989
+ RelPositionMultiHeadedAttention(
990
+ *encoder_selfattn_layer_args),
991
+ PositionwiseFeedForward(*positionwise_layer_args),
992
+ PositionwiseFeedForward(
993
+ *positionwise_layer_args) if macaron_style else None,
994
+ ConvolutionModule(
995
+ *convolution_layer_args) if use_cnn_module else None,
996
+ dropout_rate,
997
+ normalize_before,
998
+ ) for _ in range(num_blocks)
999
+ ])
1000
+
1001
+ def forward_layers(self, xs: torch.Tensor, chunk_masks: torch.Tensor,
1002
+ pos_emb: torch.Tensor,
1003
+ mask_pad: torch.Tensor) -> torch.Tensor:
1004
+ for layer in self.encoders:
1005
+ xs, chunk_masks, _, _ = layer(xs, chunk_masks, pos_emb, mask_pad)
1006
+ return xs
1007
+
1008
+ @torch.jit.unused
1009
+ def forward_layers_checkpointed(self, xs: torch.Tensor,
1010
+ chunk_masks: torch.Tensor,
1011
+ pos_emb: torch.Tensor,
1012
+ mask_pad: torch.Tensor) -> torch.Tensor:
1013
+ for layer in self.encoders:
1014
+ xs, chunk_masks, _, _ = ckpt.checkpoint(layer.__call__, xs,
1015
+ chunk_masks, pos_emb,
1016
+ mask_pad)
1017
+ return xs
1018
+
1019
+ def forward(
1020
+ self,
1021
+ xs: torch.Tensor,
1022
+ pad_mask: torch.Tensor,
1023
+ decoding_chunk_size: int = 0,
1024
+ num_decoding_left_chunks: int = -1,
1025
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
1026
+ """Embed positions in tensor.
1027
+
1028
+ Args:
1029
+ xs: padded input tensor (B, T, D)
1030
+ xs_lens: input length (B)
1031
+ decoding_chunk_size: decoding chunk size for dynamic chunk
1032
+ 0: default for training, use random dynamic chunk.
1033
+ <0: for decoding, use full chunk.
1034
+ >0: for decoding, use fixed chunk size as set.
1035
+ num_decoding_left_chunks: number of left chunks, this is for decoding,
1036
+ the chunk size is decoding_chunk_size.
1037
+ >=0: use num_decoding_left_chunks
1038
+ <0: use all left chunks
1039
+ Returns:
1040
+ encoder output tensor xs, and subsampled masks
1041
+ xs: padded output tensor (B, T' ~= T/subsample_rate, D)
1042
+ masks: torch.Tensor batch padding mask after subsample
1043
+ (B, 1, T' ~= T/subsample_rate)
1044
+ NOTE(xcsong):
1045
+ We pass the `__call__` method of the modules instead of `forward` to the
1046
+ checkpointing API because `__call__` attaches all the hooks of the module.
1047
+ https://discuss.pytorch.org/t/any-different-between-model-input-and-model-forward-input/3690/2
1048
+ """
1049
+ T = xs.size(1)
1050
+ masks = pad_mask.to(torch.bool).unsqueeze(1) # (B, 1, T)
1051
+ xs, pos_emb = self.embed(xs)
1052
+ mask_pad = masks # (B, 1, T/subsample_rate)
1053
+ chunk_masks = add_optional_chunk_mask(xs, masks,
1054
+ self.use_dynamic_chunk,
1055
+ self.use_dynamic_left_chunk,
1056
+ decoding_chunk_size,
1057
+ self.static_chunk_size,
1058
+ num_decoding_left_chunks)
1059
+ if self.gradient_checkpointing and self.training:
1060
+ xs = self.forward_layers_checkpointed(xs, chunk_masks, pos_emb,
1061
+ mask_pad)
1062
+ else:
1063
+ xs = self.forward_layers(xs, chunk_masks, pos_emb, mask_pad)
1064
+ if self.normalize_before:
1065
+ xs = self.after_norm(xs)
1066
+ # Here we assume the mask is not changed in encoder layers, so just
1067
+ # return the masks before encoder layers, and the masks will be used
1068
+ # for cross attention with decoder later
1069
+ return xs, masks
1070
+
1071
+
songgen/lyrics_utils/lyrics_tokenizer.py ADDED
@@ -0,0 +1,874 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import re
3
+ import textwrap
4
+ from functools import cached_property
5
+
6
+ import pypinyin
7
+ import torch
8
+ from hangul_romanize import Transliter
9
+ from hangul_romanize.rule import academic
10
+ from num2words import num2words
11
+ from spacy.lang.ar import Arabic
12
+ from spacy.lang.en import English
13
+ from spacy.lang.es import Spanish
14
+ from spacy.lang.ja import Japanese
15
+ from spacy.lang.zh import Chinese
16
+ from tokenizers import Tokenizer
17
+
18
+ from .zh_num2words import TextNorm as zh_num2words
19
+ from typing import Dict, List, Optional, Set, Union
20
+
21
+
22
+ #copy from https://github.com/coqui-ai/TTS/blob/dbf1a08a0d4e47fdad6172e433eeb34bc6b13b4e/TTS/tts/layers/xtts/tokenizer.py
23
+ def get_spacy_lang(lang):
24
+ if lang == "zh":
25
+ return Chinese()
26
+ elif lang == "ja":
27
+ return Japanese()
28
+ elif lang == "ar":
29
+ return Arabic()
30
+ elif lang == "es":
31
+ return Spanish()
32
+ else:
33
+ # For most languages, Enlish does the job
34
+ return English()
35
+
36
+
37
+ def split_sentence(text, lang, text_split_length=250):
38
+ """Preprocess the input text"""
39
+ text_splits = []
40
+ if text_split_length is not None and len(text) >= text_split_length:
41
+ text_splits.append("")
42
+ nlp = get_spacy_lang(lang)
43
+ nlp.add_pipe("sentencizer")
44
+ doc = nlp(text)
45
+ for sentence in doc.sents:
46
+ if len(text_splits[-1]) + len(str(sentence)) <= text_split_length:
47
+ # if the last sentence + the current sentence is less than the text_split_length
48
+ # then add the current sentence to the last sentence
49
+ text_splits[-1] += " " + str(sentence)
50
+ text_splits[-1] = text_splits[-1].lstrip()
51
+ elif len(str(sentence)) > text_split_length:
52
+ # if the current sentence is greater than the text_split_length
53
+ for line in textwrap.wrap(
54
+ str(sentence),
55
+ width=text_split_length,
56
+ drop_whitespace=True,
57
+ break_on_hyphens=False,
58
+ tabsize=1,
59
+ ):
60
+ text_splits.append(str(line))
61
+ else:
62
+ text_splits.append(str(sentence))
63
+
64
+ if len(text_splits) > 1:
65
+ if text_splits[0] == "":
66
+ del text_splits[0]
67
+ else:
68
+ text_splits = [text.lstrip()]
69
+
70
+ return text_splits
71
+
72
+
73
+ _whitespace_re = re.compile(r"\s+")
74
+
75
+ # List of (regular expression, replacement) pairs for abbreviations:
76
+ _abbreviations = {
77
+ "en": [
78
+ (re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
79
+ for x in [
80
+ ("mrs", "misess"),
81
+ ("mr", "mister"),
82
+ ("dr", "doctor"),
83
+ ("st", "saint"),
84
+ ("co", "company"),
85
+ ("jr", "junior"),
86
+ ("maj", "major"),
87
+ ("gen", "general"),
88
+ ("drs", "doctors"),
89
+ ("rev", "reverend"),
90
+ ("lt", "lieutenant"),
91
+ ("hon", "honorable"),
92
+ ("sgt", "sergeant"),
93
+ ("capt", "captain"),
94
+ ("esq", "esquire"),
95
+ ("ltd", "limited"),
96
+ ("col", "colonel"),
97
+ ("ft", "fort"),
98
+ ]
99
+ ],
100
+ "es": [
101
+ (re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
102
+ for x in [
103
+ ("sra", "señora"),
104
+ ("sr", "señor"),
105
+ ("dr", "doctor"),
106
+ ("dra", "doctora"),
107
+ ("st", "santo"),
108
+ ("co", "compañía"),
109
+ ("jr", "junior"),
110
+ ("ltd", "limitada"),
111
+ ]
112
+ ],
113
+ "fr": [
114
+ (re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
115
+ for x in [
116
+ ("mme", "madame"),
117
+ ("mr", "monsieur"),
118
+ ("dr", "docteur"),
119
+ ("st", "saint"),
120
+ ("co", "compagnie"),
121
+ ("jr", "junior"),
122
+ ("ltd", "limitée"),
123
+ ]
124
+ ],
125
+ "de": [
126
+ (re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
127
+ for x in [
128
+ ("fr", "frau"),
129
+ ("dr", "doktor"),
130
+ ("st", "sankt"),
131
+ ("co", "firma"),
132
+ ("jr", "junior"),
133
+ ]
134
+ ],
135
+ "pt": [
136
+ (re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
137
+ for x in [
138
+ ("sra", "senhora"),
139
+ ("sr", "senhor"),
140
+ ("dr", "doutor"),
141
+ ("dra", "doutora"),
142
+ ("st", "santo"),
143
+ ("co", "companhia"),
144
+ ("jr", "júnior"),
145
+ ("ltd", "limitada"),
146
+ ]
147
+ ],
148
+ "it": [
149
+ (re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
150
+ for x in [
151
+ # ("sig.ra", "signora"),
152
+ ("sig", "signore"),
153
+ ("dr", "dottore"),
154
+ ("st", "santo"),
155
+ ("co", "compagnia"),
156
+ ("jr", "junior"),
157
+ ("ltd", "limitata"),
158
+ ]
159
+ ],
160
+ "pl": [
161
+ (re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
162
+ for x in [
163
+ ("p", "pani"),
164
+ ("m", "pan"),
165
+ ("dr", "doktor"),
166
+ ("sw", "święty"),
167
+ ("jr", "junior"),
168
+ ]
169
+ ],
170
+ "ar": [
171
+ (re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
172
+ for x in [
173
+ # There are not many common abbreviations in Arabic as in English.
174
+ ]
175
+ ],
176
+ "zh": [
177
+ (re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
178
+ for x in [
179
+ # Chinese doesn't typically use abbreviations in the same way as Latin-based scripts.
180
+ ]
181
+ ],
182
+ "cs": [
183
+ (re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
184
+ for x in [
185
+ ("dr", "doktor"), # doctor
186
+ ("ing", "inženýr"), # engineer
187
+ ("p", "pan"), # Could also map to pani for woman but no easy way to do it
188
+ # Other abbreviations would be specialized and not as common.
189
+ ]
190
+ ],
191
+ "ru": [
192
+ (re.compile("\\b%s\\b" % x[0], re.IGNORECASE), x[1])
193
+ for x in [
194
+ ("г-жа", "госпожа"), # Mrs.
195
+ ("г-н", "господин"), # Mr.
196
+ ("д-р", "доктор"), # doctor
197
+ # Other abbreviations are less common or specialized.
198
+ ]
199
+ ],
200
+ "nl": [
201
+ (re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
202
+ for x in [
203
+ ("dhr", "de heer"), # Mr.
204
+ ("mevr", "mevrouw"), # Mrs.
205
+ ("dr", "dokter"), # doctor
206
+ ("jhr", "jonkheer"), # young lord or nobleman
207
+ # Dutch uses more abbreviations, but these are the most common ones.
208
+ ]
209
+ ],
210
+ "tr": [
211
+ (re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
212
+ for x in [
213
+ ("b", "bay"), # Mr.
214
+ ("byk", "büyük"), # büyük
215
+ ("dr", "doktor"), # doctor
216
+ # Add other Turkish abbreviations here if needed.
217
+ ]
218
+ ],
219
+ "hu": [
220
+ (re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
221
+ for x in [
222
+ ("dr", "doktor"), # doctor
223
+ ("b", "bácsi"), # Mr.
224
+ ("nőv", "nővér"), # nurse
225
+ # Add other Hungarian abbreviations here if needed.
226
+ ]
227
+ ],
228
+ "ko": [
229
+ (re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
230
+ for x in [
231
+ # Korean doesn't typically use abbreviations in the same way as Latin-based scripts.
232
+ ]
233
+ ],
234
+ }
235
+
236
+
237
+ def expand_abbreviations_multilingual(text, lang="en"):
238
+ for regex, replacement in _abbreviations[lang]:
239
+ text = re.sub(regex, replacement, text)
240
+ return text
241
+
242
+
243
+ _symbols_multilingual = {
244
+ "en": [
245
+ (re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
246
+ for x in [
247
+ ("&", " and "),
248
+ ("@", " at "),
249
+ ("%", " percent "),
250
+ ("#", " hash "),
251
+ ("$", " dollar "),
252
+ ("£", " pound "),
253
+ ("°", " degree "),
254
+ ]
255
+ ],
256
+ "es": [
257
+ (re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
258
+ for x in [
259
+ ("&", " y "),
260
+ ("@", " arroba "),
261
+ ("%", " por ciento "),
262
+ ("#", " numeral "),
263
+ ("$", " dolar "),
264
+ ("£", " libra "),
265
+ ("°", " grados "),
266
+ ]
267
+ ],
268
+ "fr": [
269
+ (re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
270
+ for x in [
271
+ ("&", " et "),
272
+ ("@", " arobase "),
273
+ ("%", " pour cent "),
274
+ ("#", " dièse "),
275
+ ("$", " dollar "),
276
+ ("£", " livre "),
277
+ ("°", " degrés "),
278
+ ]
279
+ ],
280
+ "de": [
281
+ (re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
282
+ for x in [
283
+ ("&", " und "),
284
+ ("@", " at "),
285
+ ("%", " prozent "),
286
+ ("#", " raute "),
287
+ ("$", " dollar "),
288
+ ("£", " pfund "),
289
+ ("°", " grad "),
290
+ ]
291
+ ],
292
+ "pt": [
293
+ (re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
294
+ for x in [
295
+ ("&", " e "),
296
+ ("@", " arroba "),
297
+ ("%", " por cento "),
298
+ ("#", " cardinal "),
299
+ ("$", " dólar "),
300
+ ("£", " libra "),
301
+ ("°", " graus "),
302
+ ]
303
+ ],
304
+ "it": [
305
+ (re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
306
+ for x in [
307
+ ("&", " e "),
308
+ ("@", " chiocciola "),
309
+ ("%", " per cento "),
310
+ ("#", " cancelletto "),
311
+ ("$", " dollaro "),
312
+ ("£", " sterlina "),
313
+ ("°", " gradi "),
314
+ ]
315
+ ],
316
+ "pl": [
317
+ (re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
318
+ for x in [
319
+ ("&", " i "),
320
+ ("@", " małpa "),
321
+ ("%", " procent "),
322
+ ("#", " krzyżyk "),
323
+ ("$", " dolar "),
324
+ ("£", " funt "),
325
+ ("°", " stopnie "),
326
+ ]
327
+ ],
328
+ "ar": [
329
+ # Arabic
330
+ (re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
331
+ for x in [
332
+ ("&", " و "),
333
+ ("@", " على "),
334
+ ("%", " في المئة "),
335
+ ("#", " رقم "),
336
+ ("$", " دولار "),
337
+ ("£", " جنيه "),
338
+ ("°", " درجة "),
339
+ ]
340
+ ],
341
+ "zh": [
342
+ # Chinese
343
+ (re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
344
+ for x in [
345
+ ("&", " 和 "),
346
+ ("@", " 在 "),
347
+ ("%", " 百分之 "),
348
+ ("#", " 号 "),
349
+ ("$", " 美元 "),
350
+ ("£", " 英镑 "),
351
+ ("°", " 度 "),
352
+ ]
353
+ ],
354
+ "cs": [
355
+ # Czech
356
+ (re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
357
+ for x in [
358
+ ("&", " a "),
359
+ ("@", " na "),
360
+ ("%", " procento "),
361
+ ("#", " křížek "),
362
+ ("$", " dolar "),
363
+ ("£", " libra "),
364
+ ("°", " stupně "),
365
+ ]
366
+ ],
367
+ "ru": [
368
+ # Russian
369
+ (re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
370
+ for x in [
371
+ ("&", " и "),
372
+ ("@", " собака "),
373
+ ("%", " процентов "),
374
+ ("#", " номер "),
375
+ ("$", " доллар "),
376
+ ("£", " фунт "),
377
+ ("°", " градус "),
378
+ ]
379
+ ],
380
+ "nl": [
381
+ # Dutch
382
+ (re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
383
+ for x in [
384
+ ("&", " en "),
385
+ ("@", " bij "),
386
+ ("%", " procent "),
387
+ ("#", " hekje "),
388
+ ("$", " dollar "),
389
+ ("£", " pond "),
390
+ ("°", " graden "),
391
+ ]
392
+ ],
393
+ "tr": [
394
+ (re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
395
+ for x in [
396
+ ("&", " ve "),
397
+ ("@", " at "),
398
+ ("%", " yüzde "),
399
+ ("#", " diyez "),
400
+ ("$", " dolar "),
401
+ ("£", " sterlin "),
402
+ ("°", " derece "),
403
+ ]
404
+ ],
405
+ "hu": [
406
+ (re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
407
+ for x in [
408
+ ("&", " és "),
409
+ ("@", " kukac "),
410
+ ("%", " százalék "),
411
+ ("#", " kettőskereszt "),
412
+ ("$", " dollár "),
413
+ ("£", " font "),
414
+ ("°", " fok "),
415
+ ]
416
+ ],
417
+ "ko": [
418
+ # Korean
419
+ (re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
420
+ for x in [
421
+ ("&", " 그리고 "),
422
+ ("@", " 에 "),
423
+ ("%", " 퍼센트 "),
424
+ ("#", " 번호 "),
425
+ ("$", " 달러 "),
426
+ ("£", " 파운드 "),
427
+ ("°", " 도 "),
428
+ ]
429
+ ],
430
+ }
431
+
432
+
433
+ def expand_symbols_multilingual(text, lang="en"):
434
+ for regex, replacement in _symbols_multilingual[lang]:
435
+ text = re.sub(regex, replacement, text)
436
+ text = text.replace(" ", " ") # Ensure there are no double spaces
437
+ return text.strip()
438
+
439
+
440
+ _ordinal_re = {
441
+ "en": re.compile(r"([0-9]+)(st|nd|rd|th)"),
442
+ "es": re.compile(r"([0-9]+)(º|ª|er|o|a|os|as)"),
443
+ "fr": re.compile(r"([0-9]+)(º|ª|er|re|e|ème)"),
444
+ "de": re.compile(r"([0-9]+)(st|nd|rd|th|º|ª|\.(?=\s|$))"),
445
+ "pt": re.compile(r"([0-9]+)(º|ª|o|a|os|as)"),
446
+ "it": re.compile(r"([0-9]+)(º|°|ª|o|a|i|e)"),
447
+ "pl": re.compile(r"([0-9]+)(º|ª|st|nd|rd|th)"),
448
+ "ar": re.compile(r"([0-9]+)(ون|ين|ث|ر|ى)"),
449
+ "cs": re.compile(r"([0-9]+)\.(?=\s|$)"), # In Czech, a dot is often used after the number to indicate ordinals.
450
+ "ru": re.compile(r"([0-9]+)(-й|-я|-е|-ое|-ье|-го)"),
451
+ "nl": re.compile(r"([0-9]+)(de|ste|e)"),
452
+ "tr": re.compile(r"([0-9]+)(\.|inci|nci|uncu|üncü|\.)"),
453
+ "hu": re.compile(r"([0-9]+)(\.|adik|edik|odik|edik|ödik|ödike|ik)"),
454
+ "ko": re.compile(r"([0-9]+)(번째|번|차|째)"),
455
+ }
456
+ _number_re = re.compile(r"[0-9]+")
457
+ _currency_re = {
458
+ "USD": re.compile(r"((\$[0-9\.\,]*[0-9]+)|([0-9\.\,]*[0-9]+\$))"),
459
+ "GBP": re.compile(r"((£[0-9\.\,]*[0-9]+)|([0-9\.\,]*[0-9]+£))"),
460
+ "EUR": re.compile(r"(([0-9\.\,]*[0-9]+€)|((€[0-9\.\,]*[0-9]+)))"),
461
+ }
462
+
463
+ _comma_number_re = re.compile(r"\b\d{1,3}(,\d{3})*(\.\d+)?\b")
464
+ _dot_number_re = re.compile(r"\b\d{1,3}(.\d{3})*(\,\d+)?\b")
465
+ _decimal_number_re = re.compile(r"([0-9]+[.,][0-9]+)")
466
+
467
+
468
+ def _remove_commas(m):
469
+ text = m.group(0)
470
+ if "," in text:
471
+ text = text.replace(",", "")
472
+ return text
473
+
474
+
475
+ def _remove_dots(m):
476
+ text = m.group(0)
477
+ if "." in text:
478
+ text = text.replace(".", "")
479
+ return text
480
+
481
+
482
+ def _expand_decimal_point(m, lang="en"):
483
+ amount = m.group(1).replace(",", ".")
484
+ return num2words(float(amount), lang=lang if lang != "cs" else "cz")
485
+
486
+
487
+ def _expand_currency(m, lang="en", currency="USD"):
488
+ amount = float((re.sub(r"[^\d.]", "", m.group(0).replace(",", "."))))
489
+ full_amount = num2words(amount, to="currency", currency=currency, lang=lang if lang != "cs" else "cz")
490
+
491
+ and_equivalents = {
492
+ "en": ", ",
493
+ "es": " con ",
494
+ "fr": " et ",
495
+ "de": " und ",
496
+ "pt": " e ",
497
+ "it": " e ",
498
+ "pl": ", ",
499
+ "cs": ", ",
500
+ "ru": ", ",
501
+ "nl": ", ",
502
+ "ar": ", ",
503
+ "tr": ", ",
504
+ "hu": ", ",
505
+ "ko": ", ",
506
+ }
507
+
508
+ if amount.is_integer():
509
+ last_and = full_amount.rfind(and_equivalents[lang])
510
+ if last_and != -1:
511
+ full_amount = full_amount[:last_and]
512
+
513
+ return full_amount
514
+
515
+
516
+ def _expand_ordinal(m, lang="en"):
517
+ return num2words(int(m.group(1)), ordinal=True, lang=lang if lang != "cs" else "cz")
518
+
519
+
520
+ def _expand_number(m, lang="en"):
521
+ return num2words(int(m.group(0)), lang=lang if lang != "cs" else "cz")
522
+
523
+
524
+ def expand_numbers_multilingual(text, lang="en"):
525
+ if lang == "zh":
526
+ text = zh_num2words()(text)
527
+ else:
528
+ if lang in ["en", "ru"]:
529
+ text = re.sub(_comma_number_re, _remove_commas, text)
530
+ else:
531
+ text = re.sub(_dot_number_re, _remove_dots, text)
532
+ try:
533
+ text = re.sub(_currency_re["GBP"], lambda m: _expand_currency(m, lang, "GBP"), text)
534
+ text = re.sub(_currency_re["USD"], lambda m: _expand_currency(m, lang, "USD"), text)
535
+ text = re.sub(_currency_re["EUR"], lambda m: _expand_currency(m, lang, "EUR"), text)
536
+ except:
537
+ pass
538
+ if lang != "tr":
539
+ text = re.sub(_decimal_number_re, lambda m: _expand_decimal_point(m, lang), text)
540
+ text = re.sub(_ordinal_re[lang], lambda m: _expand_ordinal(m, lang), text)
541
+ text = re.sub(_number_re, lambda m: _expand_number(m, lang), text)
542
+ return text
543
+
544
+
545
+ def lowercase(text):
546
+ return text.lower()
547
+
548
+
549
+ def collapse_whitespace(text):
550
+ return re.sub(_whitespace_re, " ", text)
551
+
552
+
553
+ def multilingual_cleaners(text, lang):
554
+ text = text.replace('"', "")
555
+ if lang == "tr":
556
+ text = text.replace("İ", "i")
557
+ text = text.replace("Ö", "ö")
558
+ text = text.replace("Ü", "ü")
559
+ text = lowercase(text)
560
+ text = expand_numbers_multilingual(text, lang)
561
+ text = expand_abbreviations_multilingual(text, lang)
562
+ text = expand_symbols_multilingual(text, lang=lang)
563
+ text = collapse_whitespace(text)
564
+ return text
565
+
566
+
567
+ def basic_cleaners(text):
568
+ """Basic pipeline that lowercases and collapses whitespace without transliteration."""
569
+ text = lowercase(text)
570
+ text = collapse_whitespace(text)
571
+ return text
572
+
573
+
574
+ def chinese_transliterate(text):
575
+ return "".join(
576
+ [p[0] for p in pypinyin.pinyin(text, style=pypinyin.Style.TONE3, heteronym=False, neutral_tone_with_five=True)]
577
+ )
578
+
579
+
580
+ def japanese_cleaners(text, katsu):
581
+ text = katsu.romaji(text)
582
+ text = lowercase(text)
583
+ return text
584
+
585
+
586
+ def korean_transliterate(text):
587
+ r = Transliter(academic)
588
+ return r.translit(text)
589
+
590
+
591
+ DEFAULT_VOCAB_FILE = os.path.join(os.path.dirname(os.path.realpath(__file__)), "vocab.json")
592
+
593
+
594
+ class VoiceBpeTokenizer:
595
+ def __init__(self, vocab_file=DEFAULT_VOCAB_FILE):
596
+ self.tokenizer = None
597
+ if vocab_file is not None:
598
+ self.tokenizer = Tokenizer.from_file(vocab_file)
599
+ self.char_limits = {
600
+ "en": 10000,
601
+ "de": 253,
602
+ "fr": 273,
603
+ "es": 239,
604
+ "it": 213,
605
+ "pt": 203,
606
+ "pl": 224,
607
+ "zh": 82,
608
+ "ar": 166,
609
+ "cs": 186,
610
+ "ru": 182,
611
+ "nl": 251,
612
+ "tr": 226,
613
+ "ja": 71,
614
+ "hu": 224,
615
+ "ko": 95,
616
+ }
617
+
618
+ @cached_property
619
+ def katsu(self):
620
+ import cutlet
621
+
622
+ return cutlet.Cutlet()
623
+
624
+ def check_input_length(self, txt, lang):
625
+ lang = lang.split("-")[0] # remove the region
626
+ limit = self.char_limits.get(lang, 250)
627
+ if len(txt) > limit:
628
+ print(
629
+ f"[!] Warning: The text length exceeds the character limit of {limit} for language '{lang}', this might cause truncated audio."
630
+ )
631
+
632
+ def preprocess_text(self, txt, lang):
633
+ if lang in {"ar", "cs", "de", "en", "es", "fr", "hu", "it", "nl", "pl", "pt", "ru", "tr", "zh", "ko"}:
634
+ txt = multilingual_cleaners(txt, lang)
635
+ if lang == "zh":
636
+ txt = chinese_transliterate(txt)
637
+ if lang == "ko":
638
+ txt = korean_transliterate(txt)
639
+ elif lang == "ja":
640
+ txt = japanese_cleaners(txt, self.katsu)
641
+ elif lang == "hi":
642
+ # @manmay will implement this
643
+ txt = basic_cleaners(txt)
644
+ else:
645
+ raise NotImplementedError(f"Language '{lang}' is not supported.")
646
+ return txt
647
+
648
+ def encode(self, txt, lang):
649
+ lang = lang.split("-")[0] # remove the region
650
+ self.check_input_length(txt, lang)
651
+ txt = self.preprocess_text(txt, lang)
652
+ lang = "zh-cn" if lang == "zh" else lang
653
+ txt = f"[{lang}]{txt}"
654
+ txt = txt.replace(" ", "[SPACE]")
655
+ return self.tokenizer.encode(txt).ids
656
+
657
+ def decode(self, seq, skip_special_tokens=False):
658
+ if isinstance(seq, torch.Tensor):
659
+ seq = seq.cpu().numpy()
660
+ txt = self.tokenizer.decode(seq, skip_special_tokens=False).replace(" ", "")
661
+ txt = txt.replace("[SPACE]", " ")
662
+ txt = txt.replace("[STOP]", "")
663
+ # txt = txt.replace("[UNK]", "")
664
+ return txt
665
+
666
+
667
+ #copy from https://github.com/huggingface/transformers/blob/main/src/transformers/tokenization_utils_base.py#L3936
668
+ def batch_decode(
669
+ self,
670
+ sequences: Union[List[int], List[List[int]], "np.ndarray", "torch.Tensor", "tf.Tensor"],
671
+ skip_special_tokens: bool = False,
672
+ ) -> List[str]:
673
+ """
674
+ Convert a list of lists of token ids into a list of strings by calling decode.
675
+
676
+ Args:
677
+ sequences (`Union[List[int], List[List[int]], np.ndarray, torch.Tensor, tf.Tensor]`):
678
+ List of tokenized input ids. Can be obtained using the `__call__` method.
679
+ skip_special_tokens (`bool`, *optional*, defaults to `False`):
680
+ Whether or not to remove special tokens in the decoding.
681
+ kwargs (additional keyword arguments, *optional*):
682
+ Will be passed to the underlying model specific decode method.
683
+
684
+ Returns:
685
+ `List[str]`: The list of decoded sentences.
686
+ """
687
+ return [
688
+ self.decode(seq)
689
+ for seq in sequences
690
+ ]
691
+
692
+ #https://github.com/coqui-ai/TTS/blob/dev/TTS/tts/layers/xtts/trainer/dataset.py#L202
693
+ # def pad(self):
694
+
695
+ def __len__(self):
696
+ return self.tokenizer.get_vocab_size()
697
+
698
+ def get_number_tokens(self):
699
+ return max(self.tokenizer.get_vocab().values()) + 1
700
+
701
+
702
+ def test_expand_numbers_multilingual():
703
+ test_cases = [
704
+ # English
705
+ ("In 12.5 seconds.", "In twelve point five seconds.", "en"),
706
+ ("There were 50 soldiers.", "There were fifty soldiers.", "en"),
707
+ ("This is a 1st test", "This is a first test", "en"),
708
+ ("That will be $20 sir.", "That will be twenty dollars sir.", "en"),
709
+ ("That will be 20€ sir.", "That will be twenty euro sir.", "en"),
710
+ ("That will be 20.15€ sir.", "That will be twenty euro, fifteen cents sir.", "en"),
711
+ ("That's 100,000.5.", "That's one hundred thousand point five.", "en"),
712
+ # French
713
+ ("En 12,5 secondes.", "En douze virgule cinq secondes.", "fr"),
714
+ ("Il y avait 50 soldats.", "Il y avait cinquante soldats.", "fr"),
715
+ ("Ceci est un 1er test", "Ceci est un premier test", "fr"),
716
+ ("Cela vous fera $20 monsieur.", "Cela vous fera vingt dollars monsieur.", "fr"),
717
+ ("Cela vous fera 20€ monsieur.", "Cela vous fera vingt euros monsieur.", "fr"),
718
+ ("Cela vous fera 20,15€ monsieur.", "Cela vous fera vingt euros et quinze centimes monsieur.", "fr"),
719
+ ("Ce sera 100.000,5.", "Ce sera cent mille virgule cinq.", "fr"),
720
+ # German
721
+ ("In 12,5 Sekunden.", "In zwölf Komma fünf Sekunden.", "de"),
722
+ ("Es gab 50 Soldaten.", "Es gab fünfzig Soldaten.", "de"),
723
+ ("Dies ist ein 1. Test", "Dies ist ein erste Test", "de"), # Issue with gender
724
+ ("Das macht $20 Herr.", "Das macht zwanzig Dollar Herr.", "de"),
725
+ ("Das macht 20€ Herr.", "Das macht zwanzig Euro Herr.", "de"),
726
+ ("Das macht 20,15€ Herr.", "Das macht zwanzig Euro und fünfzehn Cent Herr.", "de"),
727
+ # Spanish
728
+ ("En 12,5 segundos.", "En doce punto cinco segundos.", "es"),
729
+ ("Había 50 soldados.", "Había cincuenta soldados.", "es"),
730
+ ("Este es un 1er test", "Este es un primero test", "es"),
731
+ ("Eso le costará $20 señor.", "Eso le costará veinte dólares señor.", "es"),
732
+ ("Eso le costará 20€ señor.", "Eso le costará veinte euros señor.", "es"),
733
+ ("Eso le costará 20,15€ señor.", "Eso le costará veinte euros con quince céntimos señor.", "es"),
734
+ # Italian
735
+ ("In 12,5 secondi.", "In dodici virgola cinque secondi.", "it"),
736
+ ("C'erano 50 soldati.", "C'erano cinquanta soldati.", "it"),
737
+ ("Questo è un 1° test", "Questo è un primo test", "it"),
738
+ ("Ti costerà $20 signore.", "Ti costerà venti dollari signore.", "it"),
739
+ ("Ti costerà 20€ signore.", "Ti costerà venti euro signore.", "it"),
740
+ ("Ti costerà 20,15€ signore.", "Ti costerà venti euro e quindici centesimi signore.", "it"),
741
+ # Portuguese
742
+ ("Em 12,5 segundos.", "Em doze vírgula cinco segundos.", "pt"),
743
+ ("Havia 50 soldados.", "Havia cinquenta soldados.", "pt"),
744
+ ("Este é um 1º teste", "Este é um primeiro teste", "pt"),
745
+ ("Isso custará $20 senhor.", "Isso custará vinte dólares senhor.", "pt"),
746
+ ("Isso custará 20€ senhor.", "Isso custará vinte euros senhor.", "pt"),
747
+ (
748
+ "Isso custará 20,15€ senhor.",
749
+ "Isso custará vinte euros e quinze cêntimos senhor.",
750
+ "pt",
751
+ ), # "cêntimos" should be "centavos" num2words issue
752
+ # Polish
753
+ ("W 12,5 sekundy.", "W dwanaście przecinek pięć sekundy.", "pl"),
754
+ ("Było 50 żołnierzy.", "Było pięćdziesiąt żołnierzy.", "pl"),
755
+ ("To będzie kosztować 20€ panie.", "To będzie kosztować dwadzieścia euro panie.", "pl"),
756
+ ("To będzie kosztować 20,15€ panie.", "To będzie kosztować dwadzieścia euro, piętnaście centów panie.", "pl"),
757
+ # Arabic
758
+ ("في الـ 12,5 ثانية.", "في الـ اثنا عشر , خمسون ثانية.", "ar"),
759
+ ("كان هناك 50 جنديًا.", "كان هناك خمسون جنديًا.", "ar"),
760
+ # ("ستكون النتيجة $20 يا سيد.", 'ستكون النتيجة عشرون دولار يا سيد.', 'ar'), # $ and € are mising from num2words
761
+ # ("ستكون النتيجة 20€ يا سيد.", 'ستكون النتيجة عشرون يورو يا سيد.', 'ar'),
762
+ # Czech
763
+ ("Za 12,5 vteřiny.", "Za dvanáct celá pět vteřiny.", "cs"),
764
+ ("Bylo tam 50 vojáků.", "Bylo tam padesát vojáků.", "cs"),
765
+ ("To bude stát 20€ pane.", "To bude stát dvacet euro pane.", "cs"),
766
+ ("To bude 20.15€ pane.", "To bude dvacet euro, patnáct centů pane.", "cs"),
767
+ # Russian
768
+ ("Через 12.5 секунды.", "Через двенадцать запятая пять секунды.", "ru"),
769
+ ("Там было 50 солдат.", "Там было пятьдесят солдат.", "ru"),
770
+ ("Это будет 20.15€ сэр.", "Это будет двадцать евро, пятнадцать центов сэр.", "ru"),
771
+ ("Это будет стоить 20€ господин.", "Это будет стоить двадцать евро господин.", "ru"),
772
+ # Dutch
773
+ ("In 12,5 seconden.", "In twaalf komma vijf seconden.", "nl"),
774
+ ("Er waren 50 soldaten.", "Er waren vijftig soldaten.", "nl"),
775
+ ("Dat wordt dan $20 meneer.", "Dat wordt dan twintig dollar meneer.", "nl"),
776
+ ("Dat wordt dan 20€ meneer.", "Dat wordt dan twintig euro meneer.", "nl"),
777
+ # Chinese (Simplified)
778
+ ("在12.5秒内", "在十二点五秒内", "zh"),
779
+ ("有50名士兵", "有五十名士兵", "zh"),
780
+ # ("那将是$20先生", '那将是二十美元先生', 'zh'), currency doesn't work
781
+ # ("那将是20€先生", '那将是二十欧元先生', 'zh'),
782
+ # Turkish
783
+ # ("12,5 saniye içinde.", 'On iki virgül beş saniye içinde.', 'tr'), # decimal doesn't work for TR
784
+ ("50 asker vardı.", "elli asker vardı.", "tr"),
785
+ ("Bu 1. test", "Bu birinci test", "tr"),
786
+ # ("Bu 100.000,5.", 'Bu yüz bin virgül beş.', 'tr'),
787
+ # Hungarian
788
+ ("12,5 másodperc alatt.", "tizenkettő egész öt tized másodperc alatt.", "hu"),
789
+ ("50 katona volt.", "ötven katona volt.", "hu"),
790
+ ("Ez az 1. teszt", "Ez az első teszt", "hu"),
791
+ # Korean
792
+ ("12.5 초 안에.", "십이 점 다섯 초 안에.", "ko"),
793
+ ("50 명의 병사가 있었다.", "오십 명의 병사가 있었다.", "ko"),
794
+ ("이것은 1 번째 테스트입니다", "이것은 첫 번째 테스트입니다", "ko"),
795
+ ]
796
+ for a, b, lang in test_cases:
797
+ out = expand_numbers_multilingual(a, lang=lang)
798
+ assert out == b, f"'{out}' vs '{b}'"
799
+
800
+
801
+ def test_abbreviations_multilingual():
802
+ test_cases = [
803
+ # English
804
+ ("Hello Mr. Smith.", "Hello mister Smith.", "en"),
805
+ ("Dr. Jones is here.", "doctor Jones is here.", "en"),
806
+ # Spanish
807
+ ("Hola Sr. Garcia.", "Hola señor Garcia.", "es"),
808
+ ("La Dra. Martinez es muy buena.", "La doctora Martinez es muy buena.", "es"),
809
+ # French
810
+ ("Bonjour Mr. Dupond.", "Bonjour monsieur Dupond.", "fr"),
811
+ ("Mme. Moreau est absente aujourd'hui.", "madame Moreau est absente aujourd'hui.", "fr"),
812
+ # German
813
+ ("Frau Dr. Müller ist sehr klug.", "Frau doktor Müller ist sehr klug.", "de"),
814
+ # Portuguese
815
+ ("Olá Sr. Silva.", "Olá senhor Silva.", "pt"),
816
+ ("Dra. Costa, você está disponível?", "doutora Costa, você está disponível?", "pt"),
817
+ # Italian
818
+ ("Buongiorno, Sig. Rossi.", "Buongiorno, signore Rossi.", "it"),
819
+ # ("Sig.ra Bianchi, posso aiutarti?", 'signora Bianchi, posso aiutarti?', 'it'), # Issue with matching that pattern
820
+ # Polish
821
+ ("Dzień dobry, P. Kowalski.", "Dzień dobry, pani Kowalski.", "pl"),
822
+ ("M. Nowak, czy mogę zadać pytanie?", "pan Nowak, czy mogę zadać pytanie?", "pl"),
823
+ # Czech
824
+ ("P. Novák", "pan Novák", "cs"),
825
+ ("Dr. Vojtěch", "doktor Vojtěch", "cs"),
826
+ # Dutch
827
+ ("Dhr. Jansen", "de heer Jansen", "nl"),
828
+ ("Mevr. de Vries", "mevrouw de Vries", "nl"),
829
+ # Russian
830
+ ("Здравствуйте Г-н Иванов.", "Здравствуйте господин Иванов.", "ru"),
831
+ ("Д-р Смирнов здесь, что��ы увидеть вас.", "доктор Смирнов здесь, чтобы увидеть вас.", "ru"),
832
+ # Turkish
833
+ ("Merhaba B. Yılmaz.", "Merhaba bay Yılmaz.", "tr"),
834
+ ("Dr. Ayşe burada.", "doktor Ayşe burada.", "tr"),
835
+ # Hungarian
836
+ ("Dr. Szabó itt van.", "doktor Szabó itt van.", "hu"),
837
+ ]
838
+
839
+ for a, b, lang in test_cases:
840
+ out = expand_abbreviations_multilingual(a, lang=lang)
841
+ assert out == b, f"'{out}' vs '{b}'"
842
+
843
+
844
+ def test_symbols_multilingual():
845
+ test_cases = [
846
+ ("I have 14% battery", "I have 14 percent battery", "en"),
847
+ ("Te veo @ la fiesta", "Te veo arroba la fiesta", "es"),
848
+ ("J'ai 14° de fièvre", "J'ai 14 degrés de fièvre", "fr"),
849
+ ("Die Rechnung beträgt £ 20", "Die Rechnung beträgt pfund 20", "de"),
850
+ ("O meu email é ana&joao@gmail.com", "O meu email é ana e joao arroba gmail.com", "pt"),
851
+ ("linguaggio di programmazione C#", "linguaggio di programmazione C cancelletto", "it"),
852
+ ("Moja temperatura to 36.6°", "Moja temperatura to 36.6 stopnie", "pl"),
853
+ ("Mám 14% baterie", "Mám 14 procento baterie", "cs"),
854
+ ("Těším se na tebe @ party", "Těším se na tebe na party", "cs"),
855
+ ("У меня 14% заряда", "У меня 14 процентов заряда", "ru"),
856
+ ("Я буду @ дома", "Я буду собака дома", "ru"),
857
+ ("Ik heb 14% batterij", "Ik heb 14 procent batterij", "nl"),
858
+ ("Ik zie je @ het feest", "Ik zie je bij het feest", "nl"),
859
+ ("لدي 14% في البطارية", "لدي 14 في المئة في البطارية", "ar"),
860
+ ("我的电量为 14%", "我的电量为 14 百分之", "zh"),
861
+ ("Pilim %14 dolu.", "Pilim yüzde 14 dolu.", "tr"),
862
+ ("Az akkumulátorom töltöttsége 14%", "Az akkumulátorom töltöttsége 14 százalék", "hu"),
863
+ ("배터리 잔량이 14%입니다.", "배터리 잔량이 14 퍼센트입니다.", "ko"),
864
+ ]
865
+
866
+ for a, b, lang in test_cases:
867
+ out = expand_symbols_multilingual(a, lang=lang)
868
+ assert out == b, f"'{out}' vs '{b}'"
869
+
870
+
871
+ if __name__ == "__main__":
872
+ test_expand_numbers_multilingual()
873
+ test_abbreviations_multilingual()
874
+ test_symbols_multilingual()
songgen/lyrics_utils/zh_num2words.py ADDED
@@ -0,0 +1,1209 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Authors:
2
+ # 2019.5 Zhiyang Zhou (https://github.com/Joee1995/chn_text_norm.git)
3
+ # 2019.9 - 2022 Jiayu DU
4
+ #copy from https://github.com/coqui-ai/TTS/blob/dbf1a08a0d4e47fdad6172e433eeb34bc6b13b4e/TTS/tts/layers/xtts/zh_num2words.py
5
+ import argparse
6
+ import csv
7
+ import os
8
+ import re
9
+ import string
10
+ import sys
11
+
12
+ # fmt: off
13
+
14
+ # ================================================================================ #
15
+ # basic constant
16
+ # ================================================================================ #
17
+ CHINESE_DIGIS = "零一二三四五六七八九"
18
+ BIG_CHINESE_DIGIS_SIMPLIFIED = "零壹贰叁肆伍陆柒捌玖"
19
+ BIG_CHINESE_DIGIS_TRADITIONAL = "零壹貳參肆伍陸柒捌玖"
20
+ SMALLER_BIG_CHINESE_UNITS_SIMPLIFIED = "十百千万"
21
+ SMALLER_BIG_CHINESE_UNITS_TRADITIONAL = "拾佰仟萬"
22
+ LARGER_CHINESE_NUMERING_UNITS_SIMPLIFIED = "亿兆京垓秭穰沟涧正载"
23
+ LARGER_CHINESE_NUMERING_UNITS_TRADITIONAL = "億兆京垓秭穰溝澗正載"
24
+ SMALLER_CHINESE_NUMERING_UNITS_SIMPLIFIED = "十百千万"
25
+ SMALLER_CHINESE_NUMERING_UNITS_TRADITIONAL = "拾佰仟萬"
26
+
27
+ ZERO_ALT = "〇"
28
+ ONE_ALT = "幺"
29
+ TWO_ALTS = ["两", "兩"]
30
+
31
+ POSITIVE = ["正", "正"]
32
+ NEGATIVE = ["负", "負"]
33
+ POINT = ["点", "點"]
34
+ # PLUS = [u'加', u'加']
35
+ # SIL = [u'杠', u'槓']
36
+
37
+ FILLER_CHARS = ["呃", "啊"]
38
+
39
+ ER_WHITELIST = (
40
+ "(儿女|儿子|儿孙|女儿|儿媳|妻儿|"
41
+ "胎儿|婴儿|新生儿|婴幼儿|幼儿|少儿|小儿|儿歌|儿童|儿科|托儿所|孤儿|"
42
+ "儿戏|儿化|台儿庄|鹿儿岛|正儿八经|吊儿郎当|生儿育女|托儿带女|养儿防老|痴儿呆女|"
43
+ "佳儿佳妇|儿怜兽扰|儿无常父|儿不嫌母丑|儿行千里母担忧|儿大不由爷|苏乞儿)"
44
+ )
45
+ ER_WHITELIST_PATTERN = re.compile(ER_WHITELIST)
46
+
47
+ # 中文数字系统类型
48
+ NUMBERING_TYPES = ["low", "mid", "high"]
49
+
50
+ CURRENCY_NAMES = "(人民币|美元|日元|英镑|欧元|马克|法郎|加拿大元|澳元|港币|先令|芬兰马克|爱尔兰镑|" "里拉|荷兰盾|埃斯库多|比塞塔|印尼盾|林吉特|新西兰元|比索|卢布|新加坡元|韩元|泰铢)"
51
+ CURRENCY_UNITS = "((亿|千万|百万|万|千|百)|(亿|千万|百万|万|千|百|)元|(亿|千万|百万|万|千|百|)块|角|毛|分)"
52
+ COM_QUANTIFIERS = (
53
+ "(匹|张|座|回|场|尾|条|个|首|阙|阵|网|炮|顶|丘|棵|只|支|袭|辆|挑|担|颗|壳|窠|曲|墙|群|腔|"
54
+ "砣|座|客|贯|扎|捆|刀|令|打|手|罗|坡|山|岭|江|溪|钟|队|单|双|对|出|口|头|脚|板|跳|枝|件|贴|"
55
+ "针|线|管|名|位|身|堂|课|本|页|家|户|层|丝|毫|厘|分|钱|两|斤|担|铢|石|钧|锱|忽|(千|毫|微)克|"
56
+ "毫|厘|分|寸|尺|丈|里|寻|常|铺|程|(千|分|厘|毫|微)米|撮|勺|合|升|斗|石|盘|碗|碟|叠|桶|笼|盆|"
57
+ "盒|杯|钟|斛|锅|簋|篮|盘|桶|罐|瓶|壶|卮|盏|箩|箱|煲|啖|袋|钵|年|月|日|季|刻|时|周|天|秒|分|旬|"
58
+ "纪|岁|世|更|夜|春|夏|秋|冬|代|伏|辈|丸|泡|粒|颗|幢|堆|条|根|支|道|面|片|张|颗|块)"
59
+ )
60
+
61
+
62
+ # Punctuation information are based on Zhon project (https://github.com/tsroten/zhon.git)
63
+ CN_PUNCS_STOP = "!?。。"
64
+ CN_PUNCS_NONSTOP = ""#$%&'()*+,-/:;<=>@[\]^_`{|}~⦅⦆「」、、〃《》「」『』【】〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘’‛“”„‟…‧﹏·〈〉-"
65
+ CN_PUNCS = CN_PUNCS_STOP + CN_PUNCS_NONSTOP
66
+
67
+ PUNCS = CN_PUNCS + string.punctuation
68
+ PUNCS_TRANSFORM = str.maketrans(PUNCS, "," * len(PUNCS), "") # replace puncs with English comma
69
+
70
+
71
+ # https://zh.wikipedia.org/wiki/全行和半行
72
+ QJ2BJ = {
73
+ " ": " ",
74
+ "!": "!",
75
+ """: '"',
76
+ "#": "#",
77
+ "$": "$",
78
+ "%": "%",
79
+ "&": "&",
80
+ "'": "'",
81
+ "(": "(",
82
+ ")": ")",
83
+ "*": "*",
84
+ "+": "+",
85
+ ",": ",",
86
+ "-": "-",
87
+ ".": ".",
88
+ "/": "/",
89
+ "0": "0",
90
+ "1": "1",
91
+ "2": "2",
92
+ "3": "3",
93
+ "4": "4",
94
+ "5": "5",
95
+ "6": "6",
96
+ "7": "7",
97
+ "8": "8",
98
+ "9": "9",
99
+ ":": ":",
100
+ ";": ";",
101
+ "<": "<",
102
+ "=": "=",
103
+ ">": ">",
104
+ "?": "?",
105
+ "@": "@",
106
+ "A": "A",
107
+ "B": "B",
108
+ "C": "C",
109
+ "D": "D",
110
+ "E": "E",
111
+ "F": "F",
112
+ "G": "G",
113
+ "H": "H",
114
+ "I": "I",
115
+ "J": "J",
116
+ "K": "K",
117
+ "L": "L",
118
+ "M": "M",
119
+ "N": "N",
120
+ "O": "O",
121
+ "P": "P",
122
+ "Q": "Q",
123
+ "R": "R",
124
+ "S": "S",
125
+ "T": "T",
126
+ "U": "U",
127
+ "V": "V",
128
+ "W": "W",
129
+ "X": "X",
130
+ "Y": "Y",
131
+ "Z": "Z",
132
+ "[": "[",
133
+ "\": "\\",
134
+ "]": "]",
135
+ "^": "^",
136
+ "_": "_",
137
+ "`": "`",
138
+ "a": "a",
139
+ "b": "b",
140
+ "c": "c",
141
+ "d": "d",
142
+ "e": "e",
143
+ "f": "f",
144
+ "g": "g",
145
+ "h": "h",
146
+ "i": "i",
147
+ "j": "j",
148
+ "k": "k",
149
+ "l": "l",
150
+ "m": "m",
151
+ "n": "n",
152
+ "o": "o",
153
+ "p": "p",
154
+ "q": "q",
155
+ "r": "r",
156
+ "s": "s",
157
+ "t": "t",
158
+ "u": "u",
159
+ "v": "v",
160
+ "w": "w",
161
+ "x": "x",
162
+ "y": "y",
163
+ "z": "z",
164
+ "{": "{",
165
+ "|": "|",
166
+ "}": "}",
167
+ "~": "~",
168
+ }
169
+ QJ2BJ_TRANSFORM = str.maketrans("".join(QJ2BJ.keys()), "".join(QJ2BJ.values()), "")
170
+
171
+
172
+ # 2013 China National Standard: https://zh.wikipedia.org/wiki/通用规范汉字表, raw resources:
173
+ # https://github.com/mozillazg/pinyin-data/blob/master/kMandarin_8105.txt with 8105 chinese chars in total
174
+ CN_CHARS_COMMON = (
175
+ "一丁七万丈三上下不与丏丐丑专且丕世丘丙业丛东丝丞丢两严丧个丫中丰串临丸丹为主丽举"
176
+ "乂乃久么义之乌乍乎乏乐乒乓乔乖乘乙乜九乞也习乡书乩买乱乳乸乾了予争事二亍于亏云互"
177
+ "亓五井亘亚些亟亡亢交亥亦产亨亩享京亭亮亲亳亵亶亸亹人亿什仁仂仃仄仅仆仇仉今介仍从"
178
+ "仑仓仔仕他仗付仙仝仞仟仡代令以仨仪仫们仰仲仳仵件价任份仿企伈伉伊伋伍伎伏伐休众优"
179
+ "伙会伛伞伟传伢伣伤伥伦伧伪伫伭伯估伲伴伶伸伺似伽伾佁佃但位低住佐佑体何佖佗佘余佚"
180
+ "佛作佝佞佟你佣佤佥佩佬佯佰佳佴佶佸佺佻佼佽佾使侁侂侃侄侈侉例侍侏侑侔侗侘供依侠侣"
181
+ "侥侦侧侨侩侪侬侮侯侴侵侹便促俄俅俊俍俎俏俐俑俗俘俙俚俜保俞俟信俣俦俨俩俪俫俭修俯"
182
+ "俱俳俵俶俸俺俾倌倍倏倒倓倔倕倘候倚倜倞借倡倥倦倧倨倩倪倬倭倮倴债倻值倾偁偃假偈偌"
183
+ "偎偏偓偕做停偡健偬偭偰偲偶偷偻偾偿傀傃傅傈傉傍傒傕傣傥傧储傩催傲傺傻僇僎像僔僖僚"
184
+ "僦僧僬僭僮僰僳僵僻儆儇儋儒儡儦儳儴儿兀允元兄充兆先光克免兑兔兕兖党兜兢入全八公六"
185
+ "兮兰共关兴兵其具典兹养兼兽冀冁内冈冉册再冏冒冔冕冗写军农冠冢冤冥冬冮冯冰冱冲决况"
186
+ "冶冷冻冼冽净凄准凇凉凋凌减凑凓凘凛凝几凡凤凫凭凯凰凳凶凸凹出击凼函凿刀刁刃分切刈"
187
+ "刊刍刎刑划刖列刘则刚创初删判刨利别刬刭刮到刳制刷券刹刺刻刽刿剀剁剂剃剅削剋剌前剐"
188
+ "剑剔剕剖剜剞剟剡剥剧剩剪副割剽剿劁劂劄劈劐劓力劝办功加务劢劣动助努劫劬劭励劲劳劼"
189
+ "劾势勃勇勉勋勍勐勒勔勖勘勚募勠勤勰勺勾勿匀包匆匈匍匏匐匕化北匙匜匝匠匡匣匦匪匮匹"
190
+ "区医匼匾匿十千卅升午卉半华协卑卒卓单卖南博卜卞卟占卡卢卣卤卦卧卫卬卮卯印危即却卵"
191
+ "卷卸卺卿厂厄厅历厉压厌厍厕厖厘厚厝原厢厣厥厦厨厩厮去厾县叁参叆叇又叉及友双反发叔"
192
+ "叕取受变叙叚叛叟叠口古句另叨叩只叫召叭叮可台叱史右叵叶号司叹叻叼叽吁吃各吆合吉吊"
193
+ "同名后吏吐向吒吓吕吖吗君吝吞吟吠吡吣否吧吨吩含听吭吮启吱吲吴吵吸吹吻吼吽吾呀呃呆"
194
+ "呇呈告呋呐呒呓呔呕呖呗员呙呛呜呢呣呤呦周呱呲味呵呶呷呸呻呼命咀咂咄咆咇咉咋和咍咎"
195
+ "咏咐咒咔咕咖咙咚咛咝咡咣咤咥咦咧咨咩咪咫咬咯咱咳咴咸咺咻咽咿哀品哂哃哄哆哇哈哉哌"
196
+ "响哎哏哐哑哒哓哔哕哗哙哚哝哞哟哢哥哦哧哨哩哪哭哮哱哲哳哺哼哽哿唁唆唇唉唏唐唑唔唛"
197
+ "唝唠唢唣唤唧唪唬售唯唰唱唳唵唷唼唾唿啁啃啄商啉啊啐啕啖啜啡啤啥啦啧啪啫啬啭啮啰啴"
198
+ "啵啶啷啸啻啼啾喀喁喂喃善喆喇喈喉喊喋喏喑喔喘喙喜喝喟喤喧喱喳喵喷喹喻喽喾嗄嗅嗉嗌"
199
+ "嗍嗐嗑嗒嗓嗔嗖嗜嗝嗞嗟嗡嗣嗤嗥嗦嗨嗪嗫嗬嗯嗲嗳嗵嗷嗽嗾嘀嘁嘈嘉嘌嘎嘏嘘嘚嘛嘞嘟嘡"
200
+ "嘣嘤嘧嘬嘭嘱嘲嘴嘶嘹嘻嘿噀噂噇噌噍噎噔噗噘噙噜噢噤器噩噪噫噬噱噶噻噼嚄嚅嚆嚎嚏嚓"
201
+ "嚚嚣嚭嚯嚷嚼囊囔囚四回囟因囡团囤囫园困囱围囵囷囹固国图囿圃圄圆圈圉圊圌圐圙圜土圢"
202
+ "圣在圩圪圫圬圭圮圯地圲圳圹场圻圾址坂均坉坊坋坌坍坎坏坐坑坒块坚坛坜坝坞坟坠坡坤坥"
203
+ "坦坨坩坪坫坬坭坯坰坳坷坻坼坽垂垃垄垆垈型垌垍垎垏垒垓垕垙垚垛垞垟垠垡垢垣垤垦垧垩"
204
+ "垫垭垮垯垱垲垴垵垸垺垾垿埂埃埆埇埋埌城埏埒埔埕埗埘埙埚埝域埠埤埪埫埭埯埴埵埸培基"
205
+ "埼埽堂堃堆堇堉堋堌堍堎堐堑堕堙堞堠堡堤堧堨堪堰堲堵堼堽堾塄塅塆塌塍塑塔塘塝塞塥填"
206
+ "塬塱塾墀墁境墅墈墉墐墒墓墕墘墙墚增墟墡墣墦墨墩墼壁壅壑壕壤士壬壮声壳壶壸壹处备复"
207
+ "夏夐夔夕外夙多夜够夤夥大天太夫夬夭央夯失头夷夸夹夺夼奁奂奄奇奈奉奋奎奏契奓奔奕奖"
208
+ "套奘奚奠奡奢奥奭女奴奶奸她好妁如妃妄妆妇妈妊妍妒妓妖妗妘妙妞妣妤妥妧妨妩妪妫妭妮"
209
+ "妯妲妹妻妾姆姈姊始姐姑姒姓委姗姘姚姜姝姞姣姤姥姨姬姮姱姶姹姻姽姿娀威娃娄娅娆娇娈"
210
+ "娉娌娑娓娘娜娟娠娣娥娩娱娲娴娵娶娼婀婆婉婊婌婍婕婘婚婞婠婢婤婧婪婫婳婴婵婶婷婺���"
211
+ "婼婿媂媄媆媒媓媖媚媛媞媪媭媱媲媳媵媸媾嫁嫂嫄嫉嫌嫒嫔嫕嫖嫘嫚嫜嫠嫡嫣嫦嫩嫪嫫嫭嫱"
212
+ "嫽嬉嬖嬗嬛嬥嬬嬴嬷嬿孀孅子孑孓孔孕孖字存孙孚孛孜孝孟孢季孤孥学孩孪孬孰孱孳孵孺孽"
213
+ "宁它宄宅宇守安宋完宏宓宕宗官宙定宛宜宝实宠审客宣室宥宦宧宪宫宬宰害宴宵家宸容宽宾"
214
+ "宿寁寂寄寅密寇富寐寒寓寝寞察寡寤寥寨寮寰寸对寺寻导寿封射将尉尊小少尔尕尖尘尚尜尝"
215
+ "尢尤尥尧尨尪尬就尴尸尹尺尻尼尽尾尿局屁层屃居屈屉届屋屎屏屐屑展屙属屠屡屣履屦屯山"
216
+ "屹屺屼屾屿岁岂岈岊岌岍岐岑岔岖岗岘岙岚岛岜岞岠岢岣岨岩岫岬岭岱岳岵岷岸岽岿峁峂峃"
217
+ "峄峋峒峗峘峙峛峡峣峤峥峦峧峨峪峭峰峱峻峿崀崁崂崃崄崆崇崌崎崒崔崖崚崛崞崟崡崤崦崧"
218
+ "崩崭崮崴崶崽崾崿嵁嵅嵇嵊嵋嵌嵎嵖嵘嵚嵛嵝嵩嵫嵬嵯嵲嵴嶂嶅嶍嶒嶓嶙嶝嶟嶦嶲嶷巅巇巉"
219
+ "巍川州巡巢工左巧巨巩巫差巯己已巳巴巷巽巾币市布帅帆师希帏帐帑帔帕帖帘帙帚帛帜帝帡"
220
+ "带帧帨席帮帱帷常帻帼帽幂幄幅幌幔幕幖幛幞幡幢幪干平年并幸幺幻幼幽广庄庆庇床庋序庐"
221
+ "庑库应底庖店庙庚府庞废庠庤庥度座庭庱庳庵庶康庸庹庼庾廆廉廊廋廑廒廓廖廙廛廨廪延廷"
222
+ "建廿开弁异弃弄弆弇弈弊弋式弑弓引弗弘弛弟张弢弥弦弧弨弩弭弯弱弶弸弹强弼彀归当录彖"
223
+ "彗彘彝彟形彤彦彧彩彪彬彭彰影彳彷役彻彼往征徂径待徇很徉徊律徐徒徕得徘徙徛徜御徨循"
224
+ "徭微徵德徼徽心必忆忉忌忍忏忐忑忒忖志忘忙忝忞忠忡忤忧忪快忭忮忱忳念忸忺忻忽忾忿怀"
225
+ "态怂怃怄怅怆怊怍怎怏怒怔怕怖怙怛怜思怠怡急怦性怨怩怪怫怯怵总怼怿恁恂恃恋恍恐恒恓"
226
+ "恔恕恙恚恝恢恣恤恧恨恩恪恫恬恭息恰恳恶恸恹恺恻恼恽恿悃悄悆悈悉悌悍悒悔悖悚悛悝悟"
227
+ "悠悢患悦您悫悬悭悯悰悱悲悴悸悻悼情惆惇惊惋惎惑惔惕惘惙惚惛惜惝惟惠惦惧惨惩惫惬惭"
228
+ "惮惯惰想惴惶惹惺愀愁愃愆愈愉愍愎意愐愔愕愚感愠愣愤愦愧愫愭愿慆慈慊慌慎慑慕慝慢慥"
229
+ "慧慨慬慭慰慵慷憋憎憔憕憙憧憨憩憬憭憷憺憾懂懈懊懋懑懒懔懦懵懿戆戈戊戋戌戍戎戏成我"
230
+ "戒戕或戗战戚戛戟戡戢戣戤戥截戬戭戮戳戴户戽戾房所扁扂扃扅扆扇扈扉扊手才扎扑扒打扔"
231
+ "托扛扞扣扦执扩扪扫扬扭扮扯扰扳扶批扺扼扽找承技抃抄抉把抑抒抓抔投抖抗折抚抛抟抠抡"
232
+ "抢护报抨披抬抱抵抹抻押抽抿拂拃拄担拆拇拈拉拊拌拍拎拐拒拓拔拖拗拘拙招拜拟拢拣拤拥"
233
+ "拦拧拨择括拭拮拯拱拳拴拶拷拼拽拾拿持挂指挈按挎挑挓挖挚挛挝挞挟挠挡挣挤挥挦挨挪挫"
234
+ "振挲挹挺挽捂捃捅捆捉捋捌捍捎捏捐捕捞损捡换捣捧捩捭据捯捶捷捺捻捽掀掂掇授掉掊掌掎"
235
+ "掏掐排掖掘掞掠探掣接控推掩措掬掭掮掰掳掴掷掸掺掼掾揄揆揉揍描提插揕揖揠握揣揩揪揭"
236
+ "揳援揶揸揽揿搀搁搂搅搋搌搏搐搒搓搔搛搜搞搠搡搦搪搬搭搴携搽摁摄摅摆摇摈摊摏摒摔摘"
237
+ "摛摞摧摩摭摴摸摹摽撂撄撅撇撑撒撕撖撙撞撤撩撬播撮撰撵撷撸撺撼擀擂擅操擎擐擒擘擞擢"
238
+ "擤擦擿攀攉攒攘攥攫攮支收攸改攻攽放政故效敉敌敏救敔敕敖教敛敝敞敢散敦敩敫敬数敲整"
239
+ "敷文斋斌斐斑斓斗料斛斜斝斟斠斡斤斥斧斩斫断斯新斶方於施旁旃旄旅旆旋旌旎族旐旒旖旗"
240
+ "旞无既日旦旧旨早旬旭旮旯旰旱旴旵时旷旸旺旻旿昀昂昃昄昆昇昈昉昊昌明昏昒易昔昕昙昝"
241
+ "星映昡昣昤春昧昨昪昫昭是昱昳昴昵昶昺昼昽显晁晃晅晊晋晌晏晐晒晓晔晕晖晗晙晚晞晟晡"
242
+ "晢晤晦晨晪晫普景晰晱晴晶晷智晾暂暄暅暇暌暑暕暖暗暝暧暨暮暲暴暵暶暹暾暿曈曌曙曛曜"
243
+ "曝曦曩曰曲曳更曷曹曼曾替最月有朋服朏朐朓朔朕朗望朝期朦木未末本札术朱朳朴朵朸机朽"
244
+ "杀杂权杄杆杈杉杌李杏材村杓杕杖杙杜杞束杠条来杧杨杩杪杭杯杰杲杳杵杷杻杼松板极构枅"
245
+ "枇枉枋枍析枕林枘枚果枝枞枢枣枥枧枨枪枫枭枯枰枲枳枵架枷枸枹柁柃柄柈柊柏某柑柒染柔"
246
+ "柖柘柙柚柜柝柞柠柢查柩柬柯柰柱柳柴柷柽柿栀栅标栈栉栊栋栌栎栏栐树栒栓栖栗栝栟校栩"
247
+ "株栲栳栴样核根栻格栽栾桀桁桂桃桄桅框案桉桊桌桎桐桑桓桔桕桠桡桢档桤桥桦桧桨桩桫桯"
248
+ "桲桴桶桷桹梁梃梅梆梌梏梓梗梠梢梣梦梧梨梭梯械梳梴梵梼梽梾梿检棁棂棉棋棍棐棒棓棕棘"
249
+ "棚棠棣棤棨棪棫棬森棰棱棵棹棺棻棼棽椀椁椅椆椋植椎椐椑椒椓椟椠椤椪椭椰椴椸椹椽椿楂"
250
+ "楒楔楗楙楚楝楞楠楣楦楩楪楫楮楯楷楸楹楼概榃榄榅榆榇榈榉榍榑榔榕榖榛榜榧榨榫榭榰���"
251
+ "榴榷榻槁槃槊槌槎槐槔槚槛槜槟槠槭槱槲槽槿樊樗樘樟模樨横樯樱樵樽樾橄橇橐橑橘橙橛橞"
252
+ "橡橥橦橱橹橼檀檄檎檐檑檗檞檠檩檫檬櫆欂欠次欢欣欤欧欲欸欹欺欻款歃歅歆歇歉歌歙止正"
253
+ "此步武歧歪歹死歼殁殂殃殄殆殇殉殊残殍殒殓殖殚殛殡殣殪殳殴段殷殿毁毂毅毋毌母每毐毒"
254
+ "毓比毕毖毗毙毛毡毪毫毯毳毵毹毽氅氆氇氍氏氐民氓气氕氖氘氙氚氛氟氡氢氤氦氧氨氩氪氮"
255
+ "氯氰氲水永氾氿汀汁求汆汇汈汉汊汋汐汔汕汗汛汜汝汞江池污汤汧汨汩汪汫汭汰汲汴汶汹汽"
256
+ "汾沁沂沃沄沅沆沇沈沉沌沏沐沓沔沘沙沚沛沟没沣沤沥沦沧沨沩沪沫沭沮沱河沸油沺治沼沽"
257
+ "沾沿泂泃泄泅泇泉泊泌泐泓泔法泖泗泙泚泛泜泞泠泡波泣泥注泪泫泮泯泰泱泳泵泷泸泺泻泼"
258
+ "泽泾洁洄洇洈洋洌洎洑洒洓洗洘洙洚洛洞洢洣津洧洨洪洫洭洮洱洲洳洴洵洸洹洺活洼洽派洿"
259
+ "流浃浅浆浇浈浉浊测浍济浏浐浑浒浓浔浕浙浚浛浜浞浟浠浡浣浥浦浩浪浬浭浮浯浰浲浴海浸"
260
+ "浼涂涄涅消涉涌涍涎涐涑涓涔涕涘涛涝涞涟涠涡涢涣涤润涧涨涩涪涫涮涯液涴涵涸涿淀淄淅"
261
+ "淆淇淋淌淏淑淖淘淙淜淝淞淟淠淡淤淦淫淬淮淯深淳淴混淹添淼清渊渌渍渎渐渑渔渗渚渝渟"
262
+ "渠渡渣渤渥温渫渭港渰渲渴游渺渼湃湄湉湍湎湑湓湔湖湘湛湜湝湟湣湫湮湲湴湾湿溁溃溅溆"
263
+ "溇溉溍溏源溘溚溜溞溟溠溢溥溦溧溪溯溱溲溴溵溶溷溹溺溻溽滁滂滃滆滇滉滋滍滏滑滓滔滕"
264
+ "滗滘滚滞滟滠满滢滤滥滦滧滨滩滪滫滴滹漂漆漈漉漋漏漓演漕漖漠漤漦漩漪漫漭漯漱漳漴漶"
265
+ "漷漹漻漼漾潆潇潋潍潏潖潘潜潞潟潢潦潩潭潮潲潴潵潸潺潼潽潾澂澄澈澉澌澍澎澛澜澡澥澧"
266
+ "澪澭澳澴澶澹澼澽激濂濉濋濑濒濞濠濡濩濮濯瀌瀍瀑瀔瀚瀛瀣瀱瀵瀹瀼灈灌灏灞火灭灯灰灵"
267
+ "灶灸灼灾灿炀炅炆炉炊炌炎炒炔炕炖炘炙炜炝炟炣炫炬炭炮炯炱炳炷炸点炻炼炽烀烁烂烃烈"
268
+ "烊烔烘烙烛烜烝烟烠烤烦烧烨烩烫烬热烯烶烷烹烺烻烽焆焉焊焌焐焓焕焖焗焘焙焚焜焞焦焯"
269
+ "焰焱然煁煃煅煊煋煌煎煓煜煞煟煤煦照煨煮煲煳煴煸煺煽熄熇熊熏熔熘熙熛熜熟熠熥熨熬熵"
270
+ "熹熻燃燊燋燎燏燔燕燚燠燥燧燮燹爆爇爔爚爝爟爨爪爬爰爱爵父爷爸爹爻爽爿牁牂片版牌牍"
271
+ "牒牖牙牚牛牝牟牡牢牤牥牦牧物牮牯牲牵特牺牻牾牿犀犁犄犇犊犋犍犏犒犟犨犬犯犰犴状犷"
272
+ "犸犹狁狂狃狄狈狉狍狎狐狒狗狙狝狞狠狡狨狩独狭狮狯狰狱狲狳狴狷狸狺狻狼猁猃猄猇猊猎"
273
+ "猕猖猗猛猜猝猞猡猢猥猩猪猫猬献猯猰猱猴猷猹猺猾猿獍獐獒獗獠獬獭獯獴獾玃玄率玉王玎"
274
+ "玑玒玓玕玖玘玙玚玛玞玟玠玡玢玤玥玦玩玫玭玮环现玱玲玳玶玷玹玺玻玼玿珀珂珅珇珈珉珊"
275
+ "珋珌珍珏珐珑珒珕珖珙珛珝珞珠珢珣珥珦珧珩珪珫班珰珲珵珷珸珹珺珽琀球琄琅理琇琈琉琊"
276
+ "琎琏琐琔琚琛琟琡琢琤琥琦琨琪琫琬琭琮琯琰琲琳琴琵琶琼瑀瑁瑂瑃瑄瑅瑆瑑瑓瑔瑕瑖瑗瑙"
277
+ "瑚瑛瑜瑝瑞瑟瑢瑧瑨瑬瑭瑰瑱瑳瑶瑷瑾璀璁璃璆璇璈璋璎璐璒璘璜璞璟璠璥璧璨璩璪璬璮璱"
278
+ "璲璺瓀瓒瓖瓘瓜瓞瓠瓢瓣瓤瓦瓮瓯瓴瓶瓷瓻瓿甄甍甏甑甓甗甘甚甜生甡甥甦用甩甪甫甬甭甯"
279
+ "田由甲申电男甸町画甾畀畅畈畋界畎畏畔畖留畚畛畜畤略畦番畬畯畲畴畸畹畿疁疃疆疍疏疐"
280
+ "疑疔疖疗疙疚疝疟疠疡疢疣疤疥疫疬疭疮疯疰疱疲疳疴疵疸疹疼疽疾痂痃痄病症痈痉痊痍痒"
281
+ "痓痔痕痘痛痞痢痣痤痦痧痨痪痫痰痱痴痹痼痿瘀瘁瘃瘅瘆瘊瘌瘐瘕瘗瘘瘙瘛瘟瘠瘢瘤瘥瘦瘩"
282
+ "瘪瘫瘭瘰瘳瘴瘵瘸瘼瘾瘿癀癃癌癍癔癖癗癜癞癣癫癯癸登白百癿皂的皆皇皈皋皎皑皓皕皖皙"
283
+ "皛皞皤皦皭皮皱皲皴皿盂盅盆盈盉益盍盎盏盐监盒盔盖盗盘盛盟盥盦目盯盱盲直盷相盹盼盾"
284
+ "省眄眇眈眉眊看眍眙眚真眠眢眦眨眩眬眭眯眵眶眷眸眺眼着睁睃睄睇睎睐睑睚睛睡睢督睥睦"
285
+ "睨睫睬睹睽睾睿瞀瞄瞅瞋瞌瞍瞎瞑瞒瞟瞠瞢瞥瞧瞩瞪瞫瞬瞭瞰瞳瞵瞻瞽瞿矍矗矛矜矞矢矣知"
286
+ "矧矩矫矬短矮矰石矶矸矻矼矾矿砀码砂砄砆砉砌砍砑砒研砖砗砘砚砜砝砟砠砣砥砧砫砬砭砮"
287
+ "砰破砵砷砸砹砺砻砼砾础硁硅硇硊硌硍硎硐硒硔硕硖硗硙硚硝硪硫硬硭确硼硿碃碇碈碉碌碍"
288
+ "碎碏碑碓碗碘碚碛碜碟碡碣碥碧碨碰碱碲碳碴碶碹碾磁磅磉磊磋磏磐磔磕磙磜磡磨磬磲磴磷"
289
+ "磹磻礁礅礌礓礞礴礵示礼社祀祁祃祆祇祈祉祊祋祎祏祐祓祕祖祗祚祛祜祝神祟祠祢祥祧票祭"
290
+ "祯祲祷祸祺祼祾禀禁禄禅禊禋福禒禔禘禚禛禤禧禳禹禺离禽禾秀私秃秆秉秋种科秒秕秘租���"
291
+ "秤秦秧秩秫秬秭积称秸移秽秾稀稂稃稆程稌稍税稑稔稗稙稚稞稠稣稳稷稹稻稼稽稿穄穆穑穗"
292
+ "穙穜穟穰穴究穷穸穹空穿窀突窃窄窅窈窊窍窎窑窒窕窖窗窘窜窝窟窠窣窥窦窨窬窭窳窸窿立"
293
+ "竑竖竘站竞竟章竣童竦竫竭端竹竺竽竿笃笄笆笈笊笋笏笑笔笕笙笛笞笠笤笥符笨笪笫第笮笯"
294
+ "笱笳笸笺笼笾筀筅筇等筋筌筏筐筑筒答策筘筚筛筜筝筠筢筤筥筦筮筱筲筵筶筷筹筻筼签简箅"
295
+ "箍箐箓箔箕箖算箜管箢箦箧箨箩箪箫箬箭箱箴箸篁篆篇篌篑篓篙篚篝篡篥篦篪篮篯篱篷篼篾"
296
+ "簃簇簉簋簌簏簕簖簝簟簠簧簪簰簸簿籀籁籍籥米籴类籼籽粉粑粒粕粗粘粜粝粞粟粢粤粥粪粮"
297
+ "粱粲粳粹粼粽精粿糁糅糇糈糊糌糍糒糕糖糗糙糜糟糠糨糯糵系紊素索紧紫累絜絮絷綦綮縠縢"
298
+ "縻繁繄繇纂纛纠纡红纣纤纥约级纨纩纪纫纬纭纮纯纰纱纲纳纴纵纶纷纸纹纺纻纼纽纾线绀绁"
299
+ "绂练组绅细织终绉绊绋绌绍绎经绐绑绒结绔绕绖绗绘给绚绛络绝绞统绠绡绢绣绤绥绦继绨绩"
300
+ "绪绫续绮绯绰绱绲绳维绵绶绷绸绹绺绻综绽绾绿缀缁缂缃缄缅缆缇缈缉缊缌缎缐缑缒缓缔缕"
301
+ "编缗缘缙缚缛缜缝缞缟缠缡缢缣缤缥缦缧缨缩缪缫缬缭缮缯缰缱缲缳缴缵缶缸缺罂罄罅罍罐"
302
+ "网罔罕罗罘罚罟罡罢罨罩罪置罱署罴罶罹罽罾羁羊羌美羑羓羔羕羖羚羝羞羟羡群羧羯羰羱羲"
303
+ "羸羹羼羽羿翀翁翂翃翅翈翊翌翎翔翕翘翙翚翛翟翠翡翥翦翩翮翯翰翱翳翷翻翼翾耀老考耄者"
304
+ "耆耇耋而耍耏耐耑耒耔耕耖耗耘耙耜耠耢耤耥耦耧耨耩耪耰耱耳耵耶耷耸耻耽耿聂聃聆聊聋"
305
+ "职聍聒联聘聚聩聪聱聿肃肄肆肇肉肋肌肓肖肘肚肛肝肟肠股肢肤肥肩肪肫肭肮肯肱育肴肷肸"
306
+ "肺肼肽肾肿胀胁胂胃胄胆胈背胍胎胖胗胙胚胛胜胝胞胠胡胣胤胥胧胨胩胪胫胬胭胯胰胱胲胳"
307
+ "胴胶胸胺胼能脂脆脉脊脍脎脏脐脑脒脓脔脖脘脚脞脟脩脬脯脱脲脶脸脾脿腆腈腊腋腌腐腑腒"
308
+ "腓腔腕腘腙腚腠腥腧腨腩腭腮腯腰腱腴腹腺腻腼腽腾腿膀膂膈膊膏膑膘膙膛膜膝膦膨膳膺膻"
309
+ "臀臂臃臆臊臌臑臜臣臧自臬臭至致臻臼臾舀舁舂舄舅舆舌舍舐舒舔舛舜舞舟舠舢舣舥航舫般"
310
+ "舭舯舰舱舲舳舴舵舶舷舸船舻舾艄艅艇艉艋艎艏艘艚艟艨艮良艰色艳艴艺艽艾艿节芃芄芈芊"
311
+ "芋芍芎芏芑芒芗芘芙芜芝芟芠芡芣芤芥芦芨芩芪芫芬芭芮芯芰花芳芴芷芸芹芼芽芾苁苄苇苈"
312
+ "苉苊苋苌苍苎苏苑苒苓苔苕苗苘苛苜苞苟苠苡苣苤若苦苧苫苯英苴苷苹苻苾茀茁茂范茄茅茆"
313
+ "茈茉茋茌茎茏茑茓茔茕茗茚茛茜茝茧茨茫茬茭茯茱茳茴茵茶茸茹茺茼茽荀荁荃荄荆荇草荏荐"
314
+ "荑荒荓荔荖荙荚荛荜荞荟荠荡荣荤荥荦荧荨荩荪荫荬荭荮药荷荸荻荼荽莅莆莉莎莒莓莘莙莛"
315
+ "莜莝莞莠莨莩莪莫莰莱莲莳莴莶获莸莹莺莼莽莿菀菁菂菅菇菉菊菌菍菏菔菖菘菜菝菟菠菡菥"
316
+ "菩菪菰菱菲菹菼菽萁萃萄萆萋萌萍萎萏萑萘萚萜萝萣萤营萦萧萨萩萱萳萸萹萼落葆葎葑葖著"
317
+ "葙葚葛葜葡董葩葫葬葭葰葱葳葴葵葶葸葺蒂蒄蒇蒈蒉蒋蒌蒎蒐蒗蒙蒜蒟蒡蒨蒯蒱蒲蒴蒸蒹蒺"
318
+ "蒻蒽蒿蓁蓂蓄蓇蓉蓊蓍蓏蓐蓑蓓蓖蓝蓟蓠蓢蓣蓥蓦蓬蓰蓼蓿蔀蔃蔈蔊蔌蔑蔓蔗蔚蔟蔡蔫蔬蔷"
319
+ "蔸蔹蔺蔻蔼蔽蕃蕈蕉蕊蕖蕗蕙蕞蕤蕨蕰蕲蕴蕹蕺蕻蕾薁薄薅薇薏薛薜薢薤薨薪薮薯薰薳薷薸"
320
+ "薹薿藁藉藏藐藓藕藜藟藠藤藦藨藩藻藿蘅蘑蘖蘘蘧蘩蘸蘼虎虏虐虑虒虓虔虚虞虢虤虫虬虮虱"
321
+ "虷虸虹虺虻虼虽虾虿蚀蚁蚂蚄蚆蚊蚋蚌蚍蚓蚕蚜蚝蚣蚤蚧蚨蚩蚪蚬蚯蚰蚱蚲蚴蚶蚺蛀蛃蛄蛆"
322
+ "蛇蛉蛊蛋蛎蛏蛐蛑蛔蛘蛙蛛蛞蛟蛤蛩蛭蛮蛰蛱蛲蛳蛴蛸蛹蛾蜀蜂蜃蜇蜈蜉蜊蜍蜎蜐蜒蜓蜕蜗"
323
+ "蜘蜚蜜蜞蜡蜢蜣蜥蜩蜮蜱蜴蜷蜻蜾蜿蝇蝈蝉蝌蝎蝓蝗蝘蝙蝠蝣蝤蝥蝮蝰蝲蝴蝶蝻蝼蝽蝾螂螃"
324
+ "螅螈螋融螗螟螠螣螨螫螬螭螯螱螳螵螺螽蟀蟆蟊蟋蟏蟑蟒蟛蟠蟥蟪蟫蟮蟹蟾蠃蠊蠋蠓蠕蠖蠡"
325
+ "蠢蠲蠹蠼血衃衄衅行衍衎衒衔街衙衠衡衢衣补表衩衫衬衮衰衲衷衽衾衿袁袂袄袅袆袈袋袍袒"
326
+ "袖袗袜袢袤袪被袭袯袱袷袼裁裂装裆裈裉裎裒裔裕裘裙裛裟裢裣裤裥裨裰裱裳裴裸裹裼裾褂"
327
+ "褊褐褒褓褕褙褚褛褟褡褥褪褫褯褰褴褶襁襄襕襚襜襞襟襦襫襻西要覃覆见观觃规觅视觇览觉"
328
+ "觊觋觌觎觏觐觑角觖觚觜觞觟解觥触觫觭觯觱觳觿言訄訇訚訾詈詟詹誉誊誓謇警譬计订讣认"
329
+ "讥讦讧讨让讪讫训议讯记讱讲讳讴讵讶讷许讹论讻讼讽设访诀证诂诃评诅识诇诈诉诊诋诌词"
330
+ "诎诏诐译诒诓诔试诖诗诘诙诚诛诜话诞诟诠诡询诣诤该详诧诨诩诫诬语诮误诰诱诲诳说诵���"
331
+ "诸诹诺读诼诽课诿谀谁谂调谄谅谆谇谈谊谋谌谍谎谏谐谑谒谓谔谕谖谗谙谚谛谜谝谞谟谠谡"
332
+ "谢谣谤谥谦谧谨谩谪谫谬谭谮谯谰谱谲谳谴谵谶谷谼谿豁豆豇豉豌豕豚象豢豨豪豫豮豳豸豹"
333
+ "豺貂貅貆貉貊貌貔貘贝贞负贡财责贤败账货质贩贪贫贬购贮贯贰贱贲贳贴贵贶贷贸费贺贻贼"
334
+ "贽贾贿赀赁赂赃资赅赆赇赈赉赊赋赌赍赎赏赐赑赒赓赔赕赖赗赘赙赚赛赜赝赞赟赠赡赢赣赤"
335
+ "赦赧赪赫赭走赳赴赵赶起趁趄超越趋趑趔趟趣趯趱足趴趵趸趺趼趾趿跂跃跄跆跋跌跎跏跐跑"
336
+ "跖跗跚跛距跞跟跣跤跨跪跬路跱跳践跶跷跸跹跺跻跽踅踉踊踌踏踒踔踝踞踟踢踣踦踩踪踬踮"
337
+ "踯踱踵踶踹踺踽蹀蹁蹂蹄蹅蹇蹈蹉蹊蹋蹐蹑蹒蹙蹚蹜蹢蹦蹩蹬蹭蹯蹰蹲蹴蹶蹼蹽蹾蹿躁躅躇"
338
+ "躏躐躔躜躞身躬躯躲躺车轧轨轩轪轫转轭轮软轰轱轲轳轴轵轶轷轸轹轺轻轼载轾轿辀辁辂较"
339
+ "辄辅辆辇辈辉辊辋辌辍辎辏辐辑辒输辔辕辖辗辘辙辚辛辜辞辟辣辨辩辫辰辱边辽达辿迁迂迄"
340
+ "迅过迈迎运近迓返迕还这进远违连迟迢迤迥迦迨迩迪迫迭迮述迳迷迸迹迺追退送适逃逄逅逆"
341
+ "选逊逋逍透逐逑递途逖逗通逛逝逞速造逡逢逦逭逮逯逴逵逶逸逻逼逾遁遂遄遆遇遍遏遐遑遒"
342
+ "道遗遘遛遢遣遥遨遭遮遴遵遹遽避邀邂邃邈邋邑邓邕邗邘邙邛邝邠邡邢那邦邨邪邬邮邯邰邱"
343
+ "邲邳邴邵邶邸邹邺邻邽邾邿郁郃郄郅郇郈郊郎郏郐郑郓郗郚郛郜郝郡郢郤郦郧部郪郫郭郯郴"
344
+ "郸都郾郿鄀鄂鄃鄄鄅鄌鄑鄗鄘鄙鄚鄜鄞鄠鄢鄣鄫鄯鄱鄹酂酃酅酆酉酊酋酌配酎酏酐酒酗酚酝"
345
+ "酞酡酢酣酤酥酦酩酪酬酮酯酰酱酲酴酵酶酷酸酹酺酽酾酿醅醇醉醋醌醍醐醑醒醚醛醢醨醪醭"
346
+ "醮醯醴醵醺醾采釉释里重野量釐金釜鉴銎銮鋆鋈錾鍪鎏鏊鏖鐾鑫钆钇针钉钊钋钌钍钎钏钐钒"
347
+ "钓钔钕钖钗钘钙钚钛钜钝钞钟钠钡钢钣钤钥钦钧钨钩钪钫钬钭钮钯钰钱钲钳钴钵钷钹钺钻钼"
348
+ "钽钾钿铀铁铂铃铄铅铆铈铉铊铋铌铍铎铏铐铑铒铕铖铗铘铙铚铛铜铝铞铟铠铡铢铣铤铥铧铨"
349
+ "铩铪铫铬铭铮铯铰铱铲铳铴铵银铷铸铹铺铻铼铽链铿销锁锂锃锄锅锆锇锈锉锊锋锌锍锎锏锐"
350
+ "锑锒锓锔锕锖锗锘错锚锛锜锝锞锟锡锢锣锤锥锦锧锨锩锪锫锬锭键锯锰锱锲锳锴锵锶锷锸锹"
351
+ "锺锻锼锽锾锿镀镁镂镃镄镅镆镇镈镉镊镋镌镍镎镏镐镑镒镓镔镕镖镗镘镚镛镜镝镞镠镡镢镣"
352
+ "镤镥镦镧镨镩镪镫镬镭镮镯镰镱镲镳镴镵镶长门闩闪闫闭问闯闰闱闲闳间闵闶闷闸闹闺闻闼"
353
+ "闽闾闿阀阁阂阃阄阅阆阇阈阉阊阋阌阍阎阏阐阑阒阔阕阖阗阘阙阚阜队阡阪阮阱防阳阴阵阶"
354
+ "阻阼阽阿陀陂附际陆陇陈陉陋陌降陎限陑陔陕陛陞陟陡院除陧陨险陪陬陲陴陵陶陷隃隅隆隈"
355
+ "隋隍随隐隔隗隘隙障隧隩隰隳隶隹隺隼隽难雀雁雄雅集雇雉雊雌雍雎雏雒雕雠雨雩雪雯雱雳"
356
+ "零雷雹雾需霁霄霅霆震霈霉霍霎霏霓霖霜霞霨霪霭霰露霸霹霾青靓靖静靛非靠靡面靥革靬靰"
357
+ "靳靴靶靸靺靼靽靿鞁鞅鞋鞍鞑鞒鞔鞘鞠鞡鞣鞧鞨鞫鞬鞭鞮鞯鞲鞳鞴韂韦韧韨韩韪韫韬韭音韵"
358
+ "韶页顶顷顸项顺须顼顽顾顿颀颁颂颃预颅领颇颈颉颊颋颌颍颎颏颐频颓颔颖颗题颙颚颛颜额"
359
+ "颞颟颠颡颢颤颥颦颧风飏飐飑飒飓飔飕飗飘飙飞食飧飨餍餐餮饔饕饥饧饨饩饪饫饬饭饮饯饰"
360
+ "饱饲饳饴饵饶饷饸饹饺饻饼饽饿馁馃馄馅馆馇馈馉馊馋馌馍馏馐馑馒馓馔馕首馗馘香馝馞馥"
361
+ "馧馨马驭驮驯驰驱驲驳驴驵驶驷驸驹驺驻驼驽驾驿骀骁骂骃骄骅骆骇骈骉骊骋验骍骎骏骐骑"
362
+ "骒骓骕骖骗骘骙骚骛骜骝骞骟骠骡骢骣骤骥骦骧骨骰骱骶骷骸骺骼髀髁髂髃髅髋髌髎髑髓高"
363
+ "髡髢髦髫髭髯髹髻髽鬃鬈鬏鬒鬓鬘鬟鬣鬯鬲鬶鬷鬻鬼魁魂魃魄魅魆魇魈魉魋魍魏魑魔鱼鱽鱾"
364
+ "鱿鲀鲁鲂鲃鲅鲆鲇鲈鲉鲊鲋鲌鲍鲎鲏鲐鲑鲒鲔鲕鲖鲗鲘鲙鲚鲛鲜鲝鲞鲟鲠鲡鲢鲣鲤鲥鲦鲧鲨"
365
+ "鲩鲪鲫鲬鲭鲮鲯鲰鲱鲲鲳鲴鲵鲷鲸鲹鲺鲻鲼鲽鲾鲿鳀鳁鳂鳃鳄鳅鳇鳈鳉鳊鳌鳍鳎鳏鳐鳑鳒鳓"
366
+ "鳔鳕鳖鳗鳘鳙鳚鳛鳜鳝鳞鳟鳠鳡鳢鳣鳤鸟鸠鸡鸢鸣鸤鸥鸦鸧鸨鸩鸪鸫鸬鸭鸮鸯鸰鸱鸲鸳鸵鸶"
367
+ "鸷鸸鸹鸺鸻鸼鸽鸾鸿鹀鹁鹂鹃鹄鹅鹆鹇鹈鹉鹊鹋鹌鹍鹎鹏鹐鹑鹒鹔鹕鹖鹗鹘鹙鹚鹛鹜鹝鹞鹟"
368
+ "鹠鹡鹢鹣鹤鹦鹧鹨鹩鹪鹫鹬鹭鹮鹯鹰鹱鹲鹳鹴鹾鹿麀麂麇麈麋麑麒麓麖麝麟麦麸麹麻麽麾黄"
369
+ "黇黉黍黎黏黑黔默黛黜黝黟黠黡黢黥黧黩黪黯黹黻黼黾鼋鼍鼎鼐鼒鼓鼗鼙鼠鼢鼩鼫鼬鼯鼱鼷"
370
+ "鼹鼻鼽鼾齁齇齉齐齑齿龀龁龂龃龄龅龆龇龈龉龊龋龌龙龚龛龟龠龢鿍鿎鿏㑇㑊㕮㘎㙍㙘㙦���"
371
+ "㛚㛹㟃㠇㠓㤘㥄㧐㧑㧟㫰㬊㬎㬚㭎㭕㮾㰀㳇㳘㳚㴔㵐㶲㸆㸌㺄㻬㽏㿠䁖䂮䃅䃎䅟䌹䎃䎖䏝䏡"
372
+ "䏲䐃䓖䓛䓨䓫䓬䗖䗛䗪䗴䜣䝙䢺䢼䣘䥽䦃䲟䲠䲢䴓䴔䴕䴖䴗䴘䴙䶮𠅤𠙶𠳐𡎚𡐓𣗋𣲗𣲘𣸣𤧛𤩽"
373
+ "𤫉𥔲𥕢𥖨𥻗𦈡𦒍𦙶𦝼𦭜𦰡𧿹𨐈𨙸𨚕𨟠𨭉𨱇𨱏𨱑𨱔𨺙𩽾𩾃𩾌𪟝𪣻𪤗𪨰𪨶𪩘𪾢𫄧𫄨𫄷𫄸𫇭𫌀𫍣𫍯"
374
+ "𫍲𫍽𫐄𫐐𫐓𫑡𫓧𫓯𫓶𫓹𫔍𫔎𫔶𫖮𫖯𫖳𫗧𫗴𫘜𫘝𫘦𫘧𫘨𫘪𫘬𫚕𫚖𫚭𫛭𫞩𫟅𫟦𫟹𫟼𫠆𫠊𫠜𫢸𫫇𫭟"
375
+ "𫭢𫭼𫮃𫰛𫵷𫶇𫷷𫸩𬀩𬀪𬂩𬃊𬇕𬇙𬇹𬉼𬊈𬊤𬌗𬍛𬍡𬍤𬒈𬒔𬒗𬕂𬘓𬘘𬘡𬘩𬘫𬘬𬘭𬘯𬙂𬙊𬙋𬜬𬜯𬞟"
376
+ "𬟁𬟽𬣙𬣞𬣡𬣳𬤇𬤊𬤝𬨂𬨎𬩽𬪩𬬩𬬭𬬮𬬱𬬸𬬹𬬻𬬿𬭁𬭊𬭎𬭚𬭛𬭤𬭩𬭬𬭯𬭳𬭶𬭸𬭼𬮱𬮿𬯀𬯎𬱖𬱟"
377
+ "𬳵𬳶𬳽𬳿𬴂𬴃𬴊𬶋𬶍𬶏𬶐𬶟𬶠𬶨𬶭𬶮𬷕𬸘𬸚𬸣𬸦𬸪𬹼𬺈𬺓"
378
+ )
379
+ CN_CHARS_EXT = "吶诶屌囧飚屄"
380
+
381
+ CN_CHARS = CN_CHARS_COMMON + CN_CHARS_EXT
382
+ IN_CH_CHARS = {c: True for c in CN_CHARS}
383
+
384
+ EN_CHARS = string.ascii_letters + string.digits
385
+ IN_EN_CHARS = {c: True for c in EN_CHARS}
386
+
387
+ VALID_CHARS = CN_CHARS + EN_CHARS + " "
388
+ IN_VALID_CHARS = {c: True for c in VALID_CHARS}
389
+
390
+
391
+ # ================================================================================ #
392
+ # basic class
393
+ # ================================================================================ #
394
+ class ChineseChar(object):
395
+ """
396
+ 中文字符
397
+ 每个字符对应简体和繁体,
398
+ e.g. 简体 = '负', 繁体 = '負'
399
+ 转换时可转换为简体或繁体
400
+ """
401
+
402
+ def __init__(self, simplified, traditional):
403
+ self.simplified = simplified
404
+ self.traditional = traditional
405
+ # self.__repr__ = self.__str__
406
+
407
+ def __str__(self):
408
+ return self.simplified or self.traditional or None
409
+
410
+ def __repr__(self):
411
+ return self.__str__()
412
+
413
+
414
+ class ChineseNumberUnit(ChineseChar):
415
+ """
416
+ 中文数字/数位字符
417
+ 每个字符除繁简体外还有一个额外的大写字符
418
+ e.g. '陆' 和 '陸'
419
+ """
420
+
421
+ def __init__(self, power, simplified, traditional, big_s, big_t):
422
+ super(ChineseNumberUnit, self).__init__(simplified, traditional)
423
+ self.power = power
424
+ self.big_s = big_s
425
+ self.big_t = big_t
426
+
427
+ def __str__(self):
428
+ return "10^{}".format(self.power)
429
+
430
+ @classmethod
431
+ def create(cls, index, value, numbering_type=NUMBERING_TYPES[1], small_unit=False):
432
+ if small_unit:
433
+ return ChineseNumberUnit(
434
+ power=index + 1, simplified=value[0], traditional=value[1], big_s=value[1], big_t=value[1]
435
+ )
436
+ elif numbering_type == NUMBERING_TYPES[0]:
437
+ return ChineseNumberUnit(
438
+ power=index + 8, simplified=value[0], traditional=value[1], big_s=value[0], big_t=value[1]
439
+ )
440
+ elif numbering_type == NUMBERING_TYPES[1]:
441
+ return ChineseNumberUnit(
442
+ power=(index + 2) * 4, simplified=value[0], traditional=value[1], big_s=value[0], big_t=value[1]
443
+ )
444
+ elif numbering_type == NUMBERING_TYPES[2]:
445
+ return ChineseNumberUnit(
446
+ power=pow(2, index + 3), simplified=value[0], traditional=value[1], big_s=value[0], big_t=value[1]
447
+ )
448
+ else:
449
+ raise ValueError("Counting type should be in {0} ({1} provided).".format(NUMBERING_TYPES, numbering_type))
450
+
451
+
452
+ class ChineseNumberDigit(ChineseChar):
453
+ """
454
+ 中文数字字符
455
+ """
456
+
457
+ def __init__(self, value, simplified, traditional, big_s, big_t, alt_s=None, alt_t=None):
458
+ super(ChineseNumberDigit, self).__init__(simplified, traditional)
459
+ self.value = value
460
+ self.big_s = big_s
461
+ self.big_t = big_t
462
+ self.alt_s = alt_s
463
+ self.alt_t = alt_t
464
+
465
+ def __str__(self):
466
+ return str(self.value)
467
+
468
+ @classmethod
469
+ def create(cls, i, v):
470
+ return ChineseNumberDigit(i, v[0], v[1], v[2], v[3])
471
+
472
+
473
+ class ChineseMath(ChineseChar):
474
+ """
475
+ 中文数位字符
476
+ """
477
+
478
+ def __init__(self, simplified, traditional, symbol, expression=None):
479
+ super(ChineseMath, self).__init__(simplified, traditional)
480
+ self.symbol = symbol
481
+ self.expression = expression
482
+ self.big_s = simplified
483
+ self.big_t = traditional
484
+
485
+
486
+ CC, CNU, CND, CM = ChineseChar, ChineseNumberUnit, ChineseNumberDigit, ChineseMath
487
+
488
+
489
+ class NumberSystem(object):
490
+ """
491
+ 中文数字系统
492
+ """
493
+
494
+ pass
495
+
496
+
497
+ class MathSymbol(object):
498
+ """
499
+ 用于中文数字系统的数学符号 (繁/简体), e.g.
500
+ positive = ['正', '正']
501
+ negative = ['负', '負']
502
+ point = ['点', '點']
503
+ """
504
+
505
+ def __init__(self, positive, negative, point):
506
+ self.positive = positive
507
+ self.negative = negative
508
+ self.point = point
509
+
510
+ def __iter__(self):
511
+ for v in self.__dict__.values():
512
+ yield v
513
+
514
+
515
+ # class OtherSymbol(object):
516
+ # """
517
+ # 其他符号
518
+ # """
519
+ #
520
+ # def __init__(self, sil):
521
+ # self.sil = sil
522
+ #
523
+ # def __iter__(self):
524
+ # for v in self.__dict__.values():
525
+ # yield v
526
+
527
+
528
+ # ================================================================================ #
529
+ # basic utils
530
+ # ================================================================================ #
531
+ def create_system(numbering_type=NUMBERING_TYPES[1]):
532
+ """
533
+ 根据数字系统类型返回创建相应的数字系统,默认为 mid
534
+ NUMBERING_TYPES = ['low', 'mid', 'high']: 中文数字系统类型
535
+ low: '兆' = '亿' * '十' = $10^{9}$, '京' = '兆' * '十', etc.
536
+ mid: '兆' = '亿' * '万' = $10^{12}$, '京' = '兆' * '万', etc.
537
+ high: '兆' = '亿' * '亿' = $10^{16}$, '京' = '兆' * '兆', etc.
538
+ 返回对应的数字系统
539
+ """
540
+
541
+ # chinese number units of '亿' and larger
542
+ all_larger_units = zip(LARGER_CHINESE_NUMERING_UNITS_SIMPLIFIED, LARGER_CHINESE_NUMERING_UNITS_TRADITIONAL)
543
+ larger_units = [CNU.create(i, v, numbering_type, False) for i, v in enumerate(all_larger_units)]
544
+ # chinese number units of '十, 百, 千, 万'
545
+ all_smaller_units = zip(SMALLER_CHINESE_NUMERING_UNITS_SIMPLIFIED, SMALLER_CHINESE_NUMERING_UNITS_TRADITIONAL)
546
+ smaller_units = [CNU.create(i, v, small_unit=True) for i, v in enumerate(all_smaller_units)]
547
+ # digis
548
+ chinese_digis = zip(CHINESE_DIGIS, CHINESE_DIGIS, BIG_CHINESE_DIGIS_SIMPLIFIED, BIG_CHINESE_DIGIS_TRADITIONAL)
549
+ digits = [CND.create(i, v) for i, v in enumerate(chinese_digis)]
550
+ digits[0].alt_s, digits[0].alt_t = ZERO_ALT, ZERO_ALT
551
+ digits[1].alt_s, digits[1].alt_t = ONE_ALT, ONE_ALT
552
+ digits[2].alt_s, digits[2].alt_t = TWO_ALTS[0], TWO_ALTS[1]
553
+
554
+ # symbols
555
+ positive_cn = CM(POSITIVE[0], POSITIVE[1], "+", lambda x: x)
556
+ negative_cn = CM(NEGATIVE[0], NEGATIVE[1], "-", lambda x: -x)
557
+ point_cn = CM(POINT[0], POINT[1], ".", lambda x, y: float(str(x) + "." + str(y)))
558
+ # sil_cn = CM(SIL[0], SIL[1], '-', lambda x, y: float(str(x) + '-' + str(y)))
559
+ system = NumberSystem()
560
+ system.units = smaller_units + larger_units
561
+ system.digits = digits
562
+ system.math = MathSymbol(positive_cn, negative_cn, point_cn)
563
+ # system.symbols = OtherSymbol(sil_cn)
564
+ return system
565
+
566
+
567
+ def chn2num(chinese_string, numbering_type=NUMBERING_TYPES[1]):
568
+ def get_symbol(char, system):
569
+ for u in system.units:
570
+ if char in [u.traditional, u.simplified, u.big_s, u.big_t]:
571
+ return u
572
+ for d in system.digits:
573
+ if char in [d.traditional, d.simplified, d.big_s, d.big_t, d.alt_s, d.alt_t]:
574
+ return d
575
+ for m in system.math:
576
+ if char in [m.traditional, m.simplified]:
577
+ return m
578
+
579
+ def string2symbols(chinese_string, system):
580
+ int_string, dec_string = chinese_string, ""
581
+ for p in [system.math.point.simplified, system.math.point.traditional]:
582
+ if p in chinese_string:
583
+ int_string, dec_string = chinese_string.split(p)
584
+ break
585
+ return [get_symbol(c, system) for c in int_string], [get_symbol(c, system) for c in dec_string]
586
+
587
+ def correct_symbols(integer_symbols, system):
588
+ """
589
+ 一百八 to 一百八十
590
+ 一亿一千三百万 to 一亿 一千万 三百万
591
+ """
592
+
593
+ if integer_symbols and isinstance(integer_symbols[0], CNU):
594
+ if integer_symbols[0].power == 1:
595
+ integer_symbols = [system.digits[1]] + integer_symbols
596
+
597
+ if len(integer_symbols) > 1:
598
+ if isinstance(integer_symbols[-1], CND) and isinstance(integer_symbols[-2], CNU):
599
+ integer_symbols.append(CNU(integer_symbols[-2].power - 1, None, None, None, None))
600
+
601
+ result = []
602
+ unit_count = 0
603
+ for s in integer_symbols:
604
+ if isinstance(s, CND):
605
+ result.append(s)
606
+ unit_count = 0
607
+ elif isinstance(s, CNU):
608
+ current_unit = CNU(s.power, None, None, None, None)
609
+ unit_count += 1
610
+
611
+ if unit_count == 1:
612
+ result.append(current_unit)
613
+ elif unit_count > 1:
614
+ for i in range(len(result)):
615
+ if isinstance(result[-i - 1], CNU) and result[-i - 1].power < current_unit.power:
616
+ result[-i - 1] = CNU(result[-i - 1].power + current_unit.power, None, None, None, None)
617
+ return result
618
+
619
+ def compute_value(integer_symbols):
620
+ """
621
+ Compute the value.
622
+ When current unit is larger than previous unit, current unit * all previous units will be used as all previous units.
623
+ e.g. '两千万' = 2000 * 10000 not 2000 + 10000
624
+ """
625
+ value = [0]
626
+ last_power = 0
627
+ for s in integer_symbols:
628
+ if isinstance(s, CND):
629
+ value[-1] = s.value
630
+ elif isinstance(s, CNU):
631
+ value[-1] *= pow(10, s.power)
632
+ if s.power > last_power:
633
+ value[:-1] = list(map(lambda v: v * pow(10, s.power), value[:-1]))
634
+ last_power = s.power
635
+ value.append(0)
636
+ return sum(value)
637
+
638
+ system = create_system(numbering_type)
639
+ int_part, dec_part = string2symbols(chinese_string, system)
640
+ int_part = correct_symbols(int_part, system)
641
+ int_str = str(compute_value(int_part))
642
+ dec_str = "".join([str(d.value) for d in dec_part])
643
+ if dec_part:
644
+ return "{0}.{1}".format(int_str, dec_str)
645
+ else:
646
+ return int_str
647
+
648
+
649
+ def num2chn(
650
+ number_string,
651
+ numbering_type=NUMBERING_TYPES[1],
652
+ big=False,
653
+ traditional=False,
654
+ alt_zero=False,
655
+ alt_one=False,
656
+ alt_two=True,
657
+ use_zeros=True,
658
+ use_units=True,
659
+ ):
660
+ def get_value(value_string, use_zeros=True):
661
+ striped_string = value_string.lstrip("0")
662
+
663
+ # record nothing if all zeros
664
+ if not striped_string:
665
+ return []
666
+
667
+ # record one digits
668
+ elif len(striped_string) == 1:
669
+ if use_zeros and len(value_string) != len(striped_string):
670
+ return [system.digits[0], system.digits[int(striped_string)]]
671
+ else:
672
+ return [system.digits[int(striped_string)]]
673
+
674
+ # recursively record multiple digits
675
+ else:
676
+ result_unit = next(u for u in reversed(system.units) if u.power < len(striped_string))
677
+ result_string = value_string[: -result_unit.power]
678
+ return get_value(result_string) + [result_unit] + get_value(striped_string[-result_unit.power :])
679
+
680
+ system = create_system(numbering_type)
681
+
682
+ int_dec = number_string.split(".")
683
+ if len(int_dec) == 1:
684
+ int_string = int_dec[0]
685
+ dec_string = ""
686
+ elif len(int_dec) == 2:
687
+ int_string = int_dec[0]
688
+ dec_string = int_dec[1]
689
+ else:
690
+ raise ValueError("invalid input num string with more than one dot: {}".format(number_string))
691
+
692
+ if use_units and len(int_string) > 1:
693
+ result_symbols = get_value(int_string)
694
+ else:
695
+ result_symbols = [system.digits[int(c)] for c in int_string]
696
+ dec_symbols = [system.digits[int(c)] for c in dec_string]
697
+ if dec_string:
698
+ result_symbols += [system.math.point] + dec_symbols
699
+
700
+ if alt_two:
701
+ liang = CND(2, system.digits[2].alt_s, system.digits[2].alt_t, system.digits[2].big_s, system.digits[2].big_t)
702
+ for i, v in enumerate(result_symbols):
703
+ if isinstance(v, CND) and v.value == 2:
704
+ next_symbol = result_symbols[i + 1] if i < len(result_symbols) - 1 else None
705
+ previous_symbol = result_symbols[i - 1] if i > 0 else None
706
+ if isinstance(next_symbol, CNU) and isinstance(previous_symbol, (CNU, type(None))):
707
+ if next_symbol.power != 1 and ((previous_symbol is None) or (previous_symbol.power != 1)):
708
+ result_symbols[i] = liang
709
+
710
+ # if big is True, '两' will not be used and `alt_two` has no impact on output
711
+ if big:
712
+ attr_name = "big_"
713
+ if traditional:
714
+ attr_name += "t"
715
+ else:
716
+ attr_name += "s"
717
+ else:
718
+ if traditional:
719
+ attr_name = "traditional"
720
+ else:
721
+ attr_name = "simplified"
722
+
723
+ result = "".join([getattr(s, attr_name) for s in result_symbols])
724
+
725
+ # if not use_zeros:
726
+ # result = result.strip(getattr(system.digits[0], attr_name))
727
+
728
+ if alt_zero:
729
+ result = result.replace(getattr(system.digits[0], attr_name), system.digits[0].alt_s)
730
+
731
+ if alt_one:
732
+ result = result.replace(getattr(system.digits[1], attr_name), system.digits[1].alt_s)
733
+
734
+ for i, p in enumerate(POINT):
735
+ if result.startswith(p):
736
+ return CHINESE_DIGIS[0] + result
737
+
738
+ # ^10, 11, .., 19
739
+ if (
740
+ len(result) >= 2
741
+ and result[1] in [SMALLER_CHINESE_NUMERING_UNITS_SIMPLIFIED[0], SMALLER_CHINESE_NUMERING_UNITS_TRADITIONAL[0]]
742
+ and result[0] in [CHINESE_DIGIS[1], BIG_CHINESE_DIGIS_SIMPLIFIED[1], BIG_CHINESE_DIGIS_TRADITIONAL[1]]
743
+ ):
744
+ result = result[1:]
745
+
746
+ return result
747
+
748
+
749
+ # ================================================================================ #
750
+ # different types of rewriters
751
+ # ================================================================================ #
752
+ class Cardinal:
753
+ """
754
+ CARDINAL类
755
+ """
756
+
757
+ def __init__(self, cardinal=None, chntext=None):
758
+ self.cardinal = cardinal
759
+ self.chntext = chntext
760
+
761
+ def chntext2cardinal(self):
762
+ return chn2num(self.chntext)
763
+
764
+ def cardinal2chntext(self):
765
+ return num2chn(self.cardinal)
766
+
767
+
768
+ class Digit:
769
+ """
770
+ DIGIT类
771
+ """
772
+
773
+ def __init__(self, digit=None, chntext=None):
774
+ self.digit = digit
775
+ self.chntext = chntext
776
+
777
+ # def chntext2digit(self):
778
+ # return chn2num(self.chntext)
779
+
780
+ def digit2chntext(self):
781
+ return num2chn(self.digit, alt_two=False, use_units=False)
782
+
783
+
784
+ class TelePhone:
785
+ """
786
+ TELEPHONE类
787
+ """
788
+
789
+ def __init__(self, telephone=None, raw_chntext=None, chntext=None):
790
+ self.telephone = telephone
791
+ self.raw_chntext = raw_chntext
792
+ self.chntext = chntext
793
+
794
+ # def chntext2telephone(self):
795
+ # sil_parts = self.raw_chntext.split('<SIL>')
796
+ # self.telephone = '-'.join([
797
+ # str(chn2num(p)) for p in sil_parts
798
+ # ])
799
+ # return self.telephone
800
+
801
+ def telephone2chntext(self, fixed=False):
802
+ if fixed:
803
+ sil_parts = self.telephone.split("-")
804
+ self.raw_chntext = "<SIL>".join([num2chn(part, alt_two=False, use_units=False) for part in sil_parts])
805
+ self.chntext = self.raw_chntext.replace("<SIL>", "")
806
+ else:
807
+ sp_parts = self.telephone.strip("+").split()
808
+ self.raw_chntext = "<SP>".join([num2chn(part, alt_two=False, use_units=False) for part in sp_parts])
809
+ self.chntext = self.raw_chntext.replace("<SP>", "")
810
+ return self.chntext
811
+
812
+
813
+ class Fraction:
814
+ """
815
+ FRACTION类
816
+ """
817
+
818
+ def __init__(self, fraction=None, chntext=None):
819
+ self.fraction = fraction
820
+ self.chntext = chntext
821
+
822
+ def chntext2fraction(self):
823
+ denominator, numerator = self.chntext.split("分之")
824
+ return chn2num(numerator) + "/" + chn2num(denominator)
825
+
826
+ def fraction2chntext(self):
827
+ numerator, denominator = self.fraction.split("/")
828
+ return num2chn(denominator) + "分之" + num2chn(numerator)
829
+
830
+
831
+ class Date:
832
+ """
833
+ DATE类
834
+ """
835
+
836
+ def __init__(self, date=None, chntext=None):
837
+ self.date = date
838
+ self.chntext = chntext
839
+
840
+ # def chntext2date(self):
841
+ # chntext = self.chntext
842
+ # try:
843
+ # year, other = chntext.strip().split('年', maxsplit=1)
844
+ # year = Digit(chntext=year).digit2chntext() + '年'
845
+ # except ValueError:
846
+ # other = chntext
847
+ # year = ''
848
+ # if other:
849
+ # try:
850
+ # month, day = other.strip().split('月', maxsplit=1)
851
+ # month = Cardinal(chntext=month).chntext2cardinal() + '月'
852
+ # except ValueError:
853
+ # day = chntext
854
+ # month = ''
855
+ # if day:
856
+ # day = Cardinal(chntext=day[:-1]).chntext2cardinal() + day[-1]
857
+ # else:
858
+ # month = ''
859
+ # day = ''
860
+ # date = year + month + day
861
+ # self.date = date
862
+ # return self.date
863
+
864
+ def date2chntext(self):
865
+ date = self.date
866
+ try:
867
+ year, other = date.strip().split("年", 1)
868
+ year = Digit(digit=year).digit2chntext() + "年"
869
+ except ValueError:
870
+ other = date
871
+ year = ""
872
+ if other:
873
+ try:
874
+ month, day = other.strip().split("月", 1)
875
+ month = Cardinal(cardinal=month).cardinal2chntext() + "月"
876
+ except ValueError:
877
+ day = date
878
+ month = ""
879
+ if day:
880
+ day = Cardinal(cardinal=day[:-1]).cardinal2chntext() + day[-1]
881
+ else:
882
+ month = ""
883
+ day = ""
884
+ chntext = year + month + day
885
+ self.chntext = chntext
886
+ return self.chntext
887
+
888
+
889
+ class Money:
890
+ """
891
+ MONEY类
892
+ """
893
+
894
+ def __init__(self, money=None, chntext=None):
895
+ self.money = money
896
+ self.chntext = chntext
897
+
898
+ # def chntext2money(self):
899
+ # return self.money
900
+
901
+ def money2chntext(self):
902
+ money = self.money
903
+ pattern = re.compile(r"(\d+(\.\d+)?)")
904
+ matchers = pattern.findall(money)
905
+ if matchers:
906
+ for matcher in matchers:
907
+ money = money.replace(matcher[0], Cardinal(cardinal=matcher[0]).cardinal2chntext())
908
+ self.chntext = money
909
+ return self.chntext
910
+
911
+
912
+ class Percentage:
913
+ """
914
+ PERCENTAGE类
915
+ """
916
+
917
+ def __init__(self, percentage=None, chntext=None):
918
+ self.percentage = percentage
919
+ self.chntext = chntext
920
+
921
+ def chntext2percentage(self):
922
+ return chn2num(self.chntext.strip().strip("百分之")) + "%"
923
+
924
+ def percentage2chntext(self):
925
+ return "百分之" + num2chn(self.percentage.strip().strip("%"))
926
+
927
+
928
+ def normalize_nsw(raw_text):
929
+ text = "^" + raw_text + "$"
930
+
931
+ # 规范化日期
932
+ pattern = re.compile(r"\D+((([089]\d|(19|20)\d{2})年)?(\d{1,2}月(\d{1,2}[日号])?)?)")
933
+ matchers = pattern.findall(text)
934
+ if matchers:
935
+ # print('date')
936
+ for matcher in matchers:
937
+ text = text.replace(matcher[0], Date(date=matcher[0]).date2chntext(), 1)
938
+
939
+ # 规范化金钱
940
+ pattern = re.compile(r"\D+((\d+(\.\d+)?)[多余几]?" + CURRENCY_UNITS + r"(\d" + CURRENCY_UNITS + r"?)?)")
941
+ matchers = pattern.findall(text)
942
+ if matchers:
943
+ # print('money')
944
+ for matcher in matchers:
945
+ text = text.replace(matcher[0], Money(money=matcher[0]).money2chntext(), 1)
946
+
947
+ # 规范化固话/手机号码
948
+ # 手机
949
+ # http://www.jihaoba.com/news/show/13680
950
+ # 移动:139、138、137、136、135、134、159、158、157、150、151、152、188、187、182、183、184、178、198
951
+ # 联通:130、131、132、156、155、186、185、176
952
+ # 电信:133、153、189、180、181、177
953
+ pattern = re.compile(r"\D((\+?86 ?)?1([38]\d|5[0-35-9]|7[678]|9[89])\d{8})\D")
954
+ matchers = pattern.findall(text)
955
+ if matchers:
956
+ # print('telephone')
957
+ for matcher in matchers:
958
+ text = text.replace(matcher[0], TelePhone(telephone=matcher[0]).telephone2chntext(), 1)
959
+ # 固话
960
+ pattern = re.compile(r"\D((0(10|2[1-3]|[3-9]\d{2})-?)?[1-9]\d{6,7})\D")
961
+ matchers = pattern.findall(text)
962
+ if matchers:
963
+ # print('fixed telephone')
964
+ for matcher in matchers:
965
+ text = text.replace(matcher[0], TelePhone(telephone=matcher[0]).telephone2chntext(fixed=True), 1)
966
+
967
+ # 规范化分数
968
+ pattern = re.compile(r"(\d+/\d+)")
969
+ matchers = pattern.findall(text)
970
+ if matchers:
971
+ # print('fraction')
972
+ for matcher in matchers:
973
+ text = text.replace(matcher, Fraction(fraction=matcher).fraction2chntext(), 1)
974
+
975
+ # 规范化百分数
976
+ text = text.replace("%", "%")
977
+ pattern = re.compile(r"(\d+(\.\d+)?%)")
978
+ matchers = pattern.findall(text)
979
+ if matchers:
980
+ # print('percentage')
981
+ for matcher in matchers:
982
+ text = text.replace(matcher[0], Percentage(percentage=matcher[0]).percentage2chntext(), 1)
983
+
984
+ # 规范化纯数+量词
985
+ pattern = re.compile(r"(\d+(\.\d+)?)[多余几]?" + COM_QUANTIFIERS)
986
+ matchers = pattern.findall(text)
987
+ if matchers:
988
+ # print('cardinal+quantifier')
989
+ for matcher in matchers:
990
+ text = text.replace(matcher[0], Cardinal(cardinal=matcher[0]).cardinal2chntext(), 1)
991
+
992
+ # 规范化数字编号
993
+ pattern = re.compile(r"(\d{4,32})")
994
+ matchers = pattern.findall(text)
995
+ if matchers:
996
+ # print('digit')
997
+ for matcher in matchers:
998
+ text = text.replace(matcher, Digit(digit=matcher).digit2chntext(), 1)
999
+
1000
+ # 规范化纯数
1001
+ pattern = re.compile(r"(\d+(\.\d+)?)")
1002
+ matchers = pattern.findall(text)
1003
+ if matchers:
1004
+ # print('cardinal')
1005
+ for matcher in matchers:
1006
+ text = text.replace(matcher[0], Cardinal(cardinal=matcher[0]).cardinal2chntext(), 1)
1007
+
1008
+ # restore P2P, O2O, B2C, B2B etc
1009
+ pattern = re.compile(r"(([a-zA-Z]+)二([a-zA-Z]+))")
1010
+ matchers = pattern.findall(text)
1011
+ if matchers:
1012
+ # print('particular')
1013
+ for matcher in matchers:
1014
+ text = text.replace(matcher[0], matcher[1] + "2" + matcher[2], 1)
1015
+
1016
+ return text.lstrip("^").rstrip("$")
1017
+
1018
+
1019
+ def remove_erhua(text):
1020
+ """
1021
+ 去除儿化音词中的儿:
1022
+ 他女儿在那边儿 -> 他女儿在那边
1023
+ """
1024
+
1025
+ new_str = ""
1026
+ while re.search("儿", text):
1027
+ a = re.search("儿", text).span()
1028
+ remove_er_flag = 0
1029
+
1030
+ if ER_WHITELIST_PATTERN.search(text):
1031
+ b = ER_WHITELIST_PATTERN.search(text).span()
1032
+ if b[0] <= a[0]:
1033
+ remove_er_flag = 1
1034
+
1035
+ if remove_er_flag == 0:
1036
+ new_str = new_str + text[0 : a[0]]
1037
+ text = text[a[1] :]
1038
+ else:
1039
+ new_str = new_str + text[0 : b[1]]
1040
+ text = text[b[1] :]
1041
+
1042
+ text = new_str + text
1043
+ return text
1044
+
1045
+
1046
+ def remove_space(text):
1047
+ tokens = text.split()
1048
+ new = []
1049
+ for k, t in enumerate(tokens):
1050
+ if k != 0:
1051
+ if IN_EN_CHARS.get(tokens[k - 1][-1]) and IN_EN_CHARS.get(t[0]):
1052
+ new.append(" ")
1053
+ new.append(t)
1054
+ return "".join(new)
1055
+
1056
+
1057
+ class TextNorm:
1058
+ def __init__(
1059
+ self,
1060
+ to_banjiao: bool = False,
1061
+ to_upper: bool = False,
1062
+ to_lower: bool = False,
1063
+ remove_fillers: bool = False,
1064
+ remove_erhua: bool = False,
1065
+ check_chars: bool = False,
1066
+ remove_space: bool = False,
1067
+ cc_mode: str = "",
1068
+ ):
1069
+ self.to_banjiao = to_banjiao
1070
+ self.to_upper = to_upper
1071
+ self.to_lower = to_lower
1072
+ self.remove_fillers = remove_fillers
1073
+ self.remove_erhua = remove_erhua
1074
+ self.check_chars = check_chars
1075
+ self.remove_space = remove_space
1076
+
1077
+ self.cc = None
1078
+ if cc_mode:
1079
+ from opencc import OpenCC # Open Chinese Convert: pip install opencc
1080
+
1081
+ self.cc = OpenCC(cc_mode)
1082
+
1083
+ def __call__(self, text):
1084
+ if self.cc:
1085
+ text = self.cc.convert(text)
1086
+
1087
+ if self.to_banjiao:
1088
+ text = text.translate(QJ2BJ_TRANSFORM)
1089
+
1090
+ if self.to_upper:
1091
+ text = text.upper()
1092
+
1093
+ if self.to_lower:
1094
+ text = text.lower()
1095
+
1096
+ if self.remove_fillers:
1097
+ for c in FILLER_CHARS:
1098
+ text = text.replace(c, "")
1099
+
1100
+ if self.remove_erhua:
1101
+ text = remove_erhua(text)
1102
+
1103
+ text = normalize_nsw(text)
1104
+
1105
+ text = text.translate(PUNCS_TRANSFORM)
1106
+
1107
+ if self.check_chars:
1108
+ for c in text:
1109
+ if not IN_VALID_CHARS.get(c):
1110
+ print(f"WARNING: illegal char {c} in: {text}", file=sys.stderr)
1111
+ return ""
1112
+
1113
+ if self.remove_space:
1114
+ text = remove_space(text)
1115
+
1116
+ return text
1117
+
1118
+
1119
+ if __name__ == "__main__":
1120
+ p = argparse.ArgumentParser()
1121
+
1122
+ # normalizer options
1123
+ p.add_argument("--to_banjiao", action="store_true", help="convert quanjiao chars to banjiao")
1124
+ p.add_argument("--to_upper", action="store_true", help="convert to upper case")
1125
+ p.add_argument("--to_lower", action="store_true", help="convert to lower case")
1126
+ p.add_argument("--remove_fillers", action="store_true", help='remove filler chars such as "呃, 啊"')
1127
+ p.add_argument("--remove_erhua", action="store_true", help='remove erhua chars such as "他女儿在那边儿 -> 他女儿在那边"')
1128
+ p.add_argument("--check_chars", action="store_true", help="skip sentences containing illegal chars")
1129
+ p.add_argument("--remove_space", action="store_true", help="remove whitespace")
1130
+ p.add_argument(
1131
+ "--cc_mode", choices=["", "t2s", "s2t"], default="", help="convert between traditional to simplified"
1132
+ )
1133
+
1134
+ # I/O options
1135
+ p.add_argument("--log_interval", type=int, default=10000, help="log interval in number of processed lines")
1136
+ p.add_argument("--has_key", action="store_true", help="will be deprecated, set --format ark instead")
1137
+ p.add_argument("--format", type=str, choices=["txt", "ark", "tsv"], default="txt", help="input format")
1138
+ p.add_argument("ifile", help="input filename, assume utf-8 encoding")
1139
+ p.add_argument("ofile", help="output filename")
1140
+
1141
+ args = p.parse_args()
1142
+
1143
+ if args.has_key:
1144
+ args.format = "ark"
1145
+
1146
+ normalizer = TextNorm(
1147
+ to_banjiao=args.to_banjiao,
1148
+ to_upper=args.to_upper,
1149
+ to_lower=args.to_lower,
1150
+ remove_fillers=args.remove_fillers,
1151
+ remove_erhua=args.remove_erhua,
1152
+ check_chars=args.check_chars,
1153
+ remove_space=args.remove_space,
1154
+ cc_mode=args.cc_mode,
1155
+ )
1156
+
1157
+ normalizer = TextNorm(
1158
+ to_banjiao=args.to_banjiao,
1159
+ to_upper=args.to_upper,
1160
+ to_lower=args.to_lower,
1161
+ remove_fillers=args.remove_fillers,
1162
+ remove_erhua=args.remove_erhua,
1163
+ check_chars=args.check_chars,
1164
+ remove_space=args.remove_space,
1165
+ cc_mode=args.cc_mode,
1166
+ )
1167
+
1168
+ ndone = 0
1169
+ with open(args.ifile, "r", encoding="utf8") as istream, open(args.ofile, "w+", encoding="utf8") as ostream:
1170
+ if args.format == "tsv":
1171
+ reader = csv.DictReader(istream, delimiter="\t")
1172
+ assert "TEXT" in reader.fieldnames
1173
+ print("\t".join(reader.fieldnames), file=ostream)
1174
+
1175
+ for item in reader:
1176
+ text = item["TEXT"]
1177
+
1178
+ if text:
1179
+ text = normalizer(text)
1180
+
1181
+ if text:
1182
+ item["TEXT"] = text
1183
+ print("\t".join([item[f] for f in reader.fieldnames]), file=ostream)
1184
+
1185
+ ndone += 1
1186
+ if ndone % args.log_interval == 0:
1187
+ print(f"text norm: {ndone} lines done.", file=sys.stderr, flush=True)
1188
+ else:
1189
+ for l in istream:
1190
+ key, text = "", ""
1191
+ if args.format == "ark": # KALDI archive, line format: "key text"
1192
+ cols = l.strip().split(maxsplit=1)
1193
+ key, text = cols[0], cols[1] if len(cols) == 2 else ""
1194
+ else:
1195
+ text = l.strip()
1196
+
1197
+ if text:
1198
+ text = normalizer(text)
1199
+
1200
+ if text:
1201
+ if args.format == "ark":
1202
+ print(key + "\t" + text, file=ostream)
1203
+ else:
1204
+ print(text, file=ostream)
1205
+
1206
+ ndone += 1
1207
+ if ndone % args.log_interval == 0:
1208
+ print(f"text norm: {ndone} lines done.", file=sys.stderr, flush=True)
1209
+ print(f"text norm: {ndone} lines done in total.", file=sys.stderr, flush=True)
songgen/modeling_songgen_dual_track.py ADDED
@@ -0,0 +1,623 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import copy
2
+ import torch
3
+ import torch.nn as nn
4
+ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
5
+ from transformers.cache_utils import (
6
+ Cache,
7
+ DynamicCache,
8
+ EncoderDecoderCache,
9
+ SlidingWindowCache,
10
+ StaticCache,
11
+ )
12
+ from transformers.generation.configuration_utils import GenerationConfig, GenerationMode
13
+ from transformers.generation.logits_process import LogitsProcessorList
14
+ from transformers.generation.stopping_criteria import StoppingCriteriaList
15
+ from transformers.modeling_outputs import (
16
+ BaseModelOutput,
17
+ BaseModelOutputWithPastAndCrossAttentions,
18
+ ModelOutput,
19
+
20
+ )
21
+ from transformers.utils import (
22
+ add_start_docstrings,
23
+ add_start_docstrings_to_model_forward,
24
+ logging,
25
+ replace_return_docstrings,
26
+ )
27
+ from .modeling_songgen_mixed import (
28
+ SongGenMixedForConditionalGeneration,
29
+ build_delay_pattern_mask
30
+ )
31
+ from .configuration_songgen import SongGenConfig, SongGenDecoderConfig
32
+ from .logits_processors import SongGenLogitsProcessor
33
+
34
+ logger = logging.get_logger(__name__)
35
+
36
+ def split_combined_track_input_ids(combined_input_ids: torch.LongTensor, pattern: str, num_codebooks:int):
37
+ """
38
+ combined_input_ids has not applied the codebook-delayed pattern.
39
+ Splits the combined input_ids back into acc_input_ids and vocal_input_ids based on the specified pattern.
40
+ combined_input_ids: Tensor of shape (bsz, 2 * num_codebooks, seq_len) or (bsz, num_codebooks, 2*seq_len) depending on pattern
41
+ pattern: The pattern used for combining the input_ids.
42
+ """
43
+ assert len(combined_input_ids.shape) == 3, f"split_combined_track_input_ids(): combined_input_ids.shape:{combined_input_ids.shape}"
44
+ assert pattern in ['parallel_std', 'parallel_A_V', 'parallel_V_A', 'interleaving_A_V', 'interleaving_V_A'], f"Invalid pattern. Must be one of ['parallel_std', 'parallel_A_V', 'parallel_V_A', 'interleaving_A_V', 'interleaving_V_A'], but got: {pattern}"
45
+ if pattern == 'interleaving_A_V':
46
+ acc_input_ids = combined_input_ids[:, :, ::2] # Take the elements at even indices (acc_input_ids)
47
+ vocal_input_ids = combined_input_ids[:, :, 1::2] # Take the elements at odd indices (vocal_input_ids)
48
+
49
+ elif pattern == 'interleaving_V_A':
50
+ vocal_input_ids = combined_input_ids[:, :, ::2] # Take the elements at even indices (vocal_input_ids)
51
+ acc_input_ids = combined_input_ids[:, :, 1::2] # Take the elements at odd indices (acc_input_ids)
52
+
53
+ elif pattern.startswith('parallel'):
54
+ acc_input_ids = combined_input_ids[:, :num_codebooks, :]
55
+ vocal_input_ids = combined_input_ids[:, num_codebooks:, :]
56
+
57
+ else:
58
+ raise ValueError(f"Unexpected track_conbination pattern: {pattern}")
59
+
60
+ return acc_input_ids, vocal_input_ids
61
+
62
+
63
+
64
+ def combine_track_input_ids(acc_input_ids: torch.LongTensor, vocal_input_ids: torch.LongTensor, pattern: str, bos_token_id: int, pad_token_id: int, num_codebooks:int):
65
+ """
66
+ Combines the input_ids (acc_input_ids and vocal_input_ids) based on the specified pattern.
67
+ Both inpu_ids should be delayed. shape:(bsz, num_codebooks, seq_len)
68
+ # - [a, b, E, E, E, E]
69
+ # - [B, c, d, E, E, E]
70
+ # - [B, B, e, f, E, E]
71
+ # - [B, B, B, g, h, E]
72
+ pattern options:
73
+ ['parallel_std', 'parallel_A_V', 'parallel_V_A', 'interleaving_A_V', 'interleaving_V_A']
74
+ """
75
+ assert acc_input_ids.shape == vocal_input_ids.shape, f'acc_input_ids.shape: {acc_input_ids.shape}, vocal_input_ids.shape: {vocal_input_ids.shape}'
76
+ acc_input_ids = acc_input_ids.reshape(-1, num_codebooks, acc_input_ids.shape[-1]) #(bsz, num_codebook, seq_len)
77
+ vocal_input_ids = vocal_input_ids.reshape(-1, num_codebooks, vocal_input_ids.shape[-1])
78
+ bsz, n, seq_len = acc_input_ids.shape
79
+ if pattern == 'interleaving_A_V':
80
+ combined_input_ids = torch.stack([acc_input_ids, vocal_input_ids], dim=-1) # (bsz, num_codebooks, seq_len, 2)
81
+ combined_input_ids = combined_input_ids.reshape(bsz, n, 2 * seq_len)
82
+
83
+ elif pattern == 'interleaving_V_A':
84
+ combined_input_ids = torch.stack([vocal_input_ids, acc_input_ids], dim=-1) # (bsz, num_codebooks, seq_len, 2)
85
+ combined_input_ids = combined_input_ids.reshape(bsz, n, 2 * seq_len)
86
+
87
+ elif pattern == 'parallel_std':
88
+ combined_input_ids = torch.cat([acc_input_ids, vocal_input_ids], dim=1) #(bsz, 2*num_codebooks, seq_len)
89
+
90
+ elif pattern == 'parallel_A_V' or pattern == 'parallel_V_A' :
91
+ bos_ids = torch.ones((bsz, num_codebooks, 1), dtype=acc_input_ids.dtype, device=acc_input_ids.device) * bos_token_id
92
+ pad_ids = torch.ones((bsz, num_codebooks, 1), dtype=acc_input_ids.dtype, device=acc_input_ids.device) * pad_token_id
93
+
94
+ if pattern == 'parallel_A_V':
95
+ acc_input_ids = torch.cat([acc_input_ids, pad_ids], dim=-1)
96
+ vocal_input_ids = torch.cat([bos_ids, vocal_input_ids], dim=-1)
97
+ combined_input_ids = torch.cat([acc_input_ids, vocal_input_ids], dim=1) # #(bsz, 2*num_codebooks, seq_len+1)
98
+
99
+ elif pattern == 'parallel_V_A':
100
+ acc_input_ids = torch.cat([bos_ids, acc_input_ids], dim=-1)
101
+ vocal_input_ids = torch.cat([vocal_input_ids, pad_ids], dim=-1)
102
+ combined_input_ids = torch.cat([acc_input_ids, vocal_input_ids], dim=1) # #(bsz, 2*num_codebooks, seq_len+1)
103
+
104
+ else:
105
+ raise ValueError(f"Unexpected track_conbination pattern: {pattern}")
106
+
107
+ else:
108
+ raise ValueError(f"Unexpected track_conbination pattern: {pattern}")
109
+
110
+ return combined_input_ids
111
+
112
+
113
+ def build_combined_delay_pattern_mask(
114
+ acc_input_ids: torch.LongTensor, vocal_input_ids: torch.LongTensor, track_pattern: str, bos_token_id: int, pad_token_id: int, max_length: int, num_codebooks: int, device
115
+ ):
116
+ assert acc_input_ids.shape == vocal_input_ids.shape, f'acc_input_ids.shape:{acc_input_ids.shape} != vocal_input_ids.shape:{vocal_input_ids.shape}'
117
+ '''
118
+ input_ids:
119
+ - [B, a, b]
120
+ - [B, B, c]
121
+ - [B, B, B]
122
+ - [B, B, B]
123
+
124
+ delay_pattern_mask
125
+ - [B, a, b, -1, -1, P, P, P]
126
+ - [B, B, c, d, -1, -1, P, P]
127
+ - [B, B, B, e, f, -1, -1, P]
128
+ - [B, B, B, B, g, h, -1, -1]
129
+
130
+ or
131
+ input_ids:
132
+ - [B]
133
+ - [B]
134
+ - [B]
135
+ - [B]
136
+
137
+ delay_pattern_mask
138
+ - [B, -1, -1, P, P, P]
139
+ - [B, B, -1, -1, P, P]
140
+ - [B, B, B, -1, -1, P]
141
+ - [B, B, B, B, -1, -1]
142
+ '''
143
+ vocal_input_ids = vocal_input_ids.reshape(-1, num_codebooks, vocal_input_ids.shape[-1] )
144
+ acc_input_ids = acc_input_ids.reshape(-1, num_codebooks, acc_input_ids.shape[-1] )
145
+ batch_size = vocal_input_ids.shape[0]
146
+
147
+ #Here, the first column of vocal_input_ids and acc_input_ids must be bos_token_id.
148
+ vocal_delayed_input_ids, vocal_decoder_delay_pattern_mask = build_delay_pattern_mask(
149
+ vocal_input_ids,
150
+ bos_token_id=bos_token_id,
151
+ pad_token_id=pad_token_id,
152
+ max_length=max_length,
153
+ num_codebooks=num_codebooks
154
+ )
155
+
156
+ acc_delayed_input_ids, acc_decoder_delay_pattern_mask = build_delay_pattern_mask(
157
+ acc_input_ids,
158
+ bos_token_id=bos_token_id,
159
+ pad_token_id=pad_token_id,
160
+ max_length=max_length,
161
+ num_codebooks=num_codebooks
162
+ )
163
+
164
+
165
+ if track_pattern.startswith('parallel'):
166
+ decoder_input_ids_start = (
167
+ torch.ones((batch_size , num_codebooks * 2, 1), dtype=torch.long, device=device)
168
+ * bos_token_id
169
+ )
170
+ else:
171
+ decoder_input_ids_start = (
172
+ torch.ones((batch_size , num_codebooks, 1), dtype=torch.long, device=device)
173
+ * bos_token_id
174
+ )
175
+
176
+ #combine input_ids
177
+ if vocal_delayed_input_ids.shape[-1]==1 and (vocal_delayed_input_ids == bos_token_id).all().item() and \
178
+ acc_delayed_input_ids.shape[-1]==1 and (acc_delayed_input_ids == bos_token_id).all().item():
179
+ decoder_input_ids = decoder_input_ids_start
180
+ else:
181
+ combined_input_ids = combine_track_input_ids(acc_delayed_input_ids[..., 1:], vocal_delayed_input_ids[..., 1:], track_pattern, bos_token_id, pad_token_id, num_codebooks)
182
+ decoder_input_ids = torch.cat([decoder_input_ids_start, combined_input_ids], dim=-1)
183
+ if track_pattern in ['parallel_A_V', 'parallel_V_A']:
184
+ assert (decoder_input_ids[..., -1] == pad_token_id).any().item and (decoder_input_ids[..., -2] != pad_token_id).all().item, f'decoder_input_ids[..., -2:] : {decoder_input_ids[..., -2:]}'
185
+ decoder_input_ids = decoder_input_ids[..., :-1]
186
+
187
+ #combine decoder_delay_pattern_mask
188
+ assert vocal_decoder_delay_pattern_mask.shape[-1] > 1 and acc_decoder_delay_pattern_mask.shape[-1] >1, f'vocal_decoder_delay_pattern_mask:{vocal_decoder_delay_pattern_mask.shape}; acc_decoder_delay_pattern_mask:{acc_decoder_delay_pattern_mask.shape}'
189
+ combined_decoder_delay_pattern_mask = combine_track_input_ids(acc_decoder_delay_pattern_mask[..., 1:], vocal_decoder_delay_pattern_mask[..., 1:], track_pattern, bos_token_id, pad_token_id, num_codebooks)
190
+ decoder_delay_pattern_mask = torch.cat([decoder_input_ids_start, combined_decoder_delay_pattern_mask], dim=-1)
191
+
192
+ decoder_input_ids = decoder_input_ids.reshape(-1, decoder_input_ids.shape[-1])
193
+ decoder_delay_pattern_mask = decoder_delay_pattern_mask.reshape(-1, decoder_delay_pattern_mask.shape[-1])
194
+ return decoder_input_ids, decoder_delay_pattern_mask
195
+
196
+
197
+
198
+
199
+ class SongGenDualTrackForConditionalGeneration(SongGenMixedForConditionalGeneration):
200
+ def __init__(self, config: Optional[SongGenConfig] = None, **kwargs):
201
+ super().__init__(config=config, **kwargs)
202
+
203
+ def _prepare_combined_delayed_decoder_input_ids_for_generation(
204
+ self,
205
+ batch_size: int,
206
+ model_input_name: str,
207
+ model_kwargs: Dict[str, torch.Tensor],
208
+ decoder_start_token_id: int = None,
209
+ bos_token_id: int = None,
210
+ pad_token_id: int= None,
211
+ max_length: int= None,
212
+ device: torch.device = None,
213
+ ) -> Tuple[torch.LongTensor, Dict[str, torch.Tensor]]:
214
+ """Prepares `decoder_input_ids` for generation with encoder-decoder models"""
215
+ # 1. We also allow the user to pass it under `vocal_input_ids` and `acc_input_ids`, if the encoder does not use it as the main input.
216
+ track_pattern = self.decoder.config.track_pattern
217
+ if device is None:
218
+ device = self.device
219
+ vocal_input_ids = None
220
+ acc_input_ids = None
221
+ if model_kwargs is not None and "vocal_input_ids" in model_kwargs:
222
+ vocal_input_ids = model_kwargs.pop("vocal_input_ids")
223
+ if model_kwargs is not None and "acc_input_ids" in model_kwargs:
224
+ acc_input_ids = model_kwargs.pop("acc_input_ids")
225
+
226
+ # 2. Encoder-decoder models expect the `decoder_input_ids` to start with a special token. Let's ensure that.
227
+ decoder_start_token_id = self._get_decoder_start_token_id(decoder_start_token_id, bos_token_id)
228
+
229
+ decoder_input_ids_start = (
230
+ torch.ones((batch_size, self.decoder.num_codebooks, 1), dtype=torch.long, device=device)
231
+ * decoder_start_token_id
232
+ )
233
+
234
+ # no user input -> use decoder_start_token_id as decoder_input_ids
235
+ # user input but doesn't start with decoder_start_token_id -> prepend decoder_start_token_id
236
+ if vocal_input_ids is None:
237
+ vocal_input_ids = decoder_input_ids_start
238
+ elif (vocal_input_ids[..., 0] != decoder_start_token_id).all().item():
239
+ vocal_input_ids = torch.cat([decoder_input_ids_start, vocal_input_ids], dim=-1)
240
+
241
+ assert "decoder_attention_mask" not in model_kwargs, "decoder_attention_mask should not in model_kwargs"
242
+
243
+ if acc_input_ids is None:
244
+ acc_input_ids = decoder_input_ids_start
245
+ elif (acc_input_ids[..., 0] != decoder_start_token_id).all().item():
246
+ acc_input_ids = torch.cat([decoder_input_ids_start, acc_input_ids], dim=-1)
247
+
248
+ assert vocal_input_ids.shape == acc_input_ids.shape, f'vocal_input_ids: {vocal_input_ids}, acc_input_ids.shape: {acc_input_ids.shape}'
249
+
250
+ decoder_input_ids, decoder_delay_pattern_mask = build_combined_delay_pattern_mask(
251
+ acc_input_ids, vocal_input_ids, track_pattern, bos_token_id, pad_token_id, max_length, self.decoder.num_codebooks, device
252
+ )
253
+
254
+ # stash the delay mask so that we don't have to recompute in each forward pass
255
+ model_kwargs["decoder_delay_pattern_mask"] = decoder_delay_pattern_mask
256
+
257
+ prompt_hidden_states = model_kwargs.get("prompt_hidden_states",None) if not self.prompt_cross_attention else None
258
+ ref_audio_embeds = model_kwargs.get("ref_audio_embeds", None)
259
+ num_embed_tokens = self.decoder.num_codebooks * 2 if track_pattern.startswith('parallel') else self.decoder.num_codebooks
260
+ input = decoder_input_ids.reshape(-1, num_embed_tokens, decoder_input_ids.shape[-1])
261
+ inputs_embeds = sum(
262
+ [
263
+ self.decoder.model.decoder.embed_tokens[codebook](input[:, codebook])
264
+ for codebook in range(num_embed_tokens)
265
+ ]
266
+ ) / int(num_embed_tokens/self.decoder.num_codebooks)
267
+ if prompt_hidden_states is not None:
268
+ inputs_embeds = torch.cat([prompt_hidden_states, inputs_embeds], dim=1)
269
+ model_kwargs["inputs_embeds"] = inputs_embeds
270
+ return acc_input_ids, vocal_input_ids, decoder_input_ids, model_kwargs
271
+
272
+
273
+ def audio_decode_output_values(self, output_ids, batch_size, audio_scales, generation_config):
274
+ output_ids = output_ids[None, ...]
275
+ decode_sequentially = True
276
+ # (
277
+ # generation_config.bos_token_id in output_ids
278
+ # or generation_config.pad_token_id in output_ids
279
+ # or generation_config.eos_token_id in output_ids
280
+ # )
281
+
282
+ if not decode_sequentially:
283
+ output_values = self.audio_encoder.decode(
284
+ output_ids,
285
+ audio_scales=audio_scales,
286
+ ).audio_values.squeeze(1)
287
+ output_lengths = [audio.shape[0] for audio in output_values]
288
+ else:
289
+ output_values = []
290
+ for sample_id in range(batch_size):
291
+ sample = output_ids[:, sample_id]
292
+ sample_mask = (sample >= self.audio_encoder.config.codebook_size).sum(dim=(0, 1)) == 0
293
+ if sample_mask.sum() > 0:
294
+ sample = sample[:, :, sample_mask]
295
+ sample = self.audio_encoder.decode(sample[None, ...], [audio_scales[sample_id]]).audio_values
296
+ output_values.append(sample.transpose(0, 2))
297
+ else:
298
+ logger.warning('sample_mask.sum() <= 0 , generate sample: ', sample.shape, sample[0, :, :20])
299
+ output_values.append(torch.zeros((1, 1, 1)).to(self.device))
300
+ output_lengths = [audio.shape[0] for audio in output_values]
301
+ output_values = (
302
+ torch.nn.utils.rnn.pad_sequence(output_values, batch_first=True, padding_value=0)
303
+ .squeeze(-1)
304
+ .squeeze(-1)
305
+ )
306
+ return output_lengths, output_values
307
+
308
+ @torch.no_grad()
309
+ def generate(
310
+ self,
311
+ inputs: Optional[torch.Tensor] = None,
312
+ generation_config: Optional[GenerationConfig] = None,
313
+ logits_processor: Optional[LogitsProcessorList] = None,
314
+ stopping_criteria: Optional[StoppingCriteriaList] = None,
315
+ synced_gpus: Optional[bool] = None,
316
+ streamer: Optional["BaseStreamer"] = None,
317
+ **kwargs,
318
+ ):
319
+ """
320
+
321
+ Generates sequences of token ids for models with a language modeling head.
322
+
323
+ <Tip warning={true}>
324
+
325
+ Most generation-controlling parameters are set in `generation_config` which, if not passed, will be set to the
326
+ model's default generation configuration. You can override any `generation_config` by passing the corresponding
327
+ parameters to generate(), e.g. `.generate(inputs, num_beams=4, do_sample=True)`.
328
+
329
+ For an overview of generation strategies and code examples, check out the [following
330
+ guide](./generation_strategies).
331
+
332
+ </Tip>
333
+
334
+ Parameters:
335
+ inputs (`torch.Tensor` of varying shape depending on the modality, *optional*):
336
+ The sequence used as a prompt for the generation or as model inputs to the encoder. If `None` the
337
+ method initializes it with `bos_token_id` and a batch size of 1. For decoder-only models `inputs`
338
+ should be in the format `input_ids`. For encoder-decoder models *inputs* can represent any of
339
+ `input_ids`, `input_values`, `input_features`, or `pixel_values`.
340
+ generation_config (`~generation.GenerationConfig`, *optional*):
341
+ The generation configuration to be used as base parametrization for the generation call. `**kwargs`
342
+ passed to generate matching the attributes of `generation_config` will override them. If
343
+ `generation_config` is not provided, the default will be used, which had the following loading
344
+ priority: 1) from the `generation_config.json` model file, if it exists; 2) from the model
345
+ configuration. Please note that unspecified parameters will inherit [`~generation.GenerationConfig`]'s
346
+ default values, whose documentation should be checked to parameterize generation.
347
+ logits_processor (`LogitsProcessorList`, *optional*):
348
+ Custom logits processors that complement the default logits processors built from arguments and
349
+ generation config. If a logit processor is passed that is already created with the arguments or a
350
+ generation config an error is thrown. This feature is intended for advanced users.
351
+ stopping_criteria (`StoppingCriteriaList`, *optional*):
352
+ Custom stopping criteria that complement the default stopping criteria built from arguments and a
353
+ generation config. If a stopping criteria is passed that is already created with the arguments or a
354
+ generation config an error is thrown. This feature is intended for advanced users.
355
+ synced_gpus (`bool`, *optional*, defaults to `False`):
356
+ Whether to continue running the while loop until max_length (needed for ZeRO stage 3)
357
+ streamer (`BaseStreamer`, *optional*):
358
+ Streamer object that will be used to stream the generated sequences. Generated tokens are passed
359
+ through `streamer.put(token_ids)` and the streamer is responsible for any further processing.
360
+ kwargs (`Dict[str, Any]`, *optional*):
361
+ Ad hoc parametrization of `generate_config` and/or additional model-specific kwargs that will be
362
+ forwarded to the `forward` function of the model. If the model is an encoder-decoder model, encoder
363
+ specific kwargs should not be prefixed and decoder specific kwargs should be prefixed with *decoder_*.
364
+
365
+ Return:
366
+ [`~utils.ModelOutput`] or `torch.LongTensor`: A [`~utils.ModelOutput`] (if `return_dict_in_generate=True`
367
+ or when `config.return_dict_in_generate=True`) or a `torch.FloatTensor`.
368
+
369
+ If the model is *not* an encoder-decoder model (`model.config.is_encoder_decoder=False`), the possible
370
+ [`~utils.ModelOutput`] types are:
371
+
372
+ - [`~generation.GenerateDecoderOnlyOutput`],
373
+ - [`~generation.GenerateBeamDecoderOnlyOutput`]
374
+
375
+ If the model is an encoder-decoder model (`model.config.is_encoder_decoder=True`), the possible
376
+ [`~utils.ModelOutput`] types are:
377
+
378
+ - [`~generation.GenerateEncoderDecoderOutput`],
379
+ - [`~generation.GenerateBeamEncoderDecoderOutput`]
380
+ """
381
+ # 1. Handle `generation_config` and kwargs that might update it, and validate the resulting objects
382
+ if generation_config is None:
383
+ generation_config = self.generation_config
384
+
385
+ generation_config = copy.deepcopy(generation_config)
386
+ model_kwargs = generation_config.update(**kwargs) # All unused kwargs must be model kwargs
387
+ generation_config.validate()
388
+ self._validate_model_kwargs(model_kwargs.copy())
389
+
390
+ if model_kwargs.get("encoder_outputs") is not None and type(model_kwargs["encoder_outputs"]) == tuple:
391
+ # wrap the unconditional outputs as a BaseModelOutput for compatibility with the rest of generate
392
+ model_kwargs["encoder_outputs"] = BaseModelOutput(last_hidden_state=model_kwargs["encoder_outputs"][0])
393
+
394
+ requires_attention_mask = "encoder_outputs" not in model_kwargs
395
+ kwargs_has_attention_mask = model_kwargs.get("attention_mask", None) is not None
396
+
397
+ # 3. Define model inputs
398
+ inputs_tensor, model_input_name, model_kwargs = self._prepare_model_inputs(
399
+ inputs, generation_config.bos_token_id, model_kwargs
400
+ )
401
+ batch_size = inputs_tensor.shape[0]
402
+ self._prepare_special_tokens(generation_config, kwargs_has_attention_mask, device=inputs_tensor.device)
403
+
404
+ track_pattern = self.decoder.config.track_pattern
405
+ if track_pattern.startswith('parallel'):
406
+ combined_num_codebooks = self.decoder.num_codebooks * 2
407
+ else:
408
+ combined_num_codebooks = self.decoder.num_codebooks
409
+ logits_processor = logits_processor if logits_processor is not None else LogitsProcessorList([SongGenLogitsProcessor(generation_config.eos_token_id, combined_num_codebooks, batch_size, inputs_tensor.device, track_pattern=track_pattern)])
410
+ stopping_criteria = stopping_criteria if stopping_criteria is not None else StoppingCriteriaList()
411
+
412
+ # 4. Define other model kwargs
413
+ model_kwargs["use_cache"] = generation_config.use_cache
414
+
415
+ if model_kwargs.get("attention_mask", None) is None and requires_attention_mask:
416
+ model_kwargs["attention_mask"] = self._prepare_attention_mask_for_generation(
417
+ inputs_tensor, generation_config._pad_token_tensor, generation_config._eos_token_tensor
418
+ )
419
+
420
+ if "encoder_outputs" not in model_kwargs:
421
+ # encoder_outputs are created and added to `model_kwargs`
422
+ model_kwargs = self._prepare_text_encoder_kwargs_for_generation(
423
+ inputs_tensor, model_kwargs, model_input_name, generation_config
424
+ )
425
+
426
+ if "prompt_hidden_states" not in model_kwargs and "prompt_input_ids" in model_kwargs:
427
+ # `prompt_hidden_states` are created and added to `model_kwargs`; remove "prompt_input_ids"
428
+ model_kwargs = self._prepare_prompt_kwargs_for_generation(
429
+ model_kwargs.pop("prompt_input_ids"),
430
+ model_kwargs,
431
+ )
432
+
433
+
434
+ if "decoder_input_ids" not in model_kwargs and "input_values" in model_kwargs:
435
+ logger.warning_once(
436
+ "generate() currently does not support the input_values parameter. Please use vocal_input_ids and acc_input_ids for now."
437
+ )
438
+
439
+ acc_input_ids, vocal_input_ids, delayed_input_ids, model_kwargs =self._prepare_combined_delayed_decoder_input_ids_for_generation(
440
+ batch_size=batch_size,
441
+ model_input_name=model_input_name,
442
+ model_kwargs=model_kwargs,
443
+ decoder_start_token_id=generation_config._decoder_start_token_tensor,
444
+ bos_token_id=generation_config._bos_token_tensor,
445
+ pad_token_id=generation_config._pad_token_tensor,
446
+ max_length=generation_config.max_length,
447
+ device=inputs_tensor.device,
448
+ )
449
+
450
+
451
+ # 6. Prepare `max_length` depending on other stopping criteria.
452
+ input_ids_length = delayed_input_ids.shape[-1]
453
+ has_default_max_length = kwargs.get("max_length") is None and generation_config.max_length is not None
454
+ has_default_min_length = kwargs.get("min_length") is None and generation_config.min_length is not None
455
+ generation_config = self._prepare_generated_length(
456
+ generation_config=generation_config,
457
+ has_default_max_length=has_default_max_length,
458
+ has_default_min_length=has_default_min_length,
459
+ model_input_name=model_input_name,
460
+ inputs_tensor=inputs_tensor,
461
+ input_ids_length=input_ids_length,
462
+ )
463
+
464
+ if generation_config.cache_implementation is not None and model_kwargs.get("past_key_values") is not None:
465
+ raise ValueError(
466
+ "Passing both `cache_implementation` (used to initialize certain caches) and `past_key_values` (a "
467
+ "Cache object) is unsupported. Please use only one of the two."
468
+ )
469
+ elif generation_config.cache_implementation is not None:
470
+ if generation_config.cache_implementation in NEED_SETUP_CACHE_CLASSES_MAPPING:
471
+ if generation_config.cache_implementation == "static" and not self._supports_static_cache:
472
+ raise ValueError(
473
+ "This model does not support `cache_implementation='static'`. Please check the following "
474
+ "issue: https://github.com/huggingface/transformers/issues/28981"
475
+ )
476
+ if not self.prompt_cross_attention:
477
+ # when we prepend prompt_hidden_state to inputs_embeds, max_cache_len needs to be actualised
478
+ # generation_config.max_length has already been increased by input_ids_length which is
479
+ # already counted in input_embeds_seq_length so we remove it
480
+ input_embeds_seq_length = model_kwargs["inputs_embeds"].shape[1]
481
+ max_cache_len = generation_config.max_length + input_embeds_seq_length - input_ids_length
482
+ else:
483
+ max_cache_len = self.generation_config.max_length
484
+
485
+ model_kwargs["past_key_values"] = self._get_cache(
486
+ generation_config.cache_implementation,
487
+ getattr(generation_config, "num_beams", 1) * batch_size,
488
+ max_cache_len,
489
+ model_kwargs,
490
+ )
491
+ elif generation_config.cache_implementation == "quantized":
492
+ raise ValueError(
493
+ "This model does not support the quantized cache. If you want your model to support quantized "
494
+ "cache, please open an issue on the Parler-TTS repository https://github.com/huggingface/parler-tts"
495
+ )
496
+ # Use DynamicCache() instance by default. This will avoid back and forth from legacy format that
497
+ # keeps copying the cache thus using much more memory
498
+ elif generation_config.cache_implementation is None and self._supports_default_dynamic_cache(): #NOTE here
499
+ past = model_kwargs.get("past_key_values", None)
500
+ requires_cross_attention_cache = (
501
+ self.config.is_encoder_decoder or model_kwargs.get("encoder_outputs") is not None
502
+ )
503
+ if past is None:
504
+ model_kwargs["past_key_values"] = (
505
+ DynamicCache()
506
+ if not requires_cross_attention_cache
507
+ else EncoderDecoderCache(DynamicCache(), DynamicCache())
508
+ )
509
+ elif isinstance(past, tuple):
510
+ model_kwargs["past_key_values"] = (
511
+ DynamicCache.from_legacy_cache(past)
512
+ if not requires_cross_attention_cache
513
+ else EncoderDecoderCache.from_legacy_cache(past)
514
+ )
515
+
516
+ # delayed_input_ids are ready to be placed on the streamer (if used)
517
+ if streamer is not None:
518
+ streamer.put(delayed_input_ids.cpu())
519
+
520
+ # 7. determine generation mode
521
+ generation_mode = generation_config.get_generation_mode()
522
+ logger.info(f'generation_mode: {generation_mode}')
523
+
524
+ # 8. prepare distribution pre_processing samplers
525
+ logits_processor = self._get_logits_processor(
526
+ generation_config=generation_config,
527
+ input_ids_seq_length=input_ids_length,
528
+ encoder_input_ids=inputs_tensor,
529
+ prefix_allowed_tokens_fn=None,
530
+ logits_processor=logits_processor,
531
+ device=delayed_input_ids.device,
532
+ )
533
+
534
+ # 9. prepare stopping criteria
535
+ stopping_criteria = self._get_stopping_criteria(
536
+ generation_config=generation_config, stopping_criteria=stopping_criteria
537
+ )
538
+
539
+ if generation_mode in (GenerationMode.SAMPLE, GenerationMode.GREEDY_SEARCH):
540
+ logits_warper = self._get_logits_warper(generation_config, device=delayed_input_ids.device)
541
+ # expand input_ids with `num_return_sequences` additional sequences per batch
542
+ delayed_input_ids, model_kwargs = self._expand_inputs_for_generation(
543
+ input_ids=delayed_input_ids,
544
+ expand_size=generation_config.num_return_sequences,
545
+ is_encoder_decoder=self.config.is_encoder_decoder,
546
+ **model_kwargs,
547
+ )
548
+
549
+ # 10. run sample
550
+ outputs = self._sample(
551
+ delayed_input_ids,
552
+ logits_processor=logits_processor,
553
+ logits_warper=logits_warper,
554
+ stopping_criteria=stopping_criteria,
555
+ generation_config=generation_config,
556
+ synced_gpus=synced_gpus,
557
+ streamer=streamer,
558
+ **model_kwargs,
559
+ )
560
+ else:
561
+ raise ValueError(
562
+ "Got incompatible mode for generation, should be one of greedy or sampling. "
563
+ "Ensure that beam search is de-activated by setting `num_beams=1` and `num_beam_groups=1`."
564
+ )
565
+
566
+
567
+ if generation_config.return_dict_in_generate:
568
+ output_ids = outputs.sequences
569
+ else:
570
+ output_ids = outputs
571
+
572
+ #output_ids.shape [bsz*num_codebooks, seq_len] or [bsz*num_codebooks*2, seq_len]
573
+ # Apply the pattern mask to the final ids
574
+ output_ids = self.decoder.apply_delay_pattern_mask(output_ids, model_kwargs["decoder_delay_pattern_mask"])
575
+
576
+ # Revert the pattern delay mask by filtering the eos and bos token ids from the delay pattern mask
577
+
578
+ if track_pattern in ['interleaving_A_V', 'interleaving_V_A']:
579
+ if output_ids.shape[-1] % 2 == 0:
580
+ output_ids = output_ids[..., :-1]
581
+ output_max_length = int((output_ids.shape[-1]-1) / 2 + 1)
582
+ elif track_pattern in ['parallel']:
583
+ output_max_length = output_ids.shape[-1]
584
+ elif track_pattern in ['parallel_A_V', 'parallel_V_A']:
585
+ output_max_length = output_ids.shape[-1] -1
586
+
587
+ _, mask = build_combined_delay_pattern_mask(
588
+ acc_input_ids=acc_input_ids,
589
+ vocal_input_ids=vocal_input_ids,
590
+ track_pattern=track_pattern,
591
+ bos_token_id=generation_config.bos_token_id,
592
+ pad_token_id=generation_config.pad_token_id,
593
+ max_length=output_max_length,
594
+ num_codebooks=self.decoder.num_codebooks,
595
+ device=output_ids.device
596
+ )
597
+
598
+ assert mask.shape == output_ids.shape, f"mask.shape: {mask.shape}, output_ids.shape: {output_ids.shape}"
599
+ mask = (mask != generation_config.bos_token_id) & (mask != generation_config.pad_token_id)
600
+ output_ids = output_ids[mask].reshape(batch_size, combined_num_codebooks, -1)
601
+ acc_output_ids, vocal_output_ids = split_combined_track_input_ids(output_ids, track_pattern, self.decoder.num_codebooks)
602
+
603
+ audio_scales = model_kwargs.get("audio_scales")
604
+ if audio_scales is None:
605
+ audio_scales = [None] * batch_size
606
+
607
+ vocal_output_lengths, vocal_output_values = self.audio_decode_output_values(vocal_output_ids, batch_size, audio_scales, generation_config)
608
+ acc_output_lengths, acc_output_values = self.audio_decode_output_values(acc_output_ids, batch_size, audio_scales, generation_config)
609
+
610
+ if generation_config.return_dict_in_generate:
611
+ outputs["acc_audios_length"] = acc_output_lengths
612
+ outputs["acc_sequences"] = acc_output_values
613
+ outputs["vocal_audios_length"] = vocal_output_lengths
614
+ outputs["vocal_sequences"] = vocal_output_values
615
+ return outputs
616
+ else:
617
+ return acc_output_values, vocal_output_values
618
+
619
+
620
+
621
+
622
+
623
+
songgen/modeling_songgen_mixed.py ADDED
The diff for this file is too large to render. See raw diff
 
songgen/processing_songgen.py ADDED
@@ -0,0 +1,141 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ lyrics/Text/Voice processor class for MusicGen
3
+ """
4
+ import os
5
+ import torch
6
+ import librosa
7
+ import soundfile as sf
8
+ from transformers import AutoTokenizer
9
+ from .lyrics_utils.lyrics_tokenizer import VoiceBpeTokenizer
10
+ from transformers.models.encodec.feature_extraction_encodec import EncodecFeatureExtractor
11
+ from transformers import Wav2Vec2FeatureExtractor
12
+ from demucs import pretrained
13
+ from demucs.apply import apply_model
14
+ from demucs.audio import convert_audio
15
+ from songgen import (
16
+ XCodecModel,
17
+ build_delay_pattern_mask,
18
+ )
19
+
20
+ class SongGenProcessor():
21
+ def __init__(self, ckpt_path, device):
22
+ """
23
+ Initializes the SongGenProcessor
24
+ """
25
+ self.device = device
26
+ self.text_tokenizer = AutoTokenizer.from_pretrained(ckpt_path, padding_side='right')
27
+ self.lyrics_tokenizer = VoiceBpeTokenizer()
28
+ mert_path = 'm-a-p/MERT-v1-330M'
29
+ self.mert_processor = Wav2Vec2FeatureExtractor.from_pretrained(mert_path)
30
+ self.demucs = pretrained.get_model("htdemucs").to(device)
31
+ self.feature_extractor = EncodecFeatureExtractor(sampling_rate=16000)
32
+ self.audio_encoder = XCodecModel()
33
+ self.audio_encoder_special_token_ids ={
34
+ 'mask': 1030,
35
+ "random": {"bos": 1031 , "eos": 1032},
36
+ "melody": {"bos": 1033, "eos": 1034},
37
+ "drum": {"bos": 1035, "eos": 1036},
38
+ "vocal": {"bos": 1037, "eos": 1038},
39
+ "acc": {"bos":1039 , "eos":1040},
40
+ }
41
+
42
+
43
+
44
+ def __call__(self, text: str, lyrics: str, refaudio_path=None, refaudio_type=None, ref_voice_path=None, start=0, separate=False, padding=True, return_tensors="pt"):
45
+
46
+ """
47
+ Processes the input text, lyrics, and audio file, and returns the tensors suitable for model input.
48
+ do not support batching yet
49
+
50
+ :param text: text description.
51
+ :param lyrics: Lyrics text. English Only.
52
+ :param ref_voice_path: Optional path to the reference voice.
53
+ :param start: The starting time for the reference voice slice.
54
+ :param separate: Whether to perform audio separation.
55
+ :param return_tensors: Whether to return the tensors as PyTorch tensors.
56
+ :return: A dictionary with the model's inputs, ready for inference.
57
+ """
58
+ # Process lyrics and convert them into token IDs. Must be english now!
59
+ prompt_input_ids = [261] + self.lyrics_tokenizer.encode(lyrics.strip().replace('\n', '.'), lang='en') + [0]
60
+
61
+ # Tokenize the lyrics and pad to max length
62
+ lyrics_inputs = self.text_tokenizer.pad(
63
+ [{"input_ids": prompt_input_ids}],
64
+ return_tensors=return_tensors,
65
+ padding="longest",
66
+ ).to(self.device)
67
+
68
+ # Tokenize the text descriptions
69
+ text_inputs = self.text_tokenizer(
70
+ text,
71
+ return_tensors=return_tensors,
72
+ padding="longest",
73
+ ).to(self.device)
74
+
75
+ model_inputs = {
76
+ **text_inputs,
77
+ "prompt_input_ids": lyrics_inputs.input_ids,
78
+ "prompt_attention_mask": lyrics_inputs.attention_mask
79
+ }
80
+
81
+ # Process reference voice (if provided)
82
+ if ref_voice_path is not None:
83
+ wav, sr = sf.read(ref_voice_path)
84
+ wav = wav.T
85
+ wav = librosa.to_mono(wav) # Convert to mono if stereo
86
+ # Slice the audio according to the start and end times
87
+ lidx = int(start * sr)
88
+ ridx = lidx + int(3 * sr) # Slice a 3-second segment
89
+ wav = wav[lidx:ridx]
90
+
91
+ if separate:
92
+ # Since our model only supports reference voices that contain vocals and does not include accompaniment, it is necessary to perform vocal separation for mixed audio.
93
+ demucs_wav = convert_audio(
94
+ torch.tensor(wav[None], device=self.device).to(torch.float32),
95
+ sr,
96
+ self.demucs.samplerate,
97
+ self.demucs.audio_channels
98
+ )
99
+ sr = self.demucs.samplerate
100
+ stems = apply_model(self.demucs, demucs_wav.unsqueeze(0))
101
+ wav = stems[0][-1:].sum(0).mean(0).cpu().numpy()
102
+
103
+ if sr != self.mert_processor.sampling_rate:
104
+ wav = librosa.resample(wav, orig_sr=sr, target_sr=self.mert_processor.sampling_rate)
105
+ sr = self.mert_processor.sampling_rate
106
+
107
+ mert_inputs = self.mert_processor(
108
+ [wav], sampling_rate=self.mert_processor.sampling_rate, return_tensors="pt", padding="max_length", max_length=3*self.mert_processor.sampling_rate
109
+ )
110
+
111
+ model_inputs['ref_voice_values'] = mert_inputs['input_values'].to(self.device)
112
+ model_inputs['ref_voice_attention_mask'] = mert_inputs['attention_mask'].to(self.device)
113
+
114
+ if refaudio_path is not None and refaudio_type is not None:
115
+ refaudio_wav, refaudio_sr = sf.read(refaudio_path)
116
+ refaudio_wav = refaudio_wav.T
117
+ refaudio_wav = librosa.to_mono(refaudio_wav)
118
+ if refaudio_sr != self.feature_extractor.sampling_rate:
119
+ refaudio_wav = librosa.resample(refaudio_wav, orig_sr=refaudio_sr, target_sr=self.feature_extractor.sampling_rate)
120
+ refaudio_sr = self.feature_extractor.sampling_rate
121
+ with torch.no_grad():
122
+ self.audio_encoder.model.to(self.device)
123
+ codes = self.audio_encoder.encode(input_values=torch.tensor(refaudio_wav, dtype=torch.float32).to(self.device).unsqueeze(0).unsqueeze(0), bandwidth=4)["audio_codes"] #(1, bsz, codebooks, seq_len)
124
+ codes = codes.to(self.device).squeeze(0) # (1, bsz, codebooks, seq_len) -> (1, codebooks, seq_len)
125
+ num_codebooks = codes.shape[-2]
126
+ #apply codebook-delay
127
+ bos = self.audio_encoder_special_token_ids[refaudio_type]['bos']
128
+ eos = self.audio_encoder_special_token_ids[refaudio_type]['eos']
129
+ ref_bos = (torch.ones((1, num_codebooks, 1)) * bos).to(self.device)
130
+ codes = torch.cat([ref_bos, codes], dim=-1)
131
+ _, delay_pattern_mask = build_delay_pattern_mask(
132
+ codes,
133
+ bos_token_id=bos,
134
+ pad_token_id=eos,
135
+ max_length=codes.shape[-1] + num_codebooks,
136
+ num_codebooks=num_codebooks,
137
+ )
138
+ delay_codes = torch.where(delay_pattern_mask == -1, eos, delay_pattern_mask) #( codebooks, seq_len)
139
+ model_inputs['ref_audio_ids'] = delay_codes.transpose(0,1).unsqueeze(0) #(bsz, seq_len, codebooks)
140
+ #Xcode codes and build delay
141
+ return model_inputs
songgen/xcodec_wrapper/__init__.py ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ from .configuration_xcodec import XCodecConfig
2
+ from .modeling_xcodec import XCodecModel
songgen/xcodec_wrapper/configuration_xcodec.py ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from transformers import PretrainedConfig
2
+
3
+
4
+ class XCodecConfig(PretrainedConfig):
5
+ model_type = "xcodec"
6
+
7
+ def __init__(
8
+ self,
9
+ num_codebooks: int = 8,
10
+ model_bitrate: int = 4, # kbps
11
+ codebook_size: int = 1024,
12
+ # latent_dim: int = 1024,
13
+ frame_rate: int = 50,
14
+ sampling_rate: int = 16000,
15
+ **kwargs,
16
+ ):
17
+ self.codebook_size = codebook_size
18
+ self.model_bitrate = model_bitrate
19
+ # self.latent_dim = latent_dim
20
+ self.num_codebooks = num_codebooks
21
+ self.frame_rate = frame_rate
22
+ self.sampling_rate = sampling_rate
23
+
24
+ super().__init__(**kwargs)
songgen/xcodec_wrapper/modeling_xcodec.py ADDED
@@ -0,0 +1,167 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from transformers import PreTrainedModel
3
+ from transformers.models.encodec.modeling_encodec import EncodecDecoderOutput, EncodecEncoderOutput
4
+ from .configuration_xcodec import XCodecConfig
5
+ import torch.nn as nn
6
+ import sys
7
+ import os
8
+ XCODEC_INFER_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'xcodec_infer')
9
+ sys.path.append(XCODEC_INFER_PATH)
10
+ from omegaconf import OmegaConf
11
+ from models.soundstream_semantic import SoundStream
12
+
13
+ # model doesn't support batching yet
14
+
15
+ def build_codec_model(config):
16
+ model = eval(config.generator.name)(**config.generator.config)
17
+ return model
18
+
19
+
20
+ class XCodecModel(nn.Module):
21
+ config_class = XCodecConfig
22
+ main_input_name = "input_values"
23
+
24
+ def __init__(self):
25
+ super().__init__()
26
+ ckpt_path = os.path.join(XCODEC_INFER_PATH , 'ckpts/general_more/xcodec_hubert_general_audio_v2.pth')
27
+ config_path = os.path.join(XCODEC_INFER_PATH ,'ckpts/general_more/config_hubert_general.yaml')
28
+ config = OmegaConf.load(config_path)
29
+ self.model = build_codec_model(config)
30
+ parameter_dict = torch.load(ckpt_path)
31
+ self.model.load_state_dict(parameter_dict)
32
+ self.model.eval()
33
+ self.num_codebooks = 8
34
+ self.codebook_size = 1024
35
+ self.config = config
36
+ self.config.codebook_size = 1024
37
+ self.config.n_codebooks = 8
38
+ self.config.return_dict = True
39
+ self.config.frame_rate = 50
40
+
41
+ def encode(
42
+ self, input_values, padding_mask=None, bandwidth=None, return_dict=None, n_quantizers=None, sample_rate=None
43
+ ):
44
+ """
45
+ Encodes the input audio waveform into discrete codes.
46
+
47
+ Args:
48
+ input_values (`torch.Tensor` of shape `(batch_size, channels, sequence_length)`):
49
+ Float values of the input audio waveform.
50
+ padding_mask (`torch.Tensor` of shape `(batch_size, channels, sequence_length)`):
51
+ Padding mask used to pad the `input_values`.
52
+ bandwidth (`float`, *optional*):
53
+ Not used, kept to have the same inferface as HF encodec.
54
+ n_quantizers (`int`, *optional*) :
55
+ Number of quantizers to use, by default None
56
+ If None, all quantizers are used.
57
+ sample_rate (`int`, *optional*) :
58
+ Signal sampling_rate
59
+
60
+ Returns:
61
+ A list of frames containing the discrete encoded codes for the input audio waveform, along with rescaling
62
+ factors for each chunk when `normalize` is True. Each frames is a tuple `(codebook, scale)`, with
63
+ `codebook` of shape `[batch_size, num_codebooks, frames]`.
64
+ Scale is not used here.
65
+
66
+ """
67
+ bsz, channels, input_length = input_values.shape
68
+
69
+ if channels < 1 or channels > 2:
70
+ raise ValueError(f"Number of audio channels must be 1 or 2, but got {channels}")
71
+
72
+ if bsz != 1:
73
+ raise ValueError(f"Number of audio batch_size must be 1 in XCodec Encode() : {input_values.shape}")
74
+
75
+ # audio_data = self.model.preprocess(input_values, sample_rate)
76
+ if channels != 1:
77
+ input_values = input_values.mean(1, keepdim=True)
78
+
79
+ if sample_rate is not None and sample_rate != 16000:
80
+ input_values = torchaudio.transforms.Resample(sample_rate, 16000)(input_values)
81
+
82
+ audio_data = input_values
83
+
84
+
85
+ return_dict = return_dict if return_dict is not None else self.config.return_dict
86
+
87
+ # TODO: for now, no chunk length
88
+
89
+ chunk_length = None # self.config.chunk_length
90
+ if chunk_length is None:
91
+ chunk_length = input_length
92
+ stride = input_length
93
+ else:
94
+ stride = self.config.chunk_stride
95
+
96
+ if padding_mask is None:
97
+ padding_mask = torch.ones_like(input_values).bool()
98
+
99
+ encoded_frames = []
100
+ scales = []
101
+
102
+ step = chunk_length - stride
103
+ if (input_length % stride) - step != 0:
104
+ raise ValueError(
105
+ "The input length is not properly padded for batched chunked decoding. Make sure to pad the input correctly."
106
+ )
107
+
108
+ for offset in range(0, input_length - step, stride):
109
+ mask = padding_mask[..., offset : offset + chunk_length].bool()
110
+ frame = audio_data[:, :, offset : offset + chunk_length] #[1,1, squeeze_len]
111
+
112
+ scale = None
113
+
114
+ encoded_frame= self.model.encode(frame, target_bw=4) # [8, 1, seq_len]
115
+ encoded_frame = encoded_frame.transpose(0 , 1) # [1, 8, seq_len]
116
+ encoded_frames.append(encoded_frame)
117
+ scales.append(scale)
118
+
119
+ encoded_frames = torch.stack(encoded_frames)
120
+
121
+ if not return_dict:
122
+ return (encoded_frames, scales)
123
+
124
+ return EncodecEncoderOutput(encoded_frames, scales)
125
+
126
+ def decode(
127
+ self,
128
+ audio_codes,
129
+ audio_scales=None,
130
+ padding_mask=None,
131
+ return_dict=None,
132
+ ):
133
+ """
134
+ Decodes the given frames into an output audio waveform.
135
+
136
+ Note that the output might be a bit bigger than the input. In that case, any extra steps at the end can be
137
+ trimmed.
138
+
139
+ Args:
140
+ audio_codes (`torch.FloatTensor` of shape `(batch_size, nb_chunks, chunk_length)`, *optional*):
141
+ Discret code embeddings computed using `model.encode`.
142
+ audio_scales (`torch.Tensor` of shape `(batch_size, nb_chunks)`, *optional*):
143
+ Not used, kept to have the same inferface as HF encodec.
144
+ padding_mask (`torch.Tensor` of shape `(batch_size, channels, sequence_length)`):
145
+ Padding mask used to pad the `input_values`.
146
+ Not used yet, kept to have the same inferface as HF encodec.
147
+ return_dict (`bool`, *optional*):
148
+ Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
149
+
150
+ """
151
+ return_dict = return_dict or self.config.return_dict
152
+
153
+ # TODO: for now, no chunk length
154
+ # input shape [1, 1, 8, seq_len]
155
+ if len(audio_codes) != 1:
156
+ raise ValueError(f"Expected one frame, got {len(audio_codes)}")
157
+
158
+ audio_codes = audio_codes.transpose(1, 2) # [1, 8, 1, seq_len]
159
+
160
+ # audio_values = self.model.quantizer.from_codes(audio_codes.squeeze(0))[0]
161
+ audio_values = self.model.decode(audio_codes.squeeze(0))
162
+ if not return_dict:
163
+ return (audio_values,)
164
+ return EncodecDecoderOutput(audio_values)
165
+
166
+ def forward(self, tensor):
167
+ raise ValueError("`XCodecModel.forward` not implemented yet")
songgen/xcodec_wrapper/xcodec_infer/ckpts/general_more/config_hubert_general.yaml ADDED
@@ -0,0 +1,137 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ generator:
2
+ name: SoundStream
3
+ config:
4
+ n_filters: 32
5
+ D: 256
6
+ target_bandwidths:
7
+ - 0.5
8
+ - 1
9
+ - 1.5
10
+ - 2
11
+ - 4
12
+ ratios:
13
+ - 8
14
+ - 5
15
+ - 4
16
+ - 2
17
+ sample_rate: 16000
18
+ bins: 1024
19
+ semantic_techer: hubert_base_general
20
+ d_list:
21
+ - mfd
22
+ mfd:
23
+ name: MultiFrequencyDiscriminator
24
+ config:
25
+ hop_lengths:
26
+ - 32
27
+ - 64
28
+ - 128
29
+ - 256
30
+ - 512
31
+ - 1024
32
+ hidden_channels:
33
+ - 64
34
+ - 128
35
+ - 256
36
+ - 512
37
+ - 512
38
+ - 512
39
+ domain: double
40
+ mel_scale: true
41
+ sample_rate: 16000
42
+ mpd:
43
+ name: MultiPeriodDiscriminator
44
+ config:
45
+ period_sizes:
46
+ - 2
47
+ - 3
48
+ - 5
49
+ - 7
50
+ - 11
51
+ period_kernel_size: 5
52
+ msd:
53
+ name: MultiScaleDiscriminator
54
+ config:
55
+ num_scales: 3
56
+ pool_kernel_size: 4
57
+ pool_stride: 2
58
+ optimizer:
59
+ g:
60
+ name: AdamW
61
+ config:
62
+ lr: 0.0002
63
+ betas:
64
+ - 0.8
65
+ - 0.99
66
+ eps: 1.0e-06
67
+ d:
68
+ name: AdamW
69
+ config:
70
+ lr: 0.0002
71
+ betas:
72
+ - 0.8
73
+ - 0.99
74
+ eps: 1.0e-06
75
+ lr_scheduler:
76
+ g:
77
+ name: ExponentialLR
78
+ config:
79
+ gamma: 0.999
80
+ d:
81
+ name: ExponentialLR
82
+ config:
83
+ gamma: 0.999
84
+ criterion:
85
+ g_criterion:
86
+ name: losses.generator_loss.GeneratorSTFTLoss
87
+ config:
88
+ use_mel_loss: false
89
+ adv_criterion: MSEGLoss
90
+ mel_loss_weight: 45
91
+ use_feature_match: true
92
+ feat_match_loss_weight: 20
93
+ use_full_stft_loss: true
94
+ use_sub_stft_loss: true
95
+ full_stft_loss_weight: 1
96
+ sub_stft_loss_weight: 1
97
+ mel_scale_loss:
98
+ sampling_rate: 16000
99
+ n_fft: 1024
100
+ num_mels: 80
101
+ hop_size: 160
102
+ win_size: 800
103
+ fmin: 0
104
+ full_multi_scale_stft_loss:
105
+ fft_sizes:
106
+ - 512
107
+ - 1024
108
+ - 2048
109
+ win_sizes:
110
+ - 480
111
+ - 960
112
+ - 1200
113
+ hop_sizes:
114
+ - 120
115
+ - 240
116
+ - 300
117
+ sub_multi_scale_stft_loss:
118
+ num_bands: 6
119
+ fft_sizes:
120
+ - 128
121
+ - 256
122
+ - 256
123
+ win_sizes:
124
+ - 80
125
+ - 120
126
+ - 200
127
+ hop_sizes:
128
+ - 20
129
+ - 40
130
+ - 50
131
+ d_criterion:
132
+ name: losses.discriminator_loss.MSEDiscriminatorLoss
133
+ config: null
134
+ commit_loss_weight: 1.0
135
+ codebook_loss_weight: 100
136
+ audio_norm_scale: 0.95
137
+
songgen/xcodec_wrapper/xcodec_infer/ckpts/general_more/xcodec_hubert_general_audio_v2.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:262b4675ad000af6b4e7a30af1d7afecc79598c098944d15204a29708752d538
3
+ size 710182186
songgen/xcodec_wrapper/xcodec_infer/descriptaudiocodec/__init__.py ADDED
File without changes
songgen/xcodec_wrapper/xcodec_infer/descriptaudiocodec/dac/__init__.py ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ __version__ = "1.0.0"
2
+
3
+ # preserved here for legacy reasons
4
+ __model_version__ = "latest"
5
+
6
+ import audiotools
7
+
8
+ audiotools.ml.BaseModel.INTERN += ["dac.**"]
9
+ audiotools.ml.BaseModel.EXTERN += ["einops"]
10
+
11
+
12
+ from . import nn
13
+ from . import model
14
+ from . import utils
15
+ from .model import DAC
16
+ from .model import DACFile
songgen/xcodec_wrapper/xcodec_infer/descriptaudiocodec/dac/model/__init__.py ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ from .base import CodecMixin
2
+ from .base import DACFile
3
+ from .dac import DAC
4
+ from .discriminator import Discriminator
songgen/xcodec_wrapper/xcodec_infer/descriptaudiocodec/dac/model/base.py ADDED
@@ -0,0 +1,294 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import math
2
+ from dataclasses import dataclass
3
+ from pathlib import Path
4
+ from typing import Union
5
+
6
+ import numpy as np
7
+ import torch
8
+ import tqdm
9
+ from audiotools import AudioSignal
10
+ from torch import nn
11
+
12
+ SUPPORTED_VERSIONS = ["1.0.0"]
13
+
14
+
15
+ @dataclass
16
+ class DACFile:
17
+ codes: torch.Tensor
18
+
19
+ # Metadata
20
+ chunk_length: int
21
+ original_length: int
22
+ input_db: float
23
+ channels: int
24
+ sample_rate: int
25
+ padding: bool
26
+ dac_version: str
27
+
28
+ def save(self, path):
29
+ artifacts = {
30
+ "codes": self.codes.numpy().astype(np.uint16),
31
+ "metadata": {
32
+ "input_db": self.input_db.numpy().astype(np.float32),
33
+ "original_length": self.original_length,
34
+ "sample_rate": self.sample_rate,
35
+ "chunk_length": self.chunk_length,
36
+ "channels": self.channels,
37
+ "padding": self.padding,
38
+ "dac_version": SUPPORTED_VERSIONS[-1],
39
+ },
40
+ }
41
+ path = Path(path).with_suffix(".dac")
42
+ with open(path, "wb") as f:
43
+ np.save(f, artifacts)
44
+ return path
45
+
46
+ @classmethod
47
+ def load(cls, path):
48
+ artifacts = np.load(path, allow_pickle=True)[()]
49
+ codes = torch.from_numpy(artifacts["codes"].astype(int))
50
+ if artifacts["metadata"].get("dac_version", None) not in SUPPORTED_VERSIONS:
51
+ raise RuntimeError(
52
+ f"Given file {path} can't be loaded with this version of descript-audio-codec."
53
+ )
54
+ return cls(codes=codes, **artifacts["metadata"])
55
+
56
+
57
+ class CodecMixin:
58
+ @property
59
+ def padding(self):
60
+ if not hasattr(self, "_padding"):
61
+ self._padding = True
62
+ return self._padding
63
+
64
+ @padding.setter
65
+ def padding(self, value):
66
+ assert isinstance(value, bool)
67
+
68
+ layers = [
69
+ l for l in self.modules() if isinstance(l, (nn.Conv1d, nn.ConvTranspose1d))
70
+ ]
71
+
72
+ for layer in layers:
73
+ if value:
74
+ if hasattr(layer, "original_padding"):
75
+ layer.padding = layer.original_padding
76
+ else:
77
+ layer.original_padding = layer.padding
78
+ layer.padding = tuple(0 for _ in range(len(layer.padding)))
79
+
80
+ self._padding = value
81
+
82
+ def get_delay(self):
83
+ # Any number works here, delay is invariant to input length
84
+ l_out = self.get_output_length(0)
85
+ L = l_out
86
+
87
+ layers = []
88
+ for layer in self.modules():
89
+ if isinstance(layer, (nn.Conv1d, nn.ConvTranspose1d)):
90
+ layers.append(layer)
91
+
92
+ for layer in reversed(layers):
93
+ d = layer.dilation[0]
94
+ k = layer.kernel_size[0]
95
+ s = layer.stride[0]
96
+
97
+ if isinstance(layer, nn.ConvTranspose1d):
98
+ L = ((L - d * (k - 1) - 1) / s) + 1
99
+ elif isinstance(layer, nn.Conv1d):
100
+ L = (L - 1) * s + d * (k - 1) + 1
101
+
102
+ L = math.ceil(L)
103
+
104
+ l_in = L
105
+
106
+ return (l_in - l_out) // 2
107
+
108
+ def get_output_length(self, input_length):
109
+ L = input_length
110
+ # Calculate output length
111
+ for layer in self.modules():
112
+ if isinstance(layer, (nn.Conv1d, nn.ConvTranspose1d)):
113
+ d = layer.dilation[0]
114
+ k = layer.kernel_size[0]
115
+ s = layer.stride[0]
116
+
117
+ if isinstance(layer, nn.Conv1d):
118
+ L = ((L - d * (k - 1) - 1) / s) + 1
119
+ elif isinstance(layer, nn.ConvTranspose1d):
120
+ L = (L - 1) * s + d * (k - 1) + 1
121
+
122
+ L = math.floor(L)
123
+ return L
124
+
125
+ @torch.no_grad()
126
+ def compress(
127
+ self,
128
+ audio_path_or_signal: Union[str, Path, AudioSignal],
129
+ win_duration: float = 1.0,
130
+ verbose: bool = False,
131
+ normalize_db: float = -16,
132
+ n_quantizers: int = None,
133
+ ) -> DACFile:
134
+ """Processes an audio signal from a file or AudioSignal object into
135
+ discrete codes. This function processes the signal in short windows,
136
+ using constant GPU memory.
137
+
138
+ Parameters
139
+ ----------
140
+ audio_path_or_signal : Union[str, Path, AudioSignal]
141
+ audio signal to reconstruct
142
+ win_duration : float, optional
143
+ window duration in seconds, by default 5.0
144
+ verbose : bool, optional
145
+ by default False
146
+ normalize_db : float, optional
147
+ normalize db, by default -16
148
+
149
+ Returns
150
+ -------
151
+ DACFile
152
+ Object containing compressed codes and metadata
153
+ required for decompression
154
+ """
155
+ audio_signal = audio_path_or_signal
156
+ if isinstance(audio_signal, (str, Path)):
157
+ audio_signal = AudioSignal.load_from_file_with_ffmpeg(str(audio_signal))
158
+
159
+ self.eval()
160
+ original_padding = self.padding
161
+ original_device = audio_signal.device
162
+
163
+ audio_signal = audio_signal.clone()
164
+ original_sr = audio_signal.sample_rate
165
+
166
+ resample_fn = audio_signal.resample
167
+ loudness_fn = audio_signal.loudness
168
+
169
+ # If audio is > 10 minutes long, use the ffmpeg versions
170
+ if audio_signal.signal_duration >= 10 * 60 * 60:
171
+ resample_fn = audio_signal.ffmpeg_resample
172
+ loudness_fn = audio_signal.ffmpeg_loudness
173
+
174
+ original_length = audio_signal.signal_length
175
+ resample_fn(self.sample_rate)
176
+ input_db = loudness_fn()
177
+
178
+ if normalize_db is not None:
179
+ audio_signal.normalize(normalize_db)
180
+ audio_signal.ensure_max_of_audio()
181
+
182
+ nb, nac, nt = audio_signal.audio_data.shape
183
+ audio_signal.audio_data = audio_signal.audio_data.reshape(nb * nac, 1, nt)
184
+ win_duration = (
185
+ audio_signal.signal_duration if win_duration is None else win_duration
186
+ )
187
+
188
+ if audio_signal.signal_duration <= win_duration:
189
+ # Unchunked compression (used if signal length < win duration)
190
+ self.padding = True
191
+ n_samples = nt
192
+ hop = nt
193
+ else:
194
+ # Chunked inference
195
+ self.padding = False
196
+ # Zero-pad signal on either side by the delay
197
+ audio_signal.zero_pad(self.delay, self.delay)
198
+ n_samples = int(win_duration * self.sample_rate)
199
+ # Round n_samples to nearest hop length multiple
200
+ n_samples = int(math.ceil(n_samples / self.hop_length) * self.hop_length)
201
+ hop = self.get_output_length(n_samples)
202
+
203
+ codes = []
204
+ range_fn = range if not verbose else tqdm.trange
205
+
206
+ for i in range_fn(0, nt, hop):
207
+ x = audio_signal[..., i : i + n_samples]
208
+ x = x.zero_pad(0, max(0, n_samples - x.shape[-1]))
209
+
210
+ audio_data = x.audio_data.to(self.device)
211
+ audio_data = self.preprocess(audio_data, self.sample_rate)
212
+ _, c, _, _, _ = self.encode(audio_data, n_quantizers)
213
+ codes.append(c.to(original_device))
214
+ chunk_length = c.shape[-1]
215
+
216
+ codes = torch.cat(codes, dim=-1)
217
+
218
+ dac_file = DACFile(
219
+ codes=codes,
220
+ chunk_length=chunk_length,
221
+ original_length=original_length,
222
+ input_db=input_db,
223
+ channels=nac,
224
+ sample_rate=original_sr,
225
+ padding=self.padding,
226
+ dac_version=SUPPORTED_VERSIONS[-1],
227
+ )
228
+
229
+ if n_quantizers is not None:
230
+ codes = codes[:, :n_quantizers, :]
231
+
232
+ self.padding = original_padding
233
+ return dac_file
234
+
235
+ @torch.no_grad()
236
+ def decompress(
237
+ self,
238
+ obj: Union[str, Path, DACFile],
239
+ verbose: bool = False,
240
+ ) -> AudioSignal:
241
+ """Reconstruct audio from a given .dac file
242
+
243
+ Parameters
244
+ ----------
245
+ obj : Union[str, Path, DACFile]
246
+ .dac file location or corresponding DACFile object.
247
+ verbose : bool, optional
248
+ Prints progress if True, by default False
249
+
250
+ Returns
251
+ -------
252
+ AudioSignal
253
+ Object with the reconstructed audio
254
+ """
255
+ self.eval()
256
+ if isinstance(obj, (str, Path)):
257
+ obj = DACFile.load(obj)
258
+
259
+ original_padding = self.padding
260
+ self.padding = obj.padding
261
+
262
+ range_fn = range if not verbose else tqdm.trange
263
+ codes = obj.codes
264
+ original_device = codes.device
265
+ chunk_length = obj.chunk_length
266
+ recons = []
267
+
268
+ for i in range_fn(0, codes.shape[-1], chunk_length):
269
+ c = codes[..., i : i + chunk_length].to(self.device)
270
+ z = self.quantizer.from_codes(c)[0]
271
+ r = self.decode(z)
272
+ recons.append(r.to(original_device))
273
+
274
+ recons = torch.cat(recons, dim=-1)
275
+ recons = AudioSignal(recons, self.sample_rate)
276
+
277
+ resample_fn = recons.resample
278
+ loudness_fn = recons.loudness
279
+
280
+ # If audio is > 10 minutes long, use the ffmpeg versions
281
+ if recons.signal_duration >= 10 * 60 * 60:
282
+ resample_fn = recons.ffmpeg_resample
283
+ loudness_fn = recons.ffmpeg_loudness
284
+
285
+ recons.normalize(obj.input_db)
286
+ resample_fn(obj.sample_rate)
287
+ recons = recons[..., : obj.original_length]
288
+ loudness_fn()
289
+ recons.audio_data = recons.audio_data.reshape(
290
+ -1, obj.channels, obj.original_length
291
+ )
292
+
293
+ self.padding = original_padding
294
+ return recons
songgen/xcodec_wrapper/xcodec_infer/descriptaudiocodec/dac/model/dac.py ADDED
@@ -0,0 +1,369 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import math
2
+ from typing import List
3
+ from typing import Union
4
+
5
+ import numpy as np
6
+ import torch
7
+ from audiotools import AudioSignal
8
+ from audiotools.ml import BaseModel
9
+ from torch import nn
10
+
11
+ from .base import CodecMixin
12
+ from dac.nn.layers import Snake1d
13
+ from dac.nn.layers import WNConv1d
14
+ from dac.nn.layers import WNConvTranspose1d
15
+ from dac.nn.quantize import ResidualVectorQuantize
16
+
17
+
18
+ def init_weights(m):
19
+ if isinstance(m, nn.Conv1d):
20
+ nn.init.trunc_normal_(m.weight, std=0.02)
21
+ nn.init.constant_(m.bias, 0)
22
+
23
+
24
+ class ResidualUnit(nn.Module):
25
+ def __init__(self, dim: int = 16, dilation: int = 1):
26
+ super().__init__()
27
+ pad = ((7 - 1) * dilation) // 2
28
+ self.block = nn.Sequential(
29
+ Snake1d(dim),
30
+ WNConv1d(dim, dim, kernel_size=7, dilation=dilation, padding=pad),
31
+ Snake1d(dim),
32
+ WNConv1d(dim, dim, kernel_size=1),
33
+ )
34
+
35
+ def forward(self, x):
36
+ y = self.block(x)
37
+ pad = (x.shape[-1] - y.shape[-1]) // 2
38
+ if pad > 0:
39
+ x = x[..., pad:-pad]
40
+ return x + y
41
+
42
+
43
+ class EncoderBlock(nn.Module):
44
+ def __init__(self, dim: int = 16, stride: int = 1):
45
+ super().__init__()
46
+ self.block = nn.Sequential(
47
+ ResidualUnit(dim // 2, dilation=1),
48
+ ResidualUnit(dim // 2, dilation=3),
49
+ ResidualUnit(dim // 2, dilation=9),
50
+ Snake1d(dim // 2),
51
+ WNConv1d(
52
+ dim // 2,
53
+ dim,
54
+ kernel_size=2 * stride,
55
+ stride=stride,
56
+ padding=math.ceil(stride / 2),
57
+ ),
58
+ )
59
+
60
+ def forward(self, x):
61
+ return self.block(x)
62
+
63
+
64
+ class Encoder(nn.Module):
65
+ def __init__(
66
+ self,
67
+ d_model: int = 64,
68
+ strides: list = [2, 4, 8, 8],
69
+ d_latent: int = 256,
70
+ ):
71
+ super().__init__()
72
+ # Create first convolution
73
+ self.block = [WNConv1d(1, d_model, kernel_size=7, padding=3)]
74
+
75
+ # Create EncoderBlocks that double channels as they downsample by `stride`
76
+ for stride in strides:
77
+ d_model *= 2
78
+ self.block += [EncoderBlock(d_model, stride=stride)]
79
+
80
+ # Create last convolution
81
+ self.block += [
82
+ Snake1d(d_model),
83
+ WNConv1d(d_model, d_latent, kernel_size=3, padding=1),
84
+ ]
85
+
86
+ # Wrap black into nn.Sequential
87
+ self.block = nn.Sequential(*self.block)
88
+ self.enc_dim = d_model
89
+
90
+ def forward(self, x):
91
+ return self.block(x)
92
+
93
+
94
+ class DecoderBlock(nn.Module):
95
+ def __init__(self, input_dim: int = 16, output_dim: int = 8, stride: int = 1,out_pad=0):
96
+ super().__init__()
97
+ self.block = nn.Sequential(
98
+ Snake1d(input_dim),
99
+ WNConvTranspose1d(
100
+ input_dim,
101
+ output_dim,
102
+ kernel_size=2 * stride,
103
+ stride=stride,
104
+ padding=math.ceil(stride / 2),
105
+ output_padding= stride % 2, #out_pad,
106
+ ),
107
+ ResidualUnit(output_dim, dilation=1),
108
+ ResidualUnit(output_dim, dilation=3),
109
+ ResidualUnit(output_dim, dilation=9),
110
+ )
111
+
112
+ def forward(self, x):
113
+ return self.block(x)
114
+
115
+
116
+ class Decoder(nn.Module):
117
+ def __init__(
118
+ self,
119
+ input_channel,
120
+ channels,
121
+ rates,
122
+ d_out: int = 1,
123
+ ):
124
+ super().__init__()
125
+
126
+ # Add first conv layer
127
+ layers = [WNConv1d(input_channel, channels, kernel_size=7, padding=3)]
128
+
129
+ # Add upsampling + MRF blocks
130
+ for i, stride in enumerate(rates):
131
+ input_dim = channels // 2**i
132
+ output_dim = channels // 2 ** (i + 1)
133
+ if i==1:
134
+ out_pad=1
135
+ else:
136
+ out_pad=0
137
+ layers += [DecoderBlock(input_dim, output_dim, stride,out_pad)]
138
+
139
+ # Add final conv layer
140
+ layers += [
141
+ Snake1d(output_dim),
142
+ WNConv1d(output_dim, d_out, kernel_size=7, padding=3),
143
+ # nn.Tanh(),
144
+ ]
145
+
146
+ self.model = nn.Sequential(*layers)
147
+
148
+ def forward(self, x):
149
+ return self.model(x)
150
+
151
+
152
+ class DAC(BaseModel, CodecMixin):
153
+ def __init__(
154
+ self,
155
+ encoder_dim: int = 64,
156
+ encoder_rates: List[int] = [2, 4, 8, 8],
157
+ latent_dim: int = None,
158
+ decoder_dim: int = 1536,
159
+ decoder_rates: List[int] = [8, 8, 4, 2],
160
+ n_codebooks: int = 9,
161
+ codebook_size: int = 1024,
162
+ codebook_dim: Union[int, list] = 8,
163
+ quantizer_dropout: bool = False,
164
+ sample_rate: int = 44100,
165
+ ):
166
+ super().__init__()
167
+
168
+ self.encoder_dim = encoder_dim
169
+ self.encoder_rates = encoder_rates
170
+ self.decoder_dim = decoder_dim
171
+ self.decoder_rates = decoder_rates
172
+ self.sample_rate = sample_rate
173
+
174
+ if latent_dim is None:
175
+ latent_dim = encoder_dim * (2 ** len(encoder_rates))
176
+
177
+ self.latent_dim = latent_dim
178
+
179
+ self.hop_length = np.prod(encoder_rates)
180
+ self.encoder = Encoder(encoder_dim, encoder_rates, latent_dim)
181
+
182
+ self.n_codebooks = n_codebooks
183
+ self.codebook_size = codebook_size
184
+ self.codebook_dim = codebook_dim
185
+ self.quantizer = ResidualVectorQuantize(
186
+ input_dim=latent_dim,
187
+ n_codebooks=n_codebooks,
188
+ codebook_size=codebook_size,
189
+ codebook_dim=codebook_dim,
190
+ quantizer_dropout=quantizer_dropout,
191
+ )
192
+
193
+ self.decoder = Decoder(
194
+ latent_dim,
195
+ decoder_dim,
196
+ decoder_rates,
197
+ )
198
+ self.sample_rate = sample_rate
199
+ self.apply(init_weights)
200
+
201
+ self.delay = self.get_delay()
202
+
203
+ def preprocess(self, audio_data, sample_rate):
204
+ if sample_rate is None:
205
+ sample_rate = self.sample_rate
206
+ assert sample_rate == self.sample_rate
207
+
208
+ length = audio_data.shape[-1]
209
+ right_pad = math.ceil(length / self.hop_length) * self.hop_length - length
210
+ audio_data = nn.functional.pad(audio_data, (0, right_pad))
211
+
212
+ return audio_data
213
+
214
+ def encode(
215
+ self,
216
+ audio_data: torch.Tensor,
217
+ n_quantizers: int = None,
218
+ ):
219
+ """Encode given audio data and return quantized latent codes
220
+
221
+ Parameters
222
+ ----------
223
+ audio_data : Tensor[B x 1 x T]
224
+ Audio data to encode
225
+ n_quantizers : int, optional
226
+ Number of quantizers to use, by default None
227
+ If None, all quantizers are used.
228
+
229
+ Returns
230
+ -------
231
+ dict
232
+ A dictionary with the following keys:
233
+ "z" : Tensor[B x D x T]
234
+ Quantized continuous representation of input
235
+ "codes" : Tensor[B x N x T]
236
+ Codebook indices for each codebook
237
+ (quantized discrete representation of input)
238
+ "latents" : Tensor[B x N*D x T]
239
+ Projected latents (continuous representation of input before quantization)
240
+ "vq/commitment_loss" : Tensor[1]
241
+ Commitment loss to train encoder to predict vectors closer to codebook
242
+ entries
243
+ "vq/codebook_loss" : Tensor[1]
244
+ Codebook loss to update the codebook
245
+ "length" : int
246
+ Number of samples in input audio
247
+ """
248
+ z = self.encoder(audio_data)
249
+ z, codes, latents, commitment_loss, codebook_loss = self.quantizer(
250
+ z, n_quantizers
251
+ )
252
+ return z, codes, latents, commitment_loss, codebook_loss
253
+
254
+ def decode(self, z: torch.Tensor):
255
+ """Decode given latent codes and return audio data
256
+
257
+ Parameters
258
+ ----------
259
+ z : Tensor[B x D x T]
260
+ Quantized continuous representation of input
261
+ length : int, optional
262
+ Number of samples in output audio, by default None
263
+
264
+ Returns
265
+ -------
266
+ dict
267
+ A dictionary with the following keys:
268
+ "audio" : Tensor[B x 1 x length]
269
+ Decoded audio data.
270
+ """
271
+ return self.decoder(z)
272
+
273
+ def forward(
274
+ self,
275
+ audio_data: torch.Tensor,
276
+ sample_rate: int = None,
277
+ n_quantizers: int = None,
278
+ ):
279
+ """Model forward pass
280
+
281
+ Parameters
282
+ ----------
283
+ audio_data : Tensor[B x 1 x T]
284
+ Audio data to encode
285
+ sample_rate : int, optional
286
+ Sample rate of audio data in Hz, by default None
287
+ If None, defaults to `self.sample_rate`
288
+ n_quantizers : int, optional
289
+ Number of quantizers to use, by default None.
290
+ If None, all quantizers are used.
291
+
292
+ Returns
293
+ -------
294
+ dict
295
+ A dictionary with the following keys:
296
+ "z" : Tensor[B x D x T]
297
+ Quantized continuous representation of input
298
+ "codes" : Tensor[B x N x T]
299
+ Codebook indices for each codebook
300
+ (quantized discrete representation of input)
301
+ "latents" : Tensor[B x N*D x T]
302
+ Projected latents (continuous representation of input before quantization)
303
+ "vq/commitment_loss" : Tensor[1]
304
+ Commitment loss to train encoder to predict vectors closer to codebook
305
+ entries
306
+ "vq/codebook_loss" : Tensor[1]
307
+ Codebook loss to update the codebook
308
+ "length" : int
309
+ Number of samples in input audio
310
+ "audio" : Tensor[B x 1 x length]
311
+ Decoded audio data.
312
+ """
313
+ length = audio_data.shape[-1]
314
+ audio_data = self.preprocess(audio_data, sample_rate)
315
+ z, codes, latents, commitment_loss, codebook_loss = self.encode(
316
+ audio_data, n_quantizers
317
+ )
318
+
319
+ x = self.decode(z)
320
+ return {
321
+ "audio": x[..., :length],
322
+ "z": z,
323
+ "codes": codes,
324
+ "latents": latents,
325
+ "vq/commitment_loss": commitment_loss,
326
+ "vq/codebook_loss": codebook_loss,
327
+ }
328
+
329
+
330
+ if __name__ == "__main__":
331
+ import numpy as np
332
+ from functools import partial
333
+
334
+ model = DAC().to("cpu")
335
+
336
+ for n, m in model.named_modules():
337
+ o = m.extra_repr()
338
+ p = sum([np.prod(p.size()) for p in m.parameters()])
339
+ fn = lambda o, p: o + f" {p/1e6:<.3f}M params."
340
+ setattr(m, "extra_repr", partial(fn, o=o, p=p))
341
+ print(model)
342
+ print("Total # of params: ", sum([np.prod(p.size()) for p in model.parameters()]))
343
+
344
+ length = 88200 * 2
345
+ x = torch.randn(1, 1, length).to(model.device)
346
+ x.requires_grad_(True)
347
+ x.retain_grad()
348
+
349
+ # Make a forward pass
350
+ out = model(x)["audio"]
351
+ print("Input shape:", x.shape)
352
+ print("Output shape:", out.shape)
353
+
354
+ # Create gradient variable
355
+ grad = torch.zeros_like(out)
356
+ grad[:, :, grad.shape[-1] // 2] = 1
357
+
358
+ # Make a backward pass
359
+ out.backward(grad)
360
+
361
+ # Check non-zero values
362
+ gradmap = x.grad.squeeze(0)
363
+ gradmap = (gradmap != 0).sum(0) # sum across features
364
+ rf = (gradmap != 0).sum()
365
+
366
+ print(f"Receptive field: {rf.item()}")
367
+
368
+ x = AudioSignal(torch.randn(1, 1, 44100 * 60), 44100)
369
+ model.decompress(model.compress(x, verbose=True), verbose=True)
songgen/xcodec_wrapper/xcodec_infer/descriptaudiocodec/dac/model/discriminator.py ADDED
@@ -0,0 +1,228 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import torch.nn as nn
3
+ import torch.nn.functional as F
4
+ from audiotools import AudioSignal
5
+ from audiotools import ml
6
+ from audiotools import STFTParams
7
+ from einops import rearrange
8
+ from torch.nn.utils import weight_norm
9
+
10
+
11
+ def WNConv1d(*args, **kwargs):
12
+ act = kwargs.pop("act", True)
13
+ conv = weight_norm(nn.Conv1d(*args, **kwargs))
14
+ if not act:
15
+ return conv
16
+ return nn.Sequential(conv, nn.LeakyReLU(0.1))
17
+
18
+
19
+ def WNConv2d(*args, **kwargs):
20
+ act = kwargs.pop("act", True)
21
+ conv = weight_norm(nn.Conv2d(*args, **kwargs))
22
+ if not act:
23
+ return conv
24
+ return nn.Sequential(conv, nn.LeakyReLU(0.1))
25
+
26
+
27
+ class MPD(nn.Module):
28
+ def __init__(self, period):
29
+ super().__init__()
30
+ self.period = period
31
+ self.convs = nn.ModuleList(
32
+ [
33
+ WNConv2d(1, 32, (5, 1), (3, 1), padding=(2, 0)),
34
+ WNConv2d(32, 128, (5, 1), (3, 1), padding=(2, 0)),
35
+ WNConv2d(128, 512, (5, 1), (3, 1), padding=(2, 0)),
36
+ WNConv2d(512, 1024, (5, 1), (3, 1), padding=(2, 0)),
37
+ WNConv2d(1024, 1024, (5, 1), 1, padding=(2, 0)),
38
+ ]
39
+ )
40
+ self.conv_post = WNConv2d(
41
+ 1024, 1, kernel_size=(3, 1), padding=(1, 0), act=False
42
+ )
43
+
44
+ def pad_to_period(self, x):
45
+ t = x.shape[-1]
46
+ x = F.pad(x, (0, self.period - t % self.period), mode="reflect")
47
+ return x
48
+
49
+ def forward(self, x):
50
+ fmap = []
51
+
52
+ x = self.pad_to_period(x)
53
+ x = rearrange(x, "b c (l p) -> b c l p", p=self.period)
54
+
55
+ for layer in self.convs:
56
+ x = layer(x)
57
+ fmap.append(x)
58
+
59
+ x = self.conv_post(x)
60
+ fmap.append(x)
61
+
62
+ return fmap
63
+
64
+
65
+ class MSD(nn.Module):
66
+ def __init__(self, rate: int = 1, sample_rate: int = 44100):
67
+ super().__init__()
68
+ self.convs = nn.ModuleList(
69
+ [
70
+ WNConv1d(1, 16, 15, 1, padding=7),
71
+ WNConv1d(16, 64, 41, 4, groups=4, padding=20),
72
+ WNConv1d(64, 256, 41, 4, groups=16, padding=20),
73
+ WNConv1d(256, 1024, 41, 4, groups=64, padding=20),
74
+ WNConv1d(1024, 1024, 41, 4, groups=256, padding=20),
75
+ WNConv1d(1024, 1024, 5, 1, padding=2),
76
+ ]
77
+ )
78
+ self.conv_post = WNConv1d(1024, 1, 3, 1, padding=1, act=False)
79
+ self.sample_rate = sample_rate
80
+ self.rate = rate
81
+
82
+ def forward(self, x):
83
+ x = AudioSignal(x, self.sample_rate)
84
+ x.resample(self.sample_rate // self.rate)
85
+ x = x.audio_data
86
+
87
+ fmap = []
88
+
89
+ for l in self.convs:
90
+ x = l(x)
91
+ fmap.append(x)
92
+ x = self.conv_post(x)
93
+ fmap.append(x)
94
+
95
+ return fmap
96
+
97
+
98
+ BANDS = [(0.0, 0.1), (0.1, 0.25), (0.25, 0.5), (0.5, 0.75), (0.75, 1.0)]
99
+
100
+
101
+ class MRD(nn.Module):
102
+ def __init__(
103
+ self,
104
+ window_length: int,
105
+ hop_factor: float = 0.25,
106
+ sample_rate: int = 44100,
107
+ bands: list = BANDS,
108
+ ):
109
+ """Complex multi-band spectrogram discriminator.
110
+ Parameters
111
+ ----------
112
+ window_length : int
113
+ Window length of STFT.
114
+ hop_factor : float, optional
115
+ Hop factor of the STFT, defaults to ``0.25 * window_length``.
116
+ sample_rate : int, optional
117
+ Sampling rate of audio in Hz, by default 44100
118
+ bands : list, optional
119
+ Bands to run discriminator over.
120
+ """
121
+ super().__init__()
122
+
123
+ self.window_length = window_length
124
+ self.hop_factor = hop_factor
125
+ self.sample_rate = sample_rate
126
+ self.stft_params = STFTParams(
127
+ window_length=window_length,
128
+ hop_length=int(window_length * hop_factor),
129
+ match_stride=True,
130
+ )
131
+
132
+ n_fft = window_length // 2 + 1
133
+ bands = [(int(b[0] * n_fft), int(b[1] * n_fft)) for b in bands]
134
+ self.bands = bands
135
+
136
+ ch = 32
137
+ convs = lambda: nn.ModuleList(
138
+ [
139
+ WNConv2d(2, ch, (3, 9), (1, 1), padding=(1, 4)),
140
+ WNConv2d(ch, ch, (3, 9), (1, 2), padding=(1, 4)),
141
+ WNConv2d(ch, ch, (3, 9), (1, 2), padding=(1, 4)),
142
+ WNConv2d(ch, ch, (3, 9), (1, 2), padding=(1, 4)),
143
+ WNConv2d(ch, ch, (3, 3), (1, 1), padding=(1, 1)),
144
+ ]
145
+ )
146
+ self.band_convs = nn.ModuleList([convs() for _ in range(len(self.bands))])
147
+ self.conv_post = WNConv2d(ch, 1, (3, 3), (1, 1), padding=(1, 1), act=False)
148
+
149
+ def spectrogram(self, x):
150
+ x = AudioSignal(x, self.sample_rate, stft_params=self.stft_params)
151
+ x = torch.view_as_real(x.stft())
152
+ x = rearrange(x, "b 1 f t c -> (b 1) c t f")
153
+ # Split into bands
154
+ x_bands = [x[..., b[0] : b[1]] for b in self.bands]
155
+ return x_bands
156
+
157
+ def forward(self, x):
158
+ x_bands = self.spectrogram(x)
159
+ fmap = []
160
+
161
+ x = []
162
+ for band, stack in zip(x_bands, self.band_convs):
163
+ for layer in stack:
164
+ band = layer(band)
165
+ fmap.append(band)
166
+ x.append(band)
167
+
168
+ x = torch.cat(x, dim=-1)
169
+ x = self.conv_post(x)
170
+ fmap.append(x)
171
+
172
+ return fmap
173
+
174
+
175
+ class Discriminator(ml.BaseModel):
176
+ def __init__(
177
+ self,
178
+ rates: list = [],
179
+ periods: list = [2, 3, 5, 7, 11],
180
+ fft_sizes: list = [2048, 1024, 512],
181
+ sample_rate: int = 44100,
182
+ bands: list = BANDS,
183
+ ):
184
+ """Discriminator that combines multiple discriminators.
185
+
186
+ Parameters
187
+ ----------
188
+ rates : list, optional
189
+ sampling rates (in Hz) to run MSD at, by default []
190
+ If empty, MSD is not used.
191
+ periods : list, optional
192
+ periods (of samples) to run MPD at, by default [2, 3, 5, 7, 11]
193
+ fft_sizes : list, optional
194
+ Window sizes of the FFT to run MRD at, by default [2048, 1024, 512]
195
+ sample_rate : int, optional
196
+ Sampling rate of audio in Hz, by default 44100
197
+ bands : list, optional
198
+ Bands to run MRD at, by default `BANDS`
199
+ """
200
+ super().__init__()
201
+ discs = []
202
+ discs += [MPD(p) for p in periods]
203
+ discs += [MSD(r, sample_rate=sample_rate) for r in rates]
204
+ discs += [MRD(f, sample_rate=sample_rate, bands=bands) for f in fft_sizes]
205
+ self.discriminators = nn.ModuleList(discs)
206
+
207
+ def preprocess(self, y):
208
+ # Remove DC offset
209
+ y = y - y.mean(dim=-1, keepdims=True)
210
+ # Peak normalize the volume of input audio
211
+ y = 0.8 * y / (y.abs().max(dim=-1, keepdim=True)[0] + 1e-9)
212
+ return y
213
+
214
+ def forward(self, x):
215
+ x = self.preprocess(x)
216
+ fmaps = [d(x) for d in self.discriminators]
217
+ return fmaps
218
+
219
+
220
+ if __name__ == "__main__":
221
+ disc = Discriminator()
222
+ x = torch.zeros(1, 1, 44100)
223
+ results = disc(x)
224
+ for i, result in enumerate(results):
225
+ print(f"disc{i}")
226
+ for i, r in enumerate(result):
227
+ print(r.shape, r.mean(), r.min(), r.max())
228
+ print()
songgen/xcodec_wrapper/xcodec_infer/descriptaudiocodec/dac/nn/__init__.py ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ from . import layers
2
+ from . import loss
3
+ from . import quantize
songgen/xcodec_wrapper/xcodec_infer/descriptaudiocodec/dac/nn/layers.py ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import numpy as np
2
+ import torch
3
+ import torch.nn as nn
4
+ import torch.nn.functional as F
5
+ from einops import rearrange
6
+ from torch.nn.utils import weight_norm
7
+
8
+
9
+ def WNConv1d(*args, **kwargs):
10
+ return weight_norm(nn.Conv1d(*args, **kwargs))
11
+
12
+
13
+ def WNConvTranspose1d(*args, **kwargs):
14
+ return weight_norm(nn.ConvTranspose1d(*args, **kwargs))
15
+
16
+
17
+ # Scripting this brings model speed up 1.4x
18
+ @torch.jit.script
19
+ def snake(x, alpha):
20
+ shape = x.shape
21
+ x = x.reshape(shape[0], shape[1], -1)
22
+ x = x + (alpha + 1e-9).reciprocal() * torch.sin(alpha * x).pow(2)
23
+ x = x.reshape(shape)
24
+ return x
25
+
26
+
27
+ class Snake1d(nn.Module):
28
+ def __init__(self, channels):
29
+ super().__init__()
30
+ self.alpha = nn.Parameter(torch.ones(1, channels, 1))
31
+
32
+ def forward(self, x):
33
+ return snake(x, self.alpha)
songgen/xcodec_wrapper/xcodec_infer/descriptaudiocodec/dac/nn/loss.py ADDED
@@ -0,0 +1,368 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import typing
2
+ from typing import List
3
+
4
+ import torch
5
+ import torch.nn.functional as F
6
+ from audiotools import AudioSignal
7
+ from audiotools import STFTParams
8
+ from torch import nn
9
+
10
+
11
+ class L1Loss(nn.L1Loss):
12
+ """L1 Loss between AudioSignals. Defaults
13
+ to comparing ``audio_data``, but any
14
+ attribute of an AudioSignal can be used.
15
+
16
+ Parameters
17
+ ----------
18
+ attribute : str, optional
19
+ Attribute of signal to compare, defaults to ``audio_data``.
20
+ weight : float, optional
21
+ Weight of this loss, defaults to 1.0.
22
+
23
+ Implementation copied from: https://github.com/descriptinc/lyrebird-audiotools/blob/961786aa1a9d628cca0c0486e5885a457fe70c1a/audiotools/metrics/distance.py
24
+ """
25
+
26
+ def __init__(self, attribute: str = "audio_data", weight: float = 1.0, **kwargs):
27
+ self.attribute = attribute
28
+ self.weight = weight
29
+ super().__init__(**kwargs)
30
+
31
+ def forward(self, x: AudioSignal, y: AudioSignal):
32
+ """
33
+ Parameters
34
+ ----------
35
+ x : AudioSignal
36
+ Estimate AudioSignal
37
+ y : AudioSignal
38
+ Reference AudioSignal
39
+
40
+ Returns
41
+ -------
42
+ torch.Tensor
43
+ L1 loss between AudioSignal attributes.
44
+ """
45
+ if isinstance(x, AudioSignal):
46
+ x = getattr(x, self.attribute)
47
+ y = getattr(y, self.attribute)
48
+ return super().forward(x, y)
49
+
50
+
51
+ class SISDRLoss(nn.Module):
52
+ """
53
+ Computes the Scale-Invariant Source-to-Distortion Ratio between a batch
54
+ of estimated and reference audio signals or aligned features.
55
+
56
+ Parameters
57
+ ----------
58
+ scaling : int, optional
59
+ Whether to use scale-invariant (True) or
60
+ signal-to-noise ratio (False), by default True
61
+ reduction : str, optional
62
+ How to reduce across the batch (either 'mean',
63
+ 'sum', or none).], by default ' mean'
64
+ zero_mean : int, optional
65
+ Zero mean the references and estimates before
66
+ computing the loss, by default True
67
+ clip_min : int, optional
68
+ The minimum possible loss value. Helps network
69
+ to not focus on making already good examples better, by default None
70
+ weight : float, optional
71
+ Weight of this loss, defaults to 1.0.
72
+
73
+ Implementation copied from: https://github.com/descriptinc/lyrebird-audiotools/blob/961786aa1a9d628cca0c0486e5885a457fe70c1a/audiotools/metrics/distance.py
74
+ """
75
+
76
+ def __init__(
77
+ self,
78
+ scaling: int = True,
79
+ reduction: str = "mean",
80
+ zero_mean: int = True,
81
+ clip_min: int = None,
82
+ weight: float = 1.0,
83
+ ):
84
+ self.scaling = scaling
85
+ self.reduction = reduction
86
+ self.zero_mean = zero_mean
87
+ self.clip_min = clip_min
88
+ self.weight = weight
89
+ super().__init__()
90
+
91
+ def forward(self, x: AudioSignal, y: AudioSignal):
92
+ eps = 1e-8
93
+ # nb, nc, nt
94
+ if isinstance(x, AudioSignal):
95
+ references = x.audio_data
96
+ estimates = y.audio_data
97
+ else:
98
+ references = x
99
+ estimates = y
100
+
101
+ nb = references.shape[0]
102
+ references = references.reshape(nb, 1, -1).permute(0, 2, 1)
103
+ estimates = estimates.reshape(nb, 1, -1).permute(0, 2, 1)
104
+
105
+ # samples now on axis 1
106
+ if self.zero_mean:
107
+ mean_reference = references.mean(dim=1, keepdim=True)
108
+ mean_estimate = estimates.mean(dim=1, keepdim=True)
109
+ else:
110
+ mean_reference = 0
111
+ mean_estimate = 0
112
+
113
+ _references = references - mean_reference
114
+ _estimates = estimates - mean_estimate
115
+
116
+ references_projection = (_references**2).sum(dim=-2) + eps
117
+ references_on_estimates = (_estimates * _references).sum(dim=-2) + eps
118
+
119
+ scale = (
120
+ (references_on_estimates / references_projection).unsqueeze(1)
121
+ if self.scaling
122
+ else 1
123
+ )
124
+
125
+ e_true = scale * _references
126
+ e_res = _estimates - e_true
127
+
128
+ signal = (e_true**2).sum(dim=1)
129
+ noise = (e_res**2).sum(dim=1)
130
+ sdr = -10 * torch.log10(signal / noise + eps)
131
+
132
+ if self.clip_min is not None:
133
+ sdr = torch.clamp(sdr, min=self.clip_min)
134
+
135
+ if self.reduction == "mean":
136
+ sdr = sdr.mean()
137
+ elif self.reduction == "sum":
138
+ sdr = sdr.sum()
139
+ return sdr
140
+
141
+
142
+ class MultiScaleSTFTLoss(nn.Module):
143
+ """Computes the multi-scale STFT loss from [1].
144
+
145
+ Parameters
146
+ ----------
147
+ window_lengths : List[int], optional
148
+ Length of each window of each STFT, by default [2048, 512]
149
+ loss_fn : typing.Callable, optional
150
+ How to compare each loss, by default nn.L1Loss()
151
+ clamp_eps : float, optional
152
+ Clamp on the log magnitude, below, by default 1e-5
153
+ mag_weight : float, optional
154
+ Weight of raw magnitude portion of loss, by default 1.0
155
+ log_weight : float, optional
156
+ Weight of log magnitude portion of loss, by default 1.0
157
+ pow : float, optional
158
+ Power to raise magnitude to before taking log, by default 2.0
159
+ weight : float, optional
160
+ Weight of this loss, by default 1.0
161
+ match_stride : bool, optional
162
+ Whether to match the stride of convolutional layers, by default False
163
+
164
+ References
165
+ ----------
166
+
167
+ 1. Engel, Jesse, Chenjie Gu, and Adam Roberts.
168
+ "DDSP: Differentiable Digital Signal Processing."
169
+ International Conference on Learning Representations. 2019.
170
+
171
+ Implementation copied from: https://github.com/descriptinc/lyrebird-audiotools/blob/961786aa1a9d628cca0c0486e5885a457fe70c1a/audiotools/metrics/spectral.py
172
+ """
173
+
174
+ def __init__(
175
+ self,
176
+ window_lengths: List[int] = [2048, 512],
177
+ loss_fn: typing.Callable = nn.L1Loss(),
178
+ clamp_eps: float = 1e-5,
179
+ mag_weight: float = 1.0,
180
+ log_weight: float = 1.0,
181
+ pow: float = 2.0,
182
+ weight: float = 1.0,
183
+ match_stride: bool = False,
184
+ window_type: str = None,
185
+ ):
186
+ super().__init__()
187
+ self.stft_params = [
188
+ STFTParams(
189
+ window_length=w,
190
+ hop_length=w // 4,
191
+ match_stride=match_stride,
192
+ window_type=window_type,
193
+ )
194
+ for w in window_lengths
195
+ ]
196
+ self.loss_fn = loss_fn
197
+ self.log_weight = log_weight
198
+ self.mag_weight = mag_weight
199
+ self.clamp_eps = clamp_eps
200
+ self.weight = weight
201
+ self.pow = pow
202
+
203
+ def forward(self, x: AudioSignal, y: AudioSignal):
204
+ """Computes multi-scale STFT between an estimate and a reference
205
+ signal.
206
+
207
+ Parameters
208
+ ----------
209
+ x : AudioSignal
210
+ Estimate signal
211
+ y : AudioSignal
212
+ Reference signal
213
+
214
+ Returns
215
+ -------
216
+ torch.Tensor
217
+ Multi-scale STFT loss.
218
+ """
219
+ loss = 0.0
220
+ for s in self.stft_params:
221
+ x.stft(s.window_length, s.hop_length, s.window_type)
222
+ y.stft(s.window_length, s.hop_length, s.window_type)
223
+ loss += self.log_weight * self.loss_fn(
224
+ x.magnitude.clamp(self.clamp_eps).pow(self.pow).log10(),
225
+ y.magnitude.clamp(self.clamp_eps).pow(self.pow).log10(),
226
+ )
227
+ loss += self.mag_weight * self.loss_fn(x.magnitude, y.magnitude)
228
+ return loss
229
+
230
+
231
+ class MelSpectrogramLoss(nn.Module):
232
+ """Compute distance between mel spectrograms. Can be used
233
+ in a multi-scale way.
234
+
235
+ Parameters
236
+ ----------
237
+ n_mels : List[int]
238
+ Number of mels per STFT, by default [150, 80],
239
+ window_lengths : List[int], optional
240
+ Length of each window of each STFT, by default [2048, 512]
241
+ loss_fn : typing.Callable, optional
242
+ How to compare each loss, by default nn.L1Loss()
243
+ clamp_eps : float, optional
244
+ Clamp on the log magnitude, below, by default 1e-5
245
+ mag_weight : float, optional
246
+ Weight of raw magnitude portion of loss, by default 1.0
247
+ log_weight : float, optional
248
+ Weight of log magnitude portion of loss, by default 1.0
249
+ pow : float, optional
250
+ Power to raise magnitude to before taking log, by default 2.0
251
+ weight : float, optional
252
+ Weight of this loss, by default 1.0
253
+ match_stride : bool, optional
254
+ Whether to match the stride of convolutional layers, by default False
255
+
256
+ Implementation copied from: https://github.com/descriptinc/lyrebird-audiotools/blob/961786aa1a9d628cca0c0486e5885a457fe70c1a/audiotools/metrics/spectral.py
257
+ """
258
+
259
+ def __init__(
260
+ self,
261
+ n_mels: List[int] = [150, 80],
262
+ window_lengths: List[int] = [2048, 512],
263
+ loss_fn: typing.Callable = nn.L1Loss(),
264
+ clamp_eps: float = 1e-5,
265
+ mag_weight: float = 1.0,
266
+ log_weight: float = 1.0,
267
+ pow: float = 2.0,
268
+ weight: float = 1.0,
269
+ match_stride: bool = False,
270
+ mel_fmin: List[float] = [0.0, 0.0],
271
+ mel_fmax: List[float] = [None, None],
272
+ window_type: str = None,
273
+ ):
274
+ super().__init__()
275
+ self.stft_params = [
276
+ STFTParams(
277
+ window_length=w,
278
+ hop_length=w // 4,
279
+ match_stride=match_stride,
280
+ window_type=window_type,
281
+ )
282
+ for w in window_lengths
283
+ ]
284
+ self.n_mels = n_mels
285
+ self.loss_fn = loss_fn
286
+ self.clamp_eps = clamp_eps
287
+ self.log_weight = log_weight
288
+ self.mag_weight = mag_weight
289
+ self.weight = weight
290
+ self.mel_fmin = mel_fmin
291
+ self.mel_fmax = mel_fmax
292
+ self.pow = pow
293
+
294
+ def forward(self, x: AudioSignal, y: AudioSignal):
295
+ """Computes mel loss between an estimate and a reference
296
+ signal.
297
+
298
+ Parameters
299
+ ----------
300
+ x : AudioSignal
301
+ Estimate signal
302
+ y : AudioSignal
303
+ Reference signal
304
+
305
+ Returns
306
+ -------
307
+ torch.Tensor
308
+ Mel loss.
309
+ """
310
+ loss = 0.0
311
+ for n_mels, fmin, fmax, s in zip(
312
+ self.n_mels, self.mel_fmin, self.mel_fmax, self.stft_params
313
+ ):
314
+ kwargs = {
315
+ "window_length": s.window_length,
316
+ "hop_length": s.hop_length,
317
+ "window_type": s.window_type,
318
+ }
319
+ x_mels = x.mel_spectrogram(n_mels, mel_fmin=fmin, mel_fmax=fmax, **kwargs)
320
+ y_mels = y.mel_spectrogram(n_mels, mel_fmin=fmin, mel_fmax=fmax, **kwargs)
321
+
322
+ loss += self.log_weight * self.loss_fn(
323
+ x_mels.clamp(self.clamp_eps).pow(self.pow).log10(),
324
+ y_mels.clamp(self.clamp_eps).pow(self.pow).log10(),
325
+ )
326
+ loss += self.mag_weight * self.loss_fn(x_mels, y_mels)
327
+ return loss
328
+
329
+
330
+ class GANLoss(nn.Module):
331
+ """
332
+ Computes a discriminator loss, given a discriminator on
333
+ generated waveforms/spectrograms compared to ground truth
334
+ waveforms/spectrograms. Computes the loss for both the
335
+ discriminator and the generator in separate functions.
336
+ """
337
+
338
+ def __init__(self, discriminator):
339
+ super().__init__()
340
+ self.discriminator = discriminator
341
+
342
+ def forward(self, fake, real):
343
+ d_fake = self.discriminator(fake.audio_data)
344
+ d_real = self.discriminator(real.audio_data)
345
+ return d_fake, d_real
346
+
347
+ def discriminator_loss(self, fake, real):
348
+ d_fake, d_real = self.forward(fake.clone().detach(), real)
349
+
350
+ loss_d = 0
351
+ for x_fake, x_real in zip(d_fake, d_real):
352
+ loss_d += torch.mean(x_fake[-1] ** 2)
353
+ loss_d += torch.mean((1 - x_real[-1]) ** 2)
354
+ return loss_d
355
+
356
+ def generator_loss(self, fake, real):
357
+ d_fake, d_real = self.forward(fake, real)
358
+
359
+ loss_g = 0
360
+ for x_fake in d_fake:
361
+ loss_g += torch.mean((1 - x_fake[-1]) ** 2)
362
+
363
+ loss_feature = 0
364
+
365
+ for i in range(len(d_fake)):
366
+ for j in range(len(d_fake[i]) - 1):
367
+ loss_feature += F.l1_loss(d_fake[i][j], d_real[i][j].detach())
368
+ return loss_g+2* loss_feature
songgen/xcodec_wrapper/xcodec_infer/descriptaudiocodec/dac/nn/quantize.py ADDED
@@ -0,0 +1,262 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Union
2
+
3
+ import numpy as np
4
+ import torch
5
+ import torch.nn as nn
6
+ import torch.nn.functional as F
7
+ from einops import rearrange
8
+ from torch.nn.utils import weight_norm
9
+
10
+ from dac.nn.layers import WNConv1d
11
+
12
+
13
+ class VectorQuantize(nn.Module):
14
+ """
15
+ Implementation of VQ similar to Karpathy's repo:
16
+ https://github.com/karpathy/deep-vector-quantization
17
+ Additionally uses following tricks from Improved VQGAN
18
+ (https://arxiv.org/pdf/2110.04627.pdf):
19
+ 1. Factorized codes: Perform nearest neighbor lookup in low-dimensional space
20
+ for improved codebook usage
21
+ 2. l2-normalized codes: Converts euclidean distance to cosine similarity which
22
+ improves training stability
23
+ """
24
+
25
+ def __init__(self, input_dim: int, codebook_size: int, codebook_dim: int):
26
+ super().__init__()
27
+ self.codebook_size = codebook_size
28
+ self.codebook_dim = codebook_dim
29
+
30
+ self.in_proj = WNConv1d(input_dim, codebook_dim, kernel_size=1)
31
+ self.out_proj = WNConv1d(codebook_dim, input_dim, kernel_size=1)
32
+ self.codebook = nn.Embedding(codebook_size, codebook_dim)
33
+
34
+ def forward(self, z):
35
+ """Quantized the input tensor using a fixed codebook and returns
36
+ the corresponding codebook vectors
37
+
38
+ Parameters
39
+ ----------
40
+ z : Tensor[B x D x T]
41
+
42
+ Returns
43
+ -------
44
+ Tensor[B x D x T]
45
+ Quantized continuous representation of input
46
+ Tensor[1]
47
+ Commitment loss to train encoder to predict vectors closer to codebook
48
+ entries
49
+ Tensor[1]
50
+ Codebook loss to update the codebook
51
+ Tensor[B x T]
52
+ Codebook indices (quantized discrete representation of input)
53
+ Tensor[B x D x T]
54
+ Projected latents (continuous representation of input before quantization)
55
+ """
56
+
57
+ # Factorized codes (ViT-VQGAN) Project input into low-dimensional space
58
+ z_e = self.in_proj(z) # z_e : (B x D x T)
59
+ z_q, indices = self.decode_latents(z_e)
60
+
61
+ commitment_loss = F.mse_loss(z_e, z_q.detach(), reduction="none").mean([1, 2])
62
+ codebook_loss = F.mse_loss(z_q, z_e.detach(), reduction="none").mean([1, 2])
63
+
64
+ z_q = (
65
+ z_e + (z_q - z_e).detach()
66
+ ) # noop in forward pass, straight-through gradient estimator in backward pass
67
+
68
+ z_q = self.out_proj(z_q)
69
+
70
+ return z_q, commitment_loss, codebook_loss, indices, z_e
71
+
72
+ def embed_code(self, embed_id):
73
+ return F.embedding(embed_id, self.codebook.weight)
74
+
75
+ def decode_code(self, embed_id):
76
+ return self.embed_code(embed_id).transpose(1, 2)
77
+
78
+ def decode_latents(self, latents):
79
+ encodings = rearrange(latents, "b d t -> (b t) d")
80
+ codebook = self.codebook.weight # codebook: (N x D)
81
+
82
+ # L2 normalize encodings and codebook (ViT-VQGAN)
83
+ encodings = F.normalize(encodings)
84
+ codebook = F.normalize(codebook)
85
+
86
+ # Compute euclidean distance with codebook
87
+ dist = (
88
+ encodings.pow(2).sum(1, keepdim=True)
89
+ - 2 * encodings @ codebook.t()
90
+ + codebook.pow(2).sum(1, keepdim=True).t()
91
+ )
92
+ indices = rearrange((-dist).max(1)[1], "(b t) -> b t", b=latents.size(0))
93
+ z_q = self.decode_code(indices)
94
+ return z_q, indices
95
+
96
+
97
+ class ResidualVectorQuantize(nn.Module):
98
+ """
99
+ Introduced in SoundStream: An end2end neural audio codec
100
+ https://arxiv.org/abs/2107.03312
101
+ """
102
+
103
+ def __init__(
104
+ self,
105
+ input_dim: int = 512,
106
+ n_codebooks: int = 9,
107
+ codebook_size: int = 1024,
108
+ codebook_dim: Union[int, list] = 8,
109
+ quantizer_dropout: float = 0.0,
110
+ ):
111
+ super().__init__()
112
+ if isinstance(codebook_dim, int):
113
+ codebook_dim = [codebook_dim for _ in range(n_codebooks)]
114
+
115
+ self.n_codebooks = n_codebooks
116
+ self.codebook_dim = codebook_dim
117
+ self.codebook_size = codebook_size
118
+
119
+ self.quantizers = nn.ModuleList(
120
+ [
121
+ VectorQuantize(input_dim, codebook_size, codebook_dim[i])
122
+ for i in range(n_codebooks)
123
+ ]
124
+ )
125
+ self.quantizer_dropout = quantizer_dropout
126
+
127
+ def forward(self, z, n_quantizers: int = None):
128
+ """Quantized the input tensor using a fixed set of `n` codebooks and returns
129
+ the corresponding codebook vectors
130
+ Parameters
131
+ ----------
132
+ z : Tensor[B x D x T]
133
+ n_quantizers : int, optional
134
+ No. of quantizers to use
135
+ (n_quantizers < self.n_codebooks ex: for quantizer dropout)
136
+ Note: if `self.quantizer_dropout` is True, this argument is ignored
137
+ when in training mode, and a random number of quantizers is used.
138
+ Returns
139
+ -------
140
+ dict
141
+ A dictionary with the following keys:
142
+
143
+ "z" : Tensor[B x D x T]
144
+ Quantized continuous representation of input
145
+ "codes" : Tensor[B x N x T]
146
+ Codebook indices for each codebook
147
+ (quantized discrete representation of input)
148
+ "latents" : Tensor[B x N*D x T]
149
+ Projected latents (continuous representation of input before quantization)
150
+ "vq/commitment_loss" : Tensor[1]
151
+ Commitment loss to train encoder to predict vectors closer to codebook
152
+ entries
153
+ "vq/codebook_loss" : Tensor[1]
154
+ Codebook loss to update the codebook
155
+ """
156
+ z_q = 0
157
+ residual = z
158
+ commitment_loss = 0
159
+ codebook_loss = 0
160
+
161
+ codebook_indices = []
162
+ latents = []
163
+
164
+ if n_quantizers is None:
165
+ n_quantizers = self.n_codebooks
166
+ if self.training:
167
+ n_quantizers = torch.ones((z.shape[0],)) * self.n_codebooks + 1
168
+ dropout = torch.randint(1, self.n_codebooks + 1, (z.shape[0],))
169
+ n_dropout = int(z.shape[0] * self.quantizer_dropout)
170
+ n_quantizers[:n_dropout] = dropout[:n_dropout]
171
+ n_quantizers = n_quantizers.to(z.device)
172
+
173
+ for i, quantizer in enumerate(self.quantizers):
174
+ if self.training is False and i >= n_quantizers:
175
+ break
176
+
177
+ z_q_i, commitment_loss_i, codebook_loss_i, indices_i, z_e_i = quantizer(
178
+ residual
179
+ )
180
+
181
+ # Create mask to apply quantizer dropout
182
+ mask = (
183
+ torch.full((z.shape[0],), fill_value=i, device=z.device) < n_quantizers
184
+ )
185
+ z_q = z_q + z_q_i * mask[:, None, None]
186
+ residual = residual - z_q_i
187
+
188
+ # Sum losses
189
+ commitment_loss += (commitment_loss_i * mask).mean()
190
+ codebook_loss += (codebook_loss_i * mask).mean()
191
+
192
+ codebook_indices.append(indices_i)
193
+ latents.append(z_e_i)
194
+
195
+ codes = torch.stack(codebook_indices, dim=1)
196
+ latents = torch.cat(latents, dim=1)
197
+
198
+ return z_q, codes, latents, commitment_loss, codebook_loss
199
+
200
+ def from_codes(self, codes: torch.Tensor):
201
+ """Given the quantized codes, reconstruct the continuous representation
202
+ Parameters
203
+ ----------
204
+ codes : Tensor[B x N x T]
205
+ Quantized discrete representation of input
206
+ Returns
207
+ -------
208
+ Tensor[B x D x T]
209
+ Quantized continuous representation of input
210
+ """
211
+ z_q = 0.0
212
+ z_p = []
213
+ n_codebooks = codes.shape[1]
214
+ for i in range(n_codebooks):
215
+ z_p_i = self.quantizers[i].decode_code(codes[:, i, :])
216
+ z_p.append(z_p_i)
217
+
218
+ z_q_i = self.quantizers[i].out_proj(z_p_i)
219
+ z_q = z_q + z_q_i
220
+ return z_q, torch.cat(z_p, dim=1), codes
221
+
222
+ def from_latents(self, latents: torch.Tensor):
223
+ """Given the unquantized latents, reconstruct the
224
+ continuous representation after quantization.
225
+
226
+ Parameters
227
+ ----------
228
+ latents : Tensor[B x N x T]
229
+ Continuous representation of input after projection
230
+
231
+ Returns
232
+ -------
233
+ Tensor[B x D x T]
234
+ Quantized representation of full-projected space
235
+ Tensor[B x D x T]
236
+ Quantized representation of latent space
237
+ """
238
+ z_q = 0
239
+ z_p = []
240
+ codes = []
241
+ dims = np.cumsum([0] + [q.codebook_dim for q in self.quantizers])
242
+
243
+ n_codebooks = np.where(dims <= latents.shape[1])[0].max(axis=0, keepdims=True)[
244
+ 0
245
+ ]
246
+ for i in range(n_codebooks):
247
+ j, k = dims[i], dims[i + 1]
248
+ z_p_i, codes_i = self.quantizers[i].decode_latents(latents[:, j:k, :])
249
+ z_p.append(z_p_i)
250
+ codes.append(codes_i)
251
+
252
+ z_q_i = self.quantizers[i].out_proj(z_p_i)
253
+ z_q = z_q + z_q_i
254
+
255
+ return z_q, torch.cat(z_p, dim=1), torch.stack(codes, dim=1)
256
+
257
+
258
+ if __name__ == "__main__":
259
+ rvq = ResidualVectorQuantize(quantizer_dropout=True)
260
+ x = torch.randn(16, 512, 80)
261
+ y = rvq(x)
262
+ print(y["latents"].shape)
songgen/xcodec_wrapper/xcodec_infer/descriptaudiocodec/dac/utils/__init__.py ADDED
@@ -0,0 +1,123 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from pathlib import Path
2
+
3
+ import argbind
4
+ from audiotools import ml
5
+
6
+ import dac
7
+
8
+ DAC = dac.model.DAC
9
+ Accelerator = ml.Accelerator
10
+
11
+ __MODEL_LATEST_TAGS__ = {
12
+ ("44khz", "8kbps"): "0.0.1",
13
+ ("24khz", "8kbps"): "0.0.4",
14
+ ("16khz", "8kbps"): "0.0.5",
15
+ ("44khz", "16kbps"): "1.0.0",
16
+ }
17
+
18
+ __MODEL_URLS__ = {
19
+ (
20
+ "44khz",
21
+ "0.0.1",
22
+ "8kbps",
23
+ ): "https://github.com/descriptinc/descript-audio-codec/releases/download/0.0.1/weights.pth",
24
+ (
25
+ "24khz",
26
+ "0.0.4",
27
+ "8kbps",
28
+ ): "https://github.com/descriptinc/descript-audio-codec/releases/download/0.0.4/weights_24khz.pth",
29
+ (
30
+ "16khz",
31
+ "0.0.5",
32
+ "8kbps",
33
+ ): "https://github.com/descriptinc/descript-audio-codec/releases/download/0.0.5/weights_16khz.pth",
34
+ (
35
+ "44khz",
36
+ "1.0.0",
37
+ "16kbps",
38
+ ): "https://github.com/descriptinc/descript-audio-codec/releases/download/1.0.0/weights_44khz_16kbps.pth",
39
+ }
40
+
41
+
42
+ @argbind.bind(group="download", positional=True, without_prefix=True)
43
+ def download(
44
+ model_type: str = "44khz", model_bitrate: str = "8kbps", tag: str = "latest"
45
+ ):
46
+ """
47
+ Function that downloads the weights file from URL if a local cache is not found.
48
+
49
+ Parameters
50
+ ----------
51
+ model_type : str
52
+ The type of model to download. Must be one of "44khz", "24khz", or "16khz". Defaults to "44khz".
53
+ model_bitrate: str
54
+ Bitrate of the model. Must be one of "8kbps", or "16kbps". Defaults to "8kbps".
55
+ Only 44khz model supports 16kbps.
56
+ tag : str
57
+ The tag of the model to download. Defaults to "latest".
58
+
59
+ Returns
60
+ -------
61
+ Path
62
+ Directory path required to load model via audiotools.
63
+ """
64
+ model_type = model_type.lower()
65
+ tag = tag.lower()
66
+
67
+ assert model_type in [
68
+ "44khz",
69
+ "24khz",
70
+ "16khz",
71
+ ], "model_type must be one of '44khz', '24khz', or '16khz'"
72
+
73
+ assert model_bitrate in [
74
+ "8kbps",
75
+ "16kbps",
76
+ ], "model_bitrate must be one of '8kbps', or '16kbps'"
77
+
78
+ if tag == "latest":
79
+ tag = __MODEL_LATEST_TAGS__[(model_type, model_bitrate)]
80
+
81
+ download_link = __MODEL_URLS__.get((model_type, tag, model_bitrate), None)
82
+
83
+ if download_link is None:
84
+ raise ValueError(
85
+ f"Could not find model with tag {tag} and model type {model_type}"
86
+ )
87
+
88
+ local_path = (
89
+ Path.home()
90
+ / ".cache"
91
+ / "descript"
92
+ / "dac"
93
+ / f"weights_{model_type}_{model_bitrate}_{tag}.pth"
94
+ )
95
+ if not local_path.exists():
96
+ local_path.parent.mkdir(parents=True, exist_ok=True)
97
+
98
+ # Download the model
99
+ import requests
100
+
101
+ response = requests.get(download_link)
102
+
103
+ if response.status_code != 200:
104
+ raise ValueError(
105
+ f"Could not download model. Received response code {response.status_code}"
106
+ )
107
+ local_path.write_bytes(response.content)
108
+
109
+ return local_path
110
+
111
+
112
+ def load_model(
113
+ model_type: str = "44khz",
114
+ model_bitrate: str = "8kbps",
115
+ tag: str = "latest",
116
+ load_path: str = None,
117
+ ):
118
+ if not load_path:
119
+ load_path = download(
120
+ model_type=model_type, model_bitrate=model_bitrate, tag=tag
121
+ )
122
+ generator = DAC.load(load_path)
123
+ return generator
songgen/xcodec_wrapper/xcodec_infer/models/__init__.py ADDED
File without changes
songgen/xcodec_wrapper/xcodec_infer/models/soundstream_semantic.py ADDED
@@ -0,0 +1,157 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+
3
+ from typing import Sequence, Optional, Union
4
+ import sys
5
+
6
+ import math
7
+ import random
8
+
9
+ import torch
10
+ import torch.nn as nn
11
+ import numpy as np
12
+ import torch.nn.functional as F
13
+
14
+ import descriptaudiocodec.dac.model.dac as dac2
15
+
16
+
17
+ from quantization import ResidualVectorQuantizer
18
+ from transformers import AutoModel
19
+
20
+ from modules.semantic_module import Encoder,Decoder
21
+
22
+
23
+
24
+ class SoundStream(nn.Module):
25
+ def __init__(
26
+ self,
27
+ n_filters: int = 32,
28
+ D: int = 128,
29
+ target_bandwidths: Sequence[Union[int, float]] = [1, 1.5, 2, 4, 6],
30
+ ratios: Sequence[int] = [8, 5, 4, 2], # downsampling by 320
31
+ sample_rate: int = 16000,
32
+ bins: int = 1024,
33
+ normalize: bool = False,
34
+ causal: bool = False,
35
+ semantic_techer: str = 'hubert_base_general'
36
+ ):
37
+ super().__init__()
38
+ self.hop_length = np.prod(ratios)
39
+
40
+ n_q = int(1000 * target_bandwidths[-1] // (math.ceil(sample_rate / self.hop_length) * 10))
41
+ self.frame_rate = math.ceil(sample_rate / np.prod(ratios)) # 50 Hz
42
+ self.bits_per_codebook = int(math.log2(bins)) # 1024 => 10
43
+ self.target_bandwidths = target_bandwidths
44
+ self.n_q = n_q
45
+ self.sample_rate = sample_rate
46
+ self.encoder = dac2.Encoder(64,ratios,D)
47
+
48
+ self.encoder_semantic = Encoder(input_channels=768,encode_channels=768)
49
+ self.decoder_semantic = Decoder(code_dim=768,output_channels=768,decode_channels=768)
50
+ # out_D=D+768
51
+ self.quantizer = ResidualVectorQuantizer(dimension=D+768, n_q=n_q, bins=bins)
52
+
53
+ self.decoder_2 = dac2.Decoder(D,1024,ratios,)
54
+
55
+ if semantic_techer=='hubert_base':
56
+ self.semantic_model = AutoModel.from_pretrained("facebook/hubert-base-ls960")
57
+ elif semantic_techer=='wavlm_base_plus':
58
+ self.semantic_model = AutoModel.from_pretrained("microsoft/wavlm-base-plus")
59
+ elif semantic_techer=='hubert_base_general':
60
+
61
+ self.semantic_model = AutoModel.from_pretrained("ZhenYe234/hubert_base_general_audio")
62
+ self.semantic_model.eval()
63
+
64
+ self.fc_prior = nn.Linear(D+768, D+768 )
65
+
66
+ self.fc_post1= nn.Linear( D+768, 768 )
67
+ self.fc_post2= nn.Linear( D+768, D)
68
+
69
+ def get_last_layer(self):
70
+ return self.decoder.layers[-1].weight
71
+
72
+ def calculate_rec_loss(self, rec, target):
73
+
74
+ target = target / target.norm(dim=-1, keepdim=True)
75
+ rec = rec / rec.norm(dim=-1, keepdim=True)
76
+ rec_loss = (1 - (target * rec).sum(-1)).mean()
77
+
78
+ return rec_loss
79
+
80
+ @torch.no_grad()
81
+ def get_regress_target(self, x ):
82
+ x= x[:,0,:]
83
+ x = F.pad(x, (160, 160))
84
+ target = self.semantic_model(x, output_hidden_states=True) .hidden_states
85
+ target = torch.stack(target, dim=1)#.transpose(-1, -2)#.flatten(start_dim=1, end_dim=2)
86
+
87
+ # average for all layers
88
+ target = target.mean(1)
89
+ # target = target[9]
90
+ return target
91
+
92
+
93
+
94
+ def forward(self, x: torch.Tensor, bw: int):
95
+ x=x.unsqueeze(1)
96
+ e_semantic_input = self.get_regress_target(x).detach()
97
+ # print('e_semantic_input:', e_semantic_input.shape )
98
+ # print('x:', x.shape )
99
+ e_semantic = self.encoder_semantic(e_semantic_input.transpose(1, 2))
100
+ e_acoustic = self.encoder(x)
101
+
102
+ # print('e_acoustic:', e_acoustic.shape )
103
+ # print('e_semantic:', e_semantic.shape )
104
+ e= torch.cat([e_acoustic, e_semantic], dim=1)
105
+
106
+ e = self.fc_prior(e.transpose(1, 2)).transpose(1, 2)
107
+
108
+
109
+ quantized, codes, bandwidth, commit_loss = self.quantizer(e, self.frame_rate, bw)
110
+
111
+ quantized_semantic = self.fc_post1(quantized.transpose(1, 2)).transpose(1, 2)
112
+ quantized_acoustic = self.fc_post2(quantized.transpose(1, 2)).transpose(1, 2)
113
+
114
+ o = self.decoder_2(quantized_acoustic)
115
+
116
+ o_semantic = self.decoder_semantic(quantized_semantic )
117
+ semantic_recon_loss = F.mse_loss(e_semantic_input.transpose(1, 2).detach(),o_semantic)
118
+
119
+ return o, commit_loss, semantic_recon_loss,None
120
+
121
+ def encode(self, x: torch.Tensor,target_bw: Optional[int] = None) -> torch.Tensor:
122
+
123
+ bw = target_bw
124
+
125
+ e_semantic_input = self.get_regress_target(x).detach()
126
+
127
+ e_semantic = self.encoder_semantic(e_semantic_input.transpose(1, 2))
128
+ e_acoustic = self.encoder(x)
129
+
130
+
131
+ if e_acoustic.shape[2] != e_semantic.shape[2]:
132
+ e_acoustic = self.encoder(F.pad(x[:,0,:], (160, 160)).unsqueeze(0))
133
+
134
+ e= torch.cat([e_acoustic, e_semantic], dim=1)
135
+
136
+ e = self.fc_prior(e.transpose(1, 2)).transpose(1, 2)
137
+
138
+
139
+ quantized, codes, bandwidth, commit_loss = self.quantizer(e, self.frame_rate, bw)
140
+ return codes
141
+
142
+ def decode(self, codes: torch.Tensor) -> torch.Tensor:
143
+ quantized = self.quantizer.decode(codes)
144
+ quantized_acoustic = self.fc_post2(quantized.transpose(1, 2)).transpose(1, 2)
145
+
146
+ o = self.decoder_2(quantized_acoustic)
147
+ return o
148
+
149
+ # test
150
+ if __name__ == '__main__':
151
+ soundstream = SoundStream(n_filters=32, D=256)
152
+
153
+ for i in range(10):
154
+ print(f"Iter {i}: ")
155
+ x = torch.rand(1, 1, 16000)
156
+ o, commit_loss, distill_loss,_= soundstream(x,soundstream.target_bandwidths[-1])
157
+ print('output', o.shape)
songgen/xcodec_wrapper/xcodec_infer/modules/__init__.py ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Meta Platforms, Inc. and affiliates.
2
+ # All rights reserved.
3
+ #
4
+ # This source code is licensed under the license found in the
5
+ # LICENSE file in the root directory of this source tree.
6
+
7
+ """Torch modules."""
8
+
9
+ # flake8: noqa
10
+ from .conv import (
11
+ pad1d,
12
+ unpad1d,
13
+ NormConv1d,
14
+ NormConvTranspose1d,
15
+ NormConv2d,
16
+ NormConvTranspose2d,
17
+ SConv1d,
18
+ SConvTranspose1d,
19
+ )
20
+ from .lstm import SLSTM
21
+ from .seanet import SEANetEncoder, SEANetDecoder
22
+ from .transformer import StreamingTransformerEncoder
songgen/xcodec_wrapper/xcodec_infer/modules/conv.py ADDED
@@ -0,0 +1,252 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Meta Platforms, Inc. and affiliates.
2
+ # All rights reserved.
3
+ #
4
+ # This source code is licensed under the license found in the
5
+ # LICENSE file in the root directory of this source tree.
6
+
7
+ """Convolutional layers wrappers and utilities."""
8
+
9
+ import math
10
+ import typing as tp
11
+ import warnings
12
+
13
+ import torch
14
+ from torch import nn
15
+ from torch.nn import functional as F
16
+ from torch.nn.utils import spectral_norm, weight_norm
17
+
18
+ from .norm import ConvLayerNorm
19
+
20
+
21
+ CONV_NORMALIZATIONS = frozenset(['none', 'weight_norm', 'spectral_norm',
22
+ 'time_layer_norm', 'layer_norm', 'time_group_norm'])
23
+
24
+
25
+ def apply_parametrization_norm(module: nn.Module, norm: str = 'none') -> nn.Module:
26
+ assert norm in CONV_NORMALIZATIONS
27
+ if norm == 'weight_norm':
28
+ return weight_norm(module)
29
+ elif norm == 'spectral_norm':
30
+ return spectral_norm(module)
31
+ else:
32
+ # We already check was in CONV_NORMALIZATION, so any other choice
33
+ # doesn't need reparametrization.
34
+ return module
35
+
36
+
37
+ def get_norm_module(module: nn.Module, causal: bool = False, norm: str = 'none', **norm_kwargs) -> nn.Module:
38
+ """Return the proper normalization module. If causal is True, this will ensure the returned
39
+ module is causal, or return an error if the normalization doesn't support causal evaluation.
40
+ """
41
+ assert norm in CONV_NORMALIZATIONS
42
+ if norm == 'layer_norm':
43
+ assert isinstance(module, nn.modules.conv._ConvNd)
44
+ return ConvLayerNorm(module.out_channels, **norm_kwargs)
45
+ elif norm == 'time_group_norm':
46
+ if causal:
47
+ raise ValueError("GroupNorm doesn't support causal evaluation.")
48
+ assert isinstance(module, nn.modules.conv._ConvNd)
49
+ return nn.GroupNorm(1, module.out_channels, **norm_kwargs)
50
+ else:
51
+ return nn.Identity()
52
+
53
+
54
+ def get_extra_padding_for_conv1d(x: torch.Tensor, kernel_size: int, stride: int,
55
+ padding_total: int = 0) -> int:
56
+ """See `pad_for_conv1d`.
57
+ """
58
+ length = x.shape[-1]
59
+ n_frames = (length - kernel_size + padding_total) / stride + 1
60
+ ideal_length = (math.ceil(n_frames) - 1) * stride + (kernel_size - padding_total)
61
+ return ideal_length - length
62
+
63
+
64
+ def pad_for_conv1d(x: torch.Tensor, kernel_size: int, stride: int, padding_total: int = 0):
65
+ """Pad for a convolution to make sure that the last window is full.
66
+ Extra padding is added at the end. This is required to ensure that we can rebuild
67
+ an output of the same length, as otherwise, even with padding, some time steps
68
+ might get removed.
69
+ For instance, with total padding = 4, kernel size = 4, stride = 2:
70
+ 0 0 1 2 3 4 5 0 0 # (0s are padding)
71
+ 1 2 3 # (output frames of a convolution, last 0 is never used)
72
+ 0 0 1 2 3 4 5 0 # (output of tr. conv., but pos. 5 is going to get removed as padding)
73
+ 1 2 3 4 # once you removed padding, we are missing one time step !
74
+ """
75
+ extra_padding = get_extra_padding_for_conv1d(x, kernel_size, stride, padding_total)
76
+ return F.pad(x, (0, extra_padding))
77
+
78
+
79
+ def pad1d(x: torch.Tensor, paddings: tp.Tuple[int, int], mode: str = 'zero', value: float = 0.):
80
+ """Tiny wrapper around F.pad, just to allow for reflect padding on small input.
81
+ If this is the case, we insert extra 0 padding to the right before the reflection happen.
82
+ """
83
+ length = x.shape[-1]
84
+ padding_left, padding_right = paddings
85
+ assert padding_left >= 0 and padding_right >= 0, (padding_left, padding_right)
86
+ if mode == 'reflect':
87
+ max_pad = max(padding_left, padding_right)
88
+ extra_pad = 0
89
+ if length <= max_pad:
90
+ extra_pad = max_pad - length + 1
91
+ x = F.pad(x, (0, extra_pad))
92
+ padded = F.pad(x, paddings, mode, value)
93
+ end = padded.shape[-1] - extra_pad
94
+ return padded[..., :end]
95
+ else:
96
+ return F.pad(x, paddings, mode, value)
97
+
98
+
99
+ def unpad1d(x: torch.Tensor, paddings: tp.Tuple[int, int]):
100
+ """Remove padding from x, handling properly zero padding. Only for 1d!"""
101
+ padding_left, padding_right = paddings
102
+ assert padding_left >= 0 and padding_right >= 0, (padding_left, padding_right)
103
+ assert (padding_left + padding_right) <= x.shape[-1]
104
+ end = x.shape[-1] - padding_right
105
+ return x[..., padding_left: end]
106
+
107
+
108
+ class NormConv1d(nn.Module):
109
+ """Wrapper around Conv1d and normalization applied to this conv
110
+ to provide a uniform interface across normalization approaches.
111
+ """
112
+ def __init__(self, *args, causal: bool = False, norm: str = 'none',
113
+ norm_kwargs: tp.Dict[str, tp.Any] = {}, **kwargs):
114
+ super().__init__()
115
+ self.conv = apply_parametrization_norm(nn.Conv1d(*args, **kwargs), norm)
116
+ self.norm = get_norm_module(self.conv, causal, norm, **norm_kwargs)
117
+ self.norm_type = norm
118
+
119
+ def forward(self, x):
120
+ x = self.conv(x)
121
+ x = self.norm(x)
122
+ return x
123
+
124
+
125
+ class NormConv2d(nn.Module):
126
+ """Wrapper around Conv2d and normalization applied to this conv
127
+ to provide a uniform interface across normalization approaches.
128
+ """
129
+ def __init__(self, *args, norm: str = 'none',
130
+ norm_kwargs: tp.Dict[str, tp.Any] = {}, **kwargs):
131
+ super().__init__()
132
+ self.conv = apply_parametrization_norm(nn.Conv2d(*args, **kwargs), norm)
133
+ self.norm = get_norm_module(self.conv, causal=False, norm=norm, **norm_kwargs)
134
+ self.norm_type = norm
135
+
136
+ def forward(self, x):
137
+ x = self.conv(x)
138
+ x = self.norm(x)
139
+ return x
140
+
141
+
142
+ class NormConvTranspose1d(nn.Module):
143
+ """Wrapper around ConvTranspose1d and normalization applied to this conv
144
+ to provide a uniform interface across normalization approaches.
145
+ """
146
+ def __init__(self, *args, causal: bool = False, norm: str = 'none',
147
+ norm_kwargs: tp.Dict[str, tp.Any] = {}, **kwargs):
148
+ super().__init__()
149
+ self.convtr = apply_parametrization_norm(nn.ConvTranspose1d(*args, **kwargs), norm)
150
+ self.norm = get_norm_module(self.convtr, causal, norm, **norm_kwargs)
151
+ self.norm_type = norm
152
+
153
+ def forward(self, x):
154
+ x = self.convtr(x)
155
+ x = self.norm(x)
156
+ return x
157
+
158
+
159
+ class NormConvTranspose2d(nn.Module):
160
+ """Wrapper around ConvTranspose2d and normalization applied to this conv
161
+ to provide a uniform interface across normalization approaches.
162
+ """
163
+ def __init__(self, *args, norm: str = 'none',
164
+ norm_kwargs: tp.Dict[str, tp.Any] = {}, **kwargs):
165
+ super().__init__()
166
+ self.convtr = apply_parametrization_norm(nn.ConvTranspose2d(*args, **kwargs), norm)
167
+ self.norm = get_norm_module(self.convtr, causal=False, norm=norm, **norm_kwargs)
168
+
169
+ def forward(self, x):
170
+ x = self.convtr(x)
171
+ x = self.norm(x)
172
+ return x
173
+
174
+
175
+ class SConv1d(nn.Module):
176
+ """Conv1d with some builtin handling of asymmetric or causal padding
177
+ and normalization.
178
+ """
179
+ def __init__(self, in_channels: int, out_channels: int,
180
+ kernel_size: int, stride: int = 1, dilation: int = 1,
181
+ groups: int = 1, bias: bool = True, causal: bool = False,
182
+ norm: str = 'none', norm_kwargs: tp.Dict[str, tp.Any] = {},
183
+ pad_mode: str = 'reflect'):
184
+ super().__init__()
185
+ # warn user on unusual setup between dilation and stride
186
+ if stride > 1 and dilation > 1:
187
+ warnings.warn('SConv1d has been initialized with stride > 1 and dilation > 1'
188
+ f' (kernel_size={kernel_size} stride={stride}, dilation={dilation}).')
189
+ self.conv = NormConv1d(in_channels, out_channels, kernel_size, stride,
190
+ dilation=dilation, groups=groups, bias=bias, causal=causal,
191
+ norm=norm, norm_kwargs=norm_kwargs)
192
+ self.causal = causal
193
+ self.pad_mode = pad_mode
194
+
195
+ def forward(self, x):
196
+ B, C, T = x.shape
197
+ kernel_size = self.conv.conv.kernel_size[0]
198
+ stride = self.conv.conv.stride[0]
199
+ dilation = self.conv.conv.dilation[0]
200
+ padding_total = (kernel_size - 1) * dilation - (stride - 1)
201
+ extra_padding = get_extra_padding_for_conv1d(x, kernel_size, stride, padding_total)
202
+ if self.causal:
203
+ # Left padding for causal
204
+ x = pad1d(x, (padding_total, extra_padding), mode=self.pad_mode)
205
+ else:
206
+ # Asymmetric padding required for odd strides
207
+ padding_right = padding_total // 2
208
+ padding_left = padding_total - padding_right
209
+ x = pad1d(x, (padding_left, padding_right + extra_padding), mode=self.pad_mode)
210
+ return self.conv(x)
211
+
212
+
213
+ class SConvTranspose1d(nn.Module):
214
+ """ConvTranspose1d with some builtin handling of asymmetric or causal padding
215
+ and normalization.
216
+ """
217
+ def __init__(self, in_channels: int, out_channels: int,
218
+ kernel_size: int, stride: int = 1, causal: bool = False,
219
+ norm: str = 'none', trim_right_ratio: float = 1.,
220
+ norm_kwargs: tp.Dict[str, tp.Any] = {}):
221
+ super().__init__()
222
+ self.convtr = NormConvTranspose1d(in_channels, out_channels, kernel_size, stride,
223
+ causal=causal, norm=norm, norm_kwargs=norm_kwargs)
224
+ self.causal = causal
225
+ self.trim_right_ratio = trim_right_ratio
226
+ assert self.causal or self.trim_right_ratio == 1., \
227
+ "`trim_right_ratio` != 1.0 only makes sense for causal convolutions"
228
+ assert self.trim_right_ratio >= 0. and self.trim_right_ratio <= 1.
229
+
230
+ def forward(self, x):
231
+ kernel_size = self.convtr.convtr.kernel_size[0]
232
+ stride = self.convtr.convtr.stride[0]
233
+ padding_total = kernel_size - stride
234
+
235
+ y = self.convtr(x)
236
+
237
+ # We will only trim fixed padding. Extra padding from `pad_for_conv1d` would be
238
+ # removed at the very end, when keeping only the right length for the output,
239
+ # as removing it here would require also passing the length at the matching layer
240
+ # in the encoder.
241
+ if self.causal:
242
+ # Trim the padding on the right according to the specified ratio
243
+ # if trim_right_ratio = 1.0, trim everything from right
244
+ padding_right = math.ceil(padding_total * self.trim_right_ratio)
245
+ padding_left = padding_total - padding_right
246
+ y = unpad1d(y, (padding_left, padding_right))
247
+ else:
248
+ # Asymmetric padding required for odd strides
249
+ padding_right = padding_total // 2
250
+ padding_left = padding_total - padding_right
251
+ y = unpad1d(y, (padding_left, padding_right))
252
+ return y
songgen/xcodec_wrapper/xcodec_infer/modules/lstm.py ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Meta Platforms, Inc. and affiliates.
2
+ # All rights reserved.
3
+ #
4
+ # This source code is licensed under the license found in the
5
+ # LICENSE file in the root directory of this source tree.
6
+
7
+ """LSTM layers module."""
8
+
9
+ from torch import nn
10
+
11
+
12
+ class SLSTM(nn.Module):
13
+ """
14
+ LSTM without worrying about the hidden state, nor the layout of the data.
15
+ Expects input as convolutional layout.
16
+ """
17
+ def __init__(self, dimension: int, num_layers: int = 2, skip: bool = True):
18
+ super().__init__()
19
+ self.skip = skip
20
+ self.lstm = nn.LSTM(dimension, dimension, num_layers)
21
+
22
+ def forward(self, x):
23
+ x = x.permute(2, 0, 1)
24
+ y, _ = self.lstm(x)
25
+ if self.skip:
26
+ y = y + x
27
+ y = y.permute(1, 2, 0)
28
+ return y
songgen/xcodec_wrapper/xcodec_infer/modules/norm.py ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Meta Platforms, Inc. and affiliates.
2
+ # All rights reserved.
3
+ #
4
+ # This source code is licensed under the license found in the
5
+ # LICENSE file in the root directory of this source tree.
6
+
7
+ """Normalization modules."""
8
+
9
+ import typing as tp
10
+
11
+ import einops
12
+ import torch
13
+ from torch import nn
14
+
15
+
16
+ class ConvLayerNorm(nn.LayerNorm):
17
+ """
18
+ Convolution-friendly LayerNorm that moves channels to last dimensions
19
+ before running the normalization and moves them back to original position right after.
20
+ """
21
+ def __init__(self, normalized_shape: tp.Union[int, tp.List[int], torch.Size], **kwargs):
22
+ super().__init__(normalized_shape, **kwargs)
23
+
24
+ def forward(self, x):
25
+ x = einops.rearrange(x, 'b ... t -> b t ...')
26
+ x = super().forward(x)
27
+ x = einops.rearrange(x, 'b t ... -> b ... t')
28
+ return
songgen/xcodec_wrapper/xcodec_infer/modules/seanet.py ADDED
@@ -0,0 +1,255 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Meta Platforms, Inc. and affiliates.
2
+ # All rights reserved.
3
+ #
4
+ # This source code is licensed under the license found in the
5
+ # LICENSE file in the root directory of this source tree.
6
+
7
+ """Encodec SEANet-based encoder and decoder implementation."""
8
+
9
+ import typing as tp
10
+
11
+ import numpy as np
12
+ import torch.nn as nn
13
+
14
+ from . import (
15
+ SConv1d,
16
+ SConvTranspose1d,
17
+ SLSTM
18
+ )
19
+
20
+
21
+ class SEANetResnetBlock(nn.Module):
22
+ """Residual block from SEANet model.
23
+ Args:
24
+ dim (int): Dimension of the input/output
25
+ kernel_sizes (list): List of kernel sizes for the convolutions.
26
+ dilations (list): List of dilations for the convolutions.
27
+ activation (str): Activation function.
28
+ activation_params (dict): Parameters to provide to the activation function
29
+ norm (str): Normalization method.
30
+ norm_params (dict): Parameters to provide to the underlying normalization used along with the convolution.
31
+ causal (bool): Whether to use fully causal convolution.
32
+ pad_mode (str): Padding mode for the convolutions.
33
+ compress (int): Reduced dimensionality in residual branches (from Demucs v3)
34
+ true_skip (bool): Whether to use true skip connection or a simple convolution as the skip connection.
35
+ """
36
+ def __init__(self, dim: int, kernel_sizes: tp.List[int] = [3, 1], dilations: tp.List[int] = [1, 1],
37
+ activation: str = 'ELU', activation_params: dict = {'alpha': 1.0},
38
+ norm: str = 'weight_norm', norm_params: tp.Dict[str, tp.Any] = {}, causal: bool = False,
39
+ pad_mode: str = 'reflect', compress: int = 2, true_skip: bool = True):
40
+ super().__init__()
41
+ assert len(kernel_sizes) == len(dilations), 'Number of kernel sizes should match number of dilations'
42
+ act = getattr(nn, activation)
43
+ hidden = dim // compress
44
+ block = []
45
+ for i, (kernel_size, dilation) in enumerate(zip(kernel_sizes, dilations)):
46
+ in_chs = dim if i == 0 else hidden
47
+ out_chs = dim if i == len(kernel_sizes) - 1 else hidden
48
+ block += [
49
+ act(**activation_params),
50
+ SConv1d(in_chs, out_chs, kernel_size=kernel_size, dilation=dilation,
51
+ norm=norm, norm_kwargs=norm_params,
52
+ causal=causal, pad_mode=pad_mode),
53
+ ]
54
+ self.block = nn.Sequential(*block)
55
+ self.shortcut: nn.Module
56
+ if true_skip:
57
+ self.shortcut = nn.Identity()
58
+ else:
59
+ self.shortcut = SConv1d(dim, dim, kernel_size=1, norm=norm, norm_kwargs=norm_params,
60
+ causal=causal, pad_mode=pad_mode)
61
+
62
+ def forward(self, x):
63
+ return self.shortcut(x) + self.block(x)
64
+
65
+
66
+ class SEANetEncoder(nn.Module):
67
+ """SEANet encoder.
68
+ Args:
69
+ channels (int): Audio channels.
70
+ dimension (int): Intermediate representation dimension.
71
+ n_filters (int): Base width for the model.
72
+ n_residual_layers (int): nb of residual layers.
73
+ ratios (Sequence[int]): kernel size and stride ratios. The encoder uses downsampling ratios instead of
74
+ upsampling ratios, hence it will use the ratios in the reverse order to the ones specified here
75
+ that must match the decoder order
76
+ activation (str): Activation function.
77
+ activation_params (dict): Parameters to provide to the activation function
78
+ norm (str): Normalization method.
79
+ norm_params (dict): Parameters to provide to the underlying normalization used along with the convolution.
80
+ kernel_size (int): Kernel size for the initial convolution.
81
+ last_kernel_size (int): Kernel size for the initial convolution.
82
+ residual_kernel_size (int): Kernel size for the residual layers.
83
+ dilation_base (int): How much to increase the dilation with each layer.
84
+ causal (bool): Whether to use fully causal convolution.
85
+ pad_mode (str): Padding mode for the convolutions.
86
+ true_skip (bool): Whether to use true skip connection or a simple
87
+ (streamable) convolution as the skip connection in the residual network blocks.
88
+ compress (int): Reduced dimensionality in residual branches (from Demucs v3).
89
+ lstm (int): Number of LSTM layers at the end of the encoder.
90
+ """
91
+ def __init__(self, channels: int = 1, dimension: int = 128, n_filters: int = 32, n_residual_layers: int = 1,
92
+ ratios: tp.List[int] = [8, 5, 4, 2], activation: str = 'ELU', activation_params: dict = {'alpha': 1.0},
93
+ norm: str = 'weight_norm', norm_params: tp.Dict[str, tp.Any] = {}, kernel_size: int = 7,
94
+ last_kernel_size: int = 7, residual_kernel_size: int = 3, dilation_base: int = 2, causal: bool = False,
95
+ pad_mode: str = 'reflect', true_skip: bool = False, compress: int = 2, lstm: int = 2):
96
+ super().__init__()
97
+ self.channels = channels
98
+ self.dimension = dimension
99
+ self.n_filters = n_filters
100
+ self.ratios = list(reversed(ratios))
101
+ del ratios
102
+ self.n_residual_layers = n_residual_layers
103
+ self.hop_length = np.prod(self.ratios) # 计算乘积
104
+
105
+ act = getattr(nn, activation)
106
+ mult = 1
107
+ model: tp.List[nn.Module] = [
108
+ SConv1d(channels, mult * n_filters, kernel_size, norm=norm, norm_kwargs=norm_params,
109
+ causal=causal, pad_mode=pad_mode)
110
+ ]
111
+ # Downsample to raw audio scale
112
+ for i, ratio in enumerate(self.ratios):
113
+ # Add residual layers
114
+ for j in range(n_residual_layers):
115
+ model += [
116
+ SEANetResnetBlock(mult * n_filters, kernel_sizes=[residual_kernel_size, 1],
117
+ dilations=[dilation_base ** j, 1],
118
+ norm=norm, norm_params=norm_params,
119
+ activation=activation, activation_params=activation_params,
120
+ causal=causal, pad_mode=pad_mode, compress=compress, true_skip=true_skip)]
121
+
122
+ # Add downsampling layers
123
+ model += [
124
+ act(**activation_params),
125
+ SConv1d(mult * n_filters, mult * n_filters * 2,
126
+ kernel_size=ratio * 2, stride=ratio,
127
+ norm=norm, norm_kwargs=norm_params,
128
+ causal=causal, pad_mode=pad_mode),
129
+ ]
130
+ mult *= 2
131
+
132
+ if lstm:
133
+ model += [SLSTM(mult * n_filters, num_layers=lstm)]
134
+
135
+ model += [
136
+ act(**activation_params),
137
+ SConv1d(mult * n_filters, dimension, last_kernel_size, norm=norm, norm_kwargs=norm_params,
138
+ causal=causal, pad_mode=pad_mode)
139
+ ]
140
+
141
+ self.model = nn.Sequential(*model)
142
+
143
+ def forward(self, x):
144
+ return self.model(x)
145
+
146
+
147
+ class SEANetDecoder(nn.Module):
148
+ """SEANet decoder.
149
+ Args:
150
+ channels (int): Audio channels.
151
+ dimension (int): Intermediate representation dimension.
152
+ n_filters (int): Base width for the model.
153
+ n_residual_layers (int): nb of residual layers.
154
+ ratios (Sequence[int]): kernel size and stride ratios
155
+ activation (str): Activation function.
156
+ activation_params (dict): Parameters to provide to the activation function
157
+ final_activation (str): Final activation function after all convolutions.
158
+ final_activation_params (dict): Parameters to provide to the activation function
159
+ norm (str): Normalization method.
160
+ norm_params (dict): Parameters to provide to the underlying normalization used along with the convolution.
161
+ kernel_size (int): Kernel size for the initial convolution.
162
+ last_kernel_size (int): Kernel size for the initial convolution.
163
+ residual_kernel_size (int): Kernel size for the residual layers.
164
+ dilation_base (int): How much to increase the dilation with each layer.
165
+ causal (bool): Whether to use fully causal convolution.
166
+ pad_mode (str): Padding mode for the convolutions.
167
+ true_skip (bool): Whether to use true skip connection or a simple
168
+ (streamable) convolution as the skip connection in the residual network blocks.
169
+ compress (int): Reduced dimensionality in residual branches (from Demucs v3).
170
+ lstm (int): Number of LSTM layers at the end of the encoder.
171
+ trim_right_ratio (float): Ratio for trimming at the right of the transposed convolution under the causal setup.
172
+ If equal to 1.0, it means that all the trimming is done at the right.
173
+ """
174
+ def __init__(self, channels: int = 1, dimension: int = 128, n_filters: int = 32, n_residual_layers: int = 1,
175
+ ratios: tp.List[int] = [8, 5, 4, 2], activation: str = 'ELU', activation_params: dict = {'alpha': 1.0},
176
+ final_activation: tp.Optional[str] = None, final_activation_params: tp.Optional[dict] = None,
177
+ norm: str = 'weight_norm', norm_params: tp.Dict[str, tp.Any] = {}, kernel_size: int = 7,
178
+ last_kernel_size: int = 7, residual_kernel_size: int = 3, dilation_base: int = 2, causal: bool = False,
179
+ pad_mode: str = 'reflect', true_skip: bool = False, compress: int = 2, lstm: int = 2,
180
+ trim_right_ratio: float = 1.0):
181
+ super().__init__()
182
+ self.dimension = dimension
183
+ self.channels = channels
184
+ self.n_filters = n_filters
185
+ self.ratios = ratios
186
+ del ratios
187
+ self.n_residual_layers = n_residual_layers
188
+ self.hop_length = np.prod(self.ratios)
189
+
190
+ act = getattr(nn, activation)
191
+ mult = int(2 ** len(self.ratios))
192
+ model: tp.List[nn.Module] = [
193
+ SConv1d(dimension, mult * n_filters, kernel_size, norm=norm, norm_kwargs=norm_params,
194
+ causal=causal, pad_mode=pad_mode)
195
+ ]
196
+
197
+ if lstm:
198
+ model += [SLSTM(mult * n_filters, num_layers=lstm)]
199
+
200
+ # Upsample to raw audio scale
201
+ for i, ratio in enumerate(self.ratios):
202
+ # Add upsampling layers
203
+ model += [
204
+ act(**activation_params),
205
+ SConvTranspose1d(mult * n_filters, mult * n_filters // 2,
206
+ kernel_size=ratio * 2, stride=ratio,
207
+ norm=norm, norm_kwargs=norm_params,
208
+ causal=causal, trim_right_ratio=trim_right_ratio),
209
+ ]
210
+ # Add residual layers
211
+ for j in range(n_residual_layers):
212
+ model += [
213
+ SEANetResnetBlock(mult * n_filters // 2, kernel_sizes=[residual_kernel_size, 1],
214
+ dilations=[dilation_base ** j, 1],
215
+ activation=activation, activation_params=activation_params,
216
+ norm=norm, norm_params=norm_params, causal=causal,
217
+ pad_mode=pad_mode, compress=compress, true_skip=true_skip)]
218
+
219
+ mult //= 2
220
+
221
+ # Add final layers
222
+ model += [
223
+ act(**activation_params),
224
+ SConv1d(n_filters, channels, last_kernel_size, norm=norm, norm_kwargs=norm_params,
225
+ causal=causal, pad_mode=pad_mode)
226
+ ]
227
+ # Add optional final activation to decoder (eg. tanh)
228
+ if final_activation is not None:
229
+ final_act = getattr(nn, final_activation)
230
+ final_activation_params = final_activation_params or {}
231
+ model += [
232
+ final_act(**final_activation_params)
233
+ ]
234
+ self.model = nn.Sequential(*model)
235
+
236
+ def forward(self, z):
237
+ y = self.model(z)
238
+ return y
239
+
240
+
241
+ def test():
242
+ import torch
243
+ encoder = SEANetEncoder()
244
+ decoder = SEANetDecoder()
245
+ x = torch.randn(1, 1, 24000)
246
+ z = encoder(x)
247
+ print('z ', z.shape)
248
+ assert 1==2
249
+ assert list(z.shape) == [1, 128, 75], z.shape
250
+ y = decoder(z)
251
+ assert y.shape == x.shape, (x.shape, y.shape)
252
+
253
+
254
+ if __name__ == '__main__':
255
+ test()
songgen/xcodec_wrapper/xcodec_infer/modules/semantic_module.py ADDED
@@ -0,0 +1,294 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ import torch
3
+ import torch.nn as nn
4
+
5
+
6
+
7
+ class Conv1d1x1(nn.Conv1d):
8
+ """1x1 Conv1d."""
9
+
10
+ def __init__(self, in_channels, out_channels, bias=True):
11
+ super(Conv1d1x1, self).__init__(in_channels, out_channels, kernel_size=1, bias=bias)
12
+
13
+
14
+ class Conv1d(nn.Module):
15
+ def __init__(
16
+ self,
17
+ in_channels: int,
18
+ out_channels: int,
19
+ kernel_size: int,
20
+ stride: int = 1,
21
+ padding: int = -1,
22
+ dilation: int = 1,
23
+ groups: int = 1,
24
+ bias: bool = True
25
+ ):
26
+ super().__init__()
27
+ self.in_channels = in_channels
28
+ self.out_channels = out_channels
29
+ self.kernel_size = kernel_size
30
+ if padding < 0:
31
+ padding = (kernel_size - 1) // 2 * dilation
32
+ self.dilation = dilation
33
+ self.conv = nn.Conv1d(
34
+ in_channels=in_channels,
35
+ out_channels=out_channels,
36
+ kernel_size=kernel_size,
37
+ stride=stride,
38
+ padding=padding,
39
+ dilation=dilation,
40
+ groups=groups,
41
+ bias=bias,
42
+ )
43
+
44
+ def forward(self, x):
45
+ """
46
+ Args:
47
+ x (Tensor): Float tensor variable with the shape (B, C, T).
48
+ Returns:
49
+ Tensor: Float tensor variable with the shape (B, C, T).
50
+ """
51
+ x = self.conv(x)
52
+ return x
53
+
54
+
55
+ class ResidualUnit(nn.Module):
56
+ def __init__(
57
+ self,
58
+ in_channels: int,
59
+ out_channels: int,
60
+ kernel_size=3,
61
+ dilation=1,
62
+ bias=False,
63
+ nonlinear_activation="ELU",
64
+ nonlinear_activation_params={},
65
+ ):
66
+ super().__init__()
67
+ self.activation = getattr(nn, nonlinear_activation)(**nonlinear_activation_params)
68
+ self.conv1 = Conv1d(
69
+ in_channels=in_channels,
70
+ out_channels=out_channels,
71
+ kernel_size=kernel_size,
72
+ stride=1,
73
+ dilation=dilation,
74
+ bias=bias,
75
+ )
76
+ self.conv2 = Conv1d1x1(out_channels, out_channels, bias)
77
+
78
+ def forward(self, x):
79
+ y = self.conv1(self.activation(x))
80
+ y = self.conv2(self.activation(y))
81
+ return x + y
82
+
83
+
84
+ class ConvTranspose1d(nn.Module):
85
+ def __init__(
86
+ self,
87
+ in_channels: int,
88
+ out_channels: int,
89
+ kernel_size: int,
90
+ stride: int,
91
+ padding=-1,
92
+ output_padding=-1,
93
+ groups=1,
94
+ bias=True,
95
+ ):
96
+ super().__init__()
97
+ if padding < 0:
98
+ padding = (stride + 1) // 2
99
+ if output_padding < 0:
100
+ output_padding = 1 if stride % 2 else 0
101
+ self.deconv = nn.ConvTranspose1d(
102
+ in_channels=in_channels,
103
+ out_channels=out_channels,
104
+ kernel_size=kernel_size,
105
+ stride=stride,
106
+ padding=padding,
107
+ output_padding=output_padding,
108
+ groups=groups,
109
+ bias=bias,
110
+ )
111
+
112
+ def forward(self, x):
113
+ """
114
+ Args:
115
+ x (Tensor): Float tensor variable with the shape (B, C, T).
116
+ Returns:
117
+ Tensor: Float tensor variable with the shape (B, C', T').
118
+ """
119
+ x = self.deconv(x)
120
+ return x
121
+
122
+ class EncoderBlock(nn.Module):
123
+ def __init__(
124
+ self,
125
+ in_channels: int,
126
+ out_channels: int,
127
+ stride: int,
128
+ dilations=(1, 1),
129
+ unit_kernel_size=3,
130
+ bias=True
131
+ ):
132
+ super().__init__()
133
+ self.res_units = torch.nn.ModuleList()
134
+ for dilation in dilations:
135
+ self.res_units += [
136
+ ResidualUnit(in_channels, in_channels,
137
+ kernel_size=unit_kernel_size,
138
+ dilation=dilation)
139
+ ]
140
+ self.num_res = len(self.res_units)
141
+
142
+ self.conv = Conv1d(
143
+ in_channels=in_channels,
144
+ out_channels=out_channels,
145
+ kernel_size=3 if stride == 1 else (2 * stride), # special case: stride=1, do not use kernel=2
146
+ stride=stride,
147
+ bias=bias,
148
+ )
149
+
150
+ def forward(self, x):
151
+ for idx in range(self.num_res):
152
+ x = self.res_units[idx](x)
153
+ x = self.conv(x)
154
+ return x
155
+
156
+
157
+ class Encoder(nn.Module):
158
+ def __init__(
159
+ self,
160
+ input_channels: int,
161
+ encode_channels: int,
162
+ channel_ratios=(1, 1),
163
+ strides=(1, 1),
164
+ kernel_size=3,
165
+ bias=True,
166
+ block_dilations=(1, 1),
167
+ unit_kernel_size=3
168
+ ):
169
+ super().__init__()
170
+ assert len(channel_ratios) == len(strides)
171
+
172
+ self.conv = Conv1d(
173
+ in_channels=input_channels,
174
+ out_channels=encode_channels,
175
+ kernel_size=kernel_size,
176
+ stride=1,
177
+ bias=False
178
+ )
179
+ self.conv_blocks = torch.nn.ModuleList()
180
+ in_channels = encode_channels
181
+ for idx, stride in enumerate(strides):
182
+ out_channels = int(encode_channels * channel_ratios[idx]) # could be float
183
+ self.conv_blocks += [
184
+ EncoderBlock(in_channels, out_channels, stride,
185
+ dilations=block_dilations, unit_kernel_size=unit_kernel_size,
186
+ bias=bias)
187
+ ]
188
+ in_channels = out_channels
189
+ self.num_blocks = len(self.conv_blocks)
190
+ self.out_channels = out_channels
191
+
192
+ def forward(self, x):
193
+ x = self.conv(x)
194
+ for i in range(self.num_blocks):
195
+ x = self.conv_blocks[i](x)
196
+ return x
197
+
198
+
199
+
200
+ class DecoderBlock(nn.Module):
201
+ """ Decoder block (no up-sampling) """
202
+
203
+ def __init__(
204
+ self,
205
+ in_channels: int,
206
+ out_channels: int,
207
+ stride: int,
208
+ dilations=(1, 1),
209
+ unit_kernel_size=3,
210
+ bias=True
211
+ ):
212
+ super().__init__()
213
+
214
+ if stride == 1:
215
+ self.conv = Conv1d(
216
+ in_channels=in_channels,
217
+ out_channels=out_channels,
218
+ kernel_size=3, # fix kernel=3 when stride=1 for unchanged shape
219
+ stride=stride,
220
+ bias=bias,
221
+ )
222
+ else:
223
+ self.conv = ConvTranspose1d(
224
+ in_channels=in_channels,
225
+ out_channels=out_channels,
226
+ kernel_size=(2 * stride),
227
+ stride=stride,
228
+ bias=bias,
229
+ )
230
+
231
+ self.res_units = torch.nn.ModuleList()
232
+ for idx, dilation in enumerate(dilations):
233
+ self.res_units += [
234
+ ResidualUnit(out_channels, out_channels,
235
+ kernel_size=unit_kernel_size,
236
+ dilation=dilation)
237
+ ]
238
+ self.num_res = len(self.res_units)
239
+
240
+ def forward(self, x):
241
+ x = self.conv(x)
242
+ for idx in range(self.num_res):
243
+ x = self.res_units[idx](x)
244
+ return x
245
+
246
+
247
+ class Decoder(nn.Module):
248
+ def __init__(
249
+ self,
250
+ code_dim: int,
251
+ output_channels: int,
252
+ decode_channels: int,
253
+ channel_ratios=(1, 1),
254
+ strides=(1, 1),
255
+ kernel_size=3,
256
+ bias=True,
257
+ block_dilations=(1, 1),
258
+ unit_kernel_size=3,
259
+ ):
260
+ super().__init__()
261
+ assert len(channel_ratios) == len(strides)
262
+
263
+ self.conv1 = Conv1d(
264
+ in_channels=code_dim,
265
+ out_channels=int(decode_channels * channel_ratios[0]),
266
+ kernel_size=kernel_size,
267
+ stride=1,
268
+ bias=False
269
+ )
270
+
271
+ self.conv_blocks = torch.nn.ModuleList()
272
+ for idx, stride in enumerate(strides):
273
+ in_channels = int(decode_channels * channel_ratios[idx])
274
+ if idx < (len(channel_ratios) - 1):
275
+ out_channels = int(decode_channels * channel_ratios[idx + 1])
276
+ else:
277
+ out_channels = decode_channels
278
+ self.conv_blocks += [
279
+ DecoderBlock(
280
+ in_channels, out_channels, stride,
281
+ dilations=block_dilations, unit_kernel_size=unit_kernel_size,
282
+ bias=bias
283
+ )
284
+ ]
285
+ self.num_blocks = len(self.conv_blocks)
286
+
287
+ self.conv2 = Conv1d(out_channels, output_channels, kernel_size, 1, bias=False)
288
+
289
+ def forward(self, z):
290
+ x = self.conv1(z)
291
+ for i in range(self.num_blocks):
292
+ x = self.conv_blocks[i](x)
293
+ x = self.conv2(x)
294
+ return x
songgen/xcodec_wrapper/xcodec_infer/modules/transformer.py ADDED
@@ -0,0 +1,119 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Meta Platforms, Inc. and affiliates.
2
+ # All rights reserved.
3
+ #
4
+ # This source code is licensed under the license found in the
5
+ # LICENSE file in the root directory of this source tree.
6
+
7
+ """A streamable transformer."""
8
+
9
+ import typing as tp
10
+
11
+ import torch
12
+ import torch.nn as nn
13
+ import torch.nn.functional as F
14
+
15
+
16
+ def create_sin_embedding(positions: torch.Tensor, dim: int, max_period: float = 10000):
17
+ """Create time embedding for the given positions, target dimension `dim`.
18
+ """
19
+ # We aim for BTC format
20
+ assert dim % 2 == 0
21
+ half_dim = dim // 2
22
+ adim = torch.arange(half_dim, device=positions.device).view(1, 1, -1)
23
+ phase = positions / (max_period ** (adim / (half_dim - 1)))
24
+ return torch.cat([
25
+ torch.cos(phase),
26
+ torch.sin(phase),
27
+ ], dim=-1)
28
+
29
+
30
+ class StreamingTransformerEncoderLayer(nn.TransformerEncoderLayer):
31
+ def forward(self, x: torch.Tensor, x_past: torch.Tensor, past_context: int): # type: ignore
32
+ if self.norm_first:
33
+ sa_input = self.norm1(x)
34
+ x = x + self._sa_block(sa_input, x_past, past_context)
35
+ x = x + self._ff_block(self.norm2(x))
36
+ else:
37
+ sa_input = x
38
+ x = self.norm1(x + self._sa_block(sa_input, x_past, past_context))
39
+ x = self.norm2(x + self._ff_block(x))
40
+
41
+ return x, sa_input
42
+
43
+ # self-attention block
44
+ def _sa_block(self, x: torch.Tensor, x_past: torch.Tensor, past_context: int): # type: ignore
45
+ _, T, _ = x.shape
46
+ _, H, _ = x_past.shape
47
+
48
+ queries = x
49
+ keys = torch.cat([x_past, x], dim=1)
50
+ values = keys
51
+
52
+ queries_pos = torch.arange(H, T + H, device=x.device).view(-1, 1)
53
+ keys_pos = torch.arange(T + H, device=x.device).view(1, -1)
54
+ delta = queries_pos - keys_pos
55
+ valid_access = (delta >= 0) & (delta <= past_context)
56
+ x = self.self_attn(queries, keys, values,
57
+ attn_mask=~valid_access,
58
+ need_weights=False)[0]
59
+ return self.dropout1(x)
60
+
61
+
62
+ class StreamingTransformerEncoder(nn.Module):
63
+ """TransformerEncoder with streaming support.
64
+
65
+ Args:
66
+ dim (int): dimension of the data.
67
+ hidden_scale (int): intermediate dimension of FF module is this times the dimension.
68
+ num_heads (int): number of heads.
69
+ num_layers (int): number of layers.
70
+ max_period (float): maxium period of cosines in the positional embedding.
71
+ past_context (int or None): receptive field for the causal mask, infinite if None.
72
+ gelu (bool): if true uses GeLUs, otherwise use ReLUs.
73
+ norm_in (bool): normalize the input.
74
+ dropout (float): dropout probability.
75
+ **kwargs: See `nn.TransformerEncoderLayer`.
76
+ """
77
+ def __init__(self, dim, hidden_scale: float = 4., num_heads: int = 8, num_layers: int = 5,
78
+ max_period: float = 10000, past_context: int = 1000, gelu: bool = True,
79
+ norm_in: bool = True, dropout: float = 0., **kwargs):
80
+ super().__init__()
81
+ assert dim % num_heads == 0
82
+ hidden_dim = int(dim * hidden_scale)
83
+
84
+ self.max_period = max_period
85
+ self.past_context = past_context
86
+ activation: tp.Any = F.gelu if gelu else F.relu
87
+
88
+ self.norm_in: nn.Module
89
+ if norm_in:
90
+ self.norm_in = nn.LayerNorm(dim)
91
+ else:
92
+ self.norm_in = nn.Identity()
93
+
94
+ self.layers = nn.ModuleList()
95
+ for idx in range(num_layers):
96
+ self.layers.append(
97
+ StreamingTransformerEncoderLayer(
98
+ dim, num_heads, hidden_dim,
99
+ activation=activation, batch_first=True, dropout=dropout, **kwargs))
100
+
101
+ def forward(self, x: torch.Tensor,
102
+ states: tp.Optional[tp.List[torch.Tensor]] = None,
103
+ offset: tp.Union[int, torch.Tensor] = 0):
104
+ B, T, C = x.shape
105
+ if states is None:
106
+ states = [torch.zeros_like(x[:, :1]) for _ in range(1 + len(self.layers))]
107
+
108
+ positions = torch.arange(T, device=x.device).view(1, -1, 1) + offset
109
+ pos_emb = create_sin_embedding(positions, C, max_period=self.max_period)
110
+
111
+ new_state: tp.List[torch.Tensor] = []
112
+ x = self.norm_in(x)
113
+ x = x + pos_emb
114
+
115
+ for layer_state, layer in zip(states, self.layers):
116
+ x, new_layer_state = layer(x, layer_state, self.past_context)
117
+ new_layer_state = torch.cat([layer_state, new_layer_state], dim=1)
118
+ new_state.append(new_layer_state[:, -self.past_context:, :])
119
+ return x, new_state, offset + T
songgen/xcodec_wrapper/xcodec_infer/quantization/__init__.py ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Meta Platforms, Inc. and affiliates.
2
+ # All rights reserved.
3
+ #
4
+ # This source code is licensed under the license found in the
5
+ # LICENSE file in the root directory of this source tree.
6
+
7
+ # flake8: noqa
8
+ from .vq import QuantizedResult, ResidualVectorQuantizer
songgen/xcodec_wrapper/xcodec_infer/quantization/core_vq_lsx_version.py ADDED
@@ -0,0 +1,433 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c)
2
+ #
3
+ # This source code is licensed under the license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+ # This implementation is inspired from
6
+ # https://github.com/rosinality/vq-vae-2-pytorch/blob/master/vqvae.py and
7
+ # https://github.com/clementchadebec/benchmark_VAE/blob/dfa0dcf6c79172df5d27769c09c860c42008baaa/src/pythae/models/vq_vae/vq_vae_utils.py#L81
8
+ #
9
+ # Copyright (c) Meta Platforms, Inc. and affiliates.
10
+ # All rights reserved.
11
+ #
12
+ # This source code is licensed under the license found in the
13
+ # LICENSE file in the root directory of this source tree.
14
+ #
15
+ # This implementation is inspired from
16
+ # https://github.com/lucidrains/vector-quantize-pytorch
17
+ # which is released under MIT License. Hereafter, the original license:
18
+ # MIT License
19
+ #
20
+ # Copyright (c) 2020 Phil Wang
21
+ #
22
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
23
+ # of this software and associated documentation files (the "Software"), to deal
24
+ # in the Software without restriction, including without limitation the rights
25
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
26
+ # copies of the Software, and to permit persons to whom the Software is
27
+ # furnished to do so, subject to the following conditions:
28
+ #
29
+ # The above copyright notice and this permission notice shall be included in all
30
+ # copies or substantial portions of the Software.
31
+ #
32
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
33
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
34
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
35
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
36
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
37
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
38
+ # SOFTWARE.
39
+
40
+ """Core vector quantization implementation."""
41
+ import typing as tp
42
+
43
+ from einops import rearrange, repeat
44
+ import torch
45
+ from torch import nn
46
+ import torch.nn.functional as F
47
+ import torch.distributed as dist
48
+
49
+ from .distrib import broadcast_tensors, rank, is_distributed
50
+ from xutils.utils import is_primary
51
+ from xutils.ddp_utils import SyncFunction
52
+
53
+
54
+ def default(val: tp.Any, d: tp.Any) -> tp.Any:
55
+ return val if val is not None else d
56
+
57
+
58
+ def ema_inplace(moving_avg, new, decay: float):
59
+ moving_avg.data.mul_(decay).add_(new, alpha=(1 - decay))
60
+
61
+
62
+ def laplace_smoothing(x, n_categories: int, epsilon: float = 1e-5):
63
+ return (x + epsilon) / (x.sum() + n_categories * epsilon)
64
+
65
+
66
+ def uniform_init(*shape: int):
67
+ t = torch.empty(shape)
68
+ nn.init.kaiming_uniform_(t)
69
+ return t
70
+
71
+
72
+ def sample_vectors(samples, num: int):
73
+ num_samples, device = samples.shape[0], samples.device
74
+
75
+ if num_samples >= num:
76
+ indices = torch.randperm(num_samples, device=device)[:num]
77
+ else:
78
+ indices = torch.randint(0, num_samples, (num,), device=device)
79
+
80
+ return samples[indices]
81
+
82
+
83
+ def kmeans(samples, num_clusters: int, num_iters: int = 10, frames_to_use: int = 10_000):
84
+ """ Run K-means clustering on samples.
85
+ Args:
86
+ samples (tensor): shape [B * T, D]
87
+ num_clusters (int): number of centroids.
88
+ num_iters (int): number of iterations.
89
+ """
90
+ dim, dtype = samples.shape[-1], samples.dtype
91
+
92
+ if frames_to_use < samples.shape[0]:
93
+ samples = sample_vectors(samples, frames_to_use)
94
+
95
+ # Init means
96
+ means = sample_vectors(samples, num_clusters)
97
+
98
+ for _ in range(num_iters):
99
+ diffs = rearrange(samples, "n d -> n () d") - rearrange(
100
+ means, "c d -> () c d"
101
+ )
102
+ dists = -(diffs ** 2).sum(dim=-1)
103
+
104
+ buckets = dists.max(dim=-1).indices
105
+ bins = torch.bincount(buckets, minlength=num_clusters)
106
+ zero_mask = bins == 0
107
+ bins_min_clamped = bins.masked_fill(zero_mask, 1)
108
+
109
+ new_means = buckets.new_zeros(num_clusters, dim, dtype=dtype)
110
+ new_means.scatter_add_(0, repeat(buckets, "n -> n d", d=dim), samples)
111
+ new_means = new_means / bins_min_clamped[..., None]
112
+
113
+ means = torch.where(zero_mask[..., None], means, new_means)
114
+
115
+ return means, bins
116
+
117
+
118
+ class EuclideanCodebook(nn.Module):
119
+ """Codebook with Euclidean distance.
120
+ Args:
121
+ dim (int): Dimension.
122
+ codebook_size (int): Codebook size.
123
+ kmeans_init (bool): Whether to use k-means to initialize the codebooks.
124
+ If set to true, run the k-means algorithm on the first training batch and use
125
+ the learned centroids as initialization.
126
+ kmeans_iters (int): Number of iterations used for k-means algorithm at initialization.
127
+ decay (float): Decay for exponential moving average over the codebooks.
128
+ epsilon (float): Epsilon value for numerical stability.
129
+ threshold_ema_dead_code (int): Threshold for dead code expiration. Replace any codes
130
+ that have an exponential moving average cluster size less than the specified threshold with
131
+ randomly selected vector from the current batch.
132
+ """
133
+ def __init__(
134
+ self,
135
+ dim: int,
136
+ codebook_size: int,
137
+ kmeans_init: int = False,
138
+ kmeans_iters: int = 10,
139
+ decay: float = 0.99,
140
+ epsilon: float = 1e-5,
141
+ threshold_ema_dead_code: int = 2,
142
+ ):
143
+ super().__init__()
144
+ self.decay = decay
145
+ init_fn: tp.Union[tp.Callable[..., torch.Tensor], tp.Any] = uniform_init if not kmeans_init else torch.zeros
146
+ embed = init_fn(codebook_size, dim)
147
+
148
+ self.codebook_size = codebook_size
149
+
150
+ self.kmeans_iters = kmeans_iters
151
+ self.epsilon = epsilon
152
+ self.threshold_ema_dead_code = threshold_ema_dead_code
153
+
154
+ # Flag variable to indicate whether the codebook is initialized
155
+ self.register_buffer("inited", torch.Tensor([not kmeans_init]))
156
+ # Runing EMA cluster size/count: N_i^t in eq. (6) in vqvae paper
157
+ self.register_buffer("cluster_size", torch.zeros(codebook_size))
158
+ # Codebook
159
+ self.register_buffer("embed", embed)
160
+ # EMA codebook: eq. (7) in vqvae paper
161
+ self.register_buffer("embed_avg", embed.clone())
162
+
163
+ @torch.jit.ignore
164
+ def init_embed_(self, data):
165
+ """ Initialize codebook.
166
+ Args:
167
+ data (tensor): [B * T, D].
168
+ """
169
+ if self.inited:
170
+ return
171
+
172
+ # if is_primary():
173
+ # print(data.shape)
174
+
175
+ ## NOTE (snippet added by Songxiang Liu): gather data from all gpus
176
+ if dist.is_available() and dist.is_initialized():
177
+ # [B * T * world_size, D]
178
+ data = SyncFunction.apply(data)
179
+
180
+ # if is_primary():
181
+ # print(data.shape)
182
+
183
+ embed, cluster_size = kmeans(data, self.codebook_size, self.kmeans_iters)
184
+ self.embed.data.copy_(embed)
185
+ self.embed_avg.data.copy_(embed.clone())
186
+ self.cluster_size.data.copy_(cluster_size)
187
+ self.inited.data.copy_(torch.Tensor([True]))
188
+ # Make sure all buffers across workers are in sync after initialization
189
+ broadcast_tensors(self.buffers())
190
+
191
+ def replace_(self, samples, mask):
192
+ modified_codebook = torch.where(
193
+ mask[..., None], sample_vectors(samples, self.codebook_size), self.embed
194
+ )
195
+ self.embed.data.copy_(modified_codebook)
196
+
197
+ def expire_codes_(self, batch_samples):
198
+ if self.threshold_ema_dead_code == 0:
199
+ return
200
+
201
+ expired_codes = self.cluster_size < self.threshold_ema_dead_code
202
+ if not torch.any(expired_codes):
203
+ return
204
+
205
+ # if is_primary():
206
+ # print(batch_samples.shape)
207
+
208
+ ## NOTE (snippet added by Songxiang Liu): gather data from all gpus
209
+ if is_distributed():
210
+ # [B * T * world_size, D]
211
+ batch_samples = SyncFunction.apply(batch_samples)
212
+
213
+ # if is_primary():
214
+ # print(batch_samples.shape)
215
+
216
+ batch_samples = rearrange(batch_samples, "... d -> (...) d")
217
+ self.replace_(batch_samples, mask=expired_codes)
218
+ broadcast_tensors(self.buffers())
219
+
220
+ def preprocess(self, x):
221
+ x = rearrange(x, "... d -> (...) d")
222
+ return x
223
+
224
+ def quantize(self, x):
225
+ embed = self.embed.t()
226
+ dist = -(
227
+ x.pow(2).sum(1, keepdim=True)
228
+ - 2 * x @ embed
229
+ + embed.pow(2).sum(0, keepdim=True)
230
+ )
231
+ embed_ind = dist.max(dim=-1).indices
232
+ return embed_ind
233
+
234
+ def postprocess_emb(self, embed_ind, shape):
235
+ return embed_ind.view(*shape[:-1])
236
+
237
+ def dequantize(self, embed_ind):
238
+ quantize = F.embedding(embed_ind, self.embed)
239
+ return quantize
240
+
241
+ def encode(self, x):
242
+ shape = x.shape
243
+ # pre-process
244
+ x = self.preprocess(x) # [B, T, D] -> [B*T, D]
245
+ # quantize
246
+ embed_ind = self.quantize(x)
247
+ # post-process
248
+ embed_ind = self.postprocess_emb(embed_ind, shape)
249
+ return embed_ind
250
+
251
+ def decode(self, embed_ind):
252
+ quantize = self.dequantize(embed_ind)
253
+ return quantize
254
+
255
+ def forward(self, x):
256
+ # shape: [B, T, D]
257
+ shape, dtype = x.shape, x.dtype
258
+ x = self.preprocess(x) # [B, T, D] -> [B*T, D]
259
+
260
+ # Initialize codebook
261
+ self.init_embed_(x)
262
+
263
+ embed_ind = self.quantize(x) # [B*T,]
264
+ embed_onehot = F.one_hot(embed_ind, self.codebook_size).type(dtype) # [B*T, cb-size]
265
+ embed_ind = self.postprocess_emb(embed_ind, shape) # [B, T]
266
+ quantize = self.dequantize(embed_ind) # [B, T, D]
267
+
268
+ if self.training:
269
+ ### Update codebook by EMA
270
+ embed_onehot_sum = embed_onehot.sum(0) # [cb-size,]
271
+ embed_sum = x.t() @ embed_onehot # [D, cb-size]
272
+ # if is_primary():
273
+ # print("--------- 1 ---------")
274
+ # print(embed_onehot_sum[:10])
275
+ if is_distributed():
276
+ dist.all_reduce(embed_onehot_sum)
277
+ dist.all_reduce(embed_sum)
278
+ # if is_primary():
279
+ # print(embed_onehot_sum[:10])
280
+ # print("--------- 2 ---------")
281
+ # Update ema cluster count N_i^t, eq. (6) in vqvae paper
282
+ self.cluster_size.data.mul_(self.decay).add_(
283
+ embed_onehot_sum, alpha=1 - self.decay
284
+ )
285
+ # Update ema embed: eq. (7) in vqvae paper
286
+ self.embed_avg.data.mul_(self.decay).add_(embed_sum.t(), alpha=1 - self.decay)
287
+ # apply laplace smoothing
288
+ n = self.cluster_size.sum()
289
+ cluster_size = (
290
+ (self.cluster_size + self.epsilon) / (n + self.codebook_size * self.epsilon) * n
291
+ )
292
+ # Update ema embed: eq. (8) in vqvae paper
293
+ embed_normalized = self.embed_avg / cluster_size.unsqueeze(1)
294
+ self.embed.data.copy_(embed_normalized)
295
+
296
+ # We do the expiry of code at that point as buffers are in sync
297
+ # and all the workers will take the same decision.
298
+ self.expire_codes_(x)
299
+
300
+ return quantize, embed_ind
301
+
302
+
303
+ class VectorQuantization(nn.Module):
304
+ """Vector quantization implementation.
305
+ Currently supports only euclidean distance.
306
+ Args:
307
+ dim (int): Dimension
308
+ codebook_size (int): Codebook size
309
+ codebook_dim (int): Codebook dimension. If not defined, uses the specified dimension in dim.
310
+ decay (float): Decay for exponential moving average over the codebooks.
311
+ epsilon (float): Epsilon value for numerical stability.
312
+ kmeans_init (bool): Whether to use kmeans to initialize the codebooks.
313
+ kmeans_iters (int): Number of iterations used for kmeans initialization.
314
+ threshold_ema_dead_code (int): Threshold for dead code expiration. Replace any codes
315
+ that have an exponential moving average cluster size less than the specified threshold with
316
+ randomly selected vector from the current batch.
317
+ commitment_weight (float): Weight for commitment loss.
318
+ """
319
+ def __init__(
320
+ self,
321
+ dim: int,
322
+ codebook_size: int,
323
+ codebook_dim: tp.Optional[int] = None,
324
+ decay: float = 0.99,
325
+ epsilon: float = 1e-5,
326
+ kmeans_init: bool = True,
327
+ kmeans_iters: int = 50,
328
+ threshold_ema_dead_code: int = 2,
329
+ commitment_weight: float = 1.,
330
+ ):
331
+ super().__init__()
332
+ _codebook_dim: int = default(codebook_dim, dim)
333
+
334
+ requires_projection = _codebook_dim != dim
335
+ self.project_in = (nn.Linear(dim, _codebook_dim) if requires_projection else nn.Identity())
336
+ self.project_out = (nn.Linear(_codebook_dim, dim) if requires_projection else nn.Identity())
337
+
338
+ self.epsilon = epsilon
339
+ self.commitment_weight = commitment_weight
340
+
341
+ self._codebook = EuclideanCodebook(dim=_codebook_dim, codebook_size=codebook_size,
342
+ kmeans_init=kmeans_init, kmeans_iters=kmeans_iters,
343
+ decay=decay, epsilon=epsilon,
344
+ threshold_ema_dead_code=threshold_ema_dead_code)
345
+ self.codebook_size = codebook_size
346
+
347
+ @property
348
+ def codebook(self):
349
+ return self._codebook.embed
350
+
351
+ def encode(self, x):
352
+ x = rearrange(x, "b d n -> b n d")
353
+ x = self.project_in(x)
354
+ embed_in = self._codebook.encode(x)
355
+ return embed_in
356
+
357
+ def decode(self, embed_ind):
358
+ quantize = self._codebook.decode(embed_ind)
359
+ quantize = self.project_out(quantize)
360
+ quantize = rearrange(quantize, "b n d -> b d n")
361
+ return quantize
362
+
363
+ def forward(self, x):
364
+ device = x.device
365
+ x = rearrange(x, "b d n -> b n d")
366
+ x = self.project_in(x)
367
+
368
+ quantize, embed_ind = self._codebook(x)
369
+
370
+ if self.training:
371
+ quantize = x + (quantize - x).detach()
372
+
373
+ loss = torch.tensor([0.0], device=device, requires_grad=self.training)
374
+
375
+ if self.training:
376
+ if self.commitment_weight > 0:
377
+ commit_loss = F.mse_loss(quantize.detach(), x)
378
+ loss = loss + commit_loss * self.commitment_weight
379
+
380
+ quantize = self.project_out(quantize)
381
+ quantize = rearrange(quantize, "b n d -> b d n")
382
+ return quantize, embed_ind, loss
383
+
384
+
385
+ class ResidualVectorQuantization(nn.Module):
386
+ """Residual vector quantization implementation.
387
+ Follows Algorithm 1. in https://arxiv.org/pdf/2107.03312.pdf
388
+ """
389
+ def __init__(self, *, num_quantizers, **kwargs):
390
+ super().__init__()
391
+ self.layers = nn.ModuleList(
392
+ [VectorQuantization(**kwargs) for _ in range(num_quantizers)]
393
+ )
394
+
395
+ def forward(self, x, n_q: tp.Optional[int] = None):
396
+ quantized_out = 0.0
397
+ residual = x
398
+
399
+ all_losses = []
400
+ all_indices = []
401
+
402
+ n_q = n_q or len(self.layers)
403
+
404
+ for layer in self.layers[:n_q]:
405
+ quantized, indices, loss = layer(residual)
406
+ residual = residual - quantized
407
+ quantized_out = quantized_out + quantized
408
+
409
+ all_indices.append(indices)
410
+ all_losses.append(loss)
411
+
412
+ out_losses, out_indices = map(torch.stack, (all_losses, all_indices))
413
+ return quantized_out, out_indices, out_losses
414
+
415
+ def encode(self, x: torch.Tensor, n_q: tp.Optional[int] = None) -> torch.Tensor:
416
+ residual = x
417
+ all_indices = []
418
+ n_q = n_q or len(self.layers)
419
+ for layer in self.layers[:n_q]:
420
+ indices = layer.encode(residual)
421
+ quantized = layer.decode(indices)
422
+ residual = residual - quantized
423
+ all_indices.append(indices)
424
+ out_indices = torch.stack(all_indices)
425
+ return out_indices
426
+
427
+ def decode(self, q_indices: torch.Tensor) -> torch.Tensor:
428
+ quantized_out = torch.tensor(0.0, device=q_indices.device)
429
+ for i, indices in enumerate(q_indices):
430
+ layer = self.layers[i]
431
+ quantized = layer.decode(indices)
432
+ quantized_out = quantized_out + quantized
433
+ return quantized_out
songgen/xcodec_wrapper/xcodec_infer/quantization/distrib.py ADDED
@@ -0,0 +1,125 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Meta Platforms, Inc. and affiliates.
2
+ # All rights reserved.
3
+ #
4
+ # This source code is licensed under the license found in the
5
+ # LICENSE file in the root directory of this source tree.
6
+
7
+ """Torch distributed utilities."""
8
+
9
+ import typing as tp
10
+
11
+ import torch
12
+
13
+
14
+ def rank():
15
+ if torch.distributed.is_initialized():
16
+ return torch.distributed.get_rank()
17
+ else:
18
+ return 0
19
+
20
+
21
+ def world_size():
22
+ if torch.distributed.is_initialized():
23
+ return torch.distributed.get_world_size()
24
+ else:
25
+ return 1
26
+
27
+
28
+ def is_distributed():
29
+ return world_size() > 1
30
+
31
+
32
+ def all_reduce(tensor: torch.Tensor, op=torch.distributed.ReduceOp.SUM):
33
+ if is_distributed():
34
+ return torch.distributed.all_reduce(tensor, op)
35
+
36
+
37
+ def _is_complex_or_float(tensor):
38
+ return torch.is_floating_point(tensor) or torch.is_complex(tensor)
39
+
40
+
41
+ def _check_number_of_params(params: tp.List[torch.Tensor]):
42
+ # utility function to check that the number of params in all workers is the same,
43
+ # and thus avoid a deadlock with distributed all reduce.
44
+ if not is_distributed() or not params:
45
+ return
46
+ #print('params[0].device ', params[0].device)
47
+ tensor = torch.tensor([len(params)], device=params[0].device, dtype=torch.long)
48
+ all_reduce(tensor)
49
+ if tensor.item() != len(params) * world_size():
50
+ # If not all the workers have the same number, for at least one of them,
51
+ # this inequality will be verified.
52
+ raise RuntimeError(f"Mismatch in number of params: ours is {len(params)}, "
53
+ "at least one worker has a different one.")
54
+
55
+
56
+ def broadcast_tensors(tensors: tp.Iterable[torch.Tensor], src: int = 0):
57
+ """Broadcast the tensors from the given parameters to all workers.
58
+ This can be used to ensure that all workers have the same model to start with.
59
+ """
60
+ if not is_distributed():
61
+ return
62
+ tensors = [tensor for tensor in tensors if _is_complex_or_float(tensor)]
63
+ _check_number_of_params(tensors)
64
+ handles = []
65
+ for tensor in tensors:
66
+ handle = torch.distributed.broadcast(tensor.data, src=src, async_op=True)
67
+ handles.append(handle)
68
+ for handle in handles:
69
+ handle.wait()
70
+
71
+
72
+ def sync_buffer(buffers, average=True):
73
+ """
74
+ Sync grad for buffers. If average is False, broadcast instead of averaging.
75
+ """
76
+ if not is_distributed():
77
+ return
78
+ handles = []
79
+ for buffer in buffers:
80
+ if torch.is_floating_point(buffer.data):
81
+ if average:
82
+ handle = torch.distributed.all_reduce(
83
+ buffer.data, op=torch.distributed.ReduceOp.SUM, async_op=True)
84
+ else:
85
+ handle = torch.distributed.broadcast(
86
+ buffer.data, src=0, async_op=True)
87
+ handles.append((buffer, handle))
88
+ for buffer, handle in handles:
89
+ handle.wait()
90
+ if average:
91
+ buffer.data /= world_size
92
+
93
+
94
+ def sync_grad(params):
95
+ """
96
+ Simpler alternative to DistributedDataParallel, that doesn't rely
97
+ on any black magic. For simple models it can also be as fast.
98
+ Just call this on your model parameters after the call to backward!
99
+ """
100
+ if not is_distributed():
101
+ return
102
+ handles = []
103
+ for p in params:
104
+ if p.grad is not None:
105
+ handle = torch.distributed.all_reduce(
106
+ p.grad.data, op=torch.distributed.ReduceOp.SUM, async_op=True)
107
+ handles.append((p, handle))
108
+ for p, handle in handles:
109
+ handle.wait()
110
+ p.grad.data /= world_size()
111
+
112
+
113
+ def average_metrics(metrics: tp.Dict[str, float], count=1.):
114
+ """Average a dictionary of metrics across all workers, using the optional
115
+ `count` as unormalized weight.
116
+ """
117
+ if not is_distributed():
118
+ return metrics
119
+ keys, values = zip(*metrics.items())
120
+ device = 'cuda' if torch.cuda.is_available() else 'cpu'
121
+ tensor = torch.tensor(list(values) + [1], device=device, dtype=torch.float32)
122
+ tensor *= count
123
+ all_reduce(tensor)
124
+ averaged = (tensor[:-1] / tensor[-1]).cpu().tolist()
125
+ return dict(zip(keys, averaged))
songgen/xcodec_wrapper/xcodec_infer/quantization/vq.py ADDED
@@ -0,0 +1,115 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Meta Platforms, Inc. and affiliates.
2
+ # All rights reserved.
3
+ #
4
+ # This source code is licensed under the license found in the
5
+ # LICENSE file in the root directory of this source tree.
6
+
7
+ """Residual vector quantizer implementation."""
8
+
9
+ from dataclasses import dataclass, field
10
+ import math
11
+ import typing as tp
12
+
13
+ import torch
14
+ from torch import nn
15
+
16
+ # from .core_vq import ResidualVectorQuantization
17
+ from .core_vq_lsx_version import ResidualVectorQuantization
18
+
19
+
20
+ @dataclass
21
+ class QuantizedResult:
22
+ quantized: torch.Tensor
23
+ codes: torch.Tensor
24
+ bandwidth: torch.Tensor # bandwidth in kb/s used, per batch item.
25
+ penalty: tp.Optional[torch.Tensor] = None
26
+ metrics: dict = field(default_factory=dict)
27
+
28
+
29
+ class ResidualVectorQuantizer(nn.Module):
30
+ """Residual Vector Quantizer.
31
+ Args:
32
+ dimension (int): Dimension of the codebooks.
33
+ n_q (int): Number of residual vector quantizers used.
34
+ bins (int): Codebook size.
35
+ decay (float): Decay for exponential moving average over the codebooks.
36
+ kmeans_init (bool): Whether to use kmeans to initialize the codebooks.
37
+ kmeans_iters (int): Number of iterations used for kmeans initialization.
38
+ threshold_ema_dead_code (int): Threshold for dead code expiration. Replace any codes
39
+ that have an exponential moving average cluster size less than the specified threshold with
40
+ randomly selected vector from the current batch.
41
+ """
42
+ def __init__(
43
+ self,
44
+ dimension: int = 256,
45
+ n_q: int = 8,
46
+ bins: int = 1024,
47
+ decay: float = 0.99,
48
+ kmeans_init: bool = True,
49
+ kmeans_iters: int = 50,
50
+ threshold_ema_dead_code: int = 2,
51
+ ):
52
+ super().__init__()
53
+ self.n_q = n_q
54
+ self.dimension = dimension
55
+ self.bins = bins
56
+ self.decay = decay
57
+ self.kmeans_init = kmeans_init
58
+ self.kmeans_iters = kmeans_iters
59
+ self.threshold_ema_dead_code = threshold_ema_dead_code
60
+ self.vq = ResidualVectorQuantization(
61
+ dim=self.dimension,
62
+ codebook_size=self.bins,
63
+ num_quantizers=self.n_q,
64
+ decay=self.decay,
65
+ kmeans_init=self.kmeans_init,
66
+ kmeans_iters=self.kmeans_iters,
67
+ threshold_ema_dead_code=self.threshold_ema_dead_code,
68
+ )
69
+
70
+ def forward(self, x: torch.Tensor, sample_rate: int, bandwidth: tp.Optional[float] = None): # -> QuantizedResult:
71
+ """Residual vector quantization on the given input tensor.
72
+ Args:
73
+ x (torch.Tensor): Input tensor.
74
+ sample_rate (int): Sample rate of the input tensor.
75
+ bandwidth (float): Target bandwidth.
76
+ Returns:
77
+ QuantizedResult:
78
+ The quantized (or approximately quantized) representation with
79
+ the associated bandwidth and any penalty term for the loss.
80
+ """
81
+ bw_per_q = self.get_bandwidth_per_quantizer(sample_rate)
82
+ n_q = self.get_num_quantizers_for_bandwidth(sample_rate, bandwidth)
83
+ quantized, codes, commit_loss = self.vq(x, n_q=n_q)
84
+ bw = torch.tensor(n_q * bw_per_q).to(x)
85
+ return quantized, codes, bw, torch.mean(commit_loss)
86
+ #return QuantizedResult(quantized, codes, bw, penalty=torch.mean(commit_loss))
87
+
88
+ def get_num_quantizers_for_bandwidth(self, sample_rate: int, bandwidth: tp.Optional[float] = None) -> int:
89
+ """Return n_q based on specified target bandwidth.
90
+ """
91
+ bw_per_q = self.get_bandwidth_per_quantizer(sample_rate)
92
+ n_q = self.n_q
93
+ if bandwidth and bandwidth > 0.:
94
+ n_q = int(max(1, math.floor(bandwidth / bw_per_q)))
95
+ return n_q
96
+
97
+ def get_bandwidth_per_quantizer(self, sample_rate: int):
98
+ """Return bandwidth per quantizer for a given input sample rate.
99
+ """
100
+ return math.log2(self.bins) * sample_rate / 1000
101
+
102
+ def encode(self, x: torch.Tensor, sample_rate: int, bandwidth: tp.Optional[float] = None) -> torch.Tensor:
103
+ """Encode a given input tensor with the specified sample rate at the given bandwidth.
104
+ The RVQ encode method sets the appropriate number of quantizer to use
105
+ and returns indices for each quantizer.
106
+ """
107
+ n_q = self.get_num_quantizers_for_bandwidth(sample_rate, bandwidth)
108
+ codes = self.vq.encode(x, n_q=n_q)
109
+ return codes
110
+
111
+ def decode(self, codes: torch.Tensor) -> torch.Tensor:
112
+ """Decode the given codes to the quantized representation.
113
+ """
114
+ quantized = self.vq.decode(codes)
115
+ return quantized
songgen/xcodec_wrapper/xcodec_infer/xutils/__init__.py ADDED
File without changes
songgen/xcodec_wrapper/xcodec_infer/xutils/ddp_utils.py ADDED
@@ -0,0 +1,200 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import logging
2
+ import random
3
+ import subprocess
4
+ from datetime import datetime
5
+
6
+ import numpy as np
7
+ import torch
8
+ import torch.distributed as dist
9
+ from torch.nn.parallel import DistributedDataParallel
10
+ from torch.nn.parallel.distributed import _find_tensors
11
+ import torch.optim
12
+ import torch.utils.data
13
+ from packaging import version
14
+ from omegaconf import OmegaConf
15
+
16
+
17
+ def set_random_seed(seed):
18
+ random.seed(seed)
19
+ np.random.seed(seed)
20
+ torch.manual_seed(seed)
21
+ torch.cuda.manual_seed_all(seed)
22
+
23
+
24
+ def is_logging_process():
25
+ return not dist.is_initialized() or dist.get_rank() == 0
26
+
27
+
28
+ def get_logger(cfg, name=None):
29
+ # log_file_path is used when unit testing
30
+ if is_logging_process():
31
+ logging.config.dictConfig(
32
+ OmegaConf.to_container(cfg.job_logging_config, resolve=True)
33
+ )
34
+ return logging.getLogger(name)
35
+
36
+
37
+ # from https://github.com/Lightning-AI/lightning-bolts/blob/5d61197cd2f491f69e238137a5edabe80ae14ad9/pl_bolts/models/self_supervised/simclr/simclr_module.py#L20
38
+ class SyncFunction(torch.autograd.Function):
39
+ @staticmethod
40
+ # @torch.no_grad()
41
+ def forward(ctx, tensor):
42
+ ctx.batch_size = tensor.shape[0]
43
+
44
+ gathered_tensor = [torch.zeros_like(tensor) for _ in range(torch.distributed.get_world_size())]
45
+
46
+ torch.distributed.all_gather(gathered_tensor, tensor)
47
+ gathered_tensor = torch.cat(gathered_tensor, 0)
48
+
49
+ return gathered_tensor
50
+
51
+ @staticmethod
52
+ def backward(ctx, grad_output):
53
+ grad_input = grad_output.clone()
54
+ torch.distributed.all_reduce(grad_input, op=torch.distributed.ReduceOp.SUM, async_op=False)
55
+
56
+ idx_from = torch.distributed.get_rank() * ctx.batch_size
57
+ idx_to = (torch.distributed.get_rank() + 1) * ctx.batch_size
58
+ return grad_input[idx_from:idx_to]
59
+
60
+
61
+ def get_timestamp():
62
+ return datetime.now().strftime("%y%m%d-%H%M%S")
63
+
64
+
65
+ def get_commit_hash():
66
+ message = subprocess.check_output(["git", "rev-parse", "--short", "HEAD"])
67
+ return message.strip().decode("utf-8")
68
+
69
+
70
+ class DDP(DistributedDataParallel):
71
+ """
72
+ Override the forward call in lightning so it goes to training and validation step respectively
73
+ """
74
+
75
+ def forward(self, *inputs, **kwargs): # pragma: no cover
76
+ if version.parse(torch.__version__[:6]) < version.parse("1.11"):
77
+ self._sync_params()
78
+ inputs, kwargs = self.scatter(inputs, kwargs, self.device_ids)
79
+ assert len(self.device_ids) == 1
80
+ if self.module.training:
81
+ output = self.module.training_step(*inputs[0], **kwargs[0])
82
+ elif self.module.testing:
83
+ output = self.module.test_step(*inputs[0], **kwargs[0])
84
+ else:
85
+ output = self.module.validation_step(*inputs[0], **kwargs[0])
86
+ if torch.is_grad_enabled():
87
+ # We'll return the output object verbatim since it is a freeform
88
+ # object. We need to find any tensors in this object, though,
89
+ # because we need to figure out which parameters were used during
90
+ # this forward pass, to ensure we short circuit reduction for any
91
+ # unused parameters. Only if `find_unused_parameters` is set.
92
+ if self.find_unused_parameters:
93
+ self.reducer.prepare_for_backward(list(_find_tensors(output)))
94
+ else:
95
+ self.reducer.prepare_for_backward([])
96
+ else:
97
+ from torch.nn.parallel.distributed import \
98
+ logging, Join, _DDPSink, _tree_flatten_with_rref, _tree_unflatten_with_rref
99
+ with torch.autograd.profiler.record_function("DistributedDataParallel.forward"):
100
+ if torch.is_grad_enabled() and self.require_backward_grad_sync:
101
+ self.logger.set_runtime_stats_and_log()
102
+ self.num_iterations += 1
103
+ self.reducer.prepare_for_forward()
104
+
105
+ # Notify the join context that this process has not joined, if
106
+ # needed
107
+ work = Join.notify_join_context(self)
108
+ if work:
109
+ self.reducer._set_forward_pass_work_handle(
110
+ work, self._divide_by_initial_world_size
111
+ )
112
+
113
+ # Calling _rebuild_buckets before forward compuation,
114
+ # It may allocate new buckets before deallocating old buckets
115
+ # inside _rebuild_buckets. To save peak memory usage,
116
+ # call _rebuild_buckets before the peak memory usage increases
117
+ # during forward computation.
118
+ # This should be called only once during whole training period.
119
+ if torch.is_grad_enabled() and self.reducer._rebuild_buckets():
120
+ logging.info("Reducer buckets have been rebuilt in this iteration.")
121
+ self._has_rebuilt_buckets = True
122
+
123
+ # sync params according to location (before/after forward) user
124
+ # specified as part of hook, if hook was specified.
125
+ buffer_hook_registered = hasattr(self, 'buffer_hook')
126
+ if self._check_sync_bufs_pre_fwd():
127
+ self._sync_buffers()
128
+
129
+ if self._join_config.enable:
130
+ # Notify joined ranks whether they should sync in backwards pass or not.
131
+ self._check_global_requires_backward_grad_sync(is_joined_rank=False)
132
+
133
+ inputs, kwargs = self.scatter(inputs, kwargs, self.device_ids)
134
+ if self.module.training:
135
+ output = self.module.training_step(*inputs[0], **kwargs[0])
136
+ elif self.module.testing:
137
+ output = self.module.test_step(*inputs[0], **kwargs[0])
138
+ else:
139
+ output = self.module.validation_step(*inputs[0], **kwargs[0])
140
+
141
+ # sync params according to location (before/after forward) user
142
+ # specified as part of hook, if hook was specified.
143
+ if self._check_sync_bufs_post_fwd():
144
+ self._sync_buffers()
145
+
146
+ if torch.is_grad_enabled() and self.require_backward_grad_sync:
147
+ self.require_forward_param_sync = True
148
+ # We'll return the output object verbatim since it is a freeform
149
+ # object. We need to find any tensors in this object, though,
150
+ # because we need to figure out which parameters were used during
151
+ # this forward pass, to ensure we short circuit reduction for any
152
+ # unused parameters. Only if `find_unused_parameters` is set.
153
+ if self.find_unused_parameters and not self.static_graph:
154
+ # Do not need to populate this for static graph.
155
+ self.reducer.prepare_for_backward(list(_find_tensors(output)))
156
+ else:
157
+ self.reducer.prepare_for_backward([])
158
+ else:
159
+ self.require_forward_param_sync = False
160
+
161
+ # TODO: DDPSink is currently enabled for unused parameter detection and
162
+ # static graph training for first iteration.
163
+ if (self.find_unused_parameters and not self.static_graph) or (
164
+ self.static_graph and self.num_iterations == 1
165
+ ):
166
+ state_dict = {
167
+ 'static_graph': self.static_graph,
168
+ 'num_iterations': self.num_iterations,
169
+ }
170
+
171
+ output_tensor_list, treespec, output_is_rref = _tree_flatten_with_rref(
172
+ output
173
+ )
174
+ output_placeholders = [None for _ in range(len(output_tensor_list))]
175
+ # Do not touch tensors that have no grad_fn, which can cause issues
176
+ # such as https://github.com/pytorch/pytorch/issues/60733
177
+ for i, output in enumerate(output_tensor_list):
178
+ if torch.is_tensor(output) and output.grad_fn is None:
179
+ output_placeholders[i] = output
180
+
181
+ # When find_unused_parameters=True, makes tensors which require grad
182
+ # run through the DDPSink backward pass. When not all outputs are
183
+ # used in loss, this makes those corresponding tensors receive
184
+ # undefined gradient which the reducer then handles to ensure
185
+ # param.grad field is not touched and we don't error out.
186
+ passthrough_tensor_list = _DDPSink.apply(
187
+ self.reducer,
188
+ state_dict,
189
+ *output_tensor_list,
190
+ )
191
+ for i in range(len(output_placeholders)):
192
+ if output_placeholders[i] is None:
193
+ output_placeholders[i] = passthrough_tensor_list[i]
194
+
195
+ # Reconstruct output data structure.
196
+ output = _tree_unflatten_with_rref(
197
+ output_placeholders, treespec, output_is_rref
198
+ )
199
+ return output
200
+
songgen/xcodec_wrapper/xcodec_infer/xutils/utils.py ADDED
@@ -0,0 +1,259 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import dataclasses
2
+ import glob
3
+ import importlib
4
+ import random
5
+ import numpy as np
6
+ import torch
7
+ import warnings
8
+ import os
9
+ import time
10
+ import torch.utils.tensorboard as tensorboard
11
+ from torch import distributed as dist
12
+ import sys
13
+ import yaml
14
+ import json
15
+ import re
16
+ import pathlib
17
+ import matplotlib
18
+ matplotlib.use("Agg")
19
+ import matplotlib.pylab as plt
20
+
21
+
22
+ def plot_spectrogram(spectrogram):
23
+ fig, ax = plt.subplots(figsize=(10, 2))
24
+ im = ax.imshow(spectrogram, aspect="auto", origin="lower",
25
+ interpolation='none')
26
+ plt.colorbar(im, ax=ax)
27
+
28
+ fig.canvas.draw()
29
+ plt.close()
30
+
31
+ return fig
32
+
33
+
34
+ def seed_everything(seed, cudnn_deterministic=False):
35
+ """
36
+ Function that sets seed for pseudo-random number generators in:
37
+ pytorch, numpy, python.random
38
+
39
+ Args:
40
+ seed: the integer value seed for global random state
41
+ """
42
+ if seed is not None:
43
+ random.seed(seed)
44
+ np.random.seed(seed)
45
+ torch.manual_seed(seed)
46
+ torch.cuda.manual_seed_all(seed)
47
+
48
+ if cudnn_deterministic:
49
+ torch.backends.cudnn.deterministic = True
50
+ warnings.warn('You have chosen to seed training. '
51
+ 'This will turn on the CUDNN deterministic setting, '
52
+ 'which can slow down your training considerably! '
53
+ 'You may see unexpected behavior when restarting '
54
+ 'from checkpoints.')
55
+
56
+ def is_primary():
57
+ return get_rank() == 0
58
+
59
+
60
+ def get_rank():
61
+ if not dist.is_available():
62
+ return 0
63
+ if not dist.is_initialized():
64
+ return 0
65
+
66
+ return dist.get_rank()
67
+
68
+
69
+ def load_yaml_config(path):
70
+ with open(path) as f:
71
+ config = yaml.full_load(f)
72
+ return config
73
+
74
+
75
+ def save_config_to_yaml(config, path):
76
+ assert path.endswith('.yaml')
77
+ with open(path, 'w') as f:
78
+ f.write(yaml.dump(config))
79
+ f.close()
80
+
81
+
82
+ def save_dict_to_json(d, path, indent=None):
83
+ json.dump(d, open(path, 'w'), indent=indent)
84
+
85
+
86
+ def load_dict_from_json(path):
87
+ return json.load(open(path, 'r'))
88
+
89
+
90
+ def write_args(args, path):
91
+ args_dict = dict((name, getattr(args, name)) for name in dir(args)if not name.startswith('_'))
92
+ with open(path, 'a') as args_file:
93
+ args_file.write('==> torch version: {}\n'.format(torch.__version__))
94
+ args_file.write('==> cudnn version: {}\n'.format(torch.backends.cudnn.version()))
95
+ args_file.write('==> Cmd:\n')
96
+ args_file.write(str(sys.argv))
97
+ args_file.write('\n==> args:\n')
98
+ for k, v in sorted(args_dict.items()):
99
+ args_file.write(' %s: %s\n' % (str(k), str(v)))
100
+ args_file.close()
101
+
102
+
103
+ class Logger(object):
104
+ def __init__(self, args):
105
+ self.args = args
106
+ self.save_dir = args.log_dir
107
+ self.is_primary = is_primary()
108
+
109
+ if self.is_primary:
110
+ os.makedirs(self.save_dir, exist_ok=True)
111
+
112
+ # save the args and config
113
+ self.config_dir = os.path.join(self.save_dir, 'configs')
114
+ os.makedirs(self.config_dir, exist_ok=True)
115
+ file_name = os.path.join(self.config_dir, 'args.txt')
116
+ write_args(args, file_name)
117
+
118
+ log_dir = os.path.join(self.save_dir, 'logs')
119
+ if not os.path.exists(log_dir):
120
+ os.makedirs(log_dir, exist_ok=True)
121
+ self.text_writer = open(os.path.join(log_dir, 'log.txt'), 'a') # 'w')
122
+ if args.tensorboard:
123
+ self.log_info('using tensorboard')
124
+ self.tb_writer = torch.utils.tensorboard.SummaryWriter(log_dir=log_dir) # tensorboard.SummaryWriter(log_dir=log_dir)
125
+ else:
126
+ self.tb_writer = None
127
+
128
+ def save_config(self, config):
129
+ if self.is_primary:
130
+ save_config_to_yaml(config, os.path.join(self.config_dir, 'config.yaml'))
131
+
132
+ def log_info(self, info, check_primary=True):
133
+ if self.is_primary or (not check_primary):
134
+ print(info)
135
+ if self.is_primary:
136
+ info = str(info)
137
+ time_str = time.strftime('%Y-%m-%d-%H-%M')
138
+ info = '{}: {}'.format(time_str, info)
139
+ if not info.endswith('\n'):
140
+ info += '\n'
141
+ self.text_writer.write(info)
142
+ self.text_writer.flush()
143
+
144
+ def add_scalar(self, **kargs):
145
+ """Log a scalar variable."""
146
+ if self.is_primary:
147
+ if self.tb_writer is not None:
148
+ self.tb_writer.add_scalar(**kargs)
149
+
150
+ def add_scalars(self, **kargs):
151
+ """Log a scalar variable."""
152
+ if self.is_primary:
153
+ if self.tb_writer is not None:
154
+ self.tb_writer.add_scalars(**kargs)
155
+
156
+ def add_image(self, **kargs):
157
+ """Log a scalar variable."""
158
+ if self.is_primary:
159
+ if self.tb_writer is not None:
160
+ self.tb_writer.add_image(**kargs)
161
+
162
+ def add_images(self, **kargs):
163
+ """Log a scalar variable."""
164
+ if self.is_primary:
165
+ if self.tb_writer is not None:
166
+ self.tb_writer.add_images(**kargs)
167
+
168
+ def close(self):
169
+ if self.is_primary:
170
+ self.text_writer.close()
171
+ self.tb_writer.close()
172
+
173
+
174
+ def cal_model_size(model, name=""):
175
+
176
+ all_size = sum(p.numel() for p in model.parameters())/1024.0/1024.0
177
+ return f'Model size of {name}: {all_size:.3f} MB'
178
+
179
+ param_size = 0
180
+ param_sum = 0
181
+ for param in model.parameters():
182
+ param_size += param.nelement() * param.element_size()
183
+ param_sum += param.nelement()
184
+ buffer_size = 0
185
+ buffer_sum = 0
186
+ for buffer in model.buffers():
187
+ buffer_size += buffer.nelement() * buffer.element_size()
188
+ buffer_sum += buffer.nelement()
189
+ all_size = (param_size + buffer_size) / 1024 / 1024
190
+
191
+ return f'Model size of {name}: {all_size:.3f} MB'
192
+ # print(f'Model size of {name}: {all_size:.3f}MB')
193
+ # return (param_size, param_sum, buffer_size, buffer_sum, all_size)
194
+
195
+
196
+ def load_obj(obj_path: str, default_obj_path: str = ''):
197
+ """ Extract an object from a given path.
198
+ Args:
199
+ obj_path: Path to an object to be extracted, including the object name.
200
+ e.g.: `src.trainers.meta_trainer.MetaTrainer`
201
+ `src.models.ada_style_speech.AdaStyleSpeechModel`
202
+ default_obj_path: Default object path.
203
+
204
+ Returns:
205
+ Extracted object.
206
+ Raises:
207
+ AttributeError: When the object does not have the given named attribute.
208
+
209
+ """
210
+ obj_path_list = obj_path.rsplit('.', 1)
211
+ obj_path = obj_path_list.pop(0) if len(obj_path_list) > 1 else default_obj_path
212
+ obj_name = obj_path_list[0]
213
+ module_obj = importlib.import_module(obj_path)
214
+ if not hasattr(module_obj, obj_name):
215
+ raise AttributeError(f'Object `{obj_name}` cannot be loaded from `{obj_path}`.')
216
+ return getattr(module_obj, obj_name)
217
+
218
+
219
+ def to_device(data, device=None, dtype=None, non_blocking=False, copy=False):
220
+ """Change the device of object recursively"""
221
+ if isinstance(data, dict):
222
+ return {
223
+ k: to_device(v, device, dtype, non_blocking, copy) for k, v in data.items()
224
+ }
225
+ elif dataclasses.is_dataclass(data) and not isinstance(data, type):
226
+ return type(data)(
227
+ *[
228
+ to_device(v, device, dtype, non_blocking, copy)
229
+ for v in dataclasses.astuple(data)
230
+ ]
231
+ )
232
+ # maybe namedtuple. I don't know the correct way to judge namedtuple.
233
+ elif isinstance(data, tuple) and type(data) is not tuple:
234
+ return type(data)(
235
+ *[to_device(o, device, dtype, non_blocking, copy) for o in data]
236
+ )
237
+ elif isinstance(data, (list, tuple)):
238
+ return type(data)(to_device(v, device, dtype, non_blocking, copy) for v in data)
239
+ elif isinstance(data, np.ndarray):
240
+ return to_device(torch.from_numpy(data), device, dtype, non_blocking, copy)
241
+ elif isinstance(data, torch.Tensor):
242
+ return data.to(device, dtype, non_blocking, copy)
243
+ else:
244
+ return data
245
+
246
+
247
+ def save_checkpoint(filepath, obj, ext='pth', num_ckpt_keep=10):
248
+ ckpts = sorted(pathlib.Path(filepath).parent.glob(f'*.{ext}'))
249
+ if len(ckpts) > num_ckpt_keep:
250
+ [os.remove(c) for c in ckpts[:-num_ckpt_keep]]
251
+ torch.save(obj, filepath)
252
+
253
+
254
+ def scan_checkpoint(cp_dir, prefix='ckpt_'):
255
+ pattern = os.path.join(cp_dir, prefix + '????????.pth')
256
+ cp_list = glob.glob(pattern)
257
+ if len(cp_list) == 0:
258
+ return None
259
+ return sorted(cp_list)[-1]