Spaces:
Running on Zero
Running on Zero
Vansh Chugh commited on
Commit ·
ad73d02
1
Parent(s): b06789a
initial deploy
Browse files- .gitignore +5 -0
- README.md +5 -4
- SOURCES.md +4 -0
- app.py +214 -0
- model.json +20 -0
- requirements.txt +19 -0
- songgen/__init__.py +31 -0
- songgen/configuration_songgen.py +250 -0
- songgen/custom_modeling_outputs.py +114 -0
- songgen/logits_processors.py +85 -0
- songgen/lyrics_utils/__init__.py +0 -0
- songgen/lyrics_utils/lyrics_encoder.py +1071 -0
- songgen/lyrics_utils/lyrics_tokenizer.py +874 -0
- songgen/lyrics_utils/zh_num2words.py +1209 -0
- songgen/modeling_songgen_dual_track.py +623 -0
- songgen/modeling_songgen_mixed.py +0 -0
- songgen/processing_songgen.py +141 -0
- songgen/xcodec_wrapper/__init__.py +2 -0
- songgen/xcodec_wrapper/configuration_xcodec.py +24 -0
- songgen/xcodec_wrapper/modeling_xcodec.py +167 -0
- songgen/xcodec_wrapper/xcodec_infer/ckpts/general_more/config_hubert_general.yaml +137 -0
- songgen/xcodec_wrapper/xcodec_infer/ckpts/general_more/xcodec_hubert_general_audio_v2.pth +3 -0
- songgen/xcodec_wrapper/xcodec_infer/descriptaudiocodec/__init__.py +0 -0
- songgen/xcodec_wrapper/xcodec_infer/descriptaudiocodec/dac/__init__.py +16 -0
- songgen/xcodec_wrapper/xcodec_infer/descriptaudiocodec/dac/model/__init__.py +4 -0
- songgen/xcodec_wrapper/xcodec_infer/descriptaudiocodec/dac/model/base.py +294 -0
- songgen/xcodec_wrapper/xcodec_infer/descriptaudiocodec/dac/model/dac.py +369 -0
- songgen/xcodec_wrapper/xcodec_infer/descriptaudiocodec/dac/model/discriminator.py +228 -0
- songgen/xcodec_wrapper/xcodec_infer/descriptaudiocodec/dac/nn/__init__.py +3 -0
- songgen/xcodec_wrapper/xcodec_infer/descriptaudiocodec/dac/nn/layers.py +33 -0
- songgen/xcodec_wrapper/xcodec_infer/descriptaudiocodec/dac/nn/loss.py +368 -0
- songgen/xcodec_wrapper/xcodec_infer/descriptaudiocodec/dac/nn/quantize.py +262 -0
- songgen/xcodec_wrapper/xcodec_infer/descriptaudiocodec/dac/utils/__init__.py +123 -0
- songgen/xcodec_wrapper/xcodec_infer/models/__init__.py +0 -0
- songgen/xcodec_wrapper/xcodec_infer/models/soundstream_semantic.py +157 -0
- songgen/xcodec_wrapper/xcodec_infer/modules/__init__.py +22 -0
- songgen/xcodec_wrapper/xcodec_infer/modules/conv.py +252 -0
- songgen/xcodec_wrapper/xcodec_infer/modules/lstm.py +28 -0
- songgen/xcodec_wrapper/xcodec_infer/modules/norm.py +28 -0
- songgen/xcodec_wrapper/xcodec_infer/modules/seanet.py +255 -0
- songgen/xcodec_wrapper/xcodec_infer/modules/semantic_module.py +294 -0
- songgen/xcodec_wrapper/xcodec_infer/modules/transformer.py +119 -0
- songgen/xcodec_wrapper/xcodec_infer/quantization/__init__.py +8 -0
- songgen/xcodec_wrapper/xcodec_infer/quantization/core_vq_lsx_version.py +433 -0
- songgen/xcodec_wrapper/xcodec_infer/quantization/distrib.py +125 -0
- songgen/xcodec_wrapper/xcodec_infer/quantization/vq.py +115 -0
- songgen/xcodec_wrapper/xcodec_infer/xutils/__init__.py +0 -0
- songgen/xcodec_wrapper/xcodec_infer/xutils/ddp_utils.py +200 -0
- songgen/xcodec_wrapper/xcodec_infer/xutils/utils.py +259 -0
.gitignore
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
__pycache__/
|
| 2 |
+
*.pyc
|
| 3 |
+
.DS_Store
|
| 4 |
+
.venv/
|
| 5 |
+
SongGen-repo/
|
README.md
CHANGED
|
@@ -1,13 +1,14 @@
|
|
| 1 |
---
|
| 2 |
title: SongGen
|
| 3 |
-
emoji:
|
| 4 |
colorFrom: purple
|
| 5 |
-
colorTo:
|
| 6 |
sdk: gradio
|
| 7 |
-
sdk_version:
|
| 8 |
-
python_version: '3.
|
| 9 |
app_file: app.py
|
| 10 |
pinned: false
|
|
|
|
| 11 |
---
|
| 12 |
|
| 13 |
Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
|
|
|
|
| 1 |
---
|
| 2 |
title: SongGen
|
| 3 |
+
emoji: 🎤
|
| 4 |
colorFrom: purple
|
| 5 |
+
colorTo: pink
|
| 6 |
sdk: gradio
|
| 7 |
+
sdk_version: 5.28.0
|
| 8 |
+
python_version: '3.11'
|
| 9 |
app_file: app.py
|
| 10 |
pinned: false
|
| 11 |
+
license: mit
|
| 12 |
---
|
| 13 |
|
| 14 |
Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
|
SOURCES.md
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Sources — SongGen
|
| 2 |
+
|
| 3 |
+
- Source repo: https://github.com/LiuZH-19/SongGen
|
| 4 |
+
- Paper: https://arxiv.org/html/2502.13128v2
|
app.py
ADDED
|
@@ -0,0 +1,214 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import sys
|
| 2 |
+
|
| 3 |
+
sys.stdout.reconfigure(line_buffering=True)
|
| 4 |
+
|
| 5 |
+
try:
|
| 6 |
+
import spaces
|
| 7 |
+
except ImportError:
|
| 8 |
+
# keep @spaces.GPU usable as a no-op; ZeroGPU requires this exact name.
|
| 9 |
+
class spaces:
|
| 10 |
+
class GPU:
|
| 11 |
+
def __init__(self, func=None, duration=60):
|
| 12 |
+
self.func = func
|
| 13 |
+
|
| 14 |
+
def __call__(self, *args, **kwargs):
|
| 15 |
+
if self.func is not None:
|
| 16 |
+
return self.func(*args, **kwargs)
|
| 17 |
+
func = args[0]
|
| 18 |
+
return func
|
| 19 |
+
|
| 20 |
+
import os
|
| 21 |
+
import shutil
|
| 22 |
+
import tempfile
|
| 23 |
+
import threading
|
| 24 |
+
|
| 25 |
+
import gradio as gr
|
| 26 |
+
import numpy as np
|
| 27 |
+
import soundfile as sf
|
| 28 |
+
import torch
|
| 29 |
+
from huggingface_hub import hf_hub_download
|
| 30 |
+
from pyharp import ModelCard, build_endpoint
|
| 31 |
+
|
| 32 |
+
from songgen import (
|
| 33 |
+
SongGenDualTrackForConditionalGeneration,
|
| 34 |
+
SongGenMixedForConditionalGeneration,
|
| 35 |
+
SongGenProcessor,
|
| 36 |
+
)
|
| 37 |
+
|
| 38 |
+
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
|
| 39 |
+
|
| 40 |
+
MODES = {
|
| 41 |
+
"Mixed": SongGenMixedForConditionalGeneration,
|
| 42 |
+
"Dual-track (vocals + accompaniment)": SongGenDualTrackForConditionalGeneration,
|
| 43 |
+
}
|
| 44 |
+
CHECKPOINTS = {
|
| 45 |
+
"Mixed": "LiuZH-19/SongGen_mixed_pro",
|
| 46 |
+
"Dual-track (vocals + accompaniment)": "LiuZH-19/SongGen_interleaving_A_V",
|
| 47 |
+
}
|
| 48 |
+
|
| 49 |
+
state = {name: {"model": None, "processor": None} for name in MODES}
|
| 50 |
+
loading_done = False
|
| 51 |
+
loading_error = None
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def fetch_xcodec_assets():
|
| 55 |
+
"""Grab the X-Codec checkpoint and its config. SongGen's XCodecModel
|
| 56 |
+
hardcodes this exact folder rather than taking a path argument, so the
|
| 57 |
+
files have to be here before any SongGen model is constructed."""
|
| 58 |
+
ckpt_dir = os.path.join(
|
| 59 |
+
os.path.dirname(__file__), "songgen", "xcodec_wrapper", "xcodec_infer", "ckpts", "general_more"
|
| 60 |
+
)
|
| 61 |
+
os.makedirs(ckpt_dir, exist_ok=True)
|
| 62 |
+
for filename in ("xcodec_hubert_general_audio_v2.pth", "config_hubert_general.yaml"):
|
| 63 |
+
dest = os.path.join(ckpt_dir, filename)
|
| 64 |
+
if not os.path.exists(dest):
|
| 65 |
+
src = hf_hub_download(repo_id="ZhenYe234/xcodec", filename=filename)
|
| 66 |
+
shutil.copy(src, dest)
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def load_all():
|
| 70 |
+
"""Download and build both modes on CPU only. Do NOT call .to(DEVICE) or
|
| 71 |
+
otherwise touch CUDA here — ZeroGPU only intercepts CUDA calls made
|
| 72 |
+
inside an @spaces.GPU-decorated call, not from a background thread. This
|
| 73 |
+
split is a no-op on CPU-only hardware (DEVICE == "cpu"), so keep it even
|
| 74 |
+
while testing on a personal CPU-tier Space, before GPU code is added later."""
|
| 75 |
+
global loading_done, loading_error
|
| 76 |
+
try:
|
| 77 |
+
fetch_xcodec_assets()
|
| 78 |
+
for mode_name, model_cls in MODES.items():
|
| 79 |
+
repo_id = CHECKPOINTS[mode_name]
|
| 80 |
+
print(f"Loading {mode_name} ({repo_id})...")
|
| 81 |
+
state[mode_name]["model"] = model_cls.from_pretrained(repo_id, attn_implementation="sdpa")
|
| 82 |
+
state[mode_name]["processor"] = SongGenProcessor(repo_id, "cpu")
|
| 83 |
+
print(f"{mode_name} loaded (CPU).")
|
| 84 |
+
except Exception as e:
|
| 85 |
+
loading_error = str(e)
|
| 86 |
+
print(f"Load error: {e}")
|
| 87 |
+
finally:
|
| 88 |
+
loading_done = True
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
threading.Thread(target=load_all, daemon=True).start()
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
model_card = ModelCard(
|
| 95 |
+
name="SongGen",
|
| 96 |
+
description=(
|
| 97 |
+
"Single-stage auto-regressive transformer for text-to-song generation. "
|
| 98 |
+
"Give it lyrics and a description of the music, plus an optional reference "
|
| 99 |
+
"voice, and it generates a full song."
|
| 100 |
+
),
|
| 101 |
+
author="Zihan Liu, Shuangrui Ding, Zhixiong Zhang, Xiaoyi Dong, Pan Zhang, Yuhang Zang, Yuhang Cao, Dahua Lin, Jiaqi Wang",
|
| 102 |
+
tags=["text-to-music", "song-generation"],
|
| 103 |
+
)
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def move_mode_to_device(mode_name):
|
| 107 |
+
"""Move one mode's model and helper submodules onto DEVICE the first time
|
| 108 |
+
it's used. Rebuilding isn't needed here since each mode's model/processor
|
| 109 |
+
pair is only ever referenced from this one spot."""
|
| 110 |
+
entry = state[mode_name]
|
| 111 |
+
model = entry["model"]
|
| 112 |
+
processor = entry["processor"]
|
| 113 |
+
if next(model.parameters()).device.type != DEVICE:
|
| 114 |
+
model.to(DEVICE)
|
| 115 |
+
processor.device = DEVICE
|
| 116 |
+
processor.demucs = processor.demucs.to(DEVICE)
|
| 117 |
+
return model, processor
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def write_wav(audio_arr, sample_rate):
|
| 121 |
+
"""Save a numpy audio array to a temp wav file and return its path."""
|
| 122 |
+
path = tempfile.NamedTemporaryFile(suffix=".wav", delete=False).name
|
| 123 |
+
sf.write(path, audio_arr, sample_rate)
|
| 124 |
+
return path
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def silent_wav(sample_rate, seconds=0.1):
|
| 128 |
+
"""A short silent placeholder for the vocals-stem output when the
|
| 129 |
+
selected mode doesn't produce a separate vocal track."""
|
| 130 |
+
return write_wav(np.zeros(int(seconds * sample_rate), dtype=np.float32), sample_rate)
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
@spaces.GPU(duration=120)
|
| 134 |
+
@torch.inference_mode()
|
| 135 |
+
def process_fn(mode_name, description, lyrics, ref_voice_path, extract_vocals, temperature):
|
| 136 |
+
"""Generate a song from a description and lyrics, optionally guided by a reference voice."""
|
| 137 |
+
if not loading_done:
|
| 138 |
+
raise gr.Error("Model is still loading, please wait a moment and try again.")
|
| 139 |
+
if loading_error:
|
| 140 |
+
raise gr.Error(f"Model failed to load: {loading_error}")
|
| 141 |
+
|
| 142 |
+
model, processor = move_mode_to_device(mode_name)
|
| 143 |
+
|
| 144 |
+
model_inputs = processor(
|
| 145 |
+
text=description,
|
| 146 |
+
lyrics=lyrics,
|
| 147 |
+
ref_voice_path=ref_voice_path if ref_voice_path else None,
|
| 148 |
+
separate=extract_vocals,
|
| 149 |
+
)
|
| 150 |
+
generation = model.generate(**model_inputs, do_sample=True, temperature=temperature)
|
| 151 |
+
sample_rate = model.config.sampling_rate
|
| 152 |
+
|
| 153 |
+
if mode_name == "Mixed":
|
| 154 |
+
song_path = write_wav(generation.cpu().numpy().squeeze(), sample_rate)
|
| 155 |
+
vocals_path = silent_wav(sample_rate)
|
| 156 |
+
else:
|
| 157 |
+
acc_array = generation[0].cpu().numpy().squeeze()
|
| 158 |
+
vocal_array = generation[1].cpu().numpy().squeeze()
|
| 159 |
+
min_len = min(vocal_array.shape[0], acc_array.shape[0])
|
| 160 |
+
acc_array = acc_array[:min_len]
|
| 161 |
+
vocal_array = vocal_array[:min_len]
|
| 162 |
+
|
| 163 |
+
song_path = write_wav(vocal_array + acc_array, sample_rate)
|
| 164 |
+
vocals_path = write_wav(vocal_array, sample_rate)
|
| 165 |
+
|
| 166 |
+
return song_path, vocals_path
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
with gr.Blocks() as demo:
|
| 170 |
+
input_components = [
|
| 171 |
+
gr.Dropdown(
|
| 172 |
+
choices=list(MODES.keys()),
|
| 173 |
+
value="Mixed",
|
| 174 |
+
label="Mode",
|
| 175 |
+
info="Mixed outputs one finished track; Dual-track also gives you an isolated vocal stem",
|
| 176 |
+
),
|
| 177 |
+
gr.Textbox(
|
| 178 |
+
label="Music Description",
|
| 179 |
+
info="Genre, mood, instruments (e.g. \"upbeat pop song with piano and drums\")",
|
| 180 |
+
),
|
| 181 |
+
gr.Textbox(
|
| 182 |
+
label="Lyrics",
|
| 183 |
+
info="English lyrics for the song (required)",
|
| 184 |
+
),
|
| 185 |
+
gr.Audio(type="filepath", label="Reference Voice (optional)").harp_required(False),
|
| 186 |
+
gr.Checkbox(
|
| 187 |
+
value=True,
|
| 188 |
+
label="Extract vocals from reference",
|
| 189 |
+
info="(default: True, per repo example) turn off only if the reference audio is already an isolated vocal, not a full mix",
|
| 190 |
+
),
|
| 191 |
+
gr.Slider(
|
| 192 |
+
minimum=0.1, maximum=2.0, step=0.1, value=1.0,
|
| 193 |
+
label="Creativity",
|
| 194 |
+
info="(default: 1.0, per repo config) sampling temperature — higher is more varied/unpredictable, lower is safer/more repetitive",
|
| 195 |
+
),
|
| 196 |
+
]
|
| 197 |
+
output_components = [
|
| 198 |
+
gr.Audio(type="filepath", label="Song").set_info(
|
| 199 |
+
"The generated song. In Dual-track mode this is vocals + accompaniment combined."
|
| 200 |
+
),
|
| 201 |
+
gr.Audio(type="filepath", label="Isolated Vocals").set_info(
|
| 202 |
+
"Vocal-only stem. Only populated in Dual-track mode; silent in Mixed mode."
|
| 203 |
+
),
|
| 204 |
+
]
|
| 205 |
+
|
| 206 |
+
build_endpoint(
|
| 207 |
+
model_card=model_card,
|
| 208 |
+
input_components=input_components,
|
| 209 |
+
output_components=output_components,
|
| 210 |
+
process_fn=process_fn,
|
| 211 |
+
)
|
| 212 |
+
|
| 213 |
+
if __name__ == "__main__":
|
| 214 |
+
demo.queue().launch(pwa=True)
|
model.json
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "SongGen",
|
| 3 |
+
"local_smoke_threshold_gb": 20.0,
|
| 4 |
+
"package_dir": "songgen",
|
| 5 |
+
"entry_point": [
|
| 6 |
+
"songgen.modeling_songgen_mixed.SongGenMixedForConditionalGeneration",
|
| 7 |
+
"songgen.modeling_songgen_dual_track.SongGenDualTrackForConditionalGeneration",
|
| 8 |
+
"songgen.processing_songgen.SongGenProcessor"
|
| 9 |
+
],
|
| 10 |
+
"checkpoint": {
|
| 11 |
+
"size_mb": 14114,
|
| 12 |
+
"note": "loaded via from_pretrained/hf_hub_download at runtime, not committed to Space repo. Both mode checkpoints held in memory simultaneously (dropdown selects mode at inference time): LiuZH-19/SongGen_mixed_pro (~5457MB) + LiuZH-19/SongGen_interleaving_A_V (~5422MB), each of which internally constructs its own full m-a-p/MERT-v1-330M model via AutoModel.from_pretrained(trust_remote_code=True) (~1262MB each, loaded twice total) for reference-voice conditioning. Plus xcodec ZhenYe234/xcodec xcodec_hubert_general_audio_v2.pth + config_hubert_general.yaml (~710MB, shared single copy) and htdemucs separation weights (~150MB) auto-downloaded by SongGenProcessor (also loaded once per mode).",
|
| 13 |
+
"repos": {
|
| 14 |
+
"mixed_pro": "LiuZH-19/SongGen_mixed_pro",
|
| 15 |
+
"dual_track": "LiuZH-19/SongGen_interleaving_A_V",
|
| 16 |
+
"xcodec": {"repo": "ZhenYe234/xcodec", "filename": "xcodec_hubert_general_audio_v2.pth", "config_filename": "config_hubert_general.yaml"},
|
| 17 |
+
"mert": "m-a-p/MERT-v1-330M"
|
| 18 |
+
}
|
| 19 |
+
}
|
| 20 |
+
}
|
requirements.txt
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
git+https://github.com/TEAMuP-dev/pyharp.git@develop
|
| 2 |
+
# model-specific deps below:
|
| 3 |
+
torch==2.8.0
|
| 4 |
+
torchaudio==2.8.0
|
| 5 |
+
transformers>=4.43.0,<=4.43.3
|
| 6 |
+
sentencepiece
|
| 7 |
+
descript-audio-codec
|
| 8 |
+
descript-audiotools>=0.7.2
|
| 9 |
+
omegaconf
|
| 10 |
+
einops
|
| 11 |
+
demucs
|
| 12 |
+
pypinyin
|
| 13 |
+
hangul_romanize
|
| 14 |
+
num2words
|
| 15 |
+
spacy
|
| 16 |
+
nnAudio
|
| 17 |
+
soundfile>=0.12.1
|
| 18 |
+
librosa
|
| 19 |
+
huggingface_hub
|
songgen/__init__.py
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
__version__ = "0.1"
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
from transformers import AutoConfig, AutoModel
|
| 5 |
+
|
| 6 |
+
from .configuration_songgen import SongGenConfig, SongGenDecoderConfig
|
| 7 |
+
from .xcodec_wrapper import XCodecConfig, XCodecModel
|
| 8 |
+
from .modeling_songgen_mixed import (
|
| 9 |
+
SongGenForCausalLM,
|
| 10 |
+
SongGenMixedForConditionalGeneration,
|
| 11 |
+
apply_delay_pattern_mask,
|
| 12 |
+
build_delay_pattern_mask,
|
| 13 |
+
)
|
| 14 |
+
|
| 15 |
+
from .modeling_songgen_dual_track import (
|
| 16 |
+
SongGenDualTrackForConditionalGeneration,
|
| 17 |
+
split_combined_track_input_ids,
|
| 18 |
+
build_combined_delay_pattern_mask,
|
| 19 |
+
combine_track_input_ids
|
| 20 |
+
)
|
| 21 |
+
|
| 22 |
+
from .processing_songgen import SongGenProcessor
|
| 23 |
+
|
| 24 |
+
# from .streamer import SongGenStreamer
|
| 25 |
+
|
| 26 |
+
from .lyrics_utils.lyrics_tokenizer import VoiceBpeTokenizer
|
| 27 |
+
|
| 28 |
+
AutoConfig.register("xcodec", XCodecConfig)
|
| 29 |
+
AutoModel.register(XCodecConfig, XCodecModel)
|
| 30 |
+
|
| 31 |
+
|
songgen/configuration_songgen.py
ADDED
|
@@ -0,0 +1,250 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# coding=utf-8
|
| 2 |
+
# Copyright 2024 and The HuggingFace Inc. team. All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 5 |
+
# you may not use this file except in compliance with the License.
|
| 6 |
+
# You may obtain a copy of the License at
|
| 7 |
+
#
|
| 8 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 9 |
+
#
|
| 10 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 11 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 12 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 13 |
+
# See the License for the specific language governing permissions and
|
| 14 |
+
# limitations under the License.
|
| 15 |
+
|
| 16 |
+
""" SongGen model configuration"""
|
| 17 |
+
|
| 18 |
+
from transformers import AutoConfig, logging
|
| 19 |
+
from transformers.configuration_utils import PretrainedConfig
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
logger = logging.get_logger(__name__)
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
class SongGenDecoderConfig(PretrainedConfig):
|
| 26 |
+
r"""
|
| 27 |
+
This is the configuration class to store the configuration of an [`SongGenDecoder`]. It is used to instantiate a
|
| 28 |
+
SongGen decoder according to the specified arguments, defining the model architecture.
|
| 29 |
+
|
| 30 |
+
Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
|
| 31 |
+
documentation from [`PretrainedConfig`] for more information.
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
Args:
|
| 35 |
+
vocab_size (`int`, *optional*, defaults to 2049):
|
| 36 |
+
Vocabulary size of the SongGenDecoder model. Defines the number of different tokens that can be
|
| 37 |
+
represented by the `inputs_ids` passed when calling [`SongGenDecoder`].
|
| 38 |
+
hidden_size (`int`, *optional*, defaults to 1024):
|
| 39 |
+
Dimensionality of the layers and the pooler layer.
|
| 40 |
+
num_hidden_layers (`int`, *optional*, defaults to 24):
|
| 41 |
+
Number of decoder layers.
|
| 42 |
+
num_attention_heads (`int`, *optional*, defaults to 16):
|
| 43 |
+
Number of attention heads for each attention layer in the Transformer block.
|
| 44 |
+
num_key_value_heads (`int`, *optional*):
|
| 45 |
+
This is the number of key_value heads that should be used to implement Grouped Query Attention. If
|
| 46 |
+
`num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
|
| 47 |
+
`num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
|
| 48 |
+
converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
|
| 49 |
+
by meanpooling all the original heads within that group. For more details checkout [this
|
| 50 |
+
paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to
|
| 51 |
+
`num_attention_heads`.
|
| 52 |
+
num_cross_attention_key_value_heads (`int`, *optional*):
|
| 53 |
+
This is the number of key_value heads that should be used to implement Grouped Query Attention in the cross-attention layers.
|
| 54 |
+
If it is not specified, will default to `num_key_value_heads`.
|
| 55 |
+
ffn_dim (`int`, *optional*, defaults to 4096):
|
| 56 |
+
Dimensionality of the "intermediate" (often named feed-forward) layer in the Transformer block.
|
| 57 |
+
activation_function (`str` or `function`, *optional*, defaults to `"gelu"`):
|
| 58 |
+
The non-linear activation function (function or string) in the decoder and pooler. If string, `"gelu"`,
|
| 59 |
+
`"relu"`, `"silu"` and `"gelu_new"` are supported.
|
| 60 |
+
dropout (`float`, *optional*, defaults to 0.1):
|
| 61 |
+
The dropout probability for all fully connected layers in the embeddings, text_encoder, and pooler.
|
| 62 |
+
attention_dropout (`float`, *optional*, defaults to 0.0):
|
| 63 |
+
The dropout ratio for the attention probabilities.
|
| 64 |
+
activation_dropout (`float`, *optional*, defaults to 0.0):
|
| 65 |
+
The dropout ratio for activations inside the fully connected layer.
|
| 66 |
+
max_position_embeddings (`int`, *optional*, defaults to 2048):
|
| 67 |
+
The maximum sequence length that this model might ever be used with. Typically, set this to something large
|
| 68 |
+
just in case (e.g., 512 or 1024 or 2048).
|
| 69 |
+
initializer_factor (`float`, *optional*, defaults to 0.02):
|
| 70 |
+
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
|
| 71 |
+
layerdrop (`float`, *optional*, defaults to 0.0):
|
| 72 |
+
The LayerDrop probability for the decoder. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556)
|
| 73 |
+
for more details.
|
| 74 |
+
scale_embedding (`bool`, *optional*, defaults to `False`):
|
| 75 |
+
Scale embeddings by diving by sqrt(hidden_size).
|
| 76 |
+
use_cache (`bool`, *optional*, defaults to `True`):
|
| 77 |
+
Whether the model should return the last key/values attentions (not used by all models)
|
| 78 |
+
num_codebooks (`int`, *optional*, defaults to 4):
|
| 79 |
+
The number of parallel codebooks forwarded to the model.
|
| 80 |
+
tie_word_embeddings(`bool`, *optional*, defaults to `False`):
|
| 81 |
+
Whether input and output word embeddings should be tied.
|
| 82 |
+
rope_embeddings (`bool`, *optional*, defaults to `False`):
|
| 83 |
+
Whether to use ROPE or absolute positional embeddings.
|
| 84 |
+
rope_theta (`float`, *optional*, defaults to 100000.0):
|
| 85 |
+
The base period of the RoPE embeddings.
|
| 86 |
+
cross_attention_implementation_strategy (`str`, *optional*):
|
| 87 |
+
If not specified, the cross-attention implementation will be the same as `_attn_implementation`. If `always_eager`, it will always be the eager implementation. If `always_sdpa`, it will always be the sdpa implementation.
|
| 88 |
+
"""
|
| 89 |
+
|
| 90 |
+
model_type = "songgen_decoder"
|
| 91 |
+
keys_to_ignore_at_inference = ["past_key_values"]
|
| 92 |
+
|
| 93 |
+
def __init__(
|
| 94 |
+
self,
|
| 95 |
+
vocab_size=1088, # vocab size = 1024 (codec vocab size) + 64
|
| 96 |
+
max_position_embeddings=6000,
|
| 97 |
+
num_hidden_layers=24,
|
| 98 |
+
ffn_dim=4096,
|
| 99 |
+
num_attention_heads=16,
|
| 100 |
+
num_key_value_heads=None,
|
| 101 |
+
num_cross_attention_key_value_heads=None,
|
| 102 |
+
layerdrop=0.0,
|
| 103 |
+
use_cache=True,
|
| 104 |
+
activation_function="gelu",
|
| 105 |
+
hidden_size=1024,
|
| 106 |
+
dropout=0.1,
|
| 107 |
+
attention_dropout=0.0,
|
| 108 |
+
activation_dropout=0.0,
|
| 109 |
+
initializer_factor=0.02,
|
| 110 |
+
scale_embedding=False,
|
| 111 |
+
num_codebooks=4,
|
| 112 |
+
pad_token_id=1024,
|
| 113 |
+
bos_token_id=1025,
|
| 114 |
+
eos_token_id=1024,
|
| 115 |
+
tie_word_embeddings=False,
|
| 116 |
+
rope_embeddings=False,
|
| 117 |
+
rope_theta=10_000.0,
|
| 118 |
+
cross_attention_implementation_strategy=None,
|
| 119 |
+
**kwargs,
|
| 120 |
+
):
|
| 121 |
+
self.vocab_size = vocab_size
|
| 122 |
+
self.max_position_embeddings = max_position_embeddings
|
| 123 |
+
self.hidden_size = hidden_size
|
| 124 |
+
self.ffn_dim = ffn_dim
|
| 125 |
+
self.num_hidden_layers = num_hidden_layers
|
| 126 |
+
self.num_attention_heads = num_attention_heads
|
| 127 |
+
if num_key_value_heads is None:
|
| 128 |
+
num_key_value_heads = num_attention_heads
|
| 129 |
+
self.num_key_value_heads = num_key_value_heads
|
| 130 |
+
if num_cross_attention_key_value_heads is None:
|
| 131 |
+
num_cross_attention_key_value_heads = num_key_value_heads
|
| 132 |
+
self.num_cross_attention_key_value_heads = num_cross_attention_key_value_heads
|
| 133 |
+
self.dropout = dropout
|
| 134 |
+
self.attention_dropout = attention_dropout
|
| 135 |
+
self.activation_dropout = activation_dropout
|
| 136 |
+
self.activation_function = activation_function
|
| 137 |
+
self.initializer_factor = initializer_factor
|
| 138 |
+
self.layerdrop = layerdrop
|
| 139 |
+
self.use_cache = use_cache
|
| 140 |
+
self.scale_embedding = scale_embedding # scale factor will be sqrt(d_model) if True
|
| 141 |
+
self.num_codebooks = num_codebooks
|
| 142 |
+
self.rope_embeddings = rope_embeddings
|
| 143 |
+
self.rope_theta = rope_theta
|
| 144 |
+
self.cross_attention_implementation_strategy = cross_attention_implementation_strategy
|
| 145 |
+
|
| 146 |
+
super().__init__(
|
| 147 |
+
pad_token_id=pad_token_id,
|
| 148 |
+
bos_token_id=bos_token_id,
|
| 149 |
+
eos_token_id=eos_token_id,
|
| 150 |
+
tie_word_embeddings=tie_word_embeddings,
|
| 151 |
+
**kwargs,
|
| 152 |
+
)
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
class SongGenConfig(PretrainedConfig):
|
| 156 |
+
r"""
|
| 157 |
+
This is the configuration class to store the configuration of a [`SongGenModel`]. It is used to instantiate a
|
| 158 |
+
SongGen model according to the specified arguments, defining the text encoder, audio encoder and SongGen decoder
|
| 159 |
+
configs.
|
| 160 |
+
|
| 161 |
+
Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
|
| 162 |
+
documentation from [`PretrainedConfig`] for more information.
|
| 163 |
+
|
| 164 |
+
Args:
|
| 165 |
+
vocab_size (`int`, *optional*, defaults to 1024):
|
| 166 |
+
Vocabulary size of the prompt token ids. Defines the number of different tokens that can be
|
| 167 |
+
represented by the `prompt_inputs_ids`.
|
| 168 |
+
prompt_cross_attention (`bool`, *optional*, defaults to `False`):
|
| 169 |
+
Whether to use cross-attention conditioning for the prompt (as well as the description).
|
| 170 |
+
kwargs (*optional*):
|
| 171 |
+
Dictionary of keyword arguments. Notably:
|
| 172 |
+
|
| 173 |
+
- **text_encoder** ([`PretrainedConfig`], *optional*) -- An instance of a configuration object that
|
| 174 |
+
defines the text encoder config.
|
| 175 |
+
- **audio_encoder** ([`PretrainedConfig`], *optional*) -- An instance of a configuration object that
|
| 176 |
+
defines the audio encoder config.
|
| 177 |
+
- **decoder** ([`PretrainedConfig`], *optional*) -- An instance of a configuration object that defines
|
| 178 |
+
the decoder config.
|
| 179 |
+
"""
|
| 180 |
+
|
| 181 |
+
model_type = "songgen"
|
| 182 |
+
is_composition = True
|
| 183 |
+
|
| 184 |
+
def __init__(self, vocab_size=1024, prompt_cross_attention=True, add_prenet=True, **kwargs):
|
| 185 |
+
super().__init__(**kwargs)
|
| 186 |
+
if "text_encoder" not in kwargs or "decoder" not in kwargs: #or "audio_encoder" not in kwargs
|
| 187 |
+
raise ValueError("Config has to be initialized with text_encoder and decoder config") #, audio_encoder
|
| 188 |
+
|
| 189 |
+
text_encoder_config = kwargs.pop("text_encoder")
|
| 190 |
+
text_encoder_model_type = text_encoder_config.pop("model_type")
|
| 191 |
+
|
| 192 |
+
# audio_encoder_config = kwargs.pop("audio_encoder")
|
| 193 |
+
# audio_encoder_model_type = audio_encoder_config.pop("model_type")
|
| 194 |
+
|
| 195 |
+
decoder_config = kwargs.pop("decoder")
|
| 196 |
+
|
| 197 |
+
self.vocab_size = vocab_size
|
| 198 |
+
self.prompt_cross_attention = prompt_cross_attention
|
| 199 |
+
self.add_prenet = add_prenet
|
| 200 |
+
self.text_encoder = AutoConfig.for_model(text_encoder_model_type, **text_encoder_config)
|
| 201 |
+
# self.audio_encoder = AutoConfig.for_model(audio_encoder_model_type, **audio_encoder_config)
|
| 202 |
+
self.decoder = SongGenDecoderConfig(**decoder_config)
|
| 203 |
+
self.is_encoder_decoder = True
|
| 204 |
+
|
| 205 |
+
|
| 206 |
+
@classmethod
|
| 207 |
+
def from_sub_models_config(
|
| 208 |
+
cls,
|
| 209 |
+
text_encoder_config: PretrainedConfig,
|
| 210 |
+
# audio_encoder_config: PretrainedConfig,
|
| 211 |
+
decoder_config: SongGenDecoderConfig,
|
| 212 |
+
**kwargs,
|
| 213 |
+
):
|
| 214 |
+
r"""
|
| 215 |
+
Instantiate a [`SongGenConfig`] (or a derived class) from text encoder, audio encoder and decoder
|
| 216 |
+
configurations.
|
| 217 |
+
|
| 218 |
+
Returns:
|
| 219 |
+
[`SongGenConfig`]: An instance of a configuration object
|
| 220 |
+
"""
|
| 221 |
+
|
| 222 |
+
return cls(
|
| 223 |
+
text_encoder=text_encoder_config.to_dict(),
|
| 224 |
+
# audio_encoder=audio_encoder_config.to_dict(),
|
| 225 |
+
decoder=decoder_config.to_dict(),
|
| 226 |
+
**kwargs,
|
| 227 |
+
)
|
| 228 |
+
|
| 229 |
+
@property
|
| 230 |
+
# This is a property because you might want to change the codec model on the fly
|
| 231 |
+
def sampling_rate(self):
|
| 232 |
+
return 16000 #self.audio_encoder.sampling_rate xcodec sampling_rate 16khz
|
| 233 |
+
|
| 234 |
+
# Copy from musicgen
|
| 235 |
+
@property
|
| 236 |
+
def _attn_implementation(self):
|
| 237 |
+
# This property is made private for now (as it cannot be changed and a PreTrainedModel.use_attn_implementation method needs to be implemented.)
|
| 238 |
+
if hasattr(self, "_attn_implementation_internal"):
|
| 239 |
+
if self._attn_implementation_internal is None:
|
| 240 |
+
# `config.attn_implementation` should never be None, for backward compatibility.
|
| 241 |
+
return "eager"
|
| 242 |
+
else:
|
| 243 |
+
return self._attn_implementation_internal
|
| 244 |
+
else:
|
| 245 |
+
return "eager"
|
| 246 |
+
|
| 247 |
+
@_attn_implementation.setter
|
| 248 |
+
def _attn_implementation(self, value):
|
| 249 |
+
self._attn_implementation_internal = value
|
| 250 |
+
self.decoder._attn_implementation = value
|
songgen/custom_modeling_outputs.py
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import warnings
|
| 2 |
+
from dataclasses import dataclass
|
| 3 |
+
from typing import Optional, Tuple, List
|
| 4 |
+
import torch
|
| 5 |
+
from transformers.utils import ModelOutput
|
| 6 |
+
|
| 7 |
+
@dataclass
|
| 8 |
+
class Seq2SeqLMOutput(ModelOutput):
|
| 9 |
+
"""
|
| 10 |
+
Base class for sequence-to-sequence language models outputs.
|
| 11 |
+
|
| 12 |
+
Args:
|
| 13 |
+
loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
|
| 14 |
+
Language modeling loss.
|
| 15 |
+
logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
|
| 16 |
+
Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
|
| 17 |
+
past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
|
| 18 |
+
Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
|
| 19 |
+
`(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape
|
| 20 |
+
`(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
|
| 21 |
+
|
| 22 |
+
Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
|
| 23 |
+
blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
|
| 24 |
+
decoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
|
| 25 |
+
Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
|
| 26 |
+
one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.
|
| 27 |
+
|
| 28 |
+
Hidden-states of the decoder at the output of each layer plus the initial embedding outputs.
|
| 29 |
+
decoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
|
| 30 |
+
Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
|
| 31 |
+
sequence_length)`.
|
| 32 |
+
|
| 33 |
+
Attentions weights of the decoder, after the attention softmax, used to compute the weighted average in the
|
| 34 |
+
self-attention heads.
|
| 35 |
+
cross_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
|
| 36 |
+
Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
|
| 37 |
+
sequence_length)`.
|
| 38 |
+
|
| 39 |
+
Attentions weights of the decoder's cross-attention layer, after the attention softmax, used to compute the
|
| 40 |
+
weighted average in the cross-attention heads.
|
| 41 |
+
encoder_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
|
| 42 |
+
Sequence of hidden-states at the output of the last layer of the encoder of the model.
|
| 43 |
+
encoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
|
| 44 |
+
Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
|
| 45 |
+
one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.
|
| 46 |
+
|
| 47 |
+
Hidden-states of the encoder at the output of each layer plus the initial embedding outputs.
|
| 48 |
+
encoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
|
| 49 |
+
Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
|
| 50 |
+
sequence_length)`.
|
| 51 |
+
|
| 52 |
+
Attentions weights of the encoder, after the attention softmax, used to compute the weighted average in the
|
| 53 |
+
self-attention heads.
|
| 54 |
+
"""
|
| 55 |
+
|
| 56 |
+
loss: Optional[torch.FloatTensor] = None
|
| 57 |
+
vocal_loss: Optional[torch.FloatTensor] = None
|
| 58 |
+
logits: torch.FloatTensor = None
|
| 59 |
+
past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None
|
| 60 |
+
decoder_hidden_states: Optional[Tuple[torch.FloatTensor, ...]] = None
|
| 61 |
+
decoder_attentions: Optional[Tuple[torch.FloatTensor, ...]] = None
|
| 62 |
+
cross_attentions: Optional[Tuple[torch.FloatTensor, ...]] = None
|
| 63 |
+
encoder_last_hidden_state: Optional[torch.FloatTensor] = None
|
| 64 |
+
encoder_hidden_states: Optional[Tuple[torch.FloatTensor, ...]] = None
|
| 65 |
+
encoder_attentions: Optional[Tuple[torch.FloatTensor, ...]] = None
|
| 66 |
+
codebook_losses: Optional[List[torch.FloatTensor]]=None
|
| 67 |
+
vocal_codebook_losses: Optional[List[torch.FloatTensor]]=None
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
@dataclass
|
| 71 |
+
class CausalLMOutputWithCrossAttentions(ModelOutput):
|
| 72 |
+
"""
|
| 73 |
+
Base class for causal language model (or autoregressive) outputs.
|
| 74 |
+
|
| 75 |
+
Args:
|
| 76 |
+
loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
|
| 77 |
+
Language modeling loss (for next-token prediction).
|
| 78 |
+
logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
|
| 79 |
+
Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
|
| 80 |
+
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
|
| 81 |
+
Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
|
| 82 |
+
one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.
|
| 83 |
+
|
| 84 |
+
Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
|
| 85 |
+
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
|
| 86 |
+
Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
|
| 87 |
+
sequence_length)`.
|
| 88 |
+
|
| 89 |
+
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
|
| 90 |
+
heads.
|
| 91 |
+
cross_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
|
| 92 |
+
Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
|
| 93 |
+
sequence_length)`.
|
| 94 |
+
|
| 95 |
+
Cross attentions weights after the attention softmax, used to compute the weighted average in the
|
| 96 |
+
cross-attention heads.
|
| 97 |
+
past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
|
| 98 |
+
Tuple of `torch.FloatTensor` tuples of length `config.n_layers`, with each tuple containing the cached key,
|
| 99 |
+
value states of the self-attention and the cross-attention layers if model is used in encoder-decoder
|
| 100 |
+
setting. Only relevant if `config.is_decoder = True`.
|
| 101 |
+
|
| 102 |
+
Contains pre-computed hidden-states (key and values in the attention blocks) that can be used (see
|
| 103 |
+
`past_key_values` input) to speed up sequential decoding.
|
| 104 |
+
"""
|
| 105 |
+
|
| 106 |
+
loss: Optional[torch.FloatTensor] = None
|
| 107 |
+
vocal_loss: Optional[torch.FloatTensor] = None
|
| 108 |
+
logits: torch.FloatTensor = None
|
| 109 |
+
past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None
|
| 110 |
+
hidden_states: Optional[Tuple[torch.FloatTensor, ...]] = None
|
| 111 |
+
attentions: Optional[Tuple[torch.FloatTensor, ...]] = None
|
| 112 |
+
cross_attentions: Optional[Tuple[torch.FloatTensor, ...]] = None
|
| 113 |
+
codebook_losses: Optional[List[torch.FloatTensor]]=None
|
| 114 |
+
vocal_codebook_losses: Optional[List[torch.FloatTensor]]=None
|
songgen/logits_processors.py
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from transformers import LogitsProcessor, LogitsProcessorList
|
| 2 |
+
import math
|
| 3 |
+
import torch
|
| 4 |
+
from typing import Union
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
def isin_mps_friendly(elements: torch.Tensor, test_elements: Union[torch.Tensor, int]) -> torch.Tensor:
|
| 8 |
+
"""
|
| 9 |
+
Same as `torch.isin` without flags, but MPS-friendly. We can remove this function when we stop supporting
|
| 10 |
+
torch <= 2.3. See https://github.com/pytorch/pytorch/issues/77764#issuecomment-2067838075
|
| 11 |
+
|
| 12 |
+
Args:
|
| 13 |
+
elements (`torch.Tensor`): Input elements
|
| 14 |
+
test_elements (`torch.Tensor` or `int`): The elements to check against.
|
| 15 |
+
|
| 16 |
+
Returns:
|
| 17 |
+
`torch.Tensor`: A boolean tensor of the same shape as `elements` that is True for `elements` in `test_elements`
|
| 18 |
+
and False otherwise
|
| 19 |
+
"""
|
| 20 |
+
|
| 21 |
+
if elements.device.type == "mps" and not is_torch_greater_or_equal_than_2_4:
|
| 22 |
+
test_elements = torch.tensor(test_elements)
|
| 23 |
+
if test_elements.ndim == 0:
|
| 24 |
+
test_elements = test_elements.unsqueeze(0)
|
| 25 |
+
return elements.tile(test_elements.shape[0], 1).eq(test_elements.unsqueeze(1)).sum(dim=0).bool().squeeze()
|
| 26 |
+
else:
|
| 27 |
+
# Note: don't use named arguments in `torch.isin`, see https://github.com/pytorch/pytorch/issues/126045
|
| 28 |
+
return torch.isin(elements, test_elements)
|
| 29 |
+
|
| 30 |
+
class SongGenLogitsProcessor(LogitsProcessor):
|
| 31 |
+
r"""This processor ensures that the delayed pattern mask constraints are respected.
|
| 32 |
+
|
| 33 |
+
<Tip warning={true}>
|
| 34 |
+
|
| 35 |
+
This logits processor is exclusively compatible with SongGen.
|
| 36 |
+
See the model documentation for examples.
|
| 37 |
+
|
| 38 |
+
</Tip>
|
| 39 |
+
|
| 40 |
+
Args:
|
| 41 |
+
eos_token_id (`Union[int, List[int], torch.Tensor]`):
|
| 42 |
+
The id(s) of the *end-of-sequence* token.
|
| 43 |
+
min_eos_p (`float`, *optional*):
|
| 44 |
+
Minimum end of speech threshold.
|
| 45 |
+
"""
|
| 46 |
+
|
| 47 |
+
def __init__(self, eos_token_id, num_codebooks: int, batch_size: int, device: str = "cpu", track_pattern='mix'):
|
| 48 |
+
if not isinstance(eos_token_id, torch.Tensor):
|
| 49 |
+
if isinstance(eos_token_id, int):
|
| 50 |
+
eos_token_id = [eos_token_id]
|
| 51 |
+
eos_token_id = torch.tensor(eos_token_id, device=device)
|
| 52 |
+
self.eos_token_id = eos_token_id
|
| 53 |
+
self.batch_size = batch_size
|
| 54 |
+
self.track_pattern = track_pattern
|
| 55 |
+
|
| 56 |
+
if torch.is_floating_point(eos_token_id) or (eos_token_id < 0).any():
|
| 57 |
+
raise ValueError(f"`eos_token_id` has to be a list of positive integers, but is {eos_token_id}")
|
| 58 |
+
|
| 59 |
+
self.num_codebooks = num_codebooks
|
| 60 |
+
self.device = device
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
self.codebook_idx = torch.arange(self.batch_size*self.num_codebooks, device=self.device)
|
| 64 |
+
|
| 65 |
+
if self.track_pattern.startswith('parallel'):
|
| 66 |
+
self.first_codebooks_unfinished = torch.arange(batch_size*2, device=device)*(num_codebooks//2)
|
| 67 |
+
max_codebooks = torch.arange(self.batch_size*2, device=self.device)*(self.num_codebooks//2) + (self.num_codebooks//2) -1
|
| 68 |
+
self.track_num_codebooks = self.num_codebooks//2
|
| 69 |
+
else:
|
| 70 |
+
self.first_codebooks_unfinished = torch.arange(batch_size, device=device)*num_codebooks
|
| 71 |
+
max_codebooks = torch.arange(self.batch_size, device=self.device)*self.num_codebooks + self.num_codebooks -1
|
| 72 |
+
self.track_num_codebooks = self.num_codebooks
|
| 73 |
+
|
| 74 |
+
self.max_codebooks = max_codebooks
|
| 75 |
+
|
| 76 |
+
def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
|
| 77 |
+
is_eos = isin_mps_friendly(input_ids, self.eos_token_id).sum(1)
|
| 78 |
+
|
| 79 |
+
self.first_codebooks_unfinished = torch.where((is_eos[self.first_codebooks_unfinished]>0) & (self.first_codebooks_unfinished<self.max_codebooks), self.first_codebooks_unfinished+1, self.first_codebooks_unfinished)
|
| 80 |
+
|
| 81 |
+
# every codebook higher than the first one unfinished will never be eos
|
| 82 |
+
eos_token_mask = self.codebook_idx > self.first_codebooks_unfinished.repeat_interleave(self.track_num_codebooks)
|
| 83 |
+
scores[eos_token_mask, self.eos_token_id] = -math.inf
|
| 84 |
+
|
| 85 |
+
return scores
|
songgen/lyrics_utils/__init__.py
ADDED
|
File without changes
|
songgen/lyrics_utils/lyrics_encoder.py
ADDED
|
@@ -0,0 +1,1071 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Optional, Tuple, Union
|
| 2 |
+
import math
|
| 3 |
+
import torch
|
| 4 |
+
from torch import nn
|
| 5 |
+
|
| 6 |
+
class ConvolutionModule(nn.Module):
|
| 7 |
+
"""ConvolutionModule in Conformer model."""
|
| 8 |
+
|
| 9 |
+
def __init__(self,
|
| 10 |
+
channels: int,
|
| 11 |
+
kernel_size: int = 15,
|
| 12 |
+
activation: nn.Module = nn.ReLU(),
|
| 13 |
+
norm: str = "batch_norm",
|
| 14 |
+
causal: bool = False,
|
| 15 |
+
bias: bool = True):
|
| 16 |
+
"""Construct an ConvolutionModule object.
|
| 17 |
+
Args:
|
| 18 |
+
channels (int): The number of channels of conv layers.
|
| 19 |
+
kernel_size (int): Kernel size of conv layers.
|
| 20 |
+
causal (int): Whether use causal convolution or not
|
| 21 |
+
"""
|
| 22 |
+
super().__init__()
|
| 23 |
+
|
| 24 |
+
self.pointwise_conv1 = nn.Conv1d(
|
| 25 |
+
channels,
|
| 26 |
+
2 * channels,
|
| 27 |
+
kernel_size=1,
|
| 28 |
+
stride=1,
|
| 29 |
+
padding=0,
|
| 30 |
+
bias=bias,
|
| 31 |
+
)
|
| 32 |
+
# self.lorder is used to distinguish if it's a causal convolution,
|
| 33 |
+
# if self.lorder > 0: it's a causal convolution, the input will be
|
| 34 |
+
# padded with self.lorder frames on the left in forward.
|
| 35 |
+
# else: it's a symmetrical convolution
|
| 36 |
+
if causal:
|
| 37 |
+
padding = 0
|
| 38 |
+
self.lorder = kernel_size - 1
|
| 39 |
+
else:
|
| 40 |
+
# kernel_size should be an odd number for none causal convolution
|
| 41 |
+
assert (kernel_size - 1) % 2 == 0
|
| 42 |
+
padding = (kernel_size - 1) // 2
|
| 43 |
+
self.lorder = 0
|
| 44 |
+
self.depthwise_conv = nn.Conv1d(
|
| 45 |
+
channels,
|
| 46 |
+
channels,
|
| 47 |
+
kernel_size,
|
| 48 |
+
stride=1,
|
| 49 |
+
padding=padding,
|
| 50 |
+
groups=channels,
|
| 51 |
+
bias=bias,
|
| 52 |
+
)
|
| 53 |
+
|
| 54 |
+
assert norm in ['batch_norm', 'layer_norm']
|
| 55 |
+
if norm == "batch_norm":
|
| 56 |
+
self.use_layer_norm = False
|
| 57 |
+
self.norm = nn.BatchNorm1d(channels)
|
| 58 |
+
else:
|
| 59 |
+
self.use_layer_norm = True
|
| 60 |
+
self.norm = nn.LayerNorm(channels)
|
| 61 |
+
|
| 62 |
+
self.pointwise_conv2 = nn.Conv1d(
|
| 63 |
+
channels,
|
| 64 |
+
channels,
|
| 65 |
+
kernel_size=1,
|
| 66 |
+
stride=1,
|
| 67 |
+
padding=0,
|
| 68 |
+
bias=bias,
|
| 69 |
+
)
|
| 70 |
+
self.activation = activation
|
| 71 |
+
|
| 72 |
+
def forward(
|
| 73 |
+
self,
|
| 74 |
+
x: torch.Tensor,
|
| 75 |
+
mask_pad: torch.Tensor = torch.ones((0, 0, 0), dtype=torch.bool),
|
| 76 |
+
cache: torch.Tensor = torch.zeros((0, 0, 0)),
|
| 77 |
+
) -> Tuple[torch.Tensor, torch.Tensor]:
|
| 78 |
+
"""Compute convolution module.
|
| 79 |
+
Args:
|
| 80 |
+
x (torch.Tensor): Input tensor (#batch, time, channels).
|
| 81 |
+
mask_pad (torch.Tensor): used for batch padding (#batch, 1, time),
|
| 82 |
+
(0, 0, 0) means fake mask.
|
| 83 |
+
cache (torch.Tensor): left context cache, it is only
|
| 84 |
+
used in causal convolution (#batch, channels, cache_t),
|
| 85 |
+
(0, 0, 0) meas fake cache.
|
| 86 |
+
Returns:
|
| 87 |
+
torch.Tensor: Output tensor (#batch, time, channels).
|
| 88 |
+
"""
|
| 89 |
+
# exchange the temporal dimension and the feature dimension
|
| 90 |
+
x = x.transpose(1, 2) # (#batch, channels, time)
|
| 91 |
+
|
| 92 |
+
# mask batch padding
|
| 93 |
+
if mask_pad.size(2) > 0: # time > 0
|
| 94 |
+
x.masked_fill_(~mask_pad, 0.0)
|
| 95 |
+
|
| 96 |
+
if self.lorder > 0:
|
| 97 |
+
if cache.size(2) == 0: # cache_t == 0
|
| 98 |
+
x = nn.functional.pad(x, (self.lorder, 0), 'constant', 0.0)
|
| 99 |
+
else:
|
| 100 |
+
assert cache.size(0) == x.size(0) # equal batch
|
| 101 |
+
assert cache.size(1) == x.size(1) # equal channel
|
| 102 |
+
x = torch.cat((cache, x), dim=2)
|
| 103 |
+
assert (x.size(2) > self.lorder)
|
| 104 |
+
new_cache = x[:, :, -self.lorder:]
|
| 105 |
+
else:
|
| 106 |
+
# It's better we just return None if no cache is required,
|
| 107 |
+
# However, for JIT export, here we just fake one tensor instead of
|
| 108 |
+
# None.
|
| 109 |
+
new_cache = torch.zeros((0, 0, 0), dtype=x.dtype, device=x.device)
|
| 110 |
+
|
| 111 |
+
# GLU mechanism
|
| 112 |
+
x = self.pointwise_conv1(x) # (batch, 2*channel, dim)
|
| 113 |
+
x = nn.functional.glu(x, dim=1) # (batch, channel, dim)
|
| 114 |
+
|
| 115 |
+
# 1D Depthwise Conv
|
| 116 |
+
x = self.depthwise_conv(x)
|
| 117 |
+
if self.use_layer_norm:
|
| 118 |
+
x = x.transpose(1, 2)
|
| 119 |
+
x = self.activation(self.norm(x))
|
| 120 |
+
if self.use_layer_norm:
|
| 121 |
+
x = x.transpose(1, 2)
|
| 122 |
+
x = self.pointwise_conv2(x)
|
| 123 |
+
# mask batch padding
|
| 124 |
+
if mask_pad.size(2) > 0: # time > 0
|
| 125 |
+
x.masked_fill_(~mask_pad, 0.0)
|
| 126 |
+
|
| 127 |
+
return x.transpose(1, 2), new_cache
|
| 128 |
+
|
| 129 |
+
class PositionwiseFeedForward(torch.nn.Module):
|
| 130 |
+
"""Positionwise feed forward layer.
|
| 131 |
+
|
| 132 |
+
FeedForward are appied on each position of the sequence.
|
| 133 |
+
The output dim is same with the input dim.
|
| 134 |
+
|
| 135 |
+
Args:
|
| 136 |
+
idim (int): Input dimenstion.
|
| 137 |
+
hidden_units (int): The number of hidden units.
|
| 138 |
+
dropout_rate (float): Dropout rate.
|
| 139 |
+
activation (torch.nn.Module): Activation function
|
| 140 |
+
"""
|
| 141 |
+
|
| 142 |
+
def __init__(
|
| 143 |
+
self,
|
| 144 |
+
idim: int,
|
| 145 |
+
hidden_units: int,
|
| 146 |
+
dropout_rate: float,
|
| 147 |
+
activation: torch.nn.Module = torch.nn.ReLU(),
|
| 148 |
+
):
|
| 149 |
+
"""Construct a PositionwiseFeedForward object."""
|
| 150 |
+
super(PositionwiseFeedForward, self).__init__()
|
| 151 |
+
self.w_1 = torch.nn.Linear(idim, hidden_units)
|
| 152 |
+
self.activation = activation
|
| 153 |
+
self.dropout = torch.nn.Dropout(dropout_rate)
|
| 154 |
+
self.w_2 = torch.nn.Linear(hidden_units, idim)
|
| 155 |
+
|
| 156 |
+
def forward(self, xs: torch.Tensor) -> torch.Tensor:
|
| 157 |
+
"""Forward function.
|
| 158 |
+
|
| 159 |
+
Args:
|
| 160 |
+
xs: input tensor (B, L, D)
|
| 161 |
+
Returns:
|
| 162 |
+
output tensor, (B, L, D)
|
| 163 |
+
"""
|
| 164 |
+
return self.w_2(self.dropout(self.activation(self.w_1(xs))))
|
| 165 |
+
|
| 166 |
+
class Swish(torch.nn.Module):
|
| 167 |
+
"""Construct an Swish object."""
|
| 168 |
+
|
| 169 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 170 |
+
"""Return Swish activation function."""
|
| 171 |
+
return x * torch.sigmoid(x)
|
| 172 |
+
|
| 173 |
+
class MultiHeadedAttention(nn.Module):
|
| 174 |
+
"""Multi-Head Attention layer.
|
| 175 |
+
|
| 176 |
+
Args:
|
| 177 |
+
n_head (int): The number of heads.
|
| 178 |
+
n_feat (int): The number of features.
|
| 179 |
+
dropout_rate (float): Dropout rate.
|
| 180 |
+
|
| 181 |
+
"""
|
| 182 |
+
|
| 183 |
+
def __init__(self,
|
| 184 |
+
n_head: int,
|
| 185 |
+
n_feat: int,
|
| 186 |
+
dropout_rate: float,
|
| 187 |
+
key_bias: bool = True):
|
| 188 |
+
"""Construct an MultiHeadedAttention object."""
|
| 189 |
+
super().__init__()
|
| 190 |
+
assert n_feat % n_head == 0
|
| 191 |
+
# We assume d_v always equals d_k
|
| 192 |
+
self.d_k = n_feat // n_head
|
| 193 |
+
self.h = n_head
|
| 194 |
+
self.linear_q = nn.Linear(n_feat, n_feat)
|
| 195 |
+
self.linear_k = nn.Linear(n_feat, n_feat, bias=key_bias)
|
| 196 |
+
self.linear_v = nn.Linear(n_feat, n_feat)
|
| 197 |
+
self.linear_out = nn.Linear(n_feat, n_feat)
|
| 198 |
+
self.dropout = nn.Dropout(p=dropout_rate)
|
| 199 |
+
|
| 200 |
+
def forward_qkv(
|
| 201 |
+
self, query: torch.Tensor, key: torch.Tensor, value: torch.Tensor
|
| 202 |
+
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
| 203 |
+
"""Transform query, key and value.
|
| 204 |
+
|
| 205 |
+
Args:
|
| 206 |
+
query (torch.Tensor): Query tensor (#batch, time1, size).
|
| 207 |
+
key (torch.Tensor): Key tensor (#batch, time2, size).
|
| 208 |
+
value (torch.Tensor): Value tensor (#batch, time2, size).
|
| 209 |
+
|
| 210 |
+
Returns:
|
| 211 |
+
torch.Tensor: Transformed query tensor, size
|
| 212 |
+
(#batch, n_head, time1, d_k).
|
| 213 |
+
torch.Tensor: Transformed key tensor, size
|
| 214 |
+
(#batch, n_head, time2, d_k).
|
| 215 |
+
torch.Tensor: Transformed value tensor, size
|
| 216 |
+
(#batch, n_head, time2, d_k).
|
| 217 |
+
|
| 218 |
+
"""
|
| 219 |
+
n_batch = query.size(0)
|
| 220 |
+
q = self.linear_q(query).view(n_batch, -1, self.h, self.d_k)
|
| 221 |
+
k = self.linear_k(key).view(n_batch, -1, self.h, self.d_k)
|
| 222 |
+
v = self.linear_v(value).view(n_batch, -1, self.h, self.d_k)
|
| 223 |
+
q = q.transpose(1, 2) # (batch, head, time1, d_k)
|
| 224 |
+
k = k.transpose(1, 2) # (batch, head, time2, d_k)
|
| 225 |
+
v = v.transpose(1, 2) # (batch, head, time2, d_k)
|
| 226 |
+
return q, k, v
|
| 227 |
+
|
| 228 |
+
def forward_attention(
|
| 229 |
+
self,
|
| 230 |
+
value: torch.Tensor,
|
| 231 |
+
scores: torch.Tensor,
|
| 232 |
+
mask: torch.Tensor = torch.ones((0, 0, 0), dtype=torch.bool)
|
| 233 |
+
) -> torch.Tensor:
|
| 234 |
+
"""Compute attention context vector.
|
| 235 |
+
|
| 236 |
+
Args:
|
| 237 |
+
value (torch.Tensor): Transformed value, size
|
| 238 |
+
(#batch, n_head, time2, d_k).
|
| 239 |
+
scores (torch.Tensor): Attention score, size
|
| 240 |
+
(#batch, n_head, time1, time2).
|
| 241 |
+
mask (torch.Tensor): Mask, size (#batch, 1, time2) or
|
| 242 |
+
(#batch, time1, time2), (0, 0, 0) means fake mask.
|
| 243 |
+
|
| 244 |
+
Returns:
|
| 245 |
+
torch.Tensor: Transformed value (#batch, time1, d_model)
|
| 246 |
+
weighted by the attention score (#batch, time1, time2).
|
| 247 |
+
|
| 248 |
+
"""
|
| 249 |
+
n_batch = value.size(0)
|
| 250 |
+
|
| 251 |
+
if mask.size(2) > 0: # time2 > 0
|
| 252 |
+
mask = mask.unsqueeze(1).eq(0) # (batch, 1, *, time2)
|
| 253 |
+
# For last chunk, time2 might be larger than scores.size(-1)
|
| 254 |
+
mask = mask[:, :, :, :scores.size(-1)] # (batch, 1, *, time2)
|
| 255 |
+
scores = scores.masked_fill(mask, -float('inf'))
|
| 256 |
+
attn = torch.softmax(scores, dim=-1).masked_fill(
|
| 257 |
+
mask, 0.0) # (batch, head, time1, time2)
|
| 258 |
+
|
| 259 |
+
else:
|
| 260 |
+
attn = torch.softmax(scores, dim=-1) # (batch, head, time1, time2)
|
| 261 |
+
|
| 262 |
+
p_attn = self.dropout(attn)
|
| 263 |
+
x = torch.matmul(p_attn, value) # (batch, head, time1, d_k)
|
| 264 |
+
x = (x.transpose(1, 2).contiguous().view(n_batch, -1,
|
| 265 |
+
self.h * self.d_k)
|
| 266 |
+
) # (batch, time1, d_model)
|
| 267 |
+
|
| 268 |
+
return self.linear_out(x) # (batch, time1, d_model)
|
| 269 |
+
|
| 270 |
+
def forward(
|
| 271 |
+
self,
|
| 272 |
+
query: torch.Tensor,
|
| 273 |
+
key: torch.Tensor,
|
| 274 |
+
value: torch.Tensor,
|
| 275 |
+
mask: torch.Tensor = torch.ones((0, 0, 0), dtype=torch.bool),
|
| 276 |
+
pos_emb: torch.Tensor = torch.empty(0),
|
| 277 |
+
cache: torch.Tensor = torch.zeros((0, 0, 0, 0))
|
| 278 |
+
) -> Tuple[torch.Tensor, torch.Tensor]:
|
| 279 |
+
"""Compute scaled dot product attention.
|
| 280 |
+
|
| 281 |
+
Args:
|
| 282 |
+
query (torch.Tensor): Query tensor (#batch, time1, size).
|
| 283 |
+
key (torch.Tensor): Key tensor (#batch, time2, size).
|
| 284 |
+
value (torch.Tensor): Value tensor (#batch, time2, size).
|
| 285 |
+
mask (torch.Tensor): Mask tensor (#batch, 1, time2) or
|
| 286 |
+
(#batch, time1, time2).
|
| 287 |
+
1.When applying cross attention between decoder and encoder,
|
| 288 |
+
the batch padding mask for input is in (#batch, 1, T) shape.
|
| 289 |
+
2.When applying self attention of encoder,
|
| 290 |
+
the mask is in (#batch, T, T) shape.
|
| 291 |
+
3.When applying self attention of decoder,
|
| 292 |
+
the mask is in (#batch, L, L) shape.
|
| 293 |
+
4.If the different position in decoder see different block
|
| 294 |
+
of the encoder, such as Mocha, the passed in mask could be
|
| 295 |
+
in (#batch, L, T) shape. But there is no such case in current
|
| 296 |
+
CosyVoice.
|
| 297 |
+
cache (torch.Tensor): Cache tensor (1, head, cache_t, d_k * 2),
|
| 298 |
+
where `cache_t == chunk_size * num_decoding_left_chunks`
|
| 299 |
+
and `head * d_k == size`
|
| 300 |
+
|
| 301 |
+
|
| 302 |
+
Returns:
|
| 303 |
+
torch.Tensor: Output tensor (#batch, time1, d_model).
|
| 304 |
+
torch.Tensor: Cache tensor (1, head, cache_t + time1, d_k * 2)
|
| 305 |
+
where `cache_t == chunk_size * num_decoding_left_chunks`
|
| 306 |
+
and `head * d_k == size`
|
| 307 |
+
|
| 308 |
+
"""
|
| 309 |
+
q, k, v = self.forward_qkv(query, key, value)
|
| 310 |
+
if cache.size(0) > 0:
|
| 311 |
+
key_cache, value_cache = torch.split(cache,
|
| 312 |
+
cache.size(-1) // 2,
|
| 313 |
+
dim=-1)
|
| 314 |
+
k = torch.cat([key_cache, k], dim=2)
|
| 315 |
+
v = torch.cat([value_cache, v], dim=2)
|
| 316 |
+
new_cache = torch.cat((k, v), dim=-1)
|
| 317 |
+
|
| 318 |
+
scores = torch.matmul(q, k.transpose(-2, -1)) / math.sqrt(self.d_k)
|
| 319 |
+
return self.forward_attention(v, scores, mask), new_cache
|
| 320 |
+
|
| 321 |
+
|
| 322 |
+
class RelPositionMultiHeadedAttention(MultiHeadedAttention):
|
| 323 |
+
"""Multi-Head Attention layer with relative position encoding.
|
| 324 |
+
Paper: https://arxiv.org/abs/1901.02860
|
| 325 |
+
Args:
|
| 326 |
+
n_head (int): The number of heads.
|
| 327 |
+
n_feat (int): The number of features.
|
| 328 |
+
dropout_rate (float): Dropout rate.
|
| 329 |
+
"""
|
| 330 |
+
|
| 331 |
+
def __init__(self,
|
| 332 |
+
n_head: int,
|
| 333 |
+
n_feat: int,
|
| 334 |
+
dropout_rate: float,
|
| 335 |
+
key_bias: bool = True):
|
| 336 |
+
"""Construct an RelPositionMultiHeadedAttention object."""
|
| 337 |
+
super().__init__(n_head, n_feat, dropout_rate, key_bias)
|
| 338 |
+
# linear transformation for positional encoding
|
| 339 |
+
self.linear_pos = nn.Linear(n_feat, n_feat, bias=False)
|
| 340 |
+
# these two learnable bias are used in matrix c and matrix d
|
| 341 |
+
# as described in https://arxiv.org/abs/1901.02860 Section 3.3
|
| 342 |
+
self.pos_bias_u = nn.Parameter(torch.Tensor(self.h, self.d_k))
|
| 343 |
+
self.pos_bias_v = nn.Parameter(torch.Tensor(self.h, self.d_k))
|
| 344 |
+
torch.nn.init.xavier_uniform_(self.pos_bias_u)
|
| 345 |
+
torch.nn.init.xavier_uniform_(self.pos_bias_v)
|
| 346 |
+
|
| 347 |
+
def rel_shift(self, x: torch.Tensor) -> torch.Tensor:
|
| 348 |
+
"""Compute relative positional encoding.
|
| 349 |
+
|
| 350 |
+
Args:
|
| 351 |
+
x (torch.Tensor): Input tensor (batch, head, time1, 2*time1-1).
|
| 352 |
+
time1 means the length of query vector.
|
| 353 |
+
|
| 354 |
+
Returns:
|
| 355 |
+
torch.Tensor: Output tensor.
|
| 356 |
+
|
| 357 |
+
"""
|
| 358 |
+
zero_pad = torch.zeros((x.size()[0], x.size()[1], x.size()[2], 1),
|
| 359 |
+
device=x.device,
|
| 360 |
+
dtype=x.dtype)
|
| 361 |
+
x_padded = torch.cat([zero_pad, x], dim=-1)
|
| 362 |
+
|
| 363 |
+
x_padded = x_padded.view(x.size()[0],
|
| 364 |
+
x.size()[1],
|
| 365 |
+
x.size(3) + 1, x.size(2))
|
| 366 |
+
x = x_padded[:, :, 1:].view_as(x)[
|
| 367 |
+
:, :, :, : x.size(-1) // 2 + 1
|
| 368 |
+
] # only keep the positions from 0 to time2
|
| 369 |
+
return x
|
| 370 |
+
|
| 371 |
+
def forward(
|
| 372 |
+
self,
|
| 373 |
+
query: torch.Tensor,
|
| 374 |
+
key: torch.Tensor,
|
| 375 |
+
value: torch.Tensor,
|
| 376 |
+
mask: torch.Tensor = torch.ones((0, 0, 0), dtype=torch.bool),
|
| 377 |
+
pos_emb: torch.Tensor = torch.empty(0),
|
| 378 |
+
cache: torch.Tensor = torch.zeros((0, 0, 0, 0))
|
| 379 |
+
) -> Tuple[torch.Tensor, torch.Tensor]:
|
| 380 |
+
"""Compute 'Scaled Dot Product Attention' with rel. positional encoding.
|
| 381 |
+
Args:
|
| 382 |
+
query (torch.Tensor): Query tensor (#batch, time1, size).
|
| 383 |
+
key (torch.Tensor): Key tensor (#batch, time2, size).
|
| 384 |
+
value (torch.Tensor): Value tensor (#batch, time2, size).
|
| 385 |
+
mask (torch.Tensor): Mask tensor (#batch, 1, time2) or
|
| 386 |
+
(#batch, time1, time2), (0, 0, 0) means fake mask.
|
| 387 |
+
pos_emb (torch.Tensor): Positional embedding tensor
|
| 388 |
+
(#batch, time2, size).
|
| 389 |
+
cache (torch.Tensor): Cache tensor (1, head, cache_t, d_k * 2),
|
| 390 |
+
where `cache_t == chunk_size * num_decoding_left_chunks`
|
| 391 |
+
and `head * d_k == size`
|
| 392 |
+
Returns:
|
| 393 |
+
torch.Tensor: Output tensor (#batch, time1, d_model).
|
| 394 |
+
torch.Tensor: Cache tensor (1, head, cache_t + time1, d_k * 2)
|
| 395 |
+
where `cache_t == chunk_size * num_decoding_left_chunks`
|
| 396 |
+
and `head * d_k == size`
|
| 397 |
+
"""
|
| 398 |
+
q, k, v = self.forward_qkv(query, key, value)
|
| 399 |
+
q = q.transpose(1, 2) # (batch, time1, head, d_k)
|
| 400 |
+
|
| 401 |
+
if cache.size(0) > 0:
|
| 402 |
+
key_cache, value_cache = torch.split(cache,
|
| 403 |
+
cache.size(-1) // 2,
|
| 404 |
+
dim=-1)
|
| 405 |
+
k = torch.cat([key_cache, k], dim=2)
|
| 406 |
+
v = torch.cat([value_cache, v], dim=2)
|
| 407 |
+
# NOTE(xcsong): We do cache slicing in encoder.forward_chunk, since it's
|
| 408 |
+
# non-trivial to calculate `next_cache_start` here.
|
| 409 |
+
new_cache = torch.cat((k, v), dim=-1)
|
| 410 |
+
|
| 411 |
+
n_batch_pos = pos_emb.size(0)
|
| 412 |
+
p = self.linear_pos(pos_emb).view(n_batch_pos, -1, self.h, self.d_k)
|
| 413 |
+
p = p.transpose(1, 2) # (batch, head, time1, d_k)
|
| 414 |
+
|
| 415 |
+
# (batch, head, time1, d_k)
|
| 416 |
+
q_with_bias_u = (q + self.pos_bias_u).transpose(1, 2)
|
| 417 |
+
# (batch, head, time1, d_k)
|
| 418 |
+
q_with_bias_v = (q + self.pos_bias_v).transpose(1, 2)
|
| 419 |
+
|
| 420 |
+
# compute attention score
|
| 421 |
+
# first compute matrix a and matrix c
|
| 422 |
+
# as described in https://arxiv.org/abs/1901.02860 Section 3.3
|
| 423 |
+
# (batch, head, time1, time2)
|
| 424 |
+
matrix_ac = torch.matmul(q_with_bias_u, k.transpose(-2, -1))
|
| 425 |
+
|
| 426 |
+
# compute matrix b and matrix d
|
| 427 |
+
# (batch, head, time1, time2)
|
| 428 |
+
matrix_bd = torch.matmul(q_with_bias_v, p.transpose(-2, -1))
|
| 429 |
+
# NOTE(Xiang Lyu): Keep rel_shift since espnet rel_pos_emb is used
|
| 430 |
+
if matrix_ac.shape != matrix_bd.shape:
|
| 431 |
+
matrix_bd = self.rel_shift(matrix_bd)
|
| 432 |
+
|
| 433 |
+
scores = (matrix_ac + matrix_bd) / math.sqrt(
|
| 434 |
+
self.d_k) # (batch, head, time1, time2)
|
| 435 |
+
|
| 436 |
+
return self.forward_attention(v, scores, mask), new_cache
|
| 437 |
+
|
| 438 |
+
|
| 439 |
+
|
| 440 |
+
def subsequent_mask(
|
| 441 |
+
size: int,
|
| 442 |
+
device: torch.device = torch.device("cpu"),
|
| 443 |
+
) -> torch.Tensor:
|
| 444 |
+
"""Create mask for subsequent steps (size, size).
|
| 445 |
+
|
| 446 |
+
This mask is used only in decoder which works in an auto-regressive mode.
|
| 447 |
+
This means the current step could only do attention with its left steps.
|
| 448 |
+
|
| 449 |
+
In encoder, fully attention is used when streaming is not necessary and
|
| 450 |
+
the sequence is not long. In this case, no attention mask is needed.
|
| 451 |
+
|
| 452 |
+
When streaming is need, chunk-based attention is used in encoder. See
|
| 453 |
+
subsequent_chunk_mask for the chunk-based attention mask.
|
| 454 |
+
|
| 455 |
+
Args:
|
| 456 |
+
size (int): size of mask
|
| 457 |
+
str device (str): "cpu" or "cuda" or torch.Tensor.device
|
| 458 |
+
dtype (torch.device): result dtype
|
| 459 |
+
|
| 460 |
+
Returns:
|
| 461 |
+
torch.Tensor: mask
|
| 462 |
+
|
| 463 |
+
Examples:
|
| 464 |
+
>>> subsequent_mask(3)
|
| 465 |
+
[[1, 0, 0],
|
| 466 |
+
[1, 1, 0],
|
| 467 |
+
[1, 1, 1]]
|
| 468 |
+
"""
|
| 469 |
+
arange = torch.arange(size, device=device)
|
| 470 |
+
mask = arange.expand(size, size)
|
| 471 |
+
arange = arange.unsqueeze(-1)
|
| 472 |
+
mask = mask <= arange
|
| 473 |
+
return mask
|
| 474 |
+
|
| 475 |
+
|
| 476 |
+
def subsequent_chunk_mask(
|
| 477 |
+
size: int,
|
| 478 |
+
chunk_size: int,
|
| 479 |
+
num_left_chunks: int = -1,
|
| 480 |
+
device: torch.device = torch.device("cpu"),
|
| 481 |
+
) -> torch.Tensor:
|
| 482 |
+
"""Create mask for subsequent steps (size, size) with chunk size,
|
| 483 |
+
this is for streaming encoder
|
| 484 |
+
|
| 485 |
+
Args:
|
| 486 |
+
size (int): size of mask
|
| 487 |
+
chunk_size (int): size of chunk
|
| 488 |
+
num_left_chunks (int): number of left chunks
|
| 489 |
+
<0: use full chunk
|
| 490 |
+
>=0: use num_left_chunks
|
| 491 |
+
device (torch.device): "cpu" or "cuda" or torch.Tensor.device
|
| 492 |
+
|
| 493 |
+
Returns:
|
| 494 |
+
torch.Tensor: mask
|
| 495 |
+
|
| 496 |
+
Examples:
|
| 497 |
+
>>> subsequent_chunk_mask(4, 2)
|
| 498 |
+
[[1, 1, 0, 0],
|
| 499 |
+
[1, 1, 0, 0],
|
| 500 |
+
[1, 1, 1, 1],
|
| 501 |
+
[1, 1, 1, 1]]
|
| 502 |
+
"""
|
| 503 |
+
ret = torch.zeros(size, size, device=device, dtype=torch.bool)
|
| 504 |
+
for i in range(size):
|
| 505 |
+
if num_left_chunks < 0:
|
| 506 |
+
start = 0
|
| 507 |
+
else:
|
| 508 |
+
start = max((i // chunk_size - num_left_chunks) * chunk_size, 0)
|
| 509 |
+
ending = min((i // chunk_size + 1) * chunk_size, size)
|
| 510 |
+
ret[i, start:ending] = True
|
| 511 |
+
return ret
|
| 512 |
+
|
| 513 |
+
def add_optional_chunk_mask(xs: torch.Tensor,
|
| 514 |
+
masks: torch.Tensor,
|
| 515 |
+
use_dynamic_chunk: bool,
|
| 516 |
+
use_dynamic_left_chunk: bool,
|
| 517 |
+
decoding_chunk_size: int,
|
| 518 |
+
static_chunk_size: int,
|
| 519 |
+
num_decoding_left_chunks: int,
|
| 520 |
+
enable_full_context: bool = True):
|
| 521 |
+
""" Apply optional mask for encoder.
|
| 522 |
+
|
| 523 |
+
Args:
|
| 524 |
+
xs (torch.Tensor): padded input, (B, L, D), L for max length
|
| 525 |
+
mask (torch.Tensor): mask for xs, (B, 1, L)
|
| 526 |
+
use_dynamic_chunk (bool): whether to use dynamic chunk or not
|
| 527 |
+
use_dynamic_left_chunk (bool): whether to use dynamic left chunk for
|
| 528 |
+
training.
|
| 529 |
+
decoding_chunk_size (int): decoding chunk size for dynamic chunk, it's
|
| 530 |
+
0: default for training, use random dynamic chunk.
|
| 531 |
+
<0: for decoding, use full chunk.
|
| 532 |
+
>0: for decoding, use fixed chunk size as set.
|
| 533 |
+
static_chunk_size (int): chunk size for static chunk training/decoding
|
| 534 |
+
if it's greater than 0, if use_dynamic_chunk is true,
|
| 535 |
+
this parameter will be ignored
|
| 536 |
+
num_decoding_left_chunks: number of left chunks, this is for decoding,
|
| 537 |
+
the chunk size is decoding_chunk_size.
|
| 538 |
+
>=0: use num_decoding_left_chunks
|
| 539 |
+
<0: use all left chunks
|
| 540 |
+
enable_full_context (bool):
|
| 541 |
+
True: chunk size is either [1, 25] or full context(max_len)
|
| 542 |
+
False: chunk size ~ U[1, 25]
|
| 543 |
+
|
| 544 |
+
Returns:
|
| 545 |
+
torch.Tensor: chunk mask of the input xs.
|
| 546 |
+
"""
|
| 547 |
+
# Whether to use chunk mask or not
|
| 548 |
+
if use_dynamic_chunk:
|
| 549 |
+
max_len = xs.size(1)
|
| 550 |
+
if decoding_chunk_size < 0:
|
| 551 |
+
chunk_size = max_len
|
| 552 |
+
num_left_chunks = -1
|
| 553 |
+
elif decoding_chunk_size > 0:
|
| 554 |
+
chunk_size = decoding_chunk_size
|
| 555 |
+
num_left_chunks = num_decoding_left_chunks
|
| 556 |
+
else:
|
| 557 |
+
# chunk size is either [1, 25] or full context(max_len).
|
| 558 |
+
# Since we use 4 times subsampling and allow up to 1s(100 frames)
|
| 559 |
+
# delay, the maximum frame is 100 / 4 = 25.
|
| 560 |
+
chunk_size = torch.randint(1, max_len, (1, )).item()
|
| 561 |
+
num_left_chunks = -1
|
| 562 |
+
if chunk_size > max_len // 2 and enable_full_context:
|
| 563 |
+
chunk_size = max_len
|
| 564 |
+
else:
|
| 565 |
+
chunk_size = chunk_size % 25 + 1
|
| 566 |
+
if use_dynamic_left_chunk:
|
| 567 |
+
max_left_chunks = (max_len - 1) // chunk_size
|
| 568 |
+
num_left_chunks = torch.randint(0, max_left_chunks,
|
| 569 |
+
(1, )).item()
|
| 570 |
+
chunk_masks = subsequent_chunk_mask(xs.size(1), chunk_size,
|
| 571 |
+
num_left_chunks,
|
| 572 |
+
xs.device) # (L, L)
|
| 573 |
+
chunk_masks = chunk_masks.unsqueeze(0) # (1, L, L)
|
| 574 |
+
chunk_masks = masks & chunk_masks # (B, L, L)
|
| 575 |
+
elif static_chunk_size > 0:
|
| 576 |
+
num_left_chunks = num_decoding_left_chunks
|
| 577 |
+
chunk_masks = subsequent_chunk_mask(xs.size(1), static_chunk_size,
|
| 578 |
+
num_left_chunks,
|
| 579 |
+
xs.device) # (L, L)
|
| 580 |
+
chunk_masks = chunk_masks.unsqueeze(0) # (1, L, L)
|
| 581 |
+
chunk_masks = masks & chunk_masks # (B, L, L)
|
| 582 |
+
else:
|
| 583 |
+
chunk_masks = masks
|
| 584 |
+
return chunk_masks
|
| 585 |
+
|
| 586 |
+
|
| 587 |
+
class ConformerEncoderLayer(nn.Module):
|
| 588 |
+
"""Encoder layer module.
|
| 589 |
+
Args:
|
| 590 |
+
size (int): Input dimension.
|
| 591 |
+
self_attn (torch.nn.Module): Self-attention module instance.
|
| 592 |
+
`MultiHeadedAttention` or `RelPositionMultiHeadedAttention`
|
| 593 |
+
instance can be used as the argument.
|
| 594 |
+
feed_forward (torch.nn.Module): Feed-forward module instance.
|
| 595 |
+
`PositionwiseFeedForward` instance can be used as the argument.
|
| 596 |
+
feed_forward_macaron (torch.nn.Module): Additional feed-forward module
|
| 597 |
+
instance.
|
| 598 |
+
`PositionwiseFeedForward` instance can be used as the argument.
|
| 599 |
+
conv_module (torch.nn.Module): Convolution module instance.
|
| 600 |
+
`ConvlutionModule` instance can be used as the argument.
|
| 601 |
+
dropout_rate (float): Dropout rate.
|
| 602 |
+
normalize_before (bool):
|
| 603 |
+
True: use layer_norm before each sub-block.
|
| 604 |
+
False: use layer_norm after each sub-block.
|
| 605 |
+
"""
|
| 606 |
+
|
| 607 |
+
def __init__(
|
| 608 |
+
self,
|
| 609 |
+
size: int,
|
| 610 |
+
self_attn: torch.nn.Module,
|
| 611 |
+
feed_forward: Optional[nn.Module] = None,
|
| 612 |
+
feed_forward_macaron: Optional[nn.Module] = None,
|
| 613 |
+
conv_module: Optional[nn.Module] = None,
|
| 614 |
+
dropout_rate: float = 0.1,
|
| 615 |
+
normalize_before: bool = True,
|
| 616 |
+
):
|
| 617 |
+
"""Construct an EncoderLayer object."""
|
| 618 |
+
super().__init__()
|
| 619 |
+
self.self_attn = self_attn
|
| 620 |
+
self.feed_forward = feed_forward
|
| 621 |
+
self.feed_forward_macaron = feed_forward_macaron
|
| 622 |
+
self.conv_module = conv_module
|
| 623 |
+
self.norm_ff = nn.LayerNorm(size, eps=1e-5) # for the FNN module
|
| 624 |
+
self.norm_mha = nn.LayerNorm(size, eps=1e-5) # for the MHA module
|
| 625 |
+
if feed_forward_macaron is not None:
|
| 626 |
+
self.norm_ff_macaron = nn.LayerNorm(size, eps=1e-5)
|
| 627 |
+
self.ff_scale = 0.5
|
| 628 |
+
else:
|
| 629 |
+
self.ff_scale = 1.0
|
| 630 |
+
if self.conv_module is not None:
|
| 631 |
+
self.norm_conv = nn.LayerNorm(size, eps=1e-5) # for the CNN module
|
| 632 |
+
self.norm_final = nn.LayerNorm(
|
| 633 |
+
size, eps=1e-5) # for the final output of the block
|
| 634 |
+
self.dropout = nn.Dropout(dropout_rate)
|
| 635 |
+
self.size = size
|
| 636 |
+
self.normalize_before = normalize_before
|
| 637 |
+
|
| 638 |
+
def forward(
|
| 639 |
+
self,
|
| 640 |
+
x: torch.Tensor,
|
| 641 |
+
mask: torch.Tensor,
|
| 642 |
+
pos_emb: torch.Tensor,
|
| 643 |
+
mask_pad: torch.Tensor = torch.ones((0, 0, 0), dtype=torch.bool),
|
| 644 |
+
att_cache: torch.Tensor = torch.zeros((0, 0, 0, 0)),
|
| 645 |
+
cnn_cache: torch.Tensor = torch.zeros((0, 0, 0, 0)),
|
| 646 |
+
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
| 647 |
+
"""Compute encoded features.
|
| 648 |
+
|
| 649 |
+
Args:
|
| 650 |
+
x (torch.Tensor): (#batch, time, size)
|
| 651 |
+
mask (torch.Tensor): Mask tensor for the input (#batch, time,time),
|
| 652 |
+
(0, 0, 0) means fake mask.
|
| 653 |
+
pos_emb (torch.Tensor): positional encoding, must not be None
|
| 654 |
+
for ConformerEncoderLayer.
|
| 655 |
+
mask_pad (torch.Tensor): batch padding mask used for conv module.
|
| 656 |
+
(#batch, 1,time), (0, 0, 0) means fake mask.
|
| 657 |
+
att_cache (torch.Tensor): Cache tensor of the KEY & VALUE
|
| 658 |
+
(#batch=1, head, cache_t1, d_k * 2), head * d_k == size.
|
| 659 |
+
cnn_cache (torch.Tensor): Convolution cache in conformer layer
|
| 660 |
+
(#batch=1, size, cache_t2)
|
| 661 |
+
Returns:
|
| 662 |
+
torch.Tensor: Output tensor (#batch, time, size).
|
| 663 |
+
torch.Tensor: Mask tensor (#batch, time, time).
|
| 664 |
+
torch.Tensor: att_cache tensor,
|
| 665 |
+
(#batch=1, head, cache_t1 + time, d_k * 2).
|
| 666 |
+
torch.Tensor: cnn_cahce tensor (#batch, size, cache_t2).
|
| 667 |
+
"""
|
| 668 |
+
|
| 669 |
+
# whether to use macaron style
|
| 670 |
+
if self.feed_forward_macaron is not None:
|
| 671 |
+
residual = x
|
| 672 |
+
if self.normalize_before:
|
| 673 |
+
x = self.norm_ff_macaron(x)
|
| 674 |
+
x = residual + self.ff_scale * self.dropout(
|
| 675 |
+
self.feed_forward_macaron(x))
|
| 676 |
+
if not self.normalize_before:
|
| 677 |
+
x = self.norm_ff_macaron(x)
|
| 678 |
+
|
| 679 |
+
# multi-headed self-attention module
|
| 680 |
+
residual = x
|
| 681 |
+
if self.normalize_before:
|
| 682 |
+
x = self.norm_mha(x)
|
| 683 |
+
x_att, new_att_cache = self.self_attn(x, x, x, mask, pos_emb,
|
| 684 |
+
att_cache)
|
| 685 |
+
x = residual + self.dropout(x_att)
|
| 686 |
+
if not self.normalize_before:
|
| 687 |
+
x = self.norm_mha(x)
|
| 688 |
+
|
| 689 |
+
# convolution module
|
| 690 |
+
# Fake new cnn cache here, and then change it in conv_module
|
| 691 |
+
new_cnn_cache = torch.zeros((0, 0, 0), dtype=x.dtype, device=x.device)
|
| 692 |
+
if self.conv_module is not None:
|
| 693 |
+
residual = x
|
| 694 |
+
if self.normalize_before:
|
| 695 |
+
x = self.norm_conv(x)
|
| 696 |
+
x, new_cnn_cache = self.conv_module(x, mask_pad, cnn_cache)
|
| 697 |
+
x = residual + self.dropout(x)
|
| 698 |
+
|
| 699 |
+
if not self.normalize_before:
|
| 700 |
+
x = self.norm_conv(x)
|
| 701 |
+
|
| 702 |
+
# feed forward module
|
| 703 |
+
residual = x
|
| 704 |
+
if self.normalize_before:
|
| 705 |
+
x = self.norm_ff(x)
|
| 706 |
+
|
| 707 |
+
x = residual + self.ff_scale * self.dropout(self.feed_forward(x))
|
| 708 |
+
if not self.normalize_before:
|
| 709 |
+
x = self.norm_ff(x)
|
| 710 |
+
|
| 711 |
+
if self.conv_module is not None:
|
| 712 |
+
x = self.norm_final(x)
|
| 713 |
+
|
| 714 |
+
return x, mask, new_att_cache, new_cnn_cache
|
| 715 |
+
|
| 716 |
+
|
| 717 |
+
|
| 718 |
+
class EspnetRelPositionalEncoding(torch.nn.Module):
|
| 719 |
+
"""Relative positional encoding module (new implementation).
|
| 720 |
+
|
| 721 |
+
Details can be found in https://github.com/espnet/espnet/pull/2816.
|
| 722 |
+
|
| 723 |
+
See : Appendix B in https://arxiv.org/abs/1901.02860
|
| 724 |
+
|
| 725 |
+
Args:
|
| 726 |
+
d_model (int): Embedding dimension.
|
| 727 |
+
dropout_rate (float): Dropout rate.
|
| 728 |
+
max_len (int): Maximum input length.
|
| 729 |
+
|
| 730 |
+
"""
|
| 731 |
+
|
| 732 |
+
def __init__(self, d_model: int, dropout_rate: float, max_len: int = 5000):
|
| 733 |
+
"""Construct an PositionalEncoding object."""
|
| 734 |
+
super(EspnetRelPositionalEncoding, self).__init__()
|
| 735 |
+
self.d_model = d_model
|
| 736 |
+
self.xscale = math.sqrt(self.d_model)
|
| 737 |
+
self.dropout = torch.nn.Dropout(p=dropout_rate)
|
| 738 |
+
self.pe = None
|
| 739 |
+
self.extend_pe(torch.tensor(0.0).expand(1, max_len))
|
| 740 |
+
|
| 741 |
+
def extend_pe(self, x: torch.Tensor):
|
| 742 |
+
"""Reset the positional encodings."""
|
| 743 |
+
if self.pe is not None:
|
| 744 |
+
# self.pe contains both positive and negative parts
|
| 745 |
+
# the length of self.pe is 2 * input_len - 1
|
| 746 |
+
if self.pe.size(1) >= x.size(1) * 2 - 1:
|
| 747 |
+
if self.pe.dtype != x.dtype or self.pe.device != x.device:
|
| 748 |
+
self.pe = self.pe.to(dtype=x.dtype, device=x.device)
|
| 749 |
+
return
|
| 750 |
+
# Suppose `i` means to the position of query vecotr and `j` means the
|
| 751 |
+
# position of key vector. We use position relative positions when keys
|
| 752 |
+
# are to the left (i>j) and negative relative positions otherwise (i<j).
|
| 753 |
+
pe_positive = torch.zeros(x.size(1), self.d_model)
|
| 754 |
+
pe_negative = torch.zeros(x.size(1), self.d_model)
|
| 755 |
+
position = torch.arange(0, x.size(1), dtype=torch.float32).unsqueeze(1)
|
| 756 |
+
div_term = torch.exp(
|
| 757 |
+
torch.arange(0, self.d_model, 2, dtype=torch.float32)
|
| 758 |
+
* -(math.log(10000.0) / self.d_model)
|
| 759 |
+
)
|
| 760 |
+
pe_positive[:, 0::2] = torch.sin(position * div_term)
|
| 761 |
+
pe_positive[:, 1::2] = torch.cos(position * div_term)
|
| 762 |
+
pe_negative[:, 0::2] = torch.sin(-1 * position * div_term)
|
| 763 |
+
pe_negative[:, 1::2] = torch.cos(-1 * position * div_term)
|
| 764 |
+
|
| 765 |
+
# Reserve the order of positive indices and concat both positive and
|
| 766 |
+
# negative indices. This is used to support the shifting trick
|
| 767 |
+
# as in https://arxiv.org/abs/1901.02860
|
| 768 |
+
pe_positive = torch.flip(pe_positive, [0]).unsqueeze(0)
|
| 769 |
+
pe_negative = pe_negative[1:].unsqueeze(0)
|
| 770 |
+
pe = torch.cat([pe_positive, pe_negative], dim=1)
|
| 771 |
+
self.pe = pe.to(device=x.device, dtype=x.dtype)
|
| 772 |
+
|
| 773 |
+
def forward(self, x: torch.Tensor, offset: Union[int, torch.Tensor] = 0) \
|
| 774 |
+
-> Tuple[torch.Tensor, torch.Tensor]:
|
| 775 |
+
"""Add positional encoding.
|
| 776 |
+
|
| 777 |
+
Args:
|
| 778 |
+
x (torch.Tensor): Input tensor (batch, time, `*`).
|
| 779 |
+
|
| 780 |
+
Returns:
|
| 781 |
+
torch.Tensor: Encoded tensor (batch, time, `*`).
|
| 782 |
+
|
| 783 |
+
"""
|
| 784 |
+
self.extend_pe(x)
|
| 785 |
+
x = x * self.xscale
|
| 786 |
+
pos_emb = self.position_encoding(size=x.size(1), offset=offset)
|
| 787 |
+
return self.dropout(x), self.dropout(pos_emb)
|
| 788 |
+
|
| 789 |
+
def position_encoding(self,
|
| 790 |
+
offset: Union[int, torch.Tensor],
|
| 791 |
+
size: int) -> torch.Tensor:
|
| 792 |
+
""" For getting encoding in a streaming fashion
|
| 793 |
+
|
| 794 |
+
Attention!!!!!
|
| 795 |
+
we apply dropout only once at the whole utterance level in a none
|
| 796 |
+
streaming way, but will call this function several times with
|
| 797 |
+
increasing input size in a streaming scenario, so the dropout will
|
| 798 |
+
be applied several times.
|
| 799 |
+
|
| 800 |
+
Args:
|
| 801 |
+
offset (int or torch.tensor): start offset
|
| 802 |
+
size (int): required size of position encoding
|
| 803 |
+
|
| 804 |
+
Returns:
|
| 805 |
+
torch.Tensor: Corresponding encoding
|
| 806 |
+
"""
|
| 807 |
+
pos_emb = self.pe[
|
| 808 |
+
:,
|
| 809 |
+
self.pe.size(1) // 2 - size + 1: self.pe.size(1) // 2 + size,
|
| 810 |
+
]
|
| 811 |
+
return pos_emb
|
| 812 |
+
|
| 813 |
+
|
| 814 |
+
|
| 815 |
+
class LinearEmbed(torch.nn.Module):
|
| 816 |
+
"""Linear transform the input without subsampling
|
| 817 |
+
|
| 818 |
+
Args:
|
| 819 |
+
idim (int): Input dimension.
|
| 820 |
+
odim (int): Output dimension.
|
| 821 |
+
dropout_rate (float): Dropout rate.
|
| 822 |
+
|
| 823 |
+
"""
|
| 824 |
+
|
| 825 |
+
def __init__(self, idim: int, odim: int, dropout_rate: float,
|
| 826 |
+
pos_enc_class: torch.nn.Module):
|
| 827 |
+
"""Construct an linear object."""
|
| 828 |
+
super().__init__()
|
| 829 |
+
self.out = torch.nn.Sequential(
|
| 830 |
+
torch.nn.Linear(idim, odim),
|
| 831 |
+
torch.nn.LayerNorm(odim, eps=1e-5),
|
| 832 |
+
torch.nn.Dropout(dropout_rate),
|
| 833 |
+
)
|
| 834 |
+
self.pos_enc = pos_enc_class #rel_pos_espnet
|
| 835 |
+
|
| 836 |
+
def position_encoding(self, offset: Union[int, torch.Tensor],
|
| 837 |
+
size: int) -> torch.Tensor:
|
| 838 |
+
return self.pos_enc.position_encoding(offset, size)
|
| 839 |
+
|
| 840 |
+
def forward(
|
| 841 |
+
self,
|
| 842 |
+
x: torch.Tensor,
|
| 843 |
+
offset: Union[int, torch.Tensor] = 0
|
| 844 |
+
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
| 845 |
+
"""Input x.
|
| 846 |
+
|
| 847 |
+
Args:
|
| 848 |
+
x (torch.Tensor): Input tensor (#batch, time, idim).
|
| 849 |
+
x_mask (torch.Tensor): Input mask (#batch, 1, time).
|
| 850 |
+
|
| 851 |
+
Returns:
|
| 852 |
+
torch.Tensor: linear input tensor (#batch, time', odim),
|
| 853 |
+
where time' = time .
|
| 854 |
+
torch.Tensor: linear input mask (#batch, 1, time'),
|
| 855 |
+
where time' = time .
|
| 856 |
+
|
| 857 |
+
"""
|
| 858 |
+
x = self.out(x)
|
| 859 |
+
x, pos_emb = self.pos_enc(x, offset)
|
| 860 |
+
return x, pos_emb
|
| 861 |
+
|
| 862 |
+
|
| 863 |
+
ATTENTION_CLASSES = {
|
| 864 |
+
"selfattn": MultiHeadedAttention,
|
| 865 |
+
"rel_selfattn": RelPositionMultiHeadedAttention,
|
| 866 |
+
}
|
| 867 |
+
|
| 868 |
+
ACTIVATION_CLASSES = {
|
| 869 |
+
"hardtanh": torch.nn.Hardtanh,
|
| 870 |
+
"tanh": torch.nn.Tanh,
|
| 871 |
+
"relu": torch.nn.ReLU,
|
| 872 |
+
"selu": torch.nn.SELU,
|
| 873 |
+
"swish": getattr(torch.nn, "SiLU", Swish),
|
| 874 |
+
"gelu": torch.nn.GELU,
|
| 875 |
+
}
|
| 876 |
+
|
| 877 |
+
|
| 878 |
+
def make_pad_mask(lengths: torch.Tensor, max_len: int = 0) -> torch.Tensor:
|
| 879 |
+
"""Make mask tensor containing indices of padded part.
|
| 880 |
+
|
| 881 |
+
See description of make_non_pad_mask.
|
| 882 |
+
|
| 883 |
+
Args:
|
| 884 |
+
lengths (torch.Tensor): Batch of lengths (B,).
|
| 885 |
+
Returns:
|
| 886 |
+
torch.Tensor: Mask tensor containing indices of padded part.
|
| 887 |
+
|
| 888 |
+
Examples:
|
| 889 |
+
>>> lengths = [5, 3, 2]
|
| 890 |
+
>>> make_pad_mask(lengths)
|
| 891 |
+
masks = [[0, 0, 0, 0 ,0],
|
| 892 |
+
[0, 0, 0, 1, 1],
|
| 893 |
+
[0, 0, 1, 1, 1]]
|
| 894 |
+
"""
|
| 895 |
+
batch_size = lengths.size(0)
|
| 896 |
+
max_len = max_len if max_len > 0 else lengths.max().item()
|
| 897 |
+
seq_range = torch.arange(0,
|
| 898 |
+
max_len,
|
| 899 |
+
dtype=torch.int64,
|
| 900 |
+
device=lengths.device)
|
| 901 |
+
seq_range_expand = seq_range.unsqueeze(0).expand(batch_size, max_len)
|
| 902 |
+
seq_length_expand = lengths.unsqueeze(-1)
|
| 903 |
+
mask = seq_range_expand >= seq_length_expand
|
| 904 |
+
return mask
|
| 905 |
+
|
| 906 |
+
#https://github.com/FunAudioLLM/CosyVoice/blob/main/examples/magicdata-read/cosyvoice/conf/cosyvoice.yaml
|
| 907 |
+
class ConformerEncoder(torch.nn.Module):
|
| 908 |
+
"""Conformer encoder module."""
|
| 909 |
+
|
| 910 |
+
def __init__(
|
| 911 |
+
self,
|
| 912 |
+
input_size: int,
|
| 913 |
+
output_size: int = 1024,
|
| 914 |
+
attention_heads: int = 16,
|
| 915 |
+
linear_units: int = 4096,
|
| 916 |
+
num_blocks: int = 6,
|
| 917 |
+
dropout_rate: float = 0.1,
|
| 918 |
+
positional_dropout_rate: float = 0.1,
|
| 919 |
+
attention_dropout_rate: float = 0.0,
|
| 920 |
+
input_layer: str = 'linear',
|
| 921 |
+
pos_enc_layer_type: str = 'rel_pos_espnet',
|
| 922 |
+
normalize_before: bool = True,
|
| 923 |
+
static_chunk_size: int = 1, # 1: causal_mask; 0: full_mask
|
| 924 |
+
use_dynamic_chunk: bool = False,
|
| 925 |
+
use_dynamic_left_chunk: bool = False,
|
| 926 |
+
positionwise_conv_kernel_size: int = 1,
|
| 927 |
+
macaron_style: bool =False,
|
| 928 |
+
selfattention_layer_type: str = "rel_selfattn",
|
| 929 |
+
activation_type: str = "swish",
|
| 930 |
+
use_cnn_module: bool = False,
|
| 931 |
+
cnn_module_kernel: int = 15,
|
| 932 |
+
causal: bool = False,
|
| 933 |
+
cnn_module_norm: str = "batch_norm",
|
| 934 |
+
key_bias: bool = True,
|
| 935 |
+
gradient_checkpointing: bool = False,
|
| 936 |
+
):
|
| 937 |
+
"""Construct ConformerEncoder
|
| 938 |
+
|
| 939 |
+
Args:
|
| 940 |
+
input_size to use_dynamic_chunk, see in BaseEncoder
|
| 941 |
+
positionwise_conv_kernel_size (int): Kernel size of positionwise
|
| 942 |
+
conv1d layer.
|
| 943 |
+
macaron_style (bool): Whether to use macaron style for
|
| 944 |
+
positionwise layer.
|
| 945 |
+
selfattention_layer_type (str): Encoder attention layer type,
|
| 946 |
+
the parameter has no effect now, it's just for configure
|
| 947 |
+
compatibility. #'rel_selfattn'
|
| 948 |
+
activation_type (str): Encoder activation function type.
|
| 949 |
+
use_cnn_module (bool): Whether to use convolution module.
|
| 950 |
+
cnn_module_kernel (int): Kernel size of convolution module.
|
| 951 |
+
causal (bool): whether to use causal convolution or not.
|
| 952 |
+
key_bias: whether use bias in attention.linear_k, False for whisper models.
|
| 953 |
+
"""
|
| 954 |
+
super().__init__()
|
| 955 |
+
self.output_size = output_size
|
| 956 |
+
self.embed = LinearEmbed(input_size, output_size, dropout_rate,
|
| 957 |
+
EspnetRelPositionalEncoding(output_size, positional_dropout_rate))
|
| 958 |
+
self.normalize_before = normalize_before
|
| 959 |
+
self.after_norm = torch.nn.LayerNorm(output_size, eps=1e-5)
|
| 960 |
+
self.gradient_checkpointing = gradient_checkpointing
|
| 961 |
+
self.use_dynamic_chunk = use_dynamic_chunk
|
| 962 |
+
|
| 963 |
+
self.static_chunk_size = static_chunk_size
|
| 964 |
+
self.use_dynamic_chunk = use_dynamic_chunk
|
| 965 |
+
self.use_dynamic_left_chunk = use_dynamic_left_chunk
|
| 966 |
+
activation = ACTIVATION_CLASSES[activation_type]()
|
| 967 |
+
|
| 968 |
+
# self-attention module definition
|
| 969 |
+
encoder_selfattn_layer_args = (
|
| 970 |
+
attention_heads,
|
| 971 |
+
output_size,
|
| 972 |
+
attention_dropout_rate,
|
| 973 |
+
key_bias,
|
| 974 |
+
)
|
| 975 |
+
# feed-forward module definition
|
| 976 |
+
positionwise_layer_args = (
|
| 977 |
+
output_size,
|
| 978 |
+
linear_units,
|
| 979 |
+
dropout_rate,
|
| 980 |
+
activation,
|
| 981 |
+
)
|
| 982 |
+
# convolution module definition
|
| 983 |
+
convolution_layer_args = (output_size, cnn_module_kernel, activation,
|
| 984 |
+
cnn_module_norm, causal)
|
| 985 |
+
|
| 986 |
+
self.encoders = torch.nn.ModuleList([
|
| 987 |
+
ConformerEncoderLayer(
|
| 988 |
+
output_size,
|
| 989 |
+
RelPositionMultiHeadedAttention(
|
| 990 |
+
*encoder_selfattn_layer_args),
|
| 991 |
+
PositionwiseFeedForward(*positionwise_layer_args),
|
| 992 |
+
PositionwiseFeedForward(
|
| 993 |
+
*positionwise_layer_args) if macaron_style else None,
|
| 994 |
+
ConvolutionModule(
|
| 995 |
+
*convolution_layer_args) if use_cnn_module else None,
|
| 996 |
+
dropout_rate,
|
| 997 |
+
normalize_before,
|
| 998 |
+
) for _ in range(num_blocks)
|
| 999 |
+
])
|
| 1000 |
+
|
| 1001 |
+
def forward_layers(self, xs: torch.Tensor, chunk_masks: torch.Tensor,
|
| 1002 |
+
pos_emb: torch.Tensor,
|
| 1003 |
+
mask_pad: torch.Tensor) -> torch.Tensor:
|
| 1004 |
+
for layer in self.encoders:
|
| 1005 |
+
xs, chunk_masks, _, _ = layer(xs, chunk_masks, pos_emb, mask_pad)
|
| 1006 |
+
return xs
|
| 1007 |
+
|
| 1008 |
+
@torch.jit.unused
|
| 1009 |
+
def forward_layers_checkpointed(self, xs: torch.Tensor,
|
| 1010 |
+
chunk_masks: torch.Tensor,
|
| 1011 |
+
pos_emb: torch.Tensor,
|
| 1012 |
+
mask_pad: torch.Tensor) -> torch.Tensor:
|
| 1013 |
+
for layer in self.encoders:
|
| 1014 |
+
xs, chunk_masks, _, _ = ckpt.checkpoint(layer.__call__, xs,
|
| 1015 |
+
chunk_masks, pos_emb,
|
| 1016 |
+
mask_pad)
|
| 1017 |
+
return xs
|
| 1018 |
+
|
| 1019 |
+
def forward(
|
| 1020 |
+
self,
|
| 1021 |
+
xs: torch.Tensor,
|
| 1022 |
+
pad_mask: torch.Tensor,
|
| 1023 |
+
decoding_chunk_size: int = 0,
|
| 1024 |
+
num_decoding_left_chunks: int = -1,
|
| 1025 |
+
) -> Tuple[torch.Tensor, torch.Tensor]:
|
| 1026 |
+
"""Embed positions in tensor.
|
| 1027 |
+
|
| 1028 |
+
Args:
|
| 1029 |
+
xs: padded input tensor (B, T, D)
|
| 1030 |
+
xs_lens: input length (B)
|
| 1031 |
+
decoding_chunk_size: decoding chunk size for dynamic chunk
|
| 1032 |
+
0: default for training, use random dynamic chunk.
|
| 1033 |
+
<0: for decoding, use full chunk.
|
| 1034 |
+
>0: for decoding, use fixed chunk size as set.
|
| 1035 |
+
num_decoding_left_chunks: number of left chunks, this is for decoding,
|
| 1036 |
+
the chunk size is decoding_chunk_size.
|
| 1037 |
+
>=0: use num_decoding_left_chunks
|
| 1038 |
+
<0: use all left chunks
|
| 1039 |
+
Returns:
|
| 1040 |
+
encoder output tensor xs, and subsampled masks
|
| 1041 |
+
xs: padded output tensor (B, T' ~= T/subsample_rate, D)
|
| 1042 |
+
masks: torch.Tensor batch padding mask after subsample
|
| 1043 |
+
(B, 1, T' ~= T/subsample_rate)
|
| 1044 |
+
NOTE(xcsong):
|
| 1045 |
+
We pass the `__call__` method of the modules instead of `forward` to the
|
| 1046 |
+
checkpointing API because `__call__` attaches all the hooks of the module.
|
| 1047 |
+
https://discuss.pytorch.org/t/any-different-between-model-input-and-model-forward-input/3690/2
|
| 1048 |
+
"""
|
| 1049 |
+
T = xs.size(1)
|
| 1050 |
+
masks = pad_mask.to(torch.bool).unsqueeze(1) # (B, 1, T)
|
| 1051 |
+
xs, pos_emb = self.embed(xs)
|
| 1052 |
+
mask_pad = masks # (B, 1, T/subsample_rate)
|
| 1053 |
+
chunk_masks = add_optional_chunk_mask(xs, masks,
|
| 1054 |
+
self.use_dynamic_chunk,
|
| 1055 |
+
self.use_dynamic_left_chunk,
|
| 1056 |
+
decoding_chunk_size,
|
| 1057 |
+
self.static_chunk_size,
|
| 1058 |
+
num_decoding_left_chunks)
|
| 1059 |
+
if self.gradient_checkpointing and self.training:
|
| 1060 |
+
xs = self.forward_layers_checkpointed(xs, chunk_masks, pos_emb,
|
| 1061 |
+
mask_pad)
|
| 1062 |
+
else:
|
| 1063 |
+
xs = self.forward_layers(xs, chunk_masks, pos_emb, mask_pad)
|
| 1064 |
+
if self.normalize_before:
|
| 1065 |
+
xs = self.after_norm(xs)
|
| 1066 |
+
# Here we assume the mask is not changed in encoder layers, so just
|
| 1067 |
+
# return the masks before encoder layers, and the masks will be used
|
| 1068 |
+
# for cross attention with decoder later
|
| 1069 |
+
return xs, masks
|
| 1070 |
+
|
| 1071 |
+
|
songgen/lyrics_utils/lyrics_tokenizer.py
ADDED
|
@@ -0,0 +1,874 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import re
|
| 3 |
+
import textwrap
|
| 4 |
+
from functools import cached_property
|
| 5 |
+
|
| 6 |
+
import pypinyin
|
| 7 |
+
import torch
|
| 8 |
+
from hangul_romanize import Transliter
|
| 9 |
+
from hangul_romanize.rule import academic
|
| 10 |
+
from num2words import num2words
|
| 11 |
+
from spacy.lang.ar import Arabic
|
| 12 |
+
from spacy.lang.en import English
|
| 13 |
+
from spacy.lang.es import Spanish
|
| 14 |
+
from spacy.lang.ja import Japanese
|
| 15 |
+
from spacy.lang.zh import Chinese
|
| 16 |
+
from tokenizers import Tokenizer
|
| 17 |
+
|
| 18 |
+
from .zh_num2words import TextNorm as zh_num2words
|
| 19 |
+
from typing import Dict, List, Optional, Set, Union
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
#copy from https://github.com/coqui-ai/TTS/blob/dbf1a08a0d4e47fdad6172e433eeb34bc6b13b4e/TTS/tts/layers/xtts/tokenizer.py
|
| 23 |
+
def get_spacy_lang(lang):
|
| 24 |
+
if lang == "zh":
|
| 25 |
+
return Chinese()
|
| 26 |
+
elif lang == "ja":
|
| 27 |
+
return Japanese()
|
| 28 |
+
elif lang == "ar":
|
| 29 |
+
return Arabic()
|
| 30 |
+
elif lang == "es":
|
| 31 |
+
return Spanish()
|
| 32 |
+
else:
|
| 33 |
+
# For most languages, Enlish does the job
|
| 34 |
+
return English()
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def split_sentence(text, lang, text_split_length=250):
|
| 38 |
+
"""Preprocess the input text"""
|
| 39 |
+
text_splits = []
|
| 40 |
+
if text_split_length is not None and len(text) >= text_split_length:
|
| 41 |
+
text_splits.append("")
|
| 42 |
+
nlp = get_spacy_lang(lang)
|
| 43 |
+
nlp.add_pipe("sentencizer")
|
| 44 |
+
doc = nlp(text)
|
| 45 |
+
for sentence in doc.sents:
|
| 46 |
+
if len(text_splits[-1]) + len(str(sentence)) <= text_split_length:
|
| 47 |
+
# if the last sentence + the current sentence is less than the text_split_length
|
| 48 |
+
# then add the current sentence to the last sentence
|
| 49 |
+
text_splits[-1] += " " + str(sentence)
|
| 50 |
+
text_splits[-1] = text_splits[-1].lstrip()
|
| 51 |
+
elif len(str(sentence)) > text_split_length:
|
| 52 |
+
# if the current sentence is greater than the text_split_length
|
| 53 |
+
for line in textwrap.wrap(
|
| 54 |
+
str(sentence),
|
| 55 |
+
width=text_split_length,
|
| 56 |
+
drop_whitespace=True,
|
| 57 |
+
break_on_hyphens=False,
|
| 58 |
+
tabsize=1,
|
| 59 |
+
):
|
| 60 |
+
text_splits.append(str(line))
|
| 61 |
+
else:
|
| 62 |
+
text_splits.append(str(sentence))
|
| 63 |
+
|
| 64 |
+
if len(text_splits) > 1:
|
| 65 |
+
if text_splits[0] == "":
|
| 66 |
+
del text_splits[0]
|
| 67 |
+
else:
|
| 68 |
+
text_splits = [text.lstrip()]
|
| 69 |
+
|
| 70 |
+
return text_splits
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
_whitespace_re = re.compile(r"\s+")
|
| 74 |
+
|
| 75 |
+
# List of (regular expression, replacement) pairs for abbreviations:
|
| 76 |
+
_abbreviations = {
|
| 77 |
+
"en": [
|
| 78 |
+
(re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
|
| 79 |
+
for x in [
|
| 80 |
+
("mrs", "misess"),
|
| 81 |
+
("mr", "mister"),
|
| 82 |
+
("dr", "doctor"),
|
| 83 |
+
("st", "saint"),
|
| 84 |
+
("co", "company"),
|
| 85 |
+
("jr", "junior"),
|
| 86 |
+
("maj", "major"),
|
| 87 |
+
("gen", "general"),
|
| 88 |
+
("drs", "doctors"),
|
| 89 |
+
("rev", "reverend"),
|
| 90 |
+
("lt", "lieutenant"),
|
| 91 |
+
("hon", "honorable"),
|
| 92 |
+
("sgt", "sergeant"),
|
| 93 |
+
("capt", "captain"),
|
| 94 |
+
("esq", "esquire"),
|
| 95 |
+
("ltd", "limited"),
|
| 96 |
+
("col", "colonel"),
|
| 97 |
+
("ft", "fort"),
|
| 98 |
+
]
|
| 99 |
+
],
|
| 100 |
+
"es": [
|
| 101 |
+
(re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
|
| 102 |
+
for x in [
|
| 103 |
+
("sra", "señora"),
|
| 104 |
+
("sr", "señor"),
|
| 105 |
+
("dr", "doctor"),
|
| 106 |
+
("dra", "doctora"),
|
| 107 |
+
("st", "santo"),
|
| 108 |
+
("co", "compañía"),
|
| 109 |
+
("jr", "junior"),
|
| 110 |
+
("ltd", "limitada"),
|
| 111 |
+
]
|
| 112 |
+
],
|
| 113 |
+
"fr": [
|
| 114 |
+
(re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
|
| 115 |
+
for x in [
|
| 116 |
+
("mme", "madame"),
|
| 117 |
+
("mr", "monsieur"),
|
| 118 |
+
("dr", "docteur"),
|
| 119 |
+
("st", "saint"),
|
| 120 |
+
("co", "compagnie"),
|
| 121 |
+
("jr", "junior"),
|
| 122 |
+
("ltd", "limitée"),
|
| 123 |
+
]
|
| 124 |
+
],
|
| 125 |
+
"de": [
|
| 126 |
+
(re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
|
| 127 |
+
for x in [
|
| 128 |
+
("fr", "frau"),
|
| 129 |
+
("dr", "doktor"),
|
| 130 |
+
("st", "sankt"),
|
| 131 |
+
("co", "firma"),
|
| 132 |
+
("jr", "junior"),
|
| 133 |
+
]
|
| 134 |
+
],
|
| 135 |
+
"pt": [
|
| 136 |
+
(re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
|
| 137 |
+
for x in [
|
| 138 |
+
("sra", "senhora"),
|
| 139 |
+
("sr", "senhor"),
|
| 140 |
+
("dr", "doutor"),
|
| 141 |
+
("dra", "doutora"),
|
| 142 |
+
("st", "santo"),
|
| 143 |
+
("co", "companhia"),
|
| 144 |
+
("jr", "júnior"),
|
| 145 |
+
("ltd", "limitada"),
|
| 146 |
+
]
|
| 147 |
+
],
|
| 148 |
+
"it": [
|
| 149 |
+
(re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
|
| 150 |
+
for x in [
|
| 151 |
+
# ("sig.ra", "signora"),
|
| 152 |
+
("sig", "signore"),
|
| 153 |
+
("dr", "dottore"),
|
| 154 |
+
("st", "santo"),
|
| 155 |
+
("co", "compagnia"),
|
| 156 |
+
("jr", "junior"),
|
| 157 |
+
("ltd", "limitata"),
|
| 158 |
+
]
|
| 159 |
+
],
|
| 160 |
+
"pl": [
|
| 161 |
+
(re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
|
| 162 |
+
for x in [
|
| 163 |
+
("p", "pani"),
|
| 164 |
+
("m", "pan"),
|
| 165 |
+
("dr", "doktor"),
|
| 166 |
+
("sw", "święty"),
|
| 167 |
+
("jr", "junior"),
|
| 168 |
+
]
|
| 169 |
+
],
|
| 170 |
+
"ar": [
|
| 171 |
+
(re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
|
| 172 |
+
for x in [
|
| 173 |
+
# There are not many common abbreviations in Arabic as in English.
|
| 174 |
+
]
|
| 175 |
+
],
|
| 176 |
+
"zh": [
|
| 177 |
+
(re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
|
| 178 |
+
for x in [
|
| 179 |
+
# Chinese doesn't typically use abbreviations in the same way as Latin-based scripts.
|
| 180 |
+
]
|
| 181 |
+
],
|
| 182 |
+
"cs": [
|
| 183 |
+
(re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
|
| 184 |
+
for x in [
|
| 185 |
+
("dr", "doktor"), # doctor
|
| 186 |
+
("ing", "inženýr"), # engineer
|
| 187 |
+
("p", "pan"), # Could also map to pani for woman but no easy way to do it
|
| 188 |
+
# Other abbreviations would be specialized and not as common.
|
| 189 |
+
]
|
| 190 |
+
],
|
| 191 |
+
"ru": [
|
| 192 |
+
(re.compile("\\b%s\\b" % x[0], re.IGNORECASE), x[1])
|
| 193 |
+
for x in [
|
| 194 |
+
("г-жа", "госпожа"), # Mrs.
|
| 195 |
+
("г-н", "господин"), # Mr.
|
| 196 |
+
("д-р", "доктор"), # doctor
|
| 197 |
+
# Other abbreviations are less common or specialized.
|
| 198 |
+
]
|
| 199 |
+
],
|
| 200 |
+
"nl": [
|
| 201 |
+
(re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
|
| 202 |
+
for x in [
|
| 203 |
+
("dhr", "de heer"), # Mr.
|
| 204 |
+
("mevr", "mevrouw"), # Mrs.
|
| 205 |
+
("dr", "dokter"), # doctor
|
| 206 |
+
("jhr", "jonkheer"), # young lord or nobleman
|
| 207 |
+
# Dutch uses more abbreviations, but these are the most common ones.
|
| 208 |
+
]
|
| 209 |
+
],
|
| 210 |
+
"tr": [
|
| 211 |
+
(re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
|
| 212 |
+
for x in [
|
| 213 |
+
("b", "bay"), # Mr.
|
| 214 |
+
("byk", "büyük"), # büyük
|
| 215 |
+
("dr", "doktor"), # doctor
|
| 216 |
+
# Add other Turkish abbreviations here if needed.
|
| 217 |
+
]
|
| 218 |
+
],
|
| 219 |
+
"hu": [
|
| 220 |
+
(re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
|
| 221 |
+
for x in [
|
| 222 |
+
("dr", "doktor"), # doctor
|
| 223 |
+
("b", "bácsi"), # Mr.
|
| 224 |
+
("nőv", "nővér"), # nurse
|
| 225 |
+
# Add other Hungarian abbreviations here if needed.
|
| 226 |
+
]
|
| 227 |
+
],
|
| 228 |
+
"ko": [
|
| 229 |
+
(re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
|
| 230 |
+
for x in [
|
| 231 |
+
# Korean doesn't typically use abbreviations in the same way as Latin-based scripts.
|
| 232 |
+
]
|
| 233 |
+
],
|
| 234 |
+
}
|
| 235 |
+
|
| 236 |
+
|
| 237 |
+
def expand_abbreviations_multilingual(text, lang="en"):
|
| 238 |
+
for regex, replacement in _abbreviations[lang]:
|
| 239 |
+
text = re.sub(regex, replacement, text)
|
| 240 |
+
return text
|
| 241 |
+
|
| 242 |
+
|
| 243 |
+
_symbols_multilingual = {
|
| 244 |
+
"en": [
|
| 245 |
+
(re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
|
| 246 |
+
for x in [
|
| 247 |
+
("&", " and "),
|
| 248 |
+
("@", " at "),
|
| 249 |
+
("%", " percent "),
|
| 250 |
+
("#", " hash "),
|
| 251 |
+
("$", " dollar "),
|
| 252 |
+
("£", " pound "),
|
| 253 |
+
("°", " degree "),
|
| 254 |
+
]
|
| 255 |
+
],
|
| 256 |
+
"es": [
|
| 257 |
+
(re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
|
| 258 |
+
for x in [
|
| 259 |
+
("&", " y "),
|
| 260 |
+
("@", " arroba "),
|
| 261 |
+
("%", " por ciento "),
|
| 262 |
+
("#", " numeral "),
|
| 263 |
+
("$", " dolar "),
|
| 264 |
+
("£", " libra "),
|
| 265 |
+
("°", " grados "),
|
| 266 |
+
]
|
| 267 |
+
],
|
| 268 |
+
"fr": [
|
| 269 |
+
(re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
|
| 270 |
+
for x in [
|
| 271 |
+
("&", " et "),
|
| 272 |
+
("@", " arobase "),
|
| 273 |
+
("%", " pour cent "),
|
| 274 |
+
("#", " dièse "),
|
| 275 |
+
("$", " dollar "),
|
| 276 |
+
("£", " livre "),
|
| 277 |
+
("°", " degrés "),
|
| 278 |
+
]
|
| 279 |
+
],
|
| 280 |
+
"de": [
|
| 281 |
+
(re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
|
| 282 |
+
for x in [
|
| 283 |
+
("&", " und "),
|
| 284 |
+
("@", " at "),
|
| 285 |
+
("%", " prozent "),
|
| 286 |
+
("#", " raute "),
|
| 287 |
+
("$", " dollar "),
|
| 288 |
+
("£", " pfund "),
|
| 289 |
+
("°", " grad "),
|
| 290 |
+
]
|
| 291 |
+
],
|
| 292 |
+
"pt": [
|
| 293 |
+
(re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
|
| 294 |
+
for x in [
|
| 295 |
+
("&", " e "),
|
| 296 |
+
("@", " arroba "),
|
| 297 |
+
("%", " por cento "),
|
| 298 |
+
("#", " cardinal "),
|
| 299 |
+
("$", " dólar "),
|
| 300 |
+
("£", " libra "),
|
| 301 |
+
("°", " graus "),
|
| 302 |
+
]
|
| 303 |
+
],
|
| 304 |
+
"it": [
|
| 305 |
+
(re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
|
| 306 |
+
for x in [
|
| 307 |
+
("&", " e "),
|
| 308 |
+
("@", " chiocciola "),
|
| 309 |
+
("%", " per cento "),
|
| 310 |
+
("#", " cancelletto "),
|
| 311 |
+
("$", " dollaro "),
|
| 312 |
+
("£", " sterlina "),
|
| 313 |
+
("°", " gradi "),
|
| 314 |
+
]
|
| 315 |
+
],
|
| 316 |
+
"pl": [
|
| 317 |
+
(re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
|
| 318 |
+
for x in [
|
| 319 |
+
("&", " i "),
|
| 320 |
+
("@", " małpa "),
|
| 321 |
+
("%", " procent "),
|
| 322 |
+
("#", " krzyżyk "),
|
| 323 |
+
("$", " dolar "),
|
| 324 |
+
("£", " funt "),
|
| 325 |
+
("°", " stopnie "),
|
| 326 |
+
]
|
| 327 |
+
],
|
| 328 |
+
"ar": [
|
| 329 |
+
# Arabic
|
| 330 |
+
(re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
|
| 331 |
+
for x in [
|
| 332 |
+
("&", " و "),
|
| 333 |
+
("@", " على "),
|
| 334 |
+
("%", " في المئة "),
|
| 335 |
+
("#", " رقم "),
|
| 336 |
+
("$", " دولار "),
|
| 337 |
+
("£", " جنيه "),
|
| 338 |
+
("°", " درجة "),
|
| 339 |
+
]
|
| 340 |
+
],
|
| 341 |
+
"zh": [
|
| 342 |
+
# Chinese
|
| 343 |
+
(re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
|
| 344 |
+
for x in [
|
| 345 |
+
("&", " 和 "),
|
| 346 |
+
("@", " 在 "),
|
| 347 |
+
("%", " 百分之 "),
|
| 348 |
+
("#", " 号 "),
|
| 349 |
+
("$", " 美元 "),
|
| 350 |
+
("£", " 英镑 "),
|
| 351 |
+
("°", " 度 "),
|
| 352 |
+
]
|
| 353 |
+
],
|
| 354 |
+
"cs": [
|
| 355 |
+
# Czech
|
| 356 |
+
(re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
|
| 357 |
+
for x in [
|
| 358 |
+
("&", " a "),
|
| 359 |
+
("@", " na "),
|
| 360 |
+
("%", " procento "),
|
| 361 |
+
("#", " křížek "),
|
| 362 |
+
("$", " dolar "),
|
| 363 |
+
("£", " libra "),
|
| 364 |
+
("°", " stupně "),
|
| 365 |
+
]
|
| 366 |
+
],
|
| 367 |
+
"ru": [
|
| 368 |
+
# Russian
|
| 369 |
+
(re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
|
| 370 |
+
for x in [
|
| 371 |
+
("&", " и "),
|
| 372 |
+
("@", " собака "),
|
| 373 |
+
("%", " процентов "),
|
| 374 |
+
("#", " номер "),
|
| 375 |
+
("$", " доллар "),
|
| 376 |
+
("£", " фунт "),
|
| 377 |
+
("°", " градус "),
|
| 378 |
+
]
|
| 379 |
+
],
|
| 380 |
+
"nl": [
|
| 381 |
+
# Dutch
|
| 382 |
+
(re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
|
| 383 |
+
for x in [
|
| 384 |
+
("&", " en "),
|
| 385 |
+
("@", " bij "),
|
| 386 |
+
("%", " procent "),
|
| 387 |
+
("#", " hekje "),
|
| 388 |
+
("$", " dollar "),
|
| 389 |
+
("£", " pond "),
|
| 390 |
+
("°", " graden "),
|
| 391 |
+
]
|
| 392 |
+
],
|
| 393 |
+
"tr": [
|
| 394 |
+
(re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
|
| 395 |
+
for x in [
|
| 396 |
+
("&", " ve "),
|
| 397 |
+
("@", " at "),
|
| 398 |
+
("%", " yüzde "),
|
| 399 |
+
("#", " diyez "),
|
| 400 |
+
("$", " dolar "),
|
| 401 |
+
("£", " sterlin "),
|
| 402 |
+
("°", " derece "),
|
| 403 |
+
]
|
| 404 |
+
],
|
| 405 |
+
"hu": [
|
| 406 |
+
(re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
|
| 407 |
+
for x in [
|
| 408 |
+
("&", " és "),
|
| 409 |
+
("@", " kukac "),
|
| 410 |
+
("%", " százalék "),
|
| 411 |
+
("#", " kettőskereszt "),
|
| 412 |
+
("$", " dollár "),
|
| 413 |
+
("£", " font "),
|
| 414 |
+
("°", " fok "),
|
| 415 |
+
]
|
| 416 |
+
],
|
| 417 |
+
"ko": [
|
| 418 |
+
# Korean
|
| 419 |
+
(re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
|
| 420 |
+
for x in [
|
| 421 |
+
("&", " 그리고 "),
|
| 422 |
+
("@", " 에 "),
|
| 423 |
+
("%", " 퍼센트 "),
|
| 424 |
+
("#", " 번호 "),
|
| 425 |
+
("$", " 달러 "),
|
| 426 |
+
("£", " 파운드 "),
|
| 427 |
+
("°", " 도 "),
|
| 428 |
+
]
|
| 429 |
+
],
|
| 430 |
+
}
|
| 431 |
+
|
| 432 |
+
|
| 433 |
+
def expand_symbols_multilingual(text, lang="en"):
|
| 434 |
+
for regex, replacement in _symbols_multilingual[lang]:
|
| 435 |
+
text = re.sub(regex, replacement, text)
|
| 436 |
+
text = text.replace(" ", " ") # Ensure there are no double spaces
|
| 437 |
+
return text.strip()
|
| 438 |
+
|
| 439 |
+
|
| 440 |
+
_ordinal_re = {
|
| 441 |
+
"en": re.compile(r"([0-9]+)(st|nd|rd|th)"),
|
| 442 |
+
"es": re.compile(r"([0-9]+)(º|ª|er|o|a|os|as)"),
|
| 443 |
+
"fr": re.compile(r"([0-9]+)(º|ª|er|re|e|ème)"),
|
| 444 |
+
"de": re.compile(r"([0-9]+)(st|nd|rd|th|º|ª|\.(?=\s|$))"),
|
| 445 |
+
"pt": re.compile(r"([0-9]+)(º|ª|o|a|os|as)"),
|
| 446 |
+
"it": re.compile(r"([0-9]+)(º|°|ª|o|a|i|e)"),
|
| 447 |
+
"pl": re.compile(r"([0-9]+)(º|ª|st|nd|rd|th)"),
|
| 448 |
+
"ar": re.compile(r"([0-9]+)(ون|ين|ث|ر|ى)"),
|
| 449 |
+
"cs": re.compile(r"([0-9]+)\.(?=\s|$)"), # In Czech, a dot is often used after the number to indicate ordinals.
|
| 450 |
+
"ru": re.compile(r"([0-9]+)(-й|-я|-е|-ое|-ье|-го)"),
|
| 451 |
+
"nl": re.compile(r"([0-9]+)(de|ste|e)"),
|
| 452 |
+
"tr": re.compile(r"([0-9]+)(\.|inci|nci|uncu|üncü|\.)"),
|
| 453 |
+
"hu": re.compile(r"([0-9]+)(\.|adik|edik|odik|edik|ödik|ödike|ik)"),
|
| 454 |
+
"ko": re.compile(r"([0-9]+)(번째|번|차|째)"),
|
| 455 |
+
}
|
| 456 |
+
_number_re = re.compile(r"[0-9]+")
|
| 457 |
+
_currency_re = {
|
| 458 |
+
"USD": re.compile(r"((\$[0-9\.\,]*[0-9]+)|([0-9\.\,]*[0-9]+\$))"),
|
| 459 |
+
"GBP": re.compile(r"((£[0-9\.\,]*[0-9]+)|([0-9\.\,]*[0-9]+£))"),
|
| 460 |
+
"EUR": re.compile(r"(([0-9\.\,]*[0-9]+€)|((€[0-9\.\,]*[0-9]+)))"),
|
| 461 |
+
}
|
| 462 |
+
|
| 463 |
+
_comma_number_re = re.compile(r"\b\d{1,3}(,\d{3})*(\.\d+)?\b")
|
| 464 |
+
_dot_number_re = re.compile(r"\b\d{1,3}(.\d{3})*(\,\d+)?\b")
|
| 465 |
+
_decimal_number_re = re.compile(r"([0-9]+[.,][0-9]+)")
|
| 466 |
+
|
| 467 |
+
|
| 468 |
+
def _remove_commas(m):
|
| 469 |
+
text = m.group(0)
|
| 470 |
+
if "," in text:
|
| 471 |
+
text = text.replace(",", "")
|
| 472 |
+
return text
|
| 473 |
+
|
| 474 |
+
|
| 475 |
+
def _remove_dots(m):
|
| 476 |
+
text = m.group(0)
|
| 477 |
+
if "." in text:
|
| 478 |
+
text = text.replace(".", "")
|
| 479 |
+
return text
|
| 480 |
+
|
| 481 |
+
|
| 482 |
+
def _expand_decimal_point(m, lang="en"):
|
| 483 |
+
amount = m.group(1).replace(",", ".")
|
| 484 |
+
return num2words(float(amount), lang=lang if lang != "cs" else "cz")
|
| 485 |
+
|
| 486 |
+
|
| 487 |
+
def _expand_currency(m, lang="en", currency="USD"):
|
| 488 |
+
amount = float((re.sub(r"[^\d.]", "", m.group(0).replace(",", "."))))
|
| 489 |
+
full_amount = num2words(amount, to="currency", currency=currency, lang=lang if lang != "cs" else "cz")
|
| 490 |
+
|
| 491 |
+
and_equivalents = {
|
| 492 |
+
"en": ", ",
|
| 493 |
+
"es": " con ",
|
| 494 |
+
"fr": " et ",
|
| 495 |
+
"de": " und ",
|
| 496 |
+
"pt": " e ",
|
| 497 |
+
"it": " e ",
|
| 498 |
+
"pl": ", ",
|
| 499 |
+
"cs": ", ",
|
| 500 |
+
"ru": ", ",
|
| 501 |
+
"nl": ", ",
|
| 502 |
+
"ar": ", ",
|
| 503 |
+
"tr": ", ",
|
| 504 |
+
"hu": ", ",
|
| 505 |
+
"ko": ", ",
|
| 506 |
+
}
|
| 507 |
+
|
| 508 |
+
if amount.is_integer():
|
| 509 |
+
last_and = full_amount.rfind(and_equivalents[lang])
|
| 510 |
+
if last_and != -1:
|
| 511 |
+
full_amount = full_amount[:last_and]
|
| 512 |
+
|
| 513 |
+
return full_amount
|
| 514 |
+
|
| 515 |
+
|
| 516 |
+
def _expand_ordinal(m, lang="en"):
|
| 517 |
+
return num2words(int(m.group(1)), ordinal=True, lang=lang if lang != "cs" else "cz")
|
| 518 |
+
|
| 519 |
+
|
| 520 |
+
def _expand_number(m, lang="en"):
|
| 521 |
+
return num2words(int(m.group(0)), lang=lang if lang != "cs" else "cz")
|
| 522 |
+
|
| 523 |
+
|
| 524 |
+
def expand_numbers_multilingual(text, lang="en"):
|
| 525 |
+
if lang == "zh":
|
| 526 |
+
text = zh_num2words()(text)
|
| 527 |
+
else:
|
| 528 |
+
if lang in ["en", "ru"]:
|
| 529 |
+
text = re.sub(_comma_number_re, _remove_commas, text)
|
| 530 |
+
else:
|
| 531 |
+
text = re.sub(_dot_number_re, _remove_dots, text)
|
| 532 |
+
try:
|
| 533 |
+
text = re.sub(_currency_re["GBP"], lambda m: _expand_currency(m, lang, "GBP"), text)
|
| 534 |
+
text = re.sub(_currency_re["USD"], lambda m: _expand_currency(m, lang, "USD"), text)
|
| 535 |
+
text = re.sub(_currency_re["EUR"], lambda m: _expand_currency(m, lang, "EUR"), text)
|
| 536 |
+
except:
|
| 537 |
+
pass
|
| 538 |
+
if lang != "tr":
|
| 539 |
+
text = re.sub(_decimal_number_re, lambda m: _expand_decimal_point(m, lang), text)
|
| 540 |
+
text = re.sub(_ordinal_re[lang], lambda m: _expand_ordinal(m, lang), text)
|
| 541 |
+
text = re.sub(_number_re, lambda m: _expand_number(m, lang), text)
|
| 542 |
+
return text
|
| 543 |
+
|
| 544 |
+
|
| 545 |
+
def lowercase(text):
|
| 546 |
+
return text.lower()
|
| 547 |
+
|
| 548 |
+
|
| 549 |
+
def collapse_whitespace(text):
|
| 550 |
+
return re.sub(_whitespace_re, " ", text)
|
| 551 |
+
|
| 552 |
+
|
| 553 |
+
def multilingual_cleaners(text, lang):
|
| 554 |
+
text = text.replace('"', "")
|
| 555 |
+
if lang == "tr":
|
| 556 |
+
text = text.replace("İ", "i")
|
| 557 |
+
text = text.replace("Ö", "ö")
|
| 558 |
+
text = text.replace("Ü", "ü")
|
| 559 |
+
text = lowercase(text)
|
| 560 |
+
text = expand_numbers_multilingual(text, lang)
|
| 561 |
+
text = expand_abbreviations_multilingual(text, lang)
|
| 562 |
+
text = expand_symbols_multilingual(text, lang=lang)
|
| 563 |
+
text = collapse_whitespace(text)
|
| 564 |
+
return text
|
| 565 |
+
|
| 566 |
+
|
| 567 |
+
def basic_cleaners(text):
|
| 568 |
+
"""Basic pipeline that lowercases and collapses whitespace without transliteration."""
|
| 569 |
+
text = lowercase(text)
|
| 570 |
+
text = collapse_whitespace(text)
|
| 571 |
+
return text
|
| 572 |
+
|
| 573 |
+
|
| 574 |
+
def chinese_transliterate(text):
|
| 575 |
+
return "".join(
|
| 576 |
+
[p[0] for p in pypinyin.pinyin(text, style=pypinyin.Style.TONE3, heteronym=False, neutral_tone_with_five=True)]
|
| 577 |
+
)
|
| 578 |
+
|
| 579 |
+
|
| 580 |
+
def japanese_cleaners(text, katsu):
|
| 581 |
+
text = katsu.romaji(text)
|
| 582 |
+
text = lowercase(text)
|
| 583 |
+
return text
|
| 584 |
+
|
| 585 |
+
|
| 586 |
+
def korean_transliterate(text):
|
| 587 |
+
r = Transliter(academic)
|
| 588 |
+
return r.translit(text)
|
| 589 |
+
|
| 590 |
+
|
| 591 |
+
DEFAULT_VOCAB_FILE = os.path.join(os.path.dirname(os.path.realpath(__file__)), "vocab.json")
|
| 592 |
+
|
| 593 |
+
|
| 594 |
+
class VoiceBpeTokenizer:
|
| 595 |
+
def __init__(self, vocab_file=DEFAULT_VOCAB_FILE):
|
| 596 |
+
self.tokenizer = None
|
| 597 |
+
if vocab_file is not None:
|
| 598 |
+
self.tokenizer = Tokenizer.from_file(vocab_file)
|
| 599 |
+
self.char_limits = {
|
| 600 |
+
"en": 10000,
|
| 601 |
+
"de": 253,
|
| 602 |
+
"fr": 273,
|
| 603 |
+
"es": 239,
|
| 604 |
+
"it": 213,
|
| 605 |
+
"pt": 203,
|
| 606 |
+
"pl": 224,
|
| 607 |
+
"zh": 82,
|
| 608 |
+
"ar": 166,
|
| 609 |
+
"cs": 186,
|
| 610 |
+
"ru": 182,
|
| 611 |
+
"nl": 251,
|
| 612 |
+
"tr": 226,
|
| 613 |
+
"ja": 71,
|
| 614 |
+
"hu": 224,
|
| 615 |
+
"ko": 95,
|
| 616 |
+
}
|
| 617 |
+
|
| 618 |
+
@cached_property
|
| 619 |
+
def katsu(self):
|
| 620 |
+
import cutlet
|
| 621 |
+
|
| 622 |
+
return cutlet.Cutlet()
|
| 623 |
+
|
| 624 |
+
def check_input_length(self, txt, lang):
|
| 625 |
+
lang = lang.split("-")[0] # remove the region
|
| 626 |
+
limit = self.char_limits.get(lang, 250)
|
| 627 |
+
if len(txt) > limit:
|
| 628 |
+
print(
|
| 629 |
+
f"[!] Warning: The text length exceeds the character limit of {limit} for language '{lang}', this might cause truncated audio."
|
| 630 |
+
)
|
| 631 |
+
|
| 632 |
+
def preprocess_text(self, txt, lang):
|
| 633 |
+
if lang in {"ar", "cs", "de", "en", "es", "fr", "hu", "it", "nl", "pl", "pt", "ru", "tr", "zh", "ko"}:
|
| 634 |
+
txt = multilingual_cleaners(txt, lang)
|
| 635 |
+
if lang == "zh":
|
| 636 |
+
txt = chinese_transliterate(txt)
|
| 637 |
+
if lang == "ko":
|
| 638 |
+
txt = korean_transliterate(txt)
|
| 639 |
+
elif lang == "ja":
|
| 640 |
+
txt = japanese_cleaners(txt, self.katsu)
|
| 641 |
+
elif lang == "hi":
|
| 642 |
+
# @manmay will implement this
|
| 643 |
+
txt = basic_cleaners(txt)
|
| 644 |
+
else:
|
| 645 |
+
raise NotImplementedError(f"Language '{lang}' is not supported.")
|
| 646 |
+
return txt
|
| 647 |
+
|
| 648 |
+
def encode(self, txt, lang):
|
| 649 |
+
lang = lang.split("-")[0] # remove the region
|
| 650 |
+
self.check_input_length(txt, lang)
|
| 651 |
+
txt = self.preprocess_text(txt, lang)
|
| 652 |
+
lang = "zh-cn" if lang == "zh" else lang
|
| 653 |
+
txt = f"[{lang}]{txt}"
|
| 654 |
+
txt = txt.replace(" ", "[SPACE]")
|
| 655 |
+
return self.tokenizer.encode(txt).ids
|
| 656 |
+
|
| 657 |
+
def decode(self, seq, skip_special_tokens=False):
|
| 658 |
+
if isinstance(seq, torch.Tensor):
|
| 659 |
+
seq = seq.cpu().numpy()
|
| 660 |
+
txt = self.tokenizer.decode(seq, skip_special_tokens=False).replace(" ", "")
|
| 661 |
+
txt = txt.replace("[SPACE]", " ")
|
| 662 |
+
txt = txt.replace("[STOP]", "")
|
| 663 |
+
# txt = txt.replace("[UNK]", "")
|
| 664 |
+
return txt
|
| 665 |
+
|
| 666 |
+
|
| 667 |
+
#copy from https://github.com/huggingface/transformers/blob/main/src/transformers/tokenization_utils_base.py#L3936
|
| 668 |
+
def batch_decode(
|
| 669 |
+
self,
|
| 670 |
+
sequences: Union[List[int], List[List[int]], "np.ndarray", "torch.Tensor", "tf.Tensor"],
|
| 671 |
+
skip_special_tokens: bool = False,
|
| 672 |
+
) -> List[str]:
|
| 673 |
+
"""
|
| 674 |
+
Convert a list of lists of token ids into a list of strings by calling decode.
|
| 675 |
+
|
| 676 |
+
Args:
|
| 677 |
+
sequences (`Union[List[int], List[List[int]], np.ndarray, torch.Tensor, tf.Tensor]`):
|
| 678 |
+
List of tokenized input ids. Can be obtained using the `__call__` method.
|
| 679 |
+
skip_special_tokens (`bool`, *optional*, defaults to `False`):
|
| 680 |
+
Whether or not to remove special tokens in the decoding.
|
| 681 |
+
kwargs (additional keyword arguments, *optional*):
|
| 682 |
+
Will be passed to the underlying model specific decode method.
|
| 683 |
+
|
| 684 |
+
Returns:
|
| 685 |
+
`List[str]`: The list of decoded sentences.
|
| 686 |
+
"""
|
| 687 |
+
return [
|
| 688 |
+
self.decode(seq)
|
| 689 |
+
for seq in sequences
|
| 690 |
+
]
|
| 691 |
+
|
| 692 |
+
#https://github.com/coqui-ai/TTS/blob/dev/TTS/tts/layers/xtts/trainer/dataset.py#L202
|
| 693 |
+
# def pad(self):
|
| 694 |
+
|
| 695 |
+
def __len__(self):
|
| 696 |
+
return self.tokenizer.get_vocab_size()
|
| 697 |
+
|
| 698 |
+
def get_number_tokens(self):
|
| 699 |
+
return max(self.tokenizer.get_vocab().values()) + 1
|
| 700 |
+
|
| 701 |
+
|
| 702 |
+
def test_expand_numbers_multilingual():
|
| 703 |
+
test_cases = [
|
| 704 |
+
# English
|
| 705 |
+
("In 12.5 seconds.", "In twelve point five seconds.", "en"),
|
| 706 |
+
("There were 50 soldiers.", "There were fifty soldiers.", "en"),
|
| 707 |
+
("This is a 1st test", "This is a first test", "en"),
|
| 708 |
+
("That will be $20 sir.", "That will be twenty dollars sir.", "en"),
|
| 709 |
+
("That will be 20€ sir.", "That will be twenty euro sir.", "en"),
|
| 710 |
+
("That will be 20.15€ sir.", "That will be twenty euro, fifteen cents sir.", "en"),
|
| 711 |
+
("That's 100,000.5.", "That's one hundred thousand point five.", "en"),
|
| 712 |
+
# French
|
| 713 |
+
("En 12,5 secondes.", "En douze virgule cinq secondes.", "fr"),
|
| 714 |
+
("Il y avait 50 soldats.", "Il y avait cinquante soldats.", "fr"),
|
| 715 |
+
("Ceci est un 1er test", "Ceci est un premier test", "fr"),
|
| 716 |
+
("Cela vous fera $20 monsieur.", "Cela vous fera vingt dollars monsieur.", "fr"),
|
| 717 |
+
("Cela vous fera 20€ monsieur.", "Cela vous fera vingt euros monsieur.", "fr"),
|
| 718 |
+
("Cela vous fera 20,15€ monsieur.", "Cela vous fera vingt euros et quinze centimes monsieur.", "fr"),
|
| 719 |
+
("Ce sera 100.000,5.", "Ce sera cent mille virgule cinq.", "fr"),
|
| 720 |
+
# German
|
| 721 |
+
("In 12,5 Sekunden.", "In zwölf Komma fünf Sekunden.", "de"),
|
| 722 |
+
("Es gab 50 Soldaten.", "Es gab fünfzig Soldaten.", "de"),
|
| 723 |
+
("Dies ist ein 1. Test", "Dies ist ein erste Test", "de"), # Issue with gender
|
| 724 |
+
("Das macht $20 Herr.", "Das macht zwanzig Dollar Herr.", "de"),
|
| 725 |
+
("Das macht 20€ Herr.", "Das macht zwanzig Euro Herr.", "de"),
|
| 726 |
+
("Das macht 20,15€ Herr.", "Das macht zwanzig Euro und fünfzehn Cent Herr.", "de"),
|
| 727 |
+
# Spanish
|
| 728 |
+
("En 12,5 segundos.", "En doce punto cinco segundos.", "es"),
|
| 729 |
+
("Había 50 soldados.", "Había cincuenta soldados.", "es"),
|
| 730 |
+
("Este es un 1er test", "Este es un primero test", "es"),
|
| 731 |
+
("Eso le costará $20 señor.", "Eso le costará veinte dólares señor.", "es"),
|
| 732 |
+
("Eso le costará 20€ señor.", "Eso le costará veinte euros señor.", "es"),
|
| 733 |
+
("Eso le costará 20,15€ señor.", "Eso le costará veinte euros con quince céntimos señor.", "es"),
|
| 734 |
+
# Italian
|
| 735 |
+
("In 12,5 secondi.", "In dodici virgola cinque secondi.", "it"),
|
| 736 |
+
("C'erano 50 soldati.", "C'erano cinquanta soldati.", "it"),
|
| 737 |
+
("Questo è un 1° test", "Questo è un primo test", "it"),
|
| 738 |
+
("Ti costerà $20 signore.", "Ti costerà venti dollari signore.", "it"),
|
| 739 |
+
("Ti costerà 20€ signore.", "Ti costerà venti euro signore.", "it"),
|
| 740 |
+
("Ti costerà 20,15€ signore.", "Ti costerà venti euro e quindici centesimi signore.", "it"),
|
| 741 |
+
# Portuguese
|
| 742 |
+
("Em 12,5 segundos.", "Em doze vírgula cinco segundos.", "pt"),
|
| 743 |
+
("Havia 50 soldados.", "Havia cinquenta soldados.", "pt"),
|
| 744 |
+
("Este é um 1º teste", "Este é um primeiro teste", "pt"),
|
| 745 |
+
("Isso custará $20 senhor.", "Isso custará vinte dólares senhor.", "pt"),
|
| 746 |
+
("Isso custará 20€ senhor.", "Isso custará vinte euros senhor.", "pt"),
|
| 747 |
+
(
|
| 748 |
+
"Isso custará 20,15€ senhor.",
|
| 749 |
+
"Isso custará vinte euros e quinze cêntimos senhor.",
|
| 750 |
+
"pt",
|
| 751 |
+
), # "cêntimos" should be "centavos" num2words issue
|
| 752 |
+
# Polish
|
| 753 |
+
("W 12,5 sekundy.", "W dwanaście przecinek pięć sekundy.", "pl"),
|
| 754 |
+
("Było 50 żołnierzy.", "Było pięćdziesiąt żołnierzy.", "pl"),
|
| 755 |
+
("To będzie kosztować 20€ panie.", "To będzie kosztować dwadzieścia euro panie.", "pl"),
|
| 756 |
+
("To będzie kosztować 20,15€ panie.", "To będzie kosztować dwadzieścia euro, piętnaście centów panie.", "pl"),
|
| 757 |
+
# Arabic
|
| 758 |
+
("في الـ 12,5 ثانية.", "في الـ اثنا عشر , خمسون ثانية.", "ar"),
|
| 759 |
+
("كان هناك 50 جنديًا.", "كان هناك خمسون جنديًا.", "ar"),
|
| 760 |
+
# ("ستكون النتيجة $20 يا سيد.", 'ستكون النتيجة عشرون دولار يا سيد.', 'ar'), # $ and € are mising from num2words
|
| 761 |
+
# ("ستكون النتيجة 20€ يا سيد.", 'ستكون النتيجة عشرون يورو يا سيد.', 'ar'),
|
| 762 |
+
# Czech
|
| 763 |
+
("Za 12,5 vteřiny.", "Za dvanáct celá pět vteřiny.", "cs"),
|
| 764 |
+
("Bylo tam 50 vojáků.", "Bylo tam padesát vojáků.", "cs"),
|
| 765 |
+
("To bude stát 20€ pane.", "To bude stát dvacet euro pane.", "cs"),
|
| 766 |
+
("To bude 20.15€ pane.", "To bude dvacet euro, patnáct centů pane.", "cs"),
|
| 767 |
+
# Russian
|
| 768 |
+
("Через 12.5 секунды.", "Через двенадцать запятая пять секунды.", "ru"),
|
| 769 |
+
("Там было 50 солдат.", "Там было пятьдесят солдат.", "ru"),
|
| 770 |
+
("Это будет 20.15€ сэр.", "Это будет двадцать евро, пятнадцать центов сэр.", "ru"),
|
| 771 |
+
("Это будет стоить 20€ господин.", "Это будет стоить двадцать евро господин.", "ru"),
|
| 772 |
+
# Dutch
|
| 773 |
+
("In 12,5 seconden.", "In twaalf komma vijf seconden.", "nl"),
|
| 774 |
+
("Er waren 50 soldaten.", "Er waren vijftig soldaten.", "nl"),
|
| 775 |
+
("Dat wordt dan $20 meneer.", "Dat wordt dan twintig dollar meneer.", "nl"),
|
| 776 |
+
("Dat wordt dan 20€ meneer.", "Dat wordt dan twintig euro meneer.", "nl"),
|
| 777 |
+
# Chinese (Simplified)
|
| 778 |
+
("在12.5秒内", "在十二点五秒内", "zh"),
|
| 779 |
+
("有50名士兵", "有五十名士兵", "zh"),
|
| 780 |
+
# ("那将是$20先生", '那将是二十美元先生', 'zh'), currency doesn't work
|
| 781 |
+
# ("那将是20€先生", '那将是二十欧元先生', 'zh'),
|
| 782 |
+
# Turkish
|
| 783 |
+
# ("12,5 saniye içinde.", 'On iki virgül beş saniye içinde.', 'tr'), # decimal doesn't work for TR
|
| 784 |
+
("50 asker vardı.", "elli asker vardı.", "tr"),
|
| 785 |
+
("Bu 1. test", "Bu birinci test", "tr"),
|
| 786 |
+
# ("Bu 100.000,5.", 'Bu yüz bin virgül beş.', 'tr'),
|
| 787 |
+
# Hungarian
|
| 788 |
+
("12,5 másodperc alatt.", "tizenkettő egész öt tized másodperc alatt.", "hu"),
|
| 789 |
+
("50 katona volt.", "ötven katona volt.", "hu"),
|
| 790 |
+
("Ez az 1. teszt", "Ez az első teszt", "hu"),
|
| 791 |
+
# Korean
|
| 792 |
+
("12.5 초 안에.", "십이 점 다섯 초 안에.", "ko"),
|
| 793 |
+
("50 명의 병사가 있었다.", "오십 명의 병사가 있었다.", "ko"),
|
| 794 |
+
("이것은 1 번째 테스트입니다", "이것은 첫 번째 테스트입니다", "ko"),
|
| 795 |
+
]
|
| 796 |
+
for a, b, lang in test_cases:
|
| 797 |
+
out = expand_numbers_multilingual(a, lang=lang)
|
| 798 |
+
assert out == b, f"'{out}' vs '{b}'"
|
| 799 |
+
|
| 800 |
+
|
| 801 |
+
def test_abbreviations_multilingual():
|
| 802 |
+
test_cases = [
|
| 803 |
+
# English
|
| 804 |
+
("Hello Mr. Smith.", "Hello mister Smith.", "en"),
|
| 805 |
+
("Dr. Jones is here.", "doctor Jones is here.", "en"),
|
| 806 |
+
# Spanish
|
| 807 |
+
("Hola Sr. Garcia.", "Hola señor Garcia.", "es"),
|
| 808 |
+
("La Dra. Martinez es muy buena.", "La doctora Martinez es muy buena.", "es"),
|
| 809 |
+
# French
|
| 810 |
+
("Bonjour Mr. Dupond.", "Bonjour monsieur Dupond.", "fr"),
|
| 811 |
+
("Mme. Moreau est absente aujourd'hui.", "madame Moreau est absente aujourd'hui.", "fr"),
|
| 812 |
+
# German
|
| 813 |
+
("Frau Dr. Müller ist sehr klug.", "Frau doktor Müller ist sehr klug.", "de"),
|
| 814 |
+
# Portuguese
|
| 815 |
+
("Olá Sr. Silva.", "Olá senhor Silva.", "pt"),
|
| 816 |
+
("Dra. Costa, você está disponível?", "doutora Costa, você está disponível?", "pt"),
|
| 817 |
+
# Italian
|
| 818 |
+
("Buongiorno, Sig. Rossi.", "Buongiorno, signore Rossi.", "it"),
|
| 819 |
+
# ("Sig.ra Bianchi, posso aiutarti?", 'signora Bianchi, posso aiutarti?', 'it'), # Issue with matching that pattern
|
| 820 |
+
# Polish
|
| 821 |
+
("Dzień dobry, P. Kowalski.", "Dzień dobry, pani Kowalski.", "pl"),
|
| 822 |
+
("M. Nowak, czy mogę zadać pytanie?", "pan Nowak, czy mogę zadać pytanie?", "pl"),
|
| 823 |
+
# Czech
|
| 824 |
+
("P. Novák", "pan Novák", "cs"),
|
| 825 |
+
("Dr. Vojtěch", "doktor Vojtěch", "cs"),
|
| 826 |
+
# Dutch
|
| 827 |
+
("Dhr. Jansen", "de heer Jansen", "nl"),
|
| 828 |
+
("Mevr. de Vries", "mevrouw de Vries", "nl"),
|
| 829 |
+
# Russian
|
| 830 |
+
("Здравствуйте Г-н Иванов.", "Здравствуйте господин Иванов.", "ru"),
|
| 831 |
+
("Д-р Смирнов здесь, что��ы увидеть вас.", "доктор Смирнов здесь, чтобы увидеть вас.", "ru"),
|
| 832 |
+
# Turkish
|
| 833 |
+
("Merhaba B. Yılmaz.", "Merhaba bay Yılmaz.", "tr"),
|
| 834 |
+
("Dr. Ayşe burada.", "doktor Ayşe burada.", "tr"),
|
| 835 |
+
# Hungarian
|
| 836 |
+
("Dr. Szabó itt van.", "doktor Szabó itt van.", "hu"),
|
| 837 |
+
]
|
| 838 |
+
|
| 839 |
+
for a, b, lang in test_cases:
|
| 840 |
+
out = expand_abbreviations_multilingual(a, lang=lang)
|
| 841 |
+
assert out == b, f"'{out}' vs '{b}'"
|
| 842 |
+
|
| 843 |
+
|
| 844 |
+
def test_symbols_multilingual():
|
| 845 |
+
test_cases = [
|
| 846 |
+
("I have 14% battery", "I have 14 percent battery", "en"),
|
| 847 |
+
("Te veo @ la fiesta", "Te veo arroba la fiesta", "es"),
|
| 848 |
+
("J'ai 14° de fièvre", "J'ai 14 degrés de fièvre", "fr"),
|
| 849 |
+
("Die Rechnung beträgt £ 20", "Die Rechnung beträgt pfund 20", "de"),
|
| 850 |
+
("O meu email é ana&joao@gmail.com", "O meu email é ana e joao arroba gmail.com", "pt"),
|
| 851 |
+
("linguaggio di programmazione C#", "linguaggio di programmazione C cancelletto", "it"),
|
| 852 |
+
("Moja temperatura to 36.6°", "Moja temperatura to 36.6 stopnie", "pl"),
|
| 853 |
+
("Mám 14% baterie", "Mám 14 procento baterie", "cs"),
|
| 854 |
+
("Těším se na tebe @ party", "Těším se na tebe na party", "cs"),
|
| 855 |
+
("У меня 14% заряда", "У меня 14 процентов заряда", "ru"),
|
| 856 |
+
("Я буду @ дома", "Я буду собака дома", "ru"),
|
| 857 |
+
("Ik heb 14% batterij", "Ik heb 14 procent batterij", "nl"),
|
| 858 |
+
("Ik zie je @ het feest", "Ik zie je bij het feest", "nl"),
|
| 859 |
+
("لدي 14% في البطارية", "لدي 14 في المئة في البطارية", "ar"),
|
| 860 |
+
("我的电量为 14%", "我的电量为 14 百分之", "zh"),
|
| 861 |
+
("Pilim %14 dolu.", "Pilim yüzde 14 dolu.", "tr"),
|
| 862 |
+
("Az akkumulátorom töltöttsége 14%", "Az akkumulátorom töltöttsége 14 százalék", "hu"),
|
| 863 |
+
("배터리 잔량이 14%입니다.", "배터리 잔량이 14 퍼센트입니다.", "ko"),
|
| 864 |
+
]
|
| 865 |
+
|
| 866 |
+
for a, b, lang in test_cases:
|
| 867 |
+
out = expand_symbols_multilingual(a, lang=lang)
|
| 868 |
+
assert out == b, f"'{out}' vs '{b}'"
|
| 869 |
+
|
| 870 |
+
|
| 871 |
+
if __name__ == "__main__":
|
| 872 |
+
test_expand_numbers_multilingual()
|
| 873 |
+
test_abbreviations_multilingual()
|
| 874 |
+
test_symbols_multilingual()
|
songgen/lyrics_utils/zh_num2words.py
ADDED
|
@@ -0,0 +1,1209 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Authors:
|
| 2 |
+
# 2019.5 Zhiyang Zhou (https://github.com/Joee1995/chn_text_norm.git)
|
| 3 |
+
# 2019.9 - 2022 Jiayu DU
|
| 4 |
+
#copy from https://github.com/coqui-ai/TTS/blob/dbf1a08a0d4e47fdad6172e433eeb34bc6b13b4e/TTS/tts/layers/xtts/zh_num2words.py
|
| 5 |
+
import argparse
|
| 6 |
+
import csv
|
| 7 |
+
import os
|
| 8 |
+
import re
|
| 9 |
+
import string
|
| 10 |
+
import sys
|
| 11 |
+
|
| 12 |
+
# fmt: off
|
| 13 |
+
|
| 14 |
+
# ================================================================================ #
|
| 15 |
+
# basic constant
|
| 16 |
+
# ================================================================================ #
|
| 17 |
+
CHINESE_DIGIS = "零一二三四五六七八九"
|
| 18 |
+
BIG_CHINESE_DIGIS_SIMPLIFIED = "零壹贰叁肆伍陆柒捌玖"
|
| 19 |
+
BIG_CHINESE_DIGIS_TRADITIONAL = "零壹貳參肆伍陸柒捌玖"
|
| 20 |
+
SMALLER_BIG_CHINESE_UNITS_SIMPLIFIED = "十百千万"
|
| 21 |
+
SMALLER_BIG_CHINESE_UNITS_TRADITIONAL = "拾佰仟萬"
|
| 22 |
+
LARGER_CHINESE_NUMERING_UNITS_SIMPLIFIED = "亿兆京垓秭穰沟涧正载"
|
| 23 |
+
LARGER_CHINESE_NUMERING_UNITS_TRADITIONAL = "億兆京垓秭穰溝澗正載"
|
| 24 |
+
SMALLER_CHINESE_NUMERING_UNITS_SIMPLIFIED = "十百千万"
|
| 25 |
+
SMALLER_CHINESE_NUMERING_UNITS_TRADITIONAL = "拾佰仟萬"
|
| 26 |
+
|
| 27 |
+
ZERO_ALT = "〇"
|
| 28 |
+
ONE_ALT = "幺"
|
| 29 |
+
TWO_ALTS = ["两", "兩"]
|
| 30 |
+
|
| 31 |
+
POSITIVE = ["正", "正"]
|
| 32 |
+
NEGATIVE = ["负", "負"]
|
| 33 |
+
POINT = ["点", "點"]
|
| 34 |
+
# PLUS = [u'加', u'加']
|
| 35 |
+
# SIL = [u'杠', u'槓']
|
| 36 |
+
|
| 37 |
+
FILLER_CHARS = ["呃", "啊"]
|
| 38 |
+
|
| 39 |
+
ER_WHITELIST = (
|
| 40 |
+
"(儿女|儿子|儿孙|女儿|儿媳|妻儿|"
|
| 41 |
+
"胎儿|婴儿|新生儿|婴幼儿|幼儿|少儿|小儿|儿歌|儿童|儿科|托儿所|孤儿|"
|
| 42 |
+
"儿戏|儿化|台儿庄|鹿儿岛|正儿八经|吊儿郎当|生儿育女|托儿带女|养儿防老|痴儿呆女|"
|
| 43 |
+
"佳儿佳妇|儿怜兽扰|儿无常父|儿不嫌母丑|儿行千里母担忧|儿大不由爷|苏乞儿)"
|
| 44 |
+
)
|
| 45 |
+
ER_WHITELIST_PATTERN = re.compile(ER_WHITELIST)
|
| 46 |
+
|
| 47 |
+
# 中文数字系统类型
|
| 48 |
+
NUMBERING_TYPES = ["low", "mid", "high"]
|
| 49 |
+
|
| 50 |
+
CURRENCY_NAMES = "(人民币|美元|日元|英镑|欧元|马克|法郎|加拿大元|澳元|港币|先令|芬兰马克|爱尔兰镑|" "里拉|荷兰盾|埃斯库多|比塞塔|印尼盾|林吉特|新西兰元|比索|卢布|新加坡元|韩元|泰铢)"
|
| 51 |
+
CURRENCY_UNITS = "((亿|千万|百万|万|千|百)|(亿|千万|百万|万|千|百|)元|(亿|千万|百万|万|千|百|)块|角|毛|分)"
|
| 52 |
+
COM_QUANTIFIERS = (
|
| 53 |
+
"(匹|张|座|回|场|尾|条|个|首|阙|阵|网|炮|顶|丘|棵|只|支|袭|辆|挑|担|颗|壳|窠|曲|墙|群|腔|"
|
| 54 |
+
"砣|座|客|贯|扎|捆|刀|令|打|手|罗|坡|山|岭|江|溪|钟|队|单|双|对|出|口|头|脚|板|跳|枝|件|贴|"
|
| 55 |
+
"针|线|管|名|位|身|堂|课|本|页|家|户|层|丝|毫|厘|分|钱|两|斤|担|铢|石|钧|锱|忽|(千|毫|微)克|"
|
| 56 |
+
"毫|厘|分|寸|尺|丈|里|寻|常|铺|程|(千|分|厘|毫|微)米|撮|勺|合|升|斗|石|盘|碗|碟|叠|桶|笼|盆|"
|
| 57 |
+
"盒|杯|钟|斛|锅|簋|篮|盘|桶|罐|瓶|壶|卮|盏|箩|箱|煲|啖|袋|钵|年|月|日|季|刻|时|周|天|秒|分|旬|"
|
| 58 |
+
"纪|岁|世|更|夜|春|夏|秋|冬|代|伏|辈|丸|泡|粒|颗|幢|堆|条|根|支|道|面|片|张|颗|块)"
|
| 59 |
+
)
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
# Punctuation information are based on Zhon project (https://github.com/tsroten/zhon.git)
|
| 63 |
+
CN_PUNCS_STOP = "!?。。"
|
| 64 |
+
CN_PUNCS_NONSTOP = ""#$%&'()*+,-/:;<=>@[\]^_`{|}~⦅⦆「」、、〃《》「」『』【】〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘’‛“”„‟…‧﹏·〈〉-"
|
| 65 |
+
CN_PUNCS = CN_PUNCS_STOP + CN_PUNCS_NONSTOP
|
| 66 |
+
|
| 67 |
+
PUNCS = CN_PUNCS + string.punctuation
|
| 68 |
+
PUNCS_TRANSFORM = str.maketrans(PUNCS, "," * len(PUNCS), "") # replace puncs with English comma
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
# https://zh.wikipedia.org/wiki/全行和半行
|
| 72 |
+
QJ2BJ = {
|
| 73 |
+
" ": " ",
|
| 74 |
+
"!": "!",
|
| 75 |
+
""": '"',
|
| 76 |
+
"#": "#",
|
| 77 |
+
"$": "$",
|
| 78 |
+
"%": "%",
|
| 79 |
+
"&": "&",
|
| 80 |
+
"'": "'",
|
| 81 |
+
"(": "(",
|
| 82 |
+
")": ")",
|
| 83 |
+
"*": "*",
|
| 84 |
+
"+": "+",
|
| 85 |
+
",": ",",
|
| 86 |
+
"-": "-",
|
| 87 |
+
".": ".",
|
| 88 |
+
"/": "/",
|
| 89 |
+
"0": "0",
|
| 90 |
+
"1": "1",
|
| 91 |
+
"2": "2",
|
| 92 |
+
"3": "3",
|
| 93 |
+
"4": "4",
|
| 94 |
+
"5": "5",
|
| 95 |
+
"6": "6",
|
| 96 |
+
"7": "7",
|
| 97 |
+
"8": "8",
|
| 98 |
+
"9": "9",
|
| 99 |
+
":": ":",
|
| 100 |
+
";": ";",
|
| 101 |
+
"<": "<",
|
| 102 |
+
"=": "=",
|
| 103 |
+
">": ">",
|
| 104 |
+
"?": "?",
|
| 105 |
+
"@": "@",
|
| 106 |
+
"A": "A",
|
| 107 |
+
"B": "B",
|
| 108 |
+
"C": "C",
|
| 109 |
+
"D": "D",
|
| 110 |
+
"E": "E",
|
| 111 |
+
"F": "F",
|
| 112 |
+
"G": "G",
|
| 113 |
+
"H": "H",
|
| 114 |
+
"I": "I",
|
| 115 |
+
"J": "J",
|
| 116 |
+
"K": "K",
|
| 117 |
+
"L": "L",
|
| 118 |
+
"M": "M",
|
| 119 |
+
"N": "N",
|
| 120 |
+
"O": "O",
|
| 121 |
+
"P": "P",
|
| 122 |
+
"Q": "Q",
|
| 123 |
+
"R": "R",
|
| 124 |
+
"S": "S",
|
| 125 |
+
"T": "T",
|
| 126 |
+
"U": "U",
|
| 127 |
+
"V": "V",
|
| 128 |
+
"W": "W",
|
| 129 |
+
"X": "X",
|
| 130 |
+
"Y": "Y",
|
| 131 |
+
"Z": "Z",
|
| 132 |
+
"[": "[",
|
| 133 |
+
"\": "\\",
|
| 134 |
+
"]": "]",
|
| 135 |
+
"^": "^",
|
| 136 |
+
"_": "_",
|
| 137 |
+
"`": "`",
|
| 138 |
+
"a": "a",
|
| 139 |
+
"b": "b",
|
| 140 |
+
"c": "c",
|
| 141 |
+
"d": "d",
|
| 142 |
+
"e": "e",
|
| 143 |
+
"f": "f",
|
| 144 |
+
"g": "g",
|
| 145 |
+
"h": "h",
|
| 146 |
+
"i": "i",
|
| 147 |
+
"j": "j",
|
| 148 |
+
"k": "k",
|
| 149 |
+
"l": "l",
|
| 150 |
+
"m": "m",
|
| 151 |
+
"n": "n",
|
| 152 |
+
"o": "o",
|
| 153 |
+
"p": "p",
|
| 154 |
+
"q": "q",
|
| 155 |
+
"r": "r",
|
| 156 |
+
"s": "s",
|
| 157 |
+
"t": "t",
|
| 158 |
+
"u": "u",
|
| 159 |
+
"v": "v",
|
| 160 |
+
"w": "w",
|
| 161 |
+
"x": "x",
|
| 162 |
+
"y": "y",
|
| 163 |
+
"z": "z",
|
| 164 |
+
"{": "{",
|
| 165 |
+
"|": "|",
|
| 166 |
+
"}": "}",
|
| 167 |
+
"~": "~",
|
| 168 |
+
}
|
| 169 |
+
QJ2BJ_TRANSFORM = str.maketrans("".join(QJ2BJ.keys()), "".join(QJ2BJ.values()), "")
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
# 2013 China National Standard: https://zh.wikipedia.org/wiki/通用规范汉字表, raw resources:
|
| 173 |
+
# https://github.com/mozillazg/pinyin-data/blob/master/kMandarin_8105.txt with 8105 chinese chars in total
|
| 174 |
+
CN_CHARS_COMMON = (
|
| 175 |
+
"一丁七万丈三上下不与丏丐丑专且丕世丘丙业丛东丝丞丢两严丧个丫中丰串临丸丹为主丽举"
|
| 176 |
+
"乂乃久么义之乌乍乎乏乐乒乓乔乖乘乙乜九乞也习乡书乩买乱乳乸乾了予争事二亍于亏云互"
|
| 177 |
+
"亓五井亘亚些亟亡亢交亥亦产亨亩享京亭亮亲亳亵亶亸亹人亿什仁仂仃仄仅仆仇仉今介仍从"
|
| 178 |
+
"仑仓仔仕他仗付仙仝仞仟仡代令以仨仪仫们仰仲仳仵件价任份仿企伈伉伊伋伍伎伏伐休众优"
|
| 179 |
+
"伙会伛伞伟传伢伣伤伥伦伧伪伫伭伯估伲伴伶伸伺似伽伾佁佃但位低住佐佑体何佖佗佘余佚"
|
| 180 |
+
"佛作佝佞佟你佣佤佥佩佬佯佰佳佴佶佸佺佻佼佽佾使侁侂侃侄侈侉例侍侏侑侔侗侘供依侠侣"
|
| 181 |
+
"侥侦侧侨侩侪侬侮侯侴侵侹便促俄俅俊俍俎俏俐俑俗俘俙俚俜保俞俟信俣俦俨俩俪俫俭修俯"
|
| 182 |
+
"俱俳俵俶俸俺俾倌倍倏倒倓倔倕倘候倚倜倞借倡倥倦倧倨倩倪倬倭倮倴债倻值倾偁偃假偈偌"
|
| 183 |
+
"偎偏偓偕做停偡健偬偭偰偲偶偷偻偾偿傀傃傅傈傉傍傒傕傣傥傧储傩催傲傺傻僇僎像僔僖僚"
|
| 184 |
+
"僦僧僬僭僮僰僳僵僻儆儇儋儒儡儦儳儴儿兀允元兄充兆先光克免兑兔兕兖党兜兢入全八公六"
|
| 185 |
+
"兮兰共关兴兵其具典兹养兼兽冀冁内冈冉册再冏冒冔冕冗写军农冠冢冤冥冬冮冯冰冱冲决况"
|
| 186 |
+
"冶冷冻冼冽净凄准凇凉凋凌减凑凓凘凛凝几凡凤凫凭凯凰凳凶凸凹出击凼函凿刀刁刃分切刈"
|
| 187 |
+
"刊刍刎刑划刖列刘则刚创初删判刨利别刬刭刮到刳制刷券刹刺刻刽刿剀剁剂剃剅削剋剌前剐"
|
| 188 |
+
"剑剔剕剖剜剞剟剡剥剧剩剪副割剽剿劁劂劄劈劐劓力劝办功加务劢劣动助努劫劬劭励劲劳劼"
|
| 189 |
+
"劾势勃勇勉勋勍勐勒勔勖勘勚募勠勤勰勺勾勿匀包匆匈匍匏匐匕化北匙匜匝匠匡匣匦匪匮匹"
|
| 190 |
+
"区医匼匾匿十千卅升午卉半华协卑卒卓单卖南博卜卞卟占卡卢卣卤卦卧卫卬卮卯印危即却卵"
|
| 191 |
+
"卷卸卺卿厂厄厅历厉压厌厍厕厖厘厚厝原厢厣厥厦厨厩厮去厾县叁参叆叇又叉及友双反发叔"
|
| 192 |
+
"叕取受变叙叚叛叟叠口古句另叨叩只叫召叭叮可台叱史右叵叶号司叹叻叼叽吁吃各吆合吉吊"
|
| 193 |
+
"同名后吏吐向吒吓吕吖吗君吝吞吟吠吡吣否吧吨吩含听吭吮启吱吲吴吵吸吹吻吼吽吾呀呃呆"
|
| 194 |
+
"呇呈告呋呐呒呓呔呕呖呗员呙呛呜呢呣呤呦周呱呲味呵呶呷呸呻呼命咀咂咄咆咇咉咋和咍咎"
|
| 195 |
+
"咏咐咒咔咕咖咙咚咛咝咡咣咤咥咦咧咨咩咪咫咬咯咱咳咴咸咺咻咽咿哀品哂哃哄哆哇哈哉哌"
|
| 196 |
+
"响哎哏哐哑哒哓哔哕哗哙哚哝哞哟哢哥哦哧哨哩哪哭哮哱哲哳哺哼哽哿唁唆唇唉唏唐唑唔唛"
|
| 197 |
+
"唝唠唢唣唤唧唪唬售唯唰唱唳唵唷唼唾唿啁啃啄商啉啊啐啕啖啜啡啤啥啦啧啪啫啬啭啮啰啴"
|
| 198 |
+
"啵啶啷啸啻啼啾喀喁喂喃善喆喇喈喉喊喋喏喑喔喘喙喜喝喟喤喧喱喳喵喷喹喻喽喾嗄嗅嗉嗌"
|
| 199 |
+
"嗍嗐嗑嗒嗓嗔嗖嗜嗝嗞嗟嗡嗣嗤嗥嗦嗨嗪嗫嗬嗯嗲嗳嗵嗷嗽嗾嘀嘁嘈嘉嘌嘎嘏嘘嘚嘛嘞嘟嘡"
|
| 200 |
+
"嘣嘤嘧嘬嘭嘱嘲嘴嘶嘹嘻嘿噀噂噇噌噍噎噔噗噘噙噜噢噤器噩噪噫噬噱噶噻噼嚄嚅嚆嚎嚏嚓"
|
| 201 |
+
"嚚嚣嚭嚯嚷嚼囊囔囚四回囟因囡团囤囫园困囱围囵囷囹固国图囿圃圄圆圈圉圊圌圐圙圜土圢"
|
| 202 |
+
"圣在圩圪圫圬圭圮圯地圲圳圹场圻圾址坂均坉坊坋坌坍坎坏坐坑坒块坚坛坜坝坞坟坠坡坤坥"
|
| 203 |
+
"坦坨坩坪坫坬坭坯坰坳坷坻坼坽垂垃垄垆垈型垌垍垎垏垒垓垕垙垚垛垞垟垠垡垢垣垤垦垧垩"
|
| 204 |
+
"垫垭垮垯垱垲垴垵垸垺垾垿埂埃埆埇埋埌城埏埒埔埕埗埘埙埚埝域埠埤埪埫埭埯埴埵埸培基"
|
| 205 |
+
"埼埽堂堃堆堇堉堋堌堍堎堐堑堕堙堞堠堡堤堧堨堪堰堲堵堼堽堾塄塅塆塌塍塑塔塘塝塞塥填"
|
| 206 |
+
"塬塱塾墀墁境墅墈墉墐墒墓墕墘墙墚增墟墡墣墦墨墩墼壁壅壑壕壤士壬壮声壳壶壸壹处备复"
|
| 207 |
+
"夏夐夔夕外夙多夜够夤夥大天太夫夬夭央夯失头夷夸夹夺夼奁奂奄奇奈奉奋奎奏契奓奔奕奖"
|
| 208 |
+
"套奘奚奠奡奢奥奭女奴奶奸她好妁如妃妄妆妇妈妊妍妒妓妖妗妘妙妞妣妤妥妧妨妩妪妫妭妮"
|
| 209 |
+
"妯妲妹妻妾姆姈姊始姐姑姒姓委姗姘姚姜姝姞姣姤姥姨姬姮姱姶姹姻姽姿娀威娃娄娅娆娇娈"
|
| 210 |
+
"娉娌娑娓娘娜娟娠娣娥娩娱娲娴娵娶娼婀婆婉婊婌婍婕婘婚婞婠婢婤婧婪婫婳婴婵婶婷婺���"
|
| 211 |
+
"婼婿媂媄媆媒媓媖媚媛媞媪媭媱媲媳媵媸媾嫁嫂嫄嫉嫌嫒嫔嫕嫖嫘嫚嫜嫠嫡嫣嫦嫩嫪嫫嫭嫱"
|
| 212 |
+
"嫽嬉嬖嬗嬛嬥嬬嬴嬷嬿孀孅子孑孓孔孕孖字存孙孚孛孜孝孟孢季孤孥学孩孪孬孰孱孳孵孺孽"
|
| 213 |
+
"宁它宄宅宇守安宋完宏宓宕宗官宙定宛宜宝实宠审客宣室宥宦宧宪宫宬宰害宴宵家宸容宽宾"
|
| 214 |
+
"宿寁寂寄寅密寇富寐寒寓寝寞察寡寤寥寨寮寰寸对寺寻导寿封射将尉尊小少尔尕尖尘尚尜尝"
|
| 215 |
+
"尢尤尥尧尨尪尬就尴尸尹尺尻尼尽尾尿局屁层屃居屈屉届屋屎屏屐屑展屙属屠屡屣履屦屯山"
|
| 216 |
+
"屹屺屼屾屿岁岂岈岊岌岍岐岑岔岖岗岘岙岚岛岜岞岠岢岣岨岩岫岬岭岱岳岵岷岸岽岿峁峂峃"
|
| 217 |
+
"峄峋峒峗峘峙峛峡峣峤峥峦峧峨峪峭峰峱峻峿崀崁崂崃崄崆崇崌崎崒崔崖崚崛崞崟崡崤崦崧"
|
| 218 |
+
"崩崭崮崴崶崽崾崿嵁嵅嵇嵊嵋嵌嵎嵖嵘嵚嵛嵝嵩嵫嵬嵯嵲嵴嶂嶅嶍嶒嶓嶙嶝嶟嶦嶲嶷巅巇巉"
|
| 219 |
+
"巍川州巡巢工左巧巨巩巫差巯己已巳巴巷巽巾币市布帅帆师希帏帐帑帔帕帖帘帙帚帛帜帝帡"
|
| 220 |
+
"带帧帨席帮帱帷常帻帼帽幂幄幅幌幔幕幖幛幞幡幢幪干平年并幸幺幻幼幽广庄庆庇床庋序庐"
|
| 221 |
+
"庑库应底庖店庙庚府庞废庠庤庥度座庭庱庳庵庶康庸庹庼庾廆廉廊廋廑廒廓廖廙廛廨廪延廷"
|
| 222 |
+
"建廿开弁异弃弄弆弇弈弊弋式弑弓引弗弘弛弟张弢弥弦弧弨弩弭弯弱弶弸弹强弼彀归当录彖"
|
| 223 |
+
"彗彘彝彟形彤彦彧彩彪彬彭彰影彳彷役彻彼往征徂径待徇很徉徊律徐徒徕得徘徙徛徜御徨循"
|
| 224 |
+
"徭微徵德徼徽心必忆忉忌忍忏忐忑忒忖志忘忙忝忞忠忡忤忧忪快忭忮忱忳念忸忺忻忽忾忿怀"
|
| 225 |
+
"态怂怃怄怅怆怊怍怎怏怒怔怕怖怙怛怜思怠怡急怦性怨怩怪怫怯怵总怼怿恁恂恃恋恍恐恒恓"
|
| 226 |
+
"恔恕恙恚恝恢恣恤恧恨恩恪恫恬恭息恰恳恶恸恹恺恻恼恽恿悃悄悆悈悉悌悍悒悔悖悚悛悝悟"
|
| 227 |
+
"悠悢患悦您悫悬悭悯悰悱悲悴悸悻悼情惆惇惊惋惎惑惔惕惘惙惚惛惜惝惟惠惦惧惨惩惫惬惭"
|
| 228 |
+
"惮惯惰想惴惶惹惺愀愁愃愆愈愉愍愎意愐愔愕愚感愠愣愤愦愧愫愭愿慆慈慊慌慎慑慕慝慢慥"
|
| 229 |
+
"慧慨慬慭慰慵慷憋憎憔憕憙憧憨憩憬憭憷憺憾懂懈懊懋懑懒懔懦懵懿戆戈戊戋戌戍戎戏成我"
|
| 230 |
+
"戒戕或戗战戚戛戟戡戢戣戤戥截戬戭戮戳戴户戽戾房所扁扂扃扅扆扇扈扉扊手才扎扑扒打扔"
|
| 231 |
+
"托扛扞扣扦执扩扪扫扬扭扮扯扰扳扶批扺扼扽找承技抃抄抉把抑抒抓抔投抖抗折抚抛抟抠抡"
|
| 232 |
+
"抢护报抨披抬抱抵抹抻押抽抿拂拃拄担拆拇拈拉拊拌拍拎拐拒拓拔拖拗拘拙招拜拟拢拣拤拥"
|
| 233 |
+
"拦拧拨择括拭拮拯拱拳拴拶拷拼拽拾拿持挂指挈按挎挑挓挖挚挛挝挞挟挠挡挣挤挥挦挨挪挫"
|
| 234 |
+
"振挲挹挺挽捂捃捅捆捉捋捌捍捎捏捐捕捞损捡换捣捧捩捭据捯捶捷捺捻捽掀掂掇授掉掊掌掎"
|
| 235 |
+
"掏掐排掖掘掞掠探掣接控推掩措掬掭掮掰掳掴掷掸掺掼掾揄揆揉揍描提插揕揖揠握揣揩揪揭"
|
| 236 |
+
"揳援揶揸揽揿搀搁搂搅搋搌搏搐搒搓搔搛搜搞搠搡搦搪搬搭搴携搽摁摄摅摆摇摈摊摏摒摔摘"
|
| 237 |
+
"摛摞摧摩摭摴摸摹摽撂撄撅撇撑撒撕撖撙撞撤撩撬播撮撰撵撷撸撺撼擀擂擅操擎擐擒擘擞擢"
|
| 238 |
+
"擤擦擿攀攉攒攘攥攫攮支收攸改攻攽放政故效敉敌敏救敔敕敖教敛敝敞敢散敦敩敫敬数敲整"
|
| 239 |
+
"敷文斋斌斐斑斓斗料斛斜斝斟斠斡斤斥斧斩斫断斯新斶方於施旁旃旄旅旆旋旌旎族旐旒旖旗"
|
| 240 |
+
"旞无既日旦旧旨早旬旭旮旯旰旱旴旵时旷旸旺旻旿昀昂昃昄昆昇昈昉昊昌明昏昒易昔昕昙昝"
|
| 241 |
+
"星映昡昣昤春昧昨昪昫昭是昱昳昴昵昶昺昼昽显晁晃晅晊晋晌晏晐晒晓晔晕晖晗晙晚晞晟晡"
|
| 242 |
+
"晢晤晦晨晪晫普景晰晱晴晶晷智晾暂暄暅暇暌暑暕暖暗暝暧暨暮暲暴暵暶暹暾暿曈曌曙曛曜"
|
| 243 |
+
"曝曦曩曰曲曳更曷曹曼曾替最月有朋服朏朐朓朔朕朗望朝期朦木未末本札术朱朳朴朵朸机朽"
|
| 244 |
+
"杀杂权杄杆杈杉杌李杏材村杓杕杖杙杜杞束杠条来杧杨杩杪杭杯杰杲杳杵杷杻杼松板极构枅"
|
| 245 |
+
"枇枉枋枍析枕林枘枚果枝枞枢枣枥枧枨枪枫枭枯枰枲枳枵架枷枸枹柁柃柄柈柊柏某柑柒染柔"
|
| 246 |
+
"柖柘柙柚柜柝柞柠柢查柩柬柯柰柱柳柴柷柽柿栀栅标栈栉栊栋栌栎栏栐树栒栓栖栗栝栟校栩"
|
| 247 |
+
"株栲栳栴样核根栻格栽栾桀桁桂桃桄桅框案桉桊桌桎桐桑桓桔桕桠桡桢档桤桥桦桧桨桩桫桯"
|
| 248 |
+
"桲桴桶桷桹梁梃梅梆梌梏梓梗梠梢梣梦梧梨梭梯械梳梴梵梼梽梾梿检棁棂棉棋棍棐棒棓棕棘"
|
| 249 |
+
"棚棠棣棤棨棪棫棬森棰棱棵棹棺棻棼棽椀椁椅椆椋植椎椐椑椒椓椟椠椤椪椭椰椴椸椹椽椿楂"
|
| 250 |
+
"楒楔楗楙楚楝楞楠楣楦楩楪楫楮楯楷楸楹楼概榃榄榅榆榇榈榉榍榑榔榕榖榛榜榧榨榫榭榰���"
|
| 251 |
+
"榴榷榻槁槃槊槌槎槐槔槚槛槜槟槠槭槱槲槽槿樊樗樘樟模樨横樯樱樵樽樾橄橇橐橑橘橙橛橞"
|
| 252 |
+
"橡橥橦橱橹橼檀檄檎檐檑檗檞檠檩檫檬櫆欂欠次欢欣欤欧欲欸欹欺欻款歃歅歆歇歉歌歙止正"
|
| 253 |
+
"此步武歧歪歹死歼殁殂殃殄殆殇殉殊残殍殒殓殖殚殛殡殣殪殳殴段殷殿毁毂毅毋毌母每毐毒"
|
| 254 |
+
"毓比毕毖毗毙毛毡毪毫毯毳毵毹毽氅氆氇氍氏氐民氓气氕氖氘氙氚氛氟氡氢氤氦氧氨氩氪氮"
|
| 255 |
+
"氯氰氲水永氾氿汀汁求汆汇汈汉汊汋汐汔汕汗汛汜汝汞江池污汤汧汨汩汪汫汭汰汲汴汶汹汽"
|
| 256 |
+
"汾沁沂沃沄沅沆沇沈沉沌沏沐沓沔沘沙沚沛沟没沣沤沥沦沧沨沩沪沫沭沮沱河沸油沺治沼沽"
|
| 257 |
+
"沾沿泂泃泄泅泇泉泊泌泐泓泔法泖泗泙泚泛泜泞泠泡波泣泥注泪泫泮泯泰泱泳泵泷泸泺泻泼"
|
| 258 |
+
"泽泾洁洄洇洈洋洌洎洑洒洓洗洘洙洚洛洞洢洣津洧洨洪洫洭洮洱洲洳洴洵洸洹洺活洼洽派洿"
|
| 259 |
+
"流浃浅浆浇浈浉浊测浍济浏浐浑浒浓浔浕浙浚浛浜浞浟浠浡浣浥浦浩浪浬浭浮浯浰浲浴海浸"
|
| 260 |
+
"浼涂涄涅消涉涌涍涎涐涑涓涔涕涘涛涝涞涟涠涡涢涣涤润涧涨涩涪涫涮涯液涴涵涸涿淀淄淅"
|
| 261 |
+
"淆淇淋淌淏淑淖淘淙淜淝淞淟淠淡淤淦淫淬淮淯深淳淴混淹添淼清渊渌渍渎渐渑渔渗渚渝渟"
|
| 262 |
+
"渠渡渣渤渥温渫渭港渰渲渴游渺渼湃湄湉湍湎湑湓湔湖湘湛湜湝湟湣湫湮湲湴湾湿溁溃溅溆"
|
| 263 |
+
"溇溉溍溏源溘溚溜溞溟溠溢溥溦溧溪溯溱溲溴溵溶溷溹溺溻溽滁滂滃滆滇滉滋滍滏滑滓滔滕"
|
| 264 |
+
"滗滘滚滞滟滠满滢滤滥滦滧滨滩滪滫滴滹漂漆漈漉漋漏漓演漕漖漠漤漦漩漪漫漭漯漱漳漴漶"
|
| 265 |
+
"漷漹漻漼漾潆潇潋潍潏潖潘潜潞潟潢潦潩潭潮潲潴潵潸潺潼潽潾澂澄澈澉澌澍澎澛澜澡澥澧"
|
| 266 |
+
"澪澭澳澴澶澹澼澽激濂濉濋濑濒濞濠濡濩濮濯瀌瀍瀑瀔瀚瀛瀣瀱瀵瀹瀼灈灌灏灞火灭灯灰灵"
|
| 267 |
+
"灶灸灼灾灿炀炅炆炉炊炌炎炒炔炕炖炘炙炜炝炟炣炫炬炭炮炯炱炳炷炸点炻炼炽烀烁烂烃烈"
|
| 268 |
+
"烊烔烘烙烛烜烝烟烠烤烦烧烨烩烫烬热烯烶烷烹烺烻烽焆焉焊焌焐焓焕焖焗焘焙焚焜焞焦焯"
|
| 269 |
+
"焰焱然煁煃煅煊煋煌煎煓煜煞煟煤煦照煨煮煲煳煴煸煺煽熄熇熊熏熔熘熙熛熜熟熠熥熨熬熵"
|
| 270 |
+
"熹熻燃燊燋燎燏燔燕燚燠燥燧燮燹爆爇爔爚爝爟爨爪爬爰爱爵父爷爸爹爻爽爿牁牂片版牌牍"
|
| 271 |
+
"牒牖牙牚牛牝牟牡牢牤牥牦牧物牮牯牲牵特牺牻牾牿犀犁犄犇犊犋犍犏犒犟犨犬犯犰犴状犷"
|
| 272 |
+
"犸犹狁狂狃狄狈狉狍狎狐狒狗狙狝狞狠狡狨狩独狭狮狯狰狱狲狳狴狷狸狺狻狼猁猃猄猇猊猎"
|
| 273 |
+
"猕猖猗猛猜猝猞猡猢猥猩猪猫猬献猯猰猱猴猷猹猺猾猿獍獐獒獗獠獬獭獯獴獾玃玄率玉王玎"
|
| 274 |
+
"玑玒玓玕玖玘玙玚玛玞玟玠玡玢玤玥玦玩玫玭玮环现玱玲玳玶玷玹玺玻玼玿珀珂珅珇珈珉珊"
|
| 275 |
+
"珋珌珍珏珐珑珒珕珖珙珛珝珞珠珢珣珥珦珧珩珪珫班珰珲珵珷珸珹珺珽琀球琄琅理琇琈琉琊"
|
| 276 |
+
"琎琏琐琔琚琛琟琡琢琤琥琦琨琪琫琬琭琮琯琰琲琳琴琵琶琼瑀瑁瑂瑃瑄瑅瑆瑑瑓瑔瑕瑖瑗瑙"
|
| 277 |
+
"瑚瑛瑜瑝瑞瑟瑢瑧瑨瑬瑭瑰瑱瑳瑶瑷瑾璀璁璃璆璇璈璋璎璐璒璘璜璞璟璠璥璧璨璩璪璬璮璱"
|
| 278 |
+
"璲璺瓀瓒瓖瓘瓜瓞瓠瓢瓣瓤瓦瓮瓯瓴瓶瓷瓻瓿甄甍甏甑甓甗甘甚甜生甡甥甦用甩甪甫甬甭甯"
|
| 279 |
+
"田由甲申电男甸町画甾畀畅畈畋界畎畏畔畖留畚畛畜畤略畦番畬畯畲畴畸畹畿疁疃疆疍疏疐"
|
| 280 |
+
"疑疔疖疗疙疚疝疟疠疡疢疣疤疥疫疬疭疮疯疰疱疲疳疴疵疸疹疼疽疾痂痃痄病症痈痉痊痍痒"
|
| 281 |
+
"痓痔痕痘痛痞痢痣痤痦痧痨痪痫痰痱痴痹痼痿瘀瘁瘃瘅瘆瘊瘌瘐瘕瘗瘘瘙瘛瘟瘠瘢瘤瘥瘦瘩"
|
| 282 |
+
"瘪瘫瘭瘰瘳瘴瘵瘸瘼瘾瘿癀癃癌癍癔癖癗癜癞癣癫癯癸登白百癿皂的皆皇皈皋皎皑皓皕皖皙"
|
| 283 |
+
"皛皞皤皦皭皮皱皲皴皿盂盅盆盈盉益盍盎盏盐监盒盔盖盗盘盛盟盥盦目盯盱盲直盷相盹盼盾"
|
| 284 |
+
"省眄眇眈眉眊看眍眙眚真眠眢眦眨眩眬眭眯眵眶眷眸眺眼着睁睃睄睇睎睐睑睚睛睡睢督睥睦"
|
| 285 |
+
"睨睫睬睹睽睾睿瞀瞄瞅瞋瞌瞍瞎瞑瞒瞟瞠瞢瞥瞧瞩瞪瞫瞬瞭瞰瞳瞵瞻瞽瞿矍矗矛矜矞矢矣知"
|
| 286 |
+
"矧矩矫矬短矮矰石矶矸矻矼矾矿砀码砂砄砆砉砌砍砑砒研砖砗砘砚砜砝砟砠砣砥砧砫砬砭砮"
|
| 287 |
+
"砰破砵砷砸砹砺砻砼砾础硁硅硇硊硌硍硎硐硒硔硕硖硗硙硚硝硪硫硬硭确硼硿碃碇碈碉碌碍"
|
| 288 |
+
"碎碏碑碓碗碘碚碛碜碟碡碣碥碧碨碰碱碲碳碴碶碹碾磁磅磉磊磋磏磐磔磕磙磜磡磨磬磲磴磷"
|
| 289 |
+
"磹磻礁礅礌礓礞礴礵示礼社祀祁祃祆祇祈祉祊祋祎祏祐祓祕祖祗祚祛祜祝神祟祠祢祥祧票祭"
|
| 290 |
+
"祯祲祷祸祺祼祾禀禁禄禅禊禋福禒禔禘禚禛禤禧禳禹禺离禽禾秀私秃秆秉秋种科秒秕秘租���"
|
| 291 |
+
"秤秦秧秩秫秬秭积称秸移秽秾稀稂稃稆程稌稍税稑稔稗稙稚稞稠稣稳稷稹稻稼稽稿穄穆穑穗"
|
| 292 |
+
"穙穜穟穰穴究穷穸穹空穿窀突窃窄窅窈窊窍窎窑窒窕窖窗窘窜窝窟窠窣窥窦窨窬窭窳窸窿立"
|
| 293 |
+
"竑竖竘站竞竟章竣童竦竫竭端竹竺竽竿笃笄笆笈笊笋笏笑笔笕笙笛笞笠笤笥符笨笪笫第笮笯"
|
| 294 |
+
"笱笳笸笺笼笾筀筅筇等筋筌筏筐筑筒答策筘筚筛筜筝筠筢筤筥筦筮筱筲筵筶筷筹筻筼签简箅"
|
| 295 |
+
"箍箐箓箔箕箖算箜管箢箦箧箨箩箪箫箬箭箱箴箸篁篆篇篌篑篓篙篚篝篡篥篦篪篮篯篱篷篼篾"
|
| 296 |
+
"簃簇簉簋簌簏簕簖簝簟簠簧簪簰簸簿籀籁籍籥米籴类籼籽粉粑粒粕粗粘粜粝粞粟粢粤粥粪粮"
|
| 297 |
+
"粱粲粳粹粼粽精粿糁糅糇糈糊糌糍糒糕糖糗糙糜糟糠糨糯糵系紊素索紧紫累絜絮絷綦綮縠縢"
|
| 298 |
+
"縻繁繄繇纂纛纠纡红纣纤纥约级纨纩纪纫纬纭纮纯纰纱纲纳纴纵纶纷纸纹纺纻纼纽纾线绀绁"
|
| 299 |
+
"绂练组绅细织终绉绊绋绌绍绎经绐绑绒结绔绕绖绗绘给绚绛络绝绞统绠绡绢绣绤绥绦继绨绩"
|
| 300 |
+
"绪绫续绮绯绰绱绲绳维绵绶绷绸绹绺绻综绽绾绿缀缁缂缃缄缅缆缇缈缉缊缌缎缐缑缒缓缔缕"
|
| 301 |
+
"编缗缘缙缚缛缜缝缞缟缠缡缢缣缤缥缦缧缨缩缪缫缬缭缮缯缰缱缲缳缴缵缶缸缺罂罄罅罍罐"
|
| 302 |
+
"网罔罕罗罘罚罟罡罢罨罩罪置罱署罴罶罹罽罾羁羊羌美羑羓羔羕羖羚羝羞羟羡群羧羯羰羱羲"
|
| 303 |
+
"羸羹羼羽羿翀翁翂翃翅翈翊翌翎翔翕翘翙翚翛翟翠翡翥翦翩翮翯翰翱翳翷翻翼翾耀老考耄者"
|
| 304 |
+
"耆耇耋而耍耏耐耑耒耔耕耖耗耘耙耜耠耢耤耥耦耧耨耩耪耰耱耳耵耶耷耸耻耽耿聂聃聆聊聋"
|
| 305 |
+
"职聍聒联聘聚聩聪聱聿肃肄肆肇肉肋肌肓肖肘肚肛肝肟肠股肢肤肥肩肪肫肭肮肯肱育肴肷肸"
|
| 306 |
+
"肺肼肽肾肿胀胁胂胃胄胆胈背胍胎胖胗胙胚胛胜胝胞胠胡胣胤胥胧胨胩胪胫胬胭胯胰胱胲胳"
|
| 307 |
+
"胴胶胸胺胼能脂脆脉脊脍脎脏脐脑脒脓脔脖脘脚脞脟脩脬脯脱脲脶脸脾脿腆腈腊腋腌腐腑腒"
|
| 308 |
+
"腓腔腕腘腙腚腠腥腧腨腩腭腮腯腰腱腴腹腺腻腼腽腾腿膀膂膈膊膏膑膘膙膛膜膝膦膨膳膺膻"
|
| 309 |
+
"臀臂臃臆臊臌臑臜臣臧自臬臭至致臻臼臾舀舁舂舄舅舆舌舍舐舒舔舛舜舞舟舠舢舣舥航舫般"
|
| 310 |
+
"舭舯舰舱舲舳舴舵舶舷舸船舻舾艄艅艇艉艋艎艏艘艚艟艨艮良艰色艳艴艺艽艾艿节芃芄芈芊"
|
| 311 |
+
"芋芍芎芏芑芒芗芘芙芜芝芟芠芡芣芤芥芦芨芩芪芫芬芭芮芯芰花芳芴芷芸芹芼芽芾苁苄苇苈"
|
| 312 |
+
"苉苊苋苌苍苎苏苑苒苓苔苕苗苘苛苜苞苟苠苡苣苤若苦苧苫苯英苴苷苹苻苾茀茁茂范茄茅茆"
|
| 313 |
+
"茈茉茋茌茎茏茑茓茔茕茗茚茛茜茝茧茨茫茬茭茯茱茳茴茵茶茸茹茺茼茽荀荁荃荄荆荇草荏荐"
|
| 314 |
+
"荑荒荓荔荖荙荚荛荜荞荟荠荡荣荤荥荦荧荨荩荪荫荬荭荮药荷荸荻荼荽莅莆莉莎莒莓莘莙莛"
|
| 315 |
+
"莜莝莞莠莨莩莪莫莰莱莲莳莴莶获莸莹莺莼莽莿菀菁菂菅菇菉菊菌菍菏菔菖菘菜菝菟菠菡菥"
|
| 316 |
+
"菩菪菰菱菲菹菼菽萁萃萄萆萋萌萍萎萏萑萘萚萜萝萣萤营萦萧萨萩萱萳萸萹萼落葆葎葑葖著"
|
| 317 |
+
"葙葚葛葜葡董葩葫葬葭葰葱葳葴葵葶葸葺蒂蒄蒇蒈蒉蒋蒌蒎蒐蒗蒙蒜蒟蒡蒨蒯蒱蒲蒴蒸蒹蒺"
|
| 318 |
+
"蒻蒽蒿蓁蓂蓄蓇蓉蓊蓍蓏蓐蓑蓓蓖蓝蓟蓠蓢蓣蓥蓦蓬蓰蓼蓿蔀蔃蔈蔊蔌蔑蔓蔗蔚蔟蔡蔫蔬蔷"
|
| 319 |
+
"蔸蔹蔺蔻蔼蔽蕃蕈蕉蕊蕖蕗蕙蕞蕤蕨蕰蕲蕴蕹蕺蕻蕾薁薄薅薇薏薛薜薢薤薨薪薮薯薰薳薷薸"
|
| 320 |
+
"薹薿藁藉藏藐藓藕藜藟藠藤藦藨藩藻藿蘅蘑蘖蘘蘧蘩蘸蘼虎虏虐虑虒虓虔虚虞虢虤虫虬虮虱"
|
| 321 |
+
"虷虸虹虺虻虼虽虾虿蚀蚁蚂蚄蚆蚊蚋蚌蚍蚓蚕蚜蚝蚣蚤蚧蚨蚩蚪蚬蚯蚰蚱蚲蚴蚶蚺蛀蛃蛄蛆"
|
| 322 |
+
"蛇蛉蛊蛋蛎蛏蛐蛑蛔蛘蛙蛛蛞蛟蛤蛩蛭蛮蛰蛱蛲蛳蛴蛸蛹蛾蜀蜂蜃蜇蜈蜉蜊蜍蜎蜐蜒蜓蜕蜗"
|
| 323 |
+
"蜘蜚蜜蜞蜡蜢蜣蜥蜩蜮蜱蜴蜷蜻蜾蜿蝇蝈蝉蝌蝎蝓蝗蝘蝙蝠蝣蝤蝥蝮蝰蝲蝴蝶蝻蝼蝽蝾螂螃"
|
| 324 |
+
"螅螈螋融螗螟螠螣螨螫螬螭螯螱螳螵螺螽蟀蟆蟊蟋蟏蟑蟒蟛蟠蟥蟪蟫蟮蟹蟾蠃蠊蠋蠓蠕蠖蠡"
|
| 325 |
+
"蠢蠲蠹蠼血衃衄衅行衍衎衒衔街衙衠衡衢衣补表衩衫衬衮衰衲衷衽衾衿袁袂袄袅袆袈袋袍袒"
|
| 326 |
+
"袖袗袜袢袤袪被袭袯袱袷袼裁裂装裆裈裉裎裒裔裕裘裙裛裟裢裣裤裥裨裰裱裳裴裸裹裼裾褂"
|
| 327 |
+
"褊褐褒褓褕褙褚褛褟褡褥褪褫褯褰褴褶襁襄襕襚襜襞襟襦襫襻西要覃覆见观觃规觅视觇览觉"
|
| 328 |
+
"觊觋觌觎觏觐觑角觖觚觜觞觟解觥触觫觭觯觱觳觿言訄訇訚訾詈詟詹誉誊誓謇警譬计订讣认"
|
| 329 |
+
"讥讦讧讨让讪讫训议讯记讱讲讳讴讵讶讷许讹论讻讼讽设访诀证诂诃评诅识诇诈诉诊诋诌词"
|
| 330 |
+
"诎诏诐译诒诓诔试诖诗诘诙诚诛诜话诞诟诠诡询诣诤该详诧诨诩诫诬语诮误诰诱诲诳说诵���"
|
| 331 |
+
"诸诹诺读诼诽课诿谀谁谂调谄谅谆谇谈谊谋谌谍谎谏谐谑谒谓谔谕谖谗谙谚谛谜谝谞谟谠谡"
|
| 332 |
+
"谢谣谤谥谦谧谨谩谪谫谬谭谮谯谰谱谲谳谴谵谶谷谼谿豁豆豇豉豌豕豚象豢豨豪豫豮豳豸豹"
|
| 333 |
+
"豺貂貅貆貉貊貌貔貘贝贞负贡财责贤败账货质贩贪贫贬购贮贯贰贱贲贳贴贵贶贷贸费贺贻贼"
|
| 334 |
+
"贽贾贿赀赁赂赃资赅赆赇赈赉赊赋赌赍赎赏赐赑赒赓赔赕赖赗赘赙赚赛赜赝赞赟赠赡赢赣赤"
|
| 335 |
+
"赦赧赪赫赭走赳赴赵赶起趁趄超越趋趑趔趟趣趯趱足趴趵趸趺趼趾趿跂跃跄跆跋跌跎跏跐跑"
|
| 336 |
+
"跖跗跚跛距跞跟跣跤跨跪跬路跱跳践跶跷跸跹跺跻跽踅踉踊踌踏踒踔踝踞踟踢踣踦踩踪踬踮"
|
| 337 |
+
"踯踱踵踶踹踺踽蹀蹁蹂蹄蹅蹇蹈蹉蹊蹋蹐蹑蹒蹙蹚蹜蹢蹦蹩蹬蹭蹯蹰蹲蹴蹶蹼蹽蹾蹿躁躅躇"
|
| 338 |
+
"躏躐躔躜躞身躬躯躲躺车轧轨轩轪轫转轭轮软轰轱轲轳轴轵轶轷轸轹轺轻轼载轾轿辀辁辂较"
|
| 339 |
+
"辄辅辆辇辈辉辊辋辌辍辎辏辐辑辒输辔辕辖辗辘辙辚辛辜辞辟辣辨辩辫辰辱边辽达辿迁迂迄"
|
| 340 |
+
"迅过迈迎运近迓返迕还这进远违连迟迢迤迥迦迨迩迪迫迭迮述迳迷迸迹迺追退送适逃逄逅逆"
|
| 341 |
+
"选逊逋逍透逐逑递途逖逗通逛逝逞速造逡逢逦逭逮逯逴逵逶逸逻逼逾遁遂遄遆遇遍遏遐遑遒"
|
| 342 |
+
"道遗遘遛遢遣遥遨遭遮遴遵遹遽避邀邂邃邈邋邑邓邕邗邘邙邛邝邠邡邢那邦邨邪邬邮邯邰邱"
|
| 343 |
+
"邲邳邴邵邶邸邹邺邻邽邾邿郁郃郄郅郇郈郊郎郏郐郑郓郗郚郛郜郝郡郢郤郦郧部郪郫郭郯郴"
|
| 344 |
+
"郸都郾郿鄀鄂鄃鄄鄅鄌鄑鄗鄘鄙鄚鄜鄞鄠鄢鄣鄫鄯鄱鄹酂酃酅酆酉酊酋酌配酎酏酐酒酗酚酝"
|
| 345 |
+
"酞酡酢酣酤酥酦酩酪酬酮酯酰酱酲酴酵酶酷酸酹酺酽酾酿醅醇醉醋醌醍醐醑醒醚醛醢醨醪醭"
|
| 346 |
+
"醮醯醴醵醺醾采釉释里重野量釐金釜鉴銎銮鋆鋈錾鍪鎏鏊鏖鐾鑫钆钇针钉钊钋钌钍钎钏钐钒"
|
| 347 |
+
"钓钔钕钖钗钘钙钚钛钜钝钞钟钠钡钢钣钤钥钦钧钨钩钪钫钬钭钮钯钰钱钲钳钴钵钷钹钺钻钼"
|
| 348 |
+
"钽钾钿铀铁铂铃铄铅铆铈铉铊铋铌铍铎铏铐铑铒铕铖铗铘铙铚铛铜铝铞铟铠铡铢铣铤铥铧铨"
|
| 349 |
+
"铩铪铫铬铭铮铯铰铱铲铳铴铵银铷铸铹铺铻铼铽链铿销锁锂锃锄锅锆锇锈锉锊锋锌锍锎锏锐"
|
| 350 |
+
"锑锒锓锔锕锖锗锘错锚锛锜锝锞锟锡锢锣锤锥锦锧锨锩锪锫锬锭键锯锰锱锲锳锴锵锶锷锸锹"
|
| 351 |
+
"锺锻锼锽锾锿镀镁镂镃镄镅镆镇镈镉镊镋镌镍镎镏镐镑镒镓镔镕镖镗镘镚镛镜镝镞镠镡镢镣"
|
| 352 |
+
"镤镥镦镧镨镩镪镫镬镭镮镯镰镱镲镳镴镵镶长门闩闪闫闭问闯闰闱闲闳间闵闶闷闸闹闺闻闼"
|
| 353 |
+
"闽闾闿阀阁阂阃阄阅阆阇阈阉阊阋阌阍阎阏阐阑阒阔阕阖阗阘阙阚阜队阡阪阮阱防阳阴阵阶"
|
| 354 |
+
"阻阼阽阿陀陂附际陆陇陈陉陋陌降陎限陑陔陕陛陞陟陡院除陧陨险陪陬陲陴陵陶陷隃隅隆隈"
|
| 355 |
+
"隋隍随隐隔隗隘隙障隧隩隰隳隶隹隺隼隽难雀雁雄雅集雇雉雊雌雍雎雏雒雕雠雨雩雪雯雱雳"
|
| 356 |
+
"零雷雹雾需霁霄霅霆震霈霉霍霎霏霓霖霜霞霨霪霭霰露霸霹霾青靓靖静靛非靠靡面靥革靬靰"
|
| 357 |
+
"靳靴靶靸靺靼靽靿鞁鞅鞋鞍鞑鞒鞔鞘鞠鞡鞣鞧鞨鞫鞬鞭鞮鞯鞲鞳鞴韂韦韧韨韩韪韫韬韭音韵"
|
| 358 |
+
"韶页顶顷顸项顺须顼顽顾顿颀颁颂颃预颅领颇颈颉颊颋颌颍颎颏颐频颓颔颖颗题颙颚颛颜额"
|
| 359 |
+
"颞颟颠颡颢颤颥颦颧风飏飐飑飒飓飔飕飗飘飙飞食飧飨餍餐餮饔饕饥饧饨饩饪饫饬饭饮饯饰"
|
| 360 |
+
"饱饲饳饴饵饶饷饸饹饺饻饼饽饿馁馃馄馅馆馇馈馉馊馋馌馍馏馐馑馒馓馔馕首馗馘香馝馞馥"
|
| 361 |
+
"馧馨马驭驮驯驰驱驲驳驴驵驶驷驸驹驺驻驼驽驾驿骀骁骂骃骄骅骆骇骈骉骊骋验骍骎骏骐骑"
|
| 362 |
+
"骒骓骕骖骗骘骙骚骛骜骝骞骟骠骡骢骣骤骥骦骧骨骰骱骶骷骸骺骼髀髁髂髃髅髋髌髎髑髓高"
|
| 363 |
+
"髡髢髦髫髭髯髹髻髽鬃鬈鬏鬒鬓鬘鬟鬣鬯鬲鬶鬷鬻鬼魁魂魃魄魅魆魇魈魉魋魍魏魑魔鱼鱽鱾"
|
| 364 |
+
"鱿鲀鲁鲂鲃鲅鲆鲇鲈鲉鲊鲋鲌鲍鲎鲏鲐鲑鲒鲔鲕鲖鲗鲘鲙鲚鲛鲜鲝鲞鲟鲠鲡鲢鲣鲤鲥鲦鲧鲨"
|
| 365 |
+
"鲩鲪鲫鲬鲭鲮鲯鲰鲱鲲鲳鲴鲵鲷鲸鲹鲺鲻鲼鲽鲾鲿鳀鳁鳂鳃鳄鳅鳇鳈鳉鳊鳌鳍鳎鳏鳐鳑鳒鳓"
|
| 366 |
+
"鳔鳕鳖鳗鳘鳙鳚鳛鳜鳝鳞鳟鳠鳡鳢鳣鳤鸟鸠鸡鸢鸣鸤鸥鸦鸧鸨鸩鸪鸫鸬鸭鸮鸯鸰鸱鸲鸳鸵鸶"
|
| 367 |
+
"鸷鸸鸹鸺鸻鸼鸽鸾鸿鹀鹁鹂鹃鹄鹅鹆鹇鹈鹉鹊鹋鹌鹍鹎鹏鹐鹑鹒鹔鹕鹖鹗鹘鹙鹚鹛鹜鹝鹞鹟"
|
| 368 |
+
"鹠鹡鹢鹣鹤鹦鹧鹨鹩鹪鹫鹬鹭鹮鹯鹰鹱鹲鹳鹴鹾鹿麀麂麇麈麋麑麒麓麖麝麟麦麸麹麻麽麾黄"
|
| 369 |
+
"黇黉黍黎黏黑黔默黛黜黝黟黠黡黢黥黧黩黪黯黹黻黼黾鼋鼍鼎鼐鼒鼓鼗鼙鼠鼢鼩鼫鼬鼯鼱鼷"
|
| 370 |
+
"鼹鼻鼽鼾齁齇齉齐齑齿龀龁龂龃龄龅龆龇龈龉龊龋龌龙龚龛龟龠龢鿍鿎鿏㑇㑊㕮㘎㙍㙘㙦���"
|
| 371 |
+
"㛚㛹㟃㠇㠓㤘㥄㧐㧑㧟㫰㬊㬎㬚㭎㭕㮾㰀㳇㳘㳚㴔㵐㶲㸆㸌㺄㻬㽏㿠䁖䂮䃅䃎䅟䌹䎃䎖䏝䏡"
|
| 372 |
+
"䏲䐃䓖䓛䓨䓫䓬䗖䗛䗪䗴䜣䝙䢺䢼䣘䥽䦃䲟䲠䲢䴓䴔䴕䴖䴗䴘䴙䶮𠅤𠙶𠳐𡎚𡐓𣗋𣲗𣲘𣸣𤧛𤩽"
|
| 373 |
+
"𤫉𥔲𥕢𥖨𥻗𦈡𦒍𦙶𦝼𦭜𦰡𧿹𨐈𨙸𨚕𨟠𨭉𨱇𨱏𨱑𨱔𨺙𩽾𩾃𩾌𪟝𪣻𪤗𪨰𪨶𪩘𪾢𫄧𫄨𫄷𫄸𫇭𫌀𫍣𫍯"
|
| 374 |
+
"𫍲𫍽𫐄𫐐𫐓𫑡𫓧𫓯𫓶𫓹𫔍𫔎𫔶𫖮𫖯𫖳𫗧𫗴𫘜𫘝𫘦𫘧𫘨𫘪𫘬𫚕𫚖𫚭𫛭𫞩𫟅𫟦𫟹𫟼𫠆𫠊𫠜𫢸𫫇𫭟"
|
| 375 |
+
"𫭢𫭼𫮃𫰛𫵷𫶇𫷷𫸩𬀩𬀪𬂩𬃊𬇕𬇙𬇹𬉼𬊈𬊤𬌗𬍛𬍡𬍤𬒈𬒔𬒗𬕂𬘓𬘘𬘡𬘩𬘫𬘬𬘭𬘯𬙂𬙊𬙋𬜬𬜯𬞟"
|
| 376 |
+
"𬟁𬟽𬣙𬣞𬣡𬣳𬤇𬤊𬤝𬨂𬨎𬩽𬪩𬬩𬬭𬬮𬬱𬬸𬬹𬬻𬬿𬭁𬭊𬭎𬭚𬭛𬭤𬭩𬭬𬭯𬭳𬭶𬭸𬭼𬮱𬮿𬯀𬯎𬱖𬱟"
|
| 377 |
+
"𬳵𬳶𬳽𬳿𬴂𬴃𬴊𬶋𬶍𬶏𬶐𬶟𬶠𬶨𬶭𬶮𬷕𬸘𬸚𬸣𬸦𬸪𬹼𬺈𬺓"
|
| 378 |
+
)
|
| 379 |
+
CN_CHARS_EXT = "吶诶屌囧飚屄"
|
| 380 |
+
|
| 381 |
+
CN_CHARS = CN_CHARS_COMMON + CN_CHARS_EXT
|
| 382 |
+
IN_CH_CHARS = {c: True for c in CN_CHARS}
|
| 383 |
+
|
| 384 |
+
EN_CHARS = string.ascii_letters + string.digits
|
| 385 |
+
IN_EN_CHARS = {c: True for c in EN_CHARS}
|
| 386 |
+
|
| 387 |
+
VALID_CHARS = CN_CHARS + EN_CHARS + " "
|
| 388 |
+
IN_VALID_CHARS = {c: True for c in VALID_CHARS}
|
| 389 |
+
|
| 390 |
+
|
| 391 |
+
# ================================================================================ #
|
| 392 |
+
# basic class
|
| 393 |
+
# ================================================================================ #
|
| 394 |
+
class ChineseChar(object):
|
| 395 |
+
"""
|
| 396 |
+
中文字符
|
| 397 |
+
每个字符对应简体和繁体,
|
| 398 |
+
e.g. 简体 = '负', 繁体 = '負'
|
| 399 |
+
转换时可转换为简体或繁体
|
| 400 |
+
"""
|
| 401 |
+
|
| 402 |
+
def __init__(self, simplified, traditional):
|
| 403 |
+
self.simplified = simplified
|
| 404 |
+
self.traditional = traditional
|
| 405 |
+
# self.__repr__ = self.__str__
|
| 406 |
+
|
| 407 |
+
def __str__(self):
|
| 408 |
+
return self.simplified or self.traditional or None
|
| 409 |
+
|
| 410 |
+
def __repr__(self):
|
| 411 |
+
return self.__str__()
|
| 412 |
+
|
| 413 |
+
|
| 414 |
+
class ChineseNumberUnit(ChineseChar):
|
| 415 |
+
"""
|
| 416 |
+
中文数字/数位字符
|
| 417 |
+
每个字符除繁简体外还有一个额外的大写字符
|
| 418 |
+
e.g. '陆' 和 '陸'
|
| 419 |
+
"""
|
| 420 |
+
|
| 421 |
+
def __init__(self, power, simplified, traditional, big_s, big_t):
|
| 422 |
+
super(ChineseNumberUnit, self).__init__(simplified, traditional)
|
| 423 |
+
self.power = power
|
| 424 |
+
self.big_s = big_s
|
| 425 |
+
self.big_t = big_t
|
| 426 |
+
|
| 427 |
+
def __str__(self):
|
| 428 |
+
return "10^{}".format(self.power)
|
| 429 |
+
|
| 430 |
+
@classmethod
|
| 431 |
+
def create(cls, index, value, numbering_type=NUMBERING_TYPES[1], small_unit=False):
|
| 432 |
+
if small_unit:
|
| 433 |
+
return ChineseNumberUnit(
|
| 434 |
+
power=index + 1, simplified=value[0], traditional=value[1], big_s=value[1], big_t=value[1]
|
| 435 |
+
)
|
| 436 |
+
elif numbering_type == NUMBERING_TYPES[0]:
|
| 437 |
+
return ChineseNumberUnit(
|
| 438 |
+
power=index + 8, simplified=value[0], traditional=value[1], big_s=value[0], big_t=value[1]
|
| 439 |
+
)
|
| 440 |
+
elif numbering_type == NUMBERING_TYPES[1]:
|
| 441 |
+
return ChineseNumberUnit(
|
| 442 |
+
power=(index + 2) * 4, simplified=value[0], traditional=value[1], big_s=value[0], big_t=value[1]
|
| 443 |
+
)
|
| 444 |
+
elif numbering_type == NUMBERING_TYPES[2]:
|
| 445 |
+
return ChineseNumberUnit(
|
| 446 |
+
power=pow(2, index + 3), simplified=value[0], traditional=value[1], big_s=value[0], big_t=value[1]
|
| 447 |
+
)
|
| 448 |
+
else:
|
| 449 |
+
raise ValueError("Counting type should be in {0} ({1} provided).".format(NUMBERING_TYPES, numbering_type))
|
| 450 |
+
|
| 451 |
+
|
| 452 |
+
class ChineseNumberDigit(ChineseChar):
|
| 453 |
+
"""
|
| 454 |
+
中文数字字符
|
| 455 |
+
"""
|
| 456 |
+
|
| 457 |
+
def __init__(self, value, simplified, traditional, big_s, big_t, alt_s=None, alt_t=None):
|
| 458 |
+
super(ChineseNumberDigit, self).__init__(simplified, traditional)
|
| 459 |
+
self.value = value
|
| 460 |
+
self.big_s = big_s
|
| 461 |
+
self.big_t = big_t
|
| 462 |
+
self.alt_s = alt_s
|
| 463 |
+
self.alt_t = alt_t
|
| 464 |
+
|
| 465 |
+
def __str__(self):
|
| 466 |
+
return str(self.value)
|
| 467 |
+
|
| 468 |
+
@classmethod
|
| 469 |
+
def create(cls, i, v):
|
| 470 |
+
return ChineseNumberDigit(i, v[0], v[1], v[2], v[3])
|
| 471 |
+
|
| 472 |
+
|
| 473 |
+
class ChineseMath(ChineseChar):
|
| 474 |
+
"""
|
| 475 |
+
中文数位字符
|
| 476 |
+
"""
|
| 477 |
+
|
| 478 |
+
def __init__(self, simplified, traditional, symbol, expression=None):
|
| 479 |
+
super(ChineseMath, self).__init__(simplified, traditional)
|
| 480 |
+
self.symbol = symbol
|
| 481 |
+
self.expression = expression
|
| 482 |
+
self.big_s = simplified
|
| 483 |
+
self.big_t = traditional
|
| 484 |
+
|
| 485 |
+
|
| 486 |
+
CC, CNU, CND, CM = ChineseChar, ChineseNumberUnit, ChineseNumberDigit, ChineseMath
|
| 487 |
+
|
| 488 |
+
|
| 489 |
+
class NumberSystem(object):
|
| 490 |
+
"""
|
| 491 |
+
中文数字系统
|
| 492 |
+
"""
|
| 493 |
+
|
| 494 |
+
pass
|
| 495 |
+
|
| 496 |
+
|
| 497 |
+
class MathSymbol(object):
|
| 498 |
+
"""
|
| 499 |
+
用于中文数字系统的数学符号 (繁/简体), e.g.
|
| 500 |
+
positive = ['正', '正']
|
| 501 |
+
negative = ['负', '負']
|
| 502 |
+
point = ['点', '點']
|
| 503 |
+
"""
|
| 504 |
+
|
| 505 |
+
def __init__(self, positive, negative, point):
|
| 506 |
+
self.positive = positive
|
| 507 |
+
self.negative = negative
|
| 508 |
+
self.point = point
|
| 509 |
+
|
| 510 |
+
def __iter__(self):
|
| 511 |
+
for v in self.__dict__.values():
|
| 512 |
+
yield v
|
| 513 |
+
|
| 514 |
+
|
| 515 |
+
# class OtherSymbol(object):
|
| 516 |
+
# """
|
| 517 |
+
# 其他符号
|
| 518 |
+
# """
|
| 519 |
+
#
|
| 520 |
+
# def __init__(self, sil):
|
| 521 |
+
# self.sil = sil
|
| 522 |
+
#
|
| 523 |
+
# def __iter__(self):
|
| 524 |
+
# for v in self.__dict__.values():
|
| 525 |
+
# yield v
|
| 526 |
+
|
| 527 |
+
|
| 528 |
+
# ================================================================================ #
|
| 529 |
+
# basic utils
|
| 530 |
+
# ================================================================================ #
|
| 531 |
+
def create_system(numbering_type=NUMBERING_TYPES[1]):
|
| 532 |
+
"""
|
| 533 |
+
根据数字系统类型返回创建相应的数字系统,默认为 mid
|
| 534 |
+
NUMBERING_TYPES = ['low', 'mid', 'high']: 中文数字系统类型
|
| 535 |
+
low: '兆' = '亿' * '十' = $10^{9}$, '京' = '兆' * '十', etc.
|
| 536 |
+
mid: '兆' = '亿' * '万' = $10^{12}$, '京' = '兆' * '万', etc.
|
| 537 |
+
high: '兆' = '亿' * '亿' = $10^{16}$, '京' = '兆' * '兆', etc.
|
| 538 |
+
返回对应的数字系统
|
| 539 |
+
"""
|
| 540 |
+
|
| 541 |
+
# chinese number units of '亿' and larger
|
| 542 |
+
all_larger_units = zip(LARGER_CHINESE_NUMERING_UNITS_SIMPLIFIED, LARGER_CHINESE_NUMERING_UNITS_TRADITIONAL)
|
| 543 |
+
larger_units = [CNU.create(i, v, numbering_type, False) for i, v in enumerate(all_larger_units)]
|
| 544 |
+
# chinese number units of '十, 百, 千, 万'
|
| 545 |
+
all_smaller_units = zip(SMALLER_CHINESE_NUMERING_UNITS_SIMPLIFIED, SMALLER_CHINESE_NUMERING_UNITS_TRADITIONAL)
|
| 546 |
+
smaller_units = [CNU.create(i, v, small_unit=True) for i, v in enumerate(all_smaller_units)]
|
| 547 |
+
# digis
|
| 548 |
+
chinese_digis = zip(CHINESE_DIGIS, CHINESE_DIGIS, BIG_CHINESE_DIGIS_SIMPLIFIED, BIG_CHINESE_DIGIS_TRADITIONAL)
|
| 549 |
+
digits = [CND.create(i, v) for i, v in enumerate(chinese_digis)]
|
| 550 |
+
digits[0].alt_s, digits[0].alt_t = ZERO_ALT, ZERO_ALT
|
| 551 |
+
digits[1].alt_s, digits[1].alt_t = ONE_ALT, ONE_ALT
|
| 552 |
+
digits[2].alt_s, digits[2].alt_t = TWO_ALTS[0], TWO_ALTS[1]
|
| 553 |
+
|
| 554 |
+
# symbols
|
| 555 |
+
positive_cn = CM(POSITIVE[0], POSITIVE[1], "+", lambda x: x)
|
| 556 |
+
negative_cn = CM(NEGATIVE[0], NEGATIVE[1], "-", lambda x: -x)
|
| 557 |
+
point_cn = CM(POINT[0], POINT[1], ".", lambda x, y: float(str(x) + "." + str(y)))
|
| 558 |
+
# sil_cn = CM(SIL[0], SIL[1], '-', lambda x, y: float(str(x) + '-' + str(y)))
|
| 559 |
+
system = NumberSystem()
|
| 560 |
+
system.units = smaller_units + larger_units
|
| 561 |
+
system.digits = digits
|
| 562 |
+
system.math = MathSymbol(positive_cn, negative_cn, point_cn)
|
| 563 |
+
# system.symbols = OtherSymbol(sil_cn)
|
| 564 |
+
return system
|
| 565 |
+
|
| 566 |
+
|
| 567 |
+
def chn2num(chinese_string, numbering_type=NUMBERING_TYPES[1]):
|
| 568 |
+
def get_symbol(char, system):
|
| 569 |
+
for u in system.units:
|
| 570 |
+
if char in [u.traditional, u.simplified, u.big_s, u.big_t]:
|
| 571 |
+
return u
|
| 572 |
+
for d in system.digits:
|
| 573 |
+
if char in [d.traditional, d.simplified, d.big_s, d.big_t, d.alt_s, d.alt_t]:
|
| 574 |
+
return d
|
| 575 |
+
for m in system.math:
|
| 576 |
+
if char in [m.traditional, m.simplified]:
|
| 577 |
+
return m
|
| 578 |
+
|
| 579 |
+
def string2symbols(chinese_string, system):
|
| 580 |
+
int_string, dec_string = chinese_string, ""
|
| 581 |
+
for p in [system.math.point.simplified, system.math.point.traditional]:
|
| 582 |
+
if p in chinese_string:
|
| 583 |
+
int_string, dec_string = chinese_string.split(p)
|
| 584 |
+
break
|
| 585 |
+
return [get_symbol(c, system) for c in int_string], [get_symbol(c, system) for c in dec_string]
|
| 586 |
+
|
| 587 |
+
def correct_symbols(integer_symbols, system):
|
| 588 |
+
"""
|
| 589 |
+
一百八 to 一百八十
|
| 590 |
+
一亿一千三百万 to 一亿 一千万 三百万
|
| 591 |
+
"""
|
| 592 |
+
|
| 593 |
+
if integer_symbols and isinstance(integer_symbols[0], CNU):
|
| 594 |
+
if integer_symbols[0].power == 1:
|
| 595 |
+
integer_symbols = [system.digits[1]] + integer_symbols
|
| 596 |
+
|
| 597 |
+
if len(integer_symbols) > 1:
|
| 598 |
+
if isinstance(integer_symbols[-1], CND) and isinstance(integer_symbols[-2], CNU):
|
| 599 |
+
integer_symbols.append(CNU(integer_symbols[-2].power - 1, None, None, None, None))
|
| 600 |
+
|
| 601 |
+
result = []
|
| 602 |
+
unit_count = 0
|
| 603 |
+
for s in integer_symbols:
|
| 604 |
+
if isinstance(s, CND):
|
| 605 |
+
result.append(s)
|
| 606 |
+
unit_count = 0
|
| 607 |
+
elif isinstance(s, CNU):
|
| 608 |
+
current_unit = CNU(s.power, None, None, None, None)
|
| 609 |
+
unit_count += 1
|
| 610 |
+
|
| 611 |
+
if unit_count == 1:
|
| 612 |
+
result.append(current_unit)
|
| 613 |
+
elif unit_count > 1:
|
| 614 |
+
for i in range(len(result)):
|
| 615 |
+
if isinstance(result[-i - 1], CNU) and result[-i - 1].power < current_unit.power:
|
| 616 |
+
result[-i - 1] = CNU(result[-i - 1].power + current_unit.power, None, None, None, None)
|
| 617 |
+
return result
|
| 618 |
+
|
| 619 |
+
def compute_value(integer_symbols):
|
| 620 |
+
"""
|
| 621 |
+
Compute the value.
|
| 622 |
+
When current unit is larger than previous unit, current unit * all previous units will be used as all previous units.
|
| 623 |
+
e.g. '两千万' = 2000 * 10000 not 2000 + 10000
|
| 624 |
+
"""
|
| 625 |
+
value = [0]
|
| 626 |
+
last_power = 0
|
| 627 |
+
for s in integer_symbols:
|
| 628 |
+
if isinstance(s, CND):
|
| 629 |
+
value[-1] = s.value
|
| 630 |
+
elif isinstance(s, CNU):
|
| 631 |
+
value[-1] *= pow(10, s.power)
|
| 632 |
+
if s.power > last_power:
|
| 633 |
+
value[:-1] = list(map(lambda v: v * pow(10, s.power), value[:-1]))
|
| 634 |
+
last_power = s.power
|
| 635 |
+
value.append(0)
|
| 636 |
+
return sum(value)
|
| 637 |
+
|
| 638 |
+
system = create_system(numbering_type)
|
| 639 |
+
int_part, dec_part = string2symbols(chinese_string, system)
|
| 640 |
+
int_part = correct_symbols(int_part, system)
|
| 641 |
+
int_str = str(compute_value(int_part))
|
| 642 |
+
dec_str = "".join([str(d.value) for d in dec_part])
|
| 643 |
+
if dec_part:
|
| 644 |
+
return "{0}.{1}".format(int_str, dec_str)
|
| 645 |
+
else:
|
| 646 |
+
return int_str
|
| 647 |
+
|
| 648 |
+
|
| 649 |
+
def num2chn(
|
| 650 |
+
number_string,
|
| 651 |
+
numbering_type=NUMBERING_TYPES[1],
|
| 652 |
+
big=False,
|
| 653 |
+
traditional=False,
|
| 654 |
+
alt_zero=False,
|
| 655 |
+
alt_one=False,
|
| 656 |
+
alt_two=True,
|
| 657 |
+
use_zeros=True,
|
| 658 |
+
use_units=True,
|
| 659 |
+
):
|
| 660 |
+
def get_value(value_string, use_zeros=True):
|
| 661 |
+
striped_string = value_string.lstrip("0")
|
| 662 |
+
|
| 663 |
+
# record nothing if all zeros
|
| 664 |
+
if not striped_string:
|
| 665 |
+
return []
|
| 666 |
+
|
| 667 |
+
# record one digits
|
| 668 |
+
elif len(striped_string) == 1:
|
| 669 |
+
if use_zeros and len(value_string) != len(striped_string):
|
| 670 |
+
return [system.digits[0], system.digits[int(striped_string)]]
|
| 671 |
+
else:
|
| 672 |
+
return [system.digits[int(striped_string)]]
|
| 673 |
+
|
| 674 |
+
# recursively record multiple digits
|
| 675 |
+
else:
|
| 676 |
+
result_unit = next(u for u in reversed(system.units) if u.power < len(striped_string))
|
| 677 |
+
result_string = value_string[: -result_unit.power]
|
| 678 |
+
return get_value(result_string) + [result_unit] + get_value(striped_string[-result_unit.power :])
|
| 679 |
+
|
| 680 |
+
system = create_system(numbering_type)
|
| 681 |
+
|
| 682 |
+
int_dec = number_string.split(".")
|
| 683 |
+
if len(int_dec) == 1:
|
| 684 |
+
int_string = int_dec[0]
|
| 685 |
+
dec_string = ""
|
| 686 |
+
elif len(int_dec) == 2:
|
| 687 |
+
int_string = int_dec[0]
|
| 688 |
+
dec_string = int_dec[1]
|
| 689 |
+
else:
|
| 690 |
+
raise ValueError("invalid input num string with more than one dot: {}".format(number_string))
|
| 691 |
+
|
| 692 |
+
if use_units and len(int_string) > 1:
|
| 693 |
+
result_symbols = get_value(int_string)
|
| 694 |
+
else:
|
| 695 |
+
result_symbols = [system.digits[int(c)] for c in int_string]
|
| 696 |
+
dec_symbols = [system.digits[int(c)] for c in dec_string]
|
| 697 |
+
if dec_string:
|
| 698 |
+
result_symbols += [system.math.point] + dec_symbols
|
| 699 |
+
|
| 700 |
+
if alt_two:
|
| 701 |
+
liang = CND(2, system.digits[2].alt_s, system.digits[2].alt_t, system.digits[2].big_s, system.digits[2].big_t)
|
| 702 |
+
for i, v in enumerate(result_symbols):
|
| 703 |
+
if isinstance(v, CND) and v.value == 2:
|
| 704 |
+
next_symbol = result_symbols[i + 1] if i < len(result_symbols) - 1 else None
|
| 705 |
+
previous_symbol = result_symbols[i - 1] if i > 0 else None
|
| 706 |
+
if isinstance(next_symbol, CNU) and isinstance(previous_symbol, (CNU, type(None))):
|
| 707 |
+
if next_symbol.power != 1 and ((previous_symbol is None) or (previous_symbol.power != 1)):
|
| 708 |
+
result_symbols[i] = liang
|
| 709 |
+
|
| 710 |
+
# if big is True, '两' will not be used and `alt_two` has no impact on output
|
| 711 |
+
if big:
|
| 712 |
+
attr_name = "big_"
|
| 713 |
+
if traditional:
|
| 714 |
+
attr_name += "t"
|
| 715 |
+
else:
|
| 716 |
+
attr_name += "s"
|
| 717 |
+
else:
|
| 718 |
+
if traditional:
|
| 719 |
+
attr_name = "traditional"
|
| 720 |
+
else:
|
| 721 |
+
attr_name = "simplified"
|
| 722 |
+
|
| 723 |
+
result = "".join([getattr(s, attr_name) for s in result_symbols])
|
| 724 |
+
|
| 725 |
+
# if not use_zeros:
|
| 726 |
+
# result = result.strip(getattr(system.digits[0], attr_name))
|
| 727 |
+
|
| 728 |
+
if alt_zero:
|
| 729 |
+
result = result.replace(getattr(system.digits[0], attr_name), system.digits[0].alt_s)
|
| 730 |
+
|
| 731 |
+
if alt_one:
|
| 732 |
+
result = result.replace(getattr(system.digits[1], attr_name), system.digits[1].alt_s)
|
| 733 |
+
|
| 734 |
+
for i, p in enumerate(POINT):
|
| 735 |
+
if result.startswith(p):
|
| 736 |
+
return CHINESE_DIGIS[0] + result
|
| 737 |
+
|
| 738 |
+
# ^10, 11, .., 19
|
| 739 |
+
if (
|
| 740 |
+
len(result) >= 2
|
| 741 |
+
and result[1] in [SMALLER_CHINESE_NUMERING_UNITS_SIMPLIFIED[0], SMALLER_CHINESE_NUMERING_UNITS_TRADITIONAL[0]]
|
| 742 |
+
and result[0] in [CHINESE_DIGIS[1], BIG_CHINESE_DIGIS_SIMPLIFIED[1], BIG_CHINESE_DIGIS_TRADITIONAL[1]]
|
| 743 |
+
):
|
| 744 |
+
result = result[1:]
|
| 745 |
+
|
| 746 |
+
return result
|
| 747 |
+
|
| 748 |
+
|
| 749 |
+
# ================================================================================ #
|
| 750 |
+
# different types of rewriters
|
| 751 |
+
# ================================================================================ #
|
| 752 |
+
class Cardinal:
|
| 753 |
+
"""
|
| 754 |
+
CARDINAL类
|
| 755 |
+
"""
|
| 756 |
+
|
| 757 |
+
def __init__(self, cardinal=None, chntext=None):
|
| 758 |
+
self.cardinal = cardinal
|
| 759 |
+
self.chntext = chntext
|
| 760 |
+
|
| 761 |
+
def chntext2cardinal(self):
|
| 762 |
+
return chn2num(self.chntext)
|
| 763 |
+
|
| 764 |
+
def cardinal2chntext(self):
|
| 765 |
+
return num2chn(self.cardinal)
|
| 766 |
+
|
| 767 |
+
|
| 768 |
+
class Digit:
|
| 769 |
+
"""
|
| 770 |
+
DIGIT类
|
| 771 |
+
"""
|
| 772 |
+
|
| 773 |
+
def __init__(self, digit=None, chntext=None):
|
| 774 |
+
self.digit = digit
|
| 775 |
+
self.chntext = chntext
|
| 776 |
+
|
| 777 |
+
# def chntext2digit(self):
|
| 778 |
+
# return chn2num(self.chntext)
|
| 779 |
+
|
| 780 |
+
def digit2chntext(self):
|
| 781 |
+
return num2chn(self.digit, alt_two=False, use_units=False)
|
| 782 |
+
|
| 783 |
+
|
| 784 |
+
class TelePhone:
|
| 785 |
+
"""
|
| 786 |
+
TELEPHONE类
|
| 787 |
+
"""
|
| 788 |
+
|
| 789 |
+
def __init__(self, telephone=None, raw_chntext=None, chntext=None):
|
| 790 |
+
self.telephone = telephone
|
| 791 |
+
self.raw_chntext = raw_chntext
|
| 792 |
+
self.chntext = chntext
|
| 793 |
+
|
| 794 |
+
# def chntext2telephone(self):
|
| 795 |
+
# sil_parts = self.raw_chntext.split('<SIL>')
|
| 796 |
+
# self.telephone = '-'.join([
|
| 797 |
+
# str(chn2num(p)) for p in sil_parts
|
| 798 |
+
# ])
|
| 799 |
+
# return self.telephone
|
| 800 |
+
|
| 801 |
+
def telephone2chntext(self, fixed=False):
|
| 802 |
+
if fixed:
|
| 803 |
+
sil_parts = self.telephone.split("-")
|
| 804 |
+
self.raw_chntext = "<SIL>".join([num2chn(part, alt_two=False, use_units=False) for part in sil_parts])
|
| 805 |
+
self.chntext = self.raw_chntext.replace("<SIL>", "")
|
| 806 |
+
else:
|
| 807 |
+
sp_parts = self.telephone.strip("+").split()
|
| 808 |
+
self.raw_chntext = "<SP>".join([num2chn(part, alt_two=False, use_units=False) for part in sp_parts])
|
| 809 |
+
self.chntext = self.raw_chntext.replace("<SP>", "")
|
| 810 |
+
return self.chntext
|
| 811 |
+
|
| 812 |
+
|
| 813 |
+
class Fraction:
|
| 814 |
+
"""
|
| 815 |
+
FRACTION类
|
| 816 |
+
"""
|
| 817 |
+
|
| 818 |
+
def __init__(self, fraction=None, chntext=None):
|
| 819 |
+
self.fraction = fraction
|
| 820 |
+
self.chntext = chntext
|
| 821 |
+
|
| 822 |
+
def chntext2fraction(self):
|
| 823 |
+
denominator, numerator = self.chntext.split("分之")
|
| 824 |
+
return chn2num(numerator) + "/" + chn2num(denominator)
|
| 825 |
+
|
| 826 |
+
def fraction2chntext(self):
|
| 827 |
+
numerator, denominator = self.fraction.split("/")
|
| 828 |
+
return num2chn(denominator) + "分之" + num2chn(numerator)
|
| 829 |
+
|
| 830 |
+
|
| 831 |
+
class Date:
|
| 832 |
+
"""
|
| 833 |
+
DATE类
|
| 834 |
+
"""
|
| 835 |
+
|
| 836 |
+
def __init__(self, date=None, chntext=None):
|
| 837 |
+
self.date = date
|
| 838 |
+
self.chntext = chntext
|
| 839 |
+
|
| 840 |
+
# def chntext2date(self):
|
| 841 |
+
# chntext = self.chntext
|
| 842 |
+
# try:
|
| 843 |
+
# year, other = chntext.strip().split('年', maxsplit=1)
|
| 844 |
+
# year = Digit(chntext=year).digit2chntext() + '年'
|
| 845 |
+
# except ValueError:
|
| 846 |
+
# other = chntext
|
| 847 |
+
# year = ''
|
| 848 |
+
# if other:
|
| 849 |
+
# try:
|
| 850 |
+
# month, day = other.strip().split('月', maxsplit=1)
|
| 851 |
+
# month = Cardinal(chntext=month).chntext2cardinal() + '月'
|
| 852 |
+
# except ValueError:
|
| 853 |
+
# day = chntext
|
| 854 |
+
# month = ''
|
| 855 |
+
# if day:
|
| 856 |
+
# day = Cardinal(chntext=day[:-1]).chntext2cardinal() + day[-1]
|
| 857 |
+
# else:
|
| 858 |
+
# month = ''
|
| 859 |
+
# day = ''
|
| 860 |
+
# date = year + month + day
|
| 861 |
+
# self.date = date
|
| 862 |
+
# return self.date
|
| 863 |
+
|
| 864 |
+
def date2chntext(self):
|
| 865 |
+
date = self.date
|
| 866 |
+
try:
|
| 867 |
+
year, other = date.strip().split("年", 1)
|
| 868 |
+
year = Digit(digit=year).digit2chntext() + "年"
|
| 869 |
+
except ValueError:
|
| 870 |
+
other = date
|
| 871 |
+
year = ""
|
| 872 |
+
if other:
|
| 873 |
+
try:
|
| 874 |
+
month, day = other.strip().split("月", 1)
|
| 875 |
+
month = Cardinal(cardinal=month).cardinal2chntext() + "月"
|
| 876 |
+
except ValueError:
|
| 877 |
+
day = date
|
| 878 |
+
month = ""
|
| 879 |
+
if day:
|
| 880 |
+
day = Cardinal(cardinal=day[:-1]).cardinal2chntext() + day[-1]
|
| 881 |
+
else:
|
| 882 |
+
month = ""
|
| 883 |
+
day = ""
|
| 884 |
+
chntext = year + month + day
|
| 885 |
+
self.chntext = chntext
|
| 886 |
+
return self.chntext
|
| 887 |
+
|
| 888 |
+
|
| 889 |
+
class Money:
|
| 890 |
+
"""
|
| 891 |
+
MONEY类
|
| 892 |
+
"""
|
| 893 |
+
|
| 894 |
+
def __init__(self, money=None, chntext=None):
|
| 895 |
+
self.money = money
|
| 896 |
+
self.chntext = chntext
|
| 897 |
+
|
| 898 |
+
# def chntext2money(self):
|
| 899 |
+
# return self.money
|
| 900 |
+
|
| 901 |
+
def money2chntext(self):
|
| 902 |
+
money = self.money
|
| 903 |
+
pattern = re.compile(r"(\d+(\.\d+)?)")
|
| 904 |
+
matchers = pattern.findall(money)
|
| 905 |
+
if matchers:
|
| 906 |
+
for matcher in matchers:
|
| 907 |
+
money = money.replace(matcher[0], Cardinal(cardinal=matcher[0]).cardinal2chntext())
|
| 908 |
+
self.chntext = money
|
| 909 |
+
return self.chntext
|
| 910 |
+
|
| 911 |
+
|
| 912 |
+
class Percentage:
|
| 913 |
+
"""
|
| 914 |
+
PERCENTAGE类
|
| 915 |
+
"""
|
| 916 |
+
|
| 917 |
+
def __init__(self, percentage=None, chntext=None):
|
| 918 |
+
self.percentage = percentage
|
| 919 |
+
self.chntext = chntext
|
| 920 |
+
|
| 921 |
+
def chntext2percentage(self):
|
| 922 |
+
return chn2num(self.chntext.strip().strip("百分之")) + "%"
|
| 923 |
+
|
| 924 |
+
def percentage2chntext(self):
|
| 925 |
+
return "百分之" + num2chn(self.percentage.strip().strip("%"))
|
| 926 |
+
|
| 927 |
+
|
| 928 |
+
def normalize_nsw(raw_text):
|
| 929 |
+
text = "^" + raw_text + "$"
|
| 930 |
+
|
| 931 |
+
# 规范化日期
|
| 932 |
+
pattern = re.compile(r"\D+((([089]\d|(19|20)\d{2})年)?(\d{1,2}月(\d{1,2}[日号])?)?)")
|
| 933 |
+
matchers = pattern.findall(text)
|
| 934 |
+
if matchers:
|
| 935 |
+
# print('date')
|
| 936 |
+
for matcher in matchers:
|
| 937 |
+
text = text.replace(matcher[0], Date(date=matcher[0]).date2chntext(), 1)
|
| 938 |
+
|
| 939 |
+
# 规范化金钱
|
| 940 |
+
pattern = re.compile(r"\D+((\d+(\.\d+)?)[多余几]?" + CURRENCY_UNITS + r"(\d" + CURRENCY_UNITS + r"?)?)")
|
| 941 |
+
matchers = pattern.findall(text)
|
| 942 |
+
if matchers:
|
| 943 |
+
# print('money')
|
| 944 |
+
for matcher in matchers:
|
| 945 |
+
text = text.replace(matcher[0], Money(money=matcher[0]).money2chntext(), 1)
|
| 946 |
+
|
| 947 |
+
# 规范化固话/手机号码
|
| 948 |
+
# 手机
|
| 949 |
+
# http://www.jihaoba.com/news/show/13680
|
| 950 |
+
# 移动:139、138、137、136、135、134、159、158、157、150、151、152、188、187、182、183、184、178、198
|
| 951 |
+
# 联通:130、131、132、156、155、186、185、176
|
| 952 |
+
# 电信:133、153、189、180、181、177
|
| 953 |
+
pattern = re.compile(r"\D((\+?86 ?)?1([38]\d|5[0-35-9]|7[678]|9[89])\d{8})\D")
|
| 954 |
+
matchers = pattern.findall(text)
|
| 955 |
+
if matchers:
|
| 956 |
+
# print('telephone')
|
| 957 |
+
for matcher in matchers:
|
| 958 |
+
text = text.replace(matcher[0], TelePhone(telephone=matcher[0]).telephone2chntext(), 1)
|
| 959 |
+
# 固话
|
| 960 |
+
pattern = re.compile(r"\D((0(10|2[1-3]|[3-9]\d{2})-?)?[1-9]\d{6,7})\D")
|
| 961 |
+
matchers = pattern.findall(text)
|
| 962 |
+
if matchers:
|
| 963 |
+
# print('fixed telephone')
|
| 964 |
+
for matcher in matchers:
|
| 965 |
+
text = text.replace(matcher[0], TelePhone(telephone=matcher[0]).telephone2chntext(fixed=True), 1)
|
| 966 |
+
|
| 967 |
+
# 规范化分数
|
| 968 |
+
pattern = re.compile(r"(\d+/\d+)")
|
| 969 |
+
matchers = pattern.findall(text)
|
| 970 |
+
if matchers:
|
| 971 |
+
# print('fraction')
|
| 972 |
+
for matcher in matchers:
|
| 973 |
+
text = text.replace(matcher, Fraction(fraction=matcher).fraction2chntext(), 1)
|
| 974 |
+
|
| 975 |
+
# 规范化百分数
|
| 976 |
+
text = text.replace("%", "%")
|
| 977 |
+
pattern = re.compile(r"(\d+(\.\d+)?%)")
|
| 978 |
+
matchers = pattern.findall(text)
|
| 979 |
+
if matchers:
|
| 980 |
+
# print('percentage')
|
| 981 |
+
for matcher in matchers:
|
| 982 |
+
text = text.replace(matcher[0], Percentage(percentage=matcher[0]).percentage2chntext(), 1)
|
| 983 |
+
|
| 984 |
+
# 规范化纯数+量词
|
| 985 |
+
pattern = re.compile(r"(\d+(\.\d+)?)[多余几]?" + COM_QUANTIFIERS)
|
| 986 |
+
matchers = pattern.findall(text)
|
| 987 |
+
if matchers:
|
| 988 |
+
# print('cardinal+quantifier')
|
| 989 |
+
for matcher in matchers:
|
| 990 |
+
text = text.replace(matcher[0], Cardinal(cardinal=matcher[0]).cardinal2chntext(), 1)
|
| 991 |
+
|
| 992 |
+
# 规范化数字编号
|
| 993 |
+
pattern = re.compile(r"(\d{4,32})")
|
| 994 |
+
matchers = pattern.findall(text)
|
| 995 |
+
if matchers:
|
| 996 |
+
# print('digit')
|
| 997 |
+
for matcher in matchers:
|
| 998 |
+
text = text.replace(matcher, Digit(digit=matcher).digit2chntext(), 1)
|
| 999 |
+
|
| 1000 |
+
# 规范化纯数
|
| 1001 |
+
pattern = re.compile(r"(\d+(\.\d+)?)")
|
| 1002 |
+
matchers = pattern.findall(text)
|
| 1003 |
+
if matchers:
|
| 1004 |
+
# print('cardinal')
|
| 1005 |
+
for matcher in matchers:
|
| 1006 |
+
text = text.replace(matcher[0], Cardinal(cardinal=matcher[0]).cardinal2chntext(), 1)
|
| 1007 |
+
|
| 1008 |
+
# restore P2P, O2O, B2C, B2B etc
|
| 1009 |
+
pattern = re.compile(r"(([a-zA-Z]+)二([a-zA-Z]+))")
|
| 1010 |
+
matchers = pattern.findall(text)
|
| 1011 |
+
if matchers:
|
| 1012 |
+
# print('particular')
|
| 1013 |
+
for matcher in matchers:
|
| 1014 |
+
text = text.replace(matcher[0], matcher[1] + "2" + matcher[2], 1)
|
| 1015 |
+
|
| 1016 |
+
return text.lstrip("^").rstrip("$")
|
| 1017 |
+
|
| 1018 |
+
|
| 1019 |
+
def remove_erhua(text):
|
| 1020 |
+
"""
|
| 1021 |
+
去除儿化音词中的儿:
|
| 1022 |
+
他女儿在那边儿 -> 他女儿在那边
|
| 1023 |
+
"""
|
| 1024 |
+
|
| 1025 |
+
new_str = ""
|
| 1026 |
+
while re.search("儿", text):
|
| 1027 |
+
a = re.search("儿", text).span()
|
| 1028 |
+
remove_er_flag = 0
|
| 1029 |
+
|
| 1030 |
+
if ER_WHITELIST_PATTERN.search(text):
|
| 1031 |
+
b = ER_WHITELIST_PATTERN.search(text).span()
|
| 1032 |
+
if b[0] <= a[0]:
|
| 1033 |
+
remove_er_flag = 1
|
| 1034 |
+
|
| 1035 |
+
if remove_er_flag == 0:
|
| 1036 |
+
new_str = new_str + text[0 : a[0]]
|
| 1037 |
+
text = text[a[1] :]
|
| 1038 |
+
else:
|
| 1039 |
+
new_str = new_str + text[0 : b[1]]
|
| 1040 |
+
text = text[b[1] :]
|
| 1041 |
+
|
| 1042 |
+
text = new_str + text
|
| 1043 |
+
return text
|
| 1044 |
+
|
| 1045 |
+
|
| 1046 |
+
def remove_space(text):
|
| 1047 |
+
tokens = text.split()
|
| 1048 |
+
new = []
|
| 1049 |
+
for k, t in enumerate(tokens):
|
| 1050 |
+
if k != 0:
|
| 1051 |
+
if IN_EN_CHARS.get(tokens[k - 1][-1]) and IN_EN_CHARS.get(t[0]):
|
| 1052 |
+
new.append(" ")
|
| 1053 |
+
new.append(t)
|
| 1054 |
+
return "".join(new)
|
| 1055 |
+
|
| 1056 |
+
|
| 1057 |
+
class TextNorm:
|
| 1058 |
+
def __init__(
|
| 1059 |
+
self,
|
| 1060 |
+
to_banjiao: bool = False,
|
| 1061 |
+
to_upper: bool = False,
|
| 1062 |
+
to_lower: bool = False,
|
| 1063 |
+
remove_fillers: bool = False,
|
| 1064 |
+
remove_erhua: bool = False,
|
| 1065 |
+
check_chars: bool = False,
|
| 1066 |
+
remove_space: bool = False,
|
| 1067 |
+
cc_mode: str = "",
|
| 1068 |
+
):
|
| 1069 |
+
self.to_banjiao = to_banjiao
|
| 1070 |
+
self.to_upper = to_upper
|
| 1071 |
+
self.to_lower = to_lower
|
| 1072 |
+
self.remove_fillers = remove_fillers
|
| 1073 |
+
self.remove_erhua = remove_erhua
|
| 1074 |
+
self.check_chars = check_chars
|
| 1075 |
+
self.remove_space = remove_space
|
| 1076 |
+
|
| 1077 |
+
self.cc = None
|
| 1078 |
+
if cc_mode:
|
| 1079 |
+
from opencc import OpenCC # Open Chinese Convert: pip install opencc
|
| 1080 |
+
|
| 1081 |
+
self.cc = OpenCC(cc_mode)
|
| 1082 |
+
|
| 1083 |
+
def __call__(self, text):
|
| 1084 |
+
if self.cc:
|
| 1085 |
+
text = self.cc.convert(text)
|
| 1086 |
+
|
| 1087 |
+
if self.to_banjiao:
|
| 1088 |
+
text = text.translate(QJ2BJ_TRANSFORM)
|
| 1089 |
+
|
| 1090 |
+
if self.to_upper:
|
| 1091 |
+
text = text.upper()
|
| 1092 |
+
|
| 1093 |
+
if self.to_lower:
|
| 1094 |
+
text = text.lower()
|
| 1095 |
+
|
| 1096 |
+
if self.remove_fillers:
|
| 1097 |
+
for c in FILLER_CHARS:
|
| 1098 |
+
text = text.replace(c, "")
|
| 1099 |
+
|
| 1100 |
+
if self.remove_erhua:
|
| 1101 |
+
text = remove_erhua(text)
|
| 1102 |
+
|
| 1103 |
+
text = normalize_nsw(text)
|
| 1104 |
+
|
| 1105 |
+
text = text.translate(PUNCS_TRANSFORM)
|
| 1106 |
+
|
| 1107 |
+
if self.check_chars:
|
| 1108 |
+
for c in text:
|
| 1109 |
+
if not IN_VALID_CHARS.get(c):
|
| 1110 |
+
print(f"WARNING: illegal char {c} in: {text}", file=sys.stderr)
|
| 1111 |
+
return ""
|
| 1112 |
+
|
| 1113 |
+
if self.remove_space:
|
| 1114 |
+
text = remove_space(text)
|
| 1115 |
+
|
| 1116 |
+
return text
|
| 1117 |
+
|
| 1118 |
+
|
| 1119 |
+
if __name__ == "__main__":
|
| 1120 |
+
p = argparse.ArgumentParser()
|
| 1121 |
+
|
| 1122 |
+
# normalizer options
|
| 1123 |
+
p.add_argument("--to_banjiao", action="store_true", help="convert quanjiao chars to banjiao")
|
| 1124 |
+
p.add_argument("--to_upper", action="store_true", help="convert to upper case")
|
| 1125 |
+
p.add_argument("--to_lower", action="store_true", help="convert to lower case")
|
| 1126 |
+
p.add_argument("--remove_fillers", action="store_true", help='remove filler chars such as "呃, 啊"')
|
| 1127 |
+
p.add_argument("--remove_erhua", action="store_true", help='remove erhua chars such as "他女儿在那边儿 -> 他女儿在那边"')
|
| 1128 |
+
p.add_argument("--check_chars", action="store_true", help="skip sentences containing illegal chars")
|
| 1129 |
+
p.add_argument("--remove_space", action="store_true", help="remove whitespace")
|
| 1130 |
+
p.add_argument(
|
| 1131 |
+
"--cc_mode", choices=["", "t2s", "s2t"], default="", help="convert between traditional to simplified"
|
| 1132 |
+
)
|
| 1133 |
+
|
| 1134 |
+
# I/O options
|
| 1135 |
+
p.add_argument("--log_interval", type=int, default=10000, help="log interval in number of processed lines")
|
| 1136 |
+
p.add_argument("--has_key", action="store_true", help="will be deprecated, set --format ark instead")
|
| 1137 |
+
p.add_argument("--format", type=str, choices=["txt", "ark", "tsv"], default="txt", help="input format")
|
| 1138 |
+
p.add_argument("ifile", help="input filename, assume utf-8 encoding")
|
| 1139 |
+
p.add_argument("ofile", help="output filename")
|
| 1140 |
+
|
| 1141 |
+
args = p.parse_args()
|
| 1142 |
+
|
| 1143 |
+
if args.has_key:
|
| 1144 |
+
args.format = "ark"
|
| 1145 |
+
|
| 1146 |
+
normalizer = TextNorm(
|
| 1147 |
+
to_banjiao=args.to_banjiao,
|
| 1148 |
+
to_upper=args.to_upper,
|
| 1149 |
+
to_lower=args.to_lower,
|
| 1150 |
+
remove_fillers=args.remove_fillers,
|
| 1151 |
+
remove_erhua=args.remove_erhua,
|
| 1152 |
+
check_chars=args.check_chars,
|
| 1153 |
+
remove_space=args.remove_space,
|
| 1154 |
+
cc_mode=args.cc_mode,
|
| 1155 |
+
)
|
| 1156 |
+
|
| 1157 |
+
normalizer = TextNorm(
|
| 1158 |
+
to_banjiao=args.to_banjiao,
|
| 1159 |
+
to_upper=args.to_upper,
|
| 1160 |
+
to_lower=args.to_lower,
|
| 1161 |
+
remove_fillers=args.remove_fillers,
|
| 1162 |
+
remove_erhua=args.remove_erhua,
|
| 1163 |
+
check_chars=args.check_chars,
|
| 1164 |
+
remove_space=args.remove_space,
|
| 1165 |
+
cc_mode=args.cc_mode,
|
| 1166 |
+
)
|
| 1167 |
+
|
| 1168 |
+
ndone = 0
|
| 1169 |
+
with open(args.ifile, "r", encoding="utf8") as istream, open(args.ofile, "w+", encoding="utf8") as ostream:
|
| 1170 |
+
if args.format == "tsv":
|
| 1171 |
+
reader = csv.DictReader(istream, delimiter="\t")
|
| 1172 |
+
assert "TEXT" in reader.fieldnames
|
| 1173 |
+
print("\t".join(reader.fieldnames), file=ostream)
|
| 1174 |
+
|
| 1175 |
+
for item in reader:
|
| 1176 |
+
text = item["TEXT"]
|
| 1177 |
+
|
| 1178 |
+
if text:
|
| 1179 |
+
text = normalizer(text)
|
| 1180 |
+
|
| 1181 |
+
if text:
|
| 1182 |
+
item["TEXT"] = text
|
| 1183 |
+
print("\t".join([item[f] for f in reader.fieldnames]), file=ostream)
|
| 1184 |
+
|
| 1185 |
+
ndone += 1
|
| 1186 |
+
if ndone % args.log_interval == 0:
|
| 1187 |
+
print(f"text norm: {ndone} lines done.", file=sys.stderr, flush=True)
|
| 1188 |
+
else:
|
| 1189 |
+
for l in istream:
|
| 1190 |
+
key, text = "", ""
|
| 1191 |
+
if args.format == "ark": # KALDI archive, line format: "key text"
|
| 1192 |
+
cols = l.strip().split(maxsplit=1)
|
| 1193 |
+
key, text = cols[0], cols[1] if len(cols) == 2 else ""
|
| 1194 |
+
else:
|
| 1195 |
+
text = l.strip()
|
| 1196 |
+
|
| 1197 |
+
if text:
|
| 1198 |
+
text = normalizer(text)
|
| 1199 |
+
|
| 1200 |
+
if text:
|
| 1201 |
+
if args.format == "ark":
|
| 1202 |
+
print(key + "\t" + text, file=ostream)
|
| 1203 |
+
else:
|
| 1204 |
+
print(text, file=ostream)
|
| 1205 |
+
|
| 1206 |
+
ndone += 1
|
| 1207 |
+
if ndone % args.log_interval == 0:
|
| 1208 |
+
print(f"text norm: {ndone} lines done.", file=sys.stderr, flush=True)
|
| 1209 |
+
print(f"text norm: {ndone} lines done in total.", file=sys.stderr, flush=True)
|
songgen/modeling_songgen_dual_track.py
ADDED
|
@@ -0,0 +1,623 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import copy
|
| 2 |
+
import torch
|
| 3 |
+
import torch.nn as nn
|
| 4 |
+
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
|
| 5 |
+
from transformers.cache_utils import (
|
| 6 |
+
Cache,
|
| 7 |
+
DynamicCache,
|
| 8 |
+
EncoderDecoderCache,
|
| 9 |
+
SlidingWindowCache,
|
| 10 |
+
StaticCache,
|
| 11 |
+
)
|
| 12 |
+
from transformers.generation.configuration_utils import GenerationConfig, GenerationMode
|
| 13 |
+
from transformers.generation.logits_process import LogitsProcessorList
|
| 14 |
+
from transformers.generation.stopping_criteria import StoppingCriteriaList
|
| 15 |
+
from transformers.modeling_outputs import (
|
| 16 |
+
BaseModelOutput,
|
| 17 |
+
BaseModelOutputWithPastAndCrossAttentions,
|
| 18 |
+
ModelOutput,
|
| 19 |
+
|
| 20 |
+
)
|
| 21 |
+
from transformers.utils import (
|
| 22 |
+
add_start_docstrings,
|
| 23 |
+
add_start_docstrings_to_model_forward,
|
| 24 |
+
logging,
|
| 25 |
+
replace_return_docstrings,
|
| 26 |
+
)
|
| 27 |
+
from .modeling_songgen_mixed import (
|
| 28 |
+
SongGenMixedForConditionalGeneration,
|
| 29 |
+
build_delay_pattern_mask
|
| 30 |
+
)
|
| 31 |
+
from .configuration_songgen import SongGenConfig, SongGenDecoderConfig
|
| 32 |
+
from .logits_processors import SongGenLogitsProcessor
|
| 33 |
+
|
| 34 |
+
logger = logging.get_logger(__name__)
|
| 35 |
+
|
| 36 |
+
def split_combined_track_input_ids(combined_input_ids: torch.LongTensor, pattern: str, num_codebooks:int):
|
| 37 |
+
"""
|
| 38 |
+
combined_input_ids has not applied the codebook-delayed pattern.
|
| 39 |
+
Splits the combined input_ids back into acc_input_ids and vocal_input_ids based on the specified pattern.
|
| 40 |
+
combined_input_ids: Tensor of shape (bsz, 2 * num_codebooks, seq_len) or (bsz, num_codebooks, 2*seq_len) depending on pattern
|
| 41 |
+
pattern: The pattern used for combining the input_ids.
|
| 42 |
+
"""
|
| 43 |
+
assert len(combined_input_ids.shape) == 3, f"split_combined_track_input_ids(): combined_input_ids.shape:{combined_input_ids.shape}"
|
| 44 |
+
assert pattern in ['parallel_std', 'parallel_A_V', 'parallel_V_A', 'interleaving_A_V', 'interleaving_V_A'], f"Invalid pattern. Must be one of ['parallel_std', 'parallel_A_V', 'parallel_V_A', 'interleaving_A_V', 'interleaving_V_A'], but got: {pattern}"
|
| 45 |
+
if pattern == 'interleaving_A_V':
|
| 46 |
+
acc_input_ids = combined_input_ids[:, :, ::2] # Take the elements at even indices (acc_input_ids)
|
| 47 |
+
vocal_input_ids = combined_input_ids[:, :, 1::2] # Take the elements at odd indices (vocal_input_ids)
|
| 48 |
+
|
| 49 |
+
elif pattern == 'interleaving_V_A':
|
| 50 |
+
vocal_input_ids = combined_input_ids[:, :, ::2] # Take the elements at even indices (vocal_input_ids)
|
| 51 |
+
acc_input_ids = combined_input_ids[:, :, 1::2] # Take the elements at odd indices (acc_input_ids)
|
| 52 |
+
|
| 53 |
+
elif pattern.startswith('parallel'):
|
| 54 |
+
acc_input_ids = combined_input_ids[:, :num_codebooks, :]
|
| 55 |
+
vocal_input_ids = combined_input_ids[:, num_codebooks:, :]
|
| 56 |
+
|
| 57 |
+
else:
|
| 58 |
+
raise ValueError(f"Unexpected track_conbination pattern: {pattern}")
|
| 59 |
+
|
| 60 |
+
return acc_input_ids, vocal_input_ids
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def combine_track_input_ids(acc_input_ids: torch.LongTensor, vocal_input_ids: torch.LongTensor, pattern: str, bos_token_id: int, pad_token_id: int, num_codebooks:int):
|
| 65 |
+
"""
|
| 66 |
+
Combines the input_ids (acc_input_ids and vocal_input_ids) based on the specified pattern.
|
| 67 |
+
Both inpu_ids should be delayed. shape:(bsz, num_codebooks, seq_len)
|
| 68 |
+
# - [a, b, E, E, E, E]
|
| 69 |
+
# - [B, c, d, E, E, E]
|
| 70 |
+
# - [B, B, e, f, E, E]
|
| 71 |
+
# - [B, B, B, g, h, E]
|
| 72 |
+
pattern options:
|
| 73 |
+
['parallel_std', 'parallel_A_V', 'parallel_V_A', 'interleaving_A_V', 'interleaving_V_A']
|
| 74 |
+
"""
|
| 75 |
+
assert acc_input_ids.shape == vocal_input_ids.shape, f'acc_input_ids.shape: {acc_input_ids.shape}, vocal_input_ids.shape: {vocal_input_ids.shape}'
|
| 76 |
+
acc_input_ids = acc_input_ids.reshape(-1, num_codebooks, acc_input_ids.shape[-1]) #(bsz, num_codebook, seq_len)
|
| 77 |
+
vocal_input_ids = vocal_input_ids.reshape(-1, num_codebooks, vocal_input_ids.shape[-1])
|
| 78 |
+
bsz, n, seq_len = acc_input_ids.shape
|
| 79 |
+
if pattern == 'interleaving_A_V':
|
| 80 |
+
combined_input_ids = torch.stack([acc_input_ids, vocal_input_ids], dim=-1) # (bsz, num_codebooks, seq_len, 2)
|
| 81 |
+
combined_input_ids = combined_input_ids.reshape(bsz, n, 2 * seq_len)
|
| 82 |
+
|
| 83 |
+
elif pattern == 'interleaving_V_A':
|
| 84 |
+
combined_input_ids = torch.stack([vocal_input_ids, acc_input_ids], dim=-1) # (bsz, num_codebooks, seq_len, 2)
|
| 85 |
+
combined_input_ids = combined_input_ids.reshape(bsz, n, 2 * seq_len)
|
| 86 |
+
|
| 87 |
+
elif pattern == 'parallel_std':
|
| 88 |
+
combined_input_ids = torch.cat([acc_input_ids, vocal_input_ids], dim=1) #(bsz, 2*num_codebooks, seq_len)
|
| 89 |
+
|
| 90 |
+
elif pattern == 'parallel_A_V' or pattern == 'parallel_V_A' :
|
| 91 |
+
bos_ids = torch.ones((bsz, num_codebooks, 1), dtype=acc_input_ids.dtype, device=acc_input_ids.device) * bos_token_id
|
| 92 |
+
pad_ids = torch.ones((bsz, num_codebooks, 1), dtype=acc_input_ids.dtype, device=acc_input_ids.device) * pad_token_id
|
| 93 |
+
|
| 94 |
+
if pattern == 'parallel_A_V':
|
| 95 |
+
acc_input_ids = torch.cat([acc_input_ids, pad_ids], dim=-1)
|
| 96 |
+
vocal_input_ids = torch.cat([bos_ids, vocal_input_ids], dim=-1)
|
| 97 |
+
combined_input_ids = torch.cat([acc_input_ids, vocal_input_ids], dim=1) # #(bsz, 2*num_codebooks, seq_len+1)
|
| 98 |
+
|
| 99 |
+
elif pattern == 'parallel_V_A':
|
| 100 |
+
acc_input_ids = torch.cat([bos_ids, acc_input_ids], dim=-1)
|
| 101 |
+
vocal_input_ids = torch.cat([vocal_input_ids, pad_ids], dim=-1)
|
| 102 |
+
combined_input_ids = torch.cat([acc_input_ids, vocal_input_ids], dim=1) # #(bsz, 2*num_codebooks, seq_len+1)
|
| 103 |
+
|
| 104 |
+
else:
|
| 105 |
+
raise ValueError(f"Unexpected track_conbination pattern: {pattern}")
|
| 106 |
+
|
| 107 |
+
else:
|
| 108 |
+
raise ValueError(f"Unexpected track_conbination pattern: {pattern}")
|
| 109 |
+
|
| 110 |
+
return combined_input_ids
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
def build_combined_delay_pattern_mask(
|
| 114 |
+
acc_input_ids: torch.LongTensor, vocal_input_ids: torch.LongTensor, track_pattern: str, bos_token_id: int, pad_token_id: int, max_length: int, num_codebooks: int, device
|
| 115 |
+
):
|
| 116 |
+
assert acc_input_ids.shape == vocal_input_ids.shape, f'acc_input_ids.shape:{acc_input_ids.shape} != vocal_input_ids.shape:{vocal_input_ids.shape}'
|
| 117 |
+
'''
|
| 118 |
+
input_ids:
|
| 119 |
+
- [B, a, b]
|
| 120 |
+
- [B, B, c]
|
| 121 |
+
- [B, B, B]
|
| 122 |
+
- [B, B, B]
|
| 123 |
+
|
| 124 |
+
delay_pattern_mask
|
| 125 |
+
- [B, a, b, -1, -1, P, P, P]
|
| 126 |
+
- [B, B, c, d, -1, -1, P, P]
|
| 127 |
+
- [B, B, B, e, f, -1, -1, P]
|
| 128 |
+
- [B, B, B, B, g, h, -1, -1]
|
| 129 |
+
|
| 130 |
+
or
|
| 131 |
+
input_ids:
|
| 132 |
+
- [B]
|
| 133 |
+
- [B]
|
| 134 |
+
- [B]
|
| 135 |
+
- [B]
|
| 136 |
+
|
| 137 |
+
delay_pattern_mask
|
| 138 |
+
- [B, -1, -1, P, P, P]
|
| 139 |
+
- [B, B, -1, -1, P, P]
|
| 140 |
+
- [B, B, B, -1, -1, P]
|
| 141 |
+
- [B, B, B, B, -1, -1]
|
| 142 |
+
'''
|
| 143 |
+
vocal_input_ids = vocal_input_ids.reshape(-1, num_codebooks, vocal_input_ids.shape[-1] )
|
| 144 |
+
acc_input_ids = acc_input_ids.reshape(-1, num_codebooks, acc_input_ids.shape[-1] )
|
| 145 |
+
batch_size = vocal_input_ids.shape[0]
|
| 146 |
+
|
| 147 |
+
#Here, the first column of vocal_input_ids and acc_input_ids must be bos_token_id.
|
| 148 |
+
vocal_delayed_input_ids, vocal_decoder_delay_pattern_mask = build_delay_pattern_mask(
|
| 149 |
+
vocal_input_ids,
|
| 150 |
+
bos_token_id=bos_token_id,
|
| 151 |
+
pad_token_id=pad_token_id,
|
| 152 |
+
max_length=max_length,
|
| 153 |
+
num_codebooks=num_codebooks
|
| 154 |
+
)
|
| 155 |
+
|
| 156 |
+
acc_delayed_input_ids, acc_decoder_delay_pattern_mask = build_delay_pattern_mask(
|
| 157 |
+
acc_input_ids,
|
| 158 |
+
bos_token_id=bos_token_id,
|
| 159 |
+
pad_token_id=pad_token_id,
|
| 160 |
+
max_length=max_length,
|
| 161 |
+
num_codebooks=num_codebooks
|
| 162 |
+
)
|
| 163 |
+
|
| 164 |
+
|
| 165 |
+
if track_pattern.startswith('parallel'):
|
| 166 |
+
decoder_input_ids_start = (
|
| 167 |
+
torch.ones((batch_size , num_codebooks * 2, 1), dtype=torch.long, device=device)
|
| 168 |
+
* bos_token_id
|
| 169 |
+
)
|
| 170 |
+
else:
|
| 171 |
+
decoder_input_ids_start = (
|
| 172 |
+
torch.ones((batch_size , num_codebooks, 1), dtype=torch.long, device=device)
|
| 173 |
+
* bos_token_id
|
| 174 |
+
)
|
| 175 |
+
|
| 176 |
+
#combine input_ids
|
| 177 |
+
if vocal_delayed_input_ids.shape[-1]==1 and (vocal_delayed_input_ids == bos_token_id).all().item() and \
|
| 178 |
+
acc_delayed_input_ids.shape[-1]==1 and (acc_delayed_input_ids == bos_token_id).all().item():
|
| 179 |
+
decoder_input_ids = decoder_input_ids_start
|
| 180 |
+
else:
|
| 181 |
+
combined_input_ids = combine_track_input_ids(acc_delayed_input_ids[..., 1:], vocal_delayed_input_ids[..., 1:], track_pattern, bos_token_id, pad_token_id, num_codebooks)
|
| 182 |
+
decoder_input_ids = torch.cat([decoder_input_ids_start, combined_input_ids], dim=-1)
|
| 183 |
+
if track_pattern in ['parallel_A_V', 'parallel_V_A']:
|
| 184 |
+
assert (decoder_input_ids[..., -1] == pad_token_id).any().item and (decoder_input_ids[..., -2] != pad_token_id).all().item, f'decoder_input_ids[..., -2:] : {decoder_input_ids[..., -2:]}'
|
| 185 |
+
decoder_input_ids = decoder_input_ids[..., :-1]
|
| 186 |
+
|
| 187 |
+
#combine decoder_delay_pattern_mask
|
| 188 |
+
assert vocal_decoder_delay_pattern_mask.shape[-1] > 1 and acc_decoder_delay_pattern_mask.shape[-1] >1, f'vocal_decoder_delay_pattern_mask:{vocal_decoder_delay_pattern_mask.shape}; acc_decoder_delay_pattern_mask:{acc_decoder_delay_pattern_mask.shape}'
|
| 189 |
+
combined_decoder_delay_pattern_mask = combine_track_input_ids(acc_decoder_delay_pattern_mask[..., 1:], vocal_decoder_delay_pattern_mask[..., 1:], track_pattern, bos_token_id, pad_token_id, num_codebooks)
|
| 190 |
+
decoder_delay_pattern_mask = torch.cat([decoder_input_ids_start, combined_decoder_delay_pattern_mask], dim=-1)
|
| 191 |
+
|
| 192 |
+
decoder_input_ids = decoder_input_ids.reshape(-1, decoder_input_ids.shape[-1])
|
| 193 |
+
decoder_delay_pattern_mask = decoder_delay_pattern_mask.reshape(-1, decoder_delay_pattern_mask.shape[-1])
|
| 194 |
+
return decoder_input_ids, decoder_delay_pattern_mask
|
| 195 |
+
|
| 196 |
+
|
| 197 |
+
|
| 198 |
+
|
| 199 |
+
class SongGenDualTrackForConditionalGeneration(SongGenMixedForConditionalGeneration):
|
| 200 |
+
def __init__(self, config: Optional[SongGenConfig] = None, **kwargs):
|
| 201 |
+
super().__init__(config=config, **kwargs)
|
| 202 |
+
|
| 203 |
+
def _prepare_combined_delayed_decoder_input_ids_for_generation(
|
| 204 |
+
self,
|
| 205 |
+
batch_size: int,
|
| 206 |
+
model_input_name: str,
|
| 207 |
+
model_kwargs: Dict[str, torch.Tensor],
|
| 208 |
+
decoder_start_token_id: int = None,
|
| 209 |
+
bos_token_id: int = None,
|
| 210 |
+
pad_token_id: int= None,
|
| 211 |
+
max_length: int= None,
|
| 212 |
+
device: torch.device = None,
|
| 213 |
+
) -> Tuple[torch.LongTensor, Dict[str, torch.Tensor]]:
|
| 214 |
+
"""Prepares `decoder_input_ids` for generation with encoder-decoder models"""
|
| 215 |
+
# 1. We also allow the user to pass it under `vocal_input_ids` and `acc_input_ids`, if the encoder does not use it as the main input.
|
| 216 |
+
track_pattern = self.decoder.config.track_pattern
|
| 217 |
+
if device is None:
|
| 218 |
+
device = self.device
|
| 219 |
+
vocal_input_ids = None
|
| 220 |
+
acc_input_ids = None
|
| 221 |
+
if model_kwargs is not None and "vocal_input_ids" in model_kwargs:
|
| 222 |
+
vocal_input_ids = model_kwargs.pop("vocal_input_ids")
|
| 223 |
+
if model_kwargs is not None and "acc_input_ids" in model_kwargs:
|
| 224 |
+
acc_input_ids = model_kwargs.pop("acc_input_ids")
|
| 225 |
+
|
| 226 |
+
# 2. Encoder-decoder models expect the `decoder_input_ids` to start with a special token. Let's ensure that.
|
| 227 |
+
decoder_start_token_id = self._get_decoder_start_token_id(decoder_start_token_id, bos_token_id)
|
| 228 |
+
|
| 229 |
+
decoder_input_ids_start = (
|
| 230 |
+
torch.ones((batch_size, self.decoder.num_codebooks, 1), dtype=torch.long, device=device)
|
| 231 |
+
* decoder_start_token_id
|
| 232 |
+
)
|
| 233 |
+
|
| 234 |
+
# no user input -> use decoder_start_token_id as decoder_input_ids
|
| 235 |
+
# user input but doesn't start with decoder_start_token_id -> prepend decoder_start_token_id
|
| 236 |
+
if vocal_input_ids is None:
|
| 237 |
+
vocal_input_ids = decoder_input_ids_start
|
| 238 |
+
elif (vocal_input_ids[..., 0] != decoder_start_token_id).all().item():
|
| 239 |
+
vocal_input_ids = torch.cat([decoder_input_ids_start, vocal_input_ids], dim=-1)
|
| 240 |
+
|
| 241 |
+
assert "decoder_attention_mask" not in model_kwargs, "decoder_attention_mask should not in model_kwargs"
|
| 242 |
+
|
| 243 |
+
if acc_input_ids is None:
|
| 244 |
+
acc_input_ids = decoder_input_ids_start
|
| 245 |
+
elif (acc_input_ids[..., 0] != decoder_start_token_id).all().item():
|
| 246 |
+
acc_input_ids = torch.cat([decoder_input_ids_start, acc_input_ids], dim=-1)
|
| 247 |
+
|
| 248 |
+
assert vocal_input_ids.shape == acc_input_ids.shape, f'vocal_input_ids: {vocal_input_ids}, acc_input_ids.shape: {acc_input_ids.shape}'
|
| 249 |
+
|
| 250 |
+
decoder_input_ids, decoder_delay_pattern_mask = build_combined_delay_pattern_mask(
|
| 251 |
+
acc_input_ids, vocal_input_ids, track_pattern, bos_token_id, pad_token_id, max_length, self.decoder.num_codebooks, device
|
| 252 |
+
)
|
| 253 |
+
|
| 254 |
+
# stash the delay mask so that we don't have to recompute in each forward pass
|
| 255 |
+
model_kwargs["decoder_delay_pattern_mask"] = decoder_delay_pattern_mask
|
| 256 |
+
|
| 257 |
+
prompt_hidden_states = model_kwargs.get("prompt_hidden_states",None) if not self.prompt_cross_attention else None
|
| 258 |
+
ref_audio_embeds = model_kwargs.get("ref_audio_embeds", None)
|
| 259 |
+
num_embed_tokens = self.decoder.num_codebooks * 2 if track_pattern.startswith('parallel') else self.decoder.num_codebooks
|
| 260 |
+
input = decoder_input_ids.reshape(-1, num_embed_tokens, decoder_input_ids.shape[-1])
|
| 261 |
+
inputs_embeds = sum(
|
| 262 |
+
[
|
| 263 |
+
self.decoder.model.decoder.embed_tokens[codebook](input[:, codebook])
|
| 264 |
+
for codebook in range(num_embed_tokens)
|
| 265 |
+
]
|
| 266 |
+
) / int(num_embed_tokens/self.decoder.num_codebooks)
|
| 267 |
+
if prompt_hidden_states is not None:
|
| 268 |
+
inputs_embeds = torch.cat([prompt_hidden_states, inputs_embeds], dim=1)
|
| 269 |
+
model_kwargs["inputs_embeds"] = inputs_embeds
|
| 270 |
+
return acc_input_ids, vocal_input_ids, decoder_input_ids, model_kwargs
|
| 271 |
+
|
| 272 |
+
|
| 273 |
+
def audio_decode_output_values(self, output_ids, batch_size, audio_scales, generation_config):
|
| 274 |
+
output_ids = output_ids[None, ...]
|
| 275 |
+
decode_sequentially = True
|
| 276 |
+
# (
|
| 277 |
+
# generation_config.bos_token_id in output_ids
|
| 278 |
+
# or generation_config.pad_token_id in output_ids
|
| 279 |
+
# or generation_config.eos_token_id in output_ids
|
| 280 |
+
# )
|
| 281 |
+
|
| 282 |
+
if not decode_sequentially:
|
| 283 |
+
output_values = self.audio_encoder.decode(
|
| 284 |
+
output_ids,
|
| 285 |
+
audio_scales=audio_scales,
|
| 286 |
+
).audio_values.squeeze(1)
|
| 287 |
+
output_lengths = [audio.shape[0] for audio in output_values]
|
| 288 |
+
else:
|
| 289 |
+
output_values = []
|
| 290 |
+
for sample_id in range(batch_size):
|
| 291 |
+
sample = output_ids[:, sample_id]
|
| 292 |
+
sample_mask = (sample >= self.audio_encoder.config.codebook_size).sum(dim=(0, 1)) == 0
|
| 293 |
+
if sample_mask.sum() > 0:
|
| 294 |
+
sample = sample[:, :, sample_mask]
|
| 295 |
+
sample = self.audio_encoder.decode(sample[None, ...], [audio_scales[sample_id]]).audio_values
|
| 296 |
+
output_values.append(sample.transpose(0, 2))
|
| 297 |
+
else:
|
| 298 |
+
logger.warning('sample_mask.sum() <= 0 , generate sample: ', sample.shape, sample[0, :, :20])
|
| 299 |
+
output_values.append(torch.zeros((1, 1, 1)).to(self.device))
|
| 300 |
+
output_lengths = [audio.shape[0] for audio in output_values]
|
| 301 |
+
output_values = (
|
| 302 |
+
torch.nn.utils.rnn.pad_sequence(output_values, batch_first=True, padding_value=0)
|
| 303 |
+
.squeeze(-1)
|
| 304 |
+
.squeeze(-1)
|
| 305 |
+
)
|
| 306 |
+
return output_lengths, output_values
|
| 307 |
+
|
| 308 |
+
@torch.no_grad()
|
| 309 |
+
def generate(
|
| 310 |
+
self,
|
| 311 |
+
inputs: Optional[torch.Tensor] = None,
|
| 312 |
+
generation_config: Optional[GenerationConfig] = None,
|
| 313 |
+
logits_processor: Optional[LogitsProcessorList] = None,
|
| 314 |
+
stopping_criteria: Optional[StoppingCriteriaList] = None,
|
| 315 |
+
synced_gpus: Optional[bool] = None,
|
| 316 |
+
streamer: Optional["BaseStreamer"] = None,
|
| 317 |
+
**kwargs,
|
| 318 |
+
):
|
| 319 |
+
"""
|
| 320 |
+
|
| 321 |
+
Generates sequences of token ids for models with a language modeling head.
|
| 322 |
+
|
| 323 |
+
<Tip warning={true}>
|
| 324 |
+
|
| 325 |
+
Most generation-controlling parameters are set in `generation_config` which, if not passed, will be set to the
|
| 326 |
+
model's default generation configuration. You can override any `generation_config` by passing the corresponding
|
| 327 |
+
parameters to generate(), e.g. `.generate(inputs, num_beams=4, do_sample=True)`.
|
| 328 |
+
|
| 329 |
+
For an overview of generation strategies and code examples, check out the [following
|
| 330 |
+
guide](./generation_strategies).
|
| 331 |
+
|
| 332 |
+
</Tip>
|
| 333 |
+
|
| 334 |
+
Parameters:
|
| 335 |
+
inputs (`torch.Tensor` of varying shape depending on the modality, *optional*):
|
| 336 |
+
The sequence used as a prompt for the generation or as model inputs to the encoder. If `None` the
|
| 337 |
+
method initializes it with `bos_token_id` and a batch size of 1. For decoder-only models `inputs`
|
| 338 |
+
should be in the format `input_ids`. For encoder-decoder models *inputs* can represent any of
|
| 339 |
+
`input_ids`, `input_values`, `input_features`, or `pixel_values`.
|
| 340 |
+
generation_config (`~generation.GenerationConfig`, *optional*):
|
| 341 |
+
The generation configuration to be used as base parametrization for the generation call. `**kwargs`
|
| 342 |
+
passed to generate matching the attributes of `generation_config` will override them. If
|
| 343 |
+
`generation_config` is not provided, the default will be used, which had the following loading
|
| 344 |
+
priority: 1) from the `generation_config.json` model file, if it exists; 2) from the model
|
| 345 |
+
configuration. Please note that unspecified parameters will inherit [`~generation.GenerationConfig`]'s
|
| 346 |
+
default values, whose documentation should be checked to parameterize generation.
|
| 347 |
+
logits_processor (`LogitsProcessorList`, *optional*):
|
| 348 |
+
Custom logits processors that complement the default logits processors built from arguments and
|
| 349 |
+
generation config. If a logit processor is passed that is already created with the arguments or a
|
| 350 |
+
generation config an error is thrown. This feature is intended for advanced users.
|
| 351 |
+
stopping_criteria (`StoppingCriteriaList`, *optional*):
|
| 352 |
+
Custom stopping criteria that complement the default stopping criteria built from arguments and a
|
| 353 |
+
generation config. If a stopping criteria is passed that is already created with the arguments or a
|
| 354 |
+
generation config an error is thrown. This feature is intended for advanced users.
|
| 355 |
+
synced_gpus (`bool`, *optional*, defaults to `False`):
|
| 356 |
+
Whether to continue running the while loop until max_length (needed for ZeRO stage 3)
|
| 357 |
+
streamer (`BaseStreamer`, *optional*):
|
| 358 |
+
Streamer object that will be used to stream the generated sequences. Generated tokens are passed
|
| 359 |
+
through `streamer.put(token_ids)` and the streamer is responsible for any further processing.
|
| 360 |
+
kwargs (`Dict[str, Any]`, *optional*):
|
| 361 |
+
Ad hoc parametrization of `generate_config` and/or additional model-specific kwargs that will be
|
| 362 |
+
forwarded to the `forward` function of the model. If the model is an encoder-decoder model, encoder
|
| 363 |
+
specific kwargs should not be prefixed and decoder specific kwargs should be prefixed with *decoder_*.
|
| 364 |
+
|
| 365 |
+
Return:
|
| 366 |
+
[`~utils.ModelOutput`] or `torch.LongTensor`: A [`~utils.ModelOutput`] (if `return_dict_in_generate=True`
|
| 367 |
+
or when `config.return_dict_in_generate=True`) or a `torch.FloatTensor`.
|
| 368 |
+
|
| 369 |
+
If the model is *not* an encoder-decoder model (`model.config.is_encoder_decoder=False`), the possible
|
| 370 |
+
[`~utils.ModelOutput`] types are:
|
| 371 |
+
|
| 372 |
+
- [`~generation.GenerateDecoderOnlyOutput`],
|
| 373 |
+
- [`~generation.GenerateBeamDecoderOnlyOutput`]
|
| 374 |
+
|
| 375 |
+
If the model is an encoder-decoder model (`model.config.is_encoder_decoder=True`), the possible
|
| 376 |
+
[`~utils.ModelOutput`] types are:
|
| 377 |
+
|
| 378 |
+
- [`~generation.GenerateEncoderDecoderOutput`],
|
| 379 |
+
- [`~generation.GenerateBeamEncoderDecoderOutput`]
|
| 380 |
+
"""
|
| 381 |
+
# 1. Handle `generation_config` and kwargs that might update it, and validate the resulting objects
|
| 382 |
+
if generation_config is None:
|
| 383 |
+
generation_config = self.generation_config
|
| 384 |
+
|
| 385 |
+
generation_config = copy.deepcopy(generation_config)
|
| 386 |
+
model_kwargs = generation_config.update(**kwargs) # All unused kwargs must be model kwargs
|
| 387 |
+
generation_config.validate()
|
| 388 |
+
self._validate_model_kwargs(model_kwargs.copy())
|
| 389 |
+
|
| 390 |
+
if model_kwargs.get("encoder_outputs") is not None and type(model_kwargs["encoder_outputs"]) == tuple:
|
| 391 |
+
# wrap the unconditional outputs as a BaseModelOutput for compatibility with the rest of generate
|
| 392 |
+
model_kwargs["encoder_outputs"] = BaseModelOutput(last_hidden_state=model_kwargs["encoder_outputs"][0])
|
| 393 |
+
|
| 394 |
+
requires_attention_mask = "encoder_outputs" not in model_kwargs
|
| 395 |
+
kwargs_has_attention_mask = model_kwargs.get("attention_mask", None) is not None
|
| 396 |
+
|
| 397 |
+
# 3. Define model inputs
|
| 398 |
+
inputs_tensor, model_input_name, model_kwargs = self._prepare_model_inputs(
|
| 399 |
+
inputs, generation_config.bos_token_id, model_kwargs
|
| 400 |
+
)
|
| 401 |
+
batch_size = inputs_tensor.shape[0]
|
| 402 |
+
self._prepare_special_tokens(generation_config, kwargs_has_attention_mask, device=inputs_tensor.device)
|
| 403 |
+
|
| 404 |
+
track_pattern = self.decoder.config.track_pattern
|
| 405 |
+
if track_pattern.startswith('parallel'):
|
| 406 |
+
combined_num_codebooks = self.decoder.num_codebooks * 2
|
| 407 |
+
else:
|
| 408 |
+
combined_num_codebooks = self.decoder.num_codebooks
|
| 409 |
+
logits_processor = logits_processor if logits_processor is not None else LogitsProcessorList([SongGenLogitsProcessor(generation_config.eos_token_id, combined_num_codebooks, batch_size, inputs_tensor.device, track_pattern=track_pattern)])
|
| 410 |
+
stopping_criteria = stopping_criteria if stopping_criteria is not None else StoppingCriteriaList()
|
| 411 |
+
|
| 412 |
+
# 4. Define other model kwargs
|
| 413 |
+
model_kwargs["use_cache"] = generation_config.use_cache
|
| 414 |
+
|
| 415 |
+
if model_kwargs.get("attention_mask", None) is None and requires_attention_mask:
|
| 416 |
+
model_kwargs["attention_mask"] = self._prepare_attention_mask_for_generation(
|
| 417 |
+
inputs_tensor, generation_config._pad_token_tensor, generation_config._eos_token_tensor
|
| 418 |
+
)
|
| 419 |
+
|
| 420 |
+
if "encoder_outputs" not in model_kwargs:
|
| 421 |
+
# encoder_outputs are created and added to `model_kwargs`
|
| 422 |
+
model_kwargs = self._prepare_text_encoder_kwargs_for_generation(
|
| 423 |
+
inputs_tensor, model_kwargs, model_input_name, generation_config
|
| 424 |
+
)
|
| 425 |
+
|
| 426 |
+
if "prompt_hidden_states" not in model_kwargs and "prompt_input_ids" in model_kwargs:
|
| 427 |
+
# `prompt_hidden_states` are created and added to `model_kwargs`; remove "prompt_input_ids"
|
| 428 |
+
model_kwargs = self._prepare_prompt_kwargs_for_generation(
|
| 429 |
+
model_kwargs.pop("prompt_input_ids"),
|
| 430 |
+
model_kwargs,
|
| 431 |
+
)
|
| 432 |
+
|
| 433 |
+
|
| 434 |
+
if "decoder_input_ids" not in model_kwargs and "input_values" in model_kwargs:
|
| 435 |
+
logger.warning_once(
|
| 436 |
+
"generate() currently does not support the input_values parameter. Please use vocal_input_ids and acc_input_ids for now."
|
| 437 |
+
)
|
| 438 |
+
|
| 439 |
+
acc_input_ids, vocal_input_ids, delayed_input_ids, model_kwargs =self._prepare_combined_delayed_decoder_input_ids_for_generation(
|
| 440 |
+
batch_size=batch_size,
|
| 441 |
+
model_input_name=model_input_name,
|
| 442 |
+
model_kwargs=model_kwargs,
|
| 443 |
+
decoder_start_token_id=generation_config._decoder_start_token_tensor,
|
| 444 |
+
bos_token_id=generation_config._bos_token_tensor,
|
| 445 |
+
pad_token_id=generation_config._pad_token_tensor,
|
| 446 |
+
max_length=generation_config.max_length,
|
| 447 |
+
device=inputs_tensor.device,
|
| 448 |
+
)
|
| 449 |
+
|
| 450 |
+
|
| 451 |
+
# 6. Prepare `max_length` depending on other stopping criteria.
|
| 452 |
+
input_ids_length = delayed_input_ids.shape[-1]
|
| 453 |
+
has_default_max_length = kwargs.get("max_length") is None and generation_config.max_length is not None
|
| 454 |
+
has_default_min_length = kwargs.get("min_length") is None and generation_config.min_length is not None
|
| 455 |
+
generation_config = self._prepare_generated_length(
|
| 456 |
+
generation_config=generation_config,
|
| 457 |
+
has_default_max_length=has_default_max_length,
|
| 458 |
+
has_default_min_length=has_default_min_length,
|
| 459 |
+
model_input_name=model_input_name,
|
| 460 |
+
inputs_tensor=inputs_tensor,
|
| 461 |
+
input_ids_length=input_ids_length,
|
| 462 |
+
)
|
| 463 |
+
|
| 464 |
+
if generation_config.cache_implementation is not None and model_kwargs.get("past_key_values") is not None:
|
| 465 |
+
raise ValueError(
|
| 466 |
+
"Passing both `cache_implementation` (used to initialize certain caches) and `past_key_values` (a "
|
| 467 |
+
"Cache object) is unsupported. Please use only one of the two."
|
| 468 |
+
)
|
| 469 |
+
elif generation_config.cache_implementation is not None:
|
| 470 |
+
if generation_config.cache_implementation in NEED_SETUP_CACHE_CLASSES_MAPPING:
|
| 471 |
+
if generation_config.cache_implementation == "static" and not self._supports_static_cache:
|
| 472 |
+
raise ValueError(
|
| 473 |
+
"This model does not support `cache_implementation='static'`. Please check the following "
|
| 474 |
+
"issue: https://github.com/huggingface/transformers/issues/28981"
|
| 475 |
+
)
|
| 476 |
+
if not self.prompt_cross_attention:
|
| 477 |
+
# when we prepend prompt_hidden_state to inputs_embeds, max_cache_len needs to be actualised
|
| 478 |
+
# generation_config.max_length has already been increased by input_ids_length which is
|
| 479 |
+
# already counted in input_embeds_seq_length so we remove it
|
| 480 |
+
input_embeds_seq_length = model_kwargs["inputs_embeds"].shape[1]
|
| 481 |
+
max_cache_len = generation_config.max_length + input_embeds_seq_length - input_ids_length
|
| 482 |
+
else:
|
| 483 |
+
max_cache_len = self.generation_config.max_length
|
| 484 |
+
|
| 485 |
+
model_kwargs["past_key_values"] = self._get_cache(
|
| 486 |
+
generation_config.cache_implementation,
|
| 487 |
+
getattr(generation_config, "num_beams", 1) * batch_size,
|
| 488 |
+
max_cache_len,
|
| 489 |
+
model_kwargs,
|
| 490 |
+
)
|
| 491 |
+
elif generation_config.cache_implementation == "quantized":
|
| 492 |
+
raise ValueError(
|
| 493 |
+
"This model does not support the quantized cache. If you want your model to support quantized "
|
| 494 |
+
"cache, please open an issue on the Parler-TTS repository https://github.com/huggingface/parler-tts"
|
| 495 |
+
)
|
| 496 |
+
# Use DynamicCache() instance by default. This will avoid back and forth from legacy format that
|
| 497 |
+
# keeps copying the cache thus using much more memory
|
| 498 |
+
elif generation_config.cache_implementation is None and self._supports_default_dynamic_cache(): #NOTE here
|
| 499 |
+
past = model_kwargs.get("past_key_values", None)
|
| 500 |
+
requires_cross_attention_cache = (
|
| 501 |
+
self.config.is_encoder_decoder or model_kwargs.get("encoder_outputs") is not None
|
| 502 |
+
)
|
| 503 |
+
if past is None:
|
| 504 |
+
model_kwargs["past_key_values"] = (
|
| 505 |
+
DynamicCache()
|
| 506 |
+
if not requires_cross_attention_cache
|
| 507 |
+
else EncoderDecoderCache(DynamicCache(), DynamicCache())
|
| 508 |
+
)
|
| 509 |
+
elif isinstance(past, tuple):
|
| 510 |
+
model_kwargs["past_key_values"] = (
|
| 511 |
+
DynamicCache.from_legacy_cache(past)
|
| 512 |
+
if not requires_cross_attention_cache
|
| 513 |
+
else EncoderDecoderCache.from_legacy_cache(past)
|
| 514 |
+
)
|
| 515 |
+
|
| 516 |
+
# delayed_input_ids are ready to be placed on the streamer (if used)
|
| 517 |
+
if streamer is not None:
|
| 518 |
+
streamer.put(delayed_input_ids.cpu())
|
| 519 |
+
|
| 520 |
+
# 7. determine generation mode
|
| 521 |
+
generation_mode = generation_config.get_generation_mode()
|
| 522 |
+
logger.info(f'generation_mode: {generation_mode}')
|
| 523 |
+
|
| 524 |
+
# 8. prepare distribution pre_processing samplers
|
| 525 |
+
logits_processor = self._get_logits_processor(
|
| 526 |
+
generation_config=generation_config,
|
| 527 |
+
input_ids_seq_length=input_ids_length,
|
| 528 |
+
encoder_input_ids=inputs_tensor,
|
| 529 |
+
prefix_allowed_tokens_fn=None,
|
| 530 |
+
logits_processor=logits_processor,
|
| 531 |
+
device=delayed_input_ids.device,
|
| 532 |
+
)
|
| 533 |
+
|
| 534 |
+
# 9. prepare stopping criteria
|
| 535 |
+
stopping_criteria = self._get_stopping_criteria(
|
| 536 |
+
generation_config=generation_config, stopping_criteria=stopping_criteria
|
| 537 |
+
)
|
| 538 |
+
|
| 539 |
+
if generation_mode in (GenerationMode.SAMPLE, GenerationMode.GREEDY_SEARCH):
|
| 540 |
+
logits_warper = self._get_logits_warper(generation_config, device=delayed_input_ids.device)
|
| 541 |
+
# expand input_ids with `num_return_sequences` additional sequences per batch
|
| 542 |
+
delayed_input_ids, model_kwargs = self._expand_inputs_for_generation(
|
| 543 |
+
input_ids=delayed_input_ids,
|
| 544 |
+
expand_size=generation_config.num_return_sequences,
|
| 545 |
+
is_encoder_decoder=self.config.is_encoder_decoder,
|
| 546 |
+
**model_kwargs,
|
| 547 |
+
)
|
| 548 |
+
|
| 549 |
+
# 10. run sample
|
| 550 |
+
outputs = self._sample(
|
| 551 |
+
delayed_input_ids,
|
| 552 |
+
logits_processor=logits_processor,
|
| 553 |
+
logits_warper=logits_warper,
|
| 554 |
+
stopping_criteria=stopping_criteria,
|
| 555 |
+
generation_config=generation_config,
|
| 556 |
+
synced_gpus=synced_gpus,
|
| 557 |
+
streamer=streamer,
|
| 558 |
+
**model_kwargs,
|
| 559 |
+
)
|
| 560 |
+
else:
|
| 561 |
+
raise ValueError(
|
| 562 |
+
"Got incompatible mode for generation, should be one of greedy or sampling. "
|
| 563 |
+
"Ensure that beam search is de-activated by setting `num_beams=1` and `num_beam_groups=1`."
|
| 564 |
+
)
|
| 565 |
+
|
| 566 |
+
|
| 567 |
+
if generation_config.return_dict_in_generate:
|
| 568 |
+
output_ids = outputs.sequences
|
| 569 |
+
else:
|
| 570 |
+
output_ids = outputs
|
| 571 |
+
|
| 572 |
+
#output_ids.shape [bsz*num_codebooks, seq_len] or [bsz*num_codebooks*2, seq_len]
|
| 573 |
+
# Apply the pattern mask to the final ids
|
| 574 |
+
output_ids = self.decoder.apply_delay_pattern_mask(output_ids, model_kwargs["decoder_delay_pattern_mask"])
|
| 575 |
+
|
| 576 |
+
# Revert the pattern delay mask by filtering the eos and bos token ids from the delay pattern mask
|
| 577 |
+
|
| 578 |
+
if track_pattern in ['interleaving_A_V', 'interleaving_V_A']:
|
| 579 |
+
if output_ids.shape[-1] % 2 == 0:
|
| 580 |
+
output_ids = output_ids[..., :-1]
|
| 581 |
+
output_max_length = int((output_ids.shape[-1]-1) / 2 + 1)
|
| 582 |
+
elif track_pattern in ['parallel']:
|
| 583 |
+
output_max_length = output_ids.shape[-1]
|
| 584 |
+
elif track_pattern in ['parallel_A_V', 'parallel_V_A']:
|
| 585 |
+
output_max_length = output_ids.shape[-1] -1
|
| 586 |
+
|
| 587 |
+
_, mask = build_combined_delay_pattern_mask(
|
| 588 |
+
acc_input_ids=acc_input_ids,
|
| 589 |
+
vocal_input_ids=vocal_input_ids,
|
| 590 |
+
track_pattern=track_pattern,
|
| 591 |
+
bos_token_id=generation_config.bos_token_id,
|
| 592 |
+
pad_token_id=generation_config.pad_token_id,
|
| 593 |
+
max_length=output_max_length,
|
| 594 |
+
num_codebooks=self.decoder.num_codebooks,
|
| 595 |
+
device=output_ids.device
|
| 596 |
+
)
|
| 597 |
+
|
| 598 |
+
assert mask.shape == output_ids.shape, f"mask.shape: {mask.shape}, output_ids.shape: {output_ids.shape}"
|
| 599 |
+
mask = (mask != generation_config.bos_token_id) & (mask != generation_config.pad_token_id)
|
| 600 |
+
output_ids = output_ids[mask].reshape(batch_size, combined_num_codebooks, -1)
|
| 601 |
+
acc_output_ids, vocal_output_ids = split_combined_track_input_ids(output_ids, track_pattern, self.decoder.num_codebooks)
|
| 602 |
+
|
| 603 |
+
audio_scales = model_kwargs.get("audio_scales")
|
| 604 |
+
if audio_scales is None:
|
| 605 |
+
audio_scales = [None] * batch_size
|
| 606 |
+
|
| 607 |
+
vocal_output_lengths, vocal_output_values = self.audio_decode_output_values(vocal_output_ids, batch_size, audio_scales, generation_config)
|
| 608 |
+
acc_output_lengths, acc_output_values = self.audio_decode_output_values(acc_output_ids, batch_size, audio_scales, generation_config)
|
| 609 |
+
|
| 610 |
+
if generation_config.return_dict_in_generate:
|
| 611 |
+
outputs["acc_audios_length"] = acc_output_lengths
|
| 612 |
+
outputs["acc_sequences"] = acc_output_values
|
| 613 |
+
outputs["vocal_audios_length"] = vocal_output_lengths
|
| 614 |
+
outputs["vocal_sequences"] = vocal_output_values
|
| 615 |
+
return outputs
|
| 616 |
+
else:
|
| 617 |
+
return acc_output_values, vocal_output_values
|
| 618 |
+
|
| 619 |
+
|
| 620 |
+
|
| 621 |
+
|
| 622 |
+
|
| 623 |
+
|
songgen/modeling_songgen_mixed.py
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
songgen/processing_songgen.py
ADDED
|
@@ -0,0 +1,141 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
lyrics/Text/Voice processor class for MusicGen
|
| 3 |
+
"""
|
| 4 |
+
import os
|
| 5 |
+
import torch
|
| 6 |
+
import librosa
|
| 7 |
+
import soundfile as sf
|
| 8 |
+
from transformers import AutoTokenizer
|
| 9 |
+
from .lyrics_utils.lyrics_tokenizer import VoiceBpeTokenizer
|
| 10 |
+
from transformers.models.encodec.feature_extraction_encodec import EncodecFeatureExtractor
|
| 11 |
+
from transformers import Wav2Vec2FeatureExtractor
|
| 12 |
+
from demucs import pretrained
|
| 13 |
+
from demucs.apply import apply_model
|
| 14 |
+
from demucs.audio import convert_audio
|
| 15 |
+
from songgen import (
|
| 16 |
+
XCodecModel,
|
| 17 |
+
build_delay_pattern_mask,
|
| 18 |
+
)
|
| 19 |
+
|
| 20 |
+
class SongGenProcessor():
|
| 21 |
+
def __init__(self, ckpt_path, device):
|
| 22 |
+
"""
|
| 23 |
+
Initializes the SongGenProcessor
|
| 24 |
+
"""
|
| 25 |
+
self.device = device
|
| 26 |
+
self.text_tokenizer = AutoTokenizer.from_pretrained(ckpt_path, padding_side='right')
|
| 27 |
+
self.lyrics_tokenizer = VoiceBpeTokenizer()
|
| 28 |
+
mert_path = 'm-a-p/MERT-v1-330M'
|
| 29 |
+
self.mert_processor = Wav2Vec2FeatureExtractor.from_pretrained(mert_path)
|
| 30 |
+
self.demucs = pretrained.get_model("htdemucs").to(device)
|
| 31 |
+
self.feature_extractor = EncodecFeatureExtractor(sampling_rate=16000)
|
| 32 |
+
self.audio_encoder = XCodecModel()
|
| 33 |
+
self.audio_encoder_special_token_ids ={
|
| 34 |
+
'mask': 1030,
|
| 35 |
+
"random": {"bos": 1031 , "eos": 1032},
|
| 36 |
+
"melody": {"bos": 1033, "eos": 1034},
|
| 37 |
+
"drum": {"bos": 1035, "eos": 1036},
|
| 38 |
+
"vocal": {"bos": 1037, "eos": 1038},
|
| 39 |
+
"acc": {"bos":1039 , "eos":1040},
|
| 40 |
+
}
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def __call__(self, text: str, lyrics: str, refaudio_path=None, refaudio_type=None, ref_voice_path=None, start=0, separate=False, padding=True, return_tensors="pt"):
|
| 45 |
+
|
| 46 |
+
"""
|
| 47 |
+
Processes the input text, lyrics, and audio file, and returns the tensors suitable for model input.
|
| 48 |
+
do not support batching yet
|
| 49 |
+
|
| 50 |
+
:param text: text description.
|
| 51 |
+
:param lyrics: Lyrics text. English Only.
|
| 52 |
+
:param ref_voice_path: Optional path to the reference voice.
|
| 53 |
+
:param start: The starting time for the reference voice slice.
|
| 54 |
+
:param separate: Whether to perform audio separation.
|
| 55 |
+
:param return_tensors: Whether to return the tensors as PyTorch tensors.
|
| 56 |
+
:return: A dictionary with the model's inputs, ready for inference.
|
| 57 |
+
"""
|
| 58 |
+
# Process lyrics and convert them into token IDs. Must be english now!
|
| 59 |
+
prompt_input_ids = [261] + self.lyrics_tokenizer.encode(lyrics.strip().replace('\n', '.'), lang='en') + [0]
|
| 60 |
+
|
| 61 |
+
# Tokenize the lyrics and pad to max length
|
| 62 |
+
lyrics_inputs = self.text_tokenizer.pad(
|
| 63 |
+
[{"input_ids": prompt_input_ids}],
|
| 64 |
+
return_tensors=return_tensors,
|
| 65 |
+
padding="longest",
|
| 66 |
+
).to(self.device)
|
| 67 |
+
|
| 68 |
+
# Tokenize the text descriptions
|
| 69 |
+
text_inputs = self.text_tokenizer(
|
| 70 |
+
text,
|
| 71 |
+
return_tensors=return_tensors,
|
| 72 |
+
padding="longest",
|
| 73 |
+
).to(self.device)
|
| 74 |
+
|
| 75 |
+
model_inputs = {
|
| 76 |
+
**text_inputs,
|
| 77 |
+
"prompt_input_ids": lyrics_inputs.input_ids,
|
| 78 |
+
"prompt_attention_mask": lyrics_inputs.attention_mask
|
| 79 |
+
}
|
| 80 |
+
|
| 81 |
+
# Process reference voice (if provided)
|
| 82 |
+
if ref_voice_path is not None:
|
| 83 |
+
wav, sr = sf.read(ref_voice_path)
|
| 84 |
+
wav = wav.T
|
| 85 |
+
wav = librosa.to_mono(wav) # Convert to mono if stereo
|
| 86 |
+
# Slice the audio according to the start and end times
|
| 87 |
+
lidx = int(start * sr)
|
| 88 |
+
ridx = lidx + int(3 * sr) # Slice a 3-second segment
|
| 89 |
+
wav = wav[lidx:ridx]
|
| 90 |
+
|
| 91 |
+
if separate:
|
| 92 |
+
# Since our model only supports reference voices that contain vocals and does not include accompaniment, it is necessary to perform vocal separation for mixed audio.
|
| 93 |
+
demucs_wav = convert_audio(
|
| 94 |
+
torch.tensor(wav[None], device=self.device).to(torch.float32),
|
| 95 |
+
sr,
|
| 96 |
+
self.demucs.samplerate,
|
| 97 |
+
self.demucs.audio_channels
|
| 98 |
+
)
|
| 99 |
+
sr = self.demucs.samplerate
|
| 100 |
+
stems = apply_model(self.demucs, demucs_wav.unsqueeze(0))
|
| 101 |
+
wav = stems[0][-1:].sum(0).mean(0).cpu().numpy()
|
| 102 |
+
|
| 103 |
+
if sr != self.mert_processor.sampling_rate:
|
| 104 |
+
wav = librosa.resample(wav, orig_sr=sr, target_sr=self.mert_processor.sampling_rate)
|
| 105 |
+
sr = self.mert_processor.sampling_rate
|
| 106 |
+
|
| 107 |
+
mert_inputs = self.mert_processor(
|
| 108 |
+
[wav], sampling_rate=self.mert_processor.sampling_rate, return_tensors="pt", padding="max_length", max_length=3*self.mert_processor.sampling_rate
|
| 109 |
+
)
|
| 110 |
+
|
| 111 |
+
model_inputs['ref_voice_values'] = mert_inputs['input_values'].to(self.device)
|
| 112 |
+
model_inputs['ref_voice_attention_mask'] = mert_inputs['attention_mask'].to(self.device)
|
| 113 |
+
|
| 114 |
+
if refaudio_path is not None and refaudio_type is not None:
|
| 115 |
+
refaudio_wav, refaudio_sr = sf.read(refaudio_path)
|
| 116 |
+
refaudio_wav = refaudio_wav.T
|
| 117 |
+
refaudio_wav = librosa.to_mono(refaudio_wav)
|
| 118 |
+
if refaudio_sr != self.feature_extractor.sampling_rate:
|
| 119 |
+
refaudio_wav = librosa.resample(refaudio_wav, orig_sr=refaudio_sr, target_sr=self.feature_extractor.sampling_rate)
|
| 120 |
+
refaudio_sr = self.feature_extractor.sampling_rate
|
| 121 |
+
with torch.no_grad():
|
| 122 |
+
self.audio_encoder.model.to(self.device)
|
| 123 |
+
codes = self.audio_encoder.encode(input_values=torch.tensor(refaudio_wav, dtype=torch.float32).to(self.device).unsqueeze(0).unsqueeze(0), bandwidth=4)["audio_codes"] #(1, bsz, codebooks, seq_len)
|
| 124 |
+
codes = codes.to(self.device).squeeze(0) # (1, bsz, codebooks, seq_len) -> (1, codebooks, seq_len)
|
| 125 |
+
num_codebooks = codes.shape[-2]
|
| 126 |
+
#apply codebook-delay
|
| 127 |
+
bos = self.audio_encoder_special_token_ids[refaudio_type]['bos']
|
| 128 |
+
eos = self.audio_encoder_special_token_ids[refaudio_type]['eos']
|
| 129 |
+
ref_bos = (torch.ones((1, num_codebooks, 1)) * bos).to(self.device)
|
| 130 |
+
codes = torch.cat([ref_bos, codes], dim=-1)
|
| 131 |
+
_, delay_pattern_mask = build_delay_pattern_mask(
|
| 132 |
+
codes,
|
| 133 |
+
bos_token_id=bos,
|
| 134 |
+
pad_token_id=eos,
|
| 135 |
+
max_length=codes.shape[-1] + num_codebooks,
|
| 136 |
+
num_codebooks=num_codebooks,
|
| 137 |
+
)
|
| 138 |
+
delay_codes = torch.where(delay_pattern_mask == -1, eos, delay_pattern_mask) #( codebooks, seq_len)
|
| 139 |
+
model_inputs['ref_audio_ids'] = delay_codes.transpose(0,1).unsqueeze(0) #(bsz, seq_len, codebooks)
|
| 140 |
+
#Xcode codes and build delay
|
| 141 |
+
return model_inputs
|
songgen/xcodec_wrapper/__init__.py
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from .configuration_xcodec import XCodecConfig
|
| 2 |
+
from .modeling_xcodec import XCodecModel
|
songgen/xcodec_wrapper/configuration_xcodec.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from transformers import PretrainedConfig
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
class XCodecConfig(PretrainedConfig):
|
| 5 |
+
model_type = "xcodec"
|
| 6 |
+
|
| 7 |
+
def __init__(
|
| 8 |
+
self,
|
| 9 |
+
num_codebooks: int = 8,
|
| 10 |
+
model_bitrate: int = 4, # kbps
|
| 11 |
+
codebook_size: int = 1024,
|
| 12 |
+
# latent_dim: int = 1024,
|
| 13 |
+
frame_rate: int = 50,
|
| 14 |
+
sampling_rate: int = 16000,
|
| 15 |
+
**kwargs,
|
| 16 |
+
):
|
| 17 |
+
self.codebook_size = codebook_size
|
| 18 |
+
self.model_bitrate = model_bitrate
|
| 19 |
+
# self.latent_dim = latent_dim
|
| 20 |
+
self.num_codebooks = num_codebooks
|
| 21 |
+
self.frame_rate = frame_rate
|
| 22 |
+
self.sampling_rate = sampling_rate
|
| 23 |
+
|
| 24 |
+
super().__init__(**kwargs)
|
songgen/xcodec_wrapper/modeling_xcodec.py
ADDED
|
@@ -0,0 +1,167 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from transformers import PreTrainedModel
|
| 3 |
+
from transformers.models.encodec.modeling_encodec import EncodecDecoderOutput, EncodecEncoderOutput
|
| 4 |
+
from .configuration_xcodec import XCodecConfig
|
| 5 |
+
import torch.nn as nn
|
| 6 |
+
import sys
|
| 7 |
+
import os
|
| 8 |
+
XCODEC_INFER_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'xcodec_infer')
|
| 9 |
+
sys.path.append(XCODEC_INFER_PATH)
|
| 10 |
+
from omegaconf import OmegaConf
|
| 11 |
+
from models.soundstream_semantic import SoundStream
|
| 12 |
+
|
| 13 |
+
# model doesn't support batching yet
|
| 14 |
+
|
| 15 |
+
def build_codec_model(config):
|
| 16 |
+
model = eval(config.generator.name)(**config.generator.config)
|
| 17 |
+
return model
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
class XCodecModel(nn.Module):
|
| 21 |
+
config_class = XCodecConfig
|
| 22 |
+
main_input_name = "input_values"
|
| 23 |
+
|
| 24 |
+
def __init__(self):
|
| 25 |
+
super().__init__()
|
| 26 |
+
ckpt_path = os.path.join(XCODEC_INFER_PATH , 'ckpts/general_more/xcodec_hubert_general_audio_v2.pth')
|
| 27 |
+
config_path = os.path.join(XCODEC_INFER_PATH ,'ckpts/general_more/config_hubert_general.yaml')
|
| 28 |
+
config = OmegaConf.load(config_path)
|
| 29 |
+
self.model = build_codec_model(config)
|
| 30 |
+
parameter_dict = torch.load(ckpt_path)
|
| 31 |
+
self.model.load_state_dict(parameter_dict)
|
| 32 |
+
self.model.eval()
|
| 33 |
+
self.num_codebooks = 8
|
| 34 |
+
self.codebook_size = 1024
|
| 35 |
+
self.config = config
|
| 36 |
+
self.config.codebook_size = 1024
|
| 37 |
+
self.config.n_codebooks = 8
|
| 38 |
+
self.config.return_dict = True
|
| 39 |
+
self.config.frame_rate = 50
|
| 40 |
+
|
| 41 |
+
def encode(
|
| 42 |
+
self, input_values, padding_mask=None, bandwidth=None, return_dict=None, n_quantizers=None, sample_rate=None
|
| 43 |
+
):
|
| 44 |
+
"""
|
| 45 |
+
Encodes the input audio waveform into discrete codes.
|
| 46 |
+
|
| 47 |
+
Args:
|
| 48 |
+
input_values (`torch.Tensor` of shape `(batch_size, channels, sequence_length)`):
|
| 49 |
+
Float values of the input audio waveform.
|
| 50 |
+
padding_mask (`torch.Tensor` of shape `(batch_size, channels, sequence_length)`):
|
| 51 |
+
Padding mask used to pad the `input_values`.
|
| 52 |
+
bandwidth (`float`, *optional*):
|
| 53 |
+
Not used, kept to have the same inferface as HF encodec.
|
| 54 |
+
n_quantizers (`int`, *optional*) :
|
| 55 |
+
Number of quantizers to use, by default None
|
| 56 |
+
If None, all quantizers are used.
|
| 57 |
+
sample_rate (`int`, *optional*) :
|
| 58 |
+
Signal sampling_rate
|
| 59 |
+
|
| 60 |
+
Returns:
|
| 61 |
+
A list of frames containing the discrete encoded codes for the input audio waveform, along with rescaling
|
| 62 |
+
factors for each chunk when `normalize` is True. Each frames is a tuple `(codebook, scale)`, with
|
| 63 |
+
`codebook` of shape `[batch_size, num_codebooks, frames]`.
|
| 64 |
+
Scale is not used here.
|
| 65 |
+
|
| 66 |
+
"""
|
| 67 |
+
bsz, channels, input_length = input_values.shape
|
| 68 |
+
|
| 69 |
+
if channels < 1 or channels > 2:
|
| 70 |
+
raise ValueError(f"Number of audio channels must be 1 or 2, but got {channels}")
|
| 71 |
+
|
| 72 |
+
if bsz != 1:
|
| 73 |
+
raise ValueError(f"Number of audio batch_size must be 1 in XCodec Encode() : {input_values.shape}")
|
| 74 |
+
|
| 75 |
+
# audio_data = self.model.preprocess(input_values, sample_rate)
|
| 76 |
+
if channels != 1:
|
| 77 |
+
input_values = input_values.mean(1, keepdim=True)
|
| 78 |
+
|
| 79 |
+
if sample_rate is not None and sample_rate != 16000:
|
| 80 |
+
input_values = torchaudio.transforms.Resample(sample_rate, 16000)(input_values)
|
| 81 |
+
|
| 82 |
+
audio_data = input_values
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
return_dict = return_dict if return_dict is not None else self.config.return_dict
|
| 86 |
+
|
| 87 |
+
# TODO: for now, no chunk length
|
| 88 |
+
|
| 89 |
+
chunk_length = None # self.config.chunk_length
|
| 90 |
+
if chunk_length is None:
|
| 91 |
+
chunk_length = input_length
|
| 92 |
+
stride = input_length
|
| 93 |
+
else:
|
| 94 |
+
stride = self.config.chunk_stride
|
| 95 |
+
|
| 96 |
+
if padding_mask is None:
|
| 97 |
+
padding_mask = torch.ones_like(input_values).bool()
|
| 98 |
+
|
| 99 |
+
encoded_frames = []
|
| 100 |
+
scales = []
|
| 101 |
+
|
| 102 |
+
step = chunk_length - stride
|
| 103 |
+
if (input_length % stride) - step != 0:
|
| 104 |
+
raise ValueError(
|
| 105 |
+
"The input length is not properly padded for batched chunked decoding. Make sure to pad the input correctly."
|
| 106 |
+
)
|
| 107 |
+
|
| 108 |
+
for offset in range(0, input_length - step, stride):
|
| 109 |
+
mask = padding_mask[..., offset : offset + chunk_length].bool()
|
| 110 |
+
frame = audio_data[:, :, offset : offset + chunk_length] #[1,1, squeeze_len]
|
| 111 |
+
|
| 112 |
+
scale = None
|
| 113 |
+
|
| 114 |
+
encoded_frame= self.model.encode(frame, target_bw=4) # [8, 1, seq_len]
|
| 115 |
+
encoded_frame = encoded_frame.transpose(0 , 1) # [1, 8, seq_len]
|
| 116 |
+
encoded_frames.append(encoded_frame)
|
| 117 |
+
scales.append(scale)
|
| 118 |
+
|
| 119 |
+
encoded_frames = torch.stack(encoded_frames)
|
| 120 |
+
|
| 121 |
+
if not return_dict:
|
| 122 |
+
return (encoded_frames, scales)
|
| 123 |
+
|
| 124 |
+
return EncodecEncoderOutput(encoded_frames, scales)
|
| 125 |
+
|
| 126 |
+
def decode(
|
| 127 |
+
self,
|
| 128 |
+
audio_codes,
|
| 129 |
+
audio_scales=None,
|
| 130 |
+
padding_mask=None,
|
| 131 |
+
return_dict=None,
|
| 132 |
+
):
|
| 133 |
+
"""
|
| 134 |
+
Decodes the given frames into an output audio waveform.
|
| 135 |
+
|
| 136 |
+
Note that the output might be a bit bigger than the input. In that case, any extra steps at the end can be
|
| 137 |
+
trimmed.
|
| 138 |
+
|
| 139 |
+
Args:
|
| 140 |
+
audio_codes (`torch.FloatTensor` of shape `(batch_size, nb_chunks, chunk_length)`, *optional*):
|
| 141 |
+
Discret code embeddings computed using `model.encode`.
|
| 142 |
+
audio_scales (`torch.Tensor` of shape `(batch_size, nb_chunks)`, *optional*):
|
| 143 |
+
Not used, kept to have the same inferface as HF encodec.
|
| 144 |
+
padding_mask (`torch.Tensor` of shape `(batch_size, channels, sequence_length)`):
|
| 145 |
+
Padding mask used to pad the `input_values`.
|
| 146 |
+
Not used yet, kept to have the same inferface as HF encodec.
|
| 147 |
+
return_dict (`bool`, *optional*):
|
| 148 |
+
Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
|
| 149 |
+
|
| 150 |
+
"""
|
| 151 |
+
return_dict = return_dict or self.config.return_dict
|
| 152 |
+
|
| 153 |
+
# TODO: for now, no chunk length
|
| 154 |
+
# input shape [1, 1, 8, seq_len]
|
| 155 |
+
if len(audio_codes) != 1:
|
| 156 |
+
raise ValueError(f"Expected one frame, got {len(audio_codes)}")
|
| 157 |
+
|
| 158 |
+
audio_codes = audio_codes.transpose(1, 2) # [1, 8, 1, seq_len]
|
| 159 |
+
|
| 160 |
+
# audio_values = self.model.quantizer.from_codes(audio_codes.squeeze(0))[0]
|
| 161 |
+
audio_values = self.model.decode(audio_codes.squeeze(0))
|
| 162 |
+
if not return_dict:
|
| 163 |
+
return (audio_values,)
|
| 164 |
+
return EncodecDecoderOutput(audio_values)
|
| 165 |
+
|
| 166 |
+
def forward(self, tensor):
|
| 167 |
+
raise ValueError("`XCodecModel.forward` not implemented yet")
|
songgen/xcodec_wrapper/xcodec_infer/ckpts/general_more/config_hubert_general.yaml
ADDED
|
@@ -0,0 +1,137 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
generator:
|
| 2 |
+
name: SoundStream
|
| 3 |
+
config:
|
| 4 |
+
n_filters: 32
|
| 5 |
+
D: 256
|
| 6 |
+
target_bandwidths:
|
| 7 |
+
- 0.5
|
| 8 |
+
- 1
|
| 9 |
+
- 1.5
|
| 10 |
+
- 2
|
| 11 |
+
- 4
|
| 12 |
+
ratios:
|
| 13 |
+
- 8
|
| 14 |
+
- 5
|
| 15 |
+
- 4
|
| 16 |
+
- 2
|
| 17 |
+
sample_rate: 16000
|
| 18 |
+
bins: 1024
|
| 19 |
+
semantic_techer: hubert_base_general
|
| 20 |
+
d_list:
|
| 21 |
+
- mfd
|
| 22 |
+
mfd:
|
| 23 |
+
name: MultiFrequencyDiscriminator
|
| 24 |
+
config:
|
| 25 |
+
hop_lengths:
|
| 26 |
+
- 32
|
| 27 |
+
- 64
|
| 28 |
+
- 128
|
| 29 |
+
- 256
|
| 30 |
+
- 512
|
| 31 |
+
- 1024
|
| 32 |
+
hidden_channels:
|
| 33 |
+
- 64
|
| 34 |
+
- 128
|
| 35 |
+
- 256
|
| 36 |
+
- 512
|
| 37 |
+
- 512
|
| 38 |
+
- 512
|
| 39 |
+
domain: double
|
| 40 |
+
mel_scale: true
|
| 41 |
+
sample_rate: 16000
|
| 42 |
+
mpd:
|
| 43 |
+
name: MultiPeriodDiscriminator
|
| 44 |
+
config:
|
| 45 |
+
period_sizes:
|
| 46 |
+
- 2
|
| 47 |
+
- 3
|
| 48 |
+
- 5
|
| 49 |
+
- 7
|
| 50 |
+
- 11
|
| 51 |
+
period_kernel_size: 5
|
| 52 |
+
msd:
|
| 53 |
+
name: MultiScaleDiscriminator
|
| 54 |
+
config:
|
| 55 |
+
num_scales: 3
|
| 56 |
+
pool_kernel_size: 4
|
| 57 |
+
pool_stride: 2
|
| 58 |
+
optimizer:
|
| 59 |
+
g:
|
| 60 |
+
name: AdamW
|
| 61 |
+
config:
|
| 62 |
+
lr: 0.0002
|
| 63 |
+
betas:
|
| 64 |
+
- 0.8
|
| 65 |
+
- 0.99
|
| 66 |
+
eps: 1.0e-06
|
| 67 |
+
d:
|
| 68 |
+
name: AdamW
|
| 69 |
+
config:
|
| 70 |
+
lr: 0.0002
|
| 71 |
+
betas:
|
| 72 |
+
- 0.8
|
| 73 |
+
- 0.99
|
| 74 |
+
eps: 1.0e-06
|
| 75 |
+
lr_scheduler:
|
| 76 |
+
g:
|
| 77 |
+
name: ExponentialLR
|
| 78 |
+
config:
|
| 79 |
+
gamma: 0.999
|
| 80 |
+
d:
|
| 81 |
+
name: ExponentialLR
|
| 82 |
+
config:
|
| 83 |
+
gamma: 0.999
|
| 84 |
+
criterion:
|
| 85 |
+
g_criterion:
|
| 86 |
+
name: losses.generator_loss.GeneratorSTFTLoss
|
| 87 |
+
config:
|
| 88 |
+
use_mel_loss: false
|
| 89 |
+
adv_criterion: MSEGLoss
|
| 90 |
+
mel_loss_weight: 45
|
| 91 |
+
use_feature_match: true
|
| 92 |
+
feat_match_loss_weight: 20
|
| 93 |
+
use_full_stft_loss: true
|
| 94 |
+
use_sub_stft_loss: true
|
| 95 |
+
full_stft_loss_weight: 1
|
| 96 |
+
sub_stft_loss_weight: 1
|
| 97 |
+
mel_scale_loss:
|
| 98 |
+
sampling_rate: 16000
|
| 99 |
+
n_fft: 1024
|
| 100 |
+
num_mels: 80
|
| 101 |
+
hop_size: 160
|
| 102 |
+
win_size: 800
|
| 103 |
+
fmin: 0
|
| 104 |
+
full_multi_scale_stft_loss:
|
| 105 |
+
fft_sizes:
|
| 106 |
+
- 512
|
| 107 |
+
- 1024
|
| 108 |
+
- 2048
|
| 109 |
+
win_sizes:
|
| 110 |
+
- 480
|
| 111 |
+
- 960
|
| 112 |
+
- 1200
|
| 113 |
+
hop_sizes:
|
| 114 |
+
- 120
|
| 115 |
+
- 240
|
| 116 |
+
- 300
|
| 117 |
+
sub_multi_scale_stft_loss:
|
| 118 |
+
num_bands: 6
|
| 119 |
+
fft_sizes:
|
| 120 |
+
- 128
|
| 121 |
+
- 256
|
| 122 |
+
- 256
|
| 123 |
+
win_sizes:
|
| 124 |
+
- 80
|
| 125 |
+
- 120
|
| 126 |
+
- 200
|
| 127 |
+
hop_sizes:
|
| 128 |
+
- 20
|
| 129 |
+
- 40
|
| 130 |
+
- 50
|
| 131 |
+
d_criterion:
|
| 132 |
+
name: losses.discriminator_loss.MSEDiscriminatorLoss
|
| 133 |
+
config: null
|
| 134 |
+
commit_loss_weight: 1.0
|
| 135 |
+
codebook_loss_weight: 100
|
| 136 |
+
audio_norm_scale: 0.95
|
| 137 |
+
|
songgen/xcodec_wrapper/xcodec_infer/ckpts/general_more/xcodec_hubert_general_audio_v2.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:262b4675ad000af6b4e7a30af1d7afecc79598c098944d15204a29708752d538
|
| 3 |
+
size 710182186
|
songgen/xcodec_wrapper/xcodec_infer/descriptaudiocodec/__init__.py
ADDED
|
File without changes
|
songgen/xcodec_wrapper/xcodec_infer/descriptaudiocodec/dac/__init__.py
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
__version__ = "1.0.0"
|
| 2 |
+
|
| 3 |
+
# preserved here for legacy reasons
|
| 4 |
+
__model_version__ = "latest"
|
| 5 |
+
|
| 6 |
+
import audiotools
|
| 7 |
+
|
| 8 |
+
audiotools.ml.BaseModel.INTERN += ["dac.**"]
|
| 9 |
+
audiotools.ml.BaseModel.EXTERN += ["einops"]
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
from . import nn
|
| 13 |
+
from . import model
|
| 14 |
+
from . import utils
|
| 15 |
+
from .model import DAC
|
| 16 |
+
from .model import DACFile
|
songgen/xcodec_wrapper/xcodec_infer/descriptaudiocodec/dac/model/__init__.py
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from .base import CodecMixin
|
| 2 |
+
from .base import DACFile
|
| 3 |
+
from .dac import DAC
|
| 4 |
+
from .discriminator import Discriminator
|
songgen/xcodec_wrapper/xcodec_infer/descriptaudiocodec/dac/model/base.py
ADDED
|
@@ -0,0 +1,294 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import math
|
| 2 |
+
from dataclasses import dataclass
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
from typing import Union
|
| 5 |
+
|
| 6 |
+
import numpy as np
|
| 7 |
+
import torch
|
| 8 |
+
import tqdm
|
| 9 |
+
from audiotools import AudioSignal
|
| 10 |
+
from torch import nn
|
| 11 |
+
|
| 12 |
+
SUPPORTED_VERSIONS = ["1.0.0"]
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
@dataclass
|
| 16 |
+
class DACFile:
|
| 17 |
+
codes: torch.Tensor
|
| 18 |
+
|
| 19 |
+
# Metadata
|
| 20 |
+
chunk_length: int
|
| 21 |
+
original_length: int
|
| 22 |
+
input_db: float
|
| 23 |
+
channels: int
|
| 24 |
+
sample_rate: int
|
| 25 |
+
padding: bool
|
| 26 |
+
dac_version: str
|
| 27 |
+
|
| 28 |
+
def save(self, path):
|
| 29 |
+
artifacts = {
|
| 30 |
+
"codes": self.codes.numpy().astype(np.uint16),
|
| 31 |
+
"metadata": {
|
| 32 |
+
"input_db": self.input_db.numpy().astype(np.float32),
|
| 33 |
+
"original_length": self.original_length,
|
| 34 |
+
"sample_rate": self.sample_rate,
|
| 35 |
+
"chunk_length": self.chunk_length,
|
| 36 |
+
"channels": self.channels,
|
| 37 |
+
"padding": self.padding,
|
| 38 |
+
"dac_version": SUPPORTED_VERSIONS[-1],
|
| 39 |
+
},
|
| 40 |
+
}
|
| 41 |
+
path = Path(path).with_suffix(".dac")
|
| 42 |
+
with open(path, "wb") as f:
|
| 43 |
+
np.save(f, artifacts)
|
| 44 |
+
return path
|
| 45 |
+
|
| 46 |
+
@classmethod
|
| 47 |
+
def load(cls, path):
|
| 48 |
+
artifacts = np.load(path, allow_pickle=True)[()]
|
| 49 |
+
codes = torch.from_numpy(artifacts["codes"].astype(int))
|
| 50 |
+
if artifacts["metadata"].get("dac_version", None) not in SUPPORTED_VERSIONS:
|
| 51 |
+
raise RuntimeError(
|
| 52 |
+
f"Given file {path} can't be loaded with this version of descript-audio-codec."
|
| 53 |
+
)
|
| 54 |
+
return cls(codes=codes, **artifacts["metadata"])
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
class CodecMixin:
|
| 58 |
+
@property
|
| 59 |
+
def padding(self):
|
| 60 |
+
if not hasattr(self, "_padding"):
|
| 61 |
+
self._padding = True
|
| 62 |
+
return self._padding
|
| 63 |
+
|
| 64 |
+
@padding.setter
|
| 65 |
+
def padding(self, value):
|
| 66 |
+
assert isinstance(value, bool)
|
| 67 |
+
|
| 68 |
+
layers = [
|
| 69 |
+
l for l in self.modules() if isinstance(l, (nn.Conv1d, nn.ConvTranspose1d))
|
| 70 |
+
]
|
| 71 |
+
|
| 72 |
+
for layer in layers:
|
| 73 |
+
if value:
|
| 74 |
+
if hasattr(layer, "original_padding"):
|
| 75 |
+
layer.padding = layer.original_padding
|
| 76 |
+
else:
|
| 77 |
+
layer.original_padding = layer.padding
|
| 78 |
+
layer.padding = tuple(0 for _ in range(len(layer.padding)))
|
| 79 |
+
|
| 80 |
+
self._padding = value
|
| 81 |
+
|
| 82 |
+
def get_delay(self):
|
| 83 |
+
# Any number works here, delay is invariant to input length
|
| 84 |
+
l_out = self.get_output_length(0)
|
| 85 |
+
L = l_out
|
| 86 |
+
|
| 87 |
+
layers = []
|
| 88 |
+
for layer in self.modules():
|
| 89 |
+
if isinstance(layer, (nn.Conv1d, nn.ConvTranspose1d)):
|
| 90 |
+
layers.append(layer)
|
| 91 |
+
|
| 92 |
+
for layer in reversed(layers):
|
| 93 |
+
d = layer.dilation[0]
|
| 94 |
+
k = layer.kernel_size[0]
|
| 95 |
+
s = layer.stride[0]
|
| 96 |
+
|
| 97 |
+
if isinstance(layer, nn.ConvTranspose1d):
|
| 98 |
+
L = ((L - d * (k - 1) - 1) / s) + 1
|
| 99 |
+
elif isinstance(layer, nn.Conv1d):
|
| 100 |
+
L = (L - 1) * s + d * (k - 1) + 1
|
| 101 |
+
|
| 102 |
+
L = math.ceil(L)
|
| 103 |
+
|
| 104 |
+
l_in = L
|
| 105 |
+
|
| 106 |
+
return (l_in - l_out) // 2
|
| 107 |
+
|
| 108 |
+
def get_output_length(self, input_length):
|
| 109 |
+
L = input_length
|
| 110 |
+
# Calculate output length
|
| 111 |
+
for layer in self.modules():
|
| 112 |
+
if isinstance(layer, (nn.Conv1d, nn.ConvTranspose1d)):
|
| 113 |
+
d = layer.dilation[0]
|
| 114 |
+
k = layer.kernel_size[0]
|
| 115 |
+
s = layer.stride[0]
|
| 116 |
+
|
| 117 |
+
if isinstance(layer, nn.Conv1d):
|
| 118 |
+
L = ((L - d * (k - 1) - 1) / s) + 1
|
| 119 |
+
elif isinstance(layer, nn.ConvTranspose1d):
|
| 120 |
+
L = (L - 1) * s + d * (k - 1) + 1
|
| 121 |
+
|
| 122 |
+
L = math.floor(L)
|
| 123 |
+
return L
|
| 124 |
+
|
| 125 |
+
@torch.no_grad()
|
| 126 |
+
def compress(
|
| 127 |
+
self,
|
| 128 |
+
audio_path_or_signal: Union[str, Path, AudioSignal],
|
| 129 |
+
win_duration: float = 1.0,
|
| 130 |
+
verbose: bool = False,
|
| 131 |
+
normalize_db: float = -16,
|
| 132 |
+
n_quantizers: int = None,
|
| 133 |
+
) -> DACFile:
|
| 134 |
+
"""Processes an audio signal from a file or AudioSignal object into
|
| 135 |
+
discrete codes. This function processes the signal in short windows,
|
| 136 |
+
using constant GPU memory.
|
| 137 |
+
|
| 138 |
+
Parameters
|
| 139 |
+
----------
|
| 140 |
+
audio_path_or_signal : Union[str, Path, AudioSignal]
|
| 141 |
+
audio signal to reconstruct
|
| 142 |
+
win_duration : float, optional
|
| 143 |
+
window duration in seconds, by default 5.0
|
| 144 |
+
verbose : bool, optional
|
| 145 |
+
by default False
|
| 146 |
+
normalize_db : float, optional
|
| 147 |
+
normalize db, by default -16
|
| 148 |
+
|
| 149 |
+
Returns
|
| 150 |
+
-------
|
| 151 |
+
DACFile
|
| 152 |
+
Object containing compressed codes and metadata
|
| 153 |
+
required for decompression
|
| 154 |
+
"""
|
| 155 |
+
audio_signal = audio_path_or_signal
|
| 156 |
+
if isinstance(audio_signal, (str, Path)):
|
| 157 |
+
audio_signal = AudioSignal.load_from_file_with_ffmpeg(str(audio_signal))
|
| 158 |
+
|
| 159 |
+
self.eval()
|
| 160 |
+
original_padding = self.padding
|
| 161 |
+
original_device = audio_signal.device
|
| 162 |
+
|
| 163 |
+
audio_signal = audio_signal.clone()
|
| 164 |
+
original_sr = audio_signal.sample_rate
|
| 165 |
+
|
| 166 |
+
resample_fn = audio_signal.resample
|
| 167 |
+
loudness_fn = audio_signal.loudness
|
| 168 |
+
|
| 169 |
+
# If audio is > 10 minutes long, use the ffmpeg versions
|
| 170 |
+
if audio_signal.signal_duration >= 10 * 60 * 60:
|
| 171 |
+
resample_fn = audio_signal.ffmpeg_resample
|
| 172 |
+
loudness_fn = audio_signal.ffmpeg_loudness
|
| 173 |
+
|
| 174 |
+
original_length = audio_signal.signal_length
|
| 175 |
+
resample_fn(self.sample_rate)
|
| 176 |
+
input_db = loudness_fn()
|
| 177 |
+
|
| 178 |
+
if normalize_db is not None:
|
| 179 |
+
audio_signal.normalize(normalize_db)
|
| 180 |
+
audio_signal.ensure_max_of_audio()
|
| 181 |
+
|
| 182 |
+
nb, nac, nt = audio_signal.audio_data.shape
|
| 183 |
+
audio_signal.audio_data = audio_signal.audio_data.reshape(nb * nac, 1, nt)
|
| 184 |
+
win_duration = (
|
| 185 |
+
audio_signal.signal_duration if win_duration is None else win_duration
|
| 186 |
+
)
|
| 187 |
+
|
| 188 |
+
if audio_signal.signal_duration <= win_duration:
|
| 189 |
+
# Unchunked compression (used if signal length < win duration)
|
| 190 |
+
self.padding = True
|
| 191 |
+
n_samples = nt
|
| 192 |
+
hop = nt
|
| 193 |
+
else:
|
| 194 |
+
# Chunked inference
|
| 195 |
+
self.padding = False
|
| 196 |
+
# Zero-pad signal on either side by the delay
|
| 197 |
+
audio_signal.zero_pad(self.delay, self.delay)
|
| 198 |
+
n_samples = int(win_duration * self.sample_rate)
|
| 199 |
+
# Round n_samples to nearest hop length multiple
|
| 200 |
+
n_samples = int(math.ceil(n_samples / self.hop_length) * self.hop_length)
|
| 201 |
+
hop = self.get_output_length(n_samples)
|
| 202 |
+
|
| 203 |
+
codes = []
|
| 204 |
+
range_fn = range if not verbose else tqdm.trange
|
| 205 |
+
|
| 206 |
+
for i in range_fn(0, nt, hop):
|
| 207 |
+
x = audio_signal[..., i : i + n_samples]
|
| 208 |
+
x = x.zero_pad(0, max(0, n_samples - x.shape[-1]))
|
| 209 |
+
|
| 210 |
+
audio_data = x.audio_data.to(self.device)
|
| 211 |
+
audio_data = self.preprocess(audio_data, self.sample_rate)
|
| 212 |
+
_, c, _, _, _ = self.encode(audio_data, n_quantizers)
|
| 213 |
+
codes.append(c.to(original_device))
|
| 214 |
+
chunk_length = c.shape[-1]
|
| 215 |
+
|
| 216 |
+
codes = torch.cat(codes, dim=-1)
|
| 217 |
+
|
| 218 |
+
dac_file = DACFile(
|
| 219 |
+
codes=codes,
|
| 220 |
+
chunk_length=chunk_length,
|
| 221 |
+
original_length=original_length,
|
| 222 |
+
input_db=input_db,
|
| 223 |
+
channels=nac,
|
| 224 |
+
sample_rate=original_sr,
|
| 225 |
+
padding=self.padding,
|
| 226 |
+
dac_version=SUPPORTED_VERSIONS[-1],
|
| 227 |
+
)
|
| 228 |
+
|
| 229 |
+
if n_quantizers is not None:
|
| 230 |
+
codes = codes[:, :n_quantizers, :]
|
| 231 |
+
|
| 232 |
+
self.padding = original_padding
|
| 233 |
+
return dac_file
|
| 234 |
+
|
| 235 |
+
@torch.no_grad()
|
| 236 |
+
def decompress(
|
| 237 |
+
self,
|
| 238 |
+
obj: Union[str, Path, DACFile],
|
| 239 |
+
verbose: bool = False,
|
| 240 |
+
) -> AudioSignal:
|
| 241 |
+
"""Reconstruct audio from a given .dac file
|
| 242 |
+
|
| 243 |
+
Parameters
|
| 244 |
+
----------
|
| 245 |
+
obj : Union[str, Path, DACFile]
|
| 246 |
+
.dac file location or corresponding DACFile object.
|
| 247 |
+
verbose : bool, optional
|
| 248 |
+
Prints progress if True, by default False
|
| 249 |
+
|
| 250 |
+
Returns
|
| 251 |
+
-------
|
| 252 |
+
AudioSignal
|
| 253 |
+
Object with the reconstructed audio
|
| 254 |
+
"""
|
| 255 |
+
self.eval()
|
| 256 |
+
if isinstance(obj, (str, Path)):
|
| 257 |
+
obj = DACFile.load(obj)
|
| 258 |
+
|
| 259 |
+
original_padding = self.padding
|
| 260 |
+
self.padding = obj.padding
|
| 261 |
+
|
| 262 |
+
range_fn = range if not verbose else tqdm.trange
|
| 263 |
+
codes = obj.codes
|
| 264 |
+
original_device = codes.device
|
| 265 |
+
chunk_length = obj.chunk_length
|
| 266 |
+
recons = []
|
| 267 |
+
|
| 268 |
+
for i in range_fn(0, codes.shape[-1], chunk_length):
|
| 269 |
+
c = codes[..., i : i + chunk_length].to(self.device)
|
| 270 |
+
z = self.quantizer.from_codes(c)[0]
|
| 271 |
+
r = self.decode(z)
|
| 272 |
+
recons.append(r.to(original_device))
|
| 273 |
+
|
| 274 |
+
recons = torch.cat(recons, dim=-1)
|
| 275 |
+
recons = AudioSignal(recons, self.sample_rate)
|
| 276 |
+
|
| 277 |
+
resample_fn = recons.resample
|
| 278 |
+
loudness_fn = recons.loudness
|
| 279 |
+
|
| 280 |
+
# If audio is > 10 minutes long, use the ffmpeg versions
|
| 281 |
+
if recons.signal_duration >= 10 * 60 * 60:
|
| 282 |
+
resample_fn = recons.ffmpeg_resample
|
| 283 |
+
loudness_fn = recons.ffmpeg_loudness
|
| 284 |
+
|
| 285 |
+
recons.normalize(obj.input_db)
|
| 286 |
+
resample_fn(obj.sample_rate)
|
| 287 |
+
recons = recons[..., : obj.original_length]
|
| 288 |
+
loudness_fn()
|
| 289 |
+
recons.audio_data = recons.audio_data.reshape(
|
| 290 |
+
-1, obj.channels, obj.original_length
|
| 291 |
+
)
|
| 292 |
+
|
| 293 |
+
self.padding = original_padding
|
| 294 |
+
return recons
|
songgen/xcodec_wrapper/xcodec_infer/descriptaudiocodec/dac/model/dac.py
ADDED
|
@@ -0,0 +1,369 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import math
|
| 2 |
+
from typing import List
|
| 3 |
+
from typing import Union
|
| 4 |
+
|
| 5 |
+
import numpy as np
|
| 6 |
+
import torch
|
| 7 |
+
from audiotools import AudioSignal
|
| 8 |
+
from audiotools.ml import BaseModel
|
| 9 |
+
from torch import nn
|
| 10 |
+
|
| 11 |
+
from .base import CodecMixin
|
| 12 |
+
from dac.nn.layers import Snake1d
|
| 13 |
+
from dac.nn.layers import WNConv1d
|
| 14 |
+
from dac.nn.layers import WNConvTranspose1d
|
| 15 |
+
from dac.nn.quantize import ResidualVectorQuantize
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def init_weights(m):
|
| 19 |
+
if isinstance(m, nn.Conv1d):
|
| 20 |
+
nn.init.trunc_normal_(m.weight, std=0.02)
|
| 21 |
+
nn.init.constant_(m.bias, 0)
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
class ResidualUnit(nn.Module):
|
| 25 |
+
def __init__(self, dim: int = 16, dilation: int = 1):
|
| 26 |
+
super().__init__()
|
| 27 |
+
pad = ((7 - 1) * dilation) // 2
|
| 28 |
+
self.block = nn.Sequential(
|
| 29 |
+
Snake1d(dim),
|
| 30 |
+
WNConv1d(dim, dim, kernel_size=7, dilation=dilation, padding=pad),
|
| 31 |
+
Snake1d(dim),
|
| 32 |
+
WNConv1d(dim, dim, kernel_size=1),
|
| 33 |
+
)
|
| 34 |
+
|
| 35 |
+
def forward(self, x):
|
| 36 |
+
y = self.block(x)
|
| 37 |
+
pad = (x.shape[-1] - y.shape[-1]) // 2
|
| 38 |
+
if pad > 0:
|
| 39 |
+
x = x[..., pad:-pad]
|
| 40 |
+
return x + y
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
class EncoderBlock(nn.Module):
|
| 44 |
+
def __init__(self, dim: int = 16, stride: int = 1):
|
| 45 |
+
super().__init__()
|
| 46 |
+
self.block = nn.Sequential(
|
| 47 |
+
ResidualUnit(dim // 2, dilation=1),
|
| 48 |
+
ResidualUnit(dim // 2, dilation=3),
|
| 49 |
+
ResidualUnit(dim // 2, dilation=9),
|
| 50 |
+
Snake1d(dim // 2),
|
| 51 |
+
WNConv1d(
|
| 52 |
+
dim // 2,
|
| 53 |
+
dim,
|
| 54 |
+
kernel_size=2 * stride,
|
| 55 |
+
stride=stride,
|
| 56 |
+
padding=math.ceil(stride / 2),
|
| 57 |
+
),
|
| 58 |
+
)
|
| 59 |
+
|
| 60 |
+
def forward(self, x):
|
| 61 |
+
return self.block(x)
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
class Encoder(nn.Module):
|
| 65 |
+
def __init__(
|
| 66 |
+
self,
|
| 67 |
+
d_model: int = 64,
|
| 68 |
+
strides: list = [2, 4, 8, 8],
|
| 69 |
+
d_latent: int = 256,
|
| 70 |
+
):
|
| 71 |
+
super().__init__()
|
| 72 |
+
# Create first convolution
|
| 73 |
+
self.block = [WNConv1d(1, d_model, kernel_size=7, padding=3)]
|
| 74 |
+
|
| 75 |
+
# Create EncoderBlocks that double channels as they downsample by `stride`
|
| 76 |
+
for stride in strides:
|
| 77 |
+
d_model *= 2
|
| 78 |
+
self.block += [EncoderBlock(d_model, stride=stride)]
|
| 79 |
+
|
| 80 |
+
# Create last convolution
|
| 81 |
+
self.block += [
|
| 82 |
+
Snake1d(d_model),
|
| 83 |
+
WNConv1d(d_model, d_latent, kernel_size=3, padding=1),
|
| 84 |
+
]
|
| 85 |
+
|
| 86 |
+
# Wrap black into nn.Sequential
|
| 87 |
+
self.block = nn.Sequential(*self.block)
|
| 88 |
+
self.enc_dim = d_model
|
| 89 |
+
|
| 90 |
+
def forward(self, x):
|
| 91 |
+
return self.block(x)
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
class DecoderBlock(nn.Module):
|
| 95 |
+
def __init__(self, input_dim: int = 16, output_dim: int = 8, stride: int = 1,out_pad=0):
|
| 96 |
+
super().__init__()
|
| 97 |
+
self.block = nn.Sequential(
|
| 98 |
+
Snake1d(input_dim),
|
| 99 |
+
WNConvTranspose1d(
|
| 100 |
+
input_dim,
|
| 101 |
+
output_dim,
|
| 102 |
+
kernel_size=2 * stride,
|
| 103 |
+
stride=stride,
|
| 104 |
+
padding=math.ceil(stride / 2),
|
| 105 |
+
output_padding= stride % 2, #out_pad,
|
| 106 |
+
),
|
| 107 |
+
ResidualUnit(output_dim, dilation=1),
|
| 108 |
+
ResidualUnit(output_dim, dilation=3),
|
| 109 |
+
ResidualUnit(output_dim, dilation=9),
|
| 110 |
+
)
|
| 111 |
+
|
| 112 |
+
def forward(self, x):
|
| 113 |
+
return self.block(x)
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
class Decoder(nn.Module):
|
| 117 |
+
def __init__(
|
| 118 |
+
self,
|
| 119 |
+
input_channel,
|
| 120 |
+
channels,
|
| 121 |
+
rates,
|
| 122 |
+
d_out: int = 1,
|
| 123 |
+
):
|
| 124 |
+
super().__init__()
|
| 125 |
+
|
| 126 |
+
# Add first conv layer
|
| 127 |
+
layers = [WNConv1d(input_channel, channels, kernel_size=7, padding=3)]
|
| 128 |
+
|
| 129 |
+
# Add upsampling + MRF blocks
|
| 130 |
+
for i, stride in enumerate(rates):
|
| 131 |
+
input_dim = channels // 2**i
|
| 132 |
+
output_dim = channels // 2 ** (i + 1)
|
| 133 |
+
if i==1:
|
| 134 |
+
out_pad=1
|
| 135 |
+
else:
|
| 136 |
+
out_pad=0
|
| 137 |
+
layers += [DecoderBlock(input_dim, output_dim, stride,out_pad)]
|
| 138 |
+
|
| 139 |
+
# Add final conv layer
|
| 140 |
+
layers += [
|
| 141 |
+
Snake1d(output_dim),
|
| 142 |
+
WNConv1d(output_dim, d_out, kernel_size=7, padding=3),
|
| 143 |
+
# nn.Tanh(),
|
| 144 |
+
]
|
| 145 |
+
|
| 146 |
+
self.model = nn.Sequential(*layers)
|
| 147 |
+
|
| 148 |
+
def forward(self, x):
|
| 149 |
+
return self.model(x)
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
class DAC(BaseModel, CodecMixin):
|
| 153 |
+
def __init__(
|
| 154 |
+
self,
|
| 155 |
+
encoder_dim: int = 64,
|
| 156 |
+
encoder_rates: List[int] = [2, 4, 8, 8],
|
| 157 |
+
latent_dim: int = None,
|
| 158 |
+
decoder_dim: int = 1536,
|
| 159 |
+
decoder_rates: List[int] = [8, 8, 4, 2],
|
| 160 |
+
n_codebooks: int = 9,
|
| 161 |
+
codebook_size: int = 1024,
|
| 162 |
+
codebook_dim: Union[int, list] = 8,
|
| 163 |
+
quantizer_dropout: bool = False,
|
| 164 |
+
sample_rate: int = 44100,
|
| 165 |
+
):
|
| 166 |
+
super().__init__()
|
| 167 |
+
|
| 168 |
+
self.encoder_dim = encoder_dim
|
| 169 |
+
self.encoder_rates = encoder_rates
|
| 170 |
+
self.decoder_dim = decoder_dim
|
| 171 |
+
self.decoder_rates = decoder_rates
|
| 172 |
+
self.sample_rate = sample_rate
|
| 173 |
+
|
| 174 |
+
if latent_dim is None:
|
| 175 |
+
latent_dim = encoder_dim * (2 ** len(encoder_rates))
|
| 176 |
+
|
| 177 |
+
self.latent_dim = latent_dim
|
| 178 |
+
|
| 179 |
+
self.hop_length = np.prod(encoder_rates)
|
| 180 |
+
self.encoder = Encoder(encoder_dim, encoder_rates, latent_dim)
|
| 181 |
+
|
| 182 |
+
self.n_codebooks = n_codebooks
|
| 183 |
+
self.codebook_size = codebook_size
|
| 184 |
+
self.codebook_dim = codebook_dim
|
| 185 |
+
self.quantizer = ResidualVectorQuantize(
|
| 186 |
+
input_dim=latent_dim,
|
| 187 |
+
n_codebooks=n_codebooks,
|
| 188 |
+
codebook_size=codebook_size,
|
| 189 |
+
codebook_dim=codebook_dim,
|
| 190 |
+
quantizer_dropout=quantizer_dropout,
|
| 191 |
+
)
|
| 192 |
+
|
| 193 |
+
self.decoder = Decoder(
|
| 194 |
+
latent_dim,
|
| 195 |
+
decoder_dim,
|
| 196 |
+
decoder_rates,
|
| 197 |
+
)
|
| 198 |
+
self.sample_rate = sample_rate
|
| 199 |
+
self.apply(init_weights)
|
| 200 |
+
|
| 201 |
+
self.delay = self.get_delay()
|
| 202 |
+
|
| 203 |
+
def preprocess(self, audio_data, sample_rate):
|
| 204 |
+
if sample_rate is None:
|
| 205 |
+
sample_rate = self.sample_rate
|
| 206 |
+
assert sample_rate == self.sample_rate
|
| 207 |
+
|
| 208 |
+
length = audio_data.shape[-1]
|
| 209 |
+
right_pad = math.ceil(length / self.hop_length) * self.hop_length - length
|
| 210 |
+
audio_data = nn.functional.pad(audio_data, (0, right_pad))
|
| 211 |
+
|
| 212 |
+
return audio_data
|
| 213 |
+
|
| 214 |
+
def encode(
|
| 215 |
+
self,
|
| 216 |
+
audio_data: torch.Tensor,
|
| 217 |
+
n_quantizers: int = None,
|
| 218 |
+
):
|
| 219 |
+
"""Encode given audio data and return quantized latent codes
|
| 220 |
+
|
| 221 |
+
Parameters
|
| 222 |
+
----------
|
| 223 |
+
audio_data : Tensor[B x 1 x T]
|
| 224 |
+
Audio data to encode
|
| 225 |
+
n_quantizers : int, optional
|
| 226 |
+
Number of quantizers to use, by default None
|
| 227 |
+
If None, all quantizers are used.
|
| 228 |
+
|
| 229 |
+
Returns
|
| 230 |
+
-------
|
| 231 |
+
dict
|
| 232 |
+
A dictionary with the following keys:
|
| 233 |
+
"z" : Tensor[B x D x T]
|
| 234 |
+
Quantized continuous representation of input
|
| 235 |
+
"codes" : Tensor[B x N x T]
|
| 236 |
+
Codebook indices for each codebook
|
| 237 |
+
(quantized discrete representation of input)
|
| 238 |
+
"latents" : Tensor[B x N*D x T]
|
| 239 |
+
Projected latents (continuous representation of input before quantization)
|
| 240 |
+
"vq/commitment_loss" : Tensor[1]
|
| 241 |
+
Commitment loss to train encoder to predict vectors closer to codebook
|
| 242 |
+
entries
|
| 243 |
+
"vq/codebook_loss" : Tensor[1]
|
| 244 |
+
Codebook loss to update the codebook
|
| 245 |
+
"length" : int
|
| 246 |
+
Number of samples in input audio
|
| 247 |
+
"""
|
| 248 |
+
z = self.encoder(audio_data)
|
| 249 |
+
z, codes, latents, commitment_loss, codebook_loss = self.quantizer(
|
| 250 |
+
z, n_quantizers
|
| 251 |
+
)
|
| 252 |
+
return z, codes, latents, commitment_loss, codebook_loss
|
| 253 |
+
|
| 254 |
+
def decode(self, z: torch.Tensor):
|
| 255 |
+
"""Decode given latent codes and return audio data
|
| 256 |
+
|
| 257 |
+
Parameters
|
| 258 |
+
----------
|
| 259 |
+
z : Tensor[B x D x T]
|
| 260 |
+
Quantized continuous representation of input
|
| 261 |
+
length : int, optional
|
| 262 |
+
Number of samples in output audio, by default None
|
| 263 |
+
|
| 264 |
+
Returns
|
| 265 |
+
-------
|
| 266 |
+
dict
|
| 267 |
+
A dictionary with the following keys:
|
| 268 |
+
"audio" : Tensor[B x 1 x length]
|
| 269 |
+
Decoded audio data.
|
| 270 |
+
"""
|
| 271 |
+
return self.decoder(z)
|
| 272 |
+
|
| 273 |
+
def forward(
|
| 274 |
+
self,
|
| 275 |
+
audio_data: torch.Tensor,
|
| 276 |
+
sample_rate: int = None,
|
| 277 |
+
n_quantizers: int = None,
|
| 278 |
+
):
|
| 279 |
+
"""Model forward pass
|
| 280 |
+
|
| 281 |
+
Parameters
|
| 282 |
+
----------
|
| 283 |
+
audio_data : Tensor[B x 1 x T]
|
| 284 |
+
Audio data to encode
|
| 285 |
+
sample_rate : int, optional
|
| 286 |
+
Sample rate of audio data in Hz, by default None
|
| 287 |
+
If None, defaults to `self.sample_rate`
|
| 288 |
+
n_quantizers : int, optional
|
| 289 |
+
Number of quantizers to use, by default None.
|
| 290 |
+
If None, all quantizers are used.
|
| 291 |
+
|
| 292 |
+
Returns
|
| 293 |
+
-------
|
| 294 |
+
dict
|
| 295 |
+
A dictionary with the following keys:
|
| 296 |
+
"z" : Tensor[B x D x T]
|
| 297 |
+
Quantized continuous representation of input
|
| 298 |
+
"codes" : Tensor[B x N x T]
|
| 299 |
+
Codebook indices for each codebook
|
| 300 |
+
(quantized discrete representation of input)
|
| 301 |
+
"latents" : Tensor[B x N*D x T]
|
| 302 |
+
Projected latents (continuous representation of input before quantization)
|
| 303 |
+
"vq/commitment_loss" : Tensor[1]
|
| 304 |
+
Commitment loss to train encoder to predict vectors closer to codebook
|
| 305 |
+
entries
|
| 306 |
+
"vq/codebook_loss" : Tensor[1]
|
| 307 |
+
Codebook loss to update the codebook
|
| 308 |
+
"length" : int
|
| 309 |
+
Number of samples in input audio
|
| 310 |
+
"audio" : Tensor[B x 1 x length]
|
| 311 |
+
Decoded audio data.
|
| 312 |
+
"""
|
| 313 |
+
length = audio_data.shape[-1]
|
| 314 |
+
audio_data = self.preprocess(audio_data, sample_rate)
|
| 315 |
+
z, codes, latents, commitment_loss, codebook_loss = self.encode(
|
| 316 |
+
audio_data, n_quantizers
|
| 317 |
+
)
|
| 318 |
+
|
| 319 |
+
x = self.decode(z)
|
| 320 |
+
return {
|
| 321 |
+
"audio": x[..., :length],
|
| 322 |
+
"z": z,
|
| 323 |
+
"codes": codes,
|
| 324 |
+
"latents": latents,
|
| 325 |
+
"vq/commitment_loss": commitment_loss,
|
| 326 |
+
"vq/codebook_loss": codebook_loss,
|
| 327 |
+
}
|
| 328 |
+
|
| 329 |
+
|
| 330 |
+
if __name__ == "__main__":
|
| 331 |
+
import numpy as np
|
| 332 |
+
from functools import partial
|
| 333 |
+
|
| 334 |
+
model = DAC().to("cpu")
|
| 335 |
+
|
| 336 |
+
for n, m in model.named_modules():
|
| 337 |
+
o = m.extra_repr()
|
| 338 |
+
p = sum([np.prod(p.size()) for p in m.parameters()])
|
| 339 |
+
fn = lambda o, p: o + f" {p/1e6:<.3f}M params."
|
| 340 |
+
setattr(m, "extra_repr", partial(fn, o=o, p=p))
|
| 341 |
+
print(model)
|
| 342 |
+
print("Total # of params: ", sum([np.prod(p.size()) for p in model.parameters()]))
|
| 343 |
+
|
| 344 |
+
length = 88200 * 2
|
| 345 |
+
x = torch.randn(1, 1, length).to(model.device)
|
| 346 |
+
x.requires_grad_(True)
|
| 347 |
+
x.retain_grad()
|
| 348 |
+
|
| 349 |
+
# Make a forward pass
|
| 350 |
+
out = model(x)["audio"]
|
| 351 |
+
print("Input shape:", x.shape)
|
| 352 |
+
print("Output shape:", out.shape)
|
| 353 |
+
|
| 354 |
+
# Create gradient variable
|
| 355 |
+
grad = torch.zeros_like(out)
|
| 356 |
+
grad[:, :, grad.shape[-1] // 2] = 1
|
| 357 |
+
|
| 358 |
+
# Make a backward pass
|
| 359 |
+
out.backward(grad)
|
| 360 |
+
|
| 361 |
+
# Check non-zero values
|
| 362 |
+
gradmap = x.grad.squeeze(0)
|
| 363 |
+
gradmap = (gradmap != 0).sum(0) # sum across features
|
| 364 |
+
rf = (gradmap != 0).sum()
|
| 365 |
+
|
| 366 |
+
print(f"Receptive field: {rf.item()}")
|
| 367 |
+
|
| 368 |
+
x = AudioSignal(torch.randn(1, 1, 44100 * 60), 44100)
|
| 369 |
+
model.decompress(model.compress(x, verbose=True), verbose=True)
|
songgen/xcodec_wrapper/xcodec_infer/descriptaudiocodec/dac/model/discriminator.py
ADDED
|
@@ -0,0 +1,228 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torch.nn as nn
|
| 3 |
+
import torch.nn.functional as F
|
| 4 |
+
from audiotools import AudioSignal
|
| 5 |
+
from audiotools import ml
|
| 6 |
+
from audiotools import STFTParams
|
| 7 |
+
from einops import rearrange
|
| 8 |
+
from torch.nn.utils import weight_norm
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def WNConv1d(*args, **kwargs):
|
| 12 |
+
act = kwargs.pop("act", True)
|
| 13 |
+
conv = weight_norm(nn.Conv1d(*args, **kwargs))
|
| 14 |
+
if not act:
|
| 15 |
+
return conv
|
| 16 |
+
return nn.Sequential(conv, nn.LeakyReLU(0.1))
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def WNConv2d(*args, **kwargs):
|
| 20 |
+
act = kwargs.pop("act", True)
|
| 21 |
+
conv = weight_norm(nn.Conv2d(*args, **kwargs))
|
| 22 |
+
if not act:
|
| 23 |
+
return conv
|
| 24 |
+
return nn.Sequential(conv, nn.LeakyReLU(0.1))
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
class MPD(nn.Module):
|
| 28 |
+
def __init__(self, period):
|
| 29 |
+
super().__init__()
|
| 30 |
+
self.period = period
|
| 31 |
+
self.convs = nn.ModuleList(
|
| 32 |
+
[
|
| 33 |
+
WNConv2d(1, 32, (5, 1), (3, 1), padding=(2, 0)),
|
| 34 |
+
WNConv2d(32, 128, (5, 1), (3, 1), padding=(2, 0)),
|
| 35 |
+
WNConv2d(128, 512, (5, 1), (3, 1), padding=(2, 0)),
|
| 36 |
+
WNConv2d(512, 1024, (5, 1), (3, 1), padding=(2, 0)),
|
| 37 |
+
WNConv2d(1024, 1024, (5, 1), 1, padding=(2, 0)),
|
| 38 |
+
]
|
| 39 |
+
)
|
| 40 |
+
self.conv_post = WNConv2d(
|
| 41 |
+
1024, 1, kernel_size=(3, 1), padding=(1, 0), act=False
|
| 42 |
+
)
|
| 43 |
+
|
| 44 |
+
def pad_to_period(self, x):
|
| 45 |
+
t = x.shape[-1]
|
| 46 |
+
x = F.pad(x, (0, self.period - t % self.period), mode="reflect")
|
| 47 |
+
return x
|
| 48 |
+
|
| 49 |
+
def forward(self, x):
|
| 50 |
+
fmap = []
|
| 51 |
+
|
| 52 |
+
x = self.pad_to_period(x)
|
| 53 |
+
x = rearrange(x, "b c (l p) -> b c l p", p=self.period)
|
| 54 |
+
|
| 55 |
+
for layer in self.convs:
|
| 56 |
+
x = layer(x)
|
| 57 |
+
fmap.append(x)
|
| 58 |
+
|
| 59 |
+
x = self.conv_post(x)
|
| 60 |
+
fmap.append(x)
|
| 61 |
+
|
| 62 |
+
return fmap
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
class MSD(nn.Module):
|
| 66 |
+
def __init__(self, rate: int = 1, sample_rate: int = 44100):
|
| 67 |
+
super().__init__()
|
| 68 |
+
self.convs = nn.ModuleList(
|
| 69 |
+
[
|
| 70 |
+
WNConv1d(1, 16, 15, 1, padding=7),
|
| 71 |
+
WNConv1d(16, 64, 41, 4, groups=4, padding=20),
|
| 72 |
+
WNConv1d(64, 256, 41, 4, groups=16, padding=20),
|
| 73 |
+
WNConv1d(256, 1024, 41, 4, groups=64, padding=20),
|
| 74 |
+
WNConv1d(1024, 1024, 41, 4, groups=256, padding=20),
|
| 75 |
+
WNConv1d(1024, 1024, 5, 1, padding=2),
|
| 76 |
+
]
|
| 77 |
+
)
|
| 78 |
+
self.conv_post = WNConv1d(1024, 1, 3, 1, padding=1, act=False)
|
| 79 |
+
self.sample_rate = sample_rate
|
| 80 |
+
self.rate = rate
|
| 81 |
+
|
| 82 |
+
def forward(self, x):
|
| 83 |
+
x = AudioSignal(x, self.sample_rate)
|
| 84 |
+
x.resample(self.sample_rate // self.rate)
|
| 85 |
+
x = x.audio_data
|
| 86 |
+
|
| 87 |
+
fmap = []
|
| 88 |
+
|
| 89 |
+
for l in self.convs:
|
| 90 |
+
x = l(x)
|
| 91 |
+
fmap.append(x)
|
| 92 |
+
x = self.conv_post(x)
|
| 93 |
+
fmap.append(x)
|
| 94 |
+
|
| 95 |
+
return fmap
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
BANDS = [(0.0, 0.1), (0.1, 0.25), (0.25, 0.5), (0.5, 0.75), (0.75, 1.0)]
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
class MRD(nn.Module):
|
| 102 |
+
def __init__(
|
| 103 |
+
self,
|
| 104 |
+
window_length: int,
|
| 105 |
+
hop_factor: float = 0.25,
|
| 106 |
+
sample_rate: int = 44100,
|
| 107 |
+
bands: list = BANDS,
|
| 108 |
+
):
|
| 109 |
+
"""Complex multi-band spectrogram discriminator.
|
| 110 |
+
Parameters
|
| 111 |
+
----------
|
| 112 |
+
window_length : int
|
| 113 |
+
Window length of STFT.
|
| 114 |
+
hop_factor : float, optional
|
| 115 |
+
Hop factor of the STFT, defaults to ``0.25 * window_length``.
|
| 116 |
+
sample_rate : int, optional
|
| 117 |
+
Sampling rate of audio in Hz, by default 44100
|
| 118 |
+
bands : list, optional
|
| 119 |
+
Bands to run discriminator over.
|
| 120 |
+
"""
|
| 121 |
+
super().__init__()
|
| 122 |
+
|
| 123 |
+
self.window_length = window_length
|
| 124 |
+
self.hop_factor = hop_factor
|
| 125 |
+
self.sample_rate = sample_rate
|
| 126 |
+
self.stft_params = STFTParams(
|
| 127 |
+
window_length=window_length,
|
| 128 |
+
hop_length=int(window_length * hop_factor),
|
| 129 |
+
match_stride=True,
|
| 130 |
+
)
|
| 131 |
+
|
| 132 |
+
n_fft = window_length // 2 + 1
|
| 133 |
+
bands = [(int(b[0] * n_fft), int(b[1] * n_fft)) for b in bands]
|
| 134 |
+
self.bands = bands
|
| 135 |
+
|
| 136 |
+
ch = 32
|
| 137 |
+
convs = lambda: nn.ModuleList(
|
| 138 |
+
[
|
| 139 |
+
WNConv2d(2, ch, (3, 9), (1, 1), padding=(1, 4)),
|
| 140 |
+
WNConv2d(ch, ch, (3, 9), (1, 2), padding=(1, 4)),
|
| 141 |
+
WNConv2d(ch, ch, (3, 9), (1, 2), padding=(1, 4)),
|
| 142 |
+
WNConv2d(ch, ch, (3, 9), (1, 2), padding=(1, 4)),
|
| 143 |
+
WNConv2d(ch, ch, (3, 3), (1, 1), padding=(1, 1)),
|
| 144 |
+
]
|
| 145 |
+
)
|
| 146 |
+
self.band_convs = nn.ModuleList([convs() for _ in range(len(self.bands))])
|
| 147 |
+
self.conv_post = WNConv2d(ch, 1, (3, 3), (1, 1), padding=(1, 1), act=False)
|
| 148 |
+
|
| 149 |
+
def spectrogram(self, x):
|
| 150 |
+
x = AudioSignal(x, self.sample_rate, stft_params=self.stft_params)
|
| 151 |
+
x = torch.view_as_real(x.stft())
|
| 152 |
+
x = rearrange(x, "b 1 f t c -> (b 1) c t f")
|
| 153 |
+
# Split into bands
|
| 154 |
+
x_bands = [x[..., b[0] : b[1]] for b in self.bands]
|
| 155 |
+
return x_bands
|
| 156 |
+
|
| 157 |
+
def forward(self, x):
|
| 158 |
+
x_bands = self.spectrogram(x)
|
| 159 |
+
fmap = []
|
| 160 |
+
|
| 161 |
+
x = []
|
| 162 |
+
for band, stack in zip(x_bands, self.band_convs):
|
| 163 |
+
for layer in stack:
|
| 164 |
+
band = layer(band)
|
| 165 |
+
fmap.append(band)
|
| 166 |
+
x.append(band)
|
| 167 |
+
|
| 168 |
+
x = torch.cat(x, dim=-1)
|
| 169 |
+
x = self.conv_post(x)
|
| 170 |
+
fmap.append(x)
|
| 171 |
+
|
| 172 |
+
return fmap
|
| 173 |
+
|
| 174 |
+
|
| 175 |
+
class Discriminator(ml.BaseModel):
|
| 176 |
+
def __init__(
|
| 177 |
+
self,
|
| 178 |
+
rates: list = [],
|
| 179 |
+
periods: list = [2, 3, 5, 7, 11],
|
| 180 |
+
fft_sizes: list = [2048, 1024, 512],
|
| 181 |
+
sample_rate: int = 44100,
|
| 182 |
+
bands: list = BANDS,
|
| 183 |
+
):
|
| 184 |
+
"""Discriminator that combines multiple discriminators.
|
| 185 |
+
|
| 186 |
+
Parameters
|
| 187 |
+
----------
|
| 188 |
+
rates : list, optional
|
| 189 |
+
sampling rates (in Hz) to run MSD at, by default []
|
| 190 |
+
If empty, MSD is not used.
|
| 191 |
+
periods : list, optional
|
| 192 |
+
periods (of samples) to run MPD at, by default [2, 3, 5, 7, 11]
|
| 193 |
+
fft_sizes : list, optional
|
| 194 |
+
Window sizes of the FFT to run MRD at, by default [2048, 1024, 512]
|
| 195 |
+
sample_rate : int, optional
|
| 196 |
+
Sampling rate of audio in Hz, by default 44100
|
| 197 |
+
bands : list, optional
|
| 198 |
+
Bands to run MRD at, by default `BANDS`
|
| 199 |
+
"""
|
| 200 |
+
super().__init__()
|
| 201 |
+
discs = []
|
| 202 |
+
discs += [MPD(p) for p in periods]
|
| 203 |
+
discs += [MSD(r, sample_rate=sample_rate) for r in rates]
|
| 204 |
+
discs += [MRD(f, sample_rate=sample_rate, bands=bands) for f in fft_sizes]
|
| 205 |
+
self.discriminators = nn.ModuleList(discs)
|
| 206 |
+
|
| 207 |
+
def preprocess(self, y):
|
| 208 |
+
# Remove DC offset
|
| 209 |
+
y = y - y.mean(dim=-1, keepdims=True)
|
| 210 |
+
# Peak normalize the volume of input audio
|
| 211 |
+
y = 0.8 * y / (y.abs().max(dim=-1, keepdim=True)[0] + 1e-9)
|
| 212 |
+
return y
|
| 213 |
+
|
| 214 |
+
def forward(self, x):
|
| 215 |
+
x = self.preprocess(x)
|
| 216 |
+
fmaps = [d(x) for d in self.discriminators]
|
| 217 |
+
return fmaps
|
| 218 |
+
|
| 219 |
+
|
| 220 |
+
if __name__ == "__main__":
|
| 221 |
+
disc = Discriminator()
|
| 222 |
+
x = torch.zeros(1, 1, 44100)
|
| 223 |
+
results = disc(x)
|
| 224 |
+
for i, result in enumerate(results):
|
| 225 |
+
print(f"disc{i}")
|
| 226 |
+
for i, r in enumerate(result):
|
| 227 |
+
print(r.shape, r.mean(), r.min(), r.max())
|
| 228 |
+
print()
|
songgen/xcodec_wrapper/xcodec_infer/descriptaudiocodec/dac/nn/__init__.py
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from . import layers
|
| 2 |
+
from . import loss
|
| 3 |
+
from . import quantize
|
songgen/xcodec_wrapper/xcodec_infer/descriptaudiocodec/dac/nn/layers.py
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import numpy as np
|
| 2 |
+
import torch
|
| 3 |
+
import torch.nn as nn
|
| 4 |
+
import torch.nn.functional as F
|
| 5 |
+
from einops import rearrange
|
| 6 |
+
from torch.nn.utils import weight_norm
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def WNConv1d(*args, **kwargs):
|
| 10 |
+
return weight_norm(nn.Conv1d(*args, **kwargs))
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def WNConvTranspose1d(*args, **kwargs):
|
| 14 |
+
return weight_norm(nn.ConvTranspose1d(*args, **kwargs))
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
# Scripting this brings model speed up 1.4x
|
| 18 |
+
@torch.jit.script
|
| 19 |
+
def snake(x, alpha):
|
| 20 |
+
shape = x.shape
|
| 21 |
+
x = x.reshape(shape[0], shape[1], -1)
|
| 22 |
+
x = x + (alpha + 1e-9).reciprocal() * torch.sin(alpha * x).pow(2)
|
| 23 |
+
x = x.reshape(shape)
|
| 24 |
+
return x
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
class Snake1d(nn.Module):
|
| 28 |
+
def __init__(self, channels):
|
| 29 |
+
super().__init__()
|
| 30 |
+
self.alpha = nn.Parameter(torch.ones(1, channels, 1))
|
| 31 |
+
|
| 32 |
+
def forward(self, x):
|
| 33 |
+
return snake(x, self.alpha)
|
songgen/xcodec_wrapper/xcodec_infer/descriptaudiocodec/dac/nn/loss.py
ADDED
|
@@ -0,0 +1,368 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import typing
|
| 2 |
+
from typing import List
|
| 3 |
+
|
| 4 |
+
import torch
|
| 5 |
+
import torch.nn.functional as F
|
| 6 |
+
from audiotools import AudioSignal
|
| 7 |
+
from audiotools import STFTParams
|
| 8 |
+
from torch import nn
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
class L1Loss(nn.L1Loss):
|
| 12 |
+
"""L1 Loss between AudioSignals. Defaults
|
| 13 |
+
to comparing ``audio_data``, but any
|
| 14 |
+
attribute of an AudioSignal can be used.
|
| 15 |
+
|
| 16 |
+
Parameters
|
| 17 |
+
----------
|
| 18 |
+
attribute : str, optional
|
| 19 |
+
Attribute of signal to compare, defaults to ``audio_data``.
|
| 20 |
+
weight : float, optional
|
| 21 |
+
Weight of this loss, defaults to 1.0.
|
| 22 |
+
|
| 23 |
+
Implementation copied from: https://github.com/descriptinc/lyrebird-audiotools/blob/961786aa1a9d628cca0c0486e5885a457fe70c1a/audiotools/metrics/distance.py
|
| 24 |
+
"""
|
| 25 |
+
|
| 26 |
+
def __init__(self, attribute: str = "audio_data", weight: float = 1.0, **kwargs):
|
| 27 |
+
self.attribute = attribute
|
| 28 |
+
self.weight = weight
|
| 29 |
+
super().__init__(**kwargs)
|
| 30 |
+
|
| 31 |
+
def forward(self, x: AudioSignal, y: AudioSignal):
|
| 32 |
+
"""
|
| 33 |
+
Parameters
|
| 34 |
+
----------
|
| 35 |
+
x : AudioSignal
|
| 36 |
+
Estimate AudioSignal
|
| 37 |
+
y : AudioSignal
|
| 38 |
+
Reference AudioSignal
|
| 39 |
+
|
| 40 |
+
Returns
|
| 41 |
+
-------
|
| 42 |
+
torch.Tensor
|
| 43 |
+
L1 loss between AudioSignal attributes.
|
| 44 |
+
"""
|
| 45 |
+
if isinstance(x, AudioSignal):
|
| 46 |
+
x = getattr(x, self.attribute)
|
| 47 |
+
y = getattr(y, self.attribute)
|
| 48 |
+
return super().forward(x, y)
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
class SISDRLoss(nn.Module):
|
| 52 |
+
"""
|
| 53 |
+
Computes the Scale-Invariant Source-to-Distortion Ratio between a batch
|
| 54 |
+
of estimated and reference audio signals or aligned features.
|
| 55 |
+
|
| 56 |
+
Parameters
|
| 57 |
+
----------
|
| 58 |
+
scaling : int, optional
|
| 59 |
+
Whether to use scale-invariant (True) or
|
| 60 |
+
signal-to-noise ratio (False), by default True
|
| 61 |
+
reduction : str, optional
|
| 62 |
+
How to reduce across the batch (either 'mean',
|
| 63 |
+
'sum', or none).], by default ' mean'
|
| 64 |
+
zero_mean : int, optional
|
| 65 |
+
Zero mean the references and estimates before
|
| 66 |
+
computing the loss, by default True
|
| 67 |
+
clip_min : int, optional
|
| 68 |
+
The minimum possible loss value. Helps network
|
| 69 |
+
to not focus on making already good examples better, by default None
|
| 70 |
+
weight : float, optional
|
| 71 |
+
Weight of this loss, defaults to 1.0.
|
| 72 |
+
|
| 73 |
+
Implementation copied from: https://github.com/descriptinc/lyrebird-audiotools/blob/961786aa1a9d628cca0c0486e5885a457fe70c1a/audiotools/metrics/distance.py
|
| 74 |
+
"""
|
| 75 |
+
|
| 76 |
+
def __init__(
|
| 77 |
+
self,
|
| 78 |
+
scaling: int = True,
|
| 79 |
+
reduction: str = "mean",
|
| 80 |
+
zero_mean: int = True,
|
| 81 |
+
clip_min: int = None,
|
| 82 |
+
weight: float = 1.0,
|
| 83 |
+
):
|
| 84 |
+
self.scaling = scaling
|
| 85 |
+
self.reduction = reduction
|
| 86 |
+
self.zero_mean = zero_mean
|
| 87 |
+
self.clip_min = clip_min
|
| 88 |
+
self.weight = weight
|
| 89 |
+
super().__init__()
|
| 90 |
+
|
| 91 |
+
def forward(self, x: AudioSignal, y: AudioSignal):
|
| 92 |
+
eps = 1e-8
|
| 93 |
+
# nb, nc, nt
|
| 94 |
+
if isinstance(x, AudioSignal):
|
| 95 |
+
references = x.audio_data
|
| 96 |
+
estimates = y.audio_data
|
| 97 |
+
else:
|
| 98 |
+
references = x
|
| 99 |
+
estimates = y
|
| 100 |
+
|
| 101 |
+
nb = references.shape[0]
|
| 102 |
+
references = references.reshape(nb, 1, -1).permute(0, 2, 1)
|
| 103 |
+
estimates = estimates.reshape(nb, 1, -1).permute(0, 2, 1)
|
| 104 |
+
|
| 105 |
+
# samples now on axis 1
|
| 106 |
+
if self.zero_mean:
|
| 107 |
+
mean_reference = references.mean(dim=1, keepdim=True)
|
| 108 |
+
mean_estimate = estimates.mean(dim=1, keepdim=True)
|
| 109 |
+
else:
|
| 110 |
+
mean_reference = 0
|
| 111 |
+
mean_estimate = 0
|
| 112 |
+
|
| 113 |
+
_references = references - mean_reference
|
| 114 |
+
_estimates = estimates - mean_estimate
|
| 115 |
+
|
| 116 |
+
references_projection = (_references**2).sum(dim=-2) + eps
|
| 117 |
+
references_on_estimates = (_estimates * _references).sum(dim=-2) + eps
|
| 118 |
+
|
| 119 |
+
scale = (
|
| 120 |
+
(references_on_estimates / references_projection).unsqueeze(1)
|
| 121 |
+
if self.scaling
|
| 122 |
+
else 1
|
| 123 |
+
)
|
| 124 |
+
|
| 125 |
+
e_true = scale * _references
|
| 126 |
+
e_res = _estimates - e_true
|
| 127 |
+
|
| 128 |
+
signal = (e_true**2).sum(dim=1)
|
| 129 |
+
noise = (e_res**2).sum(dim=1)
|
| 130 |
+
sdr = -10 * torch.log10(signal / noise + eps)
|
| 131 |
+
|
| 132 |
+
if self.clip_min is not None:
|
| 133 |
+
sdr = torch.clamp(sdr, min=self.clip_min)
|
| 134 |
+
|
| 135 |
+
if self.reduction == "mean":
|
| 136 |
+
sdr = sdr.mean()
|
| 137 |
+
elif self.reduction == "sum":
|
| 138 |
+
sdr = sdr.sum()
|
| 139 |
+
return sdr
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
class MultiScaleSTFTLoss(nn.Module):
|
| 143 |
+
"""Computes the multi-scale STFT loss from [1].
|
| 144 |
+
|
| 145 |
+
Parameters
|
| 146 |
+
----------
|
| 147 |
+
window_lengths : List[int], optional
|
| 148 |
+
Length of each window of each STFT, by default [2048, 512]
|
| 149 |
+
loss_fn : typing.Callable, optional
|
| 150 |
+
How to compare each loss, by default nn.L1Loss()
|
| 151 |
+
clamp_eps : float, optional
|
| 152 |
+
Clamp on the log magnitude, below, by default 1e-5
|
| 153 |
+
mag_weight : float, optional
|
| 154 |
+
Weight of raw magnitude portion of loss, by default 1.0
|
| 155 |
+
log_weight : float, optional
|
| 156 |
+
Weight of log magnitude portion of loss, by default 1.0
|
| 157 |
+
pow : float, optional
|
| 158 |
+
Power to raise magnitude to before taking log, by default 2.0
|
| 159 |
+
weight : float, optional
|
| 160 |
+
Weight of this loss, by default 1.0
|
| 161 |
+
match_stride : bool, optional
|
| 162 |
+
Whether to match the stride of convolutional layers, by default False
|
| 163 |
+
|
| 164 |
+
References
|
| 165 |
+
----------
|
| 166 |
+
|
| 167 |
+
1. Engel, Jesse, Chenjie Gu, and Adam Roberts.
|
| 168 |
+
"DDSP: Differentiable Digital Signal Processing."
|
| 169 |
+
International Conference on Learning Representations. 2019.
|
| 170 |
+
|
| 171 |
+
Implementation copied from: https://github.com/descriptinc/lyrebird-audiotools/blob/961786aa1a9d628cca0c0486e5885a457fe70c1a/audiotools/metrics/spectral.py
|
| 172 |
+
"""
|
| 173 |
+
|
| 174 |
+
def __init__(
|
| 175 |
+
self,
|
| 176 |
+
window_lengths: List[int] = [2048, 512],
|
| 177 |
+
loss_fn: typing.Callable = nn.L1Loss(),
|
| 178 |
+
clamp_eps: float = 1e-5,
|
| 179 |
+
mag_weight: float = 1.0,
|
| 180 |
+
log_weight: float = 1.0,
|
| 181 |
+
pow: float = 2.0,
|
| 182 |
+
weight: float = 1.0,
|
| 183 |
+
match_stride: bool = False,
|
| 184 |
+
window_type: str = None,
|
| 185 |
+
):
|
| 186 |
+
super().__init__()
|
| 187 |
+
self.stft_params = [
|
| 188 |
+
STFTParams(
|
| 189 |
+
window_length=w,
|
| 190 |
+
hop_length=w // 4,
|
| 191 |
+
match_stride=match_stride,
|
| 192 |
+
window_type=window_type,
|
| 193 |
+
)
|
| 194 |
+
for w in window_lengths
|
| 195 |
+
]
|
| 196 |
+
self.loss_fn = loss_fn
|
| 197 |
+
self.log_weight = log_weight
|
| 198 |
+
self.mag_weight = mag_weight
|
| 199 |
+
self.clamp_eps = clamp_eps
|
| 200 |
+
self.weight = weight
|
| 201 |
+
self.pow = pow
|
| 202 |
+
|
| 203 |
+
def forward(self, x: AudioSignal, y: AudioSignal):
|
| 204 |
+
"""Computes multi-scale STFT between an estimate and a reference
|
| 205 |
+
signal.
|
| 206 |
+
|
| 207 |
+
Parameters
|
| 208 |
+
----------
|
| 209 |
+
x : AudioSignal
|
| 210 |
+
Estimate signal
|
| 211 |
+
y : AudioSignal
|
| 212 |
+
Reference signal
|
| 213 |
+
|
| 214 |
+
Returns
|
| 215 |
+
-------
|
| 216 |
+
torch.Tensor
|
| 217 |
+
Multi-scale STFT loss.
|
| 218 |
+
"""
|
| 219 |
+
loss = 0.0
|
| 220 |
+
for s in self.stft_params:
|
| 221 |
+
x.stft(s.window_length, s.hop_length, s.window_type)
|
| 222 |
+
y.stft(s.window_length, s.hop_length, s.window_type)
|
| 223 |
+
loss += self.log_weight * self.loss_fn(
|
| 224 |
+
x.magnitude.clamp(self.clamp_eps).pow(self.pow).log10(),
|
| 225 |
+
y.magnitude.clamp(self.clamp_eps).pow(self.pow).log10(),
|
| 226 |
+
)
|
| 227 |
+
loss += self.mag_weight * self.loss_fn(x.magnitude, y.magnitude)
|
| 228 |
+
return loss
|
| 229 |
+
|
| 230 |
+
|
| 231 |
+
class MelSpectrogramLoss(nn.Module):
|
| 232 |
+
"""Compute distance between mel spectrograms. Can be used
|
| 233 |
+
in a multi-scale way.
|
| 234 |
+
|
| 235 |
+
Parameters
|
| 236 |
+
----------
|
| 237 |
+
n_mels : List[int]
|
| 238 |
+
Number of mels per STFT, by default [150, 80],
|
| 239 |
+
window_lengths : List[int], optional
|
| 240 |
+
Length of each window of each STFT, by default [2048, 512]
|
| 241 |
+
loss_fn : typing.Callable, optional
|
| 242 |
+
How to compare each loss, by default nn.L1Loss()
|
| 243 |
+
clamp_eps : float, optional
|
| 244 |
+
Clamp on the log magnitude, below, by default 1e-5
|
| 245 |
+
mag_weight : float, optional
|
| 246 |
+
Weight of raw magnitude portion of loss, by default 1.0
|
| 247 |
+
log_weight : float, optional
|
| 248 |
+
Weight of log magnitude portion of loss, by default 1.0
|
| 249 |
+
pow : float, optional
|
| 250 |
+
Power to raise magnitude to before taking log, by default 2.0
|
| 251 |
+
weight : float, optional
|
| 252 |
+
Weight of this loss, by default 1.0
|
| 253 |
+
match_stride : bool, optional
|
| 254 |
+
Whether to match the stride of convolutional layers, by default False
|
| 255 |
+
|
| 256 |
+
Implementation copied from: https://github.com/descriptinc/lyrebird-audiotools/blob/961786aa1a9d628cca0c0486e5885a457fe70c1a/audiotools/metrics/spectral.py
|
| 257 |
+
"""
|
| 258 |
+
|
| 259 |
+
def __init__(
|
| 260 |
+
self,
|
| 261 |
+
n_mels: List[int] = [150, 80],
|
| 262 |
+
window_lengths: List[int] = [2048, 512],
|
| 263 |
+
loss_fn: typing.Callable = nn.L1Loss(),
|
| 264 |
+
clamp_eps: float = 1e-5,
|
| 265 |
+
mag_weight: float = 1.0,
|
| 266 |
+
log_weight: float = 1.0,
|
| 267 |
+
pow: float = 2.0,
|
| 268 |
+
weight: float = 1.0,
|
| 269 |
+
match_stride: bool = False,
|
| 270 |
+
mel_fmin: List[float] = [0.0, 0.0],
|
| 271 |
+
mel_fmax: List[float] = [None, None],
|
| 272 |
+
window_type: str = None,
|
| 273 |
+
):
|
| 274 |
+
super().__init__()
|
| 275 |
+
self.stft_params = [
|
| 276 |
+
STFTParams(
|
| 277 |
+
window_length=w,
|
| 278 |
+
hop_length=w // 4,
|
| 279 |
+
match_stride=match_stride,
|
| 280 |
+
window_type=window_type,
|
| 281 |
+
)
|
| 282 |
+
for w in window_lengths
|
| 283 |
+
]
|
| 284 |
+
self.n_mels = n_mels
|
| 285 |
+
self.loss_fn = loss_fn
|
| 286 |
+
self.clamp_eps = clamp_eps
|
| 287 |
+
self.log_weight = log_weight
|
| 288 |
+
self.mag_weight = mag_weight
|
| 289 |
+
self.weight = weight
|
| 290 |
+
self.mel_fmin = mel_fmin
|
| 291 |
+
self.mel_fmax = mel_fmax
|
| 292 |
+
self.pow = pow
|
| 293 |
+
|
| 294 |
+
def forward(self, x: AudioSignal, y: AudioSignal):
|
| 295 |
+
"""Computes mel loss between an estimate and a reference
|
| 296 |
+
signal.
|
| 297 |
+
|
| 298 |
+
Parameters
|
| 299 |
+
----------
|
| 300 |
+
x : AudioSignal
|
| 301 |
+
Estimate signal
|
| 302 |
+
y : AudioSignal
|
| 303 |
+
Reference signal
|
| 304 |
+
|
| 305 |
+
Returns
|
| 306 |
+
-------
|
| 307 |
+
torch.Tensor
|
| 308 |
+
Mel loss.
|
| 309 |
+
"""
|
| 310 |
+
loss = 0.0
|
| 311 |
+
for n_mels, fmin, fmax, s in zip(
|
| 312 |
+
self.n_mels, self.mel_fmin, self.mel_fmax, self.stft_params
|
| 313 |
+
):
|
| 314 |
+
kwargs = {
|
| 315 |
+
"window_length": s.window_length,
|
| 316 |
+
"hop_length": s.hop_length,
|
| 317 |
+
"window_type": s.window_type,
|
| 318 |
+
}
|
| 319 |
+
x_mels = x.mel_spectrogram(n_mels, mel_fmin=fmin, mel_fmax=fmax, **kwargs)
|
| 320 |
+
y_mels = y.mel_spectrogram(n_mels, mel_fmin=fmin, mel_fmax=fmax, **kwargs)
|
| 321 |
+
|
| 322 |
+
loss += self.log_weight * self.loss_fn(
|
| 323 |
+
x_mels.clamp(self.clamp_eps).pow(self.pow).log10(),
|
| 324 |
+
y_mels.clamp(self.clamp_eps).pow(self.pow).log10(),
|
| 325 |
+
)
|
| 326 |
+
loss += self.mag_weight * self.loss_fn(x_mels, y_mels)
|
| 327 |
+
return loss
|
| 328 |
+
|
| 329 |
+
|
| 330 |
+
class GANLoss(nn.Module):
|
| 331 |
+
"""
|
| 332 |
+
Computes a discriminator loss, given a discriminator on
|
| 333 |
+
generated waveforms/spectrograms compared to ground truth
|
| 334 |
+
waveforms/spectrograms. Computes the loss for both the
|
| 335 |
+
discriminator and the generator in separate functions.
|
| 336 |
+
"""
|
| 337 |
+
|
| 338 |
+
def __init__(self, discriminator):
|
| 339 |
+
super().__init__()
|
| 340 |
+
self.discriminator = discriminator
|
| 341 |
+
|
| 342 |
+
def forward(self, fake, real):
|
| 343 |
+
d_fake = self.discriminator(fake.audio_data)
|
| 344 |
+
d_real = self.discriminator(real.audio_data)
|
| 345 |
+
return d_fake, d_real
|
| 346 |
+
|
| 347 |
+
def discriminator_loss(self, fake, real):
|
| 348 |
+
d_fake, d_real = self.forward(fake.clone().detach(), real)
|
| 349 |
+
|
| 350 |
+
loss_d = 0
|
| 351 |
+
for x_fake, x_real in zip(d_fake, d_real):
|
| 352 |
+
loss_d += torch.mean(x_fake[-1] ** 2)
|
| 353 |
+
loss_d += torch.mean((1 - x_real[-1]) ** 2)
|
| 354 |
+
return loss_d
|
| 355 |
+
|
| 356 |
+
def generator_loss(self, fake, real):
|
| 357 |
+
d_fake, d_real = self.forward(fake, real)
|
| 358 |
+
|
| 359 |
+
loss_g = 0
|
| 360 |
+
for x_fake in d_fake:
|
| 361 |
+
loss_g += torch.mean((1 - x_fake[-1]) ** 2)
|
| 362 |
+
|
| 363 |
+
loss_feature = 0
|
| 364 |
+
|
| 365 |
+
for i in range(len(d_fake)):
|
| 366 |
+
for j in range(len(d_fake[i]) - 1):
|
| 367 |
+
loss_feature += F.l1_loss(d_fake[i][j], d_real[i][j].detach())
|
| 368 |
+
return loss_g+2* loss_feature
|
songgen/xcodec_wrapper/xcodec_infer/descriptaudiocodec/dac/nn/quantize.py
ADDED
|
@@ -0,0 +1,262 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Union
|
| 2 |
+
|
| 3 |
+
import numpy as np
|
| 4 |
+
import torch
|
| 5 |
+
import torch.nn as nn
|
| 6 |
+
import torch.nn.functional as F
|
| 7 |
+
from einops import rearrange
|
| 8 |
+
from torch.nn.utils import weight_norm
|
| 9 |
+
|
| 10 |
+
from dac.nn.layers import WNConv1d
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class VectorQuantize(nn.Module):
|
| 14 |
+
"""
|
| 15 |
+
Implementation of VQ similar to Karpathy's repo:
|
| 16 |
+
https://github.com/karpathy/deep-vector-quantization
|
| 17 |
+
Additionally uses following tricks from Improved VQGAN
|
| 18 |
+
(https://arxiv.org/pdf/2110.04627.pdf):
|
| 19 |
+
1. Factorized codes: Perform nearest neighbor lookup in low-dimensional space
|
| 20 |
+
for improved codebook usage
|
| 21 |
+
2. l2-normalized codes: Converts euclidean distance to cosine similarity which
|
| 22 |
+
improves training stability
|
| 23 |
+
"""
|
| 24 |
+
|
| 25 |
+
def __init__(self, input_dim: int, codebook_size: int, codebook_dim: int):
|
| 26 |
+
super().__init__()
|
| 27 |
+
self.codebook_size = codebook_size
|
| 28 |
+
self.codebook_dim = codebook_dim
|
| 29 |
+
|
| 30 |
+
self.in_proj = WNConv1d(input_dim, codebook_dim, kernel_size=1)
|
| 31 |
+
self.out_proj = WNConv1d(codebook_dim, input_dim, kernel_size=1)
|
| 32 |
+
self.codebook = nn.Embedding(codebook_size, codebook_dim)
|
| 33 |
+
|
| 34 |
+
def forward(self, z):
|
| 35 |
+
"""Quantized the input tensor using a fixed codebook and returns
|
| 36 |
+
the corresponding codebook vectors
|
| 37 |
+
|
| 38 |
+
Parameters
|
| 39 |
+
----------
|
| 40 |
+
z : Tensor[B x D x T]
|
| 41 |
+
|
| 42 |
+
Returns
|
| 43 |
+
-------
|
| 44 |
+
Tensor[B x D x T]
|
| 45 |
+
Quantized continuous representation of input
|
| 46 |
+
Tensor[1]
|
| 47 |
+
Commitment loss to train encoder to predict vectors closer to codebook
|
| 48 |
+
entries
|
| 49 |
+
Tensor[1]
|
| 50 |
+
Codebook loss to update the codebook
|
| 51 |
+
Tensor[B x T]
|
| 52 |
+
Codebook indices (quantized discrete representation of input)
|
| 53 |
+
Tensor[B x D x T]
|
| 54 |
+
Projected latents (continuous representation of input before quantization)
|
| 55 |
+
"""
|
| 56 |
+
|
| 57 |
+
# Factorized codes (ViT-VQGAN) Project input into low-dimensional space
|
| 58 |
+
z_e = self.in_proj(z) # z_e : (B x D x T)
|
| 59 |
+
z_q, indices = self.decode_latents(z_e)
|
| 60 |
+
|
| 61 |
+
commitment_loss = F.mse_loss(z_e, z_q.detach(), reduction="none").mean([1, 2])
|
| 62 |
+
codebook_loss = F.mse_loss(z_q, z_e.detach(), reduction="none").mean([1, 2])
|
| 63 |
+
|
| 64 |
+
z_q = (
|
| 65 |
+
z_e + (z_q - z_e).detach()
|
| 66 |
+
) # noop in forward pass, straight-through gradient estimator in backward pass
|
| 67 |
+
|
| 68 |
+
z_q = self.out_proj(z_q)
|
| 69 |
+
|
| 70 |
+
return z_q, commitment_loss, codebook_loss, indices, z_e
|
| 71 |
+
|
| 72 |
+
def embed_code(self, embed_id):
|
| 73 |
+
return F.embedding(embed_id, self.codebook.weight)
|
| 74 |
+
|
| 75 |
+
def decode_code(self, embed_id):
|
| 76 |
+
return self.embed_code(embed_id).transpose(1, 2)
|
| 77 |
+
|
| 78 |
+
def decode_latents(self, latents):
|
| 79 |
+
encodings = rearrange(latents, "b d t -> (b t) d")
|
| 80 |
+
codebook = self.codebook.weight # codebook: (N x D)
|
| 81 |
+
|
| 82 |
+
# L2 normalize encodings and codebook (ViT-VQGAN)
|
| 83 |
+
encodings = F.normalize(encodings)
|
| 84 |
+
codebook = F.normalize(codebook)
|
| 85 |
+
|
| 86 |
+
# Compute euclidean distance with codebook
|
| 87 |
+
dist = (
|
| 88 |
+
encodings.pow(2).sum(1, keepdim=True)
|
| 89 |
+
- 2 * encodings @ codebook.t()
|
| 90 |
+
+ codebook.pow(2).sum(1, keepdim=True).t()
|
| 91 |
+
)
|
| 92 |
+
indices = rearrange((-dist).max(1)[1], "(b t) -> b t", b=latents.size(0))
|
| 93 |
+
z_q = self.decode_code(indices)
|
| 94 |
+
return z_q, indices
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
class ResidualVectorQuantize(nn.Module):
|
| 98 |
+
"""
|
| 99 |
+
Introduced in SoundStream: An end2end neural audio codec
|
| 100 |
+
https://arxiv.org/abs/2107.03312
|
| 101 |
+
"""
|
| 102 |
+
|
| 103 |
+
def __init__(
|
| 104 |
+
self,
|
| 105 |
+
input_dim: int = 512,
|
| 106 |
+
n_codebooks: int = 9,
|
| 107 |
+
codebook_size: int = 1024,
|
| 108 |
+
codebook_dim: Union[int, list] = 8,
|
| 109 |
+
quantizer_dropout: float = 0.0,
|
| 110 |
+
):
|
| 111 |
+
super().__init__()
|
| 112 |
+
if isinstance(codebook_dim, int):
|
| 113 |
+
codebook_dim = [codebook_dim for _ in range(n_codebooks)]
|
| 114 |
+
|
| 115 |
+
self.n_codebooks = n_codebooks
|
| 116 |
+
self.codebook_dim = codebook_dim
|
| 117 |
+
self.codebook_size = codebook_size
|
| 118 |
+
|
| 119 |
+
self.quantizers = nn.ModuleList(
|
| 120 |
+
[
|
| 121 |
+
VectorQuantize(input_dim, codebook_size, codebook_dim[i])
|
| 122 |
+
for i in range(n_codebooks)
|
| 123 |
+
]
|
| 124 |
+
)
|
| 125 |
+
self.quantizer_dropout = quantizer_dropout
|
| 126 |
+
|
| 127 |
+
def forward(self, z, n_quantizers: int = None):
|
| 128 |
+
"""Quantized the input tensor using a fixed set of `n` codebooks and returns
|
| 129 |
+
the corresponding codebook vectors
|
| 130 |
+
Parameters
|
| 131 |
+
----------
|
| 132 |
+
z : Tensor[B x D x T]
|
| 133 |
+
n_quantizers : int, optional
|
| 134 |
+
No. of quantizers to use
|
| 135 |
+
(n_quantizers < self.n_codebooks ex: for quantizer dropout)
|
| 136 |
+
Note: if `self.quantizer_dropout` is True, this argument is ignored
|
| 137 |
+
when in training mode, and a random number of quantizers is used.
|
| 138 |
+
Returns
|
| 139 |
+
-------
|
| 140 |
+
dict
|
| 141 |
+
A dictionary with the following keys:
|
| 142 |
+
|
| 143 |
+
"z" : Tensor[B x D x T]
|
| 144 |
+
Quantized continuous representation of input
|
| 145 |
+
"codes" : Tensor[B x N x T]
|
| 146 |
+
Codebook indices for each codebook
|
| 147 |
+
(quantized discrete representation of input)
|
| 148 |
+
"latents" : Tensor[B x N*D x T]
|
| 149 |
+
Projected latents (continuous representation of input before quantization)
|
| 150 |
+
"vq/commitment_loss" : Tensor[1]
|
| 151 |
+
Commitment loss to train encoder to predict vectors closer to codebook
|
| 152 |
+
entries
|
| 153 |
+
"vq/codebook_loss" : Tensor[1]
|
| 154 |
+
Codebook loss to update the codebook
|
| 155 |
+
"""
|
| 156 |
+
z_q = 0
|
| 157 |
+
residual = z
|
| 158 |
+
commitment_loss = 0
|
| 159 |
+
codebook_loss = 0
|
| 160 |
+
|
| 161 |
+
codebook_indices = []
|
| 162 |
+
latents = []
|
| 163 |
+
|
| 164 |
+
if n_quantizers is None:
|
| 165 |
+
n_quantizers = self.n_codebooks
|
| 166 |
+
if self.training:
|
| 167 |
+
n_quantizers = torch.ones((z.shape[0],)) * self.n_codebooks + 1
|
| 168 |
+
dropout = torch.randint(1, self.n_codebooks + 1, (z.shape[0],))
|
| 169 |
+
n_dropout = int(z.shape[0] * self.quantizer_dropout)
|
| 170 |
+
n_quantizers[:n_dropout] = dropout[:n_dropout]
|
| 171 |
+
n_quantizers = n_quantizers.to(z.device)
|
| 172 |
+
|
| 173 |
+
for i, quantizer in enumerate(self.quantizers):
|
| 174 |
+
if self.training is False and i >= n_quantizers:
|
| 175 |
+
break
|
| 176 |
+
|
| 177 |
+
z_q_i, commitment_loss_i, codebook_loss_i, indices_i, z_e_i = quantizer(
|
| 178 |
+
residual
|
| 179 |
+
)
|
| 180 |
+
|
| 181 |
+
# Create mask to apply quantizer dropout
|
| 182 |
+
mask = (
|
| 183 |
+
torch.full((z.shape[0],), fill_value=i, device=z.device) < n_quantizers
|
| 184 |
+
)
|
| 185 |
+
z_q = z_q + z_q_i * mask[:, None, None]
|
| 186 |
+
residual = residual - z_q_i
|
| 187 |
+
|
| 188 |
+
# Sum losses
|
| 189 |
+
commitment_loss += (commitment_loss_i * mask).mean()
|
| 190 |
+
codebook_loss += (codebook_loss_i * mask).mean()
|
| 191 |
+
|
| 192 |
+
codebook_indices.append(indices_i)
|
| 193 |
+
latents.append(z_e_i)
|
| 194 |
+
|
| 195 |
+
codes = torch.stack(codebook_indices, dim=1)
|
| 196 |
+
latents = torch.cat(latents, dim=1)
|
| 197 |
+
|
| 198 |
+
return z_q, codes, latents, commitment_loss, codebook_loss
|
| 199 |
+
|
| 200 |
+
def from_codes(self, codes: torch.Tensor):
|
| 201 |
+
"""Given the quantized codes, reconstruct the continuous representation
|
| 202 |
+
Parameters
|
| 203 |
+
----------
|
| 204 |
+
codes : Tensor[B x N x T]
|
| 205 |
+
Quantized discrete representation of input
|
| 206 |
+
Returns
|
| 207 |
+
-------
|
| 208 |
+
Tensor[B x D x T]
|
| 209 |
+
Quantized continuous representation of input
|
| 210 |
+
"""
|
| 211 |
+
z_q = 0.0
|
| 212 |
+
z_p = []
|
| 213 |
+
n_codebooks = codes.shape[1]
|
| 214 |
+
for i in range(n_codebooks):
|
| 215 |
+
z_p_i = self.quantizers[i].decode_code(codes[:, i, :])
|
| 216 |
+
z_p.append(z_p_i)
|
| 217 |
+
|
| 218 |
+
z_q_i = self.quantizers[i].out_proj(z_p_i)
|
| 219 |
+
z_q = z_q + z_q_i
|
| 220 |
+
return z_q, torch.cat(z_p, dim=1), codes
|
| 221 |
+
|
| 222 |
+
def from_latents(self, latents: torch.Tensor):
|
| 223 |
+
"""Given the unquantized latents, reconstruct the
|
| 224 |
+
continuous representation after quantization.
|
| 225 |
+
|
| 226 |
+
Parameters
|
| 227 |
+
----------
|
| 228 |
+
latents : Tensor[B x N x T]
|
| 229 |
+
Continuous representation of input after projection
|
| 230 |
+
|
| 231 |
+
Returns
|
| 232 |
+
-------
|
| 233 |
+
Tensor[B x D x T]
|
| 234 |
+
Quantized representation of full-projected space
|
| 235 |
+
Tensor[B x D x T]
|
| 236 |
+
Quantized representation of latent space
|
| 237 |
+
"""
|
| 238 |
+
z_q = 0
|
| 239 |
+
z_p = []
|
| 240 |
+
codes = []
|
| 241 |
+
dims = np.cumsum([0] + [q.codebook_dim for q in self.quantizers])
|
| 242 |
+
|
| 243 |
+
n_codebooks = np.where(dims <= latents.shape[1])[0].max(axis=0, keepdims=True)[
|
| 244 |
+
0
|
| 245 |
+
]
|
| 246 |
+
for i in range(n_codebooks):
|
| 247 |
+
j, k = dims[i], dims[i + 1]
|
| 248 |
+
z_p_i, codes_i = self.quantizers[i].decode_latents(latents[:, j:k, :])
|
| 249 |
+
z_p.append(z_p_i)
|
| 250 |
+
codes.append(codes_i)
|
| 251 |
+
|
| 252 |
+
z_q_i = self.quantizers[i].out_proj(z_p_i)
|
| 253 |
+
z_q = z_q + z_q_i
|
| 254 |
+
|
| 255 |
+
return z_q, torch.cat(z_p, dim=1), torch.stack(codes, dim=1)
|
| 256 |
+
|
| 257 |
+
|
| 258 |
+
if __name__ == "__main__":
|
| 259 |
+
rvq = ResidualVectorQuantize(quantizer_dropout=True)
|
| 260 |
+
x = torch.randn(16, 512, 80)
|
| 261 |
+
y = rvq(x)
|
| 262 |
+
print(y["latents"].shape)
|
songgen/xcodec_wrapper/xcodec_infer/descriptaudiocodec/dac/utils/__init__.py
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from pathlib import Path
|
| 2 |
+
|
| 3 |
+
import argbind
|
| 4 |
+
from audiotools import ml
|
| 5 |
+
|
| 6 |
+
import dac
|
| 7 |
+
|
| 8 |
+
DAC = dac.model.DAC
|
| 9 |
+
Accelerator = ml.Accelerator
|
| 10 |
+
|
| 11 |
+
__MODEL_LATEST_TAGS__ = {
|
| 12 |
+
("44khz", "8kbps"): "0.0.1",
|
| 13 |
+
("24khz", "8kbps"): "0.0.4",
|
| 14 |
+
("16khz", "8kbps"): "0.0.5",
|
| 15 |
+
("44khz", "16kbps"): "1.0.0",
|
| 16 |
+
}
|
| 17 |
+
|
| 18 |
+
__MODEL_URLS__ = {
|
| 19 |
+
(
|
| 20 |
+
"44khz",
|
| 21 |
+
"0.0.1",
|
| 22 |
+
"8kbps",
|
| 23 |
+
): "https://github.com/descriptinc/descript-audio-codec/releases/download/0.0.1/weights.pth",
|
| 24 |
+
(
|
| 25 |
+
"24khz",
|
| 26 |
+
"0.0.4",
|
| 27 |
+
"8kbps",
|
| 28 |
+
): "https://github.com/descriptinc/descript-audio-codec/releases/download/0.0.4/weights_24khz.pth",
|
| 29 |
+
(
|
| 30 |
+
"16khz",
|
| 31 |
+
"0.0.5",
|
| 32 |
+
"8kbps",
|
| 33 |
+
): "https://github.com/descriptinc/descript-audio-codec/releases/download/0.0.5/weights_16khz.pth",
|
| 34 |
+
(
|
| 35 |
+
"44khz",
|
| 36 |
+
"1.0.0",
|
| 37 |
+
"16kbps",
|
| 38 |
+
): "https://github.com/descriptinc/descript-audio-codec/releases/download/1.0.0/weights_44khz_16kbps.pth",
|
| 39 |
+
}
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
@argbind.bind(group="download", positional=True, without_prefix=True)
|
| 43 |
+
def download(
|
| 44 |
+
model_type: str = "44khz", model_bitrate: str = "8kbps", tag: str = "latest"
|
| 45 |
+
):
|
| 46 |
+
"""
|
| 47 |
+
Function that downloads the weights file from URL if a local cache is not found.
|
| 48 |
+
|
| 49 |
+
Parameters
|
| 50 |
+
----------
|
| 51 |
+
model_type : str
|
| 52 |
+
The type of model to download. Must be one of "44khz", "24khz", or "16khz". Defaults to "44khz".
|
| 53 |
+
model_bitrate: str
|
| 54 |
+
Bitrate of the model. Must be one of "8kbps", or "16kbps". Defaults to "8kbps".
|
| 55 |
+
Only 44khz model supports 16kbps.
|
| 56 |
+
tag : str
|
| 57 |
+
The tag of the model to download. Defaults to "latest".
|
| 58 |
+
|
| 59 |
+
Returns
|
| 60 |
+
-------
|
| 61 |
+
Path
|
| 62 |
+
Directory path required to load model via audiotools.
|
| 63 |
+
"""
|
| 64 |
+
model_type = model_type.lower()
|
| 65 |
+
tag = tag.lower()
|
| 66 |
+
|
| 67 |
+
assert model_type in [
|
| 68 |
+
"44khz",
|
| 69 |
+
"24khz",
|
| 70 |
+
"16khz",
|
| 71 |
+
], "model_type must be one of '44khz', '24khz', or '16khz'"
|
| 72 |
+
|
| 73 |
+
assert model_bitrate in [
|
| 74 |
+
"8kbps",
|
| 75 |
+
"16kbps",
|
| 76 |
+
], "model_bitrate must be one of '8kbps', or '16kbps'"
|
| 77 |
+
|
| 78 |
+
if tag == "latest":
|
| 79 |
+
tag = __MODEL_LATEST_TAGS__[(model_type, model_bitrate)]
|
| 80 |
+
|
| 81 |
+
download_link = __MODEL_URLS__.get((model_type, tag, model_bitrate), None)
|
| 82 |
+
|
| 83 |
+
if download_link is None:
|
| 84 |
+
raise ValueError(
|
| 85 |
+
f"Could not find model with tag {tag} and model type {model_type}"
|
| 86 |
+
)
|
| 87 |
+
|
| 88 |
+
local_path = (
|
| 89 |
+
Path.home()
|
| 90 |
+
/ ".cache"
|
| 91 |
+
/ "descript"
|
| 92 |
+
/ "dac"
|
| 93 |
+
/ f"weights_{model_type}_{model_bitrate}_{tag}.pth"
|
| 94 |
+
)
|
| 95 |
+
if not local_path.exists():
|
| 96 |
+
local_path.parent.mkdir(parents=True, exist_ok=True)
|
| 97 |
+
|
| 98 |
+
# Download the model
|
| 99 |
+
import requests
|
| 100 |
+
|
| 101 |
+
response = requests.get(download_link)
|
| 102 |
+
|
| 103 |
+
if response.status_code != 200:
|
| 104 |
+
raise ValueError(
|
| 105 |
+
f"Could not download model. Received response code {response.status_code}"
|
| 106 |
+
)
|
| 107 |
+
local_path.write_bytes(response.content)
|
| 108 |
+
|
| 109 |
+
return local_path
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
def load_model(
|
| 113 |
+
model_type: str = "44khz",
|
| 114 |
+
model_bitrate: str = "8kbps",
|
| 115 |
+
tag: str = "latest",
|
| 116 |
+
load_path: str = None,
|
| 117 |
+
):
|
| 118 |
+
if not load_path:
|
| 119 |
+
load_path = download(
|
| 120 |
+
model_type=model_type, model_bitrate=model_bitrate, tag=tag
|
| 121 |
+
)
|
| 122 |
+
generator = DAC.load(load_path)
|
| 123 |
+
return generator
|
songgen/xcodec_wrapper/xcodec_infer/models/__init__.py
ADDED
|
File without changes
|
songgen/xcodec_wrapper/xcodec_infer/models/soundstream_semantic.py
ADDED
|
@@ -0,0 +1,157 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
|
| 3 |
+
from typing import Sequence, Optional, Union
|
| 4 |
+
import sys
|
| 5 |
+
|
| 6 |
+
import math
|
| 7 |
+
import random
|
| 8 |
+
|
| 9 |
+
import torch
|
| 10 |
+
import torch.nn as nn
|
| 11 |
+
import numpy as np
|
| 12 |
+
import torch.nn.functional as F
|
| 13 |
+
|
| 14 |
+
import descriptaudiocodec.dac.model.dac as dac2
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
from quantization import ResidualVectorQuantizer
|
| 18 |
+
from transformers import AutoModel
|
| 19 |
+
|
| 20 |
+
from modules.semantic_module import Encoder,Decoder
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
class SoundStream(nn.Module):
|
| 25 |
+
def __init__(
|
| 26 |
+
self,
|
| 27 |
+
n_filters: int = 32,
|
| 28 |
+
D: int = 128,
|
| 29 |
+
target_bandwidths: Sequence[Union[int, float]] = [1, 1.5, 2, 4, 6],
|
| 30 |
+
ratios: Sequence[int] = [8, 5, 4, 2], # downsampling by 320
|
| 31 |
+
sample_rate: int = 16000,
|
| 32 |
+
bins: int = 1024,
|
| 33 |
+
normalize: bool = False,
|
| 34 |
+
causal: bool = False,
|
| 35 |
+
semantic_techer: str = 'hubert_base_general'
|
| 36 |
+
):
|
| 37 |
+
super().__init__()
|
| 38 |
+
self.hop_length = np.prod(ratios)
|
| 39 |
+
|
| 40 |
+
n_q = int(1000 * target_bandwidths[-1] // (math.ceil(sample_rate / self.hop_length) * 10))
|
| 41 |
+
self.frame_rate = math.ceil(sample_rate / np.prod(ratios)) # 50 Hz
|
| 42 |
+
self.bits_per_codebook = int(math.log2(bins)) # 1024 => 10
|
| 43 |
+
self.target_bandwidths = target_bandwidths
|
| 44 |
+
self.n_q = n_q
|
| 45 |
+
self.sample_rate = sample_rate
|
| 46 |
+
self.encoder = dac2.Encoder(64,ratios,D)
|
| 47 |
+
|
| 48 |
+
self.encoder_semantic = Encoder(input_channels=768,encode_channels=768)
|
| 49 |
+
self.decoder_semantic = Decoder(code_dim=768,output_channels=768,decode_channels=768)
|
| 50 |
+
# out_D=D+768
|
| 51 |
+
self.quantizer = ResidualVectorQuantizer(dimension=D+768, n_q=n_q, bins=bins)
|
| 52 |
+
|
| 53 |
+
self.decoder_2 = dac2.Decoder(D,1024,ratios,)
|
| 54 |
+
|
| 55 |
+
if semantic_techer=='hubert_base':
|
| 56 |
+
self.semantic_model = AutoModel.from_pretrained("facebook/hubert-base-ls960")
|
| 57 |
+
elif semantic_techer=='wavlm_base_plus':
|
| 58 |
+
self.semantic_model = AutoModel.from_pretrained("microsoft/wavlm-base-plus")
|
| 59 |
+
elif semantic_techer=='hubert_base_general':
|
| 60 |
+
|
| 61 |
+
self.semantic_model = AutoModel.from_pretrained("ZhenYe234/hubert_base_general_audio")
|
| 62 |
+
self.semantic_model.eval()
|
| 63 |
+
|
| 64 |
+
self.fc_prior = nn.Linear(D+768, D+768 )
|
| 65 |
+
|
| 66 |
+
self.fc_post1= nn.Linear( D+768, 768 )
|
| 67 |
+
self.fc_post2= nn.Linear( D+768, D)
|
| 68 |
+
|
| 69 |
+
def get_last_layer(self):
|
| 70 |
+
return self.decoder.layers[-1].weight
|
| 71 |
+
|
| 72 |
+
def calculate_rec_loss(self, rec, target):
|
| 73 |
+
|
| 74 |
+
target = target / target.norm(dim=-1, keepdim=True)
|
| 75 |
+
rec = rec / rec.norm(dim=-1, keepdim=True)
|
| 76 |
+
rec_loss = (1 - (target * rec).sum(-1)).mean()
|
| 77 |
+
|
| 78 |
+
return rec_loss
|
| 79 |
+
|
| 80 |
+
@torch.no_grad()
|
| 81 |
+
def get_regress_target(self, x ):
|
| 82 |
+
x= x[:,0,:]
|
| 83 |
+
x = F.pad(x, (160, 160))
|
| 84 |
+
target = self.semantic_model(x, output_hidden_states=True) .hidden_states
|
| 85 |
+
target = torch.stack(target, dim=1)#.transpose(-1, -2)#.flatten(start_dim=1, end_dim=2)
|
| 86 |
+
|
| 87 |
+
# average for all layers
|
| 88 |
+
target = target.mean(1)
|
| 89 |
+
# target = target[9]
|
| 90 |
+
return target
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def forward(self, x: torch.Tensor, bw: int):
|
| 95 |
+
x=x.unsqueeze(1)
|
| 96 |
+
e_semantic_input = self.get_regress_target(x).detach()
|
| 97 |
+
# print('e_semantic_input:', e_semantic_input.shape )
|
| 98 |
+
# print('x:', x.shape )
|
| 99 |
+
e_semantic = self.encoder_semantic(e_semantic_input.transpose(1, 2))
|
| 100 |
+
e_acoustic = self.encoder(x)
|
| 101 |
+
|
| 102 |
+
# print('e_acoustic:', e_acoustic.shape )
|
| 103 |
+
# print('e_semantic:', e_semantic.shape )
|
| 104 |
+
e= torch.cat([e_acoustic, e_semantic], dim=1)
|
| 105 |
+
|
| 106 |
+
e = self.fc_prior(e.transpose(1, 2)).transpose(1, 2)
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
quantized, codes, bandwidth, commit_loss = self.quantizer(e, self.frame_rate, bw)
|
| 110 |
+
|
| 111 |
+
quantized_semantic = self.fc_post1(quantized.transpose(1, 2)).transpose(1, 2)
|
| 112 |
+
quantized_acoustic = self.fc_post2(quantized.transpose(1, 2)).transpose(1, 2)
|
| 113 |
+
|
| 114 |
+
o = self.decoder_2(quantized_acoustic)
|
| 115 |
+
|
| 116 |
+
o_semantic = self.decoder_semantic(quantized_semantic )
|
| 117 |
+
semantic_recon_loss = F.mse_loss(e_semantic_input.transpose(1, 2).detach(),o_semantic)
|
| 118 |
+
|
| 119 |
+
return o, commit_loss, semantic_recon_loss,None
|
| 120 |
+
|
| 121 |
+
def encode(self, x: torch.Tensor,target_bw: Optional[int] = None) -> torch.Tensor:
|
| 122 |
+
|
| 123 |
+
bw = target_bw
|
| 124 |
+
|
| 125 |
+
e_semantic_input = self.get_regress_target(x).detach()
|
| 126 |
+
|
| 127 |
+
e_semantic = self.encoder_semantic(e_semantic_input.transpose(1, 2))
|
| 128 |
+
e_acoustic = self.encoder(x)
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
if e_acoustic.shape[2] != e_semantic.shape[2]:
|
| 132 |
+
e_acoustic = self.encoder(F.pad(x[:,0,:], (160, 160)).unsqueeze(0))
|
| 133 |
+
|
| 134 |
+
e= torch.cat([e_acoustic, e_semantic], dim=1)
|
| 135 |
+
|
| 136 |
+
e = self.fc_prior(e.transpose(1, 2)).transpose(1, 2)
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
quantized, codes, bandwidth, commit_loss = self.quantizer(e, self.frame_rate, bw)
|
| 140 |
+
return codes
|
| 141 |
+
|
| 142 |
+
def decode(self, codes: torch.Tensor) -> torch.Tensor:
|
| 143 |
+
quantized = self.quantizer.decode(codes)
|
| 144 |
+
quantized_acoustic = self.fc_post2(quantized.transpose(1, 2)).transpose(1, 2)
|
| 145 |
+
|
| 146 |
+
o = self.decoder_2(quantized_acoustic)
|
| 147 |
+
return o
|
| 148 |
+
|
| 149 |
+
# test
|
| 150 |
+
if __name__ == '__main__':
|
| 151 |
+
soundstream = SoundStream(n_filters=32, D=256)
|
| 152 |
+
|
| 153 |
+
for i in range(10):
|
| 154 |
+
print(f"Iter {i}: ")
|
| 155 |
+
x = torch.rand(1, 1, 16000)
|
| 156 |
+
o, commit_loss, distill_loss,_= soundstream(x,soundstream.target_bandwidths[-1])
|
| 157 |
+
print('output', o.shape)
|
songgen/xcodec_wrapper/xcodec_infer/modules/__init__.py
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
# All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
|
| 7 |
+
"""Torch modules."""
|
| 8 |
+
|
| 9 |
+
# flake8: noqa
|
| 10 |
+
from .conv import (
|
| 11 |
+
pad1d,
|
| 12 |
+
unpad1d,
|
| 13 |
+
NormConv1d,
|
| 14 |
+
NormConvTranspose1d,
|
| 15 |
+
NormConv2d,
|
| 16 |
+
NormConvTranspose2d,
|
| 17 |
+
SConv1d,
|
| 18 |
+
SConvTranspose1d,
|
| 19 |
+
)
|
| 20 |
+
from .lstm import SLSTM
|
| 21 |
+
from .seanet import SEANetEncoder, SEANetDecoder
|
| 22 |
+
from .transformer import StreamingTransformerEncoder
|
songgen/xcodec_wrapper/xcodec_infer/modules/conv.py
ADDED
|
@@ -0,0 +1,252 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
# All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
|
| 7 |
+
"""Convolutional layers wrappers and utilities."""
|
| 8 |
+
|
| 9 |
+
import math
|
| 10 |
+
import typing as tp
|
| 11 |
+
import warnings
|
| 12 |
+
|
| 13 |
+
import torch
|
| 14 |
+
from torch import nn
|
| 15 |
+
from torch.nn import functional as F
|
| 16 |
+
from torch.nn.utils import spectral_norm, weight_norm
|
| 17 |
+
|
| 18 |
+
from .norm import ConvLayerNorm
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
CONV_NORMALIZATIONS = frozenset(['none', 'weight_norm', 'spectral_norm',
|
| 22 |
+
'time_layer_norm', 'layer_norm', 'time_group_norm'])
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def apply_parametrization_norm(module: nn.Module, norm: str = 'none') -> nn.Module:
|
| 26 |
+
assert norm in CONV_NORMALIZATIONS
|
| 27 |
+
if norm == 'weight_norm':
|
| 28 |
+
return weight_norm(module)
|
| 29 |
+
elif norm == 'spectral_norm':
|
| 30 |
+
return spectral_norm(module)
|
| 31 |
+
else:
|
| 32 |
+
# We already check was in CONV_NORMALIZATION, so any other choice
|
| 33 |
+
# doesn't need reparametrization.
|
| 34 |
+
return module
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def get_norm_module(module: nn.Module, causal: bool = False, norm: str = 'none', **norm_kwargs) -> nn.Module:
|
| 38 |
+
"""Return the proper normalization module. If causal is True, this will ensure the returned
|
| 39 |
+
module is causal, or return an error if the normalization doesn't support causal evaluation.
|
| 40 |
+
"""
|
| 41 |
+
assert norm in CONV_NORMALIZATIONS
|
| 42 |
+
if norm == 'layer_norm':
|
| 43 |
+
assert isinstance(module, nn.modules.conv._ConvNd)
|
| 44 |
+
return ConvLayerNorm(module.out_channels, **norm_kwargs)
|
| 45 |
+
elif norm == 'time_group_norm':
|
| 46 |
+
if causal:
|
| 47 |
+
raise ValueError("GroupNorm doesn't support causal evaluation.")
|
| 48 |
+
assert isinstance(module, nn.modules.conv._ConvNd)
|
| 49 |
+
return nn.GroupNorm(1, module.out_channels, **norm_kwargs)
|
| 50 |
+
else:
|
| 51 |
+
return nn.Identity()
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def get_extra_padding_for_conv1d(x: torch.Tensor, kernel_size: int, stride: int,
|
| 55 |
+
padding_total: int = 0) -> int:
|
| 56 |
+
"""See `pad_for_conv1d`.
|
| 57 |
+
"""
|
| 58 |
+
length = x.shape[-1]
|
| 59 |
+
n_frames = (length - kernel_size + padding_total) / stride + 1
|
| 60 |
+
ideal_length = (math.ceil(n_frames) - 1) * stride + (kernel_size - padding_total)
|
| 61 |
+
return ideal_length - length
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def pad_for_conv1d(x: torch.Tensor, kernel_size: int, stride: int, padding_total: int = 0):
|
| 65 |
+
"""Pad for a convolution to make sure that the last window is full.
|
| 66 |
+
Extra padding is added at the end. This is required to ensure that we can rebuild
|
| 67 |
+
an output of the same length, as otherwise, even with padding, some time steps
|
| 68 |
+
might get removed.
|
| 69 |
+
For instance, with total padding = 4, kernel size = 4, stride = 2:
|
| 70 |
+
0 0 1 2 3 4 5 0 0 # (0s are padding)
|
| 71 |
+
1 2 3 # (output frames of a convolution, last 0 is never used)
|
| 72 |
+
0 0 1 2 3 4 5 0 # (output of tr. conv., but pos. 5 is going to get removed as padding)
|
| 73 |
+
1 2 3 4 # once you removed padding, we are missing one time step !
|
| 74 |
+
"""
|
| 75 |
+
extra_padding = get_extra_padding_for_conv1d(x, kernel_size, stride, padding_total)
|
| 76 |
+
return F.pad(x, (0, extra_padding))
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
def pad1d(x: torch.Tensor, paddings: tp.Tuple[int, int], mode: str = 'zero', value: float = 0.):
|
| 80 |
+
"""Tiny wrapper around F.pad, just to allow for reflect padding on small input.
|
| 81 |
+
If this is the case, we insert extra 0 padding to the right before the reflection happen.
|
| 82 |
+
"""
|
| 83 |
+
length = x.shape[-1]
|
| 84 |
+
padding_left, padding_right = paddings
|
| 85 |
+
assert padding_left >= 0 and padding_right >= 0, (padding_left, padding_right)
|
| 86 |
+
if mode == 'reflect':
|
| 87 |
+
max_pad = max(padding_left, padding_right)
|
| 88 |
+
extra_pad = 0
|
| 89 |
+
if length <= max_pad:
|
| 90 |
+
extra_pad = max_pad - length + 1
|
| 91 |
+
x = F.pad(x, (0, extra_pad))
|
| 92 |
+
padded = F.pad(x, paddings, mode, value)
|
| 93 |
+
end = padded.shape[-1] - extra_pad
|
| 94 |
+
return padded[..., :end]
|
| 95 |
+
else:
|
| 96 |
+
return F.pad(x, paddings, mode, value)
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
def unpad1d(x: torch.Tensor, paddings: tp.Tuple[int, int]):
|
| 100 |
+
"""Remove padding from x, handling properly zero padding. Only for 1d!"""
|
| 101 |
+
padding_left, padding_right = paddings
|
| 102 |
+
assert padding_left >= 0 and padding_right >= 0, (padding_left, padding_right)
|
| 103 |
+
assert (padding_left + padding_right) <= x.shape[-1]
|
| 104 |
+
end = x.shape[-1] - padding_right
|
| 105 |
+
return x[..., padding_left: end]
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
class NormConv1d(nn.Module):
|
| 109 |
+
"""Wrapper around Conv1d and normalization applied to this conv
|
| 110 |
+
to provide a uniform interface across normalization approaches.
|
| 111 |
+
"""
|
| 112 |
+
def __init__(self, *args, causal: bool = False, norm: str = 'none',
|
| 113 |
+
norm_kwargs: tp.Dict[str, tp.Any] = {}, **kwargs):
|
| 114 |
+
super().__init__()
|
| 115 |
+
self.conv = apply_parametrization_norm(nn.Conv1d(*args, **kwargs), norm)
|
| 116 |
+
self.norm = get_norm_module(self.conv, causal, norm, **norm_kwargs)
|
| 117 |
+
self.norm_type = norm
|
| 118 |
+
|
| 119 |
+
def forward(self, x):
|
| 120 |
+
x = self.conv(x)
|
| 121 |
+
x = self.norm(x)
|
| 122 |
+
return x
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
class NormConv2d(nn.Module):
|
| 126 |
+
"""Wrapper around Conv2d and normalization applied to this conv
|
| 127 |
+
to provide a uniform interface across normalization approaches.
|
| 128 |
+
"""
|
| 129 |
+
def __init__(self, *args, norm: str = 'none',
|
| 130 |
+
norm_kwargs: tp.Dict[str, tp.Any] = {}, **kwargs):
|
| 131 |
+
super().__init__()
|
| 132 |
+
self.conv = apply_parametrization_norm(nn.Conv2d(*args, **kwargs), norm)
|
| 133 |
+
self.norm = get_norm_module(self.conv, causal=False, norm=norm, **norm_kwargs)
|
| 134 |
+
self.norm_type = norm
|
| 135 |
+
|
| 136 |
+
def forward(self, x):
|
| 137 |
+
x = self.conv(x)
|
| 138 |
+
x = self.norm(x)
|
| 139 |
+
return x
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
class NormConvTranspose1d(nn.Module):
|
| 143 |
+
"""Wrapper around ConvTranspose1d and normalization applied to this conv
|
| 144 |
+
to provide a uniform interface across normalization approaches.
|
| 145 |
+
"""
|
| 146 |
+
def __init__(self, *args, causal: bool = False, norm: str = 'none',
|
| 147 |
+
norm_kwargs: tp.Dict[str, tp.Any] = {}, **kwargs):
|
| 148 |
+
super().__init__()
|
| 149 |
+
self.convtr = apply_parametrization_norm(nn.ConvTranspose1d(*args, **kwargs), norm)
|
| 150 |
+
self.norm = get_norm_module(self.convtr, causal, norm, **norm_kwargs)
|
| 151 |
+
self.norm_type = norm
|
| 152 |
+
|
| 153 |
+
def forward(self, x):
|
| 154 |
+
x = self.convtr(x)
|
| 155 |
+
x = self.norm(x)
|
| 156 |
+
return x
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
class NormConvTranspose2d(nn.Module):
|
| 160 |
+
"""Wrapper around ConvTranspose2d and normalization applied to this conv
|
| 161 |
+
to provide a uniform interface across normalization approaches.
|
| 162 |
+
"""
|
| 163 |
+
def __init__(self, *args, norm: str = 'none',
|
| 164 |
+
norm_kwargs: tp.Dict[str, tp.Any] = {}, **kwargs):
|
| 165 |
+
super().__init__()
|
| 166 |
+
self.convtr = apply_parametrization_norm(nn.ConvTranspose2d(*args, **kwargs), norm)
|
| 167 |
+
self.norm = get_norm_module(self.convtr, causal=False, norm=norm, **norm_kwargs)
|
| 168 |
+
|
| 169 |
+
def forward(self, x):
|
| 170 |
+
x = self.convtr(x)
|
| 171 |
+
x = self.norm(x)
|
| 172 |
+
return x
|
| 173 |
+
|
| 174 |
+
|
| 175 |
+
class SConv1d(nn.Module):
|
| 176 |
+
"""Conv1d with some builtin handling of asymmetric or causal padding
|
| 177 |
+
and normalization.
|
| 178 |
+
"""
|
| 179 |
+
def __init__(self, in_channels: int, out_channels: int,
|
| 180 |
+
kernel_size: int, stride: int = 1, dilation: int = 1,
|
| 181 |
+
groups: int = 1, bias: bool = True, causal: bool = False,
|
| 182 |
+
norm: str = 'none', norm_kwargs: tp.Dict[str, tp.Any] = {},
|
| 183 |
+
pad_mode: str = 'reflect'):
|
| 184 |
+
super().__init__()
|
| 185 |
+
# warn user on unusual setup between dilation and stride
|
| 186 |
+
if stride > 1 and dilation > 1:
|
| 187 |
+
warnings.warn('SConv1d has been initialized with stride > 1 and dilation > 1'
|
| 188 |
+
f' (kernel_size={kernel_size} stride={stride}, dilation={dilation}).')
|
| 189 |
+
self.conv = NormConv1d(in_channels, out_channels, kernel_size, stride,
|
| 190 |
+
dilation=dilation, groups=groups, bias=bias, causal=causal,
|
| 191 |
+
norm=norm, norm_kwargs=norm_kwargs)
|
| 192 |
+
self.causal = causal
|
| 193 |
+
self.pad_mode = pad_mode
|
| 194 |
+
|
| 195 |
+
def forward(self, x):
|
| 196 |
+
B, C, T = x.shape
|
| 197 |
+
kernel_size = self.conv.conv.kernel_size[0]
|
| 198 |
+
stride = self.conv.conv.stride[0]
|
| 199 |
+
dilation = self.conv.conv.dilation[0]
|
| 200 |
+
padding_total = (kernel_size - 1) * dilation - (stride - 1)
|
| 201 |
+
extra_padding = get_extra_padding_for_conv1d(x, kernel_size, stride, padding_total)
|
| 202 |
+
if self.causal:
|
| 203 |
+
# Left padding for causal
|
| 204 |
+
x = pad1d(x, (padding_total, extra_padding), mode=self.pad_mode)
|
| 205 |
+
else:
|
| 206 |
+
# Asymmetric padding required for odd strides
|
| 207 |
+
padding_right = padding_total // 2
|
| 208 |
+
padding_left = padding_total - padding_right
|
| 209 |
+
x = pad1d(x, (padding_left, padding_right + extra_padding), mode=self.pad_mode)
|
| 210 |
+
return self.conv(x)
|
| 211 |
+
|
| 212 |
+
|
| 213 |
+
class SConvTranspose1d(nn.Module):
|
| 214 |
+
"""ConvTranspose1d with some builtin handling of asymmetric or causal padding
|
| 215 |
+
and normalization.
|
| 216 |
+
"""
|
| 217 |
+
def __init__(self, in_channels: int, out_channels: int,
|
| 218 |
+
kernel_size: int, stride: int = 1, causal: bool = False,
|
| 219 |
+
norm: str = 'none', trim_right_ratio: float = 1.,
|
| 220 |
+
norm_kwargs: tp.Dict[str, tp.Any] = {}):
|
| 221 |
+
super().__init__()
|
| 222 |
+
self.convtr = NormConvTranspose1d(in_channels, out_channels, kernel_size, stride,
|
| 223 |
+
causal=causal, norm=norm, norm_kwargs=norm_kwargs)
|
| 224 |
+
self.causal = causal
|
| 225 |
+
self.trim_right_ratio = trim_right_ratio
|
| 226 |
+
assert self.causal or self.trim_right_ratio == 1., \
|
| 227 |
+
"`trim_right_ratio` != 1.0 only makes sense for causal convolutions"
|
| 228 |
+
assert self.trim_right_ratio >= 0. and self.trim_right_ratio <= 1.
|
| 229 |
+
|
| 230 |
+
def forward(self, x):
|
| 231 |
+
kernel_size = self.convtr.convtr.kernel_size[0]
|
| 232 |
+
stride = self.convtr.convtr.stride[0]
|
| 233 |
+
padding_total = kernel_size - stride
|
| 234 |
+
|
| 235 |
+
y = self.convtr(x)
|
| 236 |
+
|
| 237 |
+
# We will only trim fixed padding. Extra padding from `pad_for_conv1d` would be
|
| 238 |
+
# removed at the very end, when keeping only the right length for the output,
|
| 239 |
+
# as removing it here would require also passing the length at the matching layer
|
| 240 |
+
# in the encoder.
|
| 241 |
+
if self.causal:
|
| 242 |
+
# Trim the padding on the right according to the specified ratio
|
| 243 |
+
# if trim_right_ratio = 1.0, trim everything from right
|
| 244 |
+
padding_right = math.ceil(padding_total * self.trim_right_ratio)
|
| 245 |
+
padding_left = padding_total - padding_right
|
| 246 |
+
y = unpad1d(y, (padding_left, padding_right))
|
| 247 |
+
else:
|
| 248 |
+
# Asymmetric padding required for odd strides
|
| 249 |
+
padding_right = padding_total // 2
|
| 250 |
+
padding_left = padding_total - padding_right
|
| 251 |
+
y = unpad1d(y, (padding_left, padding_right))
|
| 252 |
+
return y
|
songgen/xcodec_wrapper/xcodec_infer/modules/lstm.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
# All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
|
| 7 |
+
"""LSTM layers module."""
|
| 8 |
+
|
| 9 |
+
from torch import nn
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class SLSTM(nn.Module):
|
| 13 |
+
"""
|
| 14 |
+
LSTM without worrying about the hidden state, nor the layout of the data.
|
| 15 |
+
Expects input as convolutional layout.
|
| 16 |
+
"""
|
| 17 |
+
def __init__(self, dimension: int, num_layers: int = 2, skip: bool = True):
|
| 18 |
+
super().__init__()
|
| 19 |
+
self.skip = skip
|
| 20 |
+
self.lstm = nn.LSTM(dimension, dimension, num_layers)
|
| 21 |
+
|
| 22 |
+
def forward(self, x):
|
| 23 |
+
x = x.permute(2, 0, 1)
|
| 24 |
+
y, _ = self.lstm(x)
|
| 25 |
+
if self.skip:
|
| 26 |
+
y = y + x
|
| 27 |
+
y = y.permute(1, 2, 0)
|
| 28 |
+
return y
|
songgen/xcodec_wrapper/xcodec_infer/modules/norm.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
# All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
|
| 7 |
+
"""Normalization modules."""
|
| 8 |
+
|
| 9 |
+
import typing as tp
|
| 10 |
+
|
| 11 |
+
import einops
|
| 12 |
+
import torch
|
| 13 |
+
from torch import nn
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
class ConvLayerNorm(nn.LayerNorm):
|
| 17 |
+
"""
|
| 18 |
+
Convolution-friendly LayerNorm that moves channels to last dimensions
|
| 19 |
+
before running the normalization and moves them back to original position right after.
|
| 20 |
+
"""
|
| 21 |
+
def __init__(self, normalized_shape: tp.Union[int, tp.List[int], torch.Size], **kwargs):
|
| 22 |
+
super().__init__(normalized_shape, **kwargs)
|
| 23 |
+
|
| 24 |
+
def forward(self, x):
|
| 25 |
+
x = einops.rearrange(x, 'b ... t -> b t ...')
|
| 26 |
+
x = super().forward(x)
|
| 27 |
+
x = einops.rearrange(x, 'b t ... -> b ... t')
|
| 28 |
+
return
|
songgen/xcodec_wrapper/xcodec_infer/modules/seanet.py
ADDED
|
@@ -0,0 +1,255 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
# All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
|
| 7 |
+
"""Encodec SEANet-based encoder and decoder implementation."""
|
| 8 |
+
|
| 9 |
+
import typing as tp
|
| 10 |
+
|
| 11 |
+
import numpy as np
|
| 12 |
+
import torch.nn as nn
|
| 13 |
+
|
| 14 |
+
from . import (
|
| 15 |
+
SConv1d,
|
| 16 |
+
SConvTranspose1d,
|
| 17 |
+
SLSTM
|
| 18 |
+
)
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
class SEANetResnetBlock(nn.Module):
|
| 22 |
+
"""Residual block from SEANet model.
|
| 23 |
+
Args:
|
| 24 |
+
dim (int): Dimension of the input/output
|
| 25 |
+
kernel_sizes (list): List of kernel sizes for the convolutions.
|
| 26 |
+
dilations (list): List of dilations for the convolutions.
|
| 27 |
+
activation (str): Activation function.
|
| 28 |
+
activation_params (dict): Parameters to provide to the activation function
|
| 29 |
+
norm (str): Normalization method.
|
| 30 |
+
norm_params (dict): Parameters to provide to the underlying normalization used along with the convolution.
|
| 31 |
+
causal (bool): Whether to use fully causal convolution.
|
| 32 |
+
pad_mode (str): Padding mode for the convolutions.
|
| 33 |
+
compress (int): Reduced dimensionality in residual branches (from Demucs v3)
|
| 34 |
+
true_skip (bool): Whether to use true skip connection or a simple convolution as the skip connection.
|
| 35 |
+
"""
|
| 36 |
+
def __init__(self, dim: int, kernel_sizes: tp.List[int] = [3, 1], dilations: tp.List[int] = [1, 1],
|
| 37 |
+
activation: str = 'ELU', activation_params: dict = {'alpha': 1.0},
|
| 38 |
+
norm: str = 'weight_norm', norm_params: tp.Dict[str, tp.Any] = {}, causal: bool = False,
|
| 39 |
+
pad_mode: str = 'reflect', compress: int = 2, true_skip: bool = True):
|
| 40 |
+
super().__init__()
|
| 41 |
+
assert len(kernel_sizes) == len(dilations), 'Number of kernel sizes should match number of dilations'
|
| 42 |
+
act = getattr(nn, activation)
|
| 43 |
+
hidden = dim // compress
|
| 44 |
+
block = []
|
| 45 |
+
for i, (kernel_size, dilation) in enumerate(zip(kernel_sizes, dilations)):
|
| 46 |
+
in_chs = dim if i == 0 else hidden
|
| 47 |
+
out_chs = dim if i == len(kernel_sizes) - 1 else hidden
|
| 48 |
+
block += [
|
| 49 |
+
act(**activation_params),
|
| 50 |
+
SConv1d(in_chs, out_chs, kernel_size=kernel_size, dilation=dilation,
|
| 51 |
+
norm=norm, norm_kwargs=norm_params,
|
| 52 |
+
causal=causal, pad_mode=pad_mode),
|
| 53 |
+
]
|
| 54 |
+
self.block = nn.Sequential(*block)
|
| 55 |
+
self.shortcut: nn.Module
|
| 56 |
+
if true_skip:
|
| 57 |
+
self.shortcut = nn.Identity()
|
| 58 |
+
else:
|
| 59 |
+
self.shortcut = SConv1d(dim, dim, kernel_size=1, norm=norm, norm_kwargs=norm_params,
|
| 60 |
+
causal=causal, pad_mode=pad_mode)
|
| 61 |
+
|
| 62 |
+
def forward(self, x):
|
| 63 |
+
return self.shortcut(x) + self.block(x)
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
class SEANetEncoder(nn.Module):
|
| 67 |
+
"""SEANet encoder.
|
| 68 |
+
Args:
|
| 69 |
+
channels (int): Audio channels.
|
| 70 |
+
dimension (int): Intermediate representation dimension.
|
| 71 |
+
n_filters (int): Base width for the model.
|
| 72 |
+
n_residual_layers (int): nb of residual layers.
|
| 73 |
+
ratios (Sequence[int]): kernel size and stride ratios. The encoder uses downsampling ratios instead of
|
| 74 |
+
upsampling ratios, hence it will use the ratios in the reverse order to the ones specified here
|
| 75 |
+
that must match the decoder order
|
| 76 |
+
activation (str): Activation function.
|
| 77 |
+
activation_params (dict): Parameters to provide to the activation function
|
| 78 |
+
norm (str): Normalization method.
|
| 79 |
+
norm_params (dict): Parameters to provide to the underlying normalization used along with the convolution.
|
| 80 |
+
kernel_size (int): Kernel size for the initial convolution.
|
| 81 |
+
last_kernel_size (int): Kernel size for the initial convolution.
|
| 82 |
+
residual_kernel_size (int): Kernel size for the residual layers.
|
| 83 |
+
dilation_base (int): How much to increase the dilation with each layer.
|
| 84 |
+
causal (bool): Whether to use fully causal convolution.
|
| 85 |
+
pad_mode (str): Padding mode for the convolutions.
|
| 86 |
+
true_skip (bool): Whether to use true skip connection or a simple
|
| 87 |
+
(streamable) convolution as the skip connection in the residual network blocks.
|
| 88 |
+
compress (int): Reduced dimensionality in residual branches (from Demucs v3).
|
| 89 |
+
lstm (int): Number of LSTM layers at the end of the encoder.
|
| 90 |
+
"""
|
| 91 |
+
def __init__(self, channels: int = 1, dimension: int = 128, n_filters: int = 32, n_residual_layers: int = 1,
|
| 92 |
+
ratios: tp.List[int] = [8, 5, 4, 2], activation: str = 'ELU', activation_params: dict = {'alpha': 1.0},
|
| 93 |
+
norm: str = 'weight_norm', norm_params: tp.Dict[str, tp.Any] = {}, kernel_size: int = 7,
|
| 94 |
+
last_kernel_size: int = 7, residual_kernel_size: int = 3, dilation_base: int = 2, causal: bool = False,
|
| 95 |
+
pad_mode: str = 'reflect', true_skip: bool = False, compress: int = 2, lstm: int = 2):
|
| 96 |
+
super().__init__()
|
| 97 |
+
self.channels = channels
|
| 98 |
+
self.dimension = dimension
|
| 99 |
+
self.n_filters = n_filters
|
| 100 |
+
self.ratios = list(reversed(ratios))
|
| 101 |
+
del ratios
|
| 102 |
+
self.n_residual_layers = n_residual_layers
|
| 103 |
+
self.hop_length = np.prod(self.ratios) # 计算乘积
|
| 104 |
+
|
| 105 |
+
act = getattr(nn, activation)
|
| 106 |
+
mult = 1
|
| 107 |
+
model: tp.List[nn.Module] = [
|
| 108 |
+
SConv1d(channels, mult * n_filters, kernel_size, norm=norm, norm_kwargs=norm_params,
|
| 109 |
+
causal=causal, pad_mode=pad_mode)
|
| 110 |
+
]
|
| 111 |
+
# Downsample to raw audio scale
|
| 112 |
+
for i, ratio in enumerate(self.ratios):
|
| 113 |
+
# Add residual layers
|
| 114 |
+
for j in range(n_residual_layers):
|
| 115 |
+
model += [
|
| 116 |
+
SEANetResnetBlock(mult * n_filters, kernel_sizes=[residual_kernel_size, 1],
|
| 117 |
+
dilations=[dilation_base ** j, 1],
|
| 118 |
+
norm=norm, norm_params=norm_params,
|
| 119 |
+
activation=activation, activation_params=activation_params,
|
| 120 |
+
causal=causal, pad_mode=pad_mode, compress=compress, true_skip=true_skip)]
|
| 121 |
+
|
| 122 |
+
# Add downsampling layers
|
| 123 |
+
model += [
|
| 124 |
+
act(**activation_params),
|
| 125 |
+
SConv1d(mult * n_filters, mult * n_filters * 2,
|
| 126 |
+
kernel_size=ratio * 2, stride=ratio,
|
| 127 |
+
norm=norm, norm_kwargs=norm_params,
|
| 128 |
+
causal=causal, pad_mode=pad_mode),
|
| 129 |
+
]
|
| 130 |
+
mult *= 2
|
| 131 |
+
|
| 132 |
+
if lstm:
|
| 133 |
+
model += [SLSTM(mult * n_filters, num_layers=lstm)]
|
| 134 |
+
|
| 135 |
+
model += [
|
| 136 |
+
act(**activation_params),
|
| 137 |
+
SConv1d(mult * n_filters, dimension, last_kernel_size, norm=norm, norm_kwargs=norm_params,
|
| 138 |
+
causal=causal, pad_mode=pad_mode)
|
| 139 |
+
]
|
| 140 |
+
|
| 141 |
+
self.model = nn.Sequential(*model)
|
| 142 |
+
|
| 143 |
+
def forward(self, x):
|
| 144 |
+
return self.model(x)
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
class SEANetDecoder(nn.Module):
|
| 148 |
+
"""SEANet decoder.
|
| 149 |
+
Args:
|
| 150 |
+
channels (int): Audio channels.
|
| 151 |
+
dimension (int): Intermediate representation dimension.
|
| 152 |
+
n_filters (int): Base width for the model.
|
| 153 |
+
n_residual_layers (int): nb of residual layers.
|
| 154 |
+
ratios (Sequence[int]): kernel size and stride ratios
|
| 155 |
+
activation (str): Activation function.
|
| 156 |
+
activation_params (dict): Parameters to provide to the activation function
|
| 157 |
+
final_activation (str): Final activation function after all convolutions.
|
| 158 |
+
final_activation_params (dict): Parameters to provide to the activation function
|
| 159 |
+
norm (str): Normalization method.
|
| 160 |
+
norm_params (dict): Parameters to provide to the underlying normalization used along with the convolution.
|
| 161 |
+
kernel_size (int): Kernel size for the initial convolution.
|
| 162 |
+
last_kernel_size (int): Kernel size for the initial convolution.
|
| 163 |
+
residual_kernel_size (int): Kernel size for the residual layers.
|
| 164 |
+
dilation_base (int): How much to increase the dilation with each layer.
|
| 165 |
+
causal (bool): Whether to use fully causal convolution.
|
| 166 |
+
pad_mode (str): Padding mode for the convolutions.
|
| 167 |
+
true_skip (bool): Whether to use true skip connection or a simple
|
| 168 |
+
(streamable) convolution as the skip connection in the residual network blocks.
|
| 169 |
+
compress (int): Reduced dimensionality in residual branches (from Demucs v3).
|
| 170 |
+
lstm (int): Number of LSTM layers at the end of the encoder.
|
| 171 |
+
trim_right_ratio (float): Ratio for trimming at the right of the transposed convolution under the causal setup.
|
| 172 |
+
If equal to 1.0, it means that all the trimming is done at the right.
|
| 173 |
+
"""
|
| 174 |
+
def __init__(self, channels: int = 1, dimension: int = 128, n_filters: int = 32, n_residual_layers: int = 1,
|
| 175 |
+
ratios: tp.List[int] = [8, 5, 4, 2], activation: str = 'ELU', activation_params: dict = {'alpha': 1.0},
|
| 176 |
+
final_activation: tp.Optional[str] = None, final_activation_params: tp.Optional[dict] = None,
|
| 177 |
+
norm: str = 'weight_norm', norm_params: tp.Dict[str, tp.Any] = {}, kernel_size: int = 7,
|
| 178 |
+
last_kernel_size: int = 7, residual_kernel_size: int = 3, dilation_base: int = 2, causal: bool = False,
|
| 179 |
+
pad_mode: str = 'reflect', true_skip: bool = False, compress: int = 2, lstm: int = 2,
|
| 180 |
+
trim_right_ratio: float = 1.0):
|
| 181 |
+
super().__init__()
|
| 182 |
+
self.dimension = dimension
|
| 183 |
+
self.channels = channels
|
| 184 |
+
self.n_filters = n_filters
|
| 185 |
+
self.ratios = ratios
|
| 186 |
+
del ratios
|
| 187 |
+
self.n_residual_layers = n_residual_layers
|
| 188 |
+
self.hop_length = np.prod(self.ratios)
|
| 189 |
+
|
| 190 |
+
act = getattr(nn, activation)
|
| 191 |
+
mult = int(2 ** len(self.ratios))
|
| 192 |
+
model: tp.List[nn.Module] = [
|
| 193 |
+
SConv1d(dimension, mult * n_filters, kernel_size, norm=norm, norm_kwargs=norm_params,
|
| 194 |
+
causal=causal, pad_mode=pad_mode)
|
| 195 |
+
]
|
| 196 |
+
|
| 197 |
+
if lstm:
|
| 198 |
+
model += [SLSTM(mult * n_filters, num_layers=lstm)]
|
| 199 |
+
|
| 200 |
+
# Upsample to raw audio scale
|
| 201 |
+
for i, ratio in enumerate(self.ratios):
|
| 202 |
+
# Add upsampling layers
|
| 203 |
+
model += [
|
| 204 |
+
act(**activation_params),
|
| 205 |
+
SConvTranspose1d(mult * n_filters, mult * n_filters // 2,
|
| 206 |
+
kernel_size=ratio * 2, stride=ratio,
|
| 207 |
+
norm=norm, norm_kwargs=norm_params,
|
| 208 |
+
causal=causal, trim_right_ratio=trim_right_ratio),
|
| 209 |
+
]
|
| 210 |
+
# Add residual layers
|
| 211 |
+
for j in range(n_residual_layers):
|
| 212 |
+
model += [
|
| 213 |
+
SEANetResnetBlock(mult * n_filters // 2, kernel_sizes=[residual_kernel_size, 1],
|
| 214 |
+
dilations=[dilation_base ** j, 1],
|
| 215 |
+
activation=activation, activation_params=activation_params,
|
| 216 |
+
norm=norm, norm_params=norm_params, causal=causal,
|
| 217 |
+
pad_mode=pad_mode, compress=compress, true_skip=true_skip)]
|
| 218 |
+
|
| 219 |
+
mult //= 2
|
| 220 |
+
|
| 221 |
+
# Add final layers
|
| 222 |
+
model += [
|
| 223 |
+
act(**activation_params),
|
| 224 |
+
SConv1d(n_filters, channels, last_kernel_size, norm=norm, norm_kwargs=norm_params,
|
| 225 |
+
causal=causal, pad_mode=pad_mode)
|
| 226 |
+
]
|
| 227 |
+
# Add optional final activation to decoder (eg. tanh)
|
| 228 |
+
if final_activation is not None:
|
| 229 |
+
final_act = getattr(nn, final_activation)
|
| 230 |
+
final_activation_params = final_activation_params or {}
|
| 231 |
+
model += [
|
| 232 |
+
final_act(**final_activation_params)
|
| 233 |
+
]
|
| 234 |
+
self.model = nn.Sequential(*model)
|
| 235 |
+
|
| 236 |
+
def forward(self, z):
|
| 237 |
+
y = self.model(z)
|
| 238 |
+
return y
|
| 239 |
+
|
| 240 |
+
|
| 241 |
+
def test():
|
| 242 |
+
import torch
|
| 243 |
+
encoder = SEANetEncoder()
|
| 244 |
+
decoder = SEANetDecoder()
|
| 245 |
+
x = torch.randn(1, 1, 24000)
|
| 246 |
+
z = encoder(x)
|
| 247 |
+
print('z ', z.shape)
|
| 248 |
+
assert 1==2
|
| 249 |
+
assert list(z.shape) == [1, 128, 75], z.shape
|
| 250 |
+
y = decoder(z)
|
| 251 |
+
assert y.shape == x.shape, (x.shape, y.shape)
|
| 252 |
+
|
| 253 |
+
|
| 254 |
+
if __name__ == '__main__':
|
| 255 |
+
test()
|
songgen/xcodec_wrapper/xcodec_infer/modules/semantic_module.py
ADDED
|
@@ -0,0 +1,294 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
import torch
|
| 3 |
+
import torch.nn as nn
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
class Conv1d1x1(nn.Conv1d):
|
| 8 |
+
"""1x1 Conv1d."""
|
| 9 |
+
|
| 10 |
+
def __init__(self, in_channels, out_channels, bias=True):
|
| 11 |
+
super(Conv1d1x1, self).__init__(in_channels, out_channels, kernel_size=1, bias=bias)
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class Conv1d(nn.Module):
|
| 15 |
+
def __init__(
|
| 16 |
+
self,
|
| 17 |
+
in_channels: int,
|
| 18 |
+
out_channels: int,
|
| 19 |
+
kernel_size: int,
|
| 20 |
+
stride: int = 1,
|
| 21 |
+
padding: int = -1,
|
| 22 |
+
dilation: int = 1,
|
| 23 |
+
groups: int = 1,
|
| 24 |
+
bias: bool = True
|
| 25 |
+
):
|
| 26 |
+
super().__init__()
|
| 27 |
+
self.in_channels = in_channels
|
| 28 |
+
self.out_channels = out_channels
|
| 29 |
+
self.kernel_size = kernel_size
|
| 30 |
+
if padding < 0:
|
| 31 |
+
padding = (kernel_size - 1) // 2 * dilation
|
| 32 |
+
self.dilation = dilation
|
| 33 |
+
self.conv = nn.Conv1d(
|
| 34 |
+
in_channels=in_channels,
|
| 35 |
+
out_channels=out_channels,
|
| 36 |
+
kernel_size=kernel_size,
|
| 37 |
+
stride=stride,
|
| 38 |
+
padding=padding,
|
| 39 |
+
dilation=dilation,
|
| 40 |
+
groups=groups,
|
| 41 |
+
bias=bias,
|
| 42 |
+
)
|
| 43 |
+
|
| 44 |
+
def forward(self, x):
|
| 45 |
+
"""
|
| 46 |
+
Args:
|
| 47 |
+
x (Tensor): Float tensor variable with the shape (B, C, T).
|
| 48 |
+
Returns:
|
| 49 |
+
Tensor: Float tensor variable with the shape (B, C, T).
|
| 50 |
+
"""
|
| 51 |
+
x = self.conv(x)
|
| 52 |
+
return x
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
class ResidualUnit(nn.Module):
|
| 56 |
+
def __init__(
|
| 57 |
+
self,
|
| 58 |
+
in_channels: int,
|
| 59 |
+
out_channels: int,
|
| 60 |
+
kernel_size=3,
|
| 61 |
+
dilation=1,
|
| 62 |
+
bias=False,
|
| 63 |
+
nonlinear_activation="ELU",
|
| 64 |
+
nonlinear_activation_params={},
|
| 65 |
+
):
|
| 66 |
+
super().__init__()
|
| 67 |
+
self.activation = getattr(nn, nonlinear_activation)(**nonlinear_activation_params)
|
| 68 |
+
self.conv1 = Conv1d(
|
| 69 |
+
in_channels=in_channels,
|
| 70 |
+
out_channels=out_channels,
|
| 71 |
+
kernel_size=kernel_size,
|
| 72 |
+
stride=1,
|
| 73 |
+
dilation=dilation,
|
| 74 |
+
bias=bias,
|
| 75 |
+
)
|
| 76 |
+
self.conv2 = Conv1d1x1(out_channels, out_channels, bias)
|
| 77 |
+
|
| 78 |
+
def forward(self, x):
|
| 79 |
+
y = self.conv1(self.activation(x))
|
| 80 |
+
y = self.conv2(self.activation(y))
|
| 81 |
+
return x + y
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
class ConvTranspose1d(nn.Module):
|
| 85 |
+
def __init__(
|
| 86 |
+
self,
|
| 87 |
+
in_channels: int,
|
| 88 |
+
out_channels: int,
|
| 89 |
+
kernel_size: int,
|
| 90 |
+
stride: int,
|
| 91 |
+
padding=-1,
|
| 92 |
+
output_padding=-1,
|
| 93 |
+
groups=1,
|
| 94 |
+
bias=True,
|
| 95 |
+
):
|
| 96 |
+
super().__init__()
|
| 97 |
+
if padding < 0:
|
| 98 |
+
padding = (stride + 1) // 2
|
| 99 |
+
if output_padding < 0:
|
| 100 |
+
output_padding = 1 if stride % 2 else 0
|
| 101 |
+
self.deconv = nn.ConvTranspose1d(
|
| 102 |
+
in_channels=in_channels,
|
| 103 |
+
out_channels=out_channels,
|
| 104 |
+
kernel_size=kernel_size,
|
| 105 |
+
stride=stride,
|
| 106 |
+
padding=padding,
|
| 107 |
+
output_padding=output_padding,
|
| 108 |
+
groups=groups,
|
| 109 |
+
bias=bias,
|
| 110 |
+
)
|
| 111 |
+
|
| 112 |
+
def forward(self, x):
|
| 113 |
+
"""
|
| 114 |
+
Args:
|
| 115 |
+
x (Tensor): Float tensor variable with the shape (B, C, T).
|
| 116 |
+
Returns:
|
| 117 |
+
Tensor: Float tensor variable with the shape (B, C', T').
|
| 118 |
+
"""
|
| 119 |
+
x = self.deconv(x)
|
| 120 |
+
return x
|
| 121 |
+
|
| 122 |
+
class EncoderBlock(nn.Module):
|
| 123 |
+
def __init__(
|
| 124 |
+
self,
|
| 125 |
+
in_channels: int,
|
| 126 |
+
out_channels: int,
|
| 127 |
+
stride: int,
|
| 128 |
+
dilations=(1, 1),
|
| 129 |
+
unit_kernel_size=3,
|
| 130 |
+
bias=True
|
| 131 |
+
):
|
| 132 |
+
super().__init__()
|
| 133 |
+
self.res_units = torch.nn.ModuleList()
|
| 134 |
+
for dilation in dilations:
|
| 135 |
+
self.res_units += [
|
| 136 |
+
ResidualUnit(in_channels, in_channels,
|
| 137 |
+
kernel_size=unit_kernel_size,
|
| 138 |
+
dilation=dilation)
|
| 139 |
+
]
|
| 140 |
+
self.num_res = len(self.res_units)
|
| 141 |
+
|
| 142 |
+
self.conv = Conv1d(
|
| 143 |
+
in_channels=in_channels,
|
| 144 |
+
out_channels=out_channels,
|
| 145 |
+
kernel_size=3 if stride == 1 else (2 * stride), # special case: stride=1, do not use kernel=2
|
| 146 |
+
stride=stride,
|
| 147 |
+
bias=bias,
|
| 148 |
+
)
|
| 149 |
+
|
| 150 |
+
def forward(self, x):
|
| 151 |
+
for idx in range(self.num_res):
|
| 152 |
+
x = self.res_units[idx](x)
|
| 153 |
+
x = self.conv(x)
|
| 154 |
+
return x
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
class Encoder(nn.Module):
|
| 158 |
+
def __init__(
|
| 159 |
+
self,
|
| 160 |
+
input_channels: int,
|
| 161 |
+
encode_channels: int,
|
| 162 |
+
channel_ratios=(1, 1),
|
| 163 |
+
strides=(1, 1),
|
| 164 |
+
kernel_size=3,
|
| 165 |
+
bias=True,
|
| 166 |
+
block_dilations=(1, 1),
|
| 167 |
+
unit_kernel_size=3
|
| 168 |
+
):
|
| 169 |
+
super().__init__()
|
| 170 |
+
assert len(channel_ratios) == len(strides)
|
| 171 |
+
|
| 172 |
+
self.conv = Conv1d(
|
| 173 |
+
in_channels=input_channels,
|
| 174 |
+
out_channels=encode_channels,
|
| 175 |
+
kernel_size=kernel_size,
|
| 176 |
+
stride=1,
|
| 177 |
+
bias=False
|
| 178 |
+
)
|
| 179 |
+
self.conv_blocks = torch.nn.ModuleList()
|
| 180 |
+
in_channels = encode_channels
|
| 181 |
+
for idx, stride in enumerate(strides):
|
| 182 |
+
out_channels = int(encode_channels * channel_ratios[idx]) # could be float
|
| 183 |
+
self.conv_blocks += [
|
| 184 |
+
EncoderBlock(in_channels, out_channels, stride,
|
| 185 |
+
dilations=block_dilations, unit_kernel_size=unit_kernel_size,
|
| 186 |
+
bias=bias)
|
| 187 |
+
]
|
| 188 |
+
in_channels = out_channels
|
| 189 |
+
self.num_blocks = len(self.conv_blocks)
|
| 190 |
+
self.out_channels = out_channels
|
| 191 |
+
|
| 192 |
+
def forward(self, x):
|
| 193 |
+
x = self.conv(x)
|
| 194 |
+
for i in range(self.num_blocks):
|
| 195 |
+
x = self.conv_blocks[i](x)
|
| 196 |
+
return x
|
| 197 |
+
|
| 198 |
+
|
| 199 |
+
|
| 200 |
+
class DecoderBlock(nn.Module):
|
| 201 |
+
""" Decoder block (no up-sampling) """
|
| 202 |
+
|
| 203 |
+
def __init__(
|
| 204 |
+
self,
|
| 205 |
+
in_channels: int,
|
| 206 |
+
out_channels: int,
|
| 207 |
+
stride: int,
|
| 208 |
+
dilations=(1, 1),
|
| 209 |
+
unit_kernel_size=3,
|
| 210 |
+
bias=True
|
| 211 |
+
):
|
| 212 |
+
super().__init__()
|
| 213 |
+
|
| 214 |
+
if stride == 1:
|
| 215 |
+
self.conv = Conv1d(
|
| 216 |
+
in_channels=in_channels,
|
| 217 |
+
out_channels=out_channels,
|
| 218 |
+
kernel_size=3, # fix kernel=3 when stride=1 for unchanged shape
|
| 219 |
+
stride=stride,
|
| 220 |
+
bias=bias,
|
| 221 |
+
)
|
| 222 |
+
else:
|
| 223 |
+
self.conv = ConvTranspose1d(
|
| 224 |
+
in_channels=in_channels,
|
| 225 |
+
out_channels=out_channels,
|
| 226 |
+
kernel_size=(2 * stride),
|
| 227 |
+
stride=stride,
|
| 228 |
+
bias=bias,
|
| 229 |
+
)
|
| 230 |
+
|
| 231 |
+
self.res_units = torch.nn.ModuleList()
|
| 232 |
+
for idx, dilation in enumerate(dilations):
|
| 233 |
+
self.res_units += [
|
| 234 |
+
ResidualUnit(out_channels, out_channels,
|
| 235 |
+
kernel_size=unit_kernel_size,
|
| 236 |
+
dilation=dilation)
|
| 237 |
+
]
|
| 238 |
+
self.num_res = len(self.res_units)
|
| 239 |
+
|
| 240 |
+
def forward(self, x):
|
| 241 |
+
x = self.conv(x)
|
| 242 |
+
for idx in range(self.num_res):
|
| 243 |
+
x = self.res_units[idx](x)
|
| 244 |
+
return x
|
| 245 |
+
|
| 246 |
+
|
| 247 |
+
class Decoder(nn.Module):
|
| 248 |
+
def __init__(
|
| 249 |
+
self,
|
| 250 |
+
code_dim: int,
|
| 251 |
+
output_channels: int,
|
| 252 |
+
decode_channels: int,
|
| 253 |
+
channel_ratios=(1, 1),
|
| 254 |
+
strides=(1, 1),
|
| 255 |
+
kernel_size=3,
|
| 256 |
+
bias=True,
|
| 257 |
+
block_dilations=(1, 1),
|
| 258 |
+
unit_kernel_size=3,
|
| 259 |
+
):
|
| 260 |
+
super().__init__()
|
| 261 |
+
assert len(channel_ratios) == len(strides)
|
| 262 |
+
|
| 263 |
+
self.conv1 = Conv1d(
|
| 264 |
+
in_channels=code_dim,
|
| 265 |
+
out_channels=int(decode_channels * channel_ratios[0]),
|
| 266 |
+
kernel_size=kernel_size,
|
| 267 |
+
stride=1,
|
| 268 |
+
bias=False
|
| 269 |
+
)
|
| 270 |
+
|
| 271 |
+
self.conv_blocks = torch.nn.ModuleList()
|
| 272 |
+
for idx, stride in enumerate(strides):
|
| 273 |
+
in_channels = int(decode_channels * channel_ratios[idx])
|
| 274 |
+
if idx < (len(channel_ratios) - 1):
|
| 275 |
+
out_channels = int(decode_channels * channel_ratios[idx + 1])
|
| 276 |
+
else:
|
| 277 |
+
out_channels = decode_channels
|
| 278 |
+
self.conv_blocks += [
|
| 279 |
+
DecoderBlock(
|
| 280 |
+
in_channels, out_channels, stride,
|
| 281 |
+
dilations=block_dilations, unit_kernel_size=unit_kernel_size,
|
| 282 |
+
bias=bias
|
| 283 |
+
)
|
| 284 |
+
]
|
| 285 |
+
self.num_blocks = len(self.conv_blocks)
|
| 286 |
+
|
| 287 |
+
self.conv2 = Conv1d(out_channels, output_channels, kernel_size, 1, bias=False)
|
| 288 |
+
|
| 289 |
+
def forward(self, z):
|
| 290 |
+
x = self.conv1(z)
|
| 291 |
+
for i in range(self.num_blocks):
|
| 292 |
+
x = self.conv_blocks[i](x)
|
| 293 |
+
x = self.conv2(x)
|
| 294 |
+
return x
|
songgen/xcodec_wrapper/xcodec_infer/modules/transformer.py
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
# All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
|
| 7 |
+
"""A streamable transformer."""
|
| 8 |
+
|
| 9 |
+
import typing as tp
|
| 10 |
+
|
| 11 |
+
import torch
|
| 12 |
+
import torch.nn as nn
|
| 13 |
+
import torch.nn.functional as F
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def create_sin_embedding(positions: torch.Tensor, dim: int, max_period: float = 10000):
|
| 17 |
+
"""Create time embedding for the given positions, target dimension `dim`.
|
| 18 |
+
"""
|
| 19 |
+
# We aim for BTC format
|
| 20 |
+
assert dim % 2 == 0
|
| 21 |
+
half_dim = dim // 2
|
| 22 |
+
adim = torch.arange(half_dim, device=positions.device).view(1, 1, -1)
|
| 23 |
+
phase = positions / (max_period ** (adim / (half_dim - 1)))
|
| 24 |
+
return torch.cat([
|
| 25 |
+
torch.cos(phase),
|
| 26 |
+
torch.sin(phase),
|
| 27 |
+
], dim=-1)
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
class StreamingTransformerEncoderLayer(nn.TransformerEncoderLayer):
|
| 31 |
+
def forward(self, x: torch.Tensor, x_past: torch.Tensor, past_context: int): # type: ignore
|
| 32 |
+
if self.norm_first:
|
| 33 |
+
sa_input = self.norm1(x)
|
| 34 |
+
x = x + self._sa_block(sa_input, x_past, past_context)
|
| 35 |
+
x = x + self._ff_block(self.norm2(x))
|
| 36 |
+
else:
|
| 37 |
+
sa_input = x
|
| 38 |
+
x = self.norm1(x + self._sa_block(sa_input, x_past, past_context))
|
| 39 |
+
x = self.norm2(x + self._ff_block(x))
|
| 40 |
+
|
| 41 |
+
return x, sa_input
|
| 42 |
+
|
| 43 |
+
# self-attention block
|
| 44 |
+
def _sa_block(self, x: torch.Tensor, x_past: torch.Tensor, past_context: int): # type: ignore
|
| 45 |
+
_, T, _ = x.shape
|
| 46 |
+
_, H, _ = x_past.shape
|
| 47 |
+
|
| 48 |
+
queries = x
|
| 49 |
+
keys = torch.cat([x_past, x], dim=1)
|
| 50 |
+
values = keys
|
| 51 |
+
|
| 52 |
+
queries_pos = torch.arange(H, T + H, device=x.device).view(-1, 1)
|
| 53 |
+
keys_pos = torch.arange(T + H, device=x.device).view(1, -1)
|
| 54 |
+
delta = queries_pos - keys_pos
|
| 55 |
+
valid_access = (delta >= 0) & (delta <= past_context)
|
| 56 |
+
x = self.self_attn(queries, keys, values,
|
| 57 |
+
attn_mask=~valid_access,
|
| 58 |
+
need_weights=False)[0]
|
| 59 |
+
return self.dropout1(x)
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
class StreamingTransformerEncoder(nn.Module):
|
| 63 |
+
"""TransformerEncoder with streaming support.
|
| 64 |
+
|
| 65 |
+
Args:
|
| 66 |
+
dim (int): dimension of the data.
|
| 67 |
+
hidden_scale (int): intermediate dimension of FF module is this times the dimension.
|
| 68 |
+
num_heads (int): number of heads.
|
| 69 |
+
num_layers (int): number of layers.
|
| 70 |
+
max_period (float): maxium period of cosines in the positional embedding.
|
| 71 |
+
past_context (int or None): receptive field for the causal mask, infinite if None.
|
| 72 |
+
gelu (bool): if true uses GeLUs, otherwise use ReLUs.
|
| 73 |
+
norm_in (bool): normalize the input.
|
| 74 |
+
dropout (float): dropout probability.
|
| 75 |
+
**kwargs: See `nn.TransformerEncoderLayer`.
|
| 76 |
+
"""
|
| 77 |
+
def __init__(self, dim, hidden_scale: float = 4., num_heads: int = 8, num_layers: int = 5,
|
| 78 |
+
max_period: float = 10000, past_context: int = 1000, gelu: bool = True,
|
| 79 |
+
norm_in: bool = True, dropout: float = 0., **kwargs):
|
| 80 |
+
super().__init__()
|
| 81 |
+
assert dim % num_heads == 0
|
| 82 |
+
hidden_dim = int(dim * hidden_scale)
|
| 83 |
+
|
| 84 |
+
self.max_period = max_period
|
| 85 |
+
self.past_context = past_context
|
| 86 |
+
activation: tp.Any = F.gelu if gelu else F.relu
|
| 87 |
+
|
| 88 |
+
self.norm_in: nn.Module
|
| 89 |
+
if norm_in:
|
| 90 |
+
self.norm_in = nn.LayerNorm(dim)
|
| 91 |
+
else:
|
| 92 |
+
self.norm_in = nn.Identity()
|
| 93 |
+
|
| 94 |
+
self.layers = nn.ModuleList()
|
| 95 |
+
for idx in range(num_layers):
|
| 96 |
+
self.layers.append(
|
| 97 |
+
StreamingTransformerEncoderLayer(
|
| 98 |
+
dim, num_heads, hidden_dim,
|
| 99 |
+
activation=activation, batch_first=True, dropout=dropout, **kwargs))
|
| 100 |
+
|
| 101 |
+
def forward(self, x: torch.Tensor,
|
| 102 |
+
states: tp.Optional[tp.List[torch.Tensor]] = None,
|
| 103 |
+
offset: tp.Union[int, torch.Tensor] = 0):
|
| 104 |
+
B, T, C = x.shape
|
| 105 |
+
if states is None:
|
| 106 |
+
states = [torch.zeros_like(x[:, :1]) for _ in range(1 + len(self.layers))]
|
| 107 |
+
|
| 108 |
+
positions = torch.arange(T, device=x.device).view(1, -1, 1) + offset
|
| 109 |
+
pos_emb = create_sin_embedding(positions, C, max_period=self.max_period)
|
| 110 |
+
|
| 111 |
+
new_state: tp.List[torch.Tensor] = []
|
| 112 |
+
x = self.norm_in(x)
|
| 113 |
+
x = x + pos_emb
|
| 114 |
+
|
| 115 |
+
for layer_state, layer in zip(states, self.layers):
|
| 116 |
+
x, new_layer_state = layer(x, layer_state, self.past_context)
|
| 117 |
+
new_layer_state = torch.cat([layer_state, new_layer_state], dim=1)
|
| 118 |
+
new_state.append(new_layer_state[:, -self.past_context:, :])
|
| 119 |
+
return x, new_state, offset + T
|
songgen/xcodec_wrapper/xcodec_infer/quantization/__init__.py
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
# All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
|
| 7 |
+
# flake8: noqa
|
| 8 |
+
from .vq import QuantizedResult, ResidualVectorQuantizer
|
songgen/xcodec_wrapper/xcodec_infer/quantization/core_vq_lsx_version.py
ADDED
|
@@ -0,0 +1,433 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c)
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
# This implementation is inspired from
|
| 6 |
+
# https://github.com/rosinality/vq-vae-2-pytorch/blob/master/vqvae.py and
|
| 7 |
+
# https://github.com/clementchadebec/benchmark_VAE/blob/dfa0dcf6c79172df5d27769c09c860c42008baaa/src/pythae/models/vq_vae/vq_vae_utils.py#L81
|
| 8 |
+
#
|
| 9 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 10 |
+
# All rights reserved.
|
| 11 |
+
#
|
| 12 |
+
# This source code is licensed under the license found in the
|
| 13 |
+
# LICENSE file in the root directory of this source tree.
|
| 14 |
+
#
|
| 15 |
+
# This implementation is inspired from
|
| 16 |
+
# https://github.com/lucidrains/vector-quantize-pytorch
|
| 17 |
+
# which is released under MIT License. Hereafter, the original license:
|
| 18 |
+
# MIT License
|
| 19 |
+
#
|
| 20 |
+
# Copyright (c) 2020 Phil Wang
|
| 21 |
+
#
|
| 22 |
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
| 23 |
+
# of this software and associated documentation files (the "Software"), to deal
|
| 24 |
+
# in the Software without restriction, including without limitation the rights
|
| 25 |
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
| 26 |
+
# copies of the Software, and to permit persons to whom the Software is
|
| 27 |
+
# furnished to do so, subject to the following conditions:
|
| 28 |
+
#
|
| 29 |
+
# The above copyright notice and this permission notice shall be included in all
|
| 30 |
+
# copies or substantial portions of the Software.
|
| 31 |
+
#
|
| 32 |
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
| 33 |
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
| 34 |
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
| 35 |
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
| 36 |
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
| 37 |
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
| 38 |
+
# SOFTWARE.
|
| 39 |
+
|
| 40 |
+
"""Core vector quantization implementation."""
|
| 41 |
+
import typing as tp
|
| 42 |
+
|
| 43 |
+
from einops import rearrange, repeat
|
| 44 |
+
import torch
|
| 45 |
+
from torch import nn
|
| 46 |
+
import torch.nn.functional as F
|
| 47 |
+
import torch.distributed as dist
|
| 48 |
+
|
| 49 |
+
from .distrib import broadcast_tensors, rank, is_distributed
|
| 50 |
+
from xutils.utils import is_primary
|
| 51 |
+
from xutils.ddp_utils import SyncFunction
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def default(val: tp.Any, d: tp.Any) -> tp.Any:
|
| 55 |
+
return val if val is not None else d
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def ema_inplace(moving_avg, new, decay: float):
|
| 59 |
+
moving_avg.data.mul_(decay).add_(new, alpha=(1 - decay))
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def laplace_smoothing(x, n_categories: int, epsilon: float = 1e-5):
|
| 63 |
+
return (x + epsilon) / (x.sum() + n_categories * epsilon)
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def uniform_init(*shape: int):
|
| 67 |
+
t = torch.empty(shape)
|
| 68 |
+
nn.init.kaiming_uniform_(t)
|
| 69 |
+
return t
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def sample_vectors(samples, num: int):
|
| 73 |
+
num_samples, device = samples.shape[0], samples.device
|
| 74 |
+
|
| 75 |
+
if num_samples >= num:
|
| 76 |
+
indices = torch.randperm(num_samples, device=device)[:num]
|
| 77 |
+
else:
|
| 78 |
+
indices = torch.randint(0, num_samples, (num,), device=device)
|
| 79 |
+
|
| 80 |
+
return samples[indices]
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
def kmeans(samples, num_clusters: int, num_iters: int = 10, frames_to_use: int = 10_000):
|
| 84 |
+
""" Run K-means clustering on samples.
|
| 85 |
+
Args:
|
| 86 |
+
samples (tensor): shape [B * T, D]
|
| 87 |
+
num_clusters (int): number of centroids.
|
| 88 |
+
num_iters (int): number of iterations.
|
| 89 |
+
"""
|
| 90 |
+
dim, dtype = samples.shape[-1], samples.dtype
|
| 91 |
+
|
| 92 |
+
if frames_to_use < samples.shape[0]:
|
| 93 |
+
samples = sample_vectors(samples, frames_to_use)
|
| 94 |
+
|
| 95 |
+
# Init means
|
| 96 |
+
means = sample_vectors(samples, num_clusters)
|
| 97 |
+
|
| 98 |
+
for _ in range(num_iters):
|
| 99 |
+
diffs = rearrange(samples, "n d -> n () d") - rearrange(
|
| 100 |
+
means, "c d -> () c d"
|
| 101 |
+
)
|
| 102 |
+
dists = -(diffs ** 2).sum(dim=-1)
|
| 103 |
+
|
| 104 |
+
buckets = dists.max(dim=-1).indices
|
| 105 |
+
bins = torch.bincount(buckets, minlength=num_clusters)
|
| 106 |
+
zero_mask = bins == 0
|
| 107 |
+
bins_min_clamped = bins.masked_fill(zero_mask, 1)
|
| 108 |
+
|
| 109 |
+
new_means = buckets.new_zeros(num_clusters, dim, dtype=dtype)
|
| 110 |
+
new_means.scatter_add_(0, repeat(buckets, "n -> n d", d=dim), samples)
|
| 111 |
+
new_means = new_means / bins_min_clamped[..., None]
|
| 112 |
+
|
| 113 |
+
means = torch.where(zero_mask[..., None], means, new_means)
|
| 114 |
+
|
| 115 |
+
return means, bins
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
class EuclideanCodebook(nn.Module):
|
| 119 |
+
"""Codebook with Euclidean distance.
|
| 120 |
+
Args:
|
| 121 |
+
dim (int): Dimension.
|
| 122 |
+
codebook_size (int): Codebook size.
|
| 123 |
+
kmeans_init (bool): Whether to use k-means to initialize the codebooks.
|
| 124 |
+
If set to true, run the k-means algorithm on the first training batch and use
|
| 125 |
+
the learned centroids as initialization.
|
| 126 |
+
kmeans_iters (int): Number of iterations used for k-means algorithm at initialization.
|
| 127 |
+
decay (float): Decay for exponential moving average over the codebooks.
|
| 128 |
+
epsilon (float): Epsilon value for numerical stability.
|
| 129 |
+
threshold_ema_dead_code (int): Threshold for dead code expiration. Replace any codes
|
| 130 |
+
that have an exponential moving average cluster size less than the specified threshold with
|
| 131 |
+
randomly selected vector from the current batch.
|
| 132 |
+
"""
|
| 133 |
+
def __init__(
|
| 134 |
+
self,
|
| 135 |
+
dim: int,
|
| 136 |
+
codebook_size: int,
|
| 137 |
+
kmeans_init: int = False,
|
| 138 |
+
kmeans_iters: int = 10,
|
| 139 |
+
decay: float = 0.99,
|
| 140 |
+
epsilon: float = 1e-5,
|
| 141 |
+
threshold_ema_dead_code: int = 2,
|
| 142 |
+
):
|
| 143 |
+
super().__init__()
|
| 144 |
+
self.decay = decay
|
| 145 |
+
init_fn: tp.Union[tp.Callable[..., torch.Tensor], tp.Any] = uniform_init if not kmeans_init else torch.zeros
|
| 146 |
+
embed = init_fn(codebook_size, dim)
|
| 147 |
+
|
| 148 |
+
self.codebook_size = codebook_size
|
| 149 |
+
|
| 150 |
+
self.kmeans_iters = kmeans_iters
|
| 151 |
+
self.epsilon = epsilon
|
| 152 |
+
self.threshold_ema_dead_code = threshold_ema_dead_code
|
| 153 |
+
|
| 154 |
+
# Flag variable to indicate whether the codebook is initialized
|
| 155 |
+
self.register_buffer("inited", torch.Tensor([not kmeans_init]))
|
| 156 |
+
# Runing EMA cluster size/count: N_i^t in eq. (6) in vqvae paper
|
| 157 |
+
self.register_buffer("cluster_size", torch.zeros(codebook_size))
|
| 158 |
+
# Codebook
|
| 159 |
+
self.register_buffer("embed", embed)
|
| 160 |
+
# EMA codebook: eq. (7) in vqvae paper
|
| 161 |
+
self.register_buffer("embed_avg", embed.clone())
|
| 162 |
+
|
| 163 |
+
@torch.jit.ignore
|
| 164 |
+
def init_embed_(self, data):
|
| 165 |
+
""" Initialize codebook.
|
| 166 |
+
Args:
|
| 167 |
+
data (tensor): [B * T, D].
|
| 168 |
+
"""
|
| 169 |
+
if self.inited:
|
| 170 |
+
return
|
| 171 |
+
|
| 172 |
+
# if is_primary():
|
| 173 |
+
# print(data.shape)
|
| 174 |
+
|
| 175 |
+
## NOTE (snippet added by Songxiang Liu): gather data from all gpus
|
| 176 |
+
if dist.is_available() and dist.is_initialized():
|
| 177 |
+
# [B * T * world_size, D]
|
| 178 |
+
data = SyncFunction.apply(data)
|
| 179 |
+
|
| 180 |
+
# if is_primary():
|
| 181 |
+
# print(data.shape)
|
| 182 |
+
|
| 183 |
+
embed, cluster_size = kmeans(data, self.codebook_size, self.kmeans_iters)
|
| 184 |
+
self.embed.data.copy_(embed)
|
| 185 |
+
self.embed_avg.data.copy_(embed.clone())
|
| 186 |
+
self.cluster_size.data.copy_(cluster_size)
|
| 187 |
+
self.inited.data.copy_(torch.Tensor([True]))
|
| 188 |
+
# Make sure all buffers across workers are in sync after initialization
|
| 189 |
+
broadcast_tensors(self.buffers())
|
| 190 |
+
|
| 191 |
+
def replace_(self, samples, mask):
|
| 192 |
+
modified_codebook = torch.where(
|
| 193 |
+
mask[..., None], sample_vectors(samples, self.codebook_size), self.embed
|
| 194 |
+
)
|
| 195 |
+
self.embed.data.copy_(modified_codebook)
|
| 196 |
+
|
| 197 |
+
def expire_codes_(self, batch_samples):
|
| 198 |
+
if self.threshold_ema_dead_code == 0:
|
| 199 |
+
return
|
| 200 |
+
|
| 201 |
+
expired_codes = self.cluster_size < self.threshold_ema_dead_code
|
| 202 |
+
if not torch.any(expired_codes):
|
| 203 |
+
return
|
| 204 |
+
|
| 205 |
+
# if is_primary():
|
| 206 |
+
# print(batch_samples.shape)
|
| 207 |
+
|
| 208 |
+
## NOTE (snippet added by Songxiang Liu): gather data from all gpus
|
| 209 |
+
if is_distributed():
|
| 210 |
+
# [B * T * world_size, D]
|
| 211 |
+
batch_samples = SyncFunction.apply(batch_samples)
|
| 212 |
+
|
| 213 |
+
# if is_primary():
|
| 214 |
+
# print(batch_samples.shape)
|
| 215 |
+
|
| 216 |
+
batch_samples = rearrange(batch_samples, "... d -> (...) d")
|
| 217 |
+
self.replace_(batch_samples, mask=expired_codes)
|
| 218 |
+
broadcast_tensors(self.buffers())
|
| 219 |
+
|
| 220 |
+
def preprocess(self, x):
|
| 221 |
+
x = rearrange(x, "... d -> (...) d")
|
| 222 |
+
return x
|
| 223 |
+
|
| 224 |
+
def quantize(self, x):
|
| 225 |
+
embed = self.embed.t()
|
| 226 |
+
dist = -(
|
| 227 |
+
x.pow(2).sum(1, keepdim=True)
|
| 228 |
+
- 2 * x @ embed
|
| 229 |
+
+ embed.pow(2).sum(0, keepdim=True)
|
| 230 |
+
)
|
| 231 |
+
embed_ind = dist.max(dim=-1).indices
|
| 232 |
+
return embed_ind
|
| 233 |
+
|
| 234 |
+
def postprocess_emb(self, embed_ind, shape):
|
| 235 |
+
return embed_ind.view(*shape[:-1])
|
| 236 |
+
|
| 237 |
+
def dequantize(self, embed_ind):
|
| 238 |
+
quantize = F.embedding(embed_ind, self.embed)
|
| 239 |
+
return quantize
|
| 240 |
+
|
| 241 |
+
def encode(self, x):
|
| 242 |
+
shape = x.shape
|
| 243 |
+
# pre-process
|
| 244 |
+
x = self.preprocess(x) # [B, T, D] -> [B*T, D]
|
| 245 |
+
# quantize
|
| 246 |
+
embed_ind = self.quantize(x)
|
| 247 |
+
# post-process
|
| 248 |
+
embed_ind = self.postprocess_emb(embed_ind, shape)
|
| 249 |
+
return embed_ind
|
| 250 |
+
|
| 251 |
+
def decode(self, embed_ind):
|
| 252 |
+
quantize = self.dequantize(embed_ind)
|
| 253 |
+
return quantize
|
| 254 |
+
|
| 255 |
+
def forward(self, x):
|
| 256 |
+
# shape: [B, T, D]
|
| 257 |
+
shape, dtype = x.shape, x.dtype
|
| 258 |
+
x = self.preprocess(x) # [B, T, D] -> [B*T, D]
|
| 259 |
+
|
| 260 |
+
# Initialize codebook
|
| 261 |
+
self.init_embed_(x)
|
| 262 |
+
|
| 263 |
+
embed_ind = self.quantize(x) # [B*T,]
|
| 264 |
+
embed_onehot = F.one_hot(embed_ind, self.codebook_size).type(dtype) # [B*T, cb-size]
|
| 265 |
+
embed_ind = self.postprocess_emb(embed_ind, shape) # [B, T]
|
| 266 |
+
quantize = self.dequantize(embed_ind) # [B, T, D]
|
| 267 |
+
|
| 268 |
+
if self.training:
|
| 269 |
+
### Update codebook by EMA
|
| 270 |
+
embed_onehot_sum = embed_onehot.sum(0) # [cb-size,]
|
| 271 |
+
embed_sum = x.t() @ embed_onehot # [D, cb-size]
|
| 272 |
+
# if is_primary():
|
| 273 |
+
# print("--------- 1 ---------")
|
| 274 |
+
# print(embed_onehot_sum[:10])
|
| 275 |
+
if is_distributed():
|
| 276 |
+
dist.all_reduce(embed_onehot_sum)
|
| 277 |
+
dist.all_reduce(embed_sum)
|
| 278 |
+
# if is_primary():
|
| 279 |
+
# print(embed_onehot_sum[:10])
|
| 280 |
+
# print("--------- 2 ---------")
|
| 281 |
+
# Update ema cluster count N_i^t, eq. (6) in vqvae paper
|
| 282 |
+
self.cluster_size.data.mul_(self.decay).add_(
|
| 283 |
+
embed_onehot_sum, alpha=1 - self.decay
|
| 284 |
+
)
|
| 285 |
+
# Update ema embed: eq. (7) in vqvae paper
|
| 286 |
+
self.embed_avg.data.mul_(self.decay).add_(embed_sum.t(), alpha=1 - self.decay)
|
| 287 |
+
# apply laplace smoothing
|
| 288 |
+
n = self.cluster_size.sum()
|
| 289 |
+
cluster_size = (
|
| 290 |
+
(self.cluster_size + self.epsilon) / (n + self.codebook_size * self.epsilon) * n
|
| 291 |
+
)
|
| 292 |
+
# Update ema embed: eq. (8) in vqvae paper
|
| 293 |
+
embed_normalized = self.embed_avg / cluster_size.unsqueeze(1)
|
| 294 |
+
self.embed.data.copy_(embed_normalized)
|
| 295 |
+
|
| 296 |
+
# We do the expiry of code at that point as buffers are in sync
|
| 297 |
+
# and all the workers will take the same decision.
|
| 298 |
+
self.expire_codes_(x)
|
| 299 |
+
|
| 300 |
+
return quantize, embed_ind
|
| 301 |
+
|
| 302 |
+
|
| 303 |
+
class VectorQuantization(nn.Module):
|
| 304 |
+
"""Vector quantization implementation.
|
| 305 |
+
Currently supports only euclidean distance.
|
| 306 |
+
Args:
|
| 307 |
+
dim (int): Dimension
|
| 308 |
+
codebook_size (int): Codebook size
|
| 309 |
+
codebook_dim (int): Codebook dimension. If not defined, uses the specified dimension in dim.
|
| 310 |
+
decay (float): Decay for exponential moving average over the codebooks.
|
| 311 |
+
epsilon (float): Epsilon value for numerical stability.
|
| 312 |
+
kmeans_init (bool): Whether to use kmeans to initialize the codebooks.
|
| 313 |
+
kmeans_iters (int): Number of iterations used for kmeans initialization.
|
| 314 |
+
threshold_ema_dead_code (int): Threshold for dead code expiration. Replace any codes
|
| 315 |
+
that have an exponential moving average cluster size less than the specified threshold with
|
| 316 |
+
randomly selected vector from the current batch.
|
| 317 |
+
commitment_weight (float): Weight for commitment loss.
|
| 318 |
+
"""
|
| 319 |
+
def __init__(
|
| 320 |
+
self,
|
| 321 |
+
dim: int,
|
| 322 |
+
codebook_size: int,
|
| 323 |
+
codebook_dim: tp.Optional[int] = None,
|
| 324 |
+
decay: float = 0.99,
|
| 325 |
+
epsilon: float = 1e-5,
|
| 326 |
+
kmeans_init: bool = True,
|
| 327 |
+
kmeans_iters: int = 50,
|
| 328 |
+
threshold_ema_dead_code: int = 2,
|
| 329 |
+
commitment_weight: float = 1.,
|
| 330 |
+
):
|
| 331 |
+
super().__init__()
|
| 332 |
+
_codebook_dim: int = default(codebook_dim, dim)
|
| 333 |
+
|
| 334 |
+
requires_projection = _codebook_dim != dim
|
| 335 |
+
self.project_in = (nn.Linear(dim, _codebook_dim) if requires_projection else nn.Identity())
|
| 336 |
+
self.project_out = (nn.Linear(_codebook_dim, dim) if requires_projection else nn.Identity())
|
| 337 |
+
|
| 338 |
+
self.epsilon = epsilon
|
| 339 |
+
self.commitment_weight = commitment_weight
|
| 340 |
+
|
| 341 |
+
self._codebook = EuclideanCodebook(dim=_codebook_dim, codebook_size=codebook_size,
|
| 342 |
+
kmeans_init=kmeans_init, kmeans_iters=kmeans_iters,
|
| 343 |
+
decay=decay, epsilon=epsilon,
|
| 344 |
+
threshold_ema_dead_code=threshold_ema_dead_code)
|
| 345 |
+
self.codebook_size = codebook_size
|
| 346 |
+
|
| 347 |
+
@property
|
| 348 |
+
def codebook(self):
|
| 349 |
+
return self._codebook.embed
|
| 350 |
+
|
| 351 |
+
def encode(self, x):
|
| 352 |
+
x = rearrange(x, "b d n -> b n d")
|
| 353 |
+
x = self.project_in(x)
|
| 354 |
+
embed_in = self._codebook.encode(x)
|
| 355 |
+
return embed_in
|
| 356 |
+
|
| 357 |
+
def decode(self, embed_ind):
|
| 358 |
+
quantize = self._codebook.decode(embed_ind)
|
| 359 |
+
quantize = self.project_out(quantize)
|
| 360 |
+
quantize = rearrange(quantize, "b n d -> b d n")
|
| 361 |
+
return quantize
|
| 362 |
+
|
| 363 |
+
def forward(self, x):
|
| 364 |
+
device = x.device
|
| 365 |
+
x = rearrange(x, "b d n -> b n d")
|
| 366 |
+
x = self.project_in(x)
|
| 367 |
+
|
| 368 |
+
quantize, embed_ind = self._codebook(x)
|
| 369 |
+
|
| 370 |
+
if self.training:
|
| 371 |
+
quantize = x + (quantize - x).detach()
|
| 372 |
+
|
| 373 |
+
loss = torch.tensor([0.0], device=device, requires_grad=self.training)
|
| 374 |
+
|
| 375 |
+
if self.training:
|
| 376 |
+
if self.commitment_weight > 0:
|
| 377 |
+
commit_loss = F.mse_loss(quantize.detach(), x)
|
| 378 |
+
loss = loss + commit_loss * self.commitment_weight
|
| 379 |
+
|
| 380 |
+
quantize = self.project_out(quantize)
|
| 381 |
+
quantize = rearrange(quantize, "b n d -> b d n")
|
| 382 |
+
return quantize, embed_ind, loss
|
| 383 |
+
|
| 384 |
+
|
| 385 |
+
class ResidualVectorQuantization(nn.Module):
|
| 386 |
+
"""Residual vector quantization implementation.
|
| 387 |
+
Follows Algorithm 1. in https://arxiv.org/pdf/2107.03312.pdf
|
| 388 |
+
"""
|
| 389 |
+
def __init__(self, *, num_quantizers, **kwargs):
|
| 390 |
+
super().__init__()
|
| 391 |
+
self.layers = nn.ModuleList(
|
| 392 |
+
[VectorQuantization(**kwargs) for _ in range(num_quantizers)]
|
| 393 |
+
)
|
| 394 |
+
|
| 395 |
+
def forward(self, x, n_q: tp.Optional[int] = None):
|
| 396 |
+
quantized_out = 0.0
|
| 397 |
+
residual = x
|
| 398 |
+
|
| 399 |
+
all_losses = []
|
| 400 |
+
all_indices = []
|
| 401 |
+
|
| 402 |
+
n_q = n_q or len(self.layers)
|
| 403 |
+
|
| 404 |
+
for layer in self.layers[:n_q]:
|
| 405 |
+
quantized, indices, loss = layer(residual)
|
| 406 |
+
residual = residual - quantized
|
| 407 |
+
quantized_out = quantized_out + quantized
|
| 408 |
+
|
| 409 |
+
all_indices.append(indices)
|
| 410 |
+
all_losses.append(loss)
|
| 411 |
+
|
| 412 |
+
out_losses, out_indices = map(torch.stack, (all_losses, all_indices))
|
| 413 |
+
return quantized_out, out_indices, out_losses
|
| 414 |
+
|
| 415 |
+
def encode(self, x: torch.Tensor, n_q: tp.Optional[int] = None) -> torch.Tensor:
|
| 416 |
+
residual = x
|
| 417 |
+
all_indices = []
|
| 418 |
+
n_q = n_q or len(self.layers)
|
| 419 |
+
for layer in self.layers[:n_q]:
|
| 420 |
+
indices = layer.encode(residual)
|
| 421 |
+
quantized = layer.decode(indices)
|
| 422 |
+
residual = residual - quantized
|
| 423 |
+
all_indices.append(indices)
|
| 424 |
+
out_indices = torch.stack(all_indices)
|
| 425 |
+
return out_indices
|
| 426 |
+
|
| 427 |
+
def decode(self, q_indices: torch.Tensor) -> torch.Tensor:
|
| 428 |
+
quantized_out = torch.tensor(0.0, device=q_indices.device)
|
| 429 |
+
for i, indices in enumerate(q_indices):
|
| 430 |
+
layer = self.layers[i]
|
| 431 |
+
quantized = layer.decode(indices)
|
| 432 |
+
quantized_out = quantized_out + quantized
|
| 433 |
+
return quantized_out
|
songgen/xcodec_wrapper/xcodec_infer/quantization/distrib.py
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
# All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
|
| 7 |
+
"""Torch distributed utilities."""
|
| 8 |
+
|
| 9 |
+
import typing as tp
|
| 10 |
+
|
| 11 |
+
import torch
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def rank():
|
| 15 |
+
if torch.distributed.is_initialized():
|
| 16 |
+
return torch.distributed.get_rank()
|
| 17 |
+
else:
|
| 18 |
+
return 0
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def world_size():
|
| 22 |
+
if torch.distributed.is_initialized():
|
| 23 |
+
return torch.distributed.get_world_size()
|
| 24 |
+
else:
|
| 25 |
+
return 1
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def is_distributed():
|
| 29 |
+
return world_size() > 1
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def all_reduce(tensor: torch.Tensor, op=torch.distributed.ReduceOp.SUM):
|
| 33 |
+
if is_distributed():
|
| 34 |
+
return torch.distributed.all_reduce(tensor, op)
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def _is_complex_or_float(tensor):
|
| 38 |
+
return torch.is_floating_point(tensor) or torch.is_complex(tensor)
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def _check_number_of_params(params: tp.List[torch.Tensor]):
|
| 42 |
+
# utility function to check that the number of params in all workers is the same,
|
| 43 |
+
# and thus avoid a deadlock with distributed all reduce.
|
| 44 |
+
if not is_distributed() or not params:
|
| 45 |
+
return
|
| 46 |
+
#print('params[0].device ', params[0].device)
|
| 47 |
+
tensor = torch.tensor([len(params)], device=params[0].device, dtype=torch.long)
|
| 48 |
+
all_reduce(tensor)
|
| 49 |
+
if tensor.item() != len(params) * world_size():
|
| 50 |
+
# If not all the workers have the same number, for at least one of them,
|
| 51 |
+
# this inequality will be verified.
|
| 52 |
+
raise RuntimeError(f"Mismatch in number of params: ours is {len(params)}, "
|
| 53 |
+
"at least one worker has a different one.")
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def broadcast_tensors(tensors: tp.Iterable[torch.Tensor], src: int = 0):
|
| 57 |
+
"""Broadcast the tensors from the given parameters to all workers.
|
| 58 |
+
This can be used to ensure that all workers have the same model to start with.
|
| 59 |
+
"""
|
| 60 |
+
if not is_distributed():
|
| 61 |
+
return
|
| 62 |
+
tensors = [tensor for tensor in tensors if _is_complex_or_float(tensor)]
|
| 63 |
+
_check_number_of_params(tensors)
|
| 64 |
+
handles = []
|
| 65 |
+
for tensor in tensors:
|
| 66 |
+
handle = torch.distributed.broadcast(tensor.data, src=src, async_op=True)
|
| 67 |
+
handles.append(handle)
|
| 68 |
+
for handle in handles:
|
| 69 |
+
handle.wait()
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def sync_buffer(buffers, average=True):
|
| 73 |
+
"""
|
| 74 |
+
Sync grad for buffers. If average is False, broadcast instead of averaging.
|
| 75 |
+
"""
|
| 76 |
+
if not is_distributed():
|
| 77 |
+
return
|
| 78 |
+
handles = []
|
| 79 |
+
for buffer in buffers:
|
| 80 |
+
if torch.is_floating_point(buffer.data):
|
| 81 |
+
if average:
|
| 82 |
+
handle = torch.distributed.all_reduce(
|
| 83 |
+
buffer.data, op=torch.distributed.ReduceOp.SUM, async_op=True)
|
| 84 |
+
else:
|
| 85 |
+
handle = torch.distributed.broadcast(
|
| 86 |
+
buffer.data, src=0, async_op=True)
|
| 87 |
+
handles.append((buffer, handle))
|
| 88 |
+
for buffer, handle in handles:
|
| 89 |
+
handle.wait()
|
| 90 |
+
if average:
|
| 91 |
+
buffer.data /= world_size
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def sync_grad(params):
|
| 95 |
+
"""
|
| 96 |
+
Simpler alternative to DistributedDataParallel, that doesn't rely
|
| 97 |
+
on any black magic. For simple models it can also be as fast.
|
| 98 |
+
Just call this on your model parameters after the call to backward!
|
| 99 |
+
"""
|
| 100 |
+
if not is_distributed():
|
| 101 |
+
return
|
| 102 |
+
handles = []
|
| 103 |
+
for p in params:
|
| 104 |
+
if p.grad is not None:
|
| 105 |
+
handle = torch.distributed.all_reduce(
|
| 106 |
+
p.grad.data, op=torch.distributed.ReduceOp.SUM, async_op=True)
|
| 107 |
+
handles.append((p, handle))
|
| 108 |
+
for p, handle in handles:
|
| 109 |
+
handle.wait()
|
| 110 |
+
p.grad.data /= world_size()
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
def average_metrics(metrics: tp.Dict[str, float], count=1.):
|
| 114 |
+
"""Average a dictionary of metrics across all workers, using the optional
|
| 115 |
+
`count` as unormalized weight.
|
| 116 |
+
"""
|
| 117 |
+
if not is_distributed():
|
| 118 |
+
return metrics
|
| 119 |
+
keys, values = zip(*metrics.items())
|
| 120 |
+
device = 'cuda' if torch.cuda.is_available() else 'cpu'
|
| 121 |
+
tensor = torch.tensor(list(values) + [1], device=device, dtype=torch.float32)
|
| 122 |
+
tensor *= count
|
| 123 |
+
all_reduce(tensor)
|
| 124 |
+
averaged = (tensor[:-1] / tensor[-1]).cpu().tolist()
|
| 125 |
+
return dict(zip(keys, averaged))
|
songgen/xcodec_wrapper/xcodec_infer/quantization/vq.py
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
# All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
|
| 7 |
+
"""Residual vector quantizer implementation."""
|
| 8 |
+
|
| 9 |
+
from dataclasses import dataclass, field
|
| 10 |
+
import math
|
| 11 |
+
import typing as tp
|
| 12 |
+
|
| 13 |
+
import torch
|
| 14 |
+
from torch import nn
|
| 15 |
+
|
| 16 |
+
# from .core_vq import ResidualVectorQuantization
|
| 17 |
+
from .core_vq_lsx_version import ResidualVectorQuantization
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
@dataclass
|
| 21 |
+
class QuantizedResult:
|
| 22 |
+
quantized: torch.Tensor
|
| 23 |
+
codes: torch.Tensor
|
| 24 |
+
bandwidth: torch.Tensor # bandwidth in kb/s used, per batch item.
|
| 25 |
+
penalty: tp.Optional[torch.Tensor] = None
|
| 26 |
+
metrics: dict = field(default_factory=dict)
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
class ResidualVectorQuantizer(nn.Module):
|
| 30 |
+
"""Residual Vector Quantizer.
|
| 31 |
+
Args:
|
| 32 |
+
dimension (int): Dimension of the codebooks.
|
| 33 |
+
n_q (int): Number of residual vector quantizers used.
|
| 34 |
+
bins (int): Codebook size.
|
| 35 |
+
decay (float): Decay for exponential moving average over the codebooks.
|
| 36 |
+
kmeans_init (bool): Whether to use kmeans to initialize the codebooks.
|
| 37 |
+
kmeans_iters (int): Number of iterations used for kmeans initialization.
|
| 38 |
+
threshold_ema_dead_code (int): Threshold for dead code expiration. Replace any codes
|
| 39 |
+
that have an exponential moving average cluster size less than the specified threshold with
|
| 40 |
+
randomly selected vector from the current batch.
|
| 41 |
+
"""
|
| 42 |
+
def __init__(
|
| 43 |
+
self,
|
| 44 |
+
dimension: int = 256,
|
| 45 |
+
n_q: int = 8,
|
| 46 |
+
bins: int = 1024,
|
| 47 |
+
decay: float = 0.99,
|
| 48 |
+
kmeans_init: bool = True,
|
| 49 |
+
kmeans_iters: int = 50,
|
| 50 |
+
threshold_ema_dead_code: int = 2,
|
| 51 |
+
):
|
| 52 |
+
super().__init__()
|
| 53 |
+
self.n_q = n_q
|
| 54 |
+
self.dimension = dimension
|
| 55 |
+
self.bins = bins
|
| 56 |
+
self.decay = decay
|
| 57 |
+
self.kmeans_init = kmeans_init
|
| 58 |
+
self.kmeans_iters = kmeans_iters
|
| 59 |
+
self.threshold_ema_dead_code = threshold_ema_dead_code
|
| 60 |
+
self.vq = ResidualVectorQuantization(
|
| 61 |
+
dim=self.dimension,
|
| 62 |
+
codebook_size=self.bins,
|
| 63 |
+
num_quantizers=self.n_q,
|
| 64 |
+
decay=self.decay,
|
| 65 |
+
kmeans_init=self.kmeans_init,
|
| 66 |
+
kmeans_iters=self.kmeans_iters,
|
| 67 |
+
threshold_ema_dead_code=self.threshold_ema_dead_code,
|
| 68 |
+
)
|
| 69 |
+
|
| 70 |
+
def forward(self, x: torch.Tensor, sample_rate: int, bandwidth: tp.Optional[float] = None): # -> QuantizedResult:
|
| 71 |
+
"""Residual vector quantization on the given input tensor.
|
| 72 |
+
Args:
|
| 73 |
+
x (torch.Tensor): Input tensor.
|
| 74 |
+
sample_rate (int): Sample rate of the input tensor.
|
| 75 |
+
bandwidth (float): Target bandwidth.
|
| 76 |
+
Returns:
|
| 77 |
+
QuantizedResult:
|
| 78 |
+
The quantized (or approximately quantized) representation with
|
| 79 |
+
the associated bandwidth and any penalty term for the loss.
|
| 80 |
+
"""
|
| 81 |
+
bw_per_q = self.get_bandwidth_per_quantizer(sample_rate)
|
| 82 |
+
n_q = self.get_num_quantizers_for_bandwidth(sample_rate, bandwidth)
|
| 83 |
+
quantized, codes, commit_loss = self.vq(x, n_q=n_q)
|
| 84 |
+
bw = torch.tensor(n_q * bw_per_q).to(x)
|
| 85 |
+
return quantized, codes, bw, torch.mean(commit_loss)
|
| 86 |
+
#return QuantizedResult(quantized, codes, bw, penalty=torch.mean(commit_loss))
|
| 87 |
+
|
| 88 |
+
def get_num_quantizers_for_bandwidth(self, sample_rate: int, bandwidth: tp.Optional[float] = None) -> int:
|
| 89 |
+
"""Return n_q based on specified target bandwidth.
|
| 90 |
+
"""
|
| 91 |
+
bw_per_q = self.get_bandwidth_per_quantizer(sample_rate)
|
| 92 |
+
n_q = self.n_q
|
| 93 |
+
if bandwidth and bandwidth > 0.:
|
| 94 |
+
n_q = int(max(1, math.floor(bandwidth / bw_per_q)))
|
| 95 |
+
return n_q
|
| 96 |
+
|
| 97 |
+
def get_bandwidth_per_quantizer(self, sample_rate: int):
|
| 98 |
+
"""Return bandwidth per quantizer for a given input sample rate.
|
| 99 |
+
"""
|
| 100 |
+
return math.log2(self.bins) * sample_rate / 1000
|
| 101 |
+
|
| 102 |
+
def encode(self, x: torch.Tensor, sample_rate: int, bandwidth: tp.Optional[float] = None) -> torch.Tensor:
|
| 103 |
+
"""Encode a given input tensor with the specified sample rate at the given bandwidth.
|
| 104 |
+
The RVQ encode method sets the appropriate number of quantizer to use
|
| 105 |
+
and returns indices for each quantizer.
|
| 106 |
+
"""
|
| 107 |
+
n_q = self.get_num_quantizers_for_bandwidth(sample_rate, bandwidth)
|
| 108 |
+
codes = self.vq.encode(x, n_q=n_q)
|
| 109 |
+
return codes
|
| 110 |
+
|
| 111 |
+
def decode(self, codes: torch.Tensor) -> torch.Tensor:
|
| 112 |
+
"""Decode the given codes to the quantized representation.
|
| 113 |
+
"""
|
| 114 |
+
quantized = self.vq.decode(codes)
|
| 115 |
+
return quantized
|
songgen/xcodec_wrapper/xcodec_infer/xutils/__init__.py
ADDED
|
File without changes
|
songgen/xcodec_wrapper/xcodec_infer/xutils/ddp_utils.py
ADDED
|
@@ -0,0 +1,200 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import logging
|
| 2 |
+
import random
|
| 3 |
+
import subprocess
|
| 4 |
+
from datetime import datetime
|
| 5 |
+
|
| 6 |
+
import numpy as np
|
| 7 |
+
import torch
|
| 8 |
+
import torch.distributed as dist
|
| 9 |
+
from torch.nn.parallel import DistributedDataParallel
|
| 10 |
+
from torch.nn.parallel.distributed import _find_tensors
|
| 11 |
+
import torch.optim
|
| 12 |
+
import torch.utils.data
|
| 13 |
+
from packaging import version
|
| 14 |
+
from omegaconf import OmegaConf
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def set_random_seed(seed):
|
| 18 |
+
random.seed(seed)
|
| 19 |
+
np.random.seed(seed)
|
| 20 |
+
torch.manual_seed(seed)
|
| 21 |
+
torch.cuda.manual_seed_all(seed)
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def is_logging_process():
|
| 25 |
+
return not dist.is_initialized() or dist.get_rank() == 0
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def get_logger(cfg, name=None):
|
| 29 |
+
# log_file_path is used when unit testing
|
| 30 |
+
if is_logging_process():
|
| 31 |
+
logging.config.dictConfig(
|
| 32 |
+
OmegaConf.to_container(cfg.job_logging_config, resolve=True)
|
| 33 |
+
)
|
| 34 |
+
return logging.getLogger(name)
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
# from https://github.com/Lightning-AI/lightning-bolts/blob/5d61197cd2f491f69e238137a5edabe80ae14ad9/pl_bolts/models/self_supervised/simclr/simclr_module.py#L20
|
| 38 |
+
class SyncFunction(torch.autograd.Function):
|
| 39 |
+
@staticmethod
|
| 40 |
+
# @torch.no_grad()
|
| 41 |
+
def forward(ctx, tensor):
|
| 42 |
+
ctx.batch_size = tensor.shape[0]
|
| 43 |
+
|
| 44 |
+
gathered_tensor = [torch.zeros_like(tensor) for _ in range(torch.distributed.get_world_size())]
|
| 45 |
+
|
| 46 |
+
torch.distributed.all_gather(gathered_tensor, tensor)
|
| 47 |
+
gathered_tensor = torch.cat(gathered_tensor, 0)
|
| 48 |
+
|
| 49 |
+
return gathered_tensor
|
| 50 |
+
|
| 51 |
+
@staticmethod
|
| 52 |
+
def backward(ctx, grad_output):
|
| 53 |
+
grad_input = grad_output.clone()
|
| 54 |
+
torch.distributed.all_reduce(grad_input, op=torch.distributed.ReduceOp.SUM, async_op=False)
|
| 55 |
+
|
| 56 |
+
idx_from = torch.distributed.get_rank() * ctx.batch_size
|
| 57 |
+
idx_to = (torch.distributed.get_rank() + 1) * ctx.batch_size
|
| 58 |
+
return grad_input[idx_from:idx_to]
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def get_timestamp():
|
| 62 |
+
return datetime.now().strftime("%y%m%d-%H%M%S")
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def get_commit_hash():
|
| 66 |
+
message = subprocess.check_output(["git", "rev-parse", "--short", "HEAD"])
|
| 67 |
+
return message.strip().decode("utf-8")
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
class DDP(DistributedDataParallel):
|
| 71 |
+
"""
|
| 72 |
+
Override the forward call in lightning so it goes to training and validation step respectively
|
| 73 |
+
"""
|
| 74 |
+
|
| 75 |
+
def forward(self, *inputs, **kwargs): # pragma: no cover
|
| 76 |
+
if version.parse(torch.__version__[:6]) < version.parse("1.11"):
|
| 77 |
+
self._sync_params()
|
| 78 |
+
inputs, kwargs = self.scatter(inputs, kwargs, self.device_ids)
|
| 79 |
+
assert len(self.device_ids) == 1
|
| 80 |
+
if self.module.training:
|
| 81 |
+
output = self.module.training_step(*inputs[0], **kwargs[0])
|
| 82 |
+
elif self.module.testing:
|
| 83 |
+
output = self.module.test_step(*inputs[0], **kwargs[0])
|
| 84 |
+
else:
|
| 85 |
+
output = self.module.validation_step(*inputs[0], **kwargs[0])
|
| 86 |
+
if torch.is_grad_enabled():
|
| 87 |
+
# We'll return the output object verbatim since it is a freeform
|
| 88 |
+
# object. We need to find any tensors in this object, though,
|
| 89 |
+
# because we need to figure out which parameters were used during
|
| 90 |
+
# this forward pass, to ensure we short circuit reduction for any
|
| 91 |
+
# unused parameters. Only if `find_unused_parameters` is set.
|
| 92 |
+
if self.find_unused_parameters:
|
| 93 |
+
self.reducer.prepare_for_backward(list(_find_tensors(output)))
|
| 94 |
+
else:
|
| 95 |
+
self.reducer.prepare_for_backward([])
|
| 96 |
+
else:
|
| 97 |
+
from torch.nn.parallel.distributed import \
|
| 98 |
+
logging, Join, _DDPSink, _tree_flatten_with_rref, _tree_unflatten_with_rref
|
| 99 |
+
with torch.autograd.profiler.record_function("DistributedDataParallel.forward"):
|
| 100 |
+
if torch.is_grad_enabled() and self.require_backward_grad_sync:
|
| 101 |
+
self.logger.set_runtime_stats_and_log()
|
| 102 |
+
self.num_iterations += 1
|
| 103 |
+
self.reducer.prepare_for_forward()
|
| 104 |
+
|
| 105 |
+
# Notify the join context that this process has not joined, if
|
| 106 |
+
# needed
|
| 107 |
+
work = Join.notify_join_context(self)
|
| 108 |
+
if work:
|
| 109 |
+
self.reducer._set_forward_pass_work_handle(
|
| 110 |
+
work, self._divide_by_initial_world_size
|
| 111 |
+
)
|
| 112 |
+
|
| 113 |
+
# Calling _rebuild_buckets before forward compuation,
|
| 114 |
+
# It may allocate new buckets before deallocating old buckets
|
| 115 |
+
# inside _rebuild_buckets. To save peak memory usage,
|
| 116 |
+
# call _rebuild_buckets before the peak memory usage increases
|
| 117 |
+
# during forward computation.
|
| 118 |
+
# This should be called only once during whole training period.
|
| 119 |
+
if torch.is_grad_enabled() and self.reducer._rebuild_buckets():
|
| 120 |
+
logging.info("Reducer buckets have been rebuilt in this iteration.")
|
| 121 |
+
self._has_rebuilt_buckets = True
|
| 122 |
+
|
| 123 |
+
# sync params according to location (before/after forward) user
|
| 124 |
+
# specified as part of hook, if hook was specified.
|
| 125 |
+
buffer_hook_registered = hasattr(self, 'buffer_hook')
|
| 126 |
+
if self._check_sync_bufs_pre_fwd():
|
| 127 |
+
self._sync_buffers()
|
| 128 |
+
|
| 129 |
+
if self._join_config.enable:
|
| 130 |
+
# Notify joined ranks whether they should sync in backwards pass or not.
|
| 131 |
+
self._check_global_requires_backward_grad_sync(is_joined_rank=False)
|
| 132 |
+
|
| 133 |
+
inputs, kwargs = self.scatter(inputs, kwargs, self.device_ids)
|
| 134 |
+
if self.module.training:
|
| 135 |
+
output = self.module.training_step(*inputs[0], **kwargs[0])
|
| 136 |
+
elif self.module.testing:
|
| 137 |
+
output = self.module.test_step(*inputs[0], **kwargs[0])
|
| 138 |
+
else:
|
| 139 |
+
output = self.module.validation_step(*inputs[0], **kwargs[0])
|
| 140 |
+
|
| 141 |
+
# sync params according to location (before/after forward) user
|
| 142 |
+
# specified as part of hook, if hook was specified.
|
| 143 |
+
if self._check_sync_bufs_post_fwd():
|
| 144 |
+
self._sync_buffers()
|
| 145 |
+
|
| 146 |
+
if torch.is_grad_enabled() and self.require_backward_grad_sync:
|
| 147 |
+
self.require_forward_param_sync = True
|
| 148 |
+
# We'll return the output object verbatim since it is a freeform
|
| 149 |
+
# object. We need to find any tensors in this object, though,
|
| 150 |
+
# because we need to figure out which parameters were used during
|
| 151 |
+
# this forward pass, to ensure we short circuit reduction for any
|
| 152 |
+
# unused parameters. Only if `find_unused_parameters` is set.
|
| 153 |
+
if self.find_unused_parameters and not self.static_graph:
|
| 154 |
+
# Do not need to populate this for static graph.
|
| 155 |
+
self.reducer.prepare_for_backward(list(_find_tensors(output)))
|
| 156 |
+
else:
|
| 157 |
+
self.reducer.prepare_for_backward([])
|
| 158 |
+
else:
|
| 159 |
+
self.require_forward_param_sync = False
|
| 160 |
+
|
| 161 |
+
# TODO: DDPSink is currently enabled for unused parameter detection and
|
| 162 |
+
# static graph training for first iteration.
|
| 163 |
+
if (self.find_unused_parameters and not self.static_graph) or (
|
| 164 |
+
self.static_graph and self.num_iterations == 1
|
| 165 |
+
):
|
| 166 |
+
state_dict = {
|
| 167 |
+
'static_graph': self.static_graph,
|
| 168 |
+
'num_iterations': self.num_iterations,
|
| 169 |
+
}
|
| 170 |
+
|
| 171 |
+
output_tensor_list, treespec, output_is_rref = _tree_flatten_with_rref(
|
| 172 |
+
output
|
| 173 |
+
)
|
| 174 |
+
output_placeholders = [None for _ in range(len(output_tensor_list))]
|
| 175 |
+
# Do not touch tensors that have no grad_fn, which can cause issues
|
| 176 |
+
# such as https://github.com/pytorch/pytorch/issues/60733
|
| 177 |
+
for i, output in enumerate(output_tensor_list):
|
| 178 |
+
if torch.is_tensor(output) and output.grad_fn is None:
|
| 179 |
+
output_placeholders[i] = output
|
| 180 |
+
|
| 181 |
+
# When find_unused_parameters=True, makes tensors which require grad
|
| 182 |
+
# run through the DDPSink backward pass. When not all outputs are
|
| 183 |
+
# used in loss, this makes those corresponding tensors receive
|
| 184 |
+
# undefined gradient which the reducer then handles to ensure
|
| 185 |
+
# param.grad field is not touched and we don't error out.
|
| 186 |
+
passthrough_tensor_list = _DDPSink.apply(
|
| 187 |
+
self.reducer,
|
| 188 |
+
state_dict,
|
| 189 |
+
*output_tensor_list,
|
| 190 |
+
)
|
| 191 |
+
for i in range(len(output_placeholders)):
|
| 192 |
+
if output_placeholders[i] is None:
|
| 193 |
+
output_placeholders[i] = passthrough_tensor_list[i]
|
| 194 |
+
|
| 195 |
+
# Reconstruct output data structure.
|
| 196 |
+
output = _tree_unflatten_with_rref(
|
| 197 |
+
output_placeholders, treespec, output_is_rref
|
| 198 |
+
)
|
| 199 |
+
return output
|
| 200 |
+
|
songgen/xcodec_wrapper/xcodec_infer/xutils/utils.py
ADDED
|
@@ -0,0 +1,259 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import dataclasses
|
| 2 |
+
import glob
|
| 3 |
+
import importlib
|
| 4 |
+
import random
|
| 5 |
+
import numpy as np
|
| 6 |
+
import torch
|
| 7 |
+
import warnings
|
| 8 |
+
import os
|
| 9 |
+
import time
|
| 10 |
+
import torch.utils.tensorboard as tensorboard
|
| 11 |
+
from torch import distributed as dist
|
| 12 |
+
import sys
|
| 13 |
+
import yaml
|
| 14 |
+
import json
|
| 15 |
+
import re
|
| 16 |
+
import pathlib
|
| 17 |
+
import matplotlib
|
| 18 |
+
matplotlib.use("Agg")
|
| 19 |
+
import matplotlib.pylab as plt
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def plot_spectrogram(spectrogram):
|
| 23 |
+
fig, ax = plt.subplots(figsize=(10, 2))
|
| 24 |
+
im = ax.imshow(spectrogram, aspect="auto", origin="lower",
|
| 25 |
+
interpolation='none')
|
| 26 |
+
plt.colorbar(im, ax=ax)
|
| 27 |
+
|
| 28 |
+
fig.canvas.draw()
|
| 29 |
+
plt.close()
|
| 30 |
+
|
| 31 |
+
return fig
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def seed_everything(seed, cudnn_deterministic=False):
|
| 35 |
+
"""
|
| 36 |
+
Function that sets seed for pseudo-random number generators in:
|
| 37 |
+
pytorch, numpy, python.random
|
| 38 |
+
|
| 39 |
+
Args:
|
| 40 |
+
seed: the integer value seed for global random state
|
| 41 |
+
"""
|
| 42 |
+
if seed is not None:
|
| 43 |
+
random.seed(seed)
|
| 44 |
+
np.random.seed(seed)
|
| 45 |
+
torch.manual_seed(seed)
|
| 46 |
+
torch.cuda.manual_seed_all(seed)
|
| 47 |
+
|
| 48 |
+
if cudnn_deterministic:
|
| 49 |
+
torch.backends.cudnn.deterministic = True
|
| 50 |
+
warnings.warn('You have chosen to seed training. '
|
| 51 |
+
'This will turn on the CUDNN deterministic setting, '
|
| 52 |
+
'which can slow down your training considerably! '
|
| 53 |
+
'You may see unexpected behavior when restarting '
|
| 54 |
+
'from checkpoints.')
|
| 55 |
+
|
| 56 |
+
def is_primary():
|
| 57 |
+
return get_rank() == 0
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def get_rank():
|
| 61 |
+
if not dist.is_available():
|
| 62 |
+
return 0
|
| 63 |
+
if not dist.is_initialized():
|
| 64 |
+
return 0
|
| 65 |
+
|
| 66 |
+
return dist.get_rank()
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def load_yaml_config(path):
|
| 70 |
+
with open(path) as f:
|
| 71 |
+
config = yaml.full_load(f)
|
| 72 |
+
return config
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def save_config_to_yaml(config, path):
|
| 76 |
+
assert path.endswith('.yaml')
|
| 77 |
+
with open(path, 'w') as f:
|
| 78 |
+
f.write(yaml.dump(config))
|
| 79 |
+
f.close()
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def save_dict_to_json(d, path, indent=None):
|
| 83 |
+
json.dump(d, open(path, 'w'), indent=indent)
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def load_dict_from_json(path):
|
| 87 |
+
return json.load(open(path, 'r'))
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def write_args(args, path):
|
| 91 |
+
args_dict = dict((name, getattr(args, name)) for name in dir(args)if not name.startswith('_'))
|
| 92 |
+
with open(path, 'a') as args_file:
|
| 93 |
+
args_file.write('==> torch version: {}\n'.format(torch.__version__))
|
| 94 |
+
args_file.write('==> cudnn version: {}\n'.format(torch.backends.cudnn.version()))
|
| 95 |
+
args_file.write('==> Cmd:\n')
|
| 96 |
+
args_file.write(str(sys.argv))
|
| 97 |
+
args_file.write('\n==> args:\n')
|
| 98 |
+
for k, v in sorted(args_dict.items()):
|
| 99 |
+
args_file.write(' %s: %s\n' % (str(k), str(v)))
|
| 100 |
+
args_file.close()
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
class Logger(object):
|
| 104 |
+
def __init__(self, args):
|
| 105 |
+
self.args = args
|
| 106 |
+
self.save_dir = args.log_dir
|
| 107 |
+
self.is_primary = is_primary()
|
| 108 |
+
|
| 109 |
+
if self.is_primary:
|
| 110 |
+
os.makedirs(self.save_dir, exist_ok=True)
|
| 111 |
+
|
| 112 |
+
# save the args and config
|
| 113 |
+
self.config_dir = os.path.join(self.save_dir, 'configs')
|
| 114 |
+
os.makedirs(self.config_dir, exist_ok=True)
|
| 115 |
+
file_name = os.path.join(self.config_dir, 'args.txt')
|
| 116 |
+
write_args(args, file_name)
|
| 117 |
+
|
| 118 |
+
log_dir = os.path.join(self.save_dir, 'logs')
|
| 119 |
+
if not os.path.exists(log_dir):
|
| 120 |
+
os.makedirs(log_dir, exist_ok=True)
|
| 121 |
+
self.text_writer = open(os.path.join(log_dir, 'log.txt'), 'a') # 'w')
|
| 122 |
+
if args.tensorboard:
|
| 123 |
+
self.log_info('using tensorboard')
|
| 124 |
+
self.tb_writer = torch.utils.tensorboard.SummaryWriter(log_dir=log_dir) # tensorboard.SummaryWriter(log_dir=log_dir)
|
| 125 |
+
else:
|
| 126 |
+
self.tb_writer = None
|
| 127 |
+
|
| 128 |
+
def save_config(self, config):
|
| 129 |
+
if self.is_primary:
|
| 130 |
+
save_config_to_yaml(config, os.path.join(self.config_dir, 'config.yaml'))
|
| 131 |
+
|
| 132 |
+
def log_info(self, info, check_primary=True):
|
| 133 |
+
if self.is_primary or (not check_primary):
|
| 134 |
+
print(info)
|
| 135 |
+
if self.is_primary:
|
| 136 |
+
info = str(info)
|
| 137 |
+
time_str = time.strftime('%Y-%m-%d-%H-%M')
|
| 138 |
+
info = '{}: {}'.format(time_str, info)
|
| 139 |
+
if not info.endswith('\n'):
|
| 140 |
+
info += '\n'
|
| 141 |
+
self.text_writer.write(info)
|
| 142 |
+
self.text_writer.flush()
|
| 143 |
+
|
| 144 |
+
def add_scalar(self, **kargs):
|
| 145 |
+
"""Log a scalar variable."""
|
| 146 |
+
if self.is_primary:
|
| 147 |
+
if self.tb_writer is not None:
|
| 148 |
+
self.tb_writer.add_scalar(**kargs)
|
| 149 |
+
|
| 150 |
+
def add_scalars(self, **kargs):
|
| 151 |
+
"""Log a scalar variable."""
|
| 152 |
+
if self.is_primary:
|
| 153 |
+
if self.tb_writer is not None:
|
| 154 |
+
self.tb_writer.add_scalars(**kargs)
|
| 155 |
+
|
| 156 |
+
def add_image(self, **kargs):
|
| 157 |
+
"""Log a scalar variable."""
|
| 158 |
+
if self.is_primary:
|
| 159 |
+
if self.tb_writer is not None:
|
| 160 |
+
self.tb_writer.add_image(**kargs)
|
| 161 |
+
|
| 162 |
+
def add_images(self, **kargs):
|
| 163 |
+
"""Log a scalar variable."""
|
| 164 |
+
if self.is_primary:
|
| 165 |
+
if self.tb_writer is not None:
|
| 166 |
+
self.tb_writer.add_images(**kargs)
|
| 167 |
+
|
| 168 |
+
def close(self):
|
| 169 |
+
if self.is_primary:
|
| 170 |
+
self.text_writer.close()
|
| 171 |
+
self.tb_writer.close()
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
def cal_model_size(model, name=""):
|
| 175 |
+
|
| 176 |
+
all_size = sum(p.numel() for p in model.parameters())/1024.0/1024.0
|
| 177 |
+
return f'Model size of {name}: {all_size:.3f} MB'
|
| 178 |
+
|
| 179 |
+
param_size = 0
|
| 180 |
+
param_sum = 0
|
| 181 |
+
for param in model.parameters():
|
| 182 |
+
param_size += param.nelement() * param.element_size()
|
| 183 |
+
param_sum += param.nelement()
|
| 184 |
+
buffer_size = 0
|
| 185 |
+
buffer_sum = 0
|
| 186 |
+
for buffer in model.buffers():
|
| 187 |
+
buffer_size += buffer.nelement() * buffer.element_size()
|
| 188 |
+
buffer_sum += buffer.nelement()
|
| 189 |
+
all_size = (param_size + buffer_size) / 1024 / 1024
|
| 190 |
+
|
| 191 |
+
return f'Model size of {name}: {all_size:.3f} MB'
|
| 192 |
+
# print(f'Model size of {name}: {all_size:.3f}MB')
|
| 193 |
+
# return (param_size, param_sum, buffer_size, buffer_sum, all_size)
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
def load_obj(obj_path: str, default_obj_path: str = ''):
|
| 197 |
+
""" Extract an object from a given path.
|
| 198 |
+
Args:
|
| 199 |
+
obj_path: Path to an object to be extracted, including the object name.
|
| 200 |
+
e.g.: `src.trainers.meta_trainer.MetaTrainer`
|
| 201 |
+
`src.models.ada_style_speech.AdaStyleSpeechModel`
|
| 202 |
+
default_obj_path: Default object path.
|
| 203 |
+
|
| 204 |
+
Returns:
|
| 205 |
+
Extracted object.
|
| 206 |
+
Raises:
|
| 207 |
+
AttributeError: When the object does not have the given named attribute.
|
| 208 |
+
|
| 209 |
+
"""
|
| 210 |
+
obj_path_list = obj_path.rsplit('.', 1)
|
| 211 |
+
obj_path = obj_path_list.pop(0) if len(obj_path_list) > 1 else default_obj_path
|
| 212 |
+
obj_name = obj_path_list[0]
|
| 213 |
+
module_obj = importlib.import_module(obj_path)
|
| 214 |
+
if not hasattr(module_obj, obj_name):
|
| 215 |
+
raise AttributeError(f'Object `{obj_name}` cannot be loaded from `{obj_path}`.')
|
| 216 |
+
return getattr(module_obj, obj_name)
|
| 217 |
+
|
| 218 |
+
|
| 219 |
+
def to_device(data, device=None, dtype=None, non_blocking=False, copy=False):
|
| 220 |
+
"""Change the device of object recursively"""
|
| 221 |
+
if isinstance(data, dict):
|
| 222 |
+
return {
|
| 223 |
+
k: to_device(v, device, dtype, non_blocking, copy) for k, v in data.items()
|
| 224 |
+
}
|
| 225 |
+
elif dataclasses.is_dataclass(data) and not isinstance(data, type):
|
| 226 |
+
return type(data)(
|
| 227 |
+
*[
|
| 228 |
+
to_device(v, device, dtype, non_blocking, copy)
|
| 229 |
+
for v in dataclasses.astuple(data)
|
| 230 |
+
]
|
| 231 |
+
)
|
| 232 |
+
# maybe namedtuple. I don't know the correct way to judge namedtuple.
|
| 233 |
+
elif isinstance(data, tuple) and type(data) is not tuple:
|
| 234 |
+
return type(data)(
|
| 235 |
+
*[to_device(o, device, dtype, non_blocking, copy) for o in data]
|
| 236 |
+
)
|
| 237 |
+
elif isinstance(data, (list, tuple)):
|
| 238 |
+
return type(data)(to_device(v, device, dtype, non_blocking, copy) for v in data)
|
| 239 |
+
elif isinstance(data, np.ndarray):
|
| 240 |
+
return to_device(torch.from_numpy(data), device, dtype, non_blocking, copy)
|
| 241 |
+
elif isinstance(data, torch.Tensor):
|
| 242 |
+
return data.to(device, dtype, non_blocking, copy)
|
| 243 |
+
else:
|
| 244 |
+
return data
|
| 245 |
+
|
| 246 |
+
|
| 247 |
+
def save_checkpoint(filepath, obj, ext='pth', num_ckpt_keep=10):
|
| 248 |
+
ckpts = sorted(pathlib.Path(filepath).parent.glob(f'*.{ext}'))
|
| 249 |
+
if len(ckpts) > num_ckpt_keep:
|
| 250 |
+
[os.remove(c) for c in ckpts[:-num_ckpt_keep]]
|
| 251 |
+
torch.save(obj, filepath)
|
| 252 |
+
|
| 253 |
+
|
| 254 |
+
def scan_checkpoint(cp_dir, prefix='ckpt_'):
|
| 255 |
+
pattern = os.path.join(cp_dir, prefix + '????????.pth')
|
| 256 |
+
cp_list = glob.glob(pattern)
|
| 257 |
+
if len(cp_list) == 0:
|
| 258 |
+
return None
|
| 259 |
+
return sorted(cp_list)[-1]
|