lushy's picture
Update FastSpeech 2 code and training notebook
f4d4ab8 verified
Raw History Blame Contribute Delete
5.98 kB
import os
import json
import torch
import numpy as np
import hifigan
import istftnet # pip install / clone rishikksh20/iSTFTNet-pytorch
from model import FastSpeech2, ScheduledOptim
def get_model(args, configs, device, train=False):
(preprocess_config, model_config, train_config) = configs
model = FastSpeech2(preprocess_config, model_config).to(device)
if args.restore_step:
ckpt_path = os.path.join(
train_config["path"]["ckpt_path"],
f"{args.restore_step}.pth.tar",
)
ckpt = torch.load(ckpt_path, map_location=device)
model.load_state_dict(ckpt["model"])
if train:
scheduled_optim = ScheduledOptim(
model, train_config, model_config, args.restore_step
)
if args.restore_step:
scheduled_optim.load_state_dict(ckpt["optimizer"])
model.train()
return model, scheduled_optim
model.eval()
model.requires_grad_(False)
return model
def get_param_num(model):
return sum(param.numel() for param in model.parameters())
def get_vocoder(config, device):
name = config["vocoder"]["model"]
speaker = config["vocoder"]["speaker"]
if name == "MelGAN":
if speaker == "LJSpeech":
vocoder = torch.hub.load(
"descriptinc/melgan-neurips", "load_melgan", "linda_johnson"
)
elif speaker == "universal":
vocoder = torch.hub.load(
"descriptinc/melgan-neurips", "load_melgan", "multi_speaker"
)
vocoder.mel2wav.eval()
vocoder.mel2wav.to(device)
elif name == "HiFi-GAN":
with open("hifigan/config.json", "r") as f:
cfg = json.load(f)
cfg = hifigan.AttrDict(cfg)
vocoder = hifigan.Generator(cfg)
ckpt_file = (
"hifigan/generator_LJSpeech.pth.tar"
if speaker == "LJSpeech"
else "hifigan/generator_universal.pth.tar"
)
ckpt = torch.load(ckpt_file, map_location=device)
vocoder.load_state_dict(ckpt["generator"])
vocoder.eval()
vocoder.remove_weight_norm()
vocoder.to(device)
elif name == "iSTFTNet":
# ---------------------------------------------------------------------------
# iSTFTNet (rishikksh20/iSTFTNet-pytorch)
#
# Expected layout inside your project:
# istftnet/
# config_v1.json (or config_v2.json / config_v3.json)
# generator_<speaker>.pth.tar
#
# The AttrDict / Generator classes are imported from the `istftnet` package
# (just clone the repo alongside your project and add it to PYTHONPATH, or
# `pip install -e .` if a setup.py is present).
#
# iSTFTNet's Generator.forward() returns (spec, phase) instead of a raw
# waveform. The iSTFT reconstruction is deferred to vocoder_infer() so
# that the same pattern as HiFi-GAN is preserved throughout the codebase.
# ---------------------------------------------------------------------------
cfg_file = config["vocoder"].get("config", "istftnet/config_v1.json")
with open(cfg_file, "r") as f:
cfg = json.load(f)
cfg = istftnet.AttrDict(cfg)
vocoder = istftnet.Generator(cfg)
ckpt_file = (
"istftnet/generator_LJSpeech.pth.tar"
if speaker == "LJSpeech"
else "istftnet/generator_universal.pth.tar"
)
ckpt = torch.load(ckpt_file, map_location=device)
# The checkpoint may store weights under "generator" (HiFi-GAN convention)
# or directly as a flat state dict — handle both.
state = ckpt.get("generator", ckpt)
vocoder.load_state_dict(state)
vocoder.eval()
vocoder.remove_weight_norm()
vocoder.to(device)
# Stash iSTFT parameters on the vocoder so vocoder_infer() can read them
# without having to pass the config object separately.
vocoder._istft_n_fft = cfg.gen_istft_n_fft
vocoder._istft_hop_length = cfg.gen_istft_hop_size
vocoder._istft_win_length = cfg.gen_istft_n_fft
else:
raise ValueError(f"Unknown vocoder model: {name!r}")
return vocoder
def vocoder_infer(mels, vocoder, model_config, preprocess_config, lengths=None):
name = model_config["vocoder"]["model"]
max_wav = preprocess_config["preprocessing"]["audio"]["max_wav_value"]
with torch.no_grad():
# Ensure mels are float32 to match vocoder weights (fixes bfloat16 mismatch)
mels = mels.float()
if name == "MelGAN":
wavs = vocoder.inverse(mels / np.log(10))
elif name == "HiFi-GAN":
wavs = vocoder(mels).squeeze(1)
elif name == "iSTFTNet":
# iSTFTNet returns (magnitude_spec, phase) — reconstruct via iSTFT.
spec, phase = vocoder(mels)
real = spec * torch.cos(phase)
imag = spec * torch.sin(phase)
stft_out = torch.complex(real, imag) # (B, n_fft//2+1, T_spec)
wavs = torch.istft(
stft_out,
n_fft=vocoder._istft_n_fft,
hop_length=vocoder._istft_hop_length,
win_length=vocoder._istft_win_length,
window=torch.hann_window(vocoder._istft_win_length, device=mels.device),
onesided=True,
) # (B, T_audio)
else:
raise ValueError(f"Unknown vocoder model: {name!r}")
wavs = (wavs.cpu().numpy() * max_wav).astype("int16")
wavs = list(wavs)
if lengths is not None:
wavs = [wav[:length] for wav, length in zip(wavs, lengths)]
return wavs