Download utils/model.py from lushy/fastspeech2-ljspeech: direct link, hf CLI and curl.
- Browser
- Download file 5.98 kB
-
https://huggingface.co/lushy/fastspeech2-ljspeech/resolve/main/utils/model.py
- Command line
-
hf download hf://lushy/fastspeech2-ljspeech/utils/model.py
-
curl -L -o model.py https://huggingface.co/lushy/fastspeech2-ljspeech/resolve/main/utils/model.py
5.98 kB
| import os | |
| import json | |
| import torch | |
| import numpy as np | |
| import hifigan | |
| import istftnet # pip install / clone rishikksh20/iSTFTNet-pytorch | |
| from model import FastSpeech2, ScheduledOptim | |
| def get_model(args, configs, device, train=False): | |
| (preprocess_config, model_config, train_config) = configs | |
| model = FastSpeech2(preprocess_config, model_config).to(device) | |
| if args.restore_step: | |
| ckpt_path = os.path.join( | |
| train_config["path"]["ckpt_path"], | |
| f"{args.restore_step}.pth.tar", | |
| ) | |
| ckpt = torch.load(ckpt_path, map_location=device) | |
| model.load_state_dict(ckpt["model"]) | |
| if train: | |
| scheduled_optim = ScheduledOptim( | |
| model, train_config, model_config, args.restore_step | |
| ) | |
| if args.restore_step: | |
| scheduled_optim.load_state_dict(ckpt["optimizer"]) | |
| model.train() | |
| return model, scheduled_optim | |
| model.eval() | |
| model.requires_grad_(False) | |
| return model | |
| def get_param_num(model): | |
| return sum(param.numel() for param in model.parameters()) | |
| def get_vocoder(config, device): | |
| name = config["vocoder"]["model"] | |
| speaker = config["vocoder"]["speaker"] | |
| if name == "MelGAN": | |
| if speaker == "LJSpeech": | |
| vocoder = torch.hub.load( | |
| "descriptinc/melgan-neurips", "load_melgan", "linda_johnson" | |
| ) | |
| elif speaker == "universal": | |
| vocoder = torch.hub.load( | |
| "descriptinc/melgan-neurips", "load_melgan", "multi_speaker" | |
| ) | |
| vocoder.mel2wav.eval() | |
| vocoder.mel2wav.to(device) | |
| elif name == "HiFi-GAN": | |
| with open("hifigan/config.json", "r") as f: | |
| cfg = json.load(f) | |
| cfg = hifigan.AttrDict(cfg) | |
| vocoder = hifigan.Generator(cfg) | |
| ckpt_file = ( | |
| "hifigan/generator_LJSpeech.pth.tar" | |
| if speaker == "LJSpeech" | |
| else "hifigan/generator_universal.pth.tar" | |
| ) | |
| ckpt = torch.load(ckpt_file, map_location=device) | |
| vocoder.load_state_dict(ckpt["generator"]) | |
| vocoder.eval() | |
| vocoder.remove_weight_norm() | |
| vocoder.to(device) | |
| elif name == "iSTFTNet": | |
| # --------------------------------------------------------------------------- | |
| # iSTFTNet (rishikksh20/iSTFTNet-pytorch) | |
| # | |
| # Expected layout inside your project: | |
| # istftnet/ | |
| # config_v1.json (or config_v2.json / config_v3.json) | |
| # generator_<speaker>.pth.tar | |
| # | |
| # The AttrDict / Generator classes are imported from the `istftnet` package | |
| # (just clone the repo alongside your project and add it to PYTHONPATH, or | |
| # `pip install -e .` if a setup.py is present). | |
| # | |
| # iSTFTNet's Generator.forward() returns (spec, phase) instead of a raw | |
| # waveform. The iSTFT reconstruction is deferred to vocoder_infer() so | |
| # that the same pattern as HiFi-GAN is preserved throughout the codebase. | |
| # --------------------------------------------------------------------------- | |
| cfg_file = config["vocoder"].get("config", "istftnet/config_v1.json") | |
| with open(cfg_file, "r") as f: | |
| cfg = json.load(f) | |
| cfg = istftnet.AttrDict(cfg) | |
| vocoder = istftnet.Generator(cfg) | |
| ckpt_file = ( | |
| "istftnet/generator_LJSpeech.pth.tar" | |
| if speaker == "LJSpeech" | |
| else "istftnet/generator_universal.pth.tar" | |
| ) | |
| ckpt = torch.load(ckpt_file, map_location=device) | |
| # The checkpoint may store weights under "generator" (HiFi-GAN convention) | |
| # or directly as a flat state dict — handle both. | |
| state = ckpt.get("generator", ckpt) | |
| vocoder.load_state_dict(state) | |
| vocoder.eval() | |
| vocoder.remove_weight_norm() | |
| vocoder.to(device) | |
| # Stash iSTFT parameters on the vocoder so vocoder_infer() can read them | |
| # without having to pass the config object separately. | |
| vocoder._istft_n_fft = cfg.gen_istft_n_fft | |
| vocoder._istft_hop_length = cfg.gen_istft_hop_size | |
| vocoder._istft_win_length = cfg.gen_istft_n_fft | |
| else: | |
| raise ValueError(f"Unknown vocoder model: {name!r}") | |
| return vocoder | |
| def vocoder_infer(mels, vocoder, model_config, preprocess_config, lengths=None): | |
| name = model_config["vocoder"]["model"] | |
| max_wav = preprocess_config["preprocessing"]["audio"]["max_wav_value"] | |
| with torch.no_grad(): | |
| # Ensure mels are float32 to match vocoder weights (fixes bfloat16 mismatch) | |
| mels = mels.float() | |
| if name == "MelGAN": | |
| wavs = vocoder.inverse(mels / np.log(10)) | |
| elif name == "HiFi-GAN": | |
| wavs = vocoder(mels).squeeze(1) | |
| elif name == "iSTFTNet": | |
| # iSTFTNet returns (magnitude_spec, phase) — reconstruct via iSTFT. | |
| spec, phase = vocoder(mels) | |
| real = spec * torch.cos(phase) | |
| imag = spec * torch.sin(phase) | |
| stft_out = torch.complex(real, imag) # (B, n_fft//2+1, T_spec) | |
| wavs = torch.istft( | |
| stft_out, | |
| n_fft=vocoder._istft_n_fft, | |
| hop_length=vocoder._istft_hop_length, | |
| win_length=vocoder._istft_win_length, | |
| window=torch.hann_window(vocoder._istft_win_length, device=mels.device), | |
| onesided=True, | |
| ) # (B, T_audio) | |
| else: | |
| raise ValueError(f"Unknown vocoder model: {name!r}") | |
| wavs = (wavs.cpu().numpy() * max_wav).astype("int16") | |
| wavs = list(wavs) | |
| if lengths is not None: | |
| wavs = [wav[:length] for wav, length in zip(wavs, lengths)] | |
| return wavs |