import os import json import torch import numpy as np import hifigan import istftnet # pip install / clone rishikksh20/iSTFTNet-pytorch from model import FastSpeech2, ScheduledOptim def get_model(args, configs, device, train=False): (preprocess_config, model_config, train_config) = configs model = FastSpeech2(preprocess_config, model_config).to(device) if args.restore_step: ckpt_path = os.path.join( train_config["path"]["ckpt_path"], f"{args.restore_step}.pth.tar", ) ckpt = torch.load(ckpt_path, map_location=device) model.load_state_dict(ckpt["model"]) if train: scheduled_optim = ScheduledOptim( model, train_config, model_config, args.restore_step ) if args.restore_step: scheduled_optim.load_state_dict(ckpt["optimizer"]) model.train() return model, scheduled_optim model.eval() model.requires_grad_(False) return model def get_param_num(model): return sum(param.numel() for param in model.parameters()) def get_vocoder(config, device): name = config["vocoder"]["model"] speaker = config["vocoder"]["speaker"] if name == "MelGAN": if speaker == "LJSpeech": vocoder = torch.hub.load( "descriptinc/melgan-neurips", "load_melgan", "linda_johnson" ) elif speaker == "universal": vocoder = torch.hub.load( "descriptinc/melgan-neurips", "load_melgan", "multi_speaker" ) vocoder.mel2wav.eval() vocoder.mel2wav.to(device) elif name == "HiFi-GAN": with open("hifigan/config.json", "r") as f: cfg = json.load(f) cfg = hifigan.AttrDict(cfg) vocoder = hifigan.Generator(cfg) ckpt_file = ( "hifigan/generator_LJSpeech.pth.tar" if speaker == "LJSpeech" else "hifigan/generator_universal.pth.tar" ) ckpt = torch.load(ckpt_file, map_location=device) vocoder.load_state_dict(ckpt["generator"]) vocoder.eval() vocoder.remove_weight_norm() vocoder.to(device) elif name == "iSTFTNet": # --------------------------------------------------------------------------- # iSTFTNet (rishikksh20/iSTFTNet-pytorch) # # Expected layout inside your project: # istftnet/ # config_v1.json (or config_v2.json / config_v3.json) # generator_.pth.tar # # The AttrDict / Generator classes are imported from the `istftnet` package # (just clone the repo alongside your project and add it to PYTHONPATH, or # `pip install -e .` if a setup.py is present). # # iSTFTNet's Generator.forward() returns (spec, phase) instead of a raw # waveform. The iSTFT reconstruction is deferred to vocoder_infer() so # that the same pattern as HiFi-GAN is preserved throughout the codebase. # --------------------------------------------------------------------------- cfg_file = config["vocoder"].get("config", "istftnet/config_v1.json") with open(cfg_file, "r") as f: cfg = json.load(f) cfg = istftnet.AttrDict(cfg) vocoder = istftnet.Generator(cfg) ckpt_file = ( "istftnet/generator_LJSpeech.pth.tar" if speaker == "LJSpeech" else "istftnet/generator_universal.pth.tar" ) ckpt = torch.load(ckpt_file, map_location=device) # The checkpoint may store weights under "generator" (HiFi-GAN convention) # or directly as a flat state dict — handle both. state = ckpt.get("generator", ckpt) vocoder.load_state_dict(state) vocoder.eval() vocoder.remove_weight_norm() vocoder.to(device) # Stash iSTFT parameters on the vocoder so vocoder_infer() can read them # without having to pass the config object separately. vocoder._istft_n_fft = cfg.gen_istft_n_fft vocoder._istft_hop_length = cfg.gen_istft_hop_size vocoder._istft_win_length = cfg.gen_istft_n_fft else: raise ValueError(f"Unknown vocoder model: {name!r}") return vocoder def vocoder_infer(mels, vocoder, model_config, preprocess_config, lengths=None): name = model_config["vocoder"]["model"] max_wav = preprocess_config["preprocessing"]["audio"]["max_wav_value"] with torch.no_grad(): # Ensure mels are float32 to match vocoder weights (fixes bfloat16 mismatch) mels = mels.float() if name == "MelGAN": wavs = vocoder.inverse(mels / np.log(10)) elif name == "HiFi-GAN": wavs = vocoder(mels).squeeze(1) elif name == "iSTFTNet": # iSTFTNet returns (magnitude_spec, phase) — reconstruct via iSTFT. spec, phase = vocoder(mels) real = spec * torch.cos(phase) imag = spec * torch.sin(phase) stft_out = torch.complex(real, imag) # (B, n_fft//2+1, T_spec) wavs = torch.istft( stft_out, n_fft=vocoder._istft_n_fft, hop_length=vocoder._istft_hop_length, win_length=vocoder._istft_win_length, window=torch.hann_window(vocoder._istft_win_length, device=mels.device), onesided=True, ) # (B, T_audio) else: raise ValueError(f"Unknown vocoder model: {name!r}") wavs = (wavs.cpu().numpy() * max_wav).astype("int16") wavs = list(wavs) if lengths is not None: wavs = [wav[:length] for wav, length in zip(wavs, lengths)] return wavs