import os import time import torch import torch.nn as nn import numpy as np import pytorch_lightning as pl from functools import partial import warnings warnings.filterwarnings("ignore") from transformers import logging logging.set_verbosity_error() from ldm.utils.util import exists, default, instantiate_from_config from ldm.utils.diffusion_util import make_beta_schedule from ldm.modules.latent_diffusion.ddim import DDIMSampler import soundfile as sf import logging __conditioning_keys__ = {"concat": "c_concat", "crossattn": "c_crossattn", "adm": "y"} class DDPM(pl.LightningModule): # classic DDPM with Gaussian diffusion, in image space def __init__( self, model_config, sampling_rate=None, timesteps=1000, beta_schedule="linear", loss_type="l2", ckpt_path=None, ignore_keys=[], load_only_unet=False, monitor="val/loss", use_ema=True, first_stage_key="image", latent_t_size=256, latent_f_size=16, channels=3, log_every_t=100, clip_denoised=True, linear_start=1e-4, linear_end=2e-2, cosine_s=8e-3, given_betas=None, original_elbo_weight=0.0, v_posterior=0.0, l_simple_weight=1.0, conditioning_key=None, parameterization="eps", scheduler_config=None, use_positional_encodings=False, learn_logvar=False, logvar_init=0.0, evaluator=None, clap_version="lain", base_learning_rate=None, clap_model=None, ): super().__init__() self.parameterization = parameterization self.sampling_rate = sampling_rate self.latent_t_size = latent_t_size self.latent_f_size = latent_f_size self.channels = channels self.model = DiffusionWrapper(model_config, conditioning_key) self.v_posterior = v_posterior if ckpt_path is not None: self.init_from_ckpt( ckpt_path, ignore_keys=ignore_keys, only_model=load_only_unet ) self.register_schedule( given_betas=given_betas, beta_schedule=beta_schedule, timesteps=timesteps, linear_start=linear_start, linear_end=linear_end, cosine_s=cosine_s, ) def register_schedule( self, given_betas=None, beta_schedule="linear", timesteps=1000, linear_start=1e-4, linear_end=2e-2, cosine_s=8e-3, ): if exists(given_betas): betas = given_betas else: betas = make_beta_schedule( beta_schedule, timesteps, linear_start=linear_start, linear_end=linear_end, cosine_s=cosine_s, ) alphas = 1.0 - betas alphas_cumprod = np.cumprod(alphas, axis=0) alphas_cumprod_prev = np.append(1.0, alphas_cumprod[:-1]) (timesteps,) = betas.shape self.num_timesteps = int(timesteps) self.linear_start = linear_start self.linear_end = linear_end assert ( alphas_cumprod.shape[0] == self.num_timesteps ), "alphas have to be defined for each timestep" to_torch = partial(torch.tensor, dtype=torch.float32) self.register_buffer("betas", to_torch(betas)) self.register_buffer("alphas_cumprod", to_torch(alphas_cumprod)) self.register_buffer("alphas_cumprod_prev", to_torch(alphas_cumprod_prev)) # calculations for diffusion q(x_t | x_{t-1}) and others self.register_buffer("sqrt_alphas_cumprod", to_torch(np.sqrt(alphas_cumprod))) self.register_buffer( "sqrt_one_minus_alphas_cumprod", to_torch(np.sqrt(1.0 - alphas_cumprod)) ) self.register_buffer( "log_one_minus_alphas_cumprod", to_torch(np.log(1.0 - alphas_cumprod)) ) self.register_buffer( "sqrt_recip_alphas_cumprod", to_torch(np.sqrt(1.0 / alphas_cumprod)) ) self.register_buffer( "sqrt_recipm1_alphas_cumprod", to_torch(np.sqrt(1.0 / alphas_cumprod - 1)) ) # calculations for posterior q(x_{t-1} | x_t, x_0) posterior_variance = (1 - self.v_posterior) * betas * ( 1.0 - alphas_cumprod_prev ) / (1.0 - alphas_cumprod) + self.v_posterior * betas # above: equal to 1. / (1. / (1. - alpha_cumprod_tm1) + alpha_t / beta_t) self.register_buffer("posterior_variance", to_torch(posterior_variance)) # below: log calculation clipped because the posterior variance is 0 at the beginning of the diffusion chain self.register_buffer( "posterior_log_variance_clipped", to_torch(np.log(np.maximum(posterior_variance, 1e-20))), ) self.register_buffer( "posterior_mean_coef1", to_torch(betas * np.sqrt(alphas_cumprod_prev) / (1.0 - alphas_cumprod)), ) self.register_buffer( "posterior_mean_coef2", to_torch( (1.0 - alphas_cumprod_prev) * np.sqrt(alphas) / (1.0 - alphas_cumprod) ), ) if self.parameterization == "eps": lvlb_weights = self.betas**2 / ( 2 * self.posterior_variance * to_torch(alphas) * (1 - self.alphas_cumprod) ) elif self.parameterization == "x0": lvlb_weights = ( 0.5 * np.sqrt(torch.Tensor(alphas_cumprod)) / (2.0 * 1 - torch.Tensor(alphas_cumprod)) ) else: raise NotImplementedError("mu not supported") # TODO how to choose this term lvlb_weights[0] = lvlb_weights[1] self.register_buffer("lvlb_weights", lvlb_weights, persistent=False) assert not torch.isnan(self.lvlb_weights).all() def init_from_ckpt(self, path, ignore_keys=list(), only_model=False): sd = torch.load(path, map_location="cpu") if "state_dict" in list(sd.keys()): sd = sd["state_dict"] keys = list(sd.keys()) for k in keys: for ik in ignore_keys: if k.startswith(ik): print("Deleting key {} from state_dict.".format(k)) del sd[k] missing, unexpected = ( self.load_state_dict(sd, strict=False) if not only_model else self.model.load_state_dict(sd, strict=False) ) print( f"Restored from {path} with {len(missing)} missing and {len(unexpected)} unexpected keys" ) if len(missing) > 0: print(f"Missing Keys: {missing}") if len(unexpected) > 0: print(f"Unexpected Keys: {unexpected}") class LatentDiffusion(DDPM): """main class""" def __init__( self, first_stage_config, cond_stage_config, num_timesteps_cond=None, cond_stage_key="image", cond_stage_trainable=False, concat_mode=True, cond_stage_forward=None, conditioning_key=None, scale_factor=1.0, scale_by_std=False, *args, **kwargs, ): self.num_timesteps_cond = default(num_timesteps_cond, 1) self.scale_by_std = scale_by_std assert self.num_timesteps_cond <= kwargs["timesteps"] self.conditioning_key = conditioning_key ckpt_path = kwargs.pop("ckpt_path", None) ignore_keys = kwargs.pop("ignore_keys", []) super().__init__(conditioning_key=conditioning_key, *args, **kwargs) self.concat_mode = concat_mode self.cond_stage_key = cond_stage_key try: self.num_downs = len(first_stage_config.params.ddconfig.ch_mult) - 1 except: self.num_downs = 0 if not scale_by_std: self.scale_factor = scale_factor else: self.register_buffer("scale_factor", torch.tensor(scale_factor)) self.instantiate_first_stage(first_stage_config) self.instantiate_cond_stage(cond_stage_config) self.cond_stage_forward = cond_stage_forward self.clip_denoised = False self.restarted_from_ckpt = False if ckpt_path is not None: self.init_from_ckpt(ckpt_path, ignore_keys) self.restarted_from_ckpt = True def make_cond_schedule( self, ): self.cond_ids = torch.full( size=(self.num_timesteps,), fill_value=self.num_timesteps - 1, dtype=torch.long, ) ids = torch.round( torch.linspace(0, self.num_timesteps - 1, self.num_timesteps_cond) ).long() self.cond_ids[: self.num_timesteps_cond] = ids def register_schedule( self, given_betas=None, beta_schedule="linear", timesteps=1000, linear_start=1e-4, linear_end=2e-2, cosine_s=8e-3, ): super().register_schedule( given_betas, beta_schedule, timesteps, linear_start, linear_end, cosine_s ) self.shorten_cond_schedule = self.num_timesteps_cond > 1 if self.shorten_cond_schedule: self.make_cond_schedule() def instantiate_first_stage(self, config): model = instantiate_from_config(config) self.first_stage_model = model.eval() for param in self.first_stage_model.parameters(): param.requires_grad = False def instantiate_cond_stage(self, config): model = instantiate_from_config(config[self.conditioning_key]) self.cond_stage_model = model @torch.no_grad() def decode_first_stage(self, z, predict_cids=False, force_not_quantize=False): z = 1.0 / self.scale_factor * z z = z.permute(0, 1, 3, 2) return self.first_stage_model.decode(z) def save_wav(self, mel, savepath=None): mel = mel.squeeze(1) waveform = self.first_stage_model.vocoder(mel) waveform = waveform[: 10 * 44100] sf.write(savepath, waveform, samplerate=44100) def apply_model(self, x_noisy, t, cond, return_ids=False): x_recon = self.model(x_noisy, t, cond=cond) return x_recon @torch.no_grad() def sample_log( self, cond, batch_size, ddim, ddim_steps, cfg=1.0, uncond=None, use_plms=False, mask=None, wav_sec=None, **kwargs, ): latent_width = self.latent_t_size shape = (self.channels, latent_width, self.latent_f_size) intermediate = None ddim_sampler = DDIMSampler(self) samples, _ = ddim_sampler.sample( ddim_steps, batch_size, shape, cond, verbose=False, unconditional_guidance_scale=cfg, unconditional_conditioning=uncond, mask=mask, **kwargs, ) return samples @torch.no_grad() def generate_sample( self, textlist=[], name=None, cfg=3.5, ddim_steps=200, outputdir="", quite=False, wav_sec=4, ): batch_size = len(textlist) save_path = os.path.join(outputdir, name + ".wav") xc = textlist cond = {} cond[self.conditioning_key] = self.cond_stage_model.get_cond(xc) uncond = {} uncond[self.conditioning_key] = self.cond_stage_model.get_uncond(batch_size) samples = self.sample_log( cond=cond, batch_size=batch_size, x_T=None, ddim=True, ddim_steps=ddim_steps, eta=1.0, cfg=cfg, uncond=uncond, use_plms=False, wav_sec=wav_sec, ) mel = self.decode_first_stage(samples) os.makedirs(outputdir, exist_ok=True) self.save_wav(mel, savepath=save_path) class DiffusionWrapper(pl.LightningModule): def __init__(self, diff_model_config, conditioning_key): super().__init__() self.diffusion_model = instantiate_from_config(diff_model_config) self.conditioning_key = conditioning_key def forward(self, x, t, cond): xc = x context, mask = cond[self.conditioning_key] out = self.diffusion_model(xc, t=t, context=context, mask=mask) return out