import os os.environ.setdefault("CUDA_VISIBLE_DEVICES", "2") from itertools import chain import torch torch.manual_seed(42) from torch import nn from encodec.msstftd import MultiScaleSTFTDiscriminator from encodec.balancer import Balancer from layers import ReconstructionLoss, SuperResolver, WaveDiscriminator from data import make_dataloader try: import pytorch_lightning as pl except ImportError: class pl: class LightningModule: pass class Callback: pass l1Loss = torch.nn.L1Loss(reduction='mean') l2Loss = torch.nn.MSELoss(reduction='mean') class NGAN(pl.LightningModule): def __init__( self, n_channels: int = 4, padding: str = 'check', batch_size: int = 32, target_sample_rate: int = 48000, segment_length: int = 24000, train_data: str = None, lr: float = 1e-4, b1: float = 0.5, b2: float = 0.9 ) -> None: # https://arxiv.org/pdf/2009.02095.pdf # 2. Method # SEANet uses Adam with lr=1e-4, beta1=0.5, beta2=0.9 super().__init__() self.save_hyperparameters() self.balancer = Balancer(weights={'1': 1, '2': 1, '3':1, '4':1}, rescale_grads=True) self.automatic_optimization = False self.superResolver = SuperResolver(n_channels, padding) self.wave_discriminators = nn.ModuleList([ WaveDiscriminator(resolution=1), WaveDiscriminator(resolution=2), WaveDiscriminator(resolution=4) ]) self.sample_rate = target_sample_rate self.rec_loss = ReconstructionLoss() self.msstft_discriminator = MultiScaleSTFTDiscriminator(filters=32, hop_lengths=[256, 512, 128], win_lengths=[1024, 2048, 512], n_ffts=[1024, 2048, 512]) def configure_optimizers(self): lr = self.hparams.lr b1 = self.hparams.b1 b2 = self.hparams.b2 optimizer_g = torch.optim.Adam( chain( self.superResolver.parameters() ), lr=lr, betas=(b1, b2)) optimizer_d = torch.optim.Adam( chain( self.wave_discriminators.parameters(), self.msstft_discriminator.parameters() ), lr=lr, betas=(b1, b2)) return [optimizer_g, optimizer_d], [] def forward(self, input): x = self.superResolver(input) return x def training_step(self, batch, batch_idx): optimizer_g, optimizer_d = self.optimizers() # train generator self.toggle_optimizer(optimizer_g) output = self.forward(batch) # output: [batch, channel, sequence] #multi-scale stft generator loss relu = torch.nn.ReLU() l_g = 0 l_feat = 0 input_orig = batch[0][:, None, :] logits_real, fmap_real = self.msstft_discriminator(input_orig) logits_fake, fmap_fake = self.msstft_discriminator(output) for tt1 in range(len(fmap_real)): l_g = l_g + torch.mean(relu(1 - logits_fake[tt1])) / len(logits_fake) for tt2 in range(len(fmap_real[tt1])): l_feat = l_feat + l1Loss(fmap_real[tt1][tt2], fmap_fake[tt1][tt2]) / torch.mean(torch.abs(fmap_real[tt1][tt2])) KL_scale = len(fmap_real)*len(fmap_real[0]) K_scale = len(fmap_real) #time domain loss, output_wav is the output of the generator l_t = l1Loss(input_orig, output) g_rec_loss = self.rec_loss(output[:, 0, :], input_orig[:, 0, :]) loss_1 = l_g/K_scale loss_2 = l_feat/KL_scale loss_3 = l_t loss_4 = g_rec_loss losses = {'1': loss_1, '2': loss_2, '3': loss_3, '4': loss_4} input_orig.requires_grad_() g_loss = loss_1 + loss_2 + loss_3 + loss_4 self.log("g_loss", g_loss, prog_bar=True) self.balancer.backward(losses, output) optimizer_g.step() optimizer_g.zero_grad() self.untoggle_optimizer(optimizer_g) # train discriminator relu = torch.nn.ReLU() logits_real, _ = self.msstft_discriminator(input_orig) logits_fake, _ = self.msstft_discriminator(output.detach()) lossd = 0 for tt1 in range(len(logits_real)): lossd = lossd + torch.mean(relu(1-logits_real[tt1])) + torch.mean(relu(1+logits_fake[tt1])) d_loss = lossd / len(logits_real) self.log("d_loss", d_loss, prog_bar=True) self.manual_backward(d_loss) optimizer_d.step() optimizer_d.zero_grad() self.untoggle_optimizer(optimizer_d) def train_dataloader(self): return make_dataloader( root=self.hparams.train_data, segment_length=self.hparams.segment_length, target_sample_rate=self.hparams.target_sample_rate, batch_size=self.hparams.batch_size) def train(): model = NGAN( batch_size=32, target_sample_rate=48000, segment_length=24000, padding='same') #During training can load weights from checkpoint ## checkpoint = torch.load('./models/generator_discrim.ckpt')['state_dict'] #Can save weights for just the geneartor for inference at end of training ## model.load_state_dict(checkpoint) ## torch.save(model.superResolver.state_dict(), 'generators/n1x.ckpt') trainer = pl.Trainer( max_epochs=100, precision='32', logger=pl.loggers.CSVLogger("."), # set file name and directory name for checkpoints callbacks=[ pl.callbacks.ModelCheckpoint(save_last=True, every_n_train_steps=50000, filename='versatile', dirpath='./versatility/weighting/') ], ) trainer.fit( model, ) return model if __name__ == "__main__": train()