NGAN / load_balanced_model.py
Kremon96's picture
Upload 103 files
218a4f3 verified
Raw History Blame Contribute Delete
5.95 kB
import os
os.environ.setdefault("CUDA_VISIBLE_DEVICES", "2")
from itertools import chain
import torch
torch.manual_seed(42)
from torch import nn
from encodec.msstftd import MultiScaleSTFTDiscriminator
from encodec.balancer import Balancer
from layers import ReconstructionLoss, SuperResolver, WaveDiscriminator
from data import make_dataloader
try:
import pytorch_lightning as pl
except ImportError:
class pl:
class LightningModule:
pass
class Callback:
pass
l1Loss = torch.nn.L1Loss(reduction='mean')
l2Loss = torch.nn.MSELoss(reduction='mean')
class NGAN(pl.LightningModule):
def __init__(
self,
n_channels: int = 4,
padding: str = 'check',
batch_size: int = 32,
target_sample_rate: int = 48000,
segment_length: int = 24000,
train_data: str = None,
lr: float = 1e-4,
b1: float = 0.5,
b2: float = 0.9
) -> None:
# https://arxiv.org/pdf/2009.02095.pdf
# 2. Method
# SEANet uses Adam with lr=1e-4, beta1=0.5, beta2=0.9
super().__init__()
self.save_hyperparameters()
self.balancer = Balancer(weights={'1': 1, '2': 1, '3':1, '4':1}, rescale_grads=True)
self.automatic_optimization = False
self.superResolver = SuperResolver(n_channels, padding)
self.wave_discriminators = nn.ModuleList([
WaveDiscriminator(resolution=1),
WaveDiscriminator(resolution=2),
WaveDiscriminator(resolution=4)
])
self.sample_rate = target_sample_rate
self.rec_loss = ReconstructionLoss()
self.msstft_discriminator = MultiScaleSTFTDiscriminator(filters=32,
hop_lengths=[256, 512, 128],
win_lengths=[1024, 2048, 512],
n_ffts=[1024, 2048, 512])
def configure_optimizers(self):
lr = self.hparams.lr
b1 = self.hparams.b1
b2 = self.hparams.b2
optimizer_g = torch.optim.Adam(
chain(
self.superResolver.parameters()
),
lr=lr, betas=(b1, b2))
optimizer_d = torch.optim.Adam(
chain(
self.wave_discriminators.parameters(),
self.msstft_discriminator.parameters()
),
lr=lr, betas=(b1, b2))
return [optimizer_g, optimizer_d], []
def forward(self, input):
x = self.superResolver(input)
return x
def training_step(self, batch, batch_idx):
optimizer_g, optimizer_d = self.optimizers()
# train generator
self.toggle_optimizer(optimizer_g)
output = self.forward(batch)
# output: [batch, channel, sequence]
#multi-scale stft generator loss
relu = torch.nn.ReLU()
l_g = 0
l_feat = 0
input_orig = batch[0][:, None, :]
logits_real, fmap_real = self.msstft_discriminator(input_orig)
logits_fake, fmap_fake = self.msstft_discriminator(output)
for tt1 in range(len(fmap_real)):
l_g = l_g + torch.mean(relu(1 - logits_fake[tt1])) / len(logits_fake)
for tt2 in range(len(fmap_real[tt1])):
l_feat = l_feat + l1Loss(fmap_real[tt1][tt2], fmap_fake[tt1][tt2]) / torch.mean(torch.abs(fmap_real[tt1][tt2]))
KL_scale = len(fmap_real)*len(fmap_real[0])
K_scale = len(fmap_real)
#time domain loss, output_wav is the output of the generator
l_t = l1Loss(input_orig, output)
g_rec_loss = self.rec_loss(output[:, 0, :], input_orig[:, 0, :])
loss_1 = l_g/K_scale
loss_2 = l_feat/KL_scale
loss_3 = l_t
loss_4 = g_rec_loss
losses = {'1': loss_1, '2': loss_2, '3': loss_3, '4': loss_4}
input_orig.requires_grad_()
g_loss = loss_1 + loss_2 + loss_3 + loss_4
self.log("g_loss", g_loss, prog_bar=True)
self.balancer.backward(losses, output)
optimizer_g.step()
optimizer_g.zero_grad()
self.untoggle_optimizer(optimizer_g)
# train discriminator
relu = torch.nn.ReLU()
logits_real, _ = self.msstft_discriminator(input_orig)
logits_fake, _ = self.msstft_discriminator(output.detach())
lossd = 0
for tt1 in range(len(logits_real)):
lossd = lossd + torch.mean(relu(1-logits_real[tt1])) + torch.mean(relu(1+logits_fake[tt1]))
d_loss = lossd / len(logits_real)
self.log("d_loss", d_loss, prog_bar=True)
self.manual_backward(d_loss)
optimizer_d.step()
optimizer_d.zero_grad()
self.untoggle_optimizer(optimizer_d)
def train_dataloader(self):
return make_dataloader(
root=self.hparams.train_data,
segment_length=self.hparams.segment_length,
target_sample_rate=self.hparams.target_sample_rate,
batch_size=self.hparams.batch_size)
def train():
model = NGAN(
batch_size=32,
target_sample_rate=48000,
segment_length=24000,
padding='same')
#During training can load weights from checkpoint
## checkpoint = torch.load('./models/generator_discrim.ckpt')['state_dict']
#Can save weights for just the geneartor for inference at end of training
## model.load_state_dict(checkpoint)
## torch.save(model.superResolver.state_dict(), 'generators/n1x.ckpt')
trainer = pl.Trainer(
max_epochs=100,
precision='32',
logger=pl.loggers.CSVLogger("."),
# set file name and directory name for checkpoints
callbacks=[
pl.callbacks.ModelCheckpoint(save_last=True, every_n_train_steps=50000, filename='versatile', dirpath='./versatility/weighting/')
],
)
trainer.fit(
model,
)
return model
if __name__ == "__main__":
train()