NGAN / layers.py
Kremon96's picture
Upload 103 files
218a4f3 verified
Raw History Blame Contribute Delete
9.07 kB
"""Shared generator/discriminator building blocks for N-GAN.
Extracted from model.py and load_balanced_model.py, which previously
defined identical copies of everything below.
"""
from typing import List, Tuple
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from scipy.interpolate import CubicSpline
def cubic_upsampler(audio_signal, input_rate, target_rate):
original_duration = audio_signal.size()[-1] / input_rate
original_time = np.linspace(0, original_duration, audio_signal.size()[-1])
target_time = np.linspace(0, original_duration, int(original_duration * target_rate))
cubic_spline = CubicSpline(original_time, audio_signal, axis=-1)
upsampled_audio = cubic_spline(target_time)
return upsampled_audio
class ResNet1d(nn.Module):
def __init__(
self,
n_channels,
kernel_size: int = 3,
padding: str = 'same',
dilation: int = 1
) -> None:
super().__init__()
assert padding in ['valid', 'same']
self.kernel_size = kernel_size
self.padding = padding
self.dilation = dilation
self._padding_size = (kernel_size // 2) * dilation
self.conv0 = nn.Conv1d(
n_channels,
n_channels,
kernel_size=kernel_size,
padding=padding,
dilation=dilation)
self.bn0 = nn.BatchNorm1d(
n_channels
)
self.conv1 = nn.Conv1d(
n_channels,
n_channels,
kernel_size=1)
self.bn1 = nn.BatchNorm1d(
n_channels
)
def forward(self, input):
y = input
x = self.conv0(input)
#x = self.bn0(x)
x = F.elu(x)
x = self.conv1(x)
#x = self.bn1(x)
if self.padding == 'valid':
y = y[:, :, self._padding_size:-self._padding_size]
x += y
x = F.elu(x)
return x
class ResNet2d(nn.Module):
def __init__(
self,
n_channels: int,
factor: int,
stride: Tuple[int, int]
) -> None:
# Method from https://arxiv.org/pdf/2005.00341.pdf
# The original paper uses layer normalization, but here
# we use batch normalization.
super().__init__()
self.conv0 = nn.Conv2d(
n_channels,
n_channels,
kernel_size=(3, 3),
padding='same')
self.bn0 = nn.BatchNorm2d(
n_channels
)
self.conv1 = nn.Conv2d(
n_channels,
factor * n_channels,
kernel_size=(stride[0] + 2, stride[1] + 2),
stride=stride)
self.bn1 = nn.BatchNorm2d(
factor * n_channels
)
self.conv2 = nn.Conv2d(
n_channels,
factor * n_channels,
kernel_size=1,
stride=stride)
self.bn2 = nn.BatchNorm2d(
factor * n_channels
)
self.pad = nn.ReflectionPad2d([
(stride[1] + 1) // 2,
(stride[1] + 2) // 2,
(stride[0] + 1) // 2,
(stride[0] + 2) // 2,
])
self.activation = nn.LeakyReLU(0.3)
def forward(self, input):
x = self.conv0(input)
x = self.bn0(x)
x = self.activation(x)
x = self.pad(x)
x = self.conv1(x)
x = self.bn1(x)
# shortcut
y = self.conv2(input)
y = self.bn2(y)
x += y
x = self.activation(x)
return x
class EncoderBlock(nn.Module):
def __init__(
self,
n_channels: int,
padding: str,
stride: int
) -> None:
super().__init__()
assert padding in ['valid', 'same']
self.layers = nn.Sequential(
ResNet1d(n_channels // 2, padding=padding, dilation=1),
ResNet1d(n_channels // 2, padding=padding, dilation=3),
ResNet1d(n_channels // 2, padding=padding, dilation=9),
nn.Conv1d(
n_channels // 2, n_channels,
kernel_size=2 * stride,
padding=stride// 2,
stride=stride),
nn.ELU(),
)
def forward(self, input: torch.Tensor) -> torch.Tensor:
return self.layers(input)
class DecoderBlock(nn.Module):
def __init__(
self,
n_channels: int,
padding: str,
stride: int
) -> None:
super().__init__()
assert padding in ['valid', 'same']
self.layers = nn.Sequential(
nn.ConvTranspose1d(
n_channels, n_channels // 2,
kernel_size=2 * stride,
padding=stride// 2,
stride=stride),
nn.ELU(),
ResNet1d(n_channels // 2, padding=padding, dilation=1),
ResNet1d(n_channels // 2, padding=padding, dilation=3),
ResNet1d(n_channels // 2, padding=padding, dilation=9),
)
def forward(self, input: torch.Tensor) -> torch.Tensor:
return self.layers(input)
class SuperResolver(nn.Module):
def __init__(self, n_channels: int, padding):
super().__init__()
assert padding in ['valid', 'same']
self.n_channels = n_channels
self.resolve = nn.Sequential(
nn.Conv1d(1, n_channels, kernel_size=7, padding=padding),
nn.ELU(),
EncoderBlock(2 * n_channels, padding=padding, stride=6),
EncoderBlock(4 * n_channels, padding=padding, stride=4),
EncoderBlock(8 * n_channels, padding=padding, stride=2),
nn.Conv1d(8 * n_channels, 8 * n_channels, kernel_size=3, padding=padding),
nn.Conv1d(8 * n_channels, 8 * n_channels, kernel_size=7, padding=padding),
nn.ELU(),
DecoderBlock(8 * n_channels, padding=padding, stride=6),
DecoderBlock(4 * n_channels, padding=padding, stride=4),
DecoderBlock(2 * n_channels, padding=padding, stride=2),
nn.Conv1d(n_channels, 1, kernel_size=7, padding=padding),
nn.Tanh(),
)
def forward(self, input: torch.Tensor) -> torch.Tensor:
x = self.resolve(input[1][:, None, :])
return x
class WaveDiscriminator(nn.Module):
"""MelGAN discriminator from https://arxiv.org/pdf/1910.06711.pdf
"""
def __init__(self, resolution: int = 1, n_channels: int = 4) -> None:
super().__init__()
assert resolution >= 1
if resolution == 1:
self.avg_pool = nn.Identity()
else:
self.avg_pool = nn.AvgPool1d(resolution * 2, stride=resolution)
self.activation = nn.LeakyReLU(0.2, inplace=True)
self.layers = nn.ModuleList([
nn.utils.weight_norm(nn.Conv1d(1, n_channels, kernel_size=15, padding=7)),
nn.utils.weight_norm(nn.Conv1d(n_channels, 4 * n_channels, kernel_size=41, stride=4, padding=20, groups=4)),
nn.utils.weight_norm(nn.Conv1d(4 * n_channels, 16 * n_channels, kernel_size=41, stride=4, padding=20, groups=16)),
nn.utils.weight_norm(nn.Conv1d(16 * n_channels, 64 * n_channels, kernel_size=41, stride=4, padding=20, groups=64)),
nn.utils.weight_norm(nn.Conv1d(64 * n_channels, 256 * n_channels, kernel_size=41, stride=4, padding=20, groups=256)),
nn.utils.weight_norm(nn.Conv1d(256 * n_channels, 256 * n_channels, kernel_size=5, padding=2)),
nn.utils.weight_norm(nn.Conv1d(256 * n_channels, 1, kernel_size=3, padding=1)),
])
def forward(self, x: torch.Tensor) -> List[torch.Tensor]:
x = self.avg_pool(x)
feats = []
for layer in self.layers[:-1]:
x = layer(x)
feats.append(x)
x = self.activation(x)
feats.append(self.layers[-1](x))
return feats
class ReconstructionLoss(nn.Module):
"""Reconstruction loss from https://arxiv.org/pdf/2107.03312.pdf
but uses STFT instead of mel-spectrogram
"""
def __init__(self, eps=1e-5):
super().__init__()
self.eps = eps
def forward(self, input, target):
loss = 0
input = input.to(torch.float32)
target = target.to(torch.float32)
for i in range(6, 12):
s = 2 ** i
alpha = (s / 2) ** 0.5
# We use STFT instead of 64-bin mel-spectrogram as n_fft=64 is too small
# for 64 bins.
x = torch.stft(input, n_fft=s, hop_length=s // 4, win_length=s, normalized=True, onesided=True, return_complex=True)
x = torch.abs(x)
y = torch.stft(target, n_fft=s, hop_length=s // 4, win_length=s, normalized=True, onesided=True, return_complex=True)
y = torch.abs(y)
if x.shape[-1] > y.shape[-1]:
x = x[:, :, :y.shape[-1]]
elif x.shape[-1] < y.shape[-1]:
y = y[:, :, :x.shape[-1]]
loss += torch.mean(torch.abs(x - y))
loss += alpha * torch.mean(torch.square(torch.log(x + self.eps) - torch.log(y + self.eps)))
return loss / (12 - 6)