PRIYANSHUDHAKED commited on
Commit
e4e2231
·
verified ·
1 Parent(s): 211fd93

Delete src/miocodec/module/istft_head.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. src/miocodec/module/istft_head.py +0 -374
src/miocodec/module/istft_head.py DELETED
@@ -1,374 +0,0 @@
1
- # Adapted from X-Codec-2.0: https://github.com/zhenye234/xcodec2
2
- # ISTFT-based waveform synthesis head for direct waveform generation
3
-
4
- import torch
5
- import torch.nn as nn
6
- import torch.nn.functional as F
7
-
8
-
9
- class SnakeBeta(nn.Module):
10
- """
11
- Snake activation with separate learnable parameters for frequency (alpha) and magnitude (beta).
12
-
13
- SnakeBeta(x) = x + (1/β) * sin²(αx)
14
-
15
- This periodic activation function is well-suited for audio synthesis as it can learn
16
- to generate periodic patterns at different frequencies and magnitudes.
17
-
18
- Adapted from X-Codec-2.0, originally from https://arxiv.org/abs/2006.08195
19
-
20
- Args:
21
- channels: Number of input channels.
22
- alpha_logscale: If True, alpha and beta are stored in log scale for more stable training.
23
- """
24
-
25
- def __init__(self, channels: int, alpha_logscale: bool = True):
26
- super().__init__()
27
- self.channels = channels
28
- self.alpha_logscale = alpha_logscale
29
-
30
- # Initialize alpha (frequency) and beta (magnitude)
31
- if alpha_logscale:
32
- # Log scale: initialized to zeros (exp(0) = 1)
33
- self.alpha = nn.Parameter(torch.zeros(channels))
34
- self.beta = nn.Parameter(torch.zeros(channels))
35
- else:
36
- # Linear scale: initialized to ones
37
- self.alpha = nn.Parameter(torch.ones(channels))
38
- self.beta = nn.Parameter(torch.ones(channels))
39
-
40
- def forward(self, x: torch.Tensor) -> torch.Tensor:
41
- """
42
- Forward pass.
43
-
44
- Args:
45
- x: Input tensor of shape (B, C, T) where C is channels.
46
-
47
- Returns:
48
- Activated tensor of the same shape.
49
- """
50
- # Reshape for broadcasting: (C,) -> (1, C, 1)
51
- alpha = self.alpha.unsqueeze(0).unsqueeze(-1)
52
- beta = self.beta.unsqueeze(0).unsqueeze(-1)
53
-
54
- if self.alpha_logscale:
55
- alpha = torch.exp(alpha)
56
- beta = torch.exp(beta)
57
-
58
- # SnakeBeta: x + (1/β) * sin²(αx)
59
- return x + (1.0 / (beta + 1e-9)) * torch.sin(alpha * x).pow(2)
60
-
61
-
62
- class ISTFT(nn.Module):
63
- """
64
- Custom implementation of ISTFT with "same" padding support.
65
-
66
- Standard torch.istft doesn't allow custom padding (other than center=True) with
67
- windowing due to NOLA (Nonzero Overlap Add) check failures at the edges.
68
- This implementation supports "same" padding for neural vocoding where we want
69
- consistent input/output lengths.
70
-
71
- Args:
72
- n_fft: Size of Fourier transform.
73
- hop_length: The distance between neighboring sliding window frames.
74
- win_length: The size of window frame and STFT filter.
75
- padding: Type of padding. Options are "center" or "same".
76
- """
77
-
78
- def __init__(self, n_fft: int, hop_length: int, win_length: int, padding: str = "same"):
79
- super().__init__()
80
- if padding not in ["center", "same"]:
81
- raise ValueError("Padding must be 'center' or 'same'.")
82
- self.padding = padding
83
- self.n_fft = n_fft
84
- self.hop_length = hop_length
85
- self.win_length = win_length
86
- window = torch.hann_window(win_length)
87
- self.register_buffer("window", window)
88
-
89
- def forward(self, spec: torch.Tensor) -> torch.Tensor:
90
- """
91
- Compute the Inverse Short Time Fourier Transform (ISTFT) of a complex spectrogram.
92
-
93
- Args:
94
- spec: Input complex spectrogram of shape (B, N, T), where B is the batch size,
95
- N is the number of frequency bins, and T is the number of time frames.
96
-
97
- Returns:
98
- Reconstructed time-domain signal of shape (B, L), where L is the output signal length.
99
- """
100
- if self.padding == "center":
101
- # Fallback to pytorch native implementation
102
- return torch.istft(spec, self.n_fft, self.hop_length, self.win_length, self.window, center=True)
103
- elif self.padding == "same":
104
- pad = (self.win_length - self.hop_length) // 2
105
- else:
106
- raise ValueError("Padding must be 'center' or 'same'.")
107
-
108
- assert spec.dim() == 3, "Expected a 3D tensor as input"
109
- B, N, T = spec.shape
110
-
111
- # Inverse FFT
112
- ifft = torch.fft.irfft(spec, self.n_fft, dim=1, norm="backward")
113
- ifft = ifft * self.window[None, :, None]
114
-
115
- # Overlap and Add
116
- output_size = (T - 1) * self.hop_length + self.win_length
117
- y = F.fold(
118
- ifft,
119
- output_size=(1, output_size),
120
- kernel_size=(1, self.win_length),
121
- stride=(1, self.hop_length),
122
- )[:, 0, 0, pad:-pad]
123
-
124
- # Window envelope
125
- window_sq = self.window.square().expand(1, T, -1).transpose(1, 2)
126
- window_envelope = F.fold(
127
- window_sq,
128
- output_size=(1, output_size),
129
- kernel_size=(1, self.win_length),
130
- stride=(1, self.hop_length),
131
- ).squeeze()[pad:-pad]
132
-
133
- # Normalize
134
- assert (window_envelope > 1e-11).all()
135
- y = y / window_envelope
136
-
137
- return y
138
-
139
-
140
- class ISTFTHead(nn.Module):
141
- """
142
- ISTFT Head module for predicting STFT complex coefficients and synthesizing waveform.
143
-
144
- This module takes a hidden representation and predicts magnitude and phase components
145
- of the STFT, then uses ISTFT to synthesize the waveform.
146
-
147
- Args:
148
- dim: Hidden dimension of the input features.
149
- n_fft: Size of Fourier transform.
150
- hop_length: The distance between neighboring sliding window frames.
151
- padding: Type of padding for ISTFT. Options are "center" or "same".
152
- """
153
-
154
- def __init__(self, dim: int, n_fft: int, hop_length: int, padding: str = "same"):
155
- super().__init__()
156
- # Output dimension: (n_fft // 2 + 1) * 2 = n_fft + 2
157
- # First half for magnitude, second half for phase
158
- out_dim = n_fft + 2
159
- self.out = nn.Linear(dim, out_dim)
160
- self.istft = ISTFT(n_fft=n_fft, hop_length=hop_length, win_length=n_fft, padding=padding)
161
-
162
- def forward(self, x: torch.Tensor) -> torch.Tensor:
163
- """
164
- Forward pass of the ISTFTHead module.
165
-
166
- Args:
167
- x: Input tensor of shape (B, L, H), where B is the batch size,
168
- L is the sequence length, and H denotes the model dimension.
169
-
170
- Returns:
171
- Reconstructed time-domain audio signal of shape (B, T), where T is the output signal length.
172
- """
173
- x = self.out(x)
174
- x = x.transpose(1, 2) # (B, out_dim, L)
175
- mag, phase = x.chunk(2, dim=1)
176
-
177
- # Convert log-magnitude to magnitude
178
- mag = torch.exp(mag)
179
- mag = torch.clamp(mag, max=1e2) # Safeguard to prevent excessively large magnitudes
180
-
181
- # Construct complex spectrogram from magnitude and phase
182
- # Using cos and sin directly is more stable than exp(1j * phase)
183
- real = mag * torch.cos(phase)
184
- imag = mag * torch.sin(phase)
185
- spec = torch.complex(real, imag)
186
-
187
- # ISTFT to waveform
188
- audio = self.istft(spec)
189
- return audio
190
-
191
-
192
- class ResNetBlock(nn.Module):
193
- """
194
- ResNet block with GroupNorm for 1D convolutions.
195
-
196
- This block is used before and after the Transformer in the wave decoder
197
- to capture local patterns (e.g., consonants, transients).
198
-
199
- Args:
200
- channels: Number of input and output channels.
201
- kernel_size: Kernel size for convolutions.
202
- dilation: Dilation factor for convolutions.
203
- num_groups: Number of groups for GroupNorm.
204
- dropout: Dropout probability.
205
- """
206
-
207
- def __init__(
208
- self,
209
- channels: int,
210
- kernel_size: int = 3,
211
- dilation: int = 1,
212
- num_groups: int = 32,
213
- dropout: float = 0.1,
214
- ):
215
- super().__init__()
216
- padding = (kernel_size - 1) * dilation // 2
217
-
218
- self.norm1 = nn.GroupNorm(num_groups=num_groups, num_channels=channels, eps=1e-6)
219
- self.conv1 = nn.Conv1d(channels, channels, kernel_size=kernel_size, padding=padding, dilation=dilation)
220
- self.norm2 = nn.GroupNorm(num_groups=num_groups, num_channels=channels, eps=1e-6)
221
- self.conv2 = nn.Conv1d(channels, channels, kernel_size=kernel_size, padding=padding, dilation=dilation)
222
- self.dropout = nn.Dropout(dropout)
223
-
224
- def forward(self, x: torch.Tensor) -> torch.Tensor:
225
- """
226
- Forward pass of the ResNetBlock.
227
-
228
- Args:
229
- x: Input tensor of shape (B, C, L) where C is channels and L is sequence length.
230
-
231
- Returns:
232
- Output tensor of shape (B, C, L).
233
- """
234
- residual = x
235
-
236
- x = self.norm1(x)
237
- x = F.silu(x)
238
- x = self.conv1(x)
239
-
240
- x = self.norm2(x)
241
- x = F.silu(x)
242
- x = self.dropout(x)
243
- x = self.conv2(x)
244
-
245
- return x + residual
246
-
247
-
248
- class ResNetStack(nn.Module):
249
- """
250
- Stack of ResNet blocks for local pattern processing.
251
-
252
- Args:
253
- channels: Number of channels.
254
- num_blocks: Number of ResNet blocks.
255
- kernel_size: Kernel size for convolutions.
256
- num_groups: Number of groups for GroupNorm.
257
- dropout: Dropout probability.
258
- """
259
-
260
- def __init__(
261
- self,
262
- channels: int,
263
- num_blocks: int = 2,
264
- kernel_size: int = 3,
265
- num_groups: int = 32,
266
- dropout: float = 0.1,
267
- ):
268
- super().__init__()
269
- self.blocks = nn.ModuleList(
270
- [ResNetBlock(channels, kernel_size, num_groups=num_groups, dropout=dropout) for _ in range(num_blocks)]
271
- )
272
-
273
- def forward(self, x: torch.Tensor) -> torch.Tensor:
274
- """
275
- Forward pass through all ResNet blocks.
276
-
277
- Args:
278
- x: Input tensor of shape (B, C, L).
279
-
280
- Returns:
281
- Output tensor of shape (B, C, L).
282
- """
283
- for block in self.blocks:
284
- x = block(x)
285
- return x
286
-
287
-
288
- class UpSamplerBlock(nn.Module):
289
- """
290
- Upsampler block using transposed convolutions, SnakeBeta activations, and ResNet blocks.
291
-
292
- This module upsamples feature embeddings in the time dimension using a series of
293
- ConvTranspose1d layers, each followed by SnakeBeta activation and a ResNet block.
294
- Adapted from X-Codec-2.0.
295
-
296
- The upsampling is used to increase the frame rate before ISTFT, enabling
297
- higher sample rate output (e.g., 44.1kHz) from lower frame rate features.
298
-
299
- Args:
300
- in_channels: Number of input channels.
301
- upsample_factors: List of upsampling factors for each stage.
302
- Total upsampling = product of all factors (e.g., [3, 3] -> 9x).
303
- kernel_sizes: List of kernel sizes for each ConvTranspose1d.
304
- If None, defaults to factor * 2 for each stage.
305
- num_groups: Number of groups for GroupNorm in ResNet blocks.
306
- """
307
-
308
- def __init__(
309
- self,
310
- in_channels: int,
311
- upsample_factors: list[int],
312
- kernel_sizes: list[int] | None = None,
313
- num_groups: int = 32,
314
- ):
315
- super().__init__()
316
- self.in_channels = in_channels
317
- self.upsample_factors = list(upsample_factors)
318
- self.kernel_sizes = list(kernel_sizes or [f * 2 for f in self.upsample_factors])
319
-
320
- if len(self.kernel_sizes) != len(self.upsample_factors):
321
- raise ValueError("kernel_sizes and upsample_factors must have the same length")
322
-
323
- self.upsample_layers = nn.ModuleList()
324
- self.snake_activations = nn.ModuleList()
325
- self.resnet_blocks = nn.ModuleList()
326
-
327
- # Final projection and activation
328
- final_channels = self.in_channels // (2 ** len(self.upsample_factors))
329
- self.out_proj = nn.Linear(final_channels, self.in_channels, bias=True)
330
- self.out_snake = SnakeBeta(self.in_channels, alpha_logscale=True)
331
-
332
- for i, (k, u) in enumerate(zip(self.kernel_sizes, self.upsample_factors)):
333
- c_in = self.in_channels // (2**i)
334
- c_out = self.in_channels // (2 ** (i + 1))
335
- # ConvTranspose1d for upsampling
336
- self.upsample_layers.append(
337
- nn.utils.parametrizations.weight_norm(
338
- nn.ConvTranspose1d(c_in, c_out, kernel_size=k, stride=u, padding=(k - u) // 2)
339
- )
340
- )
341
- # SnakeBeta activation after ConvTranspose
342
- self.snake_activations.append(SnakeBeta(c_out, alpha_logscale=True))
343
- # Adjust num_groups if c_out is smaller
344
- effective_groups = min(num_groups, c_out)
345
- self.resnet_blocks.append(
346
- ResNetBlock(channels=c_out, num_groups=effective_groups, dropout=0.0)
347
- )
348
-
349
- @property
350
- def total_upsample_factor(self) -> int:
351
- """Total upsampling factor (product of all stage factors)."""
352
- result = 1
353
- for f in self.upsample_factors:
354
- result *= f
355
- return result
356
-
357
- def forward(self, x: torch.Tensor) -> torch.Tensor:
358
- """
359
- Forward pass of the UpSamplerBlock.
360
-
361
- Args:
362
- x: Input tensor of shape (B, C, L) where C is in_channels and L is sequence length.
363
-
364
- Returns:
365
- Output tensor of shape (B, L', C) where L' = L * total_upsample_factor.
366
- Note: Output is transposed to (B, L', C) for compatibility with ISTFTHead.
367
- """
368
- for up, snake, resblk in zip(self.upsample_layers, self.snake_activations, self.resnet_blocks):
369
- x = snake(up(x)) # ConvTranspose -> SnakeBeta
370
- x = resblk(x) # ResNet block
371
- # Project back to original channel dimension, apply SnakeBeta, and transpose to (B, L', C)
372
- x = self.out_proj(x.transpose(1, 2)) # (B, L', C)
373
- x = self.out_snake(x.transpose(1, 2)) # (B, C, L') -> SnakeBeta -> (B, C, L')
374
- return x.transpose(1, 2) # (B, L', C)