Delete src/miocodec/module/istft_head.py with huggingface_hub
Browse files
src/miocodec/module/istft_head.py
DELETED
|
@@ -1,374 +0,0 @@
|
|
| 1 |
-
# Adapted from X-Codec-2.0: https://github.com/zhenye234/xcodec2
|
| 2 |
-
# ISTFT-based waveform synthesis head for direct waveform generation
|
| 3 |
-
|
| 4 |
-
import torch
|
| 5 |
-
import torch.nn as nn
|
| 6 |
-
import torch.nn.functional as F
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
class SnakeBeta(nn.Module):
|
| 10 |
-
"""
|
| 11 |
-
Snake activation with separate learnable parameters for frequency (alpha) and magnitude (beta).
|
| 12 |
-
|
| 13 |
-
SnakeBeta(x) = x + (1/β) * sin²(αx)
|
| 14 |
-
|
| 15 |
-
This periodic activation function is well-suited for audio synthesis as it can learn
|
| 16 |
-
to generate periodic patterns at different frequencies and magnitudes.
|
| 17 |
-
|
| 18 |
-
Adapted from X-Codec-2.0, originally from https://arxiv.org/abs/2006.08195
|
| 19 |
-
|
| 20 |
-
Args:
|
| 21 |
-
channels: Number of input channels.
|
| 22 |
-
alpha_logscale: If True, alpha and beta are stored in log scale for more stable training.
|
| 23 |
-
"""
|
| 24 |
-
|
| 25 |
-
def __init__(self, channels: int, alpha_logscale: bool = True):
|
| 26 |
-
super().__init__()
|
| 27 |
-
self.channels = channels
|
| 28 |
-
self.alpha_logscale = alpha_logscale
|
| 29 |
-
|
| 30 |
-
# Initialize alpha (frequency) and beta (magnitude)
|
| 31 |
-
if alpha_logscale:
|
| 32 |
-
# Log scale: initialized to zeros (exp(0) = 1)
|
| 33 |
-
self.alpha = nn.Parameter(torch.zeros(channels))
|
| 34 |
-
self.beta = nn.Parameter(torch.zeros(channels))
|
| 35 |
-
else:
|
| 36 |
-
# Linear scale: initialized to ones
|
| 37 |
-
self.alpha = nn.Parameter(torch.ones(channels))
|
| 38 |
-
self.beta = nn.Parameter(torch.ones(channels))
|
| 39 |
-
|
| 40 |
-
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 41 |
-
"""
|
| 42 |
-
Forward pass.
|
| 43 |
-
|
| 44 |
-
Args:
|
| 45 |
-
x: Input tensor of shape (B, C, T) where C is channels.
|
| 46 |
-
|
| 47 |
-
Returns:
|
| 48 |
-
Activated tensor of the same shape.
|
| 49 |
-
"""
|
| 50 |
-
# Reshape for broadcasting: (C,) -> (1, C, 1)
|
| 51 |
-
alpha = self.alpha.unsqueeze(0).unsqueeze(-1)
|
| 52 |
-
beta = self.beta.unsqueeze(0).unsqueeze(-1)
|
| 53 |
-
|
| 54 |
-
if self.alpha_logscale:
|
| 55 |
-
alpha = torch.exp(alpha)
|
| 56 |
-
beta = torch.exp(beta)
|
| 57 |
-
|
| 58 |
-
# SnakeBeta: x + (1/β) * sin²(αx)
|
| 59 |
-
return x + (1.0 / (beta + 1e-9)) * torch.sin(alpha * x).pow(2)
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
class ISTFT(nn.Module):
|
| 63 |
-
"""
|
| 64 |
-
Custom implementation of ISTFT with "same" padding support.
|
| 65 |
-
|
| 66 |
-
Standard torch.istft doesn't allow custom padding (other than center=True) with
|
| 67 |
-
windowing due to NOLA (Nonzero Overlap Add) check failures at the edges.
|
| 68 |
-
This implementation supports "same" padding for neural vocoding where we want
|
| 69 |
-
consistent input/output lengths.
|
| 70 |
-
|
| 71 |
-
Args:
|
| 72 |
-
n_fft: Size of Fourier transform.
|
| 73 |
-
hop_length: The distance between neighboring sliding window frames.
|
| 74 |
-
win_length: The size of window frame and STFT filter.
|
| 75 |
-
padding: Type of padding. Options are "center" or "same".
|
| 76 |
-
"""
|
| 77 |
-
|
| 78 |
-
def __init__(self, n_fft: int, hop_length: int, win_length: int, padding: str = "same"):
|
| 79 |
-
super().__init__()
|
| 80 |
-
if padding not in ["center", "same"]:
|
| 81 |
-
raise ValueError("Padding must be 'center' or 'same'.")
|
| 82 |
-
self.padding = padding
|
| 83 |
-
self.n_fft = n_fft
|
| 84 |
-
self.hop_length = hop_length
|
| 85 |
-
self.win_length = win_length
|
| 86 |
-
window = torch.hann_window(win_length)
|
| 87 |
-
self.register_buffer("window", window)
|
| 88 |
-
|
| 89 |
-
def forward(self, spec: torch.Tensor) -> torch.Tensor:
|
| 90 |
-
"""
|
| 91 |
-
Compute the Inverse Short Time Fourier Transform (ISTFT) of a complex spectrogram.
|
| 92 |
-
|
| 93 |
-
Args:
|
| 94 |
-
spec: Input complex spectrogram of shape (B, N, T), where B is the batch size,
|
| 95 |
-
N is the number of frequency bins, and T is the number of time frames.
|
| 96 |
-
|
| 97 |
-
Returns:
|
| 98 |
-
Reconstructed time-domain signal of shape (B, L), where L is the output signal length.
|
| 99 |
-
"""
|
| 100 |
-
if self.padding == "center":
|
| 101 |
-
# Fallback to pytorch native implementation
|
| 102 |
-
return torch.istft(spec, self.n_fft, self.hop_length, self.win_length, self.window, center=True)
|
| 103 |
-
elif self.padding == "same":
|
| 104 |
-
pad = (self.win_length - self.hop_length) // 2
|
| 105 |
-
else:
|
| 106 |
-
raise ValueError("Padding must be 'center' or 'same'.")
|
| 107 |
-
|
| 108 |
-
assert spec.dim() == 3, "Expected a 3D tensor as input"
|
| 109 |
-
B, N, T = spec.shape
|
| 110 |
-
|
| 111 |
-
# Inverse FFT
|
| 112 |
-
ifft = torch.fft.irfft(spec, self.n_fft, dim=1, norm="backward")
|
| 113 |
-
ifft = ifft * self.window[None, :, None]
|
| 114 |
-
|
| 115 |
-
# Overlap and Add
|
| 116 |
-
output_size = (T - 1) * self.hop_length + self.win_length
|
| 117 |
-
y = F.fold(
|
| 118 |
-
ifft,
|
| 119 |
-
output_size=(1, output_size),
|
| 120 |
-
kernel_size=(1, self.win_length),
|
| 121 |
-
stride=(1, self.hop_length),
|
| 122 |
-
)[:, 0, 0, pad:-pad]
|
| 123 |
-
|
| 124 |
-
# Window envelope
|
| 125 |
-
window_sq = self.window.square().expand(1, T, -1).transpose(1, 2)
|
| 126 |
-
window_envelope = F.fold(
|
| 127 |
-
window_sq,
|
| 128 |
-
output_size=(1, output_size),
|
| 129 |
-
kernel_size=(1, self.win_length),
|
| 130 |
-
stride=(1, self.hop_length),
|
| 131 |
-
).squeeze()[pad:-pad]
|
| 132 |
-
|
| 133 |
-
# Normalize
|
| 134 |
-
assert (window_envelope > 1e-11).all()
|
| 135 |
-
y = y / window_envelope
|
| 136 |
-
|
| 137 |
-
return y
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
class ISTFTHead(nn.Module):
|
| 141 |
-
"""
|
| 142 |
-
ISTFT Head module for predicting STFT complex coefficients and synthesizing waveform.
|
| 143 |
-
|
| 144 |
-
This module takes a hidden representation and predicts magnitude and phase components
|
| 145 |
-
of the STFT, then uses ISTFT to synthesize the waveform.
|
| 146 |
-
|
| 147 |
-
Args:
|
| 148 |
-
dim: Hidden dimension of the input features.
|
| 149 |
-
n_fft: Size of Fourier transform.
|
| 150 |
-
hop_length: The distance between neighboring sliding window frames.
|
| 151 |
-
padding: Type of padding for ISTFT. Options are "center" or "same".
|
| 152 |
-
"""
|
| 153 |
-
|
| 154 |
-
def __init__(self, dim: int, n_fft: int, hop_length: int, padding: str = "same"):
|
| 155 |
-
super().__init__()
|
| 156 |
-
# Output dimension: (n_fft // 2 + 1) * 2 = n_fft + 2
|
| 157 |
-
# First half for magnitude, second half for phase
|
| 158 |
-
out_dim = n_fft + 2
|
| 159 |
-
self.out = nn.Linear(dim, out_dim)
|
| 160 |
-
self.istft = ISTFT(n_fft=n_fft, hop_length=hop_length, win_length=n_fft, padding=padding)
|
| 161 |
-
|
| 162 |
-
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 163 |
-
"""
|
| 164 |
-
Forward pass of the ISTFTHead module.
|
| 165 |
-
|
| 166 |
-
Args:
|
| 167 |
-
x: Input tensor of shape (B, L, H), where B is the batch size,
|
| 168 |
-
L is the sequence length, and H denotes the model dimension.
|
| 169 |
-
|
| 170 |
-
Returns:
|
| 171 |
-
Reconstructed time-domain audio signal of shape (B, T), where T is the output signal length.
|
| 172 |
-
"""
|
| 173 |
-
x = self.out(x)
|
| 174 |
-
x = x.transpose(1, 2) # (B, out_dim, L)
|
| 175 |
-
mag, phase = x.chunk(2, dim=1)
|
| 176 |
-
|
| 177 |
-
# Convert log-magnitude to magnitude
|
| 178 |
-
mag = torch.exp(mag)
|
| 179 |
-
mag = torch.clamp(mag, max=1e2) # Safeguard to prevent excessively large magnitudes
|
| 180 |
-
|
| 181 |
-
# Construct complex spectrogram from magnitude and phase
|
| 182 |
-
# Using cos and sin directly is more stable than exp(1j * phase)
|
| 183 |
-
real = mag * torch.cos(phase)
|
| 184 |
-
imag = mag * torch.sin(phase)
|
| 185 |
-
spec = torch.complex(real, imag)
|
| 186 |
-
|
| 187 |
-
# ISTFT to waveform
|
| 188 |
-
audio = self.istft(spec)
|
| 189 |
-
return audio
|
| 190 |
-
|
| 191 |
-
|
| 192 |
-
class ResNetBlock(nn.Module):
|
| 193 |
-
"""
|
| 194 |
-
ResNet block with GroupNorm for 1D convolutions.
|
| 195 |
-
|
| 196 |
-
This block is used before and after the Transformer in the wave decoder
|
| 197 |
-
to capture local patterns (e.g., consonants, transients).
|
| 198 |
-
|
| 199 |
-
Args:
|
| 200 |
-
channels: Number of input and output channels.
|
| 201 |
-
kernel_size: Kernel size for convolutions.
|
| 202 |
-
dilation: Dilation factor for convolutions.
|
| 203 |
-
num_groups: Number of groups for GroupNorm.
|
| 204 |
-
dropout: Dropout probability.
|
| 205 |
-
"""
|
| 206 |
-
|
| 207 |
-
def __init__(
|
| 208 |
-
self,
|
| 209 |
-
channels: int,
|
| 210 |
-
kernel_size: int = 3,
|
| 211 |
-
dilation: int = 1,
|
| 212 |
-
num_groups: int = 32,
|
| 213 |
-
dropout: float = 0.1,
|
| 214 |
-
):
|
| 215 |
-
super().__init__()
|
| 216 |
-
padding = (kernel_size - 1) * dilation // 2
|
| 217 |
-
|
| 218 |
-
self.norm1 = nn.GroupNorm(num_groups=num_groups, num_channels=channels, eps=1e-6)
|
| 219 |
-
self.conv1 = nn.Conv1d(channels, channels, kernel_size=kernel_size, padding=padding, dilation=dilation)
|
| 220 |
-
self.norm2 = nn.GroupNorm(num_groups=num_groups, num_channels=channels, eps=1e-6)
|
| 221 |
-
self.conv2 = nn.Conv1d(channels, channels, kernel_size=kernel_size, padding=padding, dilation=dilation)
|
| 222 |
-
self.dropout = nn.Dropout(dropout)
|
| 223 |
-
|
| 224 |
-
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 225 |
-
"""
|
| 226 |
-
Forward pass of the ResNetBlock.
|
| 227 |
-
|
| 228 |
-
Args:
|
| 229 |
-
x: Input tensor of shape (B, C, L) where C is channels and L is sequence length.
|
| 230 |
-
|
| 231 |
-
Returns:
|
| 232 |
-
Output tensor of shape (B, C, L).
|
| 233 |
-
"""
|
| 234 |
-
residual = x
|
| 235 |
-
|
| 236 |
-
x = self.norm1(x)
|
| 237 |
-
x = F.silu(x)
|
| 238 |
-
x = self.conv1(x)
|
| 239 |
-
|
| 240 |
-
x = self.norm2(x)
|
| 241 |
-
x = F.silu(x)
|
| 242 |
-
x = self.dropout(x)
|
| 243 |
-
x = self.conv2(x)
|
| 244 |
-
|
| 245 |
-
return x + residual
|
| 246 |
-
|
| 247 |
-
|
| 248 |
-
class ResNetStack(nn.Module):
|
| 249 |
-
"""
|
| 250 |
-
Stack of ResNet blocks for local pattern processing.
|
| 251 |
-
|
| 252 |
-
Args:
|
| 253 |
-
channels: Number of channels.
|
| 254 |
-
num_blocks: Number of ResNet blocks.
|
| 255 |
-
kernel_size: Kernel size for convolutions.
|
| 256 |
-
num_groups: Number of groups for GroupNorm.
|
| 257 |
-
dropout: Dropout probability.
|
| 258 |
-
"""
|
| 259 |
-
|
| 260 |
-
def __init__(
|
| 261 |
-
self,
|
| 262 |
-
channels: int,
|
| 263 |
-
num_blocks: int = 2,
|
| 264 |
-
kernel_size: int = 3,
|
| 265 |
-
num_groups: int = 32,
|
| 266 |
-
dropout: float = 0.1,
|
| 267 |
-
):
|
| 268 |
-
super().__init__()
|
| 269 |
-
self.blocks = nn.ModuleList(
|
| 270 |
-
[ResNetBlock(channels, kernel_size, num_groups=num_groups, dropout=dropout) for _ in range(num_blocks)]
|
| 271 |
-
)
|
| 272 |
-
|
| 273 |
-
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 274 |
-
"""
|
| 275 |
-
Forward pass through all ResNet blocks.
|
| 276 |
-
|
| 277 |
-
Args:
|
| 278 |
-
x: Input tensor of shape (B, C, L).
|
| 279 |
-
|
| 280 |
-
Returns:
|
| 281 |
-
Output tensor of shape (B, C, L).
|
| 282 |
-
"""
|
| 283 |
-
for block in self.blocks:
|
| 284 |
-
x = block(x)
|
| 285 |
-
return x
|
| 286 |
-
|
| 287 |
-
|
| 288 |
-
class UpSamplerBlock(nn.Module):
|
| 289 |
-
"""
|
| 290 |
-
Upsampler block using transposed convolutions, SnakeBeta activations, and ResNet blocks.
|
| 291 |
-
|
| 292 |
-
This module upsamples feature embeddings in the time dimension using a series of
|
| 293 |
-
ConvTranspose1d layers, each followed by SnakeBeta activation and a ResNet block.
|
| 294 |
-
Adapted from X-Codec-2.0.
|
| 295 |
-
|
| 296 |
-
The upsampling is used to increase the frame rate before ISTFT, enabling
|
| 297 |
-
higher sample rate output (e.g., 44.1kHz) from lower frame rate features.
|
| 298 |
-
|
| 299 |
-
Args:
|
| 300 |
-
in_channels: Number of input channels.
|
| 301 |
-
upsample_factors: List of upsampling factors for each stage.
|
| 302 |
-
Total upsampling = product of all factors (e.g., [3, 3] -> 9x).
|
| 303 |
-
kernel_sizes: List of kernel sizes for each ConvTranspose1d.
|
| 304 |
-
If None, defaults to factor * 2 for each stage.
|
| 305 |
-
num_groups: Number of groups for GroupNorm in ResNet blocks.
|
| 306 |
-
"""
|
| 307 |
-
|
| 308 |
-
def __init__(
|
| 309 |
-
self,
|
| 310 |
-
in_channels: int,
|
| 311 |
-
upsample_factors: list[int],
|
| 312 |
-
kernel_sizes: list[int] | None = None,
|
| 313 |
-
num_groups: int = 32,
|
| 314 |
-
):
|
| 315 |
-
super().__init__()
|
| 316 |
-
self.in_channels = in_channels
|
| 317 |
-
self.upsample_factors = list(upsample_factors)
|
| 318 |
-
self.kernel_sizes = list(kernel_sizes or [f * 2 for f in self.upsample_factors])
|
| 319 |
-
|
| 320 |
-
if len(self.kernel_sizes) != len(self.upsample_factors):
|
| 321 |
-
raise ValueError("kernel_sizes and upsample_factors must have the same length")
|
| 322 |
-
|
| 323 |
-
self.upsample_layers = nn.ModuleList()
|
| 324 |
-
self.snake_activations = nn.ModuleList()
|
| 325 |
-
self.resnet_blocks = nn.ModuleList()
|
| 326 |
-
|
| 327 |
-
# Final projection and activation
|
| 328 |
-
final_channels = self.in_channels // (2 ** len(self.upsample_factors))
|
| 329 |
-
self.out_proj = nn.Linear(final_channels, self.in_channels, bias=True)
|
| 330 |
-
self.out_snake = SnakeBeta(self.in_channels, alpha_logscale=True)
|
| 331 |
-
|
| 332 |
-
for i, (k, u) in enumerate(zip(self.kernel_sizes, self.upsample_factors)):
|
| 333 |
-
c_in = self.in_channels // (2**i)
|
| 334 |
-
c_out = self.in_channels // (2 ** (i + 1))
|
| 335 |
-
# ConvTranspose1d for upsampling
|
| 336 |
-
self.upsample_layers.append(
|
| 337 |
-
nn.utils.parametrizations.weight_norm(
|
| 338 |
-
nn.ConvTranspose1d(c_in, c_out, kernel_size=k, stride=u, padding=(k - u) // 2)
|
| 339 |
-
)
|
| 340 |
-
)
|
| 341 |
-
# SnakeBeta activation after ConvTranspose
|
| 342 |
-
self.snake_activations.append(SnakeBeta(c_out, alpha_logscale=True))
|
| 343 |
-
# Adjust num_groups if c_out is smaller
|
| 344 |
-
effective_groups = min(num_groups, c_out)
|
| 345 |
-
self.resnet_blocks.append(
|
| 346 |
-
ResNetBlock(channels=c_out, num_groups=effective_groups, dropout=0.0)
|
| 347 |
-
)
|
| 348 |
-
|
| 349 |
-
@property
|
| 350 |
-
def total_upsample_factor(self) -> int:
|
| 351 |
-
"""Total upsampling factor (product of all stage factors)."""
|
| 352 |
-
result = 1
|
| 353 |
-
for f in self.upsample_factors:
|
| 354 |
-
result *= f
|
| 355 |
-
return result
|
| 356 |
-
|
| 357 |
-
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 358 |
-
"""
|
| 359 |
-
Forward pass of the UpSamplerBlock.
|
| 360 |
-
|
| 361 |
-
Args:
|
| 362 |
-
x: Input tensor of shape (B, C, L) where C is in_channels and L is sequence length.
|
| 363 |
-
|
| 364 |
-
Returns:
|
| 365 |
-
Output tensor of shape (B, L', C) where L' = L * total_upsample_factor.
|
| 366 |
-
Note: Output is transposed to (B, L', C) for compatibility with ISTFTHead.
|
| 367 |
-
"""
|
| 368 |
-
for up, snake, resblk in zip(self.upsample_layers, self.snake_activations, self.resnet_blocks):
|
| 369 |
-
x = snake(up(x)) # ConvTranspose -> SnakeBeta
|
| 370 |
-
x = resblk(x) # ResNet block
|
| 371 |
-
# Project back to original channel dimension, apply SnakeBeta, and transpose to (B, L', C)
|
| 372 |
-
x = self.out_proj(x.transpose(1, 2)) # (B, L', C)
|
| 373 |
-
x = self.out_snake(x.transpose(1, 2)) # (B, C, L') -> SnakeBeta -> (B, C, L')
|
| 374 |
-
return x.transpose(1, 2) # (B, L', C)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|