Spaces:
Running
Running
File size: 6,030 Bytes
d7013e6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 | """
Full Grassmann Transcription Model.
Audio β Mel Spectrogram features β Grassmann Layers β CTC Output
Architecture:
1. Conv frontend: projects mel features into d_model
2. Positional encoding
3. N x CausalGrassmannLayer
4. Linear CTC head β vocab_size
"""
import math
import torch
import torch.nn as nn
import torch.nn.functional as F
from .config import GrassmannConfig
from .grassmann_layer import CausalGrassmannLayer
from .ablation_layer import AblationLayer
from .conformer_block import GrassmannConformerBlock, AblationConformerBlock
class ConvFrontend(nn.Module):
"""Conv1d feature extractor for mel spectrograms (legacy)."""
def __init__(self, n_mels: int, d_model: int):
super().__init__()
self.conv = nn.Sequential(
nn.Conv1d(n_mels, d_model // 2, kernel_size=3, stride=2, padding=1),
nn.GELU(),
nn.Conv1d(d_model // 2, d_model, kernel_size=3, stride=2, padding=1),
nn.GELU(),
)
def forward(self, x: torch.Tensor) -> torch.Tensor:
out = self.conv(x) # (B, d_model, T')
return out.transpose(1, 2) # (B, T', d_model)
class Conv2dSubsampling(nn.Module):
"""
Conv2d subsampling from Conformer (Gulati et al. 2020).
Two 3x3 stride-2 convolutions: 10ms fbank frames β 40ms encoder frames.
Standard ASR frontend used by ESPnet, WeNet, etc.
"""
def __init__(self, n_mels: int, channels: int, d_model: int):
super().__init__()
self.conv1 = nn.Conv2d(1, channels, kernel_size=3, stride=2)
self.conv2 = nn.Conv2d(channels, channels, kernel_size=3, stride=2)
# After two 3x3 stride-2 convs: freq dim = ((n_mels - 1) // 2 - 1) // 2
freq_out = ((n_mels - 1) // 2 - 1) // 2
self.out = nn.Linear(channels * freq_out, d_model)
def forward(self, x: torch.Tensor) -> torch.Tensor:
"""
Args:
x: (batch, n_mels, time) - mel spectrogram
Returns:
(batch, time', d_model)
"""
# Reshape: (B, n_mels, T) β (B, 1, T, n_mels) for Conv2d
x = x.transpose(1, 2).unsqueeze(1) # (B, 1, T, n_mels)
x = F.relu(self.conv1(x))
x = F.relu(self.conv2(x))
b, c, t, f = x.shape
x = x.transpose(1, 2).reshape(b, t, c * f) # (B, T', C*F)
return self.out(x) # (B, T', d_model)
class PositionalEncoding(nn.Module):
"""Sinusoidal positional encoding."""
def __init__(self, d_model: int, max_len: int = 5000, dropout: float = 0.1):
super().__init__()
self.dropout = nn.Dropout(dropout)
pe = torch.zeros(max_len, d_model)
position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
div_term = torch.exp(
torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model)
)
pe[:, 0::2] = torch.sin(position * div_term)
pe[:, 1::2] = torch.cos(position * div_term)
self.register_buffer("pe", pe.unsqueeze(0)) # (1, max_len, d_model)
def forward(self, x: torch.Tensor) -> torch.Tensor:
x = x + self.pe[:, : x.size(1)]
return self.dropout(x)
class GrassmannTranscriptionModel(nn.Module):
"""
Full model: Audio features β Grassmann layers β CTC output.
Supports two encoder modes:
- Plain: CausalGrassmannLayer stack (original)
- Conformer: GrassmannConformerBlock stack (with ConvModule + Macaron FFN)
And two frontend modes:
- Conv1d (legacy)
- Conv2d subsampling (proper ASR, from Conformer)
"""
def __init__(self, config: GrassmannConfig):
super().__init__()
self.config = config
# Frontend: mel β d_model
if config.use_conformer:
self.frontend = Conv2dSubsampling(
config.n_mels, config.conv_channels, config.d_model
)
else:
self.frontend = ConvFrontend(config.n_mels, config.d_model)
# Positional encoding
self.pos_enc = PositionalEncoding(
config.d_model, config.max_seq_len, config.dropout
)
# Encoder layers
ablation = getattr(config, "use_ablation", False)
if config.use_conformer:
block_cls = AblationConformerBlock if ablation else GrassmannConformerBlock
self.layers = nn.ModuleList(
[block_cls(config) for _ in range(config.n_layers)]
)
else:
layer_cls = AblationLayer if ablation else CausalGrassmannLayer
self.layers = nn.ModuleList(
[layer_cls(config, i) for i in range(config.n_layers)]
)
# CTC output head
self.output_proj = nn.Linear(config.d_model, config.vocab_size)
def forward(self, mel: torch.Tensor) -> torch.Tensor:
"""
Args:
mel: (batch, n_mels, time) - mel spectrogram
Returns:
log_probs: (time', batch, vocab_size) - CTC-formatted log probs
"""
# Conv frontend
x = self.frontend(mel) # (B, T', d_model)
# Positional encoding
x = self.pos_enc(x)
# Grassmann layers
for layer in self.layers:
x = layer(x)
# CTC head
logits = self.output_proj(x) # (B, T', vocab_size)
log_probs = logits.log_softmax(dim=-1)
# CTC expects (T, B, C)
return log_probs.transpose(0, 1)
def get_output_lengths(self, input_lengths: torch.Tensor) -> torch.Tensor:
"""Calculate output sequence lengths after conv downsampling."""
lengths = input_lengths
if self.config.use_conformer:
# Conv2d subsampling: two 3x3 stride-2 convs (no padding)
# Each conv: out = (in - 3) // 2 + 1 = (in - 1) // 2
for _ in range(2):
lengths = (lengths - 1) // 2
else:
# Conv1d frontend: two stride-2 convs with padding=1
for _ in range(2):
lengths = (lengths + 1) // 2
return lengths
|