grassmann-ablation / src /models /grassmann_model.py
Nicholas Finch
feat: checkpoint persistence β€” survives container restarts
a7452cc
Raw History Blame Contribute Delete
6.03 kB
"""
Full Grassmann Transcription Model.
Audio β†’ Mel Spectrogram features β†’ Grassmann Layers β†’ CTC Output
Architecture:
1. Conv frontend: projects mel features into d_model
2. Positional encoding
3. N x CausalGrassmannLayer
4. Linear CTC head β†’ vocab_size
"""
import math
import torch
import torch.nn as nn
import torch.nn.functional as F
from .config import GrassmannConfig
from .grassmann_layer import CausalGrassmannLayer
from .ablation_layer import AblationLayer
from .conformer_block import GrassmannConformerBlock, AblationConformerBlock
class ConvFrontend(nn.Module):
"""Conv1d feature extractor for mel spectrograms (legacy)."""
def __init__(self, n_mels: int, d_model: int):
super().__init__()
self.conv = nn.Sequential(
nn.Conv1d(n_mels, d_model // 2, kernel_size=3, stride=2, padding=1),
nn.GELU(),
nn.Conv1d(d_model // 2, d_model, kernel_size=3, stride=2, padding=1),
nn.GELU(),
)
def forward(self, x: torch.Tensor) -> torch.Tensor:
out = self.conv(x) # (B, d_model, T')
return out.transpose(1, 2) # (B, T', d_model)
class Conv2dSubsampling(nn.Module):
"""
Conv2d subsampling from Conformer (Gulati et al. 2020).
Two 3x3 stride-2 convolutions: 10ms fbank frames β†’ 40ms encoder frames.
Standard ASR frontend used by ESPnet, WeNet, etc.
"""
def __init__(self, n_mels: int, channels: int, d_model: int):
super().__init__()
self.conv1 = nn.Conv2d(1, channels, kernel_size=3, stride=2)
self.conv2 = nn.Conv2d(channels, channels, kernel_size=3, stride=2)
# After two 3x3 stride-2 convs: freq dim = ((n_mels - 1) // 2 - 1) // 2
freq_out = ((n_mels - 1) // 2 - 1) // 2
self.out = nn.Linear(channels * freq_out, d_model)
def forward(self, x: torch.Tensor) -> torch.Tensor:
"""
Args:
x: (batch, n_mels, time) - mel spectrogram
Returns:
(batch, time', d_model)
"""
# Reshape: (B, n_mels, T) β†’ (B, 1, T, n_mels) for Conv2d
x = x.transpose(1, 2).unsqueeze(1) # (B, 1, T, n_mels)
x = F.relu(self.conv1(x))
x = F.relu(self.conv2(x))
b, c, t, f = x.shape
x = x.transpose(1, 2).reshape(b, t, c * f) # (B, T', C*F)
return self.out(x) # (B, T', d_model)
class PositionalEncoding(nn.Module):
"""Sinusoidal positional encoding."""
def __init__(self, d_model: int, max_len: int = 5000, dropout: float = 0.1):
super().__init__()
self.dropout = nn.Dropout(dropout)
pe = torch.zeros(max_len, d_model)
position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
div_term = torch.exp(
torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model)
)
pe[:, 0::2] = torch.sin(position * div_term)
pe[:, 1::2] = torch.cos(position * div_term)
self.register_buffer("pe", pe.unsqueeze(0)) # (1, max_len, d_model)
def forward(self, x: torch.Tensor) -> torch.Tensor:
x = x + self.pe[:, : x.size(1)]
return self.dropout(x)
class GrassmannTranscriptionModel(nn.Module):
"""
Full model: Audio features β†’ Grassmann layers β†’ CTC output.
Supports two encoder modes:
- Plain: CausalGrassmannLayer stack (original)
- Conformer: GrassmannConformerBlock stack (with ConvModule + Macaron FFN)
And two frontend modes:
- Conv1d (legacy)
- Conv2d subsampling (proper ASR, from Conformer)
"""
def __init__(self, config: GrassmannConfig):
super().__init__()
self.config = config
# Frontend: mel β†’ d_model
if config.use_conformer:
self.frontend = Conv2dSubsampling(
config.n_mels, config.conv_channels, config.d_model
)
else:
self.frontend = ConvFrontend(config.n_mels, config.d_model)
# Positional encoding
self.pos_enc = PositionalEncoding(
config.d_model, config.max_seq_len, config.dropout
)
# Encoder layers
ablation = getattr(config, "use_ablation", False)
if config.use_conformer:
block_cls = AblationConformerBlock if ablation else GrassmannConformerBlock
self.layers = nn.ModuleList(
[block_cls(config) for _ in range(config.n_layers)]
)
else:
layer_cls = AblationLayer if ablation else CausalGrassmannLayer
self.layers = nn.ModuleList(
[layer_cls(config, i) for i in range(config.n_layers)]
)
# CTC output head
self.output_proj = nn.Linear(config.d_model, config.vocab_size)
def forward(self, mel: torch.Tensor) -> torch.Tensor:
"""
Args:
mel: (batch, n_mels, time) - mel spectrogram
Returns:
log_probs: (time', batch, vocab_size) - CTC-formatted log probs
"""
# Conv frontend
x = self.frontend(mel) # (B, T', d_model)
# Positional encoding
x = self.pos_enc(x)
# Grassmann layers
for layer in self.layers:
x = layer(x)
# CTC head
logits = self.output_proj(x) # (B, T', vocab_size)
log_probs = logits.log_softmax(dim=-1)
# CTC expects (T, B, C)
return log_probs.transpose(0, 1)
def get_output_lengths(self, input_lengths: torch.Tensor) -> torch.Tensor:
"""Calculate output sequence lengths after conv downsampling."""
lengths = input_lengths
if self.config.use_conformer:
# Conv2d subsampling: two 3x3 stride-2 convs (no padding)
# Each conv: out = (in - 3) // 2 + 1 = (in - 1) // 2
for _ in range(2):
lengths = (lengths - 1) // 2
else:
# Conv1d frontend: two stride-2 convs with padding=1
for _ in range(2):
lengths = (lengths + 1) // 2
return lengths