grassmann-cache / src /models /conformer_block.py
Nicholas Finch
fix: default workers=0 (sequential) for Docker cache β€” ProcessPoolExecutor hangs on cpu-basic
d7013e6
Raw History Blame Contribute Delete
4.77 kB
"""
Conformer Block with Grassmann Mixer.
Macaron-style block from Gulati et al. 2020 (arXiv 2005.08100)
with the self-attention slot replaced by a Grassmann mixer:
x β†’ +0.5*FFN β†’ +Mixer β†’ +Conv β†’ +0.5*FFN β†’ LayerNorm
This gives the best of both worlds:
- Grassmann mixer for sequence-level subspace geometry
- ConvModule for local temporal pattern capture
- Macaron FFN sandwich for nonlinear capacity
"""
import torch
import torch.nn as nn
from .config import GrassmannConfig
from .conv_module import ConvModule
from .grassmann_layer import GrassmannMixer
from .ablation_layer import AblationMixer
def _macaron_ffn(d: int, d_ff: int, dropout: float) -> nn.Sequential:
"""Conformer-style FFN: LayerNorm β†’ Linear β†’ SiLU β†’ Dropout β†’ Linear β†’ Dropout."""
return nn.Sequential(
nn.LayerNorm(d),
nn.Linear(d, d_ff),
nn.SiLU(),
nn.Dropout(dropout),
nn.Linear(d_ff, d),
nn.Dropout(dropout),
)
class GatedGrassmannMixer(nn.Module):
"""
Two-sided Grassmann mix with sigmoid gate as a residual branch.
(1 - Ξ±)(g - u) formulation from Dhruv's implementation.
"""
def __init__(self, d: int, r: int, offsets: list[int], eps: float = 1e-8):
super().__init__()
self.mixer = GrassmannMixer(d, r, offsets, two_sided=True, eps=eps)
self.gate = nn.Linear(2 * d, d)
def forward(self, u: torch.Tensor) -> torch.Tensor:
g = self.mixer(u)
alpha = torch.sigmoid(self.gate(torch.cat([u, g], dim=-1)))
return (1 - alpha) * (g - u)
class GatedAblationMixer(nn.Module):
"""
Ablation baseline: symmetric-product mixer with the same gate.
Drop-in replacement for GatedGrassmannMixer.
"""
def __init__(self, d: int, r: int, offsets: list[int], eps: float = 1e-8):
super().__init__()
self.mixer = AblationMixer(d, r, offsets, two_sided=True, eps=eps)
self.gate = nn.Linear(2 * d, d)
def forward(self, u: torch.Tensor) -> torch.Tensor:
g = self.mixer(u)
alpha = torch.sigmoid(self.gate(torch.cat([u, g], dim=-1)))
return (1 - alpha) * (g - u)
class GrassmannConformerBlock(nn.Module):
"""
Conformer block with Grassmann mixer replacing self-attention.
Architecture (Macaron block):
x β†’ +0.5*FFN1 β†’ +GatedGrassmannMixer β†’ +ConvModule β†’ +0.5*FFN2 β†’ LayerNorm
"""
def __init__(self, config: GrassmannConfig):
super().__init__()
d = config.d_model
d_ff = config.d_ff
r = config.r_reduced
dropout = config.dropout
kernel = getattr(config, "conv_kernel_size", 31)
offsets = config.window_offsets
# Macaron FFN sandwich
self.ff1 = _macaron_ffn(d, d_ff, dropout)
self.ff2 = _macaron_ffn(d, d_ff, dropout)
# Grassmann mixer (replaces self-attention)
self.ln_mix = nn.LayerNorm(d)
self.mixer = GatedGrassmannMixer(d, r, offsets, eps=config.plucker_eps)
self.drop = nn.Dropout(dropout)
# ConvModule for local patterns
self.conv = ConvModule(d, kernel, dropout)
# Final LayerNorm
self.ln_out = nn.LayerNorm(d)
def forward(self, x: torch.Tensor) -> torch.Tensor:
"""
Args:
x: (batch, seq_len, d_model)
Returns:
(batch, seq_len, d_model)
"""
# Half-step FFN 1
x = x + 0.5 * self.ff1(x)
# Grassmann mixer (with gate as residual)
x = x + self.drop(self.mixer(self.ln_mix(x)))
# ConvModule
x = x + self.conv(x)
# Half-step FFN 2 + final norm
return self.ln_out(x + 0.5 * self.ff2(x))
class AblationConformerBlock(nn.Module):
"""
Conformer block with symmetric-product mixer (ablation baseline).
Same Macaron FFN + ConvModule, just swaps Grassmann for symmetric products.
"""
def __init__(self, config: GrassmannConfig):
super().__init__()
d = config.d_model
d_ff = config.d_ff
r = config.r_reduced
dropout = config.dropout
kernel = getattr(config, "conv_kernel_size", 31)
offsets = config.window_offsets
self.ff1 = _macaron_ffn(d, d_ff, dropout)
self.ff2 = _macaron_ffn(d, d_ff, dropout)
self.ln_mix = nn.LayerNorm(d)
self.mixer = GatedAblationMixer(d, r, offsets, eps=config.plucker_eps)
self.drop = nn.Dropout(dropout)
self.conv = ConvModule(d, kernel, dropout)
self.ln_out = nn.LayerNorm(d)
def forward(self, x: torch.Tensor) -> torch.Tensor:
x = x + 0.5 * self.ff1(x)
x = x + self.drop(self.mixer(self.ln_mix(x)))
x = x + self.conv(x)
return self.ln_out(x + 0.5 * self.ff2(x))