Spaces:
Running
Running
Nicholas Finch
fix: default workers=0 (sequential) for Docker cache β ProcessPoolExecutor hangs on cpu-basic
d7013e6 Download src/models/conformer_block.py from tisaconundrum/grassmann-cache: direct link, hf CLI and curl.
- Browser
- Download file 4.77 kB
-
https://huggingface.co/spaces/tisaconundrum/grassmann-cache/resolve/main/src/models/conformer_block.py
- Command line
-
hf download hf://spaces/tisaconundrum/grassmann-cache/src/models/conformer_block.py
-
curl -L -o conformer_block.py https://huggingface.co/spaces/tisaconundrum/grassmann-cache/resolve/main/src/models/conformer_block.py
4.77 kB
| """ | |
| Conformer Block with Grassmann Mixer. | |
| Macaron-style block from Gulati et al. 2020 (arXiv 2005.08100) | |
| with the self-attention slot replaced by a Grassmann mixer: | |
| x β +0.5*FFN β +Mixer β +Conv β +0.5*FFN β LayerNorm | |
| This gives the best of both worlds: | |
| - Grassmann mixer for sequence-level subspace geometry | |
| - ConvModule for local temporal pattern capture | |
| - Macaron FFN sandwich for nonlinear capacity | |
| """ | |
| import torch | |
| import torch.nn as nn | |
| from .config import GrassmannConfig | |
| from .conv_module import ConvModule | |
| from .grassmann_layer import GrassmannMixer | |
| from .ablation_layer import AblationMixer | |
| def _macaron_ffn(d: int, d_ff: int, dropout: float) -> nn.Sequential: | |
| """Conformer-style FFN: LayerNorm β Linear β SiLU β Dropout β Linear β Dropout.""" | |
| return nn.Sequential( | |
| nn.LayerNorm(d), | |
| nn.Linear(d, d_ff), | |
| nn.SiLU(), | |
| nn.Dropout(dropout), | |
| nn.Linear(d_ff, d), | |
| nn.Dropout(dropout), | |
| ) | |
| class GatedGrassmannMixer(nn.Module): | |
| """ | |
| Two-sided Grassmann mix with sigmoid gate as a residual branch. | |
| (1 - Ξ±)(g - u) formulation from Dhruv's implementation. | |
| """ | |
| def __init__(self, d: int, r: int, offsets: list[int], eps: float = 1e-8): | |
| super().__init__() | |
| self.mixer = GrassmannMixer(d, r, offsets, two_sided=True, eps=eps) | |
| self.gate = nn.Linear(2 * d, d) | |
| def forward(self, u: torch.Tensor) -> torch.Tensor: | |
| g = self.mixer(u) | |
| alpha = torch.sigmoid(self.gate(torch.cat([u, g], dim=-1))) | |
| return (1 - alpha) * (g - u) | |
| class GatedAblationMixer(nn.Module): | |
| """ | |
| Ablation baseline: symmetric-product mixer with the same gate. | |
| Drop-in replacement for GatedGrassmannMixer. | |
| """ | |
| def __init__(self, d: int, r: int, offsets: list[int], eps: float = 1e-8): | |
| super().__init__() | |
| self.mixer = AblationMixer(d, r, offsets, two_sided=True, eps=eps) | |
| self.gate = nn.Linear(2 * d, d) | |
| def forward(self, u: torch.Tensor) -> torch.Tensor: | |
| g = self.mixer(u) | |
| alpha = torch.sigmoid(self.gate(torch.cat([u, g], dim=-1))) | |
| return (1 - alpha) * (g - u) | |
| class GrassmannConformerBlock(nn.Module): | |
| """ | |
| Conformer block with Grassmann mixer replacing self-attention. | |
| Architecture (Macaron block): | |
| x β +0.5*FFN1 β +GatedGrassmannMixer β +ConvModule β +0.5*FFN2 β LayerNorm | |
| """ | |
| def __init__(self, config: GrassmannConfig): | |
| super().__init__() | |
| d = config.d_model | |
| d_ff = config.d_ff | |
| r = config.r_reduced | |
| dropout = config.dropout | |
| kernel = getattr(config, "conv_kernel_size", 31) | |
| offsets = config.window_offsets | |
| # Macaron FFN sandwich | |
| self.ff1 = _macaron_ffn(d, d_ff, dropout) | |
| self.ff2 = _macaron_ffn(d, d_ff, dropout) | |
| # Grassmann mixer (replaces self-attention) | |
| self.ln_mix = nn.LayerNorm(d) | |
| self.mixer = GatedGrassmannMixer(d, r, offsets, eps=config.plucker_eps) | |
| self.drop = nn.Dropout(dropout) | |
| # ConvModule for local patterns | |
| self.conv = ConvModule(d, kernel, dropout) | |
| # Final LayerNorm | |
| self.ln_out = nn.LayerNorm(d) | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| """ | |
| Args: | |
| x: (batch, seq_len, d_model) | |
| Returns: | |
| (batch, seq_len, d_model) | |
| """ | |
| # Half-step FFN 1 | |
| x = x + 0.5 * self.ff1(x) | |
| # Grassmann mixer (with gate as residual) | |
| x = x + self.drop(self.mixer(self.ln_mix(x))) | |
| # ConvModule | |
| x = x + self.conv(x) | |
| # Half-step FFN 2 + final norm | |
| return self.ln_out(x + 0.5 * self.ff2(x)) | |
| class AblationConformerBlock(nn.Module): | |
| """ | |
| Conformer block with symmetric-product mixer (ablation baseline). | |
| Same Macaron FFN + ConvModule, just swaps Grassmann for symmetric products. | |
| """ | |
| def __init__(self, config: GrassmannConfig): | |
| super().__init__() | |
| d = config.d_model | |
| d_ff = config.d_ff | |
| r = config.r_reduced | |
| dropout = config.dropout | |
| kernel = getattr(config, "conv_kernel_size", 31) | |
| offsets = config.window_offsets | |
| self.ff1 = _macaron_ffn(d, d_ff, dropout) | |
| self.ff2 = _macaron_ffn(d, d_ff, dropout) | |
| self.ln_mix = nn.LayerNorm(d) | |
| self.mixer = GatedAblationMixer(d, r, offsets, eps=config.plucker_eps) | |
| self.drop = nn.Dropout(dropout) | |
| self.conv = ConvModule(d, kernel, dropout) | |
| self.ln_out = nn.LayerNorm(d) | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| x = x + 0.5 * self.ff1(x) | |
| x = x + self.drop(self.mixer(self.ln_mix(x))) | |
| x = x + self.conv(x) | |
| return self.ln_out(x + 0.5 * self.ff2(x)) | |