Spaces:
Running
Running
File size: 4,768 Bytes
d7013e6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 | """
Conformer Block with Grassmann Mixer.
Macaron-style block from Gulati et al. 2020 (arXiv 2005.08100)
with the self-attention slot replaced by a Grassmann mixer:
x β +0.5*FFN β +Mixer β +Conv β +0.5*FFN β LayerNorm
This gives the best of both worlds:
- Grassmann mixer for sequence-level subspace geometry
- ConvModule for local temporal pattern capture
- Macaron FFN sandwich for nonlinear capacity
"""
import torch
import torch.nn as nn
from .config import GrassmannConfig
from .conv_module import ConvModule
from .grassmann_layer import GrassmannMixer
from .ablation_layer import AblationMixer
def _macaron_ffn(d: int, d_ff: int, dropout: float) -> nn.Sequential:
"""Conformer-style FFN: LayerNorm β Linear β SiLU β Dropout β Linear β Dropout."""
return nn.Sequential(
nn.LayerNorm(d),
nn.Linear(d, d_ff),
nn.SiLU(),
nn.Dropout(dropout),
nn.Linear(d_ff, d),
nn.Dropout(dropout),
)
class GatedGrassmannMixer(nn.Module):
"""
Two-sided Grassmann mix with sigmoid gate as a residual branch.
(1 - Ξ±)(g - u) formulation from Dhruv's implementation.
"""
def __init__(self, d: int, r: int, offsets: list[int], eps: float = 1e-8):
super().__init__()
self.mixer = GrassmannMixer(d, r, offsets, two_sided=True, eps=eps)
self.gate = nn.Linear(2 * d, d)
def forward(self, u: torch.Tensor) -> torch.Tensor:
g = self.mixer(u)
alpha = torch.sigmoid(self.gate(torch.cat([u, g], dim=-1)))
return (1 - alpha) * (g - u)
class GatedAblationMixer(nn.Module):
"""
Ablation baseline: symmetric-product mixer with the same gate.
Drop-in replacement for GatedGrassmannMixer.
"""
def __init__(self, d: int, r: int, offsets: list[int], eps: float = 1e-8):
super().__init__()
self.mixer = AblationMixer(d, r, offsets, two_sided=True, eps=eps)
self.gate = nn.Linear(2 * d, d)
def forward(self, u: torch.Tensor) -> torch.Tensor:
g = self.mixer(u)
alpha = torch.sigmoid(self.gate(torch.cat([u, g], dim=-1)))
return (1 - alpha) * (g - u)
class GrassmannConformerBlock(nn.Module):
"""
Conformer block with Grassmann mixer replacing self-attention.
Architecture (Macaron block):
x β +0.5*FFN1 β +GatedGrassmannMixer β +ConvModule β +0.5*FFN2 β LayerNorm
"""
def __init__(self, config: GrassmannConfig):
super().__init__()
d = config.d_model
d_ff = config.d_ff
r = config.r_reduced
dropout = config.dropout
kernel = getattr(config, "conv_kernel_size", 31)
offsets = config.window_offsets
# Macaron FFN sandwich
self.ff1 = _macaron_ffn(d, d_ff, dropout)
self.ff2 = _macaron_ffn(d, d_ff, dropout)
# Grassmann mixer (replaces self-attention)
self.ln_mix = nn.LayerNorm(d)
self.mixer = GatedGrassmannMixer(d, r, offsets, eps=config.plucker_eps)
self.drop = nn.Dropout(dropout)
# ConvModule for local patterns
self.conv = ConvModule(d, kernel, dropout)
# Final LayerNorm
self.ln_out = nn.LayerNorm(d)
def forward(self, x: torch.Tensor) -> torch.Tensor:
"""
Args:
x: (batch, seq_len, d_model)
Returns:
(batch, seq_len, d_model)
"""
# Half-step FFN 1
x = x + 0.5 * self.ff1(x)
# Grassmann mixer (with gate as residual)
x = x + self.drop(self.mixer(self.ln_mix(x)))
# ConvModule
x = x + self.conv(x)
# Half-step FFN 2 + final norm
return self.ln_out(x + 0.5 * self.ff2(x))
class AblationConformerBlock(nn.Module):
"""
Conformer block with symmetric-product mixer (ablation baseline).
Same Macaron FFN + ConvModule, just swaps Grassmann for symmetric products.
"""
def __init__(self, config: GrassmannConfig):
super().__init__()
d = config.d_model
d_ff = config.d_ff
r = config.r_reduced
dropout = config.dropout
kernel = getattr(config, "conv_kernel_size", 31)
offsets = config.window_offsets
self.ff1 = _macaron_ffn(d, d_ff, dropout)
self.ff2 = _macaron_ffn(d, d_ff, dropout)
self.ln_mix = nn.LayerNorm(d)
self.mixer = GatedAblationMixer(d, r, offsets, eps=config.plucker_eps)
self.drop = nn.Dropout(dropout)
self.conv = ConvModule(d, kernel, dropout)
self.ln_out = nn.LayerNorm(d)
def forward(self, x: torch.Tensor) -> torch.Tensor:
x = x + 0.5 * self.ff1(x)
x = x + self.drop(self.mixer(self.ln_mix(x)))
x = x + self.conv(x)
return self.ln_out(x + 0.5 * self.ff2(x))
|