Spaces:
Running on T4
Running on T4
Download src/models/grassmann_layer.py from tisaconundrum/grassmann-ablation: direct link, hf CLI and curl.
- Browser
- Download file 6.31 kB
-
https://huggingface.co/spaces/tisaconundrum/grassmann-ablation/resolve/main/src/models/grassmann_layer.py
- Command line
-
hf download hf://spaces/tisaconundrum/grassmann-ablation/src/models/grassmann_layer.py
-
curl -L -o grassmann_layer.py https://huggingface.co/spaces/tisaconundrum/grassmann-ablation/resolve/main/src/models/grassmann_layer.py
6.31 kB
| """ | |
| Causal Grassmann Mixing Layer. | |
| Implements the core architecture from "Attention Is Not What You Need" | |
| (arXiv:2512.19428). Replaces self-attention with: | |
| 1. Linear reduction: h_t ∈ R^d → z_t ∈ R^r | |
| 2. Multi-scale pairing: (z_t, z_{t+Δ}) for Δ ∈ W | |
| 3. Plücker encoding: p_{ij} = z_{t,i} * z_{t+Δ,j} - z_{t,j} * z_{t+Δ,i} | |
| 4. Projection back: g_t = W_plu @ p_hat + b_plu | |
| 5. Gated fusion: h_mix = α ⊙ h + (1-α) ⊙ g | |
| 6. Feed-forward block: standard FFN with GELU | |
| """ | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| from .config import GrassmannConfig | |
| class PluckerEncoder(nn.Module): | |
| """Computes Plücker coordinates for pairs of reduced vectors.""" | |
| def __init__(self, r: int, eps: float = 1e-8): | |
| super().__init__() | |
| self.r = r | |
| self.eps = eps | |
| # Precompute index pairs (i, j) where i < j | |
| idx_i, idx_j = [], [] | |
| for i in range(r): | |
| for j in range(i + 1, r): | |
| idx_i.append(i) | |
| idx_j.append(j) | |
| self.register_buffer("idx_i", torch.tensor(idx_i, dtype=torch.long)) | |
| self.register_buffer("idx_j", torch.tensor(idx_j, dtype=torch.long)) | |
| def forward(self, z_t: torch.Tensor, z_delta: torch.Tensor) -> torch.Tensor: | |
| """ | |
| Compute normalized Plücker coordinates. | |
| Args: | |
| z_t: (batch, seq, r) - reduced states at position t | |
| z_delta: (batch, seq, r) - reduced states at position t+Δ | |
| Returns: | |
| (batch, seq, C(r,2)) - normalized Plücker vectors | |
| """ | |
| # p_{ij} = z_{t,i} * z_{Δ,j} - z_{t,j} * z_{Δ,i} | |
| p = ( | |
| z_t[..., self.idx_i] * z_delta[..., self.idx_j] | |
| - z_t[..., self.idx_j] * z_delta[..., self.idx_i] | |
| ) | |
| # Normalize for numerical stability | |
| norm = p.norm(dim=-1, keepdim=True).clamp(min=self.eps) | |
| return p / norm | |
| class GrassmannMixer(nn.Module): | |
| """ | |
| Grassmann/Plücker mixer — the core sequence interaction. | |
| Supports: | |
| - Causal (one-sided) or bidirectional (two-sided) mixing | |
| - Mean-first (anchored) form: 3.5x faster, mathematically exact | |
| - Standard (as-written) form: project then average | |
| Two-sided mixing is appropriate for CTC/encoder mode since speech | |
| isn't generated left-to-right. Antisymmetry of Plücker coordinates | |
| naturally distinguishes past from future even in two-sided mode. | |
| """ | |
| def __init__(self, d: int, r: int, offsets: list[int], | |
| two_sided: bool = False, eps: float = 1e-8): | |
| super().__init__() | |
| self.offsets = offsets | |
| self.two_sided = two_sided | |
| self.eps = eps | |
| plucker_dim = r * (r - 1) // 2 | |
| self.W_red = nn.Linear(d, r) | |
| self.plucker = PluckerEncoder(r, eps=eps) | |
| self.W_plu = nn.Linear(plucker_dim, d) | |
| def forward(self, h: torch.Tensor) -> torch.Tensor: | |
| """ | |
| Mean-first (anchored) form: average partners before projection. | |
| Mathematically equivalent to as-written, but ~3.5x faster on GPU | |
| because it does one W_plu matmul instead of one per offset. | |
| """ | |
| B, L, D = h.shape | |
| z = self.W_red(h) # (B, L, r) | |
| # Accumulate weighted partner vectors, then compute Plücker once | |
| partner = torch.zeros_like(z) # (B, L, r) | |
| count = z.new_zeros(1, L, 1) | |
| for delta in self.offsets: | |
| if delta >= L: | |
| continue | |
| earlier = z[:, :L - delta, :] | |
| later = z[:, delta:, :] | |
| # Compute norm for weighting (from dot products, numerically stable) | |
| p = self.plucker(earlier, later) | |
| w = p.norm(dim=-1, keepdim=True).clamp(min=self.eps).reciprocal() | |
| # Assign to later position (causal: t+Δ sees t) | |
| partner[:, delta:] += w * earlier | |
| count[:, delta:] += 1 | |
| if self.two_sided: | |
| # Also let earlier position see later (bidirectional) | |
| # Plücker antisymmetry (flip sign) distinguishes direction | |
| partner[:, :L - delta] += -w * later | |
| count[:, :L - delta] += 1 | |
| # Average partners, compute Plücker features, project | |
| count = count.clamp(min=1) | |
| avg_partner = partner / count | |
| # Compute final Plücker between each position and its avg partner | |
| p_final = self.plucker(z, avg_partner) | |
| return self.W_plu(p_final) | |
| class CausalGrassmannLayer(nn.Module): | |
| """ | |
| One layer of the Causal Grassmann Transformer. | |
| Replaces the self-attention block with Grassmann/Plücker mixing, | |
| followed by the standard position-wise FFN. | |
| Supports two_sided mode for encoder/CTC use cases. | |
| """ | |
| def __init__(self, config: GrassmannConfig, layer_idx: int = 0): | |
| super().__init__() | |
| d = config.d_model | |
| r = config.r_reduced | |
| two_sided = getattr(config, "two_sided", False) | |
| # Grassmann mixer | |
| self.mixer = GrassmannMixer( | |
| d, r, config.window_offsets, | |
| two_sided=two_sided, eps=config.plucker_eps, | |
| ) | |
| # Gated fusion | |
| self.W_gate = nn.Linear(2 * d, d) | |
| # Layer norm after mixing | |
| self.norm_mix = nn.LayerNorm(d) | |
| self.dropout_mix = nn.Dropout(config.dropout) | |
| # Feed-forward block | |
| self.ffn = nn.Sequential( | |
| nn.Linear(d, config.d_ff), | |
| nn.GELU(), | |
| nn.Dropout(config.dropout), | |
| nn.Linear(config.d_ff, d), | |
| nn.Dropout(config.dropout), | |
| ) | |
| self.norm_ffn = nn.LayerNorm(d) | |
| def forward(self, h: torch.Tensor) -> torch.Tensor: | |
| """ | |
| Args: | |
| h: (batch, seq_len, d_model) | |
| Returns: | |
| h': (batch, seq_len, d_model) | |
| """ | |
| # Grassmann mixing | |
| g = self.mixer(h) # (B, L, D) | |
| # Gated fusion: (1-α)(g-h) as residual (Dhruv's variant) | |
| u = torch.cat([h, g], dim=-1) | |
| alpha = torch.sigmoid(self.W_gate(u)) | |
| h_mix = h + (1 - alpha) * (g - h) | |
| # Layer norm + dropout + residual | |
| h_hat = self.norm_mix(h + self.dropout_mix(h_mix)) | |
| # Feed-forward with residual | |
| h_out = self.norm_ffn(h_hat + self.ffn(h_hat)) | |
| return h_out | |