grassmann-ablation / src /models /grassmann_layer.py
Nicholas Finch
feat: checkpoint persistence — survives container restarts
a7452cc
Raw History Blame Contribute Delete
6.31 kB
"""
Causal Grassmann Mixing Layer.
Implements the core architecture from "Attention Is Not What You Need"
(arXiv:2512.19428). Replaces self-attention with:
1. Linear reduction: h_t ∈ R^d → z_t ∈ R^r
2. Multi-scale pairing: (z_t, z_{t+Δ}) for Δ ∈ W
3. Plücker encoding: p_{ij} = z_{t,i} * z_{t+Δ,j} - z_{t,j} * z_{t+Δ,i}
4. Projection back: g_t = W_plu @ p_hat + b_plu
5. Gated fusion: h_mix = α ⊙ h + (1-α) ⊙ g
6. Feed-forward block: standard FFN with GELU
"""
import torch
import torch.nn as nn
import torch.nn.functional as F
from .config import GrassmannConfig
class PluckerEncoder(nn.Module):
"""Computes Plücker coordinates for pairs of reduced vectors."""
def __init__(self, r: int, eps: float = 1e-8):
super().__init__()
self.r = r
self.eps = eps
# Precompute index pairs (i, j) where i < j
idx_i, idx_j = [], []
for i in range(r):
for j in range(i + 1, r):
idx_i.append(i)
idx_j.append(j)
self.register_buffer("idx_i", torch.tensor(idx_i, dtype=torch.long))
self.register_buffer("idx_j", torch.tensor(idx_j, dtype=torch.long))
def forward(self, z_t: torch.Tensor, z_delta: torch.Tensor) -> torch.Tensor:
"""
Compute normalized Plücker coordinates.
Args:
z_t: (batch, seq, r) - reduced states at position t
z_delta: (batch, seq, r) - reduced states at position t+Δ
Returns:
(batch, seq, C(r,2)) - normalized Plücker vectors
"""
# p_{ij} = z_{t,i} * z_{Δ,j} - z_{t,j} * z_{Δ,i}
p = (
z_t[..., self.idx_i] * z_delta[..., self.idx_j]
- z_t[..., self.idx_j] * z_delta[..., self.idx_i]
)
# Normalize for numerical stability
norm = p.norm(dim=-1, keepdim=True).clamp(min=self.eps)
return p / norm
class GrassmannMixer(nn.Module):
"""
Grassmann/Plücker mixer — the core sequence interaction.
Supports:
- Causal (one-sided) or bidirectional (two-sided) mixing
- Mean-first (anchored) form: 3.5x faster, mathematically exact
- Standard (as-written) form: project then average
Two-sided mixing is appropriate for CTC/encoder mode since speech
isn't generated left-to-right. Antisymmetry of Plücker coordinates
naturally distinguishes past from future even in two-sided mode.
"""
def __init__(self, d: int, r: int, offsets: list[int],
two_sided: bool = False, eps: float = 1e-8):
super().__init__()
self.offsets = offsets
self.two_sided = two_sided
self.eps = eps
plucker_dim = r * (r - 1) // 2
self.W_red = nn.Linear(d, r)
self.plucker = PluckerEncoder(r, eps=eps)
self.W_plu = nn.Linear(plucker_dim, d)
def forward(self, h: torch.Tensor) -> torch.Tensor:
"""
Mean-first (anchored) form: average partners before projection.
Mathematically equivalent to as-written, but ~3.5x faster on GPU
because it does one W_plu matmul instead of one per offset.
"""
B, L, D = h.shape
z = self.W_red(h) # (B, L, r)
# Accumulate weighted partner vectors, then compute Plücker once
partner = torch.zeros_like(z) # (B, L, r)
count = z.new_zeros(1, L, 1)
for delta in self.offsets:
if delta >= L:
continue
earlier = z[:, :L - delta, :]
later = z[:, delta:, :]
# Compute norm for weighting (from dot products, numerically stable)
p = self.plucker(earlier, later)
w = p.norm(dim=-1, keepdim=True).clamp(min=self.eps).reciprocal()
# Assign to later position (causal: t+Δ sees t)
partner[:, delta:] += w * earlier
count[:, delta:] += 1
if self.two_sided:
# Also let earlier position see later (bidirectional)
# Plücker antisymmetry (flip sign) distinguishes direction
partner[:, :L - delta] += -w * later
count[:, :L - delta] += 1
# Average partners, compute Plücker features, project
count = count.clamp(min=1)
avg_partner = partner / count
# Compute final Plücker between each position and its avg partner
p_final = self.plucker(z, avg_partner)
return self.W_plu(p_final)
class CausalGrassmannLayer(nn.Module):
"""
One layer of the Causal Grassmann Transformer.
Replaces the self-attention block with Grassmann/Plücker mixing,
followed by the standard position-wise FFN.
Supports two_sided mode for encoder/CTC use cases.
"""
def __init__(self, config: GrassmannConfig, layer_idx: int = 0):
super().__init__()
d = config.d_model
r = config.r_reduced
two_sided = getattr(config, "two_sided", False)
# Grassmann mixer
self.mixer = GrassmannMixer(
d, r, config.window_offsets,
two_sided=two_sided, eps=config.plucker_eps,
)
# Gated fusion
self.W_gate = nn.Linear(2 * d, d)
# Layer norm after mixing
self.norm_mix = nn.LayerNorm(d)
self.dropout_mix = nn.Dropout(config.dropout)
# Feed-forward block
self.ffn = nn.Sequential(
nn.Linear(d, config.d_ff),
nn.GELU(),
nn.Dropout(config.dropout),
nn.Linear(config.d_ff, d),
nn.Dropout(config.dropout),
)
self.norm_ffn = nn.LayerNorm(d)
def forward(self, h: torch.Tensor) -> torch.Tensor:
"""
Args:
h: (batch, seq_len, d_model)
Returns:
h': (batch, seq_len, d_model)
"""
# Grassmann mixing
g = self.mixer(h) # (B, L, D)
# Gated fusion: (1-α)(g-h) as residual (Dhruv's variant)
u = torch.cat([h, g], dim=-1)
alpha = torch.sigmoid(self.W_gate(u))
h_mix = h + (1 - alpha) * (g - h)
# Layer norm + dropout + residual
h_hat = self.norm_mix(h + self.dropout_mix(h_mix))
# Feed-forward with residual
h_out = self.norm_ffn(h_hat + self.ffn(h_hat))
return h_out