grassmann-ablation / src /models /ablation_layer.py
Nicholas Finch
feat: checkpoint persistence — survives container restarts
a7452cc
Raw History Blame Contribute Delete
6.08 kB
"""
Ablation Mixing Layer — Symmetric Product Baseline.
Drop-in replacement for the Grassmann/Plücker mixer that uses plain
element-wise symmetric products instead of antisymmetric Plücker
coordinates. Everything else (linear reduction, multi-scale offsets,
mean-first anchoring, gated fusion, FFN) stays identical so the only
variable is whether the Grassmann geometry matters.
Matches Dhruv's grassmann-speech implementation:
- symmetric_products: a_i * b_j + a_j * b_i (vs Plücker: a_i * b_j - a_j * b_i)
- symmetric_norm_sq: |a|²|b|² + (a·b)² - 2·Σ(aᵢ²bᵢ²) (vs Lagrange: |a|²|b|² - (a·b)²)
- flip = +1.0 in two-sided mode (vs Plücker: flip = -1.0)
"""
import torch
import torch.nn as nn
from .config import GrassmannConfig
class SymmetricEncoder(nn.Module):
"""
Computes element-wise symmetric products for pairs of reduced vectors.
Same index pairs and output dimensionality as PluckerEncoder,
but uses p_{ij} = z_{t,i} * z_{Δ,j} + z_{t,j} * z_{Δ,i}
instead of the antisymmetric Plücker form.
"""
def __init__(self, r: int, eps: float = 1e-8):
super().__init__()
self.r = r
self.eps = eps
idx_i, idx_j = [], []
for i in range(r):
for j in range(i + 1, r):
idx_i.append(i)
idx_j.append(j)
self.register_buffer("idx_i", torch.tensor(idx_i, dtype=torch.long))
self.register_buffer("idx_j", torch.tensor(idx_j, dtype=torch.long))
def forward(self, z_t: torch.Tensor, z_delta: torch.Tensor) -> torch.Tensor:
"""
Compute normalized symmetric products.
Args:
z_t: (batch, seq, r)
z_delta: (batch, seq, r)
Returns:
(batch, seq, C(r,2))
"""
# Symmetric: p_{ij} = z_{t,i} * z_{Δ,j} + z_{t,j} * z_{Δ,i}
p = (
z_t[..., self.idx_i] * z_delta[..., self.idx_j]
+ z_t[..., self.idx_j] * z_delta[..., self.idx_i]
)
norm = p.norm(dim=-1, keepdim=True).clamp(min=self.eps)
return p / norm
def _symmetric_norm_sq(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor:
"""
Squared norm of symmetric products from dot products alone.
Matches grassmann-speech/plucker.py symmetric_norm_sq:
|a|²|b|² + (a·b)² - 2·Σ(aᵢ²·bᵢ²)
"""
aa = (a * a).sum(-1, keepdim=True)
bb = (b * b).sum(-1, keepdim=True)
ab = (a * b).sum(-1, keepdim=True)
return aa * bb + ab * ab - 2 * (a * a * b * b).sum(-1, keepdim=True)
class AblationMixer(nn.Module):
"""
Symmetric-product mixer — ablation baseline for GrassmannMixer.
Identical architecture (linear reduction → multi-scale pairing →
encoding → mean-first anchoring → projection) but with symmetric
products replacing Plücker coordinates.
Key differences from GrassmannMixer (matching grassmann-speech):
1. Symmetric products instead of Plücker coordinates
2. symmetric_norm_sq for anchored weighting
3. flip = +1.0 in two-sided mode (symmetric products are symmetric
under swap, so no sign flip — use positive accumulation)
"""
def __init__(self, d: int, r: int, offsets: list[int],
two_sided: bool = False, eps: float = 1e-8):
super().__init__()
self.offsets = offsets
self.two_sided = two_sided
self.eps = eps
plucker_dim = r * (r - 1) // 2 # same dimensionality
self.W_red = nn.Linear(d, r)
self.encoder = SymmetricEncoder(r, eps=eps)
self.W_plu = nn.Linear(plucker_dim, d)
def forward(self, h: torch.Tensor) -> torch.Tensor:
"""Mean-first (anchored) form, matching grassmann-speech."""
B, L, D = h.shape
z = self.W_red(h)
partner = torch.zeros_like(z)
count = z.new_zeros(1, L, 1)
for delta in self.offsets:
if delta >= L:
continue
earlier = z[:, :L - delta, :]
later = z[:, delta:, :]
# Weight by symmetric norm (from dot products, no coordinate expansion)
w = _symmetric_norm_sq(earlier, later).clamp_min(0).sqrt().clamp_min(self.eps).reciprocal()
partner[:, delta:] += w * earlier
count[:, delta:] += 1
if self.two_sided:
# flip = +1.0 for symmetric (vs -1.0 for Plücker)
# Symmetric products are invariant under swap(a,b),
# so no sign flip needed to distinguish direction
partner[:, :L - delta] += w * later
count[:, :L - delta] += 1
count = count.clamp(min=1)
avg_partner = partner / count
p_final = self.encoder(z, avg_partner)
return self.W_plu(p_final)
class AblationLayer(nn.Module):
"""
Drop-in replacement for CausalGrassmannLayer using symmetric products.
Same gated fusion + FFN, just swaps the mixer.
"""
def __init__(self, config: GrassmannConfig, layer_idx: int = 0):
super().__init__()
d = config.d_model
r = config.r_reduced
two_sided = getattr(config, "two_sided", False)
self.mixer = AblationMixer(
d, r, config.window_offsets,
two_sided=two_sided, eps=config.plucker_eps,
)
self.W_gate = nn.Linear(2 * d, d)
self.norm_mix = nn.LayerNorm(d)
self.dropout_mix = nn.Dropout(config.dropout)
self.ffn = nn.Sequential(
nn.Linear(d, config.d_ff),
nn.GELU(),
nn.Dropout(config.dropout),
nn.Linear(config.d_ff, d),
nn.Dropout(config.dropout),
)
self.norm_ffn = nn.LayerNorm(d)
def forward(self, h: torch.Tensor) -> torch.Tensor:
g = self.mixer(h)
u = torch.cat([h, g], dim=-1)
alpha = torch.sigmoid(self.W_gate(u))
h_mix = h + (1 - alpha) * (g - h)
h_hat = self.norm_mix(h + self.dropout_mix(h_mix))
h_out = self.norm_ffn(h_hat + self.ffn(h_hat))
return h_out