Spaces:
Running
Running
File size: 6,083 Bytes
d7013e6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 | """
Ablation Mixing Layer — Symmetric Product Baseline.
Drop-in replacement for the Grassmann/Plücker mixer that uses plain
element-wise symmetric products instead of antisymmetric Plücker
coordinates. Everything else (linear reduction, multi-scale offsets,
mean-first anchoring, gated fusion, FFN) stays identical so the only
variable is whether the Grassmann geometry matters.
Matches Dhruv's grassmann-speech implementation:
- symmetric_products: a_i * b_j + a_j * b_i (vs Plücker: a_i * b_j - a_j * b_i)
- symmetric_norm_sq: |a|²|b|² + (a·b)² - 2·Σ(aᵢ²bᵢ²) (vs Lagrange: |a|²|b|² - (a·b)²)
- flip = +1.0 in two-sided mode (vs Plücker: flip = -1.0)
"""
import torch
import torch.nn as nn
from .config import GrassmannConfig
class SymmetricEncoder(nn.Module):
"""
Computes element-wise symmetric products for pairs of reduced vectors.
Same index pairs and output dimensionality as PluckerEncoder,
but uses p_{ij} = z_{t,i} * z_{Δ,j} + z_{t,j} * z_{Δ,i}
instead of the antisymmetric Plücker form.
"""
def __init__(self, r: int, eps: float = 1e-8):
super().__init__()
self.r = r
self.eps = eps
idx_i, idx_j = [], []
for i in range(r):
for j in range(i + 1, r):
idx_i.append(i)
idx_j.append(j)
self.register_buffer("idx_i", torch.tensor(idx_i, dtype=torch.long))
self.register_buffer("idx_j", torch.tensor(idx_j, dtype=torch.long))
def forward(self, z_t: torch.Tensor, z_delta: torch.Tensor) -> torch.Tensor:
"""
Compute normalized symmetric products.
Args:
z_t: (batch, seq, r)
z_delta: (batch, seq, r)
Returns:
(batch, seq, C(r,2))
"""
# Symmetric: p_{ij} = z_{t,i} * z_{Δ,j} + z_{t,j} * z_{Δ,i}
p = (
z_t[..., self.idx_i] * z_delta[..., self.idx_j]
+ z_t[..., self.idx_j] * z_delta[..., self.idx_i]
)
norm = p.norm(dim=-1, keepdim=True).clamp(min=self.eps)
return p / norm
def _symmetric_norm_sq(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor:
"""
Squared norm of symmetric products from dot products alone.
Matches grassmann-speech/plucker.py symmetric_norm_sq:
|a|²|b|² + (a·b)² - 2·Σ(aᵢ²·bᵢ²)
"""
aa = (a * a).sum(-1, keepdim=True)
bb = (b * b).sum(-1, keepdim=True)
ab = (a * b).sum(-1, keepdim=True)
return aa * bb + ab * ab - 2 * (a * a * b * b).sum(-1, keepdim=True)
class AblationMixer(nn.Module):
"""
Symmetric-product mixer — ablation baseline for GrassmannMixer.
Identical architecture (linear reduction → multi-scale pairing →
encoding → mean-first anchoring → projection) but with symmetric
products replacing Plücker coordinates.
Key differences from GrassmannMixer (matching grassmann-speech):
1. Symmetric products instead of Plücker coordinates
2. symmetric_norm_sq for anchored weighting
3. flip = +1.0 in two-sided mode (symmetric products are symmetric
under swap, so no sign flip — use positive accumulation)
"""
def __init__(self, d: int, r: int, offsets: list[int],
two_sided: bool = False, eps: float = 1e-8):
super().__init__()
self.offsets = offsets
self.two_sided = two_sided
self.eps = eps
plucker_dim = r * (r - 1) // 2 # same dimensionality
self.W_red = nn.Linear(d, r)
self.encoder = SymmetricEncoder(r, eps=eps)
self.W_plu = nn.Linear(plucker_dim, d)
def forward(self, h: torch.Tensor) -> torch.Tensor:
"""Mean-first (anchored) form, matching grassmann-speech."""
B, L, D = h.shape
z = self.W_red(h)
partner = torch.zeros_like(z)
count = z.new_zeros(1, L, 1)
for delta in self.offsets:
if delta >= L:
continue
earlier = z[:, :L - delta, :]
later = z[:, delta:, :]
# Weight by symmetric norm (from dot products, no coordinate expansion)
w = _symmetric_norm_sq(earlier, later).clamp_min(0).sqrt().clamp_min(self.eps).reciprocal()
partner[:, delta:] += w * earlier
count[:, delta:] += 1
if self.two_sided:
# flip = +1.0 for symmetric (vs -1.0 for Plücker)
# Symmetric products are invariant under swap(a,b),
# so no sign flip needed to distinguish direction
partner[:, :L - delta] += w * later
count[:, :L - delta] += 1
count = count.clamp(min=1)
avg_partner = partner / count
p_final = self.encoder(z, avg_partner)
return self.W_plu(p_final)
class AblationLayer(nn.Module):
"""
Drop-in replacement for CausalGrassmannLayer using symmetric products.
Same gated fusion + FFN, just swaps the mixer.
"""
def __init__(self, config: GrassmannConfig, layer_idx: int = 0):
super().__init__()
d = config.d_model
r = config.r_reduced
two_sided = getattr(config, "two_sided", False)
self.mixer = AblationMixer(
d, r, config.window_offsets,
two_sided=two_sided, eps=config.plucker_eps,
)
self.W_gate = nn.Linear(2 * d, d)
self.norm_mix = nn.LayerNorm(d)
self.dropout_mix = nn.Dropout(config.dropout)
self.ffn = nn.Sequential(
nn.Linear(d, config.d_ff),
nn.GELU(),
nn.Dropout(config.dropout),
nn.Linear(config.d_ff, d),
nn.Dropout(config.dropout),
)
self.norm_ffn = nn.LayerNorm(d)
def forward(self, h: torch.Tensor) -> torch.Tensor:
g = self.mixer(h)
u = torch.cat([h, g], dim=-1)
alpha = torch.sigmoid(self.W_gate(u))
h_mix = h + (1 - alpha) * (g - h)
h_hat = self.norm_mix(h + self.dropout_mix(h_mix))
h_out = self.norm_ffn(h_hat + self.ffn(h_hat))
return h_out
|