Spaces:
Running
Running
File size: 6,314 Bytes
d7013e6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 | """
Causal Grassmann Mixing Layer.
Implements the core architecture from "Attention Is Not What You Need"
(arXiv:2512.19428). Replaces self-attention with:
1. Linear reduction: h_t ∈ R^d → z_t ∈ R^r
2. Multi-scale pairing: (z_t, z_{t+Δ}) for Δ ∈ W
3. Plücker encoding: p_{ij} = z_{t,i} * z_{t+Δ,j} - z_{t,j} * z_{t+Δ,i}
4. Projection back: g_t = W_plu @ p_hat + b_plu
5. Gated fusion: h_mix = α ⊙ h + (1-α) ⊙ g
6. Feed-forward block: standard FFN with GELU
"""
import torch
import torch.nn as nn
import torch.nn.functional as F
from .config import GrassmannConfig
class PluckerEncoder(nn.Module):
"""Computes Plücker coordinates for pairs of reduced vectors."""
def __init__(self, r: int, eps: float = 1e-8):
super().__init__()
self.r = r
self.eps = eps
# Precompute index pairs (i, j) where i < j
idx_i, idx_j = [], []
for i in range(r):
for j in range(i + 1, r):
idx_i.append(i)
idx_j.append(j)
self.register_buffer("idx_i", torch.tensor(idx_i, dtype=torch.long))
self.register_buffer("idx_j", torch.tensor(idx_j, dtype=torch.long))
def forward(self, z_t: torch.Tensor, z_delta: torch.Tensor) -> torch.Tensor:
"""
Compute normalized Plücker coordinates.
Args:
z_t: (batch, seq, r) - reduced states at position t
z_delta: (batch, seq, r) - reduced states at position t+Δ
Returns:
(batch, seq, C(r,2)) - normalized Plücker vectors
"""
# p_{ij} = z_{t,i} * z_{Δ,j} - z_{t,j} * z_{Δ,i}
p = (
z_t[..., self.idx_i] * z_delta[..., self.idx_j]
- z_t[..., self.idx_j] * z_delta[..., self.idx_i]
)
# Normalize for numerical stability
norm = p.norm(dim=-1, keepdim=True).clamp(min=self.eps)
return p / norm
class GrassmannMixer(nn.Module):
"""
Grassmann/Plücker mixer — the core sequence interaction.
Supports:
- Causal (one-sided) or bidirectional (two-sided) mixing
- Mean-first (anchored) form: 3.5x faster, mathematically exact
- Standard (as-written) form: project then average
Two-sided mixing is appropriate for CTC/encoder mode since speech
isn't generated left-to-right. Antisymmetry of Plücker coordinates
naturally distinguishes past from future even in two-sided mode.
"""
def __init__(self, d: int, r: int, offsets: list[int],
two_sided: bool = False, eps: float = 1e-8):
super().__init__()
self.offsets = offsets
self.two_sided = two_sided
self.eps = eps
plucker_dim = r * (r - 1) // 2
self.W_red = nn.Linear(d, r)
self.plucker = PluckerEncoder(r, eps=eps)
self.W_plu = nn.Linear(plucker_dim, d)
def forward(self, h: torch.Tensor) -> torch.Tensor:
"""
Mean-first (anchored) form: average partners before projection.
Mathematically equivalent to as-written, but ~3.5x faster on GPU
because it does one W_plu matmul instead of one per offset.
"""
B, L, D = h.shape
z = self.W_red(h) # (B, L, r)
# Accumulate weighted partner vectors, then compute Plücker once
partner = torch.zeros_like(z) # (B, L, r)
count = z.new_zeros(1, L, 1)
for delta in self.offsets:
if delta >= L:
continue
earlier = z[:, :L - delta, :]
later = z[:, delta:, :]
# Compute norm for weighting (from dot products, numerically stable)
p = self.plucker(earlier, later)
w = p.norm(dim=-1, keepdim=True).clamp(min=self.eps).reciprocal()
# Assign to later position (causal: t+Δ sees t)
partner[:, delta:] += w * earlier
count[:, delta:] += 1
if self.two_sided:
# Also let earlier position see later (bidirectional)
# Plücker antisymmetry (flip sign) distinguishes direction
partner[:, :L - delta] += -w * later
count[:, :L - delta] += 1
# Average partners, compute Plücker features, project
count = count.clamp(min=1)
avg_partner = partner / count
# Compute final Plücker between each position and its avg partner
p_final = self.plucker(z, avg_partner)
return self.W_plu(p_final)
class CausalGrassmannLayer(nn.Module):
"""
One layer of the Causal Grassmann Transformer.
Replaces the self-attention block with Grassmann/Plücker mixing,
followed by the standard position-wise FFN.
Supports two_sided mode for encoder/CTC use cases.
"""
def __init__(self, config: GrassmannConfig, layer_idx: int = 0):
super().__init__()
d = config.d_model
r = config.r_reduced
two_sided = getattr(config, "two_sided", False)
# Grassmann mixer
self.mixer = GrassmannMixer(
d, r, config.window_offsets,
two_sided=two_sided, eps=config.plucker_eps,
)
# Gated fusion
self.W_gate = nn.Linear(2 * d, d)
# Layer norm after mixing
self.norm_mix = nn.LayerNorm(d)
self.dropout_mix = nn.Dropout(config.dropout)
# Feed-forward block
self.ffn = nn.Sequential(
nn.Linear(d, config.d_ff),
nn.GELU(),
nn.Dropout(config.dropout),
nn.Linear(config.d_ff, d),
nn.Dropout(config.dropout),
)
self.norm_ffn = nn.LayerNorm(d)
def forward(self, h: torch.Tensor) -> torch.Tensor:
"""
Args:
h: (batch, seq_len, d_model)
Returns:
h': (batch, seq_len, d_model)
"""
# Grassmann mixing
g = self.mixer(h) # (B, L, D)
# Gated fusion: (1-α)(g-h) as residual (Dhruv's variant)
u = torch.cat([h, g], dim=-1)
alpha = torch.sigmoid(self.W_gate(u))
h_mix = h + (1 - alpha) * (g - h)
# Layer norm + dropout + residual
h_hat = self.norm_mix(h + self.dropout_mix(h_mix))
# Feed-forward with residual
h_out = self.norm_ffn(h_hat + self.ffn(h_hat))
return h_out
|