"""Input-dependent GLA kernels (G=1 per-head scalar, and grouped G>1) + the gate module. Parameterization (fixed after the Lead's review; the charter's `g=log σ(W_g x+b)` with `b=log γ` is inconsistent because σ(log γ)≠γ): z = W2 (W1 x) + b low-rank bottleneck, W1: d→r, W2: r→H·G, b ∈ R^{H·G} g = min(z, 0) log-decay, fp32 exp(g) = decay per token per head/group Function-preserving init: W2 = 0, b = log γ_h (per group) => g = log γ => exp(g) = γ, exactly the scalar kernel's decay (the fast kernel also computes exp(log γ · Δ)). Identity measured: max|Δlogits| = 1.526e-05 (fp32 CPU, 20 layers converted, real tokens). If one insists on the log-σ form, the bias must be logit(γ), not log γ; this file uses the direct log-decay form. Numerics: gates, cumulative sums, masks and the scan matrices stay **fp32**; only the mask *product* with qk and the output are cast to the activation dtype (bf16 in training). A bf16 mask is wrong: the relative mask error reaches ~3 % at C=128 in bf16. Variants: * G=1 — **GLA-S**: one input-dependent scalar decay per head (cheapest; grouped GLA with one group of all 64 key channels; NOT full per-channel GLA). Kernel: single [B,H,nc,C,C] mask, 1 qk matmul. DELIVERED and measured: +14.7 % end-to-end fp32-gate eager upper bound (B), identity 1.5e-5. * G>1 — grouped/per-channel: NOT DELIVERED (A's corrected design still costs +647 %/layer); the code raises NotImplementedError by design - no crashing path and no priced G=8 branch without a tested production kernel. Math: Yang et al., Gated Linear Attention (arXiv:2312.06635), chunkwise form with the cumulative-gate mask; here C=128 single-level, exact (not the sub-chunk approximation) because the gate is computed per token before the scan. All exponents are differences of cumulative log-decays in the safe direction, so no exp(+large) overflow appears even for γ=0.1. Scope: **chunkwise training/evaluation + recurrent decode + chunkwise prefill** are implemented. `MultiScaleRetention` routes `mode="chunkwise"` through this kernel and, when the recurrent GLA path is enabled, `mode="recurrent"` through `gla_recurrent`; `RetNetForCausalLM.generate` uses chunkwise prefill for GLA models and recurrent decode afterwards. The parallel mode remains scalar, but it is not used by the GLA serving path. `gla_recurrent` is a per-token loop, not a fast batched decode kernel. """ from __future__ import annotations import torch class HeadGate(torch.nn.Module): """Low-rank input-dependent log-decay gate; parameters stay fp32 under a bf16 model.""" def __init__(self, d_model: int, n_heads: int, rank: int = 8, groups: int = 1, gamma: torch.Tensor | None = None): super().__init__() self.d_model, self.n_heads, self.rank, self.groups = d_model, n_heads, rank, groups self.w1 = torch.nn.Linear(d_model, rank, bias=False).float() self.w2 = torch.nn.Linear(rank, n_heads * groups, bias=True).float() torch.nn.init.normal_(self.w1.weight, std=0.02) torch.nn.init.zeros_(self.w2.weight) gam = gamma.detach().float().clamp(1e-4, 1 - 1e-6) if gamma is not None \ else torch.full((n_heads,), 0.9) with torch.no_grad(): self.w2.bias.copy_(torch.log(gam).repeat_interleave(groups)) def forward(self, x): # x [B,L,d] -> fp32 log decay [B,L,H,G] # FP32 parameters alone do not prevent AMP from downcasting Linear. with torch.autocast(device_type=x.device.type, enabled=False): z = self.w2(self.w1(x.float())) return torch.clamp(z, max=0.0) def extra_params(self) -> int: return sum(p.numel() for p in self.parameters()) @torch.no_grad() def init_gate_from_gamma(gate: HeadGate, gamma: torch.Tensor) -> None: g = gamma.detach().float().clamp(1e-4, 1 - 1e-6) gate.w2.weight.zero_() gate.w2.bias.copy_(torch.log(g).repeat_interleave(gate.groups)) def _split(q, k, v, chunk_size): B, H, L, kd = q.shape C = int(chunk_size) assert L % C == 0, (L, C) return B, H, L, kd, v.shape[-1], L // C, C def _causal(C, dev): ar = torch.arange(C, device=dev) return ar[:, None] >= ar[None, :] def _scan_tables(Glast, nc, dev): """Per-(batch,head,group) data-dependent scan matrices: T[i,j]=d^(i-1-j), T2[i,j]=d^(i-j), j torch.arange(nc, device=dev)[None, :] diff2 = torch.where(valid2, diff2, torch.zeros((), device=dev, dtype=diff2.dtype)) T2 = torch.exp(diff2) * valid2.to(diff2.dtype) return T, T2, Ce, Ce2 @torch.no_grad() def _cast(x, dt): return x.to(dt) def _gla_chunkwise_exact(q, k, v, g_t, state, chunk_size: int): """Chunkwise GLA with input-dependent grouped decay. q,k: [B,H,L,kd]; v: [B,H,L,vd]; g_t: [B,L,H] (G=1) or [B,L,H,G] fp32 log-decay ≤0. Returns (out [B,H,L,vd], new_state [B,H,kd,vd]). """ B, H, L, kd, vd, nc, C = _split(q, k, v, chunk_size) dev, dt = q.device, q.dtype G_ = g_t.shape[-1] if g_t.dim() == 4 else 1 if G_ == 1: g = (g_t[..., 0] if g_t.dim() == 4 else g_t).transpose(1, 2) # [B,H,L] g = g.reshape(B, H, nc, C).float() qr = q.view(B, H, nc, C, kd) kr = k.view(B, H, nc, C, kd) vr = v.view(B, H, nc, C, vd) Gc = torch.cumsum(g, dim=-1) # [B,H,nc,C] causal = _causal(C, dev) delta = Gc[..., :, None] - Gc[..., None, :] delta = torch.where(causal, delta, torch.zeros((), device=dev)) mask = (torch.exp(delta) * causal.to(delta.dtype)).to(dt) qk = torch.matmul(qr, kr.transpose(-1, -2)) inner = torch.matmul(qk * mask, vr) Glast = Gc[..., -1] val_decay = torch.exp(Glast[..., None] - Gc) # [B,H,nc,C] U = torch.matmul(kr.transpose(-1, -2), (vr.float() * val_decay[..., None]).to(dt)) Uflat = U.reshape(B, H, nc, kd * vd).float() T, T2, Ce, Ce2 = _scan_tables(Glast, nc, dev) S_prev = torch.matmul(T, Uflat).view(B, H, nc, kd, vd).to(dt) S_all = torch.matmul(T2, Uflat).view(B, H, nc + 1, kd, vd).to(dt) if state is not None: st = state.reshape(B, H, 1, kd, vd) S_prev = S_prev + torch.exp(Ce)[..., None, None].to(dt) * st S_all = S_all + torch.exp(Ce2)[..., None, None].to(dt) * st cross = torch.matmul((qr.float() * torch.exp(Gc)[..., None]).to(dt), S_prev) return (inner + cross).reshape(B, H, L, vd), S_all[:, :, nc] # ---------------- grouped variant G>1: NOT DELIVERED ---------------- # The grouped kernel is not shipped in this round: the first prototype's einsum/permute ordering # was wrong (A B5 / B B3 measured the crash) and a corrected design still costs +647 %/layer # (A's corrected vectorised prototype: 16.928 ms vs 2.265 ms) => ~+97 % end-to-end at B=1. # Raising loudly beats shipping a path that silently computes a different function. raise NotImplementedError( "gla_chunkwise: grouped G>1 is not delivered (needs a tested production kernel; " "see GLA_PLAN_v2 §3/§9). Use groups=1 (GLA-S) or the scalar fast kernel.") def gla_chunkwise(q, k, v, g_t, state, chunk_size: int): """Padded wrapper: supports prompt lengths not divisible by the chunk size. Zero k/v/g at pad positions leave the recurrent state unchanged; the returned out is trimmed. """ B, H, L, kd = q.shape vd = v.shape[-1] C = int(chunk_size) pad = (-L) % C if pad == 0: return _gla_chunkwise_exact(q, k, v, g_t, state, C) qp = torch.nn.functional.pad(q, (0, 0, 0, pad)) kp = torch.nn.functional.pad(k, (0, 0, 0, pad)) vp = torch.nn.functional.pad(v, (0, 0, 0, pad)) if g_t.dim() == 4: gp = torch.nn.functional.pad(g_t, (0, 0, 0, pad)) else: gp = torch.nn.functional.pad(g_t, (0, 0, 0, pad)) out, new_state = _gla_chunkwise_exact(qp, kp, vp, gp, state, C) return out[:, :, :L], new_state def gla_recurrent(q, k, v, g_t, state=None): return gla_recurrent_reference(q, k, v, g_t, state) def gla_recurrent_reference(q, k, v, g_t, state=None): """Slow decode reference (G=1), with fp32 decay and persistent state.""" B, H, L, kd = q.shape if (g_t.dim() == 4 and g_t.shape[-1] > 1): raise NotImplementedError("recurrent reference implemented for G=1 only") g = (g_t[..., 0] if g_t.dim() == 4 else g_t).transpose(1, 2).float() # exp(g) near one must never pass through bf16: that would erase slow # decay. Promote a chunkwise-prefill state once and keep subsequent # updates in fp32; this cannot recover precision lost during prefill. output_dtype = q.dtype q, k, v = q.float(), k.float(), v.float() if state is None: state = torch.zeros(B, H, kd, v.shape[-1], device=q.device, dtype=torch.float32) else: state = state.float() outs = [] for t in range(L): state = torch.exp(g[:, :, t])[..., None, None] * state + \ k[:, :, t, :].unsqueeze(-1) * v[:, :, t, :].unsqueeze(-2) outs.append(torch.sum(q[:, :, t, :].unsqueeze(-1) * state, dim=-2)) return torch.stack(outs, dim=2).to(output_dtype), state def count_params(d_model=1408, n_heads=22, n_layers=20, rank=8, groups=1): per = rank * d_model + rank * n_heads * groups + n_heads * groups total = per * n_layers return per, total, 100.0 * total / 504_179_896 if __name__ == "__main__": for r, gp in ((4, 1), (8, 1), (16, 1), (8, 4), (8, 8)): per, total, pct = count_params(rank=r, groups=gp) print(f"low-rank rank={r:2d} G={gp}: {per:6,}/layer {total:9,} total {pct:.4f}%") per = 1408 * 22 + 22 print(f"per-head linear (no bottleneck): {per:,}/layer {per * 20:,} total " f"{100 * per * 20 / 504179896:.3f}%") full = 20 * (1408 * 1408 + 1408) print(f"full d->H*kd: {full:,} total {100 * full / 504179896:.2f}%")