RetNet-0.5B-v21.3-chat / gla_kernel.py
Ayaka157's picture
Add collection entry, fix trust_remote_code (auto_map + gla_kernel)
8a1b5b3 verified
Raw History Blame Contribute Delete
10.8 kB
"""Input-dependent GLA kernels (G=1 per-head scalar, and grouped G>1) + the gate module.
Parameterization (fixed after the Lead's review; the charter's `g=log σ(W_g x+b)` with `b=log γ` is
inconsistent because σ(log γ)≠γ):
z = W2 (W1 x) + b low-rank bottleneck, W1: d→r, W2: r→H·G, b ∈ R^{H·G}
g = min(z, 0) log-decay, fp32
exp(g) = decay per token per head/group
Function-preserving init: W2 = 0, b = log γ_h (per group) => g = log γ => exp(g) = γ, exactly the
scalar kernel's decay (the fast kernel also computes exp(log γ · Δ)). Identity measured: max|Δlogits| =
1.526e-05 (fp32 CPU, 20 layers converted, real tokens). If one insists on the log-σ form, the bias must
be logit(γ), not log γ; this file uses the direct log-decay form.
Numerics: gates, cumulative sums, masks and the scan matrices stay **fp32**; only the mask *product*
with qk and the output are cast to the activation dtype (bf16 in training). A bf16 mask is wrong: the
relative mask error reaches ~3 % at C=128 in bf16.
Variants:
* G=1 — **GLA-S**: one input-dependent scalar decay per head (cheapest; grouped GLA with one group of
all 64 key channels; NOT full per-channel GLA). Kernel: single [B,H,nc,C,C] mask, 1 qk matmul.
DELIVERED and measured: +14.7 % end-to-end fp32-gate eager upper bound (B), identity 1.5e-5.
* G>1 — grouped/per-channel: NOT DELIVERED (A's corrected design still costs +647 %/layer); the code
raises NotImplementedError by design - no crashing path and no priced G=8 branch without a tested
production kernel.
Math: Yang et al., Gated Linear Attention (arXiv:2312.06635), chunkwise form with the cumulative-gate
mask; here C=128 single-level, exact (not the sub-chunk approximation) because the gate is computed per
token before the scan. All exponents are differences of cumulative log-decays in the safe direction, so
no exp(+large) overflow appears even for γ=0.1.
Scope: **chunkwise training/evaluation + recurrent decode + chunkwise prefill** are implemented.
`MultiScaleRetention` routes `mode="chunkwise"` through this kernel and, when the recurrent GLA path is
enabled, `mode="recurrent"` through `gla_recurrent`; `RetNetForCausalLM.generate` uses chunkwise prefill
for GLA models and recurrent decode afterwards. The parallel mode remains scalar, but it is not used by
the GLA serving path. `gla_recurrent` is a per-token loop, not a fast batched decode kernel.
"""
from __future__ import annotations
import torch
class HeadGate(torch.nn.Module):
"""Low-rank input-dependent log-decay gate; parameters stay fp32 under a bf16 model."""
def __init__(self, d_model: int, n_heads: int, rank: int = 8, groups: int = 1,
gamma: torch.Tensor | None = None):
super().__init__()
self.d_model, self.n_heads, self.rank, self.groups = d_model, n_heads, rank, groups
self.w1 = torch.nn.Linear(d_model, rank, bias=False).float()
self.w2 = torch.nn.Linear(rank, n_heads * groups, bias=True).float()
torch.nn.init.normal_(self.w1.weight, std=0.02)
torch.nn.init.zeros_(self.w2.weight)
gam = gamma.detach().float().clamp(1e-4, 1 - 1e-6) if gamma is not None \
else torch.full((n_heads,), 0.9)
with torch.no_grad():
self.w2.bias.copy_(torch.log(gam).repeat_interleave(groups))
def forward(self, x): # x [B,L,d] -> fp32 log decay [B,L,H,G]
# FP32 parameters alone do not prevent AMP from downcasting Linear.
with torch.autocast(device_type=x.device.type, enabled=False):
z = self.w2(self.w1(x.float()))
return torch.clamp(z, max=0.0)
def extra_params(self) -> int:
return sum(p.numel() for p in self.parameters())
@torch.no_grad()
def init_gate_from_gamma(gate: HeadGate, gamma: torch.Tensor) -> None:
g = gamma.detach().float().clamp(1e-4, 1 - 1e-6)
gate.w2.weight.zero_()
gate.w2.bias.copy_(torch.log(g).repeat_interleave(gate.groups))
def _split(q, k, v, chunk_size):
B, H, L, kd = q.shape
C = int(chunk_size)
assert L % C == 0, (L, C)
return B, H, L, kd, v.shape[-1], L // C, C
def _causal(C, dev):
ar = torch.arange(C, device=dev)
return ar[:, None] >= ar[None, :]
def _scan_tables(Glast, nc, dev):
"""Per-(batch,head,group) data-dependent scan matrices: T[i,j]=d^(i-1-j), T2[i,j]=d^(i-j), j<i."""
P = torch.cumsum(Glast, dim=-1) # inclusive
Ce = P - Glast # exclusive
lower = torch.tril(torch.ones(nc, nc, device=dev, dtype=torch.bool), -1)
diff = Ce[..., :, None] - P[..., None, :]
diff = torch.where(lower, diff, torch.zeros((), device=dev, dtype=diff.dtype))
T = torch.exp(diff) * lower.to(diff.dtype)
Ce2 = torch.cat([torch.zeros_like(P[..., :1]), P], dim=-1) # [.., nc+1]
diff2 = Ce2[..., :, None] - P[..., None, :]
valid2 = torch.arange(nc + 1, device=dev)[:, None] > torch.arange(nc, device=dev)[None, :]
diff2 = torch.where(valid2, diff2, torch.zeros((), device=dev, dtype=diff2.dtype))
T2 = torch.exp(diff2) * valid2.to(diff2.dtype)
return T, T2, Ce, Ce2
@torch.no_grad()
def _cast(x, dt):
return x.to(dt)
def _gla_chunkwise_exact(q, k, v, g_t, state, chunk_size: int):
"""Chunkwise GLA with input-dependent grouped decay.
q,k: [B,H,L,kd]; v: [B,H,L,vd]; g_t: [B,L,H] (G=1) or [B,L,H,G] fp32 log-decay ≤0.
Returns (out [B,H,L,vd], new_state [B,H,kd,vd]).
"""
B, H, L, kd, vd, nc, C = _split(q, k, v, chunk_size)
dev, dt = q.device, q.dtype
G_ = g_t.shape[-1] if g_t.dim() == 4 else 1
if G_ == 1:
g = (g_t[..., 0] if g_t.dim() == 4 else g_t).transpose(1, 2) # [B,H,L]
g = g.reshape(B, H, nc, C).float()
qr = q.view(B, H, nc, C, kd)
kr = k.view(B, H, nc, C, kd)
vr = v.view(B, H, nc, C, vd)
Gc = torch.cumsum(g, dim=-1) # [B,H,nc,C]
causal = _causal(C, dev)
delta = Gc[..., :, None] - Gc[..., None, :]
delta = torch.where(causal, delta, torch.zeros((), device=dev))
mask = (torch.exp(delta) * causal.to(delta.dtype)).to(dt)
qk = torch.matmul(qr, kr.transpose(-1, -2))
inner = torch.matmul(qk * mask, vr)
Glast = Gc[..., -1]
val_decay = torch.exp(Glast[..., None] - Gc) # [B,H,nc,C]
U = torch.matmul(kr.transpose(-1, -2), (vr.float() * val_decay[..., None]).to(dt))
Uflat = U.reshape(B, H, nc, kd * vd).float()
T, T2, Ce, Ce2 = _scan_tables(Glast, nc, dev)
S_prev = torch.matmul(T, Uflat).view(B, H, nc, kd, vd).to(dt)
S_all = torch.matmul(T2, Uflat).view(B, H, nc + 1, kd, vd).to(dt)
if state is not None:
st = state.reshape(B, H, 1, kd, vd)
S_prev = S_prev + torch.exp(Ce)[..., None, None].to(dt) * st
S_all = S_all + torch.exp(Ce2)[..., None, None].to(dt) * st
cross = torch.matmul((qr.float() * torch.exp(Gc)[..., None]).to(dt), S_prev)
return (inner + cross).reshape(B, H, L, vd), S_all[:, :, nc]
# ---------------- grouped variant G>1: NOT DELIVERED ----------------
# The grouped kernel is not shipped in this round: the first prototype's einsum/permute ordering
# was wrong (A B5 / B B3 measured the crash) and a corrected design still costs +647 %/layer
# (A's corrected vectorised prototype: 16.928 ms vs 2.265 ms) => ~+97 % end-to-end at B=1.
# Raising loudly beats shipping a path that silently computes a different function.
raise NotImplementedError(
"gla_chunkwise: grouped G>1 is not delivered (needs a tested production kernel; "
"see GLA_PLAN_v2 §3/§9). Use groups=1 (GLA-S) or the scalar fast kernel.")
def gla_chunkwise(q, k, v, g_t, state, chunk_size: int):
"""Padded wrapper: supports prompt lengths not divisible by the chunk size.
Zero k/v/g at pad positions leave the recurrent state unchanged; the returned out is trimmed.
"""
B, H, L, kd = q.shape
vd = v.shape[-1]
C = int(chunk_size)
pad = (-L) % C
if pad == 0:
return _gla_chunkwise_exact(q, k, v, g_t, state, C)
qp = torch.nn.functional.pad(q, (0, 0, 0, pad))
kp = torch.nn.functional.pad(k, (0, 0, 0, pad))
vp = torch.nn.functional.pad(v, (0, 0, 0, pad))
if g_t.dim() == 4:
gp = torch.nn.functional.pad(g_t, (0, 0, 0, pad))
else:
gp = torch.nn.functional.pad(g_t, (0, 0, 0, pad))
out, new_state = _gla_chunkwise_exact(qp, kp, vp, gp, state, C)
return out[:, :, :L], new_state
def gla_recurrent(q, k, v, g_t, state=None):
return gla_recurrent_reference(q, k, v, g_t, state)
def gla_recurrent_reference(q, k, v, g_t, state=None):
"""Slow decode reference (G=1), with fp32 decay and persistent state."""
B, H, L, kd = q.shape
if (g_t.dim() == 4 and g_t.shape[-1] > 1):
raise NotImplementedError("recurrent reference implemented for G=1 only")
g = (g_t[..., 0] if g_t.dim() == 4 else g_t).transpose(1, 2).float()
# exp(g) near one must never pass through bf16: that would erase slow
# decay. Promote a chunkwise-prefill state once and keep subsequent
# updates in fp32; this cannot recover precision lost during prefill.
output_dtype = q.dtype
q, k, v = q.float(), k.float(), v.float()
if state is None:
state = torch.zeros(B, H, kd, v.shape[-1], device=q.device, dtype=torch.float32)
else:
state = state.float()
outs = []
for t in range(L):
state = torch.exp(g[:, :, t])[..., None, None] * state + \
k[:, :, t, :].unsqueeze(-1) * v[:, :, t, :].unsqueeze(-2)
outs.append(torch.sum(q[:, :, t, :].unsqueeze(-1) * state, dim=-2))
return torch.stack(outs, dim=2).to(output_dtype), state
def count_params(d_model=1408, n_heads=22, n_layers=20, rank=8, groups=1):
per = rank * d_model + rank * n_heads * groups + n_heads * groups
total = per * n_layers
return per, total, 100.0 * total / 504_179_896
if __name__ == "__main__":
for r, gp in ((4, 1), (8, 1), (16, 1), (8, 4), (8, 8)):
per, total, pct = count_params(rank=r, groups=gp)
print(f"low-rank rank={r:2d} G={gp}: {per:6,}/layer {total:9,} total {pct:.4f}%")
per = 1408 * 22 + 22
print(f"per-head linear (no bottleneck): {per:,}/layer {per * 20:,} total "
f"{100 * per * 20 / 504179896:.3f}%")
full = 20 * (1408 * 1408 + 1408)
print(f"full d->H*kd: {full:,} total {100 * full / 504179896:.2f}%")