Instructions to use alexlitz/elephantine with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use alexlitz/elephantine with Transformers:
# pip install -U transformers accelerate # Load model directly from transformers import AutoModel model = AutoModel.from_pretrained("alexlitz/elephantine", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
Download modeling_elephantine.py from alexlitz/elephantine: direct link, hf CLI and curl.
- Browser
- Download file 66 kB
-
https://huggingface.co/alexlitz/elephantine/resolve/main/modeling_elephantine.py
- Command line
-
hf download hf://alexlitz/elephantine/modeling_elephantine.py
-
curl -L -o modeling_elephantine.py https://huggingface.co/alexlitz/elephantine/resolve/main/modeling_elephantine.py
66 kB
"""modeling_elephantine.py: the ONE universal elephantine architecture as a transformers model.
Self-contained remote code: the architecture below is a verbatim copy of elephantine/universal.py
(everything from `F64 = torch.float64` up to the loaders) and of `gd_recurrence` from
attn/gated_delta_layer.py, the general gated-delta recurrence the SSM branch runs. Pure torch:
no fla, mamba_ssm or causal_conv1d needed. The copy is regenerated by
`python -m elephantine.push_hub_demo --vendor`, and tests/test_hub_demo.py checks it is
identical to universal.py, source and forward.
Every layer is x += Attn(norm_a(x)); x += SSM(norm_s(x)); x += FFN(norm_f(x)), and every
op a family may or may not use is a branch with a coefficient, off by default. A target model
is a weight setting of this one class; see the model card for the settings in this repo.
Computes in float64 (the weights file stores each tensor in the narrowest dtype that holds it
exactly; they are widened to float64 at load). No KV cache: generation re-runs the prefix.
"""
from __future__ import annotations
import math
from dataclasses import dataclass, fields
import torch
import torch.nn.functional as F
from torch import nn
from transformers import GenerationMixin, PreTrainedModel
from transformers.modeling_outputs import CausalLMOutput
try:
from transformers.modeling_utils import no_init_weights
except ImportError: # newer transformers keep it in initialization
from transformers.initialization import no_init_weights
try:
from .configuration_elephantine import ElephantineConfig
except ImportError: # imported as a plain module (tests)
from configuration_elephantine import ElephantineConfig
# >>> vendored: attn/gated_delta_layer.py gd_recurrence
def gd_recurrence(q, k, v, a, gamma, w):
"""Batched general gated-delta rule (the `attn.ssm_cross.general` recurrence over B,H):
S_t = Diag(a_t) S_{t-1} - gamma_t (S_{t-1} k_t) k_t^T + w_t v_t k_t^T, y_t = S_t q_t
q,k,a:(B,H,T,dk) v:(B,H,T,dv) gamma,w:(B,H,T) -> y:(B,H,T,dv)."""
B, Hn, T, dk = k.shape
dv = v.shape[-1]
S = torch.zeros(B, Hn, dv, dk, dtype=q.dtype)
out = []
for t in range(T):
S = S * a[:, :, t, None, :] # per-key-channel decay
Sk = (S * k[:, :, t, None, :]).sum(-1) # S @ k_t -> (B,H,dv)
S = S - gamma[:, :, t, None, None] * Sk[..., None] * k[:, :, t, None, :] # delta remove
S = S + w[:, :, t, None, None] * v[:, :, t, :, None] * k[:, :, t, None, :] # write v ⊗ k
out.append((S * q[:, :, t, None, :]).sum(-1)) # S @ q_t
return torch.stack(out, 2)
# <<< vendored: attn/gated_delta_layer.py gd_recurrence
# >>> vendored: elephantine/universal.py
F64 = torch.float64
@dataclass
class UConfig:
vocab: int
d: int # hidden width
L: int # layers
heads: int = 1 # attention heads (MHA; GQA targets tile their KV)
head_dim: int = 2
ssm_in: int = 1 # in_proj width (the conv runs over all of it)
conv_k: int = 1
ssm_heads: int = 1
dk: int = 1
dv: int = 1
experts: int = 1
expert_ff: int = 1
top_k: int = 1
shared_ff: int = 1
ssm_norm_feats: int = 0 # normalized knob features (B/C/dt RMS-normed before use)
ssm_sig_feats: int = 0 # sigmoid knob features (low-rank tanh / sigmoid gates)
ssm_prod_feats: int = 0 # product features f = (Px r) * (Py r + b) (RWKV-6 ddlerp)
ssm_sig2_feats: int = 0 # a second sigmoid stage reading the product features
n_prefix: int = 0 # learned prefix (meta) tokens, prepended then stripped
attnres_block: int = 0 # Block AttnRes: depth-softmax residual over block sums (0 = off)
attnres_full: bool = False # Full AttnRes: every sub-block output is its own source (GPT-S3)
attn_norm_feats: int = 0 # attention normalized-latent features (real MLA's q / kv latent norms)
lora_adapters: int = 0 # switched LoRA adapters selected per token by a switch head (aLoRA)
lora_rank: int = 1
max_pos: int = 0 # learned absolute positions (0 = none; the spec carries both)
ssm_dconv: int = 0 # dynamic (per-token generated) causal conv on v, window W (0 = off; Jet)
ssm_dconv_rank: int = 1 # its kernel generator's hidden width
norm_gate_rank: int = 0 # gated norms: rms(x) w * c * sigmoid(down(silu(up(.)))) (GigaChat-3.5)
attn_norm_gate_rank: int = 0 # the same gate on the attention normalized-latent features
engram_slots: int = 0 # hashed n-gram memory (Engram) per layer: slots per (order, hash head); 0 = none
engram_orders: tuple = (2, 3)
engram_heads: int = 1
engram_dim: int = 1
engram_conv: int = 1
mem_attn: bool = False # a second, parallel attention per layer over a score-selected memory (EdgeRock)
mem_kv_ff: int = 0 # its key/value writer MLP width
mem_sel_rank: int = 0 # its admission-score MLP rank
stream_norms: bool = False # per-layer norms OF the residual stream itself, at layer start / end (CENO)
hc_mult: int = 0 # manifold-constrained hyper-connections: N residual streams (0 = one plain stream)
hc_iters: int = 1 # their Sinkhorn iterations
dsa_heads: int = 0 # DeepSeek sparse attention indexer heads (0 = dense attention)
dsa_dim: int = 1 # its head dim
ple_dim: int = 0 # per-layer embeddings (Gemma-3n / 4 PLE): width per layer (0 = none)
ple_vocab: int = 0 # their token table's vocabulary (0: the model's)
def _lin(i, o, bias=True):
# skip PyTorch's default init: the weights are zeroed right away
m = torch.nn.utils.skip_init(nn.Linear, i, o, bias=bias, dtype=F64)
with torch.no_grad():
m.weight.zero_()
if bias:
m.bias.zero_()
return m
def _p(*shape, fill=0.0):
return nn.Parameter(torch.full(shape, fill, dtype=F64))
def _lora(x, A, B, idx):
"""Switched LoRA: token t with adapter idx[t] = a > 0 gets B[a-1] (A[a-1] x)."""
if idx is None or A.shape[0] == 0 or not bool((idx > 0).any()):
return 0
out = 0
for e in range(A.shape[0]):
m = (idx == e + 1)[..., None].to(x.dtype)
if m.any():
out = out + m * ((x @ A[e].T) @ B[e].T)
return out
def _rms(x, eps):
return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + eps)
def _sig_gate(y, c, up, down):
"""Low-rank self-gate y * c * sigmoid(down(silu(up(y)))) (off when c = 0)."""
return y if c == 0 else y * c * torch.sigmoid(down(F.silu(up(y))))
class UNorm(nn.Module):
"""RMSNorm with a settable weight and eps; centre = 1 plus a bias makes it LayerNorm;
gate_c != 0 multiplies by a low-rank sigmoid self-gate (GigaChat-3.5's gated norm)."""
def __init__(self, d, gate_rank=0):
super().__init__()
self.weight = _p(d, fill=1.0)
self.bias, self.centre = _p(d), _p(1)
self.register_buffer("eps", torch.tensor(1e-6, dtype=F64))
self.gate_c = _p(1)
self.g_up, self.g_down = _lin(d, max(gate_rank, 1), bias=False), _lin(max(gate_rank, 1), d, bias=False)
self.pre_c, self.pre_w = _p(1), _p(d, fill=1.0) # a first RMSNorm stage (two norms in series)
self.register_buffer("pre_eps", torch.tensor(1e-6, dtype=F64))
self.ident = _p(1) # 1: no norm at all (post-norm-only layers: OLMo-2)
def forward(self, x):
if self.ident != 0:
return x
if self.pre_c != 0:
x = _rms(x, self.pre_eps) * self.pre_w
if self.centre != 0:
x = x - self.centre * x.mean(-1, keepdim=True)
return _sig_gate(_rms(x, self.eps) * self.weight + self.bias, self.gate_c, self.g_up, self.g_down)
def _rotate_half(x):
h = x.shape[-1] // 2
return torch.cat((-x[..., h:], x[..., :h]), -1)
def _select_mask(sc, chunk, cap):
"""Score-selected memory: key s is visible to query t iff s precedes t's chunk, s < c(t) =
floor(t / chunk) chunk, and s is among the `cap` highest scores of the keys before c(t)
(ties to the earlier position). sc (B,H,T) -> (B,H,T,S) bool."""
T = sc.shape[-1]
pos = torch.arange(T)
before = pos[None, :] < (pos // chunk * chunk)[:, None] # (T, S)
a, b = sc[..., :, None], sc[..., None, :] # a: s', b: s
beats = (a > b) | ((a == b) & (pos[:, None] < pos[None, :])) # s' outranks s
rank = torch.einsum("tu,bhus->bhts", before.to(sc.dtype), beats.to(sc.dtype))
return before & (rank < cap)
class UAttention(nn.Module):
def __init__(self, c: UConfig, mem=False):
super().__init__()
H, hd, d = c.heads, c.head_dim, c.d
self.H, self.hd = H, hd
Dn = c.attn_norm_feats
self.Dn = Dn # q/k/v/g read [x, nb], nb = rms_grouped(pn x) * n_w
self.pn = _lin(d, max(Dn, 1), bias=False)
self.n_avg, self.n_w = _p(max(Dn, 1), max(Dn, 1)), _p(max(Dn, 1), fill=1.0)
self.register_buffer("n_eps", torch.tensor(1e-6, dtype=F64))
gr = max(c.attn_norm_gate_rank, 1) # gated latent norms (block-diagonal up / down)
self.n_gate_c, self.n_up, self.n_down = _p(1), _lin(max(Dn, 1), gr, bias=False), _lin(gr, max(Dn, 1), bias=False)
din = d + Dn
self.q_proj, self.k_proj, self.v_proj = _lin(din, H * hd), _lin(din, H * hd), _lin(din, H * hd)
self.g_proj, self.o_proj = _lin(din, H * hd), _lin(H * hd, d)
E, r = c.lora_adapters, c.lora_rank
self.la = nn.ParameterDict({n: _p(E, r, i) for n, i in (("q", din), ("k", din), ("v", din), ("o", H * hd))})
self.lb = nn.ParameterDict({n: _p(E, o, r) for n, o in (("q", H * hd), ("k", H * hd), ("v", H * hd), ("o", d))})
self.q_nw, self.k_nw = _p(H, hd, fill=1.0), _p(H, hd, fill=1.0) # per-head qk-norm weights
self.register_buffer("qk_eps", torch.tensor([1e-6, 1e-6], dtype=F64))
self.q_coef, self.k_coef = _p(2), _p(2) # [plain, normed]
self.gate_coef = _p(2) # [constant 1, sigmoid(g_proj x)]
with torch.no_grad():
self.q_coef[0] = self.k_coef[0] = self.gate_coef[0] = 1.0
self.inv_freq = _p(hd // 2) # 0 = NoPE on that pair
self.scale = _p(1, fill=1 / math.sqrt(hd))
C = H * hd
self.o_hmix = nn.Parameter(torch.eye(H, dtype=F64)) # head mix before the output norm (DIFF attn)
self.o_coef = _p(2) # [raw, RMS-normed] attention output
self.o_avg, self.o_w = _p(C, C), _p(C, fill=1.0)
self.register_buffer("o_eps", torch.tensor(1e-6, dtype=F64))
self.kv_read, self.kv_write = _p(1), _p(1) # cross-layer KV register (post-RoPE k, v)
self.register_buffer("window", torch.tensor(float("inf"), dtype=F64)) # t - s <= window
self.register_buffer("n_sink", torch.tensor(0.0, dtype=F64)) # s < n_sink always visible
# memory-read branches (EdgeRock's independent memory; off by default):
f, r = max(c.mem_kv_ff if mem else 0, 1), max(c.mem_sel_rank if mem else 0, 1)
self.kv_c = _p(1) # k, v read z = x + kv_c down(silu(gate x) * up x)
self.kv_gate, self.kv_up, self.kv_down = _lin(din, f, bias=False), _lin(din, f, bias=False), _lin(f, din, bias=False)
self.sel_down, self.sel_up = _lin(din, r, bias=False), _lin(r, H, bias=False) # score up(silu(down x))
self.register_buffer("sel_cap", torch.tensor(float("inf"), dtype=F64)) # keys kept (inf = off)
self.register_buffer("sel_chunk", torch.tensor(1.0, dtype=F64))
self.register_buffer("null_logit", torch.full((H,), float("-inf"), dtype=F64)) # a zero-value null key
# per head (learned sinks: GPT-OSS; -inf = off)
self.register_buffer("cap", torch.tensor(float("inf"), dtype=F64)) # logit softcap cap tanh(s / cap)
self.alibi = _p(H) # ALiBi slopes: s += alibi_h (s_pos - t_pos)
self.register_buffer("qkv_clip", torch.tensor(float("inf"), dtype=F64)) # q, k, v clamped to +-clip
# qk-norm variants: qk_full = 1 normalises over the WHOLE projection (all heads: OLMo-2 / 3), qk_centre
# [q, k] = 1 makes it a LayerNorm (mean-centred, + q_nb / k_nb biases: StableLM, Persimmon, Phi);
# qk_post = 1 applies it AFTER the rotary (HunYuan)
self.qk_full, self.qk_centre, self.qk_post = _p(1), _p(2), _p(1)
self.q_nb, self.k_nb = _p(H, hd), _p(H, hd)
# the width the per-head q / k / v norms average over (< hd when a narrower target head is zero-padded)
self.register_buffer("qk_width", torch.tensor(float(hd), dtype=F64))
self.v_c, self.v_nw = _p(1), _p(H, hd, fill=1.0) # v-norm: v <- rms_head(v) v_nw (Gemma-4)
self.register_buffer("v_eps", torch.tensor(1e-6, dtype=F64))
self.register_buffer("kv_slot", torch.tensor(0.0, dtype=F64)) # which KV register (per layer type)
# Llama-4 / Ministral-3 query temperature: q_t *= 1 + beta log(1 + floor(t / M)), [beta, M] (beta = 0: off)
self.register_buffer("qtemp", torch.tensor([0.0, 1.0], dtype=F64))
self.gate_sp = _p(1) # + gate_sp softplus(g_proj x) in the output gate (Laguna)
self.hg_c, self.hg_w, self.hg_b = _p(H), _p(H, hd), _p(H) # head gate c sigmoid(<q_normed, w> + b)
# grouped differential heads (Motif-3 GDLA): o_h <- o_h - dl_c m_h sigmoid(dl_proj x)_h o_{src(h)}
self.dl_c, self.dl_proj = _p(1), _lin(din, H, bias=False)
self.register_buffer("dl_src", torch.arange(H))
self.register_buffer("dl_m", torch.zeros(H, dtype=F64))
# DSA indexer (GLM-5.3 / DeepSeek-V3.2 with k-pool compression): a key is visible iff it is in one of
# the top-scoring complete pools of `dsa_pool` keys, or in the query's incomplete tail pool
nI, dI = max(c.dsa_heads, 1), c.dsa_dim
self.dsa_on = c.dsa_heads > 0
self.iq, self.ik, self.igate = _lin(din, nI * dI, bias=False), _lin(din, dI, bias=False), _lin(din, dI, bias=False)
self.ik_w, self.ik_b, self.i_ape = _p(dI, fill=1.0), _p(dI), _p(1, dI)
self.iw = _lin(din, nI, bias=False)
self.register_buffer("ik_eps", torch.tensor(1e-6, dtype=F64))
self.register_buffer("dsa_pool", torch.tensor(1.0, dtype=F64))
self.register_buffer("dsa_topk", torch.tensor(float("inf"), dtype=F64))
self.register_buffer("dsa_tail", torch.tensor(1.0, dtype=F64))
self.i_inv = _p(max(dI // 2, 1)) # RoPE on the indexer q / k (pairs (i, i + dI/2); 0 = NoPE)
self.register_buffer("dsa_share", torch.tensor(0.0, dtype=F64)) # 1: publish the mask, 2: reuse it
with torch.no_grad():
self.o_coef[0] = 1.0
def _qk_norm(self, t, i):
"""[plain, normed] mix of q (i = 0) or k (i = 1): RMS-normed per head, or over every head
(qk_full), optionally mean-centred with a bias (qk_centre: a LayerNorm)."""
coef = (self.q_coef, self.k_coef)[i]
if coef[1] == 0:
return coef[0] * t
dims = (-2, -1) if self.qk_full != 0 else (-1,)
n = t - t.mean(dims, keepdim=True) if self.qk_centre[i] != 0 else t
width = self.qk_width * (t.shape[-2] if self.qk_full != 0 else 1)
n = n * torch.rsqrt(n.pow(2).sum(dims, keepdim=True) / width + self.qk_eps[i]) * (self.q_nw, self.k_nw)[i]
if self.qk_centre[i] != 0:
n = n + (self.q_nb, self.k_nb)[i]
return coef[0] * t + coef[1] * n
def forward(self, x, causal, reg=None):
B, T, _ = x.shape
sh = lambda t: t.view(B, T, self.H, self.hd)
if self.Dn:
n = self.pn(x)
n = n * torch.rsqrt(n.pow(2) @ self.n_avg + self.n_eps) * self.n_w
x = torch.cat((x, _sig_gate(n, self.n_gate_c, self.n_up, self.n_down)), -1)
idx = reg.get("adapter") if reg is not None else None
L = lambda n, t: _lora(t, self.la[n], self.lb[n], idx)
xk = x if self.kv_c == 0 else x + self.kv_c * self.kv_down(F.silu(self.kv_gate(x)) * self.kv_up(x))
q, k, v = sh(self.q_proj(x) + L("q", x)), sh(self.k_proj(xk) + L("k", xk)), sh(self.v_proj(xk) + L("v", xk))
if self.v_c != 0:
v = v * torch.rsqrt(v.pow(2).sum(-1, keepdim=True) / self.qk_width + self.v_eps) * self.v_nw
ang = torch.arange(T, dtype=F64)[:, None] * self.inv_freq[None]
cos = torch.cat((ang.cos(), ang.cos()), -1)[None, :, None]
sin = torch.cat((ang.sin(), ang.sin()), -1)[None, :, None]
rope = lambda t: t * cos + _rotate_half(t) * sin
if self.qk_post != 0:
q, k = rope(q), rope(k)
q, k = self._qk_norm(q, 0), self._qk_norm(k, 1)
if torch.isfinite(self.qkv_clip): # after the qk-norm (OLMoE), before the rotary
c = self.qkv_clip
q, k, v = q.clamp(-c, c), k.clamp(-c, c), v.clamp(-c, c)
qn = q
if self.qk_post == 0:
q, k = rope(q), rope(k)
if self.qtemp[0] != 0:
t_ = torch.arange(T, dtype=F64)
q = q * (1 + self.qtemp[0] * torch.log1p(torch.floor(t_ / self.qtemp[1])))[None, :, None, None]
if reg is not None:
sk, sv = ("k", "v") if self.kv_slot == 0 else (f"k{int(self.kv_slot)}", f"v{int(self.kv_slot)}")
if self.kv_read != 0:
k = (1 - self.kv_read) * k + self.kv_read * reg[sk]
v = (1 - self.kv_read) * v + self.kv_read * reg[sv]
if self.kv_write != 0:
reg[sk], reg[sv] = k, v
s = torch.einsum("bthd,bshd->bhts", q, k) * self.scale
if torch.isfinite(self.cap):
s = self.cap * torch.tanh(s / self.cap)
tt, ss = torch.arange(T)[:, None], torch.arange(T)[None]
if self.alibi.any():
s = s + self.alibi[:, None, None] * (ss - tt).to(F64)
allowed = (tt - ss <= self.window) | (ss < self.n_sink)
if causal:
allowed = allowed & (ss <= tt)
if torch.isfinite(self.sel_cap):
sc = self.sel_up(F.silu(self.sel_down(x))).transpose(1, 2)
allowed = allowed & _select_mask(sc, int(self.sel_chunk), self.sel_cap)
if self.dsa_on and (torch.isfinite(self.dsa_topk) or self.dsa_share != 0):
if self.dsa_share == 2: # GLM-5 'shared' indexer layers
vis = reg["dsa"]
elif torch.isfinite(self.dsa_topk):
vis = self._dsa_mask(x)
else:
vis = torch.ones(B, T, T, dtype=torch.bool)
if self.dsa_share == 1:
reg["dsa"] = vis
allowed = allowed & vis[:, None]
s = s.masked_fill(~allowed, float("-inf"))
if torch.isfinite(self.null_logit).any():
s = torch.cat((s, self.null_logit[:, None, None].expand(*s.shape[:-1], 1)), -1)
o = torch.einsum("bhts,bshd->bthd", s.softmax(-1)[..., :T], v)
if self.hg_c.any():
o = o * (self.hg_c * torch.sigmoid((qn * self.hg_w).sum(-1) + self.hg_b))[..., None]
if self.dl_c != 0: # signal heads minus a gated copy of their noise head
o = o - self.dl_c * (self.dl_m * torch.sigmoid(self.dl_proj(x)))[..., None] * o[:, :, self.dl_src]
if not torch.equal(self.o_hmix, torch.eye(self.H, dtype=F64)):
o = torch.einsum("bthd,hg->btgd", o, self.o_hmix)
o = o.reshape(B, T, -1)
if self.o_coef[1] != 0:
o = self.o_coef[0] * o + self.o_coef[1] * o * torch.rsqrt(o.pow(2) @ self.o_avg + self.o_eps) * self.o_w
if self.gate_sp != 0:
o = o * (self.gate_coef[0] + self.gate_coef[1] * torch.sigmoid(self.g_proj(x)) + self.gate_sp * F.softplus(self.g_proj(x)))
else:
o = o * (self.gate_coef[0] + self.gate_coef[1] * torch.sigmoid(self.g_proj(x)))
return self.o_proj(o) + L("o", o)
def _dsa_mask(self, x):
"""DSA k-pool indexer -> visible keys (B,T,S). Pools of P consecutive keys; a pool's key is a
per-channel softmax(gate x + ape) average of LayerNormed key projections; the pool score for a
query is sum_h w_h relu(<q_h, pool key> / sqrt(dI)); the top floor(topk / P) complete pools whose
last key precedes the query are kept, plus the query's own incomplete tail."""
B, T, _ = x.shape
P, nI, dI = int(self.dsa_pool), self.iw.out_features, self.ik.out_features
q = self.iq(x).view(B, T, nI, dI)
k = F.layer_norm(self.ik(x), (dI,), self.ik_w, self.ik_b, float(self.ik_eps))
if self.i_inv.any(): # indexer RoPE (DeepSeek-V3.2 / GLM-5)
ang = torch.arange(T, dtype=F64)[:, None] * self.i_inv[None]
cos, sin = torch.cat((ang.cos(), ang.cos()), -1), torch.cat((ang.sin(), ang.sin()), -1)
q = q * cos[None, :, None] + _rotate_half(q) * sin[None, :, None]
k = k * cos[None] + _rotate_half(k) * sin[None]
gs = self.igate(x)
tt = torch.arange(T)
vis = torch.zeros(B, T, T, dtype=torch.bool)
npool = T // P
if npool and int(self.dsa_topk) // P:
kk = k[:, :npool * P].view(B, npool, P, dI)
gg = gs[:, :npool * P].view(B, npool, P, dI) + self.i_ape
pk = (gg.softmax(2) * kk).sum(2) # (B, pools, dI)
sc = F.relu(torch.einsum("bthd,bpd->bthp", q, pk) / math.sqrt(dI))
isc = torch.einsum("bth,bthp->btp", self.iw(x) / math.sqrt(nI), sc)
ok = ((torch.arange(npool) * P + P - 1)[None, :] <= tt[:, None]).expand(B, T, npool)
isc = isc.masked_fill(~ok, torch.finfo(isc.dtype).min)
sel = isc.topk(min(int(self.dsa_topk) // P, npool), -1).indices
keep = torch.zeros(B, T, npool, dtype=torch.bool).scatter_(-1, sel, True) & ok
vis[:, :, :npool * P] = keep.repeat_interleave(P, -1)
if self.dsa_tail != 0: # the incomplete visible pool
start = (tt + 1) - (tt + 1) % P
vis |= ((tt[None, :] >= start[:, None]) & (tt[None, :] <= tt[:, None]))[None]
return vis
UAttention._dsa_mask = _dsa_mask
class UHC(nn.Module):
"""Manifold-constrained hyper-connections around one sub-block: from the N streams X (RMS-normed
flat) a linear map gives pre (sigmoid), post (2 sigmoid) and comb (softmax, then Sinkhorn to doubly
stochastic); the block reads sum_i pre_i X_i and the streams become post * out + comb^T X."""
def __init__(self, c: UConfig):
super().__init__()
N, d = c.hc_mult, c.d
self.N, self.iters = N, c.hc_iters
self.fn, self.base, self.scale = _p((2 + N) * N, N * d), _p((2 + N) * N), _p(3, fill=1.0)
self.register_buffer("eps", torch.tensor([1e-6, 1e-6], dtype=F64)) # [hc_eps, input-norm eps]
self.register_buffer("post_c", torch.tensor(2.0, dtype=F64)) # post = post_c sigmoid(.)
# Motif-3 variant (sk = 1): pre / post logits clamped to +-lim[0]; comb = Sinkhorn of exp(clamp(logits,
# +-lim[1])), iters x (row, column) normalisation with sums clamped >= 1e-8; read transposed
# (the streams become post * out + H_res X)
self.register_buffer("sk", torch.tensor(0.0, dtype=F64))
self.register_buffer("lim", torch.tensor([10.0, 20.0], dtype=F64))
def forward(self, X):
B, T, N, d = X.shape
flat = X.reshape(B, T, N * d)
flat = flat * torch.rsqrt(flat.pow(2).mean(-1, keepdim=True) + self.eps[1])
w = flat @ self.fn.T
pw, qw, cw = w.split([N, N, N * N], -1)
pb, qb, cb = self.base.split([N, N, N * N])
if self.sk != 0:
l0, l1 = self.lim
pre = torch.sigmoid((pw * self.scale[0] + pb).clamp(-l0, l0))
post = self.post_c * torch.sigmoid((qw * self.scale[1] + qb).clamp(-l0, l0))
m = (cw * self.scale[2] + cb).view(B, T, N, N).clamp(-l1, l1).exp()
for _ in range(self.iters):
m = m / m.sum(-1, keepdim=True).clamp(min=1e-8)
m = m / m.sum(-2, keepdim=True).clamp(min=1e-8)
return post, m.transpose(-1, -2), (pre[..., None] * X).sum(2)
e = self.eps[0]
pre = torch.sigmoid(pw * self.scale[0] + pb) + e
post = self.post_c * torch.sigmoid(qw * self.scale[1] + qb)
comb = torch.softmax((cw * self.scale[2] + cb).view(B, T, N, N), -1) + e
comb = comb / (comb.sum(-2, keepdim=True) + e)
for _ in range(self.iters - 1):
comb = comb / (comb.sum(-1, keepdim=True) + e)
comb = comb / (comb.sum(-2, keepdim=True) + e)
return post, comb, (pre[..., None] * X).sum(2)
def _rot_pairs(t, theta):
"""Rotate key-dim pairs (j, j + dk/2) by theta (B,H,dk/2)."""
h = t.shape[-1] // 2
c, s_ = theta.cos(), theta.sin()
t0, t1 = t[..., :h], t[..., h:2 * h]
return torch.cat((t0 * c - t1 * s_, t0 * s_ + t1 * c, t[..., 2 * h:]), -1)
def _recurrence(q, k, v, a, gamma, w, w_prev, dth, rm=None, pre=None, er=None):
"""gd_recurrence plus: a previous-token write added before the decay (Mamba-3's trapezoid),
q/k rotated by a cumulative angle (Mamba-3, RoPE), and RWKV-7's generalized removal
S -= c (S_{t-1} kap)(kap * beta)^T read from the PRE-decay state (rm = (c, kap, rmv)), and a
per-channel erase gate on the delta term's READ key (er: S -= gamma (S (er * k)) k^T, GDN-2)."""
B, H, T, dk = k.shape
S = torch.zeros(B, H, v.shape[-1], dk, dtype=q.dtype)
theta = torch.zeros(B, H, dk // 2, dtype=q.dtype)
kp = torch.zeros(B, H, dk, dtype=q.dtype); vp = torch.zeros(B, H, v.shape[-1], dtype=q.dtype)
out = []
for t in range(T):
qt, kt = q[:, :, t], k[:, :, t]
S0 = S
if dth is not None:
theta = theta + dth[:, :, t]
qt, kt = _rot_pairs(qt, theta), _rot_pairs(kt, theta)
if w_prev is not None:
S = S + w_prev[:, :, t, None, None] * vp[..., None] * kp[:, :, None, :]
if rm is not None:
Skap = (S * rm[1][:, :, t, None, :]).sum(-1) # pre-decay S kap
S = S * a[:, :, t, None, :]
if rm is not None:
S = S - rm[0][None, :, None, None] * Skap[..., None] * rm[2][:, :, t, None, :]
Sk = (S * (kt if er is None else kt * er[:, :, t])[:, :, None, :]).sum(-1)
S = S - gamma[:, :, t, None, None] * Sk[..., None] * kt[:, :, None, :]
S = S + w[:, :, t, None, None] * v[:, :, t, :, None] * kt[:, :, None, :]
yt = (S * qt[:, :, None, :]).sum(-1)
if pre is not None: # RWKV-6 reads S_{t-1}
yt = yt + pre[None, :, None] * ((S0 * qt[:, :, None, :]).sum(-1) - yt)
out.append(yt)
kp, vp = kt, v[:, :, t]
return torch.stack(out, 2)
def _ridge_readout(q, k, a, ratio, n_iter):
"""Gated KalmaNet's readout: the key-covariance state H_t = Diag(a_t) H_{t-1} + k_t k_t^T (the
gated-delta state with v = k, write 1, no delta term) and a fixed n_iter-step Chebyshev iteration
for (H_t + lam_t I) x = q_t, lam_t = ratio ||H_t||_F (spectrum bound ||H_t||_F):
step = 2 / (2 lam + |H|_F), rho = (|H|_F / (2 lam + |H|_F) / 2)^2, x_0 = step q, x_-1 = 0, w = 2
w <- 1 / (1 - rho w); x <- w x - (step w (x^T H + lam x - q) + (w - 1) x_prev)
q, k, a: (B,H,T,dk) -> x: (B,H,T,dk)."""
B, Hn, T, dk = k.shape
S = torch.zeros(B, Hn, dk, dk, dtype=q.dtype)
lam_r = ratio[None, :, None]
out = []
for t in range(T):
kt, qt = k[:, :, t], q[:, :, t]
S = S * a[:, :, t, None, :] + kt[..., :, None] * kt[..., None, :]
fro = S.pow(2).sum((-2, -1)).sqrt()[..., None] # (B,H,1)
lam = lam_r * fro
step = 2 / (2 * lam + fro)
rho = (fro / (2 * lam + fro) / 2) ** 2
x_prev, x, w = torch.zeros_like(qt), step * qt, 2.0
for _ in range(n_iter):
w = 1 / (1 - rho * w)
g = step * w * ((x[..., :, None] * S).sum(-2) + lam * x - qt) + (w - 1) * x_prev
x_prev, x = x, w * x - g
out.append(x)
return torch.stack(out, 2)
class USSM(nn.Module):
def __init__(self, c: UConfig):
super().__init__()
D, K, H, dk, dv, d = c.ssm_in, c.conv_k, c.ssm_heads, c.dk, c.dv, c.d
self.D, self.K, self.H, self.dk, self.dv = D, K, H, dk, dv
C = H * dv
self.in_proj = _lin(d, D)
self.conv_w, self.conv_b = _p(D, K), _p(D)
self.act_silu, self.act_lin = _p(D), _p(D) # c' = act_silu*silu(c) + act_lin*c
Dn = c.ssm_norm_feats
self.Dn = Dn
self.pn = _lin(2 * D, max(Dn, 1)) # normalized knob features nb
self.n_avg, self.n_w = _p(max(Dn, 1), max(Dn, 1)), _p(max(Dn, 1), fill=1.0)
self.register_buffer("n_eps", torch.tensor(1e-6, dtype=F64))
Dh, Dp, Dh2 = c.ssm_sig_feats, c.ssm_prod_feats, c.ssm_sig2_feats
self.Dh, self.Dp, self.Dh2 = Dh, Dp, Dh2
R0 = 2 * D + Dn
self.ph = _lin(R0, max(Dh, 1)) # stage 1: h = sigmoid(ph r)
self.pfx, self.pfy = _lin(R0 + Dh, max(Dp, 1)), _lin(R0 + Dh, max(Dp, 1)) # stage 2: f
self.ph2 = _lin(R0 + Dh + Dp, max(Dh2, 1)) # stage 3: h2 = sigmoid(ph2 r)
R = R0 + Dh + Dp + Dh2 # knobs read [u, c', nb, h, f, h2]
self.pq, self.pk = _lin(R, H * dk), _lin(R, H * dk)
self.pv, self.pz = _lin(R, C), _lin(R, C)
self.q_coef, self.k_coef = _p(2), _p(2) # [plain, l2-normed]
with torch.no_grad():
self.q_coef[0] = self.k_coef[0] = 1.0
self.register_buffer("l2_eps", torch.tensor(1e-6, dtype=F64))
self.p1, self.p2 = _lin(R, H * dk), _lin(R, H * dk)
self.A1, self.A2, self.A3 = _p(H, dk), _p(H, dk), _p(H, dk)
self.mem = _p(H, dk, fill=1.0)
self.pb, self.pe, self.pi = _lin(R, H), _lin(R, H), _lin(R, H)
self.pA, self.A4, self.a_floor = _lin(R, H), _p(H, dk), _p(H) # Mamba-3 decay term
self.p5, self.A5 = _lin(R, H * dk), _p(H, dk) # log a -= A5 exp(p5) (RWKV-6)
self.c_pre = _p(H) # read S_{t-1} q (before this step's update)
self.cg = _p(H)
self.cg0 = _p(H) # constant delta strength: gamma = cg sigmoid(pb) + cg0
self.c_er, self.pbv = _p(H), _lin(R, H * dk) # erase gate on the read key: er = sigmoid(pbv r) (GDN-2)
self.cw = _p(6, H) # softplus / sigmoid / exp / RG-LRU / const / dt*sigmoid
self.cw_prev = _p(H) # previous-token write dt*(1-sigmoid(pb))
self.pang, self.rot_c = _lin(R, H * max(dk // 2, 1)), _p(H, dk // 2) # cumulative q/k rotation
self.rot_k = _p(H, dk // 2) # constant angle step per pair (= RoPE)
# RWKV-7 generalized delta: removal reads the PRE-decay state along kap (l2-normed) and
# writes out along kap * beta; the write key is modulated by beta.
self.pkap, self.pbeta = _lin(R, H * dk), _lin(R, H * dk)
self.c_rm, self.c_ka = _p(H), _p(H, dk)
self.register_buffer("kap_eps", torch.tensor(1e-24, dtype=F64))
# value register (RWKV-7 v_first): write v, or read v <- v + read * gate * (reg - v)
self.pvr = _lin(R, H * dv)
self.v_write, self.v_read = _p(1), _p(1)
self.y_write, self.y_read = _p(1), _p(1) # SSM output register (SambaY GMU memory)
# dynamic conv on v (Jet-Nemotron): v <- act(sum_j ker_t[c, j] v_{t-W+1+j}[c]), ker = w2 silu(w1 x) + b2
Wd = c.ssm_dconv
self.dcW = Wd
self.dc_w1 = _lin(d, max(c.ssm_dconv_rank, 1), bias=False)
self.dc_w2 = _lin(max(c.ssm_dconv_rank, 1), C * max(Wd, 1))
self.dc_act = _p(1) # 1 = silu after the dynamic conv
self.reset = _p(H) # RG-LRU: first token writes sigmoid(pi)
# ridge readout (Gated KalmaNet): a second, key-covariance state H_t = Diag(a_t) H_{t-1} + k_t k_t^T
# and a FIXED `ridge_iters`-step Chebyshev solve x_t ~ (H_t + lam_t I)^-1 q_t, lam_t = ridge_ratio ||H_t||_F;
# the main state then reads q <- q + ridge_c * (pal r) * (x - q)
self.ridge_c, self.ridge_ratio, self.pal = _p(H), _p(H), _lin(R, H)
self.register_buffer("ridge_iters", torch.tensor(0))
self.register_buffer("l2_eps_out", torch.tensor(0.0, dtype=F64)) # != 0: l2 is t / (|t| + eps_out)
self.Dskip = _p(H, dv)
self.gate = _p(5, C) # silu / sigmoid / gelu-tanh / identity / const
self.avg = _p(C, C) # RMS grouping as a weight (block averages)
self.norm_w, self.norm_b = _p(C, fill=1.0), _p(C)
self.mix = nn.Parameter(torch.eye(C, dtype=F64)) # channel mix of y before the output branches
self.register_buffer("norm_eps", torch.tensor(1e-6, dtype=F64))
self.out_coef = _p(4, C) # norm-then-gate / gate-then-norm / plain / ratio
self.register_buffer("dt_min", torch.tensor(0.0, dtype=F64)) # dt = max(softplus(p1), dt_min)
self.ratio_num, self.ratio_den = _p(C, C), _p(C, C)
self.out_proj = _lin(C, d)
def _norm(self, t):
return t * torch.rsqrt(t.pow(2) @ self.avg + self.norm_eps) * self.norm_w + self.norm_b
def forward(self, x, reg=None):
B, T, _ = x.shape
H, dk, dv, K = self.H, self.dk, self.dv, self.K
u = self.in_proj(x) # (B,T,D)
up = F.pad(u.transpose(1, 2), (K - 1, 0)) # causal depthwise conv
cv = sum(self.conv_w[:, j, None] * up[:, :, j:j + T] for j in range(K)) + self.conv_b[:, None]
cv = cv.transpose(1, 2)
cv = self.act_silu * F.silu(cv) + self.act_lin * cv
r = torch.cat((u, cv), -1)
if self.Dn:
n = self.pn(r)
r = torch.cat((r, n * torch.rsqrt(n.pow(2) @ self.n_avg + self.n_eps) * self.n_w), -1)
if self.Dh:
r = torch.cat((r, torch.sigmoid(self.ph(r))), -1)
if self.Dp:
r = torch.cat((r, self.pfx(r) * self.pfy(r)), -1)
if self.Dh2:
r = torch.cat((r, torch.sigmoid(self.ph2(r))), -1)
heads = lambda t, n: t.view(B, T, H, n).transpose(1, 2) # (B,H,T,n)
q, k = heads(self.pq(r), dk), heads(self.pk(r), dk)
l2 = lambda t: t * torch.rsqrt(t.pow(2).sum(-1, keepdim=True) + self.l2_eps)
if self.l2_eps_out != 0:
l2 = lambda t: t / ((t.pow(2).sum(-1, keepdim=True) + self.l2_eps).sqrt() + self.l2_eps_out)
q = self.q_coef[0] * q + self.q_coef[1] * l2(q)
k = self.k_coef[0] * k + self.k_coef[1] * l2(k)
v = heads(self.pv(r), dv)
if self.dcW:
Wd = self.dcW
ker = self.dc_w2(F.silu(self.dc_w1(x))).view(B, T, H * dv, Wd)
vr = F.pad(v.transpose(1, 2).reshape(B, T, H * dv).transpose(1, 2), (Wd - 1, 0)).unfold(2, Wd, 1)
vr = (vr.permute(0, 2, 1, 3) * ker).sum(-1)
vr = vr + self.dc_act * (F.silu(vr) - vr)
v = heads(vr, dv)
if reg is not None and self.v_write != 0:
reg["v_ssm"] = v
if reg is not None and self.v_read != 0:
v = v + self.v_read * heads(self.pvr(r), dv) * (reg["v_ssm"] - v)
kap = rmv = None
if self.c_rm.any() or self.c_ka.any():
beta = heads(self.pbeta(r), dk)
kap = heads(self.pkap(r), dk)
kap = kap * torch.rsqrt(kap.pow(2).sum(-1, keepdim=True) + self.kap_eps)
rmv = kap * beta
k = k * (1 + self.c_ka[:, None] * (beta - 1))
p1 = heads(self.p1(r), dk)
sp = F.softplus(p1)
if self.dt_min > 0: # Mamba-2 time_step_limit floor
sp = sp.clamp(min=self.dt_min)
loga = -(self.A1[:, None] * sp
+ self.A2[:, None] * torch.sigmoid(heads(self.p2(r), dk)) + self.A3[:, None])
per_head = lambda p: p(r).transpose(1, 2) # (B,H,T)
dt = sp[..., 0] # (B,H,T)
if self.A4.any():
pa = per_head(self.pA)
ht = pa.clamp(min=0) + 1 / (1 - pa.clamp(max=0))
loga = loga - self.A4[:, None] * (dt * torch.maximum(ht, self.a_floor[:, None]))[..., None]
if self.A5.any():
loga = loga - self.A5[:, None] * heads(self.p5(r), dk).exp()
a = self.mem[:, None] * loga.exp() # (B,H,T,dk)
sb, si = torch.sigmoid(per_head(self.pb)), torch.sigmoid(per_head(self.pi))
gamma = self.cg[:, None] * sb + self.cg0[:, None]
er = (1 + self.c_er[:, None, None] * (torch.sigmoid(heads(self.pbv(r), dk)) - 1)) if self.c_er.any() else None
lru = torch.sqrt((1 - a[..., 0] ** 2).clamp_min(0))
first = torch.zeros(T, dtype=F64); first[0] = 1
lru = lru + self.reset[:, None] * first * (1 - lru)
cw = lambda i: self.cw[i][:, None]
w = (cw(0) * dt + cw(1) * sb + cw(2) * per_head(self.pe).exp()
+ cw(3) * lru * si + cw(4) + cw(5) * dt * sb)
w_prev = self.cw_prev[:, None] * dt * (1 - sb) if self.cw_prev.any() else None
dth = None
if self.rot_c.any() or self.rot_k.any():
dth = self.rot_c[:, None] * torch.tanh(heads(self.pang(r), dk // 2)) * dt[..., None] + self.rot_k[:, None]
if self.ridge_c.any():
x_r = _ridge_readout(q, k, a, self.ridge_ratio, int(self.ridge_iters))
q = q + (self.ridge_c[:, None] * per_head(self.pal))[..., None] * (x_r - q)
if w_prev is None and dth is None and kap is None and er is None and not self.c_pre.any():
y = gd_recurrence(q, k, v, a, gamma, w)
else:
rm = (self.c_rm, kap, rmv) if kap is not None else None
y = _recurrence(q, k, v, a, gamma, w, w_prev, dth, rm, self.c_pre if self.c_pre.any() else None, er)
y = y + self.Dskip[:, None] * v
if reg is not None and self.y_read != 0:
y = (1 - self.y_read) * y + self.y_read * reg["y_ssm"]
if reg is not None and self.y_write != 0:
reg["y_ssm"] = y
y = y.transpose(1, 2).reshape(B, T, H * dv) @ self.mix
z = self.pz(r)
g = self.gate
G = g[0] * F.silu(z) + g[1] * torch.sigmoid(z) + g[2] * F.gelu(z, approximate="tanh") + g[3] * z + g[4]
oc = self.out_coef
out = oc[0] * self._norm(y) * G + oc[1] * self._norm(y * G) + oc[2] * y * G
if oc[3].any():
on = oc[3] != 0 # channels that use the ratio branch
den = torch.where(on, y @ self.ratio_den, torch.ones_like(y))
out = out + oc[3] * (y @ self.ratio_num) / den * G
return self.out_proj(out)
class UFFN(nn.Module):
def __init__(self, c: UConfig):
super().__init__()
d, E, f, fs = c.d, c.experts, c.expert_ff, c.shared_ff
self.top_k = c.top_k
self.router = _lin(d, E) # a router bias (zero unless a target has one: GPT-OSS)
self.w_gate, self.w_up, self.w_down = _p(E, f, d), _p(E, f, d), _p(E, d, f)
self.b_gate, self.b_up, self.b_down = _p(E, f), _p(E, f), _p(E, d) # routed-expert biases (GPT-OSS)
# group-limited top-k (DeepSeek-V2 / V3): [groups, groups kept, group score: 0 sum of the top-2
# selection scores, 1 their max, the value experts outside the kept groups are masked to before the
# top-k: 0 (DeepSeek) or -inf (Nemotron-H)]
self.register_buffer("grp", torch.tensor([1.0, 1.0, 0.0, 0.0], dtype=F64)) # [..., masked value]
self.renorm = _p(1) # 1 = renormalize the top-k weights
self.register_buffer("renorm_eps", torch.tensor(0.0, dtype=F64)) # ... by (sum + eps)
self.score_c = _p(1) # router scores: 0 softmax, 1 sigmoid
self.sel_bias = _p(E) # added to scores for top-k SELECTION only
self.sel_logit = _p(1) # 1: select on the raw logits + sel_bias (Youtu QB)
self.register_buffer("router_cap", torch.tensor(float("inf"), dtype=F64)) # router logit softcap (Laguna)
self.route_scale = _p(1, fill=1.0)
self.exp_scale = _p(E, fill=1.0) # the top-k weights times a per-expert scale (Gemma-4)
# SparseMixer top-2 (Phi-3.5-MoE, eval): each pick's weight is a softmax over the logits within a relative
# 2 eps of that pick's maximum; smix_eps = 0: off
self.register_buffer("smix_eps", torch.tensor(0.0, dtype=F64))
# the shared / dense path carries biases (zero unless a target has them: GPT-2, GPT-NeoX, ...)
self.s_gate, self.s_up, self.s_down = _lin(d, fs), _lin(d, fs), _lin(fs, d)
self.s_router = _lin(d, 1, bias=False)
E, r = c.lora_adapters, c.lora_rank
self.la = nn.ParameterDict({"g": _p(E, r, d), "u": _p(E, r, d), "d": _p(E, r, fs)})
self.lb = nn.ParameterDict({"g": _p(E, fs, r), "u": _p(E, fs, r), "d": _p(E, d, r)})
self.c_situ = _p(1) # SiTU-GLU: capped gate/up (Mini-K3)
self.register_buffer("situ_beta", torch.tensor([4.0, 25.0], dtype=F64))
self.c_clip = _p(1) # hard SwiGLU clip: gate <= L, |up| <= L
self.c_relu2 = _p(1) # ungated squared ReLU: relu(up x)^2
self.c_relu = _p(1) # ungated ReLU: relu(up x) (OPT)
self.up_shift = _p(1) # u <- u + up_shift after the clip (swigluoai's (u + 1))
# exact GeLU / mish branches: c_gelu = 1 ungated gelu(u) (GPT-2, NeoX, ...), c_geglu = 1 gated
# gelu(g) * u (Gemma GeGLU); gelu_tanh = 1 the tanh approximation (gelu_new / gelu_pytorch_tanh /
# gelu_fast), 0 the erf GeLU; c_mish = 1 gated mish(g) * u
self.c_gelu, self.c_geglu, self.gelu_tanh, self.c_mish = _p(1), _p(1), _p(1), _p(1)
self.register_buffer("clip_lim", torch.full((2,), float("inf"), dtype=F64)) # [gate, up] limits
# ungated xIELU (Apertus): [alpha_p, alpha_n] pre-softplus (learned), [beta, eps]
self.c_xielu, self.xielu_a = _p(1), _p(2)
self.register_buffer("xielu_be", torch.tensor([0.5, -1e-6], dtype=F64))
# PolyNorm-GLU (Motif-3): (w0 n(g^3) + w1 n(g^2) + w2 n(g) + b) * u, n = RMS over the width;
# weights per routed expert / for the shared path, sigmoid(w) if poly_sig
self.c_poly, self.poly_sig = _p(1), _p(1)
self.poly_w, self.poly_b = _p(c.experts, 3), _p(c.experts, 1)
self.s_poly_w, self.s_poly_b = _p(3), _p(1)
# [eps, hidden clamp, output scale, routed bias clamp, routed result clamp, shared active width]
self.register_buffer("poly_cfg", torch.tensor([1e-6, float("inf"), 1.0, float("inf"), float("inf"),
float(fs)], dtype=F64))
self.r_c = _p(1) # post-routing: routed <- r_up(rms_grouped(routed) r_w)
self.r_avg, self.r_w = _p(d, d), _p(d, fill=1.0)
self.r_up = _lin(d, d, bias=False)
self.register_buffer("r_eps", torch.tensor(1e-6, dtype=F64))
self.s_coef = _p(2) # [constant 1, sigmoid(s_router x)]
with torch.no_grad():
self.s_coef[0] = 1.0
def _xielu(self, u):
"""xIELU (HF XIELUActivation): a_p = softplus(alpha_p), a_n = beta + softplus(alpha_n),
u > 0: a_p u^2 + beta u; else: a_n (expm1(min(u, eps)) - u) + beta u."""
ap, an = F.softplus(self.xielu_a[0]), self.xielu_be[0] + F.softplus(self.xielu_a[1])
beta, eps = self.xielu_be
return torch.where(u > 0, ap * u * u + beta * u, (torch.expm1(torch.min(u, eps)) - u) * an + beta * u)
def _poly(self, g, u, routed):
"""PolyNorm-GLU (Motif-3 PolyNormTorch / GroupedPolyNorm): g, u clamped to the hidden clamp,
(w0 n(g^3) + w1 n(g^2) + w2 n(g) + b) * u with n(x) = x / sqrt(sum(x^2) / width + eps) over the
FFN width; routed experts (own w, b per expert; b clamped) also clamp the result; then * scale.
`width` is the shared path's active width, so a zero-padded shared bank normalises exactly."""
eps, hc, scale, bc, rc, ws = self.poly_cfg
g, u = g.clamp(-hc, hc), u.clamp(-hc, hc)
if routed: # h: (B, T, E, f); per-expert (E, 1) coefficients
w, b, n = self.poly_w, self.poly_b.clamp(-bc, bc), g.shape[-1]
w = [w[:, k, None] for k in range(3)]
else:
w, b, n = self.s_poly_w, self.s_poly_b, ws
w = [wk + self.poly_sig * (torch.sigmoid(wk) - wk) for wk in w]
nrm = lambda x: x / torch.sqrt(x.pow(2).sum(-1, keepdim=True) / n + eps)
y = (w[0] * nrm(g ** 3) + w[1] * nrm(g ** 2) + w[2] * nrm(g) + b) * u
if routed:
y = y.clamp(-rc, rc)
return y * scale
def _act(self, g, u, routed=False):
"""silu(g) * u, or SiTU-GLU bg tanh(g/bg) sigmoid(g) * bu tanh(u/bu) (c_situ = 1);
c_clip = 1 first clips g <= L and u to [-L, L]; c_relu2 = 1 is the ungated relu(u)^2;
c_xielu = 1 the ungated xIELU(u); c_poly = 1 PolyNorm-GLU (routed: per-expert weights);
c_gelu = 1 the ungated gelu(u), c_geglu = 1 gelu(g) * u (erf, or tanh if gelu_tanh = 1);
c_mish = 1 mish(g) * u."""
if self.c_relu2 != 0:
return F.relu(u) ** 2
if self.c_relu != 0:
return F.relu(u)
if self.c_gelu != 0 or self.c_geglu != 0:
gelu = lambda t: F.gelu(t, approximate="tanh" if self.gelu_tanh != 0 else "none")
return gelu(u) if self.c_gelu != 0 else gelu(g) * u
if self.c_mish != 0:
return F.mish(g) * u
if self.c_xielu != 0:
return self._xielu(u)
if self.c_poly != 0:
return self._poly(g, u, routed)
if self.c_clip != 0:
Lg, Lu = self.clip_lim
g = g + self.c_clip * (g.clamp(max=Lg) - g)
u = u + self.c_clip * (u.clamp(-Lu, Lu) - u)
u = u + self.up_shift
if self.c_situ == 0:
return F.silu(g) * u
c, (bg, bu) = self.c_situ, self.situ_beta
return (((1 - c) * g + c * bg * torch.tanh(g / bg)) * torch.sigmoid(g)
* ((1 - c) * u + c * bu * torch.tanh(u / bu)))
def forward(self, x, reg=None, xe=None, xg=None, parts=False):
"""routed(x) + shared(x) * gate. xe / xg: separate inputs of the routed experts and of the router
(Gemma-4 normalises each differently); parts = True returns (routed, gated shared)."""
idx = reg.get("adapter") if reg is not None else None
L = lambda n, t: _lora(t, self.la[n], self.lb[n], idx)
xs = x
x = x if xe is None else xe
logits = self.router(x if xg is None else xg)
if torch.isfinite(self.router_cap):
logits = self.router_cap * torch.tanh(logits / self.router_cap)
p = logits.softmax(-1) if self.score_c == 0 else (
(1 - self.score_c) * logits.softmax(-1) + self.score_c * torch.sigmoid(logits))
sel = (logits if self.sel_logit != 0 else p) + self.sel_bias
G, kg = int(self.grp[0]), int(self.grp[1])
if G > 1 and kg < G: # keep the top-kg groups' experts
sg = sel.view(*sel.shape[:-1], G, -1)
gs = sg.max(-1).values if self.grp[2] != 0 else sg.topk(2, -1).values.sum(-1)
keep = torch.zeros_like(gs).scatter_(-1, gs.topk(kg, -1).indices, 1.0)
sel = sel.masked_fill(keep.repeat_interleave(sel.shape[-1] // G, -1) == 0, float(self.grp[3]))
topi = sel.topk(self.top_k, -1).indices
topv = p.gather(-1, topi)
if self.smix_eps > 0: # SparseMixer: argmax, then the next, each own-masked softmax
e2, s_, picks, ws = 2 * self.smix_eps, logits, [], []
for _ in range(2):
m_, i_ = s_.max(-1, keepdim=True)
fac = logits.abs().clamp(min=m_)
ws.append(s_.masked_fill((m_ - logits) / fac > e2, float("-inf")).softmax(-1).gather(-1, i_))
picks.append(i_)
s_ = s_.scatter(-1, i_, float("-inf"))
topi, topv = torch.cat(picks, -1), torch.cat(ws, -1)
topv = (topv + self.renorm * (topv / (topv.sum(-1, keepdim=True) + self.renorm_eps) - topv)) * self.route_scale
topv = topv * self.exp_scale[topi]
h = self._act(torch.einsum("efd,btd->btef", self.w_gate, x) + self.b_gate,
torch.einsum("efd,btd->btef", self.w_up, x) + self.b_up, True)
eo = torch.einsum("edf,btef->bted", self.w_down, h) + self.b_down # every expert, then select
routed = (eo.gather(2, topi[..., None].expand(*topi.shape, x.shape[-1])) * topv[..., None]).sum(2)
if self.r_c != 0:
rn = routed * torch.rsqrt(routed.pow(2) @ self.r_avg + self.r_eps) * self.r_w
routed = (1 - self.r_c) * routed + self.r_c * self.r_up(rn)
x = xs
hs = self._act(self.s_gate(x) + L("g", x), self.s_up(x) + L("u", x))
shared = self.s_down(hs) + L("d", hs)
sg = self.s_coef[0] + self.s_coef[1] * torch.sigmoid(self.s_router(x))
return (routed, shared * sg) if parts else routed + shared * sg
class UEngram(nn.Module):
"""Hashed n-gram memory (Engram): the token embedding generalised from the current token to
hashed n-grams of the input ids. Each (order n, hash head h) folds the last n ids with a prime,
x <- x P_h + id + 1, and reads row (x ^ (x >> shift)) mod slots of its own table; the rows are
concatenated into e. Added to the residual stream before the layer:
g = sigmoid(<rms(x) w_h, rms(key e) w_k> / sqrt(d)), v = g * val(e), out = v + causal_conv(v).
Off (c = 0) unless a target sets it; ids are the model's input ids (left-padded with `pad`)."""
PRIMES = (1000003, 999983, 1000033, 999979, 1000037, 999961, 1000039, 999953)
def __init__(self, c: UConfig):
super().__init__()
d, S, Hh, D = c.d, c.engram_slots, c.engram_heads, c.engram_dim
self.orders, self.heads, self.slots, self.pad, self.shift = tuple(c.engram_orders), Hh, S, c.vocab, 17
self.register_buffer("primes", torch.tensor(self.PRIMES, dtype=torch.long))
self.table = _p(len(self.orders) * Hh * S, D)
mem = len(self.orders) * Hh * D
self.key, self.val = _lin(mem, d, bias=False), _lin(mem, d, bias=False)
self.w_h, self.w_k = _p(d, fill=1.0), _p(d, fill=1.0)
self.register_buffer("eps", torch.tensor([1e-6, 1e-6], dtype=F64))
self.conv_w = _p(d, c.engram_conv)
self.c = _p(1)
def hash_ids(self, ids):
B, T = ids.shape
maxn = max(self.orders)
p = F.pad(ids, (maxn - 1, 0), value=self.pad)
out, off = [], 0
for n in self.orders:
grams = [p[:, maxn - 1 - j: maxn - 1 - j + T] for j in range(n)]
for h in range(self.heads):
P = self.primes[h % len(self.primes)]
x = torch.zeros_like(ids)
for g in grams:
x = x * P + g + 1
out.append(torch.remainder(x ^ (x >> self.shift), self.slots) + off)
off += self.slots
return torch.stack(out, -1)
def forward(self, x, ids):
T = x.shape[1]
e = self.table[self.hash_ids(ids)].flatten(-2)
k, v = self.key(e), self.val(e)
g = torch.sigmoid((_rms(x, self.eps[0]) * self.w_h * _rms(k, self.eps[1]) * self.w_k).sum(-1, keepdim=True)
/ math.sqrt(x.shape[-1]))
v = g * v
K = self.conv_w.shape[1]
vp = F.pad(v.transpose(1, 2), (K - 1, 0))
y = sum(self.conv_w[:, j, None] * vp[:, :, j:j + T] for j in range(K)).transpose(1, 2)
return self.c * (v + y)
class ULayer(nn.Module):
def __init__(self, c: UConfig):
super().__init__()
self.engram = UEngram(c) if c.engram_slots else None
self.mem = UAttention(c, mem=True) if c.mem_attn else None # parallel memory read, own norm
self.norm_m = UNorm(c.d) if c.mem_attn else None
self.sn_in = UNorm(c.d) if c.stream_norms else None # x <- sn_in(x) at the start (sn_c[0])
self.sn_out = UNorm(c.d) if c.stream_norms else None # x <- sn_out(x) at the end (sn_c[1])
self.sn_c = _p(2)
self.hc_a = UHC(c) if c.hc_mult else None # hyper-connections (attn+SSM / FFN)
self.hc_f = UHC(c) if c.hc_mult else None
N = lambda: UNorm(c.d, c.norm_gate_rank)
self.norm_a, self.norm_s, self.norm_f = N(), N(), N()
self.attn, self.ssm, self.ffn = UAttention(c), USSM(c), UFFN(c)
self.post_a, self.post_s, self.post_f = N(), N(), N()
self.post_coef = _p(3, 2) # per sub-block [raw, post-normed] output
# split FFN (Gemma-4): the shared path reads norm_f(x) -> post_f1, the routed experts norm_f2(x) ->
# post_f2, the router norm_r(x); their sum goes through post_f (post_coef[2]). ffn_split = 0: off
self.ffn_split = _p(1)
self.norm_f2, self.norm_r, self.post_f1, self.post_f2 = UNorm(c.d), UNorm(c.d), UNorm(c.d), UNorm(c.d)
# per-layer embedding input (Gemma-4 PLE): x <- x + ple_post(ple_proj(gelu(ple_gate x) * pli_l))
dp = max(c.ple_dim, 1)
self.ple_c, self.ple_gate, self.ple_proj = _p(1), _lin(c.d, dp, bias=False), _lin(dp, c.d, bias=False)
self.ple_post = UNorm(c.d)
self.register_buffer("ple_tanh", torch.tensor(1.0, dtype=F64)) # its GeLU: 1 tanh-approx, 0 erf
self.register_buffer("out_scale", torch.tensor(1.0, dtype=F64)) # x <- out_scale x at the end (layer_scalar)
self.register_buffer("res_c", torch.ones(2, dtype=F64)) # residual coefficients x <- c x + f(x) (MiniMax alpha)
self.idx = 0
self.res_write, self.res_mix = _p(1), _p(1) # residual snapshot: write x; x <- x + mix (snap - x)
self.seq = _p(1, fill=1.0) # 1: SSM reads x + attn (sequential); 0: parallel
self.par = _p(1) # 1: FFN reads the LAYER INPUT (a parallel block,
# GPT-J / NeoX / Cohere: x + attn(n x) + ffn(n' x))
d = c.d # Block AttnRes queries / key-norm weights
self.ar_qa, self.ar_wa, self.ar_qf, self.ar_wf = _p(d), _p(d, fill=1.0), _p(d), _p(d, fill=1.0)
self.register_buffer("ar_eps", torch.tensor(1e-6, dtype=F64))
with torch.no_grad():
self.post_coef[:, 0] = 1.0
def _out(self, i, o, post):
pc = self.post_coef[i]
return pc[0] * o + (pc[1] * post(o) if pc[1] != 0 else 0)
def mix(self, sources, q, w):
"""Depth-softmax over residual sources: sum_s softmax_s(q . rms(src_s) * w) src_s."""
st = torch.stack(sources)
sc = (_rms(st, self.ar_eps) * w) @ q
return (st * sc.softmax(0)[..., None]).sum(0)
def forward_attnres(self, blocks, partial, causal, reg=None):
"""Block AttnRes: no running residual; inputs are depth-softmax mixes of the block sums."""
xa = self.mix(blocks + ([partial] if partial is not None else []), self.ar_qa, self.ar_wa)
a = self._out(0, self.attn(self.norm_a(xa), causal, reg), self.post_a)
s = self._out(1, self.ssm(self.norm_s(xa), reg), self.post_s)
partial = a + s if partial is None else partial + a + s
xf = self.mix(blocks + [partial], self.ar_qf, self.ar_wf)
return partial + self._out(2, self.ffn(self.norm_f(xf), reg), self.post_f)
def forward_fullres(self, sources, causal, reg=None):
"""Full AttnRes: each sub-block reads a depth-softmax mix over the embedding and every earlier
sub-block output, and appends its own output as a new source."""
xa = self.mix(sources, self.ar_qa, self.ar_wa)
a = self._out(0, self.attn(self.norm_a(xa), causal, reg), self.post_a)
s = self._out(1, self.ssm(self.norm_s(xa + self.seq * a), reg), self.post_s)
sources.append(a + s)
sources.append(self._out(2, self.ffn(self.norm_f(self.mix(sources, self.ar_qf, self.ar_wf)), reg), self.post_f))
def forward_hc(self, X, causal, reg=None):
"""N residual streams (mHC): each sub-block reads a pre-weighted collapse of the streams and
writes post * out + comb^T X."""
post, comb, h = self.hc_a(X)
a = self._out(0, self.attn(self.norm_a(h), causal, reg), self.post_a)
s = self._out(1, self.ssm(self.norm_s(h + self.seq * a), reg), self.post_s)
X = post[..., None] * (a + s)[..., None, :] + comb.transpose(-1, -2) @ X
post, comb, h = self.hc_f(X)
f = self._out(2, self.ffn(self.norm_f(h), reg), self.post_f)
return post[..., None] * f[..., None, :] + comb.transpose(-1, -2) @ X
def forward(self, x, causal, reg=None):
if self.engram is not None and self.engram.c != 0:
x = x + self.engram(x, reg["ids"])
if self.res_mix != 0: # x_snap + c (x - x_snap), mix = 1 - c (LFM3-RD)
x = x + self.res_mix * (reg["x_snap"] - x)
if self.res_write != 0:
reg["x_snap"] = x
if self.sn_in is not None and self.sn_c[0] != 0:
x = self.sn_in(x)
a = self._out(0, self.attn(self.norm_a(x), causal, reg), self.post_a)
s = self._out(1, self.ssm(self.norm_s(x + self.seq * a), reg), self.post_s)
m = self.mem(self.norm_m(x), causal, reg) if self.mem is not None else 0
x_in = x
x = (x if self.res_c[0] == 1 else self.res_c[0] * x) + a + s + m
xf = x_in if self.par != 0 else x
if self.res_c[1] != 1:
x = self.res_c[1] * x
if self.ffn_split != 0:
r_, s_ = self.ffn(self.norm_f(xf), reg, xe=self.norm_f2(xf), xg=self.norm_r(xf), parts=True)
x = x + self._out(2, self.post_f1(s_) + self.post_f2(r_), self.post_f)
else:
x = x + self._out(2, self.ffn(self.norm_f(xf), reg), self.post_f)
if self.ple_c != 0:
g = F.gelu(self.ple_gate(x), approximate="tanh" if self.ple_tanh != 0 else "none")
x = x + self.ple_c * self.ple_post(self.ple_proj(g * reg["ple"][:, :, self.idx]))
if self.sn_out is not None and self.sn_c[1] != 0:
x = self.sn_out(x)
return x * self.out_scale if self.out_scale != 1 else x
class UniversalElephantine(nn.Module):
def __init__(self, c: UConfig):
super().__init__()
self.cfg = c
self.embed = nn.Embedding(c.vocab, c.d).to(F64)
self.pos = _p(c.max_pos, c.d) # learned-abs table, added to the embedding
self.prefix = _p(c.n_prefix, c.d) # meta tokens
self.ar_q_out, self.ar_w_out = _p(c.d), _p(c.d, fill=1.0) # final AttnRes mix
self.sw_score, self.sw_val = _p(c.vocab), _p(c.vocab) # switch head: per-token key / value
self.emb_norm_c, self.emb_norm = _p(1), UNorm(c.d) # a norm of the embedding itself (Bloom)
self.layers = nn.ModuleList(ULayer(c) for _ in range(c.L))
for i, layer in enumerate(self.layers):
layer.idx = i
if c.ple_dim: # per-layer embeddings: pli = rms_l(P x0) w + E[ids]
self.ple_E = _p(c.ple_vocab or c.vocab, c.L * c.ple_dim)
self.ple_P = _lin(c.d, c.L * c.ple_dim, bias=False)
self.ple_w = _p(c.ple_dim, fill=1.0)
self.register_buffer("ple_eps", torch.tensor(1e-6, dtype=F64))
self.norm = UNorm(c.d, c.norm_gate_rank)
self.lm_head = _lin(c.d, c.vocab) # an lm_head bias (zero unless a target has one: Phi)
self.register_buffer("final_cap", torch.tensor(float("inf"), dtype=F64)) # cap tanh(logits / cap)
def forward(self, ids, causal=True):
x = self.embed(ids)
if self.cfg.max_pos:
x = x + self.pos[: ids.shape[1]]
if self.emb_norm_c != 0:
x = self.emb_norm(x)
n = self.cfg.n_prefix
if n:
x = torch.cat((self.prefix.expand(x.shape[0], -1, -1), x), 1)
reg = {"ids": ids}
if self.cfg.ple_dim:
B_, T_ = ids.shape
pp = self.ple_P(x).view(B_, T_, self.cfg.L, self.cfg.ple_dim)
reg["ple"] = _rms(pp, self.ple_eps) * self.ple_w + self.ple_E[ids].view(B_, T_, self.cfg.L, self.cfg.ple_dim)
if self.cfg.lora_adapters: # adapter index = round(causal softmax-avg of values)
sc, val = self.sw_score[ids], self.sw_val[ids]
T_ = ids.shape[1]
m = torch.ones(T_, T_, dtype=torch.bool).tril()
w = sc[:, None, :].expand(-1, T_, -1).masked_fill(~m, float("-inf")).softmax(-1)
avg = (w * val[:, None, :]).sum(-1)
reg["adapter"] = avg.round().long().clamp(0, self.cfg.lora_adapters)
B = self.cfg.attnres_block
if self.cfg.attnres_full:
sources = [x]
for layer in self.layers:
layer.forward_fullres(sources, causal, reg)
x = self.layers[0].mix(sources, self.ar_q_out, self.ar_w_out)
elif self.cfg.hc_mult: # N streams, collapsed by their mean at the end
X = x[:, :, None].expand(-1, -1, self.cfg.hc_mult, -1)
for layer in self.layers:
X = layer.forward_hc(X, causal, reg)
x = X.mean(2)
elif B:
blocks, partial = [x], None
for i, layer in enumerate(self.layers):
if i > 0 and i % B == 0:
blocks.append(partial); partial = None
partial = layer.forward_attnres(blocks, partial, causal, reg)
x = self.layers[0].mix(blocks + [partial], self.ar_q_out, self.ar_w_out)
else:
for layer in self.layers:
x = layer(x, causal, reg)
out = self.lm_head(self.norm(x)[:, n:])
return out if not torch.isfinite(self.final_cap) else self.final_cap * torch.tanh(out / self.final_cap)
# <<< vendored: elephantine/universal.py
# --- the transformers wrapper ------------------------------------------------------------------
def uconfig_of(config) -> UConfig:
"""The host's UConfig from an ElephantineConfig (unknown keys are an error, not ignored)."""
names = {f.name for f in fields(UConfig)}
host = dict(config.host)
extra = set(host) - names
if extra:
raise ValueError(f"host keys this architecture does not have: {sorted(extra)}")
if "engram_orders" in host:
host["engram_orders"] = tuple(host["engram_orders"])
return UConfig(**host)
class ElephantineForCausalLM(PreTrainedModel, GenerationMixin):
"""UniversalElephantine behind the transformers API. `causal` (config default; overridable per
call) picks the causal or the full (diffusion) mask. Logits are float64."""
config_class = ElephantineConfig
base_model_prefix = "model"
main_input_name = "input_ids"
# tensors left at the architecture's default are not stored: they keep the value __init__ gives
_keys_to_ignore_on_load_missing = [r".*"]
_supports_cache_class = False
_is_stateful = True # no KV cache: never build one
def __init__(self, config: ElephantineConfig):
super().__init__(config)
self.model = UniversalElephantine(uconfig_of(config))
self.post_init()
@classmethod
def get_init_context(cls, *args, **kwargs):
# build on CPU with the architecture's own init (no meta device), so a tensor the weights file
# leaves out keeps its default value instead of uninitialised memory
return [no_init_weights()]
def _init_weights(self, module):
pass # the defaults are UniversalElephantine's own
def get_input_embeddings(self):
return self.model.embed
def set_input_embeddings(self, value):
self.model.embed = value
def forward(self, input_ids=None, attention_mask=None, labels=None, causal=None, return_dict=None, **kwargs):
if attention_mask is not None and not bool((attention_mask == 1).all()):
raise ValueError("padding masks are not supported: pass unpadded sequences of equal length")
causal = self.config.causal if causal is None else bool(causal)
logits = self.model(input_ids, causal=causal)
loss = None
if labels is not None:
lg, lb = (logits[:, :-1], labels[:, 1:]) if causal else (logits, labels)
loss = F.cross_entropy(lg.reshape(-1, lg.shape[-1]), lb.reshape(-1), ignore_index=-100)
return CausalLMOutput(loss=loss, logits=logits)
def prepare_inputs_for_generation(self, input_ids, **kwargs):
return {"input_ids": input_ids}
def _supports_default_dynamic_cache(self):
return False
"""modeling_elephantine.py: the ONE universal elephantine architecture as a transformers model.
Self-contained remote code: the architecture below is a verbatim copy of elephantine/universal.py
(everything from `F64 = torch.float64` up to the loaders) and of `gd_recurrence` from
attn/gated_delta_layer.py, the general gated-delta recurrence the SSM branch runs. Pure torch:
no fla, mamba_ssm or causal_conv1d needed. The copy is regenerated by
`python -m elephantine.push_hub_demo --vendor`, and tests/test_hub_demo.py checks it is
identical to universal.py, source and forward.
Every layer is x += Attn(norm_a(x)); x += SSM(norm_s(x)); x += FFN(norm_f(x)), and every
op a family may or may not use is a branch with a coefficient, off by default. A target model
is a weight setting of this one class; see the model card for the settings in this repo.
Computes in float64 (the weights file stores each tensor in the narrowest dtype that holds it
exactly; they are widened to float64 at load). No KV cache: generation re-runs the prefix.
"""
from __future__ import annotations
import math
from dataclasses import dataclass, fields
import torch
import torch.nn.functional as F
from torch import nn
from transformers import GenerationMixin, PreTrainedModel
from transformers.modeling_outputs import CausalLMOutput
try:
from transformers.modeling_utils import no_init_weights
except ImportError: # newer transformers keep it in initialization
from transformers.initialization import no_init_weights
try:
from .configuration_elephantine import ElephantineConfig
except ImportError: # imported as a plain module (tests)
from configuration_elephantine import ElephantineConfig
# >>> vendored: attn/gated_delta_layer.py gd_recurrence
def gd_recurrence(q, k, v, a, gamma, w):
"""Batched general gated-delta rule (the `attn.ssm_cross.general` recurrence over B,H):
S_t = Diag(a_t) S_{t-1} - gamma_t (S_{t-1} k_t) k_t^T + w_t v_t k_t^T, y_t = S_t q_t
q,k,a:(B,H,T,dk) v:(B,H,T,dv) gamma,w:(B,H,T) -> y:(B,H,T,dv)."""
B, Hn, T, dk = k.shape
dv = v.shape[-1]
S = torch.zeros(B, Hn, dv, dk, dtype=q.dtype)
out = []
for t in range(T):
S = S * a[:, :, t, None, :] # per-key-channel decay
Sk = (S * k[:, :, t, None, :]).sum(-1) # S @ k_t -> (B,H,dv)
S = S - gamma[:, :, t, None, None] * Sk[..., None] * k[:, :, t, None, :] # delta remove
S = S + w[:, :, t, None, None] * v[:, :, t, :, None] * k[:, :, t, None, :] # write v ⊗ k
out.append((S * q[:, :, t, None, :]).sum(-1)) # S @ q_t
return torch.stack(out, 2)
# <<< vendored: attn/gated_delta_layer.py gd_recurrence
# >>> vendored: elephantine/universal.py
F64 = torch.float64
class UConfig:
vocab: int
d: int # hidden width
L: int # layers
heads: int = 1 # attention heads (MHA; GQA targets tile their KV)
head_dim: int = 2
ssm_in: int = 1 # in_proj width (the conv runs over all of it)
conv_k: int = 1
ssm_heads: int = 1
dk: int = 1
dv: int = 1
experts: int = 1
expert_ff: int = 1
top_k: int = 1
shared_ff: int = 1
ssm_norm_feats: int = 0 # normalized knob features (B/C/dt RMS-normed before use)
ssm_sig_feats: int = 0 # sigmoid knob features (low-rank tanh / sigmoid gates)
ssm_prod_feats: int = 0 # product features f = (Px r) * (Py r + b) (RWKV-6 ddlerp)
ssm_sig2_feats: int = 0 # a second sigmoid stage reading the product features
n_prefix: int = 0 # learned prefix (meta) tokens, prepended then stripped
attnres_block: int = 0 # Block AttnRes: depth-softmax residual over block sums (0 = off)
attnres_full: bool = False # Full AttnRes: every sub-block output is its own source (GPT-S3)
attn_norm_feats: int = 0 # attention normalized-latent features (real MLA's q / kv latent norms)
lora_adapters: int = 0 # switched LoRA adapters selected per token by a switch head (aLoRA)
lora_rank: int = 1
max_pos: int = 0 # learned absolute positions (0 = none; the spec carries both)
ssm_dconv: int = 0 # dynamic (per-token generated) causal conv on v, window W (0 = off; Jet)
ssm_dconv_rank: int = 1 # its kernel generator's hidden width
norm_gate_rank: int = 0 # gated norms: rms(x) w * c * sigmoid(down(silu(up(.)))) (GigaChat-3.5)
attn_norm_gate_rank: int = 0 # the same gate on the attention normalized-latent features
engram_slots: int = 0 # hashed n-gram memory (Engram) per layer: slots per (order, hash head); 0 = none
engram_orders: tuple = (2, 3)
engram_heads: int = 1
engram_dim: int = 1
engram_conv: int = 1
mem_attn: bool = False # a second, parallel attention per layer over a score-selected memory (EdgeRock)
mem_kv_ff: int = 0 # its key/value writer MLP width
mem_sel_rank: int = 0 # its admission-score MLP rank