"""modeling_elephantine.py: the ONE universal elephantine architecture as a transformers model. Self-contained remote code: the architecture below is a verbatim copy of elephantine/universal.py (everything from `F64 = torch.float64` up to the loaders) and of `gd_recurrence` from attn/gated_delta_layer.py, the general gated-delta recurrence the SSM branch runs. Pure torch: no fla, mamba_ssm or causal_conv1d needed. The copy is regenerated by `python -m elephantine.push_hub_demo --vendor`, and tests/test_hub_demo.py checks it is identical to universal.py, source and forward. Every layer is x += Attn(norm_a(x)); x += SSM(norm_s(x)); x += FFN(norm_f(x)), and every op a family may or may not use is a branch with a coefficient, off by default. A target model is a weight setting of this one class; see the model card for the settings in this repo. Computes in float64 (the weights file stores each tensor in the narrowest dtype that holds it exactly; they are widened to float64 at load). No KV cache: generation re-runs the prefix. """ from __future__ import annotations import math from dataclasses import dataclass, fields import torch import torch.nn.functional as F from torch import nn from transformers import GenerationMixin, PreTrainedModel from transformers.modeling_outputs import CausalLMOutput try: from transformers.modeling_utils import no_init_weights except ImportError: # newer transformers keep it in initialization from transformers.initialization import no_init_weights try: from .configuration_elephantine import ElephantineConfig except ImportError: # imported as a plain module (tests) from configuration_elephantine import ElephantineConfig # >>> vendored: attn/gated_delta_layer.py gd_recurrence def gd_recurrence(q, k, v, a, gamma, w): """Batched general gated-delta rule (the `attn.ssm_cross.general` recurrence over B,H): S_t = Diag(a_t) S_{t-1} - gamma_t (S_{t-1} k_t) k_t^T + w_t v_t k_t^T, y_t = S_t q_t q,k,a:(B,H,T,dk) v:(B,H,T,dv) gamma,w:(B,H,T) -> y:(B,H,T,dv).""" B, Hn, T, dk = k.shape dv = v.shape[-1] S = torch.zeros(B, Hn, dv, dk, dtype=q.dtype) out = [] for t in range(T): S = S * a[:, :, t, None, :] # per-key-channel decay Sk = (S * k[:, :, t, None, :]).sum(-1) # S @ k_t -> (B,H,dv) S = S - gamma[:, :, t, None, None] * Sk[..., None] * k[:, :, t, None, :] # delta remove S = S + w[:, :, t, None, None] * v[:, :, t, :, None] * k[:, :, t, None, :] # write v ⊗ k out.append((S * q[:, :, t, None, :]).sum(-1)) # S @ q_t return torch.stack(out, 2) # <<< vendored: attn/gated_delta_layer.py gd_recurrence # >>> vendored: elephantine/universal.py F64 = torch.float64 @dataclass class UConfig: vocab: int d: int # hidden width L: int # layers heads: int = 1 # attention heads (MHA; GQA targets tile their KV) head_dim: int = 2 ssm_in: int = 1 # in_proj width (the conv runs over all of it) conv_k: int = 1 ssm_heads: int = 1 dk: int = 1 dv: int = 1 experts: int = 1 expert_ff: int = 1 top_k: int = 1 shared_ff: int = 1 ssm_norm_feats: int = 0 # normalized knob features (B/C/dt RMS-normed before use) ssm_sig_feats: int = 0 # sigmoid knob features (low-rank tanh / sigmoid gates) ssm_prod_feats: int = 0 # product features f = (Px r) * (Py r + b) (RWKV-6 ddlerp) ssm_sig2_feats: int = 0 # a second sigmoid stage reading the product features n_prefix: int = 0 # learned prefix (meta) tokens, prepended then stripped attnres_block: int = 0 # Block AttnRes: depth-softmax residual over block sums (0 = off) attnres_full: bool = False # Full AttnRes: every sub-block output is its own source (GPT-S3) attn_norm_feats: int = 0 # attention normalized-latent features (real MLA's q / kv latent norms) lora_adapters: int = 0 # switched LoRA adapters selected per token by a switch head (aLoRA) lora_rank: int = 1 max_pos: int = 0 # learned absolute positions (0 = none; the spec carries both) ssm_dconv: int = 0 # dynamic (per-token generated) causal conv on v, window W (0 = off; Jet) ssm_dconv_rank: int = 1 # its kernel generator's hidden width norm_gate_rank: int = 0 # gated norms: rms(x) w * c * sigmoid(down(silu(up(.)))) (GigaChat-3.5) attn_norm_gate_rank: int = 0 # the same gate on the attention normalized-latent features engram_slots: int = 0 # hashed n-gram memory (Engram) per layer: slots per (order, hash head); 0 = none engram_orders: tuple = (2, 3) engram_heads: int = 1 engram_dim: int = 1 engram_conv: int = 1 mem_attn: bool = False # a second, parallel attention per layer over a score-selected memory (EdgeRock) mem_kv_ff: int = 0 # its key/value writer MLP width mem_sel_rank: int = 0 # its admission-score MLP rank stream_norms: bool = False # per-layer norms OF the residual stream itself, at layer start / end (CENO) hc_mult: int = 0 # manifold-constrained hyper-connections: N residual streams (0 = one plain stream) hc_iters: int = 1 # their Sinkhorn iterations dsa_heads: int = 0 # DeepSeek sparse attention indexer heads (0 = dense attention) dsa_dim: int = 1 # its head dim ple_dim: int = 0 # per-layer embeddings (Gemma-3n / 4 PLE): width per layer (0 = none) ple_vocab: int = 0 # their token table's vocabulary (0: the model's) def _lin(i, o, bias=True): # skip PyTorch's default init: the weights are zeroed right away m = torch.nn.utils.skip_init(nn.Linear, i, o, bias=bias, dtype=F64) with torch.no_grad(): m.weight.zero_() if bias: m.bias.zero_() return m def _p(*shape, fill=0.0): return nn.Parameter(torch.full(shape, fill, dtype=F64)) def _lora(x, A, B, idx): """Switched LoRA: token t with adapter idx[t] = a > 0 gets B[a-1] (A[a-1] x).""" if idx is None or A.shape[0] == 0 or not bool((idx > 0).any()): return 0 out = 0 for e in range(A.shape[0]): m = (idx == e + 1)[..., None].to(x.dtype) if m.any(): out = out + m * ((x @ A[e].T) @ B[e].T) return out def _rms(x, eps): return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + eps) def _sig_gate(y, c, up, down): """Low-rank self-gate y * c * sigmoid(down(silu(up(y)))) (off when c = 0).""" return y if c == 0 else y * c * torch.sigmoid(down(F.silu(up(y)))) class UNorm(nn.Module): """RMSNorm with a settable weight and eps; centre = 1 plus a bias makes it LayerNorm; gate_c != 0 multiplies by a low-rank sigmoid self-gate (GigaChat-3.5's gated norm).""" def __init__(self, d, gate_rank=0): super().__init__() self.weight = _p(d, fill=1.0) self.bias, self.centre = _p(d), _p(1) self.register_buffer("eps", torch.tensor(1e-6, dtype=F64)) self.gate_c = _p(1) self.g_up, self.g_down = _lin(d, max(gate_rank, 1), bias=False), _lin(max(gate_rank, 1), d, bias=False) self.pre_c, self.pre_w = _p(1), _p(d, fill=1.0) # a first RMSNorm stage (two norms in series) self.register_buffer("pre_eps", torch.tensor(1e-6, dtype=F64)) self.ident = _p(1) # 1: no norm at all (post-norm-only layers: OLMo-2) def forward(self, x): if self.ident != 0: return x if self.pre_c != 0: x = _rms(x, self.pre_eps) * self.pre_w if self.centre != 0: x = x - self.centre * x.mean(-1, keepdim=True) return _sig_gate(_rms(x, self.eps) * self.weight + self.bias, self.gate_c, self.g_up, self.g_down) def _rotate_half(x): h = x.shape[-1] // 2 return torch.cat((-x[..., h:], x[..., :h]), -1) def _select_mask(sc, chunk, cap): """Score-selected memory: key s is visible to query t iff s precedes t's chunk, s < c(t) = floor(t / chunk) chunk, and s is among the `cap` highest scores of the keys before c(t) (ties to the earlier position). sc (B,H,T) -> (B,H,T,S) bool.""" T = sc.shape[-1] pos = torch.arange(T) before = pos[None, :] < (pos // chunk * chunk)[:, None] # (T, S) a, b = sc[..., :, None], sc[..., None, :] # a: s', b: s beats = (a > b) | ((a == b) & (pos[:, None] < pos[None, :])) # s' outranks s rank = torch.einsum("tu,bhus->bhts", before.to(sc.dtype), beats.to(sc.dtype)) return before & (rank < cap) class UAttention(nn.Module): def __init__(self, c: UConfig, mem=False): super().__init__() H, hd, d = c.heads, c.head_dim, c.d self.H, self.hd = H, hd Dn = c.attn_norm_feats self.Dn = Dn # q/k/v/g read [x, nb], nb = rms_grouped(pn x) * n_w self.pn = _lin(d, max(Dn, 1), bias=False) self.n_avg, self.n_w = _p(max(Dn, 1), max(Dn, 1)), _p(max(Dn, 1), fill=1.0) self.register_buffer("n_eps", torch.tensor(1e-6, dtype=F64)) gr = max(c.attn_norm_gate_rank, 1) # gated latent norms (block-diagonal up / down) self.n_gate_c, self.n_up, self.n_down = _p(1), _lin(max(Dn, 1), gr, bias=False), _lin(gr, max(Dn, 1), bias=False) din = d + Dn self.q_proj, self.k_proj, self.v_proj = _lin(din, H * hd), _lin(din, H * hd), _lin(din, H * hd) self.g_proj, self.o_proj = _lin(din, H * hd), _lin(H * hd, d) E, r = c.lora_adapters, c.lora_rank self.la = nn.ParameterDict({n: _p(E, r, i) for n, i in (("q", din), ("k", din), ("v", din), ("o", H * hd))}) self.lb = nn.ParameterDict({n: _p(E, o, r) for n, o in (("q", H * hd), ("k", H * hd), ("v", H * hd), ("o", d))}) self.q_nw, self.k_nw = _p(H, hd, fill=1.0), _p(H, hd, fill=1.0) # per-head qk-norm weights self.register_buffer("qk_eps", torch.tensor([1e-6, 1e-6], dtype=F64)) self.q_coef, self.k_coef = _p(2), _p(2) # [plain, normed] self.gate_coef = _p(2) # [constant 1, sigmoid(g_proj x)] with torch.no_grad(): self.q_coef[0] = self.k_coef[0] = self.gate_coef[0] = 1.0 self.inv_freq = _p(hd // 2) # 0 = NoPE on that pair self.scale = _p(1, fill=1 / math.sqrt(hd)) C = H * hd self.o_hmix = nn.Parameter(torch.eye(H, dtype=F64)) # head mix before the output norm (DIFF attn) self.o_coef = _p(2) # [raw, RMS-normed] attention output self.o_avg, self.o_w = _p(C, C), _p(C, fill=1.0) self.register_buffer("o_eps", torch.tensor(1e-6, dtype=F64)) self.kv_read, self.kv_write = _p(1), _p(1) # cross-layer KV register (post-RoPE k, v) self.register_buffer("window", torch.tensor(float("inf"), dtype=F64)) # t - s <= window self.register_buffer("n_sink", torch.tensor(0.0, dtype=F64)) # s < n_sink always visible # memory-read branches (EdgeRock's independent memory; off by default): f, r = max(c.mem_kv_ff if mem else 0, 1), max(c.mem_sel_rank if mem else 0, 1) self.kv_c = _p(1) # k, v read z = x + kv_c down(silu(gate x) * up x) self.kv_gate, self.kv_up, self.kv_down = _lin(din, f, bias=False), _lin(din, f, bias=False), _lin(f, din, bias=False) self.sel_down, self.sel_up = _lin(din, r, bias=False), _lin(r, H, bias=False) # score up(silu(down x)) self.register_buffer("sel_cap", torch.tensor(float("inf"), dtype=F64)) # keys kept (inf = off) self.register_buffer("sel_chunk", torch.tensor(1.0, dtype=F64)) self.register_buffer("null_logit", torch.full((H,), float("-inf"), dtype=F64)) # a zero-value null key # per head (learned sinks: GPT-OSS; -inf = off) self.register_buffer("cap", torch.tensor(float("inf"), dtype=F64)) # logit softcap cap tanh(s / cap) self.alibi = _p(H) # ALiBi slopes: s += alibi_h (s_pos - t_pos) self.register_buffer("qkv_clip", torch.tensor(float("inf"), dtype=F64)) # q, k, v clamped to +-clip # qk-norm variants: qk_full = 1 normalises over the WHOLE projection (all heads: OLMo-2 / 3), qk_centre # [q, k] = 1 makes it a LayerNorm (mean-centred, + q_nb / k_nb biases: StableLM, Persimmon, Phi); # qk_post = 1 applies it AFTER the rotary (HunYuan) self.qk_full, self.qk_centre, self.qk_post = _p(1), _p(2), _p(1) self.q_nb, self.k_nb = _p(H, hd), _p(H, hd) # the width the per-head q / k / v norms average over (< hd when a narrower target head is zero-padded) self.register_buffer("qk_width", torch.tensor(float(hd), dtype=F64)) self.v_c, self.v_nw = _p(1), _p(H, hd, fill=1.0) # v-norm: v <- rms_head(v) v_nw (Gemma-4) self.register_buffer("v_eps", torch.tensor(1e-6, dtype=F64)) self.register_buffer("kv_slot", torch.tensor(0.0, dtype=F64)) # which KV register (per layer type) # Llama-4 / Ministral-3 query temperature: q_t *= 1 + beta log(1 + floor(t / M)), [beta, M] (beta = 0: off) self.register_buffer("qtemp", torch.tensor([0.0, 1.0], dtype=F64)) self.gate_sp = _p(1) # + gate_sp softplus(g_proj x) in the output gate (Laguna) self.hg_c, self.hg_w, self.hg_b = _p(H), _p(H, hd), _p(H) # head gate c sigmoid( + b) # grouped differential heads (Motif-3 GDLA): o_h <- o_h - dl_c m_h sigmoid(dl_proj x)_h o_{src(h)} self.dl_c, self.dl_proj = _p(1), _lin(din, H, bias=False) self.register_buffer("dl_src", torch.arange(H)) self.register_buffer("dl_m", torch.zeros(H, dtype=F64)) # DSA indexer (GLM-5.3 / DeepSeek-V3.2 with k-pool compression): a key is visible iff it is in one of # the top-scoring complete pools of `dsa_pool` keys, or in the query's incomplete tail pool nI, dI = max(c.dsa_heads, 1), c.dsa_dim self.dsa_on = c.dsa_heads > 0 self.iq, self.ik, self.igate = _lin(din, nI * dI, bias=False), _lin(din, dI, bias=False), _lin(din, dI, bias=False) self.ik_w, self.ik_b, self.i_ape = _p(dI, fill=1.0), _p(dI), _p(1, dI) self.iw = _lin(din, nI, bias=False) self.register_buffer("ik_eps", torch.tensor(1e-6, dtype=F64)) self.register_buffer("dsa_pool", torch.tensor(1.0, dtype=F64)) self.register_buffer("dsa_topk", torch.tensor(float("inf"), dtype=F64)) self.register_buffer("dsa_tail", torch.tensor(1.0, dtype=F64)) self.i_inv = _p(max(dI // 2, 1)) # RoPE on the indexer q / k (pairs (i, i + dI/2); 0 = NoPE) self.register_buffer("dsa_share", torch.tensor(0.0, dtype=F64)) # 1: publish the mask, 2: reuse it with torch.no_grad(): self.o_coef[0] = 1.0 def _qk_norm(self, t, i): """[plain, normed] mix of q (i = 0) or k (i = 1): RMS-normed per head, or over every head (qk_full), optionally mean-centred with a bias (qk_centre: a LayerNorm).""" coef = (self.q_coef, self.k_coef)[i] if coef[1] == 0: return coef[0] * t dims = (-2, -1) if self.qk_full != 0 else (-1,) n = t - t.mean(dims, keepdim=True) if self.qk_centre[i] != 0 else t width = self.qk_width * (t.shape[-2] if self.qk_full != 0 else 1) n = n * torch.rsqrt(n.pow(2).sum(dims, keepdim=True) / width + self.qk_eps[i]) * (self.q_nw, self.k_nw)[i] if self.qk_centre[i] != 0: n = n + (self.q_nb, self.k_nb)[i] return coef[0] * t + coef[1] * n def forward(self, x, causal, reg=None): B, T, _ = x.shape sh = lambda t: t.view(B, T, self.H, self.hd) if self.Dn: n = self.pn(x) n = n * torch.rsqrt(n.pow(2) @ self.n_avg + self.n_eps) * self.n_w x = torch.cat((x, _sig_gate(n, self.n_gate_c, self.n_up, self.n_down)), -1) idx = reg.get("adapter") if reg is not None else None L = lambda n, t: _lora(t, self.la[n], self.lb[n], idx) xk = x if self.kv_c == 0 else x + self.kv_c * self.kv_down(F.silu(self.kv_gate(x)) * self.kv_up(x)) q, k, v = sh(self.q_proj(x) + L("q", x)), sh(self.k_proj(xk) + L("k", xk)), sh(self.v_proj(xk) + L("v", xk)) if self.v_c != 0: v = v * torch.rsqrt(v.pow(2).sum(-1, keepdim=True) / self.qk_width + self.v_eps) * self.v_nw ang = torch.arange(T, dtype=F64)[:, None] * self.inv_freq[None] cos = torch.cat((ang.cos(), ang.cos()), -1)[None, :, None] sin = torch.cat((ang.sin(), ang.sin()), -1)[None, :, None] rope = lambda t: t * cos + _rotate_half(t) * sin if self.qk_post != 0: q, k = rope(q), rope(k) q, k = self._qk_norm(q, 0), self._qk_norm(k, 1) if torch.isfinite(self.qkv_clip): # after the qk-norm (OLMoE), before the rotary c = self.qkv_clip q, k, v = q.clamp(-c, c), k.clamp(-c, c), v.clamp(-c, c) qn = q if self.qk_post == 0: q, k = rope(q), rope(k) if self.qtemp[0] != 0: t_ = torch.arange(T, dtype=F64) q = q * (1 + self.qtemp[0] * torch.log1p(torch.floor(t_ / self.qtemp[1])))[None, :, None, None] if reg is not None: sk, sv = ("k", "v") if self.kv_slot == 0 else (f"k{int(self.kv_slot)}", f"v{int(self.kv_slot)}") if self.kv_read != 0: k = (1 - self.kv_read) * k + self.kv_read * reg[sk] v = (1 - self.kv_read) * v + self.kv_read * reg[sv] if self.kv_write != 0: reg[sk], reg[sv] = k, v s = torch.einsum("bthd,bshd->bhts", q, k) * self.scale if torch.isfinite(self.cap): s = self.cap * torch.tanh(s / self.cap) tt, ss = torch.arange(T)[:, None], torch.arange(T)[None] if self.alibi.any(): s = s + self.alibi[:, None, None] * (ss - tt).to(F64) allowed = (tt - ss <= self.window) | (ss < self.n_sink) if causal: allowed = allowed & (ss <= tt) if torch.isfinite(self.sel_cap): sc = self.sel_up(F.silu(self.sel_down(x))).transpose(1, 2) allowed = allowed & _select_mask(sc, int(self.sel_chunk), self.sel_cap) if self.dsa_on and (torch.isfinite(self.dsa_topk) or self.dsa_share != 0): if self.dsa_share == 2: # GLM-5 'shared' indexer layers vis = reg["dsa"] elif torch.isfinite(self.dsa_topk): vis = self._dsa_mask(x) else: vis = torch.ones(B, T, T, dtype=torch.bool) if self.dsa_share == 1: reg["dsa"] = vis allowed = allowed & vis[:, None] s = s.masked_fill(~allowed, float("-inf")) if torch.isfinite(self.null_logit).any(): s = torch.cat((s, self.null_logit[:, None, None].expand(*s.shape[:-1], 1)), -1) o = torch.einsum("bhts,bshd->bthd", s.softmax(-1)[..., :T], v) if self.hg_c.any(): o = o * (self.hg_c * torch.sigmoid((qn * self.hg_w).sum(-1) + self.hg_b))[..., None] if self.dl_c != 0: # signal heads minus a gated copy of their noise head o = o - self.dl_c * (self.dl_m * torch.sigmoid(self.dl_proj(x)))[..., None] * o[:, :, self.dl_src] if not torch.equal(self.o_hmix, torch.eye(self.H, dtype=F64)): o = torch.einsum("bthd,hg->btgd", o, self.o_hmix) o = o.reshape(B, T, -1) if self.o_coef[1] != 0: o = self.o_coef[0] * o + self.o_coef[1] * o * torch.rsqrt(o.pow(2) @ self.o_avg + self.o_eps) * self.o_w if self.gate_sp != 0: o = o * (self.gate_coef[0] + self.gate_coef[1] * torch.sigmoid(self.g_proj(x)) + self.gate_sp * F.softplus(self.g_proj(x))) else: o = o * (self.gate_coef[0] + self.gate_coef[1] * torch.sigmoid(self.g_proj(x))) return self.o_proj(o) + L("o", o) def _dsa_mask(self, x): """DSA k-pool indexer -> visible keys (B,T,S). Pools of P consecutive keys; a pool's key is a per-channel softmax(gate x + ape) average of LayerNormed key projections; the pool score for a query is sum_h w_h relu( / sqrt(dI)); the top floor(topk / P) complete pools whose last key precedes the query are kept, plus the query's own incomplete tail.""" B, T, _ = x.shape P, nI, dI = int(self.dsa_pool), self.iw.out_features, self.ik.out_features q = self.iq(x).view(B, T, nI, dI) k = F.layer_norm(self.ik(x), (dI,), self.ik_w, self.ik_b, float(self.ik_eps)) if self.i_inv.any(): # indexer RoPE (DeepSeek-V3.2 / GLM-5) ang = torch.arange(T, dtype=F64)[:, None] * self.i_inv[None] cos, sin = torch.cat((ang.cos(), ang.cos()), -1), torch.cat((ang.sin(), ang.sin()), -1) q = q * cos[None, :, None] + _rotate_half(q) * sin[None, :, None] k = k * cos[None] + _rotate_half(k) * sin[None] gs = self.igate(x) tt = torch.arange(T) vis = torch.zeros(B, T, T, dtype=torch.bool) npool = T // P if npool and int(self.dsa_topk) // P: kk = k[:, :npool * P].view(B, npool, P, dI) gg = gs[:, :npool * P].view(B, npool, P, dI) + self.i_ape pk = (gg.softmax(2) * kk).sum(2) # (B, pools, dI) sc = F.relu(torch.einsum("bthd,bpd->bthp", q, pk) / math.sqrt(dI)) isc = torch.einsum("bth,bthp->btp", self.iw(x) / math.sqrt(nI), sc) ok = ((torch.arange(npool) * P + P - 1)[None, :] <= tt[:, None]).expand(B, T, npool) isc = isc.masked_fill(~ok, torch.finfo(isc.dtype).min) sel = isc.topk(min(int(self.dsa_topk) // P, npool), -1).indices keep = torch.zeros(B, T, npool, dtype=torch.bool).scatter_(-1, sel, True) & ok vis[:, :, :npool * P] = keep.repeat_interleave(P, -1) if self.dsa_tail != 0: # the incomplete visible pool start = (tt + 1) - (tt + 1) % P vis |= ((tt[None, :] >= start[:, None]) & (tt[None, :] <= tt[:, None]))[None] return vis UAttention._dsa_mask = _dsa_mask class UHC(nn.Module): """Manifold-constrained hyper-connections around one sub-block: from the N streams X (RMS-normed flat) a linear map gives pre (sigmoid), post (2 sigmoid) and comb (softmax, then Sinkhorn to doubly stochastic); the block reads sum_i pre_i X_i and the streams become post * out + comb^T X.""" def __init__(self, c: UConfig): super().__init__() N, d = c.hc_mult, c.d self.N, self.iters = N, c.hc_iters self.fn, self.base, self.scale = _p((2 + N) * N, N * d), _p((2 + N) * N), _p(3, fill=1.0) self.register_buffer("eps", torch.tensor([1e-6, 1e-6], dtype=F64)) # [hc_eps, input-norm eps] self.register_buffer("post_c", torch.tensor(2.0, dtype=F64)) # post = post_c sigmoid(.) # Motif-3 variant (sk = 1): pre / post logits clamped to +-lim[0]; comb = Sinkhorn of exp(clamp(logits, # +-lim[1])), iters x (row, column) normalisation with sums clamped >= 1e-8; read transposed # (the streams become post * out + H_res X) self.register_buffer("sk", torch.tensor(0.0, dtype=F64)) self.register_buffer("lim", torch.tensor([10.0, 20.0], dtype=F64)) def forward(self, X): B, T, N, d = X.shape flat = X.reshape(B, T, N * d) flat = flat * torch.rsqrt(flat.pow(2).mean(-1, keepdim=True) + self.eps[1]) w = flat @ self.fn.T pw, qw, cw = w.split([N, N, N * N], -1) pb, qb, cb = self.base.split([N, N, N * N]) if self.sk != 0: l0, l1 = self.lim pre = torch.sigmoid((pw * self.scale[0] + pb).clamp(-l0, l0)) post = self.post_c * torch.sigmoid((qw * self.scale[1] + qb).clamp(-l0, l0)) m = (cw * self.scale[2] + cb).view(B, T, N, N).clamp(-l1, l1).exp() for _ in range(self.iters): m = m / m.sum(-1, keepdim=True).clamp(min=1e-8) m = m / m.sum(-2, keepdim=True).clamp(min=1e-8) return post, m.transpose(-1, -2), (pre[..., None] * X).sum(2) e = self.eps[0] pre = torch.sigmoid(pw * self.scale[0] + pb) + e post = self.post_c * torch.sigmoid(qw * self.scale[1] + qb) comb = torch.softmax((cw * self.scale[2] + cb).view(B, T, N, N), -1) + e comb = comb / (comb.sum(-2, keepdim=True) + e) for _ in range(self.iters - 1): comb = comb / (comb.sum(-1, keepdim=True) + e) comb = comb / (comb.sum(-2, keepdim=True) + e) return post, comb, (pre[..., None] * X).sum(2) def _rot_pairs(t, theta): """Rotate key-dim pairs (j, j + dk/2) by theta (B,H,dk/2).""" h = t.shape[-1] // 2 c, s_ = theta.cos(), theta.sin() t0, t1 = t[..., :h], t[..., h:2 * h] return torch.cat((t0 * c - t1 * s_, t0 * s_ + t1 * c, t[..., 2 * h:]), -1) def _recurrence(q, k, v, a, gamma, w, w_prev, dth, rm=None, pre=None, er=None): """gd_recurrence plus: a previous-token write added before the decay (Mamba-3's trapezoid), q/k rotated by a cumulative angle (Mamba-3, RoPE), and RWKV-7's generalized removal S -= c (S_{t-1} kap)(kap * beta)^T read from the PRE-decay state (rm = (c, kap, rmv)), and a per-channel erase gate on the delta term's READ key (er: S -= gamma (S (er * k)) k^T, GDN-2).""" B, H, T, dk = k.shape S = torch.zeros(B, H, v.shape[-1], dk, dtype=q.dtype) theta = torch.zeros(B, H, dk // 2, dtype=q.dtype) kp = torch.zeros(B, H, dk, dtype=q.dtype); vp = torch.zeros(B, H, v.shape[-1], dtype=q.dtype) out = [] for t in range(T): qt, kt = q[:, :, t], k[:, :, t] S0 = S if dth is not None: theta = theta + dth[:, :, t] qt, kt = _rot_pairs(qt, theta), _rot_pairs(kt, theta) if w_prev is not None: S = S + w_prev[:, :, t, None, None] * vp[..., None] * kp[:, :, None, :] if rm is not None: Skap = (S * rm[1][:, :, t, None, :]).sum(-1) # pre-decay S kap S = S * a[:, :, t, None, :] if rm is not None: S = S - rm[0][None, :, None, None] * Skap[..., None] * rm[2][:, :, t, None, :] Sk = (S * (kt if er is None else kt * er[:, :, t])[:, :, None, :]).sum(-1) S = S - gamma[:, :, t, None, None] * Sk[..., None] * kt[:, :, None, :] S = S + w[:, :, t, None, None] * v[:, :, t, :, None] * kt[:, :, None, :] yt = (S * qt[:, :, None, :]).sum(-1) if pre is not None: # RWKV-6 reads S_{t-1} yt = yt + pre[None, :, None] * ((S0 * qt[:, :, None, :]).sum(-1) - yt) out.append(yt) kp, vp = kt, v[:, :, t] return torch.stack(out, 2) def _ridge_readout(q, k, a, ratio, n_iter): """Gated KalmaNet's readout: the key-covariance state H_t = Diag(a_t) H_{t-1} + k_t k_t^T (the gated-delta state with v = k, write 1, no delta term) and a fixed n_iter-step Chebyshev iteration for (H_t + lam_t I) x = q_t, lam_t = ratio ||H_t||_F (spectrum bound ||H_t||_F): step = 2 / (2 lam + |H|_F), rho = (|H|_F / (2 lam + |H|_F) / 2)^2, x_0 = step q, x_-1 = 0, w = 2 w <- 1 / (1 - rho w); x <- w x - (step w (x^T H + lam x - q) + (w - 1) x_prev) q, k, a: (B,H,T,dk) -> x: (B,H,T,dk).""" B, Hn, T, dk = k.shape S = torch.zeros(B, Hn, dk, dk, dtype=q.dtype) lam_r = ratio[None, :, None] out = [] for t in range(T): kt, qt = k[:, :, t], q[:, :, t] S = S * a[:, :, t, None, :] + kt[..., :, None] * kt[..., None, :] fro = S.pow(2).sum((-2, -1)).sqrt()[..., None] # (B,H,1) lam = lam_r * fro step = 2 / (2 * lam + fro) rho = (fro / (2 * lam + fro) / 2) ** 2 x_prev, x, w = torch.zeros_like(qt), step * qt, 2.0 for _ in range(n_iter): w = 1 / (1 - rho * w) g = step * w * ((x[..., :, None] * S).sum(-2) + lam * x - qt) + (w - 1) * x_prev x_prev, x = x, w * x - g out.append(x) return torch.stack(out, 2) class USSM(nn.Module): def __init__(self, c: UConfig): super().__init__() D, K, H, dk, dv, d = c.ssm_in, c.conv_k, c.ssm_heads, c.dk, c.dv, c.d self.D, self.K, self.H, self.dk, self.dv = D, K, H, dk, dv C = H * dv self.in_proj = _lin(d, D) self.conv_w, self.conv_b = _p(D, K), _p(D) self.act_silu, self.act_lin = _p(D), _p(D) # c' = act_silu*silu(c) + act_lin*c Dn = c.ssm_norm_feats self.Dn = Dn self.pn = _lin(2 * D, max(Dn, 1)) # normalized knob features nb self.n_avg, self.n_w = _p(max(Dn, 1), max(Dn, 1)), _p(max(Dn, 1), fill=1.0) self.register_buffer("n_eps", torch.tensor(1e-6, dtype=F64)) Dh, Dp, Dh2 = c.ssm_sig_feats, c.ssm_prod_feats, c.ssm_sig2_feats self.Dh, self.Dp, self.Dh2 = Dh, Dp, Dh2 R0 = 2 * D + Dn self.ph = _lin(R0, max(Dh, 1)) # stage 1: h = sigmoid(ph r) self.pfx, self.pfy = _lin(R0 + Dh, max(Dp, 1)), _lin(R0 + Dh, max(Dp, 1)) # stage 2: f self.ph2 = _lin(R0 + Dh + Dp, max(Dh2, 1)) # stage 3: h2 = sigmoid(ph2 r) R = R0 + Dh + Dp + Dh2 # knobs read [u, c', nb, h, f, h2] self.pq, self.pk = _lin(R, H * dk), _lin(R, H * dk) self.pv, self.pz = _lin(R, C), _lin(R, C) self.q_coef, self.k_coef = _p(2), _p(2) # [plain, l2-normed] with torch.no_grad(): self.q_coef[0] = self.k_coef[0] = 1.0 self.register_buffer("l2_eps", torch.tensor(1e-6, dtype=F64)) self.p1, self.p2 = _lin(R, H * dk), _lin(R, H * dk) self.A1, self.A2, self.A3 = _p(H, dk), _p(H, dk), _p(H, dk) self.mem = _p(H, dk, fill=1.0) self.pb, self.pe, self.pi = _lin(R, H), _lin(R, H), _lin(R, H) self.pA, self.A4, self.a_floor = _lin(R, H), _p(H, dk), _p(H) # Mamba-3 decay term self.p5, self.A5 = _lin(R, H * dk), _p(H, dk) # log a -= A5 exp(p5) (RWKV-6) self.c_pre = _p(H) # read S_{t-1} q (before this step's update) self.cg = _p(H) self.cg0 = _p(H) # constant delta strength: gamma = cg sigmoid(pb) + cg0 self.c_er, self.pbv = _p(H), _lin(R, H * dk) # erase gate on the read key: er = sigmoid(pbv r) (GDN-2) self.cw = _p(6, H) # softplus / sigmoid / exp / RG-LRU / const / dt*sigmoid self.cw_prev = _p(H) # previous-token write dt*(1-sigmoid(pb)) self.pang, self.rot_c = _lin(R, H * max(dk // 2, 1)), _p(H, dk // 2) # cumulative q/k rotation self.rot_k = _p(H, dk // 2) # constant angle step per pair (= RoPE) # RWKV-7 generalized delta: removal reads the PRE-decay state along kap (l2-normed) and # writes out along kap * beta; the write key is modulated by beta. self.pkap, self.pbeta = _lin(R, H * dk), _lin(R, H * dk) self.c_rm, self.c_ka = _p(H), _p(H, dk) self.register_buffer("kap_eps", torch.tensor(1e-24, dtype=F64)) # value register (RWKV-7 v_first): write v, or read v <- v + read * gate * (reg - v) self.pvr = _lin(R, H * dv) self.v_write, self.v_read = _p(1), _p(1) self.y_write, self.y_read = _p(1), _p(1) # SSM output register (SambaY GMU memory) # dynamic conv on v (Jet-Nemotron): v <- act(sum_j ker_t[c, j] v_{t-W+1+j}[c]), ker = w2 silu(w1 x) + b2 Wd = c.ssm_dconv self.dcW = Wd self.dc_w1 = _lin(d, max(c.ssm_dconv_rank, 1), bias=False) self.dc_w2 = _lin(max(c.ssm_dconv_rank, 1), C * max(Wd, 1)) self.dc_act = _p(1) # 1 = silu after the dynamic conv self.reset = _p(H) # RG-LRU: first token writes sigmoid(pi) # ridge readout (Gated KalmaNet): a second, key-covariance state H_t = Diag(a_t) H_{t-1} + k_t k_t^T # and a FIXED `ridge_iters`-step Chebyshev solve x_t ~ (H_t + lam_t I)^-1 q_t, lam_t = ridge_ratio ||H_t||_F; # the main state then reads q <- q + ridge_c * (pal r) * (x - q) self.ridge_c, self.ridge_ratio, self.pal = _p(H), _p(H), _lin(R, H) self.register_buffer("ridge_iters", torch.tensor(0)) self.register_buffer("l2_eps_out", torch.tensor(0.0, dtype=F64)) # != 0: l2 is t / (|t| + eps_out) self.Dskip = _p(H, dv) self.gate = _p(5, C) # silu / sigmoid / gelu-tanh / identity / const self.avg = _p(C, C) # RMS grouping as a weight (block averages) self.norm_w, self.norm_b = _p(C, fill=1.0), _p(C) self.mix = nn.Parameter(torch.eye(C, dtype=F64)) # channel mix of y before the output branches self.register_buffer("norm_eps", torch.tensor(1e-6, dtype=F64)) self.out_coef = _p(4, C) # norm-then-gate / gate-then-norm / plain / ratio self.register_buffer("dt_min", torch.tensor(0.0, dtype=F64)) # dt = max(softplus(p1), dt_min) self.ratio_num, self.ratio_den = _p(C, C), _p(C, C) self.out_proj = _lin(C, d) def _norm(self, t): return t * torch.rsqrt(t.pow(2) @ self.avg + self.norm_eps) * self.norm_w + self.norm_b def forward(self, x, reg=None): B, T, _ = x.shape H, dk, dv, K = self.H, self.dk, self.dv, self.K u = self.in_proj(x) # (B,T,D) up = F.pad(u.transpose(1, 2), (K - 1, 0)) # causal depthwise conv cv = sum(self.conv_w[:, j, None] * up[:, :, j:j + T] for j in range(K)) + self.conv_b[:, None] cv = cv.transpose(1, 2) cv = self.act_silu * F.silu(cv) + self.act_lin * cv r = torch.cat((u, cv), -1) if self.Dn: n = self.pn(r) r = torch.cat((r, n * torch.rsqrt(n.pow(2) @ self.n_avg + self.n_eps) * self.n_w), -1) if self.Dh: r = torch.cat((r, torch.sigmoid(self.ph(r))), -1) if self.Dp: r = torch.cat((r, self.pfx(r) * self.pfy(r)), -1) if self.Dh2: r = torch.cat((r, torch.sigmoid(self.ph2(r))), -1) heads = lambda t, n: t.view(B, T, H, n).transpose(1, 2) # (B,H,T,n) q, k = heads(self.pq(r), dk), heads(self.pk(r), dk) l2 = lambda t: t * torch.rsqrt(t.pow(2).sum(-1, keepdim=True) + self.l2_eps) if self.l2_eps_out != 0: l2 = lambda t: t / ((t.pow(2).sum(-1, keepdim=True) + self.l2_eps).sqrt() + self.l2_eps_out) q = self.q_coef[0] * q + self.q_coef[1] * l2(q) k = self.k_coef[0] * k + self.k_coef[1] * l2(k) v = heads(self.pv(r), dv) if self.dcW: Wd = self.dcW ker = self.dc_w2(F.silu(self.dc_w1(x))).view(B, T, H * dv, Wd) vr = F.pad(v.transpose(1, 2).reshape(B, T, H * dv).transpose(1, 2), (Wd - 1, 0)).unfold(2, Wd, 1) vr = (vr.permute(0, 2, 1, 3) * ker).sum(-1) vr = vr + self.dc_act * (F.silu(vr) - vr) v = heads(vr, dv) if reg is not None and self.v_write != 0: reg["v_ssm"] = v if reg is not None and self.v_read != 0: v = v + self.v_read * heads(self.pvr(r), dv) * (reg["v_ssm"] - v) kap = rmv = None if self.c_rm.any() or self.c_ka.any(): beta = heads(self.pbeta(r), dk) kap = heads(self.pkap(r), dk) kap = kap * torch.rsqrt(kap.pow(2).sum(-1, keepdim=True) + self.kap_eps) rmv = kap * beta k = k * (1 + self.c_ka[:, None] * (beta - 1)) p1 = heads(self.p1(r), dk) sp = F.softplus(p1) if self.dt_min > 0: # Mamba-2 time_step_limit floor sp = sp.clamp(min=self.dt_min) loga = -(self.A1[:, None] * sp + self.A2[:, None] * torch.sigmoid(heads(self.p2(r), dk)) + self.A3[:, None]) per_head = lambda p: p(r).transpose(1, 2) # (B,H,T) dt = sp[..., 0] # (B,H,T) if self.A4.any(): pa = per_head(self.pA) ht = pa.clamp(min=0) + 1 / (1 - pa.clamp(max=0)) loga = loga - self.A4[:, None] * (dt * torch.maximum(ht, self.a_floor[:, None]))[..., None] if self.A5.any(): loga = loga - self.A5[:, None] * heads(self.p5(r), dk).exp() a = self.mem[:, None] * loga.exp() # (B,H,T,dk) sb, si = torch.sigmoid(per_head(self.pb)), torch.sigmoid(per_head(self.pi)) gamma = self.cg[:, None] * sb + self.cg0[:, None] er = (1 + self.c_er[:, None, None] * (torch.sigmoid(heads(self.pbv(r), dk)) - 1)) if self.c_er.any() else None lru = torch.sqrt((1 - a[..., 0] ** 2).clamp_min(0)) first = torch.zeros(T, dtype=F64); first[0] = 1 lru = lru + self.reset[:, None] * first * (1 - lru) cw = lambda i: self.cw[i][:, None] w = (cw(0) * dt + cw(1) * sb + cw(2) * per_head(self.pe).exp() + cw(3) * lru * si + cw(4) + cw(5) * dt * sb) w_prev = self.cw_prev[:, None] * dt * (1 - sb) if self.cw_prev.any() else None dth = None if self.rot_c.any() or self.rot_k.any(): dth = self.rot_c[:, None] * torch.tanh(heads(self.pang(r), dk // 2)) * dt[..., None] + self.rot_k[:, None] if self.ridge_c.any(): x_r = _ridge_readout(q, k, a, self.ridge_ratio, int(self.ridge_iters)) q = q + (self.ridge_c[:, None] * per_head(self.pal))[..., None] * (x_r - q) if w_prev is None and dth is None and kap is None and er is None and not self.c_pre.any(): y = gd_recurrence(q, k, v, a, gamma, w) else: rm = (self.c_rm, kap, rmv) if kap is not None else None y = _recurrence(q, k, v, a, gamma, w, w_prev, dth, rm, self.c_pre if self.c_pre.any() else None, er) y = y + self.Dskip[:, None] * v if reg is not None and self.y_read != 0: y = (1 - self.y_read) * y + self.y_read * reg["y_ssm"] if reg is not None and self.y_write != 0: reg["y_ssm"] = y y = y.transpose(1, 2).reshape(B, T, H * dv) @ self.mix z = self.pz(r) g = self.gate G = g[0] * F.silu(z) + g[1] * torch.sigmoid(z) + g[2] * F.gelu(z, approximate="tanh") + g[3] * z + g[4] oc = self.out_coef out = oc[0] * self._norm(y) * G + oc[1] * self._norm(y * G) + oc[2] * y * G if oc[3].any(): on = oc[3] != 0 # channels that use the ratio branch den = torch.where(on, y @ self.ratio_den, torch.ones_like(y)) out = out + oc[3] * (y @ self.ratio_num) / den * G return self.out_proj(out) class UFFN(nn.Module): def __init__(self, c: UConfig): super().__init__() d, E, f, fs = c.d, c.experts, c.expert_ff, c.shared_ff self.top_k = c.top_k self.router = _lin(d, E) # a router bias (zero unless a target has one: GPT-OSS) self.w_gate, self.w_up, self.w_down = _p(E, f, d), _p(E, f, d), _p(E, d, f) self.b_gate, self.b_up, self.b_down = _p(E, f), _p(E, f), _p(E, d) # routed-expert biases (GPT-OSS) # group-limited top-k (DeepSeek-V2 / V3): [groups, groups kept, group score: 0 sum of the top-2 # selection scores, 1 their max, the value experts outside the kept groups are masked to before the # top-k: 0 (DeepSeek) or -inf (Nemotron-H)] self.register_buffer("grp", torch.tensor([1.0, 1.0, 0.0, 0.0], dtype=F64)) # [..., masked value] self.renorm = _p(1) # 1 = renormalize the top-k weights self.register_buffer("renorm_eps", torch.tensor(0.0, dtype=F64)) # ... by (sum + eps) self.score_c = _p(1) # router scores: 0 softmax, 1 sigmoid self.sel_bias = _p(E) # added to scores for top-k SELECTION only self.sel_logit = _p(1) # 1: select on the raw logits + sel_bias (Youtu QB) self.register_buffer("router_cap", torch.tensor(float("inf"), dtype=F64)) # router logit softcap (Laguna) self.route_scale = _p(1, fill=1.0) self.exp_scale = _p(E, fill=1.0) # the top-k weights times a per-expert scale (Gemma-4) # SparseMixer top-2 (Phi-3.5-MoE, eval): each pick's weight is a softmax over the logits within a relative # 2 eps of that pick's maximum; smix_eps = 0: off self.register_buffer("smix_eps", torch.tensor(0.0, dtype=F64)) # the shared / dense path carries biases (zero unless a target has them: GPT-2, GPT-NeoX, ...) self.s_gate, self.s_up, self.s_down = _lin(d, fs), _lin(d, fs), _lin(fs, d) self.s_router = _lin(d, 1, bias=False) E, r = c.lora_adapters, c.lora_rank self.la = nn.ParameterDict({"g": _p(E, r, d), "u": _p(E, r, d), "d": _p(E, r, fs)}) self.lb = nn.ParameterDict({"g": _p(E, fs, r), "u": _p(E, fs, r), "d": _p(E, d, r)}) self.c_situ = _p(1) # SiTU-GLU: capped gate/up (Mini-K3) self.register_buffer("situ_beta", torch.tensor([4.0, 25.0], dtype=F64)) self.c_clip = _p(1) # hard SwiGLU clip: gate <= L, |up| <= L self.c_relu2 = _p(1) # ungated squared ReLU: relu(up x)^2 self.c_relu = _p(1) # ungated ReLU: relu(up x) (OPT) self.up_shift = _p(1) # u <- u + up_shift after the clip (swigluoai's (u + 1)) # exact GeLU / mish branches: c_gelu = 1 ungated gelu(u) (GPT-2, NeoX, ...), c_geglu = 1 gated # gelu(g) * u (Gemma GeGLU); gelu_tanh = 1 the tanh approximation (gelu_new / gelu_pytorch_tanh / # gelu_fast), 0 the erf GeLU; c_mish = 1 gated mish(g) * u self.c_gelu, self.c_geglu, self.gelu_tanh, self.c_mish = _p(1), _p(1), _p(1), _p(1) self.register_buffer("clip_lim", torch.full((2,), float("inf"), dtype=F64)) # [gate, up] limits # ungated xIELU (Apertus): [alpha_p, alpha_n] pre-softplus (learned), [beta, eps] self.c_xielu, self.xielu_a = _p(1), _p(2) self.register_buffer("xielu_be", torch.tensor([0.5, -1e-6], dtype=F64)) # PolyNorm-GLU (Motif-3): (w0 n(g^3) + w1 n(g^2) + w2 n(g) + b) * u, n = RMS over the width; # weights per routed expert / for the shared path, sigmoid(w) if poly_sig self.c_poly, self.poly_sig = _p(1), _p(1) self.poly_w, self.poly_b = _p(c.experts, 3), _p(c.experts, 1) self.s_poly_w, self.s_poly_b = _p(3), _p(1) # [eps, hidden clamp, output scale, routed bias clamp, routed result clamp, shared active width] self.register_buffer("poly_cfg", torch.tensor([1e-6, float("inf"), 1.0, float("inf"), float("inf"), float(fs)], dtype=F64)) self.r_c = _p(1) # post-routing: routed <- r_up(rms_grouped(routed) r_w) self.r_avg, self.r_w = _p(d, d), _p(d, fill=1.0) self.r_up = _lin(d, d, bias=False) self.register_buffer("r_eps", torch.tensor(1e-6, dtype=F64)) self.s_coef = _p(2) # [constant 1, sigmoid(s_router x)] with torch.no_grad(): self.s_coef[0] = 1.0 def _xielu(self, u): """xIELU (HF XIELUActivation): a_p = softplus(alpha_p), a_n = beta + softplus(alpha_n), u > 0: a_p u^2 + beta u; else: a_n (expm1(min(u, eps)) - u) + beta u.""" ap, an = F.softplus(self.xielu_a[0]), self.xielu_be[0] + F.softplus(self.xielu_a[1]) beta, eps = self.xielu_be return torch.where(u > 0, ap * u * u + beta * u, (torch.expm1(torch.min(u, eps)) - u) * an + beta * u) def _poly(self, g, u, routed): """PolyNorm-GLU (Motif-3 PolyNormTorch / GroupedPolyNorm): g, u clamped to the hidden clamp, (w0 n(g^3) + w1 n(g^2) + w2 n(g) + b) * u with n(x) = x / sqrt(sum(x^2) / width + eps) over the FFN width; routed experts (own w, b per expert; b clamped) also clamp the result; then * scale. `width` is the shared path's active width, so a zero-padded shared bank normalises exactly.""" eps, hc, scale, bc, rc, ws = self.poly_cfg g, u = g.clamp(-hc, hc), u.clamp(-hc, hc) if routed: # h: (B, T, E, f); per-expert (E, 1) coefficients w, b, n = self.poly_w, self.poly_b.clamp(-bc, bc), g.shape[-1] w = [w[:, k, None] for k in range(3)] else: w, b, n = self.s_poly_w, self.s_poly_b, ws w = [wk + self.poly_sig * (torch.sigmoid(wk) - wk) for wk in w] nrm = lambda x: x / torch.sqrt(x.pow(2).sum(-1, keepdim=True) / n + eps) y = (w[0] * nrm(g ** 3) + w[1] * nrm(g ** 2) + w[2] * nrm(g) + b) * u if routed: y = y.clamp(-rc, rc) return y * scale def _act(self, g, u, routed=False): """silu(g) * u, or SiTU-GLU bg tanh(g/bg) sigmoid(g) * bu tanh(u/bu) (c_situ = 1); c_clip = 1 first clips g <= L and u to [-L, L]; c_relu2 = 1 is the ungated relu(u)^2; c_xielu = 1 the ungated xIELU(u); c_poly = 1 PolyNorm-GLU (routed: per-expert weights); c_gelu = 1 the ungated gelu(u), c_geglu = 1 gelu(g) * u (erf, or tanh if gelu_tanh = 1); c_mish = 1 mish(g) * u.""" if self.c_relu2 != 0: return F.relu(u) ** 2 if self.c_relu != 0: return F.relu(u) if self.c_gelu != 0 or self.c_geglu != 0: gelu = lambda t: F.gelu(t, approximate="tanh" if self.gelu_tanh != 0 else "none") return gelu(u) if self.c_gelu != 0 else gelu(g) * u if self.c_mish != 0: return F.mish(g) * u if self.c_xielu != 0: return self._xielu(u) if self.c_poly != 0: return self._poly(g, u, routed) if self.c_clip != 0: Lg, Lu = self.clip_lim g = g + self.c_clip * (g.clamp(max=Lg) - g) u = u + self.c_clip * (u.clamp(-Lu, Lu) - u) u = u + self.up_shift if self.c_situ == 0: return F.silu(g) * u c, (bg, bu) = self.c_situ, self.situ_beta return (((1 - c) * g + c * bg * torch.tanh(g / bg)) * torch.sigmoid(g) * ((1 - c) * u + c * bu * torch.tanh(u / bu))) def forward(self, x, reg=None, xe=None, xg=None, parts=False): """routed(x) + shared(x) * gate. xe / xg: separate inputs of the routed experts and of the router (Gemma-4 normalises each differently); parts = True returns (routed, gated shared).""" idx = reg.get("adapter") if reg is not None else None L = lambda n, t: _lora(t, self.la[n], self.lb[n], idx) xs = x x = x if xe is None else xe logits = self.router(x if xg is None else xg) if torch.isfinite(self.router_cap): logits = self.router_cap * torch.tanh(logits / self.router_cap) p = logits.softmax(-1) if self.score_c == 0 else ( (1 - self.score_c) * logits.softmax(-1) + self.score_c * torch.sigmoid(logits)) sel = (logits if self.sel_logit != 0 else p) + self.sel_bias G, kg = int(self.grp[0]), int(self.grp[1]) if G > 1 and kg < G: # keep the top-kg groups' experts sg = sel.view(*sel.shape[:-1], G, -1) gs = sg.max(-1).values if self.grp[2] != 0 else sg.topk(2, -1).values.sum(-1) keep = torch.zeros_like(gs).scatter_(-1, gs.topk(kg, -1).indices, 1.0) sel = sel.masked_fill(keep.repeat_interleave(sel.shape[-1] // G, -1) == 0, float(self.grp[3])) topi = sel.topk(self.top_k, -1).indices topv = p.gather(-1, topi) if self.smix_eps > 0: # SparseMixer: argmax, then the next, each own-masked softmax e2, s_, picks, ws = 2 * self.smix_eps, logits, [], [] for _ in range(2): m_, i_ = s_.max(-1, keepdim=True) fac = logits.abs().clamp(min=m_) ws.append(s_.masked_fill((m_ - logits) / fac > e2, float("-inf")).softmax(-1).gather(-1, i_)) picks.append(i_) s_ = s_.scatter(-1, i_, float("-inf")) topi, topv = torch.cat(picks, -1), torch.cat(ws, -1) topv = (topv + self.renorm * (topv / (topv.sum(-1, keepdim=True) + self.renorm_eps) - topv)) * self.route_scale topv = topv * self.exp_scale[topi] h = self._act(torch.einsum("efd,btd->btef", self.w_gate, x) + self.b_gate, torch.einsum("efd,btd->btef", self.w_up, x) + self.b_up, True) eo = torch.einsum("edf,btef->bted", self.w_down, h) + self.b_down # every expert, then select routed = (eo.gather(2, topi[..., None].expand(*topi.shape, x.shape[-1])) * topv[..., None]).sum(2) if self.r_c != 0: rn = routed * torch.rsqrt(routed.pow(2) @ self.r_avg + self.r_eps) * self.r_w routed = (1 - self.r_c) * routed + self.r_c * self.r_up(rn) x = xs hs = self._act(self.s_gate(x) + L("g", x), self.s_up(x) + L("u", x)) shared = self.s_down(hs) + L("d", hs) sg = self.s_coef[0] + self.s_coef[1] * torch.sigmoid(self.s_router(x)) return (routed, shared * sg) if parts else routed + shared * sg class UEngram(nn.Module): """Hashed n-gram memory (Engram): the token embedding generalised from the current token to hashed n-grams of the input ids. Each (order n, hash head h) folds the last n ids with a prime, x <- x P_h + id + 1, and reads row (x ^ (x >> shift)) mod slots of its own table; the rows are concatenated into e. Added to the residual stream before the layer: g = sigmoid( / sqrt(d)), v = g * val(e), out = v + causal_conv(v). Off (c = 0) unless a target sets it; ids are the model's input ids (left-padded with `pad`).""" PRIMES = (1000003, 999983, 1000033, 999979, 1000037, 999961, 1000039, 999953) def __init__(self, c: UConfig): super().__init__() d, S, Hh, D = c.d, c.engram_slots, c.engram_heads, c.engram_dim self.orders, self.heads, self.slots, self.pad, self.shift = tuple(c.engram_orders), Hh, S, c.vocab, 17 self.register_buffer("primes", torch.tensor(self.PRIMES, dtype=torch.long)) self.table = _p(len(self.orders) * Hh * S, D) mem = len(self.orders) * Hh * D self.key, self.val = _lin(mem, d, bias=False), _lin(mem, d, bias=False) self.w_h, self.w_k = _p(d, fill=1.0), _p(d, fill=1.0) self.register_buffer("eps", torch.tensor([1e-6, 1e-6], dtype=F64)) self.conv_w = _p(d, c.engram_conv) self.c = _p(1) def hash_ids(self, ids): B, T = ids.shape maxn = max(self.orders) p = F.pad(ids, (maxn - 1, 0), value=self.pad) out, off = [], 0 for n in self.orders: grams = [p[:, maxn - 1 - j: maxn - 1 - j + T] for j in range(n)] for h in range(self.heads): P = self.primes[h % len(self.primes)] x = torch.zeros_like(ids) for g in grams: x = x * P + g + 1 out.append(torch.remainder(x ^ (x >> self.shift), self.slots) + off) off += self.slots return torch.stack(out, -1) def forward(self, x, ids): T = x.shape[1] e = self.table[self.hash_ids(ids)].flatten(-2) k, v = self.key(e), self.val(e) g = torch.sigmoid((_rms(x, self.eps[0]) * self.w_h * _rms(k, self.eps[1]) * self.w_k).sum(-1, keepdim=True) / math.sqrt(x.shape[-1])) v = g * v K = self.conv_w.shape[1] vp = F.pad(v.transpose(1, 2), (K - 1, 0)) y = sum(self.conv_w[:, j, None] * vp[:, :, j:j + T] for j in range(K)).transpose(1, 2) return self.c * (v + y) class ULayer(nn.Module): def __init__(self, c: UConfig): super().__init__() self.engram = UEngram(c) if c.engram_slots else None self.mem = UAttention(c, mem=True) if c.mem_attn else None # parallel memory read, own norm self.norm_m = UNorm(c.d) if c.mem_attn else None self.sn_in = UNorm(c.d) if c.stream_norms else None # x <- sn_in(x) at the start (sn_c[0]) self.sn_out = UNorm(c.d) if c.stream_norms else None # x <- sn_out(x) at the end (sn_c[1]) self.sn_c = _p(2) self.hc_a = UHC(c) if c.hc_mult else None # hyper-connections (attn+SSM / FFN) self.hc_f = UHC(c) if c.hc_mult else None N = lambda: UNorm(c.d, c.norm_gate_rank) self.norm_a, self.norm_s, self.norm_f = N(), N(), N() self.attn, self.ssm, self.ffn = UAttention(c), USSM(c), UFFN(c) self.post_a, self.post_s, self.post_f = N(), N(), N() self.post_coef = _p(3, 2) # per sub-block [raw, post-normed] output # split FFN (Gemma-4): the shared path reads norm_f(x) -> post_f1, the routed experts norm_f2(x) -> # post_f2, the router norm_r(x); their sum goes through post_f (post_coef[2]). ffn_split = 0: off self.ffn_split = _p(1) self.norm_f2, self.norm_r, self.post_f1, self.post_f2 = UNorm(c.d), UNorm(c.d), UNorm(c.d), UNorm(c.d) # per-layer embedding input (Gemma-4 PLE): x <- x + ple_post(ple_proj(gelu(ple_gate x) * pli_l)) dp = max(c.ple_dim, 1) self.ple_c, self.ple_gate, self.ple_proj = _p(1), _lin(c.d, dp, bias=False), _lin(dp, c.d, bias=False) self.ple_post = UNorm(c.d) self.register_buffer("ple_tanh", torch.tensor(1.0, dtype=F64)) # its GeLU: 1 tanh-approx, 0 erf self.register_buffer("out_scale", torch.tensor(1.0, dtype=F64)) # x <- out_scale x at the end (layer_scalar) self.register_buffer("res_c", torch.ones(2, dtype=F64)) # residual coefficients x <- c x + f(x) (MiniMax alpha) self.idx = 0 self.res_write, self.res_mix = _p(1), _p(1) # residual snapshot: write x; x <- x + mix (snap - x) self.seq = _p(1, fill=1.0) # 1: SSM reads x + attn (sequential); 0: parallel self.par = _p(1) # 1: FFN reads the LAYER INPUT (a parallel block, # GPT-J / NeoX / Cohere: x + attn(n x) + ffn(n' x)) d = c.d # Block AttnRes queries / key-norm weights self.ar_qa, self.ar_wa, self.ar_qf, self.ar_wf = _p(d), _p(d, fill=1.0), _p(d), _p(d, fill=1.0) self.register_buffer("ar_eps", torch.tensor(1e-6, dtype=F64)) with torch.no_grad(): self.post_coef[:, 0] = 1.0 def _out(self, i, o, post): pc = self.post_coef[i] return pc[0] * o + (pc[1] * post(o) if pc[1] != 0 else 0) def mix(self, sources, q, w): """Depth-softmax over residual sources: sum_s softmax_s(q . rms(src_s) * w) src_s.""" st = torch.stack(sources) sc = (_rms(st, self.ar_eps) * w) @ q return (st * sc.softmax(0)[..., None]).sum(0) def forward_attnres(self, blocks, partial, causal, reg=None): """Block AttnRes: no running residual; inputs are depth-softmax mixes of the block sums.""" xa = self.mix(blocks + ([partial] if partial is not None else []), self.ar_qa, self.ar_wa) a = self._out(0, self.attn(self.norm_a(xa), causal, reg), self.post_a) s = self._out(1, self.ssm(self.norm_s(xa), reg), self.post_s) partial = a + s if partial is None else partial + a + s xf = self.mix(blocks + [partial], self.ar_qf, self.ar_wf) return partial + self._out(2, self.ffn(self.norm_f(xf), reg), self.post_f) def forward_fullres(self, sources, causal, reg=None): """Full AttnRes: each sub-block reads a depth-softmax mix over the embedding and every earlier sub-block output, and appends its own output as a new source.""" xa = self.mix(sources, self.ar_qa, self.ar_wa) a = self._out(0, self.attn(self.norm_a(xa), causal, reg), self.post_a) s = self._out(1, self.ssm(self.norm_s(xa + self.seq * a), reg), self.post_s) sources.append(a + s) sources.append(self._out(2, self.ffn(self.norm_f(self.mix(sources, self.ar_qf, self.ar_wf)), reg), self.post_f)) def forward_hc(self, X, causal, reg=None): """N residual streams (mHC): each sub-block reads a pre-weighted collapse of the streams and writes post * out + comb^T X.""" post, comb, h = self.hc_a(X) a = self._out(0, self.attn(self.norm_a(h), causal, reg), self.post_a) s = self._out(1, self.ssm(self.norm_s(h + self.seq * a), reg), self.post_s) X = post[..., None] * (a + s)[..., None, :] + comb.transpose(-1, -2) @ X post, comb, h = self.hc_f(X) f = self._out(2, self.ffn(self.norm_f(h), reg), self.post_f) return post[..., None] * f[..., None, :] + comb.transpose(-1, -2) @ X def forward(self, x, causal, reg=None): if self.engram is not None and self.engram.c != 0: x = x + self.engram(x, reg["ids"]) if self.res_mix != 0: # x_snap + c (x - x_snap), mix = 1 - c (LFM3-RD) x = x + self.res_mix * (reg["x_snap"] - x) if self.res_write != 0: reg["x_snap"] = x if self.sn_in is not None and self.sn_c[0] != 0: x = self.sn_in(x) a = self._out(0, self.attn(self.norm_a(x), causal, reg), self.post_a) s = self._out(1, self.ssm(self.norm_s(x + self.seq * a), reg), self.post_s) m = self.mem(self.norm_m(x), causal, reg) if self.mem is not None else 0 x_in = x x = (x if self.res_c[0] == 1 else self.res_c[0] * x) + a + s + m xf = x_in if self.par != 0 else x if self.res_c[1] != 1: x = self.res_c[1] * x if self.ffn_split != 0: r_, s_ = self.ffn(self.norm_f(xf), reg, xe=self.norm_f2(xf), xg=self.norm_r(xf), parts=True) x = x + self._out(2, self.post_f1(s_) + self.post_f2(r_), self.post_f) else: x = x + self._out(2, self.ffn(self.norm_f(xf), reg), self.post_f) if self.ple_c != 0: g = F.gelu(self.ple_gate(x), approximate="tanh" if self.ple_tanh != 0 else "none") x = x + self.ple_c * self.ple_post(self.ple_proj(g * reg["ple"][:, :, self.idx])) if self.sn_out is not None and self.sn_c[1] != 0: x = self.sn_out(x) return x * self.out_scale if self.out_scale != 1 else x class UniversalElephantine(nn.Module): def __init__(self, c: UConfig): super().__init__() self.cfg = c self.embed = nn.Embedding(c.vocab, c.d).to(F64) self.pos = _p(c.max_pos, c.d) # learned-abs table, added to the embedding self.prefix = _p(c.n_prefix, c.d) # meta tokens self.ar_q_out, self.ar_w_out = _p(c.d), _p(c.d, fill=1.0) # final AttnRes mix self.sw_score, self.sw_val = _p(c.vocab), _p(c.vocab) # switch head: per-token key / value self.emb_norm_c, self.emb_norm = _p(1), UNorm(c.d) # a norm of the embedding itself (Bloom) self.layers = nn.ModuleList(ULayer(c) for _ in range(c.L)) for i, layer in enumerate(self.layers): layer.idx = i if c.ple_dim: # per-layer embeddings: pli = rms_l(P x0) w + E[ids] self.ple_E = _p(c.ple_vocab or c.vocab, c.L * c.ple_dim) self.ple_P = _lin(c.d, c.L * c.ple_dim, bias=False) self.ple_w = _p(c.ple_dim, fill=1.0) self.register_buffer("ple_eps", torch.tensor(1e-6, dtype=F64)) self.norm = UNorm(c.d, c.norm_gate_rank) self.lm_head = _lin(c.d, c.vocab) # an lm_head bias (zero unless a target has one: Phi) self.register_buffer("final_cap", torch.tensor(float("inf"), dtype=F64)) # cap tanh(logits / cap) def forward(self, ids, causal=True): x = self.embed(ids) if self.cfg.max_pos: x = x + self.pos[: ids.shape[1]] if self.emb_norm_c != 0: x = self.emb_norm(x) n = self.cfg.n_prefix if n: x = torch.cat((self.prefix.expand(x.shape[0], -1, -1), x), 1) reg = {"ids": ids} if self.cfg.ple_dim: B_, T_ = ids.shape pp = self.ple_P(x).view(B_, T_, self.cfg.L, self.cfg.ple_dim) reg["ple"] = _rms(pp, self.ple_eps) * self.ple_w + self.ple_E[ids].view(B_, T_, self.cfg.L, self.cfg.ple_dim) if self.cfg.lora_adapters: # adapter index = round(causal softmax-avg of values) sc, val = self.sw_score[ids], self.sw_val[ids] T_ = ids.shape[1] m = torch.ones(T_, T_, dtype=torch.bool).tril() w = sc[:, None, :].expand(-1, T_, -1).masked_fill(~m, float("-inf")).softmax(-1) avg = (w * val[:, None, :]).sum(-1) reg["adapter"] = avg.round().long().clamp(0, self.cfg.lora_adapters) B = self.cfg.attnres_block if self.cfg.attnres_full: sources = [x] for layer in self.layers: layer.forward_fullres(sources, causal, reg) x = self.layers[0].mix(sources, self.ar_q_out, self.ar_w_out) elif self.cfg.hc_mult: # N streams, collapsed by their mean at the end X = x[:, :, None].expand(-1, -1, self.cfg.hc_mult, -1) for layer in self.layers: X = layer.forward_hc(X, causal, reg) x = X.mean(2) elif B: blocks, partial = [x], None for i, layer in enumerate(self.layers): if i > 0 and i % B == 0: blocks.append(partial); partial = None partial = layer.forward_attnres(blocks, partial, causal, reg) x = self.layers[0].mix(blocks + [partial], self.ar_q_out, self.ar_w_out) else: for layer in self.layers: x = layer(x, causal, reg) out = self.lm_head(self.norm(x)[:, n:]) return out if not torch.isfinite(self.final_cap) else self.final_cap * torch.tanh(out / self.final_cap) # <<< vendored: elephantine/universal.py # --- the transformers wrapper ------------------------------------------------------------------ def uconfig_of(config) -> UConfig: """The host's UConfig from an ElephantineConfig (unknown keys are an error, not ignored).""" names = {f.name for f in fields(UConfig)} host = dict(config.host) extra = set(host) - names if extra: raise ValueError(f"host keys this architecture does not have: {sorted(extra)}") if "engram_orders" in host: host["engram_orders"] = tuple(host["engram_orders"]) return UConfig(**host) class ElephantineForCausalLM(PreTrainedModel, GenerationMixin): """UniversalElephantine behind the transformers API. `causal` (config default; overridable per call) picks the causal or the full (diffusion) mask. Logits are float64.""" config_class = ElephantineConfig base_model_prefix = "model" main_input_name = "input_ids" # tensors left at the architecture's default are not stored: they keep the value __init__ gives _keys_to_ignore_on_load_missing = [r".*"] _supports_cache_class = False _is_stateful = True # no KV cache: never build one def __init__(self, config: ElephantineConfig): super().__init__(config) self.model = UniversalElephantine(uconfig_of(config)) self.post_init() @classmethod def get_init_context(cls, *args, **kwargs): # build on CPU with the architecture's own init (no meta device), so a tensor the weights file # leaves out keeps its default value instead of uninitialised memory return [no_init_weights()] def _init_weights(self, module): pass # the defaults are UniversalElephantine's own def get_input_embeddings(self): return self.model.embed def set_input_embeddings(self, value): self.model.embed = value def forward(self, input_ids=None, attention_mask=None, labels=None, causal=None, return_dict=None, **kwargs): if attention_mask is not None and not bool((attention_mask == 1).all()): raise ValueError("padding masks are not supported: pass unpadded sequences of equal length") causal = self.config.causal if causal is None else bool(causal) logits = self.model(input_ids, causal=causal) loss = None if labels is not None: lg, lb = (logits[:, :-1], labels[:, 1:]) if causal else (logits, labels) loss = F.cross_entropy(lg.reshape(-1, lg.shape[-1]), lb.reshape(-1), ignore_index=-100) return CausalLMOutput(loss=loss, logits=logits) def prepare_inputs_for_generation(self, input_ids, **kwargs): return {"input_ids": input_ids} def _supports_default_dynamic_cache(self): return False