"""A plain decoder transformer whose weights are programmed directly as an edit list (no training, no program counter).
Forward pass (standard): x_t = embedding(token_t) + position(t); for each layer: x += sum_heads W_O softmax(QK^T) V,
then x += W_2 ReLU(W_1 x + b_1); logits = W_U x. No LayerNorm. Two extras, both ordinary components:
* memory banks: a head may attend over an external bank (token embeddings provided by the host) instead of the
sequence; such heads are not causally masked.
* a sink slot per head: one extra key with a fixed score (a weight) and a zero value, so that "nothing matches"
is representable.
All parameters start at zero; `apply_edits` writes the edit list into them.
Edit list lines (text, one edit per line; block names refer to the residual layout or a head's own dims):
FILL [i0:i1] [j0:j1]
EYE [shift] identity (optionally shifted) between equal-size blocks
CODE | | the token's embedding code placed along a row/column
SET
"""
import os
import sys
import numpy as np
from . import vocab as vb # noqa: E402
CD, NF = vb.CODE_DIM, len(vb.FLAG_NAMES)
NPOS = 512
NULL = vb.V # code row of zeros
ACTIONS = ["A_DIST_ANCH", "A_DIST_WEB", "A_DOC", "A_DOC_PREFIX"]
class Layout:
"""Named blocks of a vector space (the residual stream, or a head's query/value space)."""
def __init__(self, blocks):
self.sl, o = {}, 0
for name, n in blocks:
self.sl[name] = slice(o, o + n)
o += n
self.dim = o
def __getitem__(self, name):
return self.sl[name]
class Head:
def __init__(self, name, layer, dq, dv, d, memory=None, sink=0.0, causal=True):
self.name, self.layer, self.memory, self.causal = name, layer, memory, causal
self.Wq = np.zeros((d, dq), np.float32)
self.Wk = np.zeros((d, dq), np.float32)
self.Wv = np.zeros((d, dv), np.float32)
self.Wo = np.zeros((dv, d), np.float32)
self.sink = np.float32(sink)
class MLP:
def __init__(self, name, layer, n, d):
self.name, self.layer = name, layer
self.W1 = np.zeros((d, n), np.float32)
self.b1 = np.zeros(n, np.float32)
self.W2 = np.zeros((n, d), np.float32)
class Model:
def __init__(self, layout, heads, mlps, n_layers, out_tokens, act_dims):
self.R, self.heads, self.mlps, self.n_layers = layout, heads, mlps, n_layers
self.C = vb_codes()
self.F = vb.flags_matrix()
self.out_tokens, self.act_dims = out_tokens, act_dims
self.trace = {}
# ------------------------------------------------------------------ embedding (fixed formula)
def embed_tokens(self, ids):
R = self.R
X = np.zeros((len(ids), R.dim), np.float32)
X[:, R["one"]] = 1.0
X[:, R["tok"]] = self.C[ids]
X[:, R["flg"]] = self.F[np.minimum(ids, vb.V)]
X[np.arange(len(ids)), R["pos"].start + np.minimum(np.arange(len(ids)), NPOS - 1)] = 1.0
return X
def embed_memory(self, mem, kind):
"""Memory bank rows in residual coordinates. M rows: code of m_j in `tok`, m_{j-1} in `prev`, m_{j-2} in
`prev2`, flags in `flg`; D rows: candidate code in `tok`, flags, log count in `logc`."""
R = self.R
if kind == "M":
ids = np.array(mem, dtype=np.int64)
X = np.zeros((len(ids), R.dim), np.float32)
if len(ids) == 0:
return X
X[:, R["one"]] = 1.0
X[:, R["tok"]] = self.C[ids]
X[:, R["flg"]] = self.F[np.minimum(ids, vb.V)]
X[1:, R["prev"]] = self.C[ids[:-1]]
X[2:, R["prev2"]] = self.C[ids[:-2]]
X[3:, R["prev3"]] = self.C[ids[:-3]]
X[4:, R["prev4"]] = self.C[ids[:-4]]
X[5:, R["prev5"]] = self.C[ids[:-5]]
X[:, R["dpos"].start + min(self.mem_pos, NPOS - 1)] = 1.0 # the position the document was fetched for
X[:, R["chat"].start] = float(self.mem_chat) # the document is a conversation reply
return X
ids = np.array([t for t, _ in mem], dtype=np.int64)
X = np.zeros((len(ids), R.dim), np.float32)
if len(ids) == 0:
return X
X[:, R["one"]] = 1.0
X[:, R["tok"]] = self.C[ids]
X[:, R["flg"]] = self.F[np.minimum(ids, vb.V)]
cnt = np.array([c for _, c in mem], dtype=np.float64)
X[:, R["logc"].start] = np.log1p(cnt) # fixed count features of a candidate row
X[:, R["ge5"].start] = (cnt >= 5).astype(np.float32)
X[:, R["dom"].start] = (cnt >= 0.5 * cnt.sum()).astype(np.float32)
X[:, R["full"].start] = float(self.dist_full)
X[:, R["slot"].start] = np.arange(len(ids), dtype=np.float32)
X[:, R["dpos"].start + min(self.dist_pos, NPOS - 1)] = 1.0 # the position the distribution was computed for
return X
# ------------------------------------------------------------------ forward
dist_pos = 0
dist_full = 0
mem_pos = 0
mem_chat = 0
last_from = 3 # layers >= 3 are computed for the last position only (their outputs are read only there)
def forward(self, ids, mem=None, dist=None, dist_pos=0, dist_full=0, mem_pos=0, mem_chat=0, keep=False):
self.dist_pos, self.dist_full, self.mem_pos, self.mem_chat = dist_pos, dist_full, mem_pos, mem_chat
X = self.embed_tokens(np.array(ids, dtype=np.int64))
banks = {"M": self.embed_memory(mem or [], "M"), "D": self.embed_memory(dist or [], "D")}
T = len(ids)
causal = np.tril(np.ones((T, T), dtype=bool))
self.trace = {}
for layer in range(1, self.n_layers + 1):
if layer == self.last_from: # from here on only the last position is read
Xfull = X # keys/values of layer-3 heads still see every position
X = X[-1:]
causal = causal[-1:]
T = 1
delta = np.zeros_like(X)
for hd in self.heads:
if hd.layer != layer:
continue
Q = X @ hd.Wq
if hd.memory:
B = banks[hd.memory]
K, V = B @ hd.Wk, B @ hd.Wv
S = Q.astype(np.float64) @ K.T.astype(np.float64)
else:
Xk = Xfull if layer >= self.last_from else X
K, V = Xk @ hd.Wk, Xk @ hd.Wv
S = Q.astype(np.float64) @ K.T.astype(np.float64)
S = np.where(causal, S, -1e30) if hd.causal else S
S = np.concatenate([S, np.full((T, 1), float(hd.sink))], axis=1) # sink slot, value 0
S -= S.max(axis=1, keepdims=True)
A = np.exp(S)
A /= A.sum(axis=1, keepdims=True)
O = (A[:, :-1] @ V.astype(np.float64)).astype(np.float32)
delta += O @ hd.Wo
if keep:
self.trace[hd.name] = A
X = X + delta
for m in self.mlps:
if m.layer == layer:
Hh = np.maximum(X @ m.W1 + m.b1, 0.0)
X = X + Hh @ m.W2
if keep:
self.trace[m.name] = Hh
return X
def logits(self, X_last):
"""Token logits = scale * ; action logits from the act dims."""
R = self.R
tok = 40.0 * (self.C[:vb.V] @ X_last[R["out"]])
act = X_last[R["act"]] * 40.0
return np.concatenate([tok, act])
def vb_codes():
t = np.arange(vb.V + 1)
a0, a1, a2 = t % vb.P, (t // vb.P) % vb.P, t // (vb.P * vb.P)
C = np.zeros((vb.V + 1, CD), dtype=np.float32)
for x in range(vb.K):
C[t, x * vb.P + (a0 + a1 * x + a2 * x * x) % vb.P] = 1.0 / np.sqrt(vb.K)
C[NULL] = 0.0
return C
# ---------------------------------------------------------------------- edit list
def apply_edits(model, lines, head_layouts):
"""Write an edit list into the zero-initialized parameters. Parameters are addressed as ., e.g.
L1.prev.Wq or L3.ctl.W1; blocks are names in the residual layout (R) or in the unit's own layout."""
units = {f"L{h.layer}.{h.name}": h for h in model.heads}
units.update({f"L{m.layer}.{m.name}": m for m in model.mlps})
C = model.C
n = 0
for raw in lines:
line = raw.split("#")[0].strip()
if not line:
continue
tok = line.split()
kind, param = tok[0], tok[1]
unit_name, mat = param.rsplit(".", 1)
unit = units[unit_name]
W = getattr(unit, mat)
lays = head_layouts.get(unit_name, {})
own = lays.get({"Wq": "q", "Wk": "q", "Wv": "v", "Wo": "v", "W1": "h", "b1": "h", "W2": "h"}.get(mat, "h"))
def block(spec, axis_is_residual):
# spec: name, name[i0:i1], or an integer index
if spec.lstrip("-").isdigit():
i = int(spec)
return slice(i, i + 1)
name, rng = (spec.split("[", 1) + [""])[:2]
lay = model.R if axis_is_residual else own
s = lay[name]
if rng:
a, b = rng.rstrip("]").split(":")
return slice(s.start + int(a), s.start + int(b))
return s
rows_res = mat in ("Wq", "Wk", "Wv", "W1") # rows index the residual stream
cols_res = mat in ("Wo", "W2") # columns index the residual stream
if kind == "FILL":
rs, cs = block(tok[2], rows_res), block(tok[3], cols_res)
W[rs, cs] = float(tok[4])
elif kind == "EYE":
rs, cs = block(tok[2], rows_res), block(tok[3], cols_res)
scale = float(tok[4])
shift = int(tok[5]) if len(tok) > 5 else 0
nr = rs.stop - rs.start
for i in range(nr):
j = i + shift
if 0 <= j < cs.stop - cs.start:
W[rs.start + i, cs.start + j] = scale
elif kind == "CODE": # code along the block of the first spec
rs, cs = block(tok[2], rows_res), block(tok[3], cols_res)
v, scale = int(tok[4]), float(tok[5])
if rs.stop - rs.start == CD:
W[rs, cs.start] += scale * C[v]
else:
W[rs.start, cs] += scale * C[v]
elif kind == "SET":
rs, cs = block(tok[2], rows_res), block(tok[3], cols_res)
W[rs.start, cs.start] = float(tok[4])
elif kind == "BIAS": # MLP bias: BIAS L3.ctl.b1 value
cs = block(tok[2], False)
unit.b1[cs] = float(tok[3])
elif kind == "SINK":
unit.sink = np.float32(float(tok[2]))
else:
raise ValueError(line)
n += 1
return n