"""A plain decoder transformer whose weights are programmed directly as an edit list (no training, no program counter). Forward pass (standard): x_t = embedding(token_t) + position(t); for each layer: x += sum_heads W_O softmax(QK^T) V, then x += W_2 ReLU(W_1 x + b_1); logits = W_U x. No LayerNorm. Two extras, both ordinary components: * memory banks: a head may attend over an external bank (token embeddings provided by the host) instead of the sequence; such heads are not causally masked. * a sink slot per head: one extra key with a fixed score (a weight) and a zero value, so that "nothing matches" is representable. All parameters start at zero; `apply_edits` writes the edit list into them. Edit list lines (text, one edit per line; block names refer to the residual layout or a head's own dims): FILL [i0:i1] [j0:j1] EYE [shift] identity (optionally shifted) between equal-size blocks CODE | | the token's embedding code placed along a row/column SET """ import os import sys import numpy as np from . import vocab as vb # noqa: E402 CD, NF = vb.CODE_DIM, len(vb.FLAG_NAMES) NPOS = 512 NULL = vb.V # code row of zeros ACTIONS = ["A_DIST_ANCH", "A_DIST_WEB", "A_DOC", "A_DOC_PREFIX"] class Layout: """Named blocks of a vector space (the residual stream, or a head's query/value space).""" def __init__(self, blocks): self.sl, o = {}, 0 for name, n in blocks: self.sl[name] = slice(o, o + n) o += n self.dim = o def __getitem__(self, name): return self.sl[name] class Head: def __init__(self, name, layer, dq, dv, d, memory=None, sink=0.0, causal=True): self.name, self.layer, self.memory, self.causal = name, layer, memory, causal self.Wq = np.zeros((d, dq), np.float32) self.Wk = np.zeros((d, dq), np.float32) self.Wv = np.zeros((d, dv), np.float32) self.Wo = np.zeros((dv, d), np.float32) self.sink = np.float32(sink) class MLP: def __init__(self, name, layer, n, d): self.name, self.layer = name, layer self.W1 = np.zeros((d, n), np.float32) self.b1 = np.zeros(n, np.float32) self.W2 = np.zeros((n, d), np.float32) class Model: def __init__(self, layout, heads, mlps, n_layers, out_tokens, act_dims): self.R, self.heads, self.mlps, self.n_layers = layout, heads, mlps, n_layers self.C = vb_codes() self.F = vb.flags_matrix() self.out_tokens, self.act_dims = out_tokens, act_dims self.trace = {} # ------------------------------------------------------------------ embedding (fixed formula) def embed_tokens(self, ids): R = self.R X = np.zeros((len(ids), R.dim), np.float32) X[:, R["one"]] = 1.0 X[:, R["tok"]] = self.C[ids] X[:, R["flg"]] = self.F[np.minimum(ids, vb.V)] X[np.arange(len(ids)), R["pos"].start + np.minimum(np.arange(len(ids)), NPOS - 1)] = 1.0 return X def embed_memory(self, mem, kind): """Memory bank rows in residual coordinates. M rows: code of m_j in `tok`, m_{j-1} in `prev`, m_{j-2} in `prev2`, flags in `flg`; D rows: candidate code in `tok`, flags, log count in `logc`.""" R = self.R if kind == "M": ids = np.array(mem, dtype=np.int64) X = np.zeros((len(ids), R.dim), np.float32) if len(ids) == 0: return X X[:, R["one"]] = 1.0 X[:, R["tok"]] = self.C[ids] X[:, R["flg"]] = self.F[np.minimum(ids, vb.V)] X[1:, R["prev"]] = self.C[ids[:-1]] X[2:, R["prev2"]] = self.C[ids[:-2]] X[3:, R["prev3"]] = self.C[ids[:-3]] X[4:, R["prev4"]] = self.C[ids[:-4]] X[5:, R["prev5"]] = self.C[ids[:-5]] X[:, R["dpos"].start + min(self.mem_pos, NPOS - 1)] = 1.0 # the position the document was fetched for X[:, R["chat"].start] = float(self.mem_chat) # the document is a conversation reply return X ids = np.array([t for t, _ in mem], dtype=np.int64) X = np.zeros((len(ids), R.dim), np.float32) if len(ids) == 0: return X X[:, R["one"]] = 1.0 X[:, R["tok"]] = self.C[ids] X[:, R["flg"]] = self.F[np.minimum(ids, vb.V)] cnt = np.array([c for _, c in mem], dtype=np.float64) X[:, R["logc"].start] = np.log1p(cnt) # fixed count features of a candidate row X[:, R["ge5"].start] = (cnt >= 5).astype(np.float32) X[:, R["dom"].start] = (cnt >= 0.5 * cnt.sum()).astype(np.float32) X[:, R["full"].start] = float(self.dist_full) X[:, R["slot"].start] = np.arange(len(ids), dtype=np.float32) X[:, R["dpos"].start + min(self.dist_pos, NPOS - 1)] = 1.0 # the position the distribution was computed for return X # ------------------------------------------------------------------ forward dist_pos = 0 dist_full = 0 mem_pos = 0 mem_chat = 0 last_from = 3 # layers >= 3 are computed for the last position only (their outputs are read only there) def forward(self, ids, mem=None, dist=None, dist_pos=0, dist_full=0, mem_pos=0, mem_chat=0, keep=False): self.dist_pos, self.dist_full, self.mem_pos, self.mem_chat = dist_pos, dist_full, mem_pos, mem_chat X = self.embed_tokens(np.array(ids, dtype=np.int64)) banks = {"M": self.embed_memory(mem or [], "M"), "D": self.embed_memory(dist or [], "D")} T = len(ids) causal = np.tril(np.ones((T, T), dtype=bool)) self.trace = {} for layer in range(1, self.n_layers + 1): if layer == self.last_from: # from here on only the last position is read Xfull = X # keys/values of layer-3 heads still see every position X = X[-1:] causal = causal[-1:] T = 1 delta = np.zeros_like(X) for hd in self.heads: if hd.layer != layer: continue Q = X @ hd.Wq if hd.memory: B = banks[hd.memory] K, V = B @ hd.Wk, B @ hd.Wv S = Q.astype(np.float64) @ K.T.astype(np.float64) else: Xk = Xfull if layer >= self.last_from else X K, V = Xk @ hd.Wk, Xk @ hd.Wv S = Q.astype(np.float64) @ K.T.astype(np.float64) S = np.where(causal, S, -1e30) if hd.causal else S S = np.concatenate([S, np.full((T, 1), float(hd.sink))], axis=1) # sink slot, value 0 S -= S.max(axis=1, keepdims=True) A = np.exp(S) A /= A.sum(axis=1, keepdims=True) O = (A[:, :-1] @ V.astype(np.float64)).astype(np.float32) delta += O @ hd.Wo if keep: self.trace[hd.name] = A X = X + delta for m in self.mlps: if m.layer == layer: Hh = np.maximum(X @ m.W1 + m.b1, 0.0) X = X + Hh @ m.W2 if keep: self.trace[m.name] = Hh return X def logits(self, X_last): """Token logits = scale * ; action logits from the act dims.""" R = self.R tok = 40.0 * (self.C[:vb.V] @ X_last[R["out"]]) act = X_last[R["act"]] * 40.0 return np.concatenate([tok, act]) def vb_codes(): t = np.arange(vb.V + 1) a0, a1, a2 = t % vb.P, (t // vb.P) % vb.P, t // (vb.P * vb.P) C = np.zeros((vb.V + 1, CD), dtype=np.float32) for x in range(vb.K): C[t, x * vb.P + (a0 + a1 * x + a2 * x * x) % vb.P] = 1.0 / np.sqrt(vb.K) C[NULL] = 0.0 return C # ---------------------------------------------------------------------- edit list def apply_edits(model, lines, head_layouts): """Write an edit list into the zero-initialized parameters. Parameters are addressed as ., e.g. L1.prev.Wq or L3.ctl.W1; blocks are names in the residual layout (R) or in the unit's own layout.""" units = {f"L{h.layer}.{h.name}": h for h in model.heads} units.update({f"L{m.layer}.{m.name}": m for m in model.mlps}) C = model.C n = 0 for raw in lines: line = raw.split("#")[0].strip() if not line: continue tok = line.split() kind, param = tok[0], tok[1] unit_name, mat = param.rsplit(".", 1) unit = units[unit_name] W = getattr(unit, mat) lays = head_layouts.get(unit_name, {}) own = lays.get({"Wq": "q", "Wk": "q", "Wv": "v", "Wo": "v", "W1": "h", "b1": "h", "W2": "h"}.get(mat, "h")) def block(spec, axis_is_residual): # spec: name, name[i0:i1], or an integer index if spec.lstrip("-").isdigit(): i = int(spec) return slice(i, i + 1) name, rng = (spec.split("[", 1) + [""])[:2] lay = model.R if axis_is_residual else own s = lay[name] if rng: a, b = rng.rstrip("]").split(":") return slice(s.start + int(a), s.start + int(b)) return s rows_res = mat in ("Wq", "Wk", "Wv", "W1") # rows index the residual stream cols_res = mat in ("Wo", "W2") # columns index the residual stream if kind == "FILL": rs, cs = block(tok[2], rows_res), block(tok[3], cols_res) W[rs, cs] = float(tok[4]) elif kind == "EYE": rs, cs = block(tok[2], rows_res), block(tok[3], cols_res) scale = float(tok[4]) shift = int(tok[5]) if len(tok) > 5 else 0 nr = rs.stop - rs.start for i in range(nr): j = i + shift if 0 <= j < cs.stop - cs.start: W[rs.start + i, cs.start + j] = scale elif kind == "CODE": # code along the block of the first spec rs, cs = block(tok[2], rows_res), block(tok[3], cols_res) v, scale = int(tok[4]), float(tok[5]) if rs.stop - rs.start == CD: W[rs, cs.start] += scale * C[v] else: W[rs.start, cs] += scale * C[v] elif kind == "SET": rs, cs = block(tok[2], rows_res), block(tok[3], cols_res) W[rs.start, cs.start] = float(tok[4]) elif kind == "BIAS": # MLP bias: BIAS L3.ctl.b1 value cs = block(tok[2], False) unit.b1[cs] = float(tok[3]) elif kind == "SINK": unit.sink = np.float32(float(tok[2])) else: raise ValueError(line) n += 1 return n