WPLM / wplm /engine.py
Leejh123e's picture
v1.3: rename WP-LLM to WPLM by editing the weight file only
96f1308 verified
Raw History Blame Contribute Delete
11.2 kB
"""A plain decoder transformer whose weights are programmed directly as an edit list (no training, no program counter).
Forward pass (standard): x_t = embedding(token_t) + position(t); for each layer: x += sum_heads W_O softmax(QK^T) V,
then x += W_2 ReLU(W_1 x + b_1); logits = W_U x. No LayerNorm. Two extras, both ordinary components:
* memory banks: a head may attend over an external bank (token embeddings provided by the host) instead of the
sequence; such heads are not causally masked.
* a sink slot per head: one extra key with a fixed score (a weight) and a zero value, so that "nothing matches"
is representable.
All parameters start at zero; `apply_edits` writes the edit list into them.
Edit list lines (text, one edit per line; block names refer to the residual layout or a head's own dims):
FILL <param> <rowblock>[i0:i1] <colblock>[j0:j1] <value>
EYE <param> <rowblock> <colblock> <scale> [shift] identity (optionally shifted) between equal-size blocks
CODE <param> <rowblock>|<i> <colblock>|<j> <token> <scale> the token's embedding code placed along a row/column
SET <param> <i> <j> <value>
"""
import os
import sys
import numpy as np
from . import vocab as vb # noqa: E402
CD, NF = vb.CODE_DIM, len(vb.FLAG_NAMES)
NPOS = 512
NULL = vb.V # code row of zeros
ACTIONS = ["A_DIST_ANCH", "A_DIST_WEB", "A_DOC", "A_DOC_PREFIX"]
class Layout:
"""Named blocks of a vector space (the residual stream, or a head's query/value space)."""
def __init__(self, blocks):
self.sl, o = {}, 0
for name, n in blocks:
self.sl[name] = slice(o, o + n)
o += n
self.dim = o
def __getitem__(self, name):
return self.sl[name]
class Head:
def __init__(self, name, layer, dq, dv, d, memory=None, sink=0.0, causal=True):
self.name, self.layer, self.memory, self.causal = name, layer, memory, causal
self.Wq = np.zeros((d, dq), np.float32)
self.Wk = np.zeros((d, dq), np.float32)
self.Wv = np.zeros((d, dv), np.float32)
self.Wo = np.zeros((dv, d), np.float32)
self.sink = np.float32(sink)
class MLP:
def __init__(self, name, layer, n, d):
self.name, self.layer = name, layer
self.W1 = np.zeros((d, n), np.float32)
self.b1 = np.zeros(n, np.float32)
self.W2 = np.zeros((n, d), np.float32)
class Model:
def __init__(self, layout, heads, mlps, n_layers, out_tokens, act_dims):
self.R, self.heads, self.mlps, self.n_layers = layout, heads, mlps, n_layers
self.C = vb_codes()
self.F = vb.flags_matrix()
self.out_tokens, self.act_dims = out_tokens, act_dims
self.trace = {}
# ------------------------------------------------------------------ embedding (fixed formula)
def embed_tokens(self, ids):
R = self.R
X = np.zeros((len(ids), R.dim), np.float32)
X[:, R["one"]] = 1.0
X[:, R["tok"]] = self.C[ids]
X[:, R["flg"]] = self.F[np.minimum(ids, vb.V)]
X[np.arange(len(ids)), R["pos"].start + np.minimum(np.arange(len(ids)), NPOS - 1)] = 1.0
return X
def embed_memory(self, mem, kind):
"""Memory bank rows in residual coordinates. M rows: code of m_j in `tok`, m_{j-1} in `prev`, m_{j-2} in
`prev2`, flags in `flg`; D rows: candidate code in `tok`, flags, log count in `logc`."""
R = self.R
if kind == "M":
ids = np.array(mem, dtype=np.int64)
X = np.zeros((len(ids), R.dim), np.float32)
if len(ids) == 0:
return X
X[:, R["one"]] = 1.0
X[:, R["tok"]] = self.C[ids]
X[:, R["flg"]] = self.F[np.minimum(ids, vb.V)]
X[1:, R["prev"]] = self.C[ids[:-1]]
X[2:, R["prev2"]] = self.C[ids[:-2]]
X[3:, R["prev3"]] = self.C[ids[:-3]]
X[4:, R["prev4"]] = self.C[ids[:-4]]
X[5:, R["prev5"]] = self.C[ids[:-5]]
X[:, R["dpos"].start + min(self.mem_pos, NPOS - 1)] = 1.0 # the position the document was fetched for
X[:, R["chat"].start] = float(self.mem_chat) # the document is a conversation reply
return X
ids = np.array([t for t, _ in mem], dtype=np.int64)
X = np.zeros((len(ids), R.dim), np.float32)
if len(ids) == 0:
return X
X[:, R["one"]] = 1.0
X[:, R["tok"]] = self.C[ids]
X[:, R["flg"]] = self.F[np.minimum(ids, vb.V)]
cnt = np.array([c for _, c in mem], dtype=np.float64)
X[:, R["logc"].start] = np.log1p(cnt) # fixed count features of a candidate row
X[:, R["ge5"].start] = (cnt >= 5).astype(np.float32)
X[:, R["dom"].start] = (cnt >= 0.5 * cnt.sum()).astype(np.float32)
X[:, R["full"].start] = float(self.dist_full)
X[:, R["slot"].start] = np.arange(len(ids), dtype=np.float32)
X[:, R["dpos"].start + min(self.dist_pos, NPOS - 1)] = 1.0 # the position the distribution was computed for
return X
# ------------------------------------------------------------------ forward
dist_pos = 0
dist_full = 0
mem_pos = 0
mem_chat = 0
last_from = 3 # layers >= 3 are computed for the last position only (their outputs are read only there)
def forward(self, ids, mem=None, dist=None, dist_pos=0, dist_full=0, mem_pos=0, mem_chat=0, keep=False):
self.dist_pos, self.dist_full, self.mem_pos, self.mem_chat = dist_pos, dist_full, mem_pos, mem_chat
X = self.embed_tokens(np.array(ids, dtype=np.int64))
banks = {"M": self.embed_memory(mem or [], "M"), "D": self.embed_memory(dist or [], "D")}
T = len(ids)
causal = np.tril(np.ones((T, T), dtype=bool))
self.trace = {}
for layer in range(1, self.n_layers + 1):
if layer == self.last_from: # from here on only the last position is read
Xfull = X # keys/values of layer-3 heads still see every position
X = X[-1:]
causal = causal[-1:]
T = 1
delta = np.zeros_like(X)
for hd in self.heads:
if hd.layer != layer:
continue
Q = X @ hd.Wq
if hd.memory:
B = banks[hd.memory]
K, V = B @ hd.Wk, B @ hd.Wv
S = Q.astype(np.float64) @ K.T.astype(np.float64)
else:
Xk = Xfull if layer >= self.last_from else X
K, V = Xk @ hd.Wk, Xk @ hd.Wv
S = Q.astype(np.float64) @ K.T.astype(np.float64)
S = np.where(causal, S, -1e30) if hd.causal else S
S = np.concatenate([S, np.full((T, 1), float(hd.sink))], axis=1) # sink slot, value 0
S -= S.max(axis=1, keepdims=True)
A = np.exp(S)
A /= A.sum(axis=1, keepdims=True)
O = (A[:, :-1] @ V.astype(np.float64)).astype(np.float32)
delta += O @ hd.Wo
if keep:
self.trace[hd.name] = A
X = X + delta
for m in self.mlps:
if m.layer == layer:
Hh = np.maximum(X @ m.W1 + m.b1, 0.0)
X = X + Hh @ m.W2
if keep:
self.trace[m.name] = Hh
return X
def logits(self, X_last):
"""Token logits = scale * <code_v, out>; action logits from the act dims."""
R = self.R
tok = 40.0 * (self.C[:vb.V] @ X_last[R["out"]])
act = X_last[R["act"]] * 40.0
return np.concatenate([tok, act])
def vb_codes():
t = np.arange(vb.V + 1)
a0, a1, a2 = t % vb.P, (t // vb.P) % vb.P, t // (vb.P * vb.P)
C = np.zeros((vb.V + 1, CD), dtype=np.float32)
for x in range(vb.K):
C[t, x * vb.P + (a0 + a1 * x + a2 * x * x) % vb.P] = 1.0 / np.sqrt(vb.K)
C[NULL] = 0.0
return C
# ---------------------------------------------------------------------- edit list
def apply_edits(model, lines, head_layouts):
"""Write an edit list into the zero-initialized parameters. Parameters are addressed as <unit>.<matrix>, e.g.
L1.prev.Wq or L3.ctl.W1; blocks are names in the residual layout (R) or in the unit's own layout."""
units = {f"L{h.layer}.{h.name}": h for h in model.heads}
units.update({f"L{m.layer}.{m.name}": m for m in model.mlps})
C = model.C
n = 0
for raw in lines:
line = raw.split("#")[0].strip()
if not line:
continue
tok = line.split()
kind, param = tok[0], tok[1]
unit_name, mat = param.rsplit(".", 1)
unit = units[unit_name]
W = getattr(unit, mat)
lays = head_layouts.get(unit_name, {})
own = lays.get({"Wq": "q", "Wk": "q", "Wv": "v", "Wo": "v", "W1": "h", "b1": "h", "W2": "h"}.get(mat, "h"))
def block(spec, axis_is_residual):
# spec: name, name[i0:i1], or an integer index
if spec.lstrip("-").isdigit():
i = int(spec)
return slice(i, i + 1)
name, rng = (spec.split("[", 1) + [""])[:2]
lay = model.R if axis_is_residual else own
s = lay[name]
if rng:
a, b = rng.rstrip("]").split(":")
return slice(s.start + int(a), s.start + int(b))
return s
rows_res = mat in ("Wq", "Wk", "Wv", "W1") # rows index the residual stream
cols_res = mat in ("Wo", "W2") # columns index the residual stream
if kind == "FILL":
rs, cs = block(tok[2], rows_res), block(tok[3], cols_res)
W[rs, cs] = float(tok[4])
elif kind == "EYE":
rs, cs = block(tok[2], rows_res), block(tok[3], cols_res)
scale = float(tok[4])
shift = int(tok[5]) if len(tok) > 5 else 0
nr = rs.stop - rs.start
for i in range(nr):
j = i + shift
if 0 <= j < cs.stop - cs.start:
W[rs.start + i, cs.start + j] = scale
elif kind == "CODE": # code along the block of the first spec
rs, cs = block(tok[2], rows_res), block(tok[3], cols_res)
v, scale = int(tok[4]), float(tok[5])
if rs.stop - rs.start == CD:
W[rs, cs.start] += scale * C[v]
else:
W[rs.start, cs] += scale * C[v]
elif kind == "SET":
rs, cs = block(tok[2], rows_res), block(tok[3], cols_res)
W[rs.start, cs.start] = float(tok[4])
elif kind == "BIAS": # MLP bias: BIAS L3.ctl.b1 <unitblock|i> value
cs = block(tok[2], False)
unit.b1[cs] = float(tok[3])
elif kind == "SINK":
unit.sink = np.float32(float(tok[2]))
else:
raise ValueError(line)
n += 1
return n