Download wplm/engine.py from Leejh123e/WPLM: direct link, hf CLI and curl.
- Browser
- Download file 11.2 kB
-
https://huggingface.co/Leejh123e/WPLM/resolve/main/wplm/engine.py
- Command line
-
hf download hf://Leejh123e/WPLM/wplm/engine.py
-
curl -L -o engine.py https://huggingface.co/Leejh123e/WPLM/resolve/main/wplm/engine.py
11.2 kB
| """A plain decoder transformer whose weights are programmed directly as an edit list (no training, no program counter). | |
| Forward pass (standard): x_t = embedding(token_t) + position(t); for each layer: x += sum_heads W_O softmax(QK^T) V, | |
| then x += W_2 ReLU(W_1 x + b_1); logits = W_U x. No LayerNorm. Two extras, both ordinary components: | |
| * memory banks: a head may attend over an external bank (token embeddings provided by the host) instead of the | |
| sequence; such heads are not causally masked. | |
| * a sink slot per head: one extra key with a fixed score (a weight) and a zero value, so that "nothing matches" | |
| is representable. | |
| All parameters start at zero; `apply_edits` writes the edit list into them. | |
| Edit list lines (text, one edit per line; block names refer to the residual layout or a head's own dims): | |
| FILL <param> <rowblock>[i0:i1] <colblock>[j0:j1] <value> | |
| EYE <param> <rowblock> <colblock> <scale> [shift] identity (optionally shifted) between equal-size blocks | |
| CODE <param> <rowblock>|<i> <colblock>|<j> <token> <scale> the token's embedding code placed along a row/column | |
| SET <param> <i> <j> <value> | |
| """ | |
| import os | |
| import sys | |
| import numpy as np | |
| from . import vocab as vb # noqa: E402 | |
| CD, NF = vb.CODE_DIM, len(vb.FLAG_NAMES) | |
| NPOS = 512 | |
| NULL = vb.V # code row of zeros | |
| ACTIONS = ["A_DIST_ANCH", "A_DIST_WEB", "A_DOC", "A_DOC_PREFIX"] | |
| class Layout: | |
| """Named blocks of a vector space (the residual stream, or a head's query/value space).""" | |
| def __init__(self, blocks): | |
| self.sl, o = {}, 0 | |
| for name, n in blocks: | |
| self.sl[name] = slice(o, o + n) | |
| o += n | |
| self.dim = o | |
| def __getitem__(self, name): | |
| return self.sl[name] | |
| class Head: | |
| def __init__(self, name, layer, dq, dv, d, memory=None, sink=0.0, causal=True): | |
| self.name, self.layer, self.memory, self.causal = name, layer, memory, causal | |
| self.Wq = np.zeros((d, dq), np.float32) | |
| self.Wk = np.zeros((d, dq), np.float32) | |
| self.Wv = np.zeros((d, dv), np.float32) | |
| self.Wo = np.zeros((dv, d), np.float32) | |
| self.sink = np.float32(sink) | |
| class MLP: | |
| def __init__(self, name, layer, n, d): | |
| self.name, self.layer = name, layer | |
| self.W1 = np.zeros((d, n), np.float32) | |
| self.b1 = np.zeros(n, np.float32) | |
| self.W2 = np.zeros((n, d), np.float32) | |
| class Model: | |
| def __init__(self, layout, heads, mlps, n_layers, out_tokens, act_dims): | |
| self.R, self.heads, self.mlps, self.n_layers = layout, heads, mlps, n_layers | |
| self.C = vb_codes() | |
| self.F = vb.flags_matrix() | |
| self.out_tokens, self.act_dims = out_tokens, act_dims | |
| self.trace = {} | |
| # ------------------------------------------------------------------ embedding (fixed formula) | |
| def embed_tokens(self, ids): | |
| R = self.R | |
| X = np.zeros((len(ids), R.dim), np.float32) | |
| X[:, R["one"]] = 1.0 | |
| X[:, R["tok"]] = self.C[ids] | |
| X[:, R["flg"]] = self.F[np.minimum(ids, vb.V)] | |
| X[np.arange(len(ids)), R["pos"].start + np.minimum(np.arange(len(ids)), NPOS - 1)] = 1.0 | |
| return X | |
| def embed_memory(self, mem, kind): | |
| """Memory bank rows in residual coordinates. M rows: code of m_j in `tok`, m_{j-1} in `prev`, m_{j-2} in | |
| `prev2`, flags in `flg`; D rows: candidate code in `tok`, flags, log count in `logc`.""" | |
| R = self.R | |
| if kind == "M": | |
| ids = np.array(mem, dtype=np.int64) | |
| X = np.zeros((len(ids), R.dim), np.float32) | |
| if len(ids) == 0: | |
| return X | |
| X[:, R["one"]] = 1.0 | |
| X[:, R["tok"]] = self.C[ids] | |
| X[:, R["flg"]] = self.F[np.minimum(ids, vb.V)] | |
| X[1:, R["prev"]] = self.C[ids[:-1]] | |
| X[2:, R["prev2"]] = self.C[ids[:-2]] | |
| X[3:, R["prev3"]] = self.C[ids[:-3]] | |
| X[4:, R["prev4"]] = self.C[ids[:-4]] | |
| X[5:, R["prev5"]] = self.C[ids[:-5]] | |
| X[:, R["dpos"].start + min(self.mem_pos, NPOS - 1)] = 1.0 # the position the document was fetched for | |
| X[:, R["chat"].start] = float(self.mem_chat) # the document is a conversation reply | |
| return X | |
| ids = np.array([t for t, _ in mem], dtype=np.int64) | |
| X = np.zeros((len(ids), R.dim), np.float32) | |
| if len(ids) == 0: | |
| return X | |
| X[:, R["one"]] = 1.0 | |
| X[:, R["tok"]] = self.C[ids] | |
| X[:, R["flg"]] = self.F[np.minimum(ids, vb.V)] | |
| cnt = np.array([c for _, c in mem], dtype=np.float64) | |
| X[:, R["logc"].start] = np.log1p(cnt) # fixed count features of a candidate row | |
| X[:, R["ge5"].start] = (cnt >= 5).astype(np.float32) | |
| X[:, R["dom"].start] = (cnt >= 0.5 * cnt.sum()).astype(np.float32) | |
| X[:, R["full"].start] = float(self.dist_full) | |
| X[:, R["slot"].start] = np.arange(len(ids), dtype=np.float32) | |
| X[:, R["dpos"].start + min(self.dist_pos, NPOS - 1)] = 1.0 # the position the distribution was computed for | |
| return X | |
| # ------------------------------------------------------------------ forward | |
| dist_pos = 0 | |
| dist_full = 0 | |
| mem_pos = 0 | |
| mem_chat = 0 | |
| last_from = 3 # layers >= 3 are computed for the last position only (their outputs are read only there) | |
| def forward(self, ids, mem=None, dist=None, dist_pos=0, dist_full=0, mem_pos=0, mem_chat=0, keep=False): | |
| self.dist_pos, self.dist_full, self.mem_pos, self.mem_chat = dist_pos, dist_full, mem_pos, mem_chat | |
| X = self.embed_tokens(np.array(ids, dtype=np.int64)) | |
| banks = {"M": self.embed_memory(mem or [], "M"), "D": self.embed_memory(dist or [], "D")} | |
| T = len(ids) | |
| causal = np.tril(np.ones((T, T), dtype=bool)) | |
| self.trace = {} | |
| for layer in range(1, self.n_layers + 1): | |
| if layer == self.last_from: # from here on only the last position is read | |
| Xfull = X # keys/values of layer-3 heads still see every position | |
| X = X[-1:] | |
| causal = causal[-1:] | |
| T = 1 | |
| delta = np.zeros_like(X) | |
| for hd in self.heads: | |
| if hd.layer != layer: | |
| continue | |
| Q = X @ hd.Wq | |
| if hd.memory: | |
| B = banks[hd.memory] | |
| K, V = B @ hd.Wk, B @ hd.Wv | |
| S = Q.astype(np.float64) @ K.T.astype(np.float64) | |
| else: | |
| Xk = Xfull if layer >= self.last_from else X | |
| K, V = Xk @ hd.Wk, Xk @ hd.Wv | |
| S = Q.astype(np.float64) @ K.T.astype(np.float64) | |
| S = np.where(causal, S, -1e30) if hd.causal else S | |
| S = np.concatenate([S, np.full((T, 1), float(hd.sink))], axis=1) # sink slot, value 0 | |
| S -= S.max(axis=1, keepdims=True) | |
| A = np.exp(S) | |
| A /= A.sum(axis=1, keepdims=True) | |
| O = (A[:, :-1] @ V.astype(np.float64)).astype(np.float32) | |
| delta += O @ hd.Wo | |
| if keep: | |
| self.trace[hd.name] = A | |
| X = X + delta | |
| for m in self.mlps: | |
| if m.layer == layer: | |
| Hh = np.maximum(X @ m.W1 + m.b1, 0.0) | |
| X = X + Hh @ m.W2 | |
| if keep: | |
| self.trace[m.name] = Hh | |
| return X | |
| def logits(self, X_last): | |
| """Token logits = scale * <code_v, out>; action logits from the act dims.""" | |
| R = self.R | |
| tok = 40.0 * (self.C[:vb.V] @ X_last[R["out"]]) | |
| act = X_last[R["act"]] * 40.0 | |
| return np.concatenate([tok, act]) | |
| def vb_codes(): | |
| t = np.arange(vb.V + 1) | |
| a0, a1, a2 = t % vb.P, (t // vb.P) % vb.P, t // (vb.P * vb.P) | |
| C = np.zeros((vb.V + 1, CD), dtype=np.float32) | |
| for x in range(vb.K): | |
| C[t, x * vb.P + (a0 + a1 * x + a2 * x * x) % vb.P] = 1.0 / np.sqrt(vb.K) | |
| C[NULL] = 0.0 | |
| return C | |
| # ---------------------------------------------------------------------- edit list | |
| def apply_edits(model, lines, head_layouts): | |
| """Write an edit list into the zero-initialized parameters. Parameters are addressed as <unit>.<matrix>, e.g. | |
| L1.prev.Wq or L3.ctl.W1; blocks are names in the residual layout (R) or in the unit's own layout.""" | |
| units = {f"L{h.layer}.{h.name}": h for h in model.heads} | |
| units.update({f"L{m.layer}.{m.name}": m for m in model.mlps}) | |
| C = model.C | |
| n = 0 | |
| for raw in lines: | |
| line = raw.split("#")[0].strip() | |
| if not line: | |
| continue | |
| tok = line.split() | |
| kind, param = tok[0], tok[1] | |
| unit_name, mat = param.rsplit(".", 1) | |
| unit = units[unit_name] | |
| W = getattr(unit, mat) | |
| lays = head_layouts.get(unit_name, {}) | |
| own = lays.get({"Wq": "q", "Wk": "q", "Wv": "v", "Wo": "v", "W1": "h", "b1": "h", "W2": "h"}.get(mat, "h")) | |
| def block(spec, axis_is_residual): | |
| # spec: name, name[i0:i1], or an integer index | |
| if spec.lstrip("-").isdigit(): | |
| i = int(spec) | |
| return slice(i, i + 1) | |
| name, rng = (spec.split("[", 1) + [""])[:2] | |
| lay = model.R if axis_is_residual else own | |
| s = lay[name] | |
| if rng: | |
| a, b = rng.rstrip("]").split(":") | |
| return slice(s.start + int(a), s.start + int(b)) | |
| return s | |
| rows_res = mat in ("Wq", "Wk", "Wv", "W1") # rows index the residual stream | |
| cols_res = mat in ("Wo", "W2") # columns index the residual stream | |
| if kind == "FILL": | |
| rs, cs = block(tok[2], rows_res), block(tok[3], cols_res) | |
| W[rs, cs] = float(tok[4]) | |
| elif kind == "EYE": | |
| rs, cs = block(tok[2], rows_res), block(tok[3], cols_res) | |
| scale = float(tok[4]) | |
| shift = int(tok[5]) if len(tok) > 5 else 0 | |
| nr = rs.stop - rs.start | |
| for i in range(nr): | |
| j = i + shift | |
| if 0 <= j < cs.stop - cs.start: | |
| W[rs.start + i, cs.start + j] = scale | |
| elif kind == "CODE": # code along the block of the first spec | |
| rs, cs = block(tok[2], rows_res), block(tok[3], cols_res) | |
| v, scale = int(tok[4]), float(tok[5]) | |
| if rs.stop - rs.start == CD: | |
| W[rs, cs.start] += scale * C[v] | |
| else: | |
| W[rs.start, cs] += scale * C[v] | |
| elif kind == "SET": | |
| rs, cs = block(tok[2], rows_res), block(tok[3], cols_res) | |
| W[rs.start, cs.start] = float(tok[4]) | |
| elif kind == "BIAS": # MLP bias: BIAS L3.ctl.b1 <unitblock|i> value | |
| cs = block(tok[2], False) | |
| unit.b1[cs] = float(tok[3]) | |
| elif kind == "SINK": | |
| unit.sink = np.float32(float(tok[2])) | |
| else: | |
| raise ValueError(line) | |
| n += 1 | |
| return n | |