Download engine.py from holland202/tiny-engine-165k: direct link, hf CLI and curl.
- Browser
- Download file 9.17 kB
-
https://huggingface.co/holland202/tiny-engine-165k/resolve/main/engine.py
- Command line
-
hf download hf://holland202/tiny-engine-165k/engine.py
-
curl -L -o engine.py https://huggingface.co/holland202/tiny-engine-165k/resolve/main/engine.py
9.17 kB
| """Tiny in-context engine: an attention-only transformer with rotary (relative) positions, trained from scratch on a | |
| mix of in-context tasks, then tested on lengths it never saw. See REGISTRATION.md. | |
| Written by Claude (Anthropic, Opus 5.5) at Chad Holland's direction, 2026-11-16. SELF-TESTED only.""" | |
| import math | |
| import numpy as np | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| N_CONTENT = 127 # content tokens 1..127 | |
| BOS, SEP, QRY = 0, 128, 129 | |
| VOCAB = 130 | |
| CFG = dict(d_model=128, n_heads=2, n_layers=2, train_lens=(32, 64, 96, 128), batch=64, lr=1e-3, steps=6000) | |
| PERIODS_SEEN = tuple(L for L in range(4, 25) if L not in (7, 11, 15, 19, 23)) | |
| PERIODS_HELD = (7, 11, 15, 19, 23) | |
| # ---------------------------------------------------------------- tasks: (tokens, loss mask), mask marks positions whose | |
| # NEXT token is determined by the context, so the score is only over learnable predictions. | |
| def task_periodic(rng, T, periods): | |
| L = int(rng.choice(periods)) | |
| seg = rng.integers(1, N_CONTENT + 1, size=L) | |
| x = np.concatenate([[BOS], np.resize(seg, T - 1)]) | |
| m = np.zeros(T, bool) | |
| m[L:T - 1] = True # predicting x[t+1] from position t, for t+1 > L | |
| return x, m | |
| KEYS = np.arange(1, 64) # keys and values come from disjoint halves of the vocabulary, so a key token can | |
| VALS = np.arange(64, N_CONTENT + 1) # never also appear as a value (that made the first version of this task ambiguous) | |
| def task_recall(rng, T): | |
| """BOS k1 v1 ... kn vn, then queries 'QRY k v' about earlier pairs, filling length T.""" | |
| n = min(len(KEYS), max(2, (T - 1) * 2 // 5 // 2 * 2 // 2)) | |
| nq = (T - 1 - 2 * n) // 3 | |
| while nq < 1: | |
| n -= 1 | |
| nq = (T - 1 - 2 * n) // 3 | |
| keys = rng.choice(KEYS, size=n, replace=False) | |
| vals = rng.choice(VALS, size=n) | |
| x = [BOS] + [t for kv in zip(keys, vals) for t in kv] | |
| m = [False] * len(x) | |
| for j in rng.integers(0, n, size=nq): | |
| x += [QRY, keys[j], vals[j]] | |
| m += [False, True, False] # position of the key predicts the value | |
| pad = T - len(x) | |
| x += list(rng.integers(1, N_CONTENT + 1, size=pad)) | |
| m += [False] * pad | |
| return np.array(x), np.array(m) | |
| def task_copy(rng, T): | |
| """BOS s SEP s: s has distinct tokens (so copying is unambiguous); length fills T.""" | |
| k = min((T - 2) // 2, N_CONTENT) | |
| s = rng.choice(np.arange(1, N_CONTENT + 1), size=k, replace=False) | |
| x = np.concatenate([[BOS], s, [SEP], s]) | |
| m = np.zeros(len(x), bool) | |
| m[k + 1:len(x) - 1] = True # from SEP onward, each next token is the copy | |
| pad = T - len(x) | |
| x = np.concatenate([x, rng.integers(1, N_CONTENT + 1, size=pad)]) | |
| m = np.concatenate([m, np.zeros(pad, bool)]) | |
| return x, m | |
| def task_copy_rep(rng, T, alpha=None): | |
| """TINY-ENGINE-003: BOS s SEP s where s draws, with replacement, from an alphabet of `alpha` distinct symbols | |
| (random size 2..127 when alpha is None), so symbols repeat inside s. Same layout and mask as task_copy.""" | |
| k = min((T - 2) // 2, N_CONTENT) | |
| a = int(rng.integers(2, N_CONTENT + 1)) if alpha is None else alpha | |
| sym = rng.choice(np.arange(1, N_CONTENT + 1), size=a, replace=False) | |
| s = rng.choice(sym, size=k, replace=True) | |
| x = np.concatenate([[BOS], s, [SEP], s]) | |
| m = np.zeros(len(x), bool) | |
| m[k + 1:len(x) - 1] = True | |
| pad = T - len(x) | |
| return (np.concatenate([x, rng.integers(1, N_CONTENT + 1, size=pad)]), | |
| np.concatenate([m, np.zeros(pad, bool)])) | |
| TASKS = ("periodic", "recall", "copy") | |
| def make_batch(rng, n, T, task=None, periods=PERIODS_SEEN, rep_frac=0.0): | |
| """rep_frac (TINY-ENGINE-003): share of copy examples drawn by task_copy_rep. At 0.0 no extra random numbers are | |
| drawn, so CFG and CFG2 batches are unchanged.""" | |
| xs, ms = [], [] | |
| for _ in range(n): | |
| t = task or TASKS[int(rng.integers(len(TASKS)))] | |
| x, m = (task_periodic(rng, T, periods) if t == "periodic" else task_recall(rng, T) if t == "recall" | |
| else task_copy_rep(rng, T) if rep_frac and rng.random() < rep_frac else task_copy(rng, T)) | |
| xs.append(x); ms.append(m) | |
| return torch.tensor(np.array(xs)).long(), torch.tensor(np.array(ms)) | |
| # ---------------------------------------------------------------- model | |
| def rope(x, base=10000.0, frac=1.0): | |
| """Rotary position embedding on the first `frac` of the last dim of (b, h, t, d); the rest is left position-free.""" | |
| if frac < 1.0: | |
| r = int(x.shape[-1] * frac) // 2 * 2 | |
| return torch.cat([rope(x[..., :r], base), x[..., r:]], -1) | |
| t, d = x.shape[-2], x.shape[-1] | |
| inv = 1.0 / base ** (torch.arange(0, d, 2, dtype=torch.float32) / d) | |
| ang = torch.arange(t, dtype=torch.float32)[:, None] * inv[None] | |
| cos, sin = ang.cos(), ang.sin() | |
| x1, x2 = x[..., 0::2], x[..., 1::2] | |
| return torch.stack([x1 * cos - x2 * sin, x1 * sin + x2 * cos], -1).flatten(-2) | |
| class Attn(nn.Module): | |
| def __init__(self, d, h, use_rope=True, rope_frac=1.0): | |
| super().__init__() | |
| self.h, self.dh, self.use_rope, self.rope_frac = h, d // h, use_rope, rope_frac | |
| self.q, self.k, self.v, self.o = (nn.Linear(d, d, bias=False) for _ in range(4)) | |
| def forward(self, x): | |
| b, t, d = x.shape | |
| sh = lambda z: z.view(b, t, self.h, self.dh).transpose(1, 2) | |
| q, k = sh(self.q(x)), sh(self.k(x)) | |
| if self.use_rope: | |
| q, k = rope(q, frac=self.rope_frac), rope(k, frac=self.rope_frac) | |
| y = F.scaled_dot_product_attention(q, k, sh(self.v(x)), is_causal=True) | |
| return self.o(y.transpose(1, 2).reshape(b, t, d)) | |
| class TinyEngine(nn.Module): | |
| """Token embedding (no position embedding: positions enter only through rotary attention), attention-only | |
| pre-LN blocks, separate (untied) unembedding. Tying the unembedding to the embedding stopped learning entirely | |
| in exploration (0.00 periodic accuracy after 1,500 steps at d_model 256; untied reached 1.00).""" | |
| def __init__(self, cfg=CFG): | |
| super().__init__() | |
| d = cfg["d_model"] | |
| self.pos_kind = cfg.get("pos", "rope") | |
| self.emb = nn.Embedding(VOCAB, d) | |
| self.pos = nn.Embedding(1024, d) if self.pos_kind == "learned" else None | |
| self.lns = nn.ModuleList(nn.LayerNorm(d) for _ in range(cfg["n_layers"])) | |
| nope = set(cfg.get("nope_layers", ())) # layers with no positional encoding at all (content-only) | |
| self.attn = nn.ModuleList(Attn(d, cfg["n_heads"], self.pos_kind == "rope" and i not in nope, | |
| cfg.get("rope_frac", 1.0)) for i in range(cfg["n_layers"])) | |
| self.lnf = nn.LayerNorm(d) | |
| self.head = None if cfg.get("tied", False) else nn.Linear(d, VOCAB, bias=False) | |
| def forward(self, x): | |
| h = self.emb(x) | |
| if self.pos is not None: | |
| h = h + self.pos(torch.arange(x.shape[1])) | |
| for ln, at in zip(self.lns, self.attn): | |
| h = h + at(ln(h)) | |
| h = self.lnf(h) | |
| return self.head(h) if self.head is not None else h @ self.emb.weight.T | |
| def n_params(m): | |
| return sum(p.numel() for p in m.parameters()) | |
| # ---------------------------------------------------------------- evaluation | |
| def accuracy(m, task, T, seed, n=256, periods=PERIODS_SEEN): | |
| x, mask = make_batch(np.random.default_rng(seed), n, T, task, periods) | |
| pred = m(x).argmax(-1)[:, :-1] | |
| ok = (pred == x[:, 1:])[mask[:, :-1]] | |
| return float(ok.float().mean()) | |
| EVAL_LENS = (128, 256, 512, 1024) # 128 = longest trained; 256, 512, 1024 = 2x, 4x and 8x beyond it | |
| # TINY-ENGINE-002: the fix. Layer 1 (the layer that matches content) gets no positional encoding, so matching an | |
| # earlier token cannot depend on how far back it is. Layer 0 keeps rotary positions (it needs order to look one | |
| # token back). Same parameter count as CFG. | |
| CFG2 = dict(CFG, nope_layers=(1,)) | |
| # TINY-ENGINE-003: CFG2 with half of the copy examples using repeated symbols (task_copy_rep). Same model and size. | |
| CFG3 = dict(CFG2, copy_rep_frac=0.5) | |
| def evaluate(m): | |
| m.eval() | |
| r = {} | |
| for L in EVAL_LENS: | |
| n = 256 if L <= 128 else 96 | |
| r[f"periodic_heldout_L{L}"] = accuracy(m, "periodic", L, 100 + L, n, periods=PERIODS_HELD) | |
| r[f"recall_L{L}"] = accuracy(m, "recall", L, 200 + L, n) | |
| r[f"copy_L{L}"] = accuracy(m, "copy", L, 300 + L, n) | |
| r["periodic_seen_L128"] = accuracy(m, "periodic", 128, 99) | |
| m.train() | |
| return r | |
| def train(seed, cfg=CFG, log=None): | |
| torch.manual_seed(seed) | |
| torch.use_deterministic_algorithms(True) | |
| rng = np.random.default_rng(seed) | |
| m = TinyEngine(cfg) | |
| opt = torch.optim.AdamW(m.parameters(), lr=cfg["lr"], weight_decay=0.0) | |
| for step in range(1, cfg["steps"] + 1): | |
| x, mask = make_batch(rng, cfg["batch"], int(rng.choice(cfg["train_lens"])), rep_frac=cfg.get("copy_rep_frac", 0.0)) | |
| logits = m(x)[:, :-1] | |
| loss = F.cross_entropy(logits[mask[:, :-1]], x[:, 1:][mask[:, :-1]]) | |
| opt.zero_grad(); loss.backward(); opt.step() | |
| if log and step % 500 == 0: | |
| log(step, float(loss.detach())) | |
| return m | |