Download eval3.py from holland202/tiny-engine-165k: direct link, hf CLI and curl.
- Browser
- Download file 1.97 kB
-
https://huggingface.co/holland202/tiny-engine-165k/resolve/main/eval3.py
- Command line
-
hf download hf://holland202/tiny-engine-165k/eval3.py
-
curl -L -o eval3.py https://huggingface.co/holland202/tiny-engine-165k/resolve/main/eval3.py
1.97 kB
| """TINY-ENGINE-003 evaluation (registered in REGISTRATION_003.md). Fixed case seeds in the 8000 range, never used before | |
| this registration. Claude (Anthropic, Opus 5.5), 2026-11-20. SELF-TESTED only.""" | |
| import numpy as np | |
| import torch | |
| import engine as E | |
| FAMS = {"distinct": None, "alpha32": 32, "alpha8": 8} | |
| KS = (63, 127) # 63 = longest trained copy, 127 = 2x | |
| N_TF, N_FREE = 256, 100 | |
| def case(rng, k, fam): | |
| T = 2 * k + 2 | |
| return E.task_copy(rng, T) if FAMS[fam] is None else E.task_copy_rep(rng, T, FAMS[fam]) | |
| def tf_score(pred, x, mask): | |
| """Share of masked next tokens predicted correctly. pred = argmax over positions 0..T-2.""" | |
| return float((pred == x[:, 1:])[mask[:, :-1]].float().mean()) | |
| def tf_acc(m, fam, k, seed, n=N_TF): | |
| rng = np.random.default_rng(seed) | |
| xs, ms = zip(*[case(rng, k, fam) for _ in range(n)]) | |
| x, mask = torch.tensor(np.array(xs)), torch.tensor(np.array(ms)) | |
| return tf_score(m(x).argmax(-1)[:, :-1], x, mask) | |
| def free_whole(m, fam, k, seed, n=N_FREE): | |
| """Share of answers exactly right when the model writes all k copy tokens itself (greedy). The prefix is | |
| BOS + source + SEP only; the target is never fed in.""" | |
| rng = np.random.default_rng(seed) | |
| xs = [case(rng, k, fam)[0] for _ in range(n)] | |
| src = torch.tensor(np.array([x[1:k + 1] for x in xs])) | |
| y = torch.tensor(np.array([x[:k + 2] for x in xs])) | |
| for _ in range(k): | |
| y = torch.cat([y, m(y)[:, -1:].argmax(-1)], 1) | |
| return float((y[:, -k:] == src).all(1).float().mean()) | |
| def evaluate3(m): | |
| m.eval() | |
| r = {} | |
| for i, fam in enumerate(FAMS): | |
| for k in KS: | |
| r[f"tf_{fam}_k{k}"] = tf_acc(m, fam, k, 8000 + 10 * i + k) | |
| r["free_distinct_k127"] = free_whole(m, "distinct", 127, 8501) | |
| r["free_alpha32_k63"] = free_whole(m, "alpha32", 63, 8502) | |
| r["free_alpha8_k63"] = free_whole(m, "alpha8", 63, 8503) | |
| m.train() | |
| return r | |