tiny-engine-165k / test_eval3.py
holland202's picture
Version 3 (TINY-ENGINE-003): repeated-symbol copy training
60a72cf verified
Raw History Blame Contribute Delete
3.98 kB
"""Anti-vacuity and fail-closed checks for eval3.py / score3.py (REGISTRATION_003.md P5).
Claude (Anthropic, Opus 5.5), 2026-11-20. SELF-TESTED only."""
import json, subprocess, sys, tempfile, unittest
from pathlib import Path
import numpy as np
import torch
import torch.nn.functional as F
import engine as E
import eval3
HERE = Path(__file__).parent
class Oracle(torch.nn.Module):
"""Predicts the copy exactly: the token k+1 positions back. shift=1 makes it off by one."""
def __init__(self, k, shift=0):
super().__init__(); self.k, self.shift = k, shift
def forward(self, x):
# position t predicts x[t+1]; in the copy, x[t+1] = x[t-k] (k source tokens plus SEP back)
return F.one_hot(torch.roll(x, self.k + self.shift, 1), E.VOCAB).float()
class Cases(unittest.TestCase):
def test_families(self):
rng = np.random.default_rng(0)
for fam, cap in (("distinct", None), ("alpha32", 32), ("alpha8", 8)):
x, m = eval3.case(rng, 63, fam)
s = x[1:64]
self.assertEqual(x[64], E.SEP); self.assertTrue((x[65:128] == s).all())
self.assertFalse(m[:64].any()); self.assertTrue(m[64:127].all())
if cap is None:
self.assertEqual(len(set(s)), 63)
else:
self.assertLessEqual(len(set(s)), cap); self.assertLess(len(set(s)), 63)
class Scores(unittest.TestCase):
def test_tf_oracle_and_offbyone(self):
for fam in eval3.FAMS:
self.assertEqual(eval3.tf_acc(Oracle(63), fam, 63, 1, n=32), 1.0)
self.assertLess(eval3.tf_acc(Oracle(63, shift=1), fam, 63, 1, n=32), 0.5)
def test_free_oracle_and_offbyone(self):
self.assertEqual(eval3.free_whole(Oracle(63), "alpha8", 63, 2, n=16), 1.0)
self.assertEqual(eval3.free_whole(Oracle(63, shift=1), "alpha8", 63, 2, n=16), 0.0)
def test_untrained_below_005(self):
torch.manual_seed(991)
r = eval3.evaluate3(E.TinyEngine(E.CFG3))
self.assertTrue(all(v < 0.05 for v in r.values()), r)
def fake(d, arm, cfg, seeds, b_wins):
p = d / arm; p.mkdir(parents=True)
for i, s in enumerate(seeds):
sc = {f"{t}_L{L}": 0.99 for L in (128, 256, 512, 1024) for t in ("periodic_heldout", "recall", "copy")}
base = 0.6 + 0.001 * i
up = (arm == "B") == b_wins
sc.update(tf_alpha8_k63=base + (0.2 if up else 0), tf_alpha8_k127=0.5 + 0.001 * i,
free_distinct_k127=base + (0.2 if up else 0))
(p / f"seed{s}.json").write_text(json.dumps({"seed": s, "cfg": cfg, "scores": sc}))
def score(d):
r = subprocess.run([sys.executable, str(HERE / "score3.py"), str(d)], capture_output=True, text=True, cwd=HERE)
return r.returncode, r.stdout
class Scorer(unittest.TestCase):
S = (11, 12, 13, 14, 15, 16)
def test_both_verdicts_reachable(self):
for b_wins, word in ((True, "HELD"), (False, "FAILED")):
with tempfile.TemporaryDirectory() as t:
d = Path(t); fake(d, "A", E.CFG2, self.S, b_wins); fake(d, "B", E.CFG3, self.S, b_wins)
rc, out = score(d)
self.assertEqual(rc, 0, out)
for line in out.splitlines():
if line.startswith(("P1", "P2")):
self.assertIn(word, line)
def test_fail_closed(self):
with tempfile.TemporaryDirectory() as t:
d = Path(t); fake(d, "A", E.CFG2, self.S, True)
self.assertEqual(score(d)[0], 2) # arm B missing
fake(d, "B", E.CFG2, self.S, True)
self.assertEqual(score(d)[0], 2) # arm B has arm A's config
with tempfile.TemporaryDirectory() as t:
d = Path(t); fake(d, "A", E.CFG2, self.S, True); fake(d, "B", E.CFG3, self.S[:5], True)
self.assertEqual(score(d)[0], 2) # a seed missing
if __name__ == "__main__":
unittest.main()