File size: 1,969 Bytes
60a72cf | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 | """TINY-ENGINE-003 evaluation (registered in REGISTRATION_003.md). Fixed case seeds in the 8000 range, never used before
this registration. Claude (Anthropic, Opus 5.5), 2026-11-20. SELF-TESTED only."""
import numpy as np
import torch
import engine as E
FAMS = {"distinct": None, "alpha32": 32, "alpha8": 8}
KS = (63, 127) # 63 = longest trained copy, 127 = 2x
N_TF, N_FREE = 256, 100
def case(rng, k, fam):
T = 2 * k + 2
return E.task_copy(rng, T) if FAMS[fam] is None else E.task_copy_rep(rng, T, FAMS[fam])
def tf_score(pred, x, mask):
"""Share of masked next tokens predicted correctly. pred = argmax over positions 0..T-2."""
return float((pred == x[:, 1:])[mask[:, :-1]].float().mean())
@torch.no_grad()
def tf_acc(m, fam, k, seed, n=N_TF):
rng = np.random.default_rng(seed)
xs, ms = zip(*[case(rng, k, fam) for _ in range(n)])
x, mask = torch.tensor(np.array(xs)), torch.tensor(np.array(ms))
return tf_score(m(x).argmax(-1)[:, :-1], x, mask)
@torch.no_grad()
def free_whole(m, fam, k, seed, n=N_FREE):
"""Share of answers exactly right when the model writes all k copy tokens itself (greedy). The prefix is
BOS + source + SEP only; the target is never fed in."""
rng = np.random.default_rng(seed)
xs = [case(rng, k, fam)[0] for _ in range(n)]
src = torch.tensor(np.array([x[1:k + 1] for x in xs]))
y = torch.tensor(np.array([x[:k + 2] for x in xs]))
for _ in range(k):
y = torch.cat([y, m(y)[:, -1:].argmax(-1)], 1)
return float((y[:, -k:] == src).all(1).float().mean())
def evaluate3(m):
m.eval()
r = {}
for i, fam in enumerate(FAMS):
for k in KS:
r[f"tf_{fam}_k{k}"] = tf_acc(m, fam, k, 8000 + 10 * i + k)
r["free_distinct_k127"] = free_whole(m, "distinct", 127, 8501)
r["free_alpha32_k63"] = free_whole(m, "alpha32", 63, 8502)
r["free_alpha8_k63"] = free_whole(m, "alpha8", 63, 8503)
m.train()
return r
|