tiny-engine-165k / guard.py
holland202's picture
v2: long-string copying fixed; works at 8x training length
360628f verified
Raw History Blame Contribute Delete
2.69 kB
"""Fail-closed loading of per-seed results for the registered scorers (score.py, score2.py).
Added 2026-11-17 after Codex (OpenAI) showed, in draft PR #163, that the scorers counted files, not distinct seeds:
five copies of one seed's result passed as five seeds. Reproduced by Claude against score2.py before this fix
(five copies of seed 1 -> every criterion HELD 5/5; two of six results -> an ordinary FAILED, not an error).
A result set is scored only if: the files are exactly seed1.json..seedN.json for the registered seeds; each file's
"seed" field matches its file name; no two files hold the same seed or identical scores; every file's config equals
the registered config; every registered score key is present and is a number in [0, 1]. Anything else exits 2
with the reason, and no verdict is printed.
Written by Claude (Anthropic, Opus 5.5). SELF-TESTED only."""
import json
import math
import sys
from pathlib import Path
REGISTERED_SEEDS = (1, 2, 3, 4, 5, 6)
class InvalidResults(Exception):
pass
def load(directory, cfg, keys, seeds=REGISTERED_SEEDS):
d = Path(directory)
files = sorted(d.glob("seed*.json"))
names = {p.name for p in files}
want = {f"seed{s}.json" for s in seeds}
if names != want:
raise InvalidResults(f"expected exactly {sorted(want)}; found {sorted(names)}")
out, seen_scores = [], set()
for s in seeds:
p = d / f"seed{s}.json"
try:
r = json.loads(p.read_text())
except (OSError, ValueError) as e:
raise InvalidResults(f"{p.name}: unreadable ({e})") from e
if r.get("seed") != s:
raise InvalidResults(f"{p.name}: seed field is {r.get('seed')!r}, expected {s}")
if json.loads(json.dumps(cfg)) != r.get("cfg"):
raise InvalidResults(f"{p.name}: config does not match the registered config")
sc = r.get("scores")
if not isinstance(sc, dict):
raise InvalidResults(f"{p.name}: no scores")
for k in keys:
v = sc.get(k)
if not isinstance(v, (int, float)) or isinstance(v, bool) or not math.isfinite(v) or not 0 <= v <= 1:
raise InvalidResults(f"{p.name}: score {k!r} missing or not a number in [0, 1] ({v!r})")
fp = json.dumps(sc, sort_keys=True)
if fp in seen_scores:
raise InvalidResults(f"{p.name}: scores identical to another seed's (duplicated result?)")
seen_scores.add(fp)
out.append(r)
return out
def load_or_exit(directory, cfg, keys):
try:
return load(directory, cfg, keys)
except InvalidResults as e:
print(f"INVALID RESULT SET, no verdict: {e}")
sys.exit(2)