File size: 2,580 Bytes
3495881 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 | #!/usr/bin/env python3
"""
PHASE 4 GATE: does self-play produce measurable learning?
Measures a trained checkpoint against:
1. its OWN random initialisation (the decisive test: identical architecture
and seed, only the weights differ)
2. a uniform-random legal agent
3. a one-ply material-greedy heuristic
If (1) is not clearly above 0.5, the self-play loop does not learn and no
architectural feature should be stacked on top of it.
"""
import argparse
import json
import os
import sys
import time
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from tinychess.arena import (MaterialAgent, RandomAgent, head_to_head,
policy_diagnostics, wilson_interval)
from tinychess.checkpoint import load_model_only
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--checkpoint", required=True)
ap.add_argument("--init", default=None, help="random-init checkpoint, same cfg")
ap.add_argument("--games", type=int, default=60)
ap.add_argument("--steps", type=int, default=4)
ap.add_argument("--max-plies", type=int, default=160)
ap.add_argument("--out", default=None)
a = ap.parse_args()
model, cfg = load_model_only(a.checkpoint)
rep = model.param_report()
res = {"checkpoint": a.checkpoint, "params": rep["total"],
"active": rep["active"], "cfg": cfg.to_dict(), "steps": a.steps,
"games_per_match": a.games}
print(f"params={rep['total']:,} steps={a.steps}")
opponents = {}
if a.init and os.path.exists(a.init):
opponents["own_random_init"] = load_model_only(a.init)[0]
opponents["random_legal"] = RandomAgent()
opponents["material_greedy"] = MaterialAgent()
for name, opp in opponents.items():
t = time.time()
r = head_to_head(model, opp, n_games=a.games, steps=a.steps,
max_plies=a.max_plies, seed=1234)
lo, hi = wilson_interval(r["wins"], r["draws"], r["games"])
r["ci95"] = [round(lo, 3), round(hi, 3)]
r["seconds"] = round(time.time() - t, 1)
res[f"vs_{name}"] = r
print(f"vs {name:18s} score={r['score']:.3f} CI[{lo:.2f},{hi:.2f}] "
f"W{r['wins']}/D{r['draws']}/L{r['losses']} elo{r['elo_diff']:+.0f} "
f"illegal={r['illegal_attempts']} ({r['seconds']}s)")
if a.out:
os.makedirs(os.path.dirname(os.path.abspath(a.out)), exist_ok=True)
json.dump(res, open(a.out, "w"), indent=2)
print(f"[saved] {a.out}")
return res
if __name__ == "__main__":
main()
|