#!/usr/bin/env python3 """ PHASE 4 GATE: does self-play produce measurable learning? Measures a trained checkpoint against: 1. its OWN random initialisation (the decisive test: identical architecture and seed, only the weights differ) 2. a uniform-random legal agent 3. a one-ply material-greedy heuristic If (1) is not clearly above 0.5, the self-play loop does not learn and no architectural feature should be stacked on top of it. """ import argparse import json import os import sys import time sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from tinychess.arena import (MaterialAgent, RandomAgent, head_to_head, policy_diagnostics, wilson_interval) from tinychess.checkpoint import load_model_only def main(): ap = argparse.ArgumentParser() ap.add_argument("--checkpoint", required=True) ap.add_argument("--init", default=None, help="random-init checkpoint, same cfg") ap.add_argument("--games", type=int, default=60) ap.add_argument("--steps", type=int, default=4) ap.add_argument("--max-plies", type=int, default=160) ap.add_argument("--out", default=None) a = ap.parse_args() model, cfg = load_model_only(a.checkpoint) rep = model.param_report() res = {"checkpoint": a.checkpoint, "params": rep["total"], "active": rep["active"], "cfg": cfg.to_dict(), "steps": a.steps, "games_per_match": a.games} print(f"params={rep['total']:,} steps={a.steps}") opponents = {} if a.init and os.path.exists(a.init): opponents["own_random_init"] = load_model_only(a.init)[0] opponents["random_legal"] = RandomAgent() opponents["material_greedy"] = MaterialAgent() for name, opp in opponents.items(): t = time.time() r = head_to_head(model, opp, n_games=a.games, steps=a.steps, max_plies=a.max_plies, seed=1234) lo, hi = wilson_interval(r["wins"], r["draws"], r["games"]) r["ci95"] = [round(lo, 3), round(hi, 3)] r["seconds"] = round(time.time() - t, 1) res[f"vs_{name}"] = r print(f"vs {name:18s} score={r['score']:.3f} CI[{lo:.2f},{hi:.2f}] " f"W{r['wins']}/D{r['draws']}/L{r['losses']} elo{r['elo_diff']:+.0f} " f"illegal={r['illegal_attempts']} ({r['seconds']}s)") if a.out: os.makedirs(os.path.dirname(os.path.abspath(a.out)), exist_ok=True) json.dump(res, open(a.out, "w"), indent=2) print(f"[saved] {a.out}") return res if __name__ == "__main__": main()