| |
| """ |
| PHASE 4 GATE: does self-play produce measurable learning? |
| |
| Measures a trained checkpoint against: |
| 1. its OWN random initialisation (the decisive test: identical architecture |
| and seed, only the weights differ) |
| 2. a uniform-random legal agent |
| 3. a one-ply material-greedy heuristic |
| |
| If (1) is not clearly above 0.5, the self-play loop does not learn and no |
| architectural feature should be stacked on top of it. |
| """ |
| import argparse |
| import json |
| import os |
| import sys |
| import time |
|
|
| sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) |
|
|
| from tinychess.arena import (MaterialAgent, RandomAgent, head_to_head, |
| policy_diagnostics, wilson_interval) |
| from tinychess.checkpoint import load_model_only |
|
|
|
|
| def main(): |
| ap = argparse.ArgumentParser() |
| ap.add_argument("--checkpoint", required=True) |
| ap.add_argument("--init", default=None, help="random-init checkpoint, same cfg") |
| ap.add_argument("--games", type=int, default=60) |
| ap.add_argument("--steps", type=int, default=4) |
| ap.add_argument("--max-plies", type=int, default=160) |
| ap.add_argument("--out", default=None) |
| a = ap.parse_args() |
|
|
| model, cfg = load_model_only(a.checkpoint) |
| rep = model.param_report() |
| res = {"checkpoint": a.checkpoint, "params": rep["total"], |
| "active": rep["active"], "cfg": cfg.to_dict(), "steps": a.steps, |
| "games_per_match": a.games} |
| print(f"params={rep['total']:,} steps={a.steps}") |
|
|
| opponents = {} |
| if a.init and os.path.exists(a.init): |
| opponents["own_random_init"] = load_model_only(a.init)[0] |
| opponents["random_legal"] = RandomAgent() |
| opponents["material_greedy"] = MaterialAgent() |
|
|
| for name, opp in opponents.items(): |
| t = time.time() |
| r = head_to_head(model, opp, n_games=a.games, steps=a.steps, |
| max_plies=a.max_plies, seed=1234) |
| lo, hi = wilson_interval(r["wins"], r["draws"], r["games"]) |
| r["ci95"] = [round(lo, 3), round(hi, 3)] |
| r["seconds"] = round(time.time() - t, 1) |
| res[f"vs_{name}"] = r |
| print(f"vs {name:18s} score={r['score']:.3f} CI[{lo:.2f},{hi:.2f}] " |
| f"W{r['wins']}/D{r['draws']}/L{r['losses']} elo{r['elo_diff']:+.0f} " |
| f"illegal={r['illegal_attempts']} ({r['seconds']}s)") |
|
|
| if a.out: |
| os.makedirs(os.path.dirname(os.path.abspath(a.out)), exist_ok=True) |
| json.dump(res, open(a.out, "w"), indent=2) |
| print(f"[saved] {a.out}") |
| return res |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
