fix: repetition feature lost in FEN round-trip (train/inference mismatch); greedy-health exp
Browse files- experiments/exp_greedy_health.py +143 -0
experiments/exp_greedy_health.py
ADDED
|
@@ -0,0 +1,143 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Temperature-0 health check: the metric the training loop does NOT measure.
|
| 3 |
+
|
| 4 |
+
Self-play stats (`sp_mean_plies`, `sp_decisive_rate`) are collected at
|
| 5 |
+
temperature 1.2 with Dirichlet noise. Measured on 2026-09-22, EVERY checkpoint
|
| 6 |
+
in this project -- including the canonical strongest one -- repeats in 100% of
|
| 7 |
+
*greedy* games:
|
| 8 |
+
|
| 9 |
+
iter269 temp 0.0 -> 33 plies, repetition 30/30
|
| 10 |
+
v4_75 temp 0.0 -> 23 plies, repetition 30/30
|
| 11 |
+
(at temp 1.0 the same models reach 153 / 182 plies)
|
| 12 |
+
|
| 13 |
+
So the training metrics systematically hide the pathology: sampling noise walks
|
| 14 |
+
the policy out of loops that its argmax cannot escape. Any claim about
|
| 15 |
+
"repetition collapse being fixed" must be checked here, at temperature 0.
|
| 16 |
+
|
| 17 |
+
Reports, per checkpoint and per temperature:
|
| 18 |
+
mean_plies, repetition rate, checkmate rate, stalemate rate, ply-cap rate,
|
| 19 |
+
mean top-1 probability and mean policy entropy (commitment), and
|
| 20 |
+
`unique_position_ratio` = distinct positions / plies, which detects shuffling
|
| 21 |
+
even when a game does not formally repeat 3x.
|
| 22 |
+
|
| 23 |
+
Rules-engine terminations only; no chess heuristics. Read-only measurement.
|
| 24 |
+
"""
|
| 25 |
+
from __future__ import annotations
|
| 26 |
+
|
| 27 |
+
import argparse
|
| 28 |
+
import json
|
| 29 |
+
import os
|
| 30 |
+
import random
|
| 31 |
+
import sys
|
| 32 |
+
|
| 33 |
+
import chess
|
| 34 |
+
import torch
|
| 35 |
+
|
| 36 |
+
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
| 37 |
+
|
| 38 |
+
from tinychess.batching import encode_batch # noqa: E402
|
| 39 |
+
from tinychess.checkpoint import load_model_only # noqa: E402
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
@torch.no_grad()
|
| 43 |
+
def _move(model, board, steps, temp, rng):
|
| 44 |
+
pb = encode_batch([board], [None])
|
| 45 |
+
lg = model(pb.squares, pb.extras, pb.glob, cand_fields=pb.cand_fields,
|
| 46 |
+
cand_mask=pb.cand_mask, steps=steps)["logits"][0]
|
| 47 |
+
mvs = pb.real_moves[0]
|
| 48 |
+
s = lg[:len(mvs)]
|
| 49 |
+
p = torch.softmax(s, -1)
|
| 50 |
+
top1 = float(p.max())
|
| 51 |
+
ent = float(-(p * p.clamp_min(1e-12).log()).sum())
|
| 52 |
+
if temp > 0:
|
| 53 |
+
q = torch.softmax(s / temp, -1)
|
| 54 |
+
g = torch.Generator().manual_seed(rng.randrange(1 << 30))
|
| 55 |
+
k = int(torch.multinomial(q, 1, generator=g))
|
| 56 |
+
else:
|
| 57 |
+
k = int(torch.argmax(s))
|
| 58 |
+
return mvs[k], top1, ent
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def play(model, n_games, cap, temp, steps, seed, open_plies):
|
| 62 |
+
rng = random.Random(seed)
|
| 63 |
+
lens, tops, ents, uniq = [], [], [], []
|
| 64 |
+
terms = {}
|
| 65 |
+
for _ in range(n_games):
|
| 66 |
+
b = chess.Board()
|
| 67 |
+
for _ in range(rng.randint(0, open_plies)):
|
| 68 |
+
ms = list(b.legal_moves)
|
| 69 |
+
if not ms:
|
| 70 |
+
break
|
| 71 |
+
b.push(rng.choice(ms))
|
| 72 |
+
seen = set()
|
| 73 |
+
while b.ply() < cap and not b.is_game_over(claim_draw=True):
|
| 74 |
+
mv, t1, e = _move(model, b, steps, temp, rng)
|
| 75 |
+
tops.append(t1)
|
| 76 |
+
ents.append(e)
|
| 77 |
+
b.push(mv)
|
| 78 |
+
seen.add(b.board_fen())
|
| 79 |
+
lens.append(b.ply())
|
| 80 |
+
uniq.append(len(seen) / max(b.ply(), 1))
|
| 81 |
+
if b.is_checkmate():
|
| 82 |
+
k = "checkmate"
|
| 83 |
+
elif b.is_stalemate():
|
| 84 |
+
k = "stalemate"
|
| 85 |
+
elif b.is_insufficient_material():
|
| 86 |
+
k = "insufficient"
|
| 87 |
+
elif b.is_repetition(3) or b.can_claim_threefold_repetition():
|
| 88 |
+
k = "repetition"
|
| 89 |
+
elif b.ply() >= cap:
|
| 90 |
+
k = "ply_cap"
|
| 91 |
+
else:
|
| 92 |
+
k = "other_draw"
|
| 93 |
+
terms[k] = terms.get(k, 0) + 1
|
| 94 |
+
n = n_games
|
| 95 |
+
return {
|
| 96 |
+
"mean_plies": round(sum(lens) / n, 1),
|
| 97 |
+
"repetition_rate": round(terms.get("repetition", 0) / n, 3),
|
| 98 |
+
"checkmate_rate": round(terms.get("checkmate", 0) / n, 3),
|
| 99 |
+
"stalemate_rate": round(terms.get("stalemate", 0) / n, 3),
|
| 100 |
+
"ply_cap_rate": round(terms.get("ply_cap", 0) / n, 3),
|
| 101 |
+
"mean_top1_prob": round(sum(tops) / max(len(tops), 1), 3),
|
| 102 |
+
"mean_policy_entropy": round(sum(ents) / max(len(ents), 1), 3),
|
| 103 |
+
"unique_position_ratio": round(sum(uniq) / n, 3),
|
| 104 |
+
"terminations": terms,
|
| 105 |
+
}
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def main():
|
| 109 |
+
ap = argparse.ArgumentParser()
|
| 110 |
+
ap.add_argument("--ckpts", required=True)
|
| 111 |
+
ap.add_argument("--names", default="")
|
| 112 |
+
ap.add_argument("--temps", default="0,0.5,1.0")
|
| 113 |
+
ap.add_argument("--games", type=int, default=30)
|
| 114 |
+
ap.add_argument("--max-plies", type=int, default=300)
|
| 115 |
+
ap.add_argument("--steps", type=int, default=4)
|
| 116 |
+
ap.add_argument("--open-plies", type=int, default=6)
|
| 117 |
+
ap.add_argument("--seed", type=int, default=0)
|
| 118 |
+
ap.add_argument("--out", default="runs/greedy_health.json")
|
| 119 |
+
a = ap.parse_args()
|
| 120 |
+
|
| 121 |
+
cks = a.ckpts.split(",")
|
| 122 |
+
names = a.names.split(",") if a.names else [
|
| 123 |
+
os.path.basename(os.path.dirname(c)) for c in cks]
|
| 124 |
+
res = {"args": vars(a), "models": {}}
|
| 125 |
+
for ck, nm in zip(cks, names):
|
| 126 |
+
model, _ = load_model_only(ck)
|
| 127 |
+
model.eval()
|
| 128 |
+
res["models"][nm] = {}
|
| 129 |
+
for temp in [float(x) for x in a.temps.split(",")]:
|
| 130 |
+
r = play(model, a.games, a.max_plies, temp, a.steps, a.seed,
|
| 131 |
+
a.open_plies)
|
| 132 |
+
res["models"][nm][str(temp)] = r
|
| 133 |
+
print(f"{nm:10s} temp={temp:<4} plies {r['mean_plies']:6.1f} "
|
| 134 |
+
f"rep {r['repetition_rate']:.2f} mate {r['checkmate_rate']:.2f} "
|
| 135 |
+
f"uniq {r['unique_position_ratio']:.2f} "
|
| 136 |
+
f"top1 {r['mean_top1_prob']:.3f}", flush=True)
|
| 137 |
+
os.makedirs(os.path.dirname(os.path.abspath(a.out)), exist_ok=True)
|
| 138 |
+
json.dump(res, open(a.out, "w"), indent=2)
|
| 139 |
+
print(f"[saved] {a.out}")
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
if __name__ == "__main__":
|
| 143 |
+
main()
|