cazyundee commited on
Commit
9bf3d64
·
verified ·
1 Parent(s): fc125bf

fix: repetition feature lost in FEN round-trip (train/inference mismatch); greedy-health exp

Browse files
Files changed (1) hide show
  1. experiments/exp_greedy_health.py +143 -0
experiments/exp_greedy_health.py ADDED
@@ -0,0 +1,143 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Temperature-0 health check: the metric the training loop does NOT measure.
3
+
4
+ Self-play stats (`sp_mean_plies`, `sp_decisive_rate`) are collected at
5
+ temperature 1.2 with Dirichlet noise. Measured on 2026-09-22, EVERY checkpoint
6
+ in this project -- including the canonical strongest one -- repeats in 100% of
7
+ *greedy* games:
8
+
9
+ iter269 temp 0.0 -> 33 plies, repetition 30/30
10
+ v4_75 temp 0.0 -> 23 plies, repetition 30/30
11
+ (at temp 1.0 the same models reach 153 / 182 plies)
12
+
13
+ So the training metrics systematically hide the pathology: sampling noise walks
14
+ the policy out of loops that its argmax cannot escape. Any claim about
15
+ "repetition collapse being fixed" must be checked here, at temperature 0.
16
+
17
+ Reports, per checkpoint and per temperature:
18
+ mean_plies, repetition rate, checkmate rate, stalemate rate, ply-cap rate,
19
+ mean top-1 probability and mean policy entropy (commitment), and
20
+ `unique_position_ratio` = distinct positions / plies, which detects shuffling
21
+ even when a game does not formally repeat 3x.
22
+
23
+ Rules-engine terminations only; no chess heuristics. Read-only measurement.
24
+ """
25
+ from __future__ import annotations
26
+
27
+ import argparse
28
+ import json
29
+ import os
30
+ import random
31
+ import sys
32
+
33
+ import chess
34
+ import torch
35
+
36
+ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
37
+
38
+ from tinychess.batching import encode_batch # noqa: E402
39
+ from tinychess.checkpoint import load_model_only # noqa: E402
40
+
41
+
42
+ @torch.no_grad()
43
+ def _move(model, board, steps, temp, rng):
44
+ pb = encode_batch([board], [None])
45
+ lg = model(pb.squares, pb.extras, pb.glob, cand_fields=pb.cand_fields,
46
+ cand_mask=pb.cand_mask, steps=steps)["logits"][0]
47
+ mvs = pb.real_moves[0]
48
+ s = lg[:len(mvs)]
49
+ p = torch.softmax(s, -1)
50
+ top1 = float(p.max())
51
+ ent = float(-(p * p.clamp_min(1e-12).log()).sum())
52
+ if temp > 0:
53
+ q = torch.softmax(s / temp, -1)
54
+ g = torch.Generator().manual_seed(rng.randrange(1 << 30))
55
+ k = int(torch.multinomial(q, 1, generator=g))
56
+ else:
57
+ k = int(torch.argmax(s))
58
+ return mvs[k], top1, ent
59
+
60
+
61
+ def play(model, n_games, cap, temp, steps, seed, open_plies):
62
+ rng = random.Random(seed)
63
+ lens, tops, ents, uniq = [], [], [], []
64
+ terms = {}
65
+ for _ in range(n_games):
66
+ b = chess.Board()
67
+ for _ in range(rng.randint(0, open_plies)):
68
+ ms = list(b.legal_moves)
69
+ if not ms:
70
+ break
71
+ b.push(rng.choice(ms))
72
+ seen = set()
73
+ while b.ply() < cap and not b.is_game_over(claim_draw=True):
74
+ mv, t1, e = _move(model, b, steps, temp, rng)
75
+ tops.append(t1)
76
+ ents.append(e)
77
+ b.push(mv)
78
+ seen.add(b.board_fen())
79
+ lens.append(b.ply())
80
+ uniq.append(len(seen) / max(b.ply(), 1))
81
+ if b.is_checkmate():
82
+ k = "checkmate"
83
+ elif b.is_stalemate():
84
+ k = "stalemate"
85
+ elif b.is_insufficient_material():
86
+ k = "insufficient"
87
+ elif b.is_repetition(3) or b.can_claim_threefold_repetition():
88
+ k = "repetition"
89
+ elif b.ply() >= cap:
90
+ k = "ply_cap"
91
+ else:
92
+ k = "other_draw"
93
+ terms[k] = terms.get(k, 0) + 1
94
+ n = n_games
95
+ return {
96
+ "mean_plies": round(sum(lens) / n, 1),
97
+ "repetition_rate": round(terms.get("repetition", 0) / n, 3),
98
+ "checkmate_rate": round(terms.get("checkmate", 0) / n, 3),
99
+ "stalemate_rate": round(terms.get("stalemate", 0) / n, 3),
100
+ "ply_cap_rate": round(terms.get("ply_cap", 0) / n, 3),
101
+ "mean_top1_prob": round(sum(tops) / max(len(tops), 1), 3),
102
+ "mean_policy_entropy": round(sum(ents) / max(len(ents), 1), 3),
103
+ "unique_position_ratio": round(sum(uniq) / n, 3),
104
+ "terminations": terms,
105
+ }
106
+
107
+
108
+ def main():
109
+ ap = argparse.ArgumentParser()
110
+ ap.add_argument("--ckpts", required=True)
111
+ ap.add_argument("--names", default="")
112
+ ap.add_argument("--temps", default="0,0.5,1.0")
113
+ ap.add_argument("--games", type=int, default=30)
114
+ ap.add_argument("--max-plies", type=int, default=300)
115
+ ap.add_argument("--steps", type=int, default=4)
116
+ ap.add_argument("--open-plies", type=int, default=6)
117
+ ap.add_argument("--seed", type=int, default=0)
118
+ ap.add_argument("--out", default="runs/greedy_health.json")
119
+ a = ap.parse_args()
120
+
121
+ cks = a.ckpts.split(",")
122
+ names = a.names.split(",") if a.names else [
123
+ os.path.basename(os.path.dirname(c)) for c in cks]
124
+ res = {"args": vars(a), "models": {}}
125
+ for ck, nm in zip(cks, names):
126
+ model, _ = load_model_only(ck)
127
+ model.eval()
128
+ res["models"][nm] = {}
129
+ for temp in [float(x) for x in a.temps.split(",")]:
130
+ r = play(model, a.games, a.max_plies, temp, a.steps, a.seed,
131
+ a.open_plies)
132
+ res["models"][nm][str(temp)] = r
133
+ print(f"{nm:10s} temp={temp:<4} plies {r['mean_plies']:6.1f} "
134
+ f"rep {r['repetition_rate']:.2f} mate {r['checkmate_rate']:.2f} "
135
+ f"uniq {r['unique_position_ratio']:.2f} "
136
+ f"top1 {r['mean_top1_prob']:.3f}", flush=True)
137
+ os.makedirs(os.path.dirname(os.path.abspath(a.out)), exist_ok=True)
138
+ json.dump(res, open(a.out, "w"), indent=2)
139
+ print(f"[saved] {a.out}")
140
+
141
+
142
+ if __name__ == "__main__":
143
+ main()