robbiethompson2018 commited on
Commit
2fef3a8
·
verified ·
1 Parent(s): f5884f8

Add 707-fable_plain-state-feedback_s1

Browse files
runs/707-fable_plain-state-feedback_s1/eval_dev.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
runs/707-fable_plain-state-feedback_s1/final.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:263422c007e14b53b5e84122214650ae8b168883bff017896f8dab60ca4e8c7e
3
+ size 38293470
runs/707-fable_plain-state-feedback_s1/found_targets.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
runs/707-fable_plain-state-feedback_s1/lean_gate.disagree.jsonl ADDED
@@ -0,0 +1 @@
 
 
1
+ {"prompt": "THM S SEQ ( ~ ( ~ ( ( S > P ) > ( ( ~ ( R > P ) ) > ( S v R ) ) ) ) ) PRF", "lean_text": "have n36 : S := h1 ; have n37 : ( ¬ ( ¬ ( ( S → P ) → ( ( ¬ ( R → P ) ) → ( S ∨ R ) ) ) ) ) := ( fun ( n38 : ( ¬ ( ( S → P ) → ( ( ¬ ( R → P ) ) → ( S ∨ R ) ) ) ) ) => by have n39 : ( ( S → P ) → ( ( ¬ ( R → P ) ) → ( S ∨ R ) ) ) := ( fun ( n40 : ( S → P ) ) => by have n41 : ( ( ¬ ( R → P ) ) → ( S ∨ R ) ) := ( fun ( n42 : ( ¬ ( R → P ) ) ) => by have n43 : ( S ∨ R ) := Or.inl n36 ; exact n43 ) ; exact n41 ) ; have n44 : False := n38 n39 ; exact n44 ) ; exact n37", "nd": "N1 S : PR ; N2 | ( ~ ( ( S > P ) > ( ( ~ ( R > P ) ) > ( S v R ) ) ) ) : AS ; N3 | | ( S > P ) : AS ; N4 | | | ( ~ ( R > P ) ) : AS ; N5 | | | ( S v R ) : ORI1 N1 ; N6 | | ( ( ~ ( R > P ) ) > ( S v R ) ) : IMPI N4 N5 ; N7 | ( ( S > P ) > ( ( ~ ( R > P ) ) > ( S v R ) ) ) : IMPI N3 N6 ; N8 | F : NEGE N7 N2 ; N9 ( ~ ( ~ ( ( S > P ) > ( ( ~ ( R > P ) ) > ( S v R ) ) ) ) ) : IMPI N2 N8 ; QED", "nd_ok": false, "lean_ok": true}
runs/707-fable_plain-state-feedback_s1/lean_gate.jsonl ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {"utc": "2026-10-08T00:51:07Z", "samples": 2000, "parse_fail": 22, "distinct_checked": 1978, "both_ok": 1916, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 62, "parse_reasons": {"formula": 17, "unbound": 3, "expected )": 2}, "lean_wall_s": 3.0279541015625, "lean_proc_s": 14.234122276306152, "nd_verify_s": 0.06845808029174805, "workers": 96, "chunk": 400}
2
+ {"utc": "2026-10-08T00:57:50Z", "samples": 48000, "parse_fail": 21033, "distinct_checked": 24516, "both_ok": 2460, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 22056, "parse_reasons": {"unbound": 11101, "formula": 5288, "expected )": 3607, "name": 352, "no-eos": 191, "expected ;": 145, "exact not last": 110, "expected (": 56, "ORE box": 51, "PR position": 44, "expected :=": 35, "expected =>": 31, "trailing tokens": 15, "eof": 3, "expected exact": 2, "expected False": 1, "expected \u27e9": 1}, "lean_wall_s": 56.102779388427734, "lean_proc_s": 3341.265270471573, "nd_verify_s": 1.074197769165039, "workers": 96, "chunk": 400}
3
+ {"utc": "2026-10-08T01:05:32Z", "samples": 48000, "parse_fail": 7353, "distinct_checked": 34652, "both_ok": 17808, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 16844, "parse_reasons": {"unbound": 4431, "expected )": 1458, "formula": 678, "expected (": 274, "exact not last": 107, "ORE box": 89, "no-eos": 85, "name": 62, "PR position": 51, "expected :=": 48, "trailing tokens": 37, "expected ;": 13, "expected =>": 11, "expected exact": 4, "expected .elim": 3, "expected False": 1, "expected fun": 1}, "lean_wall_s": 78.34923481941223, "lean_proc_s": 6475.502856731415, "nd_verify_s": 1.7144474983215332, "workers": 96, "chunk": 400}
4
+ {"utc": "2026-10-08T01:13:44Z", "samples": 48000, "parse_fail": 4040, "distinct_checked": 37199, "both_ok": 24131, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 13068, "parse_reasons": {"unbound": 2548, "expected )": 543, "formula": 411, "expected (": 241, "name": 111, "exact not last": 52, "ORE box": 44, "PR position": 32, "no-eos": 23, "expected :=": 19, "trailing tokens": 6, "expected ;": 5, "expected exact": 3, "expected =>": 1, "expected fun": 1}, "lean_wall_s": 85.71210074424744, "lean_proc_s": 7700.399682998657, "nd_verify_s": 1.979217767715454, "workers": 96, "chunk": 400}
5
+ {"utc": "2026-10-08T01:21:38Z", "samples": 48000, "parse_fail": 3522, "distinct_checked": 37692, "both_ok": 26944, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 10748, "parse_reasons": {"unbound": 2314, "expected (": 333, "formula": 314, "expected )": 256, "name": 62, "ORE box": 55, "expected :=": 50, "exact not last": 46, "no-eos": 44, "PR position": 34, "expected exact": 8, "expected =>": 3, "trailing tokens": 2, "expected ;": 1}, "lean_wall_s": 86.1987612247467, "lean_proc_s": 7838.091330766678, "nd_verify_s": 2.0754356384277344, "workers": 96, "chunk": 400}
6
+ {"utc": "2026-10-08T01:33:22Z", "samples": 70912, "parse_fail": 9971, "distinct_checked": 47764, "both_ok": 20931, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 26833, "parse_reasons": {"unbound": 7031, "formula": 813, "expected )": 770, "expected (": 416, "ORE box": 403, "name": 237, "no-eos": 95, "exact not last": 78, "PR position": 52, "expected :=": 49, "trailing tokens": 13, "expected .elim": 4, "expected exact": 4, "expected None": 2, "expected ;": 2, "expected hh": 1, "expected =>": 1}, "lean_wall_s": 127.54396843910217, "lean_proc_s": 10422.36051440239, "nd_verify_s": 2.6638307571411133, "workers": 96, "chunk": 400}
7
+ {"utc": "2026-10-08T01:33:36Z", "samples": 2000, "parse_fail": 6, "distinct_checked": 1994, "both_ok": 1973, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 21, "parse_reasons": {"unbound": 4, "formula": 1, "name": 1}, "lean_wall_s": 3.371182918548584, "lean_proc_s": 15.867420196533203, "nd_verify_s": 0.10111165046691895, "workers": 96, "chunk": 400}
8
+ {"utc": "2026-10-08T01:40:16Z", "samples": 70912, "parse_fail": 13061, "distinct_checked": 45384, "both_ok": 18533, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 1, "both_rej": 26850, "parse_reasons": {"unbound": 9691, "expected )": 886, "formula": 515, "exact not last": 400, "expected :=": 380, "expected (": 302, "name": 298, "no-eos": 181, "ORE box": 176, "trailing tokens": 82, "PR position": 69, "expected exact": 52, "expected ;": 14, "expected False": 6, "expected .elim": 4, "expected =>": 3, "expected hh": 1, "eof": 1}, "lean_wall_s": 173.86037397384644, "lean_proc_s": 15009.098974943161, "nd_verify_s": 2.4347610473632812, "workers": 96, "chunk": 400}
runs/707-fable_plain-state-feedback_s1/pretrain.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:82ea5bb3df0ee9d0f25c6a9427e27c0978b5505a2e8c447ff7eb01501ccd96dd
3
+ size 38293797
runs/707-fable_plain-state-feedback_s1/source/harness.py ADDED
@@ -0,0 +1,529 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ # imports the fork (model, tokenizer, train, ...), which is not resolvable here
3
+ # pyright: reportMissingImports=false
4
+ """FROZEN harness for the test-time-loops autoresearch loop. The research agent does not edit it.
5
+
6
+ Forked from ../autoresearch/harness.py (sha 2aa52f23e7ec) on 2026-10-07. What changed: the RL
7
+ stage belongs to the agent, behind a wall-clock budget and a sampling oracle, and the metric is
8
+ measured with the model's test-time compute knob (KNOB) set to EVAL_LEVEL.
9
+
10
+ One timed run = harness owns the clocks, the oracle and the eval; `pretrain.py` (agent-editable)
11
+ owns pretraining and RL:
12
+
13
+ 1. load frozen data (Dan's cap-6 Stage-1 file, ladder RL-target pool, ladder transfer pool)
14
+ 2. call pretrain.pretrain(train_recs, PRETRAIN_SECS, ...) -> (model, tok, stats); charged for
15
+ everything including model construction and torch.compile. Overrun past the grace -> timeout.
16
+ 3. check n_params <= PARAM_CAP, tok.mode == TOK_MODE and that the model has the KNOB attribute
17
+ 4. call pretrain.rl(model, tok, targets, train_recs, oracle, RL_SECS, ...): expert iteration in
18
+ whatever form the agent likes. The targets are statements only, and `oracle.sample` is the
19
+ only way to turn them into training data: it samples the model's own attempts and returns
20
+ the ones Lean AND nd_verify accept. RL_SECS is wall-clock inside rl() minus the time the
21
+ oracle spent in the Lean gate (pod CPUs differ; GPU work is what is rationed). Overrun past
22
+ the grace, or an oracle call after the budget is spent -> timeout.
23
+ 5. eval on the dev half of the transfer pool with the harness setting the model's KNOB:
24
+ METRIC = dev theorems solved at L_true >= 7 with EVAL_K attempts at EVAL_LEVEL.
25
+ The same eval at DIAG_LEVEL is recorded as a diagnostic, not scored.
26
+ 6. SIGALRM at TOTAL_SECS from process start kills everything -> status "timeout"
27
+
28
+ The holdout half of the transfer pool is never scored here. Everything is written to --out.
29
+
30
+ python3 harness.py --out artifacts/x # on the pod (see pod.py / remote.sh)
31
+ ND_FORK=... python3 harness.py --out /tmp/smoke --smoke # CPU, ~1 min, plumbing check
32
+ """
33
+
34
+ import argparse
35
+ import collections
36
+ import hashlib
37
+ import importlib
38
+ import json
39
+ import os
40
+ import random
41
+ import shutil
42
+ import signal
43
+ import sys
44
+ import time
45
+ import traceback
46
+ from pathlib import Path
47
+
48
+ HERE = os.path.dirname(os.path.abspath(__file__))
49
+ FORK = os.environ.get("ND_FORK", "/workspace/nd-takehome")
50
+ sys.path.insert(0, HERE)
51
+ sys.path.insert(
52
+ 0, FORK
53
+ ) # fork first: its train.py provides batch/loss_on; the agent's file is pretrain.py
54
+ import torch # noqa: E402
55
+ from eval_set import judge, wilson # noqa: E402
56
+ from sample import generate # noqa: E402
57
+
58
+ # ---------------------------------------------------------------- the rules
59
+ # (hash-checked into every result)
60
+ TOTAL_SECS = int(
61
+ os.environ.get("ND_TOTAL_SECS", 5400)
62
+ ) # process start -> summary written; SIGALRM past this. env: benchmarks only
63
+ PRETRAIN_SECS = 300 # agent's pretraining budget, compile included
64
+ PRETRAIN_GRACE = 15 # seconds the agent may overrun before the run is a "timeout"
65
+ RL_SECS = 1800 # agent's RL budget: wall-clock inside rl() minus the oracle's Lean-gate time
66
+ RL_GRACE = 60
67
+ PARAM_CAP = 10_000_000 # unique trainable params, embeddings included; checked after RL too
68
+ N_TARGETS, TEMP, MAX_NEW, SAMPLE_BATCH = 1500, 0.8, 512, 1024
69
+ EVAL_K = 64
70
+ # The model's test-time compute knob: an integer attribute the harness sets before each eval and
71
+ # rl() sets before oracle calls. A sibling loop for another architecture changes these three
72
+ # constants and nothing else in this file.
73
+ KNOB = "n_thoughts" # continuous thoughts per proof line
74
+ EVAL_LEVEL = 8 # the metric: the knob's value during the scored eval
75
+ DIAG_LEVEL = 0 # the same eval again at this value, recorded but not scored
76
+ TOK_MODE = "lean_seq" # required tokenizer mode (fork lean_tok.py)
77
+ DEV = "cuda" if torch.cuda.is_available() else "cpu"
78
+
79
+
80
+ def read(fn):
81
+ return [json.loads(line) for line in open(fn) if line.strip()]
82
+
83
+
84
+ def log(msg):
85
+ print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True)
86
+
87
+
88
+ def sha(fn):
89
+ return hashlib.sha256(open(fn, "rb").read()).hexdigest()[:12]
90
+
91
+
92
+ class Timeout(Exception):
93
+ pass
94
+
95
+
96
+ class BudgetSpent(Timeout):
97
+ """rl() asked the oracle for samples after RL_SECS was used up."""
98
+
99
+
100
+ # ---------------------------------------------------------------- bookkeeping
101
+
102
+
103
+ def save_checkpoint(a, model, tok, stage):
104
+ """Publish complete weight snapshots atomically; these are not optimizer-resume files."""
105
+ path = Path(a.out) / f"{stage}.pt"
106
+ pending = path.with_suffix(".pt.tmp")
107
+ try:
108
+ torch.save(
109
+ {
110
+ "state": model.state_dict(),
111
+ "tok_mode": getattr(tok, "mode", None),
112
+ "stage": stage,
113
+ "seed": a.seed,
114
+ "harness_sha": sha(f"{HERE}/harness.py"),
115
+ "pretrain_sha": sha(f"{HERE}/pretrain.py"),
116
+ },
117
+ pending,
118
+ )
119
+ os.replace(pending, path)
120
+ finally:
121
+ pending.unlink(missing_ok=True)
122
+
123
+
124
+ def lstar(solved_names, recs, need=5):
125
+ ls_solved = [r["n_lines"] for r in recs if r["name"] in solved_names]
126
+ ge = {L: sum(1 for x in ls_solved if x >= L) for L in range(7, 15)}
127
+ return max([L for L, c in ge.items() if c >= need], default=0), ge
128
+
129
+
130
+ def by_bin(solved_names, recs):
131
+ by = collections.defaultdict(lambda: [0, 0])
132
+ for r in recs:
133
+ by[r["n_lines"]][0] += r["name"] in solved_names
134
+ by[r["n_lines"]][1] += 1
135
+ return {
136
+ str(L): {"solved": k, "n": n, "rate": k / n, "ci": list(wilson(k, n))}
137
+ for L, (k, n) in sorted(by.items())
138
+ }
139
+
140
+
141
+ def gate_seconds():
142
+ """Seconds the fork's lean_gate has logged so far (Lean wall-clock plus nd_verify)."""
143
+ fn = os.environ.get("LEAN_GATE_LOG")
144
+ if not fn or not os.path.exists(fn):
145
+ return 0.0
146
+ return sum(rec["lean_wall_s"] + rec["nd_verify_s"] for rec in read(fn))
147
+
148
+
149
+ def sample_and_judge(model, tok, recs, k, temperature, seed, batch, max_new):
150
+ prompts = [r["prompt"] for r in recs for _ in range(k)]
151
+ t0, g0 = time.time(), gate_seconds()
152
+ flat = generate(
153
+ model,
154
+ tok,
155
+ prompts,
156
+ greedy=(k == 1 and temperature == 0),
157
+ temperature=temperature,
158
+ batch=batch,
159
+ seed=seed,
160
+ max_new=max_new,
161
+ )
162
+ secs, gate = time.time() - t0, gate_seconds() - g0
163
+ rows = judge(recs, [flat[i * k : (i + 1) * k] for i in range(len(recs))], "n_lines")
164
+ return rows, {
165
+ "samples": len(prompts),
166
+ "secs": secs,
167
+ "gate_secs": gate,
168
+ "samples_per_sec": len(prompts) / secs,
169
+ }
170
+
171
+
172
+ def greedy_rate(model, tok, recs):
173
+ rows, _ = sample_and_judge(model, tok, recs, 1, 0, 0, SAMPLE_BATCH, MAX_NEW)
174
+ return sum(r["solved"] for r in rows) / max(len(rows), 1)
175
+
176
+
177
+ def dev_split(transfer):
178
+ return [r for r in transfer if int(hashlib.sha1(r["key"].encode()).hexdigest(), 16) % 2 == 0]
179
+
180
+
181
+ def holdout_split(transfer):
182
+ """Never scored by the loop. Score final.pt on this offline when checking in."""
183
+ return [r for r in transfer if int(hashlib.sha1(r["key"].encode()).hexdigest(), 16) % 2 == 1]
184
+
185
+
186
+ # ---------------------------------------------------------------- the RL oracle
187
+
188
+
189
+ class Oracle:
190
+ """What rl() gets instead of data: the model's own attempts at the RL targets, filtered to
191
+ the ones Lean AND nd_verify accept. It also keeps the RL clock and the record of every call.
192
+ """
193
+
194
+ def __init__(self, a, tok, targets):
195
+ self.a, self.tok = a, tok
196
+ self.targets = {t["name"]: t for t in targets}
197
+ self.found = collections.defaultdict(dict) # name -> {proof text: record}
198
+ self.calls = []
199
+ self.t0 = self.last = time.time()
200
+ self.uncharged = 0.0
201
+
202
+ def used(self):
203
+ """RL seconds charged so far: wall-clock since rl() started, minus Lean-gate time."""
204
+ return time.time() - self.t0 - self.uncharged
205
+
206
+ def left(self):
207
+ return self.a.rl_secs - self.used()
208
+
209
+ def sample(self, model, targets, k, temperature=TEMP, batch=SAMPLE_BATCH):
210
+ """k attempts at each of `targets` (records rl() was given, any subset, in any order) from
211
+ `model` as it is now, in eval mode, at whatever its KNOB is set to. Returns one
212
+ dict per target, in order: name, prompt, n_lines, proofs (the distinct accepted proof
213
+ texts), n_ok (accepted attempts) and n_tried. Rejected attempts are not returned.
214
+ Raises BudgetSpent, which ends the run as a timeout, once the budget is used up."""
215
+ if self.left() <= 0:
216
+ raise BudgetSpent(f"oracle called with {self.used():.0f}s of {self.a.rl_secs}s used")
217
+ recs = [self.targets[t["name"]] for t in targets]
218
+ call = len(self.calls) + 1
219
+ model.eval()
220
+ rows, samp = sample_and_judge(
221
+ model, self.tok, recs, k, temperature, self.a.seed * 1000 + call, batch, self.a.max_new
222
+ )
223
+ self.uncharged += samp["gate_secs"]
224
+ new = 0
225
+ for t, row in zip(recs, rows, strict=True):
226
+ for p, wl, pl in zip(
227
+ row["proofs"], row["written_lens"], row["pruned_lens"], strict=True
228
+ ):
229
+ if p not in self.found[t["name"]]:
230
+ self.found[t["name"]][p] = {
231
+ "proof": p,
232
+ "written": wl,
233
+ "pruned": pl,
234
+ "round": call,
235
+ }
236
+ new += 1
237
+ solved = {n for n, d in self.found.items() if d}
238
+ ls, ge = lstar(solved, list(self.targets.values()))
239
+ now = time.time()
240
+ self.calls.append(
241
+ {
242
+ "round": call,
243
+ "level": getattr(model, KNOB, None),
244
+ "k": k,
245
+ "n_targets": len(recs),
246
+ "sample_acc": sum(x["n_ok"] for x in rows) / max(samp["samples"], 1),
247
+ "new_proofs": new,
248
+ "targets_solved": len(solved),
249
+ "lstar": ls,
250
+ "ge": ge,
251
+ "mix_rl_records": sum(len(x["proofs"]) for x in rows), # proofs handed back
252
+ "sampling": samp,
253
+ "secs": now - self.last, # since the previous call returned: includes rl()'s work
254
+ "rl_secs_used": self.used(),
255
+ }
256
+ )
257
+ self.last = now
258
+ c = self.calls[-1]
259
+ log(
260
+ f"ei r{call}: acc {c['sample_acc']:.4f}, +{new} proofs, "
261
+ f"solved {len(solved)}/{len(self.targets)}, L*={ls}, {KNOB} {c['level']}, "
262
+ f"{len(recs)} targets x {k}, {samp['samples_per_sec']:.0f} samples/s, "
263
+ f"RL clock {c['rl_secs_used']:.0f}/{self.a.rl_secs}s"
264
+ )
265
+ return [
266
+ {
267
+ "name": t["name"],
268
+ "prompt": t["prompt"],
269
+ "n_lines": t["n_lines"],
270
+ "proofs": list(row["proofs"]),
271
+ "n_ok": row["n_ok"],
272
+ "n_tried": row["n_tried"],
273
+ }
274
+ for t, row in zip(recs, rows, strict=True)
275
+ ]
276
+
277
+
278
+ # ---------------------------------------------------------------- stages
279
+
280
+
281
+ def stage_rl(a, agent, tok, model, targets, train_recs):
282
+ oracle = Oracle(a, tok, targets)
283
+ # statements only: what the agent's RL may know about a target
284
+ visible = [{"name": t["name"], "prompt": t["prompt"], "n_lines": t["n_lines"]} for t in targets]
285
+ stats = agent.rl(model, tok, visible, train_recs, oracle, a.rl_secs, DEV, a.seed, log)
286
+ if DEV == "cuda":
287
+ torch.cuda.synchronize()
288
+ used = oracle.used()
289
+ model.eval()
290
+ with open(f"{a.out}/found_targets.jsonl", "w") as f:
291
+ for t in targets:
292
+ for x in oracle.found[t["name"]].values():
293
+ f.write(
294
+ json.dumps({"name": t["name"], "thm": t["thm"], "L_true": t["n_lines"], **x})
295
+ + "\n"
296
+ )
297
+ log(f"rl returned: {used:.0f}s of {a.rl_secs}s charged, {oracle.uncharged:.0f}s Lean gate")
298
+ return {
299
+ "rounds": oracle.calls,
300
+ "secs": time.time() - oracle.t0,
301
+ "rl_secs_used": used,
302
+ "gate_secs": oracle.uncharged,
303
+ "targets": len(targets),
304
+ "stats": dict(stats or {}),
305
+ }
306
+
307
+
308
+ def eval_at(a, tok, model, dev_pool, level):
309
+ setattr(model, KNOB, level)
310
+ rows, samp = sample_and_judge(
311
+ model, tok, dev_pool, a.eval_k, TEMP, a.seed * 1000 + 777, SAMPLE_BATCH, a.max_new
312
+ )
313
+ solved = {r["name"] for r in rows if r["solved"]}
314
+ ls, ge = lstar(solved, dev_pool)
315
+ out = {
316
+ "level": level,
317
+ "attempts_per_theorem": a.eval_k,
318
+ "solved": len(solved),
319
+ "n": len(dev_pool),
320
+ "solved_ge7": ge[7],
321
+ "lstar": ls,
322
+ "ge": ge,
323
+ "by_bin": by_bin(solved, dev_pool),
324
+ "sample_acc": sum(r["n_ok"] for r in rows) / max(samp["samples"], 1),
325
+ "max_pruned": max((max(r["pruned_lens"]) for r in rows if r["solved"]), default=0),
326
+ "sampling": samp,
327
+ }
328
+ log(
329
+ f"eval at {KNOB} {level}: solved {len(solved)}/{len(dev_pool)}, ge7 {ge[7]}, L*={ls}, "
330
+ f"per-sample {out['sample_acc']:.4f}, {samp['samples_per_sec']:.0f} samples/s"
331
+ )
332
+ return out, rows
333
+
334
+
335
+ def stage_eval(a, tok, model, dev_pool, heldout_recs):
336
+ out, rows = eval_at(a, tok, model, dev_pool, a.eval_level)
337
+ out["heldout_greedy"] = greedy_rate(model, tok, heldout_recs)
338
+ with open(f"{a.out}/eval_dev.jsonl", "w") as f:
339
+ for r in rows:
340
+ keys = ("name", "n_lines", "solved", "n_ok", "proofs", "written_lens", "pruned_lens")
341
+ f.write(json.dumps({k: r[k] for k in keys}) + "\n")
342
+ diag, _ = eval_at(a, tok, model, dev_pool, a.diag_level)
343
+ out["diag"] = diag
344
+ setattr(model, KNOB, a.eval_level)
345
+ log(
346
+ f"eval: solved {out['solved']}/{len(dev_pool)}, ge7 {out['solved_ge7']} at "
347
+ f"{KNOB} {a.eval_level} ({diag['solved_ge7']} at {a.diag_level}), L*={out['lstar']}, "
348
+ f"max pruned {out['max_pruned']}, held-out greedy {out['heldout_greedy']:.3f}"
349
+ )
350
+ return out
351
+
352
+
353
+ def lean_gate_totals():
354
+ """Sum the fork lean_gate's per-generate() log lines (remote.sh sets LEAN_GATE_LOG)."""
355
+ fn = os.environ.get("LEAN_GATE_LOG")
356
+ if not fn or not os.path.exists(fn):
357
+ return None
358
+ keys = ("samples", "parse_fail", "distinct_checked", "both_ok", "nd_ok_lean_rej")
359
+ tot = dict.fromkeys(keys + ("nd_rej_lean_ok", "lean_wall_s"), 0)
360
+ for rec in read(fn):
361
+ for k in tot:
362
+ tot[k] += rec[k]
363
+ return tot
364
+
365
+
366
+ RULES = (
367
+ "TOTAL_SECS",
368
+ "PRETRAIN_SECS",
369
+ "RL_SECS",
370
+ "PARAM_CAP",
371
+ "N_TARGETS",
372
+ "EVAL_K",
373
+ "KNOB",
374
+ "EVAL_LEVEL",
375
+ "DIAG_LEVEL",
376
+ "TOK_MODE",
377
+ )
378
+
379
+
380
+ def write_summary(a, status, **kw):
381
+ s = {
382
+ "status": status,
383
+ "metric": kw.get("eval", {}).get("solved_ge7"),
384
+ "metric_diag": kw.get("eval", {}).get("diag", {}).get("solved_ge7"),
385
+ "metric_rl": (kw.get("rl", {}).get("rounds") or [{}])[-1].get(
386
+ "targets_solved"
387
+ ), # secondary: cumulative RL targets solved
388
+ "harness_sha": sha(f"{HERE}/harness.py"),
389
+ "pretrain_sha": sha(f"{HERE}/pretrain.py"),
390
+ "device": DEV,
391
+ "gpu": torch.cuda.get_device_name(0) if DEV == "cuda" else "cpu",
392
+ "torch": torch.__version__,
393
+ "lean_gate": lean_gate_totals(),
394
+ "total_secs": time.time() - a.T0,
395
+ "rules": {k: globals()[k] for k in RULES},
396
+ **kw,
397
+ }
398
+ with open(f"{a.out}/summary.json", "w") as f:
399
+ json.dump(s, f, indent=1)
400
+ log(
401
+ f"SUMMARY status={status} metric={s['metric']} metric_diag={s['metric_diag']} "
402
+ f"metric_rl={s['metric_rl']} total {s['total_secs']:.0f}s"
403
+ )
404
+ return s
405
+
406
+
407
+ def main():
408
+ ap = argparse.ArgumentParser()
409
+ ap.add_argument("--out", required=True)
410
+ ap.add_argument("--seed", type=int, default=0)
411
+ ap.add_argument(
412
+ "--pretrain-only", action="store_true", help="save pretrain.pt and stop before eval/RL"
413
+ )
414
+ ap.add_argument(
415
+ "--smoke", action="store_true", help="tiny everything, for a CPU plumbing check"
416
+ )
417
+ a = ap.parse_args()
418
+ a.pretrain_secs, a.rl_secs, a.eval_k, a.max_new, a.n_targets = (
419
+ PRETRAIN_SECS,
420
+ RL_SECS,
421
+ EVAL_K,
422
+ MAX_NEW,
423
+ N_TARGETS,
424
+ )
425
+ a.eval_level, a.diag_level = EVAL_LEVEL, DIAG_LEVEL
426
+ if a.smoke:
427
+ a.pretrain_secs, a.rl_secs, a.eval_k, a.max_new, a.n_targets = 10, 30, 2, 120, 8
428
+ os.makedirs(a.out, exist_ok=True)
429
+ # Preserve discarded variants too: weights alone do not specify the model architecture.
430
+ source_dir = Path(a.out) / "source"
431
+ source_dir.mkdir(exist_ok=True)
432
+ for filename in ("pretrain.py", "harness.py", "remote.sh"):
433
+ shutil.copy2(Path(HERE) / filename, source_dir / filename)
434
+ a.T0 = time.time()
435
+ torch.manual_seed(a.seed)
436
+
437
+ def on_alarm(signum, frame):
438
+ raise Timeout()
439
+
440
+ signal.signal(signal.SIGALRM, on_alarm)
441
+ signal.alarm(TOTAL_SECS if not a.smoke else 900)
442
+
443
+ train_recs = read(f"{FORK}/data/train.jsonl")
444
+ assert all(r["n_lines"] <= 6 for r in train_recs), "pretrain file violates cap 6"
445
+ heldout_all = read(f"{FORK}/data/heldout.jsonl")
446
+ # Length-ordered file: take 2000 evenly spaced records, so every length 2-6 is covered.
447
+ heldout_all = [heldout_all[i * len(heldout_all) // 2000] for i in range(2000)]
448
+ heldout_recs = heldout_all if not a.smoke else heldout_all[:16]
449
+ targets_all = read(f"{FORK}/data/ladder/rl_targets.jsonl")
450
+ transfer = read(f"{FORK}/data/ladder/transfer.jsonl")
451
+ dev_pool = dev_split(transfer) if not a.smoke else dev_split(transfer)[:8]
452
+ targets = random.Random(a.seed).sample(targets_all, min(a.n_targets, len(targets_all)))
453
+ tk, dk, hk = (
454
+ {t["key"] for t in targets},
455
+ {r["key"] for r in dev_pool},
456
+ {r["key"] for r in train_recs},
457
+ )
458
+ assert not (tk & dk) and not (tk & hk) and not (dk & hk), "renaming-class leak between pools"
459
+ log(
460
+ f"pools: train {len(train_recs):,}, targets {len(targets)}, dev-transfer {len(dev_pool)}, "
461
+ f"held-out {len(heldout_recs)}; "
462
+ f"device {DEV}; harness {sha(f'{HERE}/harness.py')} pretrain {sha(f'{HERE}/pretrain.py')}"
463
+ )
464
+
465
+ pre = {}
466
+ try:
467
+ agent = importlib.import_module("pretrain")
468
+ t0 = time.time()
469
+ model, tok, pre = agent.pretrain(train_recs, a.pretrain_secs, DEV, a.seed, log)
470
+ if DEV == "cuda":
471
+ torch.cuda.synchronize()
472
+ pre = dict(pre or {})
473
+ pre["secs"] = time.time() - t0
474
+ model.eval()
475
+ n_params = sum(p.numel() for p in model.parameters())
476
+ pre["n_params"] = n_params
477
+ log(
478
+ f"pretrain returned after {pre['secs']:.0f}s (budget {a.pretrain_secs}s), "
479
+ f"{n_params:,} params"
480
+ )
481
+ if pre["secs"] > a.pretrain_secs + PRETRAIN_GRACE and not a.smoke:
482
+ over = pre["secs"] - a.pretrain_secs
483
+ write_summary(
484
+ a, "timeout", pretrain=pre, reason=f"pretrain overran budget by {over:.0f}s"
485
+ )
486
+ return
487
+ if getattr(tok, "mode", None) != TOK_MODE:
488
+ write_summary(a, "crash", pretrain=pre, reason=f"tok.mode {tok.mode!r} != {TOK_MODE!r}")
489
+ return
490
+ if n_params > PARAM_CAP:
491
+ write_summary(a, "crash", pretrain=pre, reason=f"{n_params} params > cap {PARAM_CAP}")
492
+ return
493
+ if not isinstance(getattr(model, KNOB, None), int):
494
+ reason = f"model has no integer {KNOB} for the harness to set at eval"
495
+ write_summary(a, "crash", pretrain=pre, reason=reason)
496
+ return
497
+ save_checkpoint(a, model, tok, "pretrain")
498
+ if a.pretrain_only:
499
+ write_summary(a, "ok", stage="pretrain_only", pretrain=pre)
500
+ return
501
+ pre["heldout_greedy"] = greedy_rate(model, tok, heldout_recs)
502
+ log(f"pretrain held-out greedy {pre['heldout_greedy']:.3f}")
503
+ rl = stage_rl(a, agent, tok, model, targets, train_recs)
504
+ if rl["rl_secs_used"] > a.rl_secs + RL_GRACE and not a.smoke:
505
+ over = rl["rl_secs_used"] - a.rl_secs
506
+ reason = f"rl overran budget by {over:.0f}s"
507
+ write_summary(a, "timeout", pretrain=pre, rl=rl, reason=reason)
508
+ return
509
+ n_after = sum(p.numel() for p in model.parameters())
510
+ if n_after > PARAM_CAP:
511
+ reason = f"{n_after} params > cap {PARAM_CAP} after rl"
512
+ write_summary(a, "crash", pretrain=pre, rl=rl, reason=reason)
513
+ return
514
+ save_checkpoint(a, model, tok, "final")
515
+ ev = stage_eval(a, tok, model, dev_pool, heldout_recs)
516
+ write_summary(a, "ok", pretrain=pre, rl=rl, eval=ev)
517
+ except BudgetSpent as e:
518
+ write_summary(a, "timeout", pretrain=pre, reason=f"rl: {e}")
519
+ except Timeout:
520
+ write_summary(a, "timeout", pretrain=pre, reason=f"exceeded TOTAL_SECS={TOTAL_SECS}")
521
+ except Exception:
522
+ traceback.print_exc()
523
+ write_summary(a, "crash", pretrain=pre, reason=traceback.format_exc()[-2000:])
524
+ finally:
525
+ signal.alarm(0)
526
+
527
+
528
+ if __name__ == "__main__":
529
+ main()
runs/707-fable_plain-state-feedback_s1/source/pretrain.py ADDED
@@ -0,0 +1,753 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # imports the fork (model, tokenizer, train, ...), which is not resolvable here
2
+ # pyright: reportMissingImports=false
3
+ """AGENT-EDITABLE: pretraining and RL. This is the only file the autoresearch agent changes.
4
+
5
+ Contract with harness.py (frozen):
6
+
7
+ pretrain(train_recs, budget_secs, device, seed, log) -> (model, tok, stats)
8
+
9
+ train_recs list of dicts {prompt, proof, n_lines<=6, key, ...}; the frozen cap-6 Stage-1 data
10
+ budget_secs wall-clock you are charged for, from call to return, INCLUDING model construction
11
+ and torch.compile. Overrun by more than the harness grace -> the run is a timeout.
12
+ model nn.Module; <= PARAM_CAP unique params (checked again after rl). The harness samples
13
+ from it with the fork's sample.generate, so it must provide:
14
+ - forward(idx) -> logits (B,T,V) and forward(idx, pos=, mask=, caches=) with a
15
+ per-block KV cache dict, exactly like the fork's GPT (model.blocks must exist)
16
+ - an integer `n_thoughts`: continuous thoughts per proof line in eval mode
17
+ (the harness's KNOB). The harness sets it to EVAL_LEVEL for the scored eval
18
+ and to DIAG_LEVEL for the diagnostic one.
19
+ tok must provide encode_prompt, encode_proof, decode, shift_abs, pad, eos, mode, and
20
+ mode must be "lean_seq" (harness TOK_MODE): Lean text in and out, Lean-gated samples.
21
+ Decoder rule: decode() is deterministic and single-pass; it may not call the
22
+ verifier or try alternatives.
23
+ stats dict recorded in summary.json and copied into results.json. MUST contain `curve`:
24
+ a list of {secs, step, train_loss, val_loss} taken every ~CURVE_EVERY seconds
25
+ (see log_point below), so a run's trajectory survives in git. Add anything else.
26
+ val_loss is on VAL_N records held out from train_recs by this file; keep that.
27
+
28
+ rl(model, tok, targets, train_recs, oracle, budget_secs, device, seed, log) -> stats | None
29
+
30
+ targets list of dicts {name, prompt, n_lines}: the 1,500 RL target statements. No proofs.
31
+ oracle oracle.sample(model, targets_subset, k, temperature=0.8, batch=1024) samples k
32
+ attempts per target from the model in eval mode at its current n_thoughts, and
33
+ returns per target {name, prompt, n_lines, proofs, n_ok, n_tried}; `proofs` are the
34
+ distinct attempts Lean AND nd_verify accepted. This is the only source of new
35
+ training data.
36
+ oracle.used() / oracle.left(): seconds of the RL budget charged / remaining.
37
+ budget_secs RL_SECS: wall-clock inside rl() minus the oracle's Lean-gate time. Returning more
38
+ than the grace late, or calling the oracle with nothing left, is a timeout.
39
+ The model is trained in place. Everything else about RL is yours: rounds, samples per target,
40
+ thought counts when sampling, which proofs to keep, weighting, replay, optimizer, loss.
41
+
42
+ Anything imported from the fork (ND_FORK) is fair game to copy in here and modify.
43
+ """
44
+
45
+ import math
46
+ import random
47
+ import time
48
+
49
+ import torch
50
+ import torch._inductor.config # noqa: F401 (explicit, so the attribute resolves for pyright)
51
+ from lean_tok import MAXN # fork
52
+ from model import GPT, Block # fork
53
+ from normalize import norm # fork
54
+ from tokenizer import make_tokenizer # fork
55
+ from train import batch as mk_batch # fork
56
+ from train import loss_on # fork
57
+
58
+ # ---- knobs
59
+ N_LAYER, D, N_HEAD = 6, 384, 8 # run 198's untied blocks
60
+ # One thinking round before each proof line. A line starts after a trigger token: `;` between
61
+ # Lean tactics, and `by`, which ends the theorem statement and opens every nested block. After a
62
+ # trigger the model takes n thought positions whose inputs are its own reading of the proof so
63
+ # far, fed back (Coconut, Hao et al. 2024): ln_thought(the PLAIN network's final state at the
64
+ # trigger) + slot_emb[j], where the plain network is this model run with no thoughts at all (run
65
+ # 198's forward), and the last thought's output predicts the line's first token. In 701-703 the
66
+ # fed-back state was the thinking network's own, which depends on the earlier lines' thoughts, so
67
+ # training ran one pass per proof line; 704's pause slots (learned input vectors alone) trained
68
+ # in one pass but went unused (val loss and eval flat in n). The plain state depends on no
69
+ # thought, so here a training step is one forward of the plain network over the tokens, with no
70
+ # gradient (its drafts are inputs, not targets), plus ONE causal pass over the expanded sequence
71
+ # as in 704; sampling keeps a second, token-only KV cache for the plain network.
72
+ # Pretraining draws n per batch from TRAIN_THOUGHTS; rl()'s fine-tune uses FT_THOUGHTS
73
+ # (below); eval and sampling apply model.n_thoughts, which the harness sets for its evals and
74
+ # rl() sets before each oracle call. DEFAULT_THOUGHTS is used for the validation curve and the
75
+ # harness's post-pretraining greedy check.
76
+ TRAIN_THOUGHTS = (0, 1, 2, 3, 4, 5, 6, 7, 8)
77
+ DEFAULT_THOUGHTS = 8
78
+ MAX_THOUGHTS = 16 # rows of the thought-slot embedding; n_thoughts above this is an error
79
+ VAL_THOUGHTS = (0, 2, 8) # val loss at each count, once, when pretraining ends
80
+ # Sampling keeps one KV cache row per sample with its own write pointer. Each cache is sized for
81
+ # the sampler's tokens plus n_thoughts * MAX_THINKS thought positions; a sample that has used its
82
+ # share (a degenerate one emitting `;` forever) goes on without thinking.
83
+ MAX_THINKS = 48
84
+ # The plan probe: at every thinking round, a head that exists only during training must read the
85
+ # line's "move" off the last thought: the token that follows the line's `:=` or `exact` (the name
86
+ # applied, `⟨`, `Or.inl`, the `(` of a `fun`, ...). It is computed from the tokens alone, so it
87
+ # stays on during RL fine-tuning, on proofs the model found.
88
+ PROBE_WEIGHT = 0.3
89
+ # ---- RL knobs: the old harness's expert iteration (600 AdamW steps per round on up to 4 proofs
90
+ # per solved target, weighted x4, mixed with 20,000 replayed pretraining records), except that
91
+ # each round samples every target RL_K times at each thought count in RL_SAMPLE_THOUGHTS (32
92
+ # attempts per target in all, as before). Rounds repeat until another would not fit the budget.
93
+ # RL runs entirely at the eval level: every sample is drawn at 8 thoughts per line and the
94
+ # fine-tune trains at 8 only (FT_THOUGHTS), never at 0. In 701 the fine-tune drew n from 0..8 and
95
+ # the no-thought path caught up within one round (sample accuracy at 0 thoughts went from 3-6% to
96
+ # 28-33%, against 39% at 8): the proofs found with thoughts were being distilled into the path
97
+ # that does not think. Here they are only ever learned with thoughts.
98
+ RL_SAMPLE_THOUGHTS = (8,)
99
+ RL_K = 32
100
+ FT_THOUGHTS = (8,)
101
+ FT_STEPS, FT_BS, FT_LR, FT_WARMUP = 600, 128, 3e-4, 50
102
+ # A fine-tune step's 128 records are sorted by length and run 32 at a time, gradients accumulated.
103
+ # run() holds attention weights over every earlier line of every row, so memory grows with rows
104
+ # times (padded length)^2: 128 long RL proofs at once ran out of 44 GB (experiment 700).
105
+ FT_MICRO = 32
106
+ RETAIN, MAX_PER_THM, RL_WEIGHT = 20000, 4, 4
107
+ # MLP width held at 151's 1280: the fork's Block hardcodes 4 * D (1536 at D 384 would pass the
108
+ # 10M cap), so PeriLNBlock swaps fc1/fc2. 1280 != 3 * D keeps muon_step's QKV-by-shape test valid.
109
+ D_FF = 1280
110
+ BS, LR, MIN_LR_FRAC, WD = 128, 1e-3, 0.3, 0.1
111
+ MUON_LR, MUON_MOMENTUM = 0.01, 0.95
112
+ # Polar Express (Amsel et al. 2025) per-iteration quintic coefficients, as in modded-nanogpt
113
+ # (num_iters=5, safety factor 2e-2): singular values land in ~[0.86, 1.14] instead of the
114
+ # fixed quintic's ~[0.68, 1.13], at the same five iterations.
115
+ NS_COEFFS = (
116
+ (8.156554524902461, -22.48329292557795, 15.878769915207462),
117
+ (4.042929935166739, -2.808917465908714, 0.5000178451051316),
118
+ (3.8916678022926607, -2.772484153217685, 0.5060648178503393),
119
+ (3.285753657755655, -2.3681294933425376, 0.46449024233003106),
120
+ (2.3465413258596377, -1.7097828382687081, 0.42323551169305323),
121
+ )
122
+ # Ten times the AdamW group's per-step multiplicative decay (lr * wd) on the Muon matrices.
123
+ MUON_WD = 10 * WD * LR / MUON_LR
124
+ TOK_MODE = "lean_seq" # required by the harness (Lean protocol)
125
+ # Hybrid heads: half the heads get ALiBi (Press et al. 2021) with the standard slope set for
126
+ # H/2 heads, 2^(-8 i / (H/2)) = 1/4 .. 1/256 with 8 heads; the other half get slope 0, i.e. no
127
+ # positional signal at all (NoPE), so they can look citations up by content at any distance.
128
+ N_ALIBI = N_HEAD // 2
129
+ ALIBI_SLOPES = tuple(2.0 ** (-8.0 * (i + 1) / N_ALIBI) for i in range(N_ALIBI)) + (0.0,) * (
130
+ N_HEAD - N_ALIBI
131
+ )
132
+ VAL_N, CURVE_EVERY = 1024, 15 # records held out for val_loss; seconds between curve points
133
+ # Token-budget batches (fairseq max-tokens): each step takes a run of length-sorted records with
134
+ # count * padded_length <= TOK_BUDGET, so a batch of short proofs has many records and a batch of
135
+ # long ones few, and the padded token count per step is ~constant. 128 x 144 matches the real
136
+ # (unpadded) tokens of the incumbent's 128-record batches (mean record ~135 tokens), whose
137
+ # padding to the batch max (~400) made every step ~3x larger than its content.
138
+ TOK_BUDGET = 128 * 144
139
+ # Multi-token prediction (DeepSeek-V3 2024, Gloeckle et al. 2024), pretraining only: one extra
140
+ # Peri-LN block reads [trunk state at t; embedding of token t+1] and predicts token t+2 through
141
+ # the model's own ln_f/head. Its loss is added at this weight (DeepSeek-V3's early-training value).
142
+ MTP_WEIGHT = 0.3
143
+
144
+
145
+ # Muon update adapted from KellerJordan/Muon, commit
146
+ # f98f1cacc0263b04290753e32be8d498c1efc806 (muon.py).
147
+ # MIT License
148
+ # Copyright (c) 2024 Keller Jordan
149
+ #
150
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
151
+ # of this software and associated documentation files (the "Software"), to deal
152
+ # in the Software without restriction, including without limitation the rights
153
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
154
+ # copies of the Software, and to permit persons to whom the Software is
155
+ # furnished to do so, subject to the following conditions:
156
+ # The above copyright notice and this permission notice shall be included in all
157
+ # copies or substantial portions of the Software.
158
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
159
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
160
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
161
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
162
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
163
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
164
+ # SOFTWARE.
165
+ @torch.no_grad()
166
+ def muon_step(params, buffers, lr):
167
+ """Nesterov momentum plus five bf16 Polar Express steps, on hidden matrices only."""
168
+ for p, momentum in zip(params, buffers, strict=True):
169
+ if p.grad is None:
170
+ continue
171
+ momentum.lerp_(p.grad, 1 - MUON_MOMENTUM)
172
+ update = p.grad.lerp(momentum, MUON_MOMENTUM).bfloat16()
173
+ # Only QKV has shape (3D, D) in this GPT; orthogonalize its roles separately.
174
+ if update.size(0) == 3 * update.size(1):
175
+ update = update.reshape(3, update.size(1), update.size(1))
176
+ transpose = update.size(-2) > update.size(-1)
177
+ if transpose:
178
+ update = update.mT
179
+ update = update / (update.norm(dim=(-2, -1), keepdim=True) * (1 + 2e-2) + 1e-6)
180
+ for a, b, c in NS_COEFFS:
181
+ gram = update @ update.mT
182
+ polynomial = b * gram + c * gram @ gram
183
+ update = a * update + polynomial @ update
184
+ if transpose:
185
+ update = update.mT
186
+ p.mul_(1 - lr * MUON_WD)
187
+ # lr may be a 0-d tensor (CUDA-graphed step); scale in p's dtype like add_(alpha=-lr)
188
+ p.sub_(lr * update.reshape_as(p).to(p.dtype))
189
+
190
+
191
+ class PeriLNBlock(Block):
192
+ """P22 Peri-LN (Kim et al. 2025): the fork's pre-LN block plus a LayerNorm on each branch
193
+ output, so every residual write has unit scale times a learned gain. Attention has no RoPE:
194
+ `mask` is the additive ALiBi bias (B or 1, H, T, S) that LineThoughtGPT builds and cos/sin
195
+ are unused. Three ways to supply earlier keys and values: none (`cache` None, one full
196
+ pass); a list, in training, holding the keys and values of the rounds already run, which this
197
+ call extends by concatenation so gradients flow; and sample_step, in sampling, which writes
198
+ into preallocated buffers at a position of each row's own."""
199
+
200
+ def __init__(self, d, h):
201
+ super().__init__(d, h)
202
+ self.fc1, self.fc2 = torch.nn.Linear(d, D_FF), torch.nn.Linear(D_FF, d)
203
+ self.ln_attn_out, self.ln_mlp_out = torch.nn.LayerNorm(d), torch.nn.LayerNorm(d)
204
+
205
+ def attn(self, x, cos, sin, mask, cache):
206
+ B, T, D = x.shape
207
+ q, k, v = self.qkv(x).view(B, T, 3, self.h, D // self.h).permute(2, 0, 3, 1, 4)
208
+ if cache is not None:
209
+ if cache:
210
+ k, v = torch.cat((cache[0], k), 2), torch.cat((cache[1], v), 2)
211
+ cache[:] = [k, v]
212
+ y = torch.nn.functional.scaled_dot_product_attention(q, k, v, attn_mask=mask.to(q.dtype))
213
+ return self.proj(y.transpose(1, 2).reshape(B, T, D))
214
+
215
+ def forward(self, x, cos, sin, mask=None, cache=None):
216
+ x = x + self.ln_attn_out(self.attn(self.ln1(x), cos, sin, mask, cache))
217
+ return x + self.ln_mlp_out(self.fc2(torch.nn.functional.gelu(self.fc1(self.ln2(x)))))
218
+
219
+ def sample_step(self, x, bias, cache, rows, cols):
220
+ """forward() for sampling. x (b, T, D) are new positions of the samples `rows` (b, 1),
221
+ all of them if b is the batch; their keys and values go to columns `cols` (b, T) of the
222
+ buffers, and they attend to those samples' first bias.size(-1) columns."""
223
+ b, T, D = x.shape
224
+ q, k, v = self.qkv(self.ln1(x)).view(b, T, 3, self.h, D // self.h).permute(2, 0, 3, 1, 4)
225
+ if "k" not in cache:
226
+ # zeros, not empty: a row reads columns past its own pointer (masked), and a masked
227
+ # key that happens to hold inf or nan still poisons the attention output
228
+ cache["k"] = k.new_zeros(cache["B"], self.h, cache["max"], D // self.h)
229
+ cache["v"] = v.new_zeros(cache["B"], self.h, cache["max"], D // self.h)
230
+ cache["k"][rows, :, cols] = k.transpose(1, 2) # indexed dims first: (b, T, H, hd)
231
+ cache["v"][rows, :, cols] = v.transpose(1, 2)
232
+ S, every = bias.size(-1), b == cache["B"]
233
+ k = cache["k"][:, :, :S] if every else cache["k"][rows[:, 0], :, :S]
234
+ v = cache["v"][:, :, :S] if every else cache["v"][rows[:, 0], :, :S]
235
+ y = torch.nn.functional.scaled_dot_product_attention(q, k, v, attn_mask=bias.to(q.dtype))
236
+ x = x + self.ln_attn_out(self.proj(y.transpose(1, 2).reshape(b, T, D)))
237
+ return x + self.ln_mlp_out(self.fc2(torch.nn.functional.gelu(self.fc1(self.ln2(x)))))
238
+
239
+
240
+ class LineThoughtGPT(GPT):
241
+ """Run 198's model (the fork's GPT with half-ALiBi, half-NoPE heads) with one thinking round
242
+ before each proof line: n fed-back positions after every trigger.
243
+
244
+ After a trigger token the sequence gets n thought positions whose inputs are
245
+ ln_thought(h) + slot_emb[j], where h is the final state at the trigger of the PLAIN network:
246
+ the same blocks run over the tokens alone, with no thoughts (plain()). They attend to
247
+ everything before them and to each other causally, sit at the trigger's position on the
248
+ ALiBi axis (so distances between tokens do not depend on n), and the last one's output
249
+ replaces the trigger's as the predictor of the next token. With n = 0 this is exactly run
250
+ 198's network, and so is the plain network at any n.
251
+
252
+ Training (no caches) goes through run(): the plain forward (no gradient) for the thought
253
+ inputs, then one causal pass over the expanded sequence; sampling (`caches` given) through
254
+ cached(), which keeps the plain network's own token-only caches and does the thinking inside
255
+ the forward call, so the fork's sampler needs no change."""
256
+
257
+ def __init__(self, vocab, n_layer, d, n_head, triggers, seed, max_len=1024):
258
+ super().__init__(vocab, n_layer, d, n_head, max_len)
259
+ slopes = torch.tensor(ALIBI_SLOPES).view(1, n_head, 1, 1)
260
+ self.register_buffer("slopes", slopes, persistent=False)
261
+ self.register_buffer("triggers", torch.tensor(triggers), persistent=False)
262
+ # Gain 0: at initialization the thoughts are 704's pause slots, which learn fast, and the
263
+ # fed-back state enters as AdamW opens the gain. At gain 1 from the start, with the state
264
+ # detached, the bench's 60 s run was stuck near loss 0.74 against 0.09 for the slots.
265
+ self.ln_thought = torch.nn.LayerNorm(d)
266
+ torch.nn.init.zeros_(self.ln_thought.weight)
267
+ self.slot_emb = torch.nn.Parameter(torch.randn(MAX_THOUGHTS, d) * 0.02)
268
+ self.n_thoughts, self.t_rng = DEFAULT_THOUGHTS, random.Random(seed)
269
+
270
+ def slots(self, h, n):
271
+ """The n thought inputs (b, n, D) of one round from the plain network's final state h
272
+ (b, 1, D) at its trigger."""
273
+ return self.ln_thought(h) + self.slot_emb[:n]
274
+
275
+ def bias(self, qpos, kpos, allowed):
276
+ """ALiBi bias (B, H, Q, S) for queries at positions qpos (B, Q) over keys at kpos (B, S),
277
+ with -inf wherever `allowed` (B, Q, S) is False."""
278
+ rel = qpos[:, None, :, None] - kpos[:, None, None, :]
279
+ return (-self.slopes * rel).masked_fill(~allowed[:, None], float("-inf"))
280
+
281
+ def forward(self, idx, pos=None, mask=None, caches=None):
282
+ if caches is not None:
283
+ return self.head(self.ln_f(self.cached(idx, pos, mask, caches)))
284
+ n = self.t_rng.choice(TRAIN_THOUGHTS) if self.training else self.n_thoughts
285
+ return self.head(self.ln_f(self.run(idx, n)))
286
+
287
+ # ---- training: one causal pass over the expanded sequence, thoughts inserted after triggers
288
+
289
+ def plain(self, idx):
290
+ """Final states (B, S, D) of the plain network over the right-padded rows idx: no
291
+ thoughts, run 198's forward."""
292
+ ar = torch.arange(idx.size(1), device=idx.device)[None]
293
+ dtype = torch.bfloat16 if torch.is_autocast_enabled() else torch.float32
294
+ x = self.emb(idx)
295
+ bias = self.bias(ar, ar, ar[:, None, :] <= ar[:, :, None]).to(dtype)
296
+ for b in self.blocks:
297
+ x = b(x, None, None, bias, None)
298
+ return x
299
+
300
+ def run(self, idx, n):
301
+ """Final hidden states (B, S, D) of right-padded rows idx with n thoughts per line, in
302
+ idx's own layout; at a trigger it is the last thought's output, which is what predicts
303
+ the next token. The expanded layout puts token t of a row at column
304
+ t + n * (triggers before t) and a trigger's n thoughts at the columns after it; rows
305
+ with fewer triggers than the longest leave empty columns at the end, masked as keys."""
306
+ B, S = idx.shape
307
+ dev, D = idx.device, self.emb.embedding_dim
308
+ dtype = torch.bfloat16 if torch.is_autocast_enabled() else torch.float32
309
+ ar = torch.arange(S, device=dev)[None].expand(B, S)
310
+ rows = torch.arange(B, device=dev)[:, None].expand(B, S)
311
+ trig = torch.isin(idx, self.triggers)
312
+ col = ar + n * (trig.cumsum(1) - trig.long()) # each token's column
313
+ E = S + n * int(trig.sum(1).max())
314
+ e = self.emb(idx)
315
+ x, pos, valid = e.new_zeros(B, E, D), idx.new_zeros(B, E), trig.new_zeros(B, E)
316
+ x[rows, col], pos[rows, col], valid[rows, col] = e, ar, True
317
+ if n:
318
+ with torch.no_grad(): # the drafts are inputs, not targets: forward only
319
+ feed = self.plain(idx)[trig][:, None] # (triggers, 1, D)
320
+ # autocast caches the bf16 casts of the weights that pass made, and they carry no
321
+ # grad_fn: the pass with gradients below would reuse them and the block matrices
322
+ # would get no gradient (torch 2.9.1; this emptied 705's first attempt).
323
+ torch.clear_autocast_cache()
324
+ tr, tc, tp = rows[trig][:, None], col[trig][:, None], ar[trig][:, None]
325
+ tc = tc + 1 + torch.arange(n, device=dev)[None] # (triggers, n)
326
+ x[tr, tc], pos[tr, tc], valid[tr, tc] = self.slots(feed, n), tp, True
327
+ ec = torch.arange(E, device=dev)
328
+ # column 0 is a token in every row, so no query is left without a key
329
+ allowed = valid[:, None, :] & (ec[None, :] <= ec[:, None])[None]
330
+ bias = self.bias(pos, pos, allowed).to(dtype)
331
+ for b in self.blocks:
332
+ x = b(x, None, None, bias, None)
333
+ out = col + n * trig.long() # a trigger reads out at its last thought
334
+ return x.gather(1, out[..., None].expand(B, S, D))
335
+
336
+ # ---- sampling: the fork's incremental decoding, thinking after every trigger
337
+
338
+ def cached(self, idx, pos, mask, caches):
339
+ """One sampler call: the left-padded prompts first, then one new token per sample. The
340
+ tokens go through the thinking network's caches and, when n > 0, the plain network's own
341
+ token-only caches (caches[0]["plain"]) as well. After the call's last token, the samples
342
+ for which it is a trigger think, from their plain state, and their returned state is the
343
+ last thought's. Samples differ in how many thoughts they have taken, so each has its own
344
+ write pointer into the buffers and its own key positions and validity, kept in the first
345
+ dict of each cache list; the sampler's key mask is only read for the prompt's padding."""
346
+ c, (B, T), n = caches[0], idx.shape, self.n_thoughts
347
+ assert n <= MAX_THOUGHTS, n
348
+ if "ptr" not in c:
349
+ c["tokens_left"] = c["max"] - T
350
+ if n: # the plain network's caches hold tokens only
351
+ p = {"max": c["max"], "B": B}
352
+ p["ptr"], p["kpos"] = pos.new_zeros(B), pos.new_zeros(B, p["max"])
353
+ p["kvalid"] = mask.new_zeros(B, p["max"])
354
+ c["plain"] = [p] + [{"max": c["max"], "B": B} for _ in self.blocks[1:]]
355
+ for cache in caches:
356
+ cache["max"] += n * MAX_THINKS
357
+ cache["B"] = B
358
+ c["ptr"] = pos.new_zeros(B)
359
+ c["kpos"] = pos.new_zeros(B, c["max"])
360
+ c["kvalid"] = mask.new_zeros(B, c["max"])
361
+ valid = mask[:, 0, -1] # (B, T): which prompt columns are real tokens
362
+ else:
363
+ c["tokens_left"] -= T
364
+ valid = mask.new_ones(B, T)
365
+ every = torch.arange(B, device=idx.device)[:, None]
366
+ e = self.emb(idx)
367
+ h = self.cached_pass(e, pos, valid, caches, every)
368
+ think = torch.isin(idx[:, -1], self.triggers)
369
+ think &= c["ptr"] + n + c["tokens_left"] <= c["max"]
370
+ if n:
371
+ drafts = self.cached_pass(e, pos, valid, c["plain"], every)
372
+ if bool(think.any()):
373
+ rows = every[think]
374
+ x = self.slots(drafts[think, -1:], n)
375
+ real = valid.new_ones(x.shape[:2])
376
+ out = self.cached_pass(x, pos[think, -1:].expand(-1, n), real, caches, rows)
377
+ h[think, -1] = out[:, -1]
378
+ return h
379
+
380
+ def cached_pass(self, x, qpos, valid, caches, rows):
381
+ c, T = caches[0], x.size(1)
382
+ at = c["ptr"][rows[:, 0]]
383
+ cols = at[:, None] + torch.arange(T, device=x.device)[None]
384
+ c["kpos"][rows, cols], c["kvalid"][rows, cols] = qpos, valid
385
+ c["ptr"][rows[:, 0]] = at + T
386
+ S = int(at.max()) + T
387
+ col = torch.arange(S, device=x.device)[None, None, :]
388
+ kvalid = c["kvalid"][rows[:, 0], :S]
389
+ allowed = (kvalid[:, None, :] & (col <= cols[:, :, None])) | (col == cols[:, :, None])
390
+ bias = self.bias(qpos, c["kpos"][rows[:, 0], :S], allowed)
391
+ for b_, cache in zip(self.blocks, caches, strict=True):
392
+ x = b_.sample_step(x, bias, cache, rows, cols)
393
+ return x
394
+
395
+
396
+ class MTPHead(torch.nn.Module):
397
+ """The MTP module. A separate nn.Module, never registered on the model, so the harness's
398
+ parameter count, RL AdamW and sampler never see it; it is dropped when pretrain() returns."""
399
+
400
+ def __init__(self, d, h):
401
+ super().__init__()
402
+ self.ln_h, self.ln_e = torch.nn.LayerNorm(d), torch.nn.LayerNorm(d)
403
+ self.proj = torch.nn.Linear(2 * d, d, bias=False)
404
+ self.block = PeriLNBlock(d, h)
405
+
406
+
407
+ class Aux(torch.nn.Module):
408
+ """What exists only during training: the MTP module (pretraining) and the plan probe
409
+ (pretraining and RL). Never registered on the model, so the harness's parameter count and
410
+ sampler never see it. pretrain() leaves it in AUX for rl()."""
411
+
412
+ def __init__(self, d, h, vocab):
413
+ super().__init__()
414
+ self.mtp = MTPHead(d, h)
415
+ self.ln_probe, self.probe = torch.nn.LayerNorm(d), torch.nn.Linear(d, vocab)
416
+
417
+
418
+ AUX = None
419
+
420
+
421
+ def next_move(x, moves):
422
+ """For each position of the token rows x (B, S): the token after the next `:=` or `exact`
423
+ strictly to its right (-100 if there is none). At a trigger this is the coming line's move."""
424
+ B, S = x.shape
425
+ at = torch.where(torch.isin(x, moves), torch.arange(S, device=x.device)[None], S)
426
+ at = at.flip(1).cummin(1).values.flip(1) # the nearest one at or after each position
427
+ label_at = torch.cat((at[:, 1:], at.new_full((B, 1), S)), 1) + 1
428
+ label = x.gather(1, label_at.clamp(max=S - 1))
429
+ return label.masked_fill(label_at >= S, -100)
430
+
431
+
432
+ def train_loss(model, aux, x, m, tok, mtp=True, thoughts=TRAIN_THOUGHTS):
433
+ """Loss of one training batch at a thought count drawn for it from `thoughts`: the fork's
434
+ loss_on (next-token CE on proof tokens); plus, in pretraining, MTP_WEIGHT times the CE of the
435
+ MTP module's token-(t+2) prediction at the same positions; plus PROBE_WEIGHT times the plan
436
+ probe's CE at the triggers when the batch has thoughts. Returns (total, main, mtp, probe)."""
437
+ n = model.t_rng.choice(thoughts)
438
+ h = model.run(x[:, :-1], n)
439
+ logits = model.head(model.ln_f(h))
440
+ V = logits.size(-1)
441
+ ce = torch.nn.functional.cross_entropy(
442
+ logits.reshape(-1, V).float(), x[:, 1:].reshape(-1), reduction="none"
443
+ )
444
+ main = (ce * m[:, 1:].reshape(-1)).sum() / m[:, 1:].sum()
445
+ total, aux_ce, probe = main, main.new_zeros(()), main.new_zeros(())
446
+ if mtp:
447
+ # Position t (0..T-3) gets h_t and emb(x_{t+1}); causal attention over these keeps every
448
+ # prediction of x_{t+2} a function of x_{<=t+1}.
449
+ head = aux.mtp
450
+ z = head.proj(torch.cat((head.ln_h(h[:, :-1]), head.ln_e(model.emb(x[:, 1:-1]))), -1))
451
+ ar = torch.arange(z.size(1), device=z.device)[None]
452
+ z = head.block(z, None, None, model.bias(ar, ar, ar[:, None, :] <= ar[:, :, None]), None)
453
+ ce2 = torch.nn.functional.cross_entropy(
454
+ model.head(model.ln_f(z)).reshape(-1, V).float(), x[:, 2:].reshape(-1), reduction="none"
455
+ )
456
+ aux_ce = (ce2 * m[:, 2:].reshape(-1)).sum() / m[:, 2:].sum().clamp(min=1)
457
+ total = total + MTP_WEIGHT * aux_ce
458
+ if n:
459
+ moves = torch.tensor([tok.stoi[":="], tok.stoi["exact"]], device=x.device)
460
+ label = next_move(x, moves)[:, :-1]
461
+ at = torch.isin(x[:, :-1], model.triggers) & (label >= 0)
462
+ if bool(at.any()):
463
+ probe = torch.nn.functional.cross_entropy(
464
+ aux.probe(aux.ln_probe(h[at])).float(), label[at]
465
+ )
466
+ total = total + PROBE_WEIGHT * probe
467
+ return total, main, aux_ce, probe
468
+
469
+
470
+ def val_loss(model, ids, mask, ref_max, lens, tok, gen, autocast):
471
+ """Mean loss over the first VAL_N records (never trained on), no shift augmentation."""
472
+ model.eval()
473
+ tot = 0.0
474
+ with torch.no_grad(), autocast:
475
+ for i in range(0, VAL_N, BS):
476
+ idxs = torch.arange(i, min(i + BS, VAL_N), device=ids.device)
477
+ x, m = ids[idxs], mask[idxs]
478
+ T = int(lens[idxs].max())
479
+ tot += loss_on(model, x[:, :T], m[:, :T]).item() * len(idxs)
480
+ model.train()
481
+ return tot / VAL_N
482
+
483
+
484
+ def to_gpu(data, tok, device):
485
+ """Pad all (prompt, proof) records once into (N, T) ids + proof mask on the device."""
486
+ N, T = len(data), max(len(p) + len(q) for p, q in data)
487
+ ids = torch.full((N, T), tok.pad, dtype=torch.long)
488
+ mask = torch.zeros((N, T), dtype=torch.bool)
489
+ ref_max = torch.zeros(
490
+ N, dtype=torch.long
491
+ ) # largest N<k> index used in the proof, for the shift augmentation
492
+ for i, (p, q) in enumerate(data):
493
+ ids[i, : len(p) + len(q)] = torch.tensor(p + q)
494
+ mask[i, len(p) : len(p) + len(q)] = True
495
+ ref_max[i] = max((x - tok.ref0 + 1 for x in q if x >= tok.ref0), default=0)
496
+ return ids.to(device), mask.to(device), ref_max.to(device), (ids != tok.pad).sum(1).to(device)
497
+
498
+
499
+ def token_batches(lens, n_train, gen, rng):
500
+ """One epoch of token-budget batches: a random permutation of the training records, stably
501
+ sorted by length (equal lengths stay shuffled), cut greedily into runs of consecutive records
502
+ with count * max_len <= TOK_BUDGET, and the runs' order shuffled. Returns (order, cuts)."""
503
+ perm = torch.randperm(n_train, device=lens.device, generator=gen) + VAL_N
504
+ order = perm[torch.argsort(lens[perm], stable=True)]
505
+ sorted_lens = lens[order].cpu()
506
+ cuts, s = [], 0
507
+ while s < len(sorted_lens):
508
+ window = sorted_lens[s : s + TOK_BUDGET // int(sorted_lens[s])]
509
+ # count * max_len is non-decreasing along the window, so the fits form a prefix.
510
+ e = s + int((torch.arange(1, len(window) + 1) * window <= TOK_BUDGET).sum())
511
+ cuts.append((s, e, int(sorted_lens[e - 1]))) # sorted: last is the batch's max length
512
+ s = e
513
+ rng.shuffle(cuts)
514
+ return order, cuts
515
+
516
+
517
+ def gpu_batch(ids, mask, ref_max, lens, idxs, tok, gen, max_len=None):
518
+ """The fork's batch() + tok.shift_abs, vectorised: a random per-row offset is added to
519
+ every ref token."""
520
+ x, m = ids[idxs], mask[idxs]
521
+ # Known on the host for training batches: passing it avoids a GPU sync every step.
522
+ T = int(lens[idxs].max()) if max_len is None else max_len
523
+ x, m = x[:, :T].clone(), m[:, :T]
524
+ if tok.mode in ("abs", "lean_seq"): # lean_seq: names n1.. get the same start offset
525
+ room = MAXN - ref_max[idxs]
526
+ shift = (
527
+ (torch.rand(len(idxs), device=x.device, generator=gen) * (room + 1))
528
+ .long()
529
+ .clamp(max=room)
530
+ )
531
+ is_ref = (x >= tok.ref0) & m
532
+ x = torch.where(is_ref, x + shift[:, None], x)
533
+ return x, m
534
+
535
+
536
+ def pretrain(train_recs, budget_secs, device, seed, log):
537
+ global AUX
538
+ t0 = time.time()
539
+ tok = make_tokenizer(TOK_MODE)
540
+ data = [(tok.encode_prompt(r["prompt"]), tok.encode_proof(r["proof"])) for r in train_recs]
541
+ ids, mask, ref_max, lens = to_gpu(data, tok, device)
542
+ gen = torch.Generator(device=device)
543
+ gen.manual_seed(seed)
544
+ triggers = (tok.stoi[";"], tok.stoi["by"])
545
+ model = LineThoughtGPT(tok.vocab_size, N_LAYER, D, N_HEAD, triggers, seed)
546
+ model.blocks = torch.nn.ModuleList(PeriLNBlock(D, N_HEAD) for _ in range(N_LAYER))
547
+ model.blocks.apply(model._init) # fork init: zero biases
548
+ model = model.to(device)
549
+ AUX = aux = Aux(D, N_HEAD, tok.vocab_size)
550
+ aux.apply(model._init)
551
+ aux = aux.to(device)
552
+ mtp = aux.mtp
553
+ # No torch.compile: the expanded length changes every batch and the blocks hit the recompile
554
+ # limit (measured on the 2026-10-06 Coconut spike: 10.9 steps/s compiled, 10.1 eager).
555
+ muon_lr, muon_graph = torch.zeros((), device=device), None
556
+ for p in list(model.blocks.parameters()) + list(mtp.parameters()):
557
+ if p.ndim == 2:
558
+ # Orthogonal matrices have element second moment gain^2 / max(rows, cols).
559
+ torch.nn.init.orthogonal_(p, gain=0.02 * math.sqrt(max(p.shape)))
560
+ log(
561
+ f"pretrain: {model.n_params():,} params, {len(data):,} records on {device} "
562
+ f"after {time.time() - t0:.0f}s, budget {budget_secs}s"
563
+ )
564
+ # The output readout joins the hidden matrices under Muon; embeddings stay on AdamW.
565
+ trainable = list(model.parameters()) + list(aux.parameters())
566
+ matrices = [p for p in model.blocks.parameters() if p.ndim == 2] + [model.head.weight]
567
+ matrices += [p for p in mtp.parameters() if p.ndim == 2]
568
+ matrix_ids = {id(p) for p in matrices}
569
+ auxiliary = [p for p in trainable if id(p) not in matrix_ids]
570
+ momentum_buffers = [torch.zeros_like(p) for p in matrices]
571
+ opt = torch.optim.AdamW(
572
+ auxiliary, lr=LR, weight_decay=WD, betas=(0.9, 0.95), fused=device == "cuda"
573
+ )
574
+ autocast = torch.autocast("cuda", dtype=torch.bfloat16, enabled=(device == "cuda"))
575
+ model.train()
576
+ step, last_loss, records_seen = 0, float("nan"), 0
577
+ curve, recent, recent_aux, recent_probe, next_point = [], [], [], [], CURVE_EVERY
578
+ n_train = len(data) - VAL_N # records [0, VAL_N) are the val slice
579
+ rng, cuts, batches_per_epoch = random.Random(seed), [], None
580
+ order = torch.empty(0, dtype=torch.long, device=device)
581
+ while True:
582
+ frac = (
583
+ time.time() - t0
584
+ ) / budget_secs # cosine over wall-clock, so the cap is the schedule
585
+ if frac >= 1.0:
586
+ break
587
+ step += 1
588
+ # No warmup (as in modded-nanogpt): Muon's step is normalized and Adam's is ~lr per
589
+ # element from step 1, and Peri-LN normalizes every branch write.
590
+ lr = LR * MIN_LR_FRAC + 0.5 * LR * (1 - MIN_LR_FRAC) * (1 + math.cos(math.pi * frac))
591
+ for g in opt.param_groups:
592
+ g["lr"] = lr
593
+ if not cuts: # new epoch
594
+ order, cuts = token_batches(lens, n_train, gen, rng)
595
+ batches_per_epoch = len(cuts)
596
+ s, e, T = cuts.pop()
597
+ idxs = order[s:e]
598
+ records_seen += e - s
599
+ x, m = gpu_batch(ids, mask, ref_max, lens, idxs, tok, gen, T)
600
+ with autocast:
601
+ loss, main, mtp_ce, probe = train_loss(model, aux, x, m, tok)
602
+ model.zero_grad(set_to_none=False) # grads keep fixed addresses for the Muon CUDA graph
603
+ aux.zero_grad(set_to_none=False)
604
+ loss.backward()
605
+ torch.nn.utils.clip_grad_norm_(trainable, 1.0)
606
+ opt.step()
607
+ muon_lr.fill_(MUON_LR * lr / LR)
608
+ # Capture once; later steps replay one graph instead of ~400 kernel launches. CPU smoke
609
+ # runs call the step directly.
610
+ if device != "cuda":
611
+ muon_step(matrices, momentum_buffers, muon_lr)
612
+ elif muon_graph is None:
613
+ state = matrices + momentum_buffers
614
+ saved = [t.clone() for t in state]
615
+ side = torch.cuda.Stream()
616
+ side.wait_stream(torch.cuda.current_stream())
617
+ with torch.cuda.stream(side):
618
+ muon_step(matrices, momentum_buffers, muon_lr) # warm-up (lazy cuBLAS init)
619
+ torch.cuda.current_stream().wait_stream(side)
620
+ muon_graph = torch.cuda.CUDAGraph()
621
+ with torch.cuda.graph(muon_graph):
622
+ muon_step(matrices, momentum_buffers, muon_lr)
623
+ with torch.no_grad():
624
+ for t, v in zip(state, saved, strict=True):
625
+ t.copy_(v) # undo the warm-up and capture updates
626
+ muon_graph.replay()
627
+ else:
628
+ muon_graph.replay()
629
+ if step % 20 == 0:
630
+ last_loss = main.item() # the next-token CE, comparable with earlier runs
631
+ recent.append(last_loss)
632
+ recent_aux.append(mtp_ce.item())
633
+ recent_probe.append(probe.item())
634
+ if (
635
+ time.time() - t0 >= next_point
636
+ ): # curve point: mean recent train loss + val loss on the held-out slice
637
+ curve.append(
638
+ {
639
+ "secs": round(time.time() - t0),
640
+ "step": step,
641
+ "train_loss": sum(recent) / len(recent) if recent else last_loss,
642
+ "val_loss": val_loss(model, ids, mask, ref_max, lens, tok, gen, autocast),
643
+ "mtp_loss": sum(recent_aux) / len(recent_aux) if recent_aux else None,
644
+ # batches drawn with zero thoughts count as 0 here
645
+ "probe_loss": sum(recent_probe) / len(recent_probe) if recent_probe else None,
646
+ }
647
+ )
648
+ log(
649
+ f"pretrain step {step} train {curve[-1]['train_loss']:.4f} "
650
+ f"val {curve[-1]['val_loss']:.4f} lr {lr:.2e} {curve[-1]['secs']}s"
651
+ )
652
+ recent, recent_aux, recent_probe = [], [], []
653
+ next_point += CURVE_EVERY
654
+ val_by_thoughts = {}
655
+ for n in VAL_THOUGHTS if step else (): # no steps: a checkpoint rebuild, not a training run
656
+ model.n_thoughts = n
657
+ val_by_thoughts[n] = val_loss(model, ids, mask, ref_max, lens, tok, gen, autocast)
658
+ model.n_thoughts = DEFAULT_THOUGHTS
659
+ log(f"pretrain val loss by thoughts: {val_by_thoughts}")
660
+ model.eval()
661
+ return (
662
+ model,
663
+ tok,
664
+ {
665
+ "steps": step,
666
+ "records_seen": records_seen,
667
+ "epochs": records_seen / n_train,
668
+ "batches_per_epoch": batches_per_epoch,
669
+ "tok_budget": TOK_BUDGET,
670
+ "last_loss": last_loss,
671
+ "train_thoughts": TRAIN_THOUGHTS,
672
+ "default_thoughts": DEFAULT_THOUGHTS,
673
+ "val_by_thoughts": val_by_thoughts,
674
+ "feedback_gain": model.ln_thought.weight.abs().mean().item(),
675
+ "curve": curve,
676
+ },
677
+ )
678
+
679
+
680
+ def finetune(model, tok, data, rng, device):
681
+ """The old harness's RL fine-tune (FT_STEPS AdamW steps, linear warmup then cosine to lr/10)
682
+ with the plan probe's loss kept on. Every micro-batch runs at a thought count drawn from
683
+ FT_THOUGHTS: the eval level only. The probe trains along; the MTP module is not used."""
684
+ params = (
685
+ list(model.parameters()) + list(AUX.ln_probe.parameters()) + list(AUX.probe.parameters())
686
+ )
687
+ opt = torch.optim.AdamW(params, lr=FT_LR, weight_decay=0.1, betas=(0.9, 0.95))
688
+ autocast = torch.autocast("cuda", dtype=torch.bfloat16, enabled=(device == "cuda"))
689
+ model.train()
690
+ perm = []
691
+ for step in range(1, FT_STEPS + 1):
692
+ lr = (
693
+ FT_LR * step / FT_WARMUP
694
+ if step < FT_WARMUP
695
+ else FT_LR / 10 + 0.45 * FT_LR * (1 + math.cos(math.pi * step / FT_STEPS))
696
+ )
697
+ for g in opt.param_groups:
698
+ g["lr"] = lr
699
+ if len(perm) < FT_BS:
700
+ perm = list(range(len(data)))
701
+ rng.shuffle(perm)
702
+ idxs = sorted((perm.pop() for _ in range(FT_BS)), key=lambda i: sum(map(len, data[i])))
703
+ opt.zero_grad(set_to_none=True)
704
+ for j in range(0, FT_BS, FT_MICRO):
705
+ x, m = mk_batch(data, idxs[j : j + FT_MICRO], tok, rng, device)
706
+ with autocast:
707
+ loss, main, _, _ = train_loss(
708
+ model, AUX, x, m, tok, mtp=False, thoughts=FT_THOUGHTS
709
+ )
710
+ (loss * FT_MICRO / FT_BS).backward()
711
+ torch.nn.utils.clip_grad_norm_(params, 1.0)
712
+ opt.step()
713
+ model.eval()
714
+ return main.item()
715
+
716
+
717
+ def rl(model, tok, targets, train_recs, oracle, budget_secs, device, seed, log):
718
+ rng = random.Random(seed)
719
+ found = {t["name"]: {} for t in targets} # name -> {normalized proof: proof text}
720
+ rounds, round_secs = 0, 0.0
721
+ # Start a round only if one as long as the last (plus 15%) still fits in the budget.
722
+ while oracle.left() > 1.15 * round_secs:
723
+ t0 = oracle.used()
724
+ for n in RL_SAMPLE_THOUGHTS:
725
+ model.n_thoughts = n
726
+ for row in oracle.sample(model, targets, RL_K):
727
+ for proof in row["proofs"]:
728
+ found[row["name"]].setdefault(norm(proof), proof)
729
+ mix = []
730
+ for t in targets:
731
+ proofs = list(found[t["name"]].values())
732
+ rng.shuffle(proofs)
733
+ mix += [(t["prompt"], proof) for proof in proofs[:MAX_PER_THM]] * RL_WEIGHT
734
+ n_rl = len(mix)
735
+ if not n_rl:
736
+ break
737
+ mix += [(r["prompt"], r["proof"]) for r in rng.sample(train_recs, RETAIN)]
738
+ data = [(tok.encode_prompt(p), tok.encode_proof(q)) for p, q in mix]
739
+ loss = finetune(model, tok, data, rng, device)
740
+ rounds += 1
741
+ round_secs = oracle.used() - t0
742
+ log(
743
+ f"rl round {rounds}: {n_rl} RL records, fine-tune loss {loss:.4f}, "
744
+ f"round {round_secs:.0f}s, {oracle.left():.0f}s left"
745
+ )
746
+ model.n_thoughts = DEFAULT_THOUGHTS
747
+ return {
748
+ "rounds": rounds,
749
+ "sample_thoughts": RL_SAMPLE_THOUGHTS,
750
+ "k_per_count": RL_K,
751
+ "ft_thoughts": FT_THOUGHTS,
752
+ "feedback_gain": model.ln_thought.weight.abs().mean().item(),
753
+ }
runs/707-fable_plain-state-feedback_s1/source/remote.sh ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env bash
2
+ # Runs on the pod. Checks out Dan's fork at the pinned commit (frozen data + verifier + Lean
3
+ # tokenizer/gate), installs Lean, runs the harness.
4
+ # Usage: bash remote.sh <out-dir> [harness.py args...]
5
+ set -euo pipefail
6
+ FORK_COMMIT=51604b3feaa5e66821d1faba22238fe6d96d46d0 # dan-pandori/nd-takehome, branch dan_lean_seed2 (lean_seq with the BOTE fix)
7
+ export ND_FORK=/workspace/nd-takehome
8
+ OUT=$1; shift
9
+
10
+ if [ ! -d "$ND_FORK/.git" ]; then
11
+ git clone -q https://github.com/dan-pandori/nd-takehome.git "$ND_FORK"
12
+ fi
13
+ git -C "$ND_FORK" cat-file -e "$FORK_COMMIT" 2>/dev/null || git -C "$ND_FORK" fetch -q origin
14
+ git -C "$ND_FORK" checkout -q "$FORK_COMMIT"
15
+ [ -f "$ND_FORK/data/train.jsonl" ] || gunzip -k "$ND_FORK/data/train.jsonl.gz"
16
+ # Lean 4 core checks every sampled proof (fork lean_gate.py, called from sample.generate): the
17
+ # reward is Lean AND nd_verify. Installed before harness.py starts, so it is not charged to TOTAL_SECS.
18
+ [ -x ~/.elan/bin/lean ] || curl -sSf https://raw.githubusercontent.com/leanprover/elan/master/elan-init.sh \
19
+ | sh -s -- -y --default-toolchain leanprover/lean4:v4.34.0 > /dev/null
20
+ # One Lean process per CPU: on these pods (nproc 96) 12 workers took 146 s per RL-round gate
21
+ # pass (099) against 67-84 s with 96 workers (098).
22
+ export LEAN_GATE_WORKERS=$(nproc) LEAN_GATE_LOG="$OUT/lean_gate.jsonl"
23
+ nvidia-smi --query-gpu=name,memory.total --format=csv,noheader
24
+ echo "cpus $(nproc), $(~/.elan/bin/lean --version)"
25
+ # Newer RunPod images put torch under `python` and leave `python3` as bare system Python.
26
+ PY=$(for p in python python3; do "$p" -c "import torch" 2>/dev/null && echo "$p" && break; done)
27
+ echo "python $($PY --version 2>&1), torch $($PY -c 'import torch; print(torch.__version__)')"
28
+ cd /workspace/autoresearch
29
+ $PY harness.py --out "$OUT" "$@" 2>&1 | tee "$OUT.log"
runs/707-fable_plain-state-feedback_s1/summary.json ADDED
@@ -0,0 +1,560 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "status": "ok",
3
+ "metric": 652,
4
+ "metric_diag": 625,
5
+ "metric_rl": 1118,
6
+ "harness_sha": "3149b2713a6d",
7
+ "pretrain_sha": "89380bc5cc43",
8
+ "device": "cuda",
9
+ "gpu": "NVIDIA A40",
10
+ "torch": "2.9.1+cu129",
11
+ "lean_gate": {
12
+ "samples": 337824,
13
+ "parse_fail": 59008,
14
+ "distinct_checked": 231179,
15
+ "both_ok": 114696,
16
+ "nd_ok_lean_rej": 0,
17
+ "nd_rej_lean_ok": 1,
18
+ "lean_wall_s": 614.1663556098938
19
+ },
20
+ "total_secs": 3265.8480644226074,
21
+ "rules": {
22
+ "TOTAL_SECS": 5400,
23
+ "PRETRAIN_SECS": 300,
24
+ "RL_SECS": 1800,
25
+ "PARAM_CAP": 10000000,
26
+ "N_TARGETS": 1500,
27
+ "EVAL_K": 64,
28
+ "KNOB": "n_thoughts",
29
+ "EVAL_LEVEL": 8,
30
+ "DIAG_LEVEL": 0,
31
+ "TOK_MODE": "lean_seq"
32
+ },
33
+ "pretrain": {
34
+ "steps": 2937,
35
+ "records_seen": 399036,
36
+ "epochs": 2.591715053972955,
37
+ "batches_per_epoch": 1132,
38
+ "tok_budget": 18432,
39
+ "last_loss": 0.04747038334608078,
40
+ "train_thoughts": [
41
+ 0,
42
+ 1,
43
+ 2,
44
+ 3,
45
+ 4,
46
+ 5,
47
+ 6,
48
+ 7,
49
+ 8
50
+ ],
51
+ "default_thoughts": 8,
52
+ "val_by_thoughts": {
53
+ "0": 0.08232343010604382,
54
+ "2": 0.08256996516138315,
55
+ "8": 0.0831614462658763
56
+ },
57
+ "feedback_gain": 0.0411137193441391,
58
+ "curve": [
59
+ {
60
+ "secs": 16,
61
+ "step": 1,
62
+ "train_loss": NaN,
63
+ "val_loss": 3.863105356693268,
64
+ "mtp_loss": null,
65
+ "probe_loss": null
66
+ },
67
+ {
68
+ "secs": 30,
69
+ "step": 149,
70
+ "train_loss": 0.6968186944723129,
71
+ "val_loss": 0.13944070227444172,
72
+ "mtp_loss": 0.6392248507056918,
73
+ "probe_loss": 1.54788772548948
74
+ },
75
+ {
76
+ "secs": 45,
77
+ "step": 304,
78
+ "train_loss": 0.15057043731212616,
79
+ "val_loss": 0.10228868667036295,
80
+ "mtp_loss": 0.13802653551101685,
81
+ "probe_loss": 0.5565244071185589
82
+ },
83
+ {
84
+ "secs": 60,
85
+ "step": 460,
86
+ "train_loss": 0.1029130956158042,
87
+ "val_loss": 0.08322210237383842,
88
+ "mtp_loss": 0.09547224920243025,
89
+ "probe_loss": 0.2469321023672819
90
+ },
91
+ {
92
+ "secs": 75,
93
+ "step": 613,
94
+ "train_loss": 0.08814916972603117,
95
+ "val_loss": 0.08903966471552849,
96
+ "mtp_loss": 0.0796054218496595,
97
+ "probe_loss": 0.12642302364110947
98
+ },
99
+ {
100
+ "secs": 90,
101
+ "step": 769,
102
+ "train_loss": 0.08526475913822651,
103
+ "val_loss": 0.08802968822419643,
104
+ "mtp_loss": 0.08351483149453998,
105
+ "probe_loss": 0.06919845147058368
106
+ },
107
+ {
108
+ "secs": 105,
109
+ "step": 923,
110
+ "train_loss": 0.07035604398697615,
111
+ "val_loss": 0.08986930549144745,
112
+ "mtp_loss": 0.06848942209035158,
113
+ "probe_loss": 0.09544991422444582
114
+ },
115
+ {
116
+ "secs": 120,
117
+ "step": 1076,
118
+ "train_loss": 0.07671544966953141,
119
+ "val_loss": 0.09829375520348549,
120
+ "mtp_loss": 0.07479695337159294,
121
+ "probe_loss": 0.09491446215127196
122
+ },
123
+ {
124
+ "secs": 135,
125
+ "step": 1229,
126
+ "train_loss": 0.06703539425507188,
127
+ "val_loss": 0.07310204021632671,
128
+ "mtp_loss": 0.06596812512725592,
129
+ "probe_loss": 0.06717315129935741
130
+ },
131
+ {
132
+ "secs": 150,
133
+ "step": 1384,
134
+ "train_loss": 0.07287110295146704,
135
+ "val_loss": 0.09030481986701488,
136
+ "mtp_loss": 0.07374804373830557,
137
+ "probe_loss": 0.09021524735726416
138
+ },
139
+ {
140
+ "secs": 165,
141
+ "step": 1539,
142
+ "train_loss": 0.06482018317495074,
143
+ "val_loss": 0.08763973508030176,
144
+ "mtp_loss": 0.06294059327670506,
145
+ "probe_loss": 0.049281416194779534
146
+ },
147
+ {
148
+ "secs": 180,
149
+ "step": 1694,
150
+ "train_loss": 0.055237766820937395,
151
+ "val_loss": 0.08533050958067179,
152
+ "mtp_loss": 0.05451565282419324,
153
+ "probe_loss": 0.045460834400728345
154
+ },
155
+ {
156
+ "secs": 195,
157
+ "step": 1850,
158
+ "train_loss": 0.070999123621732,
159
+ "val_loss": 0.0853136433288455,
160
+ "mtp_loss": 0.07099380763247609,
161
+ "probe_loss": 0.06430730456486344
162
+ },
163
+ {
164
+ "secs": 210,
165
+ "step": 2008,
166
+ "train_loss": 0.06542239803820848,
167
+ "val_loss": 0.0789189962670207,
168
+ "mtp_loss": 0.06400936562567949,
169
+ "probe_loss": 0.04609072208404541
170
+ },
171
+ {
172
+ "secs": 225,
173
+ "step": 2163,
174
+ "train_loss": 0.05195397837087512,
175
+ "val_loss": 0.08157234825193882,
176
+ "mtp_loss": 0.05218019196763635,
177
+ "probe_loss": 0.033863209653645754
178
+ },
179
+ {
180
+ "secs": 240,
181
+ "step": 2315,
182
+ "train_loss": 0.060149705835751126,
183
+ "val_loss": 0.08262713253498077,
184
+ "mtp_loss": 0.05940220132470131,
185
+ "probe_loss": 0.06776506773063115
186
+ },
187
+ {
188
+ "secs": 255,
189
+ "step": 2467,
190
+ "train_loss": 0.07321797590702772,
191
+ "val_loss": 0.08798998687416315,
192
+ "mtp_loss": 0.07351318001747131,
193
+ "probe_loss": 0.0736613639164716
194
+ },
195
+ {
196
+ "secs": 270,
197
+ "step": 2622,
198
+ "train_loss": 0.0477787924464792,
199
+ "val_loss": 0.08348270319402218,
200
+ "mtp_loss": 0.047774621518328786,
201
+ "probe_loss": 0.046014094492420554
202
+ },
203
+ {
204
+ "secs": 285,
205
+ "step": 2780,
206
+ "train_loss": 0.04634411772713065,
207
+ "val_loss": 0.08711015898734331,
208
+ "mtp_loss": 0.04552949406206608,
209
+ "probe_loss": 0.03193037479650229
210
+ },
211
+ {
212
+ "secs": 300,
213
+ "step": 2937,
214
+ "train_loss": 0.06081975144999368,
215
+ "val_loss": 0.0831614462658763,
216
+ "mtp_loss": 0.060614355972834995,
217
+ "probe_loss": 0.04569009664867606
218
+ }
219
+ ],
220
+ "secs": 301.5383403301239,
221
+ "n_params": 9564672,
222
+ "heldout_greedy": 0.958
223
+ },
224
+ "rl": {
225
+ "rounds": [
226
+ {
227
+ "round": 1,
228
+ "level": 8,
229
+ "k": 32,
230
+ "n_targets": 1500,
231
+ "sample_acc": 0.059666666666666666,
232
+ "new_proofs": 467,
233
+ "targets_solved": 363,
234
+ "lstar": 9,
235
+ "ge": {
236
+ "7": 363,
237
+ "8": 73,
238
+ "9": 16,
239
+ "10": 0,
240
+ "11": 0,
241
+ "12": 0,
242
+ "13": 0,
243
+ "14": 0
244
+ },
245
+ "mix_rl_records": 467,
246
+ "sampling": {
247
+ "samples": 48000,
248
+ "secs": 402.4999113082886,
249
+ "gate_secs": 57.17697715759277,
250
+ "samples_per_sec": 119.25468466311075
251
+ },
252
+ "secs": 403.82336139678955,
253
+ "rl_secs_used": 346.64666295051575
254
+ },
255
+ {
256
+ "round": 2,
257
+ "level": 8,
258
+ "k": 32,
259
+ "n_targets": 1500,
260
+ "sample_acc": 0.4558333333333333,
261
+ "new_proofs": 922,
262
+ "targets_solved": 923,
263
+ "lstar": 10,
264
+ "ge": {
265
+ "7": 923,
266
+ "8": 422,
267
+ "9": 178,
268
+ "10": 25,
269
+ "11": 1,
270
+ "12": 0,
271
+ "13": 0,
272
+ "14": 0
273
+ },
274
+ "mix_rl_records": 1379,
275
+ "sampling": {
276
+ "samples": 48000,
277
+ "secs": 348.1030418872833,
278
+ "gate_secs": 80.06368231773376,
279
+ "samples_per_sec": 137.89020555454533
280
+ },
281
+ "secs": 462.8676965236664,
282
+ "rl_secs_used": 729.4506826400757
283
+ },
284
+ {
285
+ "round": 3,
286
+ "level": 8,
287
+ "k": 32,
288
+ "n_targets": 1500,
289
+ "sample_acc": 0.616,
290
+ "new_proofs": 461,
291
+ "targets_solved": 1050,
292
+ "lstar": 11,
293
+ "ge": {
294
+ "7": 1050,
295
+ "8": 517,
296
+ "9": 250,
297
+ "10": 53,
298
+ "11": 8,
299
+ "12": 0,
300
+ "13": 0,
301
+ "14": 0
302
+ },
303
+ "mix_rl_records": 1809,
304
+ "sampling": {
305
+ "samples": 48000,
306
+ "secs": 374.1362271308899,
307
+ "gate_secs": 87.69131851196289,
308
+ "samples_per_sec": 128.29551516059794
309
+ },
310
+ "secs": 492.79634523391724,
311
+ "rl_secs_used": 1134.555986404419
312
+ },
313
+ {
314
+ "round": 4,
315
+ "level": 8,
316
+ "k": 32,
317
+ "n_targets": 1500,
318
+ "sample_acc": 0.6789375,
319
+ "new_proofs": 428,
320
+ "targets_solved": 1118,
321
+ "lstar": 11,
322
+ "ge": {
323
+ "7": 1118,
324
+ "8": 573,
325
+ "9": 299,
326
+ "10": 72,
327
+ "11": 12,
328
+ "12": 0,
329
+ "13": 0,
330
+ "14": 0
331
+ },
332
+ "mix_rl_records": 2186,
333
+ "sampling": {
334
+ "samples": 48000,
335
+ "secs": 349.1784243583679,
336
+ "gate_secs": 88.27419686317444,
337
+ "samples_per_sec": 137.46553810763737
338
+ },
339
+ "secs": 473.68713760375977,
340
+ "rl_secs_used": 1519.968663930893
341
+ }
342
+ ],
343
+ "secs": 1956.7443397045135,
344
+ "rl_secs_used": 1643.5226032733917,
345
+ "gate_secs": 313.20617485046387,
346
+ "targets": 1500,
347
+ "stats": {
348
+ "rounds": 4,
349
+ "sample_thoughts": [
350
+ 8
351
+ ],
352
+ "k_per_count": 32,
353
+ "ft_thoughts": [
354
+ 8
355
+ ],
356
+ "feedback_gain": 0.04191834479570389
357
+ }
358
+ },
359
+ "eval": {
360
+ "level": 8,
361
+ "attempts_per_theorem": 64,
362
+ "solved": 652,
363
+ "n": 1108,
364
+ "solved_ge7": 652,
365
+ "lstar": 11,
366
+ "ge": {
367
+ "7": 652,
368
+ "8": 563,
369
+ "9": 456,
370
+ "10": 103,
371
+ "11": 23,
372
+ "12": 4,
373
+ "13": 0,
374
+ "14": 0
375
+ },
376
+ "by_bin": {
377
+ "7": {
378
+ "solved": 89,
379
+ "n": 150,
380
+ "rate": 0.5933333333333334,
381
+ "ci": [
382
+ 0.51334517440468,
383
+ 0.6686602129547861
384
+ ]
385
+ },
386
+ "8": {
387
+ "solved": 107,
388
+ "n": 152,
389
+ "rate": 0.7039473684210527,
390
+ "ci": [
391
+ 0.6270686253277898,
392
+ 0.7707712325792132
393
+ ]
394
+ },
395
+ "9": {
396
+ "solved": 353,
397
+ "n": 477,
398
+ "rate": 0.740041928721174,
399
+ "ci": [
400
+ 0.6988728722066776,
401
+ 0.7773754390875199
402
+ ]
403
+ },
404
+ "10": {
405
+ "solved": 80,
406
+ "n": 219,
407
+ "rate": 0.365296803652968,
408
+ "ci": [
409
+ 0.3043546327248604,
410
+ 0.4308833120753923
411
+ ]
412
+ },
413
+ "11": {
414
+ "solved": 19,
415
+ "n": 49,
416
+ "rate": 0.3877551020408163,
417
+ "ci": [
418
+ 0.26428814373960496,
419
+ 0.5275425351194757
420
+ ]
421
+ },
422
+ "12": {
423
+ "solved": 4,
424
+ "n": 54,
425
+ "rate": 0.07407407407407407,
426
+ "ci": [
427
+ 0.029181055323684174,
428
+ 0.17554357193420633
429
+ ]
430
+ },
431
+ "13": {
432
+ "solved": 0,
433
+ "n": 5,
434
+ "rate": 0.0,
435
+ "ci": [
436
+ 0.0,
437
+ 0.43449149475208104
438
+ ]
439
+ },
440
+ "14": {
441
+ "solved": 0,
442
+ "n": 2,
443
+ "rate": 0.0,
444
+ "ci": [
445
+ 0.0,
446
+ 0.6576280471103807
447
+ ]
448
+ }
449
+ },
450
+ "sample_acc": 0.4096767824909747,
451
+ "max_pruned": 13,
452
+ "sampling": {
453
+ "samples": 70912,
454
+ "secs": 577.9915318489075,
455
+ "gate_secs": 130.2077991962433,
456
+ "samples_per_sec": 122.68691856637284
457
+ },
458
+ "heldout_greedy": 0.9865,
459
+ "diag": {
460
+ "level": 0,
461
+ "attempts_per_theorem": 64,
462
+ "solved": 625,
463
+ "n": 1108,
464
+ "solved_ge7": 625,
465
+ "lstar": 11,
466
+ "ge": {
467
+ "7": 625,
468
+ "8": 535,
469
+ "9": 433,
470
+ "10": 92,
471
+ "11": 22,
472
+ "12": 3,
473
+ "13": 0,
474
+ "14": 0
475
+ },
476
+ "by_bin": {
477
+ "7": {
478
+ "solved": 90,
479
+ "n": 150,
480
+ "rate": 0.6,
481
+ "ci": [
482
+ 0.5200476798506317,
483
+ 0.6749580923202245
484
+ ]
485
+ },
486
+ "8": {
487
+ "solved": 102,
488
+ "n": 152,
489
+ "rate": 0.6710526315789473,
490
+ "ci": [
491
+ 0.5929497169743494,
492
+ 0.740722421915395
493
+ ]
494
+ },
495
+ "9": {
496
+ "solved": 341,
497
+ "n": 477,
498
+ "rate": 0.7148846960167715,
499
+ "ci": [
500
+ 0.672777713739318,
501
+ 0.7535581107816884
502
+ ]
503
+ },
504
+ "10": {
505
+ "solved": 70,
506
+ "n": 219,
507
+ "rate": 0.319634703196347,
508
+ "ci": [
509
+ 0.26143631644345877,
510
+ 0.38405177913654065
511
+ ]
512
+ },
513
+ "11": {
514
+ "solved": 19,
515
+ "n": 49,
516
+ "rate": 0.3877551020408163,
517
+ "ci": [
518
+ 0.26428814373960496,
519
+ 0.5275425351194757
520
+ ]
521
+ },
522
+ "12": {
523
+ "solved": 3,
524
+ "n": 54,
525
+ "rate": 0.05555555555555555,
526
+ "ci": [
527
+ 0.019072922503838907,
528
+ 0.15107451463482943
529
+ ]
530
+ },
531
+ "13": {
532
+ "solved": 0,
533
+ "n": 5,
534
+ "rate": 0.0,
535
+ "ci": [
536
+ 0.0,
537
+ 0.43449149475208104
538
+ ]
539
+ },
540
+ "14": {
541
+ "solved": 0,
542
+ "n": 2,
543
+ "rate": 0.0,
544
+ "ci": [
545
+ 0.0,
546
+ 0.6576280471103807
547
+ ]
548
+ }
549
+ },
550
+ "sample_acc": 0.3644235108303249,
551
+ "max_pruned": 13,
552
+ "sampling": {
553
+ "samples": 70912,
554
+ "secs": 399.5426995754242,
555
+ "gate_secs": 176.29513502120972,
556
+ "samples_per_sec": 177.48290752241238
557
+ }
558
+ }
559
+ }
560
+ }