Add long-rl-20261014-v2_untied-ei_s1
Browse files- .gitattributes +1 -0
- runs/long-rl-20261014-v2_untied-ei_s1/eval_dev.jsonl +0 -0
- runs/long-rl-20261014-v2_untied-ei_s1/eval_old_dev.jsonl +0 -0
- runs/long-rl-20261014-v2_untied-ei_s1/final.pt +3 -0
- runs/long-rl-20261014-v2_untied-ei_s1/found_targets.jsonl +3 -0
- runs/long-rl-20261014-v2_untied-ei_s1/lean_gate.jsonl +138 -0
- runs/long-rl-20261014-v2_untied-ei_s1/pretrain.pt +3 -0
- runs/long-rl-20261014-v2_untied-ei_s1/source/harness.py +435 -0
- runs/long-rl-20261014-v2_untied-ei_s1/source/lean_gate.py +82 -0
- runs/long-rl-20261014-v2_untied-ei_s1/source/lean_tok.py +319 -0
- runs/long-rl-20261014-v2_untied-ei_s1/source/pretrain.py +820 -0
- runs/long-rl-20261014-v2_untied-ei_s1/source/remote.sh +27 -0
- runs/long-rl-20261014-v2_untied-ei_s1/summary.json +0 -0
.gitattributes
CHANGED
|
@@ -40,3 +40,4 @@ runs/combo-c005_s0/found_targets.jsonl filter=lfs diff=lfs merge=lfs -text
|
|
| 40 |
runs/combo-c005_s1/found_targets.jsonl filter=lfs diff=lfs merge=lfs -text
|
| 41 |
runs/combo-c005_s2/found_targets.jsonl filter=lfs diff=lfs merge=lfs -text
|
| 42 |
runs/long-rl-20261014-v2_untied-ei_s0/found_targets.jsonl filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
| 40 |
runs/combo-c005_s1/found_targets.jsonl filter=lfs diff=lfs merge=lfs -text
|
| 41 |
runs/combo-c005_s2/found_targets.jsonl filter=lfs diff=lfs merge=lfs -text
|
| 42 |
runs/long-rl-20261014-v2_untied-ei_s0/found_targets.jsonl filter=lfs diff=lfs merge=lfs -text
|
| 43 |
+
runs/long-rl-20261014-v2_untied-ei_s1/found_targets.jsonl filter=lfs diff=lfs merge=lfs -text
|
runs/long-rl-20261014-v2_untied-ei_s1/eval_dev.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
runs/long-rl-20261014-v2_untied-ei_s1/eval_old_dev.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
runs/long-rl-20261014-v2_untied-ei_s1/final.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:cdfe421671d20c5fdb52de75dd82898b2c294f0078bd020106f5efdf764dec66
|
| 3 |
+
size 38588127
|
runs/long-rl-20261014-v2_untied-ei_s1/found_targets.jsonl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:bff843b6647e2f4427b1fd2458a13cd7def71b34769779dd69256572f6ae92d4
|
| 3 |
+
size 20908857
|
runs/long-rl-20261014-v2_untied-ei_s1/lean_gate.jsonl
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"utc": "2026-10-14T17:35:33Z", "samples": 12000, "parse_fail": 4140, "distinct_checked": 7786, "lean_ok": 1189, "lean_rej": 6597, "parse_reasons": {"unbound": 2580, "formula": 673, "name": 296, "expected :=": 134, "PR position": 102, "no-eos": 91, "expected )": 84, "expected ;": 82, "exact not last": 74, "expected (": 13, "trailing tokens": 4, "expected :": 3, "expected =>": 1, "expected False": 1, "expected exact": 1, "expected fun": 1}, "lean_wall_s": 24.277427673339844, "lean_proc_s": 452.0020716190338, "workers": 96, "chunk": 400}
|
| 2 |
+
{"utc": "2026-10-14T17:37:17Z", "samples": 10320, "parse_fail": 3577, "distinct_checked": 6544, "lean_ok": 12, "lean_rej": 6532, "parse_reasons": {"unbound": 2170, "formula": 565, "name": 301, "expected :=": 135, "PR position": 95, "expected )": 94, "expected ;": 80, "exact not last": 59, "no-eos": 57, "expected (": 11, "expected :": 5, "expected False": 2, "expected .elim": 1, "eof": 1, "expected exact": 1}, "lean_wall_s": 22.11999535560608, "lean_proc_s": 357.92804980278015, "workers": 96, "chunk": 400}
|
| 3 |
+
{"utc": "2026-10-14T17:39:06Z", "samples": 10320, "parse_fail": 4376, "distinct_checked": 5787, "lean_ok": 23, "lean_rej": 5764, "parse_reasons": {"unbound": 2706, "formula": 736, "name": 274, "no-eos": 161, "PR position": 120, "expected :=": 94, "expected ;": 92, "expected )": 85, "exact not last": 70, "expected (": 27, "expected :": 7, "eof": 2, "expected .elim": 1, "expected exact": 1}, "lean_wall_s": 21.427360773086548, "lean_proc_s": 308.0573797225952, "workers": 96, "chunk": 400}
|
| 4 |
+
{"utc": "2026-10-14T17:41:08Z", "samples": 10320, "parse_fail": 4065, "distinct_checked": 6120, "lean_ok": 31, "lean_rej": 6089, "parse_reasons": {"unbound": 2538, "formula": 731, "name": 288, "expected :=": 143, "PR position": 84, "expected )": 77, "no-eos": 72, "expected ;": 64, "exact not last": 54, "expected (": 9, "expected exact": 2, "expected fun": 1, "expected :": 1, "expected None": 1}, "lean_wall_s": 22.632764101028442, "lean_proc_s": 343.9482228755951, "workers": 96, "chunk": 400}
|
| 5 |
+
{"utc": "2026-10-14T17:43:53Z", "samples": 11992, "parse_fail": 3570, "distinct_checked": 8344, "lean_ok": 1912, "lean_rej": 6432, "parse_reasons": {"unbound": 2599, "expected (": 284, "formula": 204, "no-eos": 164, "name": 146, "PR position": 59, "exact not last": 47, "expected )": 34, "expected :=": 13, "trailing tokens": 8, "expected ;": 6, "expected .elim": 3, "expected None": 2, "eof": 1}, "lean_wall_s": 24.488558053970337, "lean_proc_s": 485.3694624900818, "workers": 96, "chunk": 400}
|
| 6 |
+
{"utc": "2026-10-14T17:45:30Z", "samples": 9312, "parse_fail": 3188, "distinct_checked": 5969, "lean_ok": 32, "lean_rej": 5937, "parse_reasons": {"unbound": 2297, "expected (": 276, "formula": 219, "no-eos": 176, "name": 108, "exact not last": 47, "expected )": 32, "expected ;": 11, "PR position": 10, "trailing tokens": 4, "expected :=": 3, "expected .elim": 2, "expected False": 1, "expected None": 1, "ORE box": 1}, "lean_wall_s": 19.815534830093384, "lean_proc_s": 293.2945351600647, "workers": 96, "chunk": 400}
|
| 7 |
+
{"utc": "2026-10-14T17:46:59Z", "samples": 9288, "parse_fail": 3298, "distinct_checked": 5841, "lean_ok": 29, "lean_rej": 5812, "parse_reasons": {"unbound": 2520, "expected (": 192, "formula": 172, "name": 150, "no-eos": 115, "exact not last": 47, "PR position": 44, "expected :=": 26, "expected )": 19, "expected ;": 7, "eof": 3, "expected by": 1, "trailing tokens": 1, "expected False": 1}, "lean_wall_s": 20.768762826919556, "lean_proc_s": 301.281254529953, "workers": 96, "chunk": 400}
|
| 8 |
+
{"utc": "2026-10-14T17:48:46Z", "samples": 9288, "parse_fail": 3383, "distinct_checked": 5786, "lean_ok": 31, "lean_rej": 5755, "parse_reasons": {"unbound": 2386, "expected (": 303, "formula": 233, "no-eos": 141, "PR position": 111, "name": 82, "exact not last": 51, "expected )": 43, "trailing tokens": 14, "expected ;": 10, "expected .elim": 3, "expected :=": 3, "expected None": 2, "ORE box": 1}, "lean_wall_s": 19.43605899810791, "lean_proc_s": 281.0176110267639, "workers": 96, "chunk": 400}
|
| 9 |
+
{"utc": "2026-10-14T17:51:12Z", "samples": 11992, "parse_fail": 3321, "distinct_checked": 8588, "lean_ok": 2783, "lean_rej": 5805, "parse_reasons": {"unbound": 2606, "no-eos": 178, "name": 137, "formula": 132, "exact not last": 65, "expected )": 56, "PR position": 52, "expected (": 45, "expected ;": 27, "expected :=": 7, "expected :": 5, "expected =>": 3, "trailing tokens": 2, "expected False": 2, "ORE box": 2, "expected None": 1, "eof": 1}, "lean_wall_s": 23.782185316085815, "lean_proc_s": 503.6529939174652, "workers": 96, "chunk": 400}
|
| 10 |
+
{"utc": "2026-10-14T17:52:39Z", "samples": 8232, "parse_fail": 3315, "distinct_checked": 4808, "lean_ok": 51, "lean_rej": 4757, "parse_reasons": {"unbound": 2574, "formula": 192, "no-eos": 178, "name": 156, "exact not last": 57, "PR position": 49, "expected )": 45, "expected (": 32, "expected ;": 22, "expected :=": 8, "expected exact": 1, "trailing tokens": 1}, "lean_wall_s": 16.81738519668579, "lean_proc_s": 199.7201166152954, "workers": 96, "chunk": 400}
|
| 11 |
+
{"utc": "2026-10-14T17:53:52Z", "samples": 8208, "parse_fail": 2814, "distinct_checked": 5264, "lean_ok": 22, "lean_rej": 5242, "parse_reasons": {"unbound": 2149, "no-eos": 208, "name": 128, "formula": 99, "PR position": 73, "exact not last": 43, "expected )": 41, "expected (": 40, "expected ;": 14, "expected :=": 13, "expected None": 3, "expected =>": 1, "expected \u27e9": 1, "expected by": 1}, "lean_wall_s": 16.650907516479492, "lean_proc_s": 217.5735433101654, "workers": 96, "chunk": 400}
|
| 12 |
+
{"utc": "2026-10-14T17:54:59Z", "samples": 8208, "parse_fail": 2744, "distinct_checked": 5330, "lean_ok": 49, "lean_rej": 5281, "parse_reasons": {"unbound": 2220, "name": 92, "no-eos": 91, "formula": 74, "exact not last": 68, "expected )": 56, "PR position": 55, "expected (": 42, "expected ;": 29, "expected :=": 8, "expected :": 5, "ORE box": 2, "expected False": 2}, "lean_wall_s": 14.037258386611938, "lean_proc_s": 185.21386337280273, "workers": 96, "chunk": 400}
|
| 13 |
+
{"utc": "2026-10-14T17:57:27Z", "samples": 12000, "parse_fail": 2870, "distinct_checked": 9024, "lean_ok": 4362, "lean_rej": 4662, "parse_reasons": {"unbound": 1954, "no-eos": 290, "expected (": 168, "formula": 152, "name": 87, "exact not last": 81, "PR position": 34, "expected )": 31, "expected ;": 28, "expected :": 22, "expected :=": 12, "ORE box": 5, "expected =>": 2, "expected None": 1, "trailing tokens": 1, "expected \u27e9": 1, "eof": 1}, "lean_wall_s": 18.170727729797363, "lean_proc_s": 406.62092995643616, "workers": 96, "chunk": 400}
|
| 14 |
+
{"utc": "2026-10-14T17:58:43Z", "samples": 6600, "parse_fail": 2646, "distinct_checked": 3868, "lean_ok": 20, "lean_rej": 3848, "parse_reasons": {"unbound": 1815, "no-eos": 231, "formula": 218, "expected (": 126, "name": 67, "exact not last": 58, "PR position": 46, "expected )": 29, "expected :": 16, "expected ;": 15, "ORE box": 11, "expected :=": 10, "expected =>": 3, "trailing tokens": 1}, "lean_wall_s": 7.838829040527344, "lean_proc_s": 75.58342289924622, "workers": 96, "chunk": 400}
|
| 15 |
+
{"utc": "2026-10-14T18:00:01Z", "samples": 6600, "parse_fail": 2599, "distinct_checked": 3932, "lean_ok": 32, "lean_rej": 3900, "parse_reasons": {"unbound": 1780, "no-eos": 252, "formula": 138, "expected (": 116, "exact not last": 100, "name": 63, "expected ;": 47, "expected :": 34, "expected )": 30, "PR position": 28, "expected :=": 4, "ORE box": 3, "expected None": 2, "trailing tokens": 1, "expected exact": 1}, "lean_wall_s": 8.007895708084106, "lean_proc_s": 78.73633074760437, "workers": 96, "chunk": 400}
|
| 16 |
+
{"utc": "2026-10-14T18:01:12Z", "samples": 6600, "parse_fail": 2475, "distinct_checked": 4043, "lean_ok": 29, "lean_rej": 4014, "parse_reasons": {"unbound": 1686, "no-eos": 244, "expected (": 165, "formula": 103, "name": 60, "exact not last": 60, "expected )": 57, "expected ;": 31, "expected :": 28, "expected :=": 20, "PR position": 13, "expected None": 3, "expected =>": 2, "ORE box": 2, "eof": 1}, "lean_wall_s": 7.977844476699829, "lean_proc_s": 80.40756607055664, "workers": 96, "chunk": 400}
|
| 17 |
+
{"utc": "2026-10-14T18:03:15Z", "samples": 11992, "parse_fail": 2322, "distinct_checked": 9552, "lean_ok": 5963, "lean_rej": 3589, "parse_reasons": {"unbound": 1587, "formula": 166, "name": 137, "expected (": 135, "no-eos": 111, "exact not last": 66, "expected )": 46, "expected ;": 28, "PR position": 28, "trailing tokens": 6, "expected :": 5, "ORE box": 2, "expected :=": 2, "expected None": 1, "expected =>": 1, "expected False": 1}, "lean_wall_s": 18.551472902297974, "lean_proc_s": 431.29912757873535, "workers": 96, "chunk": 400}
|
| 18 |
+
{"utc": "2026-10-14T18:04:04Z", "samples": 5136, "parse_fail": 2055, "distinct_checked": 3010, "lean_ok": 34, "lean_rej": 2976, "parse_reasons": {"unbound": 1437, "formula": 189, "expected (": 113, "name": 88, "no-eos": 71, "exact not last": 58, "expected )": 42, "expected ;": 24, "trailing tokens": 9, "PR position": 8, "ORE box": 7, "expected :": 6, "expected =>": 1, "expected :=": 1, "expected None": 1}, "lean_wall_s": 6.229447841644287, "lean_proc_s": 47.50319695472717, "workers": 96, "chunk": 400}
|
| 19 |
+
{"utc": "2026-10-14T18:04:53Z", "samples": 5112, "parse_fail": 2142, "distinct_checked": 2921, "lean_ok": 24, "lean_rej": 2897, "parse_reasons": {"unbound": 1457, "formula": 160, "expected (": 126, "no-eos": 115, "name": 100, "exact not last": 63, "expected )": 40, "expected ;": 39, "PR position": 31, "expected :=": 6, "trailing tokens": 2, "expected :": 2, "expected =>": 1}, "lean_wall_s": 5.7273523807525635, "lean_proc_s": 41.94000458717346, "workers": 96, "chunk": 400}
|
| 20 |
+
{"utc": "2026-10-14T18:05:38Z", "samples": 5112, "parse_fail": 2155, "distinct_checked": 2897, "lean_ok": 35, "lean_rej": 2862, "parse_reasons": {"unbound": 1500, "no-eos": 132, "expected (": 131, "formula": 124, "name": 102, "exact not last": 74, "expected )": 39, "PR position": 29, "expected ;": 17, "expected :": 4, "expected :=": 1, "trailing tokens": 1, "expected by": 1}, "lean_wall_s": 5.522989749908447, "lean_proc_s": 39.833672285079956, "workers": 96, "chunk": 400}
|
| 21 |
+
{"utc": "2026-10-14T18:08:03Z", "samples": 11992, "parse_fail": 2158, "distinct_checked": 9687, "lean_ok": 6611, "lean_rej": 3076, "parse_reasons": {"unbound": 1420, "no-eos": 196, "exact not last": 114, "name": 108, "formula": 94, "expected )": 71, "expected (": 70, "expected ;": 41, "PR position": 28, "expected :=": 7, "trailing tokens": 4, "ORE box": 3, "expected None": 1, "expected exact": 1}, "lean_wall_s": 24.799399614334106, "lean_proc_s": 591.6140356063843, "workers": 96, "chunk": 400}
|
| 22 |
+
{"utc": "2026-10-14T18:09:01Z", "samples": 4272, "parse_fail": 1915, "distinct_checked": 2309, "lean_ok": 12, "lean_rej": 2297, "parse_reasons": {"unbound": 1253, "no-eos": 218, "exact not last": 107, "expected (": 72, "expected )": 66, "formula": 65, "name": 64, "PR position": 30, "expected ;": 24, "expected :=": 11, "expected None": 3, "ORE box": 2}, "lean_wall_s": 8.865485906600952, "lean_proc_s": 51.93131899833679, "workers": 96, "chunk": 400}
|
| 23 |
+
{"utc": "2026-10-14T18:10:11Z", "samples": 4272, "parse_fail": 1801, "distinct_checked": 2418, "lean_ok": 17, "lean_rej": 2401, "parse_reasons": {"unbound": 1229, "no-eos": 148, "exact not last": 107, "name": 80, "expected )": 54, "expected (": 50, "formula": 47, "expected ;": 44, "PR position": 31, "ORE box": 6, "expected :=": 2, "expected exact": 1, "expected None": 1, "trailing tokens": 1}, "lean_wall_s": 9.594828367233276, "lean_proc_s": 56.99072480201721, "workers": 96, "chunk": 400}
|
| 24 |
+
{"utc": "2026-10-14T18:11:17Z", "samples": 4272, "parse_fail": 1942, "distinct_checked": 2278, "lean_ok": 29, "lean_rej": 2249, "parse_reasons": {"unbound": 1358, "no-eos": 139, "formula": 116, "exact not last": 91, "expected (": 78, "name": 68, "expected )": 58, "expected ;": 15, "PR position": 10, "expected False": 2, "expected :=": 2, "expected None": 2, "ORE box": 1, "expected =>": 1, "expected :": 1}, "lean_wall_s": 8.741991758346558, "lean_proc_s": 49.9824321269989, "workers": 96, "chunk": 400}
|
| 25 |
+
{"utc": "2026-10-14T18:13:38Z", "samples": 11992, "parse_fail": 2077, "distinct_checked": 9783, "lean_ok": 7103, "lean_rej": 2680, "parse_reasons": {"unbound": 1401, "no-eos": 141, "expected (": 135, "name": 108, "formula": 96, "expected )": 86, "exact not last": 54, "PR position": 34, "expected :=": 10, "expected ;": 6, "ORE box": 2, "expected None": 2, "expected exact": 1, "trailing tokens": 1}, "lean_wall_s": 23.479013442993164, "lean_proc_s": 574.0977764129639, "workers": 96, "chunk": 400}
|
| 26 |
+
{"utc": "2026-10-14T18:14:40Z", "samples": 3912, "parse_fail": 1875, "distinct_checked": 1978, "lean_ok": 23, "lean_rej": 1955, "parse_reasons": {"unbound": 1149, "expected (": 208, "no-eos": 185, "expected )": 92, "formula": 92, "name": 73, "exact not last": 53, "PR position": 10, "expected ;": 9, "expected :=": 3, "expected None": 1}, "lean_wall_s": 5.534298896789551, "lean_proc_s": 26.904463052749634, "workers": 96, "chunk": 400}
|
| 27 |
+
{"utc": "2026-10-14T18:15:42Z", "samples": 3888, "parse_fail": 1820, "distinct_checked": 2031, "lean_ok": 40, "lean_rej": 1991, "parse_reasons": {"unbound": 1376, "no-eos": 90, "expected )": 79, "formula": 69, "expected (": 65, "name": 48, "exact not last": 43, "PR position": 32, "ORE box": 10, "expected ;": 6, "expected False": 1, "expected :": 1}, "lean_wall_s": 22.230774402618408, "lean_proc_s": 112.82225179672241, "workers": 96, "chunk": 400}
|
| 28 |
+
{"utc": "2026-10-14T18:16:43Z", "samples": 3888, "parse_fail": 1784, "distinct_checked": 2047, "lean_ok": 20, "lean_rej": 2027, "parse_reasons": {"unbound": 1170, "expected (": 131, "no-eos": 110, "formula": 109, "PR position": 58, "expected )": 57, "exact not last": 57, "name": 51, "expected :=": 29, "ORE box": 7, "expected ;": 4, "trailing tokens": 1}, "lean_wall_s": 8.389191389083862, "lean_proc_s": 43.18119740486145, "workers": 96, "chunk": 400}
|
| 29 |
+
{"utc": "2026-10-14T18:19:06Z", "samples": 12000, "parse_fail": 2535, "distinct_checked": 9320, "lean_ok": 6914, "lean_rej": 2406, "parse_reasons": {"unbound": 1702, "no-eos": 201, "expected (": 141, "name": 120, "expected )": 110, "formula": 100, "exact not last": 61, "PR position": 51, "expected ;": 32, "expected :=": 11, "ORE box": 4, "expected False": 1, "eof": 1}, "lean_wall_s": 23.283909797668457, "lean_proc_s": 533.5811381340027, "workers": 96, "chunk": 400}
|
| 30 |
+
{"utc": "2026-10-14T18:19:59Z", "samples": 4128, "parse_fail": 2300, "distinct_checked": 1797, "lean_ok": 22, "lean_rej": 1775, "parse_reasons": {"unbound": 1564, "no-eos": 174, "expected (": 157, "expected )": 113, "formula": 93, "exact not last": 70, "name": 65, "PR position": 32, "expected ;": 22, "expected :=": 8, "ORE box": 2}, "lean_wall_s": 7.442631006240845, "lean_proc_s": 34.680033445358276, "workers": 96, "chunk": 400}
|
| 31 |
+
{"utc": "2026-10-14T18:21:07Z", "samples": 4128, "parse_fail": 2256, "distinct_checked": 1842, "lean_ok": 20, "lean_rej": 1822, "parse_reasons": {"unbound": 1527, "no-eos": 155, "expected (": 127, "expected )": 88, "formula": 84, "exact not last": 73, "PR position": 65, "name": 61, "expected ;": 50, "expected :=": 24, "ORE box": 2}, "lean_wall_s": 7.997331380844116, "lean_proc_s": 38.51524496078491, "workers": 96, "chunk": 400}
|
| 32 |
+
{"utc": "2026-10-14T18:22:09Z", "samples": 4128, "parse_fail": 2103, "distinct_checked": 1988, "lean_ok": 30, "lean_rej": 1958, "parse_reasons": {"unbound": 1413, "no-eos": 162, "formula": 129, "expected (": 127, "expected )": 75, "PR position": 58, "name": 57, "exact not last": 38, "expected :=": 23, "ORE box": 9, "expected ;": 9, "eof": 2, "expected None": 1}, "lean_wall_s": 8.047477722167969, "lean_proc_s": 39.940632581710815, "workers": 96, "chunk": 400}
|
| 33 |
+
{"utc": "2026-10-14T18:24:31Z", "samples": 11984, "parse_fail": 2511, "distinct_checked": 9336, "lean_ok": 7238, "lean_rej": 2098, "parse_reasons": {"unbound": 1635, "expected (": 220, "no-eos": 199, "name": 124, "formula": 95, "exact not last": 71, "expected )": 67, "expected ;": 61, "PR position": 23, "ORE box": 10, "expected :=": 4, "expected None": 1, "expected False": 1}, "lean_wall_s": 23.693827867507935, "lean_proc_s": 547.8643403053284, "workers": 96, "chunk": 400}
|
| 34 |
+
{"utc": "2026-10-14T18:25:30Z", "samples": 3768, "parse_fail": 2010, "distinct_checked": 1716, "lean_ok": 14, "lean_rej": 1702, "parse_reasons": {"unbound": 1274, "no-eos": 223, "expected (": 193, "exact not last": 75, "name": 68, "expected )": 65, "formula": 63, "expected ;": 30, "PR position": 10, "ORE box": 6, "trailing tokens": 3}, "lean_wall_s": 7.842417240142822, "lean_proc_s": 33.53033995628357, "workers": 96, "chunk": 400}
|
| 35 |
+
{"utc": "2026-10-14T18:26:27Z", "samples": 3768, "parse_fail": 2318, "distinct_checked": 1427, "lean_ok": 15, "lean_rej": 1412, "parse_reasons": {"unbound": 1618, "expected (": 195, "no-eos": 180, "formula": 66, "exact not last": 54, "PR position": 53, "name": 52, "expected ;": 48, "expected )": 44, "ORE box": 5, "expected :=": 2, "expected False": 1}, "lean_wall_s": 7.2006752490997314, "lean_proc_s": 27.39743447303772, "workers": 96, "chunk": 400}
|
| 36 |
+
{"utc": "2026-10-14T18:27:15Z", "samples": 3744, "parse_fail": 2321, "distinct_checked": 1403, "lean_ok": 20, "lean_rej": 1383, "parse_reasons": {"unbound": 1546, "expected (": 232, "formula": 128, "no-eos": 119, "expected ;": 75, "name": 71, "exact not last": 69, "expected )": 49, "PR position": 20, "expected :=": 10, "expected None": 1, "expected False": 1}, "lean_wall_s": 6.393362760543823, "lean_proc_s": 22.938522338867188, "workers": 96, "chunk": 400}
|
| 37 |
+
{"utc": "2026-10-14T18:29:39Z", "samples": 12008, "parse_fail": 2629, "distinct_checked": 9261, "lean_ok": 7333, "lean_rej": 1928, "parse_reasons": {"unbound": 1752, "no-eos": 256, "expected (": 225, "name": 129, "formula": 70, "exact not last": 66, "expected )": 52, "expected ;": 38, "ORE box": 17, "expected :=": 8, "PR position": 6, "expected None": 3, "expected =>": 3, "expected exact": 2, "trailing tokens": 1, "expected :": 1}, "lean_wall_s": 19.462053060531616, "lean_proc_s": 439.37226462364197, "workers": 96, "chunk": 400}
|
| 38 |
+
{"utc": "2026-10-14T18:30:50Z", "samples": 3720, "parse_fail": 2390, "distinct_checked": 1300, "lean_ok": 14, "lean_rej": 1286, "parse_reasons": {"unbound": 1570, "no-eos": 233, "expected (": 156, "formula": 112, "name": 98, "expected )": 68, "exact not last": 60, "expected ;": 42, "expected :=": 23, "PR position": 21, "ORE box": 5, "expected False": 2}, "lean_wall_s": 3.894397497177124, "lean_proc_s": 12.99564814567566, "workers": 96, "chunk": 400}
|
| 39 |
+
{"utc": "2026-10-14T18:31:38Z", "samples": 3720, "parse_fail": 2262, "distinct_checked": 1423, "lean_ok": 7, "lean_rej": 1416, "parse_reasons": {"unbound": 1590, "expected (": 206, "no-eos": 205, "name": 70, "formula": 57, "exact not last": 50, "expected )": 47, "expected ;": 20, "ORE box": 9, "expected False": 2, "trailing tokens": 2, "PR position": 2, "expected None": 1, "expected :=": 1}, "lean_wall_s": 4.225042343139648, "lean_proc_s": 15.215304136276245, "workers": 96, "chunk": 400}
|
| 40 |
+
{"utc": "2026-10-14T18:32:38Z", "samples": 3720, "parse_fail": 2308, "distinct_checked": 1386, "lean_ok": 17, "lean_rej": 1369, "parse_reasons": {"unbound": 1593, "expected (": 239, "no-eos": 216, "name": 70, "exact not last": 60, "expected )": 49, "expected ;": 48, "expected :=": 14, "formula": 12, "expected =>": 2, "trailing tokens": 2, "ORE box": 1, "expected exact": 1, "PR position": 1}, "lean_wall_s": 4.112773180007935, "lean_proc_s": 14.58886432647705, "workers": 96, "chunk": 400}
|
| 41 |
+
{"utc": "2026-10-14T18:34:59Z", "samples": 12008, "parse_fail": 2516, "distinct_checked": 9373, "lean_ok": 7441, "lean_rej": 1932, "parse_reasons": {"unbound": 1744, "no-eos": 234, "name": 122, "formula": 91, "expected (": 89, "exact not last": 69, "expected )": 58, "expected ;": 38, "PR position": 27, "ORE box": 17, "expected False": 9, "expected :=": 7, "expected =>": 6, "trailing tokens": 3, "expected None": 1, "eof": 1}, "lean_wall_s": 19.636128187179565, "lean_proc_s": 453.8112823963165, "workers": 96, "chunk": 400}
|
| 42 |
+
{"utc": "2026-10-14T18:36:09Z", "samples": 3432, "parse_fail": 2050, "distinct_checked": 1354, "lean_ok": 14, "lean_rej": 1340, "parse_reasons": {"unbound": 1384, "no-eos": 237, "formula": 117, "expected (": 82, "name": 52, "exact not last": 51, "expected )": 44, "PR position": 32, "expected ;": 18, "expected =>": 12, "expected :=": 11, "ORE box": 8, "expected False": 2}, "lean_wall_s": 3.7394332885742188, "lean_proc_s": 12.980172157287598, "workers": 96, "chunk": 400}
|
| 43 |
+
{"utc": "2026-10-14T18:37:08Z", "samples": 3432, "parse_fail": 2261, "distinct_checked": 1146, "lean_ok": 22, "lean_rej": 1124, "parse_reasons": {"unbound": 1608, "no-eos": 212, "expected (": 94, "formula": 75, "expected ;": 70, "exact not last": 58, "expected )": 57, "name": 56, "PR position": 13, "expected False": 6, "expected :=": 5, "ORE box": 4, "expected None": 1, "expected :": 1, "eof": 1}, "lean_wall_s": 3.475093126296997, "lean_proc_s": 10.219601154327393, "workers": 96, "chunk": 400}
|
| 44 |
+
{"utc": "2026-10-14T18:37:48Z", "samples": 3432, "parse_fail": 2127, "distinct_checked": 1288, "lean_ok": 6, "lean_rej": 1282, "parse_reasons": {"unbound": 1597, "no-eos": 160, "exact not last": 65, "expected )": 54, "formula": 54, "name": 53, "expected (": 47, "ORE box": 33, "expected ;": 30, "PR position": 21, "expected False": 8, "trailing tokens": 3, "expected exact": 1, "expected :=": 1}, "lean_wall_s": 4.293886661529541, "lean_proc_s": 13.981719732284546, "workers": 96, "chunk": 400}
|
| 45 |
+
{"utc": "2026-10-14T18:40:16Z", "samples": 12000, "parse_fail": 2615, "distinct_checked": 9285, "lean_ok": 7108, "lean_rej": 2177, "parse_reasons": {"unbound": 1661, "no-eos": 277, "expected (": 202, "name": 133, "expected )": 91, "exact not last": 63, "expected ;": 55, "formula": 55, "PR position": 43, "expected :=": 13, "ORE box": 12, "expected False": 7, "expected None": 1, "trailing tokens": 1, "expected =>": 1}, "lean_wall_s": 20.66286301612854, "lean_proc_s": 468.4430751800537, "workers": 96, "chunk": 400}
|
| 46 |
+
{"utc": "2026-10-14T18:41:29Z", "samples": 3840, "parse_fail": 2330, "distinct_checked": 1478, "lean_ok": 30, "lean_rej": 1448, "parse_reasons": {"unbound": 1608, "no-eos": 216, "expected (": 120, "expected )": 82, "name": 75, "formula": 65, "exact not last": 55, "expected ;": 53, "PR position": 42, "expected :=": 7, "ORE box": 4, "expected False": 3}, "lean_wall_s": 4.331933975219727, "lean_proc_s": 16.714000701904297, "workers": 96, "chunk": 400}
|
| 47 |
+
{"utc": "2026-10-14T18:42:25Z", "samples": 3840, "parse_fail": 2431, "distinct_checked": 1392, "lean_ok": 13, "lean_rej": 1379, "parse_reasons": {"unbound": 1484, "expected (": 298, "no-eos": 249, "name": 94, "expected )": 88, "exact not last": 66, "expected ;": 57, "PR position": 46, "formula": 22, "ORE box": 19, "expected False": 4, "expected =>": 2, "expected :=": 1, "expected None": 1}, "lean_wall_s": 4.900722026824951, "lean_proc_s": 17.308871030807495, "workers": 96, "chunk": 400}
|
| 48 |
+
{"utc": "2026-10-14T18:43:28Z", "samples": 3840, "parse_fail": 2256, "distinct_checked": 1554, "lean_ok": 19, "lean_rej": 1535, "parse_reasons": {"unbound": 1439, "no-eos": 237, "expected (": 118, "expected )": 117, "name": 112, "exact not last": 68, "expected ;": 53, "PR position": 37, "formula": 35, "expected :=": 21, "ORE box": 8, "expected False": 7, "expected =>": 2, "expected exact": 1, "eof": 1}, "lean_wall_s": 7.607847452163696, "lean_proc_s": 30.11261820793152, "workers": 96, "chunk": 400}
|
| 49 |
+
{"utc": "2026-10-14T18:46:02Z", "samples": 12008, "parse_fail": 3173, "distinct_checked": 8747, "lean_ok": 7091, "lean_rej": 1656, "parse_reasons": {"unbound": 1775, "no-eos": 594, "expected (": 236, "name": 120, "expected )": 107, "formula": 101, "exact not last": 92, "expected ;": 84, "ORE box": 31, "expected :": 12, "PR position": 8, "expected :=": 5, "expected None": 3, "expected =>": 2, "expected False": 1, "expected exact": 1, "trailing tokens": 1}, "lean_wall_s": 24.22095561027527, "lean_proc_s": 519.1075320243835, "workers": 96, "chunk": 400}
|
| 50 |
+
{"utc": "2026-10-14T18:47:11Z", "samples": 3912, "parse_fail": 2987, "distinct_checked": 916, "lean_ok": 5, "lean_rej": 911, "parse_reasons": {"unbound": 1618, "no-eos": 683, "expected (": 227, "expected )": 147, "exact not last": 87, "name": 86, "expected ;": 62, "formula": 47, "PR position": 16, "expected False": 5, "ORE box": 3, "expected :": 3, "expected =>": 1, "expected None": 1, "expected :=": 1}, "lean_wall_s": 5.147904634475708, "lean_proc_s": 12.394357919692993, "workers": 96, "chunk": 400}
|
| 51 |
+
{"utc": "2026-10-14T18:48:16Z", "samples": 3912, "parse_fail": 2814, "distinct_checked": 1074, "lean_ok": 24, "lean_rej": 1050, "parse_reasons": {"unbound": 1609, "no-eos": 449, "expected (": 276, "expected ;": 114, "formula": 108, "expected )": 91, "exact not last": 80, "name": 52, "ORE box": 11, "expected :": 9, "expected :=": 6, "expected False": 4, "PR position": 3, "trailing tokens": 1, "expected exact": 1}, "lean_wall_s": 5.34496545791626, "lean_proc_s": 14.974662780761719, "workers": 96, "chunk": 400}
|
| 52 |
+
{"utc": "2026-10-14T18:49:25Z", "samples": 3888, "parse_fail": 2774, "distinct_checked": 1101, "lean_ok": 11, "lean_rej": 1090, "parse_reasons": {"unbound": 1588, "no-eos": 459, "expected (": 220, "formula": 113, "expected )": 93, "expected ;": 81, "name": 75, "ORE box": 62, "exact not last": 53, "PR position": 16, "expected :": 7, "expected :=": 3, "expected False": 1, "eof": 1, "expected None": 1, "trailing tokens": 1}, "lean_wall_s": 5.441571235656738, "lean_proc_s": 15.899064302444458, "workers": 96, "chunk": 400}
|
| 53 |
+
{"utc": "2026-10-14T18:51:58Z", "samples": 12016, "parse_fail": 2925, "distinct_checked": 8997, "lean_ok": 7092, "lean_rej": 1905, "parse_reasons": {"unbound": 1610, "no-eos": 514, "expected (": 259, "expected ;": 141, "name": 114, "exact not last": 89, "expected )": 81, "formula": 44, "PR position": 28, "expected :=": 20, "ORE box": 10, "expected exact": 9, "trailing tokens": 2, "expected None": 2, "expected False": 2}, "lean_wall_s": 24.000633716583252, "lean_proc_s": 532.2101955413818, "workers": 96, "chunk": 400}
|
| 54 |
+
{"utc": "2026-10-14T18:53:07Z", "samples": 3864, "parse_fail": 2502, "distinct_checked": 1338, "lean_ok": 28, "lean_rej": 1310, "parse_reasons": {"unbound": 1421, "no-eos": 398, "expected (": 254, "expected ;": 135, "exact not last": 95, "name": 57, "expected )": 44, "formula": 38, "expected :=": 22, "PR position": 20, "expected exact": 5, "ORE box": 5, "trailing tokens": 2, "eof": 2, "expected False": 2, "expected =>": 1, "expected None": 1}, "lean_wall_s": 6.733992099761963, "lean_proc_s": 23.23594570159912, "workers": 96, "chunk": 400}
|
| 55 |
+
{"utc": "2026-10-14T18:53:58Z", "samples": 3864, "parse_fail": 2673, "distinct_checked": 1180, "lean_ok": 6, "lean_rej": 1174, "parse_reasons": {"unbound": 1389, "no-eos": 443, "expected (": 315, "expected ;": 147, "exact not last": 102, "expected )": 90, "name": 75, "formula": 40, "PR position": 39, "ORE box": 17, "expected :=": 8, "expected exact": 4, "expected False": 2, "trailing tokens": 1, "eof": 1}, "lean_wall_s": 6.268695116043091, "lean_proc_s": 18.668310165405273, "workers": 96, "chunk": 400}
|
| 56 |
+
{"utc": "2026-10-14T18:55:20Z", "samples": 3840, "parse_fail": 2581, "distinct_checked": 1241, "lean_ok": 7, "lean_rej": 1234, "parse_reasons": {"unbound": 1344, "no-eos": 532, "expected (": 174, "expected ;": 133, "expected )": 105, "name": 84, "exact not last": 73, "formula": 57, "expected :=": 43, "PR position": 17, "trailing tokens": 6, "ORE box": 6, "expected exact": 5, "eof": 1, "expected =>": 1}, "lean_wall_s": 6.189985752105713, "lean_proc_s": 19.218172073364258, "workers": 96, "chunk": 400}
|
| 57 |
+
{"utc": "2026-10-14T18:57:57Z", "samples": 11984, "parse_fail": 3537, "distinct_checked": 8381, "lean_ok": 6785, "lean_rej": 1596, "parse_reasons": {"unbound": 1886, "no-eos": 736, "expected (": 394, "name": 131, "exact not last": 99, "expected ;": 95, "expected )": 88, "formula": 60, "ORE box": 24, "trailing tokens": 8, "PR position": 7, "expected False": 5, "expected :=": 2, "eof": 1, "expected None": 1}, "lean_wall_s": 23.38322615623474, "lean_proc_s": 472.84154868125916, "workers": 96, "chunk": 400}
|
| 58 |
+
{"utc": "2026-10-14T18:59:20Z", "samples": 4320, "parse_fail": 3160, "distinct_checked": 1157, "lean_ok": 15, "lean_rej": 1142, "parse_reasons": {"unbound": 1761, "no-eos": 633, "expected (": 342, "exact not last": 109, "expected ;": 87, "expected )": 84, "name": 53, "formula": 48, "ORE box": 30, "PR position": 6, "expected False": 3, "trailing tokens": 2, "expected None": 1, "expected :=": 1}, "lean_wall_s": 6.010027885437012, "lean_proc_s": 17.865504026412964, "workers": 96, "chunk": 400}
|
| 59 |
+
{"utc": "2026-10-14T19:00:17Z", "samples": 4296, "parse_fail": 3136, "distinct_checked": 1148, "lean_ok": 22, "lean_rej": 1126, "parse_reasons": {"unbound": 1766, "no-eos": 612, "expected (": 369, "exact not last": 91, "name": 80, "expected ;": 80, "expected )": 64, "formula": 48, "ORE box": 17, "expected False": 3, "expected :=": 1, "expected .elim": 1, "PR position": 1, "trailing tokens": 1, "expected exact": 1, "eof": 1}, "lean_wall_s": 5.698758125305176, "lean_proc_s": 16.454867362976074, "workers": 96, "chunk": 400}
|
| 60 |
+
{"utc": "2026-10-14T19:01:25Z", "samples": 4296, "parse_fail": 3415, "distinct_checked": 868, "lean_ok": 14, "lean_rej": 854, "parse_reasons": {"unbound": 1707, "no-eos": 832, "expected (": 469, "expected ;": 105, "exact not last": 95, "expected )": 67, "formula": 56, "name": 52, "ORE box": 15, "PR position": 10, "expected :=": 2, "expected False": 2, "trailing tokens": 2, "expected None": 1}, "lean_wall_s": 4.884392738342285, "lean_proc_s": 10.984759092330933, "workers": 96, "chunk": 400}
|
| 61 |
+
{"utc": "2026-10-14T19:04:03Z", "samples": 11992, "parse_fail": 3594, "distinct_checked": 8315, "lean_ok": 6740, "lean_rej": 1575, "parse_reasons": {"unbound": 2128, "no-eos": 597, "expected (": 237, "expected ;": 178, "name": 175, "exact not last": 81, "formula": 61, "expected )": 52, "ORE box": 33, "PR position": 20, "expected :=": 14, "trailing tokens": 4, "expected :": 4, "expected None": 3, "expected =>": 3, "expected False": 2, "eof": 1, "expected exact": 1}, "lean_wall_s": 23.25137424468994, "lean_proc_s": 478.4728147983551, "workers": 96, "chunk": 400}
|
| 62 |
+
{"utc": "2026-10-14T19:05:28Z", "samples": 4200, "parse_fail": 3263, "distinct_checked": 928, "lean_ok": 8, "lean_rej": 920, "parse_reasons": {"unbound": 1804, "no-eos": 516, "expected (": 284, "expected ;": 192, "name": 140, "formula": 103, "exact not last": 99, "expected )": 55, "ORE box": 19, "PR position": 18, "expected :=": 14, "expected False": 11, "expected None": 3, "expected exact": 2, "expected :": 2, "expected =>": 1}, "lean_wall_s": 5.11273193359375, "lean_proc_s": 12.66982126235962, "workers": 96, "chunk": 400}
|
| 63 |
+
{"utc": "2026-10-14T19:06:35Z", "samples": 4200, "parse_fail": 3060, "distinct_checked": 1129, "lean_ok": 12, "lean_rej": 1117, "parse_reasons": {"unbound": 1870, "no-eos": 474, "expected (": 216, "expected ;": 168, "name": 110, "exact not last": 78, "expected )": 36, "expected :=": 29, "formula": 29, "PR position": 24, "ORE box": 19, "expected False": 3, "expected None": 3, "expected exact": 1}, "lean_wall_s": 6.068432569503784, "lean_proc_s": 17.909506797790527, "workers": 96, "chunk": 400}
|
| 64 |
+
{"utc": "2026-10-14T19:07:27Z", "samples": 4176, "parse_fail": 3196, "distinct_checked": 970, "lean_ok": 20, "lean_rej": 950, "parse_reasons": {"unbound": 2030, "no-eos": 491, "expected (": 207, "expected ;": 125, "name": 124, "exact not last": 89, "expected )": 51, "formula": 31, "ORE box": 16, "expected :=": 13, "PR position": 8, "expected False": 5, "expected exact": 2, "trailing tokens": 2, "expected :": 2}, "lean_wall_s": 5.5022008419036865, "lean_proc_s": 14.548786401748657, "workers": 96, "chunk": 400}
|
| 65 |
+
{"utc": "2026-10-14T19:10:04Z", "samples": 12000, "parse_fail": 3451, "distinct_checked": 8474, "lean_ok": 6754, "lean_rej": 1720, "parse_reasons": {"unbound": 1834, "expected (": 597, "no-eos": 497, "name": 156, "exact not last": 100, "expected )": 85, "formula": 75, "expected ;": 69, "ORE box": 14, "PR position": 11, "expected :=": 5, "expected =>": 4, "expected None": 4}, "lean_wall_s": 24.124343156814575, "lean_proc_s": 494.23524951934814, "workers": 96, "chunk": 400}
|
| 66 |
+
{"utc": "2026-10-14T19:11:14Z", "samples": 4368, "parse_fail": 3072, "distinct_checked": 1285, "lean_ok": 19, "lean_rej": 1266, "parse_reasons": {"unbound": 1708, "expected (": 574, "no-eos": 417, "formula": 89, "name": 83, "exact not last": 81, "expected ;": 62, "expected )": 35, "ORE box": 15, "expected =>": 3, "expected None": 2, "PR position": 2, "eof": 1}, "lean_wall_s": 7.050646781921387, "lean_proc_s": 23.247795820236206, "workers": 96, "chunk": 400}
|
| 67 |
+
{"utc": "2026-10-14T19:12:25Z", "samples": 4368, "parse_fail": 3275, "distinct_checked": 1081, "lean_ok": 8, "lean_rej": 1073, "parse_reasons": {"unbound": 1881, "expected (": 603, "no-eos": 357, "name": 114, "exact not last": 99, "expected ;": 84, "formula": 56, "expected )": 42, "ORE box": 20, "PR position": 9, "expected :=": 3, "expected =>": 3, "expected None": 2, "trailing tokens": 1, "eof": 1}, "lean_wall_s": 6.092702865600586, "lean_proc_s": 17.49983048439026, "workers": 96, "chunk": 400}
|
| 68 |
+
{"utc": "2026-10-14T19:13:34Z", "samples": 4368, "parse_fail": 2946, "distinct_checked": 1399, "lean_ok": 22, "lean_rej": 1377, "parse_reasons": {"unbound": 1478, "expected (": 584, "no-eos": 412, "name": 115, "expected ;": 98, "exact not last": 84, "expected )": 84, "formula": 53, "ORE box": 14, "PR position": 13, "expected :=": 9, "eof": 2}, "lean_wall_s": 7.178143739700317, "lean_proc_s": 25.999193906784058, "workers": 96, "chunk": 400}
|
| 69 |
+
{"utc": "2026-10-14T19:16:07Z", "samples": 12000, "parse_fail": 3467, "distinct_checked": 8457, "lean_ok": 6764, "lean_rej": 1693, "parse_reasons": {"unbound": 1994, "no-eos": 566, "expected (": 288, "expected ;": 157, "name": 137, "formula": 98, "expected )": 80, "exact not last": 70, "ORE box": 55, "PR position": 11, "expected False": 6, "expected None": 3, "expected exact": 1, "trailing tokens": 1}, "lean_wall_s": 22.499966382980347, "lean_proc_s": 454.0443856716156, "workers": 96, "chunk": 400}
|
| 70 |
+
{"utc": "2026-10-14T19:17:11Z", "samples": 4392, "parse_fail": 3173, "distinct_checked": 1205, "lean_ok": 9, "lean_rej": 1196, "parse_reasons": {"unbound": 1709, "no-eos": 560, "expected (": 405, "expected ;": 107, "expected )": 99, "name": 97, "formula": 68, "exact not last": 67, "ORE box": 30, "PR position": 23, "expected False": 5, "trailing tokens": 1, "expected None": 1, "expected =>": 1}, "lean_wall_s": 6.765117168426514, "lean_proc_s": 20.180474042892456, "workers": 96, "chunk": 400}
|
| 71 |
+
{"utc": "2026-10-14T19:18:14Z", "samples": 4392, "parse_fail": 2984, "distinct_checked": 1383, "lean_ok": 8, "lean_rej": 1375, "parse_reasons": {"unbound": 1807, "no-eos": 464, "expected (": 246, "expected ;": 153, "name": 82, "formula": 71, "expected )": 61, "exact not last": 52, "ORE box": 30, "expected False": 11, "PR position": 3, "trailing tokens": 2, "eof": 1, "expected None": 1}, "lean_wall_s": 6.752268552780151, "lean_proc_s": 23.62572979927063, "workers": 96, "chunk": 400}
|
| 72 |
+
{"utc": "2026-10-14T19:19:32Z", "samples": 4368, "parse_fail": 3252, "distinct_checked": 1097, "lean_ok": 30, "lean_rej": 1067, "parse_reasons": {"unbound": 1914, "no-eos": 472, "expected (": 302, "expected ;": 146, "formula": 120, "name": 107, "exact not last": 78, "ORE box": 48, "expected )": 41, "expected False": 11, "PR position": 5, "trailing tokens": 5, "expected exact": 1, "expected None": 1, "expected :=": 1}, "lean_wall_s": 5.816829442977905, "lean_proc_s": 16.50232172012329, "workers": 96, "chunk": 400}
|
| 73 |
+
{"utc": "2026-10-14T19:22:13Z", "samples": 12000, "parse_fail": 3509, "distinct_checked": 8420, "lean_ok": 6564, "lean_rej": 1856, "parse_reasons": {"unbound": 1927, "no-eos": 641, "expected (": 456, "name": 150, "expected ;": 93, "formula": 79, "expected )": 52, "exact not last": 42, "expected :=": 39, "ORE box": 17, "PR position": 10, "expected None": 1, "trailing tokens": 1, "expected False": 1}, "lean_wall_s": 24.93295454978943, "lean_proc_s": 502.5256474018097, "workers": 96, "chunk": 400}
|
| 74 |
+
{"utc": "2026-10-14T19:23:21Z", "samples": 4488, "parse_fail": 3015, "distinct_checked": 1461, "lean_ok": 3, "lean_rej": 1458, "parse_reasons": {"unbound": 1706, "no-eos": 500, "expected (": 378, "name": 117, "expected ;": 84, "exact not last": 53, "expected :=": 49, "formula": 46, "expected )": 43, "PR position": 22, "ORE box": 15, "expected None": 1, "eof": 1}, "lean_wall_s": 6.999798774719238, "lean_proc_s": 25.778701543807983, "workers": 96, "chunk": 400}
|
| 75 |
+
{"utc": "2026-10-14T19:24:45Z", "samples": 4488, "parse_fail": 3263, "distinct_checked": 1214, "lean_ok": 16, "lean_rej": 1198, "parse_reasons": {"unbound": 1728, "no-eos": 629, "expected (": 480, "name": 105, "formula": 81, "expected )": 73, "exact not last": 67, "expected ;": 60, "expected :=": 25, "ORE box": 13, "trailing tokens": 2}, "lean_wall_s": 5.765920877456665, "lean_proc_s": 17.704097986221313, "workers": 96, "chunk": 400}
|
| 76 |
+
{"utc": "2026-10-14T19:25:41Z", "samples": 4488, "parse_fail": 3129, "distinct_checked": 1347, "lean_ok": 12, "lean_rej": 1335, "parse_reasons": {"unbound": 1672, "expected (": 502, "no-eos": 476, "name": 140, "expected ;": 83, "formula": 73, "expected )": 62, "exact not last": 43, "ORE box": 38, "expected :=": 33, "PR position": 4, "trailing tokens": 2, "eof": 1}, "lean_wall_s": 7.392807483673096, "lean_proc_s": 25.12494659423828, "workers": 96, "chunk": 400}
|
| 77 |
+
{"utc": "2026-10-14T19:28:20Z", "samples": 12000, "parse_fail": 3331, "distinct_checked": 8605, "lean_ok": 6679, "lean_rej": 1926, "parse_reasons": {"unbound": 1774, "no-eos": 619, "expected (": 332, "expected ;": 150, "name": 109, "expected )": 109, "formula": 98, "exact not last": 56, "ORE box": 36, "PR position": 23, "expected :=": 18, "trailing tokens": 5, "expected None": 2}, "lean_wall_s": 23.36787748336792, "lean_proc_s": 488.05835223197937, "workers": 96, "chunk": 400}
|
| 78 |
+
{"utc": "2026-10-14T19:29:29Z", "samples": 4320, "parse_fail": 2853, "distinct_checked": 1450, "lean_ok": 20, "lean_rej": 1430, "parse_reasons": {"unbound": 1539, "no-eos": 514, "expected (": 309, "expected ;": 135, "expected )": 78, "name": 75, "formula": 70, "exact not last": 53, "expected :=": 27, "PR position": 23, "ORE box": 22, "trailing tokens": 5, "expected :": 1, "eof": 1, "expected None": 1}, "lean_wall_s": 7.7944886684417725, "lean_proc_s": 29.989126205444336, "workers": 96, "chunk": 400}
|
| 79 |
+
{"utc": "2026-10-14T19:30:35Z", "samples": 4296, "parse_fail": 2915, "distinct_checked": 1370, "lean_ok": 11, "lean_rej": 1359, "parse_reasons": {"unbound": 1418, "no-eos": 626, "expected (": 324, "expected ;": 146, "formula": 117, "name": 94, "expected )": 79, "exact not last": 59, "ORE box": 20, "expected :=": 17, "PR position": 11, "expected None": 2, "trailing tokens": 2}, "lean_wall_s": 8.014110326766968, "lean_proc_s": 28.009416103363037, "workers": 96, "chunk": 400}
|
| 80 |
+
{"utc": "2026-10-14T19:31:49Z", "samples": 4296, "parse_fail": 2834, "distinct_checked": 1444, "lean_ok": 29, "lean_rej": 1415, "parse_reasons": {"unbound": 1498, "no-eos": 456, "expected (": 360, "expected ;": 138, "expected )": 86, "name": 71, "formula": 71, "exact not last": 63, "ORE box": 50, "PR position": 30, "expected :=": 9, "trailing tokens": 2}, "lean_wall_s": 6.97657036781311, "lean_proc_s": 26.082921266555786, "workers": 96, "chunk": 400}
|
| 81 |
+
{"utc": "2026-10-14T19:34:24Z", "samples": 12008, "parse_fail": 2984, "distinct_checked": 8951, "lean_ok": 6913, "lean_rej": 2038, "parse_reasons": {"unbound": 1454, "no-eos": 652, "expected (": 395, "name": 167, "formula": 81, "expected ;": 78, "exact not last": 46, "ORE box": 45, "expected )": 41, "PR position": 12, "trailing tokens": 4, "expected :": 3, "expected =>": 3, "expected None": 1, "expected :=": 1, "expected False": 1}, "lean_wall_s": 26.723570346832275, "lean_proc_s": 577.1691539287567, "workers": 96, "chunk": 400}
|
| 82 |
+
{"utc": "2026-10-14T19:35:38Z", "samples": 3912, "parse_fail": 2343, "distinct_checked": 1549, "lean_ok": 19, "lean_rej": 1530, "parse_reasons": {"unbound": 1224, "no-eos": 409, "expected (": 363, "name": 84, "formula": 84, "expected ;": 61, "expected )": 46, "exact not last": 39, "ORE box": 23, "PR position": 8, "expected :=": 2}, "lean_wall_s": 7.762413740158081, "lean_proc_s": 30.068371534347534, "workers": 96, "chunk": 400}
|
| 83 |
+
{"utc": "2026-10-14T19:36:29Z", "samples": 3888, "parse_fail": 2550, "distinct_checked": 1326, "lean_ok": 31, "lean_rej": 1295, "parse_reasons": {"unbound": 1152, "no-eos": 606, "expected (": 388, "name": 82, "expected ;": 82, "ORE box": 69, "formula": 68, "exact not last": 44, "expected )": 30, "PR position": 22, "expected :": 4, "trailing tokens": 1, "expected False": 1, "expected None": 1}, "lean_wall_s": 8.013066291809082, "lean_proc_s": 26.74479651451111, "workers": 96, "chunk": 400}
|
| 84 |
+
{"utc": "2026-10-14T19:37:42Z", "samples": 3888, "parse_fail": 2481, "distinct_checked": 1394, "lean_ok": 13, "lean_rej": 1381, "parse_reasons": {"unbound": 1167, "no-eos": 452, "expected (": 440, "name": 119, "expected ;": 101, "formula": 62, "ORE box": 47, "expected )": 40, "exact not last": 38, "expected =>": 6, "PR position": 5, "trailing tokens": 2, "expected :": 1, "expected :=": 1}, "lean_wall_s": 6.670616865158081, "lean_proc_s": 23.45348358154297, "workers": 96, "chunk": 400}
|
| 85 |
+
{"utc": "2026-10-14T19:40:11Z", "samples": 12008, "parse_fail": 3077, "distinct_checked": 8861, "lean_ok": 6993, "lean_rej": 1868, "parse_reasons": {"unbound": 1307, "no-eos": 637, "expected (": 574, "name": 155, "expected ;": 138, "formula": 76, "exact not last": 69, "ORE box": 54, "expected )": 34, "PR position": 12, "expected :=": 9, "expected :": 5, "expected None": 3, "expected by": 1, "trailing tokens": 1, "expected False": 1, "expected =>": 1}, "lean_wall_s": 21.201660871505737, "lean_proc_s": 448.0281114578247, "workers": 96, "chunk": 400}
|
| 86 |
+
{"utc": "2026-10-14T19:41:20Z", "samples": 4008, "parse_fail": 2511, "distinct_checked": 1486, "lean_ok": 22, "lean_rej": 1464, "parse_reasons": {"unbound": 1021, "no-eos": 572, "expected (": 467, "expected ;": 117, "name": 114, "formula": 83, "exact not last": 66, "ORE box": 40, "expected )": 16, "expected :": 11, "PR position": 3, "expected :=": 1}, "lean_wall_s": 5.251039266586304, "lean_proc_s": 19.595841646194458, "workers": 96, "chunk": 400}
|
| 87 |
+
{"utc": "2026-10-14T19:42:18Z", "samples": 3984, "parse_fail": 2805, "distinct_checked": 1166, "lean_ok": 12, "lean_rej": 1154, "parse_reasons": {"unbound": 1246, "expected (": 677, "no-eos": 474, "expected ;": 106, "name": 89, "formula": 68, "exact not last": 50, "PR position": 34, "ORE box": 31, "expected :=": 15, "expected )": 13, "expected :": 1, "expected =>": 1}, "lean_wall_s": 4.141697883605957, "lean_proc_s": 12.167301893234253, "workers": 96, "chunk": 400}
|
| 88 |
+
{"utc": "2026-10-14T19:43:18Z", "samples": 3984, "parse_fail": 2728, "distinct_checked": 1241, "lean_ok": 22, "lean_rej": 1219, "parse_reasons": {"unbound": 1184, "no-eos": 518, "expected (": 495, "name": 143, "expected ;": 138, "ORE box": 81, "formula": 71, "expected )": 39, "exact not last": 33, "expected :": 12, "expected :=": 9, "PR position": 3, "trailing tokens": 1, "eof": 1}, "lean_wall_s": 4.623864650726318, "lean_proc_s": 13.87053656578064, "workers": 96, "chunk": 400}
|
| 89 |
+
{"utc": "2026-10-14T19:45:56Z", "samples": 12000, "parse_fail": 3244, "distinct_checked": 8703, "lean_ok": 6587, "lean_rej": 2116, "parse_reasons": {"unbound": 1273, "no-eos": 899, "expected (": 464, "name": 183, "expected ;": 164, "formula": 101, "expected )": 52, "exact not last": 39, "ORE box": 28, "expected :=": 16, "PR position": 12, "trailing tokens": 4, "expected None": 3, "expected :": 3, "expected False": 2, "eof": 1}, "lean_wall_s": 22.16556406021118, "lean_proc_s": 467.9134454727173, "workers": 96, "chunk": 400}
|
| 90 |
+
{"utc": "2026-10-14T19:47:17Z", "samples": 4320, "parse_fail": 2959, "distinct_checked": 1346, "lean_ok": 22, "lean_rej": 1324, "parse_reasons": {"unbound": 1184, "no-eos": 770, "expected (": 439, "name": 164, "expected ;": 133, "formula": 101, "exact not last": 43, "expected )": 43, "expected :=": 26, "ORE box": 26, "PR position": 20, "expected :": 5, "trailing tokens": 4, "expected False": 1}, "lean_wall_s": 5.150734901428223, "lean_proc_s": 16.861859798431396, "workers": 96, "chunk": 400}
|
| 91 |
+
{"utc": "2026-10-14T19:48:17Z", "samples": 4296, "parse_fail": 2796, "distinct_checked": 1479, "lean_ok": 21, "lean_rej": 1458, "parse_reasons": {"unbound": 1087, "no-eos": 735, "expected (": 532, "name": 127, "expected ;": 101, "formula": 82, "expected )": 49, "ORE box": 40, "exact not last": 23, "expected :=": 10, "PR position": 6, "trailing tokens": 2, "expected False": 1, "eof": 1}, "lean_wall_s": 5.228079080581665, "lean_proc_s": 19.448278665542603, "workers": 96, "chunk": 400}
|
| 92 |
+
{"utc": "2026-10-14T19:49:20Z", "samples": 4296, "parse_fail": 2782, "distinct_checked": 1498, "lean_ok": 28, "lean_rej": 1470, "parse_reasons": {"unbound": 1170, "no-eos": 661, "expected (": 430, "expected ;": 203, "name": 119, "formula": 51, "expected )": 46, "ORE box": 30, "expected :=": 30, "exact not last": 27, "PR position": 7, "expected :": 4, "trailing tokens": 2, "expected False": 1, "eof": 1}, "lean_wall_s": 5.452368259429932, "lean_proc_s": 21.07035994529724, "workers": 96, "chunk": 400}
|
| 93 |
+
{"utc": "2026-10-14T19:52:08Z", "samples": 12000, "parse_fail": 3381, "distinct_checked": 8559, "lean_ok": 6682, "lean_rej": 1877, "parse_reasons": {"unbound": 1466, "no-eos": 717, "expected (": 361, "expected )": 236, "expected ;": 174, "formula": 142, "name": 140, "exact not last": 49, "ORE box": 46, "expected :": 26, "expected :=": 9, "PR position": 6, "expected None": 5, "expected fun": 1, "expected False": 1, "expected =>": 1, "eof": 1}, "lean_wall_s": 27.05387592315674, "lean_proc_s": 572.7888672351837, "workers": 96, "chunk": 400}
|
| 94 |
+
{"utc": "2026-10-14T19:53:27Z", "samples": 4296, "parse_fail": 3036, "distinct_checked": 1245, "lean_ok": 31, "lean_rej": 1214, "parse_reasons": {"unbound": 1315, "no-eos": 585, "expected (": 348, "expected )": 207, "expected ;": 188, "formula": 128, "name": 119, "ORE box": 56, "exact not last": 31, "expected :": 30, "expected :=": 23, "expected None": 3, "expected False": 2, "eof": 1}, "lean_wall_s": 7.313259124755859, "lean_proc_s": 22.73902177810669, "workers": 96, "chunk": 400}
|
| 95 |
+
{"utc": "2026-10-14T19:54:41Z", "samples": 4272, "parse_fail": 3070, "distinct_checked": 1188, "lean_ok": 0, "lean_rej": 1188, "parse_reasons": {"unbound": 1216, "no-eos": 650, "expected (": 418, "expected )": 236, "expected ;": 173, "formula": 159, "name": 97, "ORE box": 41, "exact not last": 40, "expected :": 25, "PR position": 15}, "lean_wall_s": 6.31742787361145, "lean_proc_s": 18.097818851470947, "workers": 96, "chunk": 400}
|
| 96 |
+
{"utc": "2026-10-14T19:55:47Z", "samples": 4272, "parse_fail": 2921, "distinct_checked": 1338, "lean_ok": 19, "lean_rej": 1319, "parse_reasons": {"unbound": 1328, "no-eos": 511, "expected (": 360, "expected )": 217, "expected ;": 144, "name": 123, "formula": 115, "ORE box": 51, "exact not last": 40, "expected :": 18, "expected :=": 7, "expected False": 2, "expected None": 2, "PR position": 1, "expected exact": 1, "expected \u27e9": 1}, "lean_wall_s": 6.74106502532959, "lean_proc_s": 22.966657876968384, "workers": 96, "chunk": 400}
|
| 97 |
+
{"utc": "2026-10-14T19:58:32Z", "samples": 12000, "parse_fail": 3640, "distinct_checked": 8314, "lean_ok": 6422, "lean_rej": 1892, "parse_reasons": {"unbound": 1650, "no-eos": 781, "expected (": 559, "name": 219, "expected ;": 158, "formula": 82, "expected )": 74, "exact not last": 38, "ORE box": 31, "expected False": 21, "expected :=": 13, "PR position": 9, "trailing tokens": 3, "expected exact": 1, "expected None": 1}, "lean_wall_s": 28.569036722183228, "lean_proc_s": 584.0010259151459, "workers": 96, "chunk": 400}
|
| 98 |
+
{"utc": "2026-10-14T19:59:37Z", "samples": 4608, "parse_fail": 3151, "distinct_checked": 1441, "lean_ok": 10, "lean_rej": 1431, "parse_reasons": {"unbound": 1705, "no-eos": 524, "expected (": 454, "name": 143, "expected ;": 103, "expected )": 75, "formula": 49, "exact not last": 33, "ORE box": 22, "expected False": 17, "expected :=": 13, "PR position": 9, "expected None": 3, "eof": 1}, "lean_wall_s": 8.723742961883545, "lean_proc_s": 32.983858823776245, "workers": 96, "chunk": 400}
|
| 99 |
+
{"utc": "2026-10-14T20:00:58Z", "samples": 4608, "parse_fail": 3313, "distinct_checked": 1284, "lean_ok": 16, "lean_rej": 1268, "parse_reasons": {"unbound": 1551, "no-eos": 685, "expected (": 499, "name": 172, "expected ;": 142, "expected )": 77, "formula": 71, "ORE box": 39, "exact not last": 36, "expected :=": 22, "expected False": 12, "PR position": 4, "eof": 1, "expected exact": 1, "expected None": 1}, "lean_wall_s": 7.162397861480713, "lean_proc_s": 23.32475209236145, "workers": 96, "chunk": 400}
|
| 100 |
+
{"utc": "2026-10-14T20:02:06Z", "samples": 4584, "parse_fail": 3204, "distinct_checked": 1365, "lean_ok": 35, "lean_rej": 1330, "parse_reasons": {"unbound": 1330, "expected (": 667, "no-eos": 654, "expected ;": 147, "name": 126, "formula": 110, "expected )": 68, "exact not last": 34, "ORE box": 28, "expected False": 21, "PR position": 13, "expected :=": 2, "expected =>": 1, "expected exact": 1, "expected None": 1, "eof": 1}, "lean_wall_s": 7.710205078125, "lean_proc_s": 27.554754972457886, "workers": 96, "chunk": 400}
|
| 101 |
+
{"utc": "2026-10-14T20:04:50Z", "samples": 12008, "parse_fail": 3807, "distinct_checked": 8147, "lean_ok": 6103, "lean_rej": 2044, "parse_reasons": {"unbound": 1534, "no-eos": 943, "expected (": 635, "expected ;": 212, "name": 155, "formula": 136, "exact not last": 73, "ORE box": 53, "expected )": 41, "PR position": 18, "expected :=": 3, "trailing tokens": 1, "expected None": 1, "expected =>": 1, "expected exact": 1}, "lean_wall_s": 27.188081979751587, "lean_proc_s": 540.4373931884766, "workers": 96, "chunk": 400}
|
| 102 |
+
{"utc": "2026-10-14T20:06:08Z", "samples": 4800, "parse_fail": 3303, "distinct_checked": 1471, "lean_ok": 18, "lean_rej": 1453, "parse_reasons": {"unbound": 1433, "no-eos": 741, "expected (": 605, "expected ;": 185, "formula": 92, "name": 90, "ORE box": 78, "exact not last": 40, "expected )": 31, "PR position": 5, "expected :=": 1, "eof": 1, "expected exact": 1}, "lean_wall_s": 4.874628067016602, "lean_proc_s": 18.2240149974823, "workers": 96, "chunk": 400}
|
| 103 |
+
{"utc": "2026-10-14T20:07:14Z", "samples": 4800, "parse_fail": 3311, "distinct_checked": 1478, "lean_ok": 27, "lean_rej": 1451, "parse_reasons": {"unbound": 1289, "no-eos": 778, "expected (": 629, "expected ;": 186, "name": 157, "formula": 131, "exact not last": 53, "ORE box": 51, "expected )": 28, "expected :=": 5, "expected None": 3, "trailing tokens": 1}, "lean_wall_s": 8.045910120010376, "lean_proc_s": 31.24750590324402, "workers": 96, "chunk": 400}
|
| 104 |
+
{"utc": "2026-10-14T20:08:25Z", "samples": 4800, "parse_fail": 3376, "distinct_checked": 1412, "lean_ok": 17, "lean_rej": 1395, "parse_reasons": {"unbound": 1466, "expected (": 698, "no-eos": 644, "formula": 149, "expected ;": 147, "name": 74, "exact not last": 57, "ORE box": 52, "PR position": 43, "expected )": 41, "expected None": 2, "expected :": 1, "trailing tokens": 1, "expected exact": 1}, "lean_wall_s": 7.804399013519287, "lean_proc_s": 28.1549711227417, "workers": 96, "chunk": 400}
|
| 105 |
+
{"utc": "2026-10-14T20:11:12Z", "samples": 12000, "parse_fail": 3801, "distinct_checked": 8150, "lean_ok": 6105, "lean_rej": 2045, "parse_reasons": {"unbound": 1742, "no-eos": 729, "expected (": 694, "name": 181, "expected ;": 140, "formula": 103, "ORE box": 75, "exact not last": 60, "expected )": 28, "PR position": 15, "expected :=": 11, "expected :": 10, "expected False": 9, "expected None": 2, "expected exact": 1, "trailing tokens": 1}, "lean_wall_s": 27.627671480178833, "lean_proc_s": 555.0783824920654, "workers": 96, "chunk": 400}
|
| 106 |
+
{"utc": "2026-10-14T20:12:23Z", "samples": 4968, "parse_fail": 3519, "distinct_checked": 1432, "lean_ok": 23, "lean_rej": 1409, "parse_reasons": {"unbound": 1525, "expected (": 747, "no-eos": 597, "name": 151, "expected ;": 148, "formula": 103, "ORE box": 91, "exact not last": 44, "PR position": 41, "expected )": 25, "expected :=": 17, "expected :": 17, "expected =>": 6, "expected False": 5, "expected exact": 2}, "lean_wall_s": 8.424697637557983, "lean_proc_s": 33.157647371292114, "workers": 96, "chunk": 400}
|
| 107 |
+
{"utc": "2026-10-14T20:13:36Z", "samples": 4944, "parse_fail": 3474, "distinct_checked": 1453, "lean_ok": 16, "lean_rej": 1437, "parse_reasons": {"unbound": 1607, "expected (": 734, "no-eos": 559, "expected ;": 151, "name": 134, "formula": 129, "ORE box": 54, "exact not last": 40, "expected :=": 31, "expected )": 22, "PR position": 5, "expected :": 3, "trailing tokens": 2, "expected None": 1, "eof": 1, "expected False": 1}, "lean_wall_s": 8.364802122116089, "lean_proc_s": 31.246862173080444, "workers": 96, "chunk": 400}
|
| 108 |
+
{"utc": "2026-10-14T20:14:38Z", "samples": 4944, "parse_fail": 3394, "distinct_checked": 1537, "lean_ok": 16, "lean_rej": 1521, "parse_reasons": {"unbound": 1742, "no-eos": 587, "expected (": 572, "name": 131, "expected ;": 129, "formula": 85, "ORE box": 52, "exact not last": 35, "expected )": 34, "expected False": 9, "expected :": 8, "PR position": 8, "expected :=": 1, "expected exact": 1}, "lean_wall_s": 8.541127681732178, "lean_proc_s": 33.516470432281494, "workers": 96, "chunk": 400}
|
| 109 |
+
{"utc": "2026-10-14T20:17:22Z", "samples": 11992, "parse_fail": 3925, "distinct_checked": 8023, "lean_ok": 5755, "lean_rej": 2268, "parse_reasons": {"unbound": 1705, "expected (": 835, "no-eos": 712, "name": 215, "formula": 127, "expected ;": 116, "ORE box": 65, "exact not last": 41, "expected :": 39, "expected :=": 30, "expected )": 27, "PR position": 9, "expected exact": 2, "expected None": 2}, "lean_wall_s": 28.71376085281372, "lean_proc_s": 561.7388763427734, "workers": 96, "chunk": 400}
|
| 110 |
+
{"utc": "2026-10-14T20:18:44Z", "samples": 5280, "parse_fail": 3664, "distinct_checked": 1605, "lean_ok": 27, "lean_rej": 1578, "parse_reasons": {"unbound": 1637, "expected (": 775, "no-eos": 581, "name": 192, "formula": 134, "expected ;": 114, "ORE box": 55, "exact not last": 55, "expected :=": 46, "expected )": 36, "expected :": 31, "PR position": 7, "expected None": 1}, "lean_wall_s": 8.133599042892456, "lean_proc_s": 32.47133994102478, "workers": 96, "chunk": 400}
|
| 111 |
+
{"utc": "2026-10-14T20:19:41Z", "samples": 5256, "parse_fail": 3398, "distinct_checked": 1835, "lean_ok": 21, "lean_rej": 1814, "parse_reasons": {"unbound": 1482, "expected (": 839, "no-eos": 439, "name": 184, "formula": 100, "ORE box": 94, "expected ;": 80, "expected :": 74, "exact not last": 47, "expected :=": 33, "expected )": 16, "PR position": 9, "expected exact": 1}, "lean_wall_s": 9.935613870620728, "lean_proc_s": 47.463221311569214, "workers": 96, "chunk": 400}
|
| 112 |
+
{"utc": "2026-10-14T20:20:50Z", "samples": 5256, "parse_fail": 3486, "distinct_checked": 1752, "lean_ok": 4, "lean_rej": 1748, "parse_reasons": {"unbound": 1453, "expected (": 936, "no-eos": 564, "expected ;": 136, "formula": 136, "name": 126, "exact not last": 50, "ORE box": 31, "expected )": 20, "expected :": 17, "expected :=": 9, "PR position": 6, "trailing tokens": 1, "expected .elim": 1}, "lean_wall_s": 9.767505407333374, "lean_proc_s": 43.603543281555176, "workers": 96, "chunk": 400}
|
| 113 |
+
{"utc": "2026-10-14T20:23:34Z", "samples": 11992, "parse_fail": 3984, "distinct_checked": 7967, "lean_ok": 5584, "lean_rej": 2383, "parse_reasons": {"unbound": 1783, "expected (": 777, "no-eos": 640, "name": 250, "formula": 168, "expected ;": 142, "ORE box": 70, "expected :": 47, "exact not last": 46, "PR position": 29, "expected )": 21, "expected :=": 5, "trailing tokens": 3, "expected None": 2, "expected exact": 1}, "lean_wall_s": 24.8245952129364, "lean_proc_s": 477.6820650100708, "workers": 96, "chunk": 400}
|
| 114 |
+
{"utc": "2026-10-14T20:24:47Z", "samples": 5400, "parse_fail": 3560, "distinct_checked": 1823, "lean_ok": 12, "lean_rej": 1811, "parse_reasons": {"unbound": 1652, "expected (": 810, "no-eos": 464, "name": 175, "formula": 137, "ORE box": 98, "expected ;": 96, "expected :": 42, "PR position": 34, "exact not last": 26, "expected )": 21, "trailing tokens": 2, "eof": 1, "expected None": 1, "expected by": 1}, "lean_wall_s": 9.643616199493408, "lean_proc_s": 45.403026819229126, "workers": 96, "chunk": 400}
|
| 115 |
+
{"utc": "2026-10-14T20:26:02Z", "samples": 5400, "parse_fail": 3546, "distinct_checked": 1839, "lean_ok": 27, "lean_rej": 1812, "parse_reasons": {"unbound": 1447, "expected (": 691, "no-eos": 661, "formula": 236, "name": 167, "expected ;": 125, "ORE box": 62, "expected :": 49, "PR position": 46, "exact not last": 32, "expected )": 27, "trailing tokens": 1, "expected None": 1, "expected False": 1}, "lean_wall_s": 9.93486762046814, "lean_proc_s": 46.35853338241577, "workers": 96, "chunk": 400}
|
| 116 |
+
{"utc": "2026-10-14T20:27:11Z", "samples": 5376, "parse_fail": 3431, "distinct_checked": 1911, "lean_ok": 20, "lean_rej": 1891, "parse_reasons": {"unbound": 1534, "expected (": 704, "no-eos": 500, "name": 211, "expected ;": 139, "formula": 124, "ORE box": 64, "PR position": 45, "exact not last": 35, "expected )": 33, "expected :": 22, "expected :=": 17, "trailing tokens": 1, "eof": 1, "expected by": 1}, "lean_wall_s": 9.95835018157959, "lean_proc_s": 48.53930187225342, "workers": 96, "chunk": 400}
|
| 117 |
+
{"utc": "2026-10-14T20:29:50Z", "samples": 12000, "parse_fail": 4103, "distinct_checked": 7864, "lean_ok": 5156, "lean_rej": 2708, "parse_reasons": {"unbound": 1679, "expected (": 936, "no-eos": 548, "name": 342, "expected ;": 166, "ORE box": 131, "formula": 116, "expected :": 57, "expected )": 47, "exact not last": 37, "PR position": 31, "expected :=": 9, "expected False": 2, "expected None": 1, "eof": 1}, "lean_wall_s": 26.882859468460083, "lean_proc_s": 522.7353072166443, "workers": 96, "chunk": 400}
|
| 118 |
+
{"utc": "2026-10-14T20:31:09Z", "samples": 5976, "parse_fail": 3795, "distinct_checked": 2155, "lean_ok": 24, "lean_rej": 2131, "parse_reasons": {"unbound": 1541, "expected (": 917, "no-eos": 478, "name": 279, "ORE box": 140, "expected ;": 135, "formula": 92, "PR position": 57, "expected :": 50, "exact not last": 49, "expected )": 45, "expected :=": 9, "expected None": 2, "expected False": 1}, "lean_wall_s": 10.192790746688843, "lean_proc_s": 55.98514914512634, "workers": 96, "chunk": 400}
|
| 119 |
+
{"utc": "2026-10-14T20:32:28Z", "samples": 5976, "parse_fail": 3783, "distinct_checked": 2163, "lean_ok": 14, "lean_rej": 2149, "parse_reasons": {"unbound": 1595, "expected (": 987, "no-eos": 344, "name": 302, "expected ;": 137, "ORE box": 124, "formula": 105, "expected :": 65, "expected )": 44, "exact not last": 36, "PR position": 21, "expected :=": 19, "expected exact": 2, "expected None": 1, "trailing tokens": 1}, "lean_wall_s": 12.131192207336426, "lean_proc_s": 65.96758103370667, "workers": 96, "chunk": 400}
|
| 120 |
+
{"utc": "2026-10-14T20:33:31Z", "samples": 5952, "parse_fail": 3613, "distinct_checked": 2314, "lean_ok": 17, "lean_rej": 2297, "parse_reasons": {"unbound": 1486, "expected (": 898, "no-eos": 491, "name": 294, "expected ;": 161, "formula": 81, "ORE box": 55, "expected :": 47, "exact not last": 33, "PR position": 29, "expected )": 26, "expected :=": 9, "expected False": 3}, "lean_wall_s": 12.402976274490356, "lean_proc_s": 72.46223306655884, "workers": 96, "chunk": 400}
|
| 121 |
+
{"utc": "2026-10-14T20:36:12Z", "samples": 12000, "parse_fail": 4829, "distinct_checked": 7130, "lean_ok": 4703, "lean_rej": 2427, "parse_reasons": {"unbound": 2079, "no-eos": 1084, "expected (": 741, "name": 288, "expected :": 183, "expected ;": 128, "formula": 99, "expected )": 97, "ORE box": 75, "exact not last": 42, "PR position": 10, "expected :=": 2, "expected False": 1}, "lean_wall_s": 24.578913927078247, "lean_proc_s": 430.0828492641449, "workers": 96, "chunk": 400}
|
| 122 |
+
{"utc": "2026-10-14T20:37:20Z", "samples": 6480, "parse_fail": 4413, "distinct_checked": 2034, "lean_ok": 24, "lean_rej": 2010, "parse_reasons": {"unbound": 1948, "no-eos": 969, "expected (": 735, "name": 202, "expected :": 166, "expected )": 129, "expected ;": 103, "ORE box": 56, "formula": 56, "exact not last": 38, "expected False": 5, "PR position": 5, "expected None": 1}, "lean_wall_s": 11.133482217788696, "lean_proc_s": 55.83375310897827, "workers": 96, "chunk": 400}
|
| 123 |
+
{"utc": "2026-10-14T20:38:32Z", "samples": 6480, "parse_fail": 4317, "distinct_checked": 2136, "lean_ok": 30, "lean_rej": 2106, "parse_reasons": {"unbound": 1944, "no-eos": 818, "expected (": 670, "name": 202, "expected :": 160, "formula": 131, "ORE box": 112, "expected )": 105, "expected ;": 102, "exact not last": 35, "PR position": 22, "expected :=": 10, "expected False": 5, "expected None": 1}, "lean_wall_s": 11.430583715438843, "lean_proc_s": 61.5658495426178, "workers": 96, "chunk": 400}
|
| 124 |
+
{"utc": "2026-10-14T20:39:49Z", "samples": 6480, "parse_fail": 4600, "distinct_checked": 1860, "lean_ok": 21, "lean_rej": 1839, "parse_reasons": {"unbound": 1919, "no-eos": 950, "expected (": 783, "name": 274, "expected :": 233, "expected ;": 135, "formula": 101, "expected )": 97, "ORE box": 76, "exact not last": 19, "expected :=": 6, "expected False": 3, "PR position": 2, "eof": 1, "expected None": 1}, "lean_wall_s": 10.013324499130249, "lean_proc_s": 48.10184717178345, "workers": 96, "chunk": 400}
|
| 125 |
+
{"utc": "2026-10-14T20:42:33Z", "samples": 12008, "parse_fail": 4538, "distinct_checked": 7427, "lean_ok": 4932, "lean_rej": 2495, "parse_reasons": {"unbound": 1885, "expected (": 1011, "no-eos": 743, "name": 258, "ORE box": 143, "expected ;": 124, "formula": 111, "expected :": 103, "expected )": 85, "exact not last": 25, "expected :=": 23, "expected False": 13, "PR position": 9, "expected None": 2, "trailing tokens": 1, "expected =>": 1, "eof": 1}, "lean_wall_s": 26.14779806137085, "lean_proc_s": 482.1890528202057, "workers": 96, "chunk": 400}
|
| 126 |
+
{"utc": "2026-10-14T20:43:58Z", "samples": 6336, "parse_fail": 4037, "distinct_checked": 2276, "lean_ok": 27, "lean_rej": 2249, "parse_reasons": {"unbound": 1648, "expected (": 957, "no-eos": 658, "name": 235, "expected :": 98, "ORE box": 97, "formula": 96, "expected ;": 84, "expected )": 82, "exact not last": 37, "expected :=": 24, "PR position": 14, "expected False": 4, "expected =>": 3}, "lean_wall_s": 13.277703523635864, "lean_proc_s": 78.30571365356445, "workers": 96, "chunk": 400}
|
| 127 |
+
{"utc": "2026-10-14T20:45:04Z", "samples": 6336, "parse_fail": 4203, "distinct_checked": 2112, "lean_ok": 24, "lean_rej": 2088, "parse_reasons": {"unbound": 1699, "expected (": 1041, "no-eos": 616, "name": 206, "ORE box": 178, "expected ;": 113, "formula": 102, "expected )": 99, "expected :": 88, "exact not last": 38, "expected :=": 16, "expected False": 6, "PR position": 1}, "lean_wall_s": 11.668589115142822, "lean_proc_s": 61.483195781707764, "workers": 96, "chunk": 400}
|
| 128 |
+
{"utc": "2026-10-14T20:46:14Z", "samples": 6336, "parse_fail": 4343, "distinct_checked": 1968, "lean_ok": 14, "lean_rej": 1954, "parse_reasons": {"unbound": 1936, "expected (": 943, "no-eos": 598, "name": 230, "formula": 143, "expected :": 124, "expected ;": 120, "ORE box": 103, "expected )": 78, "exact not last": 33, "PR position": 14, "expected :=": 10, "expected False": 8, "expected exact": 1, "expected None": 1, "expected =>": 1}, "lean_wall_s": 7.509993553161621, "lean_proc_s": 37.14505624771118, "workers": 96, "chunk": 400}
|
| 129 |
+
{"utc": "2026-10-14T20:48:50Z", "samples": 12000, "parse_fail": 4657, "distinct_checked": 7311, "lean_ok": 4991, "lean_rej": 2320, "parse_reasons": {"unbound": 1856, "expected (": 1101, "no-eos": 896, "name": 221, "ORE box": 170, "formula": 122, "expected ;": 110, "expected :": 77, "expected )": 50, "exact not last": 41, "expected False": 5, "PR position": 3, "expected :=": 2, "expected \u27e9": 1, "expected None": 1, "trailing tokens": 1}, "lean_wall_s": 20.15104055404663, "lean_proc_s": 356.4649248123169, "workers": 96, "chunk": 400}
|
| 130 |
+
{"utc": "2026-10-14T20:49:52Z", "samples": 6216, "parse_fail": 4262, "distinct_checked": 1932, "lean_ok": 17, "lean_rej": 1915, "parse_reasons": {"unbound": 1706, "expected (": 1082, "no-eos": 713, "name": 247, "ORE box": 180, "expected ;": 100, "formula": 71, "expected )": 60, "expected :": 58, "exact not last": 28, "PR position": 9, "trailing tokens": 3, "expected False": 3, "expected None": 2}, "lean_wall_s": 6.365325927734375, "lean_proc_s": 31.387881994247437, "workers": 96, "chunk": 400}
|
| 131 |
+
{"utc": "2026-10-14T20:51:12Z", "samples": 6216, "parse_fail": 4258, "distinct_checked": 1935, "lean_ok": 8, "lean_rej": 1927, "parse_reasons": {"unbound": 1613, "expected (": 1067, "no-eos": 832, "ORE box": 191, "name": 164, "formula": 146, "expected ;": 80, "expected )": 68, "expected :": 60, "exact not last": 23, "expected False": 7, "PR position": 3, "expected :=": 2, "trailing tokens": 1, "expected None": 1}, "lean_wall_s": 5.868977308273315, "lean_proc_s": 28.670335054397583, "workers": 96, "chunk": 400}
|
| 132 |
+
{"utc": "2026-10-14T20:52:25Z", "samples": 6192, "parse_fail": 4328, "distinct_checked": 1847, "lean_ok": 27, "lean_rej": 1820, "parse_reasons": {"unbound": 1933, "expected (": 1074, "no-eos": 605, "name": 215, "expected ;": 109, "ORE box": 104, "expected :": 99, "formula": 92, "expected )": 43, "exact not last": 37, "expected False": 9, "PR position": 5, "trailing tokens": 2, "expected :=": 1}, "lean_wall_s": 6.047676086425781, "lean_proc_s": 28.51197123527527, "workers": 96, "chunk": 400}
|
| 133 |
+
{"utc": "2026-10-14T20:55:06Z", "samples": 12000, "parse_fail": 4801, "distinct_checked": 7166, "lean_ok": 5107, "lean_rej": 2059, "parse_reasons": {"unbound": 1908, "expected (": 1295, "no-eos": 928, "name": 255, "expected ;": 130, "ORE box": 100, "formula": 47, "expected :": 47, "exact not last": 43, "expected )": 23, "PR position": 12, "expected :=": 6, "trailing tokens": 5, "expected None": 2}, "lean_wall_s": 19.466183185577393, "lean_proc_s": 332.34922671318054, "workers": 96, "chunk": 400}
|
| 134 |
+
{"utc": "2026-10-14T20:56:21Z", "samples": 6096, "parse_fail": 4436, "distinct_checked": 1648, "lean_ok": 16, "lean_rej": 1632, "parse_reasons": {"unbound": 1663, "expected (": 1251, "no-eos": 933, "name": 250, "expected ;": 149, "ORE box": 58, "exact not last": 43, "formula": 34, "expected :": 21, "expected )": 17, "PR position": 12, "trailing tokens": 3, "expected :=": 1, "expected None": 1}, "lean_wall_s": 6.635918617248535, "lean_proc_s": 27.538654804229736, "workers": 96, "chunk": 400}
|
| 135 |
+
{"utc": "2026-10-14T20:57:30Z", "samples": 6096, "parse_fail": 4444, "distinct_checked": 1640, "lean_ok": 20, "lean_rej": 1620, "parse_reasons": {"unbound": 1762, "expected (": 1466, "no-eos": 557, "name": 195, "expected ;": 126, "ORE box": 123, "expected :": 78, "exact not last": 44, "formula": 38, "expected )": 29, "expected :=": 16, "PR position": 6, "trailing tokens": 2, "expected exact": 1, "expected None": 1}, "lean_wall_s": 5.893754243850708, "lean_proc_s": 24.022990465164185, "workers": 96, "chunk": 400}
|
| 136 |
+
{"utc": "2026-10-14T20:58:46Z", "samples": 6096, "parse_fail": 4257, "distinct_checked": 1820, "lean_ok": 15, "lean_rej": 1805, "parse_reasons": {"unbound": 1741, "expected (": 1002, "no-eos": 870, "name": 271, "expected ;": 99, "ORE box": 96, "formula": 60, "exact not last": 35, "expected :": 30, "expected )": 20, "PR position": 13, "expected :=": 12, "trailing tokens": 6, "expected False": 1, "expected None": 1}, "lean_wall_s": 8.757471561431885, "lean_proc_s": 40.82920718193054, "workers": 96, "chunk": 400}
|
| 137 |
+
{"utc": "2026-10-14T21:15:54Z", "samples": 53952, "parse_fail": 29669, "distinct_checked": 23289, "lean_ok": 11961, "lean_rej": 11328, "parse_reasons": {"unbound": 9630, "expected (": 6281, "no-eos": 5887, "formula": 3812, "expected ;": 1566, "name": 926, "PR position": 601, "exact not last": 575, "expected )": 195, "ORE box": 117, "expected :=": 22, "expected None": 16, "expected :": 14, "expected exact": 13, "trailing tokens": 7, "eof": 6, "expected =>": 1}, "lean_wall_s": 182.96678066253662, "lean_proc_s": 10275.160593509674, "workers": 96, "chunk": 400}
|
| 138 |
+
{"utc": "2026-10-14T21:17:26Z", "samples": 17728, "parse_fail": 3245, "distinct_checked": 14288, "lean_ok": 11020, "lean_rej": 3268, "parse_reasons": {"unbound": 1915, "no-eos": 701, "name": 202, "expected (": 138, "expected ;": 99, "ORE box": 99, "formula": 38, "expected )": 18, "exact not last": 13, "PR position": 8, "expected :": 8, "trailing tokens": 2, "expected =>": 2, "expected :=": 1, "expected None": 1}, "lean_wall_s": 43.88870143890381, "lean_proc_s": 1521.5249931812286, "workers": 96, "chunk": 400}
|
runs/long-rl-20261014-v2_untied-ei_s1/pretrain.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5d53e9dd9240306992cc2e133a1faf00c27e3d341dd2e7d190d4cb7765479714
|
| 3 |
+
size 38588451
|
runs/long-rl-20261014-v2_untied-ei_s1/source/harness.py
ADDED
|
@@ -0,0 +1,435 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
# imports the fork (model, tokenizer, train, ...), which is not resolvable here
|
| 3 |
+
# pyright: reportMissingImports=false
|
| 4 |
+
"""Harness for long_rl_20261014: pretrain on <= 6 body lines, then RL on the long-proofs set (body 7 to ~45).
|
| 5 |
+
|
| 6 |
+
Forked from the test-time-loops harness (autoresearch_loops/harness.py, sha 2eec1b61666a, the one run 590 and 512 were
|
| 7 |
+
scored with). What changed (Robbie, 2026-10-14):
|
| 8 |
+
|
| 9 |
+
- Lean is the only checker: overlay/lean_gate.py, and a sample counts iff Lean accepts its literal text. nd_verify is
|
| 10 |
+
never called; the decoded ND proof is used only to count lines.
|
| 11 |
+
- Length is body lines (proof lines after the premise block) everywhere: bands, L*, the pretraining cap.
|
| 12 |
+
- Tokenizer: overlay/lean_tok.py (ten atoms, ten premises, 160 names). The overlay directory is first on sys.path.
|
| 13 |
+
- Data (make_data.py): pretraining = Dan's cap-6 file with distractor premises; RL targets / dev / holdout = the
|
| 14 |
+
long-proofs set, band-capped. The old transfer dev pool is evaluated too (EVAL_K_OLD samples), to catch forgetting.
|
| 15 |
+
- The oracle also returns every rollout (token ids and the Lean verdict) for the GRPO arm: sample_rollouts().
|
| 16 |
+
- Sampling batch size follows the model's KV-cache depth (model.kv_layers()), so 8-loop sampling fits an A40.
|
| 17 |
+
|
| 18 |
+
1. pretrain.pretrain(train_recs, PRETRAIN_SECS, ...) -> (model, tok, stats), charged for everything
|
| 19 |
+
2. checks: params <= PARAM_CAP, tok.mode == TOK_MODE, model has an integer KNOB
|
| 20 |
+
3. pretrain.rl(model, tok, targets, train_recs, oracle, RL_SECS, ...): the arm's RL. RL_SECS is wall-clock inside rl()
|
| 21 |
+
minus Lean-gate time.
|
| 22 |
+
4. eval: dev (long-proofs set) with EVAL_K samples at EVAL_LEVEL, by band; old transfer dev with EVAL_K_OLD samples
|
| 23 |
+
5. SIGALRM at TOTAL_SECS
|
| 24 |
+
|
| 25 |
+
python3 harness.py --out artifacts/x [--seed 0] # on the pod (remote.sh)
|
| 26 |
+
ND_FORK=... python3 harness.py --out /tmp/smoke --smoke # CPU plumbing check with real Lean
|
| 27 |
+
"""
|
| 28 |
+
|
| 29 |
+
import argparse
|
| 30 |
+
import collections
|
| 31 |
+
import gzip
|
| 32 |
+
import hashlib
|
| 33 |
+
import importlib
|
| 34 |
+
import json
|
| 35 |
+
import os
|
| 36 |
+
import random
|
| 37 |
+
import shutil
|
| 38 |
+
import signal
|
| 39 |
+
import sys
|
| 40 |
+
import time
|
| 41 |
+
import traceback
|
| 42 |
+
from pathlib import Path
|
| 43 |
+
|
| 44 |
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
| 45 |
+
FORK = os.environ.get("ND_FORK", "/workspace/nd-takehome")
|
| 46 |
+
DATA = os.environ.get("ND_DATA", os.path.join(HERE, "data"))
|
| 47 |
+
# overlay first (tokenizer, Lean-only gate), then the fork (model, sample, train), then this directory (pretrain.py)
|
| 48 |
+
sys.path[:0] = [os.path.join(HERE, "overlay"), FORK, HERE]
|
| 49 |
+
import importlib.util # noqa: E402
|
| 50 |
+
|
| 51 |
+
import torch # noqa: E402
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def _overlay(name):
|
| 55 |
+
"""Load overlay/<name>.py under the module name the fork imports. Fork modules put the fork directory at
|
| 56 |
+
sys.path[0] whenever they are imported (nd2lean does, inside lean_tok), so path order alone is not enough:
|
| 57 |
+
registering the module in sys.modules first is what makes every later `import lean_tok` get the overlay."""
|
| 58 |
+
spec = importlib.util.spec_from_file_location(name, os.path.join(HERE, "overlay", f"{name}.py"))
|
| 59 |
+
mod = importlib.util.module_from_spec(spec)
|
| 60 |
+
sys.modules[name] = mod
|
| 61 |
+
spec.loader.exec_module(mod)
|
| 62 |
+
return mod
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
_overlay("lean_tok")
|
| 66 |
+
gate = _overlay("lean_gate").gate
|
| 67 |
+
from eval_set import wilson # noqa: E402 (fork; wilson interval only)
|
| 68 |
+
from nd_verify.verify import parse_proof_tokens # noqa: E402 (a parser: line counts only, never a verdict)
|
| 69 |
+
from sample import generate_ids # noqa: E402 (fork)
|
| 70 |
+
|
| 71 |
+
# ---------------------------------------------------------------- the rules
|
| 72 |
+
TOTAL_SECS = int(os.environ.get("ND_TOTAL_SECS", 6.5 * 3600)) # RL_SECS excludes Lean-gate time (~80 s a round)
|
| 73 |
+
PRETRAIN_SECS = 600 # Dan's file with distractors is ~1.6x the tokens of the plain cap-6 file
|
| 74 |
+
PRETRAIN_GRACE = 30
|
| 75 |
+
RL_SECS = int(os.environ.get("ND_RL_SECS", 3 * 3600))
|
| 76 |
+
RL_GRACE = 120
|
| 77 |
+
PARAM_CAP = 10_000_000
|
| 78 |
+
TEMP = 0.8
|
| 79 |
+
MAX_NEW = 2304 # proof tokens: approach E's 80-line references run to ~1,900; checked against the pools at load
|
| 80 |
+
KV_TOKENS = 1024 * 6 * 1700 # sampling batch * KV layers * tokens: 6 layers x 1,024 x 1,700 is ~16 GB of bf16 KV
|
| 81 |
+
EVAL_K, EVAL_K_OLD = 64, 16
|
| 82 |
+
KNOB = "n_loops"
|
| 83 |
+
EVAL_LEVEL, DIAG_LEVEL = int(os.environ.get("ND_EVAL_LEVEL", 8)), 2
|
| 84 |
+
TOK_MODE = "lean_seq"
|
| 85 |
+
BANDS = ((0, 6), (7, 9), (10, 14), (15, 19), (20, 24), (25, 34), (35, 49), (50, 200))
|
| 86 |
+
DEV = "cuda" if torch.cuda.is_available() else "cpu"
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def read(fn):
|
| 90 |
+
op = gzip.open if fn.endswith(".gz") else open
|
| 91 |
+
return [json.loads(line) for line in op(fn, "rt") if line.strip()]
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def log(msg):
|
| 95 |
+
print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True)
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def sha(fn):
|
| 99 |
+
return hashlib.sha256(open(fn, "rb").read()).hexdigest()[:12]
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
def band(body):
|
| 103 |
+
return next(f"{lo}-{hi}" if hi < 200 else f"{lo}+" for lo, hi in BANDS if lo <= body <= hi)
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
class Timeout(Exception):
|
| 107 |
+
pass
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
class BudgetSpent(Timeout):
|
| 111 |
+
pass
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
# ---------------------------------------------------------------- sampling and judging
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def body_lines(nd):
|
| 118 |
+
"""(written body lines, pruned body lines) of a decoded ND proof: lines after the premise block, and those the
|
| 119 |
+
last line depends on through citations (a box rule cites the box's first and last lines, so the closure keeps a
|
| 120 |
+
needed box's hypothesis and every line its conclusion depends on)."""
|
| 121 |
+
lines = parse_proof_tokens(nd.split())
|
| 122 |
+
by = {ln["idx"]: ln for ln in lines}
|
| 123 |
+
need, todo = set(), [lines[-1]["idx"]]
|
| 124 |
+
while todo:
|
| 125 |
+
i = todo.pop()
|
| 126 |
+
if i not in need:
|
| 127 |
+
need.add(i)
|
| 128 |
+
todo += by[i]["refs"]
|
| 129 |
+
body = [ln for ln in lines if ln["rule"] != "PR"]
|
| 130 |
+
return len(body), sum(1 for ln in body if ln["idx"] in need)
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
def kv_batch(model, n_tokens):
|
| 134 |
+
layers = model.kv_layers() if hasattr(model, "kv_layers") else len(model.blocks)
|
| 135 |
+
return max(32, min(1024, KV_TOKENS // (layers * n_tokens)))
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
def new_tokens(body, max_new):
|
| 139 |
+
"""Sampling cap for a target of `body` lines: reference proofs run up to ~23 tokens per body line (16 for most), so
|
| 140 |
+
64 + 30 per line leaves >= 1.3x room, and a sample rambling without <eos> no longer holds a whole batch to max_new.
|
| 141 |
+
Rounded up to a multiple of 128 so that a few caps group the samples into full batches."""
|
| 142 |
+
return min(max_new, -(-(64 + 30 * body) // 128) * 128)
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
def sample(model, tok, recs, k, seed, max_new):
|
| 146 |
+
"""k samples per record at temperature TEMP. Returns per record a list of (ids, decoded ND proof, accepted)."""
|
| 147 |
+
prompts = [r["prompt"] for r in recs for _ in range(k)]
|
| 148 |
+
caps = [new_tokens(r["body"], max_new) for r in recs for _ in range(k)]
|
| 149 |
+
ids = [tok.encode_prompt(p) for p in prompts]
|
| 150 |
+
gen = torch.Generator(device=DEV)
|
| 151 |
+
gen.manual_seed(seed)
|
| 152 |
+
outs, nds, texts = [None] * len(prompts), [None] * len(prompts), [None] * len(prompts)
|
| 153 |
+
for cap in sorted(set(caps)):
|
| 154 |
+
order = sorted((i for i in range(len(prompts)) if caps[i] == cap), key=lambda i: len(ids[i]))
|
| 155 |
+
batch = kv_batch(model, len(ids[order[-1]]) + cap)
|
| 156 |
+
for s in range(0, len(order), batch):
|
| 157 |
+
chunk = order[s : s + batch]
|
| 158 |
+
with torch.autocast("cuda", dtype=torch.bfloat16, enabled=(DEV == "cuda")):
|
| 159 |
+
res = generate_ids(model, tok, [ids[i] for i in chunk], False, TEMP, cap, gen)
|
| 160 |
+
for i, o in zip(chunk, res, strict=True):
|
| 161 |
+
o = o[: o.index(tok.eos) + 1] if tok.eos in o else o
|
| 162 |
+
outs[i], nds[i], texts[i] = o, tok.decode(o), tok.last_text
|
| 163 |
+
if DEV == "cuda":
|
| 164 |
+
torch.cuda.empty_cache()
|
| 165 |
+
gated = gate(tok, prompts, nds, texts)
|
| 166 |
+
flat = [(o, nd, not nd.startswith("LEAN")) for o, nd in zip(outs, gated, strict=True)]
|
| 167 |
+
return [flat[i * k : (i + 1) * k] for i in range(len(recs))]
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
def judge(recs, per):
|
| 171 |
+
rows = []
|
| 172 |
+
for r, samples in zip(recs, per, strict=True):
|
| 173 |
+
good = {}
|
| 174 |
+
for _, nd, ok in samples:
|
| 175 |
+
if ok and nd not in good:
|
| 176 |
+
good[nd] = body_lines(nd)
|
| 177 |
+
rows.append({"name": r["name"], "body": r["body"], "solved": bool(good), "n_ok": sum(ok for *_, ok in samples),
|
| 178 |
+
"n_tried": len(samples), "proofs": list(good), "written": [w for w, _ in good.values()],
|
| 179 |
+
"pruned": [p for _, p in good.values()]})
|
| 180 |
+
return rows
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
def lstar(solved, recs, need=5):
|
| 184 |
+
ls = [r["body"] for r in recs if r["name"] in solved]
|
| 185 |
+
ge = {L: sum(1 for x in ls if x >= L) for L in range(7, 81)}
|
| 186 |
+
return max([L for L, c in ge.items() if c >= need], default=0), ge
|
| 187 |
+
|
| 188 |
+
|
| 189 |
+
def by_band(solved, recs):
|
| 190 |
+
by = collections.defaultdict(lambda: [0, 0])
|
| 191 |
+
for r in recs:
|
| 192 |
+
by[band(r["body"])][0] += r["name"] in solved
|
| 193 |
+
by[band(r["body"])][1] += 1
|
| 194 |
+
return {b: {"solved": k, "n": n, "rate": k / n, "ci": list(wilson(k, n))} for b, (k, n) in sorted(by.items())}
|
| 195 |
+
|
| 196 |
+
|
| 197 |
+
def gate_seconds():
|
| 198 |
+
fn = os.environ.get("LEAN_GATE_LOG")
|
| 199 |
+
return sum(r["lean_wall_s"] for r in read(fn)) if fn and os.path.exists(fn) else 0.0
|
| 200 |
+
|
| 201 |
+
|
| 202 |
+
# ---------------------------------------------------------------- the RL oracle
|
| 203 |
+
|
| 204 |
+
|
| 205 |
+
class Oracle:
|
| 206 |
+
"""What rl() gets instead of data: the model's own attempts at the RL targets, judged by Lean."""
|
| 207 |
+
|
| 208 |
+
def __init__(self, a, tok, targets):
|
| 209 |
+
self.a, self.tok = a, tok
|
| 210 |
+
self.targets = {t["name"]: t for t in targets}
|
| 211 |
+
self.found = collections.defaultdict(dict) # name -> {proof text: {written, pruned, round}}
|
| 212 |
+
self.calls = []
|
| 213 |
+
self.t0 = self.last = time.time()
|
| 214 |
+
self.uncharged = 0.0
|
| 215 |
+
|
| 216 |
+
def used(self):
|
| 217 |
+
return time.time() - self.t0 - self.uncharged
|
| 218 |
+
|
| 219 |
+
def left(self):
|
| 220 |
+
return self.a.rl_secs - self.used()
|
| 221 |
+
|
| 222 |
+
def _sample(self, model, targets, k):
|
| 223 |
+
if self.left() <= 0:
|
| 224 |
+
raise BudgetSpent(f"oracle called with {self.used():.0f}s of {self.a.rl_secs}s used")
|
| 225 |
+
recs = [self.targets[t["name"]] for t in targets]
|
| 226 |
+
call = len(self.calls) + 1
|
| 227 |
+
model.eval()
|
| 228 |
+
t0, g0 = time.time(), gate_seconds()
|
| 229 |
+
per = sample(model, self.tok, recs, k, self.a.seed * 100_000 + call, self.a.max_new)
|
| 230 |
+
secs, gate_s = time.time() - t0, gate_seconds() - g0
|
| 231 |
+
self.uncharged += gate_s
|
| 232 |
+
rows = judge(recs, per)
|
| 233 |
+
new = 0
|
| 234 |
+
for t, row in zip(recs, rows, strict=True):
|
| 235 |
+
for p, w, pl in zip(row["proofs"], row["written"], row["pruned"], strict=True):
|
| 236 |
+
if p not in self.found[t["name"]]:
|
| 237 |
+
self.found[t["name"]][p] = {"written": w, "pruned": pl, "round": call}
|
| 238 |
+
new += 1
|
| 239 |
+
solved = {n for n, d in self.found.items() if d}
|
| 240 |
+
ls, _ = lstar(solved, list(self.targets.values()))
|
| 241 |
+
n = len(recs) * k
|
| 242 |
+
self.calls.append({"round": call, "level": getattr(model, KNOB, None), "k": k, "n_targets": len(recs),
|
| 243 |
+
"sample_acc": sum(x["n_ok"] for x in rows) / max(n, 1), "new_proofs": new,
|
| 244 |
+
"targets_solved": len(solved), "lstar": ls,
|
| 245 |
+
"solved_by_band": {b: v["solved"] for b, v in by_band(solved, list(self.targets.values())).items()},
|
| 246 |
+
"sampling": {"samples": n, "secs": secs, "gate_secs": gate_s},
|
| 247 |
+
"secs": time.time() - self.last, "rl_secs_used": self.used()})
|
| 248 |
+
self.last = time.time()
|
| 249 |
+
c = self.calls[-1]
|
| 250 |
+
log(f"oracle {call}: acc {c['sample_acc']:.4f}, +{new} proofs, solved {len(solved)}/{len(self.targets)}, "
|
| 251 |
+
f"L*={ls}, {KNOB} {c['level']}, {len(recs)} targets x {k}, {n / secs:.0f} samples/s, "
|
| 252 |
+
f"RL clock {c['rl_secs_used']:.0f}/{self.a.rl_secs}s")
|
| 253 |
+
return recs, per, rows
|
| 254 |
+
|
| 255 |
+
def sample(self, model, targets, k):
|
| 256 |
+
"""k attempts per target. Returns per target {name, prompt, body, proofs (distinct Lean-accepted ND proofs),
|
| 257 |
+
n_ok, n_tried}. Raises BudgetSpent once the budget is used up."""
|
| 258 |
+
recs, _, rows = self._sample(model, targets, k)
|
| 259 |
+
return [{"name": t["name"], "prompt": t["prompt"], "body": t["body"], "proofs": row["proofs"],
|
| 260 |
+
"n_ok": row["n_ok"], "n_tried": row["n_tried"]} for t, row in zip(recs, rows, strict=True)]
|
| 261 |
+
|
| 262 |
+
def sample_rollouts(self, model, targets, k):
|
| 263 |
+
"""As sample(), plus `rollouts`: every attempt's proof token ids (eos included when the model ended it) and
|
| 264 |
+
its Lean verdict, for policy-gradient RL."""
|
| 265 |
+
recs, per, rows = self._sample(model, targets, k)
|
| 266 |
+
return [{"name": t["name"], "prompt": t["prompt"], "body": t["body"], "proofs": row["proofs"],
|
| 267 |
+
"n_ok": row["n_ok"], "n_tried": row["n_tried"], "rollouts": [(o, ok) for o, _, ok in samples]}
|
| 268 |
+
for t, row, samples in zip(recs, rows, per, strict=True)]
|
| 269 |
+
|
| 270 |
+
|
| 271 |
+
# ---------------------------------------------------------------- stages
|
| 272 |
+
|
| 273 |
+
|
| 274 |
+
def stage_rl(a, agent, tok, model, targets, train_recs):
|
| 275 |
+
oracle = Oracle(a, tok, targets)
|
| 276 |
+
visible = [{"name": t["name"], "prompt": t["prompt"], "body": t["body"], "band": t["band"]} for t in targets]
|
| 277 |
+
stats = agent.rl(model, tok, visible, train_recs, oracle, a.rl_secs, DEV, a.seed, log)
|
| 278 |
+
if DEV == "cuda":
|
| 279 |
+
torch.cuda.synchronize()
|
| 280 |
+
model.eval()
|
| 281 |
+
with open(f"{a.out}/found_targets.jsonl", "w") as f:
|
| 282 |
+
for t in targets:
|
| 283 |
+
for p, x in oracle.found[t["name"]].items():
|
| 284 |
+
f.write(json.dumps({"name": t["name"], "prompt": t["prompt"], "body": t["body"], "proof": p, **x}) + "\n")
|
| 285 |
+
used = oracle.used()
|
| 286 |
+
log(f"rl returned: {used:.0f}s of {a.rl_secs}s charged, {oracle.uncharged:.0f}s Lean gate")
|
| 287 |
+
return {"rounds": oracle.calls, "secs": time.time() - oracle.t0, "rl_secs_used": used,
|
| 288 |
+
"gate_secs": oracle.uncharged, "targets": len(targets), "stats": dict(stats or {})}
|
| 289 |
+
|
| 290 |
+
|
| 291 |
+
def eval_pool(a, tok, model, pool, k, level, tag):
|
| 292 |
+
setattr(model, KNOB, level)
|
| 293 |
+
model.eval()
|
| 294 |
+
t0 = time.time()
|
| 295 |
+
rows = judge(pool, sample(model, tok, pool, k, a.seed * 100_000 + 7777, a.max_new))
|
| 296 |
+
solved = {r["name"] for r in rows if r["solved"]}
|
| 297 |
+
ls, _ = lstar(solved, pool)
|
| 298 |
+
out = {"level": level, "k": k, "solved": len(solved), "n": len(pool), "lstar": ls, "by_band": by_band(solved, pool),
|
| 299 |
+
"sample_acc": sum(r["n_ok"] for r in rows) / max(k * len(pool), 1),
|
| 300 |
+
"max_pruned": max((max(r["pruned"]) for r in rows if r["solved"]), default=0), "secs": time.time() - t0}
|
| 301 |
+
with open(f"{a.out}/eval_{tag}.jsonl", "w") as f:
|
| 302 |
+
for r in rows:
|
| 303 |
+
f.write(json.dumps(r) + "\n")
|
| 304 |
+
log(f"eval {tag} at {KNOB} {level}: solved {len(solved)}/{len(pool)}, L*={ls}, "
|
| 305 |
+
f"by band { {b: v['solved'] for b, v in out['by_band'].items()} }, {out['secs']:.0f}s")
|
| 306 |
+
return out
|
| 307 |
+
|
| 308 |
+
|
| 309 |
+
def stage_eval(a, tok, model, dev_pool, old_dev):
|
| 310 |
+
out = {"dev": eval_pool(a, tok, model, dev_pool, a.eval_k, a.eval_level, "dev")}
|
| 311 |
+
out["old_dev"] = eval_pool(a, tok, model, old_dev, a.eval_k_old, a.eval_level, "old_dev")
|
| 312 |
+
if not getattr(model, "untied", False):
|
| 313 |
+
out["dev_diag"] = eval_pool(a, tok, model, dev_pool, a.eval_k, a.diag_level, "dev_diag")
|
| 314 |
+
setattr(model, KNOB, a.eval_level)
|
| 315 |
+
return out
|
| 316 |
+
|
| 317 |
+
|
| 318 |
+
def write_summary(a, status, **kw):
|
| 319 |
+
s = {"status": status, "metric": kw.get("eval", {}).get("dev", {}).get("solved"),
|
| 320 |
+
"metric_old_dev": kw.get("eval", {}).get("old_dev", {}).get("solved"),
|
| 321 |
+
"metric_rl": (kw.get("rl", {}).get("rounds") or [{}])[-1].get("targets_solved"),
|
| 322 |
+
"arm": {k: os.environ.get(k) for k in ("ND_ARCH", "ND_RL")}, "harness_sha": sha(f"{HERE}/harness.py"),
|
| 323 |
+
"pretrain_sha": sha(f"{HERE}/pretrain.py"), "device": DEV,
|
| 324 |
+
"gpu": torch.cuda.get_device_name(0) if DEV == "cuda" else "cpu", "torch": torch.__version__,
|
| 325 |
+
"total_secs": time.time() - a.T0,
|
| 326 |
+
"rules": {k: globals()[k] for k in ("TOTAL_SECS", "PRETRAIN_SECS", "RL_SECS", "PARAM_CAP", "MAX_NEW",
|
| 327 |
+
"EVAL_K", "EVAL_K_OLD", "EVAL_LEVEL", "DIAG_LEVEL", "TOK_MODE")},
|
| 328 |
+
**kw}
|
| 329 |
+
with open(f"{a.out}/summary.json", "w") as f:
|
| 330 |
+
json.dump(s, f, indent=1)
|
| 331 |
+
log(f"SUMMARY status={status} metric={s['metric']} old_dev={s['metric_old_dev']} metric_rl={s['metric_rl']} "
|
| 332 |
+
f"total {s['total_secs']:.0f}s")
|
| 333 |
+
return s
|
| 334 |
+
|
| 335 |
+
|
| 336 |
+
def save_checkpoint(a, model, tok, stage):
|
| 337 |
+
path = Path(a.out) / f"{stage}.pt"
|
| 338 |
+
pending = path.with_suffix(".pt.tmp")
|
| 339 |
+
try:
|
| 340 |
+
torch.save({"state": model.state_dict(), "tok_mode": tok.mode, "stage": stage, "seed": a.seed,
|
| 341 |
+
"arm": {k: os.environ.get(k) for k in ("ND_ARCH", "ND_RL")},
|
| 342 |
+
"harness_sha": sha(f"{HERE}/harness.py"), "pretrain_sha": sha(f"{HERE}/pretrain.py")}, pending)
|
| 343 |
+
os.replace(pending, path)
|
| 344 |
+
finally:
|
| 345 |
+
pending.unlink(missing_ok=True)
|
| 346 |
+
|
| 347 |
+
|
| 348 |
+
def load_pools(a):
|
| 349 |
+
train = read(f"{DATA}/pretrain.jsonl.gz")
|
| 350 |
+
assert all(r["body"] <= 6 for r in train), "pretraining file violates the 6-body-line cap"
|
| 351 |
+
targets, dev = read(f"{DATA}/rl.jsonl"), read(f"{DATA}/dev.jsonl")
|
| 352 |
+
old = [r for r in read(f"{FORK}/data/ladder/transfer.jsonl")
|
| 353 |
+
if int(hashlib.sha1(r["key"].encode()).hexdigest(), 16) % 2 == 0] # the loops harness's dev half
|
| 354 |
+
old = [{**r, "body": r["n_lines"] - r["n_prem"]} for r in old]
|
| 355 |
+
tk, dk, pk = ({r["key"] for r in xs} for xs in (targets, dev, train))
|
| 356 |
+
assert not (tk & dk) and not (tk & pk) and not (dk & pk), "theorem leak between pools"
|
| 357 |
+
if a.smoke:
|
| 358 |
+
rng = random.Random(0)
|
| 359 |
+
train = train[:3000]
|
| 360 |
+
targets, dev, old = rng.sample(targets, 12), rng.sample(dev, 6), old[:6]
|
| 361 |
+
return train, targets, dev, old
|
| 362 |
+
|
| 363 |
+
|
| 364 |
+
def main():
|
| 365 |
+
ap = argparse.ArgumentParser()
|
| 366 |
+
ap.add_argument("--out", required=True)
|
| 367 |
+
ap.add_argument("--seed", type=int, default=0)
|
| 368 |
+
ap.add_argument("--pretrain-only", action="store_true")
|
| 369 |
+
ap.add_argument("--smoke", action="store_true", help="tiny everything, for a CPU plumbing check")
|
| 370 |
+
a = ap.parse_args()
|
| 371 |
+
a.pretrain_secs, a.rl_secs, a.eval_k, a.eval_k_old, a.max_new = PRETRAIN_SECS, RL_SECS, EVAL_K, EVAL_K_OLD, MAX_NEW
|
| 372 |
+
a.eval_level, a.diag_level = EVAL_LEVEL, DIAG_LEVEL
|
| 373 |
+
if a.smoke:
|
| 374 |
+
a.pretrain_secs, a.rl_secs, a.eval_k, a.eval_k_old, a.max_new = 20, 60, 2, 2, 160
|
| 375 |
+
os.makedirs(a.out, exist_ok=True)
|
| 376 |
+
src = Path(a.out) / "source"
|
| 377 |
+
src.mkdir(exist_ok=True)
|
| 378 |
+
for fn in ("pretrain.py", "harness.py", "remote.sh", "overlay/lean_tok.py", "overlay/lean_gate.py"):
|
| 379 |
+
shutil.copy2(Path(HERE) / fn, src / os.path.basename(fn))
|
| 380 |
+
a.T0 = time.time()
|
| 381 |
+
torch.manual_seed(a.seed)
|
| 382 |
+
|
| 383 |
+
def on_alarm(signum, frame):
|
| 384 |
+
raise Timeout()
|
| 385 |
+
|
| 386 |
+
signal.signal(signal.SIGALRM, on_alarm)
|
| 387 |
+
signal.alarm(TOTAL_SECS if not a.smoke else 1800)
|
| 388 |
+
train, targets, dev, old = load_pools(a)
|
| 389 |
+
agent = importlib.import_module("pretrain")
|
| 390 |
+
for name in ("lean_tok", "lean_gate"): # the Lean-only gate and the widened tokenizer, nothing from the fork
|
| 391 |
+
assert os.path.dirname(sys.modules[name].__file__) == os.path.join(HERE, "overlay"), sys.modules[name].__file__
|
| 392 |
+
log(f"pools: pretrain {len(train):,}, RL targets {len(targets):,}, dev {len(dev)}, old dev {len(old)}; "
|
| 393 |
+
f"arm {os.environ.get('ND_ARCH')}/{os.environ.get('ND_RL')}; device {DEV}; "
|
| 394 |
+
f"harness {sha(f'{HERE}/harness.py')} pretrain {sha(f'{HERE}/pretrain.py')}")
|
| 395 |
+
pre = {}
|
| 396 |
+
try:
|
| 397 |
+
t0 = time.time()
|
| 398 |
+
model, tok, pre = agent.pretrain(train, a.pretrain_secs, DEV, a.seed, log)
|
| 399 |
+
if DEV == "cuda":
|
| 400 |
+
torch.cuda.synchronize()
|
| 401 |
+
pre = dict(pre or {})
|
| 402 |
+
pre["secs"] = time.time() - t0
|
| 403 |
+
model.eval()
|
| 404 |
+
pre["n_params"] = n_params = sum(p.numel() for p in model.parameters())
|
| 405 |
+
longest = max(len(tok.encode_proof(r["proof"])) for r in targets + dev if r.get("proof")) # ladder: none
|
| 406 |
+
pre["longest_reference_tokens"] = longest
|
| 407 |
+
log(f"pretrain returned after {pre['secs']:.0f}s (budget {a.pretrain_secs}s), {n_params:,} params; "
|
| 408 |
+
f"longest reference proof {longest} tokens, MAX_NEW {a.max_new}")
|
| 409 |
+
assert longest <= a.max_new or a.smoke, f"a reference proof ({longest} tokens) exceeds MAX_NEW"
|
| 410 |
+
if pre["secs"] > a.pretrain_secs + PRETRAIN_GRACE and not a.smoke:
|
| 411 |
+
return write_summary(a, "timeout", pretrain=pre, reason="pretrain overran its budget")
|
| 412 |
+
if tok.mode != TOK_MODE or n_params > PARAM_CAP or not isinstance(getattr(model, KNOB, None), int):
|
| 413 |
+
return write_summary(a, "crash", pretrain=pre, reason="tok mode, param cap or knob check failed")
|
| 414 |
+
save_checkpoint(a, model, tok, "pretrain")
|
| 415 |
+
if a.pretrain_only:
|
| 416 |
+
return write_summary(a, "ok", stage="pretrain_only", pretrain=pre)
|
| 417 |
+
rl = stage_rl(a, agent, tok, model, targets, train)
|
| 418 |
+
if rl["rl_secs_used"] > a.rl_secs + RL_GRACE and not a.smoke:
|
| 419 |
+
return write_summary(a, "timeout", pretrain=pre, rl=rl, reason="rl overran its budget")
|
| 420 |
+
save_checkpoint(a, model, tok, "final")
|
| 421 |
+
ev = stage_eval(a, tok, model, dev, old)
|
| 422 |
+
write_summary(a, "ok", pretrain=pre, rl=rl, eval=ev)
|
| 423 |
+
except BudgetSpent as e:
|
| 424 |
+
write_summary(a, "timeout", pretrain=pre, reason=f"rl: {e}")
|
| 425 |
+
except Timeout:
|
| 426 |
+
write_summary(a, "timeout", pretrain=pre, reason=f"exceeded TOTAL_SECS={TOTAL_SECS}")
|
| 427 |
+
except Exception:
|
| 428 |
+
traceback.print_exc()
|
| 429 |
+
write_summary(a, "crash", pretrain=pre, reason=traceback.format_exc()[-2000:])
|
| 430 |
+
finally:
|
| 431 |
+
signal.alarm(0)
|
| 432 |
+
|
| 433 |
+
|
| 434 |
+
if __name__ == "__main__":
|
| 435 |
+
main()
|
runs/long-rl-20261014-v2_untied-ei_s1/source/lean_gate.py
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Lean as the ONLY checker (long_rl_20261014 overlay on the fork's lean_gate.py at 51604b3).
|
| 2 |
+
|
| 3 |
+
gate(tok, prompts, nd_proofs, texts) is called by sample.generate() for a LeanTokenizer. For every distinct
|
| 4 |
+
(prompt, literal Lean text) whose text parsed in the strict grammar it asks Lean (core) whether the literal text proves the
|
| 5 |
+
theorem, and returns the ND proofs with 'LEANREJ ' prefixed where Lean rejects. Unlike the fork's gate there is no
|
| 6 |
+
nd_verify: a sample counts iff Lean accepts it (Robbie, 2026-10-14). The ND proof returned is only the decoder's rendering,
|
| 7 |
+
used for line counts. lean_check() below is the fork's, unchanged.
|
| 8 |
+
|
| 9 |
+
Log: $LEAN_GATE_LOG (default artifacts/lean_gate.jsonl), one json line per generate() call.
|
| 10 |
+
"""
|
| 11 |
+
import os, re, sys, json, time, subprocess, tempfile, shutil, collections
|
| 12 |
+
from concurrent.futures import ThreadPoolExecutor
|
| 13 |
+
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
| 14 |
+
|
| 15 |
+
LEAN = os.path.expanduser('~/.elan/bin/lean')
|
| 16 |
+
CHUNK = int(os.environ.get('LEAN_GATE_CHUNK', '400'))
|
| 17 |
+
WORKERS = int(os.environ.get('LEAN_GATE_WORKERS', str(max(1, (os.cpu_count() or 2) // 2))))
|
| 18 |
+
ERR = re.compile(r'^[^\n]*?:(\d+):\d+: error', re.M)
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def _run(lines, workdir, tag, depth=0):
|
| 22 |
+
"""lines: list of one-line theorem sources -> (list of bool, cpu seconds)"""
|
| 23 |
+
fn = os.path.join(workdir, f'c_{tag}.lean')
|
| 24 |
+
with open(fn, 'w') as f:
|
| 25 |
+
f.write('set_option linter.unusedVariables false\n' + '\n'.join(lines) + '\n')
|
| 26 |
+
t0 = time.time()
|
| 27 |
+
try:
|
| 28 |
+
p = subprocess.run([LEAN, '-DmaxErrors=100000000', fn], capture_output=True, text=True, timeout=60 + 2 * len(lines))
|
| 29 |
+
o = p.stdout + p.stderr; rc = p.returncode
|
| 30 |
+
except subprocess.TimeoutExpired:
|
| 31 |
+
o = ''; rc = -9
|
| 32 |
+
cpu = time.time() - t0
|
| 33 |
+
os.remove(fn)
|
| 34 |
+
bad = {int(m.group(1)) - 2 for m in ERR.finditer(o)}
|
| 35 |
+
crashed = rc not in (0, 1) or (rc == 1 and not bad) or any(b < 0 or b >= len(lines) for b in bad)
|
| 36 |
+
if crashed:
|
| 37 |
+
if len(lines) == 1:
|
| 38 |
+
return [False], cpu
|
| 39 |
+
h = len(lines) // 2
|
| 40 |
+
a, ca = _run(lines[:h], workdir, tag + 'a', depth + 1); b, cb = _run(lines[h:], workdir, tag + 'b', depth + 1)
|
| 41 |
+
return a + b, cpu + ca + cb
|
| 42 |
+
return [k not in bad for k in range(len(lines))], cpu
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def lean_check(items):
|
| 46 |
+
"""items: list of (statement text, tactic text) -> (list of bool, wall seconds, summed process seconds)"""
|
| 47 |
+
if not items:
|
| 48 |
+
return [], 0.0, 0.0
|
| 49 |
+
lines = [f'{s.replace("theorem t ", f"theorem t{k} ", 1)} {b}' for k, (s, b) in enumerate(items)]
|
| 50 |
+
wd = tempfile.mkdtemp(prefix='leangate_')
|
| 51 |
+
t0 = time.time()
|
| 52 |
+
chunks = [(lines[i:i + CHUNK], i) for i in range(0, len(lines), CHUNK)]
|
| 53 |
+
with ThreadPoolExecutor(WORKERS) as ex:
|
| 54 |
+
res = list(ex.map(lambda c: _run(c[0], wd, str(c[1])), chunks))
|
| 55 |
+
shutil.rmtree(wd, ignore_errors=True)
|
| 56 |
+
ok = [x for r, _ in res for x in r]
|
| 57 |
+
return ok, time.time() - t0, sum(c for _, c in res)
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def gate(tok, prompts, nd_proofs, texts):
|
| 61 |
+
logfn = os.environ.get('LEAN_GATE_LOG', 'artifacts/lean_gate.jsonl')
|
| 62 |
+
keys = {}
|
| 63 |
+
for p, nd, tx in zip(prompts, nd_proofs, texts):
|
| 64 |
+
if tx is not None and not nd.startswith('LEANPARSE'):
|
| 65 |
+
keys.setdefault((p, tx), nd)
|
| 66 |
+
items = list(keys)
|
| 67 |
+
lean_ok, wall, cpu = lean_check([(tok.statement(p), tx) for p, tx in items])
|
| 68 |
+
verdict = dict(zip(items, lean_ok))
|
| 69 |
+
n_parse = sum(1 for nd in nd_proofs if nd.startswith('LEANPARSE'))
|
| 70 |
+
parse_reasons = collections.Counter(nd[10:] for nd in nd_proofs if nd.startswith('LEANPARSE'))
|
| 71 |
+
n_ok = sum(map(bool, lean_ok))
|
| 72 |
+
rec = {'utc': time.strftime('%FT%TZ', time.gmtime()), 'samples': len(prompts), 'parse_fail': n_parse,
|
| 73 |
+
'distinct_checked': len(items), 'lean_ok': n_ok, 'lean_rej': len(items) - n_ok,
|
| 74 |
+
'parse_reasons': dict(parse_reasons.most_common()), 'lean_wall_s': wall, 'lean_proc_s': cpu,
|
| 75 |
+
'workers': WORKERS, 'chunk': CHUNK}
|
| 76 |
+
os.makedirs(os.path.dirname(logfn) or '.', exist_ok=True)
|
| 77 |
+
with open(logfn, 'a') as f:
|
| 78 |
+
f.write(json.dumps(rec) + '\n')
|
| 79 |
+
print(f'[lean_gate] {len(prompts)} samples, parse-fail {n_parse}, distinct checked {len(items)}: lean ok {n_ok}; '
|
| 80 |
+
f'lean {wall:.1f}s wall ({cpu:.1f}s proc, {WORKERS} workers)', flush=True)
|
| 81 |
+
return [nd if tx is None or nd.startswith('LEANPARSE') or verdict[(p, tx)] else 'LEANREJ ' + nd
|
| 82 |
+
for p, nd, tx in zip(prompts, nd_proofs, texts)]
|
runs/long-rl-20261014-v2_untied-ei_s1/source/lean_tok.py
ADDED
|
@@ -0,0 +1,319 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Lean 4 as the training surface form for the from-scratch model (run lean-format, proposal 8). One symbol per token.
|
| 2 |
+
|
| 3 |
+
The text the model reads and writes is the nd2lean.py rendering of an ND proof, linearised on one line with `;` between
|
| 4 |
+
tactics (no indentation tokens):
|
| 5 |
+
|
| 6 |
+
prompt : theorem t ( P Q R S : Prop ) ( h1 : F1 ) ... : C := by
|
| 7 |
+
proof : have n1 : F1 := h1 ; have n7 : ( P → R ) := ( fun ( n3 : P ) => by have n4 : Q := n1 n3 ; exact n4 ) ; exact n7 <eos>
|
| 8 |
+
|
| 9 |
+
Terms (exactly nd2lean's): PR `hK`; R `nA`; ANDI `⟨ nA , nB ⟩`; ANDE `nA .1|.2`; IMPE / NEGE `nF nA`; ORI `Or.inl|Or.inr nA`;
|
| 10 |
+
BOTE `False.elim nA` (tokens `False` `.elim` `nA`; was `nA .elim` before the 2026-09-22 fix, see nd2lean.py); DN `Classical.byContradiction ( fun hh => nA hh )`; IMPI `( fun ( nS : A ) => by ... ; exact nE )`;
|
| 11 |
+
NEGI the same ending `exact ( nE : False )`; ORE `Or.elim nJ <box> <box>`. `.1 .2 .elim` are glued to the name in the text.
|
| 12 |
+
|
| 13 |
+
Hypothesis names are LABELS, not line indices. 64 name tokens n1..n64; two naming schemes (stored in the ckpt as the mode):
|
| 14 |
+
lean_rand : every training presentation draws a random injective map names -> n1..n64 (names carry no order)
|
| 15 |
+
lean_seq : names are numbered in order of first appearance in the Lean text, plus a random start offset
|
| 16 |
+
(the analogue of the token format's `abs` start-index shift)
|
| 17 |
+
decode() parses the sampled tokens with the strict grammar above and returns the ND proof (spec.md format, N1..) it
|
| 18 |
+
denotes — the "nd2lean inverse" — so nd_verify, prune, normalize and every analysis script work unchanged; a sample
|
| 19 |
+
outside the grammar decodes to 'LEANPARSE <reason>' (rejected by nd_verify). The literal Lean text of the last decoded
|
| 20 |
+
sample is kept in `last_text` and is what lean_gate.py sends to Lean.
|
| 21 |
+
"""
|
| 22 |
+
import random, sys, os
|
| 23 |
+
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
| 24 |
+
from nd_verify.verify import parse_proof_tokens, parse_formula
|
| 25 |
+
from nd2lean import parse_prompt, translate
|
| 26 |
+
|
| 27 |
+
# long_rl_20261014 overlay (on the fork's lean_tok.py at 51604b3): ten atoms (no V, which reads as
|
| 28 |
+
# `v`), up to ten premises (h1..h10), 160 names, and a statement header that declares all ten atoms,
|
| 29 |
+
# so random atom relabelling (tok.atom_ids, HEADER_LEN; pretrain.py's gpu_batch) never touches it.
|
| 30 |
+
ATOMS = ['P', 'Q', 'R', 'S', 'T', 'U', 'W', 'X', 'Y', 'Z']
|
| 31 |
+
sys.modules['nd_verify.verify'].ATOM_NAMES.update(ATOMS) # the ND parsers (formula syntax only; proofs are judged by Lean)
|
| 32 |
+
MAXN = 160
|
| 33 |
+
MAXH = 10
|
| 34 |
+
FSYMS = ['(', ')', '¬', '∧', '∨', '→'] + ATOMS + ['False']
|
| 35 |
+
HEADER_LEN = len(['theorem', 't', '(']) + len(ATOMS) + len([':', 'Prop', ')'])
|
| 36 |
+
PSYMS = ['theorem', 't', ':', 'Prop', ':=', 'by']
|
| 37 |
+
TSYMS = ['have', 'exact', ';', 'fun', '=>', '⟨', '⟩', ',', '.1', '.2', '.elim', 'Or.inl', 'Or.inr', 'Or.elim', 'Classical.byContradiction', 'hh']
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
class ParseFail(Exception):
|
| 41 |
+
pass
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def ftoks(f):
|
| 45 |
+
"""formula tuple -> Lean tokens (fully parenthesised, as nd2lean.lf)"""
|
| 46 |
+
t = f[0]
|
| 47 |
+
if t == 'atom': return [f[1]]
|
| 48 |
+
if t == 'bot': return ['False']
|
| 49 |
+
if t == 'not': return ['(', '¬'] + ftoks(f[1]) + [')']
|
| 50 |
+
op = {'and': '∧', 'or': '∨', 'imp': '→'}[t]
|
| 51 |
+
return ['('] + ftoks(f[1]) + [op] + ftoks(f[2]) + [')']
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def fnd(f):
|
| 55 |
+
"""formula tuple -> ND tokens (spec.md)"""
|
| 56 |
+
t = f[0]
|
| 57 |
+
if t == 'atom': return f[1]
|
| 58 |
+
if t == 'bot': return 'F'
|
| 59 |
+
if t == 'not': return f'( ~ {fnd(f[1])} )'
|
| 60 |
+
op = {'and': '&', 'or': 'v', 'imp': '>'}[t]
|
| 61 |
+
return f'( {fnd(f[1])} {op} {fnd(f[2])} )'
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def prompt_tokens(prompt):
|
| 65 |
+
prem, concl = parse_prompt(prompt)
|
| 66 |
+
if len(prem) > MAXH:
|
| 67 |
+
raise ValueError('too many premises')
|
| 68 |
+
out = ['theorem', 't', '('] + ATOMS + [':', 'Prop', ')']
|
| 69 |
+
for j, p in enumerate(prem):
|
| 70 |
+
out += ['(', f'h{j+1}', ':'] + ftoks(p) + [')']
|
| 71 |
+
return out + [':'] + ftoks(concl) + [':=', 'by']
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def proof_tokens(proof):
|
| 75 |
+
"""ND proof body -> Lean tokens with names as ('n', ND index). Mirrors nd2lean.translate (which must have accepted
|
| 76 |
+
the proof); a closed box that no rule cites is dropped, as in nd2lean."""
|
| 77 |
+
lines = parse_proof_tokens(proof.split())
|
| 78 |
+
stack = [[]] # token lists of the open boxes' statements; stack[0] = top level
|
| 79 |
+
hyp = [None] # (idx, formula) of each open box
|
| 80 |
+
last = [None]
|
| 81 |
+
closed = {} # start idx -> (hyp formula, stmts tokens, last idx)
|
| 82 |
+
n_pr = 0
|
| 83 |
+
|
| 84 |
+
def close():
|
| 85 |
+
s = stack.pop(); h = hyp.pop(); l = last.pop()
|
| 86 |
+
closed[h[0]] = (h[1], s, l)
|
| 87 |
+
|
| 88 |
+
def box(s, e, neg=False):
|
| 89 |
+
hf, st, l = closed[s]
|
| 90 |
+
assert l == e, 'box cite'
|
| 91 |
+
ex = ['exact', '(', ('n', e), ':', 'False', ')'] if neg else ['exact', ('n', e)]
|
| 92 |
+
return ['(', 'fun', '(', ('n', s), ':'] + ftoks(hf) + [')', '=>', 'by'] + st + ex + [')']
|
| 93 |
+
|
| 94 |
+
for ln in lines:
|
| 95 |
+
i, d, f, rule, refs = ln['idx'], ln['depth'], ln['formula'], ln['rule'], ln['refs']
|
| 96 |
+
if rule == 'AS':
|
| 97 |
+
while len(stack) - 1 >= d: close()
|
| 98 |
+
stack.append([]); hyp.append((i, f)); last.append(i)
|
| 99 |
+
continue
|
| 100 |
+
while len(stack) - 1 > d: close()
|
| 101 |
+
n = lambda j: ('n', j)
|
| 102 |
+
if rule == 'PR': n_pr += 1; term = [f'h{n_pr}']
|
| 103 |
+
elif rule == 'R': term = [n(refs[0])]
|
| 104 |
+
elif rule == 'ANDI': term = ['⟨', n(refs[0]), ',', n(refs[1]), '⟩']
|
| 105 |
+
elif rule == 'ANDE1': term = [n(refs[0]), '.1']
|
| 106 |
+
elif rule == 'ANDE2': term = [n(refs[0]), '.2']
|
| 107 |
+
elif rule == 'IMPE': term = [n(refs[0]), n(refs[1])]
|
| 108 |
+
elif rule == 'NEGE': term = [n(refs[1]), n(refs[0])]
|
| 109 |
+
elif rule == 'ORI1': term = ['Or.inl', n(refs[0])]
|
| 110 |
+
elif rule == 'ORI2': term = ['Or.inr', n(refs[0])]
|
| 111 |
+
elif rule == 'BOTE': term = ['False', '.elim', n(refs[0])]
|
| 112 |
+
elif rule == 'DN': term = ['Classical.byContradiction', '(', 'fun', 'hh', '=>', n(refs[0]), 'hh', ')']
|
| 113 |
+
elif rule == 'IMPI': term = box(refs[0], refs[1])
|
| 114 |
+
elif rule == 'NEGI': term = box(refs[0], refs[1], neg=True)
|
| 115 |
+
elif rule == 'ORE': term = ['Or.elim', n(refs[0])] + box(refs[1], refs[2]) + box(refs[3], refs[4])
|
| 116 |
+
else: raise ValueError(rule)
|
| 117 |
+
stack[-1] += ['have', n(i), ':'] + ftoks(f) + [':='] + term + [';']
|
| 118 |
+
last[-1] = i
|
| 119 |
+
assert len(stack) == 1
|
| 120 |
+
return stack[0] + ['exact', ('n', lines[-1]['idx'])]
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
class LeanTokenizer:
|
| 124 |
+
def __init__(self, mode='lean_rand'):
|
| 125 |
+
assert mode in ('lean_rand', 'lean_seq')
|
| 126 |
+
self.mode = mode
|
| 127 |
+
self.itos = ['<pad>', '<eos>'] + FSYMS + PSYMS + TSYMS + [f'h{k}' for k in range(1, MAXH + 1)]
|
| 128 |
+
self.ref0 = len(self.itos)
|
| 129 |
+
self.itos += [f'n{k}' for k in range(1, MAXN + 1)]
|
| 130 |
+
self.stoi = {s: i for i, s in enumerate(self.itos)}
|
| 131 |
+
self.atom_ids = [self.stoi[x] for x in ATOMS]
|
| 132 |
+
self.pad, self.eos = 0, 1
|
| 133 |
+
self.shift = True
|
| 134 |
+
self.last_text = None
|
| 135 |
+
|
| 136 |
+
@property
|
| 137 |
+
def vocab_size(self):
|
| 138 |
+
return len(self.itos)
|
| 139 |
+
|
| 140 |
+
def encode_prompt(self, prompt):
|
| 141 |
+
return [self.stoi[t] for t in prompt_tokens(prompt)]
|
| 142 |
+
|
| 143 |
+
def encode_proof(self, body):
|
| 144 |
+
"""ND body -> ids with names numbered by first appearance (n1, n2, ...); <eos> included."""
|
| 145 |
+
order = {}
|
| 146 |
+
out = []
|
| 147 |
+
for t in proof_tokens(body):
|
| 148 |
+
if isinstance(t, tuple):
|
| 149 |
+
k = order.setdefault(t[1], len(order) + 1)
|
| 150 |
+
if k > MAXN: raise ValueError('too many names')
|
| 151 |
+
out.append(self.ref0 + k - 1)
|
| 152 |
+
else:
|
| 153 |
+
out.append(self.stoi[t])
|
| 154 |
+
return out + [self.eos]
|
| 155 |
+
|
| 156 |
+
def shift_abs(self, ids, rng):
|
| 157 |
+
"""name augmentation (the interface name is train.py's)."""
|
| 158 |
+
if not self.shift:
|
| 159 |
+
return ids
|
| 160 |
+
mx = max((x - self.ref0 + 1 for x in ids if x >= self.ref0), default=0)
|
| 161 |
+
if mx == 0:
|
| 162 |
+
return ids
|
| 163 |
+
if self.mode == 'lean_seq':
|
| 164 |
+
s = rng.randint(0, MAXN - mx)
|
| 165 |
+
return [x + s if x >= self.ref0 else x for x in ids]
|
| 166 |
+
perm = rng.sample(range(MAXN), mx)
|
| 167 |
+
return [self.ref0 + perm[x - self.ref0] if x >= self.ref0 else x for x in ids]
|
| 168 |
+
|
| 169 |
+
# ---- text ----
|
| 170 |
+
def text(self, toks):
|
| 171 |
+
"""tokens (strings) -> Lean source text of the tactic block (one line)."""
|
| 172 |
+
out = []
|
| 173 |
+
for t in toks:
|
| 174 |
+
if t in ('.1', '.2', '.elim') and out:
|
| 175 |
+
out[-1] += t
|
| 176 |
+
else:
|
| 177 |
+
out.append(t)
|
| 178 |
+
return ' '.join(out)
|
| 179 |
+
|
| 180 |
+
def statement(self, prompt):
|
| 181 |
+
return ' '.join(prompt_tokens(prompt))
|
| 182 |
+
|
| 183 |
+
def lean_source(self, prompt, proof):
|
| 184 |
+
"""(statement, tactic text) Lean checks for an ND proof: exactly what the model would write."""
|
| 185 |
+
return self.statement(prompt), self.text([self.itos[x] for x in self.encode_proof(proof)[:-1]])
|
| 186 |
+
|
| 187 |
+
# ---- inverse ----
|
| 188 |
+
def decode(self, ids):
|
| 189 |
+
toks = []
|
| 190 |
+
ended = False
|
| 191 |
+
for x in ids:
|
| 192 |
+
if x == self.pad: continue
|
| 193 |
+
if x == self.eos: ended = True; break
|
| 194 |
+
toks.append(self.itos[x])
|
| 195 |
+
self.last_text = self.text(toks) if ended else None
|
| 196 |
+
if not ended:
|
| 197 |
+
return 'LEANPARSE no-eos'
|
| 198 |
+
try:
|
| 199 |
+
return inverse(toks)
|
| 200 |
+
except ParseFail as e:
|
| 201 |
+
return f'LEANPARSE {e}'
|
| 202 |
+
|
| 203 |
+
|
| 204 |
+
def inverse(toks):
|
| 205 |
+
"""Lean tokens (strict nd2lean grammar) -> ND proof body 'N1 ... ; ... QED'. Names resolve lexically (latest binding wins,
|
| 206 |
+
as in Lean). Raises ParseFail outside the grammar: PR lines first and in order, a box's `exact` cites the box's last line,
|
| 207 |
+
the final `exact` cites the last top-level line."""
|
| 208 |
+
pos = [0]
|
| 209 |
+
out = [] # ND lines (strings)
|
| 210 |
+
forms = {} # ND idx -> formula tuple
|
| 211 |
+
n_pr = [0]; seen_non_pr = [False]
|
| 212 |
+
|
| 213 |
+
def peek():
|
| 214 |
+
return toks[pos[0]] if pos[0] < len(toks) else None
|
| 215 |
+
|
| 216 |
+
def eat(x=None):
|
| 217 |
+
t = peek()
|
| 218 |
+
if t is None or (x is not None and t != x):
|
| 219 |
+
raise ParseFail(f'expected {x}')
|
| 220 |
+
pos[0] += 1
|
| 221 |
+
return t
|
| 222 |
+
|
| 223 |
+
def name_tok():
|
| 224 |
+
t = eat()
|
| 225 |
+
if not (t[0] == 'n' and t[1:].isdigit()):
|
| 226 |
+
raise ParseFail('name')
|
| 227 |
+
return t
|
| 228 |
+
|
| 229 |
+
def formula():
|
| 230 |
+
# re-use the ND formula parser on a token-mapped slice
|
| 231 |
+
m = {'(': '(', ')': ')', '¬': '~', '∧': '&', '∨': 'v', '→': '>', 'False': 'F', **{a: a for a in ATOMS}}
|
| 232 |
+
j = pos[0]; nd = []
|
| 233 |
+
while j < len(toks) and toks[j] in m:
|
| 234 |
+
nd.append(m[toks[j]]); j += 1
|
| 235 |
+
try:
|
| 236 |
+
f, used = parse_formula(nd + ['$'], 0)
|
| 237 |
+
except Exception:
|
| 238 |
+
raise ParseFail('formula')
|
| 239 |
+
pos[0] += used
|
| 240 |
+
return f
|
| 241 |
+
|
| 242 |
+
def ref(scope):
|
| 243 |
+
t = name_tok()
|
| 244 |
+
if t not in scope:
|
| 245 |
+
raise ParseFail('unbound')
|
| 246 |
+
return scope[t]
|
| 247 |
+
|
| 248 |
+
def emit(depth, f, rule, refs):
|
| 249 |
+
i = len(out) + 1
|
| 250 |
+
out.append(f'N{i} ' + '| ' * depth + f'{fnd(f)} : {rule}' + ''.join(f' N{r}' for r in refs) + ' ;')
|
| 251 |
+
forms[i] = f
|
| 252 |
+
return i
|
| 253 |
+
|
| 254 |
+
def box(scope, depth):
|
| 255 |
+
"""'( fun ( n : A ) => by stmts exact .. )' -> (start idx, end idx, neg)"""
|
| 256 |
+
eat('('); eat('fun'); eat('('); nm = name_tok(); eat(':'); hf = formula(); eat(')'); eat('=>'); eat('by')
|
| 257 |
+
s = emit(depth + 1, hf, 'AS', [])
|
| 258 |
+
sc = dict(scope); sc[nm] = s
|
| 259 |
+
lastl = stmts(sc, depth + 1, s)
|
| 260 |
+
eat('exact')
|
| 261 |
+
if peek() == '(':
|
| 262 |
+
eat('('); e = ref(sc); eat(':'); eat('False'); eat(')'); neg = True
|
| 263 |
+
else:
|
| 264 |
+
e = ref(sc); neg = False
|
| 265 |
+
if e != lastl:
|
| 266 |
+
raise ParseFail('exact not last')
|
| 267 |
+
eat(')')
|
| 268 |
+
return s, e, neg
|
| 269 |
+
|
| 270 |
+
def stmts(scope, depth, lastl):
|
| 271 |
+
while peek() == 'have':
|
| 272 |
+
eat('have'); nm = name_tok(); eat(':'); f = formula(); eat(':=')
|
| 273 |
+
t = peek()
|
| 274 |
+
if t is None: raise ParseFail('eof')
|
| 275 |
+
if t[0] == 'h' and t[1:].isdigit():
|
| 276 |
+
eat(); k = int(t[1:])
|
| 277 |
+
if depth != 0 or seen_non_pr[0] or k != n_pr[0] + 1: raise ParseFail('PR position')
|
| 278 |
+
n_pr[0] += 1
|
| 279 |
+
i = emit(0, f, 'PR', [])
|
| 280 |
+
else:
|
| 281 |
+
seen_non_pr[0] = True
|
| 282 |
+
if t == '⟨':
|
| 283 |
+
eat(); a = ref(scope); eat(','); b = ref(scope); eat('⟩'); rule, refs, pre = 'ANDI', [a, b], None
|
| 284 |
+
elif t in ('Or.inl', 'Or.inr'):
|
| 285 |
+
eat(); a = ref(scope); rule, refs = ('ORI1' if t == 'Or.inl' else 'ORI2'), [a]
|
| 286 |
+
elif t == 'Classical.byContradiction':
|
| 287 |
+
eat(); eat('('); eat('fun'); eat('hh'); eat('=>'); a = ref(scope); eat('hh'); eat(')'); rule, refs = 'DN', [a]
|
| 288 |
+
elif t == '(':
|
| 289 |
+
s, e, neg = box(scope, depth); rule, refs = ('NEGI' if neg else 'IMPI'), [s, e]
|
| 290 |
+
elif t == 'Or.elim':
|
| 291 |
+
eat(); j = ref(scope); s1, e1, g1 = box(scope, depth); s2, e2, g2 = box(scope, depth)
|
| 292 |
+
if g1 or g2: raise ParseFail('ORE box')
|
| 293 |
+
rule, refs = 'ORE', [j, s1, e1, s2, e2]
|
| 294 |
+
elif t == 'False':
|
| 295 |
+
eat(); eat('.elim'); a = ref(scope); rule, refs = 'BOTE', [a]
|
| 296 |
+
else:
|
| 297 |
+
a = ref(scope); t2 = peek()
|
| 298 |
+
if t2 == '.1': eat(); rule, refs = 'ANDE1', [a]
|
| 299 |
+
elif t2 == '.2': eat(); rule, refs = 'ANDE2', [a]
|
| 300 |
+
elif t2 == ';': rule, refs = 'R', [a]
|
| 301 |
+
else:
|
| 302 |
+
b = ref(scope)
|
| 303 |
+
if forms[a][0] == 'not': rule, refs = 'NEGE', [b, a]
|
| 304 |
+
else: rule, refs = 'IMPE', [a, b]
|
| 305 |
+
i = emit(depth, f, rule, refs)
|
| 306 |
+
eat(';')
|
| 307 |
+
scope[nm] = i
|
| 308 |
+
lastl = i
|
| 309 |
+
return lastl
|
| 310 |
+
|
| 311 |
+
top = {}
|
| 312 |
+
lastl = stmts(top, 0, None)
|
| 313 |
+
eat('exact')
|
| 314 |
+
e = ref(top)
|
| 315 |
+
if pos[0] != len(toks):
|
| 316 |
+
raise ParseFail('trailing tokens')
|
| 317 |
+
if lastl is None or e != lastl:
|
| 318 |
+
raise ParseFail('exact not last')
|
| 319 |
+
return ' '.join(out) + ' QED'
|
runs/long-rl-20261014-v2_untied-ei_s1/source/pretrain.py
ADDED
|
@@ -0,0 +1,820 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# imports the fork (model, tokenizer, train, ...), which is not resolvable here
|
| 2 |
+
# pyright: reportMissingImports=false
|
| 3 |
+
"""long_rl_20261014: run 512's file (autoresearch_loops ae79d23, the final RL recipe: 507 token-budget fine-tune
|
| 4 |
+
batches, 509 Muon fine-tune, 510 more and shorter rounds) with three arms picked by environment variables:
|
| 5 |
+
|
| 6 |
+
ND_ARCH=untied ND_RL=ei A: run 198's untied six blocks (control 590's patch) + the final expert-iteration recipe
|
| 7 |
+
ND_ARCH=looped ND_RL=ei B: 512's looped trunk (two alternating shared core pairs) + the same recipe
|
| 8 |
+
ND_ARCH=untied ND_RL=grpo C: run 198's untied blocks + GRPO with a found-proof pool (rl_grpo)
|
| 9 |
+
|
| 10 |
+
Changes for this experiment, all arms:
|
| 11 |
+
- tokenizer: overlay/lean_tok.py (ten atoms, ten premises h1..h10, names n1..n160)
|
| 12 |
+
- random atom relabelling: every training batch maps each row's atoms through a random permutation of the ten
|
| 13 |
+
atom tokens (gpu_batch), after the fixed statement header; the analogue of the name-offset shift
|
| 14 |
+
- targets are picked per round by body-line band, weighted toward the frontier (pick_targets), from a pool of
|
| 15 |
+
thousands instead of a fixed 1,500
|
| 16 |
+
|
| 17 |
+
The original contract (harness.py's docstring describes this experiment's changes to it):
|
| 18 |
+
|
| 19 |
+
Contract with harness.py (frozen):
|
| 20 |
+
|
| 21 |
+
pretrain(train_recs, budget_secs, device, seed, log) -> (model, tok, stats)
|
| 22 |
+
|
| 23 |
+
train_recs list of dicts {prompt, proof, n_lines<=6, key, ...}; the frozen cap-6 Stage-1 data
|
| 24 |
+
budget_secs wall-clock you are charged for, from call to return, INCLUDING model construction
|
| 25 |
+
and torch.compile. Overrun by more than the harness grace -> the run is a timeout.
|
| 26 |
+
model nn.Module; <= PARAM_CAP unique params (checked again after rl). The harness samples
|
| 27 |
+
from it with the fork's sample.generate, so it must provide:
|
| 28 |
+
- forward(idx) -> logits (B,T,V) and forward(idx, pos=, mask=, caches=) with a
|
| 29 |
+
per-block KV cache dict, exactly like the fork's GPT (model.blocks must exist)
|
| 30 |
+
- an integer `n_loops` (the harness's KNOB): the loop count applied in eval
|
| 31 |
+
mode. The harness sets it to EVAL_LEVEL for the scored eval and to DIAG_LEVEL
|
| 32 |
+
for the diagnostic one.
|
| 33 |
+
tok must provide encode_prompt, encode_proof, decode, shift_abs, pad, eos, mode, and
|
| 34 |
+
mode must be "lean_seq" (harness TOK_MODE): Lean text in and out, Lean-gated samples.
|
| 35 |
+
Decoder rule: decode() is deterministic and single-pass; it may not call the
|
| 36 |
+
verifier or try alternatives.
|
| 37 |
+
stats dict recorded in summary.json and copied into results.json. MUST contain `curve`:
|
| 38 |
+
a list of {secs, step, train_loss, val_loss} taken every ~CURVE_EVERY seconds
|
| 39 |
+
(see log_point below), so a run's trajectory survives in git. Add anything else.
|
| 40 |
+
val_loss is on VAL_N records held out from train_recs by this file; keep that.
|
| 41 |
+
|
| 42 |
+
rl(model, tok, targets, train_recs, oracle, budget_secs, device, seed, log) -> stats | None
|
| 43 |
+
|
| 44 |
+
targets list of dicts {name, prompt, n_lines}: the 1,500 RL target statements. No proofs.
|
| 45 |
+
oracle oracle.sample(model, targets_subset, k, temperature=0.8, batch=1024) samples k
|
| 46 |
+
attempts per target from the model in eval mode at its current n_loops, and returns
|
| 47 |
+
per target {name, prompt, n_lines, proofs, n_ok, n_tried}; `proofs` are the distinct
|
| 48 |
+
attempts Lean AND nd_verify accepted. This is the only source of new training data.
|
| 49 |
+
oracle.used() / oracle.left(): seconds of the RL budget charged / remaining.
|
| 50 |
+
budget_secs RL_SECS: wall-clock inside rl() minus the oracle's Lean-gate time. Returning more
|
| 51 |
+
than the grace late, or calling the oracle with nothing left, is a timeout.
|
| 52 |
+
The model is trained in place. Everything else about RL is yours: rounds, samples per target,
|
| 53 |
+
loop counts when sampling, which proofs to keep, weighting, replay, optimizer, loss.
|
| 54 |
+
|
| 55 |
+
Anything imported from the fork (ND_FORK) is fair game to copy in here and modify.
|
| 56 |
+
"""
|
| 57 |
+
|
| 58 |
+
import collections
|
| 59 |
+
import math
|
| 60 |
+
import os
|
| 61 |
+
import random
|
| 62 |
+
import time
|
| 63 |
+
|
| 64 |
+
import torch
|
| 65 |
+
import torch._inductor.config # noqa: F401 (explicit, so the attribute resolves for pyright)
|
| 66 |
+
from lean_tok import HEADER_LEN, MAXN # overlay
|
| 67 |
+
from model import GPT, Block # fork
|
| 68 |
+
from normalize import norm # fork
|
| 69 |
+
from tokenizer import make_tokenizer # fork
|
| 70 |
+
from train import loss_on # fork
|
| 71 |
+
|
| 72 |
+
# ---- knobs
|
| 73 |
+
# Unique blocks: one untied entry block, N_PAIRS pairs of N_CORE shared core blocks that the loops
|
| 74 |
+
# cycle through (loop k applies pair k % N_PAIRS, so 8 loops run A B A B A B A B and 2 loops run
|
| 75 |
+
# A B: the untied six-block topology of control 590), one untied exit block. Same compute per
|
| 76 |
+
# loop count as one shared pair; 6 unique blocks, 9.56M parameters, the untied control's count.
|
| 77 |
+
N_CORE, N_PAIRS = 2, 2
|
| 78 |
+
ARCH = os.environ.get("ND_ARCH", "untied") # untied | looped
|
| 79 |
+
RL_ALGO = os.environ.get("ND_RL", "ei") # ei | grpo
|
| 80 |
+
assert ARCH in ("untied", "looped") and RL_ALGO in ("ei", "grpo"), (ARCH, RL_ALGO)
|
| 81 |
+
N_LAYER, D, N_HEAD = 2 + N_CORE * N_PAIRS, 384, 8
|
| 82 |
+
# Core applications (loops). Training draws one count per batch from TRAIN_LOOPS, in pretraining
|
| 83 |
+
# and in rl()'s fine-tune; eval and sampling apply model.n_loops, which the harness sets for its
|
| 84 |
+
# evals and rl() sets before each oracle call. DEFAULT_LOOPS is used for the validation curve and
|
| 85 |
+
# the harness's post-pretraining greedy check.
|
| 86 |
+
TRAIN_LOOPS = (1, 2, 3, 4, 5, 6, 7, 8)
|
| 87 |
+
DEFAULT_LOOPS = 2
|
| 88 |
+
VAL_LOOPS = (1, 2, 4, 8) # val loss at each count, once, when pretraining ends
|
| 89 |
+
# ---- RL knobs: the old harness's expert iteration (600 AdamW steps per round on up to 4 proofs
|
| 90 |
+
# per solved target, weighted x4, mixed with 20,000 replayed pretraining records), except for
|
| 91 |
+
# where the samples go. Each round every target gets RL_K attempts at RL_FIRST_LOOPS; a target
|
| 92 |
+
# all of them fail on gets RL_RETRY_K more (32 in all, as before) at ONE deeper loop count from
|
| 93 |
+
# RL_RETRY_LOOPS, assigned at random, so proofs that only deeper sampling finds enter the
|
| 94 |
+
# training set and no deep sample is spent on a target one loop already solves. The random
|
| 95 |
+
# assignment makes the retry calls a controlled comparison of loop counts on resistant targets.
|
| 96 |
+
# Rounds repeat until another one would not fit in the budget.
|
| 97 |
+
RL_FIRST_LOOPS, RL_K = 1, 8
|
| 98 |
+
RL_RETRY_LOOPS, RL_RETRY_K = (2, 4, 8), 24
|
| 99 |
+
# The fine-tune sees FT_RECORDS records per round, half the old harness's 600 steps x 128: in
|
| 100 |
+
# 509 the fine-tune had grown to ~74 s of a ~137 s round, so halving it fits about 16 rounds of
|
| 101 |
+
# sampling in RL_SECS instead of 11, with 27% fewer fine-tune records per run; the mix (20-33k
|
| 102 |
+
# records) still gets at least one epoch per round. Records come in the same token-budget
|
| 103 |
+
# batches as pretraining (TOK_BUDGET below) instead of 128-record batches padded to their
|
| 104 |
+
# longest proof, which with 7-14 line RL proofs in nearly every batch was ~3x the content.
|
| 105 |
+
# It uses pretraining's optimizer split: Muon (muon_step, MUON_WD) on the block matrices and the
|
| 106 |
+
# head, AdamW at FT_LR on embeddings, norms and biases. FT_MUON_LR keeps pretraining's 10:1
|
| 107 |
+
# Muon:AdamW ratio and is where pretraining's cosine ends (MUON_LR * MIN_LR_FRAC).
|
| 108 |
+
FT_RECORDS, FT_LR, FT_MUON_LR, FT_WARMUP = 300 * 128, 3e-4, 3e-3, 50
|
| 109 |
+
RETAIN, MAX_PER_THM, RL_WEIGHT = 20000, 4, 4
|
| 110 |
+
# MLP width held at 151's 1280: the fork's Block hardcodes 4 * D (1536 at D 384 would pass the
|
| 111 |
+
# 10M cap), so PeriLNBlock swaps fc1/fc2. 1280 != 3 * D keeps muon_step's QKV-by-shape test valid.
|
| 112 |
+
D_FF = 1280
|
| 113 |
+
BS, LR, MIN_LR_FRAC, WD = 128, 1e-3, 0.3, 0.1
|
| 114 |
+
MUON_LR, MUON_MOMENTUM = 0.01, 0.95
|
| 115 |
+
# Polar Express (Amsel et al. 2025) per-iteration quintic coefficients, as in modded-nanogpt
|
| 116 |
+
# (num_iters=5, safety factor 2e-2): singular values land in ~[0.86, 1.14] instead of the
|
| 117 |
+
# fixed quintic's ~[0.68, 1.13], at the same five iterations.
|
| 118 |
+
NS_COEFFS = (
|
| 119 |
+
(8.156554524902461, -22.48329292557795, 15.878769915207462),
|
| 120 |
+
(4.042929935166739, -2.808917465908714, 0.5000178451051316),
|
| 121 |
+
(3.8916678022926607, -2.772484153217685, 0.5060648178503393),
|
| 122 |
+
(3.285753657755655, -2.3681294933425376, 0.46449024233003106),
|
| 123 |
+
(2.3465413258596377, -1.7097828382687081, 0.42323551169305323),
|
| 124 |
+
)
|
| 125 |
+
# Ten times the AdamW group's per-step multiplicative decay (lr * wd) on the Muon matrices.
|
| 126 |
+
MUON_WD = 10 * WD * LR / MUON_LR
|
| 127 |
+
TOK_MODE = "lean_seq" # required by the harness (Lean protocol)
|
| 128 |
+
# Hybrid heads: half the heads get ALiBi (Press et al. 2021) with the standard slope set for
|
| 129 |
+
# H/2 heads, 2^(-8 i / (H/2)) = 1/4 .. 1/256 with 8 heads; the other half get slope 0, i.e. no
|
| 130 |
+
# positional signal at all (NoPE), so they can look citations up by content at any distance.
|
| 131 |
+
N_ALIBI = N_HEAD // 2
|
| 132 |
+
ALIBI_SLOPES = tuple(2.0 ** (-8.0 * (i + 1) / N_ALIBI) for i in range(N_ALIBI)) + (0.0,) * (
|
| 133 |
+
N_HEAD - N_ALIBI
|
| 134 |
+
)
|
| 135 |
+
VAL_N, CURVE_EVERY = 1024, 15 # records held out for val_loss; seconds between curve points
|
| 136 |
+
# Token-budget batches (fairseq max-tokens): each step takes a run of length-sorted records with
|
| 137 |
+
# count * padded_length <= TOK_BUDGET, so a batch of short proofs has many records and a batch of
|
| 138 |
+
# long ones few, and the padded token count per step is ~constant. 128 x 144 matches the real
|
| 139 |
+
# (unpadded) tokens of the incumbent's 128-record batches (mean record ~135 tokens), whose
|
| 140 |
+
# padding to the batch max (~400) made every step ~3x larger than its content.
|
| 141 |
+
TOK_BUDGET = 128 * 144
|
| 142 |
+
# Multi-token prediction (DeepSeek-V3 2024, Gloeckle et al. 2024), pretraining only: one extra
|
| 143 |
+
# Peri-LN block reads [trunk state at t; embedding of token t+1] and predicts token t+2 through
|
| 144 |
+
# the model's own ln_f/head. Its loss is added at this weight (DeepSeek-V3's early-training value).
|
| 145 |
+
MTP_WEIGHT = 0.3
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
# Muon update adapted from KellerJordan/Muon, commit
|
| 149 |
+
# f98f1cacc0263b04290753e32be8d498c1efc806 (muon.py).
|
| 150 |
+
# MIT License
|
| 151 |
+
# Copyright (c) 2024 Keller Jordan
|
| 152 |
+
#
|
| 153 |
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
| 154 |
+
# of this software and associated documentation files (the "Software"), to deal
|
| 155 |
+
# in the Software without restriction, including without limitation the rights
|
| 156 |
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
| 157 |
+
# copies of the Software, and to permit persons to whom the Software is
|
| 158 |
+
# furnished to do so, subject to the following conditions:
|
| 159 |
+
# The above copyright notice and this permission notice shall be included in all
|
| 160 |
+
# copies or substantial portions of the Software.
|
| 161 |
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
| 162 |
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
| 163 |
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
| 164 |
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
| 165 |
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
| 166 |
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
| 167 |
+
# SOFTWARE.
|
| 168 |
+
@torch.no_grad()
|
| 169 |
+
def muon_step(params, buffers, lr):
|
| 170 |
+
"""Nesterov momentum plus five bf16 Polar Express steps, on hidden matrices only."""
|
| 171 |
+
for p, momentum in zip(params, buffers, strict=True):
|
| 172 |
+
if p.grad is None:
|
| 173 |
+
continue
|
| 174 |
+
momentum.lerp_(p.grad, 1 - MUON_MOMENTUM)
|
| 175 |
+
update = p.grad.lerp(momentum, MUON_MOMENTUM).bfloat16()
|
| 176 |
+
# Only QKV has shape (3D, D) in this GPT; orthogonalize its roles separately.
|
| 177 |
+
if update.size(0) == 3 * update.size(1):
|
| 178 |
+
update = update.reshape(3, update.size(1), update.size(1))
|
| 179 |
+
transpose = update.size(-2) > update.size(-1)
|
| 180 |
+
if transpose:
|
| 181 |
+
update = update.mT
|
| 182 |
+
update = update / (update.norm(dim=(-2, -1), keepdim=True) * (1 + 2e-2) + 1e-6)
|
| 183 |
+
for a, b, c in NS_COEFFS:
|
| 184 |
+
gram = update @ update.mT
|
| 185 |
+
polynomial = b * gram + c * gram @ gram
|
| 186 |
+
update = a * update + polynomial @ update
|
| 187 |
+
if transpose:
|
| 188 |
+
update = update.mT
|
| 189 |
+
p.mul_(1 - lr * MUON_WD)
|
| 190 |
+
# lr may be a 0-d tensor (CUDA-graphed step); scale in p's dtype like add_(alpha=-lr)
|
| 191 |
+
p.sub_(lr * update.reshape_as(p).to(p.dtype))
|
| 192 |
+
|
| 193 |
+
|
| 194 |
+
class MuonStep:
|
| 195 |
+
"""muon_step on `matrices` with fresh momentum buffers, called once per step after backward
|
| 196 |
+
(grads must keep fixed addresses: zero_grad(set_to_none=False)). On CUDA the first call
|
| 197 |
+
captures the step as one CUDA graph and later calls replay it instead of ~400 kernel
|
| 198 |
+
launches; CPU smoke runs call muon_step directly."""
|
| 199 |
+
|
| 200 |
+
def __init__(self, matrices, device):
|
| 201 |
+
self.matrices, self.device = matrices, device
|
| 202 |
+
self.buffers = [torch.zeros_like(p) for p in matrices]
|
| 203 |
+
self.lr, self.graph = torch.zeros((), device=device), None
|
| 204 |
+
|
| 205 |
+
def __call__(self, lr):
|
| 206 |
+
self.lr.fill_(lr)
|
| 207 |
+
if self.device != "cuda":
|
| 208 |
+
muon_step(self.matrices, self.buffers, self.lr)
|
| 209 |
+
return
|
| 210 |
+
if self.graph is None:
|
| 211 |
+
state = self.matrices + self.buffers
|
| 212 |
+
saved = [t.clone() for t in state]
|
| 213 |
+
side = torch.cuda.Stream()
|
| 214 |
+
side.wait_stream(torch.cuda.current_stream())
|
| 215 |
+
with torch.cuda.stream(side):
|
| 216 |
+
muon_step(self.matrices, self.buffers, self.lr) # warm-up (lazy cuBLAS init)
|
| 217 |
+
torch.cuda.current_stream().wait_stream(side)
|
| 218 |
+
self.graph = torch.cuda.CUDAGraph()
|
| 219 |
+
with torch.cuda.graph(self.graph):
|
| 220 |
+
muon_step(self.matrices, self.buffers, self.lr)
|
| 221 |
+
with torch.no_grad():
|
| 222 |
+
for t, v in zip(state, saved, strict=True):
|
| 223 |
+
t.copy_(v) # undo the warm-up and capture updates
|
| 224 |
+
self.graph.replay()
|
| 225 |
+
|
| 226 |
+
|
| 227 |
+
class PeriLNBlock(Block):
|
| 228 |
+
"""P22 Peri-LN (Kim et al. 2025): the fork's pre-LN block plus a LayerNorm on each branch
|
| 229 |
+
output, so every residual write has unit scale times a learned gain. Attention has no RoPE:
|
| 230 |
+
`mask` is the additive ALiBi bias (B or 1, H, T, S) that LoopedGPT builds once per forward
|
| 231 |
+
and cos/sin are unused."""
|
| 232 |
+
|
| 233 |
+
def __init__(self, d, h):
|
| 234 |
+
super().__init__(d, h)
|
| 235 |
+
self.fc1, self.fc2 = torch.nn.Linear(d, D_FF), torch.nn.Linear(D_FF, d)
|
| 236 |
+
self.ln_attn_out, self.ln_mlp_out = torch.nn.LayerNorm(d), torch.nn.LayerNorm(d)
|
| 237 |
+
|
| 238 |
+
def attn(self, x, cos, sin, mask, cache):
|
| 239 |
+
B, T, D = x.shape
|
| 240 |
+
q, k, v = self.qkv(x).view(B, T, 3, self.h, D // self.h).permute(2, 0, 3, 1, 4)
|
| 241 |
+
if cache is not None: # the fork's preallocated KV cache, unchanged
|
| 242 |
+
if "k" not in cache:
|
| 243 |
+
cache["k"] = k.new_empty(B, self.h, cache["max"], D // self.h)
|
| 244 |
+
cache["v"] = v.new_empty(B, self.h, cache["max"], D // self.h)
|
| 245 |
+
cache["n"] = 0
|
| 246 |
+
n0, n1 = cache["n"], cache["n"] + T
|
| 247 |
+
cache["k"][:, :, n0:n1] = k
|
| 248 |
+
cache["v"][:, :, n0:n1] = v
|
| 249 |
+
cache["n"] = n1
|
| 250 |
+
k, v = cache["k"][:, :, :n1], cache["v"][:, :, :n1]
|
| 251 |
+
y = torch.nn.functional.scaled_dot_product_attention(q, k, v, attn_mask=mask.to(q.dtype))
|
| 252 |
+
return self.proj(y.transpose(1, 2).reshape(B, T, D))
|
| 253 |
+
|
| 254 |
+
def forward(self, x, cos, sin, mask=None, cache=None):
|
| 255 |
+
x = x + self.ln_attn_out(self.attn(self.ln1(x), cos, sin, mask, cache))
|
| 256 |
+
return x + self.ln_mlp_out(self.fc2(torch.nn.functional.gelu(self.fc1(self.ln2(x)))))
|
| 257 |
+
|
| 258 |
+
|
| 259 |
+
class LoopedGPT(GPT):
|
| 260 |
+
"""Looped trunk (Universal Transformer, Dehghani et al. 2018; sandwich layout and input
|
| 261 |
+
injection as in Geiping et al. 2025): blocks[0] is the entry block, blocks[1:-1] the core
|
| 262 |
+
pairs, of which loop k applies pair k % N_PAIRS, blocks[-1] the exit block. Every core pass
|
| 263 |
+
starts from ln_loop(state) + e, with e the entry block's output, so each pass sees an input
|
| 264 |
+
of the same scale and the prompt is re-injected instead of being buried under accumulated
|
| 265 |
+
writes.
|
| 266 |
+
|
| 267 |
+
P34: the fork's GPT with ALiBi (Press et al. 2021) instead of RoPE. Head h's attention
|
| 268 |
+
logits get -slope_h * (query position - key position), slopes 2^-1 .. 2^-8 for 8 heads, so
|
| 269 |
+
no rotation of q/k anywhere. Positions come from `pos` (sample.generate's left-padded
|
| 270 |
+
batches) or arange; key positions are kept in caches[0] so cached decoding sees the same
|
| 271 |
+
bias as the full forward. Causality/padding come from `mask` when given, else from rel < 0."""
|
| 272 |
+
|
| 273 |
+
def __init__(self, vocab, n_layer, d, n_head, seed, max_len=1024):
|
| 274 |
+
super().__init__(vocab, n_layer, d, n_head, max_len)
|
| 275 |
+
slopes = torch.tensor(ALIBI_SLOPES).view(1, n_head, 1, 1)
|
| 276 |
+
self.register_buffer("slopes", slopes, persistent=False)
|
| 277 |
+
self.ln_loop = torch.nn.LayerNorm(d)
|
| 278 |
+
self.n_loops, self.loop_rng = DEFAULT_LOOPS, random.Random(seed)
|
| 279 |
+
# Control 590's patch: run 198's trunk, the six blocks independent and applied once; n_loops is ignored.
|
| 280 |
+
self.untied = ARCH == "untied"
|
| 281 |
+
|
| 282 |
+
def kv_layers(self):
|
| 283 |
+
"""KV-cache layers sampling allocates at the current n_loops (the harness sizes batches by it)."""
|
| 284 |
+
return len(self.blocks) if self.untied else 2 + N_CORE * self.n_loops
|
| 285 |
+
|
| 286 |
+
def forward(self, idx, pos=None, mask=None, caches=None):
|
| 287 |
+
return self.head(self.ln_f(self.features(idx, pos, mask, caches)[0]))
|
| 288 |
+
|
| 289 |
+
def features(self, idx, pos=None, mask=None, caches=None):
|
| 290 |
+
"""The trunk: (final hidden state before ln_f, the ALiBi bias used)."""
|
| 291 |
+
B, T = idx.shape
|
| 292 |
+
if pos is None: # training/val: one (1, H, T, T) bias broadcast over the batch
|
| 293 |
+
pos = torch.arange(T, device=idx.device)[None]
|
| 294 |
+
key_pos = pos
|
| 295 |
+
if caches is not None: # incremental decoding: keys are every cached token
|
| 296 |
+
c = caches[0]
|
| 297 |
+
if "pos" not in c:
|
| 298 |
+
c["pos"], c["n_pos"] = pos.new_empty(pos.size(0), c["max"]), 0
|
| 299 |
+
n0, n1 = c["n_pos"], c["n_pos"] + T
|
| 300 |
+
c["pos"][:, n0:n1] = pos
|
| 301 |
+
c["n_pos"] = n1
|
| 302 |
+
key_pos = c["pos"][:, :n1]
|
| 303 |
+
rel = pos[:, None, :, None] - key_pos[:, None, None, :] # (B or 1, 1, T, S)
|
| 304 |
+
bias = -self.slopes * rel.to(self.slopes.dtype)
|
| 305 |
+
bias = bias.masked_fill(rel < 0 if mask is None else ~mask, float("-inf"))
|
| 306 |
+
if self.untied:
|
| 307 |
+
x = self.emb(idx)
|
| 308 |
+
for j, b in enumerate(self.blocks):
|
| 309 |
+
x = b(x, None, None, bias, None if caches is None else caches[j])
|
| 310 |
+
return x, bias
|
| 311 |
+
n = self.loop_rng.choice(TRAIN_LOOPS) if self.training else self.n_loops
|
| 312 |
+
entry, *core, exit_block = self.blocks
|
| 313 |
+
x = e = entry(self.emb(idx), None, None, bias, None if caches is None else caches[0])
|
| 314 |
+
for loop in range(n):
|
| 315 |
+
x = self.ln_loop(x) + e
|
| 316 |
+
for i in range(N_CORE):
|
| 317 |
+
j = 1 + (loop % N_PAIRS) * N_CORE + i # index in self.blocks
|
| 318 |
+
# sample.generate makes one cache dict per unique block; a shared block's keys
|
| 319 |
+
# and values differ per pass, so each pass gets its own sub-cache inside it.
|
| 320 |
+
c = caches and caches[j].setdefault(loop, {"max": caches[j]["max"]})
|
| 321 |
+
x = core[j - 1](x, None, None, bias, c)
|
| 322 |
+
x = exit_block(x, None, None, bias, None if caches is None else caches[-1])
|
| 323 |
+
return x, bias
|
| 324 |
+
|
| 325 |
+
|
| 326 |
+
class MTPHead(torch.nn.Module):
|
| 327 |
+
"""The MTP module. A separate nn.Module, never registered on the model, so the harness's
|
| 328 |
+
parameter count, RL AdamW and sampler never see it; it is dropped when pretrain() returns."""
|
| 329 |
+
|
| 330 |
+
def __init__(self, d, h):
|
| 331 |
+
super().__init__()
|
| 332 |
+
self.ln_h, self.ln_e = torch.nn.LayerNorm(d), torch.nn.LayerNorm(d)
|
| 333 |
+
self.proj = torch.nn.Linear(2 * d, d, bias=False)
|
| 334 |
+
self.block = PeriLNBlock(d, h)
|
| 335 |
+
|
| 336 |
+
|
| 337 |
+
def mtp_loss(model, mtp, x, m):
|
| 338 |
+
"""The fork's loss_on (next-token CE on proof tokens) plus MTP_WEIGHT times the CE of the
|
| 339 |
+
MTP module's token-(t+2) prediction at the same proof positions. Returns (total, main)."""
|
| 340 |
+
h, bias = model.features(x[:, :-1])
|
| 341 |
+
logits = model.head(model.ln_f(h))
|
| 342 |
+
V = logits.size(-1)
|
| 343 |
+
ce = torch.nn.functional.cross_entropy(
|
| 344 |
+
logits.reshape(-1, V).float(), x[:, 1:].reshape(-1), reduction="none"
|
| 345 |
+
)
|
| 346 |
+
main = (ce * m[:, 1:].reshape(-1)).sum() / m[:, 1:].sum()
|
| 347 |
+
# Position t (0..T-3) gets h_t and emb(x_{t+1}); causal attention over these keeps every
|
| 348 |
+
# prediction of x_{t+2} a function of x_{<=t+1}.
|
| 349 |
+
z = mtp.proj(torch.cat((mtp.ln_h(h[:, :-1]), mtp.ln_e(model.emb(x[:, 1:-1]))), -1))
|
| 350 |
+
z = mtp.block(z, None, None, bias[..., :-1, :-1], None)
|
| 351 |
+
ce2 = torch.nn.functional.cross_entropy(
|
| 352 |
+
model.head(model.ln_f(z)).reshape(-1, V).float(), x[:, 2:].reshape(-1), reduction="none"
|
| 353 |
+
)
|
| 354 |
+
aux = (ce2 * m[:, 2:].reshape(-1)).sum() / m[:, 2:].sum().clamp(min=1)
|
| 355 |
+
return main + MTP_WEIGHT * aux, main, aux
|
| 356 |
+
|
| 357 |
+
|
| 358 |
+
def val_loss(model, ids, mask, ref_max, lens, tok, gen, autocast):
|
| 359 |
+
"""Mean loss over the first VAL_N records (never trained on), no shift augmentation."""
|
| 360 |
+
model.eval()
|
| 361 |
+
tot = 0.0
|
| 362 |
+
with torch.no_grad(), autocast:
|
| 363 |
+
for i in range(0, VAL_N, BS):
|
| 364 |
+
idxs = torch.arange(i, min(i + BS, VAL_N), device=ids.device)
|
| 365 |
+
x, m = ids[idxs], mask[idxs]
|
| 366 |
+
T = int(lens[idxs].max())
|
| 367 |
+
tot += loss_on(model, x[:, :T], m[:, :T]).item() * len(idxs)
|
| 368 |
+
model.train()
|
| 369 |
+
return tot / VAL_N
|
| 370 |
+
|
| 371 |
+
|
| 372 |
+
def to_gpu(data, tok, device):
|
| 373 |
+
"""Pad all (prompt, proof) records once into (N, T) ids + proof mask on the device."""
|
| 374 |
+
N, T = len(data), max(len(p) + len(q) for p, q in data)
|
| 375 |
+
ids = torch.full((N, T), tok.pad, dtype=torch.long)
|
| 376 |
+
mask = torch.zeros((N, T), dtype=torch.bool)
|
| 377 |
+
ref_max = torch.zeros(
|
| 378 |
+
N, dtype=torch.long
|
| 379 |
+
) # largest N<k> index used in the proof, for the shift augmentation
|
| 380 |
+
for i, (p, q) in enumerate(data):
|
| 381 |
+
ids[i, : len(p) + len(q)] = torch.tensor(p + q)
|
| 382 |
+
mask[i, len(p) : len(p) + len(q)] = True
|
| 383 |
+
ref_max[i] = max((x - tok.ref0 + 1 for x in q if x >= tok.ref0), default=0)
|
| 384 |
+
return ids.to(device), mask.to(device), ref_max.to(device), (ids != tok.pad).sum(1).to(device)
|
| 385 |
+
|
| 386 |
+
|
| 387 |
+
def token_batches(lens, n_train, gen, rng, first=VAL_N):
|
| 388 |
+
"""One epoch of token-budget batches: a random permutation of the n_train records from
|
| 389 |
+
`first` on, stably sorted by length (equal lengths stay shuffled), cut greedily into runs of
|
| 390 |
+
consecutive records with count * max_len <= TOK_BUDGET, and the runs' order shuffled.
|
| 391 |
+
Returns (order, cuts)."""
|
| 392 |
+
perm = torch.randperm(n_train, device=lens.device, generator=gen) + first
|
| 393 |
+
order = perm[torch.argsort(lens[perm], stable=True)]
|
| 394 |
+
sorted_lens = lens[order].cpu()
|
| 395 |
+
cuts, s = [], 0
|
| 396 |
+
while s < len(sorted_lens):
|
| 397 |
+
window = sorted_lens[s : s + TOK_BUDGET // int(sorted_lens[s])]
|
| 398 |
+
# count * max_len is non-decreasing along the window, so the fits form a prefix.
|
| 399 |
+
e = s + int((torch.arange(1, len(window) + 1) * window <= TOK_BUDGET).sum())
|
| 400 |
+
cuts.append((s, e, int(sorted_lens[e - 1]))) # sorted: last is the batch's max length
|
| 401 |
+
s = e
|
| 402 |
+
rng.shuffle(cuts)
|
| 403 |
+
return order, cuts
|
| 404 |
+
|
| 405 |
+
|
| 406 |
+
def gpu_batch(ids, mask, ref_max, lens, idxs, tok, gen, max_len=None, augment=True):
|
| 407 |
+
"""The fork's batch() + tok.shift_abs, vectorised: a random per-row offset is added to
|
| 408 |
+
every ref token, and each row's atoms go through a random permutation of the ten atom tokens
|
| 409 |
+
(positions after the statement header, which declares all ten in a fixed order). augment=False
|
| 410 |
+
leaves rows as sampled (policy-gradient rollouts)."""
|
| 411 |
+
x, m = ids[idxs], mask[idxs]
|
| 412 |
+
# Known on the host for training batches: passing it avoids a GPU sync every step.
|
| 413 |
+
T = int(lens[idxs].max()) if max_len is None else max_len
|
| 414 |
+
x, m = x[:, :T].clone(), m[:, :T]
|
| 415 |
+
if not augment:
|
| 416 |
+
return x, m
|
| 417 |
+
atom_ids = torch.tensor(tok.atom_ids, device=x.device)
|
| 418 |
+
slot = torch.full((tok.vocab_size,), -1, dtype=torch.long, device=x.device)
|
| 419 |
+
slot[atom_ids] = torch.arange(len(atom_ids), device=x.device)
|
| 420 |
+
a = slot[x]
|
| 421 |
+
a[:, :HEADER_LEN] = -1
|
| 422 |
+
perm = torch.rand(len(idxs), len(atom_ids), device=x.device, generator=gen).argsort(1)
|
| 423 |
+
x = torch.where(a >= 0, atom_ids[perm.gather(1, a.clamp(min=0))], x)
|
| 424 |
+
if tok.mode in ("abs", "lean_seq"): # lean_seq: names n1.. get the same start offset
|
| 425 |
+
room = MAXN - ref_max[idxs]
|
| 426 |
+
shift = (
|
| 427 |
+
(torch.rand(len(idxs), device=x.device, generator=gen) * (room + 1))
|
| 428 |
+
.long()
|
| 429 |
+
.clamp(max=room)
|
| 430 |
+
)
|
| 431 |
+
is_ref = (x >= tok.ref0) & m
|
| 432 |
+
x = torch.where(is_ref, x + shift[:, None], x)
|
| 433 |
+
return x, m
|
| 434 |
+
|
| 435 |
+
|
| 436 |
+
def pretrain(train_recs, budget_secs, device, seed, log):
|
| 437 |
+
t0 = time.time()
|
| 438 |
+
tok = make_tokenizer(TOK_MODE)
|
| 439 |
+
data = [(tok.encode_prompt(r["prompt"]), tok.encode_proof(r["proof"])) for r in train_recs]
|
| 440 |
+
ids, mask, ref_max, lens = to_gpu(data, tok, device)
|
| 441 |
+
gen = torch.Generator(device=device)
|
| 442 |
+
gen.manual_seed(seed)
|
| 443 |
+
model = LoopedGPT(tok.vocab_size, N_LAYER, D, N_HEAD, seed)
|
| 444 |
+
model.blocks = torch.nn.ModuleList(PeriLNBlock(D, N_HEAD) for _ in range(N_LAYER))
|
| 445 |
+
model.blocks.apply(model._init) # fork init: zero biases
|
| 446 |
+
model = model.to(device)
|
| 447 |
+
mtp = MTPHead(D, N_HEAD)
|
| 448 |
+
mtp.apply(model._init)
|
| 449 |
+
mtp = mtp.to(device)
|
| 450 |
+
# Compile the repeated blocks once each (torch >= 2.9; dynamic shapes, ~14 s): ~14% faster
|
| 451 |
+
# steps. The coalescing tiling analysis crashes on dynamic shapes, so it is off. Undone at
|
| 452 |
+
# handoff so the harness's RL fine-tune and KV-cache sampling run eager.
|
| 453 |
+
torch._inductor.config.triton.coalesce_tiling_analysis = False
|
| 454 |
+
compiled = list(model.blocks) + [mtp.block]
|
| 455 |
+
for b in compiled if device == "cuda" else []: # CPU smoke runs stay eager
|
| 456 |
+
b.compile(dynamic=True)
|
| 457 |
+
for p in list(model.blocks.parameters()) + list(mtp.parameters()):
|
| 458 |
+
if p.ndim == 2:
|
| 459 |
+
# Orthogonal matrices have element second moment gain^2 / max(rows, cols).
|
| 460 |
+
torch.nn.init.orthogonal_(p, gain=0.02 * math.sqrt(max(p.shape)))
|
| 461 |
+
log(
|
| 462 |
+
f"pretrain: {model.n_params():,} params, {len(data):,} records on {device} "
|
| 463 |
+
f"after {time.time() - t0:.0f}s, budget {budget_secs}s"
|
| 464 |
+
)
|
| 465 |
+
# The output readout joins the hidden matrices under Muon; embeddings stay on AdamW.
|
| 466 |
+
trainable = list(model.parameters()) + list(mtp.parameters())
|
| 467 |
+
matrices = [p for p in model.blocks.parameters() if p.ndim == 2] + [model.head.weight]
|
| 468 |
+
matrices += [p for p in mtp.parameters() if p.ndim == 2]
|
| 469 |
+
matrix_ids = {id(p) for p in matrices}
|
| 470 |
+
auxiliary = [p for p in trainable if id(p) not in matrix_ids]
|
| 471 |
+
muon = MuonStep(matrices, device)
|
| 472 |
+
opt = torch.optim.AdamW(
|
| 473 |
+
auxiliary, lr=LR, weight_decay=WD, betas=(0.9, 0.95), fused=device == "cuda"
|
| 474 |
+
)
|
| 475 |
+
autocast = torch.autocast("cuda", dtype=torch.bfloat16, enabled=(device == "cuda"))
|
| 476 |
+
model.train()
|
| 477 |
+
step, last_loss, records_seen = 0, float("nan"), 0
|
| 478 |
+
curve, recent, recent_aux, next_point = [], [], [], CURVE_EVERY
|
| 479 |
+
n_train = len(data) - VAL_N # records [0, VAL_N) are the val slice
|
| 480 |
+
rng, cuts, batches_per_epoch = random.Random(seed), [], None
|
| 481 |
+
order = torch.empty(0, dtype=torch.long, device=device)
|
| 482 |
+
while True:
|
| 483 |
+
frac = (
|
| 484 |
+
time.time() - t0
|
| 485 |
+
) / budget_secs # cosine over wall-clock, so the cap is the schedule
|
| 486 |
+
if frac >= 1.0:
|
| 487 |
+
break
|
| 488 |
+
step += 1
|
| 489 |
+
# No warmup (as in modded-nanogpt): Muon's step is normalized and Adam's is ~lr per
|
| 490 |
+
# element from step 1, and Peri-LN normalizes every branch write.
|
| 491 |
+
lr = LR * MIN_LR_FRAC + 0.5 * LR * (1 - MIN_LR_FRAC) * (1 + math.cos(math.pi * frac))
|
| 492 |
+
for g in opt.param_groups:
|
| 493 |
+
g["lr"] = lr
|
| 494 |
+
if not cuts: # new epoch
|
| 495 |
+
order, cuts = token_batches(lens, n_train, gen, rng)
|
| 496 |
+
batches_per_epoch = len(cuts)
|
| 497 |
+
s, e, T = cuts.pop()
|
| 498 |
+
idxs = order[s:e]
|
| 499 |
+
records_seen += e - s
|
| 500 |
+
x, m = gpu_batch(ids, mask, ref_max, lens, idxs, tok, gen, T)
|
| 501 |
+
with autocast:
|
| 502 |
+
loss, main, aux = mtp_loss(model, mtp, x, m)
|
| 503 |
+
model.zero_grad(set_to_none=False) # grads keep fixed addresses for the Muon CUDA graph
|
| 504 |
+
mtp.zero_grad(set_to_none=False)
|
| 505 |
+
loss.backward()
|
| 506 |
+
torch.nn.utils.clip_grad_norm_(trainable, 1.0)
|
| 507 |
+
opt.step()
|
| 508 |
+
muon(MUON_LR * lr / LR)
|
| 509 |
+
if step % 20 == 0:
|
| 510 |
+
last_loss = main.item() # the next-token CE, comparable with earlier runs
|
| 511 |
+
recent.append(last_loss)
|
| 512 |
+
recent_aux.append(aux.item())
|
| 513 |
+
if (
|
| 514 |
+
time.time() - t0 >= next_point
|
| 515 |
+
): # curve point: mean recent train loss + val loss on the held-out slice
|
| 516 |
+
curve.append(
|
| 517 |
+
{
|
| 518 |
+
"secs": round(time.time() - t0),
|
| 519 |
+
"step": step,
|
| 520 |
+
"train_loss": sum(recent) / len(recent) if recent else last_loss,
|
| 521 |
+
"val_loss": val_loss(model, ids, mask, ref_max, lens, tok, gen, autocast),
|
| 522 |
+
"mtp_loss": sum(recent_aux) / len(recent_aux) if recent_aux else None,
|
| 523 |
+
}
|
| 524 |
+
)
|
| 525 |
+
log(
|
| 526 |
+
f"pretrain step {step} train {curve[-1]['train_loss']:.4f} "
|
| 527 |
+
f"val {curve[-1]['val_loss']:.4f} lr {lr:.2e} {curve[-1]['secs']}s"
|
| 528 |
+
)
|
| 529 |
+
recent, recent_aux, next_point = [], [], next_point + CURVE_EVERY
|
| 530 |
+
val_by_loops = {}
|
| 531 |
+
for n in (VAL_LOOPS if not model.untied else (DEFAULT_LOOPS,)) if step else (): # no steps: a checkpoint rebuild, not a training run
|
| 532 |
+
model.n_loops = n
|
| 533 |
+
val_by_loops[n] = val_loss(model, ids, mask, ref_max, lens, tok, gen, autocast)
|
| 534 |
+
model.n_loops = DEFAULT_LOOPS
|
| 535 |
+
log(f"pretrain val loss by loops: {val_by_loops}")
|
| 536 |
+
for b in compiled:
|
| 537 |
+
b._compiled_call_impl = None
|
| 538 |
+
model.eval()
|
| 539 |
+
return (
|
| 540 |
+
model,
|
| 541 |
+
tok,
|
| 542 |
+
{
|
| 543 |
+
"steps": step,
|
| 544 |
+
"records_seen": records_seen,
|
| 545 |
+
"epochs": records_seen / n_train,
|
| 546 |
+
"batches_per_epoch": batches_per_epoch,
|
| 547 |
+
"tok_budget": TOK_BUDGET,
|
| 548 |
+
"last_loss": last_loss,
|
| 549 |
+
"train_loops": TRAIN_LOOPS,
|
| 550 |
+
"default_loops": DEFAULT_LOOPS,
|
| 551 |
+
"val_by_loops": val_by_loops,
|
| 552 |
+
"curve": curve,
|
| 553 |
+
},
|
| 554 |
+
)
|
| 555 |
+
|
| 556 |
+
|
| 557 |
+
def finetune(model, tok, data, rng, device):
|
| 558 |
+
"""The RL fine-tune: pretraining's optimizers (Muon on the hidden matrices, AdamW on the
|
| 559 |
+
rest), linear warmup then cosine to lr/10, over FT_RECORDS records in token-budget batches
|
| 560 |
+
(to_gpu / token_batches / gpu_batch, as pretraining). Muon's momentum restarts every round,
|
| 561 |
+
as AdamW's moments did. The model is in train mode, so every batch draws its own loop count
|
| 562 |
+
from TRAIN_LOOPS. Returns (last loss, steps)."""
|
| 563 |
+
ids, mask, ref_max, lens = to_gpu(data, tok, device)
|
| 564 |
+
gen = torch.Generator(device=device)
|
| 565 |
+
gen.manual_seed(rng.getrandbits(31))
|
| 566 |
+
plan, seen = [], 0 # (order, start, end, padded length) per step, epoch after epoch
|
| 567 |
+
while seen < FT_RECORDS:
|
| 568 |
+
order, cuts = token_batches(lens, len(data), gen, rng, first=0)
|
| 569 |
+
for s, e, T in cuts:
|
| 570 |
+
if seen >= FT_RECORDS:
|
| 571 |
+
break
|
| 572 |
+
plan.append((order, s, e, T))
|
| 573 |
+
seen += e - s
|
| 574 |
+
matrices = [p for p in model.blocks.parameters() if p.ndim == 2] + [model.head.weight]
|
| 575 |
+
matrix_ids = {id(p) for p in matrices}
|
| 576 |
+
auxiliary = [p for p in model.parameters() if id(p) not in matrix_ids]
|
| 577 |
+
opt = torch.optim.AdamW(auxiliary, lr=FT_LR, weight_decay=0.1, betas=(0.9, 0.95))
|
| 578 |
+
muon = MuonStep(matrices, device)
|
| 579 |
+
autocast = torch.autocast("cuda", dtype=torch.bfloat16, enabled=(device == "cuda"))
|
| 580 |
+
model.train()
|
| 581 |
+
for step, (order, s, e, T) in enumerate(plan, 1):
|
| 582 |
+
lr = (
|
| 583 |
+
FT_LR * step / FT_WARMUP
|
| 584 |
+
if step < FT_WARMUP
|
| 585 |
+
else FT_LR / 10 + 0.45 * FT_LR * (1 + math.cos(math.pi * step / len(plan)))
|
| 586 |
+
)
|
| 587 |
+
for g in opt.param_groups:
|
| 588 |
+
g["lr"] = lr
|
| 589 |
+
x, m = gpu_batch(ids, mask, ref_max, lens, order[s:e], tok, gen, T)
|
| 590 |
+
with autocast:
|
| 591 |
+
loss = loss_on(model, x, m)
|
| 592 |
+
model.zero_grad(set_to_none=False) # grads keep fixed addresses for the Muon CUDA graph
|
| 593 |
+
loss.backward()
|
| 594 |
+
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
|
| 595 |
+
opt.step()
|
| 596 |
+
muon(FT_MUON_LR * lr / FT_LR)
|
| 597 |
+
model.eval()
|
| 598 |
+
return loss.item(), len(plan)
|
| 599 |
+
|
| 600 |
+
|
| 601 |
+
# Targets per round, picked by pick_targets: the old harness's fixed 1,500 targets, now drawn from the
|
| 602 |
+
# whole pool (thousands, body 7 to ~45) with the samples aimed at the frontier.
|
| 603 |
+
N_ROUND = 1500
|
| 604 |
+
|
| 605 |
+
|
| 606 |
+
def pick_targets(targets, found, rng, n=N_ROUND):
|
| 607 |
+
"""n targets for this round. Bands (body lines) get weight 0.1 + their solved fraction up to one
|
| 608 |
+
band past the highest band with any solve, and 0.02 beyond it, so sampling follows the frontier
|
| 609 |
+
instead of spending most of a round on targets far out of reach; within a band, unsolved targets
|
| 610 |
+
come first."""
|
| 611 |
+
bands = sorted({t["band"] for t in targets}, key=lambda b: int(b.split("-")[0].rstrip("+")))
|
| 612 |
+
by = {b: [t for t in targets if t["band"] == b] for b in bands}
|
| 613 |
+
rate = {b: sum(bool(found[t["name"]]) for t in by[b]) / len(by[b]) for b in bands}
|
| 614 |
+
top = max((i for i, b in enumerate(bands) if rate[b] > 0), default=-1)
|
| 615 |
+
w = {b: (0.1 + rate[b]) if i <= top + 1 else 0.02 for i, b in enumerate(bands)}
|
| 616 |
+
out = []
|
| 617 |
+
for b in bands:
|
| 618 |
+
k = min(len(by[b]), round(n * w[b] / sum(w.values())))
|
| 619 |
+
unsolved = [t for t in by[b] if not found[t["name"]]]
|
| 620 |
+
solved = [t for t in by[b] if found[t["name"]]]
|
| 621 |
+
rng.shuffle(unsolved)
|
| 622 |
+
rng.shuffle(solved)
|
| 623 |
+
out += (unsolved + solved)[:k]
|
| 624 |
+
return out
|
| 625 |
+
|
| 626 |
+
|
| 627 |
+
@torch.no_grad()
|
| 628 |
+
def rollout_stats(model, tok, rows, device):
|
| 629 |
+
"""Diagnostics on a round's own rollouts, before any update on them, at the model's current n_loops: mean per-token
|
| 630 |
+
entropy of the policy at temperature 1 (sampling uses TEMP), mean -log p of the sampled tokens, mean rollout
|
| 631 |
+
length in tokens, and targets whose rollouts all pass / all fail (zero GRPO advantage). Batching uses its own
|
| 632 |
+
generator and rng, so the run's random streams are the same as without it."""
|
| 633 |
+
items = [(tok.encode_prompt(r["prompt"]), ids) for r in rows for ids, _ in r["rollouts"]]
|
| 634 |
+
ids, mask, _, lens = to_gpu(items, tok, device)
|
| 635 |
+
gen = torch.Generator(device=device)
|
| 636 |
+
gen.manual_seed(0)
|
| 637 |
+
order, cuts = token_batches(lens, len(items), gen, random.Random(0), first=0)
|
| 638 |
+
model.eval()
|
| 639 |
+
ent, nll, n = 0.0, 0.0, 0
|
| 640 |
+
for s, e, T in cuts:
|
| 641 |
+
x, m = gpu_batch(ids, mask, None, lens, order[s:e], tok, gen, T, augment=False)
|
| 642 |
+
with torch.autocast("cuda", dtype=torch.bfloat16, enabled=(device == "cuda")):
|
| 643 |
+
logits = model(x[:, :-1])
|
| 644 |
+
logp, m = torch.log_softmax(logits.float(), -1), m[:, 1:]
|
| 645 |
+
ent += ((-(logp.exp() * logp).sum(-1)) * m).sum()
|
| 646 |
+
nll += (-logp.gather(-1, x[:, 1:, None]).squeeze(-1) * m).sum()
|
| 647 |
+
n += m.sum()
|
| 648 |
+
return {"entropy": float(ent / n), "nll": float(nll / n), "tokens": float(n) / len(items),
|
| 649 |
+
"all_pass": sum(all(ok for _, ok in r["rollouts"]) for r in rows),
|
| 650 |
+
"all_fail": sum(not any(ok for _, ok in r["rollouts"]) for r in rows)}
|
| 651 |
+
|
| 652 |
+
|
| 653 |
+
def fmt_stats(rs):
|
| 654 |
+
return (f"rollout entropy {rs['entropy']:.3f}, nll {rs['nll']:.3f}, {rs['tokens']:.0f} tokens, "
|
| 655 |
+
f"all pass {rs['all_pass']}, all fail {rs['all_fail']}")
|
| 656 |
+
|
| 657 |
+
|
| 658 |
+
def rl(model, tok, targets, train_recs, oracle, budget_secs, device, seed, log):
|
| 659 |
+
if RL_ALGO == "grpo":
|
| 660 |
+
return rl_grpo(model, tok, targets, train_recs, oracle, device, seed, log)
|
| 661 |
+
return rl_ei(model, tok, targets, train_recs, oracle, device, seed, log)
|
| 662 |
+
|
| 663 |
+
|
| 664 |
+
def rl_ei(model, tok, targets, train_recs, oracle, device, seed, log):
|
| 665 |
+
rng = random.Random(seed)
|
| 666 |
+
found = {t["name"]: {} for t in targets} # name -> {normalized proof: proof text}
|
| 667 |
+
rounds, round_secs, retries, ft_steps, history = 0, 0.0, [], [], []
|
| 668 |
+
# Start a round only if one as long as the last (plus 15%) still fits in the budget.
|
| 669 |
+
while oracle.left() > 1.15 * round_secs:
|
| 670 |
+
t0 = oracle.used()
|
| 671 |
+
model.n_loops = RL_FIRST_LOOPS
|
| 672 |
+
picked = pick_targets(targets, found, rng)
|
| 673 |
+
rows = oracle.sample_rollouts(model, picked, RL_K)
|
| 674 |
+
rs = rollout_stats(model, tok, rows, device) # first pass only: comparable to the GRPO arm's rounds
|
| 675 |
+
failed = [t for t, row in zip(picked, rows, strict=True) if not row["proofs"]]
|
| 676 |
+
rng.shuffle(failed)
|
| 677 |
+
for i, n in enumerate(RL_RETRY_LOOPS):
|
| 678 |
+
part = failed[i :: len(RL_RETRY_LOOPS)]
|
| 679 |
+
if not part:
|
| 680 |
+
continue
|
| 681 |
+
model.n_loops = n
|
| 682 |
+
part_rows = oracle.sample(model, part, RL_RETRY_K)
|
| 683 |
+
cracked = sum(bool(row["proofs"]) for row in part_rows)
|
| 684 |
+
retries.append(
|
| 685 |
+
{"round": rounds + 1, "loops": n, "targets": len(part), "cracked": cracked}
|
| 686 |
+
)
|
| 687 |
+
rows += part_rows
|
| 688 |
+
for row in rows:
|
| 689 |
+
for proof in row["proofs"]:
|
| 690 |
+
found[row["name"]].setdefault(norm(proof), proof)
|
| 691 |
+
mix = []
|
| 692 |
+
for t in targets:
|
| 693 |
+
proofs = list(found[t["name"]].values())
|
| 694 |
+
rng.shuffle(proofs)
|
| 695 |
+
mix += [(t["prompt"], proof) for proof in proofs[:MAX_PER_THM]] * RL_WEIGHT
|
| 696 |
+
n_rl = len(mix)
|
| 697 |
+
# No solves yet: skip the fine-tune and sample again (the old harness stopped RL here). The round is still
|
| 698 |
+
# logged, with its rollout stats.
|
| 699 |
+
loss, steps = float("nan"), 0
|
| 700 |
+
if n_rl:
|
| 701 |
+
mix += [(r["prompt"], r["proof"]) for r in rng.sample(train_recs, RETAIN)]
|
| 702 |
+
data = [(tok.encode_prompt(p), tok.encode_proof(q)) for p, q in mix]
|
| 703 |
+
loss, steps = finetune(model, tok, data, rng, device)
|
| 704 |
+
ft_steps.append(steps)
|
| 705 |
+
rounds += 1
|
| 706 |
+
round_secs = oracle.used() - t0
|
| 707 |
+
history.append({"round": rounds, "rollouts": rs})
|
| 708 |
+
log(
|
| 709 |
+
f"rl round {rounds}: {n_rl} RL records, fine-tune loss {loss:.4f} in {steps} steps, "
|
| 710 |
+
f"{fmt_stats(rs)}, round {round_secs:.0f}s, {oracle.left():.0f}s left"
|
| 711 |
+
)
|
| 712 |
+
model.n_loops = DEFAULT_LOOPS
|
| 713 |
+
return {
|
| 714 |
+
"rounds": rounds,
|
| 715 |
+
"first_loops": RL_FIRST_LOOPS,
|
| 716 |
+
"k_first": RL_K,
|
| 717 |
+
"retry_loops": RL_RETRY_LOOPS,
|
| 718 |
+
"k_retry": RL_RETRY_K,
|
| 719 |
+
"retries": retries,
|
| 720 |
+
"ft_records": FT_RECORDS,
|
| 721 |
+
"ft_steps": ft_steps,
|
| 722 |
+
"ft_optimizer": "muon (polar express) on block matrices and head + adamw on the rest",
|
| 723 |
+
"ft_lr": FT_LR,
|
| 724 |
+
"ft_muon_lr": FT_MUON_LR,
|
| 725 |
+
"history": history,
|
| 726 |
+
}
|
| 727 |
+
|
| 728 |
+
|
| 729 |
+
# ---- GRPO arm (ND_RL=grpo). Group-relative policy gradient (GRPO, Shao et al. 2024) with Dr. GRPO's
|
| 730 |
+
# unnormalised advantages r - mean(r) and a constant length normaliser instead of each rollout's length
|
| 731 |
+
# (Liu et al. 2025), and DAPO's dropping of groups that are all pass or all fail (Yu et al. 2025), since
|
| 732 |
+
# they carry no signal. The found-proof pool: once Lean accepts a proof of a target, a later group on
|
| 733 |
+
# that target with no success gets one stored proof in place of a failed rollout, with reward 1, so a
|
| 734 |
+
# target solved once keeps giving a gradient (that member is off-policy; the rest are the model's own).
|
| 735 |
+
# Same target picking, rollouts per target (EI's first pass) and budget as the EI arm; one on-policy pass
|
| 736 |
+
# over each round's rollouts. AdamW on every parameter: Muon's normalised step would turn a noisy
|
| 737 |
+
# policy gradient into a full-size update. Untuned: lr and retain weight are first guesses.
|
| 738 |
+
GRPO_G = 8 # rollouts per target per round
|
| 739 |
+
GRPO_LR, GRPO_WARMUP = 1e-4, 20
|
| 740 |
+
GRPO_LEN_NORM = 256 # constant per-rollout token normaliser
|
| 741 |
+
GRPO_RETAIN_W = 0.5 # retained-pretraining SFT term per step, in place of EI's replayed records
|
| 742 |
+
|
| 743 |
+
|
| 744 |
+
def rl_grpo(model, tok, targets, train_recs, oracle, device, seed, log):
|
| 745 |
+
rng = random.Random(seed)
|
| 746 |
+
gen = torch.Generator(device=device)
|
| 747 |
+
gen.manual_seed(seed)
|
| 748 |
+
found = {t["name"]: {} for t in targets}
|
| 749 |
+
opt = torch.optim.AdamW(model.parameters(), lr=GRPO_LR, weight_decay=0.0, betas=(0.9, 0.95))
|
| 750 |
+
autocast = torch.autocast("cuda", dtype=torch.bfloat16, enabled=(device == "cuda"))
|
| 751 |
+
rounds, round_secs, history, step = 0, 0.0, [], 0
|
| 752 |
+
while oracle.left() > 1.15 * round_secs:
|
| 753 |
+
t0 = oracle.used()
|
| 754 |
+
model.n_loops = RL_FIRST_LOOPS
|
| 755 |
+
picked = pick_targets(targets, found, rng)
|
| 756 |
+
rows = oracle.sample_rollouts(model, picked, GRPO_G)
|
| 757 |
+
rs = rollout_stats(model, tok, rows, device) # on-policy, before the found-proof injection and the update
|
| 758 |
+
items, adv, st = [], [], collections.Counter()
|
| 759 |
+
for row in rows:
|
| 760 |
+
for p in row["proofs"]:
|
| 761 |
+
found[row["name"]].setdefault(norm(p), p)
|
| 762 |
+
group = [(ids, float(ok)) for ids, ok in row["rollouts"]]
|
| 763 |
+
if not any(r for _, r in group) and found[row["name"]]:
|
| 764 |
+
group[0] = (tok.encode_proof(rng.choice(list(found[row["name"]].values()))), 1.0)
|
| 765 |
+
st["injected"] += 1
|
| 766 |
+
mean = sum(r for _, r in group) / len(group)
|
| 767 |
+
if mean in (0.0, 1.0):
|
| 768 |
+
st["all_fail" if mean == 0 else "all_pass"] += 1
|
| 769 |
+
continue
|
| 770 |
+
prompt = tok.encode_prompt(row["prompt"])
|
| 771 |
+
items += [(prompt, ids) for ids, _ in group]
|
| 772 |
+
adv += [r - mean for _, r in group]
|
| 773 |
+
st["groups"] += 1
|
| 774 |
+
if items:
|
| 775 |
+
retain = [(tok.encode_prompt(r["prompt"]), tok.encode_proof(r["proof"]))
|
| 776 |
+
for r in rng.sample(train_recs, min(len(items), len(train_recs)))]
|
| 777 |
+
loss, n_steps = pg_update(model, tok, items, adv, retain, opt, step, gen, rng, autocast, device)
|
| 778 |
+
step += n_steps
|
| 779 |
+
st["steps"], st["items"] = n_steps, len(items)
|
| 780 |
+
rounds += 1
|
| 781 |
+
round_secs = oracle.used() - t0
|
| 782 |
+
history.append({"round": rounds, **st, "rollouts": rs})
|
| 783 |
+
log(f"grpo round {rounds}: {dict(st)}, {fmt_stats(rs)}, round {round_secs:.0f}s, {oracle.left():.0f}s left")
|
| 784 |
+
model.n_loops = DEFAULT_LOOPS
|
| 785 |
+
model.eval()
|
| 786 |
+
return {"rounds": rounds, "algo": "grpo", "g": GRPO_G, "lr": GRPO_LR, "len_norm": GRPO_LEN_NORM,
|
| 787 |
+
"retain_w": GRPO_RETAIN_W, "history": history}
|
| 788 |
+
|
| 789 |
+
|
| 790 |
+
def pg_update(model, tok, items, adv, retain, opt, step0, gen, rng, autocast, device):
|
| 791 |
+
"""One pass over the round's rollouts in token-budget batches, each step paired with a retained
|
| 792 |
+
pretraining batch (augmented, as in pretraining). Returns (last loss, steps)."""
|
| 793 |
+
ids, mask, _, lens = to_gpu(items, tok, device)
|
| 794 |
+
adv_t = torch.tensor(adv, device=device)
|
| 795 |
+
r_ids, r_mask, r_ref, r_lens = to_gpu(retain, tok, device)
|
| 796 |
+
order, cuts = token_batches(lens, len(items), gen, rng, first=0)
|
| 797 |
+
r_order, r_cuts = token_batches(r_lens, len(retain), gen, rng, first=0)
|
| 798 |
+
model.train()
|
| 799 |
+
loss = torch.zeros(())
|
| 800 |
+
for k, (s, e, T) in enumerate(cuts):
|
| 801 |
+
step = step0 + k + 1
|
| 802 |
+
for g in opt.param_groups:
|
| 803 |
+
g["lr"] = GRPO_LR * min(1.0, step / GRPO_WARMUP)
|
| 804 |
+
idxs = order[s:e]
|
| 805 |
+
x, m = gpu_batch(ids, mask, None, lens, idxs, tok, gen, T, augment=False)
|
| 806 |
+
with autocast:
|
| 807 |
+
logits = model(x[:, :-1])
|
| 808 |
+
logp = -torch.nn.functional.cross_entropy(
|
| 809 |
+
logits.reshape(-1, logits.size(-1)).float(), x[:, 1:].reshape(-1), reduction="none"
|
| 810 |
+
).view(len(idxs), -1)
|
| 811 |
+
pg = -(adv_t[idxs] * (logp * m[:, 1:]).sum(1)).sum() / (len(idxs) * GRPO_LEN_NORM)
|
| 812 |
+
rs, re_, rT = r_cuts[k % len(r_cuts)]
|
| 813 |
+
rx, rm = gpu_batch(r_ids, r_mask, r_ref, r_lens, r_order[rs:re_], tok, gen, rT)
|
| 814 |
+
loss = pg + GRPO_RETAIN_W * loss_on(model, rx, rm)
|
| 815 |
+
opt.zero_grad(set_to_none=True)
|
| 816 |
+
loss.backward()
|
| 817 |
+
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
|
| 818 |
+
opt.step()
|
| 819 |
+
model.eval()
|
| 820 |
+
return loss.item(), len(cuts)
|
runs/long-rl-20261014-v2_untied-ei_s1/source/remote.sh
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
# Runs on the pod. Checks out Dan's fork at the pinned commit (model, sampler, nd2lean; the tokenizer and the Lean
|
| 3 |
+
# gate come from overlay/), installs Lean, runs the harness.
|
| 4 |
+
# Usage: ND_ARCH=untied|looped ND_RL=ei|grpo bash remote.sh <out-dir> [harness.py args...]
|
| 5 |
+
set -euo pipefail
|
| 6 |
+
FORK_COMMIT=51604b3feaa5e66821d1faba22238fe6d96d46d0 # dan-pandori/nd-takehome, branch dan_lean_seed2
|
| 7 |
+
export ND_FORK=/workspace/nd-takehome
|
| 8 |
+
OUT=$1; shift
|
| 9 |
+
HERE=$(cd "$(dirname "$0")" && pwd)
|
| 10 |
+
|
| 11 |
+
if [ ! -d "$ND_FORK/.git" ]; then
|
| 12 |
+
git clone -q https://github.com/dan-pandori/nd-takehome.git "$ND_FORK"
|
| 13 |
+
fi
|
| 14 |
+
git -C "$ND_FORK" cat-file -e "$FORK_COMMIT" 2>/dev/null || git -C "$ND_FORK" fetch -q origin
|
| 15 |
+
git -C "$ND_FORK" checkout -q "$FORK_COMMIT"
|
| 16 |
+
[ -f "$ND_FORK/data/train.jsonl" ] || gunzip -k "$ND_FORK/data/train.jsonl.gz"
|
| 17 |
+
# Lean 4 core is the only checker (overlay/lean_gate.py). Installed before harness.py starts, so it is not charged.
|
| 18 |
+
[ -x ~/.elan/bin/lean ] || curl -sSf https://raw.githubusercontent.com/leanprover/elan/master/elan-init.sh \
|
| 19 |
+
| sh -s -- -y --default-toolchain leanprover/lean4:v4.34.0 > /dev/null
|
| 20 |
+
export LEAN_GATE_WORKERS=$(nproc) LEAN_GATE_LOG="$OUT/lean_gate.jsonl" ND_DATA="$HERE/data"
|
| 21 |
+
mkdir -p "$OUT"
|
| 22 |
+
nvidia-smi --query-gpu=name,memory.total --format=csv,noheader
|
| 23 |
+
echo "cpus $(nproc), $(~/.elan/bin/lean --version), arm $ND_ARCH/$ND_RL"
|
| 24 |
+
PY=$(for p in python python3; do "$p" -c "import torch" 2>/dev/null && echo "$p" && break; done)
|
| 25 |
+
echo "python $($PY --version 2>&1), torch $($PY -c 'import torch; print(torch.__version__)')"
|
| 26 |
+
cd "$HERE"
|
| 27 |
+
$PY harness.py --out "$OUT" "$@" 2>&1 | tee "$OUT.log"
|
runs/long-rl-20261014-v2_untied-ei_s1/summary.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|