robbiethompson2018 commited on
Commit
7cd5bdc
·
verified ·
1 Parent(s): 44e6b72

Add 590_control-untied-198_s0

Browse files
runs/590_control-untied-198_s0/eval_dev.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
runs/590_control-untied-198_s0/final.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c3952712c7b473ab557a5dd74748fa7e23ca082f7af5bc17793e509f29f7faa6
3
+ size 38268575
runs/590_control-untied-198_s0/found_targets.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
runs/590_control-untied-198_s0/lean_gate.jsonl ADDED
@@ -0,0 +1,84 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"utc": "2026-10-07T22:06:45Z", "samples": 2000, "parse_fail": 9, "distinct_checked": 1991, "both_ok": 1937, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 54, "parse_reasons": {"unbound": 7, "no-eos": 1, "name": 1}, "lean_wall_s": 3.5214762687683105, "lean_proc_s": 17.022944927215576, "nd_verify_s": 0.10487580299377441, "workers": 96, "chunk": 400}
2
+ {"utc": "2026-10-07T22:07:53Z", "samples": 12000, "parse_fail": 4055, "distinct_checked": 7716, "both_ok": 2506, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 5210, "parse_reasons": {"unbound": 3223, "formula": 295, "expected )": 152, "exact not last": 144, "name": 87, "no-eos": 49, "expected (": 36, "expected ;": 30, "PR position": 14, "expected :=": 6, "ORE box": 5, "trailing tokens": 5, "expected False": 4, "expected =>": 2, "expected .elim": 1, "expected fun": 1, "expected None": 1}, "lean_wall_s": 24.845098972320557, "lean_proc_s": 475.8859384059906, "nd_verify_s": 0.4249696731567383, "workers": 96, "chunk": 400}
3
+ {"utc": "2026-10-07T22:08:32Z", "samples": 7008, "parse_fail": 2640, "distinct_checked": 4041, "both_ok": 75, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 3966, "parse_reasons": {"unbound": 2104, "formula": 187, "expected )": 105, "exact not last": 77, "name": 61, "expected (": 41, "no-eos": 21, "expected ;": 10, "expected :=": 10, "ORE box": 9, "PR position": 9, "trailing tokens": 4, "expected False": 2}, "lean_wall_s": 14.552488327026367, "lean_proc_s": 145.86501574516296, "nd_verify_s": 0.2135484218597412, "workers": 96, "chunk": 400}
4
+ {"utc": "2026-10-07T22:09:07Z", "samples": 7008, "parse_fail": 2542, "distinct_checked": 4146, "both_ok": 46, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 4100, "parse_reasons": {"unbound": 1980, "formula": 227, "exact not last": 95, "expected )": 75, "name": 70, "no-eos": 52, "expected ;": 15, "expected (": 10, "PR position": 5, "trailing tokens": 4, "expected =>": 2, "expected False": 2, "expected .elim": 1, "expected :=": 1, "expected fun": 1, "ORE box": 1, "eof": 1}, "lean_wall_s": 15.492853164672852, "lean_proc_s": 160.28228092193604, "nd_verify_s": 0.19858956336975098, "workers": 96, "chunk": 400}
5
+ {"utc": "2026-10-07T22:09:43Z", "samples": 6984, "parse_fail": 2601, "distinct_checked": 4060, "both_ok": 86, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 3974, "parse_reasons": {"unbound": 1922, "formula": 298, "exact not last": 96, "name": 79, "expected )": 74, "no-eos": 49, "expected (": 48, "expected ;": 20, "PR position": 5, "trailing tokens": 4, "ORE box": 3, "expected =>": 1, "expected False": 1, "expected fun": 1}, "lean_wall_s": 14.698702096939087, "lean_proc_s": 148.1787965297699, "nd_verify_s": 0.197296142578125, "workers": 96, "chunk": 400}
6
+ {"utc": "2026-10-07T22:11:17Z", "samples": 12000, "parse_fail": 1250, "distinct_checked": 10264, "both_ok": 6791, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 3473, "parse_reasons": {"unbound": 875, "name": 91, "expected (": 90, "formula": 71, "expected )": 52, "exact not last": 31, "expected ;": 18, "PR position": 9, "no-eos": 6, "ORE box": 2, "expected :=": 1, "expected =>": 1, "expected fun": 1, "expected False": 1, "trailing tokens": 1}, "lean_wall_s": 35.21818518638611, "lean_proc_s": 893.2075176239014, "nd_verify_s": 0.6092596054077148, "workers": 96, "chunk": 400}
7
+ {"utc": "2026-10-07T22:11:40Z", "samples": 3888, "parse_fail": 919, "distinct_checked": 2709, "both_ok": 28, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 2681, "parse_reasons": {"unbound": 651, "name": 84, "expected (": 84, "formula": 25, "expected ;": 24, "exact not last": 21, "expected )": 13, "PR position": 9, "expected =>": 3, "ORE box": 2, "no-eos": 1, "trailing tokens": 1, "expected fun": 1}, "lean_wall_s": 10.493554592132568, "lean_proc_s": 72.69925570487976, "nd_verify_s": 0.1555500030517578, "workers": 96, "chunk": 400}
8
+ {"utc": "2026-10-07T22:11:57Z", "samples": 3888, "parse_fail": 1186, "distinct_checked": 2454, "both_ok": 22, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 2432, "parse_reasons": {"unbound": 875, "name": 86, "formula": 72, "expected (": 56, "expected )": 55, "expected ;": 26, "ORE box": 8, "exact not last": 6, "no-eos": 1, "PR position": 1}, "lean_wall_s": 8.818713665008545, "lean_proc_s": 53.735737800598145, "nd_verify_s": 0.11390161514282227, "workers": 96, "chunk": 400}
9
+ {"utc": "2026-10-07T22:12:18Z", "samples": 3888, "parse_fail": 988, "distinct_checked": 2675, "both_ok": 34, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 2641, "parse_reasons": {"unbound": 660, "expected (": 112, "name": 96, "formula": 55, "expected )": 35, "exact not last": 8, "ORE box": 7, "expected ;": 5, "expected :=": 4, "PR position": 2, "expected False": 2, "no-eos": 1, "trailing tokens": 1}, "lean_wall_s": 10.771084785461426, "lean_proc_s": 72.13880109786987, "nd_verify_s": 0.15462708473205566, "workers": 96, "chunk": 400}
10
+ {"utc": "2026-10-07T22:13:43Z", "samples": 12000, "parse_fail": 852, "distinct_checked": 10688, "both_ok": 7905, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 2783, "parse_reasons": {"unbound": 603, "expected (": 94, "name": 60, "expected )": 35, "formula": 22, "ORE box": 12, "exact not last": 8, "expected ;": 7, "expected :=": 6, "PR position": 5}, "lean_wall_s": 27.536895036697388, "lean_proc_s": 730.5064754486084, "nd_verify_s": 0.681675910949707, "workers": 96, "chunk": 400}
11
+ {"utc": "2026-10-07T22:14:00Z", "samples": 3144, "parse_fail": 679, "distinct_checked": 2257, "both_ok": 27, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 2230, "parse_reasons": {"unbound": 503, "expected (": 63, "name": 43, "expected )": 21, "formula": 17, "expected :=": 13, "ORE box": 10, "exact not last": 6, "PR position": 1, "expected ;": 1, "expected fun": 1}, "lean_wall_s": 8.917486906051636, "lean_proc_s": 51.93652367591858, "nd_verify_s": 0.10415196418762207, "workers": 96, "chunk": 400}
12
+ {"utc": "2026-10-07T22:14:18Z", "samples": 3144, "parse_fail": 707, "distinct_checked": 2220, "both_ok": 20, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 2200, "parse_reasons": {"unbound": 524, "expected (": 71, "expected )": 41, "name": 34, "formula": 27, "exact not last": 6, "expected ;": 2, "PR position": 1, "expected False": 1}, "lean_wall_s": 8.934838056564331, "lean_proc_s": 49.52054500579834, "nd_verify_s": 0.11041092872619629, "workers": 96, "chunk": 400}
13
+ {"utc": "2026-10-07T22:14:33Z", "samples": 3144, "parse_fail": 858, "distinct_checked": 2105, "both_ok": 32, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 2073, "parse_reasons": {"unbound": 614, "expected (": 110, "name": 63, "ORE box": 24, "expected )": 21, "formula": 9, "PR position": 5, "expected fun": 4, "expected ;": 3, "exact not last": 3, "expected False": 2}, "lean_wall_s": 8.76211929321289, "lean_proc_s": 46.95437216758728, "nd_verify_s": 0.11509251594543457, "workers": 96, "chunk": 400}
14
+ {"utc": "2026-10-07T22:16:05Z", "samples": 12000, "parse_fail": 958, "distinct_checked": 10558, "both_ok": 8439, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 2119, "parse_reasons": {"unbound": 637, "expected (": 119, "formula": 84, "expected )": 47, "name": 31, "exact not last": 11, "ORE box": 11, "expected ;": 9, "PR position": 4, "no-eos": 3, "expected =>": 1, "expected :=": 1}, "lean_wall_s": 30.82331871986389, "lean_proc_s": 804.2527289390564, "nd_verify_s": 0.6117749214172363, "workers": 96, "chunk": 400}
15
+ {"utc": "2026-10-07T22:16:18Z", "samples": 2616, "parse_fail": 886, "distinct_checked": 1604, "both_ok": 9, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1595, "parse_reasons": {"unbound": 568, "formula": 106, "expected (": 103, "expected )": 56, "name": 32, "exact not last": 8, "expected ;": 7, "no-eos": 3, "ORE box": 2, "PR position": 1}, "lean_wall_s": 6.5083167552948, "lean_proc_s": 25.601459741592407, "nd_verify_s": 0.11884880065917969, "workers": 96, "chunk": 400}
16
+ {"utc": "2026-10-07T22:16:32Z", "samples": 2592, "parse_fail": 826, "distinct_checked": 1636, "both_ok": 41, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1595, "parse_reasons": {"unbound": 478, "expected (": 166, "formula": 75, "expected )": 36, "name": 28, "ORE box": 16, "exact not last": 14, "expected ;": 9, "expected fun": 2, "expected :=": 2}, "lean_wall_s": 6.603259801864624, "lean_proc_s": 26.57071018218994, "nd_verify_s": 0.07048296928405762, "workers": 96, "chunk": 400}
17
+ {"utc": "2026-10-07T22:16:44Z", "samples": 2592, "parse_fail": 753, "distinct_checked": 1710, "both_ok": 3, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1707, "parse_reasons": {"unbound": 517, "expected (": 88, "expected )": 60, "name": 38, "formula": 27, "expected ;": 10, "exact not last": 5, "PR position": 4, "expected .elim": 3, "ORE box": 1}, "lean_wall_s": 7.12667179107666, "lean_proc_s": 30.669589519500732, "nd_verify_s": 0.10635018348693848, "workers": 96, "chunk": 400}
18
+ {"utc": "2026-10-07T22:18:25Z", "samples": 12000, "parse_fail": 952, "distinct_checked": 10562, "both_ok": 8788, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1774, "parse_reasons": {"unbound": 654, "expected (": 84, "formula": 79, "name": 46, "expected )": 32, "expected ;": 21, "ORE box": 10, "expected :=": 10, "exact not last": 9, "no-eos": 2, "PR position": 2, "expected =>": 1, "expected \u27e9": 1, "expected exact": 1}, "lean_wall_s": 35.68090486526489, "lean_proc_s": 930.8845624923706, "nd_verify_s": 0.6378681659698486, "workers": 96, "chunk": 400}
19
+ {"utc": "2026-10-07T22:18:36Z", "samples": 2376, "parse_fail": 837, "distinct_checked": 1391, "both_ok": 3, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1388, "parse_reasons": {"unbound": 593, "expected (": 88, "name": 45, "formula": 44, "expected ;": 23, "expected )": 20, "exact not last": 11, "ORE box": 11, "expected :=": 1, "PR position": 1}, "lean_wall_s": 5.490969657897949, "lean_proc_s": 19.718544006347656, "nd_verify_s": 0.06748843193054199, "workers": 96, "chunk": 400}
20
+ {"utc": "2026-10-07T22:18:48Z", "samples": 2352, "parse_fail": 724, "distinct_checked": 1510, "both_ok": 13, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1497, "parse_reasons": {"unbound": 476, "formula": 129, "expected (": 39, "name": 34, "expected )": 19, "expected ;": 14, "ORE box": 7, "PR position": 3, "exact not last": 3}, "lean_wall_s": 6.038282871246338, "lean_proc_s": 23.68019390106201, "nd_verify_s": 0.07426762580871582, "workers": 96, "chunk": 400}
21
+ {"utc": "2026-10-07T22:18:59Z", "samples": 2352, "parse_fail": 801, "distinct_checked": 1439, "both_ok": 3, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1436, "parse_reasons": {"unbound": 498, "expected (": 108, "name": 55, "formula": 49, "expected )": 33, "expected :=": 21, "ORE box": 17, "exact not last": 9, "no-eos": 6, "expected ;": 4, "eof": 1}, "lean_wall_s": 5.4260783195495605, "lean_proc_s": 20.484389066696167, "nd_verify_s": 0.06520771980285645, "workers": 96, "chunk": 400}
22
+ {"utc": "2026-10-07T22:20:35Z", "samples": 12000, "parse_fail": 695, "distinct_checked": 10849, "both_ok": 9014, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1835, "parse_reasons": {"unbound": 438, "name": 76, "expected (": 65, "formula": 52, "expected )": 16, "ORE box": 15, "exact not last": 13, "expected :=": 9, "expected ;": 5, "expected =>": 2, "expected by": 1, "expected None": 1, "PR position": 1, "no-eos": 1}, "lean_wall_s": 33.27284097671509, "lean_proc_s": 878.5364444255829, "nd_verify_s": 0.6728756427764893, "workers": 96, "chunk": 400}
23
+ {"utc": "2026-10-07T22:20:49Z", "samples": 2184, "parse_fail": 560, "distinct_checked": 1495, "both_ok": 2, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1493, "parse_reasons": {"unbound": 323, "formula": 83, "name": 70, "expected (": 35, "expected :=": 24, "expected )": 15, "ORE box": 4, "expected ;": 3, "PR position": 2, "no-eos": 1}, "lean_wall_s": 5.636815071105957, "lean_proc_s": 22.2003972530365, "nd_verify_s": 0.0722653865814209, "workers": 96, "chunk": 400}
24
+ {"utc": "2026-10-07T22:20:58Z", "samples": 2160, "parse_fail": 641, "distinct_checked": 1406, "both_ok": 6, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1400, "parse_reasons": {"unbound": 378, "expected (": 129, "name": 102, "ORE box": 11, "formula": 8, "expected )": 7, "exact not last": 4, "expected \u27e9": 1, "PR position": 1}, "lean_wall_s": 5.160709381103516, "lean_proc_s": 19.36012101173401, "nd_verify_s": 0.0691680908203125, "workers": 96, "chunk": 400}
25
+ {"utc": "2026-10-07T22:21:10Z", "samples": 2160, "parse_fail": 563, "distinct_checked": 1460, "both_ok": 2, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1458, "parse_reasons": {"unbound": 361, "expected (": 57, "formula": 54, "name": 49, "ORE box": 28, "expected )": 4, "no-eos": 3, "expected :=": 3, "PR position": 3, "expected None": 1}, "lean_wall_s": 5.395240783691406, "lean_proc_s": 20.39289879798889, "nd_verify_s": 0.07996606826782227, "workers": 96, "chunk": 400}
26
+ {"utc": "2026-10-07T22:22:45Z", "samples": 12000, "parse_fail": 855, "distinct_checked": 10663, "both_ok": 9137, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1526, "parse_reasons": {"unbound": 561, "expected (": 115, "name": 67, "formula": 60, "ORE box": 28, "expected )": 11, "exact not last": 6, "expected :=": 3, "expected ;": 2, "PR position": 1, "no-eos": 1}, "lean_wall_s": 32.39378643035889, "lean_proc_s": 852.9908123016357, "nd_verify_s": 0.6322927474975586, "workers": 96, "chunk": 400}
27
+ {"utc": "2026-10-07T22:22:58Z", "samples": 2088, "parse_fail": 594, "distinct_checked": 1361, "both_ok": 5, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1356, "parse_reasons": {"unbound": 412, "expected (": 57, "formula": 46, "name": 44, "ORE box": 20, "expected )": 7, "exact not last": 4, "expected :=": 3, "no-eos": 1}, "lean_wall_s": 5.149581432342529, "lean_proc_s": 18.325379848480225, "nd_verify_s": 0.0661458969116211, "workers": 96, "chunk": 400}
28
+ {"utc": "2026-10-07T22:23:10Z", "samples": 2064, "parse_fail": 711, "distinct_checked": 1230, "both_ok": 0, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1230, "parse_reasons": {"unbound": 423, "name": 115, "formula": 62, "expected (": 52, "ORE box": 38, "expected )": 8, "expected :=": 6, "trailing tokens": 3, "no-eos": 2, "exact not last": 1, "expected ;": 1}, "lean_wall_s": 4.495267868041992, "lean_proc_s": 13.392057657241821, "nd_verify_s": 0.056827545166015625, "workers": 96, "chunk": 400}
29
+ {"utc": "2026-10-07T22:23:19Z", "samples": 2064, "parse_fail": 769, "distinct_checked": 1197, "both_ok": 1, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1196, "parse_reasons": {"unbound": 412, "expected (": 236, "name": 53, "formula": 27, "ORE box": 26, "expected )": 9, "PR position": 3, "exact not last": 2, "expected ;": 1}, "lean_wall_s": 4.360255718231201, "lean_proc_s": 12.837864637374878, "nd_verify_s": 0.047799110412597656, "workers": 96, "chunk": 400}
30
+ {"utc": "2026-10-07T22:24:54Z", "samples": 12000, "parse_fail": 854, "distinct_checked": 10670, "both_ok": 9229, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1441, "parse_reasons": {"unbound": 604, "name": 79, "expected (": 68, "formula": 43, "ORE box": 28, "expected )": 17, "exact not last": 4, "expected ,": 4, "expected ;": 3, "PR position": 2, "expected fun": 1, "no-eos": 1}, "lean_wall_s": 31.666245222091675, "lean_proc_s": 836.4754900932312, "nd_verify_s": 0.6611950397491455, "workers": 96, "chunk": 400}
31
+ {"utc": "2026-10-07T22:25:05Z", "samples": 2016, "parse_fail": 825, "distinct_checked": 1105, "both_ok": 0, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1105, "parse_reasons": {"unbound": 504, "expected (": 101, "formula": 72, "name": 69, "ORE box": 47, "expected )": 22, "exact not last": 3, "expected ,": 3, "expected ;": 2, "PR position": 2}, "lean_wall_s": 4.575493335723877, "lean_proc_s": 12.941572189331055, "nd_verify_s": 0.04767131805419922, "workers": 96, "chunk": 400}
32
+ {"utc": "2026-10-07T22:25:16Z", "samples": 2016, "parse_fail": 732, "distinct_checked": 1184, "both_ok": 0, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1184, "parse_reasons": {"unbound": 508, "expected (": 67, "name": 57, "ORE box": 56, "formula": 25, "expected )": 11, "expected ;": 3, "PR position": 2, "exact not last": 2, "no-eos": 1}, "lean_wall_s": 4.416784286499023, "lean_proc_s": 13.107749223709106, "nd_verify_s": 0.048589229583740234, "workers": 96, "chunk": 400}
33
+ {"utc": "2026-10-07T22:25:25Z", "samples": 2016, "parse_fail": 686, "distinct_checked": 1210, "both_ok": 2, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1208, "parse_reasons": {"unbound": 520, "expected (": 61, "name": 61, "formula": 32, "expected )": 7, "ORE box": 2, "PR position": 2, "exact not last": 1}, "lean_wall_s": 4.630467891693115, "lean_proc_s": 13.73590087890625, "nd_verify_s": 0.0494840145111084, "workers": 96, "chunk": 400}
34
+ {"utc": "2026-10-07T22:27:06Z", "samples": 12000, "parse_fail": 849, "distinct_checked": 10749, "both_ok": 9381, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1368, "parse_reasons": {"unbound": 509, "expected (": 130, "formula": 92, "name": 40, "ORE box": 36, "exact not last": 12, "expected )": 11, "expected ;": 8, "expected :=": 4, "PR position": 3, "expected fun": 2, "no-eos": 2}, "lean_wall_s": 30.599449157714844, "lean_proc_s": 813.9485108852386, "nd_verify_s": 0.6633610725402832, "workers": 96, "chunk": 400}
35
+ {"utc": "2026-10-07T22:27:19Z", "samples": 1992, "parse_fail": 748, "distinct_checked": 1147, "both_ok": 2, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1145, "parse_reasons": {"unbound": 427, "formula": 145, "expected (": 91, "ORE box": 39, "name": 28, "expected )": 12, "PR position": 2, "expected ;": 2, "exact not last": 1, "no-eos": 1}, "lean_wall_s": 4.360078573226929, "lean_proc_s": 12.885820627212524, "nd_verify_s": 0.06374645233154297, "workers": 96, "chunk": 400}
36
+ {"utc": "2026-10-07T22:27:31Z", "samples": 1992, "parse_fail": 805, "distinct_checked": 1078, "both_ok": 3, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1075, "parse_reasons": {"unbound": 448, "expected (": 193, "formula": 72, "name": 52, "ORE box": 23, "PR position": 6, "expected )": 5, "exact not last": 3, "expected ;": 2, "no-eos": 1}, "lean_wall_s": 4.141678094863892, "lean_proc_s": 11.681383848190308, "nd_verify_s": 0.04389810562133789, "workers": 96, "chunk": 400}
37
+ {"utc": "2026-10-07T22:27:41Z", "samples": 1968, "parse_fail": 702, "distinct_checked": 1171, "both_ok": 2, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1169, "parse_reasons": {"unbound": 431, "expected (": 92, "name": 52, "ORE box": 48, "formula": 47, "expected :=": 15, "expected )": 6, "exact not last": 4, "PR position": 2, "no-eos": 2, "expected fun": 2, "expected ;": 1}, "lean_wall_s": 4.371264934539795, "lean_proc_s": 12.872344255447388, "nd_verify_s": 0.059053897857666016, "workers": 96, "chunk": 400}
38
+ {"utc": "2026-10-07T22:29:22Z", "samples": 12000, "parse_fail": 818, "distinct_checked": 10747, "both_ok": 9362, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1385, "parse_reasons": {"unbound": 553, "expected (": 134, "name": 37, "ORE box": 29, "formula": 18, "expected )": 15, "exact not last": 11, "expected ;": 7, "expected :=": 5, "no-eos": 4, "PR position": 2, "expected ,": 2, "expected \u27e9": 1}, "lean_wall_s": 31.961103677749634, "lean_proc_s": 853.4920246601105, "nd_verify_s": 0.6602957248687744, "workers": 96, "chunk": 400}
39
+ {"utc": "2026-10-07T22:29:32Z", "samples": 1920, "parse_fail": 779, "distinct_checked": 1055, "both_ok": 1, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1054, "parse_reasons": {"unbound": 496, "expected (": 149, "name": 50, "ORE box": 23, "formula": 19, "expected )": 18, "expected :=": 9, "expected ;": 7, "no-eos": 3, "exact not last": 2, "expected ,": 1, "trailing tokens": 1, "PR position": 1}, "lean_wall_s": 3.9825429916381836, "lean_proc_s": 11.346506118774414, "nd_verify_s": 0.03564763069152832, "workers": 96, "chunk": 400}
40
+ {"utc": "2026-10-07T22:29:43Z", "samples": 1896, "parse_fail": 665, "distinct_checked": 1156, "both_ok": 0, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1156, "parse_reasons": {"unbound": 463, "expected (": 114, "ORE box": 30, "name": 21, "expected )": 11, "formula": 6, "exact not last": 6, "PR position": 6, "expected ,": 4, "expected ;": 4}, "lean_wall_s": 4.536712884902954, "lean_proc_s": 13.38208293914795, "nd_verify_s": 0.05449032783508301, "workers": 96, "chunk": 400}
41
+ {"utc": "2026-10-07T22:29:52Z", "samples": 1896, "parse_fail": 711, "distinct_checked": 1095, "both_ok": 1, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1094, "parse_reasons": {"unbound": 466, "expected (": 123, "name": 46, "formula": 42, "expected )": 14, "expected ;": 12, "ORE box": 4, "expected \u27e9": 2, "expected ,": 1, "exact not last": 1}, "lean_wall_s": 4.351956367492676, "lean_proc_s": 12.382160425186157, "nd_verify_s": 0.04508709907531738, "workers": 96, "chunk": 400}
42
+ {"utc": "2026-10-07T22:31:25Z", "samples": 12000, "parse_fail": 794, "distinct_checked": 10797, "both_ok": 9457, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1340, "parse_reasons": {"unbound": 488, "expected (": 143, "name": 44, "formula": 40, "expected )": 35, "ORE box": 11, "expected ;": 7, "exact not last": 7, "expected \u27e9": 6, "expected :=": 6, "PR position": 3, "expected ,": 3, "expected False": 1}, "lean_wall_s": 30.53050208091736, "lean_proc_s": 811.528406381607, "nd_verify_s": 0.6610722541809082, "workers": 96, "chunk": 400}
43
+ {"utc": "2026-10-07T22:31:35Z", "samples": 1872, "parse_fail": 621, "distinct_checked": 1152, "both_ok": 0, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1152, "parse_reasons": {"unbound": 403, "formula": 75, "expected (": 60, "name": 41, "expected )": 25, "PR position": 6, "expected ;": 2, "expected \u27e9": 2, "expected False": 2, "exact not last": 2, "ORE box": 2, "expected ,": 1}, "lean_wall_s": 4.1689958572387695, "lean_proc_s": 12.333671569824219, "nd_verify_s": 0.04584002494812012, "workers": 96, "chunk": 400}
44
+ {"utc": "2026-10-07T22:31:45Z", "samples": 1848, "parse_fail": 731, "distinct_checked": 1026, "both_ok": 0, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1026, "parse_reasons": {"unbound": 422, "expected (": 217, "name": 24, "ORE box": 18, "expected )": 17, "exact not last": 8, "expected :=": 8, "formula": 7, "PR position": 3, "no-eos": 3, "expected ;": 2, "expected \u27e9": 1, "expected ,": 1}, "lean_wall_s": 4.288639783859253, "lean_proc_s": 11.693408727645874, "nd_verify_s": 0.07408308982849121, "workers": 96, "chunk": 400}
45
+ {"utc": "2026-10-07T22:31:54Z", "samples": 1848, "parse_fail": 683, "distinct_checked": 1069, "both_ok": 2, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1067, "parse_reasons": {"unbound": 377, "expected (": 138, "name": 53, "formula": 41, "expected )": 37, "expected ;": 13, "exact not last": 9, "ORE box": 8, "PR position": 4, "expected \u27e9": 2, "expected ,": 1}, "lean_wall_s": 4.184258222579956, "lean_proc_s": 11.743452310562134, "nd_verify_s": 0.049808502197265625, "workers": 96, "chunk": 400}
46
+ {"utc": "2026-10-07T22:33:40Z", "samples": 12000, "parse_fail": 883, "distinct_checked": 10767, "both_ok": 9601, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1166, "parse_reasons": {"unbound": 586, "expected (": 172, "name": 43, "ORE box": 23, "expected )": 22, "exact not last": 12, "expected :=": 6, "expected ,": 5, "expected ;": 4, "no-eos": 4, "formula": 3, "PR position": 2, "expected by": 1}, "lean_wall_s": 30.843714714050293, "lean_proc_s": 819.9535932540894, "nd_verify_s": 0.6709682941436768, "workers": 96, "chunk": 400}
47
+ {"utc": "2026-10-07T22:33:52Z", "samples": 1824, "parse_fail": 709, "distinct_checked": 1018, "both_ok": 2, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1016, "parse_reasons": {"unbound": 444, "expected (": 175, "name": 39, "ORE box": 16, "expected :=": 14, "expected )": 13, "formula": 4, "expected ,": 1, "no-eos": 1, "exact not last": 1, "expected ;": 1}, "lean_wall_s": 4.116172790527344, "lean_proc_s": 11.273685216903687, "nd_verify_s": 0.04662656784057617, "workers": 96, "chunk": 400}
48
+ {"utc": "2026-10-07T22:34:03Z", "samples": 1800, "parse_fail": 798, "distinct_checked": 934, "both_ok": 1, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 933, "parse_reasons": {"unbound": 555, "expected (": 133, "ORE box": 47, "name": 23, "expected )": 19, "no-eos": 4, "formula": 4, "expected ;": 4, "expected :=": 4, "exact not last": 2, "PR position": 2, "expected None": 1}, "lean_wall_s": 3.4939091205596924, "lean_proc_s": 8.653051614761353, "nd_verify_s": 0.04425644874572754, "workers": 96, "chunk": 400}
49
+ {"utc": "2026-10-07T22:34:15Z", "samples": 1800, "parse_fail": 804, "distinct_checked": 932, "both_ok": 5, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 927, "parse_reasons": {"unbound": 505, "expected (": 196, "name": 35, "expected )": 24, "ORE box": 19, "formula": 8, "exact not last": 6, "expected :=": 4, "no-eos": 3, "expected ,": 2, "PR position": 1, "expected ;": 1}, "lean_wall_s": 3.612116575241089, "lean_proc_s": 9.051259994506836, "nd_verify_s": 0.06170058250427246, "workers": 96, "chunk": 400}
50
+ {"utc": "2026-10-07T22:35:54Z", "samples": 12000, "parse_fail": 816, "distinct_checked": 10782, "both_ok": 9639, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1143, "parse_reasons": {"unbound": 536, "expected (": 117, "name": 50, "ORE box": 38, "expected )": 36, "formula": 14, "exact not last": 8, "expected ;": 8, "expected :=": 4, "expected ,": 2, "expected exact": 1, "expected \u27e9": 1, "PR position": 1}, "lean_wall_s": 30.9827663898468, "lean_proc_s": 818.4715430736542, "nd_verify_s": 0.6768157482147217, "workers": 96, "chunk": 400}
51
+ {"utc": "2026-10-07T22:36:05Z", "samples": 1776, "parse_fail": 700, "distinct_checked": 998, "both_ok": 0, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 998, "parse_reasons": {"unbound": 465, "expected (": 83, "name": 56, "expected )": 52, "ORE box": 18, "expected ;": 9, "formula": 7, "exact not last": 4, "PR position": 3, "no-eos": 3}, "lean_wall_s": 3.8764398097991943, "lean_proc_s": 10.170764207839966, "nd_verify_s": 0.0408327579498291, "workers": 96, "chunk": 400}
52
+ {"utc": "2026-10-07T22:36:15Z", "samples": 1776, "parse_fail": 781, "distinct_checked": 923, "both_ok": 1, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 922, "parse_reasons": {"unbound": 469, "expected (": 131, "ORE box": 55, "name": 55, "expected )": 37, "expected ;": 18, "formula": 7, "exact not last": 4, "PR position": 2, "expected :": 1, "expected :=": 1, "no-eos": 1}, "lean_wall_s": 3.610947608947754, "lean_proc_s": 8.643958568572998, "nd_verify_s": 0.04439234733581543, "workers": 96, "chunk": 400}
53
+ {"utc": "2026-10-07T22:36:24Z", "samples": 1752, "parse_fail": 704, "distinct_checked": 967, "both_ok": 0, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 967, "parse_reasons": {"unbound": 432, "expected (": 137, "name": 36, "expected )": 34, "ORE box": 31, "expected :=": 16, "formula": 11, "expected ,": 2, "expected ;": 2, "no-eos": 1, "exact not last": 1, "PR position": 1}, "lean_wall_s": 3.5093626976013184, "lean_proc_s": 9.01863145828247, "nd_verify_s": 0.04015231132507324, "workers": 96, "chunk": 400}
54
+ {"utc": "2026-10-07T22:38:04Z", "samples": 12000, "parse_fail": 811, "distinct_checked": 10776, "both_ok": 9630, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1146, "parse_reasons": {"unbound": 501, "expected (": 130, "name": 62, "formula": 34, "expected )": 31, "ORE box": 29, "exact not last": 9, "expected ;": 4, "expected ,": 4, "no-eos": 3, "PR position": 3, "trailing tokens": 1}, "lean_wall_s": 31.182451009750366, "lean_proc_s": 830.0188059806824, "nd_verify_s": 0.6706230640411377, "workers": 96, "chunk": 400}
55
+ {"utc": "2026-10-07T22:38:14Z", "samples": 1752, "parse_fail": 674, "distinct_checked": 976, "both_ok": 5, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 971, "parse_reasons": {"unbound": 398, "expected (": 106, "name": 58, "expected )": 51, "formula": 25, "ORE box": 22, "PR position": 5, "exact not last": 4, "expected ;": 2, "expected :=": 1, "no-eos": 1, "trailing tokens": 1}, "lean_wall_s": 3.751281499862671, "lean_proc_s": 10.283748149871826, "nd_verify_s": 0.04014134407043457, "workers": 96, "chunk": 400}
56
+ {"utc": "2026-10-07T22:38:23Z", "samples": 1752, "parse_fail": 761, "distinct_checked": 923, "both_ok": 0, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 923, "parse_reasons": {"unbound": 489, "expected (": 120, "name": 66, "expected )": 31, "formula": 27, "ORE box": 11, "expected ;": 8, "no-eos": 4, "exact not last": 2, "expected :=": 1, "PR position": 1, "expected exact": 1}, "lean_wall_s": 3.714834690093994, "lean_proc_s": 9.106945276260376, "nd_verify_s": 0.04108071327209473, "workers": 96, "chunk": 400}
57
+ {"utc": "2026-10-07T22:38:34Z", "samples": 1752, "parse_fail": 740, "distinct_checked": 931, "both_ok": 0, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 931, "parse_reasons": {"unbound": 404, "expected (": 153, "name": 59, "ORE box": 56, "expected )": 27, "formula": 25, "expected ;": 8, "no-eos": 4, "exact not last": 1, "eof": 1, "expected ,": 1, "PR position": 1}, "lean_wall_s": 3.820549249649048, "lean_proc_s": 9.806522130966187, "nd_verify_s": 0.05093264579772949, "workers": 96, "chunk": 400}
58
+ {"utc": "2026-10-07T22:40:13Z", "samples": 12000, "parse_fail": 893, "distinct_checked": 10724, "both_ok": 9662, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1062, "parse_reasons": {"unbound": 572, "expected (": 164, "name": 48, "expected )": 29, "formula": 20, "ORE box": 17, "exact not last": 16, "no-eos": 7, "expected ,": 6, "expected :=": 6, "expected ;": 5, "PR position": 3}, "lean_wall_s": 30.381449222564697, "lean_proc_s": 807.2059581279755, "nd_verify_s": 0.6879208087921143, "workers": 96, "chunk": 400}
59
+ {"utc": "2026-10-07T22:40:23Z", "samples": 1752, "parse_fail": 789, "distinct_checked": 868, "both_ok": 0, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 868, "parse_reasons": {"unbound": 517, "expected (": 165, "name": 38, "expected )": 29, "ORE box": 16, "no-eos": 10, "expected None": 4, "expected ;": 4, "PR position": 3, "exact not last": 1, "formula": 1, "expected ,": 1}, "lean_wall_s": 3.1183199882507324, "lean_proc_s": 7.024734258651733, "nd_verify_s": 0.040032386779785156, "workers": 96, "chunk": 400}
60
+ {"utc": "2026-10-07T22:40:32Z", "samples": 1728, "parse_fail": 798, "distinct_checked": 856, "both_ok": 1, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 855, "parse_reasons": {"unbound": 551, "expected (": 91, "name": 60, "ORE box": 35, "expected )": 22, "expected :=": 20, "formula": 5, "expected ,": 4, "exact not last": 4, "PR position": 3, "expected None": 2, "no-eos": 1}, "lean_wall_s": 3.2346765995025635, "lean_proc_s": 7.057374477386475, "nd_verify_s": 0.038831472396850586, "workers": 96, "chunk": 400}
61
+ {"utc": "2026-10-07T22:40:40Z", "samples": 1728, "parse_fail": 807, "distinct_checked": 850, "both_ok": 1, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 849, "parse_reasons": {"unbound": 495, "expected (": 189, "formula": 41, "name": 31, "expected )": 22, "expected ;": 13, "ORE box": 9, "expected =>": 4, "PR position": 2, "exact not last": 1}, "lean_wall_s": 3.2143030166625977, "lean_proc_s": 7.278073310852051, "nd_verify_s": 0.0407099723815918, "workers": 96, "chunk": 400}
62
+ {"utc": "2026-10-07T22:42:21Z", "samples": 12000, "parse_fail": 875, "distinct_checked": 10747, "both_ok": 9650, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1097, "parse_reasons": {"unbound": 602, "expected (": 100, "expected )": 67, "name": 28, "formula": 24, "ORE box": 20, "no-eos": 13, "exact not last": 8, "expected :=": 5, "PR position": 5, "expected None": 1, "expected ;": 1, "expected fun": 1}, "lean_wall_s": 30.67842435836792, "lean_proc_s": 815.0815408229828, "nd_verify_s": 0.6310138702392578, "workers": 96, "chunk": 400}
63
+ {"utc": "2026-10-07T22:42:32Z", "samples": 1728, "parse_fail": 758, "distinct_checked": 902, "both_ok": 5, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 897, "parse_reasons": {"unbound": 515, "expected (": 102, "expected )": 59, "no-eos": 24, "formula": 17, "ORE box": 17, "name": 15, "exact not last": 5, "expected ;": 2, "PR position": 1, "expected ,": 1}, "lean_wall_s": 3.7992937564849854, "lean_proc_s": 8.879080533981323, "nd_verify_s": 0.05792570114135742, "workers": 96, "chunk": 400}
64
+ {"utc": "2026-10-07T22:42:43Z", "samples": 1704, "parse_fail": 693, "distinct_checked": 937, "both_ok": 0, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 937, "parse_reasons": {"unbound": 416, "expected (": 135, "formula": 32, "name": 30, "expected )": 29, "ORE box": 26, "PR position": 8, "no-eos": 8, "exact not last": 6, "expected :=": 1, "expected ;": 1, "eof": 1}, "lean_wall_s": 3.6159117221832275, "lean_proc_s": 9.164945840835571, "nd_verify_s": 0.05957460403442383, "workers": 96, "chunk": 400}
65
+ {"utc": "2026-10-07T22:42:51Z", "samples": 1704, "parse_fail": 743, "distinct_checked": 907, "both_ok": 1, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 906, "parse_reasons": {"unbound": 543, "expected (": 67, "expected )": 53, "name": 34, "ORE box": 12, "expected ;": 8, "PR position": 8, "no-eos": 6, "expected :=": 5, "formula": 5, "exact not last": 2}, "lean_wall_s": 3.475057363510132, "lean_proc_s": 8.682583093643188, "nd_verify_s": 0.0645906925201416, "workers": 96, "chunk": 400}
66
+ {"utc": "2026-10-07T22:44:57Z", "samples": 12000, "parse_fail": 907, "distinct_checked": 10757, "both_ok": 9785, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 972, "parse_reasons": {"unbound": 581, "expected (": 171, "name": 48, "formula": 26, "expected )": 25, "expected :=": 15, "ORE box": 14, "exact not last": 12, "PR position": 5, "expected ;": 5, "no-eos": 4, "expected =>": 1}, "lean_wall_s": 56.22629404067993, "lean_proc_s": 1485.9576659202576, "nd_verify_s": 0.6910834312438965, "workers": 96, "chunk": 400}
67
+ {"utc": "2026-10-07T22:45:06Z", "samples": 1680, "parse_fail": 711, "distinct_checked": 903, "both_ok": 5, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 898, "parse_reasons": {"unbound": 496, "expected (": 148, "name": 37, "formula": 9, "expected )": 6, "expected :=": 5, "PR position": 4, "exact not last": 3, "expected ;": 2, "expected =>": 1}, "lean_wall_s": 3.142735242843628, "lean_proc_s": 7.392163038253784, "nd_verify_s": 0.030957460403442383, "workers": 96, "chunk": 400}
68
+ {"utc": "2026-10-07T22:45:14Z", "samples": 1656, "parse_fail": 790, "distinct_checked": 812, "both_ok": 5, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 807, "parse_reasons": {"unbound": 491, "expected (": 146, "name": 52, "formula": 46, "ORE box": 24, "expected )": 17, "PR position": 6, "exact not last": 4, "no-eos": 2, "expected ;": 2}, "lean_wall_s": 3.3853070735931396, "lean_proc_s": 6.9160943031311035, "nd_verify_s": 0.035274505615234375, "workers": 96, "chunk": 400}
69
+ {"utc": "2026-10-07T22:45:23Z", "samples": 1656, "parse_fail": 851, "distinct_checked": 751, "both_ok": 6, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 745, "parse_reasons": {"unbound": 492, "expected (": 198, "name": 48, "expected )": 31, "expected :=": 29, "formula": 24, "ORE box": 13, "expected ;": 8, "exact not last": 5, "PR position": 2, "no-eos": 1}, "lean_wall_s": 3.0288801193237305, "lean_proc_s": 5.971598863601685, "nd_verify_s": 0.027647733688354492, "workers": 96, "chunk": 400}
70
+ {"utc": "2026-10-07T22:47:16Z", "samples": 12000, "parse_fail": 850, "distinct_checked": 10765, "both_ok": 9794, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 971, "parse_reasons": {"unbound": 527, "expected (": 103, "name": 85, "formula": 47, "ORE box": 37, "expected )": 26, "exact not last": 16, "PR position": 4, "expected ;": 2, "expected None": 1, "expected :=": 1, "expected fun": 1}, "lean_wall_s": 43.73678803443909, "lean_proc_s": 1165.9451277256012, "nd_verify_s": 0.6921589374542236, "workers": 96, "chunk": 400}
71
+ {"utc": "2026-10-07T22:47:24Z", "samples": 1584, "parse_fail": 737, "distinct_checked": 793, "both_ok": 0, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 793, "parse_reasons": {"unbound": 453, "expected (": 110, "name": 56, "ORE box": 53, "formula": 33, "expected )": 22, "exact not last": 4, "expected ;": 4, "expected ,": 1, "expected \u27e9": 1}, "lean_wall_s": 3.060340642929077, "lean_proc_s": 5.999511003494263, "nd_verify_s": 0.04212760925292969, "workers": 96, "chunk": 400}
72
+ {"utc": "2026-10-07T22:47:31Z", "samples": 1584, "parse_fail": 806, "distinct_checked": 728, "both_ok": 0, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 728, "parse_reasons": {"unbound": 504, "expected (": 123, "name": 62, "formula": 44, "ORE box": 30, "expected )": 28, "PR position": 6, "exact not last": 5, "expected ;": 4}, "lean_wall_s": 2.814277410507202, "lean_proc_s": 5.591063737869263, "nd_verify_s": 0.032677650451660156, "workers": 96, "chunk": 400}
73
+ {"utc": "2026-10-07T22:47:41Z", "samples": 1584, "parse_fail": 740, "distinct_checked": 785, "both_ok": 1, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 784, "parse_reasons": {"unbound": 434, "name": 84, "formula": 78, "expected (": 71, "expected )": 25, "ORE box": 23, "PR position": 12, "exact not last": 9, "expected ;": 2, "no-eos": 1, "expected fun": 1}, "lean_wall_s": 2.999161958694458, "lean_proc_s": 5.969151973724365, "nd_verify_s": 0.036293983459472656, "workers": 96, "chunk": 400}
74
+ {"utc": "2026-10-07T22:49:26Z", "samples": 12000, "parse_fail": 968, "distinct_checked": 10690, "both_ok": 9828, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 862, "parse_reasons": {"unbound": 657, "expected (": 160, "name": 65, "formula": 33, "expected ;": 16, "expected )": 15, "ORE box": 13, "exact not last": 5, "PR position": 3, "expected exact": 1}, "lean_wall_s": 36.72743010520935, "lean_proc_s": 967.8459165096283, "nd_verify_s": 0.7485952377319336, "workers": 96, "chunk": 400}
75
+ {"utc": "2026-10-07T22:49:35Z", "samples": 1584, "parse_fail": 709, "distinct_checked": 813, "both_ok": 1, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 812, "parse_reasons": {"unbound": 427, "expected (": 166, "name": 46, "ORE box": 27, "expected ;": 19, "expected )": 16, "formula": 5, "expected .elim": 1, "exact not last": 1, "no-eos": 1}, "lean_wall_s": 3.4337949752807617, "lean_proc_s": 7.075579643249512, "nd_verify_s": 0.050041913986206055, "workers": 96, "chunk": 400}
76
+ {"utc": "2026-10-07T22:49:41Z", "samples": 1560, "parse_fail": 935, "distinct_checked": 586, "both_ok": 0, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 586, "parse_reasons": {"unbound": 597, "expected (": 166, "name": 78, "formula": 45, "expected ;": 17, "expected )": 13, "PR position": 6, "ORE box": 6, "exact not last": 4, "expected ,": 1, "expected :=": 1, "expected None": 1}, "lean_wall_s": 2.30311918258667, "lean_proc_s": 3.52762770652771, "nd_verify_s": 0.020653247833251953, "workers": 96, "chunk": 400}
77
+ {"utc": "2026-10-07T22:49:48Z", "samples": 1560, "parse_fail": 738, "distinct_checked": 752, "both_ok": 0, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 752, "parse_reasons": {"unbound": 476, "expected (": 143, "name": 50, "formula": 40, "expected ;": 10, "expected )": 9, "ORE box": 6, "PR position": 2, "exact not last": 1, "expected fun": 1}, "lean_wall_s": 3.023484230041504, "lean_proc_s": 5.83513879776001, "nd_verify_s": 0.025631189346313477, "workers": 96, "chunk": 400}
78
+ {"utc": "2026-10-07T22:51:32Z", "samples": 12000, "parse_fail": 813, "distinct_checked": 10865, "both_ok": 9873, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 992, "parse_reasons": {"unbound": 525, "expected (": 130, "name": 51, "expected )": 29, "formula": 24, "ORE box": 22, "exact not last": 13, "no-eos": 5, "PR position": 5, "expected :=": 5, "trailing tokens": 2, "expected ;": 1, "expected ,": 1}, "lean_wall_s": 30.692853689193726, "lean_proc_s": 820.8768775463104, "nd_verify_s": 0.7073726654052734, "workers": 96, "chunk": 400}
79
+ {"utc": "2026-10-07T22:51:42Z", "samples": 1584, "parse_fail": 739, "distinct_checked": 790, "both_ok": 1, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 789, "parse_reasons": {"unbound": 453, "expected (": 120, "name": 55, "formula": 32, "expected )": 25, "ORE box": 19, "no-eos": 12, "expected :=": 9, "PR position": 8, "exact not last": 3, "expected ;": 2, "eof": 1}, "lean_wall_s": 3.1218409538269043, "lean_proc_s": 6.118686676025391, "nd_verify_s": 0.033074140548706055, "workers": 96, "chunk": 400}
80
+ {"utc": "2026-10-07T22:51:52Z", "samples": 1584, "parse_fail": 713, "distinct_checked": 803, "both_ok": 2, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 801, "parse_reasons": {"unbound": 450, "expected (": 116, "name": 48, "expected )": 35, "formula": 32, "ORE box": 10, "expected :=": 10, "PR position": 6, "exact not last": 4, "expected ;": 1, "trailing tokens": 1}, "lean_wall_s": 3.216684103012085, "lean_proc_s": 6.50602650642395, "nd_verify_s": 0.04164385795593262, "workers": 96, "chunk": 400}
81
+ {"utc": "2026-10-07T22:52:00Z", "samples": 1560, "parse_fail": 598, "distinct_checked": 900, "both_ok": 1, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 899, "parse_reasons": {"unbound": 342, "expected (": 138, "name": 48, "ORE box": 44, "expected )": 13, "exact not last": 6, "PR position": 3, "expected None": 2, "formula": 1, "expected ;": 1}, "lean_wall_s": 3.2267887592315674, "lean_proc_s": 7.199268579483032, "nd_verify_s": 0.036550045013427734, "workers": 96, "chunk": 400}
82
+ {"utc": "2026-10-07T22:57:28Z", "samples": 70912, "parse_fail": 11581, "distinct_checked": 46882, "both_ok": 29532, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 17350, "parse_reasons": {"unbound": 7875, "expected (": 1722, "name": 904, "formula": 259, "ORE box": 220, "expected ;": 177, "expected )": 145, "no-eos": 87, "exact not last": 83, "PR position": 61, "expected ,": 18, "trailing tokens": 9, "expected None": 7, "expected :=": 6, "expected \u27e9": 3, "expected fun": 2, "expected exact": 2, "expected False": 1}, "lean_wall_s": 127.70648622512817, "lean_proc_s": 10237.825224161148, "nd_verify_s": 2.7987918853759766, "workers": 96, "chunk": 400}
83
+ {"utc": "2026-10-07T22:57:38Z", "samples": 2000, "parse_fail": 2, "distinct_checked": 1998, "both_ok": 1978, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 20, "parse_reasons": {"ORE box": 1, "expected (": 1}, "lean_wall_s": 3.7903358936309814, "lean_proc_s": 17.225631952285767, "nd_verify_s": 0.10003995895385742, "workers": 96, "chunk": 400}
84
+ {"utc": "2026-10-07T23:02:31Z", "samples": 70912, "parse_fail": 11581, "distinct_checked": 46882, "both_ok": 29532, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 17350, "parse_reasons": {"unbound": 7875, "expected (": 1722, "name": 904, "formula": 259, "ORE box": 220, "expected ;": 177, "expected )": 145, "no-eos": 87, "exact not last": 83, "PR position": 61, "expected ,": 18, "trailing tokens": 9, "expected None": 7, "expected :=": 6, "expected \u27e9": 3, "expected fun": 2, "expected exact": 2, "expected False": 1}, "lean_wall_s": 133.12740182876587, "lean_proc_s": 10788.583482027054, "nd_verify_s": 2.8955774307250977, "workers": 96, "chunk": 400}
runs/590_control-untied-198_s0/pretrain.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:aa73e00d8ebdc7841c86809e1e055b4989254fe4db63372db78d895e9e4de071
3
+ size 38268899
runs/590_control-untied-198_s0/source/harness.py ADDED
@@ -0,0 +1,529 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ # imports the fork (model, tokenizer, train, ...), which is not resolvable here
3
+ # pyright: reportMissingImports=false
4
+ """FROZEN harness for the test-time-loops autoresearch loop. The research agent does not edit it.
5
+
6
+ Forked from ../autoresearch/harness.py (sha 2aa52f23e7ec) on 2026-10-07. What changed: the RL
7
+ stage belongs to the agent, behind a wall-clock budget and a sampling oracle, and the metric is
8
+ measured with the model's test-time compute knob (KNOB) set to EVAL_LEVEL.
9
+
10
+ One timed run = harness owns the clocks, the oracle and the eval; `pretrain.py` (agent-editable)
11
+ owns pretraining and RL:
12
+
13
+ 1. load frozen data (Dan's cap-6 Stage-1 file, ladder RL-target pool, ladder transfer pool)
14
+ 2. call pretrain.pretrain(train_recs, PRETRAIN_SECS, ...) -> (model, tok, stats); charged for
15
+ everything including model construction and torch.compile. Overrun past the grace -> timeout.
16
+ 3. check n_params <= PARAM_CAP, tok.mode == TOK_MODE and that the model has the KNOB attribute
17
+ 4. call pretrain.rl(model, tok, targets, train_recs, oracle, RL_SECS, ...): expert iteration in
18
+ whatever form the agent likes. The targets are statements only, and `oracle.sample` is the
19
+ only way to turn them into training data: it samples the model's own attempts and returns
20
+ the ones Lean AND nd_verify accept. RL_SECS is wall-clock inside rl() minus the time the
21
+ oracle spent in the Lean gate (pod CPUs differ; GPU work is what is rationed). Overrun past
22
+ the grace, or an oracle call after the budget is spent -> timeout.
23
+ 5. eval on the dev half of the transfer pool with the harness setting the model's KNOB:
24
+ METRIC = dev theorems solved at L_true >= 7 with EVAL_K attempts at EVAL_LEVEL.
25
+ The same eval at DIAG_LEVEL is recorded as a diagnostic, not scored.
26
+ 6. SIGALRM at TOTAL_SECS from process start kills everything -> status "timeout"
27
+
28
+ The holdout half of the transfer pool is never scored here. Everything is written to --out.
29
+
30
+ python3 harness.py --out artifacts/x # on the pod (see pod.py / remote.sh)
31
+ ND_FORK=... python3 harness.py --out /tmp/smoke --smoke # CPU, ~1 min, plumbing check
32
+ """
33
+
34
+ import argparse
35
+ import collections
36
+ import hashlib
37
+ import importlib
38
+ import json
39
+ import os
40
+ import random
41
+ import shutil
42
+ import signal
43
+ import sys
44
+ import time
45
+ import traceback
46
+ from pathlib import Path
47
+
48
+ HERE = os.path.dirname(os.path.abspath(__file__))
49
+ FORK = os.environ.get("ND_FORK", "/workspace/nd-takehome")
50
+ sys.path.insert(0, HERE)
51
+ sys.path.insert(
52
+ 0, FORK
53
+ ) # fork first: its train.py provides batch/loss_on; the agent's file is pretrain.py
54
+ import torch # noqa: E402
55
+ from eval_set import judge, wilson # noqa: E402
56
+ from sample import generate # noqa: E402
57
+
58
+ # ---------------------------------------------------------------- the rules
59
+ # (hash-checked into every result)
60
+ TOTAL_SECS = int(
61
+ os.environ.get("ND_TOTAL_SECS", 5400)
62
+ ) # process start -> summary written; SIGALRM past this. env: benchmarks only
63
+ PRETRAIN_SECS = 300 # agent's pretraining budget, compile included
64
+ PRETRAIN_GRACE = 15 # seconds the agent may overrun before the run is a "timeout"
65
+ RL_SECS = 1800 # agent's RL budget: wall-clock inside rl() minus the oracle's Lean-gate time
66
+ RL_GRACE = 60
67
+ PARAM_CAP = 10_000_000 # unique trainable params, embeddings included; checked after RL too
68
+ N_TARGETS, TEMP, MAX_NEW, SAMPLE_BATCH = 1500, 0.8, 512, 1024
69
+ EVAL_K = 64
70
+ # The model's test-time compute knob: an integer attribute the harness sets before each eval and
71
+ # rl() sets before oracle calls. A sibling loop for another architecture changes these three
72
+ # constants and nothing else in this file.
73
+ KNOB = "n_loops" # loops of the shared core
74
+ EVAL_LEVEL = 8 # the metric: the knob's value during the scored eval
75
+ DIAG_LEVEL = 2 # the same eval again at this value, recorded but not scored
76
+ TOK_MODE = "lean_seq" # required tokenizer mode (fork lean_tok.py)
77
+ DEV = "cuda" if torch.cuda.is_available() else "cpu"
78
+
79
+
80
+ def read(fn):
81
+ return [json.loads(line) for line in open(fn) if line.strip()]
82
+
83
+
84
+ def log(msg):
85
+ print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True)
86
+
87
+
88
+ def sha(fn):
89
+ return hashlib.sha256(open(fn, "rb").read()).hexdigest()[:12]
90
+
91
+
92
+ class Timeout(Exception):
93
+ pass
94
+
95
+
96
+ class BudgetSpent(Timeout):
97
+ """rl() asked the oracle for samples after RL_SECS was used up."""
98
+
99
+
100
+ # ---------------------------------------------------------------- bookkeeping
101
+
102
+
103
+ def save_checkpoint(a, model, tok, stage):
104
+ """Publish complete weight snapshots atomically; these are not optimizer-resume files."""
105
+ path = Path(a.out) / f"{stage}.pt"
106
+ pending = path.with_suffix(".pt.tmp")
107
+ try:
108
+ torch.save(
109
+ {
110
+ "state": model.state_dict(),
111
+ "tok_mode": getattr(tok, "mode", None),
112
+ "stage": stage,
113
+ "seed": a.seed,
114
+ "harness_sha": sha(f"{HERE}/harness.py"),
115
+ "pretrain_sha": sha(f"{HERE}/pretrain.py"),
116
+ },
117
+ pending,
118
+ )
119
+ os.replace(pending, path)
120
+ finally:
121
+ pending.unlink(missing_ok=True)
122
+
123
+
124
+ def lstar(solved_names, recs, need=5):
125
+ ls_solved = [r["n_lines"] for r in recs if r["name"] in solved_names]
126
+ ge = {L: sum(1 for x in ls_solved if x >= L) for L in range(7, 15)}
127
+ return max([L for L, c in ge.items() if c >= need], default=0), ge
128
+
129
+
130
+ def by_bin(solved_names, recs):
131
+ by = collections.defaultdict(lambda: [0, 0])
132
+ for r in recs:
133
+ by[r["n_lines"]][0] += r["name"] in solved_names
134
+ by[r["n_lines"]][1] += 1
135
+ return {
136
+ str(L): {"solved": k, "n": n, "rate": k / n, "ci": list(wilson(k, n))}
137
+ for L, (k, n) in sorted(by.items())
138
+ }
139
+
140
+
141
+ def gate_seconds():
142
+ """Seconds the fork's lean_gate has logged so far (Lean wall-clock plus nd_verify)."""
143
+ fn = os.environ.get("LEAN_GATE_LOG")
144
+ if not fn or not os.path.exists(fn):
145
+ return 0.0
146
+ return sum(rec["lean_wall_s"] + rec["nd_verify_s"] for rec in read(fn))
147
+
148
+
149
+ def sample_and_judge(model, tok, recs, k, temperature, seed, batch, max_new):
150
+ prompts = [r["prompt"] for r in recs for _ in range(k)]
151
+ t0, g0 = time.time(), gate_seconds()
152
+ flat = generate(
153
+ model,
154
+ tok,
155
+ prompts,
156
+ greedy=(k == 1 and temperature == 0),
157
+ temperature=temperature,
158
+ batch=batch,
159
+ seed=seed,
160
+ max_new=max_new,
161
+ )
162
+ secs, gate = time.time() - t0, gate_seconds() - g0
163
+ rows = judge(recs, [flat[i * k : (i + 1) * k] for i in range(len(recs))], "n_lines")
164
+ return rows, {
165
+ "samples": len(prompts),
166
+ "secs": secs,
167
+ "gate_secs": gate,
168
+ "samples_per_sec": len(prompts) / secs,
169
+ }
170
+
171
+
172
+ def greedy_rate(model, tok, recs):
173
+ rows, _ = sample_and_judge(model, tok, recs, 1, 0, 0, SAMPLE_BATCH, MAX_NEW)
174
+ return sum(r["solved"] for r in rows) / max(len(rows), 1)
175
+
176
+
177
+ def dev_split(transfer):
178
+ return [r for r in transfer if int(hashlib.sha1(r["key"].encode()).hexdigest(), 16) % 2 == 0]
179
+
180
+
181
+ def holdout_split(transfer):
182
+ """Never scored by the loop. Score final.pt on this offline when checking in."""
183
+ return [r for r in transfer if int(hashlib.sha1(r["key"].encode()).hexdigest(), 16) % 2 == 1]
184
+
185
+
186
+ # ---------------------------------------------------------------- the RL oracle
187
+
188
+
189
+ class Oracle:
190
+ """What rl() gets instead of data: the model's own attempts at the RL targets, filtered to
191
+ the ones Lean AND nd_verify accept. It also keeps the RL clock and the record of every call.
192
+ """
193
+
194
+ def __init__(self, a, tok, targets):
195
+ self.a, self.tok = a, tok
196
+ self.targets = {t["name"]: t for t in targets}
197
+ self.found = collections.defaultdict(dict) # name -> {proof text: record}
198
+ self.calls = []
199
+ self.t0 = self.last = time.time()
200
+ self.uncharged = 0.0
201
+
202
+ def used(self):
203
+ """RL seconds charged so far: wall-clock since rl() started, minus Lean-gate time."""
204
+ return time.time() - self.t0 - self.uncharged
205
+
206
+ def left(self):
207
+ return self.a.rl_secs - self.used()
208
+
209
+ def sample(self, model, targets, k, temperature=TEMP, batch=SAMPLE_BATCH):
210
+ """k attempts at each of `targets` (records rl() was given, any subset, in any order) from
211
+ `model` as it is now, in eval mode, at whatever its KNOB is set to. Returns one
212
+ dict per target, in order: name, prompt, n_lines, proofs (the distinct accepted proof
213
+ texts), n_ok (accepted attempts) and n_tried. Rejected attempts are not returned.
214
+ Raises BudgetSpent, which ends the run as a timeout, once the budget is used up."""
215
+ if self.left() <= 0:
216
+ raise BudgetSpent(f"oracle called with {self.used():.0f}s of {self.a.rl_secs}s used")
217
+ recs = [self.targets[t["name"]] for t in targets]
218
+ call = len(self.calls) + 1
219
+ model.eval()
220
+ rows, samp = sample_and_judge(
221
+ model, self.tok, recs, k, temperature, self.a.seed * 1000 + call, batch, self.a.max_new
222
+ )
223
+ self.uncharged += samp["gate_secs"]
224
+ new = 0
225
+ for t, row in zip(recs, rows, strict=True):
226
+ for p, wl, pl in zip(
227
+ row["proofs"], row["written_lens"], row["pruned_lens"], strict=True
228
+ ):
229
+ if p not in self.found[t["name"]]:
230
+ self.found[t["name"]][p] = {
231
+ "proof": p,
232
+ "written": wl,
233
+ "pruned": pl,
234
+ "round": call,
235
+ }
236
+ new += 1
237
+ solved = {n for n, d in self.found.items() if d}
238
+ ls, ge = lstar(solved, list(self.targets.values()))
239
+ now = time.time()
240
+ self.calls.append(
241
+ {
242
+ "round": call,
243
+ "level": getattr(model, KNOB, None),
244
+ "k": k,
245
+ "n_targets": len(recs),
246
+ "sample_acc": sum(x["n_ok"] for x in rows) / max(samp["samples"], 1),
247
+ "new_proofs": new,
248
+ "targets_solved": len(solved),
249
+ "lstar": ls,
250
+ "ge": ge,
251
+ "mix_rl_records": sum(len(x["proofs"]) for x in rows), # proofs handed back
252
+ "sampling": samp,
253
+ "secs": now - self.last, # since the previous call returned: includes rl()'s work
254
+ "rl_secs_used": self.used(),
255
+ }
256
+ )
257
+ self.last = now
258
+ c = self.calls[-1]
259
+ log(
260
+ f"ei r{call}: acc {c['sample_acc']:.4f}, +{new} proofs, "
261
+ f"solved {len(solved)}/{len(self.targets)}, L*={ls}, {KNOB} {c['level']}, "
262
+ f"{len(recs)} targets x {k}, {samp['samples_per_sec']:.0f} samples/s, "
263
+ f"RL clock {c['rl_secs_used']:.0f}/{self.a.rl_secs}s"
264
+ )
265
+ return [
266
+ {
267
+ "name": t["name"],
268
+ "prompt": t["prompt"],
269
+ "n_lines": t["n_lines"],
270
+ "proofs": list(row["proofs"]),
271
+ "n_ok": row["n_ok"],
272
+ "n_tried": row["n_tried"],
273
+ }
274
+ for t, row in zip(recs, rows, strict=True)
275
+ ]
276
+
277
+
278
+ # ---------------------------------------------------------------- stages
279
+
280
+
281
+ def stage_rl(a, agent, tok, model, targets, train_recs):
282
+ oracle = Oracle(a, tok, targets)
283
+ # statements only: what the agent's RL may know about a target
284
+ visible = [{"name": t["name"], "prompt": t["prompt"], "n_lines": t["n_lines"]} for t in targets]
285
+ stats = agent.rl(model, tok, visible, train_recs, oracle, a.rl_secs, DEV, a.seed, log)
286
+ if DEV == "cuda":
287
+ torch.cuda.synchronize()
288
+ used = oracle.used()
289
+ model.eval()
290
+ with open(f"{a.out}/found_targets.jsonl", "w") as f:
291
+ for t in targets:
292
+ for x in oracle.found[t["name"]].values():
293
+ f.write(
294
+ json.dumps({"name": t["name"], "thm": t["thm"], "L_true": t["n_lines"], **x})
295
+ + "\n"
296
+ )
297
+ log(f"rl returned: {used:.0f}s of {a.rl_secs}s charged, {oracle.uncharged:.0f}s Lean gate")
298
+ return {
299
+ "rounds": oracle.calls,
300
+ "secs": time.time() - oracle.t0,
301
+ "rl_secs_used": used,
302
+ "gate_secs": oracle.uncharged,
303
+ "targets": len(targets),
304
+ "stats": dict(stats or {}),
305
+ }
306
+
307
+
308
+ def eval_at(a, tok, model, dev_pool, level):
309
+ setattr(model, KNOB, level)
310
+ rows, samp = sample_and_judge(
311
+ model, tok, dev_pool, a.eval_k, TEMP, a.seed * 1000 + 777, SAMPLE_BATCH, a.max_new
312
+ )
313
+ solved = {r["name"] for r in rows if r["solved"]}
314
+ ls, ge = lstar(solved, dev_pool)
315
+ out = {
316
+ "level": level,
317
+ "attempts_per_theorem": a.eval_k,
318
+ "solved": len(solved),
319
+ "n": len(dev_pool),
320
+ "solved_ge7": ge[7],
321
+ "lstar": ls,
322
+ "ge": ge,
323
+ "by_bin": by_bin(solved, dev_pool),
324
+ "sample_acc": sum(r["n_ok"] for r in rows) / max(samp["samples"], 1),
325
+ "max_pruned": max((max(r["pruned_lens"]) for r in rows if r["solved"]), default=0),
326
+ "sampling": samp,
327
+ }
328
+ log(
329
+ f"eval at {KNOB} {level}: solved {len(solved)}/{len(dev_pool)}, ge7 {ge[7]}, L*={ls}, "
330
+ f"per-sample {out['sample_acc']:.4f}, {samp['samples_per_sec']:.0f} samples/s"
331
+ )
332
+ return out, rows
333
+
334
+
335
+ def stage_eval(a, tok, model, dev_pool, heldout_recs):
336
+ out, rows = eval_at(a, tok, model, dev_pool, a.eval_level)
337
+ out["heldout_greedy"] = greedy_rate(model, tok, heldout_recs)
338
+ with open(f"{a.out}/eval_dev.jsonl", "w") as f:
339
+ for r in rows:
340
+ keys = ("name", "n_lines", "solved", "n_ok", "proofs", "written_lens", "pruned_lens")
341
+ f.write(json.dumps({k: r[k] for k in keys}) + "\n")
342
+ diag, _ = eval_at(a, tok, model, dev_pool, a.diag_level)
343
+ out["diag"] = diag
344
+ setattr(model, KNOB, a.eval_level)
345
+ log(
346
+ f"eval: solved {out['solved']}/{len(dev_pool)}, ge7 {out['solved_ge7']} at "
347
+ f"{KNOB} {a.eval_level} ({diag['solved_ge7']} at {a.diag_level}), L*={out['lstar']}, "
348
+ f"max pruned {out['max_pruned']}, held-out greedy {out['heldout_greedy']:.3f}"
349
+ )
350
+ return out
351
+
352
+
353
+ def lean_gate_totals():
354
+ """Sum the fork lean_gate's per-generate() log lines (remote.sh sets LEAN_GATE_LOG)."""
355
+ fn = os.environ.get("LEAN_GATE_LOG")
356
+ if not fn or not os.path.exists(fn):
357
+ return None
358
+ keys = ("samples", "parse_fail", "distinct_checked", "both_ok", "nd_ok_lean_rej")
359
+ tot = dict.fromkeys(keys + ("nd_rej_lean_ok", "lean_wall_s"), 0)
360
+ for rec in read(fn):
361
+ for k in tot:
362
+ tot[k] += rec[k]
363
+ return tot
364
+
365
+
366
+ RULES = (
367
+ "TOTAL_SECS",
368
+ "PRETRAIN_SECS",
369
+ "RL_SECS",
370
+ "PARAM_CAP",
371
+ "N_TARGETS",
372
+ "EVAL_K",
373
+ "KNOB",
374
+ "EVAL_LEVEL",
375
+ "DIAG_LEVEL",
376
+ "TOK_MODE",
377
+ )
378
+
379
+
380
+ def write_summary(a, status, **kw):
381
+ s = {
382
+ "status": status,
383
+ "metric": kw.get("eval", {}).get("solved_ge7"),
384
+ "metric_diag": kw.get("eval", {}).get("diag", {}).get("solved_ge7"),
385
+ "metric_rl": (kw.get("rl", {}).get("rounds") or [{}])[-1].get(
386
+ "targets_solved"
387
+ ), # secondary: cumulative RL targets solved
388
+ "harness_sha": sha(f"{HERE}/harness.py"),
389
+ "pretrain_sha": sha(f"{HERE}/pretrain.py"),
390
+ "device": DEV,
391
+ "gpu": torch.cuda.get_device_name(0) if DEV == "cuda" else "cpu",
392
+ "torch": torch.__version__,
393
+ "lean_gate": lean_gate_totals(),
394
+ "total_secs": time.time() - a.T0,
395
+ "rules": {k: globals()[k] for k in RULES},
396
+ **kw,
397
+ }
398
+ with open(f"{a.out}/summary.json", "w") as f:
399
+ json.dump(s, f, indent=1)
400
+ log(
401
+ f"SUMMARY status={status} metric={s['metric']} metric_diag={s['metric_diag']} "
402
+ f"metric_rl={s['metric_rl']} total {s['total_secs']:.0f}s"
403
+ )
404
+ return s
405
+
406
+
407
+ def main():
408
+ ap = argparse.ArgumentParser()
409
+ ap.add_argument("--out", required=True)
410
+ ap.add_argument("--seed", type=int, default=0)
411
+ ap.add_argument(
412
+ "--pretrain-only", action="store_true", help="save pretrain.pt and stop before eval/RL"
413
+ )
414
+ ap.add_argument(
415
+ "--smoke", action="store_true", help="tiny everything, for a CPU plumbing check"
416
+ )
417
+ a = ap.parse_args()
418
+ a.pretrain_secs, a.rl_secs, a.eval_k, a.max_new, a.n_targets = (
419
+ PRETRAIN_SECS,
420
+ RL_SECS,
421
+ EVAL_K,
422
+ MAX_NEW,
423
+ N_TARGETS,
424
+ )
425
+ a.eval_level, a.diag_level = EVAL_LEVEL, DIAG_LEVEL
426
+ if a.smoke:
427
+ a.pretrain_secs, a.rl_secs, a.eval_k, a.max_new, a.n_targets = 10, 30, 2, 120, 8
428
+ os.makedirs(a.out, exist_ok=True)
429
+ # Preserve discarded variants too: weights alone do not specify the model architecture.
430
+ source_dir = Path(a.out) / "source"
431
+ source_dir.mkdir(exist_ok=True)
432
+ for filename in ("pretrain.py", "harness.py", "remote.sh"):
433
+ shutil.copy2(Path(HERE) / filename, source_dir / filename)
434
+ a.T0 = time.time()
435
+ torch.manual_seed(a.seed)
436
+
437
+ def on_alarm(signum, frame):
438
+ raise Timeout()
439
+
440
+ signal.signal(signal.SIGALRM, on_alarm)
441
+ signal.alarm(TOTAL_SECS if not a.smoke else 900)
442
+
443
+ train_recs = read(f"{FORK}/data/train.jsonl")
444
+ assert all(r["n_lines"] <= 6 for r in train_recs), "pretrain file violates cap 6"
445
+ heldout_all = read(f"{FORK}/data/heldout.jsonl")
446
+ # Length-ordered file: take 2000 evenly spaced records, so every length 2-6 is covered.
447
+ heldout_all = [heldout_all[i * len(heldout_all) // 2000] for i in range(2000)]
448
+ heldout_recs = heldout_all if not a.smoke else heldout_all[:16]
449
+ targets_all = read(f"{FORK}/data/ladder/rl_targets.jsonl")
450
+ transfer = read(f"{FORK}/data/ladder/transfer.jsonl")
451
+ dev_pool = dev_split(transfer) if not a.smoke else dev_split(transfer)[:8]
452
+ targets = random.Random(a.seed).sample(targets_all, min(a.n_targets, len(targets_all)))
453
+ tk, dk, hk = (
454
+ {t["key"] for t in targets},
455
+ {r["key"] for r in dev_pool},
456
+ {r["key"] for r in train_recs},
457
+ )
458
+ assert not (tk & dk) and not (tk & hk) and not (dk & hk), "renaming-class leak between pools"
459
+ log(
460
+ f"pools: train {len(train_recs):,}, targets {len(targets)}, dev-transfer {len(dev_pool)}, "
461
+ f"held-out {len(heldout_recs)}; "
462
+ f"device {DEV}; harness {sha(f'{HERE}/harness.py')} pretrain {sha(f'{HERE}/pretrain.py')}"
463
+ )
464
+
465
+ pre = {}
466
+ try:
467
+ agent = importlib.import_module("pretrain")
468
+ t0 = time.time()
469
+ model, tok, pre = agent.pretrain(train_recs, a.pretrain_secs, DEV, a.seed, log)
470
+ if DEV == "cuda":
471
+ torch.cuda.synchronize()
472
+ pre = dict(pre or {})
473
+ pre["secs"] = time.time() - t0
474
+ model.eval()
475
+ n_params = sum(p.numel() for p in model.parameters())
476
+ pre["n_params"] = n_params
477
+ log(
478
+ f"pretrain returned after {pre['secs']:.0f}s (budget {a.pretrain_secs}s), "
479
+ f"{n_params:,} params"
480
+ )
481
+ if pre["secs"] > a.pretrain_secs + PRETRAIN_GRACE and not a.smoke:
482
+ over = pre["secs"] - a.pretrain_secs
483
+ write_summary(
484
+ a, "timeout", pretrain=pre, reason=f"pretrain overran budget by {over:.0f}s"
485
+ )
486
+ return
487
+ if getattr(tok, "mode", None) != TOK_MODE:
488
+ write_summary(a, "crash", pretrain=pre, reason=f"tok.mode {tok.mode!r} != {TOK_MODE!r}")
489
+ return
490
+ if n_params > PARAM_CAP:
491
+ write_summary(a, "crash", pretrain=pre, reason=f"{n_params} params > cap {PARAM_CAP}")
492
+ return
493
+ if not isinstance(getattr(model, KNOB, None), int):
494
+ reason = f"model has no integer {KNOB} for the harness to set at eval"
495
+ write_summary(a, "crash", pretrain=pre, reason=reason)
496
+ return
497
+ save_checkpoint(a, model, tok, "pretrain")
498
+ if a.pretrain_only:
499
+ write_summary(a, "ok", stage="pretrain_only", pretrain=pre)
500
+ return
501
+ pre["heldout_greedy"] = greedy_rate(model, tok, heldout_recs)
502
+ log(f"pretrain held-out greedy {pre['heldout_greedy']:.3f}")
503
+ rl = stage_rl(a, agent, tok, model, targets, train_recs)
504
+ if rl["rl_secs_used"] > a.rl_secs + RL_GRACE and not a.smoke:
505
+ over = rl["rl_secs_used"] - a.rl_secs
506
+ reason = f"rl overran budget by {over:.0f}s"
507
+ write_summary(a, "timeout", pretrain=pre, rl=rl, reason=reason)
508
+ return
509
+ n_after = sum(p.numel() for p in model.parameters())
510
+ if n_after > PARAM_CAP:
511
+ reason = f"{n_after} params > cap {PARAM_CAP} after rl"
512
+ write_summary(a, "crash", pretrain=pre, rl=rl, reason=reason)
513
+ return
514
+ save_checkpoint(a, model, tok, "final")
515
+ ev = stage_eval(a, tok, model, dev_pool, heldout_recs)
516
+ write_summary(a, "ok", pretrain=pre, rl=rl, eval=ev)
517
+ except BudgetSpent as e:
518
+ write_summary(a, "timeout", pretrain=pre, reason=f"rl: {e}")
519
+ except Timeout:
520
+ write_summary(a, "timeout", pretrain=pre, reason=f"exceeded TOTAL_SECS={TOTAL_SECS}")
521
+ except Exception:
522
+ traceback.print_exc()
523
+ write_summary(a, "crash", pretrain=pre, reason=traceback.format_exc()[-2000:])
524
+ finally:
525
+ signal.alarm(0)
526
+
527
+
528
+ if __name__ == "__main__":
529
+ main()
runs/590_control-untied-198_s0/source/pretrain.py ADDED
@@ -0,0 +1,594 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # imports the fork (model, tokenizer, train, ...), which is not resolvable here
2
+ # pyright: reportMissingImports=false
3
+ """AGENT-EDITABLE: pretraining and RL. This is the only file the autoresearch agent changes.
4
+
5
+ Contract with harness.py (frozen):
6
+
7
+ pretrain(train_recs, budget_secs, device, seed, log) -> (model, tok, stats)
8
+
9
+ train_recs list of dicts {prompt, proof, n_lines<=6, key, ...}; the frozen cap-6 Stage-1 data
10
+ budget_secs wall-clock you are charged for, from call to return, INCLUDING model construction
11
+ and torch.compile. Overrun by more than the harness grace -> the run is a timeout.
12
+ model nn.Module; <= PARAM_CAP unique params (checked again after rl). The harness samples
13
+ from it with the fork's sample.generate, so it must provide:
14
+ - forward(idx) -> logits (B,T,V) and forward(idx, pos=, mask=, caches=) with a
15
+ per-block KV cache dict, exactly like the fork's GPT (model.blocks must exist)
16
+ - an integer `n_loops` (the harness's KNOB): the loop count applied in eval
17
+ mode. The harness sets it to EVAL_LEVEL for the scored eval and to DIAG_LEVEL
18
+ for the diagnostic one.
19
+ tok must provide encode_prompt, encode_proof, decode, shift_abs, pad, eos, mode, and
20
+ mode must be "lean_seq" (harness TOK_MODE): Lean text in and out, Lean-gated samples.
21
+ Decoder rule: decode() is deterministic and single-pass; it may not call the
22
+ verifier or try alternatives.
23
+ stats dict recorded in summary.json and copied into results.json. MUST contain `curve`:
24
+ a list of {secs, step, train_loss, val_loss} taken every ~CURVE_EVERY seconds
25
+ (see log_point below), so a run's trajectory survives in git. Add anything else.
26
+ val_loss is on VAL_N records held out from train_recs by this file; keep that.
27
+
28
+ rl(model, tok, targets, train_recs, oracle, budget_secs, device, seed, log) -> stats | None
29
+
30
+ targets list of dicts {name, prompt, n_lines}: the 1,500 RL target statements. No proofs.
31
+ oracle oracle.sample(model, targets_subset, k, temperature=0.8, batch=1024) samples k
32
+ attempts per target from the model in eval mode at its current n_loops, and returns
33
+ per target {name, prompt, n_lines, proofs, n_ok, n_tried}; `proofs` are the distinct
34
+ attempts Lean AND nd_verify accepted. This is the only source of new training data.
35
+ oracle.used() / oracle.left(): seconds of the RL budget charged / remaining.
36
+ budget_secs RL_SECS: wall-clock inside rl() minus the oracle's Lean-gate time. Returning more
37
+ than the grace late, or calling the oracle with nothing left, is a timeout.
38
+ The model is trained in place. Everything else about RL is yours: rounds, samples per target,
39
+ loop counts when sampling, which proofs to keep, weighting, replay, optimizer, loss.
40
+
41
+ Anything imported from the fork (ND_FORK) is fair game to copy in here and modify.
42
+ """
43
+
44
+ import math
45
+ import random
46
+ import time
47
+
48
+ import torch
49
+ import torch._inductor.config # noqa: F401 (explicit, so the attribute resolves for pyright)
50
+ from lean_tok import MAXN # fork
51
+ from model import GPT, Block # fork
52
+ from normalize import norm # fork
53
+ from tokenizer import make_tokenizer # fork
54
+ from train import loss_on # fork
55
+
56
+ # ---- knobs
57
+ # Unique blocks: one untied entry block, N_CORE shared core blocks, one untied exit block.
58
+ N_CORE = 4
59
+ # Control: run 198's trunk. The six blocks are independent and applied once, with no re-injection;
60
+ # n_loops is accepted and ignored, so the harness's two evals and rl()'s retry depths all sample
61
+ # the same model.
62
+ UNTIED = True
63
+ N_LAYER, D, N_HEAD = 2 + N_CORE, 384, 8
64
+ # Core applications (loops). Training draws one count per batch from TRAIN_LOOPS, in pretraining
65
+ # and in rl()'s fine-tune; eval and sampling apply model.n_loops, which the harness sets for its
66
+ # evals and rl() sets before each oracle call. DEFAULT_LOOPS is used for the validation curve and
67
+ # the harness's post-pretraining greedy check.
68
+ TRAIN_LOOPS = (1, 2, 3, 4, 5, 6, 7, 8)
69
+ DEFAULT_LOOPS = 2
70
+ VAL_LOOPS = (1, 2, 4, 8) # val loss at each count, once, when pretraining ends
71
+ # ---- RL knobs: the old harness's expert iteration (600 AdamW steps per round on up to 4 proofs
72
+ # per solved target, weighted x4, mixed with 20,000 replayed pretraining records), except for
73
+ # where the samples go. Each round every target gets RL_K attempts at RL_FIRST_LOOPS; a target
74
+ # all of them fail on gets RL_RETRY_K more (32 in all, as before) at ONE deeper loop count from
75
+ # RL_RETRY_LOOPS, assigned at random, so proofs that only deeper sampling finds enter the
76
+ # training set and no deep sample is spent on a target one loop already solves. The random
77
+ # assignment makes the retry calls a controlled comparison of loop counts on resistant targets.
78
+ # Rounds repeat until another one would not fit in the budget.
79
+ RL_FIRST_LOOPS, RL_K = 1, 8
80
+ RL_RETRY_LOOPS, RL_RETRY_K = (2, 4, 8), 24
81
+ # The fine-tune sees FT_RECORDS records per round (the old 600 steps x 128 records) in the same
82
+ # token-budget batches as pretraining (TOK_BUDGET below) instead of 128-record batches padded to
83
+ # their longest proof, which with 7-14 line RL proofs in nearly every batch was ~3x the content.
84
+ FT_RECORDS, FT_LR, FT_WARMUP = 600 * 128, 3e-4, 50
85
+ RETAIN, MAX_PER_THM, RL_WEIGHT = 20000, 4, 4
86
+ # MLP width held at 151's 1280: the fork's Block hardcodes 4 * D (1536 at D 384 would pass the
87
+ # 10M cap), so PeriLNBlock swaps fc1/fc2. 1280 != 3 * D keeps muon_step's QKV-by-shape test valid.
88
+ D_FF = 1280
89
+ BS, LR, MIN_LR_FRAC, WD = 128, 1e-3, 0.3, 0.1
90
+ MUON_LR, MUON_MOMENTUM = 0.01, 0.95
91
+ # Polar Express (Amsel et al. 2025) per-iteration quintic coefficients, as in modded-nanogpt
92
+ # (num_iters=5, safety factor 2e-2): singular values land in ~[0.86, 1.14] instead of the
93
+ # fixed quintic's ~[0.68, 1.13], at the same five iterations.
94
+ NS_COEFFS = (
95
+ (8.156554524902461, -22.48329292557795, 15.878769915207462),
96
+ (4.042929935166739, -2.808917465908714, 0.5000178451051316),
97
+ (3.8916678022926607, -2.772484153217685, 0.5060648178503393),
98
+ (3.285753657755655, -2.3681294933425376, 0.46449024233003106),
99
+ (2.3465413258596377, -1.7097828382687081, 0.42323551169305323),
100
+ )
101
+ # Ten times the AdamW group's per-step multiplicative decay (lr * wd) on the Muon matrices.
102
+ MUON_WD = 10 * WD * LR / MUON_LR
103
+ TOK_MODE = "lean_seq" # required by the harness (Lean protocol)
104
+ # Hybrid heads: half the heads get ALiBi (Press et al. 2021) with the standard slope set for
105
+ # H/2 heads, 2^(-8 i / (H/2)) = 1/4 .. 1/256 with 8 heads; the other half get slope 0, i.e. no
106
+ # positional signal at all (NoPE), so they can look citations up by content at any distance.
107
+ N_ALIBI = N_HEAD // 2
108
+ ALIBI_SLOPES = tuple(2.0 ** (-8.0 * (i + 1) / N_ALIBI) for i in range(N_ALIBI)) + (0.0,) * (
109
+ N_HEAD - N_ALIBI
110
+ )
111
+ VAL_N, CURVE_EVERY = 1024, 15 # records held out for val_loss; seconds between curve points
112
+ # Token-budget batches (fairseq max-tokens): each step takes a run of length-sorted records with
113
+ # count * padded_length <= TOK_BUDGET, so a batch of short proofs has many records and a batch of
114
+ # long ones few, and the padded token count per step is ~constant. 128 x 144 matches the real
115
+ # (unpadded) tokens of the incumbent's 128-record batches (mean record ~135 tokens), whose
116
+ # padding to the batch max (~400) made every step ~3x larger than its content.
117
+ TOK_BUDGET = 128 * 144
118
+ # Multi-token prediction (DeepSeek-V3 2024, Gloeckle et al. 2024), pretraining only: one extra
119
+ # Peri-LN block reads [trunk state at t; embedding of token t+1] and predicts token t+2 through
120
+ # the model's own ln_f/head. Its loss is added at this weight (DeepSeek-V3's early-training value).
121
+ MTP_WEIGHT = 0.3
122
+
123
+
124
+ # Muon update adapted from KellerJordan/Muon, commit
125
+ # f98f1cacc0263b04290753e32be8d498c1efc806 (muon.py).
126
+ # MIT License
127
+ # Copyright (c) 2024 Keller Jordan
128
+ #
129
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
130
+ # of this software and associated documentation files (the "Software"), to deal
131
+ # in the Software without restriction, including without limitation the rights
132
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
133
+ # copies of the Software, and to permit persons to whom the Software is
134
+ # furnished to do so, subject to the following conditions:
135
+ # The above copyright notice and this permission notice shall be included in all
136
+ # copies or substantial portions of the Software.
137
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
138
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
139
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
140
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
141
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
142
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
143
+ # SOFTWARE.
144
+ @torch.no_grad()
145
+ def muon_step(params, buffers, lr):
146
+ """Nesterov momentum plus five bf16 Polar Express steps, on hidden matrices only."""
147
+ for p, momentum in zip(params, buffers, strict=True):
148
+ if p.grad is None:
149
+ continue
150
+ momentum.lerp_(p.grad, 1 - MUON_MOMENTUM)
151
+ update = p.grad.lerp(momentum, MUON_MOMENTUM).bfloat16()
152
+ # Only QKV has shape (3D, D) in this GPT; orthogonalize its roles separately.
153
+ if update.size(0) == 3 * update.size(1):
154
+ update = update.reshape(3, update.size(1), update.size(1))
155
+ transpose = update.size(-2) > update.size(-1)
156
+ if transpose:
157
+ update = update.mT
158
+ update = update / (update.norm(dim=(-2, -1), keepdim=True) * (1 + 2e-2) + 1e-6)
159
+ for a, b, c in NS_COEFFS:
160
+ gram = update @ update.mT
161
+ polynomial = b * gram + c * gram @ gram
162
+ update = a * update + polynomial @ update
163
+ if transpose:
164
+ update = update.mT
165
+ p.mul_(1 - lr * MUON_WD)
166
+ # lr may be a 0-d tensor (CUDA-graphed step); scale in p's dtype like add_(alpha=-lr)
167
+ p.sub_(lr * update.reshape_as(p).to(p.dtype))
168
+
169
+
170
+ class PeriLNBlock(Block):
171
+ """P22 Peri-LN (Kim et al. 2025): the fork's pre-LN block plus a LayerNorm on each branch
172
+ output, so every residual write has unit scale times a learned gain. Attention has no RoPE:
173
+ `mask` is the additive ALiBi bias (B or 1, H, T, S) that LoopedGPT builds once per forward
174
+ and cos/sin are unused."""
175
+
176
+ def __init__(self, d, h):
177
+ super().__init__(d, h)
178
+ self.fc1, self.fc2 = torch.nn.Linear(d, D_FF), torch.nn.Linear(D_FF, d)
179
+ self.ln_attn_out, self.ln_mlp_out = torch.nn.LayerNorm(d), torch.nn.LayerNorm(d)
180
+
181
+ def attn(self, x, cos, sin, mask, cache):
182
+ B, T, D = x.shape
183
+ q, k, v = self.qkv(x).view(B, T, 3, self.h, D // self.h).permute(2, 0, 3, 1, 4)
184
+ if cache is not None: # the fork's preallocated KV cache, unchanged
185
+ if "k" not in cache:
186
+ cache["k"] = k.new_empty(B, self.h, cache["max"], D // self.h)
187
+ cache["v"] = v.new_empty(B, self.h, cache["max"], D // self.h)
188
+ cache["n"] = 0
189
+ n0, n1 = cache["n"], cache["n"] + T
190
+ cache["k"][:, :, n0:n1] = k
191
+ cache["v"][:, :, n0:n1] = v
192
+ cache["n"] = n1
193
+ k, v = cache["k"][:, :, :n1], cache["v"][:, :, :n1]
194
+ y = torch.nn.functional.scaled_dot_product_attention(q, k, v, attn_mask=mask.to(q.dtype))
195
+ return self.proj(y.transpose(1, 2).reshape(B, T, D))
196
+
197
+ def forward(self, x, cos, sin, mask=None, cache=None):
198
+ x = x + self.ln_attn_out(self.attn(self.ln1(x), cos, sin, mask, cache))
199
+ return x + self.ln_mlp_out(self.fc2(torch.nn.functional.gelu(self.fc1(self.ln2(x)))))
200
+
201
+
202
+ class LoopedGPT(GPT):
203
+ """Looped trunk (Universal Transformer, Dehghani et al. 2018; sandwich layout and input
204
+ injection as in Geiping et al. 2025): blocks[0] is the entry block, blocks[1:-1] the shared
205
+ core applied n times, blocks[-1] the exit block. Every core pass starts from
206
+ ln_loop(state) + e, with e the entry block's output, so each pass sees an input of the same
207
+ scale and the prompt is re-injected instead of being buried under accumulated writes.
208
+
209
+ P34: the fork's GPT with ALiBi (Press et al. 2021) instead of RoPE. Head h's attention
210
+ logits get -slope_h * (query position - key position), slopes 2^-1 .. 2^-8 for 8 heads, so
211
+ no rotation of q/k anywhere. Positions come from `pos` (sample.generate's left-padded
212
+ batches) or arange; key positions are kept in caches[0] so cached decoding sees the same
213
+ bias as the full forward. Causality/padding come from `mask` when given, else from rel < 0."""
214
+
215
+ def __init__(self, vocab, n_layer, d, n_head, seed, max_len=1024):
216
+ super().__init__(vocab, n_layer, d, n_head, max_len)
217
+ slopes = torch.tensor(ALIBI_SLOPES).view(1, n_head, 1, 1)
218
+ self.register_buffer("slopes", slopes, persistent=False)
219
+ self.ln_loop = torch.nn.LayerNorm(d)
220
+ self.n_loops, self.loop_rng = DEFAULT_LOOPS, random.Random(seed)
221
+
222
+ def forward(self, idx, pos=None, mask=None, caches=None):
223
+ return self.head(self.ln_f(self.features(idx, pos, mask, caches)[0]))
224
+
225
+ def features(self, idx, pos=None, mask=None, caches=None):
226
+ """The trunk: (final hidden state before ln_f, the ALiBi bias used)."""
227
+ B, T = idx.shape
228
+ if pos is None: # training/val: one (1, H, T, T) bias broadcast over the batch
229
+ pos = torch.arange(T, device=idx.device)[None]
230
+ key_pos = pos
231
+ if caches is not None: # incremental decoding: keys are every cached token
232
+ c = caches[0]
233
+ if "pos" not in c:
234
+ c["pos"], c["n_pos"] = pos.new_empty(pos.size(0), c["max"]), 0
235
+ n0, n1 = c["n_pos"], c["n_pos"] + T
236
+ c["pos"][:, n0:n1] = pos
237
+ c["n_pos"] = n1
238
+ key_pos = c["pos"][:, :n1]
239
+ rel = pos[:, None, :, None] - key_pos[:, None, None, :] # (B or 1, 1, T, S)
240
+ bias = -self.slopes * rel.to(self.slopes.dtype)
241
+ bias = bias.masked_fill(rel < 0 if mask is None else ~mask, float("-inf"))
242
+ if UNTIED:
243
+ x = self.emb(idx)
244
+ for j, b in enumerate(self.blocks):
245
+ x = b(x, None, None, bias, None if caches is None else caches[j])
246
+ return x, bias
247
+ n = self.loop_rng.choice(TRAIN_LOOPS) if self.training else self.n_loops
248
+ entry, *core, exit_block = self.blocks
249
+ x = e = entry(self.emb(idx), None, None, bias, None if caches is None else caches[0])
250
+ for loop in range(n):
251
+ x = self.ln_loop(x) + e
252
+ for j, b in enumerate(core, 1):
253
+ # sample.generate makes one cache dict per unique block; a shared block's keys
254
+ # and values differ per pass, so each pass gets its own sub-cache inside it.
255
+ c = caches and caches[j].setdefault(loop, {"max": caches[j]["max"]})
256
+ x = b(x, None, None, bias, c)
257
+ x = exit_block(x, None, None, bias, None if caches is None else caches[-1])
258
+ return x, bias
259
+
260
+
261
+ class MTPHead(torch.nn.Module):
262
+ """The MTP module. A separate nn.Module, never registered on the model, so the harness's
263
+ parameter count, RL AdamW and sampler never see it; it is dropped when pretrain() returns."""
264
+
265
+ def __init__(self, d, h):
266
+ super().__init__()
267
+ self.ln_h, self.ln_e = torch.nn.LayerNorm(d), torch.nn.LayerNorm(d)
268
+ self.proj = torch.nn.Linear(2 * d, d, bias=False)
269
+ self.block = PeriLNBlock(d, h)
270
+
271
+
272
+ def mtp_loss(model, mtp, x, m):
273
+ """The fork's loss_on (next-token CE on proof tokens) plus MTP_WEIGHT times the CE of the
274
+ MTP module's token-(t+2) prediction at the same proof positions. Returns (total, main)."""
275
+ h, bias = model.features(x[:, :-1])
276
+ logits = model.head(model.ln_f(h))
277
+ V = logits.size(-1)
278
+ ce = torch.nn.functional.cross_entropy(
279
+ logits.reshape(-1, V).float(), x[:, 1:].reshape(-1), reduction="none"
280
+ )
281
+ main = (ce * m[:, 1:].reshape(-1)).sum() / m[:, 1:].sum()
282
+ # Position t (0..T-3) gets h_t and emb(x_{t+1}); causal attention over these keeps every
283
+ # prediction of x_{t+2} a function of x_{<=t+1}.
284
+ z = mtp.proj(torch.cat((mtp.ln_h(h[:, :-1]), mtp.ln_e(model.emb(x[:, 1:-1]))), -1))
285
+ z = mtp.block(z, None, None, bias[..., :-1, :-1], None)
286
+ ce2 = torch.nn.functional.cross_entropy(
287
+ model.head(model.ln_f(z)).reshape(-1, V).float(), x[:, 2:].reshape(-1), reduction="none"
288
+ )
289
+ aux = (ce2 * m[:, 2:].reshape(-1)).sum() / m[:, 2:].sum().clamp(min=1)
290
+ return main + MTP_WEIGHT * aux, main, aux
291
+
292
+
293
+ def val_loss(model, ids, mask, ref_max, lens, tok, gen, autocast):
294
+ """Mean loss over the first VAL_N records (never trained on), no shift augmentation."""
295
+ model.eval()
296
+ tot = 0.0
297
+ with torch.no_grad(), autocast:
298
+ for i in range(0, VAL_N, BS):
299
+ idxs = torch.arange(i, min(i + BS, VAL_N), device=ids.device)
300
+ x, m = ids[idxs], mask[idxs]
301
+ T = int(lens[idxs].max())
302
+ tot += loss_on(model, x[:, :T], m[:, :T]).item() * len(idxs)
303
+ model.train()
304
+ return tot / VAL_N
305
+
306
+
307
+ def to_gpu(data, tok, device):
308
+ """Pad all (prompt, proof) records once into (N, T) ids + proof mask on the device."""
309
+ N, T = len(data), max(len(p) + len(q) for p, q in data)
310
+ ids = torch.full((N, T), tok.pad, dtype=torch.long)
311
+ mask = torch.zeros((N, T), dtype=torch.bool)
312
+ ref_max = torch.zeros(
313
+ N, dtype=torch.long
314
+ ) # largest N<k> index used in the proof, for the shift augmentation
315
+ for i, (p, q) in enumerate(data):
316
+ ids[i, : len(p) + len(q)] = torch.tensor(p + q)
317
+ mask[i, len(p) : len(p) + len(q)] = True
318
+ ref_max[i] = max((x - tok.ref0 + 1 for x in q if x >= tok.ref0), default=0)
319
+ return ids.to(device), mask.to(device), ref_max.to(device), (ids != tok.pad).sum(1).to(device)
320
+
321
+
322
+ def token_batches(lens, n_train, gen, rng, first=VAL_N):
323
+ """One epoch of token-budget batches: a random permutation of the n_train records from
324
+ `first` on, stably sorted by length (equal lengths stay shuffled), cut greedily into runs of
325
+ consecutive records with count * max_len <= TOK_BUDGET, and the runs' order shuffled.
326
+ Returns (order, cuts)."""
327
+ perm = torch.randperm(n_train, device=lens.device, generator=gen) + first
328
+ order = perm[torch.argsort(lens[perm], stable=True)]
329
+ sorted_lens = lens[order].cpu()
330
+ cuts, s = [], 0
331
+ while s < len(sorted_lens):
332
+ window = sorted_lens[s : s + TOK_BUDGET // int(sorted_lens[s])]
333
+ # count * max_len is non-decreasing along the window, so the fits form a prefix.
334
+ e = s + int((torch.arange(1, len(window) + 1) * window <= TOK_BUDGET).sum())
335
+ cuts.append((s, e, int(sorted_lens[e - 1]))) # sorted: last is the batch's max length
336
+ s = e
337
+ rng.shuffle(cuts)
338
+ return order, cuts
339
+
340
+
341
+ def gpu_batch(ids, mask, ref_max, lens, idxs, tok, gen, max_len=None):
342
+ """The fork's batch() + tok.shift_abs, vectorised: a random per-row offset is added to
343
+ every ref token."""
344
+ x, m = ids[idxs], mask[idxs]
345
+ # Known on the host for training batches: passing it avoids a GPU sync every step.
346
+ T = int(lens[idxs].max()) if max_len is None else max_len
347
+ x, m = x[:, :T].clone(), m[:, :T]
348
+ if tok.mode in ("abs", "lean_seq"): # lean_seq: names n1.. get the same start offset
349
+ room = MAXN - ref_max[idxs]
350
+ shift = (
351
+ (torch.rand(len(idxs), device=x.device, generator=gen) * (room + 1))
352
+ .long()
353
+ .clamp(max=room)
354
+ )
355
+ is_ref = (x >= tok.ref0) & m
356
+ x = torch.where(is_ref, x + shift[:, None], x)
357
+ return x, m
358
+
359
+
360
+ def pretrain(train_recs, budget_secs, device, seed, log):
361
+ t0 = time.time()
362
+ tok = make_tokenizer(TOK_MODE)
363
+ data = [(tok.encode_prompt(r["prompt"]), tok.encode_proof(r["proof"])) for r in train_recs]
364
+ ids, mask, ref_max, lens = to_gpu(data, tok, device)
365
+ gen = torch.Generator(device=device)
366
+ gen.manual_seed(seed)
367
+ model = LoopedGPT(tok.vocab_size, N_LAYER, D, N_HEAD, seed)
368
+ model.blocks = torch.nn.ModuleList(PeriLNBlock(D, N_HEAD) for _ in range(N_LAYER))
369
+ model.blocks.apply(model._init) # fork init: zero biases
370
+ model = model.to(device)
371
+ mtp = MTPHead(D, N_HEAD)
372
+ mtp.apply(model._init)
373
+ mtp = mtp.to(device)
374
+ # Compile the repeated blocks once each (torch >= 2.9; dynamic shapes, ~14 s): ~14% faster
375
+ # steps. The coalescing tiling analysis crashes on dynamic shapes, so it is off. Undone at
376
+ # handoff so the harness's RL fine-tune and KV-cache sampling run eager.
377
+ torch._inductor.config.triton.coalesce_tiling_analysis = False
378
+ muon_lr, muon_graph = torch.zeros((), device=device), None
379
+ compiled = list(model.blocks) + [mtp.block]
380
+ for b in compiled if device == "cuda" else []: # CPU smoke runs stay eager
381
+ b.compile(dynamic=True)
382
+ for p in list(model.blocks.parameters()) + list(mtp.parameters()):
383
+ if p.ndim == 2:
384
+ # Orthogonal matrices have element second moment gain^2 / max(rows, cols).
385
+ torch.nn.init.orthogonal_(p, gain=0.02 * math.sqrt(max(p.shape)))
386
+ log(
387
+ f"pretrain: {model.n_params():,} params, {len(data):,} records on {device} "
388
+ f"after {time.time() - t0:.0f}s, budget {budget_secs}s"
389
+ )
390
+ # The output readout joins the hidden matrices under Muon; embeddings stay on AdamW.
391
+ trainable = list(model.parameters()) + list(mtp.parameters())
392
+ matrices = [p for p in model.blocks.parameters() if p.ndim == 2] + [model.head.weight]
393
+ matrices += [p for p in mtp.parameters() if p.ndim == 2]
394
+ matrix_ids = {id(p) for p in matrices}
395
+ auxiliary = [p for p in trainable if id(p) not in matrix_ids]
396
+ momentum_buffers = [torch.zeros_like(p) for p in matrices]
397
+ opt = torch.optim.AdamW(
398
+ auxiliary, lr=LR, weight_decay=WD, betas=(0.9, 0.95), fused=device == "cuda"
399
+ )
400
+ autocast = torch.autocast("cuda", dtype=torch.bfloat16, enabled=(device == "cuda"))
401
+ model.train()
402
+ step, last_loss, records_seen = 0, float("nan"), 0
403
+ curve, recent, recent_aux, next_point = [], [], [], CURVE_EVERY
404
+ n_train = len(data) - VAL_N # records [0, VAL_N) are the val slice
405
+ rng, cuts, batches_per_epoch = random.Random(seed), [], None
406
+ order = torch.empty(0, dtype=torch.long, device=device)
407
+ while True:
408
+ frac = (
409
+ time.time() - t0
410
+ ) / budget_secs # cosine over wall-clock, so the cap is the schedule
411
+ if frac >= 1.0:
412
+ break
413
+ step += 1
414
+ # No warmup (as in modded-nanogpt): Muon's step is normalized and Adam's is ~lr per
415
+ # element from step 1, and Peri-LN normalizes every branch write.
416
+ lr = LR * MIN_LR_FRAC + 0.5 * LR * (1 - MIN_LR_FRAC) * (1 + math.cos(math.pi * frac))
417
+ for g in opt.param_groups:
418
+ g["lr"] = lr
419
+ if not cuts: # new epoch
420
+ order, cuts = token_batches(lens, n_train, gen, rng)
421
+ batches_per_epoch = len(cuts)
422
+ s, e, T = cuts.pop()
423
+ idxs = order[s:e]
424
+ records_seen += e - s
425
+ x, m = gpu_batch(ids, mask, ref_max, lens, idxs, tok, gen, T)
426
+ with autocast:
427
+ loss, main, aux = mtp_loss(model, mtp, x, m)
428
+ model.zero_grad(set_to_none=False) # grads keep fixed addresses for the Muon CUDA graph
429
+ mtp.zero_grad(set_to_none=False)
430
+ loss.backward()
431
+ torch.nn.utils.clip_grad_norm_(trainable, 1.0)
432
+ opt.step()
433
+ muon_lr.fill_(MUON_LR * lr / LR)
434
+ # Capture once; later steps replay one graph instead of ~400 kernel launches. CPU smoke
435
+ # runs call the step directly.
436
+ if device != "cuda":
437
+ muon_step(matrices, momentum_buffers, muon_lr)
438
+ elif muon_graph is None:
439
+ state = matrices + momentum_buffers
440
+ saved = [t.clone() for t in state]
441
+ side = torch.cuda.Stream()
442
+ side.wait_stream(torch.cuda.current_stream())
443
+ with torch.cuda.stream(side):
444
+ muon_step(matrices, momentum_buffers, muon_lr) # warm-up (lazy cuBLAS init)
445
+ torch.cuda.current_stream().wait_stream(side)
446
+ muon_graph = torch.cuda.CUDAGraph()
447
+ with torch.cuda.graph(muon_graph):
448
+ muon_step(matrices, momentum_buffers, muon_lr)
449
+ with torch.no_grad():
450
+ for t, v in zip(state, saved, strict=True):
451
+ t.copy_(v) # undo the warm-up and capture updates
452
+ muon_graph.replay()
453
+ else:
454
+ muon_graph.replay()
455
+ if step % 20 == 0:
456
+ last_loss = main.item() # the next-token CE, comparable with earlier runs
457
+ recent.append(last_loss)
458
+ recent_aux.append(aux.item())
459
+ if (
460
+ time.time() - t0 >= next_point
461
+ ): # curve point: mean recent train loss + val loss on the held-out slice
462
+ curve.append(
463
+ {
464
+ "secs": round(time.time() - t0),
465
+ "step": step,
466
+ "train_loss": sum(recent) / len(recent) if recent else last_loss,
467
+ "val_loss": val_loss(model, ids, mask, ref_max, lens, tok, gen, autocast),
468
+ "mtp_loss": sum(recent_aux) / len(recent_aux) if recent_aux else None,
469
+ }
470
+ )
471
+ log(
472
+ f"pretrain step {step} train {curve[-1]['train_loss']:.4f} "
473
+ f"val {curve[-1]['val_loss']:.4f} lr {lr:.2e} {curve[-1]['secs']}s"
474
+ )
475
+ recent, recent_aux, next_point = [], [], next_point + CURVE_EVERY
476
+ val_by_loops = {}
477
+ for n in VAL_LOOPS if step else (): # no steps: a checkpoint rebuild, not a training run
478
+ model.n_loops = n
479
+ val_by_loops[n] = val_loss(model, ids, mask, ref_max, lens, tok, gen, autocast)
480
+ model.n_loops = DEFAULT_LOOPS
481
+ log(f"pretrain val loss by loops: {val_by_loops}")
482
+ for b in compiled:
483
+ b._compiled_call_impl = None
484
+ model.eval()
485
+ return (
486
+ model,
487
+ tok,
488
+ {
489
+ "steps": step,
490
+ "records_seen": records_seen,
491
+ "epochs": records_seen / n_train,
492
+ "batches_per_epoch": batches_per_epoch,
493
+ "tok_budget": TOK_BUDGET,
494
+ "last_loss": last_loss,
495
+ "train_loops": TRAIN_LOOPS,
496
+ "default_loops": DEFAULT_LOOPS,
497
+ "val_by_loops": val_by_loops,
498
+ "curve": curve,
499
+ },
500
+ )
501
+
502
+
503
+ def finetune(model, tok, data, rng, device):
504
+ """The old harness's RL fine-tune: AdamW, linear warmup then cosine to lr/10, over FT_RECORDS
505
+ records in token-budget batches (to_gpu / token_batches / gpu_batch, as pretraining). The
506
+ model is in train mode, so every batch draws its own loop count from TRAIN_LOOPS. Returns
507
+ (last loss, steps)."""
508
+ ids, mask, ref_max, lens = to_gpu(data, tok, device)
509
+ gen = torch.Generator(device=device)
510
+ gen.manual_seed(rng.getrandbits(31))
511
+ plan, seen = [], 0 # (order, start, end, padded length) per step, epoch after epoch
512
+ while seen < FT_RECORDS:
513
+ order, cuts = token_batches(lens, len(data), gen, rng, first=0)
514
+ for s, e, T in cuts:
515
+ if seen >= FT_RECORDS:
516
+ break
517
+ plan.append((order, s, e, T))
518
+ seen += e - s
519
+ opt = torch.optim.AdamW(model.parameters(), lr=FT_LR, weight_decay=0.1, betas=(0.9, 0.95))
520
+ autocast = torch.autocast("cuda", dtype=torch.bfloat16, enabled=(device == "cuda"))
521
+ model.train()
522
+ for step, (order, s, e, T) in enumerate(plan, 1):
523
+ lr = (
524
+ FT_LR * step / FT_WARMUP
525
+ if step < FT_WARMUP
526
+ else FT_LR / 10 + 0.45 * FT_LR * (1 + math.cos(math.pi * step / len(plan)))
527
+ )
528
+ for g in opt.param_groups:
529
+ g["lr"] = lr
530
+ x, m = gpu_batch(ids, mask, ref_max, lens, order[s:e], tok, gen, T)
531
+ with autocast:
532
+ loss = loss_on(model, x, m)
533
+ opt.zero_grad(set_to_none=True)
534
+ loss.backward()
535
+ torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
536
+ opt.step()
537
+ model.eval()
538
+ return loss.item(), len(plan)
539
+
540
+
541
+ def rl(model, tok, targets, train_recs, oracle, budget_secs, device, seed, log):
542
+ rng = random.Random(seed)
543
+ found = {t["name"]: {} for t in targets} # name -> {normalized proof: proof text}
544
+ rounds, round_secs, retries, ft_steps = 0, 0.0, [], []
545
+ # Start a round only if one as long as the last (plus 15%) still fits in the budget.
546
+ while oracle.left() > 1.15 * round_secs:
547
+ t0 = oracle.used()
548
+ model.n_loops = RL_FIRST_LOOPS
549
+ rows = oracle.sample(model, targets, RL_K)
550
+ failed = [t for t, row in zip(targets, rows, strict=True) if not row["proofs"]]
551
+ rng.shuffle(failed)
552
+ for i, n in enumerate(RL_RETRY_LOOPS):
553
+ part = failed[i :: len(RL_RETRY_LOOPS)]
554
+ if not part:
555
+ continue
556
+ model.n_loops = n
557
+ part_rows = oracle.sample(model, part, RL_RETRY_K)
558
+ cracked = sum(bool(row["proofs"]) for row in part_rows)
559
+ retries.append(
560
+ {"round": rounds + 1, "loops": n, "targets": len(part), "cracked": cracked}
561
+ )
562
+ rows += part_rows
563
+ for row in rows:
564
+ for proof in row["proofs"]:
565
+ found[row["name"]].setdefault(norm(proof), proof)
566
+ mix = []
567
+ for t in targets:
568
+ proofs = list(found[t["name"]].values())
569
+ rng.shuffle(proofs)
570
+ mix += [(t["prompt"], proof) for proof in proofs[:MAX_PER_THM]] * RL_WEIGHT
571
+ n_rl = len(mix)
572
+ if not n_rl:
573
+ break
574
+ mix += [(r["prompt"], r["proof"]) for r in rng.sample(train_recs, RETAIN)]
575
+ data = [(tok.encode_prompt(p), tok.encode_proof(q)) for p, q in mix]
576
+ loss, steps = finetune(model, tok, data, rng, device)
577
+ ft_steps.append(steps)
578
+ rounds += 1
579
+ round_secs = oracle.used() - t0
580
+ log(
581
+ f"rl round {rounds}: {n_rl} RL records, fine-tune loss {loss:.4f} in {steps} steps, "
582
+ f"round {round_secs:.0f}s, {oracle.left():.0f}s left"
583
+ )
584
+ model.n_loops = DEFAULT_LOOPS
585
+ return {
586
+ "rounds": rounds,
587
+ "first_loops": RL_FIRST_LOOPS,
588
+ "k_first": RL_K,
589
+ "retry_loops": RL_RETRY_LOOPS,
590
+ "k_retry": RL_RETRY_K,
591
+ "retries": retries,
592
+ "ft_records": FT_RECORDS,
593
+ "ft_steps": ft_steps,
594
+ }
runs/590_control-untied-198_s0/source/remote.sh ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env bash
2
+ # Runs on the pod. Checks out Dan's fork at the pinned commit (frozen data + verifier + Lean
3
+ # tokenizer/gate), installs Lean, runs the harness.
4
+ # Usage: bash remote.sh <out-dir> [harness.py args...]
5
+ set -euo pipefail
6
+ FORK_COMMIT=51604b3feaa5e66821d1faba22238fe6d96d46d0 # dan-pandori/nd-takehome, branch dan_lean_seed2 (lean_seq with the BOTE fix)
7
+ export ND_FORK=/workspace/nd-takehome
8
+ OUT=$1; shift
9
+
10
+ if [ ! -d "$ND_FORK/.git" ]; then
11
+ git clone -q https://github.com/dan-pandori/nd-takehome.git "$ND_FORK"
12
+ fi
13
+ git -C "$ND_FORK" cat-file -e "$FORK_COMMIT" 2>/dev/null || git -C "$ND_FORK" fetch -q origin
14
+ git -C "$ND_FORK" checkout -q "$FORK_COMMIT"
15
+ [ -f "$ND_FORK/data/train.jsonl" ] || gunzip -k "$ND_FORK/data/train.jsonl.gz"
16
+ # Lean 4 core checks every sampled proof (fork lean_gate.py, called from sample.generate): the
17
+ # reward is Lean AND nd_verify. Installed before harness.py starts, so it is not charged to TOTAL_SECS.
18
+ [ -x ~/.elan/bin/lean ] || curl -sSf https://raw.githubusercontent.com/leanprover/elan/master/elan-init.sh \
19
+ | sh -s -- -y --default-toolchain leanprover/lean4:v4.34.0 > /dev/null
20
+ # One Lean process per CPU: on these pods (nproc 96) 12 workers took 146 s per RL-round gate
21
+ # pass (099) against 67-84 s with 96 workers (098).
22
+ export LEAN_GATE_WORKERS=$(nproc) LEAN_GATE_LOG="$OUT/lean_gate.jsonl"
23
+ nvidia-smi --query-gpu=name,memory.total --format=csv,noheader
24
+ echo "cpus $(nproc), $(~/.elan/bin/lean --version)"
25
+ # Newer RunPod images put torch under `python` and leave `python3` as bare system Python.
26
+ PY=$(for p in python python3; do "$p" -c "import torch" 2>/dev/null && echo "$p" && break; done)
27
+ echo "python $($PY --version 2>&1), torch $($PY -c 'import torch; print(torch.__version__)')"
28
+ cd /workspace/autoresearch
29
+ $PY harness.py --out "$OUT" "$@" 2>&1 | tee "$OUT.log"
runs/590_control-untied-198_s0/summary.json ADDED
@@ -0,0 +1,3128 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "status": "ok",
3
+ "metric": 790,
4
+ "metric_diag": 790,
5
+ "metric_rl": 1309,
6
+ "harness_sha": "2eec1b61666a",
7
+ "pretrain_sha": "b27dd2cb03ba",
8
+ "device": "cuda",
9
+ "gpu": "NVIDIA A40",
10
+ "torch": "2.9.1+cu129",
11
+ "lean_gate": {
12
+ "samples": 524304,
13
+ "parse_fail": 94720,
14
+ "distinct_checked": 389781,
15
+ "both_ok": 240965,
16
+ "nd_ok_lean_rej": 0,
17
+ "nd_rej_lean_ok": 0,
18
+ "lean_wall_s": 1245.6879568099976
19
+ },
20
+ "total_secs": 3660.8543887138367,
21
+ "rules": {
22
+ "TOTAL_SECS": 5400,
23
+ "PRETRAIN_SECS": 300,
24
+ "RL_SECS": 1800,
25
+ "PARAM_CAP": 10000000,
26
+ "N_TARGETS": 1500,
27
+ "EVAL_K": 64,
28
+ "KNOB": "n_loops",
29
+ "EVAL_LEVEL": 8,
30
+ "DIAG_LEVEL": 2,
31
+ "TOK_MODE": "lean_seq"
32
+ },
33
+ "pretrain": {
34
+ "steps": 5322,
35
+ "records_seen": 723768,
36
+ "epochs": 4.700830053388411,
37
+ "batches_per_epoch": 1132,
38
+ "tok_budget": 18432,
39
+ "last_loss": 0.0643298402428627,
40
+ "train_loops": [
41
+ 1,
42
+ 2,
43
+ 3,
44
+ 4,
45
+ 5,
46
+ 6,
47
+ 7,
48
+ 8
49
+ ],
50
+ "default_loops": 2,
51
+ "val_by_loops": {
52
+ "1": 0.0866707032546401,
53
+ "2": 0.0866707032546401,
54
+ "4": 0.0866707032546401,
55
+ "8": 0.0866707032546401
56
+ },
57
+ "curve": [
58
+ {
59
+ "secs": 31,
60
+ "step": 1,
61
+ "train_loss": NaN,
62
+ "val_loss": 3.9265004098415375,
63
+ "mtp_loss": null
64
+ },
65
+ {
66
+ "secs": 34,
67
+ "step": 2,
68
+ "train_loss": NaN,
69
+ "val_loss": 3.5213595032691956,
70
+ "mtp_loss": null
71
+ },
72
+ {
73
+ "secs": 45,
74
+ "step": 234,
75
+ "train_loss": 0.4438230293718251,
76
+ "val_loss": 0.10797532740980387,
77
+ "mtp_loss": 0.45858401534232224
78
+ },
79
+ {
80
+ "secs": 60,
81
+ "step": 537,
82
+ "train_loss": 0.09316873500744502,
83
+ "val_loss": 0.09572682809084654,
84
+ "mtp_loss": 0.09360948950052261
85
+ },
86
+ {
87
+ "secs": 75,
88
+ "step": 840,
89
+ "train_loss": 0.07020179787650704,
90
+ "val_loss": 0.08384304493665695,
91
+ "mtp_loss": 0.07230285461992025
92
+ },
93
+ {
94
+ "secs": 90,
95
+ "step": 1140,
96
+ "train_loss": 0.059316099435091016,
97
+ "val_loss": 0.09674938209354877,
98
+ "mtp_loss": 0.061977747827768326
99
+ },
100
+ {
101
+ "secs": 105,
102
+ "step": 1444,
103
+ "train_loss": 0.06523487865924835,
104
+ "val_loss": 0.08504640311002731,
105
+ "mtp_loss": 0.06617915580670039
106
+ },
107
+ {
108
+ "secs": 120,
109
+ "step": 1743,
110
+ "train_loss": 0.06727594335873922,
111
+ "val_loss": 0.07866689842194319,
112
+ "mtp_loss": 0.06769198700785636
113
+ },
114
+ {
115
+ "secs": 135,
116
+ "step": 2042,
117
+ "train_loss": 0.0691841093202432,
118
+ "val_loss": 0.08289193827658892,
119
+ "mtp_loss": 0.06978676989674568
120
+ },
121
+ {
122
+ "secs": 150,
123
+ "step": 2342,
124
+ "train_loss": 0.06363775730133056,
125
+ "val_loss": 0.0940767228603363,
126
+ "mtp_loss": 0.06482930605610211
127
+ },
128
+ {
129
+ "secs": 165,
130
+ "step": 2643,
131
+ "train_loss": 0.05806758925318718,
132
+ "val_loss": 0.07991388812661171,
133
+ "mtp_loss": 0.058483662207921346
134
+ },
135
+ {
136
+ "secs": 180,
137
+ "step": 2943,
138
+ "train_loss": 0.07072806904713312,
139
+ "val_loss": 0.08869772125035524,
140
+ "mtp_loss": 0.07184548750519752
141
+ },
142
+ {
143
+ "secs": 195,
144
+ "step": 3242,
145
+ "train_loss": 0.056170069798827174,
146
+ "val_loss": 0.0838435497134924,
147
+ "mtp_loss": 0.05944442500670751
148
+ },
149
+ {
150
+ "secs": 210,
151
+ "step": 3540,
152
+ "train_loss": 0.06035705991089344,
153
+ "val_loss": 0.08335433062165976,
154
+ "mtp_loss": 0.060601692150036496
155
+ },
156
+ {
157
+ "secs": 225,
158
+ "step": 3840,
159
+ "train_loss": 0.05841459110379219,
160
+ "val_loss": 0.07814371213316917,
161
+ "mtp_loss": 0.058488543579975766
162
+ },
163
+ {
164
+ "secs": 240,
165
+ "step": 4140,
166
+ "train_loss": 0.05674752692381541,
167
+ "val_loss": 0.07864161394536495,
168
+ "mtp_loss": 0.056829093893369036
169
+ },
170
+ {
171
+ "secs": 255,
172
+ "step": 4439,
173
+ "train_loss": 0.05531486363283226,
174
+ "val_loss": 0.09093722235411406,
175
+ "mtp_loss": 0.05732316550399576
176
+ },
177
+ {
178
+ "secs": 270,
179
+ "step": 4724,
180
+ "train_loss": 0.04911263796190421,
181
+ "val_loss": 0.0790712758898735,
182
+ "mtp_loss": 0.04921042968829473
183
+ },
184
+ {
185
+ "secs": 285,
186
+ "step": 5024,
187
+ "train_loss": 0.060920559366544086,
188
+ "val_loss": 0.08792904019355774,
189
+ "mtp_loss": 0.060834589103857674
190
+ },
191
+ {
192
+ "secs": 300,
193
+ "step": 5322,
194
+ "train_loss": 0.05323498733341694,
195
+ "val_loss": 0.0866707032546401,
196
+ "mtp_loss": 0.053421760971347494
197
+ }
198
+ ],
199
+ "secs": 300.7043650150299,
200
+ "n_params": 9558528,
201
+ "heldout_greedy": 0.9685
202
+ },
203
+ "rl": {
204
+ "rounds": [
205
+ {
206
+ "round": 1,
207
+ "level": 1,
208
+ "k": 8,
209
+ "n_targets": 1500,
210
+ "sample_acc": 0.21783333333333332,
211
+ "new_proofs": 713,
212
+ "targets_solved": 625,
213
+ "lstar": 9,
214
+ "ge": {
215
+ "7": 625,
216
+ "8": 241,
217
+ "9": 63,
218
+ "10": 1,
219
+ "11": 0,
220
+ "12": 0,
221
+ "13": 0,
222
+ "14": 0
223
+ },
224
+ "mix_rl_records": 713,
225
+ "sampling": {
226
+ "samples": 12000,
227
+ "secs": 67.77599048614502,
228
+ "gate_secs": 25.270068645477295,
229
+ "samples_per_sec": 177.05384921601518
230
+ },
231
+ "secs": 68.23632264137268,
232
+ "rl_secs_used": 42.96647357940674
233
+ },
234
+ {
235
+ "round": 2,
236
+ "level": 2,
237
+ "k": 24,
238
+ "n_targets": 292,
239
+ "sample_acc": 0.01127283105022831,
240
+ "new_proofs": 36,
241
+ "targets_solved": 660,
242
+ "lstar": 9,
243
+ "ge": {
244
+ "7": 660,
245
+ "8": 263,
246
+ "9": 75,
247
+ "10": 3,
248
+ "11": 0,
249
+ "12": 0,
250
+ "13": 0,
251
+ "14": 0
252
+ },
253
+ "mix_rl_records": 36,
254
+ "sampling": {
255
+ "samples": 7008,
256
+ "secs": 38.23188781738281,
257
+ "gate_secs": 14.766036748886108,
258
+ "samples_per_sec": 183.30248387090336
259
+ },
260
+ "secs": 38.45504450798035,
261
+ "rl_secs_used": 66.65531277656555
262
+ },
263
+ {
264
+ "round": 3,
265
+ "level": 4,
266
+ "k": 24,
267
+ "n_targets": 292,
268
+ "sample_acc": 0.006563926940639269,
269
+ "new_proofs": 29,
270
+ "targets_solved": 688,
271
+ "lstar": 10,
272
+ "ge": {
273
+ "7": 688,
274
+ "8": 278,
275
+ "9": 85,
276
+ "10": 5,
277
+ "11": 0,
278
+ "12": 0,
279
+ "13": 0,
280
+ "14": 0
281
+ },
282
+ "mix_rl_records": 29,
283
+ "sampling": {
284
+ "samples": 7008,
285
+ "secs": 35.475021839141846,
286
+ "gate_secs": 15.691442728042603,
287
+ "samples_per_sec": 197.5474470960756
288
+ },
289
+ "secs": 35.69817113876343,
290
+ "rl_secs_used": 86.66204929351807
291
+ },
292
+ {
293
+ "round": 4,
294
+ "level": 8,
295
+ "k": 24,
296
+ "n_targets": 291,
297
+ "sample_acc": 0.012600229095074456,
298
+ "new_proofs": 48,
299
+ "targets_solved": 732,
300
+ "lstar": 10,
301
+ "ge": {
302
+ "7": 732,
303
+ "8": 306,
304
+ "9": 100,
305
+ "10": 6,
306
+ "11": 0,
307
+ "12": 0,
308
+ "13": 0,
309
+ "14": 0
310
+ },
311
+ "mix_rl_records": 48,
312
+ "sampling": {
313
+ "samples": 6984,
314
+ "secs": 35.888651609420776,
315
+ "gate_secs": 14.895998239517212,
316
+ "samples_per_sec": 194.60190580597626
317
+ },
318
+ "secs": 36.10332417488098,
319
+ "rl_secs_used": 107.86937427520752
320
+ },
321
+ {
322
+ "round": 5,
323
+ "level": 1,
324
+ "k": 8,
325
+ "n_targets": 1500,
326
+ "sample_acc": 0.5983333333333334,
327
+ "new_proofs": 523,
328
+ "targets_solved": 1014,
329
+ "lstar": 10,
330
+ "ge": {
331
+ "7": 1014,
332
+ "8": 512,
333
+ "9": 245,
334
+ "10": 33,
335
+ "11": 2,
336
+ "12": 1,
337
+ "13": 0,
338
+ "14": 0
339
+ },
340
+ "mix_rl_records": 1331,
341
+ "sampling": {
342
+ "samples": 12000,
343
+ "secs": 62.51512789726257,
344
+ "gate_secs": 35.82744479179382,
345
+ "samples_per_sec": 191.95353834548357
346
+ },
347
+ "secs": 94.49270796775818,
348
+ "rl_secs_used": 166.5347602367401
349
+ },
350
+ {
351
+ "round": 6,
352
+ "level": 2,
353
+ "k": 24,
354
+ "n_targets": 162,
355
+ "sample_acc": 0.007716049382716049,
356
+ "new_proofs": 13,
357
+ "targets_solved": 1026,
358
+ "lstar": 10,
359
+ "ge": {
360
+ "7": 1026,
361
+ "8": 523,
362
+ "9": 252,
363
+ "10": 35,
364
+ "11": 2,
365
+ "12": 1,
366
+ "13": 0,
367
+ "14": 0
368
+ },
369
+ "mix_rl_records": 13,
370
+ "sampling": {
371
+ "samples": 3888,
372
+ "secs": 21.616540908813477,
373
+ "gate_secs": 10.649104595184326,
374
+ "samples_per_sec": 179.86226456864745
375
+ },
376
+ "secs": 21.765689849853516,
377
+ "rl_secs_used": 177.6511993408203
378
+ },
379
+ {
380
+ "round": 7,
381
+ "level": 4,
382
+ "k": 24,
383
+ "n_targets": 162,
384
+ "sample_acc": 0.00565843621399177,
385
+ "new_proofs": 7,
386
+ "targets_solved": 1033,
387
+ "lstar": 10,
388
+ "ge": {
389
+ "7": 1033,
390
+ "8": 530,
391
+ "9": 259,
392
+ "10": 37,
393
+ "11": 2,
394
+ "12": 1,
395
+ "13": 0,
396
+ "14": 0
397
+ },
398
+ "mix_rl_records": 7,
399
+ "sampling": {
400
+ "samples": 3888,
401
+ "secs": 17.423007011413574,
402
+ "gate_secs": 8.932615280151367,
403
+ "samples_per_sec": 223.15321330313557
404
+ },
405
+ "secs": 17.55361533164978,
406
+ "rl_secs_used": 186.27219939231873
407
+ },
408
+ {
409
+ "round": 8,
410
+ "level": 8,
411
+ "k": 24,
412
+ "n_targets": 162,
413
+ "sample_acc": 0.009259259259259259,
414
+ "new_proofs": 19,
415
+ "targets_solved": 1047,
416
+ "lstar": 10,
417
+ "ge": {
418
+ "7": 1047,
419
+ "8": 539,
420
+ "9": 266,
421
+ "10": 40,
422
+ "11": 2,
423
+ "12": 1,
424
+ "13": 0,
425
+ "14": 0
426
+ },
427
+ "mix_rl_records": 19,
428
+ "sampling": {
429
+ "samples": 3888,
430
+ "secs": 20.20773720741272,
431
+ "gate_secs": 10.925711870193481,
432
+ "samples_per_sec": 192.40155194485513
433
+ },
434
+ "secs": 20.36957311630249,
435
+ "rl_secs_used": 195.71607875823975
436
+ },
437
+ {
438
+ "round": 9,
439
+ "level": 1,
440
+ "k": 8,
441
+ "n_targets": 1500,
442
+ "sample_acc": 0.691,
443
+ "new_proofs": 252,
444
+ "targets_solved": 1107,
445
+ "lstar": 11,
446
+ "ge": {
447
+ "7": 1107,
448
+ "8": 587,
449
+ "9": 299,
450
+ "10": 55,
451
+ "11": 5,
452
+ "12": 1,
453
+ "13": 0,
454
+ "14": 0
455
+ },
456
+ "mix_rl_records": 1545,
457
+ "sampling": {
458
+ "samples": 12000,
459
+ "secs": 51.692243576049805,
460
+ "gate_secs": 28.218570947647095,
461
+ "samples_per_sec": 232.14314508027803
462
+ },
463
+ "secs": 86.44683027267456,
464
+ "rl_secs_used": 253.94453406333923
465
+ },
466
+ {
467
+ "round": 10,
468
+ "level": 2,
469
+ "k": 24,
470
+ "n_targets": 131,
471
+ "sample_acc": 0.008587786259541985,
472
+ "new_proofs": 14,
473
+ "targets_solved": 1117,
474
+ "lstar": 11,
475
+ "ge": {
476
+ "7": 1117,
477
+ "8": 592,
478
+ "9": 304,
479
+ "10": 56,
480
+ "11": 5,
481
+ "12": 1,
482
+ "13": 0,
483
+ "14": 0
484
+ },
485
+ "mix_rl_records": 14,
486
+ "sampling": {
487
+ "samples": 3144,
488
+ "secs": 15.908243894577026,
489
+ "gate_secs": 9.021638870239258,
490
+ "samples_per_sec": 197.63337932427353
491
+ },
492
+ "secs": 16.025395154953003,
493
+ "rl_secs_used": 260.9480698108673
494
+ },
495
+ {
496
+ "round": 11,
497
+ "level": 4,
498
+ "k": 24,
499
+ "n_targets": 131,
500
+ "sample_acc": 0.006361323155216285,
501
+ "new_proofs": 11,
502
+ "targets_solved": 1128,
503
+ "lstar": 11,
504
+ "ge": {
505
+ "7": 1128,
506
+ "8": 600,
507
+ "9": 311,
508
+ "10": 59,
509
+ "11": 6,
510
+ "12": 1,
511
+ "13": 0,
512
+ "14": 0
513
+ },
514
+ "mix_rl_records": 11,
515
+ "sampling": {
516
+ "samples": 3144,
517
+ "secs": 17.698871850967407,
518
+ "gate_secs": 9.045248985290527,
519
+ "samples_per_sec": 177.63844082684577
520
+ },
521
+ "secs": 17.816609859466553,
522
+ "rl_secs_used": 269.71942925453186
523
+ },
524
+ {
525
+ "round": 12,
526
+ "level": 8,
527
+ "k": 24,
528
+ "n_targets": 131,
529
+ "sample_acc": 0.0111323155216285,
530
+ "new_proofs": 16,
531
+ "targets_solved": 1141,
532
+ "lstar": 11,
533
+ "ge": {
534
+ "7": 1141,
535
+ "8": 611,
536
+ "9": 322,
537
+ "10": 64,
538
+ "11": 8,
539
+ "12": 1,
540
+ "13": 0,
541
+ "14": 0
542
+ },
543
+ "mix_rl_records": 16,
544
+ "sampling": {
545
+ "samples": 3144,
546
+ "secs": 14.799442291259766,
547
+ "gate_secs": 8.877211809158325,
548
+ "samples_per_sec": 212.44043783033496
549
+ },
550
+ "secs": 14.911684513092041,
551
+ "rl_secs_used": 275.75390100479126
552
+ },
553
+ {
554
+ "round": 13,
555
+ "level": 1,
556
+ "k": 8,
557
+ "n_targets": 1500,
558
+ "sample_acc": 0.7393333333333333,
559
+ "new_proofs": 219,
560
+ "targets_solved": 1176,
561
+ "lstar": 11,
562
+ "ge": {
563
+ "7": 1176,
564
+ "8": 636,
565
+ "9": 337,
566
+ "10": 77,
567
+ "11": 9,
568
+ "12": 1,
569
+ "13": 0,
570
+ "14": 0
571
+ },
572
+ "mix_rl_records": 1751,
573
+ "sampling": {
574
+ "samples": 12000,
575
+ "secs": 56.74859642982483,
576
+ "gate_secs": 31.435093641281128,
577
+ "samples_per_sec": 211.45897440545812
578
+ },
579
+ "secs": 92.67984652519226,
580
+ "rl_secs_used": 336.99888610839844
581
+ },
582
+ {
583
+ "round": 14,
584
+ "level": 2,
585
+ "k": 24,
586
+ "n_targets": 109,
587
+ "sample_acc": 0.0034403669724770644,
588
+ "new_proofs": 5,
589
+ "targets_solved": 1179,
590
+ "lstar": 11,
591
+ "ge": {
592
+ "7": 1179,
593
+ "8": 638,
594
+ "9": 339,
595
+ "10": 78,
596
+ "11": 10,
597
+ "12": 1,
598
+ "13": 0,
599
+ "14": 0
600
+ },
601
+ "mix_rl_records": 5,
602
+ "sampling": {
603
+ "samples": 2616,
604
+ "secs": 12.687414407730103,
605
+ "gate_secs": 6.6271655559539795,
606
+ "samples_per_sec": 206.18858310532846
607
+ },
608
+ "secs": 12.77810025215149,
609
+ "rl_secs_used": 343.14958453178406
610
+ },
611
+ {
612
+ "round": 15,
613
+ "level": 4,
614
+ "k": 24,
615
+ "n_targets": 108,
616
+ "sample_acc": 0.017746913580246913,
617
+ "new_proofs": 18,
618
+ "targets_solved": 1190,
619
+ "lstar": 11,
620
+ "ge": {
621
+ "7": 1190,
622
+ "8": 648,
623
+ "9": 346,
624
+ "10": 80,
625
+ "11": 10,
626
+ "12": 1,
627
+ "13": 0,
628
+ "14": 0
629
+ },
630
+ "mix_rl_records": 18,
631
+ "sampling": {
632
+ "samples": 2592,
633
+ "secs": 13.075745105743408,
634
+ "gate_secs": 6.673742771148682,
635
+ "samples_per_sec": 198.22962125970827
636
+ },
637
+ "secs": 13.16294264793396,
638
+ "rl_secs_used": 349.6388018131256
639
+ },
640
+ {
641
+ "round": 16,
642
+ "level": 8,
643
+ "k": 24,
644
+ "n_targets": 108,
645
+ "sample_acc": 0.0011574074074074073,
646
+ "new_proofs": 3,
647
+ "targets_solved": 1193,
648
+ "lstar": 11,
649
+ "ge": {
650
+ "7": 1193,
651
+ "8": 650,
652
+ "9": 347,
653
+ "10": 81,
654
+ "11": 10,
655
+ "12": 1,
656
+ "13": 0,
657
+ "14": 0
658
+ },
659
+ "mix_rl_records": 3,
660
+ "sampling": {
661
+ "samples": 2592,
662
+ "secs": 12.169060230255127,
663
+ "gate_secs": 7.233021974563599,
664
+ "samples_per_sec": 212.99919229224312
665
+ },
666
+ "secs": 12.258346557617188,
667
+ "rl_secs_used": 354.66411542892456
668
+ },
669
+ {
670
+ "round": 17,
671
+ "level": 1,
672
+ "k": 8,
673
+ "n_targets": 1500,
674
+ "sample_acc": 0.769,
675
+ "new_proofs": 181,
676
+ "targets_solved": 1206,
677
+ "lstar": 11,
678
+ "ge": {
679
+ "7": 1206,
680
+ "8": 662,
681
+ "9": 357,
682
+ "10": 89,
683
+ "11": 12,
684
+ "12": 2,
685
+ "13": 0,
686
+ "14": 0
687
+ },
688
+ "mix_rl_records": 1905,
689
+ "sampling": {
690
+ "samples": 12000,
691
+ "secs": 64.74264144897461,
692
+ "gate_secs": 36.31877303123474,
693
+ "samples_per_sec": 185.3492494503413
694
+ },
695
+ "secs": 101.42965579032898,
696
+ "rl_secs_used": 419.77515053749084
697
+ },
698
+ {
699
+ "round": 18,
700
+ "level": 2,
701
+ "k": 24,
702
+ "n_targets": 99,
703
+ "sample_acc": 0.0012626262626262627,
704
+ "new_proofs": 2,
705
+ "targets_solved": 1208,
706
+ "lstar": 11,
707
+ "ge": {
708
+ "7": 1208,
709
+ "8": 662,
710
+ "9": 357,
711
+ "10": 89,
712
+ "11": 12,
713
+ "12": 2,
714
+ "13": 0,
715
+ "14": 0
716
+ },
717
+ "mix_rl_records": 2,
718
+ "sampling": {
719
+ "samples": 2376,
720
+ "secs": 10.839726209640503,
721
+ "gate_secs": 5.558458089828491,
722
+ "samples_per_sec": 219.19372814848978
723
+ },
724
+ "secs": 10.918500661849976,
725
+ "rl_secs_used": 425.135036945343
726
+ },
727
+ {
728
+ "round": 19,
729
+ "level": 4,
730
+ "k": 24,
731
+ "n_targets": 98,
732
+ "sample_acc": 0.0055272108843537416,
733
+ "new_proofs": 9,
734
+ "targets_solved": 1216,
735
+ "lstar": 11,
736
+ "ge": {
737
+ "7": 1216,
738
+ "8": 670,
739
+ "9": 362,
740
+ "10": 90,
741
+ "11": 13,
742
+ "12": 2,
743
+ "13": 0,
744
+ "14": 0
745
+ },
746
+ "mix_rl_records": 9,
747
+ "sampling": {
748
+ "samples": 2352,
749
+ "secs": 11.605263710021973,
750
+ "gate_secs": 6.112550497055054,
751
+ "samples_per_sec": 202.666657024681
752
+ },
753
+ "secs": 11.686203718185425,
754
+ "rl_secs_used": 430.70869851112366
755
+ },
756
+ {
757
+ "round": 20,
758
+ "level": 8,
759
+ "k": 24,
760
+ "n_targets": 98,
761
+ "sample_acc": 0.0012755102040816326,
762
+ "new_proofs": 2,
763
+ "targets_solved": 1218,
764
+ "lstar": 11,
765
+ "ge": {
766
+ "7": 1218,
767
+ "8": 672,
768
+ "9": 363,
769
+ "10": 90,
770
+ "11": 13,
771
+ "12": 2,
772
+ "13": 0,
773
+ "14": 0
774
+ },
775
+ "mix_rl_records": 2,
776
+ "sampling": {
777
+ "samples": 2352,
778
+ "secs": 11.557843685150146,
779
+ "gate_secs": 5.491286039352417,
780
+ "samples_per_sec": 203.49816661925598
781
+ },
782
+ "secs": 11.630718231201172,
783
+ "rl_secs_used": 436.84812211990356
784
+ },
785
+ {
786
+ "round": 21,
787
+ "level": 1,
788
+ "k": 8,
789
+ "n_targets": 1500,
790
+ "sample_acc": 0.785,
791
+ "new_proofs": 185,
792
+ "targets_solved": 1229,
793
+ "lstar": 11,
794
+ "ge": {
795
+ "7": 1229,
796
+ "8": 681,
797
+ "9": 370,
798
+ "10": 91,
799
+ "11": 14,
800
+ "12": 2,
801
+ "13": 0,
802
+ "14": 0
803
+ },
804
+ "mix_rl_records": 2034,
805
+ "sampling": {
806
+ "samples": 12000,
807
+ "secs": 58.953579902648926,
808
+ "gate_secs": 33.94571661949158,
809
+ "samples_per_sec": 203.54997982846518
810
+ },
811
+ "secs": 95.96905970573425,
812
+ "rl_secs_used": 498.87162160873413
813
+ },
814
+ {
815
+ "round": 22,
816
+ "level": 2,
817
+ "k": 24,
818
+ "n_targets": 91,
819
+ "sample_acc": 0.0009157509157509158,
820
+ "new_proofs": 1,
821
+ "targets_solved": 1230,
822
+ "lstar": 11,
823
+ "ge": {
824
+ "7": 1230,
825
+ "8": 681,
826
+ "9": 370,
827
+ "10": 91,
828
+ "11": 14,
829
+ "12": 2,
830
+ "13": 0,
831
+ "14": 0
832
+ },
833
+ "mix_rl_records": 1,
834
+ "sampling": {
835
+ "samples": 2184,
836
+ "secs": 13.13532567024231,
837
+ "gate_secs": 5.709080457687378,
838
+ "samples_per_sec": 166.2691930773964
839
+ },
840
+ "secs": 13.216411590576172,
841
+ "rl_secs_used": 506.3788034915924
842
+ },
843
+ {
844
+ "round": 23,
845
+ "level": 4,
846
+ "k": 24,
847
+ "n_targets": 90,
848
+ "sample_acc": 0.002777777777777778,
849
+ "new_proofs": 4,
850
+ "targets_solved": 1233,
851
+ "lstar": 11,
852
+ "ge": {
853
+ "7": 1233,
854
+ "8": 684,
855
+ "9": 373,
856
+ "10": 93,
857
+ "11": 15,
858
+ "12": 3,
859
+ "13": 0,
860
+ "14": 0
861
+ },
862
+ "mix_rl_records": 4,
863
+ "sampling": {
864
+ "samples": 2160,
865
+ "secs": 9.675426006317139,
866
+ "gate_secs": 5.229877471923828,
867
+ "samples_per_sec": 223.2459840620686
868
+ },
869
+ "secs": 9.747728824615479,
870
+ "rl_secs_used": 510.896657705307
871
+ },
872
+ {
873
+ "round": 24,
874
+ "level": 8,
875
+ "k": 24,
876
+ "n_targets": 90,
877
+ "sample_acc": 0.000925925925925926,
878
+ "new_proofs": 2,
879
+ "targets_solved": 1235,
880
+ "lstar": 11,
881
+ "ge": {
882
+ "7": 1235,
883
+ "8": 686,
884
+ "9": 375,
885
+ "10": 93,
886
+ "11": 15,
887
+ "12": 3,
888
+ "13": 0,
889
+ "14": 0
890
+ },
891
+ "mix_rl_records": 2,
892
+ "sampling": {
893
+ "samples": 2160,
894
+ "secs": 11.80867075920105,
895
+ "gate_secs": 5.4752068519592285,
896
+ "samples_per_sec": 182.9164386107536
897
+ },
898
+ "secs": 11.884621143341064,
899
+ "rl_secs_used": 517.3060626983643
900
+ },
901
+ {
902
+ "round": 25,
903
+ "level": 1,
904
+ "k": 8,
905
+ "n_targets": 1500,
906
+ "sample_acc": 0.7980833333333334,
907
+ "new_proofs": 148,
908
+ "targets_solved": 1241,
909
+ "lstar": 11,
910
+ "ge": {
911
+ "7": 1241,
912
+ "8": 690,
913
+ "9": 378,
914
+ "10": 95,
915
+ "11": 16,
916
+ "12": 3,
917
+ "13": 0,
918
+ "14": 0
919
+ },
920
+ "mix_rl_records": 2128,
921
+ "sampling": {
922
+ "samples": 12000,
923
+ "secs": 57.482112646102905,
924
+ "gate_secs": 33.026079177856445,
925
+ "samples_per_sec": 208.76059434140438
926
+ },
927
+ "secs": 95.11074233055115,
928
+ "rl_secs_used": 579.390962600708
929
+ },
930
+ {
931
+ "round": 26,
932
+ "level": 2,
933
+ "k": 24,
934
+ "n_targets": 87,
935
+ "sample_acc": 0.0023946360153256703,
936
+ "new_proofs": 4,
937
+ "targets_solved": 1245,
938
+ "lstar": 11,
939
+ "ge": {
940
+ "7": 1245,
941
+ "8": 693,
942
+ "9": 381,
943
+ "10": 96,
944
+ "11": 16,
945
+ "12": 3,
946
+ "13": 0,
947
+ "14": 0
948
+ },
949
+ "mix_rl_records": 4,
950
+ "sampling": {
951
+ "samples": 2088,
952
+ "secs": 12.137623071670532,
953
+ "gate_secs": 5.21572732925415,
954
+ "samples_per_sec": 172.02709193313441
955
+ },
956
+ "secs": 12.210636854171753,
957
+ "rl_secs_used": 586.3856315612793
958
+ },
959
+ {
960
+ "round": 27,
961
+ "level": 4,
962
+ "k": 24,
963
+ "n_targets": 86,
964
+ "sample_acc": 0.0,
965
+ "new_proofs": 0,
966
+ "targets_solved": 1245,
967
+ "lstar": 11,
968
+ "ge": {
969
+ "7": 1245,
970
+ "8": 693,
971
+ "9": 381,
972
+ "10": 96,
973
+ "11": 16,
974
+ "12": 3,
975
+ "13": 0,
976
+ "14": 0
977
+ },
978
+ "mix_rl_records": 0,
979
+ "sampling": {
980
+ "samples": 2064,
981
+ "secs": 11.939735412597656,
982
+ "gate_secs": 4.552095413208008,
983
+ "samples_per_sec": 172.8681523228954
984
+ },
985
+ "secs": 12.005858421325684,
986
+ "rl_secs_used": 593.8393940925598
987
+ },
988
+ {
989
+ "round": 28,
990
+ "level": 8,
991
+ "k": 24,
992
+ "n_targets": 86,
993
+ "sample_acc": 0.00048449612403100775,
994
+ "new_proofs": 1,
995
+ "targets_solved": 1246,
996
+ "lstar": 11,
997
+ "ge": {
998
+ "7": 1246,
999
+ "8": 694,
1000
+ "9": 382,
1001
+ "10": 97,
1002
+ "11": 16,
1003
+ "12": 3,
1004
+ "13": 0,
1005
+ "14": 0
1006
+ },
1007
+ "mix_rl_records": 1,
1008
+ "sampling": {
1009
+ "samples": 2064,
1010
+ "secs": 9.376678466796875,
1011
+ "gate_secs": 4.408054828643799,
1012
+ "samples_per_sec": 220.1205903890905
1013
+ },
1014
+ "secs": 9.437121391296387,
1015
+ "rl_secs_used": 598.8684594631195
1016
+ },
1017
+ {
1018
+ "round": 29,
1019
+ "level": 1,
1020
+ "k": 8,
1021
+ "n_targets": 1500,
1022
+ "sample_acc": 0.8060833333333334,
1023
+ "new_proofs": 129,
1024
+ "targets_solved": 1248,
1025
+ "lstar": 11,
1026
+ "ge": {
1027
+ "7": 1248,
1028
+ "8": 695,
1029
+ "9": 383,
1030
+ "10": 97,
1031
+ "11": 16,
1032
+ "12": 3,
1033
+ "13": 0,
1034
+ "14": 0
1035
+ },
1036
+ "mix_rl_records": 2220,
1037
+ "sampling": {
1038
+ "samples": 12000,
1039
+ "secs": 57.369160890579224,
1040
+ "gate_secs": 32.32744026184082,
1041
+ "samples_per_sec": 209.17161439554118
1042
+ },
1043
+ "secs": 95.50686025619507,
1044
+ "rl_secs_used": 662.0481176376343
1045
+ },
1046
+ {
1047
+ "round": 30,
1048
+ "level": 2,
1049
+ "k": 24,
1050
+ "n_targets": 84,
1051
+ "sample_acc": 0.0,
1052
+ "new_proofs": 0,
1053
+ "targets_solved": 1248,
1054
+ "lstar": 11,
1055
+ "ge": {
1056
+ "7": 1248,
1057
+ "8": 695,
1058
+ "9": 383,
1059
+ "10": 97,
1060
+ "11": 16,
1061
+ "12": 3,
1062
+ "13": 0,
1063
+ "14": 0
1064
+ },
1065
+ "mix_rl_records": 0,
1066
+ "sampling": {
1067
+ "samples": 2016,
1068
+ "secs": 10.358954429626465,
1069
+ "gate_secs": 4.623164653778076,
1070
+ "samples_per_sec": 194.61423579915248
1071
+ },
1072
+ "secs": 10.423393726348877,
1073
+ "rl_secs_used": 667.8481087684631
1074
+ },
1075
+ {
1076
+ "round": 31,
1077
+ "level": 4,
1078
+ "k": 24,
1079
+ "n_targets": 84,
1080
+ "sample_acc": 0.0,
1081
+ "new_proofs": 0,
1082
+ "targets_solved": 1248,
1083
+ "lstar": 11,
1084
+ "ge": {
1085
+ "7": 1248,
1086
+ "8": 695,
1087
+ "9": 383,
1088
+ "10": 97,
1089
+ "11": 16,
1090
+ "12": 3,
1091
+ "13": 0,
1092
+ "14": 0
1093
+ },
1094
+ "mix_rl_records": 0,
1095
+ "sampling": {
1096
+ "samples": 2016,
1097
+ "secs": 11.444467306137085,
1098
+ "gate_secs": 4.465373516082764,
1099
+ "samples_per_sec": 176.15498791445907
1100
+ },
1101
+ "secs": 11.505950927734375,
1102
+ "rl_secs_used": 674.8886890411377
1103
+ },
1104
+ {
1105
+ "round": 32,
1106
+ "level": 8,
1107
+ "k": 24,
1108
+ "n_targets": 84,
1109
+ "sample_acc": 0.000992063492063492,
1110
+ "new_proofs": 2,
1111
+ "targets_solved": 1250,
1112
+ "lstar": 11,
1113
+ "ge": {
1114
+ "7": 1250,
1115
+ "8": 697,
1116
+ "9": 385,
1117
+ "10": 97,
1118
+ "11": 16,
1119
+ "12": 3,
1120
+ "13": 0,
1121
+ "14": 0
1122
+ },
1123
+ "mix_rl_records": 2,
1124
+ "sampling": {
1125
+ "samples": 2016,
1126
+ "secs": 8.455175161361694,
1127
+ "gate_secs": 4.679951906204224,
1128
+ "samples_per_sec": 238.4338540037207
1129
+ },
1130
+ "secs": 8.518441677093506,
1131
+ "rl_secs_used": 678.7271831035614
1132
+ },
1133
+ {
1134
+ "round": 33,
1135
+ "level": 1,
1136
+ "k": 8,
1137
+ "n_targets": 1500,
1138
+ "sample_acc": 0.81275,
1139
+ "new_proofs": 178,
1140
+ "targets_solved": 1252,
1141
+ "lstar": 11,
1142
+ "ge": {
1143
+ "7": 1252,
1144
+ "8": 699,
1145
+ "9": 387,
1146
+ "10": 98,
1147
+ "11": 17,
1148
+ "12": 4,
1149
+ "13": 1,
1150
+ "14": 0
1151
+ },
1152
+ "mix_rl_records": 2388,
1153
+ "sampling": {
1154
+ "samples": 12000,
1155
+ "secs": 63.13167691230774,
1156
+ "gate_secs": 31.262810230255127,
1157
+ "samples_per_sec": 190.07890471004674
1158
+ },
1159
+ "secs": 101.95304489135742,
1160
+ "rl_secs_used": 749.4176428318024
1161
+ },
1162
+ {
1163
+ "round": 34,
1164
+ "level": 2,
1165
+ "k": 24,
1166
+ "n_targets": 83,
1167
+ "sample_acc": 0.001004016064257028,
1168
+ "new_proofs": 2,
1169
+ "targets_solved": 1254,
1170
+ "lstar": 11,
1171
+ "ge": {
1172
+ "7": 1254,
1173
+ "8": 700,
1174
+ "9": 388,
1175
+ "10": 99,
1176
+ "11": 18,
1177
+ "12": 4,
1178
+ "13": 1,
1179
+ "14": 0
1180
+ },
1181
+ "mix_rl_records": 2,
1182
+ "sampling": {
1183
+ "samples": 1992,
1184
+ "secs": 11.663103103637695,
1185
+ "gate_secs": 4.423825025558472,
1186
+ "samples_per_sec": 170.79502618635857
1187
+ },
1188
+ "secs": 11.728440046310425,
1189
+ "rl_secs_used": 756.7220275402069
1190
+ },
1191
+ {
1192
+ "round": 35,
1193
+ "level": 4,
1194
+ "k": 24,
1195
+ "n_targets": 83,
1196
+ "sample_acc": 0.0015060240963855422,
1197
+ "new_proofs": 3,
1198
+ "targets_solved": 1257,
1199
+ "lstar": 11,
1200
+ "ge": {
1201
+ "7": 1257,
1202
+ "8": 702,
1203
+ "9": 389,
1204
+ "10": 99,
1205
+ "11": 18,
1206
+ "12": 4,
1207
+ "13": 1,
1208
+ "14": 0
1209
+ },
1210
+ "mix_rl_records": 3,
1211
+ "sampling": {
1212
+ "samples": 1992,
1213
+ "secs": 11.743556261062622,
1214
+ "gate_secs": 4.1855762004852295,
1215
+ "samples_per_sec": 169.6249377715974
1216
+ },
1217
+ "secs": 11.801961898803711,
1218
+ "rl_secs_used": 764.3384103775024
1219
+ },
1220
+ {
1221
+ "round": 36,
1222
+ "level": 8,
1223
+ "k": 24,
1224
+ "n_targets": 82,
1225
+ "sample_acc": 0.0010162601626016261,
1226
+ "new_proofs": 1,
1227
+ "targets_solved": 1258,
1228
+ "lstar": 11,
1229
+ "ge": {
1230
+ "7": 1258,
1231
+ "8": 702,
1232
+ "9": 389,
1233
+ "10": 99,
1234
+ "11": 18,
1235
+ "12": 4,
1236
+ "13": 1,
1237
+ "14": 0
1238
+ },
1239
+ "mix_rl_records": 1,
1240
+ "sampling": {
1241
+ "samples": 1968,
1242
+ "secs": 10.082074642181396,
1243
+ "gate_secs": 4.430318832397461,
1244
+ "samples_per_sec": 195.19792005568766
1245
+ },
1246
+ "secs": 10.144181251525879,
1247
+ "rl_secs_used": 770.0522747039795
1248
+ },
1249
+ {
1250
+ "round": 37,
1251
+ "level": 1,
1252
+ "k": 8,
1253
+ "n_targets": 1500,
1254
+ "sample_acc": 0.8143333333333334,
1255
+ "new_proofs": 187,
1256
+ "targets_solved": 1262,
1257
+ "lstar": 11,
1258
+ "ge": {
1259
+ "7": 1262,
1260
+ "8": 705,
1261
+ "9": 392,
1262
+ "10": 101,
1263
+ "11": 18,
1264
+ "12": 4,
1265
+ "13": 1,
1266
+ "14": 0
1267
+ },
1268
+ "mix_rl_records": 2476,
1269
+ "sampling": {
1270
+ "samples": 12000,
1271
+ "secs": 62.811208963394165,
1272
+ "gate_secs": 32.62139940261841,
1273
+ "samples_per_sec": 191.048702899406
1274
+ },
1275
+ "secs": 102.03536176681519,
1276
+ "rl_secs_used": 839.4664659500122
1277
+ },
1278
+ {
1279
+ "round": 38,
1280
+ "level": 2,
1281
+ "k": 24,
1282
+ "n_targets": 80,
1283
+ "sample_acc": 0.0005208333333333333,
1284
+ "new_proofs": 1,
1285
+ "targets_solved": 1263,
1286
+ "lstar": 11,
1287
+ "ge": {
1288
+ "7": 1263,
1289
+ "8": 706,
1290
+ "9": 393,
1291
+ "10": 102,
1292
+ "11": 18,
1293
+ "12": 4,
1294
+ "13": 1,
1295
+ "14": 0
1296
+ },
1297
+ "mix_rl_records": 1,
1298
+ "sampling": {
1299
+ "samples": 1920,
1300
+ "secs": 9.2752845287323,
1301
+ "gate_secs": 4.018190622329712,
1302
+ "samples_per_sec": 207.0017360709921
1303
+ },
1304
+ "secs": 9.324950695037842,
1305
+ "rl_secs_used": 844.77299284935
1306
+ },
1307
+ {
1308
+ "round": 39,
1309
+ "level": 4,
1310
+ "k": 24,
1311
+ "n_targets": 79,
1312
+ "sample_acc": 0.0,
1313
+ "new_proofs": 0,
1314
+ "targets_solved": 1263,
1315
+ "lstar": 11,
1316
+ "ge": {
1317
+ "7": 1263,
1318
+ "8": 706,
1319
+ "9": 393,
1320
+ "10": 102,
1321
+ "11": 18,
1322
+ "12": 4,
1323
+ "13": 1,
1324
+ "14": 0
1325
+ },
1326
+ "mix_rl_records": 0,
1327
+ "sampling": {
1328
+ "samples": 1896,
1329
+ "secs": 11.218536615371704,
1330
+ "gate_secs": 4.591203212738037,
1331
+ "samples_per_sec": 169.00600006974972
1332
+ },
1333
+ "secs": 11.278528690338135,
1334
+ "rl_secs_used": 851.4603216648102
1335
+ },
1336
+ {
1337
+ "round": 40,
1338
+ "level": 8,
1339
+ "k": 24,
1340
+ "n_targets": 79,
1341
+ "sample_acc": 0.0005274261603375527,
1342
+ "new_proofs": 1,
1343
+ "targets_solved": 1264,
1344
+ "lstar": 11,
1345
+ "ge": {
1346
+ "7": 1264,
1347
+ "8": 707,
1348
+ "9": 394,
1349
+ "10": 102,
1350
+ "11": 18,
1351
+ "12": 4,
1352
+ "13": 1,
1353
+ "14": 0
1354
+ },
1355
+ "mix_rl_records": 1,
1356
+ "sampling": {
1357
+ "samples": 1896,
1358
+ "secs": 8.67183804512024,
1359
+ "gate_secs": 4.397043466567993,
1360
+ "samples_per_sec": 218.6387695589985
1361
+ },
1362
+ "secs": 8.72948169708252,
1363
+ "rl_secs_used": 855.7927589416504
1364
+ },
1365
+ {
1366
+ "round": 41,
1367
+ "level": 1,
1368
+ "k": 8,
1369
+ "n_targets": 1500,
1370
+ "sample_acc": 0.8198333333333333,
1371
+ "new_proofs": 160,
1372
+ "targets_solved": 1268,
1373
+ "lstar": 11,
1374
+ "ge": {
1375
+ "7": 1268,
1376
+ "8": 710,
1377
+ "9": 397,
1378
+ "10": 102,
1379
+ "11": 18,
1380
+ "12": 4,
1381
+ "13": 1,
1382
+ "14": 0
1383
+ },
1384
+ "mix_rl_records": 2583,
1385
+ "sampling": {
1386
+ "samples": 12000,
1387
+ "secs": 54.82120609283447,
1388
+ "gate_secs": 31.191574335098267,
1389
+ "samples_per_sec": 218.89339646557843
1390
+ },
1391
+ "secs": 93.82205104827881,
1392
+ "rl_secs_used": 918.4234540462494
1393
+ },
1394
+ {
1395
+ "round": 42,
1396
+ "level": 2,
1397
+ "k": 24,
1398
+ "n_targets": 78,
1399
+ "sample_acc": 0.0,
1400
+ "new_proofs": 0,
1401
+ "targets_solved": 1268,
1402
+ "lstar": 11,
1403
+ "ge": {
1404
+ "7": 1268,
1405
+ "8": 710,
1406
+ "9": 397,
1407
+ "10": 102,
1408
+ "11": 18,
1409
+ "12": 4,
1410
+ "13": 1,
1411
+ "14": 0
1412
+ },
1413
+ "mix_rl_records": 0,
1414
+ "sampling": {
1415
+ "samples": 1872,
1416
+ "secs": 8.76870059967041,
1417
+ "gate_secs": 4.21483588218689,
1418
+ "samples_per_sec": 213.48659116840676
1419
+ },
1420
+ "secs": 8.828282356262207,
1421
+ "rl_secs_used": 923.036682844162
1422
+ },
1423
+ {
1424
+ "round": 43,
1425
+ "level": 4,
1426
+ "k": 24,
1427
+ "n_targets": 77,
1428
+ "sample_acc": 0.0,
1429
+ "new_proofs": 0,
1430
+ "targets_solved": 1268,
1431
+ "lstar": 11,
1432
+ "ge": {
1433
+ "7": 1268,
1434
+ "8": 710,
1435
+ "9": 397,
1436
+ "10": 102,
1437
+ "11": 18,
1438
+ "12": 4,
1439
+ "13": 1,
1440
+ "14": 0
1441
+ },
1442
+ "mix_rl_records": 0,
1443
+ "sampling": {
1444
+ "samples": 1848,
1445
+ "secs": 10.38786244392395,
1446
+ "gate_secs": 4.362722873687744,
1447
+ "samples_per_sec": 177.8999298436926
1448
+ },
1449
+ "secs": 10.475448369979858,
1450
+ "rl_secs_used": 929.1494071483612
1451
+ },
1452
+ {
1453
+ "round": 44,
1454
+ "level": 8,
1455
+ "k": 24,
1456
+ "n_targets": 77,
1457
+ "sample_acc": 0.0010822510822510823,
1458
+ "new_proofs": 1,
1459
+ "targets_solved": 1269,
1460
+ "lstar": 11,
1461
+ "ge": {
1462
+ "7": 1269,
1463
+ "8": 711,
1464
+ "9": 398,
1465
+ "10": 102,
1466
+ "11": 18,
1467
+ "12": 4,
1468
+ "13": 1,
1469
+ "14": 0
1470
+ },
1471
+ "mix_rl_records": 1,
1472
+ "sampling": {
1473
+ "samples": 1848,
1474
+ "secs": 9.193273305892944,
1475
+ "gate_secs": 4.234066724777222,
1476
+ "samples_per_sec": 201.01654095450647
1477
+ },
1478
+ "secs": 9.250768899917603,
1479
+ "rl_secs_used": 934.1661093235016
1480
+ },
1481
+ {
1482
+ "round": 45,
1483
+ "level": 1,
1484
+ "k": 8,
1485
+ "n_targets": 1500,
1486
+ "sample_acc": 0.8270833333333333,
1487
+ "new_proofs": 156,
1488
+ "targets_solved": 1274,
1489
+ "lstar": 11,
1490
+ "ge": {
1491
+ "7": 1274,
1492
+ "8": 714,
1493
+ "9": 401,
1494
+ "10": 103,
1495
+ "11": 18,
1496
+ "12": 4,
1497
+ "13": 1,
1498
+ "14": 0
1499
+ },
1500
+ "mix_rl_records": 2685,
1501
+ "sampling": {
1502
+ "samples": 12000,
1503
+ "secs": 65.98701977729797,
1504
+ "gate_secs": 31.51468300819397,
1505
+ "samples_per_sec": 181.8539470413915
1506
+ },
1507
+ "secs": 106.23185753822327,
1508
+ "rl_secs_used": 1008.8835937976837
1509
+ },
1510
+ {
1511
+ "round": 46,
1512
+ "level": 2,
1513
+ "k": 24,
1514
+ "n_targets": 76,
1515
+ "sample_acc": 0.0010964912280701754,
1516
+ "new_proofs": 1,
1517
+ "targets_solved": 1275,
1518
+ "lstar": 11,
1519
+ "ge": {
1520
+ "7": 1275,
1521
+ "8": 715,
1522
+ "9": 402,
1523
+ "10": 103,
1524
+ "11": 18,
1525
+ "12": 4,
1526
+ "13": 1,
1527
+ "14": 0
1528
+ },
1529
+ "mix_rl_records": 1,
1530
+ "sampling": {
1531
+ "samples": 1824,
1532
+ "secs": 11.321188926696777,
1533
+ "gate_secs": 4.16279935836792,
1534
+ "samples_per_sec": 161.11382044855557
1535
+ },
1536
+ "secs": 11.381211519241333,
1537
+ "rl_secs_used": 1016.1017034053802
1538
+ },
1539
+ {
1540
+ "round": 47,
1541
+ "level": 4,
1542
+ "k": 24,
1543
+ "n_targets": 75,
1544
+ "sample_acc": 0.0005555555555555556,
1545
+ "new_proofs": 1,
1546
+ "targets_solved": 1276,
1547
+ "lstar": 11,
1548
+ "ge": {
1549
+ "7": 1276,
1550
+ "8": 716,
1551
+ "9": 403,
1552
+ "10": 104,
1553
+ "11": 18,
1554
+ "12": 4,
1555
+ "13": 1,
1556
+ "14": 0
1557
+ },
1558
+ "mix_rl_records": 1,
1559
+ "sampling": {
1560
+ "samples": 1800,
1561
+ "secs": 11.093557357788086,
1562
+ "gate_secs": 3.53816556930542,
1563
+ "samples_per_sec": 162.2563386970126
1564
+ },
1565
+ "secs": 11.145751953125,
1566
+ "rl_secs_used": 1023.709291934967
1567
+ },
1568
+ {
1569
+ "round": 48,
1570
+ "level": 8,
1571
+ "k": 24,
1572
+ "n_targets": 75,
1573
+ "sample_acc": 0.002777777777777778,
1574
+ "new_proofs": 4,
1575
+ "targets_solved": 1279,
1576
+ "lstar": 12,
1577
+ "ge": {
1578
+ "7": 1279,
1579
+ "8": 719,
1580
+ "9": 406,
1581
+ "10": 105,
1582
+ "11": 19,
1583
+ "12": 5,
1584
+ "13": 2,
1585
+ "14": 1
1586
+ },
1587
+ "mix_rl_records": 4,
1588
+ "sampling": {
1589
+ "samples": 1800,
1590
+ "secs": 11.368651866912842,
1591
+ "gate_secs": 3.6738171577453613,
1592
+ "samples_per_sec": 158.3301187398212
1593
+ },
1594
+ "secs": 11.425716638565063,
1595
+ "rl_secs_used": 1031.4611945152283
1596
+ },
1597
+ {
1598
+ "round": 49,
1599
+ "level": 1,
1600
+ "k": 8,
1601
+ "n_targets": 1500,
1602
+ "sample_acc": 0.83475,
1603
+ "new_proofs": 154,
1604
+ "targets_solved": 1280,
1605
+ "lstar": 12,
1606
+ "ge": {
1607
+ "7": 1280,
1608
+ "8": 720,
1609
+ "9": 407,
1610
+ "10": 105,
1611
+ "11": 19,
1612
+ "12": 5,
1613
+ "13": 2,
1614
+ "14": 1
1615
+ },
1616
+ "mix_rl_records": 2763,
1617
+ "sampling": {
1618
+ "samples": 12000,
1619
+ "secs": 60.34134888648987,
1620
+ "gate_secs": 31.659582138061523,
1621
+ "samples_per_sec": 198.86860704048235
1622
+ },
1623
+ "secs": 100.36369943618774,
1624
+ "rl_secs_used": 1100.165493965149
1625
+ },
1626
+ {
1627
+ "round": 50,
1628
+ "level": 2,
1629
+ "k": 24,
1630
+ "n_targets": 74,
1631
+ "sample_acc": 0.0,
1632
+ "new_proofs": 0,
1633
+ "targets_solved": 1280,
1634
+ "lstar": 12,
1635
+ "ge": {
1636
+ "7": 1280,
1637
+ "8": 720,
1638
+ "9": 407,
1639
+ "10": 105,
1640
+ "11": 19,
1641
+ "12": 5,
1642
+ "13": 2,
1643
+ "14": 1
1644
+ },
1645
+ "mix_rl_records": 0,
1646
+ "sampling": {
1647
+ "samples": 1776,
1648
+ "secs": 10.07914924621582,
1649
+ "gate_secs": 3.9172725677490234,
1650
+ "samples_per_sec": 176.20534795303212
1651
+ },
1652
+ "secs": 10.135313510894775,
1653
+ "rl_secs_used": 1106.383339881897
1654
+ },
1655
+ {
1656
+ "round": 51,
1657
+ "level": 4,
1658
+ "k": 24,
1659
+ "n_targets": 74,
1660
+ "sample_acc": 0.0005630630630630631,
1661
+ "new_proofs": 0,
1662
+ "targets_solved": 1280,
1663
+ "lstar": 12,
1664
+ "ge": {
1665
+ "7": 1280,
1666
+ "8": 720,
1667
+ "9": 407,
1668
+ "10": 105,
1669
+ "11": 19,
1670
+ "12": 5,
1671
+ "13": 2,
1672
+ "14": 1
1673
+ },
1674
+ "mix_rl_records": 1,
1675
+ "sampling": {
1676
+ "samples": 1776,
1677
+ "secs": 9.937037944793701,
1678
+ "gate_secs": 3.6553399562835693,
1679
+ "samples_per_sec": 178.72529116490867
1680
+ },
1681
+ "secs": 9.990573644638062,
1682
+ "rl_secs_used": 1112.7185723781586
1683
+ },
1684
+ {
1685
+ "round": 52,
1686
+ "level": 8,
1687
+ "k": 24,
1688
+ "n_targets": 73,
1689
+ "sample_acc": 0.0,
1690
+ "new_proofs": 0,
1691
+ "targets_solved": 1280,
1692
+ "lstar": 12,
1693
+ "ge": {
1694
+ "7": 1280,
1695
+ "8": 720,
1696
+ "9": 407,
1697
+ "10": 105,
1698
+ "11": 19,
1699
+ "12": 5,
1700
+ "13": 2,
1701
+ "14": 1
1702
+ },
1703
+ "mix_rl_records": 0,
1704
+ "sampling": {
1705
+ "samples": 1752,
1706
+ "secs": 9.329228162765503,
1707
+ "gate_secs": 3.5495150089263916,
1708
+ "samples_per_sec": 187.79688624108505
1709
+ },
1710
+ "secs": 9.381494760513306,
1711
+ "rl_secs_used": 1118.5505533218384
1712
+ },
1713
+ {
1714
+ "round": 53,
1715
+ "level": 1,
1716
+ "k": 8,
1717
+ "n_targets": 1500,
1718
+ "sample_acc": 0.834,
1719
+ "new_proofs": 144,
1720
+ "targets_solved": 1281,
1721
+ "lstar": 12,
1722
+ "ge": {
1723
+ "7": 1281,
1724
+ "8": 721,
1725
+ "9": 408,
1726
+ "10": 105,
1727
+ "11": 19,
1728
+ "12": 5,
1729
+ "13": 2,
1730
+ "14": 1
1731
+ },
1732
+ "mix_rl_records": 2851,
1733
+ "sampling": {
1734
+ "samples": 12000,
1735
+ "secs": 60.05979084968567,
1736
+ "gate_secs": 31.853074073791504,
1737
+ "samples_per_sec": 199.80089557809046
1738
+ },
1739
+ "secs": 100.43740677833557,
1740
+ "rl_secs_used": 1187.1351177692413
1741
+ },
1742
+ {
1743
+ "round": 54,
1744
+ "level": 2,
1745
+ "k": 24,
1746
+ "n_targets": 73,
1747
+ "sample_acc": 0.0028538812785388126,
1748
+ "new_proofs": 3,
1749
+ "targets_solved": 1283,
1750
+ "lstar": 12,
1751
+ "ge": {
1752
+ "7": 1283,
1753
+ "8": 723,
1754
+ "9": 410,
1755
+ "10": 107,
1756
+ "11": 21,
1757
+ "12": 6,
1758
+ "13": 2,
1759
+ "14": 1
1760
+ },
1761
+ "mix_rl_records": 3,
1762
+ "sampling": {
1763
+ "samples": 1752,
1764
+ "secs": 8.604615926742554,
1765
+ "gate_secs": 3.7914228439331055,
1766
+ "samples_per_sec": 203.61164460053408
1767
+ },
1768
+ "secs": 8.660674571990967,
1769
+ "rl_secs_used": 1192.0041372776031
1770
+ },
1771
+ {
1772
+ "round": 55,
1773
+ "level": 4,
1774
+ "k": 24,
1775
+ "n_targets": 73,
1776
+ "sample_acc": 0.0,
1777
+ "new_proofs": 0,
1778
+ "targets_solved": 1283,
1779
+ "lstar": 12,
1780
+ "ge": {
1781
+ "7": 1283,
1782
+ "8": 723,
1783
+ "9": 410,
1784
+ "10": 107,
1785
+ "11": 21,
1786
+ "12": 6,
1787
+ "13": 2,
1788
+ "14": 1
1789
+ },
1790
+ "mix_rl_records": 0,
1791
+ "sampling": {
1792
+ "samples": 1752,
1793
+ "secs": 9.214615821838379,
1794
+ "gate_secs": 3.755915403366089,
1795
+ "samples_per_sec": 190.13272325991167
1796
+ },
1797
+ "secs": 9.269420862197876,
1798
+ "rl_secs_used": 1197.517643213272
1799
+ },
1800
+ {
1801
+ "round": 56,
1802
+ "level": 8,
1803
+ "k": 24,
1804
+ "n_targets": 73,
1805
+ "sample_acc": 0.0,
1806
+ "new_proofs": 0,
1807
+ "targets_solved": 1283,
1808
+ "lstar": 12,
1809
+ "ge": {
1810
+ "7": 1283,
1811
+ "8": 723,
1812
+ "9": 410,
1813
+ "10": 107,
1814
+ "11": 21,
1815
+ "12": 6,
1816
+ "13": 2,
1817
+ "14": 1
1818
+ },
1819
+ "mix_rl_records": 0,
1820
+ "sampling": {
1821
+ "samples": 1752,
1822
+ "secs": 11.32279634475708,
1823
+ "gate_secs": 3.8714818954467773,
1824
+ "samples_per_sec": 154.73209502803147
1825
+ },
1826
+ "secs": 11.373117923736572,
1827
+ "rl_secs_used": 1205.0192773342133
1828
+ },
1829
+ {
1830
+ "round": 57,
1831
+ "level": 1,
1832
+ "k": 8,
1833
+ "n_targets": 1500,
1834
+ "sample_acc": 0.835,
1835
+ "new_proofs": 160,
1836
+ "targets_solved": 1283,
1837
+ "lstar": 12,
1838
+ "ge": {
1839
+ "7": 1283,
1840
+ "8": 723,
1841
+ "9": 410,
1842
+ "10": 107,
1843
+ "11": 21,
1844
+ "12": 6,
1845
+ "13": 2,
1846
+ "14": 1
1847
+ },
1848
+ "mix_rl_records": 2959,
1849
+ "sampling": {
1850
+ "samples": 12000,
1851
+ "secs": 60.068341970443726,
1852
+ "gate_secs": 31.06937003135681,
1853
+ "samples_per_sec": 199.77245261579768
1854
+ },
1855
+ "secs": 99.95769906044006,
1856
+ "rl_secs_used": 1273.9078090190887
1857
+ },
1858
+ {
1859
+ "round": 58,
1860
+ "level": 2,
1861
+ "k": 24,
1862
+ "n_targets": 73,
1863
+ "sample_acc": 0.0,
1864
+ "new_proofs": 0,
1865
+ "targets_solved": 1283,
1866
+ "lstar": 12,
1867
+ "ge": {
1868
+ "7": 1283,
1869
+ "8": 723,
1870
+ "9": 410,
1871
+ "10": 107,
1872
+ "11": 21,
1873
+ "12": 6,
1874
+ "13": 2,
1875
+ "14": 1
1876
+ },
1877
+ "mix_rl_records": 0,
1878
+ "sampling": {
1879
+ "samples": 1752,
1880
+ "secs": 8.726410388946533,
1881
+ "gate_secs": 3.1583523750305176,
1882
+ "samples_per_sec": 200.76983798735878
1883
+ },
1884
+ "secs": 8.780374526977539,
1885
+ "rl_secs_used": 1279.5296349525452
1886
+ },
1887
+ {
1888
+ "round": 59,
1889
+ "level": 4,
1890
+ "k": 24,
1891
+ "n_targets": 72,
1892
+ "sample_acc": 0.0005787037037037037,
1893
+ "new_proofs": 1,
1894
+ "targets_solved": 1284,
1895
+ "lstar": 12,
1896
+ "ge": {
1897
+ "7": 1284,
1898
+ "8": 724,
1899
+ "9": 411,
1900
+ "10": 107,
1901
+ "11": 21,
1902
+ "12": 6,
1903
+ "13": 2,
1904
+ "14": 1
1905
+ },
1906
+ "mix_rl_records": 1,
1907
+ "sampling": {
1908
+ "samples": 1728,
1909
+ "secs": 9.223639726638794,
1910
+ "gate_secs": 3.273508071899414,
1911
+ "samples_per_sec": 187.34469810321875
1912
+ },
1913
+ "secs": 9.27296757698059,
1914
+ "rl_secs_used": 1285.529099702835
1915
+ },
1916
+ {
1917
+ "round": 60,
1918
+ "level": 8,
1919
+ "k": 24,
1920
+ "n_targets": 72,
1921
+ "sample_acc": 0.0005787037037037037,
1922
+ "new_proofs": 1,
1923
+ "targets_solved": 1285,
1924
+ "lstar": 12,
1925
+ "ge": {
1926
+ "7": 1285,
1927
+ "8": 724,
1928
+ "9": 411,
1929
+ "10": 107,
1930
+ "11": 21,
1931
+ "12": 6,
1932
+ "13": 2,
1933
+ "14": 1
1934
+ },
1935
+ "mix_rl_records": 1,
1936
+ "sampling": {
1937
+ "samples": 1728,
1938
+ "secs": 7.401629686355591,
1939
+ "gate_secs": 3.2550129890441895,
1940
+ "samples_per_sec": 233.46209864909244
1941
+ },
1942
+ "secs": 7.45253586769104,
1943
+ "rl_secs_used": 1289.7266209125519
1944
+ },
1945
+ {
1946
+ "round": 61,
1947
+ "level": 1,
1948
+ "k": 8,
1949
+ "n_targets": 1500,
1950
+ "sample_acc": 0.8341666666666666,
1951
+ "new_proofs": 205,
1952
+ "targets_solved": 1286,
1953
+ "lstar": 12,
1954
+ "ge": {
1955
+ "7": 1286,
1956
+ "8": 725,
1957
+ "9": 412,
1958
+ "10": 108,
1959
+ "11": 22,
1960
+ "12": 6,
1961
+ "13": 2,
1962
+ "14": 1
1963
+ },
1964
+ "mix_rl_records": 3023,
1965
+ "sampling": {
1966
+ "samples": 12000,
1967
+ "secs": 61.02544951438904,
1968
+ "gate_secs": 31.309438228607178,
1969
+ "samples_per_sec": 196.63927255743604
1970
+ },
1971
+ "secs": 101.87276816368103,
1972
+ "rl_secs_used": 1360.2902128696442
1973
+ },
1974
+ {
1975
+ "round": 62,
1976
+ "level": 2,
1977
+ "k": 24,
1978
+ "n_targets": 72,
1979
+ "sample_acc": 0.0028935185185185184,
1980
+ "new_proofs": 3,
1981
+ "targets_solved": 1289,
1982
+ "lstar": 12,
1983
+ "ge": {
1984
+ "7": 1289,
1985
+ "8": 726,
1986
+ "9": 413,
1987
+ "10": 109,
1988
+ "11": 22,
1989
+ "12": 6,
1990
+ "13": 2,
1991
+ "14": 1
1992
+ },
1993
+ "mix_rl_records": 3,
1994
+ "sampling": {
1995
+ "samples": 1728,
1996
+ "secs": 10.04113483428955,
1997
+ "gate_secs": 3.8572194576263428,
1998
+ "samples_per_sec": 172.09210199020922
1999
+ },
2000
+ "secs": 10.098647356033325,
2001
+ "rl_secs_used": 1366.5313792228699
2002
+ },
2003
+ {
2004
+ "round": 63,
2005
+ "level": 4,
2006
+ "k": 24,
2007
+ "n_targets": 71,
2008
+ "sample_acc": 0.0,
2009
+ "new_proofs": 0,
2010
+ "targets_solved": 1289,
2011
+ "lstar": 12,
2012
+ "ge": {
2013
+ "7": 1289,
2014
+ "8": 726,
2015
+ "9": 413,
2016
+ "10": 109,
2017
+ "11": 22,
2018
+ "12": 6,
2019
+ "13": 2,
2020
+ "14": 1
2021
+ },
2022
+ "mix_rl_records": 0,
2023
+ "sampling": {
2024
+ "samples": 1704,
2025
+ "secs": 10.859347820281982,
2026
+ "gate_secs": 3.6754863262176514,
2027
+ "samples_per_sec": 156.91550065441706
2028
+ },
2029
+ "secs": 10.912447690963745,
2030
+ "rl_secs_used": 1373.768336057663
2031
+ },
2032
+ {
2033
+ "round": 64,
2034
+ "level": 8,
2035
+ "k": 24,
2036
+ "n_targets": 71,
2037
+ "sample_acc": 0.0005868544600938967,
2038
+ "new_proofs": 1,
2039
+ "targets_solved": 1290,
2040
+ "lstar": 12,
2041
+ "ge": {
2042
+ "7": 1290,
2043
+ "8": 726,
2044
+ "9": 413,
2045
+ "10": 109,
2046
+ "11": 22,
2047
+ "12": 6,
2048
+ "13": 2,
2049
+ "14": 1
2050
+ },
2051
+ "mix_rl_records": 1,
2052
+ "sampling": {
2053
+ "samples": 1704,
2054
+ "secs": 8.173127889633179,
2055
+ "gate_secs": 3.5396480560302734,
2056
+ "samples_per_sec": 208.48811165201013
2057
+ },
2058
+ "secs": 8.24605417251587,
2059
+ "rl_secs_used": 1378.4747486114502
2060
+ },
2061
+ {
2062
+ "round": 65,
2063
+ "level": 1,
2064
+ "k": 8,
2065
+ "n_targets": 1500,
2066
+ "sample_acc": 0.8424166666666667,
2067
+ "new_proofs": 155,
2068
+ "targets_solved": 1294,
2069
+ "lstar": 12,
2070
+ "ge": {
2071
+ "7": 1294,
2072
+ "8": 726,
2073
+ "9": 413,
2074
+ "10": 109,
2075
+ "11": 22,
2076
+ "12": 6,
2077
+ "13": 2,
2078
+ "14": 1
2079
+ },
2080
+ "mix_rl_records": 3139,
2081
+ "sampling": {
2082
+ "samples": 12000,
2083
+ "secs": 86.09665894508362,
2084
+ "gate_secs": 56.91737747192383,
2085
+ "samples_per_sec": 139.3782307819186
2086
+ },
2087
+ "secs": 127.23849725723267,
2088
+ "rl_secs_used": 1448.7961819171906
2089
+ },
2090
+ {
2091
+ "round": 66,
2092
+ "level": 2,
2093
+ "k": 24,
2094
+ "n_targets": 70,
2095
+ "sample_acc": 0.002976190476190476,
2096
+ "new_proofs": 2,
2097
+ "targets_solved": 1296,
2098
+ "lstar": 12,
2099
+ "ge": {
2100
+ "7": 1296,
2101
+ "8": 728,
2102
+ "9": 414,
2103
+ "10": 109,
2104
+ "11": 22,
2105
+ "12": 6,
2106
+ "13": 2,
2107
+ "14": 1
2108
+ },
2109
+ "mix_rl_records": 3,
2110
+ "sampling": {
2111
+ "samples": 1680,
2112
+ "secs": 7.904250383377075,
2113
+ "gate_secs": 3.1736927032470703,
2114
+ "samples_per_sec": 212.5438743100928
2115
+ },
2116
+ "secs": 7.951009511947632,
2117
+ "rl_secs_used": 1453.573173046112
2118
+ },
2119
+ {
2120
+ "round": 67,
2121
+ "level": 4,
2122
+ "k": 24,
2123
+ "n_targets": 69,
2124
+ "sample_acc": 0.0030193236714975845,
2125
+ "new_proofs": 1,
2126
+ "targets_solved": 1297,
2127
+ "lstar": 12,
2128
+ "ge": {
2129
+ "7": 1297,
2130
+ "8": 728,
2131
+ "9": 414,
2132
+ "10": 109,
2133
+ "11": 22,
2134
+ "12": 6,
2135
+ "13": 2,
2136
+ "14": 1
2137
+ },
2138
+ "mix_rl_records": 1,
2139
+ "sampling": {
2140
+ "samples": 1656,
2141
+ "secs": 8.08107590675354,
2142
+ "gate_secs": 3.420581579208374,
2143
+ "samples_per_sec": 204.92320813569427
2144
+ },
2145
+ "secs": 8.128207683563232,
2146
+ "rl_secs_used": 1458.2807984352112
2147
+ },
2148
+ {
2149
+ "round": 68,
2150
+ "level": 8,
2151
+ "k": 24,
2152
+ "n_targets": 69,
2153
+ "sample_acc": 0.0036231884057971015,
2154
+ "new_proofs": 1,
2155
+ "targets_solved": 1298,
2156
+ "lstar": 12,
2157
+ "ge": {
2158
+ "7": 1298,
2159
+ "8": 728,
2160
+ "9": 414,
2161
+ "10": 109,
2162
+ "11": 22,
2163
+ "12": 6,
2164
+ "13": 2,
2165
+ "14": 1
2166
+ },
2167
+ "mix_rl_records": 1,
2168
+ "sampling": {
2169
+ "samples": 1656,
2170
+ "secs": 8.900831460952759,
2171
+ "gate_secs": 3.056527853012085,
2172
+ "samples_per_sec": 186.05003445630226
2173
+ },
2174
+ "secs": 8.939305305480957,
2175
+ "rl_secs_used": 1464.1635749340057
2176
+ },
2177
+ {
2178
+ "round": 69,
2179
+ "level": 1,
2180
+ "k": 8,
2181
+ "n_targets": 1500,
2182
+ "sample_acc": 0.8468333333333333,
2183
+ "new_proofs": 178,
2184
+ "targets_solved": 1302,
2185
+ "lstar": 12,
2186
+ "ge": {
2187
+ "7": 1302,
2188
+ "8": 728,
2189
+ "9": 414,
2190
+ "10": 109,
2191
+ "11": 22,
2192
+ "12": 6,
2193
+ "13": 2,
2194
+ "14": 1
2195
+ },
2196
+ "mix_rl_records": 3220,
2197
+ "sampling": {
2198
+ "samples": 12000,
2199
+ "secs": 72.287841796875,
2200
+ "gate_secs": 44.42894697189331,
2201
+ "samples_per_sec": 166.00301934202662
2202
+ },
2203
+ "secs": 113.25818109512329,
2204
+ "rl_secs_used": 1532.99302816391
2205
+ },
2206
+ {
2207
+ "round": 70,
2208
+ "level": 2,
2209
+ "k": 24,
2210
+ "n_targets": 66,
2211
+ "sample_acc": 0.0,
2212
+ "new_proofs": 0,
2213
+ "targets_solved": 1302,
2214
+ "lstar": 12,
2215
+ "ge": {
2216
+ "7": 1302,
2217
+ "8": 728,
2218
+ "9": 414,
2219
+ "10": 109,
2220
+ "11": 22,
2221
+ "12": 6,
2222
+ "13": 2,
2223
+ "14": 1
2224
+ },
2225
+ "mix_rl_records": 0,
2226
+ "sampling": {
2227
+ "samples": 1584,
2228
+ "secs": 7.666700601577759,
2229
+ "gate_secs": 3.102468252182007,
2230
+ "samples_per_sec": 206.6077811456499
2231
+ },
2232
+ "secs": 7.7166478633880615,
2233
+ "rl_secs_used": 1537.606997013092
2234
+ },
2235
+ {
2236
+ "round": 71,
2237
+ "level": 4,
2238
+ "k": 24,
2239
+ "n_targets": 66,
2240
+ "sample_acc": 0.0,
2241
+ "new_proofs": 0,
2242
+ "targets_solved": 1302,
2243
+ "lstar": 12,
2244
+ "ge": {
2245
+ "7": 1302,
2246
+ "8": 728,
2247
+ "9": 414,
2248
+ "10": 109,
2249
+ "11": 22,
2250
+ "12": 6,
2251
+ "13": 2,
2252
+ "14": 1
2253
+ },
2254
+ "mix_rl_records": 0,
2255
+ "sampling": {
2256
+ "samples": 1584,
2257
+ "secs": 7.162403106689453,
2258
+ "gate_secs": 2.8469550609588623,
2259
+ "samples_per_sec": 221.15482421264383
2260
+ },
2261
+ "secs": 7.2067930698394775,
2262
+ "rl_secs_used": 1541.9668273925781
2263
+ },
2264
+ {
2265
+ "round": 72,
2266
+ "level": 8,
2267
+ "k": 24,
2268
+ "n_targets": 66,
2269
+ "sample_acc": 0.0006313131313131314,
2270
+ "new_proofs": 1,
2271
+ "targets_solved": 1303,
2272
+ "lstar": 12,
2273
+ "ge": {
2274
+ "7": 1303,
2275
+ "8": 728,
2276
+ "9": 414,
2277
+ "10": 109,
2278
+ "11": 22,
2279
+ "12": 6,
2280
+ "13": 2,
2281
+ "14": 1
2282
+ },
2283
+ "mix_rl_records": 1,
2284
+ "sampling": {
2285
+ "samples": 1584,
2286
+ "secs": 9.242881059646606,
2287
+ "gate_secs": 3.0354559421539307,
2288
+ "samples_per_sec": 171.3751361483562
2289
+ },
2290
+ "secs": 9.28851842880249,
2291
+ "rl_secs_used": 1548.2198905944824
2292
+ },
2293
+ {
2294
+ "round": 73,
2295
+ "level": 1,
2296
+ "k": 8,
2297
+ "n_targets": 1500,
2298
+ "sample_acc": 0.8461666666666666,
2299
+ "new_proofs": 211,
2300
+ "targets_solved": 1304,
2301
+ "lstar": 12,
2302
+ "ge": {
2303
+ "7": 1304,
2304
+ "8": 729,
2305
+ "9": 415,
2306
+ "10": 110,
2307
+ "11": 23,
2308
+ "12": 7,
2309
+ "13": 2,
2310
+ "14": 1
2311
+ },
2312
+ "mix_rl_records": 3330,
2313
+ "sampling": {
2314
+ "samples": 12000,
2315
+ "secs": 64.04320573806763,
2316
+ "gate_secs": 37.476025342941284,
2317
+ "samples_per_sec": 187.3735060839894
2318
+ },
2319
+ "secs": 105.84708571434021,
2320
+ "rl_secs_used": 1616.5911128520966
2321
+ },
2322
+ {
2323
+ "round": 74,
2324
+ "level": 2,
2325
+ "k": 24,
2326
+ "n_targets": 66,
2327
+ "sample_acc": 0.0006313131313131314,
2328
+ "new_proofs": 1,
2329
+ "targets_solved": 1305,
2330
+ "lstar": 12,
2331
+ "ge": {
2332
+ "7": 1305,
2333
+ "8": 730,
2334
+ "9": 416,
2335
+ "10": 111,
2336
+ "11": 24,
2337
+ "12": 7,
2338
+ "13": 2,
2339
+ "14": 1
2340
+ },
2341
+ "mix_rl_records": 1,
2342
+ "sampling": {
2343
+ "samples": 1584,
2344
+ "secs": 8.173463106155396,
2345
+ "gate_secs": 3.4838368892669678,
2346
+ "samples_per_sec": 193.79790174951634
2347
+ },
2348
+ "secs": 8.2254478931427,
2349
+ "rl_secs_used": 1621.33256316185
2350
+ },
2351
+ {
2352
+ "round": 75,
2353
+ "level": 4,
2354
+ "k": 24,
2355
+ "n_targets": 65,
2356
+ "sample_acc": 0.0,
2357
+ "new_proofs": 0,
2358
+ "targets_solved": 1305,
2359
+ "lstar": 12,
2360
+ "ge": {
2361
+ "7": 1305,
2362
+ "8": 730,
2363
+ "9": 416,
2364
+ "10": 111,
2365
+ "11": 24,
2366
+ "12": 7,
2367
+ "13": 2,
2368
+ "14": 1
2369
+ },
2370
+ "mix_rl_records": 0,
2371
+ "sampling": {
2372
+ "samples": 1560,
2373
+ "secs": 6.802579164505005,
2374
+ "gate_secs": 2.323772430419922,
2375
+ "samples_per_sec": 229.32478436118495
2376
+ },
2377
+ "secs": 6.834012031555176,
2378
+ "rl_secs_used": 1625.8428010940552
2379
+ },
2380
+ {
2381
+ "round": 76,
2382
+ "level": 8,
2383
+ "k": 24,
2384
+ "n_targets": 65,
2385
+ "sample_acc": 0.0,
2386
+ "new_proofs": 0,
2387
+ "targets_solved": 1305,
2388
+ "lstar": 12,
2389
+ "ge": {
2390
+ "7": 1305,
2391
+ "8": 730,
2392
+ "9": 416,
2393
+ "10": 111,
2394
+ "11": 24,
2395
+ "12": 7,
2396
+ "13": 2,
2397
+ "14": 1
2398
+ },
2399
+ "mix_rl_records": 0,
2400
+ "sampling": {
2401
+ "samples": 1560,
2402
+ "secs": 6.4656782150268555,
2403
+ "gate_secs": 3.0491154193878174,
2404
+ "samples_per_sec": 241.2739929392729
2405
+ },
2406
+ "secs": 6.501502990722656,
2407
+ "rl_secs_used": 1629.2951867580414
2408
+ },
2409
+ {
2410
+ "round": 77,
2411
+ "level": 1,
2412
+ "k": 8,
2413
+ "n_targets": 1500,
2414
+ "sample_acc": 0.84775,
2415
+ "new_proofs": 159,
2416
+ "targets_solved": 1305,
2417
+ "lstar": 12,
2418
+ "ge": {
2419
+ "7": 1305,
2420
+ "8": 730,
2421
+ "9": 416,
2422
+ "10": 111,
2423
+ "11": 24,
2424
+ "12": 7,
2425
+ "13": 2,
2426
+ "14": 1
2427
+ },
2428
+ "mix_rl_records": 3341,
2429
+ "sampling": {
2430
+ "samples": 12000,
2431
+ "secs": 63.612189531326294,
2432
+ "gate_secs": 31.400226354599,
2433
+ "samples_per_sec": 188.64309008088003
2434
+ },
2435
+ "secs": 104.60654807090759,
2436
+ "rl_secs_used": 1702.501704454422
2437
+ },
2438
+ {
2439
+ "round": 78,
2440
+ "level": 2,
2441
+ "k": 24,
2442
+ "n_targets": 66,
2443
+ "sample_acc": 0.0006313131313131314,
2444
+ "new_proofs": 1,
2445
+ "targets_solved": 1306,
2446
+ "lstar": 12,
2447
+ "ge": {
2448
+ "7": 1306,
2449
+ "8": 730,
2450
+ "9": 416,
2451
+ "10": 111,
2452
+ "11": 24,
2453
+ "12": 7,
2454
+ "13": 2,
2455
+ "14": 1
2456
+ },
2457
+ "mix_rl_records": 1,
2458
+ "sampling": {
2459
+ "samples": 1584,
2460
+ "secs": 9.80240535736084,
2461
+ "gate_secs": 3.1549150943756104,
2462
+ "samples_per_sec": 161.59299092957218
2463
+ },
2464
+ "secs": 9.850524425506592,
2465
+ "rl_secs_used": 1709.197124004364
2466
+ },
2467
+ {
2468
+ "round": 79,
2469
+ "level": 4,
2470
+ "k": 24,
2471
+ "n_targets": 66,
2472
+ "sample_acc": 0.0012626262626262627,
2473
+ "new_proofs": 2,
2474
+ "targets_solved": 1308,
2475
+ "lstar": 12,
2476
+ "ge": {
2477
+ "7": 1308,
2478
+ "8": 732,
2479
+ "9": 418,
2480
+ "10": 112,
2481
+ "11": 25,
2482
+ "12": 8,
2483
+ "13": 3,
2484
+ "14": 1
2485
+ },
2486
+ "mix_rl_records": 2,
2487
+ "sampling": {
2488
+ "samples": 1584,
2489
+ "secs": 9.975895404815674,
2490
+ "gate_secs": 3.2583279609680176,
2491
+ "samples_per_sec": 158.78273936546628
2492
+ },
2493
+ "secs": 10.023608446121216,
2494
+ "rl_secs_used": 1715.9624004364014
2495
+ },
2496
+ {
2497
+ "round": 80,
2498
+ "level": 8,
2499
+ "k": 24,
2500
+ "n_targets": 65,
2501
+ "sample_acc": 0.000641025641025641,
2502
+ "new_proofs": 1,
2503
+ "targets_solved": 1309,
2504
+ "lstar": 12,
2505
+ "ge": {
2506
+ "7": 1309,
2507
+ "8": 733,
2508
+ "9": 418,
2509
+ "10": 112,
2510
+ "11": 25,
2511
+ "12": 8,
2512
+ "13": 3,
2513
+ "14": 1
2514
+ },
2515
+ "mix_rl_records": 1,
2516
+ "sampling": {
2517
+ "samples": 1560,
2518
+ "secs": 7.9195191860198975,
2519
+ "gate_secs": 3.263338804244995,
2520
+ "samples_per_sec": 196.98165549669022
2521
+ },
2522
+ "secs": 7.966829538345337,
2523
+ "rl_secs_used": 1720.665892124176
2524
+ }
2525
+ ],
2526
+ "secs": 2756.16139960289,
2527
+ "rl_secs_used": 1761.497284412384,
2528
+ "gate_secs": 994.6351890563965,
2529
+ "targets": 1500,
2530
+ "stats": {
2531
+ "rounds": 20,
2532
+ "first_loops": 1,
2533
+ "k_first": 8,
2534
+ "retry_loops": [
2535
+ 2,
2536
+ 4,
2537
+ 8
2538
+ ],
2539
+ "k_retry": 24,
2540
+ "retries": [
2541
+ {
2542
+ "round": 1,
2543
+ "loops": 2,
2544
+ "targets": 292,
2545
+ "cracked": 35
2546
+ },
2547
+ {
2548
+ "round": 1,
2549
+ "loops": 4,
2550
+ "targets": 292,
2551
+ "cracked": 28
2552
+ },
2553
+ {
2554
+ "round": 1,
2555
+ "loops": 8,
2556
+ "targets": 291,
2557
+ "cracked": 44
2558
+ },
2559
+ {
2560
+ "round": 2,
2561
+ "loops": 2,
2562
+ "targets": 162,
2563
+ "cracked": 12
2564
+ },
2565
+ {
2566
+ "round": 2,
2567
+ "loops": 4,
2568
+ "targets": 162,
2569
+ "cracked": 7
2570
+ },
2571
+ {
2572
+ "round": 2,
2573
+ "loops": 8,
2574
+ "targets": 162,
2575
+ "cracked": 14
2576
+ },
2577
+ {
2578
+ "round": 3,
2579
+ "loops": 2,
2580
+ "targets": 131,
2581
+ "cracked": 10
2582
+ },
2583
+ {
2584
+ "round": 3,
2585
+ "loops": 4,
2586
+ "targets": 131,
2587
+ "cracked": 11
2588
+ },
2589
+ {
2590
+ "round": 3,
2591
+ "loops": 8,
2592
+ "targets": 131,
2593
+ "cracked": 13
2594
+ },
2595
+ {
2596
+ "round": 4,
2597
+ "loops": 2,
2598
+ "targets": 109,
2599
+ "cracked": 3
2600
+ },
2601
+ {
2602
+ "round": 4,
2603
+ "loops": 4,
2604
+ "targets": 108,
2605
+ "cracked": 11
2606
+ },
2607
+ {
2608
+ "round": 4,
2609
+ "loops": 8,
2610
+ "targets": 108,
2611
+ "cracked": 3
2612
+ },
2613
+ {
2614
+ "round": 5,
2615
+ "loops": 2,
2616
+ "targets": 99,
2617
+ "cracked": 2
2618
+ },
2619
+ {
2620
+ "round": 5,
2621
+ "loops": 4,
2622
+ "targets": 98,
2623
+ "cracked": 8
2624
+ },
2625
+ {
2626
+ "round": 5,
2627
+ "loops": 8,
2628
+ "targets": 98,
2629
+ "cracked": 2
2630
+ },
2631
+ {
2632
+ "round": 6,
2633
+ "loops": 2,
2634
+ "targets": 91,
2635
+ "cracked": 1
2636
+ },
2637
+ {
2638
+ "round": 6,
2639
+ "loops": 4,
2640
+ "targets": 90,
2641
+ "cracked": 3
2642
+ },
2643
+ {
2644
+ "round": 6,
2645
+ "loops": 8,
2646
+ "targets": 90,
2647
+ "cracked": 2
2648
+ },
2649
+ {
2650
+ "round": 7,
2651
+ "loops": 2,
2652
+ "targets": 87,
2653
+ "cracked": 4
2654
+ },
2655
+ {
2656
+ "round": 7,
2657
+ "loops": 4,
2658
+ "targets": 86,
2659
+ "cracked": 0
2660
+ },
2661
+ {
2662
+ "round": 7,
2663
+ "loops": 8,
2664
+ "targets": 86,
2665
+ "cracked": 1
2666
+ },
2667
+ {
2668
+ "round": 8,
2669
+ "loops": 2,
2670
+ "targets": 84,
2671
+ "cracked": 0
2672
+ },
2673
+ {
2674
+ "round": 8,
2675
+ "loops": 4,
2676
+ "targets": 84,
2677
+ "cracked": 0
2678
+ },
2679
+ {
2680
+ "round": 8,
2681
+ "loops": 8,
2682
+ "targets": 84,
2683
+ "cracked": 2
2684
+ },
2685
+ {
2686
+ "round": 9,
2687
+ "loops": 2,
2688
+ "targets": 83,
2689
+ "cracked": 2
2690
+ },
2691
+ {
2692
+ "round": 9,
2693
+ "loops": 4,
2694
+ "targets": 83,
2695
+ "cracked": 3
2696
+ },
2697
+ {
2698
+ "round": 9,
2699
+ "loops": 8,
2700
+ "targets": 82,
2701
+ "cracked": 1
2702
+ },
2703
+ {
2704
+ "round": 10,
2705
+ "loops": 2,
2706
+ "targets": 80,
2707
+ "cracked": 1
2708
+ },
2709
+ {
2710
+ "round": 10,
2711
+ "loops": 4,
2712
+ "targets": 79,
2713
+ "cracked": 0
2714
+ },
2715
+ {
2716
+ "round": 10,
2717
+ "loops": 8,
2718
+ "targets": 79,
2719
+ "cracked": 1
2720
+ },
2721
+ {
2722
+ "round": 11,
2723
+ "loops": 2,
2724
+ "targets": 78,
2725
+ "cracked": 0
2726
+ },
2727
+ {
2728
+ "round": 11,
2729
+ "loops": 4,
2730
+ "targets": 77,
2731
+ "cracked": 0
2732
+ },
2733
+ {
2734
+ "round": 11,
2735
+ "loops": 8,
2736
+ "targets": 77,
2737
+ "cracked": 1
2738
+ },
2739
+ {
2740
+ "round": 12,
2741
+ "loops": 2,
2742
+ "targets": 76,
2743
+ "cracked": 1
2744
+ },
2745
+ {
2746
+ "round": 12,
2747
+ "loops": 4,
2748
+ "targets": 75,
2749
+ "cracked": 1
2750
+ },
2751
+ {
2752
+ "round": 12,
2753
+ "loops": 8,
2754
+ "targets": 75,
2755
+ "cracked": 3
2756
+ },
2757
+ {
2758
+ "round": 13,
2759
+ "loops": 2,
2760
+ "targets": 74,
2761
+ "cracked": 0
2762
+ },
2763
+ {
2764
+ "round": 13,
2765
+ "loops": 4,
2766
+ "targets": 74,
2767
+ "cracked": 1
2768
+ },
2769
+ {
2770
+ "round": 13,
2771
+ "loops": 8,
2772
+ "targets": 73,
2773
+ "cracked": 0
2774
+ },
2775
+ {
2776
+ "round": 14,
2777
+ "loops": 2,
2778
+ "targets": 73,
2779
+ "cracked": 2
2780
+ },
2781
+ {
2782
+ "round": 14,
2783
+ "loops": 4,
2784
+ "targets": 73,
2785
+ "cracked": 0
2786
+ },
2787
+ {
2788
+ "round": 14,
2789
+ "loops": 8,
2790
+ "targets": 73,
2791
+ "cracked": 0
2792
+ },
2793
+ {
2794
+ "round": 15,
2795
+ "loops": 2,
2796
+ "targets": 73,
2797
+ "cracked": 0
2798
+ },
2799
+ {
2800
+ "round": 15,
2801
+ "loops": 4,
2802
+ "targets": 72,
2803
+ "cracked": 1
2804
+ },
2805
+ {
2806
+ "round": 15,
2807
+ "loops": 8,
2808
+ "targets": 72,
2809
+ "cracked": 1
2810
+ },
2811
+ {
2812
+ "round": 16,
2813
+ "loops": 2,
2814
+ "targets": 72,
2815
+ "cracked": 3
2816
+ },
2817
+ {
2818
+ "round": 16,
2819
+ "loops": 4,
2820
+ "targets": 71,
2821
+ "cracked": 0
2822
+ },
2823
+ {
2824
+ "round": 16,
2825
+ "loops": 8,
2826
+ "targets": 71,
2827
+ "cracked": 1
2828
+ },
2829
+ {
2830
+ "round": 17,
2831
+ "loops": 2,
2832
+ "targets": 70,
2833
+ "cracked": 3
2834
+ },
2835
+ {
2836
+ "round": 17,
2837
+ "loops": 4,
2838
+ "targets": 69,
2839
+ "cracked": 1
2840
+ },
2841
+ {
2842
+ "round": 17,
2843
+ "loops": 8,
2844
+ "targets": 69,
2845
+ "cracked": 1
2846
+ },
2847
+ {
2848
+ "round": 18,
2849
+ "loops": 2,
2850
+ "targets": 66,
2851
+ "cracked": 0
2852
+ },
2853
+ {
2854
+ "round": 18,
2855
+ "loops": 4,
2856
+ "targets": 66,
2857
+ "cracked": 0
2858
+ },
2859
+ {
2860
+ "round": 18,
2861
+ "loops": 8,
2862
+ "targets": 66,
2863
+ "cracked": 1
2864
+ },
2865
+ {
2866
+ "round": 19,
2867
+ "loops": 2,
2868
+ "targets": 66,
2869
+ "cracked": 1
2870
+ },
2871
+ {
2872
+ "round": 19,
2873
+ "loops": 4,
2874
+ "targets": 65,
2875
+ "cracked": 0
2876
+ },
2877
+ {
2878
+ "round": 19,
2879
+ "loops": 8,
2880
+ "targets": 65,
2881
+ "cracked": 0
2882
+ },
2883
+ {
2884
+ "round": 20,
2885
+ "loops": 2,
2886
+ "targets": 66,
2887
+ "cracked": 1
2888
+ },
2889
+ {
2890
+ "round": 20,
2891
+ "loops": 4,
2892
+ "targets": 66,
2893
+ "cracked": 2
2894
+ },
2895
+ {
2896
+ "round": 20,
2897
+ "loops": 8,
2898
+ "targets": 65,
2899
+ "cracked": 1
2900
+ }
2901
+ ],
2902
+ "ft_records": 76800,
2903
+ "ft_steps": [
2904
+ 626,
2905
+ 660,
2906
+ 679,
2907
+ 693,
2908
+ 691,
2909
+ 705,
2910
+ 713,
2911
+ 724,
2912
+ 722,
2913
+ 726,
2914
+ 739,
2915
+ 740,
2916
+ 738,
2917
+ 747,
2918
+ 742,
2919
+ 756,
2920
+ 757,
2921
+ 760,
2922
+ 757,
2923
+ 760
2924
+ ]
2925
+ }
2926
+ },
2927
+ "eval": {
2928
+ "level": 8,
2929
+ "attempts_per_theorem": 64,
2930
+ "solved": 790,
2931
+ "n": 1108,
2932
+ "solved_ge7": 790,
2933
+ "lstar": 12,
2934
+ "ge": {
2935
+ "7": 790,
2936
+ "8": 655,
2937
+ "9": 521,
2938
+ "10": 134,
2939
+ "11": 31,
2940
+ "12": 7,
2941
+ "13": 1,
2942
+ "14": 0
2943
+ },
2944
+ "by_bin": {
2945
+ "7": {
2946
+ "solved": 135,
2947
+ "n": 150,
2948
+ "rate": 0.9,
2949
+ "ci": [
2950
+ 0.8415639255193982,
2951
+ 0.9384591631640268
2952
+ ]
2953
+ },
2954
+ "8": {
2955
+ "solved": 134,
2956
+ "n": 152,
2957
+ "rate": 0.881578947368421,
2958
+ "ci": [
2959
+ 0.8205787323018284,
2960
+ 0.9237668082983707
2961
+ ]
2962
+ },
2963
+ "9": {
2964
+ "solved": 387,
2965
+ "n": 477,
2966
+ "rate": 0.8113207547169812,
2967
+ "ci": [
2968
+ 0.7737737427372532,
2969
+ 0.8438932810809855
2970
+ ]
2971
+ },
2972
+ "10": {
2973
+ "solved": 103,
2974
+ "n": 219,
2975
+ "rate": 0.4703196347031963,
2976
+ "ci": [
2977
+ 0.40529609337642064,
2978
+ 0.5363665046304147
2979
+ ]
2980
+ },
2981
+ "11": {
2982
+ "solved": 24,
2983
+ "n": 49,
2984
+ "rate": 0.4897959183673469,
2985
+ "ci": [
2986
+ 0.3557488315940086,
2987
+ 0.6253266846659078
2988
+ ]
2989
+ },
2990
+ "12": {
2991
+ "solved": 6,
2992
+ "n": 54,
2993
+ "rate": 0.1111111111111111,
2994
+ "ci": [
2995
+ 0.05192951817556753,
2996
+ 0.2219494893207673
2997
+ ]
2998
+ },
2999
+ "13": {
3000
+ "solved": 1,
3001
+ "n": 5,
3002
+ "rate": 0.2,
3003
+ "ci": [
3004
+ 0.03622316096978745,
3005
+ 0.6244717358814613
3006
+ ]
3007
+ },
3008
+ "14": {
3009
+ "solved": 0,
3010
+ "n": 2,
3011
+ "rate": 0.0,
3012
+ "ci": [
3013
+ 0.0,
3014
+ 0.6576280471103807
3015
+ ]
3016
+ }
3017
+ },
3018
+ "sample_acc": 0.5484544223826715,
3019
+ "max_pruned": 15,
3020
+ "sampling": {
3021
+ "samples": 70912,
3022
+ "secs": 287.0320768356323,
3023
+ "gate_secs": 130.50527811050415,
3024
+ "samples_per_sec": 247.05252730553684
3025
+ },
3026
+ "heldout_greedy": 0.989,
3027
+ "diag": {
3028
+ "level": 2,
3029
+ "attempts_per_theorem": 64,
3030
+ "solved": 790,
3031
+ "n": 1108,
3032
+ "solved_ge7": 790,
3033
+ "lstar": 12,
3034
+ "ge": {
3035
+ "7": 790,
3036
+ "8": 655,
3037
+ "9": 521,
3038
+ "10": 134,
3039
+ "11": 31,
3040
+ "12": 7,
3041
+ "13": 1,
3042
+ "14": 0
3043
+ },
3044
+ "by_bin": {
3045
+ "7": {
3046
+ "solved": 135,
3047
+ "n": 150,
3048
+ "rate": 0.9,
3049
+ "ci": [
3050
+ 0.8415639255193982,
3051
+ 0.9384591631640268
3052
+ ]
3053
+ },
3054
+ "8": {
3055
+ "solved": 134,
3056
+ "n": 152,
3057
+ "rate": 0.881578947368421,
3058
+ "ci": [
3059
+ 0.8205787323018284,
3060
+ 0.9237668082983707
3061
+ ]
3062
+ },
3063
+ "9": {
3064
+ "solved": 387,
3065
+ "n": 477,
3066
+ "rate": 0.8113207547169812,
3067
+ "ci": [
3068
+ 0.7737737427372532,
3069
+ 0.8438932810809855
3070
+ ]
3071
+ },
3072
+ "10": {
3073
+ "solved": 103,
3074
+ "n": 219,
3075
+ "rate": 0.4703196347031963,
3076
+ "ci": [
3077
+ 0.40529609337642064,
3078
+ 0.5363665046304147
3079
+ ]
3080
+ },
3081
+ "11": {
3082
+ "solved": 24,
3083
+ "n": 49,
3084
+ "rate": 0.4897959183673469,
3085
+ "ci": [
3086
+ 0.3557488315940086,
3087
+ 0.6253266846659078
3088
+ ]
3089
+ },
3090
+ "12": {
3091
+ "solved": 6,
3092
+ "n": 54,
3093
+ "rate": 0.1111111111111111,
3094
+ "ci": [
3095
+ 0.05192951817556753,
3096
+ 0.2219494893207673
3097
+ ]
3098
+ },
3099
+ "13": {
3100
+ "solved": 1,
3101
+ "n": 5,
3102
+ "rate": 0.2,
3103
+ "ci": [
3104
+ 0.03622316096978745,
3105
+ 0.6244717358814613
3106
+ ]
3107
+ },
3108
+ "14": {
3109
+ "solved": 0,
3110
+ "n": 2,
3111
+ "rate": 0.0,
3112
+ "ci": [
3113
+ 0.0,
3114
+ 0.6576280471103807
3115
+ ]
3116
+ }
3117
+ },
3118
+ "sample_acc": 0.5484544223826715,
3119
+ "max_pruned": 15,
3120
+ "sampling": {
3121
+ "samples": 70912,
3122
+ "secs": 292.38331151008606,
3123
+ "gate_secs": 136.02297925949097,
3124
+ "samples_per_sec": 242.53094211758327
3125
+ }
3126
+ }
3127
+ }
3128
+ }