Add 590_control-untied-198_s0
Browse files- runs/590_control-untied-198_s0/eval_dev.jsonl +0 -0
- runs/590_control-untied-198_s0/final.pt +3 -0
- runs/590_control-untied-198_s0/found_targets.jsonl +0 -0
- runs/590_control-untied-198_s0/lean_gate.jsonl +84 -0
- runs/590_control-untied-198_s0/pretrain.pt +3 -0
- runs/590_control-untied-198_s0/source/harness.py +529 -0
- runs/590_control-untied-198_s0/source/pretrain.py +594 -0
- runs/590_control-untied-198_s0/source/remote.sh +29 -0
- runs/590_control-untied-198_s0/summary.json +3128 -0
runs/590_control-untied-198_s0/eval_dev.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
runs/590_control-untied-198_s0/final.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c3952712c7b473ab557a5dd74748fa7e23ca082f7af5bc17793e509f29f7faa6
|
| 3 |
+
size 38268575
|
runs/590_control-untied-198_s0/found_targets.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
runs/590_control-untied-198_s0/lean_gate.jsonl
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"utc": "2026-10-07T22:06:45Z", "samples": 2000, "parse_fail": 9, "distinct_checked": 1991, "both_ok": 1937, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 54, "parse_reasons": {"unbound": 7, "no-eos": 1, "name": 1}, "lean_wall_s": 3.5214762687683105, "lean_proc_s": 17.022944927215576, "nd_verify_s": 0.10487580299377441, "workers": 96, "chunk": 400}
|
| 2 |
+
{"utc": "2026-10-07T22:07:53Z", "samples": 12000, "parse_fail": 4055, "distinct_checked": 7716, "both_ok": 2506, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 5210, "parse_reasons": {"unbound": 3223, "formula": 295, "expected )": 152, "exact not last": 144, "name": 87, "no-eos": 49, "expected (": 36, "expected ;": 30, "PR position": 14, "expected :=": 6, "ORE box": 5, "trailing tokens": 5, "expected False": 4, "expected =>": 2, "expected .elim": 1, "expected fun": 1, "expected None": 1}, "lean_wall_s": 24.845098972320557, "lean_proc_s": 475.8859384059906, "nd_verify_s": 0.4249696731567383, "workers": 96, "chunk": 400}
|
| 3 |
+
{"utc": "2026-10-07T22:08:32Z", "samples": 7008, "parse_fail": 2640, "distinct_checked": 4041, "both_ok": 75, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 3966, "parse_reasons": {"unbound": 2104, "formula": 187, "expected )": 105, "exact not last": 77, "name": 61, "expected (": 41, "no-eos": 21, "expected ;": 10, "expected :=": 10, "ORE box": 9, "PR position": 9, "trailing tokens": 4, "expected False": 2}, "lean_wall_s": 14.552488327026367, "lean_proc_s": 145.86501574516296, "nd_verify_s": 0.2135484218597412, "workers": 96, "chunk": 400}
|
| 4 |
+
{"utc": "2026-10-07T22:09:07Z", "samples": 7008, "parse_fail": 2542, "distinct_checked": 4146, "both_ok": 46, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 4100, "parse_reasons": {"unbound": 1980, "formula": 227, "exact not last": 95, "expected )": 75, "name": 70, "no-eos": 52, "expected ;": 15, "expected (": 10, "PR position": 5, "trailing tokens": 4, "expected =>": 2, "expected False": 2, "expected .elim": 1, "expected :=": 1, "expected fun": 1, "ORE box": 1, "eof": 1}, "lean_wall_s": 15.492853164672852, "lean_proc_s": 160.28228092193604, "nd_verify_s": 0.19858956336975098, "workers": 96, "chunk": 400}
|
| 5 |
+
{"utc": "2026-10-07T22:09:43Z", "samples": 6984, "parse_fail": 2601, "distinct_checked": 4060, "both_ok": 86, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 3974, "parse_reasons": {"unbound": 1922, "formula": 298, "exact not last": 96, "name": 79, "expected )": 74, "no-eos": 49, "expected (": 48, "expected ;": 20, "PR position": 5, "trailing tokens": 4, "ORE box": 3, "expected =>": 1, "expected False": 1, "expected fun": 1}, "lean_wall_s": 14.698702096939087, "lean_proc_s": 148.1787965297699, "nd_verify_s": 0.197296142578125, "workers": 96, "chunk": 400}
|
| 6 |
+
{"utc": "2026-10-07T22:11:17Z", "samples": 12000, "parse_fail": 1250, "distinct_checked": 10264, "both_ok": 6791, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 3473, "parse_reasons": {"unbound": 875, "name": 91, "expected (": 90, "formula": 71, "expected )": 52, "exact not last": 31, "expected ;": 18, "PR position": 9, "no-eos": 6, "ORE box": 2, "expected :=": 1, "expected =>": 1, "expected fun": 1, "expected False": 1, "trailing tokens": 1}, "lean_wall_s": 35.21818518638611, "lean_proc_s": 893.2075176239014, "nd_verify_s": 0.6092596054077148, "workers": 96, "chunk": 400}
|
| 7 |
+
{"utc": "2026-10-07T22:11:40Z", "samples": 3888, "parse_fail": 919, "distinct_checked": 2709, "both_ok": 28, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 2681, "parse_reasons": {"unbound": 651, "name": 84, "expected (": 84, "formula": 25, "expected ;": 24, "exact not last": 21, "expected )": 13, "PR position": 9, "expected =>": 3, "ORE box": 2, "no-eos": 1, "trailing tokens": 1, "expected fun": 1}, "lean_wall_s": 10.493554592132568, "lean_proc_s": 72.69925570487976, "nd_verify_s": 0.1555500030517578, "workers": 96, "chunk": 400}
|
| 8 |
+
{"utc": "2026-10-07T22:11:57Z", "samples": 3888, "parse_fail": 1186, "distinct_checked": 2454, "both_ok": 22, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 2432, "parse_reasons": {"unbound": 875, "name": 86, "formula": 72, "expected (": 56, "expected )": 55, "expected ;": 26, "ORE box": 8, "exact not last": 6, "no-eos": 1, "PR position": 1}, "lean_wall_s": 8.818713665008545, "lean_proc_s": 53.735737800598145, "nd_verify_s": 0.11390161514282227, "workers": 96, "chunk": 400}
|
| 9 |
+
{"utc": "2026-10-07T22:12:18Z", "samples": 3888, "parse_fail": 988, "distinct_checked": 2675, "both_ok": 34, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 2641, "parse_reasons": {"unbound": 660, "expected (": 112, "name": 96, "formula": 55, "expected )": 35, "exact not last": 8, "ORE box": 7, "expected ;": 5, "expected :=": 4, "PR position": 2, "expected False": 2, "no-eos": 1, "trailing tokens": 1}, "lean_wall_s": 10.771084785461426, "lean_proc_s": 72.13880109786987, "nd_verify_s": 0.15462708473205566, "workers": 96, "chunk": 400}
|
| 10 |
+
{"utc": "2026-10-07T22:13:43Z", "samples": 12000, "parse_fail": 852, "distinct_checked": 10688, "both_ok": 7905, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 2783, "parse_reasons": {"unbound": 603, "expected (": 94, "name": 60, "expected )": 35, "formula": 22, "ORE box": 12, "exact not last": 8, "expected ;": 7, "expected :=": 6, "PR position": 5}, "lean_wall_s": 27.536895036697388, "lean_proc_s": 730.5064754486084, "nd_verify_s": 0.681675910949707, "workers": 96, "chunk": 400}
|
| 11 |
+
{"utc": "2026-10-07T22:14:00Z", "samples": 3144, "parse_fail": 679, "distinct_checked": 2257, "both_ok": 27, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 2230, "parse_reasons": {"unbound": 503, "expected (": 63, "name": 43, "expected )": 21, "formula": 17, "expected :=": 13, "ORE box": 10, "exact not last": 6, "PR position": 1, "expected ;": 1, "expected fun": 1}, "lean_wall_s": 8.917486906051636, "lean_proc_s": 51.93652367591858, "nd_verify_s": 0.10415196418762207, "workers": 96, "chunk": 400}
|
| 12 |
+
{"utc": "2026-10-07T22:14:18Z", "samples": 3144, "parse_fail": 707, "distinct_checked": 2220, "both_ok": 20, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 2200, "parse_reasons": {"unbound": 524, "expected (": 71, "expected )": 41, "name": 34, "formula": 27, "exact not last": 6, "expected ;": 2, "PR position": 1, "expected False": 1}, "lean_wall_s": 8.934838056564331, "lean_proc_s": 49.52054500579834, "nd_verify_s": 0.11041092872619629, "workers": 96, "chunk": 400}
|
| 13 |
+
{"utc": "2026-10-07T22:14:33Z", "samples": 3144, "parse_fail": 858, "distinct_checked": 2105, "both_ok": 32, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 2073, "parse_reasons": {"unbound": 614, "expected (": 110, "name": 63, "ORE box": 24, "expected )": 21, "formula": 9, "PR position": 5, "expected fun": 4, "expected ;": 3, "exact not last": 3, "expected False": 2}, "lean_wall_s": 8.76211929321289, "lean_proc_s": 46.95437216758728, "nd_verify_s": 0.11509251594543457, "workers": 96, "chunk": 400}
|
| 14 |
+
{"utc": "2026-10-07T22:16:05Z", "samples": 12000, "parse_fail": 958, "distinct_checked": 10558, "both_ok": 8439, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 2119, "parse_reasons": {"unbound": 637, "expected (": 119, "formula": 84, "expected )": 47, "name": 31, "exact not last": 11, "ORE box": 11, "expected ;": 9, "PR position": 4, "no-eos": 3, "expected =>": 1, "expected :=": 1}, "lean_wall_s": 30.82331871986389, "lean_proc_s": 804.2527289390564, "nd_verify_s": 0.6117749214172363, "workers": 96, "chunk": 400}
|
| 15 |
+
{"utc": "2026-10-07T22:16:18Z", "samples": 2616, "parse_fail": 886, "distinct_checked": 1604, "both_ok": 9, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1595, "parse_reasons": {"unbound": 568, "formula": 106, "expected (": 103, "expected )": 56, "name": 32, "exact not last": 8, "expected ;": 7, "no-eos": 3, "ORE box": 2, "PR position": 1}, "lean_wall_s": 6.5083167552948, "lean_proc_s": 25.601459741592407, "nd_verify_s": 0.11884880065917969, "workers": 96, "chunk": 400}
|
| 16 |
+
{"utc": "2026-10-07T22:16:32Z", "samples": 2592, "parse_fail": 826, "distinct_checked": 1636, "both_ok": 41, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1595, "parse_reasons": {"unbound": 478, "expected (": 166, "formula": 75, "expected )": 36, "name": 28, "ORE box": 16, "exact not last": 14, "expected ;": 9, "expected fun": 2, "expected :=": 2}, "lean_wall_s": 6.603259801864624, "lean_proc_s": 26.57071018218994, "nd_verify_s": 0.07048296928405762, "workers": 96, "chunk": 400}
|
| 17 |
+
{"utc": "2026-10-07T22:16:44Z", "samples": 2592, "parse_fail": 753, "distinct_checked": 1710, "both_ok": 3, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1707, "parse_reasons": {"unbound": 517, "expected (": 88, "expected )": 60, "name": 38, "formula": 27, "expected ;": 10, "exact not last": 5, "PR position": 4, "expected .elim": 3, "ORE box": 1}, "lean_wall_s": 7.12667179107666, "lean_proc_s": 30.669589519500732, "nd_verify_s": 0.10635018348693848, "workers": 96, "chunk": 400}
|
| 18 |
+
{"utc": "2026-10-07T22:18:25Z", "samples": 12000, "parse_fail": 952, "distinct_checked": 10562, "both_ok": 8788, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1774, "parse_reasons": {"unbound": 654, "expected (": 84, "formula": 79, "name": 46, "expected )": 32, "expected ;": 21, "ORE box": 10, "expected :=": 10, "exact not last": 9, "no-eos": 2, "PR position": 2, "expected =>": 1, "expected \u27e9": 1, "expected exact": 1}, "lean_wall_s": 35.68090486526489, "lean_proc_s": 930.8845624923706, "nd_verify_s": 0.6378681659698486, "workers": 96, "chunk": 400}
|
| 19 |
+
{"utc": "2026-10-07T22:18:36Z", "samples": 2376, "parse_fail": 837, "distinct_checked": 1391, "both_ok": 3, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1388, "parse_reasons": {"unbound": 593, "expected (": 88, "name": 45, "formula": 44, "expected ;": 23, "expected )": 20, "exact not last": 11, "ORE box": 11, "expected :=": 1, "PR position": 1}, "lean_wall_s": 5.490969657897949, "lean_proc_s": 19.718544006347656, "nd_verify_s": 0.06748843193054199, "workers": 96, "chunk": 400}
|
| 20 |
+
{"utc": "2026-10-07T22:18:48Z", "samples": 2352, "parse_fail": 724, "distinct_checked": 1510, "both_ok": 13, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1497, "parse_reasons": {"unbound": 476, "formula": 129, "expected (": 39, "name": 34, "expected )": 19, "expected ;": 14, "ORE box": 7, "PR position": 3, "exact not last": 3}, "lean_wall_s": 6.038282871246338, "lean_proc_s": 23.68019390106201, "nd_verify_s": 0.07426762580871582, "workers": 96, "chunk": 400}
|
| 21 |
+
{"utc": "2026-10-07T22:18:59Z", "samples": 2352, "parse_fail": 801, "distinct_checked": 1439, "both_ok": 3, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1436, "parse_reasons": {"unbound": 498, "expected (": 108, "name": 55, "formula": 49, "expected )": 33, "expected :=": 21, "ORE box": 17, "exact not last": 9, "no-eos": 6, "expected ;": 4, "eof": 1}, "lean_wall_s": 5.4260783195495605, "lean_proc_s": 20.484389066696167, "nd_verify_s": 0.06520771980285645, "workers": 96, "chunk": 400}
|
| 22 |
+
{"utc": "2026-10-07T22:20:35Z", "samples": 12000, "parse_fail": 695, "distinct_checked": 10849, "both_ok": 9014, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1835, "parse_reasons": {"unbound": 438, "name": 76, "expected (": 65, "formula": 52, "expected )": 16, "ORE box": 15, "exact not last": 13, "expected :=": 9, "expected ;": 5, "expected =>": 2, "expected by": 1, "expected None": 1, "PR position": 1, "no-eos": 1}, "lean_wall_s": 33.27284097671509, "lean_proc_s": 878.5364444255829, "nd_verify_s": 0.6728756427764893, "workers": 96, "chunk": 400}
|
| 23 |
+
{"utc": "2026-10-07T22:20:49Z", "samples": 2184, "parse_fail": 560, "distinct_checked": 1495, "both_ok": 2, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1493, "parse_reasons": {"unbound": 323, "formula": 83, "name": 70, "expected (": 35, "expected :=": 24, "expected )": 15, "ORE box": 4, "expected ;": 3, "PR position": 2, "no-eos": 1}, "lean_wall_s": 5.636815071105957, "lean_proc_s": 22.2003972530365, "nd_verify_s": 0.0722653865814209, "workers": 96, "chunk": 400}
|
| 24 |
+
{"utc": "2026-10-07T22:20:58Z", "samples": 2160, "parse_fail": 641, "distinct_checked": 1406, "both_ok": 6, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1400, "parse_reasons": {"unbound": 378, "expected (": 129, "name": 102, "ORE box": 11, "formula": 8, "expected )": 7, "exact not last": 4, "expected \u27e9": 1, "PR position": 1}, "lean_wall_s": 5.160709381103516, "lean_proc_s": 19.36012101173401, "nd_verify_s": 0.0691680908203125, "workers": 96, "chunk": 400}
|
| 25 |
+
{"utc": "2026-10-07T22:21:10Z", "samples": 2160, "parse_fail": 563, "distinct_checked": 1460, "both_ok": 2, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1458, "parse_reasons": {"unbound": 361, "expected (": 57, "formula": 54, "name": 49, "ORE box": 28, "expected )": 4, "no-eos": 3, "expected :=": 3, "PR position": 3, "expected None": 1}, "lean_wall_s": 5.395240783691406, "lean_proc_s": 20.39289879798889, "nd_verify_s": 0.07996606826782227, "workers": 96, "chunk": 400}
|
| 26 |
+
{"utc": "2026-10-07T22:22:45Z", "samples": 12000, "parse_fail": 855, "distinct_checked": 10663, "both_ok": 9137, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1526, "parse_reasons": {"unbound": 561, "expected (": 115, "name": 67, "formula": 60, "ORE box": 28, "expected )": 11, "exact not last": 6, "expected :=": 3, "expected ;": 2, "PR position": 1, "no-eos": 1}, "lean_wall_s": 32.39378643035889, "lean_proc_s": 852.9908123016357, "nd_verify_s": 0.6322927474975586, "workers": 96, "chunk": 400}
|
| 27 |
+
{"utc": "2026-10-07T22:22:58Z", "samples": 2088, "parse_fail": 594, "distinct_checked": 1361, "both_ok": 5, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1356, "parse_reasons": {"unbound": 412, "expected (": 57, "formula": 46, "name": 44, "ORE box": 20, "expected )": 7, "exact not last": 4, "expected :=": 3, "no-eos": 1}, "lean_wall_s": 5.149581432342529, "lean_proc_s": 18.325379848480225, "nd_verify_s": 0.0661458969116211, "workers": 96, "chunk": 400}
|
| 28 |
+
{"utc": "2026-10-07T22:23:10Z", "samples": 2064, "parse_fail": 711, "distinct_checked": 1230, "both_ok": 0, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1230, "parse_reasons": {"unbound": 423, "name": 115, "formula": 62, "expected (": 52, "ORE box": 38, "expected )": 8, "expected :=": 6, "trailing tokens": 3, "no-eos": 2, "exact not last": 1, "expected ;": 1}, "lean_wall_s": 4.495267868041992, "lean_proc_s": 13.392057657241821, "nd_verify_s": 0.056827545166015625, "workers": 96, "chunk": 400}
|
| 29 |
+
{"utc": "2026-10-07T22:23:19Z", "samples": 2064, "parse_fail": 769, "distinct_checked": 1197, "both_ok": 1, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1196, "parse_reasons": {"unbound": 412, "expected (": 236, "name": 53, "formula": 27, "ORE box": 26, "expected )": 9, "PR position": 3, "exact not last": 2, "expected ;": 1}, "lean_wall_s": 4.360255718231201, "lean_proc_s": 12.837864637374878, "nd_verify_s": 0.047799110412597656, "workers": 96, "chunk": 400}
|
| 30 |
+
{"utc": "2026-10-07T22:24:54Z", "samples": 12000, "parse_fail": 854, "distinct_checked": 10670, "both_ok": 9229, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1441, "parse_reasons": {"unbound": 604, "name": 79, "expected (": 68, "formula": 43, "ORE box": 28, "expected )": 17, "exact not last": 4, "expected ,": 4, "expected ;": 3, "PR position": 2, "expected fun": 1, "no-eos": 1}, "lean_wall_s": 31.666245222091675, "lean_proc_s": 836.4754900932312, "nd_verify_s": 0.6611950397491455, "workers": 96, "chunk": 400}
|
| 31 |
+
{"utc": "2026-10-07T22:25:05Z", "samples": 2016, "parse_fail": 825, "distinct_checked": 1105, "both_ok": 0, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1105, "parse_reasons": {"unbound": 504, "expected (": 101, "formula": 72, "name": 69, "ORE box": 47, "expected )": 22, "exact not last": 3, "expected ,": 3, "expected ;": 2, "PR position": 2}, "lean_wall_s": 4.575493335723877, "lean_proc_s": 12.941572189331055, "nd_verify_s": 0.04767131805419922, "workers": 96, "chunk": 400}
|
| 32 |
+
{"utc": "2026-10-07T22:25:16Z", "samples": 2016, "parse_fail": 732, "distinct_checked": 1184, "both_ok": 0, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1184, "parse_reasons": {"unbound": 508, "expected (": 67, "name": 57, "ORE box": 56, "formula": 25, "expected )": 11, "expected ;": 3, "PR position": 2, "exact not last": 2, "no-eos": 1}, "lean_wall_s": 4.416784286499023, "lean_proc_s": 13.107749223709106, "nd_verify_s": 0.048589229583740234, "workers": 96, "chunk": 400}
|
| 33 |
+
{"utc": "2026-10-07T22:25:25Z", "samples": 2016, "parse_fail": 686, "distinct_checked": 1210, "both_ok": 2, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1208, "parse_reasons": {"unbound": 520, "expected (": 61, "name": 61, "formula": 32, "expected )": 7, "ORE box": 2, "PR position": 2, "exact not last": 1}, "lean_wall_s": 4.630467891693115, "lean_proc_s": 13.73590087890625, "nd_verify_s": 0.0494840145111084, "workers": 96, "chunk": 400}
|
| 34 |
+
{"utc": "2026-10-07T22:27:06Z", "samples": 12000, "parse_fail": 849, "distinct_checked": 10749, "both_ok": 9381, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1368, "parse_reasons": {"unbound": 509, "expected (": 130, "formula": 92, "name": 40, "ORE box": 36, "exact not last": 12, "expected )": 11, "expected ;": 8, "expected :=": 4, "PR position": 3, "expected fun": 2, "no-eos": 2}, "lean_wall_s": 30.599449157714844, "lean_proc_s": 813.9485108852386, "nd_verify_s": 0.6633610725402832, "workers": 96, "chunk": 400}
|
| 35 |
+
{"utc": "2026-10-07T22:27:19Z", "samples": 1992, "parse_fail": 748, "distinct_checked": 1147, "both_ok": 2, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1145, "parse_reasons": {"unbound": 427, "formula": 145, "expected (": 91, "ORE box": 39, "name": 28, "expected )": 12, "PR position": 2, "expected ;": 2, "exact not last": 1, "no-eos": 1}, "lean_wall_s": 4.360078573226929, "lean_proc_s": 12.885820627212524, "nd_verify_s": 0.06374645233154297, "workers": 96, "chunk": 400}
|
| 36 |
+
{"utc": "2026-10-07T22:27:31Z", "samples": 1992, "parse_fail": 805, "distinct_checked": 1078, "both_ok": 3, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1075, "parse_reasons": {"unbound": 448, "expected (": 193, "formula": 72, "name": 52, "ORE box": 23, "PR position": 6, "expected )": 5, "exact not last": 3, "expected ;": 2, "no-eos": 1}, "lean_wall_s": 4.141678094863892, "lean_proc_s": 11.681383848190308, "nd_verify_s": 0.04389810562133789, "workers": 96, "chunk": 400}
|
| 37 |
+
{"utc": "2026-10-07T22:27:41Z", "samples": 1968, "parse_fail": 702, "distinct_checked": 1171, "both_ok": 2, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1169, "parse_reasons": {"unbound": 431, "expected (": 92, "name": 52, "ORE box": 48, "formula": 47, "expected :=": 15, "expected )": 6, "exact not last": 4, "PR position": 2, "no-eos": 2, "expected fun": 2, "expected ;": 1}, "lean_wall_s": 4.371264934539795, "lean_proc_s": 12.872344255447388, "nd_verify_s": 0.059053897857666016, "workers": 96, "chunk": 400}
|
| 38 |
+
{"utc": "2026-10-07T22:29:22Z", "samples": 12000, "parse_fail": 818, "distinct_checked": 10747, "both_ok": 9362, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1385, "parse_reasons": {"unbound": 553, "expected (": 134, "name": 37, "ORE box": 29, "formula": 18, "expected )": 15, "exact not last": 11, "expected ;": 7, "expected :=": 5, "no-eos": 4, "PR position": 2, "expected ,": 2, "expected \u27e9": 1}, "lean_wall_s": 31.961103677749634, "lean_proc_s": 853.4920246601105, "nd_verify_s": 0.6602957248687744, "workers": 96, "chunk": 400}
|
| 39 |
+
{"utc": "2026-10-07T22:29:32Z", "samples": 1920, "parse_fail": 779, "distinct_checked": 1055, "both_ok": 1, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1054, "parse_reasons": {"unbound": 496, "expected (": 149, "name": 50, "ORE box": 23, "formula": 19, "expected )": 18, "expected :=": 9, "expected ;": 7, "no-eos": 3, "exact not last": 2, "expected ,": 1, "trailing tokens": 1, "PR position": 1}, "lean_wall_s": 3.9825429916381836, "lean_proc_s": 11.346506118774414, "nd_verify_s": 0.03564763069152832, "workers": 96, "chunk": 400}
|
| 40 |
+
{"utc": "2026-10-07T22:29:43Z", "samples": 1896, "parse_fail": 665, "distinct_checked": 1156, "both_ok": 0, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1156, "parse_reasons": {"unbound": 463, "expected (": 114, "ORE box": 30, "name": 21, "expected )": 11, "formula": 6, "exact not last": 6, "PR position": 6, "expected ,": 4, "expected ;": 4}, "lean_wall_s": 4.536712884902954, "lean_proc_s": 13.38208293914795, "nd_verify_s": 0.05449032783508301, "workers": 96, "chunk": 400}
|
| 41 |
+
{"utc": "2026-10-07T22:29:52Z", "samples": 1896, "parse_fail": 711, "distinct_checked": 1095, "both_ok": 1, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1094, "parse_reasons": {"unbound": 466, "expected (": 123, "name": 46, "formula": 42, "expected )": 14, "expected ;": 12, "ORE box": 4, "expected \u27e9": 2, "expected ,": 1, "exact not last": 1}, "lean_wall_s": 4.351956367492676, "lean_proc_s": 12.382160425186157, "nd_verify_s": 0.04508709907531738, "workers": 96, "chunk": 400}
|
| 42 |
+
{"utc": "2026-10-07T22:31:25Z", "samples": 12000, "parse_fail": 794, "distinct_checked": 10797, "both_ok": 9457, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1340, "parse_reasons": {"unbound": 488, "expected (": 143, "name": 44, "formula": 40, "expected )": 35, "ORE box": 11, "expected ;": 7, "exact not last": 7, "expected \u27e9": 6, "expected :=": 6, "PR position": 3, "expected ,": 3, "expected False": 1}, "lean_wall_s": 30.53050208091736, "lean_proc_s": 811.528406381607, "nd_verify_s": 0.6610722541809082, "workers": 96, "chunk": 400}
|
| 43 |
+
{"utc": "2026-10-07T22:31:35Z", "samples": 1872, "parse_fail": 621, "distinct_checked": 1152, "both_ok": 0, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1152, "parse_reasons": {"unbound": 403, "formula": 75, "expected (": 60, "name": 41, "expected )": 25, "PR position": 6, "expected ;": 2, "expected \u27e9": 2, "expected False": 2, "exact not last": 2, "ORE box": 2, "expected ,": 1}, "lean_wall_s": 4.1689958572387695, "lean_proc_s": 12.333671569824219, "nd_verify_s": 0.04584002494812012, "workers": 96, "chunk": 400}
|
| 44 |
+
{"utc": "2026-10-07T22:31:45Z", "samples": 1848, "parse_fail": 731, "distinct_checked": 1026, "both_ok": 0, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1026, "parse_reasons": {"unbound": 422, "expected (": 217, "name": 24, "ORE box": 18, "expected )": 17, "exact not last": 8, "expected :=": 8, "formula": 7, "PR position": 3, "no-eos": 3, "expected ;": 2, "expected \u27e9": 1, "expected ,": 1}, "lean_wall_s": 4.288639783859253, "lean_proc_s": 11.693408727645874, "nd_verify_s": 0.07408308982849121, "workers": 96, "chunk": 400}
|
| 45 |
+
{"utc": "2026-10-07T22:31:54Z", "samples": 1848, "parse_fail": 683, "distinct_checked": 1069, "both_ok": 2, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1067, "parse_reasons": {"unbound": 377, "expected (": 138, "name": 53, "formula": 41, "expected )": 37, "expected ;": 13, "exact not last": 9, "ORE box": 8, "PR position": 4, "expected \u27e9": 2, "expected ,": 1}, "lean_wall_s": 4.184258222579956, "lean_proc_s": 11.743452310562134, "nd_verify_s": 0.049808502197265625, "workers": 96, "chunk": 400}
|
| 46 |
+
{"utc": "2026-10-07T22:33:40Z", "samples": 12000, "parse_fail": 883, "distinct_checked": 10767, "both_ok": 9601, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1166, "parse_reasons": {"unbound": 586, "expected (": 172, "name": 43, "ORE box": 23, "expected )": 22, "exact not last": 12, "expected :=": 6, "expected ,": 5, "expected ;": 4, "no-eos": 4, "formula": 3, "PR position": 2, "expected by": 1}, "lean_wall_s": 30.843714714050293, "lean_proc_s": 819.9535932540894, "nd_verify_s": 0.6709682941436768, "workers": 96, "chunk": 400}
|
| 47 |
+
{"utc": "2026-10-07T22:33:52Z", "samples": 1824, "parse_fail": 709, "distinct_checked": 1018, "both_ok": 2, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1016, "parse_reasons": {"unbound": 444, "expected (": 175, "name": 39, "ORE box": 16, "expected :=": 14, "expected )": 13, "formula": 4, "expected ,": 1, "no-eos": 1, "exact not last": 1, "expected ;": 1}, "lean_wall_s": 4.116172790527344, "lean_proc_s": 11.273685216903687, "nd_verify_s": 0.04662656784057617, "workers": 96, "chunk": 400}
|
| 48 |
+
{"utc": "2026-10-07T22:34:03Z", "samples": 1800, "parse_fail": 798, "distinct_checked": 934, "both_ok": 1, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 933, "parse_reasons": {"unbound": 555, "expected (": 133, "ORE box": 47, "name": 23, "expected )": 19, "no-eos": 4, "formula": 4, "expected ;": 4, "expected :=": 4, "exact not last": 2, "PR position": 2, "expected None": 1}, "lean_wall_s": 3.4939091205596924, "lean_proc_s": 8.653051614761353, "nd_verify_s": 0.04425644874572754, "workers": 96, "chunk": 400}
|
| 49 |
+
{"utc": "2026-10-07T22:34:15Z", "samples": 1800, "parse_fail": 804, "distinct_checked": 932, "both_ok": 5, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 927, "parse_reasons": {"unbound": 505, "expected (": 196, "name": 35, "expected )": 24, "ORE box": 19, "formula": 8, "exact not last": 6, "expected :=": 4, "no-eos": 3, "expected ,": 2, "PR position": 1, "expected ;": 1}, "lean_wall_s": 3.612116575241089, "lean_proc_s": 9.051259994506836, "nd_verify_s": 0.06170058250427246, "workers": 96, "chunk": 400}
|
| 50 |
+
{"utc": "2026-10-07T22:35:54Z", "samples": 12000, "parse_fail": 816, "distinct_checked": 10782, "both_ok": 9639, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1143, "parse_reasons": {"unbound": 536, "expected (": 117, "name": 50, "ORE box": 38, "expected )": 36, "formula": 14, "exact not last": 8, "expected ;": 8, "expected :=": 4, "expected ,": 2, "expected exact": 1, "expected \u27e9": 1, "PR position": 1}, "lean_wall_s": 30.9827663898468, "lean_proc_s": 818.4715430736542, "nd_verify_s": 0.6768157482147217, "workers": 96, "chunk": 400}
|
| 51 |
+
{"utc": "2026-10-07T22:36:05Z", "samples": 1776, "parse_fail": 700, "distinct_checked": 998, "both_ok": 0, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 998, "parse_reasons": {"unbound": 465, "expected (": 83, "name": 56, "expected )": 52, "ORE box": 18, "expected ;": 9, "formula": 7, "exact not last": 4, "PR position": 3, "no-eos": 3}, "lean_wall_s": 3.8764398097991943, "lean_proc_s": 10.170764207839966, "nd_verify_s": 0.0408327579498291, "workers": 96, "chunk": 400}
|
| 52 |
+
{"utc": "2026-10-07T22:36:15Z", "samples": 1776, "parse_fail": 781, "distinct_checked": 923, "both_ok": 1, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 922, "parse_reasons": {"unbound": 469, "expected (": 131, "ORE box": 55, "name": 55, "expected )": 37, "expected ;": 18, "formula": 7, "exact not last": 4, "PR position": 2, "expected :": 1, "expected :=": 1, "no-eos": 1}, "lean_wall_s": 3.610947608947754, "lean_proc_s": 8.643958568572998, "nd_verify_s": 0.04439234733581543, "workers": 96, "chunk": 400}
|
| 53 |
+
{"utc": "2026-10-07T22:36:24Z", "samples": 1752, "parse_fail": 704, "distinct_checked": 967, "both_ok": 0, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 967, "parse_reasons": {"unbound": 432, "expected (": 137, "name": 36, "expected )": 34, "ORE box": 31, "expected :=": 16, "formula": 11, "expected ,": 2, "expected ;": 2, "no-eos": 1, "exact not last": 1, "PR position": 1}, "lean_wall_s": 3.5093626976013184, "lean_proc_s": 9.01863145828247, "nd_verify_s": 0.04015231132507324, "workers": 96, "chunk": 400}
|
| 54 |
+
{"utc": "2026-10-07T22:38:04Z", "samples": 12000, "parse_fail": 811, "distinct_checked": 10776, "both_ok": 9630, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1146, "parse_reasons": {"unbound": 501, "expected (": 130, "name": 62, "formula": 34, "expected )": 31, "ORE box": 29, "exact not last": 9, "expected ;": 4, "expected ,": 4, "no-eos": 3, "PR position": 3, "trailing tokens": 1}, "lean_wall_s": 31.182451009750366, "lean_proc_s": 830.0188059806824, "nd_verify_s": 0.6706230640411377, "workers": 96, "chunk": 400}
|
| 55 |
+
{"utc": "2026-10-07T22:38:14Z", "samples": 1752, "parse_fail": 674, "distinct_checked": 976, "both_ok": 5, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 971, "parse_reasons": {"unbound": 398, "expected (": 106, "name": 58, "expected )": 51, "formula": 25, "ORE box": 22, "PR position": 5, "exact not last": 4, "expected ;": 2, "expected :=": 1, "no-eos": 1, "trailing tokens": 1}, "lean_wall_s": 3.751281499862671, "lean_proc_s": 10.283748149871826, "nd_verify_s": 0.04014134407043457, "workers": 96, "chunk": 400}
|
| 56 |
+
{"utc": "2026-10-07T22:38:23Z", "samples": 1752, "parse_fail": 761, "distinct_checked": 923, "both_ok": 0, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 923, "parse_reasons": {"unbound": 489, "expected (": 120, "name": 66, "expected )": 31, "formula": 27, "ORE box": 11, "expected ;": 8, "no-eos": 4, "exact not last": 2, "expected :=": 1, "PR position": 1, "expected exact": 1}, "lean_wall_s": 3.714834690093994, "lean_proc_s": 9.106945276260376, "nd_verify_s": 0.04108071327209473, "workers": 96, "chunk": 400}
|
| 57 |
+
{"utc": "2026-10-07T22:38:34Z", "samples": 1752, "parse_fail": 740, "distinct_checked": 931, "both_ok": 0, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 931, "parse_reasons": {"unbound": 404, "expected (": 153, "name": 59, "ORE box": 56, "expected )": 27, "formula": 25, "expected ;": 8, "no-eos": 4, "exact not last": 1, "eof": 1, "expected ,": 1, "PR position": 1}, "lean_wall_s": 3.820549249649048, "lean_proc_s": 9.806522130966187, "nd_verify_s": 0.05093264579772949, "workers": 96, "chunk": 400}
|
| 58 |
+
{"utc": "2026-10-07T22:40:13Z", "samples": 12000, "parse_fail": 893, "distinct_checked": 10724, "both_ok": 9662, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1062, "parse_reasons": {"unbound": 572, "expected (": 164, "name": 48, "expected )": 29, "formula": 20, "ORE box": 17, "exact not last": 16, "no-eos": 7, "expected ,": 6, "expected :=": 6, "expected ;": 5, "PR position": 3}, "lean_wall_s": 30.381449222564697, "lean_proc_s": 807.2059581279755, "nd_verify_s": 0.6879208087921143, "workers": 96, "chunk": 400}
|
| 59 |
+
{"utc": "2026-10-07T22:40:23Z", "samples": 1752, "parse_fail": 789, "distinct_checked": 868, "both_ok": 0, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 868, "parse_reasons": {"unbound": 517, "expected (": 165, "name": 38, "expected )": 29, "ORE box": 16, "no-eos": 10, "expected None": 4, "expected ;": 4, "PR position": 3, "exact not last": 1, "formula": 1, "expected ,": 1}, "lean_wall_s": 3.1183199882507324, "lean_proc_s": 7.024734258651733, "nd_verify_s": 0.040032386779785156, "workers": 96, "chunk": 400}
|
| 60 |
+
{"utc": "2026-10-07T22:40:32Z", "samples": 1728, "parse_fail": 798, "distinct_checked": 856, "both_ok": 1, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 855, "parse_reasons": {"unbound": 551, "expected (": 91, "name": 60, "ORE box": 35, "expected )": 22, "expected :=": 20, "formula": 5, "expected ,": 4, "exact not last": 4, "PR position": 3, "expected None": 2, "no-eos": 1}, "lean_wall_s": 3.2346765995025635, "lean_proc_s": 7.057374477386475, "nd_verify_s": 0.038831472396850586, "workers": 96, "chunk": 400}
|
| 61 |
+
{"utc": "2026-10-07T22:40:40Z", "samples": 1728, "parse_fail": 807, "distinct_checked": 850, "both_ok": 1, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 849, "parse_reasons": {"unbound": 495, "expected (": 189, "formula": 41, "name": 31, "expected )": 22, "expected ;": 13, "ORE box": 9, "expected =>": 4, "PR position": 2, "exact not last": 1}, "lean_wall_s": 3.2143030166625977, "lean_proc_s": 7.278073310852051, "nd_verify_s": 0.0407099723815918, "workers": 96, "chunk": 400}
|
| 62 |
+
{"utc": "2026-10-07T22:42:21Z", "samples": 12000, "parse_fail": 875, "distinct_checked": 10747, "both_ok": 9650, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 1097, "parse_reasons": {"unbound": 602, "expected (": 100, "expected )": 67, "name": 28, "formula": 24, "ORE box": 20, "no-eos": 13, "exact not last": 8, "expected :=": 5, "PR position": 5, "expected None": 1, "expected ;": 1, "expected fun": 1}, "lean_wall_s": 30.67842435836792, "lean_proc_s": 815.0815408229828, "nd_verify_s": 0.6310138702392578, "workers": 96, "chunk": 400}
|
| 63 |
+
{"utc": "2026-10-07T22:42:32Z", "samples": 1728, "parse_fail": 758, "distinct_checked": 902, "both_ok": 5, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 897, "parse_reasons": {"unbound": 515, "expected (": 102, "expected )": 59, "no-eos": 24, "formula": 17, "ORE box": 17, "name": 15, "exact not last": 5, "expected ;": 2, "PR position": 1, "expected ,": 1}, "lean_wall_s": 3.7992937564849854, "lean_proc_s": 8.879080533981323, "nd_verify_s": 0.05792570114135742, "workers": 96, "chunk": 400}
|
| 64 |
+
{"utc": "2026-10-07T22:42:43Z", "samples": 1704, "parse_fail": 693, "distinct_checked": 937, "both_ok": 0, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 937, "parse_reasons": {"unbound": 416, "expected (": 135, "formula": 32, "name": 30, "expected )": 29, "ORE box": 26, "PR position": 8, "no-eos": 8, "exact not last": 6, "expected :=": 1, "expected ;": 1, "eof": 1}, "lean_wall_s": 3.6159117221832275, "lean_proc_s": 9.164945840835571, "nd_verify_s": 0.05957460403442383, "workers": 96, "chunk": 400}
|
| 65 |
+
{"utc": "2026-10-07T22:42:51Z", "samples": 1704, "parse_fail": 743, "distinct_checked": 907, "both_ok": 1, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 906, "parse_reasons": {"unbound": 543, "expected (": 67, "expected )": 53, "name": 34, "ORE box": 12, "expected ;": 8, "PR position": 8, "no-eos": 6, "expected :=": 5, "formula": 5, "exact not last": 2}, "lean_wall_s": 3.475057363510132, "lean_proc_s": 8.682583093643188, "nd_verify_s": 0.0645906925201416, "workers": 96, "chunk": 400}
|
| 66 |
+
{"utc": "2026-10-07T22:44:57Z", "samples": 12000, "parse_fail": 907, "distinct_checked": 10757, "both_ok": 9785, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 972, "parse_reasons": {"unbound": 581, "expected (": 171, "name": 48, "formula": 26, "expected )": 25, "expected :=": 15, "ORE box": 14, "exact not last": 12, "PR position": 5, "expected ;": 5, "no-eos": 4, "expected =>": 1}, "lean_wall_s": 56.22629404067993, "lean_proc_s": 1485.9576659202576, "nd_verify_s": 0.6910834312438965, "workers": 96, "chunk": 400}
|
| 67 |
+
{"utc": "2026-10-07T22:45:06Z", "samples": 1680, "parse_fail": 711, "distinct_checked": 903, "both_ok": 5, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 898, "parse_reasons": {"unbound": 496, "expected (": 148, "name": 37, "formula": 9, "expected )": 6, "expected :=": 5, "PR position": 4, "exact not last": 3, "expected ;": 2, "expected =>": 1}, "lean_wall_s": 3.142735242843628, "lean_proc_s": 7.392163038253784, "nd_verify_s": 0.030957460403442383, "workers": 96, "chunk": 400}
|
| 68 |
+
{"utc": "2026-10-07T22:45:14Z", "samples": 1656, "parse_fail": 790, "distinct_checked": 812, "both_ok": 5, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 807, "parse_reasons": {"unbound": 491, "expected (": 146, "name": 52, "formula": 46, "ORE box": 24, "expected )": 17, "PR position": 6, "exact not last": 4, "no-eos": 2, "expected ;": 2}, "lean_wall_s": 3.3853070735931396, "lean_proc_s": 6.9160943031311035, "nd_verify_s": 0.035274505615234375, "workers": 96, "chunk": 400}
|
| 69 |
+
{"utc": "2026-10-07T22:45:23Z", "samples": 1656, "parse_fail": 851, "distinct_checked": 751, "both_ok": 6, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 745, "parse_reasons": {"unbound": 492, "expected (": 198, "name": 48, "expected )": 31, "expected :=": 29, "formula": 24, "ORE box": 13, "expected ;": 8, "exact not last": 5, "PR position": 2, "no-eos": 1}, "lean_wall_s": 3.0288801193237305, "lean_proc_s": 5.971598863601685, "nd_verify_s": 0.027647733688354492, "workers": 96, "chunk": 400}
|
| 70 |
+
{"utc": "2026-10-07T22:47:16Z", "samples": 12000, "parse_fail": 850, "distinct_checked": 10765, "both_ok": 9794, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 971, "parse_reasons": {"unbound": 527, "expected (": 103, "name": 85, "formula": 47, "ORE box": 37, "expected )": 26, "exact not last": 16, "PR position": 4, "expected ;": 2, "expected None": 1, "expected :=": 1, "expected fun": 1}, "lean_wall_s": 43.73678803443909, "lean_proc_s": 1165.9451277256012, "nd_verify_s": 0.6921589374542236, "workers": 96, "chunk": 400}
|
| 71 |
+
{"utc": "2026-10-07T22:47:24Z", "samples": 1584, "parse_fail": 737, "distinct_checked": 793, "both_ok": 0, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 793, "parse_reasons": {"unbound": 453, "expected (": 110, "name": 56, "ORE box": 53, "formula": 33, "expected )": 22, "exact not last": 4, "expected ;": 4, "expected ,": 1, "expected \u27e9": 1}, "lean_wall_s": 3.060340642929077, "lean_proc_s": 5.999511003494263, "nd_verify_s": 0.04212760925292969, "workers": 96, "chunk": 400}
|
| 72 |
+
{"utc": "2026-10-07T22:47:31Z", "samples": 1584, "parse_fail": 806, "distinct_checked": 728, "both_ok": 0, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 728, "parse_reasons": {"unbound": 504, "expected (": 123, "name": 62, "formula": 44, "ORE box": 30, "expected )": 28, "PR position": 6, "exact not last": 5, "expected ;": 4}, "lean_wall_s": 2.814277410507202, "lean_proc_s": 5.591063737869263, "nd_verify_s": 0.032677650451660156, "workers": 96, "chunk": 400}
|
| 73 |
+
{"utc": "2026-10-07T22:47:41Z", "samples": 1584, "parse_fail": 740, "distinct_checked": 785, "both_ok": 1, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 784, "parse_reasons": {"unbound": 434, "name": 84, "formula": 78, "expected (": 71, "expected )": 25, "ORE box": 23, "PR position": 12, "exact not last": 9, "expected ;": 2, "no-eos": 1, "expected fun": 1}, "lean_wall_s": 2.999161958694458, "lean_proc_s": 5.969151973724365, "nd_verify_s": 0.036293983459472656, "workers": 96, "chunk": 400}
|
| 74 |
+
{"utc": "2026-10-07T22:49:26Z", "samples": 12000, "parse_fail": 968, "distinct_checked": 10690, "both_ok": 9828, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 862, "parse_reasons": {"unbound": 657, "expected (": 160, "name": 65, "formula": 33, "expected ;": 16, "expected )": 15, "ORE box": 13, "exact not last": 5, "PR position": 3, "expected exact": 1}, "lean_wall_s": 36.72743010520935, "lean_proc_s": 967.8459165096283, "nd_verify_s": 0.7485952377319336, "workers": 96, "chunk": 400}
|
| 75 |
+
{"utc": "2026-10-07T22:49:35Z", "samples": 1584, "parse_fail": 709, "distinct_checked": 813, "both_ok": 1, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 812, "parse_reasons": {"unbound": 427, "expected (": 166, "name": 46, "ORE box": 27, "expected ;": 19, "expected )": 16, "formula": 5, "expected .elim": 1, "exact not last": 1, "no-eos": 1}, "lean_wall_s": 3.4337949752807617, "lean_proc_s": 7.075579643249512, "nd_verify_s": 0.050041913986206055, "workers": 96, "chunk": 400}
|
| 76 |
+
{"utc": "2026-10-07T22:49:41Z", "samples": 1560, "parse_fail": 935, "distinct_checked": 586, "both_ok": 0, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 586, "parse_reasons": {"unbound": 597, "expected (": 166, "name": 78, "formula": 45, "expected ;": 17, "expected )": 13, "PR position": 6, "ORE box": 6, "exact not last": 4, "expected ,": 1, "expected :=": 1, "expected None": 1}, "lean_wall_s": 2.30311918258667, "lean_proc_s": 3.52762770652771, "nd_verify_s": 0.020653247833251953, "workers": 96, "chunk": 400}
|
| 77 |
+
{"utc": "2026-10-07T22:49:48Z", "samples": 1560, "parse_fail": 738, "distinct_checked": 752, "both_ok": 0, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 752, "parse_reasons": {"unbound": 476, "expected (": 143, "name": 50, "formula": 40, "expected ;": 10, "expected )": 9, "ORE box": 6, "PR position": 2, "exact not last": 1, "expected fun": 1}, "lean_wall_s": 3.023484230041504, "lean_proc_s": 5.83513879776001, "nd_verify_s": 0.025631189346313477, "workers": 96, "chunk": 400}
|
| 78 |
+
{"utc": "2026-10-07T22:51:32Z", "samples": 12000, "parse_fail": 813, "distinct_checked": 10865, "both_ok": 9873, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 992, "parse_reasons": {"unbound": 525, "expected (": 130, "name": 51, "expected )": 29, "formula": 24, "ORE box": 22, "exact not last": 13, "no-eos": 5, "PR position": 5, "expected :=": 5, "trailing tokens": 2, "expected ;": 1, "expected ,": 1}, "lean_wall_s": 30.692853689193726, "lean_proc_s": 820.8768775463104, "nd_verify_s": 0.7073726654052734, "workers": 96, "chunk": 400}
|
| 79 |
+
{"utc": "2026-10-07T22:51:42Z", "samples": 1584, "parse_fail": 739, "distinct_checked": 790, "both_ok": 1, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 789, "parse_reasons": {"unbound": 453, "expected (": 120, "name": 55, "formula": 32, "expected )": 25, "ORE box": 19, "no-eos": 12, "expected :=": 9, "PR position": 8, "exact not last": 3, "expected ;": 2, "eof": 1}, "lean_wall_s": 3.1218409538269043, "lean_proc_s": 6.118686676025391, "nd_verify_s": 0.033074140548706055, "workers": 96, "chunk": 400}
|
| 80 |
+
{"utc": "2026-10-07T22:51:52Z", "samples": 1584, "parse_fail": 713, "distinct_checked": 803, "both_ok": 2, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 801, "parse_reasons": {"unbound": 450, "expected (": 116, "name": 48, "expected )": 35, "formula": 32, "ORE box": 10, "expected :=": 10, "PR position": 6, "exact not last": 4, "expected ;": 1, "trailing tokens": 1}, "lean_wall_s": 3.216684103012085, "lean_proc_s": 6.50602650642395, "nd_verify_s": 0.04164385795593262, "workers": 96, "chunk": 400}
|
| 81 |
+
{"utc": "2026-10-07T22:52:00Z", "samples": 1560, "parse_fail": 598, "distinct_checked": 900, "both_ok": 1, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 899, "parse_reasons": {"unbound": 342, "expected (": 138, "name": 48, "ORE box": 44, "expected )": 13, "exact not last": 6, "PR position": 3, "expected None": 2, "formula": 1, "expected ;": 1}, "lean_wall_s": 3.2267887592315674, "lean_proc_s": 7.199268579483032, "nd_verify_s": 0.036550045013427734, "workers": 96, "chunk": 400}
|
| 82 |
+
{"utc": "2026-10-07T22:57:28Z", "samples": 70912, "parse_fail": 11581, "distinct_checked": 46882, "both_ok": 29532, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 17350, "parse_reasons": {"unbound": 7875, "expected (": 1722, "name": 904, "formula": 259, "ORE box": 220, "expected ;": 177, "expected )": 145, "no-eos": 87, "exact not last": 83, "PR position": 61, "expected ,": 18, "trailing tokens": 9, "expected None": 7, "expected :=": 6, "expected \u27e9": 3, "expected fun": 2, "expected exact": 2, "expected False": 1}, "lean_wall_s": 127.70648622512817, "lean_proc_s": 10237.825224161148, "nd_verify_s": 2.7987918853759766, "workers": 96, "chunk": 400}
|
| 83 |
+
{"utc": "2026-10-07T22:57:38Z", "samples": 2000, "parse_fail": 2, "distinct_checked": 1998, "both_ok": 1978, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 20, "parse_reasons": {"ORE box": 1, "expected (": 1}, "lean_wall_s": 3.7903358936309814, "lean_proc_s": 17.225631952285767, "nd_verify_s": 0.10003995895385742, "workers": 96, "chunk": 400}
|
| 84 |
+
{"utc": "2026-10-07T23:02:31Z", "samples": 70912, "parse_fail": 11581, "distinct_checked": 46882, "both_ok": 29532, "nd_ok_lean_rej": 0, "nd_rej_lean_ok": 0, "both_rej": 17350, "parse_reasons": {"unbound": 7875, "expected (": 1722, "name": 904, "formula": 259, "ORE box": 220, "expected ;": 177, "expected )": 145, "no-eos": 87, "exact not last": 83, "PR position": 61, "expected ,": 18, "trailing tokens": 9, "expected None": 7, "expected :=": 6, "expected \u27e9": 3, "expected fun": 2, "expected exact": 2, "expected False": 1}, "lean_wall_s": 133.12740182876587, "lean_proc_s": 10788.583482027054, "nd_verify_s": 2.8955774307250977, "workers": 96, "chunk": 400}
|
runs/590_control-untied-198_s0/pretrain.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:aa73e00d8ebdc7841c86809e1e055b4989254fe4db63372db78d895e9e4de071
|
| 3 |
+
size 38268899
|
runs/590_control-untied-198_s0/source/harness.py
ADDED
|
@@ -0,0 +1,529 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
# imports the fork (model, tokenizer, train, ...), which is not resolvable here
|
| 3 |
+
# pyright: reportMissingImports=false
|
| 4 |
+
"""FROZEN harness for the test-time-loops autoresearch loop. The research agent does not edit it.
|
| 5 |
+
|
| 6 |
+
Forked from ../autoresearch/harness.py (sha 2aa52f23e7ec) on 2026-10-07. What changed: the RL
|
| 7 |
+
stage belongs to the agent, behind a wall-clock budget and a sampling oracle, and the metric is
|
| 8 |
+
measured with the model's test-time compute knob (KNOB) set to EVAL_LEVEL.
|
| 9 |
+
|
| 10 |
+
One timed run = harness owns the clocks, the oracle and the eval; `pretrain.py` (agent-editable)
|
| 11 |
+
owns pretraining and RL:
|
| 12 |
+
|
| 13 |
+
1. load frozen data (Dan's cap-6 Stage-1 file, ladder RL-target pool, ladder transfer pool)
|
| 14 |
+
2. call pretrain.pretrain(train_recs, PRETRAIN_SECS, ...) -> (model, tok, stats); charged for
|
| 15 |
+
everything including model construction and torch.compile. Overrun past the grace -> timeout.
|
| 16 |
+
3. check n_params <= PARAM_CAP, tok.mode == TOK_MODE and that the model has the KNOB attribute
|
| 17 |
+
4. call pretrain.rl(model, tok, targets, train_recs, oracle, RL_SECS, ...): expert iteration in
|
| 18 |
+
whatever form the agent likes. The targets are statements only, and `oracle.sample` is the
|
| 19 |
+
only way to turn them into training data: it samples the model's own attempts and returns
|
| 20 |
+
the ones Lean AND nd_verify accept. RL_SECS is wall-clock inside rl() minus the time the
|
| 21 |
+
oracle spent in the Lean gate (pod CPUs differ; GPU work is what is rationed). Overrun past
|
| 22 |
+
the grace, or an oracle call after the budget is spent -> timeout.
|
| 23 |
+
5. eval on the dev half of the transfer pool with the harness setting the model's KNOB:
|
| 24 |
+
METRIC = dev theorems solved at L_true >= 7 with EVAL_K attempts at EVAL_LEVEL.
|
| 25 |
+
The same eval at DIAG_LEVEL is recorded as a diagnostic, not scored.
|
| 26 |
+
6. SIGALRM at TOTAL_SECS from process start kills everything -> status "timeout"
|
| 27 |
+
|
| 28 |
+
The holdout half of the transfer pool is never scored here. Everything is written to --out.
|
| 29 |
+
|
| 30 |
+
python3 harness.py --out artifacts/x # on the pod (see pod.py / remote.sh)
|
| 31 |
+
ND_FORK=... python3 harness.py --out /tmp/smoke --smoke # CPU, ~1 min, plumbing check
|
| 32 |
+
"""
|
| 33 |
+
|
| 34 |
+
import argparse
|
| 35 |
+
import collections
|
| 36 |
+
import hashlib
|
| 37 |
+
import importlib
|
| 38 |
+
import json
|
| 39 |
+
import os
|
| 40 |
+
import random
|
| 41 |
+
import shutil
|
| 42 |
+
import signal
|
| 43 |
+
import sys
|
| 44 |
+
import time
|
| 45 |
+
import traceback
|
| 46 |
+
from pathlib import Path
|
| 47 |
+
|
| 48 |
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
| 49 |
+
FORK = os.environ.get("ND_FORK", "/workspace/nd-takehome")
|
| 50 |
+
sys.path.insert(0, HERE)
|
| 51 |
+
sys.path.insert(
|
| 52 |
+
0, FORK
|
| 53 |
+
) # fork first: its train.py provides batch/loss_on; the agent's file is pretrain.py
|
| 54 |
+
import torch # noqa: E402
|
| 55 |
+
from eval_set import judge, wilson # noqa: E402
|
| 56 |
+
from sample import generate # noqa: E402
|
| 57 |
+
|
| 58 |
+
# ---------------------------------------------------------------- the rules
|
| 59 |
+
# (hash-checked into every result)
|
| 60 |
+
TOTAL_SECS = int(
|
| 61 |
+
os.environ.get("ND_TOTAL_SECS", 5400)
|
| 62 |
+
) # process start -> summary written; SIGALRM past this. env: benchmarks only
|
| 63 |
+
PRETRAIN_SECS = 300 # agent's pretraining budget, compile included
|
| 64 |
+
PRETRAIN_GRACE = 15 # seconds the agent may overrun before the run is a "timeout"
|
| 65 |
+
RL_SECS = 1800 # agent's RL budget: wall-clock inside rl() minus the oracle's Lean-gate time
|
| 66 |
+
RL_GRACE = 60
|
| 67 |
+
PARAM_CAP = 10_000_000 # unique trainable params, embeddings included; checked after RL too
|
| 68 |
+
N_TARGETS, TEMP, MAX_NEW, SAMPLE_BATCH = 1500, 0.8, 512, 1024
|
| 69 |
+
EVAL_K = 64
|
| 70 |
+
# The model's test-time compute knob: an integer attribute the harness sets before each eval and
|
| 71 |
+
# rl() sets before oracle calls. A sibling loop for another architecture changes these three
|
| 72 |
+
# constants and nothing else in this file.
|
| 73 |
+
KNOB = "n_loops" # loops of the shared core
|
| 74 |
+
EVAL_LEVEL = 8 # the metric: the knob's value during the scored eval
|
| 75 |
+
DIAG_LEVEL = 2 # the same eval again at this value, recorded but not scored
|
| 76 |
+
TOK_MODE = "lean_seq" # required tokenizer mode (fork lean_tok.py)
|
| 77 |
+
DEV = "cuda" if torch.cuda.is_available() else "cpu"
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def read(fn):
|
| 81 |
+
return [json.loads(line) for line in open(fn) if line.strip()]
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
def log(msg):
|
| 85 |
+
print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True)
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def sha(fn):
|
| 89 |
+
return hashlib.sha256(open(fn, "rb").read()).hexdigest()[:12]
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
class Timeout(Exception):
|
| 93 |
+
pass
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
class BudgetSpent(Timeout):
|
| 97 |
+
"""rl() asked the oracle for samples after RL_SECS was used up."""
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
# ---------------------------------------------------------------- bookkeeping
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def save_checkpoint(a, model, tok, stage):
|
| 104 |
+
"""Publish complete weight snapshots atomically; these are not optimizer-resume files."""
|
| 105 |
+
path = Path(a.out) / f"{stage}.pt"
|
| 106 |
+
pending = path.with_suffix(".pt.tmp")
|
| 107 |
+
try:
|
| 108 |
+
torch.save(
|
| 109 |
+
{
|
| 110 |
+
"state": model.state_dict(),
|
| 111 |
+
"tok_mode": getattr(tok, "mode", None),
|
| 112 |
+
"stage": stage,
|
| 113 |
+
"seed": a.seed,
|
| 114 |
+
"harness_sha": sha(f"{HERE}/harness.py"),
|
| 115 |
+
"pretrain_sha": sha(f"{HERE}/pretrain.py"),
|
| 116 |
+
},
|
| 117 |
+
pending,
|
| 118 |
+
)
|
| 119 |
+
os.replace(pending, path)
|
| 120 |
+
finally:
|
| 121 |
+
pending.unlink(missing_ok=True)
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
def lstar(solved_names, recs, need=5):
|
| 125 |
+
ls_solved = [r["n_lines"] for r in recs if r["name"] in solved_names]
|
| 126 |
+
ge = {L: sum(1 for x in ls_solved if x >= L) for L in range(7, 15)}
|
| 127 |
+
return max([L for L, c in ge.items() if c >= need], default=0), ge
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
def by_bin(solved_names, recs):
|
| 131 |
+
by = collections.defaultdict(lambda: [0, 0])
|
| 132 |
+
for r in recs:
|
| 133 |
+
by[r["n_lines"]][0] += r["name"] in solved_names
|
| 134 |
+
by[r["n_lines"]][1] += 1
|
| 135 |
+
return {
|
| 136 |
+
str(L): {"solved": k, "n": n, "rate": k / n, "ci": list(wilson(k, n))}
|
| 137 |
+
for L, (k, n) in sorted(by.items())
|
| 138 |
+
}
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
def gate_seconds():
|
| 142 |
+
"""Seconds the fork's lean_gate has logged so far (Lean wall-clock plus nd_verify)."""
|
| 143 |
+
fn = os.environ.get("LEAN_GATE_LOG")
|
| 144 |
+
if not fn or not os.path.exists(fn):
|
| 145 |
+
return 0.0
|
| 146 |
+
return sum(rec["lean_wall_s"] + rec["nd_verify_s"] for rec in read(fn))
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
def sample_and_judge(model, tok, recs, k, temperature, seed, batch, max_new):
|
| 150 |
+
prompts = [r["prompt"] for r in recs for _ in range(k)]
|
| 151 |
+
t0, g0 = time.time(), gate_seconds()
|
| 152 |
+
flat = generate(
|
| 153 |
+
model,
|
| 154 |
+
tok,
|
| 155 |
+
prompts,
|
| 156 |
+
greedy=(k == 1 and temperature == 0),
|
| 157 |
+
temperature=temperature,
|
| 158 |
+
batch=batch,
|
| 159 |
+
seed=seed,
|
| 160 |
+
max_new=max_new,
|
| 161 |
+
)
|
| 162 |
+
secs, gate = time.time() - t0, gate_seconds() - g0
|
| 163 |
+
rows = judge(recs, [flat[i * k : (i + 1) * k] for i in range(len(recs))], "n_lines")
|
| 164 |
+
return rows, {
|
| 165 |
+
"samples": len(prompts),
|
| 166 |
+
"secs": secs,
|
| 167 |
+
"gate_secs": gate,
|
| 168 |
+
"samples_per_sec": len(prompts) / secs,
|
| 169 |
+
}
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
def greedy_rate(model, tok, recs):
|
| 173 |
+
rows, _ = sample_and_judge(model, tok, recs, 1, 0, 0, SAMPLE_BATCH, MAX_NEW)
|
| 174 |
+
return sum(r["solved"] for r in rows) / max(len(rows), 1)
|
| 175 |
+
|
| 176 |
+
|
| 177 |
+
def dev_split(transfer):
|
| 178 |
+
return [r for r in transfer if int(hashlib.sha1(r["key"].encode()).hexdigest(), 16) % 2 == 0]
|
| 179 |
+
|
| 180 |
+
|
| 181 |
+
def holdout_split(transfer):
|
| 182 |
+
"""Never scored by the loop. Score final.pt on this offline when checking in."""
|
| 183 |
+
return [r for r in transfer if int(hashlib.sha1(r["key"].encode()).hexdigest(), 16) % 2 == 1]
|
| 184 |
+
|
| 185 |
+
|
| 186 |
+
# ---------------------------------------------------------------- the RL oracle
|
| 187 |
+
|
| 188 |
+
|
| 189 |
+
class Oracle:
|
| 190 |
+
"""What rl() gets instead of data: the model's own attempts at the RL targets, filtered to
|
| 191 |
+
the ones Lean AND nd_verify accept. It also keeps the RL clock and the record of every call.
|
| 192 |
+
"""
|
| 193 |
+
|
| 194 |
+
def __init__(self, a, tok, targets):
|
| 195 |
+
self.a, self.tok = a, tok
|
| 196 |
+
self.targets = {t["name"]: t for t in targets}
|
| 197 |
+
self.found = collections.defaultdict(dict) # name -> {proof text: record}
|
| 198 |
+
self.calls = []
|
| 199 |
+
self.t0 = self.last = time.time()
|
| 200 |
+
self.uncharged = 0.0
|
| 201 |
+
|
| 202 |
+
def used(self):
|
| 203 |
+
"""RL seconds charged so far: wall-clock since rl() started, minus Lean-gate time."""
|
| 204 |
+
return time.time() - self.t0 - self.uncharged
|
| 205 |
+
|
| 206 |
+
def left(self):
|
| 207 |
+
return self.a.rl_secs - self.used()
|
| 208 |
+
|
| 209 |
+
def sample(self, model, targets, k, temperature=TEMP, batch=SAMPLE_BATCH):
|
| 210 |
+
"""k attempts at each of `targets` (records rl() was given, any subset, in any order) from
|
| 211 |
+
`model` as it is now, in eval mode, at whatever its KNOB is set to. Returns one
|
| 212 |
+
dict per target, in order: name, prompt, n_lines, proofs (the distinct accepted proof
|
| 213 |
+
texts), n_ok (accepted attempts) and n_tried. Rejected attempts are not returned.
|
| 214 |
+
Raises BudgetSpent, which ends the run as a timeout, once the budget is used up."""
|
| 215 |
+
if self.left() <= 0:
|
| 216 |
+
raise BudgetSpent(f"oracle called with {self.used():.0f}s of {self.a.rl_secs}s used")
|
| 217 |
+
recs = [self.targets[t["name"]] for t in targets]
|
| 218 |
+
call = len(self.calls) + 1
|
| 219 |
+
model.eval()
|
| 220 |
+
rows, samp = sample_and_judge(
|
| 221 |
+
model, self.tok, recs, k, temperature, self.a.seed * 1000 + call, batch, self.a.max_new
|
| 222 |
+
)
|
| 223 |
+
self.uncharged += samp["gate_secs"]
|
| 224 |
+
new = 0
|
| 225 |
+
for t, row in zip(recs, rows, strict=True):
|
| 226 |
+
for p, wl, pl in zip(
|
| 227 |
+
row["proofs"], row["written_lens"], row["pruned_lens"], strict=True
|
| 228 |
+
):
|
| 229 |
+
if p not in self.found[t["name"]]:
|
| 230 |
+
self.found[t["name"]][p] = {
|
| 231 |
+
"proof": p,
|
| 232 |
+
"written": wl,
|
| 233 |
+
"pruned": pl,
|
| 234 |
+
"round": call,
|
| 235 |
+
}
|
| 236 |
+
new += 1
|
| 237 |
+
solved = {n for n, d in self.found.items() if d}
|
| 238 |
+
ls, ge = lstar(solved, list(self.targets.values()))
|
| 239 |
+
now = time.time()
|
| 240 |
+
self.calls.append(
|
| 241 |
+
{
|
| 242 |
+
"round": call,
|
| 243 |
+
"level": getattr(model, KNOB, None),
|
| 244 |
+
"k": k,
|
| 245 |
+
"n_targets": len(recs),
|
| 246 |
+
"sample_acc": sum(x["n_ok"] for x in rows) / max(samp["samples"], 1),
|
| 247 |
+
"new_proofs": new,
|
| 248 |
+
"targets_solved": len(solved),
|
| 249 |
+
"lstar": ls,
|
| 250 |
+
"ge": ge,
|
| 251 |
+
"mix_rl_records": sum(len(x["proofs"]) for x in rows), # proofs handed back
|
| 252 |
+
"sampling": samp,
|
| 253 |
+
"secs": now - self.last, # since the previous call returned: includes rl()'s work
|
| 254 |
+
"rl_secs_used": self.used(),
|
| 255 |
+
}
|
| 256 |
+
)
|
| 257 |
+
self.last = now
|
| 258 |
+
c = self.calls[-1]
|
| 259 |
+
log(
|
| 260 |
+
f"ei r{call}: acc {c['sample_acc']:.4f}, +{new} proofs, "
|
| 261 |
+
f"solved {len(solved)}/{len(self.targets)}, L*={ls}, {KNOB} {c['level']}, "
|
| 262 |
+
f"{len(recs)} targets x {k}, {samp['samples_per_sec']:.0f} samples/s, "
|
| 263 |
+
f"RL clock {c['rl_secs_used']:.0f}/{self.a.rl_secs}s"
|
| 264 |
+
)
|
| 265 |
+
return [
|
| 266 |
+
{
|
| 267 |
+
"name": t["name"],
|
| 268 |
+
"prompt": t["prompt"],
|
| 269 |
+
"n_lines": t["n_lines"],
|
| 270 |
+
"proofs": list(row["proofs"]),
|
| 271 |
+
"n_ok": row["n_ok"],
|
| 272 |
+
"n_tried": row["n_tried"],
|
| 273 |
+
}
|
| 274 |
+
for t, row in zip(recs, rows, strict=True)
|
| 275 |
+
]
|
| 276 |
+
|
| 277 |
+
|
| 278 |
+
# ---------------------------------------------------------------- stages
|
| 279 |
+
|
| 280 |
+
|
| 281 |
+
def stage_rl(a, agent, tok, model, targets, train_recs):
|
| 282 |
+
oracle = Oracle(a, tok, targets)
|
| 283 |
+
# statements only: what the agent's RL may know about a target
|
| 284 |
+
visible = [{"name": t["name"], "prompt": t["prompt"], "n_lines": t["n_lines"]} for t in targets]
|
| 285 |
+
stats = agent.rl(model, tok, visible, train_recs, oracle, a.rl_secs, DEV, a.seed, log)
|
| 286 |
+
if DEV == "cuda":
|
| 287 |
+
torch.cuda.synchronize()
|
| 288 |
+
used = oracle.used()
|
| 289 |
+
model.eval()
|
| 290 |
+
with open(f"{a.out}/found_targets.jsonl", "w") as f:
|
| 291 |
+
for t in targets:
|
| 292 |
+
for x in oracle.found[t["name"]].values():
|
| 293 |
+
f.write(
|
| 294 |
+
json.dumps({"name": t["name"], "thm": t["thm"], "L_true": t["n_lines"], **x})
|
| 295 |
+
+ "\n"
|
| 296 |
+
)
|
| 297 |
+
log(f"rl returned: {used:.0f}s of {a.rl_secs}s charged, {oracle.uncharged:.0f}s Lean gate")
|
| 298 |
+
return {
|
| 299 |
+
"rounds": oracle.calls,
|
| 300 |
+
"secs": time.time() - oracle.t0,
|
| 301 |
+
"rl_secs_used": used,
|
| 302 |
+
"gate_secs": oracle.uncharged,
|
| 303 |
+
"targets": len(targets),
|
| 304 |
+
"stats": dict(stats or {}),
|
| 305 |
+
}
|
| 306 |
+
|
| 307 |
+
|
| 308 |
+
def eval_at(a, tok, model, dev_pool, level):
|
| 309 |
+
setattr(model, KNOB, level)
|
| 310 |
+
rows, samp = sample_and_judge(
|
| 311 |
+
model, tok, dev_pool, a.eval_k, TEMP, a.seed * 1000 + 777, SAMPLE_BATCH, a.max_new
|
| 312 |
+
)
|
| 313 |
+
solved = {r["name"] for r in rows if r["solved"]}
|
| 314 |
+
ls, ge = lstar(solved, dev_pool)
|
| 315 |
+
out = {
|
| 316 |
+
"level": level,
|
| 317 |
+
"attempts_per_theorem": a.eval_k,
|
| 318 |
+
"solved": len(solved),
|
| 319 |
+
"n": len(dev_pool),
|
| 320 |
+
"solved_ge7": ge[7],
|
| 321 |
+
"lstar": ls,
|
| 322 |
+
"ge": ge,
|
| 323 |
+
"by_bin": by_bin(solved, dev_pool),
|
| 324 |
+
"sample_acc": sum(r["n_ok"] for r in rows) / max(samp["samples"], 1),
|
| 325 |
+
"max_pruned": max((max(r["pruned_lens"]) for r in rows if r["solved"]), default=0),
|
| 326 |
+
"sampling": samp,
|
| 327 |
+
}
|
| 328 |
+
log(
|
| 329 |
+
f"eval at {KNOB} {level}: solved {len(solved)}/{len(dev_pool)}, ge7 {ge[7]}, L*={ls}, "
|
| 330 |
+
f"per-sample {out['sample_acc']:.4f}, {samp['samples_per_sec']:.0f} samples/s"
|
| 331 |
+
)
|
| 332 |
+
return out, rows
|
| 333 |
+
|
| 334 |
+
|
| 335 |
+
def stage_eval(a, tok, model, dev_pool, heldout_recs):
|
| 336 |
+
out, rows = eval_at(a, tok, model, dev_pool, a.eval_level)
|
| 337 |
+
out["heldout_greedy"] = greedy_rate(model, tok, heldout_recs)
|
| 338 |
+
with open(f"{a.out}/eval_dev.jsonl", "w") as f:
|
| 339 |
+
for r in rows:
|
| 340 |
+
keys = ("name", "n_lines", "solved", "n_ok", "proofs", "written_lens", "pruned_lens")
|
| 341 |
+
f.write(json.dumps({k: r[k] for k in keys}) + "\n")
|
| 342 |
+
diag, _ = eval_at(a, tok, model, dev_pool, a.diag_level)
|
| 343 |
+
out["diag"] = diag
|
| 344 |
+
setattr(model, KNOB, a.eval_level)
|
| 345 |
+
log(
|
| 346 |
+
f"eval: solved {out['solved']}/{len(dev_pool)}, ge7 {out['solved_ge7']} at "
|
| 347 |
+
f"{KNOB} {a.eval_level} ({diag['solved_ge7']} at {a.diag_level}), L*={out['lstar']}, "
|
| 348 |
+
f"max pruned {out['max_pruned']}, held-out greedy {out['heldout_greedy']:.3f}"
|
| 349 |
+
)
|
| 350 |
+
return out
|
| 351 |
+
|
| 352 |
+
|
| 353 |
+
def lean_gate_totals():
|
| 354 |
+
"""Sum the fork lean_gate's per-generate() log lines (remote.sh sets LEAN_GATE_LOG)."""
|
| 355 |
+
fn = os.environ.get("LEAN_GATE_LOG")
|
| 356 |
+
if not fn or not os.path.exists(fn):
|
| 357 |
+
return None
|
| 358 |
+
keys = ("samples", "parse_fail", "distinct_checked", "both_ok", "nd_ok_lean_rej")
|
| 359 |
+
tot = dict.fromkeys(keys + ("nd_rej_lean_ok", "lean_wall_s"), 0)
|
| 360 |
+
for rec in read(fn):
|
| 361 |
+
for k in tot:
|
| 362 |
+
tot[k] += rec[k]
|
| 363 |
+
return tot
|
| 364 |
+
|
| 365 |
+
|
| 366 |
+
RULES = (
|
| 367 |
+
"TOTAL_SECS",
|
| 368 |
+
"PRETRAIN_SECS",
|
| 369 |
+
"RL_SECS",
|
| 370 |
+
"PARAM_CAP",
|
| 371 |
+
"N_TARGETS",
|
| 372 |
+
"EVAL_K",
|
| 373 |
+
"KNOB",
|
| 374 |
+
"EVAL_LEVEL",
|
| 375 |
+
"DIAG_LEVEL",
|
| 376 |
+
"TOK_MODE",
|
| 377 |
+
)
|
| 378 |
+
|
| 379 |
+
|
| 380 |
+
def write_summary(a, status, **kw):
|
| 381 |
+
s = {
|
| 382 |
+
"status": status,
|
| 383 |
+
"metric": kw.get("eval", {}).get("solved_ge7"),
|
| 384 |
+
"metric_diag": kw.get("eval", {}).get("diag", {}).get("solved_ge7"),
|
| 385 |
+
"metric_rl": (kw.get("rl", {}).get("rounds") or [{}])[-1].get(
|
| 386 |
+
"targets_solved"
|
| 387 |
+
), # secondary: cumulative RL targets solved
|
| 388 |
+
"harness_sha": sha(f"{HERE}/harness.py"),
|
| 389 |
+
"pretrain_sha": sha(f"{HERE}/pretrain.py"),
|
| 390 |
+
"device": DEV,
|
| 391 |
+
"gpu": torch.cuda.get_device_name(0) if DEV == "cuda" else "cpu",
|
| 392 |
+
"torch": torch.__version__,
|
| 393 |
+
"lean_gate": lean_gate_totals(),
|
| 394 |
+
"total_secs": time.time() - a.T0,
|
| 395 |
+
"rules": {k: globals()[k] for k in RULES},
|
| 396 |
+
**kw,
|
| 397 |
+
}
|
| 398 |
+
with open(f"{a.out}/summary.json", "w") as f:
|
| 399 |
+
json.dump(s, f, indent=1)
|
| 400 |
+
log(
|
| 401 |
+
f"SUMMARY status={status} metric={s['metric']} metric_diag={s['metric_diag']} "
|
| 402 |
+
f"metric_rl={s['metric_rl']} total {s['total_secs']:.0f}s"
|
| 403 |
+
)
|
| 404 |
+
return s
|
| 405 |
+
|
| 406 |
+
|
| 407 |
+
def main():
|
| 408 |
+
ap = argparse.ArgumentParser()
|
| 409 |
+
ap.add_argument("--out", required=True)
|
| 410 |
+
ap.add_argument("--seed", type=int, default=0)
|
| 411 |
+
ap.add_argument(
|
| 412 |
+
"--pretrain-only", action="store_true", help="save pretrain.pt and stop before eval/RL"
|
| 413 |
+
)
|
| 414 |
+
ap.add_argument(
|
| 415 |
+
"--smoke", action="store_true", help="tiny everything, for a CPU plumbing check"
|
| 416 |
+
)
|
| 417 |
+
a = ap.parse_args()
|
| 418 |
+
a.pretrain_secs, a.rl_secs, a.eval_k, a.max_new, a.n_targets = (
|
| 419 |
+
PRETRAIN_SECS,
|
| 420 |
+
RL_SECS,
|
| 421 |
+
EVAL_K,
|
| 422 |
+
MAX_NEW,
|
| 423 |
+
N_TARGETS,
|
| 424 |
+
)
|
| 425 |
+
a.eval_level, a.diag_level = EVAL_LEVEL, DIAG_LEVEL
|
| 426 |
+
if a.smoke:
|
| 427 |
+
a.pretrain_secs, a.rl_secs, a.eval_k, a.max_new, a.n_targets = 10, 30, 2, 120, 8
|
| 428 |
+
os.makedirs(a.out, exist_ok=True)
|
| 429 |
+
# Preserve discarded variants too: weights alone do not specify the model architecture.
|
| 430 |
+
source_dir = Path(a.out) / "source"
|
| 431 |
+
source_dir.mkdir(exist_ok=True)
|
| 432 |
+
for filename in ("pretrain.py", "harness.py", "remote.sh"):
|
| 433 |
+
shutil.copy2(Path(HERE) / filename, source_dir / filename)
|
| 434 |
+
a.T0 = time.time()
|
| 435 |
+
torch.manual_seed(a.seed)
|
| 436 |
+
|
| 437 |
+
def on_alarm(signum, frame):
|
| 438 |
+
raise Timeout()
|
| 439 |
+
|
| 440 |
+
signal.signal(signal.SIGALRM, on_alarm)
|
| 441 |
+
signal.alarm(TOTAL_SECS if not a.smoke else 900)
|
| 442 |
+
|
| 443 |
+
train_recs = read(f"{FORK}/data/train.jsonl")
|
| 444 |
+
assert all(r["n_lines"] <= 6 for r in train_recs), "pretrain file violates cap 6"
|
| 445 |
+
heldout_all = read(f"{FORK}/data/heldout.jsonl")
|
| 446 |
+
# Length-ordered file: take 2000 evenly spaced records, so every length 2-6 is covered.
|
| 447 |
+
heldout_all = [heldout_all[i * len(heldout_all) // 2000] for i in range(2000)]
|
| 448 |
+
heldout_recs = heldout_all if not a.smoke else heldout_all[:16]
|
| 449 |
+
targets_all = read(f"{FORK}/data/ladder/rl_targets.jsonl")
|
| 450 |
+
transfer = read(f"{FORK}/data/ladder/transfer.jsonl")
|
| 451 |
+
dev_pool = dev_split(transfer) if not a.smoke else dev_split(transfer)[:8]
|
| 452 |
+
targets = random.Random(a.seed).sample(targets_all, min(a.n_targets, len(targets_all)))
|
| 453 |
+
tk, dk, hk = (
|
| 454 |
+
{t["key"] for t in targets},
|
| 455 |
+
{r["key"] for r in dev_pool},
|
| 456 |
+
{r["key"] for r in train_recs},
|
| 457 |
+
)
|
| 458 |
+
assert not (tk & dk) and not (tk & hk) and not (dk & hk), "renaming-class leak between pools"
|
| 459 |
+
log(
|
| 460 |
+
f"pools: train {len(train_recs):,}, targets {len(targets)}, dev-transfer {len(dev_pool)}, "
|
| 461 |
+
f"held-out {len(heldout_recs)}; "
|
| 462 |
+
f"device {DEV}; harness {sha(f'{HERE}/harness.py')} pretrain {sha(f'{HERE}/pretrain.py')}"
|
| 463 |
+
)
|
| 464 |
+
|
| 465 |
+
pre = {}
|
| 466 |
+
try:
|
| 467 |
+
agent = importlib.import_module("pretrain")
|
| 468 |
+
t0 = time.time()
|
| 469 |
+
model, tok, pre = agent.pretrain(train_recs, a.pretrain_secs, DEV, a.seed, log)
|
| 470 |
+
if DEV == "cuda":
|
| 471 |
+
torch.cuda.synchronize()
|
| 472 |
+
pre = dict(pre or {})
|
| 473 |
+
pre["secs"] = time.time() - t0
|
| 474 |
+
model.eval()
|
| 475 |
+
n_params = sum(p.numel() for p in model.parameters())
|
| 476 |
+
pre["n_params"] = n_params
|
| 477 |
+
log(
|
| 478 |
+
f"pretrain returned after {pre['secs']:.0f}s (budget {a.pretrain_secs}s), "
|
| 479 |
+
f"{n_params:,} params"
|
| 480 |
+
)
|
| 481 |
+
if pre["secs"] > a.pretrain_secs + PRETRAIN_GRACE and not a.smoke:
|
| 482 |
+
over = pre["secs"] - a.pretrain_secs
|
| 483 |
+
write_summary(
|
| 484 |
+
a, "timeout", pretrain=pre, reason=f"pretrain overran budget by {over:.0f}s"
|
| 485 |
+
)
|
| 486 |
+
return
|
| 487 |
+
if getattr(tok, "mode", None) != TOK_MODE:
|
| 488 |
+
write_summary(a, "crash", pretrain=pre, reason=f"tok.mode {tok.mode!r} != {TOK_MODE!r}")
|
| 489 |
+
return
|
| 490 |
+
if n_params > PARAM_CAP:
|
| 491 |
+
write_summary(a, "crash", pretrain=pre, reason=f"{n_params} params > cap {PARAM_CAP}")
|
| 492 |
+
return
|
| 493 |
+
if not isinstance(getattr(model, KNOB, None), int):
|
| 494 |
+
reason = f"model has no integer {KNOB} for the harness to set at eval"
|
| 495 |
+
write_summary(a, "crash", pretrain=pre, reason=reason)
|
| 496 |
+
return
|
| 497 |
+
save_checkpoint(a, model, tok, "pretrain")
|
| 498 |
+
if a.pretrain_only:
|
| 499 |
+
write_summary(a, "ok", stage="pretrain_only", pretrain=pre)
|
| 500 |
+
return
|
| 501 |
+
pre["heldout_greedy"] = greedy_rate(model, tok, heldout_recs)
|
| 502 |
+
log(f"pretrain held-out greedy {pre['heldout_greedy']:.3f}")
|
| 503 |
+
rl = stage_rl(a, agent, tok, model, targets, train_recs)
|
| 504 |
+
if rl["rl_secs_used"] > a.rl_secs + RL_GRACE and not a.smoke:
|
| 505 |
+
over = rl["rl_secs_used"] - a.rl_secs
|
| 506 |
+
reason = f"rl overran budget by {over:.0f}s"
|
| 507 |
+
write_summary(a, "timeout", pretrain=pre, rl=rl, reason=reason)
|
| 508 |
+
return
|
| 509 |
+
n_after = sum(p.numel() for p in model.parameters())
|
| 510 |
+
if n_after > PARAM_CAP:
|
| 511 |
+
reason = f"{n_after} params > cap {PARAM_CAP} after rl"
|
| 512 |
+
write_summary(a, "crash", pretrain=pre, rl=rl, reason=reason)
|
| 513 |
+
return
|
| 514 |
+
save_checkpoint(a, model, tok, "final")
|
| 515 |
+
ev = stage_eval(a, tok, model, dev_pool, heldout_recs)
|
| 516 |
+
write_summary(a, "ok", pretrain=pre, rl=rl, eval=ev)
|
| 517 |
+
except BudgetSpent as e:
|
| 518 |
+
write_summary(a, "timeout", pretrain=pre, reason=f"rl: {e}")
|
| 519 |
+
except Timeout:
|
| 520 |
+
write_summary(a, "timeout", pretrain=pre, reason=f"exceeded TOTAL_SECS={TOTAL_SECS}")
|
| 521 |
+
except Exception:
|
| 522 |
+
traceback.print_exc()
|
| 523 |
+
write_summary(a, "crash", pretrain=pre, reason=traceback.format_exc()[-2000:])
|
| 524 |
+
finally:
|
| 525 |
+
signal.alarm(0)
|
| 526 |
+
|
| 527 |
+
|
| 528 |
+
if __name__ == "__main__":
|
| 529 |
+
main()
|
runs/590_control-untied-198_s0/source/pretrain.py
ADDED
|
@@ -0,0 +1,594 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# imports the fork (model, tokenizer, train, ...), which is not resolvable here
|
| 2 |
+
# pyright: reportMissingImports=false
|
| 3 |
+
"""AGENT-EDITABLE: pretraining and RL. This is the only file the autoresearch agent changes.
|
| 4 |
+
|
| 5 |
+
Contract with harness.py (frozen):
|
| 6 |
+
|
| 7 |
+
pretrain(train_recs, budget_secs, device, seed, log) -> (model, tok, stats)
|
| 8 |
+
|
| 9 |
+
train_recs list of dicts {prompt, proof, n_lines<=6, key, ...}; the frozen cap-6 Stage-1 data
|
| 10 |
+
budget_secs wall-clock you are charged for, from call to return, INCLUDING model construction
|
| 11 |
+
and torch.compile. Overrun by more than the harness grace -> the run is a timeout.
|
| 12 |
+
model nn.Module; <= PARAM_CAP unique params (checked again after rl). The harness samples
|
| 13 |
+
from it with the fork's sample.generate, so it must provide:
|
| 14 |
+
- forward(idx) -> logits (B,T,V) and forward(idx, pos=, mask=, caches=) with a
|
| 15 |
+
per-block KV cache dict, exactly like the fork's GPT (model.blocks must exist)
|
| 16 |
+
- an integer `n_loops` (the harness's KNOB): the loop count applied in eval
|
| 17 |
+
mode. The harness sets it to EVAL_LEVEL for the scored eval and to DIAG_LEVEL
|
| 18 |
+
for the diagnostic one.
|
| 19 |
+
tok must provide encode_prompt, encode_proof, decode, shift_abs, pad, eos, mode, and
|
| 20 |
+
mode must be "lean_seq" (harness TOK_MODE): Lean text in and out, Lean-gated samples.
|
| 21 |
+
Decoder rule: decode() is deterministic and single-pass; it may not call the
|
| 22 |
+
verifier or try alternatives.
|
| 23 |
+
stats dict recorded in summary.json and copied into results.json. MUST contain `curve`:
|
| 24 |
+
a list of {secs, step, train_loss, val_loss} taken every ~CURVE_EVERY seconds
|
| 25 |
+
(see log_point below), so a run's trajectory survives in git. Add anything else.
|
| 26 |
+
val_loss is on VAL_N records held out from train_recs by this file; keep that.
|
| 27 |
+
|
| 28 |
+
rl(model, tok, targets, train_recs, oracle, budget_secs, device, seed, log) -> stats | None
|
| 29 |
+
|
| 30 |
+
targets list of dicts {name, prompt, n_lines}: the 1,500 RL target statements. No proofs.
|
| 31 |
+
oracle oracle.sample(model, targets_subset, k, temperature=0.8, batch=1024) samples k
|
| 32 |
+
attempts per target from the model in eval mode at its current n_loops, and returns
|
| 33 |
+
per target {name, prompt, n_lines, proofs, n_ok, n_tried}; `proofs` are the distinct
|
| 34 |
+
attempts Lean AND nd_verify accepted. This is the only source of new training data.
|
| 35 |
+
oracle.used() / oracle.left(): seconds of the RL budget charged / remaining.
|
| 36 |
+
budget_secs RL_SECS: wall-clock inside rl() minus the oracle's Lean-gate time. Returning more
|
| 37 |
+
than the grace late, or calling the oracle with nothing left, is a timeout.
|
| 38 |
+
The model is trained in place. Everything else about RL is yours: rounds, samples per target,
|
| 39 |
+
loop counts when sampling, which proofs to keep, weighting, replay, optimizer, loss.
|
| 40 |
+
|
| 41 |
+
Anything imported from the fork (ND_FORK) is fair game to copy in here and modify.
|
| 42 |
+
"""
|
| 43 |
+
|
| 44 |
+
import math
|
| 45 |
+
import random
|
| 46 |
+
import time
|
| 47 |
+
|
| 48 |
+
import torch
|
| 49 |
+
import torch._inductor.config # noqa: F401 (explicit, so the attribute resolves for pyright)
|
| 50 |
+
from lean_tok import MAXN # fork
|
| 51 |
+
from model import GPT, Block # fork
|
| 52 |
+
from normalize import norm # fork
|
| 53 |
+
from tokenizer import make_tokenizer # fork
|
| 54 |
+
from train import loss_on # fork
|
| 55 |
+
|
| 56 |
+
# ---- knobs
|
| 57 |
+
# Unique blocks: one untied entry block, N_CORE shared core blocks, one untied exit block.
|
| 58 |
+
N_CORE = 4
|
| 59 |
+
# Control: run 198's trunk. The six blocks are independent and applied once, with no re-injection;
|
| 60 |
+
# n_loops is accepted and ignored, so the harness's two evals and rl()'s retry depths all sample
|
| 61 |
+
# the same model.
|
| 62 |
+
UNTIED = True
|
| 63 |
+
N_LAYER, D, N_HEAD = 2 + N_CORE, 384, 8
|
| 64 |
+
# Core applications (loops). Training draws one count per batch from TRAIN_LOOPS, in pretraining
|
| 65 |
+
# and in rl()'s fine-tune; eval and sampling apply model.n_loops, which the harness sets for its
|
| 66 |
+
# evals and rl() sets before each oracle call. DEFAULT_LOOPS is used for the validation curve and
|
| 67 |
+
# the harness's post-pretraining greedy check.
|
| 68 |
+
TRAIN_LOOPS = (1, 2, 3, 4, 5, 6, 7, 8)
|
| 69 |
+
DEFAULT_LOOPS = 2
|
| 70 |
+
VAL_LOOPS = (1, 2, 4, 8) # val loss at each count, once, when pretraining ends
|
| 71 |
+
# ---- RL knobs: the old harness's expert iteration (600 AdamW steps per round on up to 4 proofs
|
| 72 |
+
# per solved target, weighted x4, mixed with 20,000 replayed pretraining records), except for
|
| 73 |
+
# where the samples go. Each round every target gets RL_K attempts at RL_FIRST_LOOPS; a target
|
| 74 |
+
# all of them fail on gets RL_RETRY_K more (32 in all, as before) at ONE deeper loop count from
|
| 75 |
+
# RL_RETRY_LOOPS, assigned at random, so proofs that only deeper sampling finds enter the
|
| 76 |
+
# training set and no deep sample is spent on a target one loop already solves. The random
|
| 77 |
+
# assignment makes the retry calls a controlled comparison of loop counts on resistant targets.
|
| 78 |
+
# Rounds repeat until another one would not fit in the budget.
|
| 79 |
+
RL_FIRST_LOOPS, RL_K = 1, 8
|
| 80 |
+
RL_RETRY_LOOPS, RL_RETRY_K = (2, 4, 8), 24
|
| 81 |
+
# The fine-tune sees FT_RECORDS records per round (the old 600 steps x 128 records) in the same
|
| 82 |
+
# token-budget batches as pretraining (TOK_BUDGET below) instead of 128-record batches padded to
|
| 83 |
+
# their longest proof, which with 7-14 line RL proofs in nearly every batch was ~3x the content.
|
| 84 |
+
FT_RECORDS, FT_LR, FT_WARMUP = 600 * 128, 3e-4, 50
|
| 85 |
+
RETAIN, MAX_PER_THM, RL_WEIGHT = 20000, 4, 4
|
| 86 |
+
# MLP width held at 151's 1280: the fork's Block hardcodes 4 * D (1536 at D 384 would pass the
|
| 87 |
+
# 10M cap), so PeriLNBlock swaps fc1/fc2. 1280 != 3 * D keeps muon_step's QKV-by-shape test valid.
|
| 88 |
+
D_FF = 1280
|
| 89 |
+
BS, LR, MIN_LR_FRAC, WD = 128, 1e-3, 0.3, 0.1
|
| 90 |
+
MUON_LR, MUON_MOMENTUM = 0.01, 0.95
|
| 91 |
+
# Polar Express (Amsel et al. 2025) per-iteration quintic coefficients, as in modded-nanogpt
|
| 92 |
+
# (num_iters=5, safety factor 2e-2): singular values land in ~[0.86, 1.14] instead of the
|
| 93 |
+
# fixed quintic's ~[0.68, 1.13], at the same five iterations.
|
| 94 |
+
NS_COEFFS = (
|
| 95 |
+
(8.156554524902461, -22.48329292557795, 15.878769915207462),
|
| 96 |
+
(4.042929935166739, -2.808917465908714, 0.5000178451051316),
|
| 97 |
+
(3.8916678022926607, -2.772484153217685, 0.5060648178503393),
|
| 98 |
+
(3.285753657755655, -2.3681294933425376, 0.46449024233003106),
|
| 99 |
+
(2.3465413258596377, -1.7097828382687081, 0.42323551169305323),
|
| 100 |
+
)
|
| 101 |
+
# Ten times the AdamW group's per-step multiplicative decay (lr * wd) on the Muon matrices.
|
| 102 |
+
MUON_WD = 10 * WD * LR / MUON_LR
|
| 103 |
+
TOK_MODE = "lean_seq" # required by the harness (Lean protocol)
|
| 104 |
+
# Hybrid heads: half the heads get ALiBi (Press et al. 2021) with the standard slope set for
|
| 105 |
+
# H/2 heads, 2^(-8 i / (H/2)) = 1/4 .. 1/256 with 8 heads; the other half get slope 0, i.e. no
|
| 106 |
+
# positional signal at all (NoPE), so they can look citations up by content at any distance.
|
| 107 |
+
N_ALIBI = N_HEAD // 2
|
| 108 |
+
ALIBI_SLOPES = tuple(2.0 ** (-8.0 * (i + 1) / N_ALIBI) for i in range(N_ALIBI)) + (0.0,) * (
|
| 109 |
+
N_HEAD - N_ALIBI
|
| 110 |
+
)
|
| 111 |
+
VAL_N, CURVE_EVERY = 1024, 15 # records held out for val_loss; seconds between curve points
|
| 112 |
+
# Token-budget batches (fairseq max-tokens): each step takes a run of length-sorted records with
|
| 113 |
+
# count * padded_length <= TOK_BUDGET, so a batch of short proofs has many records and a batch of
|
| 114 |
+
# long ones few, and the padded token count per step is ~constant. 128 x 144 matches the real
|
| 115 |
+
# (unpadded) tokens of the incumbent's 128-record batches (mean record ~135 tokens), whose
|
| 116 |
+
# padding to the batch max (~400) made every step ~3x larger than its content.
|
| 117 |
+
TOK_BUDGET = 128 * 144
|
| 118 |
+
# Multi-token prediction (DeepSeek-V3 2024, Gloeckle et al. 2024), pretraining only: one extra
|
| 119 |
+
# Peri-LN block reads [trunk state at t; embedding of token t+1] and predicts token t+2 through
|
| 120 |
+
# the model's own ln_f/head. Its loss is added at this weight (DeepSeek-V3's early-training value).
|
| 121 |
+
MTP_WEIGHT = 0.3
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
# Muon update adapted from KellerJordan/Muon, commit
|
| 125 |
+
# f98f1cacc0263b04290753e32be8d498c1efc806 (muon.py).
|
| 126 |
+
# MIT License
|
| 127 |
+
# Copyright (c) 2024 Keller Jordan
|
| 128 |
+
#
|
| 129 |
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
| 130 |
+
# of this software and associated documentation files (the "Software"), to deal
|
| 131 |
+
# in the Software without restriction, including without limitation the rights
|
| 132 |
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
| 133 |
+
# copies of the Software, and to permit persons to whom the Software is
|
| 134 |
+
# furnished to do so, subject to the following conditions:
|
| 135 |
+
# The above copyright notice and this permission notice shall be included in all
|
| 136 |
+
# copies or substantial portions of the Software.
|
| 137 |
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
| 138 |
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
| 139 |
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
| 140 |
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
| 141 |
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
| 142 |
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
| 143 |
+
# SOFTWARE.
|
| 144 |
+
@torch.no_grad()
|
| 145 |
+
def muon_step(params, buffers, lr):
|
| 146 |
+
"""Nesterov momentum plus five bf16 Polar Express steps, on hidden matrices only."""
|
| 147 |
+
for p, momentum in zip(params, buffers, strict=True):
|
| 148 |
+
if p.grad is None:
|
| 149 |
+
continue
|
| 150 |
+
momentum.lerp_(p.grad, 1 - MUON_MOMENTUM)
|
| 151 |
+
update = p.grad.lerp(momentum, MUON_MOMENTUM).bfloat16()
|
| 152 |
+
# Only QKV has shape (3D, D) in this GPT; orthogonalize its roles separately.
|
| 153 |
+
if update.size(0) == 3 * update.size(1):
|
| 154 |
+
update = update.reshape(3, update.size(1), update.size(1))
|
| 155 |
+
transpose = update.size(-2) > update.size(-1)
|
| 156 |
+
if transpose:
|
| 157 |
+
update = update.mT
|
| 158 |
+
update = update / (update.norm(dim=(-2, -1), keepdim=True) * (1 + 2e-2) + 1e-6)
|
| 159 |
+
for a, b, c in NS_COEFFS:
|
| 160 |
+
gram = update @ update.mT
|
| 161 |
+
polynomial = b * gram + c * gram @ gram
|
| 162 |
+
update = a * update + polynomial @ update
|
| 163 |
+
if transpose:
|
| 164 |
+
update = update.mT
|
| 165 |
+
p.mul_(1 - lr * MUON_WD)
|
| 166 |
+
# lr may be a 0-d tensor (CUDA-graphed step); scale in p's dtype like add_(alpha=-lr)
|
| 167 |
+
p.sub_(lr * update.reshape_as(p).to(p.dtype))
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
class PeriLNBlock(Block):
|
| 171 |
+
"""P22 Peri-LN (Kim et al. 2025): the fork's pre-LN block plus a LayerNorm on each branch
|
| 172 |
+
output, so every residual write has unit scale times a learned gain. Attention has no RoPE:
|
| 173 |
+
`mask` is the additive ALiBi bias (B or 1, H, T, S) that LoopedGPT builds once per forward
|
| 174 |
+
and cos/sin are unused."""
|
| 175 |
+
|
| 176 |
+
def __init__(self, d, h):
|
| 177 |
+
super().__init__(d, h)
|
| 178 |
+
self.fc1, self.fc2 = torch.nn.Linear(d, D_FF), torch.nn.Linear(D_FF, d)
|
| 179 |
+
self.ln_attn_out, self.ln_mlp_out = torch.nn.LayerNorm(d), torch.nn.LayerNorm(d)
|
| 180 |
+
|
| 181 |
+
def attn(self, x, cos, sin, mask, cache):
|
| 182 |
+
B, T, D = x.shape
|
| 183 |
+
q, k, v = self.qkv(x).view(B, T, 3, self.h, D // self.h).permute(2, 0, 3, 1, 4)
|
| 184 |
+
if cache is not None: # the fork's preallocated KV cache, unchanged
|
| 185 |
+
if "k" not in cache:
|
| 186 |
+
cache["k"] = k.new_empty(B, self.h, cache["max"], D // self.h)
|
| 187 |
+
cache["v"] = v.new_empty(B, self.h, cache["max"], D // self.h)
|
| 188 |
+
cache["n"] = 0
|
| 189 |
+
n0, n1 = cache["n"], cache["n"] + T
|
| 190 |
+
cache["k"][:, :, n0:n1] = k
|
| 191 |
+
cache["v"][:, :, n0:n1] = v
|
| 192 |
+
cache["n"] = n1
|
| 193 |
+
k, v = cache["k"][:, :, :n1], cache["v"][:, :, :n1]
|
| 194 |
+
y = torch.nn.functional.scaled_dot_product_attention(q, k, v, attn_mask=mask.to(q.dtype))
|
| 195 |
+
return self.proj(y.transpose(1, 2).reshape(B, T, D))
|
| 196 |
+
|
| 197 |
+
def forward(self, x, cos, sin, mask=None, cache=None):
|
| 198 |
+
x = x + self.ln_attn_out(self.attn(self.ln1(x), cos, sin, mask, cache))
|
| 199 |
+
return x + self.ln_mlp_out(self.fc2(torch.nn.functional.gelu(self.fc1(self.ln2(x)))))
|
| 200 |
+
|
| 201 |
+
|
| 202 |
+
class LoopedGPT(GPT):
|
| 203 |
+
"""Looped trunk (Universal Transformer, Dehghani et al. 2018; sandwich layout and input
|
| 204 |
+
injection as in Geiping et al. 2025): blocks[0] is the entry block, blocks[1:-1] the shared
|
| 205 |
+
core applied n times, blocks[-1] the exit block. Every core pass starts from
|
| 206 |
+
ln_loop(state) + e, with e the entry block's output, so each pass sees an input of the same
|
| 207 |
+
scale and the prompt is re-injected instead of being buried under accumulated writes.
|
| 208 |
+
|
| 209 |
+
P34: the fork's GPT with ALiBi (Press et al. 2021) instead of RoPE. Head h's attention
|
| 210 |
+
logits get -slope_h * (query position - key position), slopes 2^-1 .. 2^-8 for 8 heads, so
|
| 211 |
+
no rotation of q/k anywhere. Positions come from `pos` (sample.generate's left-padded
|
| 212 |
+
batches) or arange; key positions are kept in caches[0] so cached decoding sees the same
|
| 213 |
+
bias as the full forward. Causality/padding come from `mask` when given, else from rel < 0."""
|
| 214 |
+
|
| 215 |
+
def __init__(self, vocab, n_layer, d, n_head, seed, max_len=1024):
|
| 216 |
+
super().__init__(vocab, n_layer, d, n_head, max_len)
|
| 217 |
+
slopes = torch.tensor(ALIBI_SLOPES).view(1, n_head, 1, 1)
|
| 218 |
+
self.register_buffer("slopes", slopes, persistent=False)
|
| 219 |
+
self.ln_loop = torch.nn.LayerNorm(d)
|
| 220 |
+
self.n_loops, self.loop_rng = DEFAULT_LOOPS, random.Random(seed)
|
| 221 |
+
|
| 222 |
+
def forward(self, idx, pos=None, mask=None, caches=None):
|
| 223 |
+
return self.head(self.ln_f(self.features(idx, pos, mask, caches)[0]))
|
| 224 |
+
|
| 225 |
+
def features(self, idx, pos=None, mask=None, caches=None):
|
| 226 |
+
"""The trunk: (final hidden state before ln_f, the ALiBi bias used)."""
|
| 227 |
+
B, T = idx.shape
|
| 228 |
+
if pos is None: # training/val: one (1, H, T, T) bias broadcast over the batch
|
| 229 |
+
pos = torch.arange(T, device=idx.device)[None]
|
| 230 |
+
key_pos = pos
|
| 231 |
+
if caches is not None: # incremental decoding: keys are every cached token
|
| 232 |
+
c = caches[0]
|
| 233 |
+
if "pos" not in c:
|
| 234 |
+
c["pos"], c["n_pos"] = pos.new_empty(pos.size(0), c["max"]), 0
|
| 235 |
+
n0, n1 = c["n_pos"], c["n_pos"] + T
|
| 236 |
+
c["pos"][:, n0:n1] = pos
|
| 237 |
+
c["n_pos"] = n1
|
| 238 |
+
key_pos = c["pos"][:, :n1]
|
| 239 |
+
rel = pos[:, None, :, None] - key_pos[:, None, None, :] # (B or 1, 1, T, S)
|
| 240 |
+
bias = -self.slopes * rel.to(self.slopes.dtype)
|
| 241 |
+
bias = bias.masked_fill(rel < 0 if mask is None else ~mask, float("-inf"))
|
| 242 |
+
if UNTIED:
|
| 243 |
+
x = self.emb(idx)
|
| 244 |
+
for j, b in enumerate(self.blocks):
|
| 245 |
+
x = b(x, None, None, bias, None if caches is None else caches[j])
|
| 246 |
+
return x, bias
|
| 247 |
+
n = self.loop_rng.choice(TRAIN_LOOPS) if self.training else self.n_loops
|
| 248 |
+
entry, *core, exit_block = self.blocks
|
| 249 |
+
x = e = entry(self.emb(idx), None, None, bias, None if caches is None else caches[0])
|
| 250 |
+
for loop in range(n):
|
| 251 |
+
x = self.ln_loop(x) + e
|
| 252 |
+
for j, b in enumerate(core, 1):
|
| 253 |
+
# sample.generate makes one cache dict per unique block; a shared block's keys
|
| 254 |
+
# and values differ per pass, so each pass gets its own sub-cache inside it.
|
| 255 |
+
c = caches and caches[j].setdefault(loop, {"max": caches[j]["max"]})
|
| 256 |
+
x = b(x, None, None, bias, c)
|
| 257 |
+
x = exit_block(x, None, None, bias, None if caches is None else caches[-1])
|
| 258 |
+
return x, bias
|
| 259 |
+
|
| 260 |
+
|
| 261 |
+
class MTPHead(torch.nn.Module):
|
| 262 |
+
"""The MTP module. A separate nn.Module, never registered on the model, so the harness's
|
| 263 |
+
parameter count, RL AdamW and sampler never see it; it is dropped when pretrain() returns."""
|
| 264 |
+
|
| 265 |
+
def __init__(self, d, h):
|
| 266 |
+
super().__init__()
|
| 267 |
+
self.ln_h, self.ln_e = torch.nn.LayerNorm(d), torch.nn.LayerNorm(d)
|
| 268 |
+
self.proj = torch.nn.Linear(2 * d, d, bias=False)
|
| 269 |
+
self.block = PeriLNBlock(d, h)
|
| 270 |
+
|
| 271 |
+
|
| 272 |
+
def mtp_loss(model, mtp, x, m):
|
| 273 |
+
"""The fork's loss_on (next-token CE on proof tokens) plus MTP_WEIGHT times the CE of the
|
| 274 |
+
MTP module's token-(t+2) prediction at the same proof positions. Returns (total, main)."""
|
| 275 |
+
h, bias = model.features(x[:, :-1])
|
| 276 |
+
logits = model.head(model.ln_f(h))
|
| 277 |
+
V = logits.size(-1)
|
| 278 |
+
ce = torch.nn.functional.cross_entropy(
|
| 279 |
+
logits.reshape(-1, V).float(), x[:, 1:].reshape(-1), reduction="none"
|
| 280 |
+
)
|
| 281 |
+
main = (ce * m[:, 1:].reshape(-1)).sum() / m[:, 1:].sum()
|
| 282 |
+
# Position t (0..T-3) gets h_t and emb(x_{t+1}); causal attention over these keeps every
|
| 283 |
+
# prediction of x_{t+2} a function of x_{<=t+1}.
|
| 284 |
+
z = mtp.proj(torch.cat((mtp.ln_h(h[:, :-1]), mtp.ln_e(model.emb(x[:, 1:-1]))), -1))
|
| 285 |
+
z = mtp.block(z, None, None, bias[..., :-1, :-1], None)
|
| 286 |
+
ce2 = torch.nn.functional.cross_entropy(
|
| 287 |
+
model.head(model.ln_f(z)).reshape(-1, V).float(), x[:, 2:].reshape(-1), reduction="none"
|
| 288 |
+
)
|
| 289 |
+
aux = (ce2 * m[:, 2:].reshape(-1)).sum() / m[:, 2:].sum().clamp(min=1)
|
| 290 |
+
return main + MTP_WEIGHT * aux, main, aux
|
| 291 |
+
|
| 292 |
+
|
| 293 |
+
def val_loss(model, ids, mask, ref_max, lens, tok, gen, autocast):
|
| 294 |
+
"""Mean loss over the first VAL_N records (never trained on), no shift augmentation."""
|
| 295 |
+
model.eval()
|
| 296 |
+
tot = 0.0
|
| 297 |
+
with torch.no_grad(), autocast:
|
| 298 |
+
for i in range(0, VAL_N, BS):
|
| 299 |
+
idxs = torch.arange(i, min(i + BS, VAL_N), device=ids.device)
|
| 300 |
+
x, m = ids[idxs], mask[idxs]
|
| 301 |
+
T = int(lens[idxs].max())
|
| 302 |
+
tot += loss_on(model, x[:, :T], m[:, :T]).item() * len(idxs)
|
| 303 |
+
model.train()
|
| 304 |
+
return tot / VAL_N
|
| 305 |
+
|
| 306 |
+
|
| 307 |
+
def to_gpu(data, tok, device):
|
| 308 |
+
"""Pad all (prompt, proof) records once into (N, T) ids + proof mask on the device."""
|
| 309 |
+
N, T = len(data), max(len(p) + len(q) for p, q in data)
|
| 310 |
+
ids = torch.full((N, T), tok.pad, dtype=torch.long)
|
| 311 |
+
mask = torch.zeros((N, T), dtype=torch.bool)
|
| 312 |
+
ref_max = torch.zeros(
|
| 313 |
+
N, dtype=torch.long
|
| 314 |
+
) # largest N<k> index used in the proof, for the shift augmentation
|
| 315 |
+
for i, (p, q) in enumerate(data):
|
| 316 |
+
ids[i, : len(p) + len(q)] = torch.tensor(p + q)
|
| 317 |
+
mask[i, len(p) : len(p) + len(q)] = True
|
| 318 |
+
ref_max[i] = max((x - tok.ref0 + 1 for x in q if x >= tok.ref0), default=0)
|
| 319 |
+
return ids.to(device), mask.to(device), ref_max.to(device), (ids != tok.pad).sum(1).to(device)
|
| 320 |
+
|
| 321 |
+
|
| 322 |
+
def token_batches(lens, n_train, gen, rng, first=VAL_N):
|
| 323 |
+
"""One epoch of token-budget batches: a random permutation of the n_train records from
|
| 324 |
+
`first` on, stably sorted by length (equal lengths stay shuffled), cut greedily into runs of
|
| 325 |
+
consecutive records with count * max_len <= TOK_BUDGET, and the runs' order shuffled.
|
| 326 |
+
Returns (order, cuts)."""
|
| 327 |
+
perm = torch.randperm(n_train, device=lens.device, generator=gen) + first
|
| 328 |
+
order = perm[torch.argsort(lens[perm], stable=True)]
|
| 329 |
+
sorted_lens = lens[order].cpu()
|
| 330 |
+
cuts, s = [], 0
|
| 331 |
+
while s < len(sorted_lens):
|
| 332 |
+
window = sorted_lens[s : s + TOK_BUDGET // int(sorted_lens[s])]
|
| 333 |
+
# count * max_len is non-decreasing along the window, so the fits form a prefix.
|
| 334 |
+
e = s + int((torch.arange(1, len(window) + 1) * window <= TOK_BUDGET).sum())
|
| 335 |
+
cuts.append((s, e, int(sorted_lens[e - 1]))) # sorted: last is the batch's max length
|
| 336 |
+
s = e
|
| 337 |
+
rng.shuffle(cuts)
|
| 338 |
+
return order, cuts
|
| 339 |
+
|
| 340 |
+
|
| 341 |
+
def gpu_batch(ids, mask, ref_max, lens, idxs, tok, gen, max_len=None):
|
| 342 |
+
"""The fork's batch() + tok.shift_abs, vectorised: a random per-row offset is added to
|
| 343 |
+
every ref token."""
|
| 344 |
+
x, m = ids[idxs], mask[idxs]
|
| 345 |
+
# Known on the host for training batches: passing it avoids a GPU sync every step.
|
| 346 |
+
T = int(lens[idxs].max()) if max_len is None else max_len
|
| 347 |
+
x, m = x[:, :T].clone(), m[:, :T]
|
| 348 |
+
if tok.mode in ("abs", "lean_seq"): # lean_seq: names n1.. get the same start offset
|
| 349 |
+
room = MAXN - ref_max[idxs]
|
| 350 |
+
shift = (
|
| 351 |
+
(torch.rand(len(idxs), device=x.device, generator=gen) * (room + 1))
|
| 352 |
+
.long()
|
| 353 |
+
.clamp(max=room)
|
| 354 |
+
)
|
| 355 |
+
is_ref = (x >= tok.ref0) & m
|
| 356 |
+
x = torch.where(is_ref, x + shift[:, None], x)
|
| 357 |
+
return x, m
|
| 358 |
+
|
| 359 |
+
|
| 360 |
+
def pretrain(train_recs, budget_secs, device, seed, log):
|
| 361 |
+
t0 = time.time()
|
| 362 |
+
tok = make_tokenizer(TOK_MODE)
|
| 363 |
+
data = [(tok.encode_prompt(r["prompt"]), tok.encode_proof(r["proof"])) for r in train_recs]
|
| 364 |
+
ids, mask, ref_max, lens = to_gpu(data, tok, device)
|
| 365 |
+
gen = torch.Generator(device=device)
|
| 366 |
+
gen.manual_seed(seed)
|
| 367 |
+
model = LoopedGPT(tok.vocab_size, N_LAYER, D, N_HEAD, seed)
|
| 368 |
+
model.blocks = torch.nn.ModuleList(PeriLNBlock(D, N_HEAD) for _ in range(N_LAYER))
|
| 369 |
+
model.blocks.apply(model._init) # fork init: zero biases
|
| 370 |
+
model = model.to(device)
|
| 371 |
+
mtp = MTPHead(D, N_HEAD)
|
| 372 |
+
mtp.apply(model._init)
|
| 373 |
+
mtp = mtp.to(device)
|
| 374 |
+
# Compile the repeated blocks once each (torch >= 2.9; dynamic shapes, ~14 s): ~14% faster
|
| 375 |
+
# steps. The coalescing tiling analysis crashes on dynamic shapes, so it is off. Undone at
|
| 376 |
+
# handoff so the harness's RL fine-tune and KV-cache sampling run eager.
|
| 377 |
+
torch._inductor.config.triton.coalesce_tiling_analysis = False
|
| 378 |
+
muon_lr, muon_graph = torch.zeros((), device=device), None
|
| 379 |
+
compiled = list(model.blocks) + [mtp.block]
|
| 380 |
+
for b in compiled if device == "cuda" else []: # CPU smoke runs stay eager
|
| 381 |
+
b.compile(dynamic=True)
|
| 382 |
+
for p in list(model.blocks.parameters()) + list(mtp.parameters()):
|
| 383 |
+
if p.ndim == 2:
|
| 384 |
+
# Orthogonal matrices have element second moment gain^2 / max(rows, cols).
|
| 385 |
+
torch.nn.init.orthogonal_(p, gain=0.02 * math.sqrt(max(p.shape)))
|
| 386 |
+
log(
|
| 387 |
+
f"pretrain: {model.n_params():,} params, {len(data):,} records on {device} "
|
| 388 |
+
f"after {time.time() - t0:.0f}s, budget {budget_secs}s"
|
| 389 |
+
)
|
| 390 |
+
# The output readout joins the hidden matrices under Muon; embeddings stay on AdamW.
|
| 391 |
+
trainable = list(model.parameters()) + list(mtp.parameters())
|
| 392 |
+
matrices = [p for p in model.blocks.parameters() if p.ndim == 2] + [model.head.weight]
|
| 393 |
+
matrices += [p for p in mtp.parameters() if p.ndim == 2]
|
| 394 |
+
matrix_ids = {id(p) for p in matrices}
|
| 395 |
+
auxiliary = [p for p in trainable if id(p) not in matrix_ids]
|
| 396 |
+
momentum_buffers = [torch.zeros_like(p) for p in matrices]
|
| 397 |
+
opt = torch.optim.AdamW(
|
| 398 |
+
auxiliary, lr=LR, weight_decay=WD, betas=(0.9, 0.95), fused=device == "cuda"
|
| 399 |
+
)
|
| 400 |
+
autocast = torch.autocast("cuda", dtype=torch.bfloat16, enabled=(device == "cuda"))
|
| 401 |
+
model.train()
|
| 402 |
+
step, last_loss, records_seen = 0, float("nan"), 0
|
| 403 |
+
curve, recent, recent_aux, next_point = [], [], [], CURVE_EVERY
|
| 404 |
+
n_train = len(data) - VAL_N # records [0, VAL_N) are the val slice
|
| 405 |
+
rng, cuts, batches_per_epoch = random.Random(seed), [], None
|
| 406 |
+
order = torch.empty(0, dtype=torch.long, device=device)
|
| 407 |
+
while True:
|
| 408 |
+
frac = (
|
| 409 |
+
time.time() - t0
|
| 410 |
+
) / budget_secs # cosine over wall-clock, so the cap is the schedule
|
| 411 |
+
if frac >= 1.0:
|
| 412 |
+
break
|
| 413 |
+
step += 1
|
| 414 |
+
# No warmup (as in modded-nanogpt): Muon's step is normalized and Adam's is ~lr per
|
| 415 |
+
# element from step 1, and Peri-LN normalizes every branch write.
|
| 416 |
+
lr = LR * MIN_LR_FRAC + 0.5 * LR * (1 - MIN_LR_FRAC) * (1 + math.cos(math.pi * frac))
|
| 417 |
+
for g in opt.param_groups:
|
| 418 |
+
g["lr"] = lr
|
| 419 |
+
if not cuts: # new epoch
|
| 420 |
+
order, cuts = token_batches(lens, n_train, gen, rng)
|
| 421 |
+
batches_per_epoch = len(cuts)
|
| 422 |
+
s, e, T = cuts.pop()
|
| 423 |
+
idxs = order[s:e]
|
| 424 |
+
records_seen += e - s
|
| 425 |
+
x, m = gpu_batch(ids, mask, ref_max, lens, idxs, tok, gen, T)
|
| 426 |
+
with autocast:
|
| 427 |
+
loss, main, aux = mtp_loss(model, mtp, x, m)
|
| 428 |
+
model.zero_grad(set_to_none=False) # grads keep fixed addresses for the Muon CUDA graph
|
| 429 |
+
mtp.zero_grad(set_to_none=False)
|
| 430 |
+
loss.backward()
|
| 431 |
+
torch.nn.utils.clip_grad_norm_(trainable, 1.0)
|
| 432 |
+
opt.step()
|
| 433 |
+
muon_lr.fill_(MUON_LR * lr / LR)
|
| 434 |
+
# Capture once; later steps replay one graph instead of ~400 kernel launches. CPU smoke
|
| 435 |
+
# runs call the step directly.
|
| 436 |
+
if device != "cuda":
|
| 437 |
+
muon_step(matrices, momentum_buffers, muon_lr)
|
| 438 |
+
elif muon_graph is None:
|
| 439 |
+
state = matrices + momentum_buffers
|
| 440 |
+
saved = [t.clone() for t in state]
|
| 441 |
+
side = torch.cuda.Stream()
|
| 442 |
+
side.wait_stream(torch.cuda.current_stream())
|
| 443 |
+
with torch.cuda.stream(side):
|
| 444 |
+
muon_step(matrices, momentum_buffers, muon_lr) # warm-up (lazy cuBLAS init)
|
| 445 |
+
torch.cuda.current_stream().wait_stream(side)
|
| 446 |
+
muon_graph = torch.cuda.CUDAGraph()
|
| 447 |
+
with torch.cuda.graph(muon_graph):
|
| 448 |
+
muon_step(matrices, momentum_buffers, muon_lr)
|
| 449 |
+
with torch.no_grad():
|
| 450 |
+
for t, v in zip(state, saved, strict=True):
|
| 451 |
+
t.copy_(v) # undo the warm-up and capture updates
|
| 452 |
+
muon_graph.replay()
|
| 453 |
+
else:
|
| 454 |
+
muon_graph.replay()
|
| 455 |
+
if step % 20 == 0:
|
| 456 |
+
last_loss = main.item() # the next-token CE, comparable with earlier runs
|
| 457 |
+
recent.append(last_loss)
|
| 458 |
+
recent_aux.append(aux.item())
|
| 459 |
+
if (
|
| 460 |
+
time.time() - t0 >= next_point
|
| 461 |
+
): # curve point: mean recent train loss + val loss on the held-out slice
|
| 462 |
+
curve.append(
|
| 463 |
+
{
|
| 464 |
+
"secs": round(time.time() - t0),
|
| 465 |
+
"step": step,
|
| 466 |
+
"train_loss": sum(recent) / len(recent) if recent else last_loss,
|
| 467 |
+
"val_loss": val_loss(model, ids, mask, ref_max, lens, tok, gen, autocast),
|
| 468 |
+
"mtp_loss": sum(recent_aux) / len(recent_aux) if recent_aux else None,
|
| 469 |
+
}
|
| 470 |
+
)
|
| 471 |
+
log(
|
| 472 |
+
f"pretrain step {step} train {curve[-1]['train_loss']:.4f} "
|
| 473 |
+
f"val {curve[-1]['val_loss']:.4f} lr {lr:.2e} {curve[-1]['secs']}s"
|
| 474 |
+
)
|
| 475 |
+
recent, recent_aux, next_point = [], [], next_point + CURVE_EVERY
|
| 476 |
+
val_by_loops = {}
|
| 477 |
+
for n in VAL_LOOPS if step else (): # no steps: a checkpoint rebuild, not a training run
|
| 478 |
+
model.n_loops = n
|
| 479 |
+
val_by_loops[n] = val_loss(model, ids, mask, ref_max, lens, tok, gen, autocast)
|
| 480 |
+
model.n_loops = DEFAULT_LOOPS
|
| 481 |
+
log(f"pretrain val loss by loops: {val_by_loops}")
|
| 482 |
+
for b in compiled:
|
| 483 |
+
b._compiled_call_impl = None
|
| 484 |
+
model.eval()
|
| 485 |
+
return (
|
| 486 |
+
model,
|
| 487 |
+
tok,
|
| 488 |
+
{
|
| 489 |
+
"steps": step,
|
| 490 |
+
"records_seen": records_seen,
|
| 491 |
+
"epochs": records_seen / n_train,
|
| 492 |
+
"batches_per_epoch": batches_per_epoch,
|
| 493 |
+
"tok_budget": TOK_BUDGET,
|
| 494 |
+
"last_loss": last_loss,
|
| 495 |
+
"train_loops": TRAIN_LOOPS,
|
| 496 |
+
"default_loops": DEFAULT_LOOPS,
|
| 497 |
+
"val_by_loops": val_by_loops,
|
| 498 |
+
"curve": curve,
|
| 499 |
+
},
|
| 500 |
+
)
|
| 501 |
+
|
| 502 |
+
|
| 503 |
+
def finetune(model, tok, data, rng, device):
|
| 504 |
+
"""The old harness's RL fine-tune: AdamW, linear warmup then cosine to lr/10, over FT_RECORDS
|
| 505 |
+
records in token-budget batches (to_gpu / token_batches / gpu_batch, as pretraining). The
|
| 506 |
+
model is in train mode, so every batch draws its own loop count from TRAIN_LOOPS. Returns
|
| 507 |
+
(last loss, steps)."""
|
| 508 |
+
ids, mask, ref_max, lens = to_gpu(data, tok, device)
|
| 509 |
+
gen = torch.Generator(device=device)
|
| 510 |
+
gen.manual_seed(rng.getrandbits(31))
|
| 511 |
+
plan, seen = [], 0 # (order, start, end, padded length) per step, epoch after epoch
|
| 512 |
+
while seen < FT_RECORDS:
|
| 513 |
+
order, cuts = token_batches(lens, len(data), gen, rng, first=0)
|
| 514 |
+
for s, e, T in cuts:
|
| 515 |
+
if seen >= FT_RECORDS:
|
| 516 |
+
break
|
| 517 |
+
plan.append((order, s, e, T))
|
| 518 |
+
seen += e - s
|
| 519 |
+
opt = torch.optim.AdamW(model.parameters(), lr=FT_LR, weight_decay=0.1, betas=(0.9, 0.95))
|
| 520 |
+
autocast = torch.autocast("cuda", dtype=torch.bfloat16, enabled=(device == "cuda"))
|
| 521 |
+
model.train()
|
| 522 |
+
for step, (order, s, e, T) in enumerate(plan, 1):
|
| 523 |
+
lr = (
|
| 524 |
+
FT_LR * step / FT_WARMUP
|
| 525 |
+
if step < FT_WARMUP
|
| 526 |
+
else FT_LR / 10 + 0.45 * FT_LR * (1 + math.cos(math.pi * step / len(plan)))
|
| 527 |
+
)
|
| 528 |
+
for g in opt.param_groups:
|
| 529 |
+
g["lr"] = lr
|
| 530 |
+
x, m = gpu_batch(ids, mask, ref_max, lens, order[s:e], tok, gen, T)
|
| 531 |
+
with autocast:
|
| 532 |
+
loss = loss_on(model, x, m)
|
| 533 |
+
opt.zero_grad(set_to_none=True)
|
| 534 |
+
loss.backward()
|
| 535 |
+
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
|
| 536 |
+
opt.step()
|
| 537 |
+
model.eval()
|
| 538 |
+
return loss.item(), len(plan)
|
| 539 |
+
|
| 540 |
+
|
| 541 |
+
def rl(model, tok, targets, train_recs, oracle, budget_secs, device, seed, log):
|
| 542 |
+
rng = random.Random(seed)
|
| 543 |
+
found = {t["name"]: {} for t in targets} # name -> {normalized proof: proof text}
|
| 544 |
+
rounds, round_secs, retries, ft_steps = 0, 0.0, [], []
|
| 545 |
+
# Start a round only if one as long as the last (plus 15%) still fits in the budget.
|
| 546 |
+
while oracle.left() > 1.15 * round_secs:
|
| 547 |
+
t0 = oracle.used()
|
| 548 |
+
model.n_loops = RL_FIRST_LOOPS
|
| 549 |
+
rows = oracle.sample(model, targets, RL_K)
|
| 550 |
+
failed = [t for t, row in zip(targets, rows, strict=True) if not row["proofs"]]
|
| 551 |
+
rng.shuffle(failed)
|
| 552 |
+
for i, n in enumerate(RL_RETRY_LOOPS):
|
| 553 |
+
part = failed[i :: len(RL_RETRY_LOOPS)]
|
| 554 |
+
if not part:
|
| 555 |
+
continue
|
| 556 |
+
model.n_loops = n
|
| 557 |
+
part_rows = oracle.sample(model, part, RL_RETRY_K)
|
| 558 |
+
cracked = sum(bool(row["proofs"]) for row in part_rows)
|
| 559 |
+
retries.append(
|
| 560 |
+
{"round": rounds + 1, "loops": n, "targets": len(part), "cracked": cracked}
|
| 561 |
+
)
|
| 562 |
+
rows += part_rows
|
| 563 |
+
for row in rows:
|
| 564 |
+
for proof in row["proofs"]:
|
| 565 |
+
found[row["name"]].setdefault(norm(proof), proof)
|
| 566 |
+
mix = []
|
| 567 |
+
for t in targets:
|
| 568 |
+
proofs = list(found[t["name"]].values())
|
| 569 |
+
rng.shuffle(proofs)
|
| 570 |
+
mix += [(t["prompt"], proof) for proof in proofs[:MAX_PER_THM]] * RL_WEIGHT
|
| 571 |
+
n_rl = len(mix)
|
| 572 |
+
if not n_rl:
|
| 573 |
+
break
|
| 574 |
+
mix += [(r["prompt"], r["proof"]) for r in rng.sample(train_recs, RETAIN)]
|
| 575 |
+
data = [(tok.encode_prompt(p), tok.encode_proof(q)) for p, q in mix]
|
| 576 |
+
loss, steps = finetune(model, tok, data, rng, device)
|
| 577 |
+
ft_steps.append(steps)
|
| 578 |
+
rounds += 1
|
| 579 |
+
round_secs = oracle.used() - t0
|
| 580 |
+
log(
|
| 581 |
+
f"rl round {rounds}: {n_rl} RL records, fine-tune loss {loss:.4f} in {steps} steps, "
|
| 582 |
+
f"round {round_secs:.0f}s, {oracle.left():.0f}s left"
|
| 583 |
+
)
|
| 584 |
+
model.n_loops = DEFAULT_LOOPS
|
| 585 |
+
return {
|
| 586 |
+
"rounds": rounds,
|
| 587 |
+
"first_loops": RL_FIRST_LOOPS,
|
| 588 |
+
"k_first": RL_K,
|
| 589 |
+
"retry_loops": RL_RETRY_LOOPS,
|
| 590 |
+
"k_retry": RL_RETRY_K,
|
| 591 |
+
"retries": retries,
|
| 592 |
+
"ft_records": FT_RECORDS,
|
| 593 |
+
"ft_steps": ft_steps,
|
| 594 |
+
}
|
runs/590_control-untied-198_s0/source/remote.sh
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
# Runs on the pod. Checks out Dan's fork at the pinned commit (frozen data + verifier + Lean
|
| 3 |
+
# tokenizer/gate), installs Lean, runs the harness.
|
| 4 |
+
# Usage: bash remote.sh <out-dir> [harness.py args...]
|
| 5 |
+
set -euo pipefail
|
| 6 |
+
FORK_COMMIT=51604b3feaa5e66821d1faba22238fe6d96d46d0 # dan-pandori/nd-takehome, branch dan_lean_seed2 (lean_seq with the BOTE fix)
|
| 7 |
+
export ND_FORK=/workspace/nd-takehome
|
| 8 |
+
OUT=$1; shift
|
| 9 |
+
|
| 10 |
+
if [ ! -d "$ND_FORK/.git" ]; then
|
| 11 |
+
git clone -q https://github.com/dan-pandori/nd-takehome.git "$ND_FORK"
|
| 12 |
+
fi
|
| 13 |
+
git -C "$ND_FORK" cat-file -e "$FORK_COMMIT" 2>/dev/null || git -C "$ND_FORK" fetch -q origin
|
| 14 |
+
git -C "$ND_FORK" checkout -q "$FORK_COMMIT"
|
| 15 |
+
[ -f "$ND_FORK/data/train.jsonl" ] || gunzip -k "$ND_FORK/data/train.jsonl.gz"
|
| 16 |
+
# Lean 4 core checks every sampled proof (fork lean_gate.py, called from sample.generate): the
|
| 17 |
+
# reward is Lean AND nd_verify. Installed before harness.py starts, so it is not charged to TOTAL_SECS.
|
| 18 |
+
[ -x ~/.elan/bin/lean ] || curl -sSf https://raw.githubusercontent.com/leanprover/elan/master/elan-init.sh \
|
| 19 |
+
| sh -s -- -y --default-toolchain leanprover/lean4:v4.34.0 > /dev/null
|
| 20 |
+
# One Lean process per CPU: on these pods (nproc 96) 12 workers took 146 s per RL-round gate
|
| 21 |
+
# pass (099) against 67-84 s with 96 workers (098).
|
| 22 |
+
export LEAN_GATE_WORKERS=$(nproc) LEAN_GATE_LOG="$OUT/lean_gate.jsonl"
|
| 23 |
+
nvidia-smi --query-gpu=name,memory.total --format=csv,noheader
|
| 24 |
+
echo "cpus $(nproc), $(~/.elan/bin/lean --version)"
|
| 25 |
+
# Newer RunPod images put torch under `python` and leave `python3` as bare system Python.
|
| 26 |
+
PY=$(for p in python python3; do "$p" -c "import torch" 2>/dev/null && echo "$p" && break; done)
|
| 27 |
+
echo "python $($PY --version 2>&1), torch $($PY -c 'import torch; print(torch.__version__)')"
|
| 28 |
+
cd /workspace/autoresearch
|
| 29 |
+
$PY harness.py --out "$OUT" "$@" 2>&1 | tee "$OUT.log"
|
runs/590_control-untied-198_s0/summary.json
ADDED
|
@@ -0,0 +1,3128 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "ok",
|
| 3 |
+
"metric": 790,
|
| 4 |
+
"metric_diag": 790,
|
| 5 |
+
"metric_rl": 1309,
|
| 6 |
+
"harness_sha": "2eec1b61666a",
|
| 7 |
+
"pretrain_sha": "b27dd2cb03ba",
|
| 8 |
+
"device": "cuda",
|
| 9 |
+
"gpu": "NVIDIA A40",
|
| 10 |
+
"torch": "2.9.1+cu129",
|
| 11 |
+
"lean_gate": {
|
| 12 |
+
"samples": 524304,
|
| 13 |
+
"parse_fail": 94720,
|
| 14 |
+
"distinct_checked": 389781,
|
| 15 |
+
"both_ok": 240965,
|
| 16 |
+
"nd_ok_lean_rej": 0,
|
| 17 |
+
"nd_rej_lean_ok": 0,
|
| 18 |
+
"lean_wall_s": 1245.6879568099976
|
| 19 |
+
},
|
| 20 |
+
"total_secs": 3660.8543887138367,
|
| 21 |
+
"rules": {
|
| 22 |
+
"TOTAL_SECS": 5400,
|
| 23 |
+
"PRETRAIN_SECS": 300,
|
| 24 |
+
"RL_SECS": 1800,
|
| 25 |
+
"PARAM_CAP": 10000000,
|
| 26 |
+
"N_TARGETS": 1500,
|
| 27 |
+
"EVAL_K": 64,
|
| 28 |
+
"KNOB": "n_loops",
|
| 29 |
+
"EVAL_LEVEL": 8,
|
| 30 |
+
"DIAG_LEVEL": 2,
|
| 31 |
+
"TOK_MODE": "lean_seq"
|
| 32 |
+
},
|
| 33 |
+
"pretrain": {
|
| 34 |
+
"steps": 5322,
|
| 35 |
+
"records_seen": 723768,
|
| 36 |
+
"epochs": 4.700830053388411,
|
| 37 |
+
"batches_per_epoch": 1132,
|
| 38 |
+
"tok_budget": 18432,
|
| 39 |
+
"last_loss": 0.0643298402428627,
|
| 40 |
+
"train_loops": [
|
| 41 |
+
1,
|
| 42 |
+
2,
|
| 43 |
+
3,
|
| 44 |
+
4,
|
| 45 |
+
5,
|
| 46 |
+
6,
|
| 47 |
+
7,
|
| 48 |
+
8
|
| 49 |
+
],
|
| 50 |
+
"default_loops": 2,
|
| 51 |
+
"val_by_loops": {
|
| 52 |
+
"1": 0.0866707032546401,
|
| 53 |
+
"2": 0.0866707032546401,
|
| 54 |
+
"4": 0.0866707032546401,
|
| 55 |
+
"8": 0.0866707032546401
|
| 56 |
+
},
|
| 57 |
+
"curve": [
|
| 58 |
+
{
|
| 59 |
+
"secs": 31,
|
| 60 |
+
"step": 1,
|
| 61 |
+
"train_loss": NaN,
|
| 62 |
+
"val_loss": 3.9265004098415375,
|
| 63 |
+
"mtp_loss": null
|
| 64 |
+
},
|
| 65 |
+
{
|
| 66 |
+
"secs": 34,
|
| 67 |
+
"step": 2,
|
| 68 |
+
"train_loss": NaN,
|
| 69 |
+
"val_loss": 3.5213595032691956,
|
| 70 |
+
"mtp_loss": null
|
| 71 |
+
},
|
| 72 |
+
{
|
| 73 |
+
"secs": 45,
|
| 74 |
+
"step": 234,
|
| 75 |
+
"train_loss": 0.4438230293718251,
|
| 76 |
+
"val_loss": 0.10797532740980387,
|
| 77 |
+
"mtp_loss": 0.45858401534232224
|
| 78 |
+
},
|
| 79 |
+
{
|
| 80 |
+
"secs": 60,
|
| 81 |
+
"step": 537,
|
| 82 |
+
"train_loss": 0.09316873500744502,
|
| 83 |
+
"val_loss": 0.09572682809084654,
|
| 84 |
+
"mtp_loss": 0.09360948950052261
|
| 85 |
+
},
|
| 86 |
+
{
|
| 87 |
+
"secs": 75,
|
| 88 |
+
"step": 840,
|
| 89 |
+
"train_loss": 0.07020179787650704,
|
| 90 |
+
"val_loss": 0.08384304493665695,
|
| 91 |
+
"mtp_loss": 0.07230285461992025
|
| 92 |
+
},
|
| 93 |
+
{
|
| 94 |
+
"secs": 90,
|
| 95 |
+
"step": 1140,
|
| 96 |
+
"train_loss": 0.059316099435091016,
|
| 97 |
+
"val_loss": 0.09674938209354877,
|
| 98 |
+
"mtp_loss": 0.061977747827768326
|
| 99 |
+
},
|
| 100 |
+
{
|
| 101 |
+
"secs": 105,
|
| 102 |
+
"step": 1444,
|
| 103 |
+
"train_loss": 0.06523487865924835,
|
| 104 |
+
"val_loss": 0.08504640311002731,
|
| 105 |
+
"mtp_loss": 0.06617915580670039
|
| 106 |
+
},
|
| 107 |
+
{
|
| 108 |
+
"secs": 120,
|
| 109 |
+
"step": 1743,
|
| 110 |
+
"train_loss": 0.06727594335873922,
|
| 111 |
+
"val_loss": 0.07866689842194319,
|
| 112 |
+
"mtp_loss": 0.06769198700785636
|
| 113 |
+
},
|
| 114 |
+
{
|
| 115 |
+
"secs": 135,
|
| 116 |
+
"step": 2042,
|
| 117 |
+
"train_loss": 0.0691841093202432,
|
| 118 |
+
"val_loss": 0.08289193827658892,
|
| 119 |
+
"mtp_loss": 0.06978676989674568
|
| 120 |
+
},
|
| 121 |
+
{
|
| 122 |
+
"secs": 150,
|
| 123 |
+
"step": 2342,
|
| 124 |
+
"train_loss": 0.06363775730133056,
|
| 125 |
+
"val_loss": 0.0940767228603363,
|
| 126 |
+
"mtp_loss": 0.06482930605610211
|
| 127 |
+
},
|
| 128 |
+
{
|
| 129 |
+
"secs": 165,
|
| 130 |
+
"step": 2643,
|
| 131 |
+
"train_loss": 0.05806758925318718,
|
| 132 |
+
"val_loss": 0.07991388812661171,
|
| 133 |
+
"mtp_loss": 0.058483662207921346
|
| 134 |
+
},
|
| 135 |
+
{
|
| 136 |
+
"secs": 180,
|
| 137 |
+
"step": 2943,
|
| 138 |
+
"train_loss": 0.07072806904713312,
|
| 139 |
+
"val_loss": 0.08869772125035524,
|
| 140 |
+
"mtp_loss": 0.07184548750519752
|
| 141 |
+
},
|
| 142 |
+
{
|
| 143 |
+
"secs": 195,
|
| 144 |
+
"step": 3242,
|
| 145 |
+
"train_loss": 0.056170069798827174,
|
| 146 |
+
"val_loss": 0.0838435497134924,
|
| 147 |
+
"mtp_loss": 0.05944442500670751
|
| 148 |
+
},
|
| 149 |
+
{
|
| 150 |
+
"secs": 210,
|
| 151 |
+
"step": 3540,
|
| 152 |
+
"train_loss": 0.06035705991089344,
|
| 153 |
+
"val_loss": 0.08335433062165976,
|
| 154 |
+
"mtp_loss": 0.060601692150036496
|
| 155 |
+
},
|
| 156 |
+
{
|
| 157 |
+
"secs": 225,
|
| 158 |
+
"step": 3840,
|
| 159 |
+
"train_loss": 0.05841459110379219,
|
| 160 |
+
"val_loss": 0.07814371213316917,
|
| 161 |
+
"mtp_loss": 0.058488543579975766
|
| 162 |
+
},
|
| 163 |
+
{
|
| 164 |
+
"secs": 240,
|
| 165 |
+
"step": 4140,
|
| 166 |
+
"train_loss": 0.05674752692381541,
|
| 167 |
+
"val_loss": 0.07864161394536495,
|
| 168 |
+
"mtp_loss": 0.056829093893369036
|
| 169 |
+
},
|
| 170 |
+
{
|
| 171 |
+
"secs": 255,
|
| 172 |
+
"step": 4439,
|
| 173 |
+
"train_loss": 0.05531486363283226,
|
| 174 |
+
"val_loss": 0.09093722235411406,
|
| 175 |
+
"mtp_loss": 0.05732316550399576
|
| 176 |
+
},
|
| 177 |
+
{
|
| 178 |
+
"secs": 270,
|
| 179 |
+
"step": 4724,
|
| 180 |
+
"train_loss": 0.04911263796190421,
|
| 181 |
+
"val_loss": 0.0790712758898735,
|
| 182 |
+
"mtp_loss": 0.04921042968829473
|
| 183 |
+
},
|
| 184 |
+
{
|
| 185 |
+
"secs": 285,
|
| 186 |
+
"step": 5024,
|
| 187 |
+
"train_loss": 0.060920559366544086,
|
| 188 |
+
"val_loss": 0.08792904019355774,
|
| 189 |
+
"mtp_loss": 0.060834589103857674
|
| 190 |
+
},
|
| 191 |
+
{
|
| 192 |
+
"secs": 300,
|
| 193 |
+
"step": 5322,
|
| 194 |
+
"train_loss": 0.05323498733341694,
|
| 195 |
+
"val_loss": 0.0866707032546401,
|
| 196 |
+
"mtp_loss": 0.053421760971347494
|
| 197 |
+
}
|
| 198 |
+
],
|
| 199 |
+
"secs": 300.7043650150299,
|
| 200 |
+
"n_params": 9558528,
|
| 201 |
+
"heldout_greedy": 0.9685
|
| 202 |
+
},
|
| 203 |
+
"rl": {
|
| 204 |
+
"rounds": [
|
| 205 |
+
{
|
| 206 |
+
"round": 1,
|
| 207 |
+
"level": 1,
|
| 208 |
+
"k": 8,
|
| 209 |
+
"n_targets": 1500,
|
| 210 |
+
"sample_acc": 0.21783333333333332,
|
| 211 |
+
"new_proofs": 713,
|
| 212 |
+
"targets_solved": 625,
|
| 213 |
+
"lstar": 9,
|
| 214 |
+
"ge": {
|
| 215 |
+
"7": 625,
|
| 216 |
+
"8": 241,
|
| 217 |
+
"9": 63,
|
| 218 |
+
"10": 1,
|
| 219 |
+
"11": 0,
|
| 220 |
+
"12": 0,
|
| 221 |
+
"13": 0,
|
| 222 |
+
"14": 0
|
| 223 |
+
},
|
| 224 |
+
"mix_rl_records": 713,
|
| 225 |
+
"sampling": {
|
| 226 |
+
"samples": 12000,
|
| 227 |
+
"secs": 67.77599048614502,
|
| 228 |
+
"gate_secs": 25.270068645477295,
|
| 229 |
+
"samples_per_sec": 177.05384921601518
|
| 230 |
+
},
|
| 231 |
+
"secs": 68.23632264137268,
|
| 232 |
+
"rl_secs_used": 42.96647357940674
|
| 233 |
+
},
|
| 234 |
+
{
|
| 235 |
+
"round": 2,
|
| 236 |
+
"level": 2,
|
| 237 |
+
"k": 24,
|
| 238 |
+
"n_targets": 292,
|
| 239 |
+
"sample_acc": 0.01127283105022831,
|
| 240 |
+
"new_proofs": 36,
|
| 241 |
+
"targets_solved": 660,
|
| 242 |
+
"lstar": 9,
|
| 243 |
+
"ge": {
|
| 244 |
+
"7": 660,
|
| 245 |
+
"8": 263,
|
| 246 |
+
"9": 75,
|
| 247 |
+
"10": 3,
|
| 248 |
+
"11": 0,
|
| 249 |
+
"12": 0,
|
| 250 |
+
"13": 0,
|
| 251 |
+
"14": 0
|
| 252 |
+
},
|
| 253 |
+
"mix_rl_records": 36,
|
| 254 |
+
"sampling": {
|
| 255 |
+
"samples": 7008,
|
| 256 |
+
"secs": 38.23188781738281,
|
| 257 |
+
"gate_secs": 14.766036748886108,
|
| 258 |
+
"samples_per_sec": 183.30248387090336
|
| 259 |
+
},
|
| 260 |
+
"secs": 38.45504450798035,
|
| 261 |
+
"rl_secs_used": 66.65531277656555
|
| 262 |
+
},
|
| 263 |
+
{
|
| 264 |
+
"round": 3,
|
| 265 |
+
"level": 4,
|
| 266 |
+
"k": 24,
|
| 267 |
+
"n_targets": 292,
|
| 268 |
+
"sample_acc": 0.006563926940639269,
|
| 269 |
+
"new_proofs": 29,
|
| 270 |
+
"targets_solved": 688,
|
| 271 |
+
"lstar": 10,
|
| 272 |
+
"ge": {
|
| 273 |
+
"7": 688,
|
| 274 |
+
"8": 278,
|
| 275 |
+
"9": 85,
|
| 276 |
+
"10": 5,
|
| 277 |
+
"11": 0,
|
| 278 |
+
"12": 0,
|
| 279 |
+
"13": 0,
|
| 280 |
+
"14": 0
|
| 281 |
+
},
|
| 282 |
+
"mix_rl_records": 29,
|
| 283 |
+
"sampling": {
|
| 284 |
+
"samples": 7008,
|
| 285 |
+
"secs": 35.475021839141846,
|
| 286 |
+
"gate_secs": 15.691442728042603,
|
| 287 |
+
"samples_per_sec": 197.5474470960756
|
| 288 |
+
},
|
| 289 |
+
"secs": 35.69817113876343,
|
| 290 |
+
"rl_secs_used": 86.66204929351807
|
| 291 |
+
},
|
| 292 |
+
{
|
| 293 |
+
"round": 4,
|
| 294 |
+
"level": 8,
|
| 295 |
+
"k": 24,
|
| 296 |
+
"n_targets": 291,
|
| 297 |
+
"sample_acc": 0.012600229095074456,
|
| 298 |
+
"new_proofs": 48,
|
| 299 |
+
"targets_solved": 732,
|
| 300 |
+
"lstar": 10,
|
| 301 |
+
"ge": {
|
| 302 |
+
"7": 732,
|
| 303 |
+
"8": 306,
|
| 304 |
+
"9": 100,
|
| 305 |
+
"10": 6,
|
| 306 |
+
"11": 0,
|
| 307 |
+
"12": 0,
|
| 308 |
+
"13": 0,
|
| 309 |
+
"14": 0
|
| 310 |
+
},
|
| 311 |
+
"mix_rl_records": 48,
|
| 312 |
+
"sampling": {
|
| 313 |
+
"samples": 6984,
|
| 314 |
+
"secs": 35.888651609420776,
|
| 315 |
+
"gate_secs": 14.895998239517212,
|
| 316 |
+
"samples_per_sec": 194.60190580597626
|
| 317 |
+
},
|
| 318 |
+
"secs": 36.10332417488098,
|
| 319 |
+
"rl_secs_used": 107.86937427520752
|
| 320 |
+
},
|
| 321 |
+
{
|
| 322 |
+
"round": 5,
|
| 323 |
+
"level": 1,
|
| 324 |
+
"k": 8,
|
| 325 |
+
"n_targets": 1500,
|
| 326 |
+
"sample_acc": 0.5983333333333334,
|
| 327 |
+
"new_proofs": 523,
|
| 328 |
+
"targets_solved": 1014,
|
| 329 |
+
"lstar": 10,
|
| 330 |
+
"ge": {
|
| 331 |
+
"7": 1014,
|
| 332 |
+
"8": 512,
|
| 333 |
+
"9": 245,
|
| 334 |
+
"10": 33,
|
| 335 |
+
"11": 2,
|
| 336 |
+
"12": 1,
|
| 337 |
+
"13": 0,
|
| 338 |
+
"14": 0
|
| 339 |
+
},
|
| 340 |
+
"mix_rl_records": 1331,
|
| 341 |
+
"sampling": {
|
| 342 |
+
"samples": 12000,
|
| 343 |
+
"secs": 62.51512789726257,
|
| 344 |
+
"gate_secs": 35.82744479179382,
|
| 345 |
+
"samples_per_sec": 191.95353834548357
|
| 346 |
+
},
|
| 347 |
+
"secs": 94.49270796775818,
|
| 348 |
+
"rl_secs_used": 166.5347602367401
|
| 349 |
+
},
|
| 350 |
+
{
|
| 351 |
+
"round": 6,
|
| 352 |
+
"level": 2,
|
| 353 |
+
"k": 24,
|
| 354 |
+
"n_targets": 162,
|
| 355 |
+
"sample_acc": 0.007716049382716049,
|
| 356 |
+
"new_proofs": 13,
|
| 357 |
+
"targets_solved": 1026,
|
| 358 |
+
"lstar": 10,
|
| 359 |
+
"ge": {
|
| 360 |
+
"7": 1026,
|
| 361 |
+
"8": 523,
|
| 362 |
+
"9": 252,
|
| 363 |
+
"10": 35,
|
| 364 |
+
"11": 2,
|
| 365 |
+
"12": 1,
|
| 366 |
+
"13": 0,
|
| 367 |
+
"14": 0
|
| 368 |
+
},
|
| 369 |
+
"mix_rl_records": 13,
|
| 370 |
+
"sampling": {
|
| 371 |
+
"samples": 3888,
|
| 372 |
+
"secs": 21.616540908813477,
|
| 373 |
+
"gate_secs": 10.649104595184326,
|
| 374 |
+
"samples_per_sec": 179.86226456864745
|
| 375 |
+
},
|
| 376 |
+
"secs": 21.765689849853516,
|
| 377 |
+
"rl_secs_used": 177.6511993408203
|
| 378 |
+
},
|
| 379 |
+
{
|
| 380 |
+
"round": 7,
|
| 381 |
+
"level": 4,
|
| 382 |
+
"k": 24,
|
| 383 |
+
"n_targets": 162,
|
| 384 |
+
"sample_acc": 0.00565843621399177,
|
| 385 |
+
"new_proofs": 7,
|
| 386 |
+
"targets_solved": 1033,
|
| 387 |
+
"lstar": 10,
|
| 388 |
+
"ge": {
|
| 389 |
+
"7": 1033,
|
| 390 |
+
"8": 530,
|
| 391 |
+
"9": 259,
|
| 392 |
+
"10": 37,
|
| 393 |
+
"11": 2,
|
| 394 |
+
"12": 1,
|
| 395 |
+
"13": 0,
|
| 396 |
+
"14": 0
|
| 397 |
+
},
|
| 398 |
+
"mix_rl_records": 7,
|
| 399 |
+
"sampling": {
|
| 400 |
+
"samples": 3888,
|
| 401 |
+
"secs": 17.423007011413574,
|
| 402 |
+
"gate_secs": 8.932615280151367,
|
| 403 |
+
"samples_per_sec": 223.15321330313557
|
| 404 |
+
},
|
| 405 |
+
"secs": 17.55361533164978,
|
| 406 |
+
"rl_secs_used": 186.27219939231873
|
| 407 |
+
},
|
| 408 |
+
{
|
| 409 |
+
"round": 8,
|
| 410 |
+
"level": 8,
|
| 411 |
+
"k": 24,
|
| 412 |
+
"n_targets": 162,
|
| 413 |
+
"sample_acc": 0.009259259259259259,
|
| 414 |
+
"new_proofs": 19,
|
| 415 |
+
"targets_solved": 1047,
|
| 416 |
+
"lstar": 10,
|
| 417 |
+
"ge": {
|
| 418 |
+
"7": 1047,
|
| 419 |
+
"8": 539,
|
| 420 |
+
"9": 266,
|
| 421 |
+
"10": 40,
|
| 422 |
+
"11": 2,
|
| 423 |
+
"12": 1,
|
| 424 |
+
"13": 0,
|
| 425 |
+
"14": 0
|
| 426 |
+
},
|
| 427 |
+
"mix_rl_records": 19,
|
| 428 |
+
"sampling": {
|
| 429 |
+
"samples": 3888,
|
| 430 |
+
"secs": 20.20773720741272,
|
| 431 |
+
"gate_secs": 10.925711870193481,
|
| 432 |
+
"samples_per_sec": 192.40155194485513
|
| 433 |
+
},
|
| 434 |
+
"secs": 20.36957311630249,
|
| 435 |
+
"rl_secs_used": 195.71607875823975
|
| 436 |
+
},
|
| 437 |
+
{
|
| 438 |
+
"round": 9,
|
| 439 |
+
"level": 1,
|
| 440 |
+
"k": 8,
|
| 441 |
+
"n_targets": 1500,
|
| 442 |
+
"sample_acc": 0.691,
|
| 443 |
+
"new_proofs": 252,
|
| 444 |
+
"targets_solved": 1107,
|
| 445 |
+
"lstar": 11,
|
| 446 |
+
"ge": {
|
| 447 |
+
"7": 1107,
|
| 448 |
+
"8": 587,
|
| 449 |
+
"9": 299,
|
| 450 |
+
"10": 55,
|
| 451 |
+
"11": 5,
|
| 452 |
+
"12": 1,
|
| 453 |
+
"13": 0,
|
| 454 |
+
"14": 0
|
| 455 |
+
},
|
| 456 |
+
"mix_rl_records": 1545,
|
| 457 |
+
"sampling": {
|
| 458 |
+
"samples": 12000,
|
| 459 |
+
"secs": 51.692243576049805,
|
| 460 |
+
"gate_secs": 28.218570947647095,
|
| 461 |
+
"samples_per_sec": 232.14314508027803
|
| 462 |
+
},
|
| 463 |
+
"secs": 86.44683027267456,
|
| 464 |
+
"rl_secs_used": 253.94453406333923
|
| 465 |
+
},
|
| 466 |
+
{
|
| 467 |
+
"round": 10,
|
| 468 |
+
"level": 2,
|
| 469 |
+
"k": 24,
|
| 470 |
+
"n_targets": 131,
|
| 471 |
+
"sample_acc": 0.008587786259541985,
|
| 472 |
+
"new_proofs": 14,
|
| 473 |
+
"targets_solved": 1117,
|
| 474 |
+
"lstar": 11,
|
| 475 |
+
"ge": {
|
| 476 |
+
"7": 1117,
|
| 477 |
+
"8": 592,
|
| 478 |
+
"9": 304,
|
| 479 |
+
"10": 56,
|
| 480 |
+
"11": 5,
|
| 481 |
+
"12": 1,
|
| 482 |
+
"13": 0,
|
| 483 |
+
"14": 0
|
| 484 |
+
},
|
| 485 |
+
"mix_rl_records": 14,
|
| 486 |
+
"sampling": {
|
| 487 |
+
"samples": 3144,
|
| 488 |
+
"secs": 15.908243894577026,
|
| 489 |
+
"gate_secs": 9.021638870239258,
|
| 490 |
+
"samples_per_sec": 197.63337932427353
|
| 491 |
+
},
|
| 492 |
+
"secs": 16.025395154953003,
|
| 493 |
+
"rl_secs_used": 260.9480698108673
|
| 494 |
+
},
|
| 495 |
+
{
|
| 496 |
+
"round": 11,
|
| 497 |
+
"level": 4,
|
| 498 |
+
"k": 24,
|
| 499 |
+
"n_targets": 131,
|
| 500 |
+
"sample_acc": 0.006361323155216285,
|
| 501 |
+
"new_proofs": 11,
|
| 502 |
+
"targets_solved": 1128,
|
| 503 |
+
"lstar": 11,
|
| 504 |
+
"ge": {
|
| 505 |
+
"7": 1128,
|
| 506 |
+
"8": 600,
|
| 507 |
+
"9": 311,
|
| 508 |
+
"10": 59,
|
| 509 |
+
"11": 6,
|
| 510 |
+
"12": 1,
|
| 511 |
+
"13": 0,
|
| 512 |
+
"14": 0
|
| 513 |
+
},
|
| 514 |
+
"mix_rl_records": 11,
|
| 515 |
+
"sampling": {
|
| 516 |
+
"samples": 3144,
|
| 517 |
+
"secs": 17.698871850967407,
|
| 518 |
+
"gate_secs": 9.045248985290527,
|
| 519 |
+
"samples_per_sec": 177.63844082684577
|
| 520 |
+
},
|
| 521 |
+
"secs": 17.816609859466553,
|
| 522 |
+
"rl_secs_used": 269.71942925453186
|
| 523 |
+
},
|
| 524 |
+
{
|
| 525 |
+
"round": 12,
|
| 526 |
+
"level": 8,
|
| 527 |
+
"k": 24,
|
| 528 |
+
"n_targets": 131,
|
| 529 |
+
"sample_acc": 0.0111323155216285,
|
| 530 |
+
"new_proofs": 16,
|
| 531 |
+
"targets_solved": 1141,
|
| 532 |
+
"lstar": 11,
|
| 533 |
+
"ge": {
|
| 534 |
+
"7": 1141,
|
| 535 |
+
"8": 611,
|
| 536 |
+
"9": 322,
|
| 537 |
+
"10": 64,
|
| 538 |
+
"11": 8,
|
| 539 |
+
"12": 1,
|
| 540 |
+
"13": 0,
|
| 541 |
+
"14": 0
|
| 542 |
+
},
|
| 543 |
+
"mix_rl_records": 16,
|
| 544 |
+
"sampling": {
|
| 545 |
+
"samples": 3144,
|
| 546 |
+
"secs": 14.799442291259766,
|
| 547 |
+
"gate_secs": 8.877211809158325,
|
| 548 |
+
"samples_per_sec": 212.44043783033496
|
| 549 |
+
},
|
| 550 |
+
"secs": 14.911684513092041,
|
| 551 |
+
"rl_secs_used": 275.75390100479126
|
| 552 |
+
},
|
| 553 |
+
{
|
| 554 |
+
"round": 13,
|
| 555 |
+
"level": 1,
|
| 556 |
+
"k": 8,
|
| 557 |
+
"n_targets": 1500,
|
| 558 |
+
"sample_acc": 0.7393333333333333,
|
| 559 |
+
"new_proofs": 219,
|
| 560 |
+
"targets_solved": 1176,
|
| 561 |
+
"lstar": 11,
|
| 562 |
+
"ge": {
|
| 563 |
+
"7": 1176,
|
| 564 |
+
"8": 636,
|
| 565 |
+
"9": 337,
|
| 566 |
+
"10": 77,
|
| 567 |
+
"11": 9,
|
| 568 |
+
"12": 1,
|
| 569 |
+
"13": 0,
|
| 570 |
+
"14": 0
|
| 571 |
+
},
|
| 572 |
+
"mix_rl_records": 1751,
|
| 573 |
+
"sampling": {
|
| 574 |
+
"samples": 12000,
|
| 575 |
+
"secs": 56.74859642982483,
|
| 576 |
+
"gate_secs": 31.435093641281128,
|
| 577 |
+
"samples_per_sec": 211.45897440545812
|
| 578 |
+
},
|
| 579 |
+
"secs": 92.67984652519226,
|
| 580 |
+
"rl_secs_used": 336.99888610839844
|
| 581 |
+
},
|
| 582 |
+
{
|
| 583 |
+
"round": 14,
|
| 584 |
+
"level": 2,
|
| 585 |
+
"k": 24,
|
| 586 |
+
"n_targets": 109,
|
| 587 |
+
"sample_acc": 0.0034403669724770644,
|
| 588 |
+
"new_proofs": 5,
|
| 589 |
+
"targets_solved": 1179,
|
| 590 |
+
"lstar": 11,
|
| 591 |
+
"ge": {
|
| 592 |
+
"7": 1179,
|
| 593 |
+
"8": 638,
|
| 594 |
+
"9": 339,
|
| 595 |
+
"10": 78,
|
| 596 |
+
"11": 10,
|
| 597 |
+
"12": 1,
|
| 598 |
+
"13": 0,
|
| 599 |
+
"14": 0
|
| 600 |
+
},
|
| 601 |
+
"mix_rl_records": 5,
|
| 602 |
+
"sampling": {
|
| 603 |
+
"samples": 2616,
|
| 604 |
+
"secs": 12.687414407730103,
|
| 605 |
+
"gate_secs": 6.6271655559539795,
|
| 606 |
+
"samples_per_sec": 206.18858310532846
|
| 607 |
+
},
|
| 608 |
+
"secs": 12.77810025215149,
|
| 609 |
+
"rl_secs_used": 343.14958453178406
|
| 610 |
+
},
|
| 611 |
+
{
|
| 612 |
+
"round": 15,
|
| 613 |
+
"level": 4,
|
| 614 |
+
"k": 24,
|
| 615 |
+
"n_targets": 108,
|
| 616 |
+
"sample_acc": 0.017746913580246913,
|
| 617 |
+
"new_proofs": 18,
|
| 618 |
+
"targets_solved": 1190,
|
| 619 |
+
"lstar": 11,
|
| 620 |
+
"ge": {
|
| 621 |
+
"7": 1190,
|
| 622 |
+
"8": 648,
|
| 623 |
+
"9": 346,
|
| 624 |
+
"10": 80,
|
| 625 |
+
"11": 10,
|
| 626 |
+
"12": 1,
|
| 627 |
+
"13": 0,
|
| 628 |
+
"14": 0
|
| 629 |
+
},
|
| 630 |
+
"mix_rl_records": 18,
|
| 631 |
+
"sampling": {
|
| 632 |
+
"samples": 2592,
|
| 633 |
+
"secs": 13.075745105743408,
|
| 634 |
+
"gate_secs": 6.673742771148682,
|
| 635 |
+
"samples_per_sec": 198.22962125970827
|
| 636 |
+
},
|
| 637 |
+
"secs": 13.16294264793396,
|
| 638 |
+
"rl_secs_used": 349.6388018131256
|
| 639 |
+
},
|
| 640 |
+
{
|
| 641 |
+
"round": 16,
|
| 642 |
+
"level": 8,
|
| 643 |
+
"k": 24,
|
| 644 |
+
"n_targets": 108,
|
| 645 |
+
"sample_acc": 0.0011574074074074073,
|
| 646 |
+
"new_proofs": 3,
|
| 647 |
+
"targets_solved": 1193,
|
| 648 |
+
"lstar": 11,
|
| 649 |
+
"ge": {
|
| 650 |
+
"7": 1193,
|
| 651 |
+
"8": 650,
|
| 652 |
+
"9": 347,
|
| 653 |
+
"10": 81,
|
| 654 |
+
"11": 10,
|
| 655 |
+
"12": 1,
|
| 656 |
+
"13": 0,
|
| 657 |
+
"14": 0
|
| 658 |
+
},
|
| 659 |
+
"mix_rl_records": 3,
|
| 660 |
+
"sampling": {
|
| 661 |
+
"samples": 2592,
|
| 662 |
+
"secs": 12.169060230255127,
|
| 663 |
+
"gate_secs": 7.233021974563599,
|
| 664 |
+
"samples_per_sec": 212.99919229224312
|
| 665 |
+
},
|
| 666 |
+
"secs": 12.258346557617188,
|
| 667 |
+
"rl_secs_used": 354.66411542892456
|
| 668 |
+
},
|
| 669 |
+
{
|
| 670 |
+
"round": 17,
|
| 671 |
+
"level": 1,
|
| 672 |
+
"k": 8,
|
| 673 |
+
"n_targets": 1500,
|
| 674 |
+
"sample_acc": 0.769,
|
| 675 |
+
"new_proofs": 181,
|
| 676 |
+
"targets_solved": 1206,
|
| 677 |
+
"lstar": 11,
|
| 678 |
+
"ge": {
|
| 679 |
+
"7": 1206,
|
| 680 |
+
"8": 662,
|
| 681 |
+
"9": 357,
|
| 682 |
+
"10": 89,
|
| 683 |
+
"11": 12,
|
| 684 |
+
"12": 2,
|
| 685 |
+
"13": 0,
|
| 686 |
+
"14": 0
|
| 687 |
+
},
|
| 688 |
+
"mix_rl_records": 1905,
|
| 689 |
+
"sampling": {
|
| 690 |
+
"samples": 12000,
|
| 691 |
+
"secs": 64.74264144897461,
|
| 692 |
+
"gate_secs": 36.31877303123474,
|
| 693 |
+
"samples_per_sec": 185.3492494503413
|
| 694 |
+
},
|
| 695 |
+
"secs": 101.42965579032898,
|
| 696 |
+
"rl_secs_used": 419.77515053749084
|
| 697 |
+
},
|
| 698 |
+
{
|
| 699 |
+
"round": 18,
|
| 700 |
+
"level": 2,
|
| 701 |
+
"k": 24,
|
| 702 |
+
"n_targets": 99,
|
| 703 |
+
"sample_acc": 0.0012626262626262627,
|
| 704 |
+
"new_proofs": 2,
|
| 705 |
+
"targets_solved": 1208,
|
| 706 |
+
"lstar": 11,
|
| 707 |
+
"ge": {
|
| 708 |
+
"7": 1208,
|
| 709 |
+
"8": 662,
|
| 710 |
+
"9": 357,
|
| 711 |
+
"10": 89,
|
| 712 |
+
"11": 12,
|
| 713 |
+
"12": 2,
|
| 714 |
+
"13": 0,
|
| 715 |
+
"14": 0
|
| 716 |
+
},
|
| 717 |
+
"mix_rl_records": 2,
|
| 718 |
+
"sampling": {
|
| 719 |
+
"samples": 2376,
|
| 720 |
+
"secs": 10.839726209640503,
|
| 721 |
+
"gate_secs": 5.558458089828491,
|
| 722 |
+
"samples_per_sec": 219.19372814848978
|
| 723 |
+
},
|
| 724 |
+
"secs": 10.918500661849976,
|
| 725 |
+
"rl_secs_used": 425.135036945343
|
| 726 |
+
},
|
| 727 |
+
{
|
| 728 |
+
"round": 19,
|
| 729 |
+
"level": 4,
|
| 730 |
+
"k": 24,
|
| 731 |
+
"n_targets": 98,
|
| 732 |
+
"sample_acc": 0.0055272108843537416,
|
| 733 |
+
"new_proofs": 9,
|
| 734 |
+
"targets_solved": 1216,
|
| 735 |
+
"lstar": 11,
|
| 736 |
+
"ge": {
|
| 737 |
+
"7": 1216,
|
| 738 |
+
"8": 670,
|
| 739 |
+
"9": 362,
|
| 740 |
+
"10": 90,
|
| 741 |
+
"11": 13,
|
| 742 |
+
"12": 2,
|
| 743 |
+
"13": 0,
|
| 744 |
+
"14": 0
|
| 745 |
+
},
|
| 746 |
+
"mix_rl_records": 9,
|
| 747 |
+
"sampling": {
|
| 748 |
+
"samples": 2352,
|
| 749 |
+
"secs": 11.605263710021973,
|
| 750 |
+
"gate_secs": 6.112550497055054,
|
| 751 |
+
"samples_per_sec": 202.666657024681
|
| 752 |
+
},
|
| 753 |
+
"secs": 11.686203718185425,
|
| 754 |
+
"rl_secs_used": 430.70869851112366
|
| 755 |
+
},
|
| 756 |
+
{
|
| 757 |
+
"round": 20,
|
| 758 |
+
"level": 8,
|
| 759 |
+
"k": 24,
|
| 760 |
+
"n_targets": 98,
|
| 761 |
+
"sample_acc": 0.0012755102040816326,
|
| 762 |
+
"new_proofs": 2,
|
| 763 |
+
"targets_solved": 1218,
|
| 764 |
+
"lstar": 11,
|
| 765 |
+
"ge": {
|
| 766 |
+
"7": 1218,
|
| 767 |
+
"8": 672,
|
| 768 |
+
"9": 363,
|
| 769 |
+
"10": 90,
|
| 770 |
+
"11": 13,
|
| 771 |
+
"12": 2,
|
| 772 |
+
"13": 0,
|
| 773 |
+
"14": 0
|
| 774 |
+
},
|
| 775 |
+
"mix_rl_records": 2,
|
| 776 |
+
"sampling": {
|
| 777 |
+
"samples": 2352,
|
| 778 |
+
"secs": 11.557843685150146,
|
| 779 |
+
"gate_secs": 5.491286039352417,
|
| 780 |
+
"samples_per_sec": 203.49816661925598
|
| 781 |
+
},
|
| 782 |
+
"secs": 11.630718231201172,
|
| 783 |
+
"rl_secs_used": 436.84812211990356
|
| 784 |
+
},
|
| 785 |
+
{
|
| 786 |
+
"round": 21,
|
| 787 |
+
"level": 1,
|
| 788 |
+
"k": 8,
|
| 789 |
+
"n_targets": 1500,
|
| 790 |
+
"sample_acc": 0.785,
|
| 791 |
+
"new_proofs": 185,
|
| 792 |
+
"targets_solved": 1229,
|
| 793 |
+
"lstar": 11,
|
| 794 |
+
"ge": {
|
| 795 |
+
"7": 1229,
|
| 796 |
+
"8": 681,
|
| 797 |
+
"9": 370,
|
| 798 |
+
"10": 91,
|
| 799 |
+
"11": 14,
|
| 800 |
+
"12": 2,
|
| 801 |
+
"13": 0,
|
| 802 |
+
"14": 0
|
| 803 |
+
},
|
| 804 |
+
"mix_rl_records": 2034,
|
| 805 |
+
"sampling": {
|
| 806 |
+
"samples": 12000,
|
| 807 |
+
"secs": 58.953579902648926,
|
| 808 |
+
"gate_secs": 33.94571661949158,
|
| 809 |
+
"samples_per_sec": 203.54997982846518
|
| 810 |
+
},
|
| 811 |
+
"secs": 95.96905970573425,
|
| 812 |
+
"rl_secs_used": 498.87162160873413
|
| 813 |
+
},
|
| 814 |
+
{
|
| 815 |
+
"round": 22,
|
| 816 |
+
"level": 2,
|
| 817 |
+
"k": 24,
|
| 818 |
+
"n_targets": 91,
|
| 819 |
+
"sample_acc": 0.0009157509157509158,
|
| 820 |
+
"new_proofs": 1,
|
| 821 |
+
"targets_solved": 1230,
|
| 822 |
+
"lstar": 11,
|
| 823 |
+
"ge": {
|
| 824 |
+
"7": 1230,
|
| 825 |
+
"8": 681,
|
| 826 |
+
"9": 370,
|
| 827 |
+
"10": 91,
|
| 828 |
+
"11": 14,
|
| 829 |
+
"12": 2,
|
| 830 |
+
"13": 0,
|
| 831 |
+
"14": 0
|
| 832 |
+
},
|
| 833 |
+
"mix_rl_records": 1,
|
| 834 |
+
"sampling": {
|
| 835 |
+
"samples": 2184,
|
| 836 |
+
"secs": 13.13532567024231,
|
| 837 |
+
"gate_secs": 5.709080457687378,
|
| 838 |
+
"samples_per_sec": 166.2691930773964
|
| 839 |
+
},
|
| 840 |
+
"secs": 13.216411590576172,
|
| 841 |
+
"rl_secs_used": 506.3788034915924
|
| 842 |
+
},
|
| 843 |
+
{
|
| 844 |
+
"round": 23,
|
| 845 |
+
"level": 4,
|
| 846 |
+
"k": 24,
|
| 847 |
+
"n_targets": 90,
|
| 848 |
+
"sample_acc": 0.002777777777777778,
|
| 849 |
+
"new_proofs": 4,
|
| 850 |
+
"targets_solved": 1233,
|
| 851 |
+
"lstar": 11,
|
| 852 |
+
"ge": {
|
| 853 |
+
"7": 1233,
|
| 854 |
+
"8": 684,
|
| 855 |
+
"9": 373,
|
| 856 |
+
"10": 93,
|
| 857 |
+
"11": 15,
|
| 858 |
+
"12": 3,
|
| 859 |
+
"13": 0,
|
| 860 |
+
"14": 0
|
| 861 |
+
},
|
| 862 |
+
"mix_rl_records": 4,
|
| 863 |
+
"sampling": {
|
| 864 |
+
"samples": 2160,
|
| 865 |
+
"secs": 9.675426006317139,
|
| 866 |
+
"gate_secs": 5.229877471923828,
|
| 867 |
+
"samples_per_sec": 223.2459840620686
|
| 868 |
+
},
|
| 869 |
+
"secs": 9.747728824615479,
|
| 870 |
+
"rl_secs_used": 510.896657705307
|
| 871 |
+
},
|
| 872 |
+
{
|
| 873 |
+
"round": 24,
|
| 874 |
+
"level": 8,
|
| 875 |
+
"k": 24,
|
| 876 |
+
"n_targets": 90,
|
| 877 |
+
"sample_acc": 0.000925925925925926,
|
| 878 |
+
"new_proofs": 2,
|
| 879 |
+
"targets_solved": 1235,
|
| 880 |
+
"lstar": 11,
|
| 881 |
+
"ge": {
|
| 882 |
+
"7": 1235,
|
| 883 |
+
"8": 686,
|
| 884 |
+
"9": 375,
|
| 885 |
+
"10": 93,
|
| 886 |
+
"11": 15,
|
| 887 |
+
"12": 3,
|
| 888 |
+
"13": 0,
|
| 889 |
+
"14": 0
|
| 890 |
+
},
|
| 891 |
+
"mix_rl_records": 2,
|
| 892 |
+
"sampling": {
|
| 893 |
+
"samples": 2160,
|
| 894 |
+
"secs": 11.80867075920105,
|
| 895 |
+
"gate_secs": 5.4752068519592285,
|
| 896 |
+
"samples_per_sec": 182.9164386107536
|
| 897 |
+
},
|
| 898 |
+
"secs": 11.884621143341064,
|
| 899 |
+
"rl_secs_used": 517.3060626983643
|
| 900 |
+
},
|
| 901 |
+
{
|
| 902 |
+
"round": 25,
|
| 903 |
+
"level": 1,
|
| 904 |
+
"k": 8,
|
| 905 |
+
"n_targets": 1500,
|
| 906 |
+
"sample_acc": 0.7980833333333334,
|
| 907 |
+
"new_proofs": 148,
|
| 908 |
+
"targets_solved": 1241,
|
| 909 |
+
"lstar": 11,
|
| 910 |
+
"ge": {
|
| 911 |
+
"7": 1241,
|
| 912 |
+
"8": 690,
|
| 913 |
+
"9": 378,
|
| 914 |
+
"10": 95,
|
| 915 |
+
"11": 16,
|
| 916 |
+
"12": 3,
|
| 917 |
+
"13": 0,
|
| 918 |
+
"14": 0
|
| 919 |
+
},
|
| 920 |
+
"mix_rl_records": 2128,
|
| 921 |
+
"sampling": {
|
| 922 |
+
"samples": 12000,
|
| 923 |
+
"secs": 57.482112646102905,
|
| 924 |
+
"gate_secs": 33.026079177856445,
|
| 925 |
+
"samples_per_sec": 208.76059434140438
|
| 926 |
+
},
|
| 927 |
+
"secs": 95.11074233055115,
|
| 928 |
+
"rl_secs_used": 579.390962600708
|
| 929 |
+
},
|
| 930 |
+
{
|
| 931 |
+
"round": 26,
|
| 932 |
+
"level": 2,
|
| 933 |
+
"k": 24,
|
| 934 |
+
"n_targets": 87,
|
| 935 |
+
"sample_acc": 0.0023946360153256703,
|
| 936 |
+
"new_proofs": 4,
|
| 937 |
+
"targets_solved": 1245,
|
| 938 |
+
"lstar": 11,
|
| 939 |
+
"ge": {
|
| 940 |
+
"7": 1245,
|
| 941 |
+
"8": 693,
|
| 942 |
+
"9": 381,
|
| 943 |
+
"10": 96,
|
| 944 |
+
"11": 16,
|
| 945 |
+
"12": 3,
|
| 946 |
+
"13": 0,
|
| 947 |
+
"14": 0
|
| 948 |
+
},
|
| 949 |
+
"mix_rl_records": 4,
|
| 950 |
+
"sampling": {
|
| 951 |
+
"samples": 2088,
|
| 952 |
+
"secs": 12.137623071670532,
|
| 953 |
+
"gate_secs": 5.21572732925415,
|
| 954 |
+
"samples_per_sec": 172.02709193313441
|
| 955 |
+
},
|
| 956 |
+
"secs": 12.210636854171753,
|
| 957 |
+
"rl_secs_used": 586.3856315612793
|
| 958 |
+
},
|
| 959 |
+
{
|
| 960 |
+
"round": 27,
|
| 961 |
+
"level": 4,
|
| 962 |
+
"k": 24,
|
| 963 |
+
"n_targets": 86,
|
| 964 |
+
"sample_acc": 0.0,
|
| 965 |
+
"new_proofs": 0,
|
| 966 |
+
"targets_solved": 1245,
|
| 967 |
+
"lstar": 11,
|
| 968 |
+
"ge": {
|
| 969 |
+
"7": 1245,
|
| 970 |
+
"8": 693,
|
| 971 |
+
"9": 381,
|
| 972 |
+
"10": 96,
|
| 973 |
+
"11": 16,
|
| 974 |
+
"12": 3,
|
| 975 |
+
"13": 0,
|
| 976 |
+
"14": 0
|
| 977 |
+
},
|
| 978 |
+
"mix_rl_records": 0,
|
| 979 |
+
"sampling": {
|
| 980 |
+
"samples": 2064,
|
| 981 |
+
"secs": 11.939735412597656,
|
| 982 |
+
"gate_secs": 4.552095413208008,
|
| 983 |
+
"samples_per_sec": 172.8681523228954
|
| 984 |
+
},
|
| 985 |
+
"secs": 12.005858421325684,
|
| 986 |
+
"rl_secs_used": 593.8393940925598
|
| 987 |
+
},
|
| 988 |
+
{
|
| 989 |
+
"round": 28,
|
| 990 |
+
"level": 8,
|
| 991 |
+
"k": 24,
|
| 992 |
+
"n_targets": 86,
|
| 993 |
+
"sample_acc": 0.00048449612403100775,
|
| 994 |
+
"new_proofs": 1,
|
| 995 |
+
"targets_solved": 1246,
|
| 996 |
+
"lstar": 11,
|
| 997 |
+
"ge": {
|
| 998 |
+
"7": 1246,
|
| 999 |
+
"8": 694,
|
| 1000 |
+
"9": 382,
|
| 1001 |
+
"10": 97,
|
| 1002 |
+
"11": 16,
|
| 1003 |
+
"12": 3,
|
| 1004 |
+
"13": 0,
|
| 1005 |
+
"14": 0
|
| 1006 |
+
},
|
| 1007 |
+
"mix_rl_records": 1,
|
| 1008 |
+
"sampling": {
|
| 1009 |
+
"samples": 2064,
|
| 1010 |
+
"secs": 9.376678466796875,
|
| 1011 |
+
"gate_secs": 4.408054828643799,
|
| 1012 |
+
"samples_per_sec": 220.1205903890905
|
| 1013 |
+
},
|
| 1014 |
+
"secs": 9.437121391296387,
|
| 1015 |
+
"rl_secs_used": 598.8684594631195
|
| 1016 |
+
},
|
| 1017 |
+
{
|
| 1018 |
+
"round": 29,
|
| 1019 |
+
"level": 1,
|
| 1020 |
+
"k": 8,
|
| 1021 |
+
"n_targets": 1500,
|
| 1022 |
+
"sample_acc": 0.8060833333333334,
|
| 1023 |
+
"new_proofs": 129,
|
| 1024 |
+
"targets_solved": 1248,
|
| 1025 |
+
"lstar": 11,
|
| 1026 |
+
"ge": {
|
| 1027 |
+
"7": 1248,
|
| 1028 |
+
"8": 695,
|
| 1029 |
+
"9": 383,
|
| 1030 |
+
"10": 97,
|
| 1031 |
+
"11": 16,
|
| 1032 |
+
"12": 3,
|
| 1033 |
+
"13": 0,
|
| 1034 |
+
"14": 0
|
| 1035 |
+
},
|
| 1036 |
+
"mix_rl_records": 2220,
|
| 1037 |
+
"sampling": {
|
| 1038 |
+
"samples": 12000,
|
| 1039 |
+
"secs": 57.369160890579224,
|
| 1040 |
+
"gate_secs": 32.32744026184082,
|
| 1041 |
+
"samples_per_sec": 209.17161439554118
|
| 1042 |
+
},
|
| 1043 |
+
"secs": 95.50686025619507,
|
| 1044 |
+
"rl_secs_used": 662.0481176376343
|
| 1045 |
+
},
|
| 1046 |
+
{
|
| 1047 |
+
"round": 30,
|
| 1048 |
+
"level": 2,
|
| 1049 |
+
"k": 24,
|
| 1050 |
+
"n_targets": 84,
|
| 1051 |
+
"sample_acc": 0.0,
|
| 1052 |
+
"new_proofs": 0,
|
| 1053 |
+
"targets_solved": 1248,
|
| 1054 |
+
"lstar": 11,
|
| 1055 |
+
"ge": {
|
| 1056 |
+
"7": 1248,
|
| 1057 |
+
"8": 695,
|
| 1058 |
+
"9": 383,
|
| 1059 |
+
"10": 97,
|
| 1060 |
+
"11": 16,
|
| 1061 |
+
"12": 3,
|
| 1062 |
+
"13": 0,
|
| 1063 |
+
"14": 0
|
| 1064 |
+
},
|
| 1065 |
+
"mix_rl_records": 0,
|
| 1066 |
+
"sampling": {
|
| 1067 |
+
"samples": 2016,
|
| 1068 |
+
"secs": 10.358954429626465,
|
| 1069 |
+
"gate_secs": 4.623164653778076,
|
| 1070 |
+
"samples_per_sec": 194.61423579915248
|
| 1071 |
+
},
|
| 1072 |
+
"secs": 10.423393726348877,
|
| 1073 |
+
"rl_secs_used": 667.8481087684631
|
| 1074 |
+
},
|
| 1075 |
+
{
|
| 1076 |
+
"round": 31,
|
| 1077 |
+
"level": 4,
|
| 1078 |
+
"k": 24,
|
| 1079 |
+
"n_targets": 84,
|
| 1080 |
+
"sample_acc": 0.0,
|
| 1081 |
+
"new_proofs": 0,
|
| 1082 |
+
"targets_solved": 1248,
|
| 1083 |
+
"lstar": 11,
|
| 1084 |
+
"ge": {
|
| 1085 |
+
"7": 1248,
|
| 1086 |
+
"8": 695,
|
| 1087 |
+
"9": 383,
|
| 1088 |
+
"10": 97,
|
| 1089 |
+
"11": 16,
|
| 1090 |
+
"12": 3,
|
| 1091 |
+
"13": 0,
|
| 1092 |
+
"14": 0
|
| 1093 |
+
},
|
| 1094 |
+
"mix_rl_records": 0,
|
| 1095 |
+
"sampling": {
|
| 1096 |
+
"samples": 2016,
|
| 1097 |
+
"secs": 11.444467306137085,
|
| 1098 |
+
"gate_secs": 4.465373516082764,
|
| 1099 |
+
"samples_per_sec": 176.15498791445907
|
| 1100 |
+
},
|
| 1101 |
+
"secs": 11.505950927734375,
|
| 1102 |
+
"rl_secs_used": 674.8886890411377
|
| 1103 |
+
},
|
| 1104 |
+
{
|
| 1105 |
+
"round": 32,
|
| 1106 |
+
"level": 8,
|
| 1107 |
+
"k": 24,
|
| 1108 |
+
"n_targets": 84,
|
| 1109 |
+
"sample_acc": 0.000992063492063492,
|
| 1110 |
+
"new_proofs": 2,
|
| 1111 |
+
"targets_solved": 1250,
|
| 1112 |
+
"lstar": 11,
|
| 1113 |
+
"ge": {
|
| 1114 |
+
"7": 1250,
|
| 1115 |
+
"8": 697,
|
| 1116 |
+
"9": 385,
|
| 1117 |
+
"10": 97,
|
| 1118 |
+
"11": 16,
|
| 1119 |
+
"12": 3,
|
| 1120 |
+
"13": 0,
|
| 1121 |
+
"14": 0
|
| 1122 |
+
},
|
| 1123 |
+
"mix_rl_records": 2,
|
| 1124 |
+
"sampling": {
|
| 1125 |
+
"samples": 2016,
|
| 1126 |
+
"secs": 8.455175161361694,
|
| 1127 |
+
"gate_secs": 4.679951906204224,
|
| 1128 |
+
"samples_per_sec": 238.4338540037207
|
| 1129 |
+
},
|
| 1130 |
+
"secs": 8.518441677093506,
|
| 1131 |
+
"rl_secs_used": 678.7271831035614
|
| 1132 |
+
},
|
| 1133 |
+
{
|
| 1134 |
+
"round": 33,
|
| 1135 |
+
"level": 1,
|
| 1136 |
+
"k": 8,
|
| 1137 |
+
"n_targets": 1500,
|
| 1138 |
+
"sample_acc": 0.81275,
|
| 1139 |
+
"new_proofs": 178,
|
| 1140 |
+
"targets_solved": 1252,
|
| 1141 |
+
"lstar": 11,
|
| 1142 |
+
"ge": {
|
| 1143 |
+
"7": 1252,
|
| 1144 |
+
"8": 699,
|
| 1145 |
+
"9": 387,
|
| 1146 |
+
"10": 98,
|
| 1147 |
+
"11": 17,
|
| 1148 |
+
"12": 4,
|
| 1149 |
+
"13": 1,
|
| 1150 |
+
"14": 0
|
| 1151 |
+
},
|
| 1152 |
+
"mix_rl_records": 2388,
|
| 1153 |
+
"sampling": {
|
| 1154 |
+
"samples": 12000,
|
| 1155 |
+
"secs": 63.13167691230774,
|
| 1156 |
+
"gate_secs": 31.262810230255127,
|
| 1157 |
+
"samples_per_sec": 190.07890471004674
|
| 1158 |
+
},
|
| 1159 |
+
"secs": 101.95304489135742,
|
| 1160 |
+
"rl_secs_used": 749.4176428318024
|
| 1161 |
+
},
|
| 1162 |
+
{
|
| 1163 |
+
"round": 34,
|
| 1164 |
+
"level": 2,
|
| 1165 |
+
"k": 24,
|
| 1166 |
+
"n_targets": 83,
|
| 1167 |
+
"sample_acc": 0.001004016064257028,
|
| 1168 |
+
"new_proofs": 2,
|
| 1169 |
+
"targets_solved": 1254,
|
| 1170 |
+
"lstar": 11,
|
| 1171 |
+
"ge": {
|
| 1172 |
+
"7": 1254,
|
| 1173 |
+
"8": 700,
|
| 1174 |
+
"9": 388,
|
| 1175 |
+
"10": 99,
|
| 1176 |
+
"11": 18,
|
| 1177 |
+
"12": 4,
|
| 1178 |
+
"13": 1,
|
| 1179 |
+
"14": 0
|
| 1180 |
+
},
|
| 1181 |
+
"mix_rl_records": 2,
|
| 1182 |
+
"sampling": {
|
| 1183 |
+
"samples": 1992,
|
| 1184 |
+
"secs": 11.663103103637695,
|
| 1185 |
+
"gate_secs": 4.423825025558472,
|
| 1186 |
+
"samples_per_sec": 170.79502618635857
|
| 1187 |
+
},
|
| 1188 |
+
"secs": 11.728440046310425,
|
| 1189 |
+
"rl_secs_used": 756.7220275402069
|
| 1190 |
+
},
|
| 1191 |
+
{
|
| 1192 |
+
"round": 35,
|
| 1193 |
+
"level": 4,
|
| 1194 |
+
"k": 24,
|
| 1195 |
+
"n_targets": 83,
|
| 1196 |
+
"sample_acc": 0.0015060240963855422,
|
| 1197 |
+
"new_proofs": 3,
|
| 1198 |
+
"targets_solved": 1257,
|
| 1199 |
+
"lstar": 11,
|
| 1200 |
+
"ge": {
|
| 1201 |
+
"7": 1257,
|
| 1202 |
+
"8": 702,
|
| 1203 |
+
"9": 389,
|
| 1204 |
+
"10": 99,
|
| 1205 |
+
"11": 18,
|
| 1206 |
+
"12": 4,
|
| 1207 |
+
"13": 1,
|
| 1208 |
+
"14": 0
|
| 1209 |
+
},
|
| 1210 |
+
"mix_rl_records": 3,
|
| 1211 |
+
"sampling": {
|
| 1212 |
+
"samples": 1992,
|
| 1213 |
+
"secs": 11.743556261062622,
|
| 1214 |
+
"gate_secs": 4.1855762004852295,
|
| 1215 |
+
"samples_per_sec": 169.6249377715974
|
| 1216 |
+
},
|
| 1217 |
+
"secs": 11.801961898803711,
|
| 1218 |
+
"rl_secs_used": 764.3384103775024
|
| 1219 |
+
},
|
| 1220 |
+
{
|
| 1221 |
+
"round": 36,
|
| 1222 |
+
"level": 8,
|
| 1223 |
+
"k": 24,
|
| 1224 |
+
"n_targets": 82,
|
| 1225 |
+
"sample_acc": 0.0010162601626016261,
|
| 1226 |
+
"new_proofs": 1,
|
| 1227 |
+
"targets_solved": 1258,
|
| 1228 |
+
"lstar": 11,
|
| 1229 |
+
"ge": {
|
| 1230 |
+
"7": 1258,
|
| 1231 |
+
"8": 702,
|
| 1232 |
+
"9": 389,
|
| 1233 |
+
"10": 99,
|
| 1234 |
+
"11": 18,
|
| 1235 |
+
"12": 4,
|
| 1236 |
+
"13": 1,
|
| 1237 |
+
"14": 0
|
| 1238 |
+
},
|
| 1239 |
+
"mix_rl_records": 1,
|
| 1240 |
+
"sampling": {
|
| 1241 |
+
"samples": 1968,
|
| 1242 |
+
"secs": 10.082074642181396,
|
| 1243 |
+
"gate_secs": 4.430318832397461,
|
| 1244 |
+
"samples_per_sec": 195.19792005568766
|
| 1245 |
+
},
|
| 1246 |
+
"secs": 10.144181251525879,
|
| 1247 |
+
"rl_secs_used": 770.0522747039795
|
| 1248 |
+
},
|
| 1249 |
+
{
|
| 1250 |
+
"round": 37,
|
| 1251 |
+
"level": 1,
|
| 1252 |
+
"k": 8,
|
| 1253 |
+
"n_targets": 1500,
|
| 1254 |
+
"sample_acc": 0.8143333333333334,
|
| 1255 |
+
"new_proofs": 187,
|
| 1256 |
+
"targets_solved": 1262,
|
| 1257 |
+
"lstar": 11,
|
| 1258 |
+
"ge": {
|
| 1259 |
+
"7": 1262,
|
| 1260 |
+
"8": 705,
|
| 1261 |
+
"9": 392,
|
| 1262 |
+
"10": 101,
|
| 1263 |
+
"11": 18,
|
| 1264 |
+
"12": 4,
|
| 1265 |
+
"13": 1,
|
| 1266 |
+
"14": 0
|
| 1267 |
+
},
|
| 1268 |
+
"mix_rl_records": 2476,
|
| 1269 |
+
"sampling": {
|
| 1270 |
+
"samples": 12000,
|
| 1271 |
+
"secs": 62.811208963394165,
|
| 1272 |
+
"gate_secs": 32.62139940261841,
|
| 1273 |
+
"samples_per_sec": 191.048702899406
|
| 1274 |
+
},
|
| 1275 |
+
"secs": 102.03536176681519,
|
| 1276 |
+
"rl_secs_used": 839.4664659500122
|
| 1277 |
+
},
|
| 1278 |
+
{
|
| 1279 |
+
"round": 38,
|
| 1280 |
+
"level": 2,
|
| 1281 |
+
"k": 24,
|
| 1282 |
+
"n_targets": 80,
|
| 1283 |
+
"sample_acc": 0.0005208333333333333,
|
| 1284 |
+
"new_proofs": 1,
|
| 1285 |
+
"targets_solved": 1263,
|
| 1286 |
+
"lstar": 11,
|
| 1287 |
+
"ge": {
|
| 1288 |
+
"7": 1263,
|
| 1289 |
+
"8": 706,
|
| 1290 |
+
"9": 393,
|
| 1291 |
+
"10": 102,
|
| 1292 |
+
"11": 18,
|
| 1293 |
+
"12": 4,
|
| 1294 |
+
"13": 1,
|
| 1295 |
+
"14": 0
|
| 1296 |
+
},
|
| 1297 |
+
"mix_rl_records": 1,
|
| 1298 |
+
"sampling": {
|
| 1299 |
+
"samples": 1920,
|
| 1300 |
+
"secs": 9.2752845287323,
|
| 1301 |
+
"gate_secs": 4.018190622329712,
|
| 1302 |
+
"samples_per_sec": 207.0017360709921
|
| 1303 |
+
},
|
| 1304 |
+
"secs": 9.324950695037842,
|
| 1305 |
+
"rl_secs_used": 844.77299284935
|
| 1306 |
+
},
|
| 1307 |
+
{
|
| 1308 |
+
"round": 39,
|
| 1309 |
+
"level": 4,
|
| 1310 |
+
"k": 24,
|
| 1311 |
+
"n_targets": 79,
|
| 1312 |
+
"sample_acc": 0.0,
|
| 1313 |
+
"new_proofs": 0,
|
| 1314 |
+
"targets_solved": 1263,
|
| 1315 |
+
"lstar": 11,
|
| 1316 |
+
"ge": {
|
| 1317 |
+
"7": 1263,
|
| 1318 |
+
"8": 706,
|
| 1319 |
+
"9": 393,
|
| 1320 |
+
"10": 102,
|
| 1321 |
+
"11": 18,
|
| 1322 |
+
"12": 4,
|
| 1323 |
+
"13": 1,
|
| 1324 |
+
"14": 0
|
| 1325 |
+
},
|
| 1326 |
+
"mix_rl_records": 0,
|
| 1327 |
+
"sampling": {
|
| 1328 |
+
"samples": 1896,
|
| 1329 |
+
"secs": 11.218536615371704,
|
| 1330 |
+
"gate_secs": 4.591203212738037,
|
| 1331 |
+
"samples_per_sec": 169.00600006974972
|
| 1332 |
+
},
|
| 1333 |
+
"secs": 11.278528690338135,
|
| 1334 |
+
"rl_secs_used": 851.4603216648102
|
| 1335 |
+
},
|
| 1336 |
+
{
|
| 1337 |
+
"round": 40,
|
| 1338 |
+
"level": 8,
|
| 1339 |
+
"k": 24,
|
| 1340 |
+
"n_targets": 79,
|
| 1341 |
+
"sample_acc": 0.0005274261603375527,
|
| 1342 |
+
"new_proofs": 1,
|
| 1343 |
+
"targets_solved": 1264,
|
| 1344 |
+
"lstar": 11,
|
| 1345 |
+
"ge": {
|
| 1346 |
+
"7": 1264,
|
| 1347 |
+
"8": 707,
|
| 1348 |
+
"9": 394,
|
| 1349 |
+
"10": 102,
|
| 1350 |
+
"11": 18,
|
| 1351 |
+
"12": 4,
|
| 1352 |
+
"13": 1,
|
| 1353 |
+
"14": 0
|
| 1354 |
+
},
|
| 1355 |
+
"mix_rl_records": 1,
|
| 1356 |
+
"sampling": {
|
| 1357 |
+
"samples": 1896,
|
| 1358 |
+
"secs": 8.67183804512024,
|
| 1359 |
+
"gate_secs": 4.397043466567993,
|
| 1360 |
+
"samples_per_sec": 218.6387695589985
|
| 1361 |
+
},
|
| 1362 |
+
"secs": 8.72948169708252,
|
| 1363 |
+
"rl_secs_used": 855.7927589416504
|
| 1364 |
+
},
|
| 1365 |
+
{
|
| 1366 |
+
"round": 41,
|
| 1367 |
+
"level": 1,
|
| 1368 |
+
"k": 8,
|
| 1369 |
+
"n_targets": 1500,
|
| 1370 |
+
"sample_acc": 0.8198333333333333,
|
| 1371 |
+
"new_proofs": 160,
|
| 1372 |
+
"targets_solved": 1268,
|
| 1373 |
+
"lstar": 11,
|
| 1374 |
+
"ge": {
|
| 1375 |
+
"7": 1268,
|
| 1376 |
+
"8": 710,
|
| 1377 |
+
"9": 397,
|
| 1378 |
+
"10": 102,
|
| 1379 |
+
"11": 18,
|
| 1380 |
+
"12": 4,
|
| 1381 |
+
"13": 1,
|
| 1382 |
+
"14": 0
|
| 1383 |
+
},
|
| 1384 |
+
"mix_rl_records": 2583,
|
| 1385 |
+
"sampling": {
|
| 1386 |
+
"samples": 12000,
|
| 1387 |
+
"secs": 54.82120609283447,
|
| 1388 |
+
"gate_secs": 31.191574335098267,
|
| 1389 |
+
"samples_per_sec": 218.89339646557843
|
| 1390 |
+
},
|
| 1391 |
+
"secs": 93.82205104827881,
|
| 1392 |
+
"rl_secs_used": 918.4234540462494
|
| 1393 |
+
},
|
| 1394 |
+
{
|
| 1395 |
+
"round": 42,
|
| 1396 |
+
"level": 2,
|
| 1397 |
+
"k": 24,
|
| 1398 |
+
"n_targets": 78,
|
| 1399 |
+
"sample_acc": 0.0,
|
| 1400 |
+
"new_proofs": 0,
|
| 1401 |
+
"targets_solved": 1268,
|
| 1402 |
+
"lstar": 11,
|
| 1403 |
+
"ge": {
|
| 1404 |
+
"7": 1268,
|
| 1405 |
+
"8": 710,
|
| 1406 |
+
"9": 397,
|
| 1407 |
+
"10": 102,
|
| 1408 |
+
"11": 18,
|
| 1409 |
+
"12": 4,
|
| 1410 |
+
"13": 1,
|
| 1411 |
+
"14": 0
|
| 1412 |
+
},
|
| 1413 |
+
"mix_rl_records": 0,
|
| 1414 |
+
"sampling": {
|
| 1415 |
+
"samples": 1872,
|
| 1416 |
+
"secs": 8.76870059967041,
|
| 1417 |
+
"gate_secs": 4.21483588218689,
|
| 1418 |
+
"samples_per_sec": 213.48659116840676
|
| 1419 |
+
},
|
| 1420 |
+
"secs": 8.828282356262207,
|
| 1421 |
+
"rl_secs_used": 923.036682844162
|
| 1422 |
+
},
|
| 1423 |
+
{
|
| 1424 |
+
"round": 43,
|
| 1425 |
+
"level": 4,
|
| 1426 |
+
"k": 24,
|
| 1427 |
+
"n_targets": 77,
|
| 1428 |
+
"sample_acc": 0.0,
|
| 1429 |
+
"new_proofs": 0,
|
| 1430 |
+
"targets_solved": 1268,
|
| 1431 |
+
"lstar": 11,
|
| 1432 |
+
"ge": {
|
| 1433 |
+
"7": 1268,
|
| 1434 |
+
"8": 710,
|
| 1435 |
+
"9": 397,
|
| 1436 |
+
"10": 102,
|
| 1437 |
+
"11": 18,
|
| 1438 |
+
"12": 4,
|
| 1439 |
+
"13": 1,
|
| 1440 |
+
"14": 0
|
| 1441 |
+
},
|
| 1442 |
+
"mix_rl_records": 0,
|
| 1443 |
+
"sampling": {
|
| 1444 |
+
"samples": 1848,
|
| 1445 |
+
"secs": 10.38786244392395,
|
| 1446 |
+
"gate_secs": 4.362722873687744,
|
| 1447 |
+
"samples_per_sec": 177.8999298436926
|
| 1448 |
+
},
|
| 1449 |
+
"secs": 10.475448369979858,
|
| 1450 |
+
"rl_secs_used": 929.1494071483612
|
| 1451 |
+
},
|
| 1452 |
+
{
|
| 1453 |
+
"round": 44,
|
| 1454 |
+
"level": 8,
|
| 1455 |
+
"k": 24,
|
| 1456 |
+
"n_targets": 77,
|
| 1457 |
+
"sample_acc": 0.0010822510822510823,
|
| 1458 |
+
"new_proofs": 1,
|
| 1459 |
+
"targets_solved": 1269,
|
| 1460 |
+
"lstar": 11,
|
| 1461 |
+
"ge": {
|
| 1462 |
+
"7": 1269,
|
| 1463 |
+
"8": 711,
|
| 1464 |
+
"9": 398,
|
| 1465 |
+
"10": 102,
|
| 1466 |
+
"11": 18,
|
| 1467 |
+
"12": 4,
|
| 1468 |
+
"13": 1,
|
| 1469 |
+
"14": 0
|
| 1470 |
+
},
|
| 1471 |
+
"mix_rl_records": 1,
|
| 1472 |
+
"sampling": {
|
| 1473 |
+
"samples": 1848,
|
| 1474 |
+
"secs": 9.193273305892944,
|
| 1475 |
+
"gate_secs": 4.234066724777222,
|
| 1476 |
+
"samples_per_sec": 201.01654095450647
|
| 1477 |
+
},
|
| 1478 |
+
"secs": 9.250768899917603,
|
| 1479 |
+
"rl_secs_used": 934.1661093235016
|
| 1480 |
+
},
|
| 1481 |
+
{
|
| 1482 |
+
"round": 45,
|
| 1483 |
+
"level": 1,
|
| 1484 |
+
"k": 8,
|
| 1485 |
+
"n_targets": 1500,
|
| 1486 |
+
"sample_acc": 0.8270833333333333,
|
| 1487 |
+
"new_proofs": 156,
|
| 1488 |
+
"targets_solved": 1274,
|
| 1489 |
+
"lstar": 11,
|
| 1490 |
+
"ge": {
|
| 1491 |
+
"7": 1274,
|
| 1492 |
+
"8": 714,
|
| 1493 |
+
"9": 401,
|
| 1494 |
+
"10": 103,
|
| 1495 |
+
"11": 18,
|
| 1496 |
+
"12": 4,
|
| 1497 |
+
"13": 1,
|
| 1498 |
+
"14": 0
|
| 1499 |
+
},
|
| 1500 |
+
"mix_rl_records": 2685,
|
| 1501 |
+
"sampling": {
|
| 1502 |
+
"samples": 12000,
|
| 1503 |
+
"secs": 65.98701977729797,
|
| 1504 |
+
"gate_secs": 31.51468300819397,
|
| 1505 |
+
"samples_per_sec": 181.8539470413915
|
| 1506 |
+
},
|
| 1507 |
+
"secs": 106.23185753822327,
|
| 1508 |
+
"rl_secs_used": 1008.8835937976837
|
| 1509 |
+
},
|
| 1510 |
+
{
|
| 1511 |
+
"round": 46,
|
| 1512 |
+
"level": 2,
|
| 1513 |
+
"k": 24,
|
| 1514 |
+
"n_targets": 76,
|
| 1515 |
+
"sample_acc": 0.0010964912280701754,
|
| 1516 |
+
"new_proofs": 1,
|
| 1517 |
+
"targets_solved": 1275,
|
| 1518 |
+
"lstar": 11,
|
| 1519 |
+
"ge": {
|
| 1520 |
+
"7": 1275,
|
| 1521 |
+
"8": 715,
|
| 1522 |
+
"9": 402,
|
| 1523 |
+
"10": 103,
|
| 1524 |
+
"11": 18,
|
| 1525 |
+
"12": 4,
|
| 1526 |
+
"13": 1,
|
| 1527 |
+
"14": 0
|
| 1528 |
+
},
|
| 1529 |
+
"mix_rl_records": 1,
|
| 1530 |
+
"sampling": {
|
| 1531 |
+
"samples": 1824,
|
| 1532 |
+
"secs": 11.321188926696777,
|
| 1533 |
+
"gate_secs": 4.16279935836792,
|
| 1534 |
+
"samples_per_sec": 161.11382044855557
|
| 1535 |
+
},
|
| 1536 |
+
"secs": 11.381211519241333,
|
| 1537 |
+
"rl_secs_used": 1016.1017034053802
|
| 1538 |
+
},
|
| 1539 |
+
{
|
| 1540 |
+
"round": 47,
|
| 1541 |
+
"level": 4,
|
| 1542 |
+
"k": 24,
|
| 1543 |
+
"n_targets": 75,
|
| 1544 |
+
"sample_acc": 0.0005555555555555556,
|
| 1545 |
+
"new_proofs": 1,
|
| 1546 |
+
"targets_solved": 1276,
|
| 1547 |
+
"lstar": 11,
|
| 1548 |
+
"ge": {
|
| 1549 |
+
"7": 1276,
|
| 1550 |
+
"8": 716,
|
| 1551 |
+
"9": 403,
|
| 1552 |
+
"10": 104,
|
| 1553 |
+
"11": 18,
|
| 1554 |
+
"12": 4,
|
| 1555 |
+
"13": 1,
|
| 1556 |
+
"14": 0
|
| 1557 |
+
},
|
| 1558 |
+
"mix_rl_records": 1,
|
| 1559 |
+
"sampling": {
|
| 1560 |
+
"samples": 1800,
|
| 1561 |
+
"secs": 11.093557357788086,
|
| 1562 |
+
"gate_secs": 3.53816556930542,
|
| 1563 |
+
"samples_per_sec": 162.2563386970126
|
| 1564 |
+
},
|
| 1565 |
+
"secs": 11.145751953125,
|
| 1566 |
+
"rl_secs_used": 1023.709291934967
|
| 1567 |
+
},
|
| 1568 |
+
{
|
| 1569 |
+
"round": 48,
|
| 1570 |
+
"level": 8,
|
| 1571 |
+
"k": 24,
|
| 1572 |
+
"n_targets": 75,
|
| 1573 |
+
"sample_acc": 0.002777777777777778,
|
| 1574 |
+
"new_proofs": 4,
|
| 1575 |
+
"targets_solved": 1279,
|
| 1576 |
+
"lstar": 12,
|
| 1577 |
+
"ge": {
|
| 1578 |
+
"7": 1279,
|
| 1579 |
+
"8": 719,
|
| 1580 |
+
"9": 406,
|
| 1581 |
+
"10": 105,
|
| 1582 |
+
"11": 19,
|
| 1583 |
+
"12": 5,
|
| 1584 |
+
"13": 2,
|
| 1585 |
+
"14": 1
|
| 1586 |
+
},
|
| 1587 |
+
"mix_rl_records": 4,
|
| 1588 |
+
"sampling": {
|
| 1589 |
+
"samples": 1800,
|
| 1590 |
+
"secs": 11.368651866912842,
|
| 1591 |
+
"gate_secs": 3.6738171577453613,
|
| 1592 |
+
"samples_per_sec": 158.3301187398212
|
| 1593 |
+
},
|
| 1594 |
+
"secs": 11.425716638565063,
|
| 1595 |
+
"rl_secs_used": 1031.4611945152283
|
| 1596 |
+
},
|
| 1597 |
+
{
|
| 1598 |
+
"round": 49,
|
| 1599 |
+
"level": 1,
|
| 1600 |
+
"k": 8,
|
| 1601 |
+
"n_targets": 1500,
|
| 1602 |
+
"sample_acc": 0.83475,
|
| 1603 |
+
"new_proofs": 154,
|
| 1604 |
+
"targets_solved": 1280,
|
| 1605 |
+
"lstar": 12,
|
| 1606 |
+
"ge": {
|
| 1607 |
+
"7": 1280,
|
| 1608 |
+
"8": 720,
|
| 1609 |
+
"9": 407,
|
| 1610 |
+
"10": 105,
|
| 1611 |
+
"11": 19,
|
| 1612 |
+
"12": 5,
|
| 1613 |
+
"13": 2,
|
| 1614 |
+
"14": 1
|
| 1615 |
+
},
|
| 1616 |
+
"mix_rl_records": 2763,
|
| 1617 |
+
"sampling": {
|
| 1618 |
+
"samples": 12000,
|
| 1619 |
+
"secs": 60.34134888648987,
|
| 1620 |
+
"gate_secs": 31.659582138061523,
|
| 1621 |
+
"samples_per_sec": 198.86860704048235
|
| 1622 |
+
},
|
| 1623 |
+
"secs": 100.36369943618774,
|
| 1624 |
+
"rl_secs_used": 1100.165493965149
|
| 1625 |
+
},
|
| 1626 |
+
{
|
| 1627 |
+
"round": 50,
|
| 1628 |
+
"level": 2,
|
| 1629 |
+
"k": 24,
|
| 1630 |
+
"n_targets": 74,
|
| 1631 |
+
"sample_acc": 0.0,
|
| 1632 |
+
"new_proofs": 0,
|
| 1633 |
+
"targets_solved": 1280,
|
| 1634 |
+
"lstar": 12,
|
| 1635 |
+
"ge": {
|
| 1636 |
+
"7": 1280,
|
| 1637 |
+
"8": 720,
|
| 1638 |
+
"9": 407,
|
| 1639 |
+
"10": 105,
|
| 1640 |
+
"11": 19,
|
| 1641 |
+
"12": 5,
|
| 1642 |
+
"13": 2,
|
| 1643 |
+
"14": 1
|
| 1644 |
+
},
|
| 1645 |
+
"mix_rl_records": 0,
|
| 1646 |
+
"sampling": {
|
| 1647 |
+
"samples": 1776,
|
| 1648 |
+
"secs": 10.07914924621582,
|
| 1649 |
+
"gate_secs": 3.9172725677490234,
|
| 1650 |
+
"samples_per_sec": 176.20534795303212
|
| 1651 |
+
},
|
| 1652 |
+
"secs": 10.135313510894775,
|
| 1653 |
+
"rl_secs_used": 1106.383339881897
|
| 1654 |
+
},
|
| 1655 |
+
{
|
| 1656 |
+
"round": 51,
|
| 1657 |
+
"level": 4,
|
| 1658 |
+
"k": 24,
|
| 1659 |
+
"n_targets": 74,
|
| 1660 |
+
"sample_acc": 0.0005630630630630631,
|
| 1661 |
+
"new_proofs": 0,
|
| 1662 |
+
"targets_solved": 1280,
|
| 1663 |
+
"lstar": 12,
|
| 1664 |
+
"ge": {
|
| 1665 |
+
"7": 1280,
|
| 1666 |
+
"8": 720,
|
| 1667 |
+
"9": 407,
|
| 1668 |
+
"10": 105,
|
| 1669 |
+
"11": 19,
|
| 1670 |
+
"12": 5,
|
| 1671 |
+
"13": 2,
|
| 1672 |
+
"14": 1
|
| 1673 |
+
},
|
| 1674 |
+
"mix_rl_records": 1,
|
| 1675 |
+
"sampling": {
|
| 1676 |
+
"samples": 1776,
|
| 1677 |
+
"secs": 9.937037944793701,
|
| 1678 |
+
"gate_secs": 3.6553399562835693,
|
| 1679 |
+
"samples_per_sec": 178.72529116490867
|
| 1680 |
+
},
|
| 1681 |
+
"secs": 9.990573644638062,
|
| 1682 |
+
"rl_secs_used": 1112.7185723781586
|
| 1683 |
+
},
|
| 1684 |
+
{
|
| 1685 |
+
"round": 52,
|
| 1686 |
+
"level": 8,
|
| 1687 |
+
"k": 24,
|
| 1688 |
+
"n_targets": 73,
|
| 1689 |
+
"sample_acc": 0.0,
|
| 1690 |
+
"new_proofs": 0,
|
| 1691 |
+
"targets_solved": 1280,
|
| 1692 |
+
"lstar": 12,
|
| 1693 |
+
"ge": {
|
| 1694 |
+
"7": 1280,
|
| 1695 |
+
"8": 720,
|
| 1696 |
+
"9": 407,
|
| 1697 |
+
"10": 105,
|
| 1698 |
+
"11": 19,
|
| 1699 |
+
"12": 5,
|
| 1700 |
+
"13": 2,
|
| 1701 |
+
"14": 1
|
| 1702 |
+
},
|
| 1703 |
+
"mix_rl_records": 0,
|
| 1704 |
+
"sampling": {
|
| 1705 |
+
"samples": 1752,
|
| 1706 |
+
"secs": 9.329228162765503,
|
| 1707 |
+
"gate_secs": 3.5495150089263916,
|
| 1708 |
+
"samples_per_sec": 187.79688624108505
|
| 1709 |
+
},
|
| 1710 |
+
"secs": 9.381494760513306,
|
| 1711 |
+
"rl_secs_used": 1118.5505533218384
|
| 1712 |
+
},
|
| 1713 |
+
{
|
| 1714 |
+
"round": 53,
|
| 1715 |
+
"level": 1,
|
| 1716 |
+
"k": 8,
|
| 1717 |
+
"n_targets": 1500,
|
| 1718 |
+
"sample_acc": 0.834,
|
| 1719 |
+
"new_proofs": 144,
|
| 1720 |
+
"targets_solved": 1281,
|
| 1721 |
+
"lstar": 12,
|
| 1722 |
+
"ge": {
|
| 1723 |
+
"7": 1281,
|
| 1724 |
+
"8": 721,
|
| 1725 |
+
"9": 408,
|
| 1726 |
+
"10": 105,
|
| 1727 |
+
"11": 19,
|
| 1728 |
+
"12": 5,
|
| 1729 |
+
"13": 2,
|
| 1730 |
+
"14": 1
|
| 1731 |
+
},
|
| 1732 |
+
"mix_rl_records": 2851,
|
| 1733 |
+
"sampling": {
|
| 1734 |
+
"samples": 12000,
|
| 1735 |
+
"secs": 60.05979084968567,
|
| 1736 |
+
"gate_secs": 31.853074073791504,
|
| 1737 |
+
"samples_per_sec": 199.80089557809046
|
| 1738 |
+
},
|
| 1739 |
+
"secs": 100.43740677833557,
|
| 1740 |
+
"rl_secs_used": 1187.1351177692413
|
| 1741 |
+
},
|
| 1742 |
+
{
|
| 1743 |
+
"round": 54,
|
| 1744 |
+
"level": 2,
|
| 1745 |
+
"k": 24,
|
| 1746 |
+
"n_targets": 73,
|
| 1747 |
+
"sample_acc": 0.0028538812785388126,
|
| 1748 |
+
"new_proofs": 3,
|
| 1749 |
+
"targets_solved": 1283,
|
| 1750 |
+
"lstar": 12,
|
| 1751 |
+
"ge": {
|
| 1752 |
+
"7": 1283,
|
| 1753 |
+
"8": 723,
|
| 1754 |
+
"9": 410,
|
| 1755 |
+
"10": 107,
|
| 1756 |
+
"11": 21,
|
| 1757 |
+
"12": 6,
|
| 1758 |
+
"13": 2,
|
| 1759 |
+
"14": 1
|
| 1760 |
+
},
|
| 1761 |
+
"mix_rl_records": 3,
|
| 1762 |
+
"sampling": {
|
| 1763 |
+
"samples": 1752,
|
| 1764 |
+
"secs": 8.604615926742554,
|
| 1765 |
+
"gate_secs": 3.7914228439331055,
|
| 1766 |
+
"samples_per_sec": 203.61164460053408
|
| 1767 |
+
},
|
| 1768 |
+
"secs": 8.660674571990967,
|
| 1769 |
+
"rl_secs_used": 1192.0041372776031
|
| 1770 |
+
},
|
| 1771 |
+
{
|
| 1772 |
+
"round": 55,
|
| 1773 |
+
"level": 4,
|
| 1774 |
+
"k": 24,
|
| 1775 |
+
"n_targets": 73,
|
| 1776 |
+
"sample_acc": 0.0,
|
| 1777 |
+
"new_proofs": 0,
|
| 1778 |
+
"targets_solved": 1283,
|
| 1779 |
+
"lstar": 12,
|
| 1780 |
+
"ge": {
|
| 1781 |
+
"7": 1283,
|
| 1782 |
+
"8": 723,
|
| 1783 |
+
"9": 410,
|
| 1784 |
+
"10": 107,
|
| 1785 |
+
"11": 21,
|
| 1786 |
+
"12": 6,
|
| 1787 |
+
"13": 2,
|
| 1788 |
+
"14": 1
|
| 1789 |
+
},
|
| 1790 |
+
"mix_rl_records": 0,
|
| 1791 |
+
"sampling": {
|
| 1792 |
+
"samples": 1752,
|
| 1793 |
+
"secs": 9.214615821838379,
|
| 1794 |
+
"gate_secs": 3.755915403366089,
|
| 1795 |
+
"samples_per_sec": 190.13272325991167
|
| 1796 |
+
},
|
| 1797 |
+
"secs": 9.269420862197876,
|
| 1798 |
+
"rl_secs_used": 1197.517643213272
|
| 1799 |
+
},
|
| 1800 |
+
{
|
| 1801 |
+
"round": 56,
|
| 1802 |
+
"level": 8,
|
| 1803 |
+
"k": 24,
|
| 1804 |
+
"n_targets": 73,
|
| 1805 |
+
"sample_acc": 0.0,
|
| 1806 |
+
"new_proofs": 0,
|
| 1807 |
+
"targets_solved": 1283,
|
| 1808 |
+
"lstar": 12,
|
| 1809 |
+
"ge": {
|
| 1810 |
+
"7": 1283,
|
| 1811 |
+
"8": 723,
|
| 1812 |
+
"9": 410,
|
| 1813 |
+
"10": 107,
|
| 1814 |
+
"11": 21,
|
| 1815 |
+
"12": 6,
|
| 1816 |
+
"13": 2,
|
| 1817 |
+
"14": 1
|
| 1818 |
+
},
|
| 1819 |
+
"mix_rl_records": 0,
|
| 1820 |
+
"sampling": {
|
| 1821 |
+
"samples": 1752,
|
| 1822 |
+
"secs": 11.32279634475708,
|
| 1823 |
+
"gate_secs": 3.8714818954467773,
|
| 1824 |
+
"samples_per_sec": 154.73209502803147
|
| 1825 |
+
},
|
| 1826 |
+
"secs": 11.373117923736572,
|
| 1827 |
+
"rl_secs_used": 1205.0192773342133
|
| 1828 |
+
},
|
| 1829 |
+
{
|
| 1830 |
+
"round": 57,
|
| 1831 |
+
"level": 1,
|
| 1832 |
+
"k": 8,
|
| 1833 |
+
"n_targets": 1500,
|
| 1834 |
+
"sample_acc": 0.835,
|
| 1835 |
+
"new_proofs": 160,
|
| 1836 |
+
"targets_solved": 1283,
|
| 1837 |
+
"lstar": 12,
|
| 1838 |
+
"ge": {
|
| 1839 |
+
"7": 1283,
|
| 1840 |
+
"8": 723,
|
| 1841 |
+
"9": 410,
|
| 1842 |
+
"10": 107,
|
| 1843 |
+
"11": 21,
|
| 1844 |
+
"12": 6,
|
| 1845 |
+
"13": 2,
|
| 1846 |
+
"14": 1
|
| 1847 |
+
},
|
| 1848 |
+
"mix_rl_records": 2959,
|
| 1849 |
+
"sampling": {
|
| 1850 |
+
"samples": 12000,
|
| 1851 |
+
"secs": 60.068341970443726,
|
| 1852 |
+
"gate_secs": 31.06937003135681,
|
| 1853 |
+
"samples_per_sec": 199.77245261579768
|
| 1854 |
+
},
|
| 1855 |
+
"secs": 99.95769906044006,
|
| 1856 |
+
"rl_secs_used": 1273.9078090190887
|
| 1857 |
+
},
|
| 1858 |
+
{
|
| 1859 |
+
"round": 58,
|
| 1860 |
+
"level": 2,
|
| 1861 |
+
"k": 24,
|
| 1862 |
+
"n_targets": 73,
|
| 1863 |
+
"sample_acc": 0.0,
|
| 1864 |
+
"new_proofs": 0,
|
| 1865 |
+
"targets_solved": 1283,
|
| 1866 |
+
"lstar": 12,
|
| 1867 |
+
"ge": {
|
| 1868 |
+
"7": 1283,
|
| 1869 |
+
"8": 723,
|
| 1870 |
+
"9": 410,
|
| 1871 |
+
"10": 107,
|
| 1872 |
+
"11": 21,
|
| 1873 |
+
"12": 6,
|
| 1874 |
+
"13": 2,
|
| 1875 |
+
"14": 1
|
| 1876 |
+
},
|
| 1877 |
+
"mix_rl_records": 0,
|
| 1878 |
+
"sampling": {
|
| 1879 |
+
"samples": 1752,
|
| 1880 |
+
"secs": 8.726410388946533,
|
| 1881 |
+
"gate_secs": 3.1583523750305176,
|
| 1882 |
+
"samples_per_sec": 200.76983798735878
|
| 1883 |
+
},
|
| 1884 |
+
"secs": 8.780374526977539,
|
| 1885 |
+
"rl_secs_used": 1279.5296349525452
|
| 1886 |
+
},
|
| 1887 |
+
{
|
| 1888 |
+
"round": 59,
|
| 1889 |
+
"level": 4,
|
| 1890 |
+
"k": 24,
|
| 1891 |
+
"n_targets": 72,
|
| 1892 |
+
"sample_acc": 0.0005787037037037037,
|
| 1893 |
+
"new_proofs": 1,
|
| 1894 |
+
"targets_solved": 1284,
|
| 1895 |
+
"lstar": 12,
|
| 1896 |
+
"ge": {
|
| 1897 |
+
"7": 1284,
|
| 1898 |
+
"8": 724,
|
| 1899 |
+
"9": 411,
|
| 1900 |
+
"10": 107,
|
| 1901 |
+
"11": 21,
|
| 1902 |
+
"12": 6,
|
| 1903 |
+
"13": 2,
|
| 1904 |
+
"14": 1
|
| 1905 |
+
},
|
| 1906 |
+
"mix_rl_records": 1,
|
| 1907 |
+
"sampling": {
|
| 1908 |
+
"samples": 1728,
|
| 1909 |
+
"secs": 9.223639726638794,
|
| 1910 |
+
"gate_secs": 3.273508071899414,
|
| 1911 |
+
"samples_per_sec": 187.34469810321875
|
| 1912 |
+
},
|
| 1913 |
+
"secs": 9.27296757698059,
|
| 1914 |
+
"rl_secs_used": 1285.529099702835
|
| 1915 |
+
},
|
| 1916 |
+
{
|
| 1917 |
+
"round": 60,
|
| 1918 |
+
"level": 8,
|
| 1919 |
+
"k": 24,
|
| 1920 |
+
"n_targets": 72,
|
| 1921 |
+
"sample_acc": 0.0005787037037037037,
|
| 1922 |
+
"new_proofs": 1,
|
| 1923 |
+
"targets_solved": 1285,
|
| 1924 |
+
"lstar": 12,
|
| 1925 |
+
"ge": {
|
| 1926 |
+
"7": 1285,
|
| 1927 |
+
"8": 724,
|
| 1928 |
+
"9": 411,
|
| 1929 |
+
"10": 107,
|
| 1930 |
+
"11": 21,
|
| 1931 |
+
"12": 6,
|
| 1932 |
+
"13": 2,
|
| 1933 |
+
"14": 1
|
| 1934 |
+
},
|
| 1935 |
+
"mix_rl_records": 1,
|
| 1936 |
+
"sampling": {
|
| 1937 |
+
"samples": 1728,
|
| 1938 |
+
"secs": 7.401629686355591,
|
| 1939 |
+
"gate_secs": 3.2550129890441895,
|
| 1940 |
+
"samples_per_sec": 233.46209864909244
|
| 1941 |
+
},
|
| 1942 |
+
"secs": 7.45253586769104,
|
| 1943 |
+
"rl_secs_used": 1289.7266209125519
|
| 1944 |
+
},
|
| 1945 |
+
{
|
| 1946 |
+
"round": 61,
|
| 1947 |
+
"level": 1,
|
| 1948 |
+
"k": 8,
|
| 1949 |
+
"n_targets": 1500,
|
| 1950 |
+
"sample_acc": 0.8341666666666666,
|
| 1951 |
+
"new_proofs": 205,
|
| 1952 |
+
"targets_solved": 1286,
|
| 1953 |
+
"lstar": 12,
|
| 1954 |
+
"ge": {
|
| 1955 |
+
"7": 1286,
|
| 1956 |
+
"8": 725,
|
| 1957 |
+
"9": 412,
|
| 1958 |
+
"10": 108,
|
| 1959 |
+
"11": 22,
|
| 1960 |
+
"12": 6,
|
| 1961 |
+
"13": 2,
|
| 1962 |
+
"14": 1
|
| 1963 |
+
},
|
| 1964 |
+
"mix_rl_records": 3023,
|
| 1965 |
+
"sampling": {
|
| 1966 |
+
"samples": 12000,
|
| 1967 |
+
"secs": 61.02544951438904,
|
| 1968 |
+
"gate_secs": 31.309438228607178,
|
| 1969 |
+
"samples_per_sec": 196.63927255743604
|
| 1970 |
+
},
|
| 1971 |
+
"secs": 101.87276816368103,
|
| 1972 |
+
"rl_secs_used": 1360.2902128696442
|
| 1973 |
+
},
|
| 1974 |
+
{
|
| 1975 |
+
"round": 62,
|
| 1976 |
+
"level": 2,
|
| 1977 |
+
"k": 24,
|
| 1978 |
+
"n_targets": 72,
|
| 1979 |
+
"sample_acc": 0.0028935185185185184,
|
| 1980 |
+
"new_proofs": 3,
|
| 1981 |
+
"targets_solved": 1289,
|
| 1982 |
+
"lstar": 12,
|
| 1983 |
+
"ge": {
|
| 1984 |
+
"7": 1289,
|
| 1985 |
+
"8": 726,
|
| 1986 |
+
"9": 413,
|
| 1987 |
+
"10": 109,
|
| 1988 |
+
"11": 22,
|
| 1989 |
+
"12": 6,
|
| 1990 |
+
"13": 2,
|
| 1991 |
+
"14": 1
|
| 1992 |
+
},
|
| 1993 |
+
"mix_rl_records": 3,
|
| 1994 |
+
"sampling": {
|
| 1995 |
+
"samples": 1728,
|
| 1996 |
+
"secs": 10.04113483428955,
|
| 1997 |
+
"gate_secs": 3.8572194576263428,
|
| 1998 |
+
"samples_per_sec": 172.09210199020922
|
| 1999 |
+
},
|
| 2000 |
+
"secs": 10.098647356033325,
|
| 2001 |
+
"rl_secs_used": 1366.5313792228699
|
| 2002 |
+
},
|
| 2003 |
+
{
|
| 2004 |
+
"round": 63,
|
| 2005 |
+
"level": 4,
|
| 2006 |
+
"k": 24,
|
| 2007 |
+
"n_targets": 71,
|
| 2008 |
+
"sample_acc": 0.0,
|
| 2009 |
+
"new_proofs": 0,
|
| 2010 |
+
"targets_solved": 1289,
|
| 2011 |
+
"lstar": 12,
|
| 2012 |
+
"ge": {
|
| 2013 |
+
"7": 1289,
|
| 2014 |
+
"8": 726,
|
| 2015 |
+
"9": 413,
|
| 2016 |
+
"10": 109,
|
| 2017 |
+
"11": 22,
|
| 2018 |
+
"12": 6,
|
| 2019 |
+
"13": 2,
|
| 2020 |
+
"14": 1
|
| 2021 |
+
},
|
| 2022 |
+
"mix_rl_records": 0,
|
| 2023 |
+
"sampling": {
|
| 2024 |
+
"samples": 1704,
|
| 2025 |
+
"secs": 10.859347820281982,
|
| 2026 |
+
"gate_secs": 3.6754863262176514,
|
| 2027 |
+
"samples_per_sec": 156.91550065441706
|
| 2028 |
+
},
|
| 2029 |
+
"secs": 10.912447690963745,
|
| 2030 |
+
"rl_secs_used": 1373.768336057663
|
| 2031 |
+
},
|
| 2032 |
+
{
|
| 2033 |
+
"round": 64,
|
| 2034 |
+
"level": 8,
|
| 2035 |
+
"k": 24,
|
| 2036 |
+
"n_targets": 71,
|
| 2037 |
+
"sample_acc": 0.0005868544600938967,
|
| 2038 |
+
"new_proofs": 1,
|
| 2039 |
+
"targets_solved": 1290,
|
| 2040 |
+
"lstar": 12,
|
| 2041 |
+
"ge": {
|
| 2042 |
+
"7": 1290,
|
| 2043 |
+
"8": 726,
|
| 2044 |
+
"9": 413,
|
| 2045 |
+
"10": 109,
|
| 2046 |
+
"11": 22,
|
| 2047 |
+
"12": 6,
|
| 2048 |
+
"13": 2,
|
| 2049 |
+
"14": 1
|
| 2050 |
+
},
|
| 2051 |
+
"mix_rl_records": 1,
|
| 2052 |
+
"sampling": {
|
| 2053 |
+
"samples": 1704,
|
| 2054 |
+
"secs": 8.173127889633179,
|
| 2055 |
+
"gate_secs": 3.5396480560302734,
|
| 2056 |
+
"samples_per_sec": 208.48811165201013
|
| 2057 |
+
},
|
| 2058 |
+
"secs": 8.24605417251587,
|
| 2059 |
+
"rl_secs_used": 1378.4747486114502
|
| 2060 |
+
},
|
| 2061 |
+
{
|
| 2062 |
+
"round": 65,
|
| 2063 |
+
"level": 1,
|
| 2064 |
+
"k": 8,
|
| 2065 |
+
"n_targets": 1500,
|
| 2066 |
+
"sample_acc": 0.8424166666666667,
|
| 2067 |
+
"new_proofs": 155,
|
| 2068 |
+
"targets_solved": 1294,
|
| 2069 |
+
"lstar": 12,
|
| 2070 |
+
"ge": {
|
| 2071 |
+
"7": 1294,
|
| 2072 |
+
"8": 726,
|
| 2073 |
+
"9": 413,
|
| 2074 |
+
"10": 109,
|
| 2075 |
+
"11": 22,
|
| 2076 |
+
"12": 6,
|
| 2077 |
+
"13": 2,
|
| 2078 |
+
"14": 1
|
| 2079 |
+
},
|
| 2080 |
+
"mix_rl_records": 3139,
|
| 2081 |
+
"sampling": {
|
| 2082 |
+
"samples": 12000,
|
| 2083 |
+
"secs": 86.09665894508362,
|
| 2084 |
+
"gate_secs": 56.91737747192383,
|
| 2085 |
+
"samples_per_sec": 139.3782307819186
|
| 2086 |
+
},
|
| 2087 |
+
"secs": 127.23849725723267,
|
| 2088 |
+
"rl_secs_used": 1448.7961819171906
|
| 2089 |
+
},
|
| 2090 |
+
{
|
| 2091 |
+
"round": 66,
|
| 2092 |
+
"level": 2,
|
| 2093 |
+
"k": 24,
|
| 2094 |
+
"n_targets": 70,
|
| 2095 |
+
"sample_acc": 0.002976190476190476,
|
| 2096 |
+
"new_proofs": 2,
|
| 2097 |
+
"targets_solved": 1296,
|
| 2098 |
+
"lstar": 12,
|
| 2099 |
+
"ge": {
|
| 2100 |
+
"7": 1296,
|
| 2101 |
+
"8": 728,
|
| 2102 |
+
"9": 414,
|
| 2103 |
+
"10": 109,
|
| 2104 |
+
"11": 22,
|
| 2105 |
+
"12": 6,
|
| 2106 |
+
"13": 2,
|
| 2107 |
+
"14": 1
|
| 2108 |
+
},
|
| 2109 |
+
"mix_rl_records": 3,
|
| 2110 |
+
"sampling": {
|
| 2111 |
+
"samples": 1680,
|
| 2112 |
+
"secs": 7.904250383377075,
|
| 2113 |
+
"gate_secs": 3.1736927032470703,
|
| 2114 |
+
"samples_per_sec": 212.5438743100928
|
| 2115 |
+
},
|
| 2116 |
+
"secs": 7.951009511947632,
|
| 2117 |
+
"rl_secs_used": 1453.573173046112
|
| 2118 |
+
},
|
| 2119 |
+
{
|
| 2120 |
+
"round": 67,
|
| 2121 |
+
"level": 4,
|
| 2122 |
+
"k": 24,
|
| 2123 |
+
"n_targets": 69,
|
| 2124 |
+
"sample_acc": 0.0030193236714975845,
|
| 2125 |
+
"new_proofs": 1,
|
| 2126 |
+
"targets_solved": 1297,
|
| 2127 |
+
"lstar": 12,
|
| 2128 |
+
"ge": {
|
| 2129 |
+
"7": 1297,
|
| 2130 |
+
"8": 728,
|
| 2131 |
+
"9": 414,
|
| 2132 |
+
"10": 109,
|
| 2133 |
+
"11": 22,
|
| 2134 |
+
"12": 6,
|
| 2135 |
+
"13": 2,
|
| 2136 |
+
"14": 1
|
| 2137 |
+
},
|
| 2138 |
+
"mix_rl_records": 1,
|
| 2139 |
+
"sampling": {
|
| 2140 |
+
"samples": 1656,
|
| 2141 |
+
"secs": 8.08107590675354,
|
| 2142 |
+
"gate_secs": 3.420581579208374,
|
| 2143 |
+
"samples_per_sec": 204.92320813569427
|
| 2144 |
+
},
|
| 2145 |
+
"secs": 8.128207683563232,
|
| 2146 |
+
"rl_secs_used": 1458.2807984352112
|
| 2147 |
+
},
|
| 2148 |
+
{
|
| 2149 |
+
"round": 68,
|
| 2150 |
+
"level": 8,
|
| 2151 |
+
"k": 24,
|
| 2152 |
+
"n_targets": 69,
|
| 2153 |
+
"sample_acc": 0.0036231884057971015,
|
| 2154 |
+
"new_proofs": 1,
|
| 2155 |
+
"targets_solved": 1298,
|
| 2156 |
+
"lstar": 12,
|
| 2157 |
+
"ge": {
|
| 2158 |
+
"7": 1298,
|
| 2159 |
+
"8": 728,
|
| 2160 |
+
"9": 414,
|
| 2161 |
+
"10": 109,
|
| 2162 |
+
"11": 22,
|
| 2163 |
+
"12": 6,
|
| 2164 |
+
"13": 2,
|
| 2165 |
+
"14": 1
|
| 2166 |
+
},
|
| 2167 |
+
"mix_rl_records": 1,
|
| 2168 |
+
"sampling": {
|
| 2169 |
+
"samples": 1656,
|
| 2170 |
+
"secs": 8.900831460952759,
|
| 2171 |
+
"gate_secs": 3.056527853012085,
|
| 2172 |
+
"samples_per_sec": 186.05003445630226
|
| 2173 |
+
},
|
| 2174 |
+
"secs": 8.939305305480957,
|
| 2175 |
+
"rl_secs_used": 1464.1635749340057
|
| 2176 |
+
},
|
| 2177 |
+
{
|
| 2178 |
+
"round": 69,
|
| 2179 |
+
"level": 1,
|
| 2180 |
+
"k": 8,
|
| 2181 |
+
"n_targets": 1500,
|
| 2182 |
+
"sample_acc": 0.8468333333333333,
|
| 2183 |
+
"new_proofs": 178,
|
| 2184 |
+
"targets_solved": 1302,
|
| 2185 |
+
"lstar": 12,
|
| 2186 |
+
"ge": {
|
| 2187 |
+
"7": 1302,
|
| 2188 |
+
"8": 728,
|
| 2189 |
+
"9": 414,
|
| 2190 |
+
"10": 109,
|
| 2191 |
+
"11": 22,
|
| 2192 |
+
"12": 6,
|
| 2193 |
+
"13": 2,
|
| 2194 |
+
"14": 1
|
| 2195 |
+
},
|
| 2196 |
+
"mix_rl_records": 3220,
|
| 2197 |
+
"sampling": {
|
| 2198 |
+
"samples": 12000,
|
| 2199 |
+
"secs": 72.287841796875,
|
| 2200 |
+
"gate_secs": 44.42894697189331,
|
| 2201 |
+
"samples_per_sec": 166.00301934202662
|
| 2202 |
+
},
|
| 2203 |
+
"secs": 113.25818109512329,
|
| 2204 |
+
"rl_secs_used": 1532.99302816391
|
| 2205 |
+
},
|
| 2206 |
+
{
|
| 2207 |
+
"round": 70,
|
| 2208 |
+
"level": 2,
|
| 2209 |
+
"k": 24,
|
| 2210 |
+
"n_targets": 66,
|
| 2211 |
+
"sample_acc": 0.0,
|
| 2212 |
+
"new_proofs": 0,
|
| 2213 |
+
"targets_solved": 1302,
|
| 2214 |
+
"lstar": 12,
|
| 2215 |
+
"ge": {
|
| 2216 |
+
"7": 1302,
|
| 2217 |
+
"8": 728,
|
| 2218 |
+
"9": 414,
|
| 2219 |
+
"10": 109,
|
| 2220 |
+
"11": 22,
|
| 2221 |
+
"12": 6,
|
| 2222 |
+
"13": 2,
|
| 2223 |
+
"14": 1
|
| 2224 |
+
},
|
| 2225 |
+
"mix_rl_records": 0,
|
| 2226 |
+
"sampling": {
|
| 2227 |
+
"samples": 1584,
|
| 2228 |
+
"secs": 7.666700601577759,
|
| 2229 |
+
"gate_secs": 3.102468252182007,
|
| 2230 |
+
"samples_per_sec": 206.6077811456499
|
| 2231 |
+
},
|
| 2232 |
+
"secs": 7.7166478633880615,
|
| 2233 |
+
"rl_secs_used": 1537.606997013092
|
| 2234 |
+
},
|
| 2235 |
+
{
|
| 2236 |
+
"round": 71,
|
| 2237 |
+
"level": 4,
|
| 2238 |
+
"k": 24,
|
| 2239 |
+
"n_targets": 66,
|
| 2240 |
+
"sample_acc": 0.0,
|
| 2241 |
+
"new_proofs": 0,
|
| 2242 |
+
"targets_solved": 1302,
|
| 2243 |
+
"lstar": 12,
|
| 2244 |
+
"ge": {
|
| 2245 |
+
"7": 1302,
|
| 2246 |
+
"8": 728,
|
| 2247 |
+
"9": 414,
|
| 2248 |
+
"10": 109,
|
| 2249 |
+
"11": 22,
|
| 2250 |
+
"12": 6,
|
| 2251 |
+
"13": 2,
|
| 2252 |
+
"14": 1
|
| 2253 |
+
},
|
| 2254 |
+
"mix_rl_records": 0,
|
| 2255 |
+
"sampling": {
|
| 2256 |
+
"samples": 1584,
|
| 2257 |
+
"secs": 7.162403106689453,
|
| 2258 |
+
"gate_secs": 2.8469550609588623,
|
| 2259 |
+
"samples_per_sec": 221.15482421264383
|
| 2260 |
+
},
|
| 2261 |
+
"secs": 7.2067930698394775,
|
| 2262 |
+
"rl_secs_used": 1541.9668273925781
|
| 2263 |
+
},
|
| 2264 |
+
{
|
| 2265 |
+
"round": 72,
|
| 2266 |
+
"level": 8,
|
| 2267 |
+
"k": 24,
|
| 2268 |
+
"n_targets": 66,
|
| 2269 |
+
"sample_acc": 0.0006313131313131314,
|
| 2270 |
+
"new_proofs": 1,
|
| 2271 |
+
"targets_solved": 1303,
|
| 2272 |
+
"lstar": 12,
|
| 2273 |
+
"ge": {
|
| 2274 |
+
"7": 1303,
|
| 2275 |
+
"8": 728,
|
| 2276 |
+
"9": 414,
|
| 2277 |
+
"10": 109,
|
| 2278 |
+
"11": 22,
|
| 2279 |
+
"12": 6,
|
| 2280 |
+
"13": 2,
|
| 2281 |
+
"14": 1
|
| 2282 |
+
},
|
| 2283 |
+
"mix_rl_records": 1,
|
| 2284 |
+
"sampling": {
|
| 2285 |
+
"samples": 1584,
|
| 2286 |
+
"secs": 9.242881059646606,
|
| 2287 |
+
"gate_secs": 3.0354559421539307,
|
| 2288 |
+
"samples_per_sec": 171.3751361483562
|
| 2289 |
+
},
|
| 2290 |
+
"secs": 9.28851842880249,
|
| 2291 |
+
"rl_secs_used": 1548.2198905944824
|
| 2292 |
+
},
|
| 2293 |
+
{
|
| 2294 |
+
"round": 73,
|
| 2295 |
+
"level": 1,
|
| 2296 |
+
"k": 8,
|
| 2297 |
+
"n_targets": 1500,
|
| 2298 |
+
"sample_acc": 0.8461666666666666,
|
| 2299 |
+
"new_proofs": 211,
|
| 2300 |
+
"targets_solved": 1304,
|
| 2301 |
+
"lstar": 12,
|
| 2302 |
+
"ge": {
|
| 2303 |
+
"7": 1304,
|
| 2304 |
+
"8": 729,
|
| 2305 |
+
"9": 415,
|
| 2306 |
+
"10": 110,
|
| 2307 |
+
"11": 23,
|
| 2308 |
+
"12": 7,
|
| 2309 |
+
"13": 2,
|
| 2310 |
+
"14": 1
|
| 2311 |
+
},
|
| 2312 |
+
"mix_rl_records": 3330,
|
| 2313 |
+
"sampling": {
|
| 2314 |
+
"samples": 12000,
|
| 2315 |
+
"secs": 64.04320573806763,
|
| 2316 |
+
"gate_secs": 37.476025342941284,
|
| 2317 |
+
"samples_per_sec": 187.3735060839894
|
| 2318 |
+
},
|
| 2319 |
+
"secs": 105.84708571434021,
|
| 2320 |
+
"rl_secs_used": 1616.5911128520966
|
| 2321 |
+
},
|
| 2322 |
+
{
|
| 2323 |
+
"round": 74,
|
| 2324 |
+
"level": 2,
|
| 2325 |
+
"k": 24,
|
| 2326 |
+
"n_targets": 66,
|
| 2327 |
+
"sample_acc": 0.0006313131313131314,
|
| 2328 |
+
"new_proofs": 1,
|
| 2329 |
+
"targets_solved": 1305,
|
| 2330 |
+
"lstar": 12,
|
| 2331 |
+
"ge": {
|
| 2332 |
+
"7": 1305,
|
| 2333 |
+
"8": 730,
|
| 2334 |
+
"9": 416,
|
| 2335 |
+
"10": 111,
|
| 2336 |
+
"11": 24,
|
| 2337 |
+
"12": 7,
|
| 2338 |
+
"13": 2,
|
| 2339 |
+
"14": 1
|
| 2340 |
+
},
|
| 2341 |
+
"mix_rl_records": 1,
|
| 2342 |
+
"sampling": {
|
| 2343 |
+
"samples": 1584,
|
| 2344 |
+
"secs": 8.173463106155396,
|
| 2345 |
+
"gate_secs": 3.4838368892669678,
|
| 2346 |
+
"samples_per_sec": 193.79790174951634
|
| 2347 |
+
},
|
| 2348 |
+
"secs": 8.2254478931427,
|
| 2349 |
+
"rl_secs_used": 1621.33256316185
|
| 2350 |
+
},
|
| 2351 |
+
{
|
| 2352 |
+
"round": 75,
|
| 2353 |
+
"level": 4,
|
| 2354 |
+
"k": 24,
|
| 2355 |
+
"n_targets": 65,
|
| 2356 |
+
"sample_acc": 0.0,
|
| 2357 |
+
"new_proofs": 0,
|
| 2358 |
+
"targets_solved": 1305,
|
| 2359 |
+
"lstar": 12,
|
| 2360 |
+
"ge": {
|
| 2361 |
+
"7": 1305,
|
| 2362 |
+
"8": 730,
|
| 2363 |
+
"9": 416,
|
| 2364 |
+
"10": 111,
|
| 2365 |
+
"11": 24,
|
| 2366 |
+
"12": 7,
|
| 2367 |
+
"13": 2,
|
| 2368 |
+
"14": 1
|
| 2369 |
+
},
|
| 2370 |
+
"mix_rl_records": 0,
|
| 2371 |
+
"sampling": {
|
| 2372 |
+
"samples": 1560,
|
| 2373 |
+
"secs": 6.802579164505005,
|
| 2374 |
+
"gate_secs": 2.323772430419922,
|
| 2375 |
+
"samples_per_sec": 229.32478436118495
|
| 2376 |
+
},
|
| 2377 |
+
"secs": 6.834012031555176,
|
| 2378 |
+
"rl_secs_used": 1625.8428010940552
|
| 2379 |
+
},
|
| 2380 |
+
{
|
| 2381 |
+
"round": 76,
|
| 2382 |
+
"level": 8,
|
| 2383 |
+
"k": 24,
|
| 2384 |
+
"n_targets": 65,
|
| 2385 |
+
"sample_acc": 0.0,
|
| 2386 |
+
"new_proofs": 0,
|
| 2387 |
+
"targets_solved": 1305,
|
| 2388 |
+
"lstar": 12,
|
| 2389 |
+
"ge": {
|
| 2390 |
+
"7": 1305,
|
| 2391 |
+
"8": 730,
|
| 2392 |
+
"9": 416,
|
| 2393 |
+
"10": 111,
|
| 2394 |
+
"11": 24,
|
| 2395 |
+
"12": 7,
|
| 2396 |
+
"13": 2,
|
| 2397 |
+
"14": 1
|
| 2398 |
+
},
|
| 2399 |
+
"mix_rl_records": 0,
|
| 2400 |
+
"sampling": {
|
| 2401 |
+
"samples": 1560,
|
| 2402 |
+
"secs": 6.4656782150268555,
|
| 2403 |
+
"gate_secs": 3.0491154193878174,
|
| 2404 |
+
"samples_per_sec": 241.2739929392729
|
| 2405 |
+
},
|
| 2406 |
+
"secs": 6.501502990722656,
|
| 2407 |
+
"rl_secs_used": 1629.2951867580414
|
| 2408 |
+
},
|
| 2409 |
+
{
|
| 2410 |
+
"round": 77,
|
| 2411 |
+
"level": 1,
|
| 2412 |
+
"k": 8,
|
| 2413 |
+
"n_targets": 1500,
|
| 2414 |
+
"sample_acc": 0.84775,
|
| 2415 |
+
"new_proofs": 159,
|
| 2416 |
+
"targets_solved": 1305,
|
| 2417 |
+
"lstar": 12,
|
| 2418 |
+
"ge": {
|
| 2419 |
+
"7": 1305,
|
| 2420 |
+
"8": 730,
|
| 2421 |
+
"9": 416,
|
| 2422 |
+
"10": 111,
|
| 2423 |
+
"11": 24,
|
| 2424 |
+
"12": 7,
|
| 2425 |
+
"13": 2,
|
| 2426 |
+
"14": 1
|
| 2427 |
+
},
|
| 2428 |
+
"mix_rl_records": 3341,
|
| 2429 |
+
"sampling": {
|
| 2430 |
+
"samples": 12000,
|
| 2431 |
+
"secs": 63.612189531326294,
|
| 2432 |
+
"gate_secs": 31.400226354599,
|
| 2433 |
+
"samples_per_sec": 188.64309008088003
|
| 2434 |
+
},
|
| 2435 |
+
"secs": 104.60654807090759,
|
| 2436 |
+
"rl_secs_used": 1702.501704454422
|
| 2437 |
+
},
|
| 2438 |
+
{
|
| 2439 |
+
"round": 78,
|
| 2440 |
+
"level": 2,
|
| 2441 |
+
"k": 24,
|
| 2442 |
+
"n_targets": 66,
|
| 2443 |
+
"sample_acc": 0.0006313131313131314,
|
| 2444 |
+
"new_proofs": 1,
|
| 2445 |
+
"targets_solved": 1306,
|
| 2446 |
+
"lstar": 12,
|
| 2447 |
+
"ge": {
|
| 2448 |
+
"7": 1306,
|
| 2449 |
+
"8": 730,
|
| 2450 |
+
"9": 416,
|
| 2451 |
+
"10": 111,
|
| 2452 |
+
"11": 24,
|
| 2453 |
+
"12": 7,
|
| 2454 |
+
"13": 2,
|
| 2455 |
+
"14": 1
|
| 2456 |
+
},
|
| 2457 |
+
"mix_rl_records": 1,
|
| 2458 |
+
"sampling": {
|
| 2459 |
+
"samples": 1584,
|
| 2460 |
+
"secs": 9.80240535736084,
|
| 2461 |
+
"gate_secs": 3.1549150943756104,
|
| 2462 |
+
"samples_per_sec": 161.59299092957218
|
| 2463 |
+
},
|
| 2464 |
+
"secs": 9.850524425506592,
|
| 2465 |
+
"rl_secs_used": 1709.197124004364
|
| 2466 |
+
},
|
| 2467 |
+
{
|
| 2468 |
+
"round": 79,
|
| 2469 |
+
"level": 4,
|
| 2470 |
+
"k": 24,
|
| 2471 |
+
"n_targets": 66,
|
| 2472 |
+
"sample_acc": 0.0012626262626262627,
|
| 2473 |
+
"new_proofs": 2,
|
| 2474 |
+
"targets_solved": 1308,
|
| 2475 |
+
"lstar": 12,
|
| 2476 |
+
"ge": {
|
| 2477 |
+
"7": 1308,
|
| 2478 |
+
"8": 732,
|
| 2479 |
+
"9": 418,
|
| 2480 |
+
"10": 112,
|
| 2481 |
+
"11": 25,
|
| 2482 |
+
"12": 8,
|
| 2483 |
+
"13": 3,
|
| 2484 |
+
"14": 1
|
| 2485 |
+
},
|
| 2486 |
+
"mix_rl_records": 2,
|
| 2487 |
+
"sampling": {
|
| 2488 |
+
"samples": 1584,
|
| 2489 |
+
"secs": 9.975895404815674,
|
| 2490 |
+
"gate_secs": 3.2583279609680176,
|
| 2491 |
+
"samples_per_sec": 158.78273936546628
|
| 2492 |
+
},
|
| 2493 |
+
"secs": 10.023608446121216,
|
| 2494 |
+
"rl_secs_used": 1715.9624004364014
|
| 2495 |
+
},
|
| 2496 |
+
{
|
| 2497 |
+
"round": 80,
|
| 2498 |
+
"level": 8,
|
| 2499 |
+
"k": 24,
|
| 2500 |
+
"n_targets": 65,
|
| 2501 |
+
"sample_acc": 0.000641025641025641,
|
| 2502 |
+
"new_proofs": 1,
|
| 2503 |
+
"targets_solved": 1309,
|
| 2504 |
+
"lstar": 12,
|
| 2505 |
+
"ge": {
|
| 2506 |
+
"7": 1309,
|
| 2507 |
+
"8": 733,
|
| 2508 |
+
"9": 418,
|
| 2509 |
+
"10": 112,
|
| 2510 |
+
"11": 25,
|
| 2511 |
+
"12": 8,
|
| 2512 |
+
"13": 3,
|
| 2513 |
+
"14": 1
|
| 2514 |
+
},
|
| 2515 |
+
"mix_rl_records": 1,
|
| 2516 |
+
"sampling": {
|
| 2517 |
+
"samples": 1560,
|
| 2518 |
+
"secs": 7.9195191860198975,
|
| 2519 |
+
"gate_secs": 3.263338804244995,
|
| 2520 |
+
"samples_per_sec": 196.98165549669022
|
| 2521 |
+
},
|
| 2522 |
+
"secs": 7.966829538345337,
|
| 2523 |
+
"rl_secs_used": 1720.665892124176
|
| 2524 |
+
}
|
| 2525 |
+
],
|
| 2526 |
+
"secs": 2756.16139960289,
|
| 2527 |
+
"rl_secs_used": 1761.497284412384,
|
| 2528 |
+
"gate_secs": 994.6351890563965,
|
| 2529 |
+
"targets": 1500,
|
| 2530 |
+
"stats": {
|
| 2531 |
+
"rounds": 20,
|
| 2532 |
+
"first_loops": 1,
|
| 2533 |
+
"k_first": 8,
|
| 2534 |
+
"retry_loops": [
|
| 2535 |
+
2,
|
| 2536 |
+
4,
|
| 2537 |
+
8
|
| 2538 |
+
],
|
| 2539 |
+
"k_retry": 24,
|
| 2540 |
+
"retries": [
|
| 2541 |
+
{
|
| 2542 |
+
"round": 1,
|
| 2543 |
+
"loops": 2,
|
| 2544 |
+
"targets": 292,
|
| 2545 |
+
"cracked": 35
|
| 2546 |
+
},
|
| 2547 |
+
{
|
| 2548 |
+
"round": 1,
|
| 2549 |
+
"loops": 4,
|
| 2550 |
+
"targets": 292,
|
| 2551 |
+
"cracked": 28
|
| 2552 |
+
},
|
| 2553 |
+
{
|
| 2554 |
+
"round": 1,
|
| 2555 |
+
"loops": 8,
|
| 2556 |
+
"targets": 291,
|
| 2557 |
+
"cracked": 44
|
| 2558 |
+
},
|
| 2559 |
+
{
|
| 2560 |
+
"round": 2,
|
| 2561 |
+
"loops": 2,
|
| 2562 |
+
"targets": 162,
|
| 2563 |
+
"cracked": 12
|
| 2564 |
+
},
|
| 2565 |
+
{
|
| 2566 |
+
"round": 2,
|
| 2567 |
+
"loops": 4,
|
| 2568 |
+
"targets": 162,
|
| 2569 |
+
"cracked": 7
|
| 2570 |
+
},
|
| 2571 |
+
{
|
| 2572 |
+
"round": 2,
|
| 2573 |
+
"loops": 8,
|
| 2574 |
+
"targets": 162,
|
| 2575 |
+
"cracked": 14
|
| 2576 |
+
},
|
| 2577 |
+
{
|
| 2578 |
+
"round": 3,
|
| 2579 |
+
"loops": 2,
|
| 2580 |
+
"targets": 131,
|
| 2581 |
+
"cracked": 10
|
| 2582 |
+
},
|
| 2583 |
+
{
|
| 2584 |
+
"round": 3,
|
| 2585 |
+
"loops": 4,
|
| 2586 |
+
"targets": 131,
|
| 2587 |
+
"cracked": 11
|
| 2588 |
+
},
|
| 2589 |
+
{
|
| 2590 |
+
"round": 3,
|
| 2591 |
+
"loops": 8,
|
| 2592 |
+
"targets": 131,
|
| 2593 |
+
"cracked": 13
|
| 2594 |
+
},
|
| 2595 |
+
{
|
| 2596 |
+
"round": 4,
|
| 2597 |
+
"loops": 2,
|
| 2598 |
+
"targets": 109,
|
| 2599 |
+
"cracked": 3
|
| 2600 |
+
},
|
| 2601 |
+
{
|
| 2602 |
+
"round": 4,
|
| 2603 |
+
"loops": 4,
|
| 2604 |
+
"targets": 108,
|
| 2605 |
+
"cracked": 11
|
| 2606 |
+
},
|
| 2607 |
+
{
|
| 2608 |
+
"round": 4,
|
| 2609 |
+
"loops": 8,
|
| 2610 |
+
"targets": 108,
|
| 2611 |
+
"cracked": 3
|
| 2612 |
+
},
|
| 2613 |
+
{
|
| 2614 |
+
"round": 5,
|
| 2615 |
+
"loops": 2,
|
| 2616 |
+
"targets": 99,
|
| 2617 |
+
"cracked": 2
|
| 2618 |
+
},
|
| 2619 |
+
{
|
| 2620 |
+
"round": 5,
|
| 2621 |
+
"loops": 4,
|
| 2622 |
+
"targets": 98,
|
| 2623 |
+
"cracked": 8
|
| 2624 |
+
},
|
| 2625 |
+
{
|
| 2626 |
+
"round": 5,
|
| 2627 |
+
"loops": 8,
|
| 2628 |
+
"targets": 98,
|
| 2629 |
+
"cracked": 2
|
| 2630 |
+
},
|
| 2631 |
+
{
|
| 2632 |
+
"round": 6,
|
| 2633 |
+
"loops": 2,
|
| 2634 |
+
"targets": 91,
|
| 2635 |
+
"cracked": 1
|
| 2636 |
+
},
|
| 2637 |
+
{
|
| 2638 |
+
"round": 6,
|
| 2639 |
+
"loops": 4,
|
| 2640 |
+
"targets": 90,
|
| 2641 |
+
"cracked": 3
|
| 2642 |
+
},
|
| 2643 |
+
{
|
| 2644 |
+
"round": 6,
|
| 2645 |
+
"loops": 8,
|
| 2646 |
+
"targets": 90,
|
| 2647 |
+
"cracked": 2
|
| 2648 |
+
},
|
| 2649 |
+
{
|
| 2650 |
+
"round": 7,
|
| 2651 |
+
"loops": 2,
|
| 2652 |
+
"targets": 87,
|
| 2653 |
+
"cracked": 4
|
| 2654 |
+
},
|
| 2655 |
+
{
|
| 2656 |
+
"round": 7,
|
| 2657 |
+
"loops": 4,
|
| 2658 |
+
"targets": 86,
|
| 2659 |
+
"cracked": 0
|
| 2660 |
+
},
|
| 2661 |
+
{
|
| 2662 |
+
"round": 7,
|
| 2663 |
+
"loops": 8,
|
| 2664 |
+
"targets": 86,
|
| 2665 |
+
"cracked": 1
|
| 2666 |
+
},
|
| 2667 |
+
{
|
| 2668 |
+
"round": 8,
|
| 2669 |
+
"loops": 2,
|
| 2670 |
+
"targets": 84,
|
| 2671 |
+
"cracked": 0
|
| 2672 |
+
},
|
| 2673 |
+
{
|
| 2674 |
+
"round": 8,
|
| 2675 |
+
"loops": 4,
|
| 2676 |
+
"targets": 84,
|
| 2677 |
+
"cracked": 0
|
| 2678 |
+
},
|
| 2679 |
+
{
|
| 2680 |
+
"round": 8,
|
| 2681 |
+
"loops": 8,
|
| 2682 |
+
"targets": 84,
|
| 2683 |
+
"cracked": 2
|
| 2684 |
+
},
|
| 2685 |
+
{
|
| 2686 |
+
"round": 9,
|
| 2687 |
+
"loops": 2,
|
| 2688 |
+
"targets": 83,
|
| 2689 |
+
"cracked": 2
|
| 2690 |
+
},
|
| 2691 |
+
{
|
| 2692 |
+
"round": 9,
|
| 2693 |
+
"loops": 4,
|
| 2694 |
+
"targets": 83,
|
| 2695 |
+
"cracked": 3
|
| 2696 |
+
},
|
| 2697 |
+
{
|
| 2698 |
+
"round": 9,
|
| 2699 |
+
"loops": 8,
|
| 2700 |
+
"targets": 82,
|
| 2701 |
+
"cracked": 1
|
| 2702 |
+
},
|
| 2703 |
+
{
|
| 2704 |
+
"round": 10,
|
| 2705 |
+
"loops": 2,
|
| 2706 |
+
"targets": 80,
|
| 2707 |
+
"cracked": 1
|
| 2708 |
+
},
|
| 2709 |
+
{
|
| 2710 |
+
"round": 10,
|
| 2711 |
+
"loops": 4,
|
| 2712 |
+
"targets": 79,
|
| 2713 |
+
"cracked": 0
|
| 2714 |
+
},
|
| 2715 |
+
{
|
| 2716 |
+
"round": 10,
|
| 2717 |
+
"loops": 8,
|
| 2718 |
+
"targets": 79,
|
| 2719 |
+
"cracked": 1
|
| 2720 |
+
},
|
| 2721 |
+
{
|
| 2722 |
+
"round": 11,
|
| 2723 |
+
"loops": 2,
|
| 2724 |
+
"targets": 78,
|
| 2725 |
+
"cracked": 0
|
| 2726 |
+
},
|
| 2727 |
+
{
|
| 2728 |
+
"round": 11,
|
| 2729 |
+
"loops": 4,
|
| 2730 |
+
"targets": 77,
|
| 2731 |
+
"cracked": 0
|
| 2732 |
+
},
|
| 2733 |
+
{
|
| 2734 |
+
"round": 11,
|
| 2735 |
+
"loops": 8,
|
| 2736 |
+
"targets": 77,
|
| 2737 |
+
"cracked": 1
|
| 2738 |
+
},
|
| 2739 |
+
{
|
| 2740 |
+
"round": 12,
|
| 2741 |
+
"loops": 2,
|
| 2742 |
+
"targets": 76,
|
| 2743 |
+
"cracked": 1
|
| 2744 |
+
},
|
| 2745 |
+
{
|
| 2746 |
+
"round": 12,
|
| 2747 |
+
"loops": 4,
|
| 2748 |
+
"targets": 75,
|
| 2749 |
+
"cracked": 1
|
| 2750 |
+
},
|
| 2751 |
+
{
|
| 2752 |
+
"round": 12,
|
| 2753 |
+
"loops": 8,
|
| 2754 |
+
"targets": 75,
|
| 2755 |
+
"cracked": 3
|
| 2756 |
+
},
|
| 2757 |
+
{
|
| 2758 |
+
"round": 13,
|
| 2759 |
+
"loops": 2,
|
| 2760 |
+
"targets": 74,
|
| 2761 |
+
"cracked": 0
|
| 2762 |
+
},
|
| 2763 |
+
{
|
| 2764 |
+
"round": 13,
|
| 2765 |
+
"loops": 4,
|
| 2766 |
+
"targets": 74,
|
| 2767 |
+
"cracked": 1
|
| 2768 |
+
},
|
| 2769 |
+
{
|
| 2770 |
+
"round": 13,
|
| 2771 |
+
"loops": 8,
|
| 2772 |
+
"targets": 73,
|
| 2773 |
+
"cracked": 0
|
| 2774 |
+
},
|
| 2775 |
+
{
|
| 2776 |
+
"round": 14,
|
| 2777 |
+
"loops": 2,
|
| 2778 |
+
"targets": 73,
|
| 2779 |
+
"cracked": 2
|
| 2780 |
+
},
|
| 2781 |
+
{
|
| 2782 |
+
"round": 14,
|
| 2783 |
+
"loops": 4,
|
| 2784 |
+
"targets": 73,
|
| 2785 |
+
"cracked": 0
|
| 2786 |
+
},
|
| 2787 |
+
{
|
| 2788 |
+
"round": 14,
|
| 2789 |
+
"loops": 8,
|
| 2790 |
+
"targets": 73,
|
| 2791 |
+
"cracked": 0
|
| 2792 |
+
},
|
| 2793 |
+
{
|
| 2794 |
+
"round": 15,
|
| 2795 |
+
"loops": 2,
|
| 2796 |
+
"targets": 73,
|
| 2797 |
+
"cracked": 0
|
| 2798 |
+
},
|
| 2799 |
+
{
|
| 2800 |
+
"round": 15,
|
| 2801 |
+
"loops": 4,
|
| 2802 |
+
"targets": 72,
|
| 2803 |
+
"cracked": 1
|
| 2804 |
+
},
|
| 2805 |
+
{
|
| 2806 |
+
"round": 15,
|
| 2807 |
+
"loops": 8,
|
| 2808 |
+
"targets": 72,
|
| 2809 |
+
"cracked": 1
|
| 2810 |
+
},
|
| 2811 |
+
{
|
| 2812 |
+
"round": 16,
|
| 2813 |
+
"loops": 2,
|
| 2814 |
+
"targets": 72,
|
| 2815 |
+
"cracked": 3
|
| 2816 |
+
},
|
| 2817 |
+
{
|
| 2818 |
+
"round": 16,
|
| 2819 |
+
"loops": 4,
|
| 2820 |
+
"targets": 71,
|
| 2821 |
+
"cracked": 0
|
| 2822 |
+
},
|
| 2823 |
+
{
|
| 2824 |
+
"round": 16,
|
| 2825 |
+
"loops": 8,
|
| 2826 |
+
"targets": 71,
|
| 2827 |
+
"cracked": 1
|
| 2828 |
+
},
|
| 2829 |
+
{
|
| 2830 |
+
"round": 17,
|
| 2831 |
+
"loops": 2,
|
| 2832 |
+
"targets": 70,
|
| 2833 |
+
"cracked": 3
|
| 2834 |
+
},
|
| 2835 |
+
{
|
| 2836 |
+
"round": 17,
|
| 2837 |
+
"loops": 4,
|
| 2838 |
+
"targets": 69,
|
| 2839 |
+
"cracked": 1
|
| 2840 |
+
},
|
| 2841 |
+
{
|
| 2842 |
+
"round": 17,
|
| 2843 |
+
"loops": 8,
|
| 2844 |
+
"targets": 69,
|
| 2845 |
+
"cracked": 1
|
| 2846 |
+
},
|
| 2847 |
+
{
|
| 2848 |
+
"round": 18,
|
| 2849 |
+
"loops": 2,
|
| 2850 |
+
"targets": 66,
|
| 2851 |
+
"cracked": 0
|
| 2852 |
+
},
|
| 2853 |
+
{
|
| 2854 |
+
"round": 18,
|
| 2855 |
+
"loops": 4,
|
| 2856 |
+
"targets": 66,
|
| 2857 |
+
"cracked": 0
|
| 2858 |
+
},
|
| 2859 |
+
{
|
| 2860 |
+
"round": 18,
|
| 2861 |
+
"loops": 8,
|
| 2862 |
+
"targets": 66,
|
| 2863 |
+
"cracked": 1
|
| 2864 |
+
},
|
| 2865 |
+
{
|
| 2866 |
+
"round": 19,
|
| 2867 |
+
"loops": 2,
|
| 2868 |
+
"targets": 66,
|
| 2869 |
+
"cracked": 1
|
| 2870 |
+
},
|
| 2871 |
+
{
|
| 2872 |
+
"round": 19,
|
| 2873 |
+
"loops": 4,
|
| 2874 |
+
"targets": 65,
|
| 2875 |
+
"cracked": 0
|
| 2876 |
+
},
|
| 2877 |
+
{
|
| 2878 |
+
"round": 19,
|
| 2879 |
+
"loops": 8,
|
| 2880 |
+
"targets": 65,
|
| 2881 |
+
"cracked": 0
|
| 2882 |
+
},
|
| 2883 |
+
{
|
| 2884 |
+
"round": 20,
|
| 2885 |
+
"loops": 2,
|
| 2886 |
+
"targets": 66,
|
| 2887 |
+
"cracked": 1
|
| 2888 |
+
},
|
| 2889 |
+
{
|
| 2890 |
+
"round": 20,
|
| 2891 |
+
"loops": 4,
|
| 2892 |
+
"targets": 66,
|
| 2893 |
+
"cracked": 2
|
| 2894 |
+
},
|
| 2895 |
+
{
|
| 2896 |
+
"round": 20,
|
| 2897 |
+
"loops": 8,
|
| 2898 |
+
"targets": 65,
|
| 2899 |
+
"cracked": 1
|
| 2900 |
+
}
|
| 2901 |
+
],
|
| 2902 |
+
"ft_records": 76800,
|
| 2903 |
+
"ft_steps": [
|
| 2904 |
+
626,
|
| 2905 |
+
660,
|
| 2906 |
+
679,
|
| 2907 |
+
693,
|
| 2908 |
+
691,
|
| 2909 |
+
705,
|
| 2910 |
+
713,
|
| 2911 |
+
724,
|
| 2912 |
+
722,
|
| 2913 |
+
726,
|
| 2914 |
+
739,
|
| 2915 |
+
740,
|
| 2916 |
+
738,
|
| 2917 |
+
747,
|
| 2918 |
+
742,
|
| 2919 |
+
756,
|
| 2920 |
+
757,
|
| 2921 |
+
760,
|
| 2922 |
+
757,
|
| 2923 |
+
760
|
| 2924 |
+
]
|
| 2925 |
+
}
|
| 2926 |
+
},
|
| 2927 |
+
"eval": {
|
| 2928 |
+
"level": 8,
|
| 2929 |
+
"attempts_per_theorem": 64,
|
| 2930 |
+
"solved": 790,
|
| 2931 |
+
"n": 1108,
|
| 2932 |
+
"solved_ge7": 790,
|
| 2933 |
+
"lstar": 12,
|
| 2934 |
+
"ge": {
|
| 2935 |
+
"7": 790,
|
| 2936 |
+
"8": 655,
|
| 2937 |
+
"9": 521,
|
| 2938 |
+
"10": 134,
|
| 2939 |
+
"11": 31,
|
| 2940 |
+
"12": 7,
|
| 2941 |
+
"13": 1,
|
| 2942 |
+
"14": 0
|
| 2943 |
+
},
|
| 2944 |
+
"by_bin": {
|
| 2945 |
+
"7": {
|
| 2946 |
+
"solved": 135,
|
| 2947 |
+
"n": 150,
|
| 2948 |
+
"rate": 0.9,
|
| 2949 |
+
"ci": [
|
| 2950 |
+
0.8415639255193982,
|
| 2951 |
+
0.9384591631640268
|
| 2952 |
+
]
|
| 2953 |
+
},
|
| 2954 |
+
"8": {
|
| 2955 |
+
"solved": 134,
|
| 2956 |
+
"n": 152,
|
| 2957 |
+
"rate": 0.881578947368421,
|
| 2958 |
+
"ci": [
|
| 2959 |
+
0.8205787323018284,
|
| 2960 |
+
0.9237668082983707
|
| 2961 |
+
]
|
| 2962 |
+
},
|
| 2963 |
+
"9": {
|
| 2964 |
+
"solved": 387,
|
| 2965 |
+
"n": 477,
|
| 2966 |
+
"rate": 0.8113207547169812,
|
| 2967 |
+
"ci": [
|
| 2968 |
+
0.7737737427372532,
|
| 2969 |
+
0.8438932810809855
|
| 2970 |
+
]
|
| 2971 |
+
},
|
| 2972 |
+
"10": {
|
| 2973 |
+
"solved": 103,
|
| 2974 |
+
"n": 219,
|
| 2975 |
+
"rate": 0.4703196347031963,
|
| 2976 |
+
"ci": [
|
| 2977 |
+
0.40529609337642064,
|
| 2978 |
+
0.5363665046304147
|
| 2979 |
+
]
|
| 2980 |
+
},
|
| 2981 |
+
"11": {
|
| 2982 |
+
"solved": 24,
|
| 2983 |
+
"n": 49,
|
| 2984 |
+
"rate": 0.4897959183673469,
|
| 2985 |
+
"ci": [
|
| 2986 |
+
0.3557488315940086,
|
| 2987 |
+
0.6253266846659078
|
| 2988 |
+
]
|
| 2989 |
+
},
|
| 2990 |
+
"12": {
|
| 2991 |
+
"solved": 6,
|
| 2992 |
+
"n": 54,
|
| 2993 |
+
"rate": 0.1111111111111111,
|
| 2994 |
+
"ci": [
|
| 2995 |
+
0.05192951817556753,
|
| 2996 |
+
0.2219494893207673
|
| 2997 |
+
]
|
| 2998 |
+
},
|
| 2999 |
+
"13": {
|
| 3000 |
+
"solved": 1,
|
| 3001 |
+
"n": 5,
|
| 3002 |
+
"rate": 0.2,
|
| 3003 |
+
"ci": [
|
| 3004 |
+
0.03622316096978745,
|
| 3005 |
+
0.6244717358814613
|
| 3006 |
+
]
|
| 3007 |
+
},
|
| 3008 |
+
"14": {
|
| 3009 |
+
"solved": 0,
|
| 3010 |
+
"n": 2,
|
| 3011 |
+
"rate": 0.0,
|
| 3012 |
+
"ci": [
|
| 3013 |
+
0.0,
|
| 3014 |
+
0.6576280471103807
|
| 3015 |
+
]
|
| 3016 |
+
}
|
| 3017 |
+
},
|
| 3018 |
+
"sample_acc": 0.5484544223826715,
|
| 3019 |
+
"max_pruned": 15,
|
| 3020 |
+
"sampling": {
|
| 3021 |
+
"samples": 70912,
|
| 3022 |
+
"secs": 287.0320768356323,
|
| 3023 |
+
"gate_secs": 130.50527811050415,
|
| 3024 |
+
"samples_per_sec": 247.05252730553684
|
| 3025 |
+
},
|
| 3026 |
+
"heldout_greedy": 0.989,
|
| 3027 |
+
"diag": {
|
| 3028 |
+
"level": 2,
|
| 3029 |
+
"attempts_per_theorem": 64,
|
| 3030 |
+
"solved": 790,
|
| 3031 |
+
"n": 1108,
|
| 3032 |
+
"solved_ge7": 790,
|
| 3033 |
+
"lstar": 12,
|
| 3034 |
+
"ge": {
|
| 3035 |
+
"7": 790,
|
| 3036 |
+
"8": 655,
|
| 3037 |
+
"9": 521,
|
| 3038 |
+
"10": 134,
|
| 3039 |
+
"11": 31,
|
| 3040 |
+
"12": 7,
|
| 3041 |
+
"13": 1,
|
| 3042 |
+
"14": 0
|
| 3043 |
+
},
|
| 3044 |
+
"by_bin": {
|
| 3045 |
+
"7": {
|
| 3046 |
+
"solved": 135,
|
| 3047 |
+
"n": 150,
|
| 3048 |
+
"rate": 0.9,
|
| 3049 |
+
"ci": [
|
| 3050 |
+
0.8415639255193982,
|
| 3051 |
+
0.9384591631640268
|
| 3052 |
+
]
|
| 3053 |
+
},
|
| 3054 |
+
"8": {
|
| 3055 |
+
"solved": 134,
|
| 3056 |
+
"n": 152,
|
| 3057 |
+
"rate": 0.881578947368421,
|
| 3058 |
+
"ci": [
|
| 3059 |
+
0.8205787323018284,
|
| 3060 |
+
0.9237668082983707
|
| 3061 |
+
]
|
| 3062 |
+
},
|
| 3063 |
+
"9": {
|
| 3064 |
+
"solved": 387,
|
| 3065 |
+
"n": 477,
|
| 3066 |
+
"rate": 0.8113207547169812,
|
| 3067 |
+
"ci": [
|
| 3068 |
+
0.7737737427372532,
|
| 3069 |
+
0.8438932810809855
|
| 3070 |
+
]
|
| 3071 |
+
},
|
| 3072 |
+
"10": {
|
| 3073 |
+
"solved": 103,
|
| 3074 |
+
"n": 219,
|
| 3075 |
+
"rate": 0.4703196347031963,
|
| 3076 |
+
"ci": [
|
| 3077 |
+
0.40529609337642064,
|
| 3078 |
+
0.5363665046304147
|
| 3079 |
+
]
|
| 3080 |
+
},
|
| 3081 |
+
"11": {
|
| 3082 |
+
"solved": 24,
|
| 3083 |
+
"n": 49,
|
| 3084 |
+
"rate": 0.4897959183673469,
|
| 3085 |
+
"ci": [
|
| 3086 |
+
0.3557488315940086,
|
| 3087 |
+
0.6253266846659078
|
| 3088 |
+
]
|
| 3089 |
+
},
|
| 3090 |
+
"12": {
|
| 3091 |
+
"solved": 6,
|
| 3092 |
+
"n": 54,
|
| 3093 |
+
"rate": 0.1111111111111111,
|
| 3094 |
+
"ci": [
|
| 3095 |
+
0.05192951817556753,
|
| 3096 |
+
0.2219494893207673
|
| 3097 |
+
]
|
| 3098 |
+
},
|
| 3099 |
+
"13": {
|
| 3100 |
+
"solved": 1,
|
| 3101 |
+
"n": 5,
|
| 3102 |
+
"rate": 0.2,
|
| 3103 |
+
"ci": [
|
| 3104 |
+
0.03622316096978745,
|
| 3105 |
+
0.6244717358814613
|
| 3106 |
+
]
|
| 3107 |
+
},
|
| 3108 |
+
"14": {
|
| 3109 |
+
"solved": 0,
|
| 3110 |
+
"n": 2,
|
| 3111 |
+
"rate": 0.0,
|
| 3112 |
+
"ci": [
|
| 3113 |
+
0.0,
|
| 3114 |
+
0.6576280471103807
|
| 3115 |
+
]
|
| 3116 |
+
}
|
| 3117 |
+
},
|
| 3118 |
+
"sample_acc": 0.5484544223826715,
|
| 3119 |
+
"max_pruned": 15,
|
| 3120 |
+
"sampling": {
|
| 3121 |
+
"samples": 70912,
|
| 3122 |
+
"secs": 292.38331151008606,
|
| 3123 |
+
"gate_secs": 136.02297925949097,
|
| 3124 |
+
"samples_per_sec": 242.53094211758327
|
| 3125 |
+
}
|
| 3126 |
+
}
|
| 3127 |
+
}
|
| 3128 |
+
}
|