File size: 51,978 Bytes
dca70d8
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
"""Generate an executable-Python corpus, verified by running the code.



## Why this exists



v86 scores 0.779 across 21 tasks and has **zero** coding ability, for the

simple reason that there is no code in the corpus. Nothing here has ever asked

the model to read a snippet.



Code is the cheapest thing in this repository to verify exactly: you *run it*.

That is the same contract `build_omni_corpus` has with `nexus_solver`, with

CPython as the checker instead. A generator builds a snippet from a template,

works the answer out step by step in the model's own format, and the

interpreter then executes the snippet it actually shows the model. Disagreement

drops the row.



The check is not circular, and it is worth being precise about why, because the

drop rate is 0.00% and a 0% drop rate usually means a verifier that passes

everything. The generator computes its answer with Python *expressions*

(`s + n * k`); the checker executes the *string it renders into the prompt*.

Those two agree only if the rendered snippet says what the generator thinks it

says. An off-by-one in a `range`, a variable renamed in the template but not in

the derivation, a loop bound written from the wrong parameter -- each of those

is a mismatch between expression and string, and each is caught.

`test_code_corpus.py` mutates a template and asserts the row is dropped, which

is the only way to know the checker is load-bearing rather than decorative.



## What the envelope allowed, and what it ruled out



Every constraint from V81_WHAT_THE_MODEL_CAN_LEARN applies unchanged, and two

of them removed whole task families before a line was written:



**Character-level string work is unlearnable here and is not attempted.**

`s = "banana"; r = s.count("a")` cannot be learned by this model at all: the

tokenizer's `[A-Za-z]+` rule makes `banana` a *single* token, so the letters

are not visible to the model in any representation. It would be a task whose

answer is a function of information the input does not carry. Counting *list

elements* (`len(words)`) is fine and appears in `code_index`, because each word

is its own token.



**Snippets are one statement per line and never nest more than one level.**

Tokens carry their leading whitespace, so `"\\n    x"` and `" x"` are different

symbols and a deeply indented corpus builds a parallel vocabulary of

indent-prefixed identifiers. The generators here emit exactly six line-initial

forms in total (`\\nfor`, `\\n    for`, `\\nwhile`, `\\nif`, `\\nelse`, `\\nr`),

all of them high frequency; `test_code_corpus.py` pins that set so a future

task cannot quietly widen it.



**Every intermediate stays two-digit.** Loop counts, steps and list values are

bounded so no running total passes 99 and no product leaves

two-digit-by-one-digit. `code_nested_loop` caps both ranges at 9 for a reason

that is easy to miss: `decompose_product` writes `10 x 7 = 70, 2 x 7 = 14` and

deliberately does *not* write the total, because that is the form that scored

0.93. That is fine when the product is the answer, and wrong when it is an

operand for a later step -- the model would have to carry an unwritten number.

So the outer range is single-digit and the iteration count is one written step.



## Repetition, and where this family is weakest



v74's multiplication task holds 712 distinct problems repeated ~56x and scored

0.93; the same operation with 24,000 unique problems scored 0.03. Distinct

capacity is therefore a design parameter, not an accident, and

`distinct_capacity()` reports it per task. At `--per_task 20000` the family

runs from 28x repetition (`code_divmod`) down to 3.9x (`code_index`), and

`code_index` is the one to narrow first if the family underperforms: its list

carries a distractor element that multiplies the prompt space without adding a

step to the procedure.

"""

from __future__ import annotations

import argparse
import ast
import json
import math
import random
import sys
import threading
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple

SOURCE_DIR = Path(__file__).resolve().parent
if str(SOURCE_DIR) not in sys.path:
    sys.path.insert(0, str(SOURCE_DIR))

# Imported rather than re-implemented on purpose. `decompose_product` is the
# exact working format that scored 0.93 on v74's multiplication task, down to
# whether the addition of the partial products is written out; a second copy of
# it here would drift from that measurement the first time either file changed.
from build_omni_corpus import (  # noqa: E402
    DEFAULT_SEQUENCE_LENGTH,
    decompose_product,
    extract_answer,
    token_budget_report,
)

#: Wall clock a single snippet gets before it is abandoned.
#:
#: Every snippet these generators emit is bounded by construction -- the
#: longest runs 270 iterations -- so this never fires in practice. It exists
#: because "the templates are all safe" is an assumption, and an assumption
#: that hangs the build for good is worse than one that drops a row.
DEFAULT_TIMEOUT_SECONDS = 2.0

#: Refuse a snippet longer than this before parsing it. Generated snippets top
#: out near 90 characters.
MAX_SOURCE_CHARS = 400

#: Consecutive failures to produce a new prompt before a task is judged
#: exhausted, matching `build_omni_corpus.EXHAUSTION_MISSES`.
EXHAUSTION_MISSES = 5000


# -- the checker ------------------------------------------------------------
#
# Running generated code is the whole point, so the sandbox is written to be
# read: an allowlist of syntax, an allowlist of names, no builtins beyond six
# pure functions, and a wall clock. Nothing here ever executes a string that
# came from outside this module -- `verify` runs `problem.canonical`, and the
# only thing that writes a `canonical` is a generator in `TASKS`.


class SnippetRejected(Exception):
    """The snippet uses syntax the corpus does not permit."""


#: Node types a corpus snippet may contain. Matched by class *name* rather than
#: by class object so this survives the AST changes between Python versions
#: (`ast.Index` disappeared from parse output in 3.9).
ALLOWED_NODES = frozenset({
    "Module", "Expr", "Assign", "AugAssign", "For", "While", "If", "Break",
    "Continue", "Pass", "Name", "Store", "Load", "Constant", "BinOp", "UnaryOp",
    "USub", "UAdd", "Add", "Sub", "Mult", "FloorDiv", "Mod", "Compare", "Eq",
    "NotEq", "Lt", "LtE", "Gt", "GtE", "List", "Subscript", "Index", "Call",
    # `IfExp` is the conditional expression `a if cond else b`, used by
    # `code_conditional` because the three-line block form left a 34-token
    # prompt where the model's 128-token context allows 32 alongside a
    # full-length generation.
    #
    # Worth recording how this was found: adding the ternary without this entry
    # sent the build's drop rate from 0.000 to 0.111 -- exactly one task in
    # nine -- because every `code_conditional` row failed validation and was
    # discarded. The verifier caught a change its author had not thought
    # through, which is the whole argument for validating snippets this module
    # generates itself.
    "IfExp",
    # `Attribute` is admitted for v93's `code_list_count` (`nums.count(v)`),
    # and only in the shape `validate_snippet` checks below: a method named in
    # `SAFE_METHODS`, called on a plain name. An attribute that is not a call
    # on an allowlisted method -- `().__class__`, `x.__dict__` -- is still
    # refused, so the sandbox-escape surface this node type usually opens is
    # not opened here.
    "Attribute",
})

#: The only callables a snippet may name. All six are pure, total on the
#: arguments these templates pass, and cannot reach the filesystem, the network
#: or the interpreter's own state.
SAFE_BUILTINS: Dict[str, Any] = {
    "abs": abs,
    "len": len,
    "max": max,
    "min": min,
    "range": range,
    "sum": sum,
}

#: The only methods a snippet may call, by attribute name. `list.count` is
#: pure and total; it is the one v93 needs.
SAFE_METHODS = frozenset({"count"})


def validate_snippet(source: str) -> ast.Module:
    """Parse `source` and refuse anything outside the allowlist.



    Belt and braces: these snippets are built from templates in this file, so

    in normal operation nothing can reach here that is not already safe. The

    validator is what makes that a *checked* claim instead of a stated one --

    if someone later adds a template with an attribute access or an import, the

    build fails loudly at that template rather than silently gaining a

    capability.

    """

    if len(source) > MAX_SOURCE_CHARS:
        raise SnippetRejected(f"snippet is {len(source)} chars, limit {MAX_SOURCE_CHARS}")
    try:
        tree = ast.parse(source, filename="<code-corpus>", mode="exec")
    except SyntaxError as exc:  # a template that does not parse is a bug, not a row
        raise SnippetRejected(f"syntax error: {exc}") from exc

    for node in ast.walk(tree):
        name = type(node).__name__
        if name not in ALLOWED_NODES:
            raise SnippetRejected(f"disallowed syntax: {name}")
        if isinstance(node, ast.Name):
            # `_`-prefixed names are how sandbox escapes are written
            # (`().__class__`, `__builtins__`); no template needs one.
            if node.id.startswith("_"):
                raise SnippetRejected(f"disallowed name: {node.id}")
        if isinstance(node, ast.Attribute):
            # Only `<name>.<safe method>`; anything else is the escape shape.
            if node.attr not in SAFE_METHODS or not isinstance(node.value, ast.Name):
                raise SnippetRejected(f"disallowed attribute: {node.attr}")
        if isinstance(node, ast.Call):
            if isinstance(node.func, ast.Attribute):
                pass  # checked as an Attribute above
            elif not isinstance(node.func, ast.Name):
                raise SnippetRejected("only direct calls to allowed builtins")
            elif node.func.id not in SAFE_BUILTINS:
                raise SnippetRejected(f"disallowed call: {node.func.id}")
            if node.keywords:
                raise SnippetRejected("keyword arguments are not permitted")
    return tree


@dataclass
class SnippetResult:
    """What happened when the interpreter ran a snippet."""

    ok: bool
    value: Optional[float] = None
    reason: str = ""


def run_snippet(source: str, target: str,

                timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS) -> SnippetResult:
    """Execute `source` and read `target` out of the resulting namespace.



    Restricted three ways: the AST allowlist above, a `__builtins__` mapping

    holding six pure functions, and a wall clock. The snippet runs on a daemon

    thread so a hang cannot wedge the build -- a thread that overruns is

    abandoned rather than joined, which is the honest thing to do given CPython

    offers no way to kill one, and is safe here because the thread can only

    touch its own namespace.



    A non-numeric or non-integral result is a failure, not a row: the whole

    corpus answers with the last integer in the reply.

    """

    try:
        validate_snippet(source)
        compiled = compile(source, "<code-corpus>", "exec")
    except SnippetRejected as exc:
        return SnippetResult(False, None, str(exc))
    except (SyntaxError, ValueError) as exc:
        return SnippetResult(False, None, f"compile failed: {exc}")

    outcome: Dict[str, Any] = {}

    def _run() -> None:
        namespace: Dict[str, Any] = {"__builtins__": dict(SAFE_BUILTINS)}
        try:
            exec(compiled, namespace)  # noqa: S102 - the point of the module
        except BaseException as exc:  # noqa: BLE001 - any failure drops the row
            outcome["error"] = f"{type(exc).__name__}: {exc}"
            return
        if target not in namespace:
            outcome["error"] = f"snippet never assigned {target}"
            return
        outcome["value"] = namespace[target]

    thread = threading.Thread(target=_run, daemon=True)
    thread.start()
    thread.join(timeout_seconds)
    if thread.is_alive():
        return SnippetResult(False, None, f"timed out after {timeout_seconds}s")
    if "error" in outcome:
        return SnippetResult(False, None, outcome["error"])

    value = outcome.get("value")
    if isinstance(value, bool) or not isinstance(value, int):
        # bool is an int subclass; `r = True` would silently score as 1.
        return SnippetResult(False, None, f"{target} is {type(value).__name__}, not int")
    return SnippetResult(True, float(value), "")


@dataclass
class CodeProblem:
    """One snippet, its worked derivation, and the value the interpreter got.



    `canonical` is the executable source itself, which is the exact parallel to

    `OmniProblem.canonical` holding the solver's own query: it is the form the

    checker consumes, and it is decoupled from the phrasing the model sees.

    """

    task: str
    domain: str
    prompt: str
    response: str
    answer: float
    unit: str
    canonical: str
    target: str
    params: Dict[str, float] = field(default_factory=dict)

    def to_row(self, keep_canonical: bool = False) -> Dict[str, str]:
        row = {
            "user": self.prompt,
            "assistant": self.response,
            "domain": self.domain,
            "task": self.task,
        }
        if keep_canonical:
            # Carrying the source makes a shipped row re-runnable months later
            # without regenerating it, which is what the omni corpus wanted
            # `keep_canonical` for and could only get for 41.9% of its rows.
            # Here it is 100%: the checker parses exactly what it emitted.
            row["canonical"] = self.canonical
            row["target"] = self.target
        return row


DOMAIN = "programming"


def _pick(rng: random.Random, templates: Sequence[str], **values: str) -> str:
    """Choose one phrasing. Values are pre-rendered strings, never numbers.



    v74 carried one template per task, scored 0.894 on its own benchmark and

    answered 0 of 5 naturally-typed questions. Every task below carries five.

    """

    return rng.choice(list(templates)).format(**values)


#: The five ways a trace question gets asked. Two put the code first and three
#: put it last, because a model that only ever saw `<instruction><code>` learns
#: the position as part of the task.
#: Every template wraps the snippet on ONE side only, never both.
#:
#: The budget here is not the 128-token turn -- it is the model's context. A
#: prompt must leave room for a full-length generation, because
#: `max_position_embeddings` is 128 and a reply that runs to the cap would ask
#: for a position the rotary table does not have. `test_eval_v82` asserts
#: `longest_prompt + DEFAULT_MAX_NEW_TOKENS <= 128`, so at a 96-token cap the
#: longest prompt must fit in 32.
#:
#: The dropped template was "Run this snippet in your head. {code} Give the
#: value of {var}." -- a lead-in AND a trailing instruction around a two-line
#: snippet, which reached 39 tokens on `code_conditional`. One wrapper says the
#: same thing. Phrasing variety matters (v74 had one template per task and
#: scored 0 of 5 on naturally-typed questions), so the count is kept at five by
#: replacing it rather than deleting it.
TRACE_TEMPLATES = (
    "Trace this Python and give {var}. {code}",
    "{code} What is {var} after this runs?",
    "What does {var} hold at the end? {code}",
    "{code} Give the value of {var}.",
    "Python: {code} What is {var}?",
)

#: The same question, worded tightly, for tasks whose snippet is already long.
#:
#: Measured, and the result was not what it looked like. `code_nested_loop`
#: reached 35 tokens against the 32 the context allows beside a full-length
#: generation, and the obvious suspects cost nothing at all: dropping the
#: four-space indent to one space saved 0 tokens, and fixing the start value at
#: 0 saved 0. The tokenizer normalises runs of whitespace, and every digit is
#: already its own token under `--digit_tokens`.
#:
#: The entire overhead was the wrapper. "What does c hold at the end?" costs
#: four tokens more than "What is c?", which is the whole overrun. So the
#: snippet is left exactly as it is -- it is the thing being taught -- and only
#: the words around it are trimmed.
#:
#: Kept at five entries because phrasing variety is what stops the model
#: learning a template instead of a task: v74 had one phrasing per task and
#: answered 0 of 5 naturally-typed questions.
COMPACT_TRACE_TEMPLATES = (
    "{code} What is {var}?",
    "{code} Give {var}.",
    "Python: {code} {var}?",
    "Trace this. {code} {var}?",
    "{code} Final {var}?",
)


# -- task generators --------------------------------------------------------


def _loop_add(rng: random.Random) -> CodeProblem:
    """`x = s` then `n` additions of `k`. The plainest accumulator there is.



    `n` stops at 7 and `s + n * k` at 72 for two separate reasons: the running

    total must stay two-digit (rule 1), and each written step costs about eight

    tokens, so an eighth step pushed the longest turns past the 128-token block

    where turn-aligned packing drops them outright.

    """

    start = rng.randint(0, 9)
    steps = rng.randint(3, 7)
    increment = rng.randint(2, 9)
    source = f"x = {start}\nfor i in range({steps}): x = x + {increment}"
    prompt = _pick(rng, TRACE_TEMPLATES, code=source, var="x")

    value = start
    pieces = [f"the loop runs {steps} times", f"x starts at {start}"]
    for _ in range(steps):
        pieces.append(f"{value} + {increment} = {value + increment}")
        value += increment
    pieces.append(f"x is {value}")
    response = ", ".join(pieces) + f", total {value}"
    return CodeProblem("code_loop_add", DOMAIN, prompt, response, float(value), "",
                       source, "x",
                       {"start": start, "steps": steps, "increment": increment})


def _loop_subtract(rng: random.Random) -> CodeProblem:
    """The same accumulator downwards, so the model does not learn `for` = `+`.



    The start is redrawn until the final value is non-negative: a corpus of

    negative results would teach the sign as a property of the loop rather than

    of the arithmetic, and the benchmark's `-?\\d+` extraction makes a stray

    minus a wrong answer rather than a near miss.

    """

    steps = rng.randint(3, 7)
    decrement = rng.randint(2, 9)
    span = steps * decrement
    start = rng.randint(span, 99)
    source = f"x = {start}\nfor i in range({steps}): x = x - {decrement}"
    prompt = _pick(rng, TRACE_TEMPLATES, code=source, var="x")

    value = start
    pieces = [f"the loop runs {steps} times", f"x starts at {start}"]
    for _ in range(steps):
        pieces.append(f"{value} - {decrement} = {value - decrement}")
        value -= decrement
    pieces.append(f"x is {value}")
    response = ", ".join(pieces) + f", total {value}"
    return CodeProblem("code_loop_subtract", DOMAIN, prompt, response, float(value), "",
                       source, "x",
                       {"start": start, "steps": steps, "decrement": decrement})


#: Values a literal list is built from. Three drawn without replacement out of
#: fifteen gives 2,730 distinct lists -- 7.3x repetition at `--per_task 20000`,
#: the same order as v74's winning 56x rather than the omni build's 1.7x.
LIST_POOL = range(2, 17)

#: `code_index` draws from a narrower pool because its prompt space is already
#: multiplied by the choice of index pair; see the module docstring.
INDEX_POOL = range(2, 14)


def _list_sum(rng: random.Random) -> CodeProblem:
    """`sum` over a three-item literal, with **the operands written out**.



    This is the task the omni corpus got wrong. v86's `average` writes

    `sum: 48 then 137 then 170` -- a running total with no operands -- and its

    individual additions are correct 1.5% of the time while its division is

    correct 69%. Writing `48 + 89 = 137` instead is the entire difference, and

    it is why a list-sum task earns its place next to an arithmetic one.

    """

    values = rng.sample(list(LIST_POOL), 3)
    literal = ", ".join(str(v) for v in values)
    source = f"nums = [{literal}]\nr = sum(nums)"
    prompt = _pick(rng, COMPACT_TRACE_TEMPLATES, code=source, var="r")

    running = values[0]
    pieces = ["sum adds the items in order", f"r starts at {running}"]
    for value in values[1:]:
        pieces.append(f"{running} + {value} = {running + value}")
        running += value
    pieces.append(f"r is {running}")
    response = ", ".join(pieces) + f", total {running}"
    return CodeProblem("code_list_sum", DOMAIN, prompt, response, float(running), "",
                       source, "r", {f"v{i}": v for i, v in enumerate(values)})


def _list_extreme(rng: random.Random) -> CodeProblem:
    """`max` or `min` over the same literal, as a chain of pairwise compares.



    Both directions in one task rather than two, so the model has to read which

    builtin was called instead of learning "a list question means the biggest

    number". The canonical source records which was asked; the phrasing does

    not mention it at all.

    """

    values = rng.sample(list(LIST_POOL), 3)
    literal = ", ".join(str(v) for v in values)
    wants_max = rng.random() < 0.5
    builtin = "max" if wants_max else "min"
    word = "larger" if wants_max else "smaller"
    source = f"nums = [{literal}]\nr = {builtin}(nums)"
    prompt = _pick(rng, COMPACT_TRACE_TEMPLATES, code=source, var="r")

    best = values[0]
    pieces = [f"{builtin} keeps the {word} of each pair"]
    for value in values[1:]:
        chosen = max(best, value) if wants_max else min(best, value)
        pieces.append(f"compare {best} and {value}, the {word} is {chosen}")
        best = chosen
    pieces.append(f"r is {best}")
    response = ", ".join(pieces) + f", total {best}"
    return CodeProblem("code_list_extreme", DOMAIN, prompt, response, float(best), "",
                       source, "r",
                       {f"v{i}": v for i, v in enumerate(values)})


def _index(rng: random.Random) -> CodeProblem:
    """Two positions read out of a list and added.



    The derivation states `count positions from zero` every time. Zero-based

    indexing is the one fact in this family that is a convention rather than a

    computation, and a convention the derivation never names is a convention

    the model has to infer from the numbers.

    """

    values = rng.sample(list(INDEX_POOL), 3)
    literal = ", ".join(str(v) for v in values)
    first, second = sorted(rng.sample(range(3), 2))
    source = f"nums = [{literal}]\nr = nums[{first}] + nums[{second}]"
    prompt = _pick(rng, COMPACT_TRACE_TEMPLATES, code=source, var="r")

    left, right = values[first], values[second]
    answer = left + right
    response = (f"count positions from zero, "
                f"nums [ {first} ] is {left}, nums [ {second} ] is {right}, "
                f"{left} + {right} = {answer}, r is {answer}, total {answer}")
    return CodeProblem("code_index", DOMAIN, prompt, response, float(answer), "",
                       source, "r",
                       {f"v{i}": v for i, v in enumerate(values)},)


def _conditional(rng: random.Random) -> CodeProblem:
    """`if a > b` picking which subtraction to run.



    Both operands stay two-digit and the branch is the only decision, so a

    wrong answer localises: taking the wrong branch gives a negative number,

    which is visibly different from an arithmetic slip.

    """

    a = rng.randint(11, 60)
    b = rng.randint(11, 60)
    while b == a:
        b = rng.randint(11, 60)
    # A conditional *expression*, not a three-line if/else block.
    #
    # The constraint this hit is the model's CONTEXT, not the turn budget:
    # `max_position_embeddings` is 128, so a prompt must leave room for a
    # full-length generation, or a reply running to the 96-token cap asks the
    # rotary table for a position it does not have. The block form reached 34
    # tokens against the 32 that leaves, and joining its lines with spaces is
    # not valid Python -- the interpreter rejects it, which is the executor
    # doing its job.
    #
    # `--digit_tokens` is why this is tight at all: every digit is its own
    # token, so `a = 59; b = 50` spends six tokens on the numbers alone.
    #
    # The ternary keeps what the task teaches -- both branches are visible, and
    # taking the wrong one still yields a negative number, so a branch error
    # stays distinguishable from an arithmetic slip.
    source = f"a = {a}; b = {b}\nr = a - b if a > b else b - a"
    prompt = _pick(rng, COMPACT_TRACE_TEMPLATES, code=source, var="r")

    if a > b:
        branch = (f"{a} is greater than {b}, so the if branch runs, r = a - b")
        answer = a - b
        step = f"{a} - {b} = {answer}"
    else:
        branch = (f"{a} is not greater than {b}, so the else branch runs, r = b - a")
        answer = b - a
        step = f"{b} - {a} = {answer}"
    response = (f"compare a and b, {branch}, {step}, "
                f"r is {answer}, total {answer}")
    return CodeProblem("code_conditional", DOMAIN, prompt, response, float(answer), "",
                       source, "r", {"a": a, "b": b})


def _divmod_task(rng: random.Random) -> CodeProblem:
    """`a // b` or `a % b`, worked as one multiplication and one subtraction.



    `a` is capped at `10 * b - 1` so the quotient is always a single digit.

    That is not tidiness: `decompose_product` writes two partial products and

    stops, by design, so a two-digit quotient would put the product itself

    (`40 x 2 = 80, 9 x 2 = 18` -> 98) nowhere in the text while the next step

    subtracts it. Every number this derivation uses is a number it wrote.

    """

    b = rng.randint(3, 9)
    a = rng.randint(10, min(99, 10 * b - 1))
    quotient, remainder = divmod(a, b)
    floor_division = rng.random() < 0.5
    operator = "//" if floor_division else "%"
    source = f"a = {a}; b = {b}; r = a {operator} b"
    prompt = _pick(rng, COMPACT_TRACE_TEMPLATES, code=source, var="r")

    working = (f"{a} {operator} {b}, count whole {b} s inside {a}, "
               f"{decompose_product(quotient, b)}, "
               f"{a} - {quotient * b} = {remainder}, "
               f"{remainder} is less than {b}")
    if floor_division:
        answer = quotient
        response = (f"{working}, the quotient is {quotient}, "
                    f"r is {quotient}, total {quotient}")
    else:
        answer = remainder
        response = (f"{working}, the remainder is {remainder}, "
                    f"r is {remainder}, total {remainder}")
    return CodeProblem("code_divmod", DOMAIN, prompt, response, float(answer), "",
                       source, "r", {"a": a, "b": b, "q": quotient})


def _nested_loop(rng: random.Random) -> CodeProblem:
    """How many times a doubly-nested body runs, plus a starting count.



    Both ranges are single-digit so `decompose_product(outer, inner)` is one

    written step and the iteration count is a number on the page before the

    final addition consumes it. `start + outer * inner` is capped at 99 by

    redrawing, keeping that addition a two-digit single step.

    """

    outer = rng.randint(2, 9)
    inner = rng.randint(2, 9)
    start = rng.randint(0, 9)
    while start + outer * inner > 99:
        outer = rng.randint(2, 9)
        inner = rng.randint(2, 9)
    iterations = outer * inner
    answer = start + iterations
    source = (f"c = {start}\n"
              f"for i in range({outer}):\n"
              f"    for j in range({inner}): c = c + 1")
    prompt = _pick(rng, COMPACT_TRACE_TEMPLATES, code=source, var="c")

    response = (f"the inner loop runs {inner} times for each outer step, "
                f"the outer loop runs {outer} times, "
                f"{decompose_product(outer, inner)} iterations, "
                f"c starts at {start}, {start} + {iterations} = {answer}, "
                f"c is {answer}, total {answer}")
    return CodeProblem("code_nested_loop", DOMAIN, prompt, response, float(answer), "",
                       source, "c",
                       {"outer": outer, "inner": inner, "start": start})


def _while_accumulate(rng: random.Random) -> CodeProblem:
    """A `while` that stops the first time the total passes a threshold.



    Written as `while` rather than `for ... if ...: break` for a tokenizer

    reason, not a semantic one: the `break` form needs an indented two-statement

    body, which adds `\\n    x` and `\\n    if` to the vocabulary as symbols

    unrelated to ` x` and ` if`. The `while` form is one line and teaches the

    same thing -- accumulate, test, exit.



    The comparison is restated after *every* step. Emitting only the final one

    would leave the stopping rule implicit, and rule 4 of the envelope is that

    a scratchpad helps only where it decomposes the operation being performed.

    """

    # Three or four iterations, not three to five.
    #
    # Each iteration emits two pieces -- the addition and the restated
    # comparison -- so the reply grows about twice as fast per step as the other
    # code tasks. Measured over 1,080 rows with the v86 tokenizer, five
    # iterations reached a 129-token turn against the 128-token budget, and
    # `_build_turn_aligned_tensors` drops a turn longer than the block *without
    # reporting it*. Five of 1,080 rows were being silently discarded.
    #
    # Capping iterations rather than dropping the restated comparison is
    # deliberate: the comparison is what makes the stopping rule explicit, and
    # rule 4 of the envelope is that a scratchpad helps only where it decomposes
    # the operation. Losing a little capacity is the cheaper trade.
    step = rng.randint(5, 19)
    iterations = rng.randint(3, 4)
    threshold = rng.randint((iterations - 1) * step, iterations * step - 1)
    source = f"x = 0\nwhile x <= {threshold}: x = x + {step}"
    prompt = _pick(rng, TRACE_TEMPLATES, code=source, var="x")

    value = 0
    pieces = ["x starts at 0"]
    for _ in range(iterations):
        nxt = value + step
        pieces.append(f"{value} + {step} = {nxt}")
        value = nxt
        if value <= threshold:
            pieces.append(f"{value} is not past {threshold}")
        else:
            pieces.append(f"{value} is past {threshold}, the loop stops")
    pieces.append(f"x is {value}")
    response = ", ".join(pieces) + f", total {value}"
    return CodeProblem("code_while_accumulate", DOMAIN, prompt, response, float(value), "",
                       source, "x",
                       {"step": step, "threshold": threshold, "iterations": iterations})


# -- v93 generators ----------------------------------------------------------
#
# Three more tasks (docs/V93_NEUROGENESIS_TWO_HEMISPHERES.md, D7), verified by
# execution like the nine above. They live in `V93_TASKS` rather than `TASKS`
# for the reason `build_omni_corpus` gives: `build` draws every task from one
# RNG in `chosen` order, so a default build stays byte-identical only if the
# default list does; and the benchmark registry built from `TASKS` carries the
# fingerprint every published receipt is paired on.

#: The most terms a `code_range_sum` trace writes out. Measured with the
#: training segmentation against the benchmark's 96-token generation cap
#: (`eval_problem_solving.DEFAULT_MAX_NEW_TOKENS`): the longest ten-term
#: trace, `range(3, 13)`, is 95 reply tokens including EOS, and an eleven-term
#: one is 97 or more. One step per addition is the rule this family rests on,
#: so the term count is bounded rather than the trace compressed. The map's
#: suggestion of `range(13)` alone would have needed thirteen terms and 115
#: tokens, a reply the benchmark could never finish.
RANGE_SUM_MAX_TERMS = 10

#: `range` stops below this. Sums stay two-digit: `sum(range(1, 13)) = 78`.
RANGE_SUM_MAX_STOP = 13


def _range_sum(rng: random.Random) -> CodeProblem:
    """`r = sum(range(n))` or `r = sum(range(a, n))`, every addition written.



    The one-argument form alone has eight legal instances under the term

    bound, which is a lookup table rather than a procedure. The two-argument

    form is what makes it a procedure: the trace has to read where `range`

    starts as well as where it stops, and the sum then depends on both. A

    start of 0 is written as `range(n)`, so both spellings the model will meet

    are taught, and the trace names the first and last values so nothing about

    the half-open interval is left to be inferred.

    """

    stop = rng.randint(3, RANGE_SUM_MAX_STOP)
    # The start must leave at least two terms and at most the trace budget.
    start = rng.randint(max(0, stop - RANGE_SUM_MAX_TERMS), stop - 2)
    call = f"range({stop})" if start == 0 else f"range({start}, {stop})"
    source = f"r = sum({call})"
    prompt = _pick(rng, TRACE_TEMPLATES, code=source, var="r")

    terms = list(range(start, stop))
    running = terms[0]
    pieces = [f"range counts from {start} up to {stop - 1}", f"r starts at {running}"]
    for term in terms[1:]:
        pieces.append(f"{running} + {term} = {running + term}")
        running += term
    pieces.append(f"r is {running}")
    response = ", ".join(pieces) + f", total {running}"
    return CodeProblem("code_range_sum", DOMAIN, prompt, response, float(running), "",
                       source, "r", {"start": start, "stop": stop})


#: Values a `code_list_count` list is built from. Six values in four slots keep
#: the space near the family's other list tasks (4,020 distinct problems,
#: 5.0x repetition at 20,000 rows) rather than the 96,040 a five-slot list
#: over eight values would give.
COUNT_POOL = range(2, 8)
COUNT_LENGTH = 4


def _list_count(rng: random.Random) -> CodeProblem:
    """`r = nums.count(v)` over a four-item literal, one verdict per item.



    The counted value appears one, two or three times; the other slots hold

    values that differ from it, so every `matches` / `does not` in the trace is

    decidable from the two numbers on the line. `.count` is the first method

    call in this family, admitted through `SAFE_METHODS` rather than by

    widening the call rule in general.

    """

    value = rng.choice(list(COUNT_POOL))
    hits = rng.randint(1, 3)
    positions = set(rng.sample(range(COUNT_LENGTH), hits))
    others = [v for v in COUNT_POOL if v != value]
    items = [value if i in positions else rng.choice(others) for i in range(COUNT_LENGTH)]
    literal = ", ".join(str(v) for v in items)
    source = f"nums = [{literal}]\nr = nums.count({value})"
    prompt = _pick(rng, COMPACT_TRACE_TEMPLATES, code=source, var="r")

    pieces = [f"count checks each item against {value}"]
    for item in items:
        pieces.append(f"{item} matches" if item == value else f"{item} does not")
    pieces.append(f"r is {hits}")
    response = ", ".join(pieces) + f", total {hits}"
    return CodeProblem("code_list_count", DOMAIN, prompt, response, float(hits), "",
                       source, "r", {"value": value, "hits": hits,
                                     **{f"v{i}": v for i, v in enumerate(items)}})


def _neg_index(rng: random.Random) -> CodeProblem:
    """`r = nums[-k]`: one position read from the end of a three-item list.



    `code_index` states `count positions from zero` on every row because

    zero-based indexing is a convention the numbers alone do not reveal. The

    negative form is the same kind of fact from the other end, and the trace

    walks back one position at a time so `nums [ -2 ]` is reached from

    `nums [ -1 ]` rather than asserted.

    """

    values = rng.sample(list(INDEX_POOL), 3)
    literal = ", ".join(str(v) for v in values)
    k = rng.randint(1, 3)
    source = f"nums = [{literal}]\nr = nums[-{k}]"
    prompt = _pick(rng, COMPACT_TRACE_TEMPLATES, code=source, var="r")

    pieces = ["negative positions count from the end"]
    for back in range(1, k + 1):
        pieces.append(f"nums [ -{back} ] is {values[-back]}")
    answer = values[-k]
    pieces.append(f"r is {answer}")
    response = ", ".join(pieces) + f", total {answer}"
    return CodeProblem("code_neg_index", DOMAIN, prompt, response, float(answer), "",
                       source, "r", {"k": k, **{f"v{i}": v for i, v in enumerate(values)}})


#: Every v89 generator, by task name. The `code_` prefix keeps them
#: distinguishable in a receipt and guarantees they never shadow an arithmetic
#: or omni task when `eval_problem_solving` merges the three families into one
#: dict. Frozen: see the note above `RANGE_SUM_MAX_TERMS`.
TASKS: Dict[str, Callable[[random.Random], CodeProblem]] = {
    "code_loop_add": _loop_add,
    "code_loop_subtract": _loop_subtract,
    "code_list_sum": _list_sum,
    "code_list_extreme": _list_extreme,
    "code_index": _index,
    "code_conditional": _conditional,
    "code_divmod": _divmod_task,
    "code_nested_loop": _nested_loop,
    "code_while_accumulate": _while_accumulate,
}

#: The tasks added for v93. Built only when named with `--task`.
V93_TASKS: Dict[str, Callable[[random.Random], CodeProblem]] = {
    "code_range_sum": _range_sum,
    "code_list_count": _list_count,
    "code_neg_index": _neg_index,
}

#: Every generator this module can build, for lookup by name.
ALL_TASKS: Dict[str, Callable[[random.Random], CodeProblem]] = {**TASKS, **V93_TASKS}


#: How many distinct problems each generator can produce, counted from the
#: parameter ranges in its body rather than sampled.
#:
#: This is reported because v74's multiplication task scored 0.93 with 712
#: distinct problems and 0.03 with 24,000 of the same operation. A task whose
#: capacity drifts upward silently is a task whose score will fall for a reason
#: nobody attributes to the corpus. `test_code_corpus.py` samples each
#: generator and asserts the number below is not an undercount.
def distinct_capacity() -> Dict[str, int]:
    pool = len(LIST_POOL)
    index_pool = len(INDEX_POOL)
    return {
        # start(10) x steps(5) x increment(8)
        "code_loop_add": 10 * 5 * 8,
        # sum over steps(3..7), decrement(2..9) of the legal starts 99-span+1
        "code_loop_subtract": sum(
            max(0, 99 - steps * dec + 1)
            for steps in range(3, 8) for dec in range(2, 10)
        ),
        # ordered 3-subsets of the pool
        "code_list_sum": pool * (pool - 1) * (pool - 2),
        # the same, doubled by max/min
        "code_list_extreme": 2 * pool * (pool - 1) * (pool - 2),
        # ordered 3-subsets of the narrower pool x 3 index pairs
        "code_index": index_pool * (index_pool - 1) * (index_pool - 2) * 3,
        # a(50) x b(50) minus the diagonal
        "code_conditional": 50 * 50 - 50,
        # sum over b of the legal a range, doubled by // and %
        "code_divmod": 2 * sum(min(99, 10 * b - 1) - 10 + 1 for b in range(3, 10)),
        # (outer, inner) pairs with a legal start
        "code_nested_loop": sum(
            1 for outer in range(2, 10) for inner in range(2, 10)
            for start in range(10) if start + outer * inner <= 99
        ),
        # step(15) x iterations(3) x the `step` thresholds each admits
        "code_while_accumulate": sum(
            step for step in range(5, 20) for _ in range(3)
        ),
        # -- v93 --
        # (start, stop) pairs with 2..RANGE_SUM_MAX_TERMS terms; 71 today
        "code_range_sum": sum(
            (stop - 2) - max(0, stop - RANGE_SUM_MAX_TERMS) + 1
            for stop in range(3, RANGE_SUM_MAX_STOP + 1)
        ),
        # value(6) x sum over hits of C(4, hits) x 5^(4 - hits)
        "code_list_count": len(COUNT_POOL) * sum(
            math.comb(COUNT_LENGTH, hits) * (len(COUNT_POOL) - 1) ** (COUNT_LENGTH - hits)
            for hits in range(1, 4)
        ),
        # ordered 3-subsets of the narrower pool x 3 offsets
        "code_neg_index": index_pool * (index_pool - 1) * (index_pool - 2) * 3,
    }


# -- verification -----------------------------------------------------------


def verify(problem: CodeProblem,

           timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS) -> SnippetResult:
    """Run the snippet the prompt shows and compare it to the worked answer.



    Returns the full `SnippetResult` rather than a bool so the build can report

    *why* rows were dropped. A verifier whose failures are an unlabelled count

    cannot tell a broken template from an exhausted parameter range.

    """

    result = run_snippet(problem.canonical, problem.target, timeout_seconds)
    if not result.ok:
        return result
    if result.value != problem.answer:
        return SnippetResult(
            False, result.value,
            f"interpreter got {result.value}, derivation says {problem.answer}",
        )
    return result


#: The words a snippet line may begin with. Pinned because tokens carry their
#: leading whitespace: `"\n    for"` is a different symbol from `" for"`, so
#: every new line-initial form is a new vocabulary entry the model must learn
#: separately. Six is cheap; an indented multi-statement body would not be.
LINE_INITIAL_FORMS = frozenset({
    "for", "    for", "while", "if", "else", "r",
})


def line_initial_forms(rows: Sequence[Dict[str, str]]) -> Dict[str, int]:
    """Count the distinct indent-prefixed line starts a corpus contains."""

    counts: Dict[str, int] = {}
    for row in rows:
        for line in row["user"].split("\n")[1:]:
            word = line.split(" ")[0] if line.strip() else ""
            # Keep the indentation, because that is what the tokenizer keeps.
            leading = len(line) - len(line.lstrip(" "))
            form = " " * leading + word.strip()
            if form:
                counts[form] = counts.get(form, 0) + 1
    return dict(sorted(counts.items()))


# -- corpus construction ----------------------------------------------------


def build(per_task: int, seed: int, tasks: Optional[Sequence[str]] = None,

          repeat: bool = True, keep_canonical: bool = False,

          timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS,

          sequence_length: int = DEFAULT_SEQUENCE_LENGTH,

          token_budget: bool = False) -> Tuple[List[Dict[str, str]], Dict[str, Any]]:
    """Generate, execute, verify, and return (rows, report).



    `repeat=True` is the default for the reason `build_omni_corpus` documents

    at length: repetition is how this model learns a procedure, and the

    benchmark draws unseen parameters from the same space so recall cannot pass

    it. `repeat=False` emits distinct prompts only and reports the shortfall

    when a generator's space runs out.

    """

    rng = random.Random(seed)
    # The default is the v89 nine; a v93 task is built only when named.
    chosen = list(tasks or TASKS)
    unknown = [name for name in chosen if name not in ALL_TASKS]
    if unknown:
        raise ValueError(f"unknown task(s): {', '.join(unknown)}")

    rows: List[Dict[str, str]] = []
    counts: Dict[str, int] = {}
    dropped: Dict[str, int] = {}
    reasons: Dict[str, Dict[str, int]] = {}
    distinct: Dict[str, int] = {}
    short: Dict[str, Dict[str, object]] = {}
    attempts: Dict[str, int] = {}

    def draw(name: str) -> Optional[CodeProblem]:
        """One executed-and-agreed problem, or None. Counts its own drops."""

        attempts[name] = attempts.get(name, 0) + 1
        problem = ALL_TASKS[name](rng)
        result = verify(problem, timeout_seconds)
        parsed = extract_answer(problem.response)
        if not result.ok:
            dropped[name] = dropped.get(name, 0) + 1
            bucket = reasons.setdefault(name, {})
            bucket[result.reason] = bucket.get(result.reason, 0) + 1
            return None
        if parsed != problem.answer:
            # A right answer the benchmark cannot read scores as wrong, so this
            # is as fatal as an execution mismatch and is counted separately.
            dropped[name] = dropped.get(name, 0) + 1
            bucket = reasons.setdefault(name, {})
            key = f"response parses to {parsed}, answer is {problem.answer}"
            bucket[key] = bucket.get(key, 0) + 1
            return None
        return problem

    for name in chosen:
        made = 0
        seen: set = set()
        if repeat:
            for _ in range(per_task):
                problem = draw(name)
                if problem is None:
                    continue
                seen.add(problem.prompt)
                rows.append(problem.to_row(keep_canonical=keep_canonical))
                made += 1
        else:
            misses = 0
            while made < per_task and misses < EXHAUSTION_MISSES:
                problem = draw(name)
                if problem is None or problem.prompt in seen:
                    misses += 1
                    continue
                seen.add(problem.prompt)
                rows.append(problem.to_row(keep_canonical=keep_canonical))
                made += 1
                misses = 0
            if made < per_task:
                short[name] = {"asked": per_task, "produced": made,
                               "reason": "parameter space exhausted"}
        counts[name] = made
        distinct[name] = len(seen)

    total_attempts = sum(attempts.values())
    total_dropped = sum(dropped.values())
    report: Dict[str, Any] = {
        "schema": "supermix-v87-code-corpus-v1",
        "seed": seed,
        "rows": len(rows),
        "tasks": len(chosen),
        "per_task": counts,
        "distinct_prompts": distinct,
        "repetition": {k: round(counts[k] / v, 1) for k, v in distinct.items() if v},
        "distinct_capacity": {k: v for k, v in distinct_capacity().items()
                              if k in counts},
        "attempts": attempts,
        "dropped_failing_execution": dropped,
        "drop_reasons": reasons,
        "drop_rate": round(total_dropped / total_attempts, 6) if total_attempts else 0.0,
        "short_of_requested": short,
        "verified_by": "cpython exec in a restricted namespace",
        "verification": {
            "builtins": sorted(SAFE_BUILTINS),
            "methods": sorted(SAFE_METHODS),
            "timeout_seconds": timeout_seconds,
            "allowed_syntax": sorted(ALLOWED_NODES),
            "note": (
                "the checker executes the snippet the PROMPT shows, not the "
                "expression the generator used, so a template that renders "
                "something other than what it computes is dropped"
            ),
        },
        "line_initial_forms": line_initial_forms(rows),
        "options": {
            "repeat": repeat,
            "keep_canonical": keep_canonical,
        },
    }
    if token_budget:
        report["token_budget"] = token_budget_report(rows, sequence_length)
    return rows, report


def build_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    parser.add_argument("--per_task", type=int, default=20000)
    parser.add_argument("--seed", type=int, default=87)
    parser.add_argument("--output", default="datasets/v87/v87_code.jsonl")
    parser.add_argument("--report", default=None)
    parser.add_argument("--unique", action="store_true",
                        help=("emit only distinct prompts. Off by default: v74's "
                              "multiplication task repeated 712 pairs 56x each and "
                              "scored 0.93, while a 24,000-unique build of the same "
                              "operation scored 0.03"))
    parser.add_argument("--task", action="append", default=[],
                        help=("restrict to these tasks; repeatable. The default is "
                              "the nine v89 tasks; the v93 tasks "
                              f"({', '.join(V93_TASKS)}) are built only when named"))
    parser.add_argument("--keep_canonical", action="store_true",
                        help="ship the executable source and target variable per row")
    parser.add_argument("--timeout", type=float, default=DEFAULT_TIMEOUT_SECONDS,
                        help="wall clock a single snippet gets before it is dropped")
    parser.add_argument("--token_budget_report", action="store_true",
                        help=("measure per-task response and turn lengths and what "
                              "turn-aligned packing would drop"))
    parser.add_argument("--sequence_length", type=int,
                        default=DEFAULT_SEQUENCE_LENGTH,
                        help="the block size the token budget is measured against")
    return parser


def main(argv: Optional[Sequence[str]] = None) -> int:
    args = build_parser().parse_args(argv)
    rows, report = build(args.per_task, args.seed, args.task or None,
                         repeat=not args.unique,
                         keep_canonical=args.keep_canonical,
                         timeout_seconds=args.timeout,
                         sequence_length=args.sequence_length,
                         token_budget=args.token_budget_report)

    output = Path(args.output)
    output.parent.mkdir(parents=True, exist_ok=True)
    with output.open("w", encoding="utf-8") as handle:
        for row in rows:
            handle.write(json.dumps(row) + "\n")

    report["output"] = str(output)
    report_path = Path(args.report) if args.report else output.with_suffix(".report.json")
    report_path.write_text(json.dumps(report, indent=2), encoding="utf-8")

    print(f"wrote {len(rows):,} rows to {output}")
    print(f"drop rate {report['drop_rate']:.4%} "
          f"({sum(report['dropped_failing_execution'].values()):,} of "
          f"{sum(report['attempts'].values()):,} attempts)")
    print(f"  {'task':<24} {'rows':>7} {'distinct':>9} {'x':>6} {'capacity':>9} "
          f"{'dropped':>8}")
    capacity = report["distinct_capacity"]
    for name, count in sorted(report["per_task"].items()):
        print(f"  {name:<24} {count:>7,} {report['distinct_prompts'][name]:>9,} "
              f"{report['repetition'].get(name, 0):>6.1f} "
              f"{capacity.get(name, 0):>9,} "
              f"{report['dropped_failing_execution'].get(name, 0):>8,}")

    budget = report.get("token_budget")
    if budget:
        print(f"\ntoken budget at sequence_length {budget['sequence_length']}")
        print(f"  {'task':<24} {'resp med':>8} {'p95':>5} {'max':>5} "
              f"{'turn max':>9} {'dropped':>8}")
        for name, stats in budget["tasks"].items():
            print(f"  {name:<24} {stats['response_median']:>8} "
                  f"{stats['response_p95']:>5} {stats['response_max']:>5} "
                  f"{stats['turn_max']:>9} {stats['dropped_fraction']:>8.4f}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())