File size: 49,419 Bytes
dca70d8
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
"""Measure problem solving by whether the answer is *right*.



Every metric in this repo so far scores how probable the training text was. V64

showed why that is not enough: verbatim reproduction of training data is the

lowest-loss behaviour available, so perplexity actively prefers a model that

recites over one that reasons.



Arithmetic does not have that problem. The answer to "what is 617 + 288" is

checkable, and a model that recites a remembered answer to a *novel* problem is

simply wrong. Exact-match accuracy on freshly generated problems is therefore a

recitation-proof measure of problem solving, which is what this module provides.



The decisive comparison is the pair:



* **seen** -- problems drawn verbatim from the training corpus.

* **novel** -- problems in identical phrasing with operands generated here.



A model that has learned arithmetic scores similarly on both. A model that has

memorised the corpus scores well on *seen* and near zero on *novel*, and the gap

between them is the measurement. Aggregate accuracy alone cannot tell those two

models apart, which is the whole point.



    python source/eval_problem_solving.py --checkpoint output/v64_meaning/v64_meaning.partial.pt



v82 changed three things about *how* this measures, none of them about the

model:



1. **Each task draws from its own RNG.** Until v82 `generate_novel` round-robined

   over `GENERATORS` with one shared `random.Random`, so adding a task,

   reordering `build_omni_corpus.TASKS`, or changing how many draws a generator

   makes shifted every *later* problem for the same seed. The commit before this

   one removed `combination`'s `k` draw, which means the pre- and post-commit

   n=630 numbers were never paired. `--legacy_shared_rng` reproduces the old

   draw exactly, and that is how the v80 receipt is still checkable.

2. **The generation cap rose from 40 to 96 tokens.** Measured with the v80

   tokenizer over `datasets/v80/v80_combined.jsonl` (400 rows per task), reply

   lengths run to a median of 92 tokens for `arithmetic_series` and 88 for

   `work`; eleven of twenty-two tasks exceed 40 tokens and seven exceed 64. At

   40 the benchmark was scoring the cap, not the model.

3. **Two scores are reported, not one.** Exact match is kept unchanged, and an

   abstention-aware score (correct +1, no answer 0, wrong -1) is reported

   beside it, because a binary-graded benchmark rewards confident guessing

   (OpenAI, "Why Language Models Hallucinate", arXiv 2509.04664). Wilson score

   intervals accompany every rate: at n=30 per task the 95% interval is

   [0.332, 0.668] at its widest (p=0.5), so +-17 points -- which is the

   difference between a real per-task change and noise.

"""

from __future__ import annotations

import argparse
import hashlib
import json
import math
import random
import re
import sys
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple

SOURCE_DIR = Path(__file__).resolve().parent
if str(SOURCE_DIR) not in sys.path:
    sys.path.append(str(SOURCE_DIR))

from train_mimomix_talk import generate_reply, load_talk_checkpoint  # noqa: E402

#: v1 receipts (v65..v81) carry only exact match and no settings block. They
#: are still readable; they are not comparable to a v2 receipt without checking
#: the generator fingerprint by hand, which is exactly why v2 records one.
RECEIPT_SCHEMA_V1 = "supermix-v65-problem-solving-accuracy-v1"
RECEIPT_SCHEMA = "supermix-v82-problem-solving-accuracy-v2"

#: Answers are floats; comparison needs a tolerance rather than equality.
TOLERANCE = 1e-6

#: The cap in force from v65 to v81. Kept as a name so a re-run against an old
#: receipt is a flag rather than a magic number.
LEGACY_MAX_NEW_TOKENS = 40

#: Measured, not chosen for roundness. Two constraints bracket it:
#:
#: * **Below**: the longest replies the model was trained to produce. Over 400
#:   rows per task of `datasets/v80/v80_combined.jsonl`, encoded with the v80
#:   tokenizer, the longest reply is 99 tokens (`momentum`), the 95th
#:   percentile per task tops out at 97 (`arithmetic_series`, `work`); 8 of the
#:   22 task labels in that corpus have a *median* above 40, and 11 produce
#:   replies over 40 at all.
#: * **Above**: `max_position_embeddings` is 128 in the v80 config, and the
#:   longest benchmark prompt measured across all 21 tasks is 30 tokens
#:   (`average`, `word_problem`). 30 + 96 = 126 fits; 30 + 112 would not.
#:
#: 96 is therefore the largest cap that never runs the model past the context
#: it was trained on. It does not eliminate truncation -- a 99-token
#: `momentum` reply still hits it -- which is why truncation is now counted and
#: reported rather than silently scored wrong.
DEFAULT_MAX_NEW_TOKENS = 96

#: Every corpus reply ends "... total <number>". A reply that stops without one
#: was cut off rather than finished.
_TERMINAL = re.compile(r"total\s+-?\d+(?:\.\d+)?\s*\.?\s*$", re.I)

_NUMBER = re.compile(r"-?\d+(?:\.\d+)?(?:/\d+)?")


@dataclass
class Problem:
    task: str
    prompt: str
    answer: float
    source: str  # "novel" or "seen"


# -- generators ------------------------------------------------------------
#
# Phrasing is copied from `build_english_math_dataset.py` so the *format* is
# in-distribution and only the operands are new. A model that failed here purely
# because the wording was unfamiliar would be measuring the wrong thing.


def _arithmetic(rng: random.Random) -> Problem:
    a, b = rng.randint(100, 999), rng.randint(10, 999)
    op = rng.choice(["+", "-"])
    answer = a + b if op == "+" else a - b
    return Problem(
        "arithmetic", f"Solve this basic math problem: {a} {op} {b}", float(answer), "novel"
    )


def _percent(rng: random.Random) -> Problem:
    pct, base = rng.choice([5, 10, 12, 15, 20, 25]), rng.randint(20, 2000)
    return Problem("percent", f"What is {pct}% of {base}?", pct * base / 100.0, "novel")


def _average(rng: random.Random) -> Problem:
    values = [rng.randint(5, 99) for _ in range(rng.choice([4, 5, 6]))]
    joined = ", ".join(str(v) for v in values)
    return Problem(
        "average",
        f"Find the average (mean) of these numbers: {joined}",
        sum(values) / len(values),
        "novel",
    )


def _algebra(rng: random.Random) -> Problem:
    x, b = rng.randint(-30, 30), rng.randint(-30, 30)
    return Problem("algebra_one_step", f"Solve for x: x + {b} = {x + b}", float(x), "novel")


def _word_problem(rng: random.Random) -> Problem:
    start, gain, lose = rng.randint(20, 99), rng.randint(5, 60), rng.randint(5, 60)
    item = rng.choice(["notebooks", "cookies", "marbles", "stickers"])
    return Problem(
        "word_problem",
        f"A student has {start} {item}. They get {gain} more and then give away "
        f"{lose}. How many {item} do they have now?",
        float(start + gain - lose),
        "novel",
    )



# -- v74 task types ---------------------------------------------------------
#
# Added alongside the corpus generators, not after the fact. A capability the
# benchmark cannot see is a capability nobody can claim.


def _multiplication(rng: random.Random) -> Problem:
    a, b = rng.randint(11, 99), rng.randint(2, 9)
    return Problem("multiplication", f"Solve this basic math problem: {a} x {b}",
                   float(a * b), "novel")


def _division(rng: random.Random) -> Problem:
    b, quotient = rng.randint(2, 9), rng.randint(11, 60)
    return Problem("division", f"Solve this basic math problem: {b * quotient} / {b}",
                   float(quotient), "novel")


def _sequence(rng: random.Random) -> Problem:
    start, step = rng.randint(3, 40), rng.randint(2, 12)
    terms = [start + step * i for i in range(4)]
    joined = ", ".join(str(t) for t in terms)
    return Problem("sequence", f"What comes next in the sequence: {joined}?",
                   float(terms[-1] + step), "novel")


def _two_step(rng: random.Random) -> Problem:
    pct = rng.choice([10, 20, 25, 50])
    base = rng.choice([x for x in range(40, 900) if (x * pct) % 100 == 0])
    first = base * pct // 100
    delta = rng.randint(5, 60)
    add = rng.random() < 0.5
    answer = first + delta if add else first - delta
    word = "add" if add else "subtract"
    return Problem("two_step", f"What is {pct}% of {base}, then {word} {delta}?",
                   float(answer), "novel")


GENERATORS: Dict[str, Callable[[random.Random], Problem]] = {
    "arithmetic": _arithmetic,
    "percent": _percent,
    "average": _average,
    "algebra_one_step": _algebra,
    "word_problem": _word_problem,
    "multiplication": _multiplication,
    "division": _division,
    "sequence": _sequence,
    "two_step": _two_step,
}


def _register_omni_generators() -> List[str]:
    """Add the solver-verified science tasks to this benchmark.



    v79 trains on physics, chemistry and mathematics generated by

    `build_omni_corpus`, every row checked against `nexus_solver`. Those tasks

    need measuring on the *same* benchmark as the arithmetic ones, or v79

    cannot be compared with v74 at all -- two report formats would make the

    comparison a matter of interpretation rather than a number.



    The adapter is thin on purpose: the omni generator already produces a

    prompt and an exactly-verified answer, so this only reshapes it. Import

    failures are swallowed and the science tasks simply do not appear, because

    a benchmark that cannot run at all is worse than one measuring less.

    """

    try:
        import build_omni_corpus as omni
    except Exception:  # noqa: BLE001 - optional; arithmetic tasks still run
        return []

    def adapt(name):
        def generate(rng: random.Random) -> Problem:
            problem = omni.TASKS[name](rng)
            return Problem(name, problem.prompt, problem.answer, "novel")
        return generate

    added = []
    for name in omni.TASKS:
        if name not in GENERATORS:      # never shadow an arithmetic task
            GENERATORS[name] = adapt(name)
            added.append(name)
    return added


#: Task names contributed by the solver-verified science corpus.
OMNI_TASKS: List[str] = _register_omni_generators()


def _register_code_generators() -> List[str]:
    """Add the execution-verified code-tracing tasks to this benchmark.



    Same argument as the omni adapter above, and the same shape. What differs

    is the oracle: an omni row is checked against `nexus_solver`, a code row is

    checked by **running the snippet**. The interpreter is the exact checker,

    which is why this family belongs in a corpus whose whole premise is that

    every row is verified before it ships.



    The `code_` prefix means these can never shadow an arithmetic or omni task,

    so the guard below is belt-and-braces rather than load-bearing.



    Registered unconditionally, unlike the corpus-side flags. A benchmark task

    is not a training decision: leaving it out would mean a corpus built with

    code rows had no matching benchmark, which is exactly the silent

    train/eval split `--combination_in_envelope` exists to prevent.

    """

    try:
        import build_code_corpus as code
    except Exception:  # noqa: BLE001 - optional; every other task still runs
        return []

    def adapt(name):
        def generate(rng: random.Random) -> Problem:
            problem = code.TASKS[name](rng)
            return Problem(name, problem.prompt, problem.answer, "novel")
        return generate

    added = []
    for name in code.TASKS:
        if name not in GENERATORS:
            GENERATORS[name] = adapt(name)
            added.append(name)
    return added


#: Task names contributed by the execution-verified code corpus.
CODE_TASKS: List[str] = _register_code_generators()


#: The thirty tasks of the v89 benchmark, in the registry order of the day
#: v89 was scored: nine arithmetic templates, `build_omni_corpus.TASKS`, then
#: `build_code_corpus.TASKS`. Frozen as a literal rather than read from the
#: registry, so that registering a task can never move it. `--task_set v89`
#: expands to exactly this list, and `generator_fingerprint` over it is
#: 3b99a446cd533be9bc5f8ae57d1310b4 -- the value every v87, v88, v89 and v91
#: receipt carries, which `test_v93_corpus.py` pins. A v93 receipt over this
#: set pairs with those by exact McNemar; one over any other set does not.
V89_BENCHMARK_TASKS: Tuple[str, ...] = (
    "arithmetic", "percent", "average", "algebra_one_step", "word_problem",
    "multiplication", "division", "sequence", "two_step",
    "force", "acceleration", "momentum", "kinetic_energy", "work", "power",
    "voltage", "electrical_power", "wave_speed", "molarity", "combination",
    "arithmetic_series",
    "code_loop_add", "code_loop_subtract", "code_list_sum", "code_list_extreme",
    "code_index", "code_conditional", "code_divmod", "code_nested_loop",
    "code_while_accumulate",
)

#: The tasks v93 adds (docs/V93_NEUROGENESIS_TWO_HEMISPHERES.md, D7): five
#: solver-verified science tasks, three execution-verified code tasks and
#: three connectome-knowledge lookups. `--task_set new` expands to this list.
#: Their scores have no v89 baseline -- v89 cannot answer them -- so a number
#: over this set is descriptive, and the connectome three are recall (see
#: `NON_CLAIMS`).
V93_NEW_TASKS: Tuple[str, ...] = (
    "impulse", "ohms_current", "spring_energy", "permutations", "final_velocity",
    "code_range_sum", "code_list_count", "code_neg_index",
    "cns_type_count", "cns_side_count", "cns_pair_synapses",
)

#: What `--task_set` expands to. `all` is the v89 thirty followed by the
#: eleven new tasks, which is also the registry's own order today.
TASK_SETS: Dict[str, Tuple[str, ...]] = {
    "v89": V89_BENCHMARK_TASKS,
    "new": V93_NEW_TASKS,
    "all": V89_BENCHMARK_TASKS + V93_NEW_TASKS,
}


def _register_v93_generators() -> List[str]:
    """Add the v93 families to this benchmark, after the thirty above.



    Three registries, one adapter shape each, exactly as the two adapters

    above: `build_omni_corpus.V93_TASKS` (solver-verified),

    `build_code_corpus.V93_TASKS` (execution-verified) and

    `build_connectome_corpus.TASKS` (looked up in the CC-BY male-CNS tables).

    They are kept out of the builders' `TASKS` dicts so a default corpus build

    and the two adapters above stay byte-identical; this is where they enter

    the benchmark, and the trainer's probe, instead.



    Registration never shadows: a name already in `GENERATORS` is skipped, so

    the v89 thirty cannot be redefined from here. Per-task RNGs mean the

    thirty's problems do not move either (`test_v93_corpus.py` asserts both).



    The connectome tasks need `datasets/v91_malecns/malecns_types.npz` and a

    side source on disk; where those are absent the family is left out, as an

    import failure leaves a builder's family out. A `--task_set new` run on

    such a machine then fails loudly at the unknown-task check rather than

    silently scoring eight tasks as eleven.

    """

    added: List[str] = []

    def adapt(table, name):
        def generate(rng: random.Random) -> Problem:
            problem = table[name](rng)
            return Problem(name, problem.prompt, problem.answer, "novel")
        return generate

    try:
        import build_omni_corpus as omni
        omni_v93 = getattr(omni, "V93_TASKS", {})
    except Exception:  # noqa: BLE001 - optional, as for the twelve above
        omni_v93 = {}
    try:
        import build_code_corpus as code
        code_v93 = getattr(code, "V93_TASKS", {})
    except Exception:  # noqa: BLE001
        code_v93 = {}
    try:
        import build_connectome_corpus as cns
        cns_tasks = cns.TASKS if cns.data_available() else {}
    except Exception:  # noqa: BLE001
        cns_tasks = {}

    for table in (omni_v93, code_v93, cns_tasks):
        for name in table:
            if name not in GENERATORS:      # never shadow an existing task
                GENERATORS[name] = adapt(table, name)
                added.append(name)
    return added


#: Task names the v93 adapter actually registered on this machine. Equal to
#: `V93_NEW_TASKS` wherever the connectome data is present.
V93_REGISTERED_TASKS: List[str] = _register_v93_generators()

# The three v93 code tasks are execution-verified code tasks like the nine
# before them, and `CODE_TASKS` is how the live checker's tests find every
# task that `answer_check._code_trace` must run; so they are listed there too.
# `OMNI_TASKS` stays the twelve of v89 (it names what `_register_omni_generators`
# added), and the five new science tasks are found through `V93_NEW_TASKS`.
CODE_TASKS.extend(name for name in V93_REGISTERED_TASKS if name.startswith("code_"))


def expand_task_set(name: str) -> List[str]:
    """The ordered task list a `--task_set` name stands for."""

    if name not in TASK_SETS:
        raise KeyError(f"unknown task set {name!r}; choose from {', '.join(TASK_SETS)}")
    return list(TASK_SETS[name])


def task_rng(task: str, seed: int) -> random.Random:
    """A generator's own RNG, derived from its *name* and the master seed.



    Deriving from the name rather than from an index is the whole point: a

    task's stream then does not depend on how many tasks exist, what order they

    are in, or how many draws its neighbours make.

    """

    digest = hashlib.blake2b(f"{task}:{seed}".encode("utf-8"), digest_size=8).digest()
    return random.Random(int.from_bytes(digest, "big"))


def generate_novel(

    count: int,

    seed: int = 65,

    tasks: Optional[Sequence[str]] = None,

    shared_rng: bool = False,

) -> List[Problem]:
    """`count` problems, round-robined over the task list.



    The interleaving order is unchanged from v65. What changed in v82 is where

    the randomness comes from.



    Until v82 one `random.Random(seed)` fed every generator in turn, so the

    stream a task saw depended on every task before it in `GENERATORS`. Adding

    `combination` to `build_omni_corpus.TASKS`, or removing a single `rng`

    draw from it -- which the commit before v82 did -- silently changed every

    problem *after* that task for the same seed. Two receipts with the same

    seed were therefore not necessarily comparable, and the pre- and

    post-commit v80 numbers are not paired.



    With `shared_rng=False` (the default from v82) each task derives its own

    RNG from its name, so tasks are independent: adding a task changes only

    that task's problems. Pass `shared_rng=True` to reproduce a pre-v82 draw

    exactly -- that is how the v80 receipt's recorded examples are still

    checkable, and `test_eval_v82.py` asserts it.

    """

    names = list(tasks or GENERATORS)
    if not names:
        return []
    if shared_rng:
        rng = random.Random(seed)
        return [GENERATORS[names[i % len(names)]](rng) for i in range(count)]
    streams = {name: task_rng(name, seed) for name in names}
    problems: List[Problem] = []
    for index in range(count):
        name = names[index % len(names)]
        problems.append(GENERATORS[name](streams[name]))
    return problems


def generator_fingerprint(

    tasks: Optional[Sequence[str]] = None, probe: int = 3, seed: int = 0

) -> str:
    """A short digest of *what the generators produce*, for receipt comparison.



    Two receipts that share a seed, a task list and this fingerprint drew the

    same problems. Two that differ in it did not, whatever their seeds say. It

    hashes each task's first few prompts and answers under a fixed probe seed,

    so a reworded template, a changed operand range, or a different number of

    rng draws all move it -- which is what the v80 comparison needed and did

    not have.



    It deliberately does not hash source code: a comment change would move that

    and would not change a single problem.

    """

    digest = hashlib.blake2b(digest_size=16)
    for name in sorted(tasks or GENERATORS):
        rng = task_rng(name, seed)
        digest.update(name.encode("utf-8"))
        for _ in range(probe):
            problem = GENERATORS[name](rng)
            digest.update(f"|{problem.prompt}|{problem.answer!r}".encode("utf-8"))
    return digest.hexdigest()


# -- seen problems ---------------------------------------------------------


def load_seen(

    path: Path, count: int, seed: int = 65, tasks: Optional[Sequence[str]] = None

) -> List[Problem]:
    """Problems lifted verbatim from the corpus, with their stated answers.



    These are the memorisation control. Rows whose answer cannot be parsed are

    skipped rather than guessed at -- a wrong ground truth would understate the

    model and make the comparison meaningless in the flattering direction.

    """

    wanted = set(tasks or GENERATORS)
    rows: List[Problem] = []
    with path.open(encoding="utf-8") as handle:
        for line in handle:
            line = line.strip()
            if not line:
                continue
            try:
                record = json.loads(line)
            except json.JSONDecodeError:
                continue
            if record.get("task") not in wanted:
                continue
            answer = extract_answer(str(record.get("assistant", "")))
            if answer is None:
                continue
            rows.append(Problem(record["task"], str(record["user"]), answer, "seen"))
    random.Random(seed).shuffle(rows)
    return rows[:count]


# -- scoring ---------------------------------------------------------------


def extract_answer(text: str) -> Optional[float]:
    """The model's answer is the last number it produces.



    The corpus answers in several shapes -- "79", "376 - 17 = 359",

    "Calculate directly. Answer: 10.68." -- and in every one the final number is

    the answer. Taking the last rather than the first is what makes

    "376 - 17 = 359" score as 359 instead of 376.

    """

    matches = _NUMBER.findall(text.replace(",", ""))
    if not matches:
        return None
    raw = matches[-1].rstrip(".")
    try:
        if "/" in raw:
            numerator, denominator = raw.split("/", 1)
            return float(numerator) / float(denominator)
        return float(raw)
    except (ValueError, ZeroDivisionError):
        return None


def is_correct(predicted: Optional[float], expected: float) -> bool:
    if predicted is None:
        return False
    return abs(predicted - expected) <= max(TOLERANCE, abs(expected) * 1e-6)


def wilson_interval(correct: int, n: int, z: float = 1.96) -> Tuple[float, float]:
    """95% Wilson score interval for a proportion.



    Wilson rather than the normal approximation because this benchmark

    routinely reports 0/30 and 30/30, where `p +- z*sqrt(p(1-p)/n)` is a

    zero-width interval and therefore a lie. Wilson gives (0.000, 0.113) for

    0/30 and (0.885, 1.000) for 30/30.



    The trainer already refuses to *select* on fewer than

    `train_mimomix_generalisation.MIN_SELECTION_PROBLEMS = 100` problems for

    this reason, citing v73's 20-problem probe reading 0.15 where a

    60-problem evaluation of the same checkpoint read 0.467. This benchmark

    runs 30 per task, so per-task numbers here carry roughly a +-18 point

    interval and must not be read as rankings. The aggregate over 21 tasks is

    the number with enough n to move.

    """

    if n <= 0:
        return (0.0, 0.0)
    p = correct / n
    denominator = 1.0 + z * z / n
    centre = (p + z * z / (2 * n)) / denominator
    spread = (z / denominator) * math.sqrt(p * (1.0 - p) / n + z * z / (4.0 * n * n))
    return (max(0.0, centre - spread), min(1.0, centre + spread))


def looks_terminated(text: str) -> bool:
    """True when the reply ends in the corpus's terminal `total <number>`."""

    return _TERMINAL.search(text.strip()) is not None


def is_truncated(text: str, tokens: Optional[int], max_new_tokens: int) -> bool:
    """The reply spent its whole budget and never reached a final answer.



    Both halves matter. A reply that stops early without "total N" simply

    failed; a reply that ends on "total 905" at exactly the cap finished. Only

    the conjunction -- budget exhausted *and* no terminal total -- means the

    harness cut the model off, and that is a fact about the benchmark rather

    than about the model, so it is reported separately instead of being scored

    as a wrong answer.

    """

    if tokens is None:
        return False
    return tokens >= max_new_tokens and not looks_terminated(text)


def abstention_score(correct: int, wrong: int, abstained: int) -> Optional[float]:
    """Correct +1, no answer 0, wrong -1, divided by n.



    Exact match is a binary grade, and a binary grade pays a model to guess:

    a wrong answer and a refusal both score zero, so guessing is free upside

    (OpenAI, "Why Language Models Hallucinate", 2025, arXiv 2509.04664). This

    is reported *beside* exact match, never instead of it -- the two answer

    different questions and this repo's headline number is still exact match.



    The range is [-1, +1]: 1.0 is every problem right, 0.0 is either total

    abstention or right-as-often-as-wrong, and negatives mean the model is

    wrong more often than right and would score better by saying nothing.

    """

    n = correct + wrong + abstained
    if n <= 0:
        return None
    return (correct - wrong) / n


def evaluate(

    checkpoint: Path,

    problems: Sequence[Problem],

    max_new_tokens: int = DEFAULT_MAX_NEW_TOKENS,

    settings: Optional[Dict[str, Any]] = None,

    transcript: Optional[List[Dict[str, Any]]] = None,

) -> Dict[str, Any]:
    """Score a checkpoint and return a v2 receipt.



    `max_new_tokens` defaults to 96 from v82 (it was 40 from v65 to v81); see

    `DEFAULT_MAX_NEW_TOKENS` for the measurement behind the number. Pass

    `LEGACY_MAX_NEW_TOKENS` to reproduce a pre-v82 receipt.



    `settings` is copied verbatim into the receipt's `settings` block. Callers

    should put the seed, the task list and the generator fingerprint there --

    `main` does -- so that two receipts can be compared without guessing

    whether they drew the same problems.



    Pass a list as `transcript` to collect every reply in full. The receipt's

    own `examples` block keeps twelve replies clipped to 90 characters, which

    is enough to see what a reply looks like and not enough to check whether

    its working supports its answer -- and after v86 was caught writing

    ``320 / 7 = 60`` on the way to a correct 60, that is a question worth being

    able to ask of a whole run rather than of twelve rows.

    """

    model, tokenizer, payload = load_talk_checkpoint(checkpoint)
    model.eval()

    per_task: Dict[Tuple[str, str], Dict[str, Any]] = {}
    unparsed = 0
    truncated = 0
    examples: List[Dict[str, Any]] = []
    for problem in problems:
        reply = generate_reply(model, tokenizer, problem.prompt, max_new_tokens=max_new_tokens)
        if isinstance(reply, dict):
            text = reply["reply"]
            tokens = reply.get("tokens")
        else:
            text, tokens = str(reply), None
        predicted = extract_answer(text)
        if predicted is None:
            unparsed += 1
        correct = is_correct(predicted, problem.answer)
        cut_off = is_truncated(text, tokens, max_new_tokens)
        if cut_off:
            truncated += 1

        bucket = per_task.setdefault(
            (problem.source, problem.task),
            {"n": 0, "correct": 0, "errors": [], "truncated": 0, "abstained": 0},
        )
        bucket["n"] += 1
        bucket["correct"] += int(correct)
        bucket["truncated"] += int(cut_off)
        # An abstention is a reply that states no answer. A reply the harness
        # cut off states no answer *of the model's choosing*, so it is scored
        # 0 rather than -1: charging the model for the benchmark's own token
        # budget would make the abstention score a measure of the cap.
        bucket["abstained"] += int(predicted is None or (cut_off and not correct))
        # Exact match alone cannot tell "computed approximately" from "produced
        # noise", and the difference is the whole question for a model this
        # size. Relative error separates them: v65 answers 51.5 where the truth
        # is 51.333, which is a different failure from answering "Synonym: glad".
        if predicted is not None:
            bucket["errors"].append(
                abs(predicted - problem.answer) / max(1.0, abs(problem.answer))
            )
        if transcript is not None:
            transcript.append(
                {
                    "source": problem.source,
                    "task": problem.task,
                    "prompt": problem.prompt,
                    "reply": text,
                    "expected": problem.answer,
                    "predicted": predicted,
                    "correct": correct,
                    "tokens": tokens,
                    "truncated": cut_off,
                }
            )
        if len(examples) < 12:
            examples.append(
                {
                    "source": problem.source,
                    "task": problem.task,
                    "prompt": problem.prompt[:90],
                    "reply": text[:90],
                    "expected": problem.answer,
                    "predicted": predicted,
                    "correct": correct,
                    "tokens": tokens,
                    "truncated": cut_off,
                }
            )

    by_source: Dict[str, Dict[str, Any]] = {}
    for (source, task), counts in sorted(per_task.items()):
        entry = by_source.setdefault(
            source, {"n": 0, "correct": 0, "truncated": 0, "abstained": 0, "tasks": {}}
        )
        entry["n"] += counts["n"]
        entry["correct"] += counts["correct"]
        entry["truncated"] += counts["truncated"]
        entry["abstained"] += counts["abstained"]
        errors = sorted(counts["errors"])
        entry.setdefault("errors", []).extend(errors)
        n = counts["n"]
        wrong = n - counts["correct"] - counts["abstained"]
        low, high = wilson_interval(counts["correct"], n)
        untruncated = n - counts["truncated"]
        entry["tasks"][task] = {
            "n": n,
            "correct": counts["correct"],
            "accuracy": round(counts["correct"] / max(1, n), 4),
            "accuracy_95ci": [round(low, 4), round(high, 4)],
            "truncated": counts["truncated"],
            "abstained": counts["abstained"],
            "abstention_score": (
                None
                if (score := abstention_score(counts["correct"], wrong, counts["abstained"]))
                is None
                else round(score, 4)
            ),
            # Truncated replies are the harness's fault, so accuracy with them
            # removed from the denominator is reported too. It is an upper
            # bound in the same way `accuracy` is a lower one: neither is the
            # number, and quoting only the flattering one would be the
            # dishonesty this receipt exists to prevent.
            "accuracy_untruncated": (
                round(counts["correct"] / untruncated, 4) if untruncated > 0 else None
            ),
            "median_relative_error": (
                round(errors[len(errors) // 2], 4) if errors else None
            ),
            "within_10_percent": round(
                sum(1 for e in errors if e <= 0.10) / max(1, n), 4
            ),
        }
    for entry in by_source.values():
        n = entry["n"]
        entry["accuracy"] = round(entry["correct"] / max(1, n), 4)
        low, high = wilson_interval(entry["correct"], n)
        entry["accuracy_95ci"] = [round(low, 4), round(high, 4)]
        wrong = n - entry["correct"] - entry["abstained"]
        score = abstention_score(entry["correct"], wrong, entry["abstained"])
        entry["abstention_score"] = None if score is None else round(score, 4)
        entry["wrong"] = wrong
        untruncated = n - entry["truncated"]
        entry["accuracy_untruncated"] = (
            round(entry["correct"] / untruncated, 4) if untruncated > 0 else None
        )
        errors = sorted(entry.pop("errors", []))
        entry["median_relative_error"] = (
            round(errors[len(errors) // 2], 4) if errors else None
        )
        entry["within_10_percent"] = round(
            sum(1 for e in errors if e <= 0.10) / max(1, n), 4
        )

    seen = by_source.get("seen", {}).get("accuracy")
    novel = by_source.get("novel", {}).get("accuracy")
    receipt_settings: Dict[str, Any] = {
        "max_new_tokens": max_new_tokens,
        "legacy_max_new_tokens": LEGACY_MAX_NEW_TOKENS,
        "tolerance": TOLERANCE,
    }
    receipt_settings.update(settings or {})

    # A checkpoint written from v85 onward records the corpus it trained on. If
    # the seen arm was drawn from a different file the gap is not a memorisation
    # measurement, so say which two files disagreed rather than publishing it.
    trained_on = (payload.get("extra") or {}).get("corpus_jsonl")
    scored_against = receipt_settings.get("corpus")
    corpus_mismatch = None
    if seen is not None and trained_on and scored_against:
        if Path(trained_on).name != Path(scored_against).name:
            corpus_mismatch = (
                f"seen rows came from {Path(scored_against).name}, but this "
                f"checkpoint trained on {Path(trained_on).name}"
            )
            print(f"memorisation gap withheld: {corpus_mismatch}", file=sys.stderr)
    receipt_settings["checkpoint_trained_on"] = trained_on

    return {
        "schema": RECEIPT_SCHEMA,
        "checkpoint": str(checkpoint),
        "dev_loss": (payload.get("extra") or {}).get("best_dev_loss"),
        "problems": len(problems),
        "unparsed_replies": unparsed,
        "truncated_replies": truncated,
        "settings": receipt_settings,
        "by_source": by_source,
        # Only a gap between a model and its OWN training rows means anything.
        # Where the checkpoint records the corpus it trained on and the seen arm
        # was drawn from a different one, the number is withheld and the reason
        # given: a wrong gap reads as a finding, and a missing one reads as a
        # missing measurement, which is what it is.
        "memorisation_gap": (
            round(seen - novel, 4)
            if seen is not None and novel is not None and not corpus_mismatch
            else None
        ),
        "memorisation_gap_withheld": corpus_mismatch,
        "examples": examples,
        "non_claims": NON_CLAIMS,
    }


#: What this benchmark does *not* establish. Rewritten in v82: the previous
#: text still said "five task types with small operands" on what had become a
#: 21-task, 630-problem run, which understated the coverage while overstating
#: nothing -- but a receipt that describes a different experiment than the one
#: it ran is not honest in either direction.
NON_CLAIMS: List[str] = [
    "Accuracy on generated problems is not general problem solving. The 21 "
    "tasks are nine arithmetic templates (two- and three-digit operands, one "
    "or two steps) and twelve single-formula science and mathematics "
    "templates. Every prompt comes from a handful of fixed phrasings, so this "
    "measures a narrow, in-distribution slice and says nothing about "
    "multi-step reasoning, unfamiliar wording, or any task not listed.",
    "A high 'seen' score with a low 'novel' score is recall, not skill; the "
    "gap is the finding, not either number alone.",
    "Answer extraction takes the last number in the reply. A reply that "
    "reasons correctly and then trails off into another number scores wrong, "
    "so this is a lower bound on the model.",
    # This line used to assert "n=30 by default". It is not the default: --novel
    # is 100 over 21 registered tasks, which is 4 or 5 per task, where the widest
    # Wilson interval is +-33 points rather than +-17. A non-claims block that
    # understates its own uncertainty by a factor of two is worse than none, so
    # the number is now computed from the run instead of asserted.
    "A per-task row's uncertainty depends on that row's n, which is "
    "novel_requested divided across the task list -- at the --novel 100 default "
    "over 21 tasks that is 4 or 5 problems per task, where the widest 95% "
    "Wilson interval spans about +-33 points, not the +-17 that n=30 gives. "
    "Every row carries its own interval in the receipt; read those rather than "
    "any rule of thumb. The trainer's MIN_SELECTION_PROBLEMS = 100 is the same "
    "argument applied to checkpoint selection.",
    "Two receipts are comparable only if their seed, task list and "
    "generator_fingerprint all match. Before v82 tasks shared one RNG, so "
    "changing any generator silently changed every later task's problems for "
    "the same seed -- v80's pre- and post-commit n=630 receipts are not "
    "paired, and no delta between them should be quoted.",
    "truncated_replies counts replies that spent the whole token budget "
    "without reaching a terminal 'total <number>'. Those are a limit of this "
    "harness, not a measured failure of the model, and 'accuracy' still counts "
    "them as wrong. Read 'accuracy' as a lower bound and "
    "'accuracy_untruncated' as an upper one.",
    "The abstention score is an alternative grading of the same replies, not "
    "a second measurement. It cannot reward calibrated uncertainty in a model "
    "that was never trained to express any -- these models emit a number or "
    "nothing.",
    # v93. The eleven new tasks enter the default registry, so a run with no
    # --task_set or --tasks now draws over 41 names. The line above about
    # comparability already covers that; this one covers the three tasks
    # whose score is not skill of any kind.
    "The cns_* tasks (cns_type_count, cns_side_count, cns_pair_synapses) "
    "measure recall over the same population of types and type pairs the "
    "corpus was generated from: benchmark and corpus draw from one "
    "`build_connectome_corpus.population()`, and there is no held-out type. "
    "A score there is how much of the male-CNS table the model retained, not "
    "generalisation to a type it never saw, and it says nothing about the "
    "10,000-odd types outside the population. Use --task_set v89 for the "
    "thirty tasks that pair with v87-v91 receipts and --task_set new for the "
    "eleven that have no baseline.",
]


def print_summary(report: Dict[str, Any]) -> None:
    settings = report.get("settings") or {}
    print(f"checkpoint  {report['checkpoint']}")
    print(f"dev loss    {report['dev_loss']}")
    print(
        f"problems    {report['problems']}   unparsed {report['unparsed_replies']}"
        f"   truncated {report.get('truncated_replies', 0)}"
    )
    print(
        f"settings    seed {settings.get('seed')}  max_new_tokens "
        f"{settings.get('max_new_tokens')}  shared_rng {settings.get('shared_rng')}"
    )
    print(f"fingerprint {settings.get('generator_fingerprint')}")
    print()
    header = (
        f"{'source':8s} {'task':18s} {'n':>5s} {'corr':>5s} {'acc':>7s} "
        f"{'95% CI':>17s} {'trunc':>6s} {'abst':>6s}"
    )
    print(header)
    print("-" * len(header))
    for source in ("seen", "novel"):
        entry = report["by_source"].get(source)
        if not entry:
            continue
        for task, counts in sorted(entry["tasks"].items()):
            low, high = counts.get("accuracy_95ci", [0.0, 0.0])
            print(
                f"{source:8s} {task:18s} {counts['n']:5d} {counts['correct']:5d} "
                f"{counts['accuracy']:7.3f} [{low:6.3f},{high:6.3f}] "
                f"{counts.get('truncated', 0):6d} {counts.get('abstained', 0):6d}"
            )
        low, high = entry.get("accuracy_95ci", [0.0, 0.0])
        print(
            f"{source:8s} {'ALL':18s} {entry['n']:5d} {entry['correct']:5d} "
            f"{entry['accuracy']:7.3f} [{low:6.3f},{high:6.3f}] "
            f"{entry.get('truncated', 0):6d} {entry.get('abstained', 0):6d}"
        )
        mre = entry.get("median_relative_error")
        untruncated = entry.get("accuracy_untruncated")
        score = entry.get("abstention_score")
        score_text = "n/a" if score is None else f"{score:+.4f}"
        untruncated_text = "n/a" if untruncated is None else f"{untruncated:.3f}"
        mre_text = "n/a" if mre is None else f"{mre:.3f}"
        print(
            f"{'':8s} {'':18s} abstention score {score_text}"
            f"   acc excl. truncated {untruncated_text}"
            f"   median rel.err {mre_text}"
            f"   within10% {entry['within_10_percent']:.3f}"
        )
        print()
    gap = report.get("memorisation_gap")
    if gap is not None:
        print(f"memorisation gap (seen - novel): {gap:+.4f}")


def build_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    parser.add_argument("--checkpoint", required=True)
    parser.add_argument(
        "--corpus",
        default=None,
        help=(
            "corpus to draw 'seen' problems from. It must be the corpus the "
            "CHECKPOINT WAS TRAINED ON, or the memorisation gap is meaningless. "
            "There is no default: this used to be hard-coded to "
            "datasets/v62/english_math_40k.jsonl, so every run after v62 "
            "silently measured its 'seen' arm against a corpus the model had "
            "never seen. Omit it and the seen arm is skipped and the gap "
            "reported as null, which is honest; pass the wrong one and the "
            "number is worse than missing."
        ),
    )
    parser.add_argument("--novel", type=int, default=100)
    parser.add_argument("--seen", type=int, default=100)
    parser.add_argument("--seed", type=int, default=65)
    parser.add_argument(
        "--max_new_tokens",
        type=int,
        default=DEFAULT_MAX_NEW_TOKENS,
        help=(
            f"generation cap (default {DEFAULT_MAX_NEW_TOKENS}; it was "
            f"{LEGACY_MAX_NEW_TOKENS} through v81, which truncated 11 of 22 "
            "task shapes)"
        ),
    )
    task_group = parser.add_mutually_exclusive_group()
    task_group.add_argument(
        "--tasks",
        default=None,
        help="comma-separated subset of tasks; default is every registered task",
    )
    task_group.add_argument(
        "--task_set",
        choices=sorted(TASK_SETS),
        default=None,
        help=(
            "a named task list instead of --tasks: 'v89' is the thirty tasks "
            "every v87-v91 receipt was scored on, in their order, so the "
            "fingerprint reproduces 3b99a446cd533be9bc5f8ae57d1310b4 and "
            "--novel 630 gives 21 per task; 'new' is the eleven v93 tasks, "
            "which have no baseline; 'all' is both. With neither flag the run "
            "draws over every registered task, which since v93 is 41 names "
            "and a fingerprint that pairs with nothing published."
        ),
    )
    parser.add_argument(
        "--legacy_shared_rng",
        action="store_true",
        help=(
            "draw problems the pre-v82 way, from one RNG shared across tasks. "
            "Only for reproducing a receipt written before v82; the draw is "
            "not stable against adding or reordering a task."
        ),
    )
    parser.add_argument(
        "--combination_in_envelope",
        action="store_true",
        help=(
            "score `combination` with the narrowed generator that the corpus "
            "builder's --combination_in_envelope produces. The twelve omni "
            "tasks are ADAPTED from build_omni_corpus.TASKS, so a corpus built "
            "with that flag must be scored with this one, or the model is "
            "tested on problems its corpus never contained. Without it the "
            "global is always False in this process and the mismatch is "
            "invisible. The nine arithmetic tasks keep their own generators "
            "here and no corpus flag reaches them, which is why only this one "
            "needs mirroring."
        ),
    )
    parser.add_argument(
        "--dump_replies",
        default=None,
        help=(
            "write every reply in full to this path as JSONL. The receipt's "
            "`examples` block keeps twelve replies clipped to 90 characters, "
            "which cannot answer whether a reply's working supports its "
            "answer. `step_audit` reads this file."
        ),
    )
    parser.add_argument("--output", default=None)
    return parser


def main(argv: Optional[Sequence[str]] = None) -> int:
    args = build_parser().parse_args(argv)
    task_set = getattr(args, "task_set", None)
    if task_set:
        tasks: Optional[List[str]] = expand_task_set(task_set)
    else:
        tasks = (
            [name.strip() for name in args.tasks.split(",") if name.strip()]
            if args.tasks
            else None
        )
    unknown = [name for name in (tasks or []) if name not in GENERATORS]
    if unknown:
        raise SystemExit(f"unknown task(s): {', '.join(unknown)}")

    # Set before any problem is drawn AND before the fingerprint is computed, so
    # the digest reflects the generator actually used. The fingerprint does
    # distinguish the two shapes; what it could not do until now is ever see the
    # narrowed one, because nothing in this process could set the global.
    if getattr(args, "combination_in_envelope", False):
        try:
            import build_omni_corpus as _omni

            _omni.COMBINATION_IN_ENVELOPE = True
        except ImportError:  # the omni tasks are simply absent; nothing to narrow
            pass

    problems = list(
        generate_novel(
            args.novel, seed=args.seed, tasks=tasks, shared_rng=args.legacy_shared_rng
        )
    )
    # The seen arm is the memorisation control, and it is only a control if its
    # rows are rows the checkpoint actually trained on. Requiring the path makes
    # a wrong one a decision someone made rather than a default they inherited.
    corpus = Path(args.corpus) if args.corpus else None
    seen_skipped = None
    if not args.seen:
        seen_skipped = "not requested"
    elif corpus is None:
        seen_skipped = (
            "no --corpus given; the seen arm and the memorisation gap need the "
            "checkpoint's own training corpus"
        )
    elif not corpus.is_file():
        seen_skipped = f"corpus not found: {corpus}"
    if seen_skipped is None:
        problems.extend(load_seen(corpus, args.seen, seed=args.seed, tasks=tasks))
    else:
        print(f"seen arm skipped: {seen_skipped}", file=sys.stderr)
    settings = {
        "seed": args.seed,
        "tasks": list(tasks or GENERATORS),
        "task_set": task_set,
        "shared_rng": bool(args.legacy_shared_rng),
        "generator_fingerprint": generator_fingerprint(tasks),
        "novel_requested": args.novel,
        "seen_requested": args.seen,
        "corpus": str(corpus) if corpus else None,
        "seen_skipped": seen_skipped,
    }
    transcript: Optional[List[Dict[str, Any]]] = [] if args.dump_replies else None
    report = evaluate(
        Path(args.checkpoint), problems, args.max_new_tokens, settings=settings,
        transcript=transcript,
    )
    print_summary(report)
    if transcript is not None:
        replies = Path(args.dump_replies)
        replies.parent.mkdir(parents=True, exist_ok=True)
        with replies.open("w", encoding="utf-8") as handle:
            for record in transcript:
                handle.write(json.dumps(record, ensure_ascii=False) + "\n")
        print(f"replies -> {replies} ({len(transcript)} rows)")
    if args.output:
        destination = Path(args.output)
        destination.parent.mkdir(parents=True, exist_ok=True)
        destination.write_text(
            json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
        )
        print(f"\nreceipt -> {destination}")
    return 0


if __name__ == "__main__":  # pragma: no cover
    raise SystemExit(main())