File size: 51,978 Bytes
326fe47 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 | """Generate an executable-Python corpus, verified by running the code.
## Why this exists
v86 scores 0.779 across 21 tasks and has **zero** coding ability, for the
simple reason that there is no code in the corpus. Nothing here has ever asked
the model to read a snippet.
Code is the cheapest thing in this repository to verify exactly: you *run it*.
That is the same contract `build_omni_corpus` has with `nexus_solver`, with
CPython as the checker instead. A generator builds a snippet from a template,
works the answer out step by step in the model's own format, and the
interpreter then executes the snippet it actually shows the model. Disagreement
drops the row.
The check is not circular, and it is worth being precise about why, because the
drop rate is 0.00% and a 0% drop rate usually means a verifier that passes
everything. The generator computes its answer with Python *expressions*
(`s + n * k`); the checker executes the *string it renders into the prompt*.
Those two agree only if the rendered snippet says what the generator thinks it
says. An off-by-one in a `range`, a variable renamed in the template but not in
the derivation, a loop bound written from the wrong parameter -- each of those
is a mismatch between expression and string, and each is caught.
`test_code_corpus.py` mutates a template and asserts the row is dropped, which
is the only way to know the checker is load-bearing rather than decorative.
## What the envelope allowed, and what it ruled out
Every constraint from V81_WHAT_THE_MODEL_CAN_LEARN applies unchanged, and two
of them removed whole task families before a line was written:
**Character-level string work is unlearnable here and is not attempted.**
`s = "banana"; r = s.count("a")` cannot be learned by this model at all: the
tokenizer's `[A-Za-z]+` rule makes `banana` a *single* token, so the letters
are not visible to the model in any representation. It would be a task whose
answer is a function of information the input does not carry. Counting *list
elements* (`len(words)`) is fine and appears in `code_index`, because each word
is its own token.
**Snippets are one statement per line and never nest more than one level.**
Tokens carry their leading whitespace, so `"\\n x"` and `" x"` are different
symbols and a deeply indented corpus builds a parallel vocabulary of
indent-prefixed identifiers. The generators here emit exactly six line-initial
forms in total (`\\nfor`, `\\n for`, `\\nwhile`, `\\nif`, `\\nelse`, `\\nr`),
all of them high frequency; `test_code_corpus.py` pins that set so a future
task cannot quietly widen it.
**Every intermediate stays two-digit.** Loop counts, steps and list values are
bounded so no running total passes 99 and no product leaves
two-digit-by-one-digit. `code_nested_loop` caps both ranges at 9 for a reason
that is easy to miss: `decompose_product` writes `10 x 7 = 70, 2 x 7 = 14` and
deliberately does *not* write the total, because that is the form that scored
0.93. That is fine when the product is the answer, and wrong when it is an
operand for a later step -- the model would have to carry an unwritten number.
So the outer range is single-digit and the iteration count is one written step.
## Repetition, and where this family is weakest
v74's multiplication task holds 712 distinct problems repeated ~56x and scored
0.93; the same operation with 24,000 unique problems scored 0.03. Distinct
capacity is therefore a design parameter, not an accident, and
`distinct_capacity()` reports it per task. At `--per_task 20000` the family
runs from 28x repetition (`code_divmod`) down to 3.9x (`code_index`), and
`code_index` is the one to narrow first if the family underperforms: its list
carries a distractor element that multiplies the prompt space without adding a
step to the procedure.
"""
from __future__ import annotations
import argparse
import ast
import json
import math
import random
import sys
import threading
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple
SOURCE_DIR = Path(__file__).resolve().parent
if str(SOURCE_DIR) not in sys.path:
sys.path.insert(0, str(SOURCE_DIR))
# Imported rather than re-implemented on purpose. `decompose_product` is the
# exact working format that scored 0.93 on v74's multiplication task, down to
# whether the addition of the partial products is written out; a second copy of
# it here would drift from that measurement the first time either file changed.
from build_omni_corpus import ( # noqa: E402
DEFAULT_SEQUENCE_LENGTH,
decompose_product,
extract_answer,
token_budget_report,
)
#: Wall clock a single snippet gets before it is abandoned.
#:
#: Every snippet these generators emit is bounded by construction -- the
#: longest runs 270 iterations -- so this never fires in practice. It exists
#: because "the templates are all safe" is an assumption, and an assumption
#: that hangs the build for good is worse than one that drops a row.
DEFAULT_TIMEOUT_SECONDS = 2.0
#: Refuse a snippet longer than this before parsing it. Generated snippets top
#: out near 90 characters.
MAX_SOURCE_CHARS = 400
#: Consecutive failures to produce a new prompt before a task is judged
#: exhausted, matching `build_omni_corpus.EXHAUSTION_MISSES`.
EXHAUSTION_MISSES = 5000
# -- the checker ------------------------------------------------------------
#
# Running generated code is the whole point, so the sandbox is written to be
# read: an allowlist of syntax, an allowlist of names, no builtins beyond six
# pure functions, and a wall clock. Nothing here ever executes a string that
# came from outside this module -- `verify` runs `problem.canonical`, and the
# only thing that writes a `canonical` is a generator in `TASKS`.
class SnippetRejected(Exception):
"""The snippet uses syntax the corpus does not permit."""
#: Node types a corpus snippet may contain. Matched by class *name* rather than
#: by class object so this survives the AST changes between Python versions
#: (`ast.Index` disappeared from parse output in 3.9).
ALLOWED_NODES = frozenset({
"Module", "Expr", "Assign", "AugAssign", "For", "While", "If", "Break",
"Continue", "Pass", "Name", "Store", "Load", "Constant", "BinOp", "UnaryOp",
"USub", "UAdd", "Add", "Sub", "Mult", "FloorDiv", "Mod", "Compare", "Eq",
"NotEq", "Lt", "LtE", "Gt", "GtE", "List", "Subscript", "Index", "Call",
# `IfExp` is the conditional expression `a if cond else b`, used by
# `code_conditional` because the three-line block form left a 34-token
# prompt where the model's 128-token context allows 32 alongside a
# full-length generation.
#
# Worth recording how this was found: adding the ternary without this entry
# sent the build's drop rate from 0.000 to 0.111 -- exactly one task in
# nine -- because every `code_conditional` row failed validation and was
# discarded. The verifier caught a change its author had not thought
# through, which is the whole argument for validating snippets this module
# generates itself.
"IfExp",
# `Attribute` is admitted for v93's `code_list_count` (`nums.count(v)`),
# and only in the shape `validate_snippet` checks below: a method named in
# `SAFE_METHODS`, called on a plain name. An attribute that is not a call
# on an allowlisted method -- `().__class__`, `x.__dict__` -- is still
# refused, so the sandbox-escape surface this node type usually opens is
# not opened here.
"Attribute",
})
#: The only callables a snippet may name. All six are pure, total on the
#: arguments these templates pass, and cannot reach the filesystem, the network
#: or the interpreter's own state.
SAFE_BUILTINS: Dict[str, Any] = {
"abs": abs,
"len": len,
"max": max,
"min": min,
"range": range,
"sum": sum,
}
#: The only methods a snippet may call, by attribute name. `list.count` is
#: pure and total; it is the one v93 needs.
SAFE_METHODS = frozenset({"count"})
def validate_snippet(source: str) -> ast.Module:
"""Parse `source` and refuse anything outside the allowlist.
Belt and braces: these snippets are built from templates in this file, so
in normal operation nothing can reach here that is not already safe. The
validator is what makes that a *checked* claim instead of a stated one --
if someone later adds a template with an attribute access or an import, the
build fails loudly at that template rather than silently gaining a
capability.
"""
if len(source) > MAX_SOURCE_CHARS:
raise SnippetRejected(f"snippet is {len(source)} chars, limit {MAX_SOURCE_CHARS}")
try:
tree = ast.parse(source, filename="<code-corpus>", mode="exec")
except SyntaxError as exc: # a template that does not parse is a bug, not a row
raise SnippetRejected(f"syntax error: {exc}") from exc
for node in ast.walk(tree):
name = type(node).__name__
if name not in ALLOWED_NODES:
raise SnippetRejected(f"disallowed syntax: {name}")
if isinstance(node, ast.Name):
# `_`-prefixed names are how sandbox escapes are written
# (`().__class__`, `__builtins__`); no template needs one.
if node.id.startswith("_"):
raise SnippetRejected(f"disallowed name: {node.id}")
if isinstance(node, ast.Attribute):
# Only `<name>.<safe method>`; anything else is the escape shape.
if node.attr not in SAFE_METHODS or not isinstance(node.value, ast.Name):
raise SnippetRejected(f"disallowed attribute: {node.attr}")
if isinstance(node, ast.Call):
if isinstance(node.func, ast.Attribute):
pass # checked as an Attribute above
elif not isinstance(node.func, ast.Name):
raise SnippetRejected("only direct calls to allowed builtins")
elif node.func.id not in SAFE_BUILTINS:
raise SnippetRejected(f"disallowed call: {node.func.id}")
if node.keywords:
raise SnippetRejected("keyword arguments are not permitted")
return tree
@dataclass
class SnippetResult:
"""What happened when the interpreter ran a snippet."""
ok: bool
value: Optional[float] = None
reason: str = ""
def run_snippet(source: str, target: str,
timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS) -> SnippetResult:
"""Execute `source` and read `target` out of the resulting namespace.
Restricted three ways: the AST allowlist above, a `__builtins__` mapping
holding six pure functions, and a wall clock. The snippet runs on a daemon
thread so a hang cannot wedge the build -- a thread that overruns is
abandoned rather than joined, which is the honest thing to do given CPython
offers no way to kill one, and is safe here because the thread can only
touch its own namespace.
A non-numeric or non-integral result is a failure, not a row: the whole
corpus answers with the last integer in the reply.
"""
try:
validate_snippet(source)
compiled = compile(source, "<code-corpus>", "exec")
except SnippetRejected as exc:
return SnippetResult(False, None, str(exc))
except (SyntaxError, ValueError) as exc:
return SnippetResult(False, None, f"compile failed: {exc}")
outcome: Dict[str, Any] = {}
def _run() -> None:
namespace: Dict[str, Any] = {"__builtins__": dict(SAFE_BUILTINS)}
try:
exec(compiled, namespace) # noqa: S102 - the point of the module
except BaseException as exc: # noqa: BLE001 - any failure drops the row
outcome["error"] = f"{type(exc).__name__}: {exc}"
return
if target not in namespace:
outcome["error"] = f"snippet never assigned {target}"
return
outcome["value"] = namespace[target]
thread = threading.Thread(target=_run, daemon=True)
thread.start()
thread.join(timeout_seconds)
if thread.is_alive():
return SnippetResult(False, None, f"timed out after {timeout_seconds}s")
if "error" in outcome:
return SnippetResult(False, None, outcome["error"])
value = outcome.get("value")
if isinstance(value, bool) or not isinstance(value, int):
# bool is an int subclass; `r = True` would silently score as 1.
return SnippetResult(False, None, f"{target} is {type(value).__name__}, not int")
return SnippetResult(True, float(value), "")
@dataclass
class CodeProblem:
"""One snippet, its worked derivation, and the value the interpreter got.
`canonical` is the executable source itself, which is the exact parallel to
`OmniProblem.canonical` holding the solver's own query: it is the form the
checker consumes, and it is decoupled from the phrasing the model sees.
"""
task: str
domain: str
prompt: str
response: str
answer: float
unit: str
canonical: str
target: str
params: Dict[str, float] = field(default_factory=dict)
def to_row(self, keep_canonical: bool = False) -> Dict[str, str]:
row = {
"user": self.prompt,
"assistant": self.response,
"domain": self.domain,
"task": self.task,
}
if keep_canonical:
# Carrying the source makes a shipped row re-runnable months later
# without regenerating it, which is what the omni corpus wanted
# `keep_canonical` for and could only get for 41.9% of its rows.
# Here it is 100%: the checker parses exactly what it emitted.
row["canonical"] = self.canonical
row["target"] = self.target
return row
DOMAIN = "programming"
def _pick(rng: random.Random, templates: Sequence[str], **values: str) -> str:
"""Choose one phrasing. Values are pre-rendered strings, never numbers.
v74 carried one template per task, scored 0.894 on its own benchmark and
answered 0 of 5 naturally-typed questions. Every task below carries five.
"""
return rng.choice(list(templates)).format(**values)
#: The five ways a trace question gets asked. Two put the code first and three
#: put it last, because a model that only ever saw `<instruction><code>` learns
#: the position as part of the task.
#: Every template wraps the snippet on ONE side only, never both.
#:
#: The budget here is not the 128-token turn -- it is the model's context. A
#: prompt must leave room for a full-length generation, because
#: `max_position_embeddings` is 128 and a reply that runs to the cap would ask
#: for a position the rotary table does not have. `test_eval_v82` asserts
#: `longest_prompt + DEFAULT_MAX_NEW_TOKENS <= 128`, so at a 96-token cap the
#: longest prompt must fit in 32.
#:
#: The dropped template was "Run this snippet in your head. {code} Give the
#: value of {var}." -- a lead-in AND a trailing instruction around a two-line
#: snippet, which reached 39 tokens on `code_conditional`. One wrapper says the
#: same thing. Phrasing variety matters (v74 had one template per task and
#: scored 0 of 5 on naturally-typed questions), so the count is kept at five by
#: replacing it rather than deleting it.
TRACE_TEMPLATES = (
"Trace this Python and give {var}. {code}",
"{code} What is {var} after this runs?",
"What does {var} hold at the end? {code}",
"{code} Give the value of {var}.",
"Python: {code} What is {var}?",
)
#: The same question, worded tightly, for tasks whose snippet is already long.
#:
#: Measured, and the result was not what it looked like. `code_nested_loop`
#: reached 35 tokens against the 32 the context allows beside a full-length
#: generation, and the obvious suspects cost nothing at all: dropping the
#: four-space indent to one space saved 0 tokens, and fixing the start value at
#: 0 saved 0. The tokenizer normalises runs of whitespace, and every digit is
#: already its own token under `--digit_tokens`.
#:
#: The entire overhead was the wrapper. "What does c hold at the end?" costs
#: four tokens more than "What is c?", which is the whole overrun. So the
#: snippet is left exactly as it is -- it is the thing being taught -- and only
#: the words around it are trimmed.
#:
#: Kept at five entries because phrasing variety is what stops the model
#: learning a template instead of a task: v74 had one phrasing per task and
#: answered 0 of 5 naturally-typed questions.
COMPACT_TRACE_TEMPLATES = (
"{code} What is {var}?",
"{code} Give {var}.",
"Python: {code} {var}?",
"Trace this. {code} {var}?",
"{code} Final {var}?",
)
# -- task generators --------------------------------------------------------
def _loop_add(rng: random.Random) -> CodeProblem:
"""`x = s` then `n` additions of `k`. The plainest accumulator there is.
`n` stops at 7 and `s + n * k` at 72 for two separate reasons: the running
total must stay two-digit (rule 1), and each written step costs about eight
tokens, so an eighth step pushed the longest turns past the 128-token block
where turn-aligned packing drops them outright.
"""
start = rng.randint(0, 9)
steps = rng.randint(3, 7)
increment = rng.randint(2, 9)
source = f"x = {start}\nfor i in range({steps}): x = x + {increment}"
prompt = _pick(rng, TRACE_TEMPLATES, code=source, var="x")
value = start
pieces = [f"the loop runs {steps} times", f"x starts at {start}"]
for _ in range(steps):
pieces.append(f"{value} + {increment} = {value + increment}")
value += increment
pieces.append(f"x is {value}")
response = ", ".join(pieces) + f", total {value}"
return CodeProblem("code_loop_add", DOMAIN, prompt, response, float(value), "",
source, "x",
{"start": start, "steps": steps, "increment": increment})
def _loop_subtract(rng: random.Random) -> CodeProblem:
"""The same accumulator downwards, so the model does not learn `for` = `+`.
The start is redrawn until the final value is non-negative: a corpus of
negative results would teach the sign as a property of the loop rather than
of the arithmetic, and the benchmark's `-?\\d+` extraction makes a stray
minus a wrong answer rather than a near miss.
"""
steps = rng.randint(3, 7)
decrement = rng.randint(2, 9)
span = steps * decrement
start = rng.randint(span, 99)
source = f"x = {start}\nfor i in range({steps}): x = x - {decrement}"
prompt = _pick(rng, TRACE_TEMPLATES, code=source, var="x")
value = start
pieces = [f"the loop runs {steps} times", f"x starts at {start}"]
for _ in range(steps):
pieces.append(f"{value} - {decrement} = {value - decrement}")
value -= decrement
pieces.append(f"x is {value}")
response = ", ".join(pieces) + f", total {value}"
return CodeProblem("code_loop_subtract", DOMAIN, prompt, response, float(value), "",
source, "x",
{"start": start, "steps": steps, "decrement": decrement})
#: Values a literal list is built from. Three drawn without replacement out of
#: fifteen gives 2,730 distinct lists -- 7.3x repetition at `--per_task 20000`,
#: the same order as v74's winning 56x rather than the omni build's 1.7x.
LIST_POOL = range(2, 17)
#: `code_index` draws from a narrower pool because its prompt space is already
#: multiplied by the choice of index pair; see the module docstring.
INDEX_POOL = range(2, 14)
def _list_sum(rng: random.Random) -> CodeProblem:
"""`sum` over a three-item literal, with **the operands written out**.
This is the task the omni corpus got wrong. v86's `average` writes
`sum: 48 then 137 then 170` -- a running total with no operands -- and its
individual additions are correct 1.5% of the time while its division is
correct 69%. Writing `48 + 89 = 137` instead is the entire difference, and
it is why a list-sum task earns its place next to an arithmetic one.
"""
values = rng.sample(list(LIST_POOL), 3)
literal = ", ".join(str(v) for v in values)
source = f"nums = [{literal}]\nr = sum(nums)"
prompt = _pick(rng, COMPACT_TRACE_TEMPLATES, code=source, var="r")
running = values[0]
pieces = ["sum adds the items in order", f"r starts at {running}"]
for value in values[1:]:
pieces.append(f"{running} + {value} = {running + value}")
running += value
pieces.append(f"r is {running}")
response = ", ".join(pieces) + f", total {running}"
return CodeProblem("code_list_sum", DOMAIN, prompt, response, float(running), "",
source, "r", {f"v{i}": v for i, v in enumerate(values)})
def _list_extreme(rng: random.Random) -> CodeProblem:
"""`max` or `min` over the same literal, as a chain of pairwise compares.
Both directions in one task rather than two, so the model has to read which
builtin was called instead of learning "a list question means the biggest
number". The canonical source records which was asked; the phrasing does
not mention it at all.
"""
values = rng.sample(list(LIST_POOL), 3)
literal = ", ".join(str(v) for v in values)
wants_max = rng.random() < 0.5
builtin = "max" if wants_max else "min"
word = "larger" if wants_max else "smaller"
source = f"nums = [{literal}]\nr = {builtin}(nums)"
prompt = _pick(rng, COMPACT_TRACE_TEMPLATES, code=source, var="r")
best = values[0]
pieces = [f"{builtin} keeps the {word} of each pair"]
for value in values[1:]:
chosen = max(best, value) if wants_max else min(best, value)
pieces.append(f"compare {best} and {value}, the {word} is {chosen}")
best = chosen
pieces.append(f"r is {best}")
response = ", ".join(pieces) + f", total {best}"
return CodeProblem("code_list_extreme", DOMAIN, prompt, response, float(best), "",
source, "r",
{f"v{i}": v for i, v in enumerate(values)})
def _index(rng: random.Random) -> CodeProblem:
"""Two positions read out of a list and added.
The derivation states `count positions from zero` every time. Zero-based
indexing is the one fact in this family that is a convention rather than a
computation, and a convention the derivation never names is a convention
the model has to infer from the numbers.
"""
values = rng.sample(list(INDEX_POOL), 3)
literal = ", ".join(str(v) for v in values)
first, second = sorted(rng.sample(range(3), 2))
source = f"nums = [{literal}]\nr = nums[{first}] + nums[{second}]"
prompt = _pick(rng, COMPACT_TRACE_TEMPLATES, code=source, var="r")
left, right = values[first], values[second]
answer = left + right
response = (f"count positions from zero, "
f"nums [ {first} ] is {left}, nums [ {second} ] is {right}, "
f"{left} + {right} = {answer}, r is {answer}, total {answer}")
return CodeProblem("code_index", DOMAIN, prompt, response, float(answer), "",
source, "r",
{f"v{i}": v for i, v in enumerate(values)},)
def _conditional(rng: random.Random) -> CodeProblem:
"""`if a > b` picking which subtraction to run.
Both operands stay two-digit and the branch is the only decision, so a
wrong answer localises: taking the wrong branch gives a negative number,
which is visibly different from an arithmetic slip.
"""
a = rng.randint(11, 60)
b = rng.randint(11, 60)
while b == a:
b = rng.randint(11, 60)
# A conditional *expression*, not a three-line if/else block.
#
# The constraint this hit is the model's CONTEXT, not the turn budget:
# `max_position_embeddings` is 128, so a prompt must leave room for a
# full-length generation, or a reply running to the 96-token cap asks the
# rotary table for a position it does not have. The block form reached 34
# tokens against the 32 that leaves, and joining its lines with spaces is
# not valid Python -- the interpreter rejects it, which is the executor
# doing its job.
#
# `--digit_tokens` is why this is tight at all: every digit is its own
# token, so `a = 59; b = 50` spends six tokens on the numbers alone.
#
# The ternary keeps what the task teaches -- both branches are visible, and
# taking the wrong one still yields a negative number, so a branch error
# stays distinguishable from an arithmetic slip.
source = f"a = {a}; b = {b}\nr = a - b if a > b else b - a"
prompt = _pick(rng, COMPACT_TRACE_TEMPLATES, code=source, var="r")
if a > b:
branch = (f"{a} is greater than {b}, so the if branch runs, r = a - b")
answer = a - b
step = f"{a} - {b} = {answer}"
else:
branch = (f"{a} is not greater than {b}, so the else branch runs, r = b - a")
answer = b - a
step = f"{b} - {a} = {answer}"
response = (f"compare a and b, {branch}, {step}, "
f"r is {answer}, total {answer}")
return CodeProblem("code_conditional", DOMAIN, prompt, response, float(answer), "",
source, "r", {"a": a, "b": b})
def _divmod_task(rng: random.Random) -> CodeProblem:
"""`a // b` or `a % b`, worked as one multiplication and one subtraction.
`a` is capped at `10 * b - 1` so the quotient is always a single digit.
That is not tidiness: `decompose_product` writes two partial products and
stops, by design, so a two-digit quotient would put the product itself
(`40 x 2 = 80, 9 x 2 = 18` -> 98) nowhere in the text while the next step
subtracts it. Every number this derivation uses is a number it wrote.
"""
b = rng.randint(3, 9)
a = rng.randint(10, min(99, 10 * b - 1))
quotient, remainder = divmod(a, b)
floor_division = rng.random() < 0.5
operator = "//" if floor_division else "%"
source = f"a = {a}; b = {b}; r = a {operator} b"
prompt = _pick(rng, COMPACT_TRACE_TEMPLATES, code=source, var="r")
working = (f"{a} {operator} {b}, count whole {b} s inside {a}, "
f"{decompose_product(quotient, b)}, "
f"{a} - {quotient * b} = {remainder}, "
f"{remainder} is less than {b}")
if floor_division:
answer = quotient
response = (f"{working}, the quotient is {quotient}, "
f"r is {quotient}, total {quotient}")
else:
answer = remainder
response = (f"{working}, the remainder is {remainder}, "
f"r is {remainder}, total {remainder}")
return CodeProblem("code_divmod", DOMAIN, prompt, response, float(answer), "",
source, "r", {"a": a, "b": b, "q": quotient})
def _nested_loop(rng: random.Random) -> CodeProblem:
"""How many times a doubly-nested body runs, plus a starting count.
Both ranges are single-digit so `decompose_product(outer, inner)` is one
written step and the iteration count is a number on the page before the
final addition consumes it. `start + outer * inner` is capped at 99 by
redrawing, keeping that addition a two-digit single step.
"""
outer = rng.randint(2, 9)
inner = rng.randint(2, 9)
start = rng.randint(0, 9)
while start + outer * inner > 99:
outer = rng.randint(2, 9)
inner = rng.randint(2, 9)
iterations = outer * inner
answer = start + iterations
source = (f"c = {start}\n"
f"for i in range({outer}):\n"
f" for j in range({inner}): c = c + 1")
prompt = _pick(rng, COMPACT_TRACE_TEMPLATES, code=source, var="c")
response = (f"the inner loop runs {inner} times for each outer step, "
f"the outer loop runs {outer} times, "
f"{decompose_product(outer, inner)} iterations, "
f"c starts at {start}, {start} + {iterations} = {answer}, "
f"c is {answer}, total {answer}")
return CodeProblem("code_nested_loop", DOMAIN, prompt, response, float(answer), "",
source, "c",
{"outer": outer, "inner": inner, "start": start})
def _while_accumulate(rng: random.Random) -> CodeProblem:
"""A `while` that stops the first time the total passes a threshold.
Written as `while` rather than `for ... if ...: break` for a tokenizer
reason, not a semantic one: the `break` form needs an indented two-statement
body, which adds `\\n x` and `\\n if` to the vocabulary as symbols
unrelated to ` x` and ` if`. The `while` form is one line and teaches the
same thing -- accumulate, test, exit.
The comparison is restated after *every* step. Emitting only the final one
would leave the stopping rule implicit, and rule 4 of the envelope is that
a scratchpad helps only where it decomposes the operation being performed.
"""
# Three or four iterations, not three to five.
#
# Each iteration emits two pieces -- the addition and the restated
# comparison -- so the reply grows about twice as fast per step as the other
# code tasks. Measured over 1,080 rows with the v86 tokenizer, five
# iterations reached a 129-token turn against the 128-token budget, and
# `_build_turn_aligned_tensors` drops a turn longer than the block *without
# reporting it*. Five of 1,080 rows were being silently discarded.
#
# Capping iterations rather than dropping the restated comparison is
# deliberate: the comparison is what makes the stopping rule explicit, and
# rule 4 of the envelope is that a scratchpad helps only where it decomposes
# the operation. Losing a little capacity is the cheaper trade.
step = rng.randint(5, 19)
iterations = rng.randint(3, 4)
threshold = rng.randint((iterations - 1) * step, iterations * step - 1)
source = f"x = 0\nwhile x <= {threshold}: x = x + {step}"
prompt = _pick(rng, TRACE_TEMPLATES, code=source, var="x")
value = 0
pieces = ["x starts at 0"]
for _ in range(iterations):
nxt = value + step
pieces.append(f"{value} + {step} = {nxt}")
value = nxt
if value <= threshold:
pieces.append(f"{value} is not past {threshold}")
else:
pieces.append(f"{value} is past {threshold}, the loop stops")
pieces.append(f"x is {value}")
response = ", ".join(pieces) + f", total {value}"
return CodeProblem("code_while_accumulate", DOMAIN, prompt, response, float(value), "",
source, "x",
{"step": step, "threshold": threshold, "iterations": iterations})
# -- v93 generators ----------------------------------------------------------
#
# Three more tasks (docs/V93_NEUROGENESIS_TWO_HEMISPHERES.md, D7), verified by
# execution like the nine above. They live in `V93_TASKS` rather than `TASKS`
# for the reason `build_omni_corpus` gives: `build` draws every task from one
# RNG in `chosen` order, so a default build stays byte-identical only if the
# default list does; and the benchmark registry built from `TASKS` carries the
# fingerprint every published receipt is paired on.
#: The most terms a `code_range_sum` trace writes out. Measured with the
#: training segmentation against the benchmark's 96-token generation cap
#: (`eval_problem_solving.DEFAULT_MAX_NEW_TOKENS`): the longest ten-term
#: trace, `range(3, 13)`, is 95 reply tokens including EOS, and an eleven-term
#: one is 97 or more. One step per addition is the rule this family rests on,
#: so the term count is bounded rather than the trace compressed. The map's
#: suggestion of `range(13)` alone would have needed thirteen terms and 115
#: tokens, a reply the benchmark could never finish.
RANGE_SUM_MAX_TERMS = 10
#: `range` stops below this. Sums stay two-digit: `sum(range(1, 13)) = 78`.
RANGE_SUM_MAX_STOP = 13
def _range_sum(rng: random.Random) -> CodeProblem:
"""`r = sum(range(n))` or `r = sum(range(a, n))`, every addition written.
The one-argument form alone has eight legal instances under the term
bound, which is a lookup table rather than a procedure. The two-argument
form is what makes it a procedure: the trace has to read where `range`
starts as well as where it stops, and the sum then depends on both. A
start of 0 is written as `range(n)`, so both spellings the model will meet
are taught, and the trace names the first and last values so nothing about
the half-open interval is left to be inferred.
"""
stop = rng.randint(3, RANGE_SUM_MAX_STOP)
# The start must leave at least two terms and at most the trace budget.
start = rng.randint(max(0, stop - RANGE_SUM_MAX_TERMS), stop - 2)
call = f"range({stop})" if start == 0 else f"range({start}, {stop})"
source = f"r = sum({call})"
prompt = _pick(rng, TRACE_TEMPLATES, code=source, var="r")
terms = list(range(start, stop))
running = terms[0]
pieces = [f"range counts from {start} up to {stop - 1}", f"r starts at {running}"]
for term in terms[1:]:
pieces.append(f"{running} + {term} = {running + term}")
running += term
pieces.append(f"r is {running}")
response = ", ".join(pieces) + f", total {running}"
return CodeProblem("code_range_sum", DOMAIN, prompt, response, float(running), "",
source, "r", {"start": start, "stop": stop})
#: Values a `code_list_count` list is built from. Six values in four slots keep
#: the space near the family's other list tasks (4,020 distinct problems,
#: 5.0x repetition at 20,000 rows) rather than the 96,040 a five-slot list
#: over eight values would give.
COUNT_POOL = range(2, 8)
COUNT_LENGTH = 4
def _list_count(rng: random.Random) -> CodeProblem:
"""`r = nums.count(v)` over a four-item literal, one verdict per item.
The counted value appears one, two or three times; the other slots hold
values that differ from it, so every `matches` / `does not` in the trace is
decidable from the two numbers on the line. `.count` is the first method
call in this family, admitted through `SAFE_METHODS` rather than by
widening the call rule in general.
"""
value = rng.choice(list(COUNT_POOL))
hits = rng.randint(1, 3)
positions = set(rng.sample(range(COUNT_LENGTH), hits))
others = [v for v in COUNT_POOL if v != value]
items = [value if i in positions else rng.choice(others) for i in range(COUNT_LENGTH)]
literal = ", ".join(str(v) for v in items)
source = f"nums = [{literal}]\nr = nums.count({value})"
prompt = _pick(rng, COMPACT_TRACE_TEMPLATES, code=source, var="r")
pieces = [f"count checks each item against {value}"]
for item in items:
pieces.append(f"{item} matches" if item == value else f"{item} does not")
pieces.append(f"r is {hits}")
response = ", ".join(pieces) + f", total {hits}"
return CodeProblem("code_list_count", DOMAIN, prompt, response, float(hits), "",
source, "r", {"value": value, "hits": hits,
**{f"v{i}": v for i, v in enumerate(items)}})
def _neg_index(rng: random.Random) -> CodeProblem:
"""`r = nums[-k]`: one position read from the end of a three-item list.
`code_index` states `count positions from zero` on every row because
zero-based indexing is a convention the numbers alone do not reveal. The
negative form is the same kind of fact from the other end, and the trace
walks back one position at a time so `nums [ -2 ]` is reached from
`nums [ -1 ]` rather than asserted.
"""
values = rng.sample(list(INDEX_POOL), 3)
literal = ", ".join(str(v) for v in values)
k = rng.randint(1, 3)
source = f"nums = [{literal}]\nr = nums[-{k}]"
prompt = _pick(rng, COMPACT_TRACE_TEMPLATES, code=source, var="r")
pieces = ["negative positions count from the end"]
for back in range(1, k + 1):
pieces.append(f"nums [ -{back} ] is {values[-back]}")
answer = values[-k]
pieces.append(f"r is {answer}")
response = ", ".join(pieces) + f", total {answer}"
return CodeProblem("code_neg_index", DOMAIN, prompt, response, float(answer), "",
source, "r", {"k": k, **{f"v{i}": v for i, v in enumerate(values)}})
#: Every v89 generator, by task name. The `code_` prefix keeps them
#: distinguishable in a receipt and guarantees they never shadow an arithmetic
#: or omni task when `eval_problem_solving` merges the three families into one
#: dict. Frozen: see the note above `RANGE_SUM_MAX_TERMS`.
TASKS: Dict[str, Callable[[random.Random], CodeProblem]] = {
"code_loop_add": _loop_add,
"code_loop_subtract": _loop_subtract,
"code_list_sum": _list_sum,
"code_list_extreme": _list_extreme,
"code_index": _index,
"code_conditional": _conditional,
"code_divmod": _divmod_task,
"code_nested_loop": _nested_loop,
"code_while_accumulate": _while_accumulate,
}
#: The tasks added for v93. Built only when named with `--task`.
V93_TASKS: Dict[str, Callable[[random.Random], CodeProblem]] = {
"code_range_sum": _range_sum,
"code_list_count": _list_count,
"code_neg_index": _neg_index,
}
#: Every generator this module can build, for lookup by name.
ALL_TASKS: Dict[str, Callable[[random.Random], CodeProblem]] = {**TASKS, **V93_TASKS}
#: How many distinct problems each generator can produce, counted from the
#: parameter ranges in its body rather than sampled.
#:
#: This is reported because v74's multiplication task scored 0.93 with 712
#: distinct problems and 0.03 with 24,000 of the same operation. A task whose
#: capacity drifts upward silently is a task whose score will fall for a reason
#: nobody attributes to the corpus. `test_code_corpus.py` samples each
#: generator and asserts the number below is not an undercount.
def distinct_capacity() -> Dict[str, int]:
pool = len(LIST_POOL)
index_pool = len(INDEX_POOL)
return {
# start(10) x steps(5) x increment(8)
"code_loop_add": 10 * 5 * 8,
# sum over steps(3..7), decrement(2..9) of the legal starts 99-span+1
"code_loop_subtract": sum(
max(0, 99 - steps * dec + 1)
for steps in range(3, 8) for dec in range(2, 10)
),
# ordered 3-subsets of the pool
"code_list_sum": pool * (pool - 1) * (pool - 2),
# the same, doubled by max/min
"code_list_extreme": 2 * pool * (pool - 1) * (pool - 2),
# ordered 3-subsets of the narrower pool x 3 index pairs
"code_index": index_pool * (index_pool - 1) * (index_pool - 2) * 3,
# a(50) x b(50) minus the diagonal
"code_conditional": 50 * 50 - 50,
# sum over b of the legal a range, doubled by // and %
"code_divmod": 2 * sum(min(99, 10 * b - 1) - 10 + 1 for b in range(3, 10)),
# (outer, inner) pairs with a legal start
"code_nested_loop": sum(
1 for outer in range(2, 10) for inner in range(2, 10)
for start in range(10) if start + outer * inner <= 99
),
# step(15) x iterations(3) x the `step` thresholds each admits
"code_while_accumulate": sum(
step for step in range(5, 20) for _ in range(3)
),
# -- v93 --
# (start, stop) pairs with 2..RANGE_SUM_MAX_TERMS terms; 71 today
"code_range_sum": sum(
(stop - 2) - max(0, stop - RANGE_SUM_MAX_TERMS) + 1
for stop in range(3, RANGE_SUM_MAX_STOP + 1)
),
# value(6) x sum over hits of C(4, hits) x 5^(4 - hits)
"code_list_count": len(COUNT_POOL) * sum(
math.comb(COUNT_LENGTH, hits) * (len(COUNT_POOL) - 1) ** (COUNT_LENGTH - hits)
for hits in range(1, 4)
),
# ordered 3-subsets of the narrower pool x 3 offsets
"code_neg_index": index_pool * (index_pool - 1) * (index_pool - 2) * 3,
}
# -- verification -----------------------------------------------------------
def verify(problem: CodeProblem,
timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS) -> SnippetResult:
"""Run the snippet the prompt shows and compare it to the worked answer.
Returns the full `SnippetResult` rather than a bool so the build can report
*why* rows were dropped. A verifier whose failures are an unlabelled count
cannot tell a broken template from an exhausted parameter range.
"""
result = run_snippet(problem.canonical, problem.target, timeout_seconds)
if not result.ok:
return result
if result.value != problem.answer:
return SnippetResult(
False, result.value,
f"interpreter got {result.value}, derivation says {problem.answer}",
)
return result
#: The words a snippet line may begin with. Pinned because tokens carry their
#: leading whitespace: `"\n for"` is a different symbol from `" for"`, so
#: every new line-initial form is a new vocabulary entry the model must learn
#: separately. Six is cheap; an indented multi-statement body would not be.
LINE_INITIAL_FORMS = frozenset({
"for", " for", "while", "if", "else", "r",
})
def line_initial_forms(rows: Sequence[Dict[str, str]]) -> Dict[str, int]:
"""Count the distinct indent-prefixed line starts a corpus contains."""
counts: Dict[str, int] = {}
for row in rows:
for line in row["user"].split("\n")[1:]:
word = line.split(" ")[0] if line.strip() else ""
# Keep the indentation, because that is what the tokenizer keeps.
leading = len(line) - len(line.lstrip(" "))
form = " " * leading + word.strip()
if form:
counts[form] = counts.get(form, 0) + 1
return dict(sorted(counts.items()))
# -- corpus construction ----------------------------------------------------
def build(per_task: int, seed: int, tasks: Optional[Sequence[str]] = None,
repeat: bool = True, keep_canonical: bool = False,
timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS,
sequence_length: int = DEFAULT_SEQUENCE_LENGTH,
token_budget: bool = False) -> Tuple[List[Dict[str, str]], Dict[str, Any]]:
"""Generate, execute, verify, and return (rows, report).
`repeat=True` is the default for the reason `build_omni_corpus` documents
at length: repetition is how this model learns a procedure, and the
benchmark draws unseen parameters from the same space so recall cannot pass
it. `repeat=False` emits distinct prompts only and reports the shortfall
when a generator's space runs out.
"""
rng = random.Random(seed)
# The default is the v89 nine; a v93 task is built only when named.
chosen = list(tasks or TASKS)
unknown = [name for name in chosen if name not in ALL_TASKS]
if unknown:
raise ValueError(f"unknown task(s): {', '.join(unknown)}")
rows: List[Dict[str, str]] = []
counts: Dict[str, int] = {}
dropped: Dict[str, int] = {}
reasons: Dict[str, Dict[str, int]] = {}
distinct: Dict[str, int] = {}
short: Dict[str, Dict[str, object]] = {}
attempts: Dict[str, int] = {}
def draw(name: str) -> Optional[CodeProblem]:
"""One executed-and-agreed problem, or None. Counts its own drops."""
attempts[name] = attempts.get(name, 0) + 1
problem = ALL_TASKS[name](rng)
result = verify(problem, timeout_seconds)
parsed = extract_answer(problem.response)
if not result.ok:
dropped[name] = dropped.get(name, 0) + 1
bucket = reasons.setdefault(name, {})
bucket[result.reason] = bucket.get(result.reason, 0) + 1
return None
if parsed != problem.answer:
# A right answer the benchmark cannot read scores as wrong, so this
# is as fatal as an execution mismatch and is counted separately.
dropped[name] = dropped.get(name, 0) + 1
bucket = reasons.setdefault(name, {})
key = f"response parses to {parsed}, answer is {problem.answer}"
bucket[key] = bucket.get(key, 0) + 1
return None
return problem
for name in chosen:
made = 0
seen: set = set()
if repeat:
for _ in range(per_task):
problem = draw(name)
if problem is None:
continue
seen.add(problem.prompt)
rows.append(problem.to_row(keep_canonical=keep_canonical))
made += 1
else:
misses = 0
while made < per_task and misses < EXHAUSTION_MISSES:
problem = draw(name)
if problem is None or problem.prompt in seen:
misses += 1
continue
seen.add(problem.prompt)
rows.append(problem.to_row(keep_canonical=keep_canonical))
made += 1
misses = 0
if made < per_task:
short[name] = {"asked": per_task, "produced": made,
"reason": "parameter space exhausted"}
counts[name] = made
distinct[name] = len(seen)
total_attempts = sum(attempts.values())
total_dropped = sum(dropped.values())
report: Dict[str, Any] = {
"schema": "supermix-v87-code-corpus-v1",
"seed": seed,
"rows": len(rows),
"tasks": len(chosen),
"per_task": counts,
"distinct_prompts": distinct,
"repetition": {k: round(counts[k] / v, 1) for k, v in distinct.items() if v},
"distinct_capacity": {k: v for k, v in distinct_capacity().items()
if k in counts},
"attempts": attempts,
"dropped_failing_execution": dropped,
"drop_reasons": reasons,
"drop_rate": round(total_dropped / total_attempts, 6) if total_attempts else 0.0,
"short_of_requested": short,
"verified_by": "cpython exec in a restricted namespace",
"verification": {
"builtins": sorted(SAFE_BUILTINS),
"methods": sorted(SAFE_METHODS),
"timeout_seconds": timeout_seconds,
"allowed_syntax": sorted(ALLOWED_NODES),
"note": (
"the checker executes the snippet the PROMPT shows, not the "
"expression the generator used, so a template that renders "
"something other than what it computes is dropped"
),
},
"line_initial_forms": line_initial_forms(rows),
"options": {
"repeat": repeat,
"keep_canonical": keep_canonical,
},
}
if token_budget:
report["token_budget"] = token_budget_report(rows, sequence_length)
return rows, report
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("--per_task", type=int, default=20000)
parser.add_argument("--seed", type=int, default=87)
parser.add_argument("--output", default="datasets/v87/v87_code.jsonl")
parser.add_argument("--report", default=None)
parser.add_argument("--unique", action="store_true",
help=("emit only distinct prompts. Off by default: v74's "
"multiplication task repeated 712 pairs 56x each and "
"scored 0.93, while a 24,000-unique build of the same "
"operation scored 0.03"))
parser.add_argument("--task", action="append", default=[],
help=("restrict to these tasks; repeatable. The default is "
"the nine v89 tasks; the v93 tasks "
f"({', '.join(V93_TASKS)}) are built only when named"))
parser.add_argument("--keep_canonical", action="store_true",
help="ship the executable source and target variable per row")
parser.add_argument("--timeout", type=float, default=DEFAULT_TIMEOUT_SECONDS,
help="wall clock a single snippet gets before it is dropped")
parser.add_argument("--token_budget_report", action="store_true",
help=("measure per-task response and turn lengths and what "
"turn-aligned packing would drop"))
parser.add_argument("--sequence_length", type=int,
default=DEFAULT_SEQUENCE_LENGTH,
help="the block size the token budget is measured against")
return parser
def main(argv: Optional[Sequence[str]] = None) -> int:
args = build_parser().parse_args(argv)
rows, report = build(args.per_task, args.seed, args.task or None,
repeat=not args.unique,
keep_canonical=args.keep_canonical,
timeout_seconds=args.timeout,
sequence_length=args.sequence_length,
token_budget=args.token_budget_report)
output = Path(args.output)
output.parent.mkdir(parents=True, exist_ok=True)
with output.open("w", encoding="utf-8") as handle:
for row in rows:
handle.write(json.dumps(row) + "\n")
report["output"] = str(output)
report_path = Path(args.report) if args.report else output.with_suffix(".report.json")
report_path.write_text(json.dumps(report, indent=2), encoding="utf-8")
print(f"wrote {len(rows):,} rows to {output}")
print(f"drop rate {report['drop_rate']:.4%} "
f"({sum(report['dropped_failing_execution'].values()):,} of "
f"{sum(report['attempts'].values()):,} attempts)")
print(f" {'task':<24} {'rows':>7} {'distinct':>9} {'x':>6} {'capacity':>9} "
f"{'dropped':>8}")
capacity = report["distinct_capacity"]
for name, count in sorted(report["per_task"].items()):
print(f" {name:<24} {count:>7,} {report['distinct_prompts'][name]:>9,} "
f"{report['repetition'].get(name, 0):>6.1f} "
f"{capacity.get(name, 0):>9,} "
f"{report['dropped_failing_execution'].get(name, 0):>8,}")
budget = report.get("token_budget")
if budget:
print(f"\ntoken budget at sequence_length {budget['sequence_length']}")
print(f" {'task':<24} {'resp med':>8} {'p95':>5} {'max':>5} "
f"{'turn max':>9} {'dropped':>8}")
for name, stats in budget["tasks"].items():
print(f" {name:<24} {stats['response_median']:>8} "
f"{stats['response_p95']:>5} {stats['response_max']:>5} "
f"{stats['turn_max']:>9} {stats['dropped_fraction']:>8.4f}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
|