Supermix v93: two male-CNS hemispheres bonded to the trunk, neurogenesis, 41 tasks; results and ablations in README
dca70d8 verified Download src/coverage_audit.py from Kai9987kai/supermix-v93: direct link, hf CLI and curl.
- Browser
- Download file 9.98 kB
-
https://huggingface.co/Kai9987kai/supermix-v93/resolve/main/src/coverage_audit.py
- Command line
-
hf download hf://Kai9987kai/supermix-v93/src/coverage_audit.py
-
curl -L -o coverage_audit.py https://huggingface.co/Kai9987kai/supermix-v93/resolve/main/src/coverage_audit.py
9.98 kB
| """Which values does the benchmark ask about that the corpus never teaches? | |
| `percent` scored 0.533 on v86 and was read as a hard task for eight versions. | |
| It is not. Split by the percentage asked for: | |
| 5, 10, 20, 25 percent 16/17 = 0.941 | |
| 12, 15 percent 0/13 = 0.000 | |
| `build_scratchpad_math._scratchpad_percent` draws its percentage from | |
| ``[5, 10, 20, 25, 50]`` and `eval_problem_solving._percent` draws from | |
| ``[5, 10, 12, 15, 20, 25]``. Twelve and fifteen appear in the benchmark and in | |
| none of the corpus's forty thousand percent rows, and fifty appears in the | |
| corpus and never in the benchmark. Nothing reported this: the two generators | |
| live in different files, neither imports the other, and the score they produce | |
| together looks like a middling task rather than two tasks averaged. | |
| This module compares the two sides directly. For each task it draws prompts | |
| from both generators, pulls every number out of them, and reports values the | |
| benchmark asks about that the corpus never contains. | |
| It compares *values*, not ranges, because that is what catches a categorical | |
| hole. A range check would have to know which numbers in a prompt are operands | |
| and which are incidental, and would still have called ``[5, 10, 20, 25, 50]`` | |
| and ``[5, 10, 12, 15, 20, 25]`` compatible on the grounds that both run 5 to 50. | |
| What it cannot do is tell a real hole from a rare draw, so it reports a | |
| frequency alongside every value and leaves the judgement where it belongs. A | |
| value the benchmark asks for in a third of its problems and the corpus never | |
| contains is a defect; one that turns up once in a thousand is a tail. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import collections | |
| import json | |
| import random | |
| import re | |
| from typing import Callable, Dict, Iterable, List, Optional, Sequence | |
| NUMBER = re.compile(r"-?\d+(?:\.\d+)?") | |
| #: Corpus task names that the benchmark knows under a different name. | |
| #: `eval_problem_solving._arithmetic` emits both signs under one task; the | |
| #: corpus splits them into two builders. | |
| CORPUS_ALIASES = {"addition": "arithmetic", "subtraction": "arithmetic"} | |
| #: Benchmark tasks this audit names as uncompared on purpose, with the reason | |
| #: the report carries. The v93 connectome lookups are one generator on both | |
| #: sides: `eval_problem_solving` adapts `build_connectome_corpus.TASKS` | |
| #: directly, and every type and pair either side can draw comes from the one | |
| #: `build_connectome_corpus.population()`, so value coverage is complete by | |
| #: construction and there is nothing to compare. Running the comparison | |
| #: anyway would be worse than skipping it: the only numbers in a connectome | |
| #: prompt are the digits inside type names (`T4a`, `IN17A052`), and reading | |
| #: those as operands would make a coverage table out of noise. | |
| NOT_COMPARED_BY_DESIGN = { | |
| name: ( | |
| "lookup task: benchmark and corpus draw from the same " | |
| "build_connectome_corpus.population(), so every asked type or pair is " | |
| "taught by construction; the digits in type names are not values" | |
| ) | |
| for name in ("cns_type_count", "cns_side_count", "cns_pair_synapses") | |
| } | |
| def numbers_in(prompt: str) -> List[float]: | |
| return [float(match) for match in NUMBER.findall(prompt)] | |
| def _profile(prompts: Iterable[str]) -> Dict[str, object]: | |
| counts: collections.Counter = collections.Counter() | |
| total = 0 | |
| for prompt in prompts: | |
| total += 1 | |
| counts.update(numbers_in(prompt)) | |
| return {"prompts": total, "values": counts} | |
| def compare(corpus_prompts: Sequence[str], benchmark_prompts: Sequence[str], | |
| threshold: float = 0.02) -> Dict[str, object]: | |
| """Values the benchmark asks about often and the corpus never contains. | |
| `threshold` is the share of benchmark prompts a missing value must appear | |
| in before it is called a hole rather than a tail. Two percent of thirty | |
| problems is under one problem, so the default errs towards reporting. | |
| """ | |
| corpus = _profile(corpus_prompts) | |
| benchmark = _profile(benchmark_prompts) | |
| seen = set(corpus["values"]) | |
| holes = [] | |
| for value, count in benchmark["values"].most_common(): | |
| if value in seen: | |
| continue | |
| share = count / max(1, benchmark["prompts"]) | |
| if share >= threshold: | |
| holes.append({"value": value, "benchmark_prompts": count, | |
| "share_of_benchmark": round(share, 4)}) | |
| # The reverse direction is not a defect, but it is worth knowing: corpus | |
| # values the benchmark never asks about are training the model spends | |
| # capacity on and the score never sees. | |
| unasked = [] | |
| asked = set(benchmark["values"]) | |
| for value, count in corpus["values"].most_common(20): | |
| if value in asked: | |
| continue | |
| share = count / max(1, corpus["prompts"]) | |
| if share >= threshold: | |
| unasked.append({"value": value, "corpus_prompts": count, | |
| "share_of_corpus": round(share, 4)}) | |
| return { | |
| "corpus_prompts": corpus["prompts"], | |
| "benchmark_prompts": benchmark["prompts"], | |
| "distinct_corpus_values": len(corpus["values"]), | |
| "distinct_benchmark_values": len(benchmark["values"]), | |
| "asked_but_never_taught": holes, | |
| "taught_but_never_asked": unasked, | |
| } | |
| def audit(samples: int = 4000, seed: int = 87, threshold: float = 0.02, | |
| tasks: Optional[Sequence[str]] = None) -> Dict[str, object]: | |
| """Run the comparison over every task both sides define.""" | |
| import build_code_corpus as code | |
| import build_omni_corpus as omni | |
| import build_scratchpad_math as scratch | |
| import eval_problem_solving as solving | |
| corpus_generators: Dict[str, Callable[[random.Random], str]] = {} | |
| # v93's tasks live in each builder's `V93_TASKS` rather than `TASKS`, so | |
| # a default build stays byte-identical; the benchmark adapts both, and so | |
| # must this, or a value hole in a new task would never be reported. | |
| for table in (omni.TASKS, getattr(omni, "V93_TASKS", {})): | |
| for name, generator in table.items(): | |
| corpus_generators[name] = lambda rng, g=generator: g(rng).prompt | |
| for table in (code.TASKS, getattr(code, "V93_TASKS", {})): | |
| for name, generator in table.items(): | |
| corpus_generators[name] = lambda rng, g=generator: g(rng).prompt | |
| # `build_scratchpad_math.GENERATORS` is a tuple of functions with no names | |
| # attached; each row it produces carries its own `task`. Calling each once | |
| # is how the mapping is recovered, and it is also a check that every | |
| # generator runs at all. | |
| probe = random.Random(0) | |
| for generator in scratch.GENERATORS: | |
| name = str(generator(probe)["task"]) | |
| target = CORPUS_ALIASES.get(name, name) | |
| corpus_generators.setdefault( | |
| target, lambda rng, g=generator: _scratchpad_prompt(g(rng))) | |
| report: Dict[str, object] = { | |
| "schema": "supermix-v87-coverage-audit-v1", | |
| "samples_per_side": samples, | |
| "seed": seed, | |
| "threshold": threshold, | |
| "tasks": {}, | |
| "not_compared": [], | |
| "not_compared_reason": {}, | |
| } | |
| names = list(tasks or solving.GENERATORS) | |
| for name in sorted(names): | |
| if name in NOT_COMPARED_BY_DESIGN: | |
| report["not_compared"].append(name) | |
| report["not_compared_reason"][name] = NOT_COMPARED_BY_DESIGN[name] | |
| continue | |
| if name not in corpus_generators: | |
| report["not_compared"].append(name) | |
| report["not_compared_reason"][name] = "no corpus generator" | |
| continue | |
| corpus_rng = random.Random(f"corpus:{name}:{seed}") | |
| bench_rng = solving.task_rng(name, seed) | |
| corpus_prompts = [corpus_generators[name](corpus_rng) for _ in range(samples)] | |
| bench_prompts = [solving.GENERATORS[name](bench_rng).prompt | |
| for _ in range(samples)] | |
| report["tasks"][name] = compare(corpus_prompts, bench_prompts, threshold) | |
| return report | |
| def _scratchpad_prompt(row: Dict[str, object]) -> str: | |
| """`build_scratchpad_math` returns a dict whose prompt field varies.""" | |
| expression = str(row.get("expression", "")) | |
| if row.get("raw_prompt"): | |
| return expression | |
| return f"Solve this basic math problem: {expression}" | |
| def main(argv: Optional[Sequence[str]] = None) -> int: | |
| parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) | |
| parser.add_argument("--samples", type=int, default=4000) | |
| parser.add_argument("--seed", type=int, default=87) | |
| parser.add_argument("--threshold", type=float, default=0.02) | |
| parser.add_argument("--tasks", default=None) | |
| parser.add_argument("--output", default=None) | |
| args = parser.parse_args(argv) | |
| tasks = ([t.strip() for t in args.tasks.split(",") if t.strip()] | |
| if args.tasks else None) | |
| report = audit(args.samples, args.seed, args.threshold, tasks) | |
| flagged = 0 | |
| for name, entry in report["tasks"].items(): | |
| holes = entry["asked_but_never_taught"] | |
| if not holes: | |
| continue | |
| flagged += 1 | |
| share = sum(h["share_of_benchmark"] for h in holes) | |
| print(f"{name}: {len(holes)} value(s) the corpus never teaches, " | |
| f"in up to {share:.1%} of benchmark prompts") | |
| for hole in holes[:8]: | |
| print(f" {hole['value']:g} in {hole['share_of_benchmark']:.1%} " | |
| f"of benchmark prompts") | |
| if not flagged: | |
| print("no task asks about a value its corpus never teaches") | |
| if report["not_compared"]: | |
| print("\nnot compared:") | |
| for name in report["not_compared"]: | |
| print(f" {name}: {report['not_compared_reason'].get(name, '')}") | |
| if args.output: | |
| with open(args.output, "w", encoding="utf-8") as handle: | |
| json.dump(report, handle, indent=2) | |
| print(f"\nreceipt -> {args.output}") | |
| return 0 | |
| if __name__ == "__main__": # pragma: no cover | |
| raise SystemExit(main()) | |