{ "experiment": "CV4 Phase 1 \u2014 capability calibration across the ladder, all six families", "prereg": "docs/experiments/preregs/CV4_capability_ladder.md", "status": "MEASUREMENT ONLY \u2014 Phase 1 gates which model later phases target; CV4's registered rule is adjudicated in Phase 3, not here", "readout_convention": "STRIPPED (corrected)", "n_items_total": 551, "n_items_with_target": 255, "measures": { "answer_rank": "rank of the expected answer's first token; defined only where `target` is non-null (multihop, order-ops, multilingual)", "prompt_surprisal": "mean per-token NLL of the prompt; defined for all six families. NOT a task-competence measure \u2014 it separates 'cannot do the task' from 'cannot parse the prompt'" }, "note_on_undefined": "poetry / typo / association carry target: null. The annotated intermediate is the object of interest and there is no separate answer, so ANSWER COMPETENCE IS UNDEFINED for them, not zero.", "interval": "95% Wilson score interval on every rate", "by_model": { "70m": { "multihop": { "n_items": 93, "mean_prompt_surprisal": 4.742897197764407, "median_prompt_surprisal": 4.761600017547607, "n_scored_for_answer": 93, "top1_count": 1, "top1_rate": 0.010752688172043012, "top1_wilson95": [ 0.001900593097955601, 0.05842059118648415 ], "top10_count": 13, "top10_rate": 0.13978494623655913, "top10_wilson95": [ 0.08354643741721157, 0.22460212683616715 ], "median_rank": 80 }, "multilingual": { "n_items": 107, "mean_prompt_surprisal": 5.743064506031643, "median_prompt_surprisal": 5.629157066345215, "n_scored_for_answer": 107, "top1_count": 1, "top1_rate": 0.009345794392523364, "top1_wilson95": [ 0.0016516291511190044, 0.05105060538898146 ], "top10_count": 9, "top10_rate": 0.08411214953271028, "top10_wilson95": [ 0.044881436613206184, 0.15217093361602183 ], "median_rank": 3436 }, "order-ops": { "n_items": 55, "mean_prompt_surprisal": 4.685458686135032, "median_prompt_surprisal": 4.4597907066345215, "n_scored_for_answer": 55, "top1_count": 2, "top1_rate": 0.03636363636363636, "top1_wilson95": [ 0.010029330855724924, 0.1232369297490173 ], "top10_count": 21, "top10_rate": 0.38181818181818183, "top10_wilson95": [ 0.26515766317415235, 0.513910207176076 ], "median_rank": 18 }, "poetry": { "n_items": 98, "mean_prompt_surprisal": 5.154343254712163, "median_prompt_surprisal": 5.116718769073486, "answer_competence": "UNDEFINED \u2014 target is null for this family" }, "typo": { "n_items": 96, "mean_prompt_surprisal": 6.320249448219935, "median_prompt_surprisal": 6.34007453918457, "answer_competence": "UNDEFINED \u2014 target is null for this family" }, "association": { "n_items": 102, "mean_prompt_surprisal": 4.577553527027953, "median_prompt_surprisal": 4.60029411315918, "answer_competence": "UNDEFINED \u2014 target is null for this family" }, "_pooled_answer": { "n": 255, "top1_count": 4, "top1_rate": 0.01568627450980392, "top1_wilson95": [ 0.006116456526536161, 0.03963198576403846 ], "top10_rate": 0.16862745098039217, "top10_wilson95": [ 0.12766685656458393, 0.2194241828973882 ] }, "_pooled_surprisal_all_six": 5.203927769981855 }, "160m": { "multihop": { "n_items": 93, "mean_prompt_surprisal": 4.323448388807235, "median_prompt_surprisal": 4.265374183654785, "n_scored_for_answer": 93, "top1_count": 3, "top1_rate": 0.03225806451612903, "top1_wilson95": [ 0.011030412935116203, 0.09059533467748004 ], "top10_count": 24, "top10_rate": 0.25806451612903225, "top10_wilson95": [ 0.17998399469266027, 0.35533966786557925 ], "median_rank": 42 }, "multilingual": { "n_items": 107, "mean_prompt_surprisal": 6.095417096236042, "median_prompt_surprisal": 5.124428749084473, "n_scored_for_answer": 107, "top1_count": 1, "top1_rate": 0.009345794392523364, "top1_wilson95": [ 0.0016516291511190044, 0.05105060538898146 ], "top10_count": 9, "top10_rate": 0.08411214953271028, "top10_wilson95": [ 0.044881436613206184, 0.15217093361602183 ], "median_rank": 3788 }, "order-ops": { "n_items": 55, "mean_prompt_surprisal": 4.33481783433394, "median_prompt_surprisal": 4.353231430053711, "n_scored_for_answer": 55, "top1_count": 0, "top1_rate": 0.0, "top1_wilson95": [ 0.0, 0.06528714378942788 ], "top10_count": 24, "top10_rate": 0.43636363636363634, "top10_wilson95": [ 0.31373238636001305, 0.5673041592131869 ], "median_rank": 13 }, "poetry": { "n_items": 98, "mean_prompt_surprisal": 4.700447340400851, "median_prompt_surprisal": 4.699031829833984, "answer_competence": "UNDEFINED \u2014 target is null for this family" }, "typo": { "n_items": 96, "mean_prompt_surprisal": 5.784147493541241, "median_prompt_surprisal": 5.752850532531738, "answer_competence": "UNDEFINED \u2014 target is null for this family" }, "association": { "n_items": 102, "mean_prompt_surprisal": 4.123158244525685, "median_prompt_surprisal": 4.120728969573975, "answer_competence": "UNDEFINED \u2014 target is null for this family" }, "_pooled_answer": { "n": 255, "top1_count": 4, "top1_rate": 0.01568627450980392, "top1_wilson95": [ 0.006116456526536161, 0.03963198576403846 ], "top10_rate": 0.2235294117647059, "top10_wilson95": [ 0.1767132210847063, 0.27855208402853665 ] }, "_pooled_surprisal_all_six": 4.893572732974166 }, "410m": { "multihop": { "n_items": 93, "mean_prompt_surprisal": 3.8603221395964264, "median_prompt_surprisal": 3.814849376678467, "n_scored_for_answer": 93, "top1_count": 6, "top1_rate": 0.06451612903225806, "top1_wilson95": [ 0.029901024142029796, 0.13368156846280116 ], "top10_count": 31, "top10_rate": 0.3333333333333333, "top10_wilson95": [ 0.24582243182201943, 0.4340672023847677 ], "median_rank": 31 }, "multilingual": { "n_items": 107, "mean_prompt_surprisal": 4.675777234763743, "median_prompt_surprisal": 4.748653411865234, "n_scored_for_answer": 107, "top1_count": 1, "top1_rate": 0.009345794392523364, "top1_wilson95": [ 0.0016516291511190044, 0.05105060538898146 ], "top10_count": 6, "top10_rate": 0.056074766355140186, "top10_wilson95": [ 0.025950008789329404, 0.11697106055647577 ], "median_rank": 3044 }, "order-ops": { "n_items": 55, "mean_prompt_surprisal": 4.2336343418468125, "median_prompt_surprisal": 4.275657653808594, "n_scored_for_answer": 55, "top1_count": 5, "top1_rate": 0.09090909090909091, "top1_wilson95": [ 0.039457573336373924, 0.19577736249133978 ], "top10_count": 30, "top10_rate": 0.5454545454545454, "top10_wilson95": [ 0.41522461348360484, 0.669749282535538 ], "median_rank": 9 }, "poetry": { "n_items": 98, "mean_prompt_surprisal": 4.148344592172272, "median_prompt_surprisal": 4.126254558563232, "answer_competence": "UNDEFINED \u2014 target is null for this family" }, "typo": { "n_items": 96, "mean_prompt_surprisal": 5.49502004434665, "median_prompt_surprisal": 5.293105602264404, "answer_competence": "UNDEFINED \u2014 target is null for this family" }, "association": { "n_items": 102, "mean_prompt_surprisal": 3.8331370657565547, "median_prompt_surprisal": 3.8432066440582275, "answer_competence": "UNDEFINED \u2014 target is null for this family" }, "_pooled_answer": { "n": 255, "top1_count": 12, "top1_rate": 0.047058823529411764, "top1_wilson95": [ 0.02712132140633029, 0.08044098697068483 ], "top10_rate": 0.2627450980392157, "top10_wilson95": [ 0.21253225697118083, 0.3200003807501127 ] }, "_pooled_surprisal_all_six": 4.374372569747076 }, "1b": { "multihop": { "n_items": 93, "mean_prompt_surprisal": 3.7122991469598587, "median_prompt_surprisal": 3.713099241256714, "n_scored_for_answer": 93, "top1_count": 7, "top1_rate": 0.07526881720430108, "top1_wilson95": [ 0.036936214085016475, 0.14729866018389276 ], "top10_count": 31, "top10_rate": 0.3333333333333333, "top10_wilson95": [ 0.24582243182201943, 0.4340672023847677 ], "median_rank": 18 }, "multilingual": { "n_items": 107, "mean_prompt_surprisal": 4.4941605663745205, "median_prompt_surprisal": 4.559407711029053, "n_scored_for_answer": 107, "top1_count": 0, "top1_rate": 0.0, "top1_wilson95": [ 0.0, 0.03465846757895952 ], "top10_count": 7, "top10_rate": 0.06542056074766354, "top10_wilson95": [ 0.032047896397421095, 0.12891693990952502 ], "median_rank": 1649 }, "order-ops": { "n_items": 55, "mean_prompt_surprisal": 4.145381216569381, "median_prompt_surprisal": 4.083983898162842, "n_scored_for_answer": 55, "top1_count": 4, "top1_rate": 0.07272727272727272, "top1_wilson95": [ 0.028643596503688543, 0.17260178091636796 ], "top10_count": 37, "top10_rate": 0.6727272727272727, "top10_wilson95": [ 0.5410297617609947, 0.7818710431117484 ], "median_rank": 7 }, "poetry": { "n_items": 98, "mean_prompt_surprisal": 3.935050759996687, "median_prompt_surprisal": 3.910858154296875, "answer_competence": "UNDEFINED \u2014 target is null for this family" }, "typo": { "n_items": 96, "mean_prompt_surprisal": 5.290679027636846, "median_prompt_surprisal": 5.2639923095703125, "answer_competence": "UNDEFINED \u2014 target is null for this family" }, "association": { "n_items": 102, "mean_prompt_surprisal": 3.6671034354789582, "median_prompt_surprisal": 3.6480932235717773, "answer_competence": "UNDEFINED \u2014 target is null for this family" }, "_pooled_answer": { "n": 255, "top1_count": 11, "top1_rate": 0.043137254901960784, "top1_wilson95": [ 0.024254970635041635, 0.07558060448116843 ], "top10_rate": 0.29411764705882354, "top10_wilson95": [ 0.24157991671293233, 0.3527665870948018 ] }, "_pooled_surprisal_all_six": 4.207445692169375 }, "1.4b": { "multihop": { "n_items": 93, "mean_prompt_surprisal": 3.5891012683991463, "median_prompt_surprisal": 3.6213531494140625, "n_scored_for_answer": 93, "top1_count": 5, "top1_rate": 0.053763440860215055, "top1_wilson95": [ 0.023180332422992958, 0.11974997851775977 ], "top10_count": 33, "top10_rate": 0.3548387096774194, "top10_wilson95": [ 0.26512726316736557, 0.45606693436757817 ], "median_rank": 24 }, "multilingual": { "n_items": 107, "mean_prompt_surprisal": 4.305595877014588, "median_prompt_surprisal": 4.307002067565918, "n_scored_for_answer": 107, "top1_count": 0, "top1_rate": 0.0, "top1_wilson95": [ 0.0, 0.03465846757895952 ], "top10_count": 8, "top10_rate": 0.07476635514018691, "top10_wilson95": [ 0.03836964356083345, 0.1406389597072536 ], "median_rank": 1345 }, "order-ops": { "n_items": 55, "mean_prompt_surprisal": 4.166462430087003, "median_prompt_surprisal": 4.244797229766846, "n_scored_for_answer": 55, "top1_count": 3, "top1_rate": 0.05454545454545454, "top1_wilson95": [ 0.01872287596750008, 0.14853294304489928 ], "top10_count": 29, "top10_rate": 0.5272727272727272, "top10_wilson95": [ 0.39791319071629383, 0.6530711468951917 ], "median_rank": 7 }, "poetry": { "n_items": 98, "mean_prompt_surprisal": 3.6910850977411074, "median_prompt_surprisal": 3.6510846614837646, "answer_competence": "UNDEFINED \u2014 target is null for this family" }, "typo": { "n_items": 96, "mean_prompt_surprisal": 5.277833836774032, "median_prompt_surprisal": 5.16269588470459, "answer_competence": "UNDEFINED \u2014 target is null for this family" }, "association": { "n_items": 102, "mean_prompt_surprisal": 3.6086690916734585, "median_prompt_surprisal": 3.6544148921966553, "answer_competence": "UNDEFINED \u2014 target is null for this family" }, "_pooled_answer": { "n": 255, "top1_count": 8, "top1_rate": 0.03137254901960784, "top1_wilson95": [ 0.01598082278729794, 0.060674552546496924 ], "top10_rate": 0.27450980392156865, "top10_wilson95": [ 0.22338671792303816, 0.33232611958067076 ] }, "_pooled_surprisal_all_six": 4.106457933614889 }, "2.8b": { "multihop": { "n_items": 93, "mean_prompt_surprisal": 3.5109470018776516, "median_prompt_surprisal": 3.531287670135498, "n_scored_for_answer": 93, "top1_count": 6, "top1_rate": 0.06451612903225806, "top1_wilson95": [ 0.029901024142029796, 0.13368156846280116 ], "top10_count": 40, "top10_rate": 0.43010752688172044, "top10_wilson95": [ 0.33423358533237524, 0.5315265838511162 ], "median_rank": 14 }, "multilingual": { "n_items": 107, "mean_prompt_surprisal": 4.153321266174316, "median_prompt_surprisal": 4.131369590759277, "n_scored_for_answer": 107, "top1_count": 1, "top1_rate": 0.009345794392523364, "top1_wilson95": [ 0.0016516291511190044, 0.05105060538898146 ], "top10_count": 10, "top10_rate": 0.09345794392523364, "top10_wilson95": [ 0.05155771001191554, 0.1635384271784534 ], "median_rank": 578 }, "order-ops": { "n_items": 55, "mean_prompt_surprisal": 4.071762145649303, "median_prompt_surprisal": 4.097884178161621, "n_scored_for_answer": 55, "top1_count": 8, "top1_rate": 0.14545454545454545, "top1_wilson95": [ 0.07559199016771778, 0.2616116208829674 ], "top10_count": 34, "top10_rate": 0.6181818181818182, "top10_wilson95": [ 0.4860897928239239, 0.7348423368258477 ], "median_rank": 7 }, "poetry": { "n_items": 98, "mean_prompt_surprisal": 3.6883874936979644, "median_prompt_surprisal": 3.6747243404388428, "answer_competence": "UNDEFINED \u2014 target is null for this family" }, "typo": { "n_items": 96, "mean_prompt_surprisal": 5.161082645257314, "median_prompt_surprisal": 5.04867696762085, "answer_competence": "UNDEFINED \u2014 target is null for this family" }, "association": { "n_items": 102, "mean_prompt_surprisal": 3.4943248594508454, "median_prompt_surprisal": 3.5100271701812744, "answer_competence": "UNDEFINED \u2014 target is null for this family" }, "_pooled_answer": { "n": 255, "top1_count": 15, "top1_rate": 0.058823529411764705, "top1_wilson95": [ 0.03596805781169063, 0.09477445034773968 ], "top10_rate": 0.32941176470588235, "top10_wilson95": [ 0.2746295550144033, 0.3892575481405764 ] }, "_pooled_surprisal_all_six": 4.0133042353512325 } }, "VERDICT": "Answer competence, 255 items with a target, 95% Wilson: 70m: top1 1.6% [0.6,4.0] | 160m: top1 1.6% [0.6,4.0] | 410m: top1 4.7% [2.7,8.0] | 1b: top1 4.3% [2.4,7.6] | 1.4b: top1 3.1% [1.6,6.1] | 2.8b: top1 5.9% [3.6,9.5]. Prompt surprisal covers all 551 items in all six families.", "provenance": { "schema_version": 1, "experiment": "CV4P1", "utc": "2026-08-30T06:30:11+00:00", "script": { "path": "experiments/cv4_phase1_capability.py", "sha256": "3aaadc49c285d30fe1aafb92714633b4d5e694caac40f82b0afbfa964a424cfb", "n_lines": 173 }, "git": { "commit": "c9c49697e295c0fec4225552836fc9733b279f75", "commit_short": "c9c49697e295", "git_repository_present": true, "branch": "pythia", "dirty": true, "dirty_files": [ "CLAUDE.md", "README.md", "corpora/manifest_ood.json", "docs/HANDOFF.md", "docs/REPO_REVIEW_PROMPT.md", "docs/context/AGGREGATION_POLICY.md", "docs/context/CONTEXT.md", "docs/context/IMPLEMENTATION_RULES.md", "docs/context/STATE.md", "docs/experiments/INDEX.md", "docs/experiments/METRICS.md", "docs/experiments/PROPOSED.md", "docs/experiments/PROVENANCE.md", "docs/experiments/descriptions/E64.md", "docs/experiments/descriptions/EN1.md", "docs/experiments/preregs/CV3_margins.md", "docs/experiments/preregs/CV4_capability_ladder.md", "docs/experiments/preregs/CV5_metric_sensitivity.md", "docs/experiments/preregs/CV6_per_family_ladder.md", "docs/experiments/preregs/CV7_1b_rung.md", "docs/experiments/preregs/DN1_design_nulls.md", "docs/experiments/preregs/E33_logit_baseline.md", "docs/experiments/preregs/E36_qladder.md", "docs/experiments/preregs/E48_crossover.md", "docs/experiments/preregs/E48b_containment.md", "docs/experiments/preregs/E48c_exposure_vs_read.md", "docs/experiments/preregs/E51_interaction.md", "docs/experiments/preregs/E52_factorial.md", "docs/experiments/preregs/E53_ladder_summary.md", "docs/experiments/preregs/E55_matrix_robustness.md", "docs/experiments/preregs/E56_predictor_registry.md", "docs/experiments/preregs/E57_grid_variance_ci.md", "docs/experiments/preregs/E58_algebra_audit.md", "docs/experiments/preregs/E59_read_dose_response.md", "docs/experiments/preregs/E60_fitter_determinism.md", "docs/experiments/preregs/E61_randomized_network_null.md", "docs/experiments/preregs/E62_ladder1b_corrected_band.md", "docs/experiments/preregs/E65_training_axis_floor.md", "docs/experiments/preregs/E66b_determinism_floor.md", "docs/experiments/preregs/OS1_operator_space.md", "docs/experiments/preregs/P0_t52_parallel_port.md", "docs/experiments/preregs/R1_grid_rstrip.md", "docs/experiments/preregs/R2_e48_rstrip_arm.md", "docs/experiments/preregs/R3_ladder410_cpu.md", "docs/experiments/preregs/R4_corrections.md", "docs/experiments/preregs/R4b_e36_rstrip.md", "docs/experiments/preregs/R5_corpus_axis_uncertainty.md", "docs/experiments/preregs/R6_within_source_resampling.md", "docs/experiments/preregs/R8_ladder_rstrip_S2.md", "docs/experiments/preregs/R9_permutation_calibrated_min.md", "docs/experiments/preregs/superseded/PREREG_E48_CROSSOVER.md", "docs/reproducibility/ARTIFACTS.md", "docs/reproducibility/PAYLOAD_REDACTIONS.md", "docs/reproducibility/RELEASE.md", "docs/reproducibility/REVIEWER_REPRO.md", "docs/reproducibility/T1_FINDING_E33_BAND.md", "docs/reproducibility/T1_SWEEP_PREDICTIONS.md", "docs/reproducibility/T1_SWEEP_SPEC.md", "docs/reproducibility/review-snapshot-2026-08-19/MANIFEST.sha256", "docs/reproducibility/review-snapshot-2026-08-19/PROVENANCE.md", "docs/reproducibility/review-snapshot-2026-08-19/REDACTIONS.md", "docs/reproducibility/review-snapshot-2026-08-19/review/EXTERNAL_REVIEW.md", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/FINDINGS.md", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/PHASE_A_ACCOUNT.md" ], "dirty_diff_sha256": "ec61dde1905616f407f51d5f6fe3ea37c9c6e3e1ade6998cd0644527d080c600", "describe": "c9c4969-dirty" }, "argv": [ "experiments/cv4_phase1_capability.py", "--models", "70m,160m,410m,1b,1.4b,2.8b" ], "cwd": ".", "inputs": [ { "path": "jacobian-lens/data/evaluations/lens-eval-multihop.json", "sha256": "50b7e4c9255291c0ca2a8e94615be9f44531fa57bb1a844e4f9616056d987416", "bytes": 21869, "exists": true }, { "path": "jacobian-lens/data/evaluations/lens-eval-multilingual.json", "sha256": "fa70b9bd89416a6d8d985a80dc628b109ae6fd3b25b9275c0fc5065d7ff4a0ef", "bytes": 24284, "exists": true }, { "path": "jacobian-lens/data/evaluations/lens-eval-order-ops.json", "sha256": "b203206d16ff628152cc86f3838604e06cb54776f3e14fa1c34f150db8bc7560", "bytes": 9589, "exists": true }, { "path": "jacobian-lens/data/evaluations/lens-eval-poetry.json", "sha256": "6aeb3415c5a5c3f3827c9efe63f006de02f5ef39a816bbac68e15e733aba60cc", "bytes": 21533, "exists": true }, { "path": "jacobian-lens/data/evaluations/lens-eval-typo.json", "sha256": "9d05e16b7234a57d0773d120a4e1c4e94fd3bc2235a8125d4200a70e60ab17aa", "bytes": 14646, "exists": true }, { "path": "jacobian-lens/data/evaluations/lens-eval-association.json", "sha256": "d1a98cd4911b594282e74168091c77d849dae18ffe2acb5761074853f327d71c", "bytes": 24228, "exists": true } ], "env": { "python": "3.14.2", "torch": "2.13.0", "transformers": "5.14.1", "numpy": "2.5.1", "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", "machine": "arm64", "hostname": "", "tf32": { "allow_tf32_matmul": false, "allow_tf32_cudnn": true, "NVIDIA_TF32_OVERRIDE": null }, "threads": { "torch_num_threads": 10, "torch_num_interop_threads": 14, "OMP_NUM_THREADS": null, "os_cpu_count": 14 } }, "payload_sha256": "43b21658bcc6a1acf619b614a5eb74db9252372b3ff7f151fbbde0ce7fbd9bad" } }