{ "experiment": "CV1 \u2014 answer competence: can the model do the task at all?", "status": "MEASUREMENT ONLY \u2014 no registered decision rule", "why": "existing capability_* fields measure the INTERMEDIATE's rank, not the ANSWER's; t36_qladder.py:483-489 records the registered competence gate as degenerate and unable to fire. This supplies the missing number.", "readout_convention": "STRIPPED (corrected) \u2014 rank of the target's first token at the final position of the rstripped prompt", "sets_with_target": [ "multihop", "order-ops", "multilingual" ], "sets_without_target": [ "poetry", "typo", "association" ], "note_on_undefined": "poetry/typo/association carry target: null. The annotated intermediate is the object of interest and there is no separate answer, so answer competence is UNDEFINED for them \u2014 not zero.", "by_model": { "70m": { "multihop": { "n_scored": 93, "n_multitoken_answers_scored_on_first_token": 3, "top1_rate": 0.010752688172043012, "top10_rate": 0.13978494623655913, "median_rank": 80, "mean_rank": 1106.5806451612902, "median_prob": 0.0015416988171637058 }, "order-ops": { "n_scored": 55, "n_multitoken_answers_scored_on_first_token": 0, "top1_rate": 0.03636363636363636, "top10_rate": 0.38181818181818183, "median_rank": 18, "mean_rank": 22.09090909090909, "median_prob": 0.006405120715498924 }, "multilingual": { "n_scored": 107, "n_multitoken_answers_scored_on_first_token": 92, "top1_rate": 0.009345794392523364, "top10_rate": 0.08411214953271028, "median_rank": 3436, "mean_rank": 4615.915887850468, "median_prob": 2.3302003683056682e-05 }, "_pooled": { "n": 255, "top1_rate": 0.01568627450980392, "top10_rate": 0.16862745098039217 } }, "160m": { "multihop": { "n_scored": 93, "n_multitoken_answers_scored_on_first_token": 3, "top1_rate": 0.03225806451612903, "top10_rate": 0.25806451612903225, "median_rank": 42, "mean_rank": 628.3655913978495, "median_prob": 0.0031900564208626747 }, "order-ops": { "n_scored": 55, "n_multitoken_answers_scored_on_first_token": 0, "top1_rate": 0.0, "top10_rate": 0.43636363636363634, "median_rank": 13, "mean_rank": 14.49090909090909, "median_prob": 0.01120622269809246 }, "multilingual": { "n_scored": 107, "n_multitoken_answers_scored_on_first_token": 92, "top1_rate": 0.009345794392523364, "top10_rate": 0.08411214953271028, "median_rank": 3788, "mean_rank": 4630.429906542056, "median_prob": 1.6619400412309915e-05 }, "_pooled": { "n": 255, "top1_rate": 0.01568627450980392, "top10_rate": 0.2235294117647059 } }, "410m": { "multihop": { "n_scored": 93, "n_multitoken_answers_scored_on_first_token": 3, "top1_rate": 0.06451612903225806, "top10_rate": 0.3333333333333333, "median_rank": 31, "mean_rank": 460.9139784946237, "median_prob": 0.005166209302842617 }, "order-ops": { "n_scored": 55, "n_multitoken_answers_scored_on_first_token": 0, "top1_rate": 0.09090909090909091, "top10_rate": 0.5454545454545454, "median_rank": 9, "mean_rank": 11.909090909090908, "median_prob": 0.026608118787407875 }, "multilingual": { "n_scored": 107, "n_multitoken_answers_scored_on_first_token": 92, "top1_rate": 0.009345794392523364, "top10_rate": 0.056074766355140186, "median_rank": 3044, "mean_rank": 4031.233644859813, "median_prob": 1.0294868843629956e-05 }, "_pooled": { "n": 255, "top1_rate": 0.047058823529411764, "top10_rate": 0.2627450980392157 } }, "1b": { "multihop": { "n_scored": 93, "n_multitoken_answers_scored_on_first_token": 3, "top1_rate": 0.07526881720430108, "top10_rate": 0.3333333333333333, "median_rank": 18, "mean_rank": 316.0967741935484, "median_prob": 0.011097713373601437 }, "order-ops": { "n_scored": 55, "n_multitoken_answers_scored_on_first_token": 0, "top1_rate": 0.07272727272727272, "top10_rate": 0.6727272727272727, "median_rank": 7, "mean_rank": 9.163636363636364, "median_prob": 0.029681021347641945 }, "multilingual": { "n_scored": 107, "n_multitoken_answers_scored_on_first_token": 92, "top1_rate": 0.0, "top10_rate": 0.06542056074766354, "median_rank": 1649, "mean_rank": 2892.6168224299067, "median_prob": 3.0693336157128215e-05 }, "_pooled": { "n": 255, "top1_rate": 0.043137254901960784, "top10_rate": 0.29411764705882354 } } }, "VERDICT": "Answer competence on the three sets carrying a target. 70m: top1 1.6%, top10 16.9% (n=255 items with a target) | 160m: top1 1.6%, top10 22.4% (n=255 items with a target) | 410m: top1 4.7%, top10 26.3% (n=255 items with a target) | 1b: top1 4.3%, top10 29.4% (n=255 items with a target)", "provenance": { "schema_version": 1, "experiment": "CV1", "utc": "2026-08-30T06:32:53+00:00", "script": { "path": "experiments/cv1_answer_competence.py", "sha256": "29279dcb364dec36ae9f32438ab9ac607cc8b89983cf20c5796662e9a56a7a21", "n_lines": 138 }, "git": { "commit": "c9c49697e295c0fec4225552836fc9733b279f75", "commit_short": "c9c49697e295", "git_repository_present": true, "branch": "pythia", "dirty": true, "dirty_files": [ "CLAUDE.md", "README.md", "corpora/manifest_ood.json", "docs/HANDOFF.md", "docs/REPO_REVIEW_PROMPT.md", "docs/context/AGGREGATION_POLICY.md", "docs/context/CONTEXT.md", "docs/context/IMPLEMENTATION_RULES.md", "docs/context/STATE.md", "docs/experiments/INDEX.md", "docs/experiments/METRICS.md", "docs/experiments/PROPOSED.md", "docs/experiments/PROVENANCE.md", "docs/experiments/descriptions/E64.md", "docs/experiments/descriptions/EN1.md", "docs/experiments/preregs/CV3_margins.md", "docs/experiments/preregs/CV4_capability_ladder.md", "docs/experiments/preregs/CV5_metric_sensitivity.md", "docs/experiments/preregs/CV6_per_family_ladder.md", "docs/experiments/preregs/CV7_1b_rung.md", "docs/experiments/preregs/DN1_design_nulls.md", "docs/experiments/preregs/E33_logit_baseline.md", "docs/experiments/preregs/E36_qladder.md", "docs/experiments/preregs/E48_crossover.md", "docs/experiments/preregs/E48b_containment.md", "docs/experiments/preregs/E48c_exposure_vs_read.md", "docs/experiments/preregs/E51_interaction.md", "docs/experiments/preregs/E52_factorial.md", "docs/experiments/preregs/E53_ladder_summary.md", "docs/experiments/preregs/E55_matrix_robustness.md", "docs/experiments/preregs/E56_predictor_registry.md", "docs/experiments/preregs/E57_grid_variance_ci.md", "docs/experiments/preregs/E58_algebra_audit.md", "docs/experiments/preregs/E59_read_dose_response.md", "docs/experiments/preregs/E60_fitter_determinism.md", "docs/experiments/preregs/E61_randomized_network_null.md", "docs/experiments/preregs/E62_ladder1b_corrected_band.md", "docs/experiments/preregs/E65_training_axis_floor.md", "docs/experiments/preregs/E66b_determinism_floor.md", "docs/experiments/preregs/OS1_operator_space.md", "docs/experiments/preregs/P0_t52_parallel_port.md", "docs/experiments/preregs/R1_grid_rstrip.md", "docs/experiments/preregs/R2_e48_rstrip_arm.md", "docs/experiments/preregs/R3_ladder410_cpu.md", "docs/experiments/preregs/R4_corrections.md", "docs/experiments/preregs/R4b_e36_rstrip.md", "docs/experiments/preregs/R5_corpus_axis_uncertainty.md", "docs/experiments/preregs/R6_within_source_resampling.md", "docs/experiments/preregs/R8_ladder_rstrip_S2.md", "docs/experiments/preregs/R9_permutation_calibrated_min.md", "docs/experiments/preregs/superseded/PREREG_E48_CROSSOVER.md", "docs/reproducibility/ARTIFACTS.md", "docs/reproducibility/PAYLOAD_REDACTIONS.md", "docs/reproducibility/RELEASE.md", "docs/reproducibility/REVIEWER_REPRO.md", "docs/reproducibility/T1_FINDING_E33_BAND.md", "docs/reproducibility/T1_SWEEP_PREDICTIONS.md", "docs/reproducibility/T1_SWEEP_SPEC.md", "docs/reproducibility/review-snapshot-2026-08-19/MANIFEST.sha256", "docs/reproducibility/review-snapshot-2026-08-19/PROVENANCE.md", "docs/reproducibility/review-snapshot-2026-08-19/REDACTIONS.md", "docs/reproducibility/review-snapshot-2026-08-19/review/EXTERNAL_REVIEW.md", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/FINDINGS.md", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/PHASE_A_ACCOUNT.md" ], "dirty_diff_sha256": "ec61dde1905616f407f51d5f6fe3ea37c9c6e3e1ade6998cd0644527d080c600", "describe": "c9c4969-dirty" }, "argv": [ "experiments/cv1_answer_competence.py", "--models", "70m,160m,410m,1b" ], "cwd": ".", "inputs": [ { "path": "jacobian-lens/data/evaluations/lens-eval-multihop.json", "sha256": "50b7e4c9255291c0ca2a8e94615be9f44531fa57bb1a844e4f9616056d987416", "bytes": 21869, "exists": true }, { "path": "jacobian-lens/data/evaluations/lens-eval-order-ops.json", "sha256": "b203206d16ff628152cc86f3838604e06cb54776f3e14fa1c34f150db8bc7560", "bytes": 9589, "exists": true }, { "path": "jacobian-lens/data/evaluations/lens-eval-multilingual.json", "sha256": "fa70b9bd89416a6d8d985a80dc628b109ae6fd3b25b9275c0fc5065d7ff4a0ef", "bytes": 24284, "exists": true } ], "env": { "python": "3.14.2", "torch": "2.13.0", "transformers": "5.14.1", "numpy": "2.5.1", "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", "machine": "arm64", "hostname": "", "tf32": { "allow_tf32_matmul": false, "allow_tf32_cudnn": true, "NVIDIA_TF32_OVERRIDE": null }, "threads": { "torch_num_threads": 10, "torch_num_interop_threads": 14, "OMP_NUM_THREADS": null, "os_cpu_count": 14 } }, "payload_sha256": "8809019f865fe0939cee28bd5d201f7a842b85b9cb5929440541804f3786e74d" } }