{ "experiment": "CV2 \u2014 positional support audit", "status": "MEASUREMENT ONLY \u2014 no registered decision rule; this establishes or refutes the mismatch flagged in docs/validity/CONSTRUCT_VALIDITY.md section 4.2", "model": "EleutherAI/pythia-410m-deduped", "fitting_support": { "source": "src/fastfit.py:116 -> jlens.fitting.valid_position_mask(seq_len, skip_first)", "skip_first": 16, "max_seq_len": 128, "excludes": "first skip_first positions (attention sink) and the final position (no next-token target)", "inclusive_range": [ 16, 126 ] }, "readout_convention": "STRIPPED (corrected). Position computed on the item alone, then shifted by the prefix length \u2014 matches t36_qladder.py:27-31", "by_set": { "association": { "n_items": 102, "readout_rule": "last", "median_prompt_tokens": 28, "Q0": { "median_read_pos": 28, "min": 19, "max": 49, "n_in_support": 102, "frac_in_support": 1.0 }, "prefixed": { "median_read_pos": 156, "min": 147, "max": 177, "n_in_support": 0, "frac_in_support": 0.0 } }, "multihop": { "n_items": 93, "readout_rule": "last", "median_prompt_tokens": 15, "Q0": { "median_read_pos": 15, "min": 8, "max": 40, "n_in_support": 45, "frac_in_support": 0.4838709677419355 }, "prefixed": { "median_read_pos": 143, "min": 136, "max": 168, "n_in_support": 0, "frac_in_support": 0.0 } }, "multilingual": { "n_items": 107, "readout_rule": "last", "median_prompt_tokens": 10, "Q0": { "median_read_pos": 10, "min": 5, "max": 26, "n_in_support": 6, "frac_in_support": 0.056074766355140186 }, "prefixed": { "median_read_pos": 138, "min": 133, "max": 154, "n_in_support": 0, "frac_in_support": 0.0 } }, "order-ops": { "n_items": 55, "readout_rule": "last", "median_prompt_tokens": 8, "Q0": { "median_read_pos": 8, "min": 6, "max": 18, "n_in_support": 1, "frac_in_support": 0.01818181818181818 }, "prefixed": { "median_read_pos": 136, "min": 134, "max": 146, "n_in_support": 0, "frac_in_support": 0.0 } }, "poetry": { "n_items": 98, "readout_rule": "last_newline", "median_prompt_tokens": 26, "Q0": { "median_read_pos": 18, "min": 16, "max": 22, "n_in_support": 98, "frac_in_support": 1.0 }, "prefixed": { "median_read_pos": 146, "min": 144, "max": 150, "n_in_support": 0, "frac_in_support": 0.0 } }, "typo": { "n_items": 96, "readout_rule": "last", "median_prompt_tokens": 8, "Q0": { "median_read_pos": 8, "min": 6, "max": 14, "n_in_support": 0, "frac_in_support": 0.0 }, "prefixed": { "median_read_pos": 136, "min": 134, "max": 142, "n_in_support": 0, "frac_in_support": 0.0 } } }, "overall": { "Q0": { "n": 551, "median_read_pos": 14, "min": 5, "max": 49, "n_in_support": 252, "frac_in_support": 0.4573502722323049 }, "prefixed": { "n": 551, "median_read_pos": 142, "min": 133, "max": 177, "n_in_support": 0, "frac_in_support": 0.0 } }, "VERDICT": "Q0 reads inside the fitting support on 45.7% of items (median position 14); prefixed rungs on 0.0% (median 142). Fitting support is positions 16..126.", "provenance": { "schema_version": 1, "experiment": "CV2", "utc": "2026-08-30T00:19:04+00:00", "script": { "path": "experiments/cv2_position_support.py", "sha256": "fee6460d8d930d3c9ebc138a29def2533ca5bd8fc2392daceed2b0644e20abbc", "n_lines": 145 }, "git": { "commit": "c9c49697e295c0fec4225552836fc9733b279f75", "commit_short": "c9c49697e295", "git_repository_present": true, "branch": "pythia", "dirty": true, "dirty_files": [ "CLAUDE.md", "README.md", "corpora/manifest_ood.json", "docs/HANDOFF.md", "docs/REPO_REVIEW_PROMPT.md", "docs/context/AGGREGATION_POLICY.md", "docs/context/CONTEXT.md", "docs/context/IMPLEMENTATION_RULES.md", "docs/context/STATE.md", "docs/experiments/INDEX.md", "docs/experiments/METRICS.md", "docs/experiments/PROPOSED.md", "docs/experiments/PROVENANCE.md", "docs/experiments/descriptions/E64.md", "docs/experiments/descriptions/EN1.md", "docs/experiments/preregs/CV3_margins.md", "docs/experiments/preregs/CV4_capability_ladder.md", "docs/experiments/preregs/CV5_metric_sensitivity.md", "docs/experiments/preregs/CV6_per_family_ladder.md", "docs/experiments/preregs/CV7_1b_rung.md", "docs/experiments/preregs/DN1_design_nulls.md", "docs/experiments/preregs/E33_logit_baseline.md", "docs/experiments/preregs/E36_qladder.md", "docs/experiments/preregs/E48_crossover.md", "docs/experiments/preregs/E48b_containment.md", "docs/experiments/preregs/E48c_exposure_vs_read.md", "docs/experiments/preregs/E51_interaction.md", "docs/experiments/preregs/E52_factorial.md", "docs/experiments/preregs/E53_ladder_summary.md", "docs/experiments/preregs/E55_matrix_robustness.md", "docs/experiments/preregs/E56_predictor_registry.md", "docs/experiments/preregs/E57_grid_variance_ci.md", "docs/experiments/preregs/E58_algebra_audit.md", "docs/experiments/preregs/E59_read_dose_response.md", "docs/experiments/preregs/E60_fitter_determinism.md", "docs/experiments/preregs/E61_randomized_network_null.md", "docs/experiments/preregs/E62_ladder1b_corrected_band.md", "docs/experiments/preregs/E65_training_axis_floor.md", "docs/experiments/preregs/E66b_determinism_floor.md", "docs/experiments/preregs/OS1_operator_space.md", "docs/experiments/preregs/P0_t52_parallel_port.md", "docs/experiments/preregs/R1_grid_rstrip.md", "docs/experiments/preregs/R2_e48_rstrip_arm.md", "docs/experiments/preregs/R3_ladder410_cpu.md", "docs/experiments/preregs/R4_corrections.md", "docs/experiments/preregs/R4b_e36_rstrip.md", "docs/experiments/preregs/R5_corpus_axis_uncertainty.md", "docs/experiments/preregs/R6_within_source_resampling.md", "docs/experiments/preregs/R8_ladder_rstrip_S2.md", "docs/experiments/preregs/R9_permutation_calibrated_min.md", "docs/experiments/preregs/superseded/PREREG_E48_CROSSOVER.md", "docs/reproducibility/ARTIFACTS.md", "docs/reproducibility/PAYLOAD_REDACTIONS.md", "docs/reproducibility/RELEASE.md", "docs/reproducibility/REVIEWER_REPRO.md", "docs/reproducibility/T1_SWEEP_SPEC.md", "docs/reproducibility/review-snapshot-2026-08-19/MANIFEST.sha256", "docs/reproducibility/review-snapshot-2026-08-19/PROVENANCE.md", "docs/reproducibility/review-snapshot-2026-08-19/REDACTIONS.md", "docs/reproducibility/review-snapshot-2026-08-19/review/EXTERNAL_REVIEW.md", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/FINDINGS.md", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/PHASE_A_ACCOUNT.md", "docs/reviews/REVIEW_INSIGHTS_2026-08-25.md", "docs/validity/CONSTRUCT_VALIDITY.md" ], "dirty_diff_sha256": "6b143c292229b3d2480d1b73243675f0db5e23b27f66e4215e0a9c1947a761a8", "describe": "c9c4969-dirty" }, "argv": [ "experiments/cv2_position_support.py" ], "cwd": ".", "inputs": [ { "path": "jacobian-lens/data/evaluations/lens-eval-association.json", "sha256": "d1a98cd4911b594282e74168091c77d849dae18ffe2acb5761074853f327d71c", "bytes": 24228, "exists": true }, { "path": "jacobian-lens/data/evaluations/lens-eval-multihop.json", "sha256": "50b7e4c9255291c0ca2a8e94615be9f44531fa57bb1a844e4f9616056d987416", "bytes": 21869, "exists": true }, { "path": "jacobian-lens/data/evaluations/lens-eval-multilingual.json", "sha256": "fa70b9bd89416a6d8d985a80dc628b109ae6fd3b25b9275c0fc5065d7ff4a0ef", "bytes": 24284, "exists": true }, { "path": "jacobian-lens/data/evaluations/lens-eval-order-ops.json", "sha256": "b203206d16ff628152cc86f3838604e06cb54776f3e14fa1c34f150db8bc7560", "bytes": 9589, "exists": true }, { "path": "jacobian-lens/data/evaluations/lens-eval-poetry.json", "sha256": "6aeb3415c5a5c3f3827c9efe63f006de02f5ef39a816bbac68e15e733aba60cc", "bytes": 21533, "exists": true }, { "path": "jacobian-lens/data/evaluations/lens-eval-typo.json", "sha256": "9d05e16b7234a57d0773d120a4e1c4e94fd3bc2235a8125d4200a70e60ab17aa", "bytes": 14646, "exists": true } ], "env": { "python": "3.14.2", "torch": "2.13.0", "transformers": "5.14.1", "numpy": "2.5.1", "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", "machine": "arm64", "hostname": "", "tf32": { "allow_tf32_matmul": false, "allow_tf32_cudnn": true, "NVIDIA_TF32_OVERRIDE": null }, "threads": { "torch_num_threads": 10, "torch_num_interop_threads": 14, "OMP_NUM_THREADS": null, "os_cpu_count": 14 } }, "payload_sha256": "1eb1ab9b76e77d6060876485b34291cf9e1c525a755ae0166d5f7ec98f6d0d92" } }