| { |
| "experiment": "CV2 \u2014 positional support audit", |
| "status": "MEASUREMENT ONLY \u2014 no registered decision rule; this establishes or refutes the mismatch flagged in docs/validity/CONSTRUCT_VALIDITY.md section 4.2", |
| "model": "EleutherAI/pythia-410m-deduped", |
| "fitting_support": { |
| "source": "src/fastfit.py:116 -> jlens.fitting.valid_position_mask(seq_len, skip_first)", |
| "skip_first": 16, |
| "max_seq_len": 128, |
| "excludes": "first skip_first positions (attention sink) and the final position (no next-token target)", |
| "inclusive_range": [ |
| 16, |
| 126 |
| ] |
| }, |
| "readout_convention": "STRIPPED (corrected). Position computed on the item alone, then shifted by the prefix length \u2014 matches t36_qladder.py:27-31", |
| "by_set": { |
| "association": { |
| "n_items": 102, |
| "readout_rule": "last", |
| "median_prompt_tokens": 28, |
| "Q0": { |
| "median_read_pos": 28, |
| "min": 19, |
| "max": 49, |
| "n_in_support": 102, |
| "frac_in_support": 1.0 |
| }, |
| "prefixed": { |
| "median_read_pos": 156, |
| "min": 147, |
| "max": 177, |
| "n_in_support": 0, |
| "frac_in_support": 0.0 |
| } |
| }, |
| "multihop": { |
| "n_items": 93, |
| "readout_rule": "last", |
| "median_prompt_tokens": 15, |
| "Q0": { |
| "median_read_pos": 15, |
| "min": 8, |
| "max": 40, |
| "n_in_support": 45, |
| "frac_in_support": 0.4838709677419355 |
| }, |
| "prefixed": { |
| "median_read_pos": 143, |
| "min": 136, |
| "max": 168, |
| "n_in_support": 0, |
| "frac_in_support": 0.0 |
| } |
| }, |
| "multilingual": { |
| "n_items": 107, |
| "readout_rule": "last", |
| "median_prompt_tokens": 10, |
| "Q0": { |
| "median_read_pos": 10, |
| "min": 5, |
| "max": 26, |
| "n_in_support": 6, |
| "frac_in_support": 0.056074766355140186 |
| }, |
| "prefixed": { |
| "median_read_pos": 138, |
| "min": 133, |
| "max": 154, |
| "n_in_support": 0, |
| "frac_in_support": 0.0 |
| } |
| }, |
| "order-ops": { |
| "n_items": 55, |
| "readout_rule": "last", |
| "median_prompt_tokens": 8, |
| "Q0": { |
| "median_read_pos": 8, |
| "min": 6, |
| "max": 18, |
| "n_in_support": 1, |
| "frac_in_support": 0.01818181818181818 |
| }, |
| "prefixed": { |
| "median_read_pos": 136, |
| "min": 134, |
| "max": 146, |
| "n_in_support": 0, |
| "frac_in_support": 0.0 |
| } |
| }, |
| "poetry": { |
| "n_items": 98, |
| "readout_rule": "last_newline", |
| "median_prompt_tokens": 26, |
| "Q0": { |
| "median_read_pos": 18, |
| "min": 16, |
| "max": 22, |
| "n_in_support": 98, |
| "frac_in_support": 1.0 |
| }, |
| "prefixed": { |
| "median_read_pos": 146, |
| "min": 144, |
| "max": 150, |
| "n_in_support": 0, |
| "frac_in_support": 0.0 |
| } |
| }, |
| "typo": { |
| "n_items": 96, |
| "readout_rule": "last", |
| "median_prompt_tokens": 8, |
| "Q0": { |
| "median_read_pos": 8, |
| "min": 6, |
| "max": 14, |
| "n_in_support": 0, |
| "frac_in_support": 0.0 |
| }, |
| "prefixed": { |
| "median_read_pos": 136, |
| "min": 134, |
| "max": 142, |
| "n_in_support": 0, |
| "frac_in_support": 0.0 |
| } |
| } |
| }, |
| "overall": { |
| "Q0": { |
| "n": 551, |
| "median_read_pos": 14, |
| "min": 5, |
| "max": 49, |
| "n_in_support": 252, |
| "frac_in_support": 0.4573502722323049 |
| }, |
| "prefixed": { |
| "n": 551, |
| "median_read_pos": 142, |
| "min": 133, |
| "max": 177, |
| "n_in_support": 0, |
| "frac_in_support": 0.0 |
| } |
| }, |
| "VERDICT": "Q0 reads inside the fitting support on 45.7% of items (median position 14); prefixed rungs on 0.0% (median 142). Fitting support is positions 16..126.", |
| "provenance": { |
| "schema_version": 1, |
| "experiment": "CV2", |
| "utc": "2026-08-30T00:19:04+00:00", |
| "script": { |
| "path": "experiments/cv2_position_support.py", |
| "sha256": "fee6460d8d930d3c9ebc138a29def2533ca5bd8fc2392daceed2b0644e20abbc", |
| "n_lines": 145 |
| }, |
| "git": { |
| "commit": "c9c49697e295c0fec4225552836fc9733b279f75", |
| "commit_short": "c9c49697e295", |
| "git_repository_present": true, |
| "branch": "pythia", |
| "dirty": true, |
| "dirty_files": [ |
| "CLAUDE.md", |
| "README.md", |
| "corpora/manifest_ood.json", |
| "docs/HANDOFF.md", |
| "docs/REPO_REVIEW_PROMPT.md", |
| "docs/context/AGGREGATION_POLICY.md", |
| "docs/context/CONTEXT.md", |
| "docs/context/IMPLEMENTATION_RULES.md", |
| "docs/context/STATE.md", |
| "docs/experiments/INDEX.md", |
| "docs/experiments/METRICS.md", |
| "docs/experiments/PROPOSED.md", |
| "docs/experiments/PROVENANCE.md", |
| "docs/experiments/descriptions/E64.md", |
| "docs/experiments/descriptions/EN1.md", |
| "docs/experiments/preregs/CV3_margins.md", |
| "docs/experiments/preregs/CV4_capability_ladder.md", |
| "docs/experiments/preregs/CV5_metric_sensitivity.md", |
| "docs/experiments/preregs/CV6_per_family_ladder.md", |
| "docs/experiments/preregs/CV7_1b_rung.md", |
| "docs/experiments/preregs/DN1_design_nulls.md", |
| "docs/experiments/preregs/E33_logit_baseline.md", |
| "docs/experiments/preregs/E36_qladder.md", |
| "docs/experiments/preregs/E48_crossover.md", |
| "docs/experiments/preregs/E48b_containment.md", |
| "docs/experiments/preregs/E48c_exposure_vs_read.md", |
| "docs/experiments/preregs/E51_interaction.md", |
| "docs/experiments/preregs/E52_factorial.md", |
| "docs/experiments/preregs/E53_ladder_summary.md", |
| "docs/experiments/preregs/E55_matrix_robustness.md", |
| "docs/experiments/preregs/E56_predictor_registry.md", |
| "docs/experiments/preregs/E57_grid_variance_ci.md", |
| "docs/experiments/preregs/E58_algebra_audit.md", |
| "docs/experiments/preregs/E59_read_dose_response.md", |
| "docs/experiments/preregs/E60_fitter_determinism.md", |
| "docs/experiments/preregs/E61_randomized_network_null.md", |
| "docs/experiments/preregs/E62_ladder1b_corrected_band.md", |
| "docs/experiments/preregs/E65_training_axis_floor.md", |
| "docs/experiments/preregs/E66b_determinism_floor.md", |
| "docs/experiments/preregs/OS1_operator_space.md", |
| "docs/experiments/preregs/P0_t52_parallel_port.md", |
| "docs/experiments/preregs/R1_grid_rstrip.md", |
| "docs/experiments/preregs/R2_e48_rstrip_arm.md", |
| "docs/experiments/preregs/R3_ladder410_cpu.md", |
| "docs/experiments/preregs/R4_corrections.md", |
| "docs/experiments/preregs/R4b_e36_rstrip.md", |
| "docs/experiments/preregs/R5_corpus_axis_uncertainty.md", |
| "docs/experiments/preregs/R6_within_source_resampling.md", |
| "docs/experiments/preregs/R8_ladder_rstrip_S2.md", |
| "docs/experiments/preregs/R9_permutation_calibrated_min.md", |
| "docs/experiments/preregs/superseded/PREREG_E48_CROSSOVER.md", |
| "docs/reproducibility/ARTIFACTS.md", |
| "docs/reproducibility/PAYLOAD_REDACTIONS.md", |
| "docs/reproducibility/RELEASE.md", |
| "docs/reproducibility/REVIEWER_REPRO.md", |
| "docs/reproducibility/T1_SWEEP_SPEC.md", |
| "docs/reproducibility/review-snapshot-2026-08-19/MANIFEST.sha256", |
| "docs/reproducibility/review-snapshot-2026-08-19/PROVENANCE.md", |
| "docs/reproducibility/review-snapshot-2026-08-19/REDACTIONS.md", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/EXTERNAL_REVIEW.md", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/FINDINGS.md", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/PHASE_A_ACCOUNT.md", |
| "docs/reviews/REVIEW_INSIGHTS_2026-08-25.md", |
| "docs/validity/CONSTRUCT_VALIDITY.md" |
| ], |
| "dirty_diff_sha256": "6b143c292229b3d2480d1b73243675f0db5e23b27f66e4215e0a9c1947a761a8", |
| "describe": "c9c4969-dirty" |
| }, |
| "argv": [ |
| "experiments/cv2_position_support.py" |
| ], |
| "cwd": ".", |
| "inputs": [ |
| { |
| "path": "jacobian-lens/data/evaluations/lens-eval-association.json", |
| "sha256": "d1a98cd4911b594282e74168091c77d849dae18ffe2acb5761074853f327d71c", |
| "bytes": 24228, |
| "exists": true |
| }, |
| { |
| "path": "jacobian-lens/data/evaluations/lens-eval-multihop.json", |
| "sha256": "50b7e4c9255291c0ca2a8e94615be9f44531fa57bb1a844e4f9616056d987416", |
| "bytes": 21869, |
| "exists": true |
| }, |
| { |
| "path": "jacobian-lens/data/evaluations/lens-eval-multilingual.json", |
| "sha256": "fa70b9bd89416a6d8d985a80dc628b109ae6fd3b25b9275c0fc5065d7ff4a0ef", |
| "bytes": 24284, |
| "exists": true |
| }, |
| { |
| "path": "jacobian-lens/data/evaluations/lens-eval-order-ops.json", |
| "sha256": "b203206d16ff628152cc86f3838604e06cb54776f3e14fa1c34f150db8bc7560", |
| "bytes": 9589, |
| "exists": true |
| }, |
| { |
| "path": "jacobian-lens/data/evaluations/lens-eval-poetry.json", |
| "sha256": "6aeb3415c5a5c3f3827c9efe63f006de02f5ef39a816bbac68e15e733aba60cc", |
| "bytes": 21533, |
| "exists": true |
| }, |
| { |
| "path": "jacobian-lens/data/evaluations/lens-eval-typo.json", |
| "sha256": "9d05e16b7234a57d0773d120a4e1c4e94fd3bc2235a8125d4200a70e60ab17aa", |
| "bytes": 14646, |
| "exists": true |
| } |
| ], |
| "env": { |
| "python": "3.14.2", |
| "torch": "2.13.0", |
| "transformers": "5.14.1", |
| "numpy": "2.5.1", |
| "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", |
| "machine": "arm64", |
| "hostname": "<redacted for double-blind review>", |
| "tf32": { |
| "allow_tf32_matmul": false, |
| "allow_tf32_cudnn": true, |
| "NVIDIA_TF32_OVERRIDE": null |
| }, |
| "threads": { |
| "torch_num_threads": 10, |
| "torch_num_interop_threads": 14, |
| "OMP_NUM_THREADS": null, |
| "os_cpu_count": 14 |
| } |
| }, |
| "payload_sha256": "1eb1ab9b76e77d6060876485b34291cf9e1c525a755ae0166d5f7ec98f6d0d92" |
| } |
| } |
