{ "experiment": "CV8 \u2014 positional extrapolation: is the corpus effect an extrapolation artifact?", "prereg": "docs/experiments/preregs/CV8_positional_extrapolation.md", "status": "PRE-REGISTERED", "model": "EleutherAI/pythia-410m-deduped", "band": [ 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21 ], "device": "cpu", "smoke": false, "n_sequences_for_G": 128, "prefix_corpus": "Github", "probe_positions": [ 16, 24, 32, 48, 64, 80, 96, 112, 126, 133, 142, 150, 160, 177 ], "p_read": 142, "in_support_positions": [ 16, 24, 32, 48, 64, 80, 96, 112, 126 ], "out_of_support_positions": [ 133, 142, 150, 160, 177 ], "alignment_by_corpus_by_position": { "Github": { "16": 0.8199622410112316, "24": 0.8649228662025228, "32": 0.8823327600490843, "48": 0.9208955206003056, "64": 0.9392306171621081, "80": 0.9484260866168507, "96": 0.9504478884880948, "112": 0.9423929234741516, "126": 0.9310935599306583, "133": 0.8484558265334022, "142": 0.8928635661375124, "150": 0.8991461850577471, "160": 0.863023156475018, "177": 0.8091136683178206 }, "Pile-CC": { "16": 0.7388377966330032, "24": 0.7623900261264125, "32": 0.7401483099189051, "48": 0.8078829412811408, "64": 0.8026415395543312, "80": 0.8074120082320871, "96": 0.8188838080903893, "112": 0.8275872410947619, "126": 0.8261497692178901, "133": 0.8825382212439484, "142": 0.8410997638017006, "150": 0.784536015586303, "160": 0.737921293108437, "177": 0.7043631819379887 }, "StackExchange": { "16": 0.7868950844126045, "24": 0.8172987029771465, "32": 0.7958271283933123, "48": 0.8631015421781737, "64": 0.8521835524590238, "80": 0.8567536478592028, "96": 0.865747722698461, "112": 0.869334769490467, "126": 0.8632586365641963, "133": 0.8908658940965233, "142": 0.8680869951100778, "150": 0.8141380848365287, "160": 0.7637898622813123, "177": 0.72162493963662 }, "USPTO_Backgrounds": { "16": 0.7250339105496532, "24": 0.7466318620527843, "32": 0.7217538165312588, "48": 0.7893456531556848, "64": 0.7835063101091788, "80": 0.7881745586910742, "96": 0.8035122217701587, "112": 0.8106891547649236, "126": 0.8059748422771726, "133": 0.8468851304825057, "142": 0.8175291862732885, "150": 0.765009421906371, "160": 0.7211020727707582, "177": 0.6890500060955719 }, "Wikipedia_en": { "16": 0.714737290764607, "24": 0.7414217829325931, "32": 0.7282756412878298, "48": 0.7893573136174032, "64": 0.7913875814589846, "80": 0.8014941703581413, "96": 0.8124232223799928, "112": 0.8217904107612356, "126": 0.820110692737911, "133": 0.8634235496340769, "142": 0.8326451279763215, "150": 0.7839842876667025, "160": 0.7441040846658752, "177": 0.7115479365349467 }, "OOD_arXiv_2023": { "16": 0.7314797724797859, "24": 0.7538180293815879, "32": 0.7296685194098056, "48": 0.7965723238808289, "64": 0.7904637129490234, "80": 0.792464811600057, "96": 0.8055670247052423, "112": 0.8121151935399167, "126": 0.8085203811055055, "133": 0.8561239410946141, "142": 0.8231678802397187, "150": 0.7658832922857227, "160": 0.7208183538688316, "177": 0.6866824113541039 }, "OOD_CommonPile": { "16": 0.7274252644273965, "24": 0.7487318124617317, "32": 0.7244637094978772, "48": 0.7910418690218445, "64": 0.786705942925807, "80": 0.7893214492024896, "96": 0.8042403273110403, "112": 0.8103238690236166, "126": 0.8063952401312486, "133": 0.8562419725480844, "142": 0.8221690899817258, "150": 0.7646649434641972, "160": 0.7195846531182958, "177": 0.6856726030758462 }, "OOD_News_2024": { "16": 0.6930364953152686, "24": 0.714579362305328, "32": 0.6958332436342336, "48": 0.759202188142498, "64": 0.7562811167705011, "80": 0.7640614480227068, "96": 0.7759056468667923, "112": 0.7856216585197024, "126": 0.7848264476594219, "133": 0.8403997315239466, "142": 0.7984675936770528, "150": 0.7452955111728733, "160": 0.7056315182522144, "177": 0.675980543031208 } }, "read_scores_min_from_r9": { "Github": 0.24049734957516195, "Pile-CC": 0.26239902824163436, "StackExchange": 0.26430597975850106, "USPTO_Backgrounds": 0.26112590581178663, "Wikipedia_en": 0.24160844944417476, "OOD_arXiv_2023": 0.2692730328068137, "OOD_CommonPile": 0.2528811445459723, "OOD_News_2024": 0.25006329715251924 }, "PRIMARY": { "statistic": "Spearman rho between alignment at p_read and the stored min read score", "rho": -0.047619047619047616, "exact_two_sided_p": 0.9348710317460317, "n_permutations": 40320, "critical_abs_rho_at_alpha_0.05": 0.7142857142857143, "minimum_detectable": "n=8 detects only |rho| >= 0.714 at alpha=0.05 two-sided; a moderate relationship would NOT be detected by this design", "corpora_in_order": [ "Github", "OOD_CommonPile", "OOD_News_2024", "OOD_arXiv_2023", "Pile-CC", "StackExchange", "USPTO_Backgrounds", "Wikipedia_en" ], "alignment_at_p_read": [ 0.8928635661375124, 0.8221690899817258, 0.7984675936770528, 0.8231678802397187, 0.8410997638017006, 0.8680869951100778, 0.8175291862732885, 0.8326451279763215 ], "read_score": [ 0.24049734957516195, 0.2528811445459723, 0.25006329715251924, 0.2692730328068137, 0.26239902824163436, 0.26430597975850106, 0.26112590581178663, 0.24160844944417476 ] }, "SECONDARY_alignment_spread": { "between_corpus_sd_at_p_read": 0.030232561695707077, "between_corpus_sd_in_support_p64": 0.05773061221794703, "ratio": 0.5236833723774111, "note": "descriptive; no threshold is attached, per the registration" }, "controls": { "C1_scale_invariance": { "required": "cos_F unchanged to < 1e-9 when G is multiplied by alpha in {0.5, 2, 100}; the control for the target-count artifact that motivated a scale-free metric. AMENDED from '== 0.0', which is unreachable in floating point.", "max_abs_difference": 7.949996216893851e-11, "tolerance": 1e-09, "fires": true }, "C2_metric_is_not_vacuous": { "required": "in-support alignment >= 0.30 for all 8 corpora AND |alignment| vs a Gaussian matrix <= 0.05. IF THIS FAILS THE INSTRUMENT IS BROKEN AND NO VERDICT ISSUES.", "min_in_support_p64": 0.7562811167705011, "max_abs_vs_random": 0.00020370669755364056, "per_corpus_in_support": { "Github": 0.9392306171621081, "Pile-CC": 0.8026415395543312, "StackExchange": 0.8521835524590238, "USPTO_Backgrounds": 0.7835063101091788, "Wikipedia_en": 0.7913875814589846, "OOD_arXiv_2023": 0.7904637129490234, "OOD_CommonPile": 0.786705942925807, "OOD_News_2024": 0.7562811167705011 }, "per_corpus_vs_random": { "Github": 8.715056257069297e-05, "Pile-CC": 8.34612596484493e-05, "StackExchange": 0.00020370669755364056, "USPTO_Backgrounds": 7.459009373812329e-05, "Wikipedia_en": 5.197126198105234e-05, "OOD_arXiv_2023": 3.00259731882602e-05, "OOD_CommonPile": -2.5453768332120378e-05, "OOD_News_2024": -0.00011232876610156356 }, "fires": true }, "C3_no_leakage": { "required": "the items building G and the items scored are disjoint", "n_overlap": 0, "n_build": 128, "n_score": 270, "fires": true }, "C4_probe_matches_the_real_read": { "required": "p_read inside CV2's measured prefixed read range, and CV2's n_in_support == 0", "p_read": 142, "cv2_min": 133, "cv2_max": 177, "cv2_median": 142, "cv2_n_in_support": 0, "fires": true }, "C5_band_rule": { "required": "band == floor(0.38L)..floor(0.92L) intersected with layers strictly below the penultimate target layer, at L=24", "band": [ 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21 ], "rule_band": [ 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21 ], "target_layer": 22, "fires": true } }, "controls_fired": { "C1_scale_invariance": true, "C2_metric_is_not_vacuous": true, "C3_no_leakage": true, "C4_probe_matches_the_real_read": true, "C5_band_rule": true }, "VERDICT": "EXTRAPOLATION DOES NOT EXPLAIN THE ORDERING \u2014 exact two-sided p = 0.9349 >= 0.20, rho = -0.048, below the alpha=0.05 critical |rho| of 0.714 for n=8. No STRONG monotone relationship between alignment at position 142 and the read. The positional hedge is retired; this design could not have detected a moderate relationship and the claim is bounded accordingly.", "provenance": { "schema_version": 1, "experiment": "CV8", "utc": "2026-08-31T05:58:45+00:00", "script": { "path": "experiments/cv8_positional_extrapolation.py", "sha256": "7bc5d7908a74bbd6713744b4af830c963396d9402c51b1fb289d7c92d23edb8d", "n_lines": 408 }, "git": { "commit": "9770d208af381e9579ae259d49d95c0c0a5b9a33", "commit_short": "9770d208af38", "git_repository_present": true, "branch": "pythia", "dirty": true, "dirty_files": [ "research-paper-writing/" ], "dirty_diff_sha256": null, "describe": "9770d20" }, "argv": [ "experiments/cv8_positional_extrapolation.py", "--prefix-corpus", "Github", "--out", "results/cv8_positional_extrapolation_github.json" ], "cwd": ".", "inputs": [ { "path": "results/r9_permutation_calibrated_min.json", "sha256": "eea61cccda689ed7903ca0f24af1961beb18f104fc7226a21f61a2563e3a9ba2", "bytes": 20471, "exists": true }, { "path": "results/cv2_position_support.json", "sha256": "e61248b05d2e8fa4db0262b325e2748579f873326414b1992d5bd4476c136b9c", "bytes": 9152, "exists": true } ], "env": { "python": "3.14.2", "torch": "2.13.0", "transformers": "5.14.1", "numpy": "2.5.1", "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", "machine": "arm64", "hostname": "", "tf32": { "allow_tf32_matmul": false, "allow_tf32_cudnn": false, "NVIDIA_TF32_OVERRIDE": null }, "threads": { "torch_num_threads": 8, "torch_num_interop_threads": 14, "OMP_NUM_THREADS": null, "os_cpu_count": 14 } }, "payload_sha256": "db838d5ea323fb6ef2b73cec85b2d427d891d37340a801e138f2bf20f0b01d15" } }