{ "experiment": "CV8 \u2014 positional extrapolation: is the corpus effect an extrapolation artifact?", "prereg": "docs/experiments/preregs/CV8_positional_extrapolation.md", "status": "PRE-REGISTERED", "model": "EleutherAI/pythia-410m-deduped", "band": [ 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21 ], "device": "cpu", "smoke": false, "n_sequences_for_G": 128, "prefix_corpus": "Pile-CC", "probe_positions": [ 16, 24, 32, 48, 64, 80, 96, 112, 126, 133, 142, 150, 160, 177 ], "p_read": 142, "in_support_positions": [ 16, 24, 32, 48, 64, 80, 96, 112, 126 ], "out_of_support_positions": [ 133, 142, 150, 160, 177 ], "alignment_by_corpus_by_position": { "Github": { "16": 0.6996252853536956, "24": 0.7195726800527922, "32": 0.7419920931750147, "48": 0.7890212811960222, "64": 0.7941302109133536, "80": 0.8161464913360663, "96": 0.8183671340138088, "112": 0.8229096214682946, "126": 0.8238566421789222, "133": 0.7944473162024065, "142": 0.8388732396731006, "150": 0.8087969634902892, "160": 0.7728870509309129, "177": 0.7256898382376776 }, "Pile-CC": { "16": 0.8655362229126606, "24": 0.8794407644581727, "32": 0.9045990717899588, "48": 0.9609246582531124, "64": 0.9600123937570153, "80": 0.9735561251166504, "96": 0.9736115131621349, "112": 0.9587698643649143, "126": 0.9315715191158161, "133": 0.8844150766354874, "142": 0.8829077025264984, "150": 0.812117650005082, "160": 0.750308456655454, "177": 0.6929892962431711 }, "StackExchange": { "16": 0.8036795767735414, "24": 0.823674148552886, "32": 0.8445555689011814, "48": 0.8826739930722733, "64": 0.8859453230341394, "80": 0.898833215588331, "96": 0.904113715176498, "112": 0.8884164602671668, "126": 0.8710085702476517, "133": 0.8486331253630209, "142": 0.8570446480097353, "150": 0.7892426785265432, "160": 0.7309038381482675, "177": 0.6775588355322955 }, "USPTO_Backgrounds": { "16": 0.7907249900285207, "24": 0.8090482826892232, "32": 0.8344965816173697, "48": 0.8756122774890299, "64": 0.8785620695065707, "80": 0.8933423001911186, "96": 0.8913479278432859, "112": 0.8787880476980111, "126": 0.8573401928967778, "133": 0.827182917778604, "142": 0.8381376088564241, "150": 0.7732960251599772, "160": 0.7201225439786406, "177": 0.6684383565635394 }, "Wikipedia_en": { "16": 0.8116737040343527, "24": 0.8243563752414974, "32": 0.8515287521634032, "48": 0.9105007011285444, "64": 0.9187448191728964, "80": 0.929348254154488, "96": 0.925303206745383, "112": 0.9200323122113367, "126": 0.9014905392400295, "133": 0.8630461395728837, "142": 0.8731856807813917, "150": 0.8077957514475436, "160": 0.7556025082473946, "177": 0.6978742364606902 }, "OOD_arXiv_2023": { "16": 0.7916525423537559, "24": 0.8102770192869214, "32": 0.8335794211271619, "48": 0.8782709376178942, "64": 0.8771631566419636, "80": 0.8915520081502442, "96": 0.8949200856470252, "112": 0.8814222544321175, "126": 0.8556494233519906, "133": 0.8358392088399716, "142": 0.8390375985784466, "150": 0.7702991962986523, "160": 0.7144830722075272, "177": 0.6603754919946767 }, "OOD_CommonPile": { "16": 0.781236677486555, "24": 0.7988611712590944, "32": 0.8232537507439353, "48": 0.8654877791238893, "64": 0.8656894410263634, "80": 0.8812748843918979, "96": 0.8850409193797165, "112": 0.8723165202545101, "126": 0.8502470394124511, "133": 0.833852988611821, "142": 0.8369051828581651, "150": 0.7682897033917105, "160": 0.7138111346577654, "177": 0.6595057776158877 }, "OOD_News_2024": { "16": 0.827480165969923, "24": 0.837726723624863, "32": 0.8632292155316605, "48": 0.9210869498301496, "64": 0.9247255673704081, "80": 0.9362241489898391, "96": 0.9349798506919959, "112": 0.9225137382747429, "126": 0.8975991050504516, "133": 0.8515537919652221, "142": 0.8455001686641376, "150": 0.7801679271847001, "160": 0.7259258340690871, "177": 0.6713449714937855 } }, "read_scores_min_from_r9": { "Github": 0.24049734957516195, "Pile-CC": 0.26239902824163436, "StackExchange": 0.26430597975850106, "USPTO_Backgrounds": 0.26112590581178663, "Wikipedia_en": 0.24160844944417476, "OOD_arXiv_2023": 0.2692730328068137, "OOD_CommonPile": 0.2528811445459723, "OOD_News_2024": 0.25006329715251924 }, "PRIMARY": { "statistic": "Spearman rho between alignment at p_read and the stored min read score", "rho": 0.14285714285714285, "exact_two_sided_p": 0.7520337301587302, "n_permutations": 40320, "critical_abs_rho_at_alpha_0.05": 0.7142857142857143, "minimum_detectable": "n=8 detects only |rho| >= 0.714 at alpha=0.05 two-sided; a moderate relationship would NOT be detected by this design", "corpora_in_order": [ "Github", "OOD_CommonPile", "OOD_News_2024", "OOD_arXiv_2023", "Pile-CC", "StackExchange", "USPTO_Backgrounds", "Wikipedia_en" ], "alignment_at_p_read": [ 0.8388732396731006, 0.8369051828581651, 0.8455001686641376, 0.8390375985784466, 0.8829077025264984, 0.8570446480097353, 0.8381376088564241, 0.8731856807813917 ], "read_score": [ 0.24049734957516195, 0.2528811445459723, 0.25006329715251924, 0.2692730328068137, 0.26239902824163436, 0.26430597975850106, 0.26112590581178663, 0.24160844944417476 ] }, "SECONDARY_alignment_spread": { "between_corpus_sd_at_p_read": 0.01784753488872414, "between_corpus_sd_in_support_p64": 0.049280116927836765, "ratio": 0.3621650272230308, "note": "descriptive; no threshold is attached, per the registration" }, "controls": { "C1_scale_invariance": { "required": "cos_F unchanged to < 1e-9 when G is multiplied by alpha in {0.5, 2, 100}; the control for the target-count artifact that motivated a scale-free metric. AMENDED from '== 0.0', which is unreachable in floating point.", "max_abs_difference": 4.8365533800165394e-11, "tolerance": 1e-09, "fires": true }, "C2_metric_is_not_vacuous": { "required": "in-support alignment >= 0.30 for all 8 corpora AND |alignment| vs a Gaussian matrix <= 0.05. IF THIS FAILS THE INSTRUMENT IS BROKEN AND NO VERDICT ISSUES.", "min_in_support_p64": 0.7941302109133536, "max_abs_vs_random": 0.00020370669755364056, "per_corpus_in_support": { "Github": 0.7941302109133536, "Pile-CC": 0.9600123937570153, "StackExchange": 0.8859453230341394, "USPTO_Backgrounds": 0.8785620695065707, "Wikipedia_en": 0.9187448191728964, "OOD_arXiv_2023": 0.8771631566419636, "OOD_CommonPile": 0.8656894410263634, "OOD_News_2024": 0.9247255673704081 }, "per_corpus_vs_random": { "Github": 8.715056257069297e-05, "Pile-CC": 8.34612596484493e-05, "StackExchange": 0.00020370669755364056, "USPTO_Backgrounds": 7.459009373812329e-05, "Wikipedia_en": 5.197126198105234e-05, "OOD_arXiv_2023": 3.00259731882602e-05, "OOD_CommonPile": -2.5453768332120378e-05, "OOD_News_2024": -0.00011232876610156356 }, "fires": true }, "C3_no_leakage": { "required": "the items building G and the items scored are disjoint", "n_overlap": 0, "n_build": 128, "n_score": 270, "fires": true }, "C4_probe_matches_the_real_read": { "required": "p_read inside CV2's measured prefixed read range, and CV2's n_in_support == 0", "p_read": 142, "cv2_min": 133, "cv2_max": 177, "cv2_median": 142, "cv2_n_in_support": 0, "fires": true }, "C5_band_rule": { "required": "band == floor(0.38L)..floor(0.92L) intersected with layers strictly below the penultimate target layer, at L=24", "band": [ 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21 ], "rule_band": [ 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21 ], "target_layer": 22, "fires": true } }, "controls_fired": { "C1_scale_invariance": true, "C2_metric_is_not_vacuous": true, "C3_no_leakage": true, "C4_probe_matches_the_real_read": true, "C5_band_rule": true }, "VERDICT": "EXTRAPOLATION DOES NOT EXPLAIN THE ORDERING \u2014 exact two-sided p = 0.7520 >= 0.20, rho = +0.143, below the alpha=0.05 critical |rho| of 0.714 for n=8. No STRONG monotone relationship between alignment at position 142 and the read. The positional hedge is retired; this design could not have detected a moderate relationship and the claim is bounded accordingly.", "provenance": { "schema_version": 1, "experiment": "CV8", "utc": "2026-08-31T03:39:53+00:00", "script": { "path": "experiments/cv8_positional_extrapolation.py", "sha256": "7bc5d7908a74bbd6713744b4af830c963396d9402c51b1fb289d7c92d23edb8d", "n_lines": 408 }, "git": { "commit": "99a6dd48ac17cd06f63d77e1e2fa51296cd189f7", "commit_short": "99a6dd48ac17", "git_repository_present": true, "branch": "pythia", "dirty": true, "dirty_files": [ "PAPER_REWRITE_SCOPE.md", "xperiments/e66b_determinism_floor.py" ], "dirty_diff_sha256": "01b0d0f1f9e05dca6bc8dca13e923a4920f8bf53c42c0d1a3cca9d266754795c", "describe": "99a6dd4-dirty" }, "argv": [ "experiments/cv8_positional_extrapolation.py" ], "cwd": ".", "inputs": [ { "path": "results/r9_permutation_calibrated_min.json", "sha256": "eea61cccda689ed7903ca0f24af1961beb18f104fc7226a21f61a2563e3a9ba2", "bytes": 20471, "exists": true }, { "path": "results/cv2_position_support.json", "sha256": "e61248b05d2e8fa4db0262b325e2748579f873326414b1992d5bd4476c136b9c", "bytes": 9152, "exists": true } ], "env": { "python": "3.14.2", "torch": "2.13.0", "transformers": "5.14.1", "numpy": "2.5.1", "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", "machine": "arm64", "hostname": "", "tf32": { "allow_tf32_matmul": false, "allow_tf32_cudnn": false, "NVIDIA_TF32_OVERRIDE": null }, "threads": { "torch_num_threads": 8, "torch_num_interop_threads": 14, "OMP_NUM_THREADS": null, "os_cpu_count": 14 } }, "payload_sha256": "a950c3b88b5401cefca4116adf35303854131b5bd45dfd084c0efe00325e4501" } }