| { |
| "experiment": "CV8 \u2014 positional extrapolation: is the corpus effect an extrapolation artifact?", |
| "prereg": "docs/experiments/preregs/CV8_positional_extrapolation.md", |
| "status": "PRE-REGISTERED", |
| "model": "EleutherAI/pythia-410m-deduped", |
| "band": [ |
| 9, |
| 10, |
| 11, |
| 12, |
| 13, |
| 14, |
| 15, |
| 16, |
| 17, |
| 18, |
| 19, |
| 20, |
| 21 |
| ], |
| "device": "cpu", |
| "smoke": false, |
| "n_sequences_for_G": 128, |
| "prefix_corpus": "Pile-CC", |
| "probe_positions": [ |
| 16, |
| 24, |
| 32, |
| 48, |
| 64, |
| 80, |
| 96, |
| 112, |
| 126, |
| 133, |
| 142, |
| 150, |
| 160, |
| 177 |
| ], |
| "p_read": 142, |
| "in_support_positions": [ |
| 16, |
| 24, |
| 32, |
| 48, |
| 64, |
| 80, |
| 96, |
| 112, |
| 126 |
| ], |
| "out_of_support_positions": [ |
| 133, |
| 142, |
| 150, |
| 160, |
| 177 |
| ], |
| "alignment_by_corpus_by_position": { |
| "Github": { |
| "16": 0.6996252853536956, |
| "24": 0.7195726800527922, |
| "32": 0.7419920931750147, |
| "48": 0.7890212811960222, |
| "64": 0.7941302109133536, |
| "80": 0.8161464913360663, |
| "96": 0.8183671340138088, |
| "112": 0.8229096214682946, |
| "126": 0.8238566421789222, |
| "133": 0.7944473162024065, |
| "142": 0.8388732396731006, |
| "150": 0.8087969634902892, |
| "160": 0.7728870509309129, |
| "177": 0.7256898382376776 |
| }, |
| "Pile-CC": { |
| "16": 0.8655362229126606, |
| "24": 0.8794407644581727, |
| "32": 0.9045990717899588, |
| "48": 0.9609246582531124, |
| "64": 0.9600123937570153, |
| "80": 0.9735561251166504, |
| "96": 0.9736115131621349, |
| "112": 0.9587698643649143, |
| "126": 0.9315715191158161, |
| "133": 0.8844150766354874, |
| "142": 0.8829077025264984, |
| "150": 0.812117650005082, |
| "160": 0.750308456655454, |
| "177": 0.6929892962431711 |
| }, |
| "StackExchange": { |
| "16": 0.8036795767735414, |
| "24": 0.823674148552886, |
| "32": 0.8445555689011814, |
| "48": 0.8826739930722733, |
| "64": 0.8859453230341394, |
| "80": 0.898833215588331, |
| "96": 0.904113715176498, |
| "112": 0.8884164602671668, |
| "126": 0.8710085702476517, |
| "133": 0.8486331253630209, |
| "142": 0.8570446480097353, |
| "150": 0.7892426785265432, |
| "160": 0.7309038381482675, |
| "177": 0.6775588355322955 |
| }, |
| "USPTO_Backgrounds": { |
| "16": 0.7907249900285207, |
| "24": 0.8090482826892232, |
| "32": 0.8344965816173697, |
| "48": 0.8756122774890299, |
| "64": 0.8785620695065707, |
| "80": 0.8933423001911186, |
| "96": 0.8913479278432859, |
| "112": 0.8787880476980111, |
| "126": 0.8573401928967778, |
| "133": 0.827182917778604, |
| "142": 0.8381376088564241, |
| "150": 0.7732960251599772, |
| "160": 0.7201225439786406, |
| "177": 0.6684383565635394 |
| }, |
| "Wikipedia_en": { |
| "16": 0.8116737040343527, |
| "24": 0.8243563752414974, |
| "32": 0.8515287521634032, |
| "48": 0.9105007011285444, |
| "64": 0.9187448191728964, |
| "80": 0.929348254154488, |
| "96": 0.925303206745383, |
| "112": 0.9200323122113367, |
| "126": 0.9014905392400295, |
| "133": 0.8630461395728837, |
| "142": 0.8731856807813917, |
| "150": 0.8077957514475436, |
| "160": 0.7556025082473946, |
| "177": 0.6978742364606902 |
| }, |
| "OOD_arXiv_2023": { |
| "16": 0.7916525423537559, |
| "24": 0.8102770192869214, |
| "32": 0.8335794211271619, |
| "48": 0.8782709376178942, |
| "64": 0.8771631566419636, |
| "80": 0.8915520081502442, |
| "96": 0.8949200856470252, |
| "112": 0.8814222544321175, |
| "126": 0.8556494233519906, |
| "133": 0.8358392088399716, |
| "142": 0.8390375985784466, |
| "150": 0.7702991962986523, |
| "160": 0.7144830722075272, |
| "177": 0.6603754919946767 |
| }, |
| "OOD_CommonPile": { |
| "16": 0.781236677486555, |
| "24": 0.7988611712590944, |
| "32": 0.8232537507439353, |
| "48": 0.8654877791238893, |
| "64": 0.8656894410263634, |
| "80": 0.8812748843918979, |
| "96": 0.8850409193797165, |
| "112": 0.8723165202545101, |
| "126": 0.8502470394124511, |
| "133": 0.833852988611821, |
| "142": 0.8369051828581651, |
| "150": 0.7682897033917105, |
| "160": 0.7138111346577654, |
| "177": 0.6595057776158877 |
| }, |
| "OOD_News_2024": { |
| "16": 0.827480165969923, |
| "24": 0.837726723624863, |
| "32": 0.8632292155316605, |
| "48": 0.9210869498301496, |
| "64": 0.9247255673704081, |
| "80": 0.9362241489898391, |
| "96": 0.9349798506919959, |
| "112": 0.9225137382747429, |
| "126": 0.8975991050504516, |
| "133": 0.8515537919652221, |
| "142": 0.8455001686641376, |
| "150": 0.7801679271847001, |
| "160": 0.7259258340690871, |
| "177": 0.6713449714937855 |
| } |
| }, |
| "read_scores_min_from_r9": { |
| "Github": 0.24049734957516195, |
| "Pile-CC": 0.26239902824163436, |
| "StackExchange": 0.26430597975850106, |
| "USPTO_Backgrounds": 0.26112590581178663, |
| "Wikipedia_en": 0.24160844944417476, |
| "OOD_arXiv_2023": 0.2692730328068137, |
| "OOD_CommonPile": 0.2528811445459723, |
| "OOD_News_2024": 0.25006329715251924 |
| }, |
| "PRIMARY": { |
| "statistic": "Spearman rho between alignment at p_read and the stored min read score", |
| "rho": 0.14285714285714285, |
| "exact_two_sided_p": 0.7520337301587302, |
| "n_permutations": 40320, |
| "critical_abs_rho_at_alpha_0.05": 0.7142857142857143, |
| "minimum_detectable": "n=8 detects only |rho| >= 0.714 at alpha=0.05 two-sided; a moderate relationship would NOT be detected by this design", |
| "corpora_in_order": [ |
| "Github", |
| "OOD_CommonPile", |
| "OOD_News_2024", |
| "OOD_arXiv_2023", |
| "Pile-CC", |
| "StackExchange", |
| "USPTO_Backgrounds", |
| "Wikipedia_en" |
| ], |
| "alignment_at_p_read": [ |
| 0.8388732396731006, |
| 0.8369051828581651, |
| 0.8455001686641376, |
| 0.8390375985784466, |
| 0.8829077025264984, |
| 0.8570446480097353, |
| 0.8381376088564241, |
| 0.8731856807813917 |
| ], |
| "read_score": [ |
| 0.24049734957516195, |
| 0.2528811445459723, |
| 0.25006329715251924, |
| 0.2692730328068137, |
| 0.26239902824163436, |
| 0.26430597975850106, |
| 0.26112590581178663, |
| 0.24160844944417476 |
| ] |
| }, |
| "SECONDARY_alignment_spread": { |
| "between_corpus_sd_at_p_read": 0.01784753488872414, |
| "between_corpus_sd_in_support_p64": 0.049280116927836765, |
| "ratio": 0.3621650272230308, |
| "note": "descriptive; no threshold is attached, per the registration" |
| }, |
| "controls": { |
| "C1_scale_invariance": { |
| "required": "cos_F unchanged to < 1e-9 when G is multiplied by alpha in {0.5, 2, 100}; the control for the target-count artifact that motivated a scale-free metric. AMENDED from '== 0.0', which is unreachable in floating point.", |
| "max_abs_difference": 4.8365533800165394e-11, |
| "tolerance": 1e-09, |
| "fires": true |
| }, |
| "C2_metric_is_not_vacuous": { |
| "required": "in-support alignment >= 0.30 for all 8 corpora AND |alignment| vs a Gaussian matrix <= 0.05. IF THIS FAILS THE INSTRUMENT IS BROKEN AND NO VERDICT ISSUES.", |
| "min_in_support_p64": 0.7941302109133536, |
| "max_abs_vs_random": 0.00020370669755364056, |
| "per_corpus_in_support": { |
| "Github": 0.7941302109133536, |
| "Pile-CC": 0.9600123937570153, |
| "StackExchange": 0.8859453230341394, |
| "USPTO_Backgrounds": 0.8785620695065707, |
| "Wikipedia_en": 0.9187448191728964, |
| "OOD_arXiv_2023": 0.8771631566419636, |
| "OOD_CommonPile": 0.8656894410263634, |
| "OOD_News_2024": 0.9247255673704081 |
| }, |
| "per_corpus_vs_random": { |
| "Github": 8.715056257069297e-05, |
| "Pile-CC": 8.34612596484493e-05, |
| "StackExchange": 0.00020370669755364056, |
| "USPTO_Backgrounds": 7.459009373812329e-05, |
| "Wikipedia_en": 5.197126198105234e-05, |
| "OOD_arXiv_2023": 3.00259731882602e-05, |
| "OOD_CommonPile": -2.5453768332120378e-05, |
| "OOD_News_2024": -0.00011232876610156356 |
| }, |
| "fires": true |
| }, |
| "C3_no_leakage": { |
| "required": "the items building G and the items scored are disjoint", |
| "n_overlap": 0, |
| "n_build": 128, |
| "n_score": 270, |
| "fires": true |
| }, |
| "C4_probe_matches_the_real_read": { |
| "required": "p_read inside CV2's measured prefixed read range, and CV2's n_in_support == 0", |
| "p_read": 142, |
| "cv2_min": 133, |
| "cv2_max": 177, |
| "cv2_median": 142, |
| "cv2_n_in_support": 0, |
| "fires": true |
| }, |
| "C5_band_rule": { |
| "required": "band == floor(0.38L)..floor(0.92L) intersected with layers strictly below the penultimate target layer, at L=24", |
| "band": [ |
| 9, |
| 10, |
| 11, |
| 12, |
| 13, |
| 14, |
| 15, |
| 16, |
| 17, |
| 18, |
| 19, |
| 20, |
| 21 |
| ], |
| "rule_band": [ |
| 9, |
| 10, |
| 11, |
| 12, |
| 13, |
| 14, |
| 15, |
| 16, |
| 17, |
| 18, |
| 19, |
| 20, |
| 21 |
| ], |
| "target_layer": 22, |
| "fires": true |
| } |
| }, |
| "controls_fired": { |
| "C1_scale_invariance": true, |
| "C2_metric_is_not_vacuous": true, |
| "C3_no_leakage": true, |
| "C4_probe_matches_the_real_read": true, |
| "C5_band_rule": true |
| }, |
| "VERDICT": "EXTRAPOLATION DOES NOT EXPLAIN THE ORDERING \u2014 exact two-sided p = 0.7520 >= 0.20, rho = +0.143, below the alpha=0.05 critical |rho| of 0.714 for n=8. No STRONG monotone relationship between alignment at position 142 and the read. The positional hedge is retired; this design could not have detected a moderate relationship and the claim is bounded accordingly.", |
| "provenance": { |
| "schema_version": 1, |
| "experiment": "CV8", |
| "utc": "2026-08-31T03:39:53+00:00", |
| "script": { |
| "path": "experiments/cv8_positional_extrapolation.py", |
| "sha256": "7bc5d7908a74bbd6713744b4af830c963396d9402c51b1fb289d7c92d23edb8d", |
| "n_lines": 408 |
| }, |
| "git": { |
| "commit": "99a6dd48ac17cd06f63d77e1e2fa51296cd189f7", |
| "commit_short": "99a6dd48ac17", |
| "git_repository_present": true, |
| "branch": "pythia", |
| "dirty": true, |
| "dirty_files": [ |
| "PAPER_REWRITE_SCOPE.md", |
| "xperiments/e66b_determinism_floor.py" |
| ], |
| "dirty_diff_sha256": "01b0d0f1f9e05dca6bc8dca13e923a4920f8bf53c42c0d1a3cca9d266754795c", |
| "describe": "99a6dd4-dirty" |
| }, |
| "argv": [ |
| "experiments/cv8_positional_extrapolation.py" |
| ], |
| "cwd": ".", |
| "inputs": [ |
| { |
| "path": "results/r9_permutation_calibrated_min.json", |
| "sha256": "eea61cccda689ed7903ca0f24af1961beb18f104fc7226a21f61a2563e3a9ba2", |
| "bytes": 20471, |
| "exists": true |
| }, |
| { |
| "path": "results/cv2_position_support.json", |
| "sha256": "e61248b05d2e8fa4db0262b325e2748579f873326414b1992d5bd4476c136b9c", |
| "bytes": 9152, |
| "exists": true |
| } |
| ], |
| "env": { |
| "python": "3.14.2", |
| "torch": "2.13.0", |
| "transformers": "5.14.1", |
| "numpy": "2.5.1", |
| "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", |
| "machine": "arm64", |
| "hostname": "<redacted for double-blind review>", |
| "tf32": { |
| "allow_tf32_matmul": false, |
| "allow_tf32_cudnn": false, |
| "NVIDIA_TF32_OVERRIDE": null |
| }, |
| "threads": { |
| "torch_num_threads": 8, |
| "torch_num_interop_threads": 14, |
| "OMP_NUM_THREADS": null, |
| "os_cpu_count": 14 |
| } |
| }, |
| "payload_sha256": "a950c3b88b5401cefca4116adf35303854131b5bd45dfd084c0efe00325e4501" |
| } |
| } |
