| { |
| "experiment": "CV8 \u2014 positional extrapolation: is the corpus effect an extrapolation artifact?", |
| "prereg": "docs/experiments/preregs/CV8_positional_extrapolation.md", |
| "status": "PRE-REGISTERED", |
| "model": "EleutherAI/pythia-410m-deduped", |
| "band": [ |
| 9, |
| 10, |
| 11, |
| 12, |
| 13, |
| 14, |
| 15, |
| 16, |
| 17, |
| 18, |
| 19, |
| 20, |
| 21 |
| ], |
| "device": "cpu", |
| "smoke": false, |
| "n_sequences_for_G": 128, |
| "prefix_corpus": "Github", |
| "probe_positions": [ |
| 16, |
| 24, |
| 32, |
| 48, |
| 64, |
| 80, |
| 96, |
| 112, |
| 126, |
| 133, |
| 142, |
| 150, |
| 160, |
| 177 |
| ], |
| "p_read": 142, |
| "in_support_positions": [ |
| 16, |
| 24, |
| 32, |
| 48, |
| 64, |
| 80, |
| 96, |
| 112, |
| 126 |
| ], |
| "out_of_support_positions": [ |
| 133, |
| 142, |
| 150, |
| 160, |
| 177 |
| ], |
| "alignment_by_corpus_by_position": { |
| "Github": { |
| "16": 0.8199622410112316, |
| "24": 0.8649228662025228, |
| "32": 0.8823327600490843, |
| "48": 0.9208955206003056, |
| "64": 0.9392306171621081, |
| "80": 0.9484260866168507, |
| "96": 0.9504478884880948, |
| "112": 0.9423929234741516, |
| "126": 0.9310935599306583, |
| "133": 0.8484558265334022, |
| "142": 0.8928635661375124, |
| "150": 0.8991461850577471, |
| "160": 0.863023156475018, |
| "177": 0.8091136683178206 |
| }, |
| "Pile-CC": { |
| "16": 0.7388377966330032, |
| "24": 0.7623900261264125, |
| "32": 0.7401483099189051, |
| "48": 0.8078829412811408, |
| "64": 0.8026415395543312, |
| "80": 0.8074120082320871, |
| "96": 0.8188838080903893, |
| "112": 0.8275872410947619, |
| "126": 0.8261497692178901, |
| "133": 0.8825382212439484, |
| "142": 0.8410997638017006, |
| "150": 0.784536015586303, |
| "160": 0.737921293108437, |
| "177": 0.7043631819379887 |
| }, |
| "StackExchange": { |
| "16": 0.7868950844126045, |
| "24": 0.8172987029771465, |
| "32": 0.7958271283933123, |
| "48": 0.8631015421781737, |
| "64": 0.8521835524590238, |
| "80": 0.8567536478592028, |
| "96": 0.865747722698461, |
| "112": 0.869334769490467, |
| "126": 0.8632586365641963, |
| "133": 0.8908658940965233, |
| "142": 0.8680869951100778, |
| "150": 0.8141380848365287, |
| "160": 0.7637898622813123, |
| "177": 0.72162493963662 |
| }, |
| "USPTO_Backgrounds": { |
| "16": 0.7250339105496532, |
| "24": 0.7466318620527843, |
| "32": 0.7217538165312588, |
| "48": 0.7893456531556848, |
| "64": 0.7835063101091788, |
| "80": 0.7881745586910742, |
| "96": 0.8035122217701587, |
| "112": 0.8106891547649236, |
| "126": 0.8059748422771726, |
| "133": 0.8468851304825057, |
| "142": 0.8175291862732885, |
| "150": 0.765009421906371, |
| "160": 0.7211020727707582, |
| "177": 0.6890500060955719 |
| }, |
| "Wikipedia_en": { |
| "16": 0.714737290764607, |
| "24": 0.7414217829325931, |
| "32": 0.7282756412878298, |
| "48": 0.7893573136174032, |
| "64": 0.7913875814589846, |
| "80": 0.8014941703581413, |
| "96": 0.8124232223799928, |
| "112": 0.8217904107612356, |
| "126": 0.820110692737911, |
| "133": 0.8634235496340769, |
| "142": 0.8326451279763215, |
| "150": 0.7839842876667025, |
| "160": 0.7441040846658752, |
| "177": 0.7115479365349467 |
| }, |
| "OOD_arXiv_2023": { |
| "16": 0.7314797724797859, |
| "24": 0.7538180293815879, |
| "32": 0.7296685194098056, |
| "48": 0.7965723238808289, |
| "64": 0.7904637129490234, |
| "80": 0.792464811600057, |
| "96": 0.8055670247052423, |
| "112": 0.8121151935399167, |
| "126": 0.8085203811055055, |
| "133": 0.8561239410946141, |
| "142": 0.8231678802397187, |
| "150": 0.7658832922857227, |
| "160": 0.7208183538688316, |
| "177": 0.6866824113541039 |
| }, |
| "OOD_CommonPile": { |
| "16": 0.7274252644273965, |
| "24": 0.7487318124617317, |
| "32": 0.7244637094978772, |
| "48": 0.7910418690218445, |
| "64": 0.786705942925807, |
| "80": 0.7893214492024896, |
| "96": 0.8042403273110403, |
| "112": 0.8103238690236166, |
| "126": 0.8063952401312486, |
| "133": 0.8562419725480844, |
| "142": 0.8221690899817258, |
| "150": 0.7646649434641972, |
| "160": 0.7195846531182958, |
| "177": 0.6856726030758462 |
| }, |
| "OOD_News_2024": { |
| "16": 0.6930364953152686, |
| "24": 0.714579362305328, |
| "32": 0.6958332436342336, |
| "48": 0.759202188142498, |
| "64": 0.7562811167705011, |
| "80": 0.7640614480227068, |
| "96": 0.7759056468667923, |
| "112": 0.7856216585197024, |
| "126": 0.7848264476594219, |
| "133": 0.8403997315239466, |
| "142": 0.7984675936770528, |
| "150": 0.7452955111728733, |
| "160": 0.7056315182522144, |
| "177": 0.675980543031208 |
| } |
| }, |
| "read_scores_min_from_r9": { |
| "Github": 0.24049734957516195, |
| "Pile-CC": 0.26239902824163436, |
| "StackExchange": 0.26430597975850106, |
| "USPTO_Backgrounds": 0.26112590581178663, |
| "Wikipedia_en": 0.24160844944417476, |
| "OOD_arXiv_2023": 0.2692730328068137, |
| "OOD_CommonPile": 0.2528811445459723, |
| "OOD_News_2024": 0.25006329715251924 |
| }, |
| "PRIMARY": { |
| "statistic": "Spearman rho between alignment at p_read and the stored min read score", |
| "rho": -0.047619047619047616, |
| "exact_two_sided_p": 0.9348710317460317, |
| "n_permutations": 40320, |
| "critical_abs_rho_at_alpha_0.05": 0.7142857142857143, |
| "minimum_detectable": "n=8 detects only |rho| >= 0.714 at alpha=0.05 two-sided; a moderate relationship would NOT be detected by this design", |
| "corpora_in_order": [ |
| "Github", |
| "OOD_CommonPile", |
| "OOD_News_2024", |
| "OOD_arXiv_2023", |
| "Pile-CC", |
| "StackExchange", |
| "USPTO_Backgrounds", |
| "Wikipedia_en" |
| ], |
| "alignment_at_p_read": [ |
| 0.8928635661375124, |
| 0.8221690899817258, |
| 0.7984675936770528, |
| 0.8231678802397187, |
| 0.8410997638017006, |
| 0.8680869951100778, |
| 0.8175291862732885, |
| 0.8326451279763215 |
| ], |
| "read_score": [ |
| 0.24049734957516195, |
| 0.2528811445459723, |
| 0.25006329715251924, |
| 0.2692730328068137, |
| 0.26239902824163436, |
| 0.26430597975850106, |
| 0.26112590581178663, |
| 0.24160844944417476 |
| ] |
| }, |
| "SECONDARY_alignment_spread": { |
| "between_corpus_sd_at_p_read": 0.030232561695707077, |
| "between_corpus_sd_in_support_p64": 0.05773061221794703, |
| "ratio": 0.5236833723774111, |
| "note": "descriptive; no threshold is attached, per the registration" |
| }, |
| "controls": { |
| "C1_scale_invariance": { |
| "required": "cos_F unchanged to < 1e-9 when G is multiplied by alpha in {0.5, 2, 100}; the control for the target-count artifact that motivated a scale-free metric. AMENDED from '== 0.0', which is unreachable in floating point.", |
| "max_abs_difference": 7.949996216893851e-11, |
| "tolerance": 1e-09, |
| "fires": true |
| }, |
| "C2_metric_is_not_vacuous": { |
| "required": "in-support alignment >= 0.30 for all 8 corpora AND |alignment| vs a Gaussian matrix <= 0.05. IF THIS FAILS THE INSTRUMENT IS BROKEN AND NO VERDICT ISSUES.", |
| "min_in_support_p64": 0.7562811167705011, |
| "max_abs_vs_random": 0.00020370669755364056, |
| "per_corpus_in_support": { |
| "Github": 0.9392306171621081, |
| "Pile-CC": 0.8026415395543312, |
| "StackExchange": 0.8521835524590238, |
| "USPTO_Backgrounds": 0.7835063101091788, |
| "Wikipedia_en": 0.7913875814589846, |
| "OOD_arXiv_2023": 0.7904637129490234, |
| "OOD_CommonPile": 0.786705942925807, |
| "OOD_News_2024": 0.7562811167705011 |
| }, |
| "per_corpus_vs_random": { |
| "Github": 8.715056257069297e-05, |
| "Pile-CC": 8.34612596484493e-05, |
| "StackExchange": 0.00020370669755364056, |
| "USPTO_Backgrounds": 7.459009373812329e-05, |
| "Wikipedia_en": 5.197126198105234e-05, |
| "OOD_arXiv_2023": 3.00259731882602e-05, |
| "OOD_CommonPile": -2.5453768332120378e-05, |
| "OOD_News_2024": -0.00011232876610156356 |
| }, |
| "fires": true |
| }, |
| "C3_no_leakage": { |
| "required": "the items building G and the items scored are disjoint", |
| "n_overlap": 0, |
| "n_build": 128, |
| "n_score": 270, |
| "fires": true |
| }, |
| "C4_probe_matches_the_real_read": { |
| "required": "p_read inside CV2's measured prefixed read range, and CV2's n_in_support == 0", |
| "p_read": 142, |
| "cv2_min": 133, |
| "cv2_max": 177, |
| "cv2_median": 142, |
| "cv2_n_in_support": 0, |
| "fires": true |
| }, |
| "C5_band_rule": { |
| "required": "band == floor(0.38L)..floor(0.92L) intersected with layers strictly below the penultimate target layer, at L=24", |
| "band": [ |
| 9, |
| 10, |
| 11, |
| 12, |
| 13, |
| 14, |
| 15, |
| 16, |
| 17, |
| 18, |
| 19, |
| 20, |
| 21 |
| ], |
| "rule_band": [ |
| 9, |
| 10, |
| 11, |
| 12, |
| 13, |
| 14, |
| 15, |
| 16, |
| 17, |
| 18, |
| 19, |
| 20, |
| 21 |
| ], |
| "target_layer": 22, |
| "fires": true |
| } |
| }, |
| "controls_fired": { |
| "C1_scale_invariance": true, |
| "C2_metric_is_not_vacuous": true, |
| "C3_no_leakage": true, |
| "C4_probe_matches_the_real_read": true, |
| "C5_band_rule": true |
| }, |
| "VERDICT": "EXTRAPOLATION DOES NOT EXPLAIN THE ORDERING \u2014 exact two-sided p = 0.9349 >= 0.20, rho = -0.048, below the alpha=0.05 critical |rho| of 0.714 for n=8. No STRONG monotone relationship between alignment at position 142 and the read. The positional hedge is retired; this design could not have detected a moderate relationship and the claim is bounded accordingly.", |
| "provenance": { |
| "schema_version": 1, |
| "experiment": "CV8", |
| "utc": "2026-08-31T05:58:45+00:00", |
| "script": { |
| "path": "experiments/cv8_positional_extrapolation.py", |
| "sha256": "7bc5d7908a74bbd6713744b4af830c963396d9402c51b1fb289d7c92d23edb8d", |
| "n_lines": 408 |
| }, |
| "git": { |
| "commit": "9770d208af381e9579ae259d49d95c0c0a5b9a33", |
| "commit_short": "9770d208af38", |
| "git_repository_present": true, |
| "branch": "pythia", |
| "dirty": true, |
| "dirty_files": [ |
| "research-paper-writing/" |
| ], |
| "dirty_diff_sha256": null, |
| "describe": "9770d20" |
| }, |
| "argv": [ |
| "experiments/cv8_positional_extrapolation.py", |
| "--prefix-corpus", |
| "Github", |
| "--out", |
| "results/cv8_positional_extrapolation_github.json" |
| ], |
| "cwd": ".", |
| "inputs": [ |
| { |
| "path": "results/r9_permutation_calibrated_min.json", |
| "sha256": "eea61cccda689ed7903ca0f24af1961beb18f104fc7226a21f61a2563e3a9ba2", |
| "bytes": 20471, |
| "exists": true |
| }, |
| { |
| "path": "results/cv2_position_support.json", |
| "sha256": "e61248b05d2e8fa4db0262b325e2748579f873326414b1992d5bd4476c136b9c", |
| "bytes": 9152, |
| "exists": true |
| } |
| ], |
| "env": { |
| "python": "3.14.2", |
| "torch": "2.13.0", |
| "transformers": "5.14.1", |
| "numpy": "2.5.1", |
| "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", |
| "machine": "arm64", |
| "hostname": "<redacted for double-blind review>", |
| "tf32": { |
| "allow_tf32_matmul": false, |
| "allow_tf32_cudnn": false, |
| "NVIDIA_TF32_OVERRIDE": null |
| }, |
| "threads": { |
| "torch_num_threads": 8, |
| "torch_num_interop_threads": 14, |
| "OMP_NUM_THREADS": null, |
| "os_cpu_count": 14 |
| } |
| }, |
| "payload_sha256": "db838d5ea323fb6ef2b73cec85b2d427d891d37340a801e138f2bf20f0b01d15" |
| } |
| } |
