{ "experiment": "DN1 \u2014 null distributions of three already-registered bars", "prereg": "docs/experiments/preregs/DN1_design_nulls.md", "status": "PRE-REGISTERED", "measures_nothing_new": true, "scope": "Calibrates thresholds, not measurements. Cannot overturn CV6, CV7 or R9; prices the bars they were judged against.", "n_sim": 1000000, "n_perm": 100000, "A1_design_null": { "definition": "R = range(group means) / sqrt(mean(per-group sample var)), G=5 groups of n=3", "distribution": "iid normal, scale-free in sigma and location-free in mu", "mean": 1.4558214009529349, "sd": 0.6683661758236041, "percentiles": { "p50": 1.3504426642962697, "p90": 2.3197482513624257, "p95": 2.685370632595311, "p99": 3.539988075003452, "p99.9": 4.87893156220862 }, "P_R_ge_10": 2e-06, "P_R_ge_10_exceedances_observed": 2, "note": "A one-sided exceedance of 0 in n_sim draws is reported as < 1/n_sim, not as 0. The bar is calibrated against this column." }, "C1_scale_freedom": { "required": "|p99(sigma=1) - p99(sigma=100)| <= 0.02", "p99_sigma_1": 3.539988075003452, "p99_sigma_100": 3.5461058294242056, "abs_diff": 0.006117754420753574, "fires": true, "falsification_witness": { "what": "same code with the denominator replaced by the constant 1.0", "p99_sigma_1": 2.6567914459131163, "p99_sigma_100": 265.1280524472683, "abs_diff": 262.4712610013552, "control_would_fire": false, "note": "this must be False, or C1 cannot distinguish a scale-free statistic from any other" } }, "A2_permutation_null_410m": { "source": "results/d3_corpus_by_family_410m.json -> per_arm.|s..z", "unit": "15 z values per family, permuted across 5 corpus labels, 3 per label", "by_family": { "multihop": { "observed_R": 25.361005428912275, "null_mean": 1.4354219513761668, "null_sd": 0.8281876972206043, "null_p50": 1.277465107028326, "null_p95": 2.515198551887543, "null_p99": 5.61098563909792, "null_max": 25.43264217979271, "n_null_ge_bar": 23, "P_null_ge_bar": 0.00023, "n_null_ge_observed": 2, "p_observed_one_sided": 2.999970000299997e-05 }, "multilingual": { "observed_R": 19.426787847831193, "null_mean": 1.4472181163211184, "null_sd": 0.7043406387033073, "null_p50": 1.3324022652684364, "null_p95": 2.660570482190145, "null_p99": 3.8541903041621643, "null_max": 15.066763195787532, "n_null_ge_bar": 10, "P_null_ge_bar": 0.0001, "n_null_ge_observed": 0, "p_observed_one_sided": 9.99990000099999e-06 }, "order-ops": { "observed_R": 24.632511354365725, "null_mean": 1.4600481090175972, "null_sd": 0.724071411525985, "null_p50": 1.341316486899336, "null_p95": 2.7854524342316607, "null_p99": 3.4035854960502476, "null_max": 24.632511354365725, "n_null_ge_bar": 10, "P_null_ge_bar": 0.0001, "n_null_ge_observed": 1, "p_observed_one_sided": 1.999980000199998e-05 }, "poetry": { "observed_R": 42.686224720210525, "null_mean": 1.4207336216576643, "null_sd": 1.1226321014484546, "null_p50": 1.0953311671022932, "null_p95": 2.289972367428842, "null_p99": 8.799193358965976, "null_max": 42.686224720210525, "n_null_ge_bar": 571, "P_null_ge_bar": 0.00571, "n_null_ge_observed": 1, "p_observed_one_sided": 1.999980000199998e-05 }, "typo": { "observed_R": 27.014210896207846, "null_mean": 1.4510950885052967, "null_sd": 0.7238398592296635, "null_p50": 1.3394268870958137, "null_p95": 2.644590252011299, "null_p99": 4.196756766414129, "null_max": 11.924406661562124, "n_null_ge_bar": 11, "P_null_ge_bar": 0.00011, "n_null_ge_observed": 0, "p_observed_one_sided": 9.99990000099999e-06 } } }, "C2_pooled_sd_recomputes": { "required": "<= 1e-12 on all 5 families", "by_family": { "multihop": { "recomputed": 0.017453481001782855, "stored": 0.017453481001782855, "abs_diff": 0.0 }, "multilingual": { "recomputed": 0.019695881058294912, "stored": 0.019695881058294912, "abs_diff": 0.0 }, "order-ops": { "recomputed": 0.012061420042855183, "stored": 0.012061420042855183, "abs_diff": 0.0 }, "poetry": { "recomputed": 0.005578110253433368, "stored": 0.005578110253433368, "abs_diff": 0.0 }, "typo": { "recomputed": 0.019451275697770446, "stored": 0.019451275697770446, "abs_diff": 0.0 } }, "max_abs_diff": 0.0, "fires": true }, "C3_R_recomputes": { "required": "<= 1e-12 on all 5 families", "by_family": { "multihop": { "recomputed": 25.361005428912275, "stored": 25.36100542891225, "abs_diff": 2.4868995751603507e-14 }, "multilingual": { "recomputed": 19.426787847831193, "stored": 19.426787847831193, "abs_diff": 0.0 }, "order-ops": { "recomputed": 24.632511354365725, "stored": 24.632511354365725, "abs_diff": 0.0 }, "poetry": { "recomputed": 42.686224720210525, "stored": 42.686224720210525, "abs_diff": 0.0 }, "typo": { "recomputed": 27.014210896207846, "stored": 27.014210896207846, "abs_diff": 0.0 } }, "max_abs_diff": 2.4868995751603507e-14, "fires": true }, "C4_identity_in_support": { "required": "the unpermuted assignment reproduces the observed R exactly", "by_family": { "multihop": { "identity_R": 25.361005428912275, "observed_R": 25.361005428912275, "abs_diff": 0.0 }, "multilingual": { "identity_R": 19.426787847831193, "observed_R": 19.426787847831193, "abs_diff": 0.0 }, "order-ops": { "identity_R": 24.632511354365725, "observed_R": 24.632511354365725, "abs_diff": 0.0 }, "poetry": { "identity_R": 42.686224720210525, "observed_R": 42.686224720210525, "abs_diff": 0.0 }, "typo": { "identity_R": 27.014210896207846, "observed_R": 27.014210896207846, "abs_diff": 0.0 } }, "max_abs_diff": 0.0, "fires": true }, "C5_design_vs_permutation": { "required": "the two 99th percentiles agree to within a factor of 1.25", "p99_design": 3.539988075003452, "p99_permutation_worst_family": 8.799193358965976, "ratio": 2.485656214804451, "fires": false }, "B_tau_null_exact": { "n": 5, "n_orderings": 120, "bar": 0.6, "n_orderings_clearing_bar": 14, "P_single_family_tau_ge_bar": 0.11666666666666667, "distinct_tau_values": [ -1.0, -0.8, -0.6, -0.4, -0.2, 0.0, 0.2, 0.4, 0.6, 0.8, 1.0 ], "P_k_of_5_families_clear_under_independence": { "0": 0.5378028459362139, "1": 0.3551528227880658, "2": 0.09381395318930041, "3": 0.012390522119341565, "4": 0.0008182420267489713, "5": 2.1613940329218108e-05 }, "P_at_least_3_of_5_under_independence": 0.013230378086419754, "DEPENDENCE_CAVEAT": "the five families share the same five operators and one activation cache, so they are not independent; the independence figure is an UPPER bound on the evidence, not an estimate of it" }, "C_r9_count_null_exact": { "design": "8 corpora, 15 own-derangement draws each", "P_one_corpus_clears_all_draws_under_H0": 0.0625, "expected_count_under_H0": 0.5, "P_count_equals_zero_under_H0": 0.5967194738332182, "observed_count": 0, "reading": "a count of 0 is the modal outcome under the exact null, so the count alone carries almost no information; the direction does", "sign_test_on_direction": { "statistic": "number of corpora whose real z_vs_null is negative, min aggregation", "n_negative": 7, "n_total": 8, "z_by_corpus": { "Wikipedia_en": -1.6709216685460513, "USPTO_Backgrounds": -0.7050665067014444, "Pile-CC": -1.9874216306081265, "StackExchange": -0.09090121377251528, "Github": 0.593144297659444, "OOD_News_2024": -0.7490796022292463, "OOD_arXiv_2023": -0.0881460749226426, "OOD_CommonPile": -0.7064942600047919 }, "p_one_sided_exact_binomial": 0.03515625, "DEPENDENCE_CAVEAT": "the eight corpora share one activation cache and one 551-item battery; the sign test assumes independence and therefore understates the p-value" }, "positive_control": { "what": "persist on the same draws returns 8 of 8", "n_beating": 8, "why_it_matters": "the design CAN return the positive branch, so min's 0 of 8 is a failure of the statistic and not a lack of power in the null" } }, "controls_fired": { "C1_scale_freedom": true, "C2_pooled_sd_recomputes": true, "C3_R_recomputes": true, "C4_identity_in_support": true, "C5_design_vs_permutation": false }, "VERDICT": "UNCLEAR \u2014 a control did not fire: C5_design_vs_permutation", "provenance": { "schema_version": 1, "experiment": "DN1", "utc": "2026-08-31T23:19:44+00:00", "script": { "path": "experiments/dn1_design_nulls.py", "sha256": "06e93d9dbd43feadaf2975d5854fd7252ba55264109a66909b2f4df18ea7f6e6", "n_lines": 305 }, "git": { "commit": "9987d82887ed339bf69b5e7fa1e4fc98d5bbb4ab", "commit_short": "9987d82887ed", "git_repository_present": true, "branch": "pythia", "dirty": true, "dirty_files": [ "CONTEXT.md", "NEELNANDA_REP.md", "docs/experiments/COVERAGE.md", "docs/experiments/METRICS.md", "docs/experiments/descriptions/D1.md", "docs/experiments/descriptions/E33.md", "docs/experiments/descriptions/E34A.md", "docs/experiments/descriptions/E34B.md", "docs/experiments/descriptions/E36.md", "docs/experiments/descriptions/E37.md", "docs/experiments/descriptions/E44.md", "docs/experiments/descriptions/E58.md", "docs/experiments/preregs/E66d_s_quantization_limited.md", "docs/experiments/preregs/superseded/PREREG_PYTHIA_T7_v2.md", "docs/reproducibility/ARTIFACTS.md", "docs/reproducibility/PAYLOAD_REDACTIONS.md", "docs/reproducibility/SCRIPT_PROVENANCE.md", "docs/reproducibility/review-snapshot-2026-08-19/MANIFEST.sha256", "docs/reproducibility/review-snapshot-2026-08-19/PROVENANCE.md", "docs/reproducibility/review-snapshot-2026-08-19/REDACTIONS.md", "docs/reproducibility/review-snapshot-2026-08-19/review/EXTERNAL_REVIEW.md", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/AGENT_REPORT.md", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/CONTEXT_UNDER_REVIEW.md", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/FINDINGS.md", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/JUSTIFIABILITY.md", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/LEDGER.md", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/PHASE_A_ACCOUNT.md", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/SUSPICIONS.md", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s0.json", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s1.json", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s2.json", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s0.json", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s1.json", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s2.json", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s0.json", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s1.json", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s2.json", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s0.json", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s1.json", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s2.json", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s0.json", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s1.json", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s2.json", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/balance_test.py", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/check_payload.py", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/coverage2.py", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/derange.py", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/derangement_result.json", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/logit_base.py", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/logit_baseline.json", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_anova.py", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_e52.py", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_rescore.py", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/prereg2.py", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/prereg_audit.py", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/rescore_all.py", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/rstrip_anova.py", "docs/validity/AUDIT_READINESS.md", "docs/validity/audit-2026-08-22/DIGEST.json", "experiments/d1_min_union_diagnostic.py", "experiments/e28_eval_410m.sh", "experiments/e28_ladder_410m.py", "experiments/exploratory/descriptions/T0.md", "experiments/exploratory/descriptions/T12.md" ], "dirty_diff_sha256": "aff7ed44a7f13d5d00e837c12a7446366431d5a5ef0602ddfcfb051e2a0ddd20", "describe": "9987d82-dirty" }, "argv": [ "experiments/dn1_design_nulls.py" ], "cwd": ".", "inputs": [ { "path": "results/d3_corpus_by_family_410m.json", "sha256": "795676289554af8111a4b4464617203271848f210fe1ce29a9946d44ab5dad5a", "bytes": 23102, "exists": true }, { "path": "results/r9_permutation_calibrated_min.json", "sha256": "773e2a6423db3022d96f71373c859fd0ae313f529abf569c396fc7de56461b89", "bytes": 22358, "exists": true } ], "env": { "python": "3.14.2", "torch": "2.13.0", "transformers": "5.14.1", "numpy": "2.5.1", "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", "machine": "arm64", "tf32": { "allow_tf32_matmul": false, "allow_tf32_cudnn": true, "NVIDIA_TF32_OVERRIDE": null }, "threads": { "torch_num_threads": 10, "torch_num_interop_threads": 14, "OMP_NUM_THREADS": null, "os_cpu_count": 14 } }, "payload_sha256": "201a8d6db3dfd8bcb12d81b23aa253b0ee6e7795a039ddb802b39ef668518dfe" } }