| { |
| "experiment": "DN1 \u2014 null distributions of three already-registered bars", |
| "prereg": "docs/experiments/preregs/DN1_design_nulls.md", |
| "status": "PRE-REGISTERED", |
| "measures_nothing_new": true, |
| "scope": "Calibrates thresholds, not measurements. Cannot overturn CV6, CV7 or R9; prices the bars they were judged against.", |
| "n_sim": 1000000, |
| "n_perm": 100000, |
| "A1_design_null": { |
| "definition": "R = range(group means) / sqrt(mean(per-group sample var)), G=5 groups of n=3", |
| "distribution": "iid normal, scale-free in sigma and location-free in mu", |
| "mean": 1.4558214009529349, |
| "sd": 0.6683661758236041, |
| "percentiles": { |
| "p50": 1.3504426642962697, |
| "p90": 2.3197482513624257, |
| "p95": 2.685370632595311, |
| "p99": 3.539988075003452, |
| "p99.9": 4.87893156220862 |
| }, |
| "P_R_ge_10": 2e-06, |
| "P_R_ge_10_exceedances_observed": 2, |
| "note": "A one-sided exceedance of 0 in n_sim draws is reported as < 1/n_sim, not as 0. The bar is calibrated against this column." |
| }, |
| "C1_scale_freedom": { |
| "required": "|p99(sigma=1) - p99(sigma=100)| <= 0.02", |
| "p99_sigma_1": 3.539988075003452, |
| "p99_sigma_100": 3.5461058294242056, |
| "abs_diff": 0.006117754420753574, |
| "fires": true, |
| "falsification_witness": { |
| "what": "same code with the denominator replaced by the constant 1.0", |
| "p99_sigma_1": 2.6567914459131163, |
| "p99_sigma_100": 265.1280524472683, |
| "abs_diff": 262.4712610013552, |
| "control_would_fire": false, |
| "note": "this must be False, or C1 cannot distinguish a scale-free statistic from any other" |
| } |
| }, |
| "A2_permutation_null_410m": { |
| "source": "results/d3_corpus_by_family_410m.json -> per_arm.<corpus>|s<seed>.<family>.z", |
| "unit": "15 z values per family, permuted across 5 corpus labels, 3 per label", |
| "by_family": { |
| "multihop": { |
| "observed_R": 25.361005428912275, |
| "null_mean": 1.4354219513761668, |
| "null_sd": 0.8281876972206043, |
| "null_p50": 1.277465107028326, |
| "null_p95": 2.515198551887543, |
| "null_p99": 5.61098563909792, |
| "null_max": 25.43264217979271, |
| "n_null_ge_bar": 23, |
| "P_null_ge_bar": 0.00023, |
| "n_null_ge_observed": 2, |
| "p_observed_one_sided": 2.999970000299997e-05 |
| }, |
| "multilingual": { |
| "observed_R": 19.426787847831193, |
| "null_mean": 1.4472181163211184, |
| "null_sd": 0.7043406387033073, |
| "null_p50": 1.3324022652684364, |
| "null_p95": 2.660570482190145, |
| "null_p99": 3.8541903041621643, |
| "null_max": 15.066763195787532, |
| "n_null_ge_bar": 10, |
| "P_null_ge_bar": 0.0001, |
| "n_null_ge_observed": 0, |
| "p_observed_one_sided": 9.99990000099999e-06 |
| }, |
| "order-ops": { |
| "observed_R": 24.632511354365725, |
| "null_mean": 1.4600481090175972, |
| "null_sd": 0.724071411525985, |
| "null_p50": 1.341316486899336, |
| "null_p95": 2.7854524342316607, |
| "null_p99": 3.4035854960502476, |
| "null_max": 24.632511354365725, |
| "n_null_ge_bar": 10, |
| "P_null_ge_bar": 0.0001, |
| "n_null_ge_observed": 1, |
| "p_observed_one_sided": 1.999980000199998e-05 |
| }, |
| "poetry": { |
| "observed_R": 42.686224720210525, |
| "null_mean": 1.4207336216576643, |
| "null_sd": 1.1226321014484546, |
| "null_p50": 1.0953311671022932, |
| "null_p95": 2.289972367428842, |
| "null_p99": 8.799193358965976, |
| "null_max": 42.686224720210525, |
| "n_null_ge_bar": 571, |
| "P_null_ge_bar": 0.00571, |
| "n_null_ge_observed": 1, |
| "p_observed_one_sided": 1.999980000199998e-05 |
| }, |
| "typo": { |
| "observed_R": 27.014210896207846, |
| "null_mean": 1.4510950885052967, |
| "null_sd": 0.7238398592296635, |
| "null_p50": 1.3394268870958137, |
| "null_p95": 2.644590252011299, |
| "null_p99": 4.196756766414129, |
| "null_max": 11.924406661562124, |
| "n_null_ge_bar": 11, |
| "P_null_ge_bar": 0.00011, |
| "n_null_ge_observed": 0, |
| "p_observed_one_sided": 9.99990000099999e-06 |
| } |
| } |
| }, |
| "C2_pooled_sd_recomputes": { |
| "required": "<= 1e-12 on all 5 families", |
| "by_family": { |
| "multihop": { |
| "recomputed": 0.017453481001782855, |
| "stored": 0.017453481001782855, |
| "abs_diff": 0.0 |
| }, |
| "multilingual": { |
| "recomputed": 0.019695881058294912, |
| "stored": 0.019695881058294912, |
| "abs_diff": 0.0 |
| }, |
| "order-ops": { |
| "recomputed": 0.012061420042855183, |
| "stored": 0.012061420042855183, |
| "abs_diff": 0.0 |
| }, |
| "poetry": { |
| "recomputed": 0.005578110253433368, |
| "stored": 0.005578110253433368, |
| "abs_diff": 0.0 |
| }, |
| "typo": { |
| "recomputed": 0.019451275697770446, |
| "stored": 0.019451275697770446, |
| "abs_diff": 0.0 |
| } |
| }, |
| "max_abs_diff": 0.0, |
| "fires": true |
| }, |
| "C3_R_recomputes": { |
| "required": "<= 1e-12 on all 5 families", |
| "by_family": { |
| "multihop": { |
| "recomputed": 25.361005428912275, |
| "stored": 25.36100542891225, |
| "abs_diff": 2.4868995751603507e-14 |
| }, |
| "multilingual": { |
| "recomputed": 19.426787847831193, |
| "stored": 19.426787847831193, |
| "abs_diff": 0.0 |
| }, |
| "order-ops": { |
| "recomputed": 24.632511354365725, |
| "stored": 24.632511354365725, |
| "abs_diff": 0.0 |
| }, |
| "poetry": { |
| "recomputed": 42.686224720210525, |
| "stored": 42.686224720210525, |
| "abs_diff": 0.0 |
| }, |
| "typo": { |
| "recomputed": 27.014210896207846, |
| "stored": 27.014210896207846, |
| "abs_diff": 0.0 |
| } |
| }, |
| "max_abs_diff": 2.4868995751603507e-14, |
| "fires": true |
| }, |
| "C4_identity_in_support": { |
| "required": "the unpermuted assignment reproduces the observed R exactly", |
| "by_family": { |
| "multihop": { |
| "identity_R": 25.361005428912275, |
| "observed_R": 25.361005428912275, |
| "abs_diff": 0.0 |
| }, |
| "multilingual": { |
| "identity_R": 19.426787847831193, |
| "observed_R": 19.426787847831193, |
| "abs_diff": 0.0 |
| }, |
| "order-ops": { |
| "identity_R": 24.632511354365725, |
| "observed_R": 24.632511354365725, |
| "abs_diff": 0.0 |
| }, |
| "poetry": { |
| "identity_R": 42.686224720210525, |
| "observed_R": 42.686224720210525, |
| "abs_diff": 0.0 |
| }, |
| "typo": { |
| "identity_R": 27.014210896207846, |
| "observed_R": 27.014210896207846, |
| "abs_diff": 0.0 |
| } |
| }, |
| "max_abs_diff": 0.0, |
| "fires": true |
| }, |
| "C5_design_vs_permutation": { |
| "required": "the two 99th percentiles agree to within a factor of 1.25", |
| "p99_design": 3.539988075003452, |
| "p99_permutation_worst_family": 8.799193358965976, |
| "ratio": 2.485656214804451, |
| "fires": false |
| }, |
| "B_tau_null_exact": { |
| "n": 5, |
| "n_orderings": 120, |
| "bar": 0.6, |
| "n_orderings_clearing_bar": 14, |
| "P_single_family_tau_ge_bar": 0.11666666666666667, |
| "distinct_tau_values": [ |
| -1.0, |
| -0.8, |
| -0.6, |
| -0.4, |
| -0.2, |
| 0.0, |
| 0.2, |
| 0.4, |
| 0.6, |
| 0.8, |
| 1.0 |
| ], |
| "P_k_of_5_families_clear_under_independence": { |
| "0": 0.5378028459362139, |
| "1": 0.3551528227880658, |
| "2": 0.09381395318930041, |
| "3": 0.012390522119341565, |
| "4": 0.0008182420267489713, |
| "5": 2.1613940329218108e-05 |
| }, |
| "P_at_least_3_of_5_under_independence": 0.013230378086419754, |
| "DEPENDENCE_CAVEAT": "the five families share the same five operators and one activation cache, so they are not independent; the independence figure is an UPPER bound on the evidence, not an estimate of it" |
| }, |
| "C_r9_count_null_exact": { |
| "design": "8 corpora, 15 own-derangement draws each", |
| "P_one_corpus_clears_all_draws_under_H0": 0.0625, |
| "expected_count_under_H0": 0.5, |
| "P_count_equals_zero_under_H0": 0.5967194738332182, |
| "observed_count": 0, |
| "reading": "a count of 0 is the modal outcome under the exact null, so the count alone carries almost no information; the direction does", |
| "sign_test_on_direction": { |
| "statistic": "number of corpora whose real z_vs_null is negative, min aggregation", |
| "n_negative": 7, |
| "n_total": 8, |
| "z_by_corpus": { |
| "Wikipedia_en": -1.6709216685460513, |
| "USPTO_Backgrounds": -0.7050665067014444, |
| "Pile-CC": -1.9874216306081265, |
| "StackExchange": -0.09090121377251528, |
| "Github": 0.593144297659444, |
| "OOD_News_2024": -0.7490796022292463, |
| "OOD_arXiv_2023": -0.0881460749226426, |
| "OOD_CommonPile": -0.7064942600047919 |
| }, |
| "p_one_sided_exact_binomial": 0.03515625, |
| "DEPENDENCE_CAVEAT": "the eight corpora share one activation cache and one 551-item battery; the sign test assumes independence and therefore understates the p-value" |
| }, |
| "positive_control": { |
| "what": "persist on the same draws returns 8 of 8", |
| "n_beating": 8, |
| "why_it_matters": "the design CAN return the positive branch, so min's 0 of 8 is a failure of the statistic and not a lack of power in the null" |
| } |
| }, |
| "controls_fired": { |
| "C1_scale_freedom": true, |
| "C2_pooled_sd_recomputes": true, |
| "C3_R_recomputes": true, |
| "C4_identity_in_support": true, |
| "C5_design_vs_permutation": false |
| }, |
| "VERDICT": "UNCLEAR \u2014 a control did not fire: C5_design_vs_permutation", |
| "provenance": { |
| "schema_version": 1, |
| "experiment": "DN1", |
| "utc": "2026-08-31T23:19:44+00:00", |
| "script": { |
| "path": "experiments/dn1_design_nulls.py", |
| "sha256": "06e93d9dbd43feadaf2975d5854fd7252ba55264109a66909b2f4df18ea7f6e6", |
| "n_lines": 305 |
| }, |
| "git": { |
| "commit": "9987d82887ed339bf69b5e7fa1e4fc98d5bbb4ab", |
| "commit_short": "9987d82887ed", |
| "git_repository_present": true, |
| "branch": "pythia", |
| "dirty": true, |
| "dirty_files": [ |
| "CONTEXT.md", |
| "NEELNANDA_REP.md", |
| "docs/experiments/COVERAGE.md", |
| "docs/experiments/METRICS.md", |
| "docs/experiments/descriptions/D1.md", |
| "docs/experiments/descriptions/E33.md", |
| "docs/experiments/descriptions/E34A.md", |
| "docs/experiments/descriptions/E34B.md", |
| "docs/experiments/descriptions/E36.md", |
| "docs/experiments/descriptions/E37.md", |
| "docs/experiments/descriptions/E44.md", |
| "docs/experiments/descriptions/E58.md", |
| "docs/experiments/preregs/E66d_s_quantization_limited.md", |
| "docs/experiments/preregs/superseded/PREREG_PYTHIA_T7_v2.md", |
| "docs/reproducibility/ARTIFACTS.md", |
| "docs/reproducibility/PAYLOAD_REDACTIONS.md", |
| "docs/reproducibility/SCRIPT_PROVENANCE.md", |
| "docs/reproducibility/review-snapshot-2026-08-19/MANIFEST.sha256", |
| "docs/reproducibility/review-snapshot-2026-08-19/PROVENANCE.md", |
| "docs/reproducibility/review-snapshot-2026-08-19/REDACTIONS.md", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/EXTERNAL_REVIEW.md", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/AGENT_REPORT.md", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/CONTEXT_UNDER_REVIEW.md", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/FINDINGS.md", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/JUSTIFIABILITY.md", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/LEDGER.md", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/PHASE_A_ACCOUNT.md", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/SUSPICIONS.md", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s0.json", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s1.json", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s2.json", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s0.json", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s1.json", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s2.json", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s0.json", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s1.json", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s2.json", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s0.json", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s1.json", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s2.json", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s0.json", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s1.json", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s2.json", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/balance_test.py", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/check_payload.py", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/coverage2.py", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/derange.py", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/derangement_result.json", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/logit_base.py", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/logit_baseline.json", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_anova.py", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_e52.py", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_rescore.py", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/prereg2.py", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/prereg_audit.py", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/rescore_all.py", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/rstrip_anova.py", |
| "docs/validity/AUDIT_READINESS.md", |
| "docs/validity/audit-2026-08-22/DIGEST.json", |
| "experiments/d1_min_union_diagnostic.py", |
| "experiments/e28_eval_410m.sh", |
| "experiments/e28_ladder_410m.py", |
| "experiments/exploratory/descriptions/T0.md", |
| "experiments/exploratory/descriptions/T12.md" |
| ], |
| "dirty_diff_sha256": "aff7ed44a7f13d5d00e837c12a7446366431d5a5ef0602ddfcfb051e2a0ddd20", |
| "describe": "9987d82-dirty" |
| }, |
| "argv": [ |
| "experiments/dn1_design_nulls.py" |
| ], |
| "cwd": ".", |
| "inputs": [ |
| { |
| "path": "results/d3_corpus_by_family_410m.json", |
| "sha256": "795676289554af8111a4b4464617203271848f210fe1ce29a9946d44ab5dad5a", |
| "bytes": 23102, |
| "exists": true |
| }, |
| { |
| "path": "results/r9_permutation_calibrated_min.json", |
| "sha256": "773e2a6423db3022d96f71373c859fd0ae313f529abf569c396fc7de56461b89", |
| "bytes": 22358, |
| "exists": true |
| } |
| ], |
| "env": { |
| "python": "3.14.2", |
| "torch": "2.13.0", |
| "transformers": "5.14.1", |
| "numpy": "2.5.1", |
| "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", |
| "machine": "arm64", |
| "tf32": { |
| "allow_tf32_matmul": false, |
| "allow_tf32_cudnn": true, |
| "NVIDIA_TF32_OVERRIDE": null |
| }, |
| "threads": { |
| "torch_num_threads": 10, |
| "torch_num_interop_threads": 14, |
| "OMP_NUM_THREADS": null, |
| "os_cpu_count": 14 |
| } |
| }, |
| "payload_sha256": "201a8d6db3dfd8bcb12d81b23aa253b0ee6e7795a039ddb802b39ef668518dfe" |
| } |
| } |
