{ "experiment": "CTRLBF1 \u2014 backfill of three already-completed, previously unrecorded controls", "prereg": [ "docs/experiments/preregs/superseded/PREREG_E36_QLADDER.md (section E35)", "docs/experiments/preregs/P0_t52_parallel_port.md" ], "status": "RECORDED FROM EXISTING OUTPUTS \u2014 no new experiment, no new threshold", "recomputes_not_remeasures": true, "why": "CLAUDE.md rule 10 requires a control to be named with the number it produced. These three were satisfied by data already on disk but were recorded as 'NOT RECORDED' (E35 M1-C3, M1-C4) or '(full-grid arms outstanding)' (P0 C2).", "E35_M1_C3": { "requirement_verbatim": "post-2020 text | must score below M1-C1. If not, either the date argument or the index is wrong", "m1_c1_operationalised": "M1-C1 is 'a known Pile component ... high containment'. The bar used is the LOWEST of the five in-stream Pile components at k=32, 20/20 coverage \u2014 the most demanding reading of 'below M1-C1'.", "k": 32, "coverage": "20/20", "in_stream_final": { "Pile-CC": 0.9276060773863086, "Github": 0.9340004374453194, "StackExchange": 0.9179914195561899, "USPTO_Backgrounds": 0.9267616923398619, "Wikipedia_en": 0.9215951727072293 }, "lowest_in_stream": 0.9179914195561899, "post_2020": { "CONTROL_PubMed_2023": { "containment_20of20": 0.7927273508312213, "below_lowest_in_stream": true }, "OOD_Wikipedia_2023": { "containment_20of20": 0.26957534299550123, "below_lowest_in_stream": true }, "OOD_arXiv_2023": { "containment_20of20": 8.598932872430531e-05, "below_lowest_in_stream": true }, "OOD_News_2024": { "containment_20of20": 0.0007487928942136383, "below_lowest_in_stream": true } }, "fires": true, "source": "results/e48b_exposure_growth.json -> containment_by_coverage" }, "E35_M1_C4": { "requirement_verbatim": "one shard vs twenty | containment on 1/20 of the stream must be a strict lower bound on containment over 20/20", "k": 32, "by_corpus": { "CONTROL_PubMed_2023": { "containment_1of20": 0.061166716783366985, "containment_20of20": 0.7927273508312213, "is_lower_bound": true, "monotone_nondecreasing": true, "n_coverage_points": 20 }, "Github": { "containment_1of20": 0.3742096158434741, "containment_20of20": 0.9340004374453194, "is_lower_bound": true, "monotone_nondecreasing": true, "n_coverage_points": 20 }, "OOD_CommonPile": { "containment_1of20": 0.0, "containment_20of20": 0.00125, "is_lower_bound": true, "monotone_nondecreasing": true, "n_coverage_points": 20 }, "OOD_News_2024": { "containment_1of20": 1.936533347104237e-05, "containment_20of20": 0.0007487928942136383, "is_lower_bound": true, "monotone_nondecreasing": true, "n_coverage_points": 20 }, "OOD_Wikipedia_2023": { "containment_1of20": 0.022133038856352236, "containment_20of20": 0.26957534299550123, "is_lower_bound": true, "monotone_nondecreasing": true, "n_coverage_points": 20 }, "OOD_arXiv_2023": { "containment_1of20": 0.0, "containment_20of20": 8.598932872430531e-05, "is_lower_bound": true, "monotone_nondecreasing": true, "n_coverage_points": 20 }, "Pile-CC": { "containment_1of20": 0.09955354052744514, "containment_20of20": 0.9276060773863086, "is_lower_bound": true, "monotone_nondecreasing": true, "n_coverage_points": 20 }, "StackExchange": { "containment_1of20": 0.07686719226900293, "containment_20of20": 0.9179914195561899, "is_lower_bound": true, "monotone_nondecreasing": true, "n_coverage_points": 20 }, "TRAP_FineWeb": { "containment_1of20": 0.007237811980212063, "containment_20of20": 0.06309446548070848, "is_lower_bound": true, "monotone_nondecreasing": true, "n_coverage_points": 20 }, "USPTO_Backgrounds": { "containment_1of20": 0.12033439046490556, "containment_20of20": 0.9267616923398619, "is_lower_bound": true, "monotone_nondecreasing": true, "n_coverage_points": 20 }, "Wikipedia_en": { "containment_1of20": 0.07066448338254318, "containment_20of20": 0.9215951727072293, "is_lower_bound": true, "monotone_nondecreasing": true, "n_coverage_points": 20 }, "_CONTROL_random": { "containment_1of20": 0.0, "containment_20of20": 0.0, "is_lower_bound": true, "monotone_nondecreasing": true, "n_coverage_points": 20 } }, "strict_increase_for_in_stream_corpora": { "Pile-CC": true, "Github": true, "StackExchange": true, "USPTO_Backgrounds": true, "Wikipedia_en": true }, "fires": true, "note": "'strict' is checked as strict only for corpora actually present in the stream; a genuinely absent corpus pinned at 0.0 at every coverage satisfies the lower-bound requirement with equality, which is the intended behaviour and is what E48b's growth-rate reading rests on.", "source": "results/e48b_exposure_growth.json -> containment_by_coverage" }, "P0_C2": { "requirement_verbatim": "C2 - worker-count invariance. --workers 1, 2 and 8 must give identical output. Required: max_abs_diff = 0.0 between every pair, on the same 128 values.", "arms_found": { "w1": { "file": "e52_factorial_410m_rstrip_w1.json", "workers": 1, "D": 0.000845372218579525 }, "w2": { "file": "e52_factorial_410m_rstrip_w2.json", "workers": 2, "D": 0.000845372218579525 }, "w8": { "file": "e52_factorial_410m_rstrip_w8.json", "workers": 8, "D": 0.000845372218579525 }, "w6": { "file": "e52_factorial_410m_rstrip.json", "workers": 6, "D": 0.000845372218579525 } }, "n_values_compared": 128, "same_key_set_across_arms": true, "pairwise_max_abs_diff": { "w1_vs_w2": 0.0, "w1_vs_w6": 0.0, "w1_vs_w8": 0.0, "w2_vs_w6": 0.0, "w2_vs_w8": 0.0, "w6_vs_w8": 0.0 }, "pairwise_max_abs_diff_registered_arms_only": { "w1_vs_w2": 0.0, "w1_vs_w8": 0.0, "w2_vs_w8": 0.0 }, "max_abs_diff_registered": 0.0, "max_abs_diff_including_w6_reference": 0.0, "fires": true, "note": "The 128 values are the 64 fit x read cells under each of the two aggregations. w6 is the reference run (--workers 6) and is reported alongside but is not one of the registered 1/2/8 arms.", "source": "results/e52_factorial_410m_rstrip{,_w1,_w2,_w8}.json -> by_aggregation.{persist,min}.matrix" }, "controls_fired": { "E35_M1_C3": true, "E35_M1_C4": true, "P0_C2": true }, "VERDICT": "ALL THREE RECORDED. E35 M1-C3 FIRES: all 4 post-2020 panels sit below the lowest in-stream Pile component (0.9180) at k=32, 20/20 \u2014 CONTROL_PubMed_2023 0.79273, OOD_Wikipedia_2023 0.26958, OOD_arXiv_2023 0.00009, OOD_News_2024 0.00075. E35 M1-C4 FIRES: containment at 1/20 coverage is a lower bound on 20/20 for 12/12 corpora and the series is monotone non-decreasing in all of them. P0 C2 FIRES: max_abs_diff = 0.0 across every pair of --workers 1/2/8 on all 128 values. No threshold was introduced here; each requirement is quoted from its own registration.", "provenance": { "schema_version": 1, "experiment": "CTRLBF1 \u2014 backfill of three already-completed, previously unrecorded controls", "utc": "2026-09-01T20:35:05+00:00", "script": { "path": "experiments/ctrlbf1_control_backfill.py", "sha256": "6b944b4756c5d93fc012771d2bf57553b52192cc5832ee7bff360489515fdfb4", "n_lines": 156 }, "git": { "commit": "9987d82887ed339bf69b5e7fa1e4fc98d5bbb4ab", "commit_short": "9987d82887ed", "git_repository_present": true, "branch": "pythia", "dirty": true, "dirty_files": [ "CONTEXT.md", "NEELNANDA_REP.md", "corpora/dataset_revisions.json", "corpora/dataset_revisions.json.bak", "corpora/manifest_ood.json", "corpora/manifest_ood.json.bak", "docs/experiments/COVERAGE.md", "docs/experiments/METRICS.md", "docs/experiments/descriptions/D1.md", "docs/experiments/descriptions/E33.md", "docs/experiments/descriptions/E34A.md", "docs/experiments/descriptions/E34B.md", "docs/experiments/descriptions/E36.md", "docs/experiments/descriptions/E37.md", "docs/experiments/descriptions/E44.md", "docs/experiments/descriptions/E58.md", "docs/experiments/preregs/E65R_phase1_rstrip_rescore.md", "docs/experiments/preregs/E65T_threshold_defect_adjudication.md", "docs/experiments/preregs/E66d_s_quantization_limited.md", "docs/experiments/preregs/EFFRANK1_effective_rank_adjudication.md", "docs/experiments/preregs/superseded/PREREG_PYTHIA_T7_v2.md", "docs/reproducibility/ARTIFACTS.md", "docs/reproducibility/PAYLOAD_REDACTIONS.md", "docs/reproducibility/SCRIPT_PROVENANCE.md", "docs/reproducibility/review-snapshot-2026-08-19/MANIFEST.sha256", "docs/reproducibility/review-snapshot-2026-08-19/PROVENANCE.md", "docs/reproducibility/review-snapshot-2026-08-19/REDACTIONS.md", "docs/reproducibility/review-snapshot-2026-08-19/review/EXTERNAL_REVIEW.md", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/AGENT_REPORT.md", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/CONTEXT_UNDER_REVIEW.md", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/FINDINGS.md", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/JUSTIFIABILITY.md", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/LEDGER.md", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/PHASE_A_ACCOUNT.md", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/SUSPICIONS.md", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s0.json", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s1.json", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s2.json", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s0.json", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s1.json", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s2.json", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s0.json", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s1.json", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s2.json", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s0.json", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s1.json", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s2.json", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s0.json", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s1.json", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s2.json", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/balance_test.py", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/check_payload.py", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/coverage2.py", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/derange.py", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/derangement_result.json", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/logit_base.py", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/logit_baseline.json", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_anova.py", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_e52.py", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_rescore.py", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/prereg2.py", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/prereg_audit.py", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/rescore_all.py", "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/rstrip_anova.py" ], "dirty_diff_sha256": "719949b81d9871a26c26430713cafa9ad3b949cc5eea0ad080ae38307d75f63d", "describe": "9987d82-dirty" }, "argv": [ "experiments/ctrlbf1_control_backfill.py" ], "cwd": ".", "inputs": [], "env": { "python": "3.14.2", "torch": "2.13.0", "transformers": "5.14.1", "numpy": "2.5.1", "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", "machine": "arm64", "tf32": { "allow_tf32_matmul": false, "allow_tf32_cudnn": true, "NVIDIA_TF32_OVERRIDE": null }, "threads": { "torch_num_threads": 10, "torch_num_interop_threads": 14, "OMP_NUM_THREADS": null, "os_cpu_count": 14 } }, "payload_sha256": "244eafca442da2a20faa694df90c22fe3bdd41e299594c22382a37b5825e4962" } }