| { |
| "experiment": "CTRLBF1 \u2014 backfill of three already-completed, previously unrecorded controls", |
| "prereg": [ |
| "docs/experiments/preregs/superseded/PREREG_E36_QLADDER.md (section E35)", |
| "docs/experiments/preregs/P0_t52_parallel_port.md" |
| ], |
| "status": "RECORDED FROM EXISTING OUTPUTS \u2014 no new experiment, no new threshold", |
| "recomputes_not_remeasures": true, |
| "why": "CLAUDE.md rule 10 requires a control to be named with the number it produced. These three were satisfied by data already on disk but were recorded as 'NOT RECORDED' (E35 M1-C3, M1-C4) or '(full-grid arms outstanding)' (P0 C2).", |
| "E35_M1_C3": { |
| "requirement_verbatim": "post-2020 text | must score below M1-C1. If not, either the date argument or the index is wrong", |
| "m1_c1_operationalised": "M1-C1 is 'a known Pile component ... high containment'. The bar used is the LOWEST of the five in-stream Pile components at k=32, 20/20 coverage \u2014 the most demanding reading of 'below M1-C1'.", |
| "k": 32, |
| "coverage": "20/20", |
| "in_stream_final": { |
| "Pile-CC": 0.9276060773863086, |
| "Github": 0.9340004374453194, |
| "StackExchange": 0.9179914195561899, |
| "USPTO_Backgrounds": 0.9267616923398619, |
| "Wikipedia_en": 0.9215951727072293 |
| }, |
| "lowest_in_stream": 0.9179914195561899, |
| "post_2020": { |
| "CONTROL_PubMed_2023": { |
| "containment_20of20": 0.7927273508312213, |
| "below_lowest_in_stream": true |
| }, |
| "OOD_Wikipedia_2023": { |
| "containment_20of20": 0.26957534299550123, |
| "below_lowest_in_stream": true |
| }, |
| "OOD_arXiv_2023": { |
| "containment_20of20": 8.598932872430531e-05, |
| "below_lowest_in_stream": true |
| }, |
| "OOD_News_2024": { |
| "containment_20of20": 0.0007487928942136383, |
| "below_lowest_in_stream": true |
| } |
| }, |
| "fires": true, |
| "source": "results/e48b_exposure_growth.json -> containment_by_coverage" |
| }, |
| "E35_M1_C4": { |
| "requirement_verbatim": "one shard vs twenty | containment on 1/20 of the stream must be a strict lower bound on containment over 20/20", |
| "k": 32, |
| "by_corpus": { |
| "CONTROL_PubMed_2023": { |
| "containment_1of20": 0.061166716783366985, |
| "containment_20of20": 0.7927273508312213, |
| "is_lower_bound": true, |
| "monotone_nondecreasing": true, |
| "n_coverage_points": 20 |
| }, |
| "Github": { |
| "containment_1of20": 0.3742096158434741, |
| "containment_20of20": 0.9340004374453194, |
| "is_lower_bound": true, |
| "monotone_nondecreasing": true, |
| "n_coverage_points": 20 |
| }, |
| "OOD_CommonPile": { |
| "containment_1of20": 0.0, |
| "containment_20of20": 0.00125, |
| "is_lower_bound": true, |
| "monotone_nondecreasing": true, |
| "n_coverage_points": 20 |
| }, |
| "OOD_News_2024": { |
| "containment_1of20": 1.936533347104237e-05, |
| "containment_20of20": 0.0007487928942136383, |
| "is_lower_bound": true, |
| "monotone_nondecreasing": true, |
| "n_coverage_points": 20 |
| }, |
| "OOD_Wikipedia_2023": { |
| "containment_1of20": 0.022133038856352236, |
| "containment_20of20": 0.26957534299550123, |
| "is_lower_bound": true, |
| "monotone_nondecreasing": true, |
| "n_coverage_points": 20 |
| }, |
| "OOD_arXiv_2023": { |
| "containment_1of20": 0.0, |
| "containment_20of20": 8.598932872430531e-05, |
| "is_lower_bound": true, |
| "monotone_nondecreasing": true, |
| "n_coverage_points": 20 |
| }, |
| "Pile-CC": { |
| "containment_1of20": 0.09955354052744514, |
| "containment_20of20": 0.9276060773863086, |
| "is_lower_bound": true, |
| "monotone_nondecreasing": true, |
| "n_coverage_points": 20 |
| }, |
| "StackExchange": { |
| "containment_1of20": 0.07686719226900293, |
| "containment_20of20": 0.9179914195561899, |
| "is_lower_bound": true, |
| "monotone_nondecreasing": true, |
| "n_coverage_points": 20 |
| }, |
| "TRAP_FineWeb": { |
| "containment_1of20": 0.007237811980212063, |
| "containment_20of20": 0.06309446548070848, |
| "is_lower_bound": true, |
| "monotone_nondecreasing": true, |
| "n_coverage_points": 20 |
| }, |
| "USPTO_Backgrounds": { |
| "containment_1of20": 0.12033439046490556, |
| "containment_20of20": 0.9267616923398619, |
| "is_lower_bound": true, |
| "monotone_nondecreasing": true, |
| "n_coverage_points": 20 |
| }, |
| "Wikipedia_en": { |
| "containment_1of20": 0.07066448338254318, |
| "containment_20of20": 0.9215951727072293, |
| "is_lower_bound": true, |
| "monotone_nondecreasing": true, |
| "n_coverage_points": 20 |
| }, |
| "_CONTROL_random": { |
| "containment_1of20": 0.0, |
| "containment_20of20": 0.0, |
| "is_lower_bound": true, |
| "monotone_nondecreasing": true, |
| "n_coverage_points": 20 |
| } |
| }, |
| "strict_increase_for_in_stream_corpora": { |
| "Pile-CC": true, |
| "Github": true, |
| "StackExchange": true, |
| "USPTO_Backgrounds": true, |
| "Wikipedia_en": true |
| }, |
| "fires": true, |
| "note": "'strict' is checked as strict only for corpora actually present in the stream; a genuinely absent corpus pinned at 0.0 at every coverage satisfies the lower-bound requirement with equality, which is the intended behaviour and is what E48b's growth-rate reading rests on.", |
| "source": "results/e48b_exposure_growth.json -> containment_by_coverage" |
| }, |
| "P0_C2": { |
| "requirement_verbatim": "C2 - worker-count invariance. --workers 1, 2 and 8 must give identical output. Required: max_abs_diff = 0.0 between every pair, on the same 128 values.", |
| "arms_found": { |
| "w1": { |
| "file": "e52_factorial_410m_rstrip_w1.json", |
| "workers": 1, |
| "D": 0.000845372218579525 |
| }, |
| "w2": { |
| "file": "e52_factorial_410m_rstrip_w2.json", |
| "workers": 2, |
| "D": 0.000845372218579525 |
| }, |
| "w8": { |
| "file": "e52_factorial_410m_rstrip_w8.json", |
| "workers": 8, |
| "D": 0.000845372218579525 |
| }, |
| "w6": { |
| "file": "e52_factorial_410m_rstrip.json", |
| "workers": 6, |
| "D": 0.000845372218579525 |
| } |
| }, |
| "n_values_compared": 128, |
| "same_key_set_across_arms": true, |
| "pairwise_max_abs_diff": { |
| "w1_vs_w2": 0.0, |
| "w1_vs_w6": 0.0, |
| "w1_vs_w8": 0.0, |
| "w2_vs_w6": 0.0, |
| "w2_vs_w8": 0.0, |
| "w6_vs_w8": 0.0 |
| }, |
| "pairwise_max_abs_diff_registered_arms_only": { |
| "w1_vs_w2": 0.0, |
| "w1_vs_w8": 0.0, |
| "w2_vs_w8": 0.0 |
| }, |
| "max_abs_diff_registered": 0.0, |
| "max_abs_diff_including_w6_reference": 0.0, |
| "fires": true, |
| "note": "The 128 values are the 64 fit x read cells under each of the two aggregations. w6 is the reference run (--workers 6) and is reported alongside but is not one of the registered 1/2/8 arms.", |
| "source": "results/e52_factorial_410m_rstrip{,_w1,_w2,_w8}.json -> by_aggregation.{persist,min}.matrix" |
| }, |
| "controls_fired": { |
| "E35_M1_C3": true, |
| "E35_M1_C4": true, |
| "P0_C2": true |
| }, |
| "VERDICT": "ALL THREE RECORDED. E35 M1-C3 FIRES: all 4 post-2020 panels sit below the lowest in-stream Pile component (0.9180) at k=32, 20/20 \u2014 CONTROL_PubMed_2023 0.79273, OOD_Wikipedia_2023 0.26958, OOD_arXiv_2023 0.00009, OOD_News_2024 0.00075. E35 M1-C4 FIRES: containment at 1/20 coverage is a lower bound on 20/20 for 12/12 corpora and the series is monotone non-decreasing in all of them. P0 C2 FIRES: max_abs_diff = 0.0 across every pair of --workers 1/2/8 on all 128 values. No threshold was introduced here; each requirement is quoted from its own registration.", |
| "provenance": { |
| "schema_version": 1, |
| "experiment": "CTRLBF1 \u2014 backfill of three already-completed, previously unrecorded controls", |
| "utc": "2026-09-01T20:35:05+00:00", |
| "script": { |
| "path": "experiments/ctrlbf1_control_backfill.py", |
| "sha256": "6b944b4756c5d93fc012771d2bf57553b52192cc5832ee7bff360489515fdfb4", |
| "n_lines": 156 |
| }, |
| "git": { |
| "commit": "9987d82887ed339bf69b5e7fa1e4fc98d5bbb4ab", |
| "commit_short": "9987d82887ed", |
| "git_repository_present": true, |
| "branch": "pythia", |
| "dirty": true, |
| "dirty_files": [ |
| "CONTEXT.md", |
| "NEELNANDA_REP.md", |
| "corpora/dataset_revisions.json", |
| "corpora/dataset_revisions.json.bak", |
| "corpora/manifest_ood.json", |
| "corpora/manifest_ood.json.bak", |
| "docs/experiments/COVERAGE.md", |
| "docs/experiments/METRICS.md", |
| "docs/experiments/descriptions/D1.md", |
| "docs/experiments/descriptions/E33.md", |
| "docs/experiments/descriptions/E34A.md", |
| "docs/experiments/descriptions/E34B.md", |
| "docs/experiments/descriptions/E36.md", |
| "docs/experiments/descriptions/E37.md", |
| "docs/experiments/descriptions/E44.md", |
| "docs/experiments/descriptions/E58.md", |
| "docs/experiments/preregs/E65R_phase1_rstrip_rescore.md", |
| "docs/experiments/preregs/E65T_threshold_defect_adjudication.md", |
| "docs/experiments/preregs/E66d_s_quantization_limited.md", |
| "docs/experiments/preregs/EFFRANK1_effective_rank_adjudication.md", |
| "docs/experiments/preregs/superseded/PREREG_PYTHIA_T7_v2.md", |
| "docs/reproducibility/ARTIFACTS.md", |
| "docs/reproducibility/PAYLOAD_REDACTIONS.md", |
| "docs/reproducibility/SCRIPT_PROVENANCE.md", |
| "docs/reproducibility/review-snapshot-2026-08-19/MANIFEST.sha256", |
| "docs/reproducibility/review-snapshot-2026-08-19/PROVENANCE.md", |
| "docs/reproducibility/review-snapshot-2026-08-19/REDACTIONS.md", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/EXTERNAL_REVIEW.md", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/AGENT_REPORT.md", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/CONTEXT_UNDER_REVIEW.md", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/FINDINGS.md", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/JUSTIFIABILITY.md", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/LEDGER.md", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/PHASE_A_ACCOUNT.md", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/SUSPICIONS.md", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s0.json", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s1.json", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s2.json", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s0.json", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s1.json", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s2.json", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s0.json", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s1.json", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s2.json", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s0.json", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s1.json", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s2.json", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s0.json", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s1.json", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s2.json", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/balance_test.py", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/check_payload.py", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/coverage2.py", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/derange.py", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/derangement_result.json", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/logit_base.py", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/logit_baseline.json", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_anova.py", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_e52.py", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_rescore.py", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/prereg2.py", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/prereg_audit.py", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/rescore_all.py", |
| "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/rstrip_anova.py" |
| ], |
| "dirty_diff_sha256": "719949b81d9871a26c26430713cafa9ad3b949cc5eea0ad080ae38307d75f63d", |
| "describe": "9987d82-dirty" |
| }, |
| "argv": [ |
| "experiments/ctrlbf1_control_backfill.py" |
| ], |
| "cwd": ".", |
| "inputs": [], |
| "env": { |
| "python": "3.14.2", |
| "torch": "2.13.0", |
| "transformers": "5.14.1", |
| "numpy": "2.5.1", |
| "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", |
| "machine": "arm64", |
| "tf32": { |
| "allow_tf32_matmul": false, |
| "allow_tf32_cudnn": true, |
| "NVIDIA_TF32_OVERRIDE": null |
| }, |
| "threads": { |
| "torch_num_threads": 10, |
| "torch_num_interop_threads": 14, |
| "OMP_NUM_THREADS": null, |
| "os_cpu_count": 14 |
| } |
| }, |
| "payload_sha256": "244eafca442da2a20faa694df90c22fe3bdd41e299594c22382a37b5825e4962" |
| } |
| } |
