| """Fail closed when Study 2--4 evidence and manuscript claims drift apart.""" |
|
|
| from __future__ import annotations |
|
|
| import json |
| from pathlib import Path |
| from typing import Any |
|
|
|
|
| ROOT = Path(__file__).resolve().parents[1] |
|
|
|
|
| def _require(condition: bool, message: str) -> None: |
| if not condition: |
| raise AssertionError(message) |
|
|
|
|
| def audit(root: Path = ROOT) -> dict[str, Any]: |
| analysis_path = root / "results" / "derived" / "study2" / "e08_analysis.json" |
| paper_path = root / "paper" / "main.tex" |
| report = json.loads(analysis_path.read_text(encoding="utf-8")) |
| study3 = json.loads( |
| (root / "results/derived/study3/e09_analysis.json").read_text(encoding="utf-8") |
| ) |
| study4 = json.loads( |
| (root / "results/derived/study4/e10_analysis.json").read_text(encoding="utf-8") |
| ) |
| ancillary = json.loads( |
| (root / "results/derived/study4_ancillary/e11_e12_analysis.json").read_text( |
| encoding="utf-8" |
| ) |
| ) |
| paper = paper_path.read_text(encoding="utf-8") |
|
|
| manifest = report["analysis_manifest"] |
| run_audit = report["audit"] |
| contrasts = report["contrasts"] |
| primary = next(row for row in contrasts if row["family"] == "confirmatory_primary") |
| secondary = [row for row in contrasts if row["family"] != "confirmatory_primary"] |
| failures = report["failure_analysis"]["failure_stages"] |
| reliability = report["reliability"] |
| diagnostic = report["exploratory_tool_protocol_diagnostic"]["by_model"] |
|
|
| _require(manifest["input_cells"] == 912, "analysis must consume 912 E08 cells") |
| _require(run_audit["main_cells"] == 840, "main cell count drift") |
| _require(run_audit["reliability_cells"] == 72, "reliability cell count drift") |
| _require(run_audit["unique_run_ids"] == 912, "run IDs are not unique/complete") |
| _require(run_audit["model_keys"] == { |
| "openai/gpt-oss-20b": 456, |
| "qwen/qwen3.6-35b-a3b": 456, |
| }, "model assignment is not balanced") |
| _require(run_audit["execution_revision"] == |
| "58933d6fa8af09fcc5a832fb3b523ffb4182bc50", "execution revision drift") |
|
|
| _require(primary["contrast"] == "P1_H007_vs_H000_M002", "primary contrast drift") |
| _require(primary["tasks"] == 60, "primary task count drift") |
| _require(primary["left_resolved"] == primary["right_resolved"] == 2, |
| "primary resolution counts drift") |
| _require(primary["paired_risk_difference"] == 0.0, "primary effect drift") |
| _require(primary["risk_difference_ci_low"] == -0.05 and |
| primary["risk_difference_ci_high"] == 0.05, "primary CI drift") |
| _require(primary["mcnemar_p"] == 1.0, "primary exact p-value drift") |
| _require(len(secondary) == 13, "secondary contrast-family size drift") |
| _require(all(row["mcnemar_p_holm"] == 1.0 for row in secondary), |
| "a secondary Holm-adjusted p-value changed") |
|
|
| _require(failures == {"empty_patch": 804, "resolved": 16, "tests": 20}, |
| "main endpoint accounting drift") |
| _require(reliability["observations"] == 72 and reliability["successes"] == 0, |
| "reliability outcome drift") |
| _require(reliability["fleiss_kappa"] is None and |
| reliability["krippendorff_alpha_nominal"] is None and |
| reliability["icc_1_1"] is None, "degenerate reliability must remain undefined") |
| _require(diagnostic["M003"]["patch_attempts"] == 887 and |
| diagnostic["M003"]["patch_protocol_errors"] == 877, |
| "exploratory protocol diagnostic drift") |
|
|
| study3_primary = next( |
| row for row in study3["contrasts"] if row["family"] == "confirmatory_primary" |
| ) |
| _require(study3["audit"]["cells"] == 540, "E09 cell count drift") |
| _require( |
| study3_primary["left_count"] == 27 |
| and study3_primary["right_count"] == 0 |
| and study3_primary["paired_risk_difference"] == 0.45, |
| "E09 primary result drift", |
| ) |
| _require( |
| {model: value["selected_interface"] for model, value in |
| study3["compatibility_gate"]["models"].items()} |
| == {"M002": "P002", "M003": "P003", "M004": "P003"}, |
| "E09 compatibility gate drift", |
| ) |
| _require(study4["audit"]["input_cells"] == 180, "E10 cell count drift") |
| _require( |
| study4["primary"]["h007_count"] == 1 |
| and study4["primary"]["h000_count"] == 0 |
| and study4["primary"]["exact_task_cluster_sign_flip_p"] == 1.0, |
| "E10 primary result drift", |
| ) |
| _require( |
| all(row["mcnemar_p_holm"] == 1.0 for row in study4["secondary_contrasts"]), |
| "E10 secondary family drift", |
| ) |
| _require(ancillary["input_cells"] == 30, "E11/E12 cell count drift") |
| _require( |
| ancillary["reliability"]["unanimous_resolution_rate"] == 1.0 |
| and ancillary["reliability"]["unanimous_trajectory_rate"] == 0.0, |
| "E11 reliability result drift", |
| ) |
| _require( |
| ancillary["context_scarcity"]["resolution_by_context"] |
| == {"16384": 0, "65536": 0}, |
| "E12 context result drift", |
| ) |
|
|
| manuscript_markers = [ |
| "5,453 scored cells", |
| "resolve 2/60 tasks", |
| "13 secondary exact tests", |
| "Only 36/840 main cells", |
| "877/887", |
| "912/912 finalized identities", |
| "exact replacement produces accepted edits on 27/60", |
| "one H007 resolution and zero H000", |
| "trajectories match in 0/6", |
| manifest["analysis_code_revision"][:12], |
| run_audit["execution_revision"][:12], |
| ] |
| missing = [marker for marker in manuscript_markers if marker not in paper] |
| _require(not missing, f"manuscript is missing generated-evidence markers: {missing}") |
|
|
| return { |
| "status": "pass", |
| "analysis_revision": manifest["analysis_code_revision"], |
| "execution_revision": run_audit["execution_revision"], |
| "cells": manifest["input_cells"], |
| "primary": primary["contrast"], |
| "secondary_tests": len(secondary), |
| "study3_cells": study3["audit"]["cells"], |
| "study4_main_cells": study4["audit"]["input_cells"], |
| "study4_ancillary_cells": ancillary["input_cells"], |
| "paper_markers_checked": len(manuscript_markers), |
| } |
|
|
|
|
| def main() -> None: |
| print(json.dumps(audit(), indent=2, sort_keys=True)) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
