| |
| """Fail if Study 5 derived evidence and manuscript claims drift apart.""" |
|
|
| from __future__ import annotations |
|
|
| import json |
| from pathlib import Path |
|
|
|
|
| ROOT = Path(__file__).resolve().parents[1] |
|
|
|
|
| def require(condition: bool, message: str) -> None: |
| if not condition: |
| raise RuntimeError(message) |
|
|
|
|
| def audit(root: Path = ROOT) -> dict[str, object]: |
| derived = root / "results" / "derived" / "study5" |
| audit = json.loads((derived / "artifact_audit.json").read_text()) |
| stats = json.loads((derived / "statistical_results.json").read_text()) |
| paper = (root / "paper" / "main.tex").read_text() |
| expected = { |
| "E13": (1440, 624, 610, 148), |
| "E14": (540, 143, 140, 0), |
| "E15": (540, 206, 199, 0), |
| "E16": (306, 126, 124, 1), |
| } |
| require(audit["passed"] and audit["total_cells"] == 2826, "Study 5 audit/count drift") |
| for experiment, values in expected.items(): |
| row = audit["experiments"][experiment] |
| observed = ( |
| row["run_count"], row["accepted_edit_count"], |
| row["applicable_patch_count"], row["resolved_count"], |
| ) |
| require(observed == values, f"{experiment} funnel drift: {observed}") |
| rank = stats["e16_held_out"]["rank_stability"] |
| require(abs(rank["accepted"]["spearman_rank_correlation"] - 0.04411764705882353) < 1e-12, |
| "E16 accepted rank-correlation drift") |
| require(abs(rank["applicable"]["spearman_rank_correlation"] + 0.2125118592516207) < 1e-12, |
| "E16 applicable rank-correlation drift") |
| require(abs(rank["tokens"]["spearman_rank_correlation"] - 0.9428571428571428) < 1e-12, |
| "E16 token rank-correlation drift") |
| markers = ( |
| "5,453 scored experimental cells", "2,826 new cells", |
| "624 executor-accepted", "143 accepted edits", "206 accepted", |
| "126 accepted edits", "$\\rho=.04$ for accepted edits", "$-.21$ for applicable patches", |
| "mean-token rank is highly stable ($\\rho=.94$)", |
| "Only one held-out cell resolves", |
| ) |
| missing = [marker for marker in markers if marker not in paper] |
| require(not missing, f"manuscript is missing Study 5 evidence markers: {missing}") |
| require("When Better Retrieval Does Not Produce Better Patches" not in paper, |
| "legacy caveat title remains in manuscript") |
| return { |
| "status": "pass", "study5_cells": audit["total_cells"], |
| "audit_sha256": audit["audit_sha256"], "markers_checked": len(markers), |
| } |
|
|
|
|
| def main() -> None: |
| print(json.dumps(audit(), indent=2, sort_keys=True)) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
