#!/usr/bin/env python """Mutation check for the reviewer-R2 fixes: each mutant reverts ONE fix in a private copy of the tree and tests/test_p2_r2_review.py must go red. .venv/bin/python scripts/p2/mutation_check_r2.py Same method as scripts/p2/mutation_check_integration.py (scratch tree under .tmp; the working tree is never modified). Exit status 1 if any mutant survives or any mutation does not apply exactly once. """ from __future__ import annotations import json import os import shutil import subprocess import sys import tempfile from pathlib import Path REPO = Path(__file__).resolve().parents[2] MUTANTS = [ # (id, file, old, new, test selector) ("D1 guard: borrowed lock not tied to the holder", "align/utils/guard.py", " _die_with_parent(int(parent_pid))\n", "", "borrowed"), ("D2 guard: pdeathsig failure ignored", "align/utils/guard.py", " if rc != 0:\n raise PreflightRefused(\n f\"inherit_heavy_job_lock: prctl", " if False:\n raise PreflightRefused(\n f\"inherit_heavy_job_lock: prctl", "pdeathsig_failure"), ("D3 guard: parent death before prctl ignored", "align/utils/guard.py", " if os.getppid() != parent_pid:\n raise PreflightRefused(f\"inherit_heavy_job_lock: parent", " if False:\n raise PreflightRefused(f\"inherit_heavy_job_lock: parent", "parent_that_died"), ("S1 supervisor: stop signal before child recorded ignored", "scripts/p2/launch_baseline.py", " if self.stop:\n # A stop signal that landed between", " if False:\n # A stop signal that landed between", "stop_signal"), ("S2 supervisor: zombie leader counted as alive", "scripts/p2/launch_baseline.py", "if int(rest[2]) == pgid and rest[0] != \"Z\":", "if int(rest[2]) == pgid:", "zombie"), ("S3 supervisor: group liveness ignores non-leader members", "scripts/p2/launch_baseline.py", "if int(rest[2]) == pgid and rest[0] != \"Z\":", "if int(entry) == pgid and rest[0] != \"Z\":", "live_member"), ("R1 lean_gen: model_rev not compared on resume", "extras/vllm_rollout/lean_gen.py", "if r.get(\"model_rev\") != model_rev:", "if False:", "another_models_output"), ("R2 lean_gen: seed not compared on resume", "extras/vllm_rollout/lean_gen.py", "elif r.get(\"sampling\", {}).get(\"seed\") != want:", "elif False:", "base_seed"), ("R3 lean_gen: sampling params not compared on resume", "extras/vllm_rollout/lean_gen.py", "elif r.get(\"sampling\", {}).get(\"temperature\") != temperature or r.get(\"sampling\", {}).get(\"top_p\") != top_p:", "elif False:", "sampling_parameters"), ("R4 lean_gen: conflict count truncated", "extras/vllm_rollout/lean_gen.py", " if n_bad > len(bad):", " if False:", "bounded"), ("R5 lean_gen: prompt module change ignored", "extras/vllm_rollout/lean_gen.py", "if s.get(\"lean_prompts_sha256\") != prompts_sha256})", "if False})", "prompt_module_edited"), ("R6 lean_gen: resume check not wired into main", "extras/vllm_rollout/lean_gen.py", " conflicts = resume_conflicts(a.out, rev, a.seed)\n", " conflicts = []\n", "main_refuses"), ("P1 sft.py: repo root not put on sys.path", "scripts/sft.py", "sys.path.insert(0, str(Path(__file__).resolve().parent.parent))\n", "", "sft_script"), ("H1 shm sweep: segments a live process holds are removed", "scripts/p2/launch_baseline.py", "or path in held:", "or False:", "sweep_removes"), ("H2 shm sweep: no time bound (other jobs' old segments removed)", "scripts/p2/launch_baseline.py", "or st.st_mtime < since", "or False", "sweep_removes"), ("H3 shm sweep: any file name removed", "scripts/p2/launch_baseline.py", "if not re.fullmatch(SHM_NAME_RE, name):", "if False:", "sweep_removes"), ("H4 shm sweep: not run after the child exits", "scripts/p2/launch_baseline.py", " gone, nbytes = sweep_orphan_shm(started_at)\n", " gone, nbytes = [], 0\n", "run_child_sweeps"), ("T1 conftest: a held heavy-job lock is not seen by gpu tests", "tests/conftest.py", " return None if f\"pid={os.getpid()}\" in label.split() else (label or \"unknown holder\")", " return None", "gpu_test_guard"), ("T2 conftest: own lock blocks own tests", "tests/conftest.py", "None if f\"pid={os.getpid()}\" in label.split() else (label or \"unknown holder\")", "(label or \"unknown holder\")", "gpu_test_guard"), ] def main() -> int: (REPO / ".tmp").mkdir(exist_ok=True) work = Path(tempfile.mkdtemp(dir=REPO / ".tmp", prefix="mutr2_")) results = [] try: for mid, rel, old, new, sel in MUTANTS: tree = work / mid.split()[0] shutil.copytree(REPO, tree, ignore=shutil.ignore_patterns(".tmp", ".venv", ".venv-rollout", ".cache", "__pycache__", ".git", "hf_cache"), dirs_exist_ok=True) src = (tree / rel).read_text() if src.count(old) != 1: results.append({"mutant": mid, "status": "BAD_MUTANT", "count": src.count(old)}) print(results[-1], flush=True) continue (tree / rel).write_text(src.replace(old, new)) env = dict(os.environ, PYTHONPATH=str(tree)) r = subprocess.run([str(REPO / ".venv/bin/python"), "-m", "pytest", "tests/test_p2_r2_review.py", "-q", "-x", "-k", sel, "-p", "no:cacheprovider"], cwd=tree, capture_output=True, text=True, env=env, timeout=600) results.append({"mutant": mid, "status": "killed" if r.returncode == 1 else ("SURVIVED" if r.returncode == 0 else f"BAD_SELECTOR_rc{r.returncode}"), "tail": r.stdout.strip().splitlines()[-1:]}) print(results[-1], flush=True) shutil.rmtree(tree, ignore_errors=True) finally: shutil.rmtree(work, ignore_errors=True) bad = [r for r in results if r["status"] != "killed"] print(json.dumps({"mutants": len(results), "killed": len(results) - len(bad), "not_killed": bad}, indent=1)) return 1 if bad else 0 if __name__ == "__main__": raise SystemExit(main())