File size: 5,841 Bytes
63f41a6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 | #!/usr/bin/env python
"""Mutation check for the P2 integration seams: each mutant removes ONE guard from a private copy of the
file under test and tests/test_p2_integration.py must go red.
.venv/bin/python scripts/p2/mutation_check_integration.py
The mutated file is a copy placed in a scratch tree that mirrors the repo layout (scripts/p2/, align/utils/);
nothing in the working tree is modified, so a concurrent job importing the real files is never exposed to
a mutant. Exit status 1 if any mutant survives.
"""
from __future__ import annotations
import json
import os
import shutil
import subprocess
import sys
import tempfile
from pathlib import Path
REPO = Path(__file__).resolve().parents[2]
MUTANTS = [
# (id, file, old, new, test selector)
("S1 keys: unverified samples allowed", "scripts/p2/check_seams.py", "if missing or extra:", "if False:", "keys_seam"),
("S2 keys: stale verdict allowed", "scripts/p2/check_seams.py", "if stale:", "if False:", "stale_verdict"),
("S3 keys: accepted truncated allowed", "scripts/p2/check_seams.py", "if bad:\n raise SeamError(f\"{len(bad)} length", "if False:\n raise SeamError(f\"{len(bad)} length", "truncated"),
("S4 sft: rejected sample may be trained on", "scripts/p2/check_seams.py", "if not (key in vers and vers[key][\"accepted\"]):", "if False:", "rejected_sample"),
("S5 sft: completion not compared", "scripts/p2/check_seams.py", "if not np.array_equal(trained, np.asarray(g[\"completion_ids\"], dtype=trained.dtype)):", "if False:", "differs_by_one_token"),
("S6 sft: prompt not compared", "scripts/p2/check_seams.py", "if p != len(g[\"prompt_ids\"]) or not np.array_equal(ids[:p], np.asarray(g[\"prompt_ids\"])):", "if False:", "different_prompt"),
("S7 sft: empty shard set passes", "scripts/p2/check_seams.py", "if rows == 0:", "if False:", "vacuously"),
("L1 launch: exit code trusted over completeness", "scripts/p2/launch_baseline.py", "if rc == 0 and ok:", "if rc == 0:", "exits_zero"),
("L2 launch: retry without progress", "scripts/p2/launch_baseline.py", "if size <= last:", "if False:", "crash_loop"),
("L3 launch: finished phases rerun", "scripts/p2/launch_baseline.py", "if marker.is_file():", "if False:", "skipped_on_rerun"),
("L4 launch: orphans not reaped", "scripts/p2/launch_baseline.py", " self.kill_group(self.child, grace=15.0)\n self.child = None", " self.child = None", "reaped"),
("L5 launch: torn tail counted", "scripts/p2/launch_baseline.py", "if not line.endswith(b\"\\n\"):\n break", "if False:\n break", "torn_last_line"),
("L6 launch: children do not inherit the lock", "scripts/p2/launch_baseline.py", "\"--inherit-lock\", str(parent_pid),\n \"--label\"", "\"--label\"", "inherits_the_lock"),
("L7 launch: ver complete on any non-empty file", "scripts/p2/launch_baseline.py", "return bool(gkeys) and keys == gkeys,", "return bool(gkeys) and bool(keys),", "key_set"),
# R2: `_die_with_parent` re-reads getppid() (the parent may die between the ownership check and the prctl), which makes the
# first check redundant for a claimant that is not the holder's child; the mutant must therefore remove BOTH occurrences.
("G1 guard: parent pid unchecked", "align/utils/guard.py",
["if os.getppid() != int(parent_pid):", " if os.getppid() != parent_pid:\n raise PreflightRefused(f\"inherit_heavy_job_lock: parent"],
["if False:", " if False:\n raise PreflightRefused(f\"inherit_heavy_job_lock: parent"], "inherit"),
("G2 guard: holder label unchecked", "align/utils/guard.py", "if f\"pid={int(parent_pid)}\" not in holder.split():", "if False:", "inherit"),
("G3 guard: flock held not verified", "align/utils/guard.py", " raise PreflightRefused(\n f\"lock file names pid={parent_pid} but nobody", " return True\n raise PreflightRefused(\n f\"lock file names pid={parent_pid} but nobody", "inherit"),
("E1 export: group not signalled after result", "scripts/qwen3_export_check.py", "os.killpg(proc.pid, sig)", "pass", "vllm_child"),
]
def main() -> int:
work = Path(tempfile.mkdtemp(dir=REPO / ".tmp", prefix="mutint_"))
results = []
try:
for mid, rel, old, new, sel in MUTANTS:
tree = work / mid.split()[0]
shutil.copytree(REPO, tree, ignore=shutil.ignore_patterns(".tmp", ".venv", ".venv-rollout", ".cache", "__pycache__", ".git", "hf_cache"), dirs_exist_ok=True)
src = (tree / rel).read_text()
pairs = list(zip(old, new)) if isinstance(old, (list, tuple)) else [(old, new)]
if any(src.count(o) != 1 for o, _ in pairs):
results.append({"mutant": mid, "status": "BAD_MUTANT", "count": [src.count(o) for o, _ in pairs]})
continue
for o, n in pairs:
src = src.replace(o, n)
(tree / rel).write_text(src)
env = dict(os.environ, PYTHONPATH=str(tree))
r = subprocess.run([str(REPO / ".venv/bin/python"), "-m", "pytest", "tests/test_p2_integration.py", "-q", "-x", "-k", sel,
"-p", "no:cacheprovider"], cwd=tree, capture_output=True, text=True, env=env, timeout=600)
results.append({"mutant": mid, "status": "killed" if r.returncode != 0 else "SURVIVED", "tail": r.stdout.strip().splitlines()[-1:]})
print(results[-1], flush=True)
shutil.rmtree(tree, ignore_errors=True)
finally:
shutil.rmtree(work, ignore_errors=True)
bad = [r for r in results if r["status"] != "killed"]
print(json.dumps({"mutants": len(results), "killed": len(results) - len(bad), "not_killed": bad}, indent=1))
return 1 if bad else 0
if __name__ == "__main__":
raise SystemExit(main())
|