File size: 6,362 Bytes
63f41a6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 | #!/usr/bin/env python
"""Hand-written mutation check for the expert-iteration driver and weight averaging.
Each mutant removes ONE rule in a private COPY of the repo subset and re-runs the targeted tests; a mutant
must make them fail ("KILLED"). The live tree is never touched. A SURVIVED or BAD-MUTANT line fails the run.
CUDA_VISIBLE_DEVICES= .venv/bin/python scripts/p2/mutation_check_ei.py
"""
import json, os, shutil, subprocess, sys
from pathlib import Path
ROOT = str(Path(__file__).resolve().parents[2])
MR = f"{ROOT}/.tmp/p2/mutroot_ei"
PY = sys.executable
EI = "scripts/p2/expert_iteration.py"
WA = "align/model/weight_average.py"
SD = "scripts/p2/build_sft_data.py"
TESTS = ["tests/test_expert_iteration.py", "tests/test_weight_average.py"]
M = [
("M01 band lower bound inclusive", EI, [('elif m["p"] <= lo:', 'elif m["p"] < lo:')]),
("M02 band upper bound exclusive", EI, [('elif m["p"] > hi:', 'elif m["p"] >= hi:')]),
("M03 unsolved problems always resampled", EI, [(" if not include_zero:\n for pid in groups[\"unsolved\"]:", " if False:\n for pid in groups[\"unsolved\"]:")]),
("M04 shortest rank ignored by the SFT builder", SD, [('if getattr(a, "rank", "order") == "shortest":', 'if False:')]),
("M05 shortest picks the LONGEST", SD, [('first.sort(key=lambda k: (idx.meta[k]["n_completion"], k))', 'first.sort(key=lambda k: (-idx.meta[k]["n_completion"], k))')]),
("M06 stale marker accepted (config change not detected)", EI, [(' if mk.get("chain") != s.chain:\n return "stale"', ' if False:\n return "stale"')]),
("M07 non-zero exit still writes the marker", EI, [(" if rc != 0:\n raise DriverError", " if False:\n raise DriverError")]),
("M08 partial outputs not cleaned before a re-run", EI, [(" for p in s.pre_clean:\n if Path(p).exists():", " for p in []:\n if Path(p).exists():")]),
("M09 resume re-runs finished stages", EI, [(' if status == "done":\n say(f"[round {s.r}] {s.name}: done (skipped)")\n return "skipped"', ' if False:\n return "skipped"')]),
("M10 export picks the OLDEST checkpoint", EI, [(" return cands[-1] if cands else None", " return cands[0] if cands else None")]),
("M11 export merges onto the hard-coded snapshot", EI, [('{**env_py, "P2_EXPORT_BASE_SNAPSHOT": gen_model}', "dict(env_py)")]),
("M12 pass-rate table not carried across rounds", EI, [(" table = dict(prev)\n for pid, m in new.items():", " table = {}\n for pid, m in new.items():")]),
("M13 regression boundary off by one", EI, [("last[\"solved\"] <= prior_best - max_regress", "last[\"solved\"] < prior_best - max_regress")]),
("M14 plateau needs only one flat round", EI, [(" if flat == patience:", " if flat >= 1:")]),
("M15 stop rule compares across different n/cap", EI, [('comp = [h for h in history if (h["n_eval"], h["cap"]) == (last["n_eval"], last["cap"])]', "comp = list(history)")]),
("M16 average: storage-dtype arithmetic", WA, [("fa[lo:hi].to(torch.float64)", "fa[lo:hi]"), ("fb[lo:hi].to(torch.float64)", "fb[lo:hi]")]),
("M17 average: weights swapped", WA, [("((1.0 - w) * fa[lo:hi]", "((w) * fa[lo:hi]"), ("+ w * fb[lo:hi]", "+ (1.0 - w) * fb[lo:hi]")]),
("M18 average: last element of each chunk dropped", WA, [("hi = min(lo + chunk_elems, fa.numel())", "hi = min(lo + chunk_elems, fa.numel()) - (1 if fa.numel() > chunk_elems and lo + chunk_elems < fa.numel() else 0)")]),
("M19 average: shape mismatch tolerated", WA, [(" if ha[n][1] != hb[n][1] or ha[n][2] != hb[n][2]:", " if False:")]),
("M20 average: partial output left behind on a crash", WA, [(" handles.clear()\n shutil.rmtree(part, ignore_errors=True)\n raise", " handles.clear()\n raise")]),
]
def setup():
shutil.rmtree(MR, ignore_errors=True)
os.makedirs(MR)
shutil.copytree(f"{ROOT}/align", f"{MR}/align", ignore=shutil.ignore_patterns("__pycache__"))
shutil.copytree(f"{ROOT}/scripts/p2", f"{MR}/scripts/p2", ignore=shutil.ignore_patterns("__pycache__"))
shutil.copy(f"{ROOT}/scripts/qwen3_export_check.py", f"{MR}/scripts/qwen3_export_check.py")
os.makedirs(f"{MR}/tests")
for f in ("conftest.py", "test_p2_support.py", *[Path(t).name for t in TESTS]):
shutil.copy(f"{ROOT}/tests/{f}", f"{MR}/tests/{f}")
os.makedirs(f"{MR}/extras/vllm_rollout")
shutil.copy(f"{ROOT}/extras/vllm_rollout/lean_prompts.py", f"{MR}/extras/vllm_rollout/lean_prompts.py")
shutil.copy(f"{ROOT}/pyproject.toml", MR)
os.makedirs(f"{MR}/docs")
shutil.copy(f"{ROOT}/docs/P2.md", f"{MR}/docs/P2.md")
def run_tests():
env = dict(os.environ, CUDA_VISIBLE_DEVICES="", PYTHONDONTWRITEBYTECODE="1")
r = subprocess.run([PY, "-m", "pytest", *TESTS, "-q", "-x", "-p", "no:cacheprovider"], cwd=MR, env=env, capture_output=True, text=True)
return r.returncode, r.stdout[-600:]
def main():
setup()
rc, out = run_tests()
if rc != 0:
print("BASELINE FAILS in the copy:\n", out)
return 2
results, bad = [], 0
for name, rel, subs in M:
path = f"{MR}/{rel}"
orig = open(path).read()
mut = orig
ok = True
for old, new in subs:
if old not in mut:
ok = False
break
mut = mut.replace(old, new, 1)
if not ok:
print(f"BAD-MUTANT {name}: pattern not found in {rel}")
results.append({"mutant": name, "result": "BAD-MUTANT"})
bad += 1
continue
open(path, "w").write(mut)
try:
rc, out = run_tests()
finally:
open(path, "w").write(orig)
res = "KILLED" if rc != 0 else "SURVIVED"
bad += res == "SURVIVED"
failing = [l for l in out.splitlines() if l.startswith("FAILED")][:1]
print(f"{res:9s} {name} {failing[0][:110] if failing else ''}")
results.append({"mutant": name, "result": res})
os.makedirs(f"{ROOT}/.tmp/p2", exist_ok=True)
json.dump(results, open(f"{ROOT}/.tmp/p2/mutants_result_ei.json", "w"), indent=1)
print(f"{sum(r['result'] == 'KILLED' for r in results)}/{len(M)} killed")
return 1 if bad else 0
if __name__ == "__main__":
sys.exit(main())
|