File size: 4,945 Bytes
4738a0c 660c9f6 4738a0c 660c9f6 6c7e6d8 4738a0c 660c9f6 4738a0c 660c9f6 4738a0c 660c9f6 4738a0c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 | #!/usr/bin/env python3
"""Gated edit applier — the SkillOpt validation gate ENFORCED BY THE PROGRAM, not by hand.
This is the piece that was missing from the harness: a way to *edit* a skill/law document where
the edit is accepted ONLY if it survives the held-out gate. You propose a bounded edit
(append / replace / delete); the program scores the held-out lint coverage BEFORE and AFTER and:
- VALIDATION_ACCEPTED : coverage did not drop AND no previously-passing scenario regressed.
With --apply, the edit is written to the target.
- REJECTED_BUFFER : the edit regressed a scenario (or dropped coverage). The target is left
UNCHANGED and the rejected edit + reason is appended to the rejected buffer
(negative feedback, SkillOpt-style). Never hand-wave a regressing edit in.
Default is dry-run; pass --apply to actually write an accepted edit. That is the discipline: a
skill/law edit goes through THIS, not a manual editor.
Usage:
python scripts/apply_edit.py --target SKILL.md --scenarios examples/asolaria-scenarios.json \
--op append --text "new rule line" [--apply]
python scripts/apply_edit.py --target SKILL.md --scenarios ... --op replace --find OLD --text NEW
python scripts/apply_edit.py --target SKILL.md --scenarios ... --op delete --find PHRASE
"""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
from score_skill import score # reuse the v1 held-out lint scorer
def load_scenarios(path):
data = json.loads(Path(path).read_text(encoding="utf-8"))
return data["scenarios"] if isinstance(data, dict) and "scenarios" in data else data
def apply_op(text, op, find, repl):
if op == "append":
sep = "" if text.endswith("\n") else "\n"
return text + sep + repl + "\n"
if op == "replace":
if find not in text:
raise ValueError(f"replace: find text not present: {find[:50]!r}")
return text.replace(find, repl)
if op == "delete":
if find not in text:
raise ValueError(f"delete: find text not present: {find[:50]!r}")
return text.replace(find, "")
raise ValueError(f"unknown op: {op}")
def coverage(text, scenarios):
r = score(text, scenarios)
return r["passed"], r["total"], {x["id"]: x["ok"] for x in r["results"]}
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--target", required=True, help="skill/law file to edit")
ap.add_argument("--scenarios", required=True, help="canonical held-out set")
ap.add_argument("--op", required=True, choices=["append", "replace", "delete"])
ap.add_argument("--text", default="")
ap.add_argument("--find", default="")
ap.add_argument("--buffer", default="rejected-edits.log", help="rejected-edit buffer (negative feedback)")
ap.add_argument("--require-improve", default="",
help="scenario id that MUST flip FAIL->PASS (strict-improve gate; matches SkillOpt). "
"Without it the gate only requires non-regression.")
ap.add_argument("--apply", action="store_true", help="write the edit IF accepted (default: dry-run)")
a = ap.parse_args()
scenarios = load_scenarios(a.scenarios)
before = Path(a.target).read_text(encoding="utf-8")
bp, bt, bmap = coverage(before, scenarios)
try:
after = apply_op(before, a.op, a.find, a.text)
except Exception as exc: # noqa: BLE001
print(f"EDIT|verdict=ERROR|detail={exc}", file=sys.stderr)
return 2
ap_pass, at, amap = coverage(after, scenarios)
regressed = sorted(i for i in bmap if bmap[i] and not amap.get(i))
ri = a.require_improve
target_flipped = (ri in bmap and not bmap[ri] and bool(amap.get(ri))) if ri else None
improved_ok = (target_flipped is True) if ri else True
accept = at > 0 and (ap_pass >= bp) and not regressed and improved_ok # reject empty/vacuous held-out set (total=0)
verdict = "VALIDATION_ACCEPTED" if accept else "REJECTED_BUFFER"
applied = bool(accept and a.apply)
if applied:
Path(a.target).write_text(after, encoding="utf-8")
if not accept:
rec = {"op": a.op, "find": a.find[:120], "text": a.text[:120],
"before": f"{bp}/{bt}", "after": f"{ap_pass}/{at}", "regressed": regressed,
"require_improve": ri, "target_flipped": target_flipped}
with open(a.buffer, "a", encoding="utf-8") as f:
f.write(json.dumps(rec) + "\n")
flip = "n/a" if ri == "" else ("yes" if target_flipped else "no")
print(f"EDIT|verdict={verdict}|before={bp}/{bt}|after={ap_pass}/{at}|delta={ap_pass - bp}"
f"|regressed={','.join(regressed) or 'none'}|require_improve={ri or 'none'}|target_flipped={flip}|applied={applied}")
return 0 if accept else 1
if __name__ == "__main__":
raise SystemExit(main())
|