File size: 2,948 Bytes
c8af058 6c7e6d8 c8af058 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 | #!/usr/bin/env python3
"""Deterministic v1 SkillOpt-style held-out scorer for skill/law text."""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
def norm(text: str) -> str:
return " ".join(text.lower().split())
def phrase_present(text: str, phrase: str) -> bool:
return norm(phrase) in text
def score(skill_text: str, scenarios: list[dict]) -> dict:
text = norm(skill_text)
results = []
passed = 0
for scenario in scenarios:
missing_groups = []
for group in scenario.get("must_include_any", []):
if not any(phrase_present(text, phrase) for phrase in group):
missing_groups.append(group)
forbidden_hits = [
phrase
for phrase in scenario.get("must_not_include", [])
if phrase_present(text, phrase)
]
ok = not missing_groups and not forbidden_hits
passed += 1 if ok else 0
results.append(
{
"id": scenario.get("id"),
"ok": ok,
"missing_groups": missing_groups,
"forbidden_hits": forbidden_hits,
"failure": scenario.get("failure", ""),
}
)
return {
"ok": len(scenarios) > 0 and passed == len(scenarios),
"passed": passed,
"total": len(scenarios),
"score": passed / len(scenarios) if scenarios else 0.0,
"results": results,
}
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--skill", required=True, help="Path to skill/law markdown")
parser.add_argument("--scenarios", required=True, help="Path to scenario JSON")
parser.add_argument("--report", help="Optional JSON report output path")
args = parser.parse_args()
try:
skill_text = Path(args.skill).read_text(encoding="utf-8")
scenarios = json.loads(Path(args.scenarios).read_text(encoding="utf-8"))
if not isinstance(scenarios, list):
raise ValueError("scenario file must contain a JSON list")
report = score(skill_text, scenarios)
except Exception as exc:
print(f"HARNESSEDIT|ok=0|error={type(exc).__name__}|detail={exc}", file=sys.stderr)
return 2
if args.report:
out = Path(args.report)
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
print(
"HARNESSEDIT|ok={ok}|passed={passed}|total={total}|score={score:.3f}".format(
ok=1 if report["ok"] else 0,
passed=report["passed"],
total=report["total"],
score=report["score"],
)
)
for item in report["results"]:
print(f"HARNESSEDITCASE|id={item['id']}|ok={1 if item['ok'] else 0}")
return 0 if report["ok"] else 1
if __name__ == "__main__":
raise SystemExit(main())
|