| |
| """Offline check (no GPU, no network): feed each example's recorded model output through the v2.9 decision |
| path and assert the documented decision. Also runs the API self-test and the v2.9 guard unit tests. |
| Usage: python3 tests/test_examples.py""" |
| import json, os, subprocess, sys |
| from pathlib import Path |
|
|
| ROOT = Path(__file__).resolve().parents[1] |
| os.environ["KD_BACKEND"] = "v29" |
| os.environ.setdefault("KD_SCRIPTS", str(ROOT / "scripts")) |
| sys.path.insert(0, str(ROOT / "serve")) |
| import kd9b_api as A |
|
|
| fails = 0 |
| for p in sorted((ROOT / "examples").glob("*.json")): |
| e = json.loads(p.read_text()) |
| rq = e["request"] |
| r = A.judge(e["model_output"], rq["doc"], rq["q"], rq["options"]) |
| if e.get("finish_reason") == "length" and r["abstain_reason"] in ("parse_fail", "schema_fail"): |
| r["abstain_reason"] = "truncated" |
| want = e["expected"]["v29"] |
| got = {"decision": r["decision"], "level": r["level"], "abstain_reason": r["abstain_reason"]} |
| ok = got == want |
| fails += not ok |
| print(("OK " if ok else "FAIL"), p.stem, got if ok else f"{got} != {want}") |
| for cmd in ([sys.executable, str(ROOT / "serve/kd9b_api.py"), "selftest"], [sys.executable, str(ROOT / "scripts/test_esg_v29_guard.py")]): |
| rc = subprocess.run(cmd, env=dict(os.environ), capture_output=True, text=True) |
| print(("OK " if rc.returncode == 0 else "FAIL"), Path(cmd[1]).name, (rc.stdout.strip().splitlines() or [""])[-1]) |
| fails += rc.returncode != 0 |
| print("PASS" if not fails else f"FAIL {fails}") |
| sys.exit(1 if fails else 0) |
|
|
