Spaces:
Running
Running
File size: 4,206 Bytes
898a901 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 | """AuditKIT agent_eval over a student run: independent outcome verification.
Each logged run row becomes a recorded AgentEpisode (user prompt -> assistant raw
output). The outcome oracle is a CustomPredicate that RE-EXECUTES the model's SQL
against the real database and compares with the gold result - AuditKIT's verdict is a
fresh execution, not a read of our logged status. We then cross-check AuditKIT's
verdicts against the harness log (they must agree 100%).
Usage: python audit_eval.py --run baseline-test --split-file ../../data/splits/test.jsonl
"""
from __future__ import annotations
import argparse
import json
import os
import sys
from collections import Counter
HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)
from execcheck import check # noqa: E402
from student import PROMPT # noqa: E402
RUNS_REPO = "HideIron/selfheal-sql-runs"
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("--run", required=True)
ap.add_argument("--split-file", required=True)
ap.add_argument("--out", default=None)
ap.add_argument("--no-push", action="store_true")
args = ap.parse_args()
from huggingface_hub import HfApi, hf_hub_download
from auditkit.agent_eval import (
AgentCase,
AgentEvalRunner,
AgentEvalSpec,
CustomPredicate,
episode_from_openai_messages,
)
rp = hf_hub_download(RUNS_REPO, f"runs/{args.run}/results.jsonl", repo_type="dataset")
rows = [json.loads(l) for l in open(rp, encoding="utf-8")]
splits = {r["qid"]: r for r in map(json.loads, open(args.split_file, encoding="utf-8"))}
def exec_predicate(episode, case):
s = splits[case.id]
res = check(s["db_path"], episode.final_answer or "", s["gold_sql"])
return bool(res.correct)
episodes, cases = [], []
for r in rows:
s = splits[r["qid"]]
episodes.append(
episode_from_openai_messages(
[
{"role": "user",
"content": PROMPT.format(schema=s["schema"], question=s["question"])},
{"role": "assistant", "content": r["raw_output"]},
],
case_id=r["qid"],
final_answer=r["raw_output"],
)
)
cases.append(
AgentCase(
id=r["qid"], task=s["question"], category="text_to_sql",
outcome=CustomPredicate(exec_predicate, version="execcheck-v1",
name="spider_execution_accuracy"),
metadata={"db_id": s["db_id"]},
)
)
spec = AgentEvalSpec(cases=cases, mode="recorded", episodes=episodes,
scorers=[], trials=1)
result = AgentEvalRunner().run(spec)
verdicts = {row.case_id: row.outcome["verdict"] for row in result.rows}
logged = {r["qid"]: ("success" if r["status"] == "correct" else "failure") for r in rows}
agree = sum(verdicts[q] == logged[q] for q in verdicts)
out = {
"run": args.run,
"auditkit_summary": result.summary(),
"verdict_counts": dict(Counter(verdicts.values())),
"n": len(verdicts),
"success_rate": round(sum(v == "success" for v in verdicts.values()) / len(verdicts), 4),
"crosscheck_agree_with_harness": f"{agree}/{len(verdicts)}",
"oracle": "CustomPredicate(spider_execution_accuracy, version=execcheck-v1) - fresh re-execution",
}
out_dir = args.out or os.path.join(HERE, "..", "..", "data", "audit", args.run)
os.makedirs(out_dir, exist_ok=True)
with open(os.path.join(out_dir, "auditkit_report.json"), "w", encoding="utf-8") as f:
json.dump(out, f, indent=2, default=str)
print(json.dumps(out, indent=2, default=str))
if not args.no_push:
HfApi().upload_folder(repo_id=RUNS_REPO, repo_type="dataset", folder_path=out_dir,
path_in_repo=f"audit/{args.run}",
commit_message=f"auditkit report {args.run}")
if __name__ == "__main__":
main()
|