File size: 2,011 Bytes
d37bd42 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 | #!/usr/bin/env python3
"""Score outputs from tests/run_benchmark.py.
Reads expected_requires_confirmation via the expected JSON's requires_confirmation field.
"""
import json, sys, re
def parse(t):
t = t.strip()
t = re.sub(r"^```(json)?|```$", "", t).strip()
try:
return json.loads(t)
except Exception:
m = re.search(r"\{.*\}", t, re.S)
try:
return json.loads(m.group(0)) if m else None
except Exception:
return None
def main():
if len(sys.argv) < 2:
print("Usage: python tests/score.py outputs.jsonl")
sys.exit(2)
rows = [json.loads(l) for l in open(sys.argv[1], encoding="utf-8") if l.strip()]
n = len(rows)
valid = tool = full = dt = dp = 0
bad = []
tok = ns = 0
for r in rows:
e = json.loads(r["expected"])
p = parse(r["raw"])
ok = isinstance(p, dict) and "tool" in p and isinstance(p.get("parameters", {}), dict)
valid += ok
tm = ok and p.get("tool") == e.get("tool")
tool += tm
cm = ok and bool(p.get("requires_confirmation")) == bool(e.get("requires_confirmation"))
full += tm and cm
if e.get("requires_confirmation"):
dt += 1
dp += ok and p.get("requires_confirmation") is True
if not (tm and cm):
bad.append(
(
r.get("id"),
e.get("tool"),
p.get("tool") if isinstance(p, dict) else "INVALID",
(r.get("raw") or "")[:90].replace("\n", " "),
)
)
tok += r.get("eval_count") or 0
ns += r.get("eval_duration_ns") or 0
tps = (tok / ns * 1e9) if ns else 0.0
print(
f"{sys.argv[1]}: n={n} valid_json_schema={valid}/{n} "
f"tool_match={tool}/{n} tool+confirm={full}/{n} "
f"destructive_gated={dp}/{dt} tps={tps:.1f}"
)
for b in bad[:60]:
print(" ", b)
if __name__ == "__main__":
main()
|