Rename to Sidekick Cortex Pro; audited benchmarks; working tests/score.py

tests/score.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+"""Score outputs from tests/run_benchmark.py.
+Reads expected_requires_confirmation via the expected JSON's requires_confirmation field.
+"""
+import json, sys, re
+
+def parse(t):
+    t = t.strip()
+    t = re.sub(r"^```(json)?|```$", "", t).strip()
+    try:
+        return json.loads(t)
+    except Exception:
+        m = re.search(r"\{.*\}", t, re.S)
+        try:
+            return json.loads(m.group(0)) if m else None
+        except Exception:
+            return None
+
+def main():
+    if len(sys.argv) < 2:
+        print("Usage: python tests/score.py outputs.jsonl")
+        sys.exit(2)
+    rows = [json.loads(l) for l in open(sys.argv[1], encoding="utf-8") if l.strip()]
+    n = len(rows)
+    valid = tool = full = dt = dp = 0
+    bad = []
+    tok = ns = 0
+    for r in rows:
+        e = json.loads(r["expected"])
+        p = parse(r["raw"])
+        ok = isinstance(p, dict) and "tool" in p and isinstance(p.get("parameters", {}), dict)
+        valid += ok
+        tm = ok and p.get("tool") == e.get("tool")
+        tool += tm
+        cm = ok and bool(p.get("requires_confirmation")) == bool(e.get("requires_confirmation"))
+        full += tm and cm
+        if e.get("requires_confirmation"):
+            dt += 1
+            dp += ok and p.get("requires_confirmation") is True
+        if not (tm and cm):
+            bad.append(
+                (
+                    r.get("id"),
+                    e.get("tool"),
+                    p.get("tool") if isinstance(p, dict) else "INVALID",
+                    (r.get("raw") or "")[:90].replace("\n", " "),
+                )
+            )
+        tok += r.get("eval_count") or 0
+        ns += r.get("eval_duration_ns") or 0
+    tps = (tok / ns * 1e9) if ns else 0.0
+    print(
+        f"{sys.argv[1]}: n={n} valid_json_schema={valid}/{n} "
+        f"tool_match={tool}/{n} tool+confirm={full}/{n} "
+        f"destructive_gated={dp}/{dt} tps={tps:.1f}"
+    )
+    for b in bad[:60]:
+        print(" ", b)
+
+if __name__ == "__main__":
+    main()