neojev / jevbench_via_server.py
dowands's picture
Add TypeSafe-compatible server.py, LRU schema cache; JevBench numbers via JevBench's own adapter
d3771d5 verified
Raw History Blame Contribute Delete
1.33 kB
"""Run JevBench's own TypeSafe adapter (unmodified) against a NeoJev server on the 231 public items.
usage: jevbench_via_server.py <endpoint> <out.jsonl>"""
import json
import os
import sys
ROOT = os.path.join(os.path.dirname(__file__), "..")
JB = os.environ.get("JB_DIR", os.path.join(ROOT, "reviews", "jevbench"))
sys.path.insert(0, JB)
from jevbench.adapters.typesafe import TypeSafeAdapter # noqa: E402
from jevbench.scoring import score_task # noqa: E402
class T:
def __init__(self, r):
self.state, self.question, self.labels, self.expected = r["state"], r["question"], r["labels"], r["expected"]
ep, out = sys.argv[1], sys.argv[2]
ad = TypeSafeAdapter(endpoint=ep, model="neojev", key_env=None)
rows = []
for tier in ["easy", "original", "hard"]:
rows += [json.loads(l) for l in open(os.path.join(JB, "datasets", "public", f"{tier}.jsonl"))]
ok = 0
with open(out, "w") as f:
for r in rows:
res = ad.run(T(r))
s = score_task(res.probs, T(r)) if res.ok else {"correct": False, "valid": False}
ok += res.ok
f.write(json.dumps({"id": r["id"], "tier": r["id"].split("-")[0], "ok": res.ok, "error": res.error,
"latency_s": round(res.latency_s, 4), "correct": s["correct"], "probs": res.probs}) + "\n")
print("requests ok", ok, "/", len(rows))