"""Run JevBench's own TypeSafe adapter (unmodified) against a NeoJev server on the 231 public items. usage: jevbench_via_server.py """ import json import os import sys ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..") # a clone of https://github.com/fstandhartinger/jevbench: $JB_DIR, else third_party/jevbench in this project, else ./jevbench JB = os.environ.get("JB_DIR") or next((p for p in [os.path.join(ROOT, "third_party", "jevbench"), "jevbench"] if os.path.isdir(p)), "jevbench") sys.path.insert(0, JB) from jevbench.adapters.typesafe import TypeSafeAdapter # noqa: E402 from jevbench.scoring import score_task # noqa: E402 class T: def __init__(self, r): self.state, self.question, self.labels, self.expected = r["state"], r["question"], r["labels"], r["expected"] ep, out = sys.argv[1], sys.argv[2] ad = TypeSafeAdapter(endpoint=ep, model="neojev", key_env=None) rows = [] for tier in ["easy", "original", "hard"]: rows += [json.loads(l) for l in open(os.path.join(JB, "datasets", "public", f"{tier}.jsonl"))] ok = 0 with open(out, "w") as f: for r in rows: res = ad.run(T(r)) s = score_task(res.probs, T(r)) if res.ok else {"correct": False, "valid": False} ok += res.ok f.write(json.dumps({"id": r["id"], "tier": r["id"].split("-")[0], "ok": res.ok, "error": res.error, "latency_s": round(res.latency_s, 4), "correct": s["correct"], "probs": res.probs}) + "\n") print("requests ok", ok, "/", len(rows))