neojev-mlx / jevbench_via_server.py
dowands's picture
Add files using upload-large-folder tool
74630d5 verified
Raw History Blame Contribute Delete
1.59 kB
"""Run JevBench's own TypeSafe adapter (unmodified) against a NeoJev server on the 231 public items.
usage: jevbench_via_server.py <endpoint> <out.jsonl>"""
import json
import os
import sys
ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
# a clone of https://github.com/fstandhartinger/jevbench: $JB_DIR, else third_party/jevbench in this project, else ./jevbench
JB = os.environ.get("JB_DIR") or next((p for p in [os.path.join(ROOT, "third_party", "jevbench"), "jevbench"]
if os.path.isdir(p)), "jevbench")
sys.path.insert(0, JB)
from jevbench.adapters.typesafe import TypeSafeAdapter # noqa: E402
from jevbench.scoring import score_task # noqa: E402
class T:
def __init__(self, r):
self.state, self.question, self.labels, self.expected = r["state"], r["question"], r["labels"], r["expected"]
ep, out = sys.argv[1], sys.argv[2]
ad = TypeSafeAdapter(endpoint=ep, model="neojev", key_env=None)
rows = []
for tier in ["easy", "original", "hard"]:
rows += [json.loads(l) for l in open(os.path.join(JB, "datasets", "public", f"{tier}.jsonl"))]
ok = 0
with open(out, "w") as f:
for r in rows:
res = ad.run(T(r))
s = score_task(res.probs, T(r)) if res.ok else {"correct": False, "valid": False}
ok += res.ok
f.write(json.dumps({"id": r["id"], "tier": r["id"].split("-")[0], "ok": res.ok, "error": res.error,
"latency_s": round(res.latency_s, 4), "correct": s["correct"], "probs": res.probs}) + "\n")
print("requests ok", ok, "/", len(rows))