File size: 2,063 Bytes
07ef007 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 | #!/usr/bin/env python3
"""Independent Sidekick Cortex Pro (formerly Axon) benchmark runner.
Uses Ollama /api/chat with the dataset's persona system message (temp 0).
Usage:
python tests/run_benchmark.py lastbrowser/sidekick-cortex-pro:latest \\
data/axon_handwritten_testset_50.jsonl outputs_hw50.jsonl
python tests/score.py outputs_hw50.jsonl
"""
import json, time, sys, requests
def main():
if len(sys.argv) < 4:
print(__doc__)
sys.exit(2)
model, src, out = sys.argv[1], sys.argv[2], sys.argv[3]
base = sys.argv[4] if len(sys.argv) > 4 else "http://localhost:11434"
samples = [json.loads(l) for l in open(src, encoding="utf-8") if l.strip()]
with open(out, "w", encoding="utf-8") as fo:
for i, s in enumerate(samples, 1):
msgs = [m for m in s["messages"] if m["role"] != "assistant"]
t0 = time.time()
r = requests.post(
f"{base}/api/chat",
json={
"model": model,
"messages": msgs,
"stream": False,
"options": {"temperature": 0, "num_ctx": 4096, "num_predict": 256},
},
timeout=300,
).json()
exp = next(m["content"] for m in s["messages"] if m["role"] == "assistant")
fo.write(
json.dumps(
{
"id": s.get("id", f"sample_{i}"),
"prompt": msgs[-1]["content"],
"expected": exp,
"raw": r.get("message", {}).get("content", ""),
"eval_count": r.get("eval_count"),
"eval_duration_ns": r.get("eval_duration"),
"wall_s": time.time() - t0,
},
ensure_ascii=False,
)
+ "\n"
)
fo.flush()
print(f"[{i}/{len(samples)}] {s.get('id', i)} done", flush=True)
if __name__ == "__main__":
main()
|