"""Quick eval of a backend on a labelled JSONL (text, label) with the example labelset. uv run python scripts/eval.py --backend rules uv run python scripts/eval.py --backend basal --basal-url http://127.0.0.1:8000 """ import argparse, asyncio, json, time from pathlib import Path from xaddons_classifier.backends import get_backend from xaddons_classifier.spec import LabelSet, Post ROOT = Path(__file__).resolve().parents[1] async def main(): ap = argparse.ArgumentParser() ap.add_argument("--backend", default="rules") ap.add_argument("--basal-url", default="http://127.0.0.1:8000") ap.add_argument("--labelset", default=str(ROOT.parent / "shared" / "labelset.example.json")) ap.add_argument("--data", default=str(ROOT / "data" / "eval.sample.jsonl")) ap.add_argument("--no-examples", action="store_true", help="zero-shot: drop examples from the labelset") a = ap.parse_args() ls = LabelSet.model_validate_json(Path(a.labelset).read_text()) if a.no_examples: ls.examples = [] rows = [json.loads(l) for l in Path(a.data).read_text().splitlines() if l.strip()] kw = {"url": a.basal_url} if a.backend == "basal" else {} be = get_backend(a.backend, **kw) posts = [Post(id=str(i), text=r["text"]) for i, r in enumerate(rows)] t0 = time.time() res = await be.classify(ls, posts) dt = time.time() - t0 ok = 0 for i, r in enumerate(rows): p = res[str(i)] ok += p.label == r["label"] mark = "✓" if p.label == r["label"] else "✗" print(f"{mark} gold={r['label']:<13} pred={p.label:<13} conf={p.confidence:.2f} {r['text'][:60]}") print(f"\nbackend={a.backend} examples={'no' if a.no_examples else len(ls.examples)} accuracy={ok}/{len(rows)} = {ok/len(rows):.2f} time={dt:.1f}s ({dt/len(rows)*1000:.0f} ms/post)") asyncio.run(main())