botezx's picture
Grok Bot
FlagTLDR 0.5.0 (snapshot of local f98bb37)
043267b
Raw History Blame Contribute Delete
1.86 kB
"""Quick eval of a backend on a labelled JSONL (text, label) with the example labelset.
uv run python scripts/eval.py --backend rules
uv run python scripts/eval.py --backend basal --basal-url http://127.0.0.1:8000
"""
import argparse, asyncio, json, time
from pathlib import Path
from xaddons_classifier.backends import get_backend
from xaddons_classifier.spec import LabelSet, Post
ROOT = Path(__file__).resolve().parents[1]
async def main():
ap = argparse.ArgumentParser()
ap.add_argument("--backend", default="rules")
ap.add_argument("--basal-url", default="http://127.0.0.1:8000")
ap.add_argument("--labelset", default=str(ROOT.parent / "shared" / "labelset.example.json"))
ap.add_argument("--data", default=str(ROOT / "data" / "eval.sample.jsonl"))
ap.add_argument("--no-examples", action="store_true", help="zero-shot: drop examples from the labelset")
a = ap.parse_args()
ls = LabelSet.model_validate_json(Path(a.labelset).read_text())
if a.no_examples:
ls.examples = []
rows = [json.loads(l) for l in Path(a.data).read_text().splitlines() if l.strip()]
kw = {"url": a.basal_url} if a.backend == "basal" else {}
be = get_backend(a.backend, **kw)
posts = [Post(id=str(i), text=r["text"]) for i, r in enumerate(rows)]
t0 = time.time()
res = await be.classify(ls, posts)
dt = time.time() - t0
ok = 0
for i, r in enumerate(rows):
p = res[str(i)]
ok += p.label == r["label"]
mark = "✓" if p.label == r["label"] else "✗"
print(f"{mark} gold={r['label']:<13} pred={p.label:<13} conf={p.confidence:.2f} {r['text'][:60]}")
print(f"\nbackend={a.backend} examples={'no' if a.no_examples else len(ls.examples)} accuracy={ok}/{len(rows)} = {ok/len(rows):.2f} time={dt:.1f}s ({dt/len(rows)*1000:.0f} ms/post)")
asyncio.run(main())