File size: 1,861 Bytes
043267b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
"""Quick eval of a backend on a labelled JSONL (text, label) with the example labelset.

    uv run python scripts/eval.py --backend rules
    uv run python scripts/eval.py --backend basal --basal-url http://127.0.0.1:8000
"""
import argparse, asyncio, json, time
from pathlib import Path

from xaddons_classifier.backends import get_backend
from xaddons_classifier.spec import LabelSet, Post

ROOT = Path(__file__).resolve().parents[1]


async def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--backend", default="rules")
    ap.add_argument("--basal-url", default="http://127.0.0.1:8000")
    ap.add_argument("--labelset", default=str(ROOT.parent / "shared" / "labelset.example.json"))
    ap.add_argument("--data", default=str(ROOT / "data" / "eval.sample.jsonl"))
    ap.add_argument("--no-examples", action="store_true", help="zero-shot: drop examples from the labelset")
    a = ap.parse_args()
    ls = LabelSet.model_validate_json(Path(a.labelset).read_text())
    if a.no_examples:
        ls.examples = []
    rows = [json.loads(l) for l in Path(a.data).read_text().splitlines() if l.strip()]
    kw = {"url": a.basal_url} if a.backend == "basal" else {}
    be = get_backend(a.backend, **kw)
    posts = [Post(id=str(i), text=r["text"]) for i, r in enumerate(rows)]
    t0 = time.time()
    res = await be.classify(ls, posts)
    dt = time.time() - t0
    ok = 0
    for i, r in enumerate(rows):
        p = res[str(i)]
        ok += p.label == r["label"]
        mark = "✓" if p.label == r["label"] else "✗"
        print(f"{mark} gold={r['label']:<13} pred={p.label:<13} conf={p.confidence:.2f}  {r['text'][:60]}")
    print(f"\nbackend={a.backend} examples={'no' if a.no_examples else len(ls.examples)} accuracy={ok}/{len(rows)} = {ok/len(rows):.2f}  time={dt:.1f}s ({dt/len(rows)*1000:.0f} ms/post)")


asyncio.run(main())