File size: 2,579 Bytes
b38d524 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 | """Evaluate transformers sentiment classifiers on sentiment/evals. python sentiment/eval_sent.py --models A,B --evals DIR --out R.json
Each model's labels are mapped to negative/neutral/positive by name (very negative -> negative; 1-2 stars -> negative,
3 stars -> neutral, 4-5 stars -> positive; LABEL_0/1/2 -> negative/neutral/positive) and probabilities summed per class.
"""
import argparse, json, os, re, sys
import numpy as np, torch
from transformers import AutoModelForSequenceClassification, AutoTokenizer
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from common import load_evals, score, summarize
ap = argparse.ArgumentParser()
ap.add_argument("--models", required=True); ap.add_argument("--evals", required=True); ap.add_argument("--out", required=True)
ap.add_argument("--bs", type=int, default=64); ap.add_argument("--max_len", type=int, default=512)
args = ap.parse_args()
def cls_of(name, n):
s = name.lower()
if "neg" in s: return 0
if "neu" in s: return 1
if "pos" in s: return 2
m = re.match(r"(\d) star", s)
if m: return 0 if int(m.group(1)) <= 2 else 1 if int(m.group(1)) == 3 else 2
m = re.match(r"label_(\d)", s)
if m and n == 3: return int(m.group(1))
raise ValueError(f"unmapped label {name}")
EV = load_evals(args.evals)
res = json.load(open(args.out)) if os.path.exists(args.out) else {}
for spec in args.models.split(","):
tok = AutoTokenizer.from_pretrained(spec)
m = AutoModelForSequenceClassification.from_pretrained(spec, torch_dtype=torch.bfloat16).cuda().eval()
id2 = m.config.id2label; M = torch.zeros(len(id2), 3)
for i, l in id2.items():
M[int(i), cls_of(l, len(id2))] = 1
print(spec, {l: cls_of(l, len(id2)) for l in id2.values()}, flush=True)
M = M.cuda(); r = {}
for name, df in EV.items():
texts = df.text.tolist(); order = np.argsort([len(t) for t in texts]); P = np.zeros((len(texts), 3))
with torch.no_grad():
for s in range(0, len(texts), args.bs):
b = order[s:s + args.bs]
enc = tok([texts[i] for i in b], truncation=True, max_length=args.max_len, padding=True, return_tensors="pt")
lo = m(**{k: v.cuda() for k, v in enc.items() if k in ("input_ids", "attention_mask")}).logits.float()
P[b] = (torch.softmax(lo, -1) @ M).cpu().numpy()
r[name] = score(df, P)
r.update(summarize(r)); print(spec, {k: round(v, 4) for k, v in r.items() if k.startswith("_")}, flush=True)
res[spec] = r; json.dump(res, open(args.out, "w"), indent=1)
|