| |
| """BritEval: ARC, PIQA, TruthfulQA and XNLI translated into Welsh, Irish, Scots and Scottish Gaelic (britllm/* on Hugging Face). |
| Zero-shot log-likelihood scoring, following lm-evaluation-harness's prompt formats: |
| ARC, PIQA: "Question: {q}\\nAnswer:" + " {choice}" acc and acc_norm (log-likelihood per character) |
| TruthfulQA: "Q: {q}\\nA:" + " {choice}" mc1 (best answer correct) and mc2 (probability on true answers) |
| XNLI: "{premise}, right? {Yes|Also|No}, {hypothesis}" acc, whole-sentence log-likelihood |
| COPA-cy: "{premise} oherwydd|felly" + " {choice}" acc (Welsh only; "oherwydd" = because, "felly" = so). Not in the mean |
| Differences from the harness: TruthfulQA here is zero-shot (the harness uses a 6-shot primer), and XNLI uses English |
| connector words because the harness has none for these languages. So scores are comparable between models run with |
| this script, not directly with other published numbers. |
| python run.py NAME MODEL_PATH [--limit N]""" |
| import argparse, glob, io, json, os, sys, time, zipfile |
| import pyarrow.parquet as pq, torch |
| from transformers import AutoModelForCausalLM, AutoTokenizer |
| D = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data") |
| LANGS = {"welsh": "cy", "irish": "ga", "scots": "sco", "scottish_gaelic": "gd"} |
|
|
| def tasks(lang, limit): |
| """Yield (task, item_id, [(context, continuation)], gold) per question.""" |
| for split in ("ARC-Easy", "ARC-Challenge"): |
| rows = pq.read_table(f"{D}/arc_{lang}/{split}/test-00000-of-00001.parquet").to_pylist()[:limit] |
| for i, r in enumerate(rows): |
| labels = list(r["choices"]["label"]) |
| if r["answerKey"] not in labels: continue |
| yield ("arc_easy" if split == "ARC-Easy" else "arc_challenge"), i, [(f"Question: {r['question']}\nAnswer:", f" {c}") for c in r["choices"]["text"]], labels.index(r["answerKey"]) |
| z = zipfile.ZipFile(glob.glob(f"{D}/piqa_{lang}/*.zip")[0]); name = [n for n in z.namelist() if n.endswith("dev.jsonl")][0] |
| rows = [json.loads(l) for l in z.read(name).decode().splitlines() if l.strip()]; gold = [int(l) for l in z.read(name.replace("dev.jsonl", "dev-labels.lst")).decode().split()] |
| for i, (r, g) in enumerate(list(zip(rows, gold))[:limit]): |
| yield "piqa", i, [(f"Question: {r['goal']}\nAnswer:", f" {r['sol1']}"), (f"Question: {r['goal']}\nAnswer:", f" {r['sol2']}")], g |
| rows = pq.read_table(f"{D}/truthfulqa_{lang}/multiple_choice/validation-00000-of-00001.parquet").to_pylist()[:limit] |
| for i, r in enumerate(rows): |
| yield "truthfulqa_mc1", i, [(f"Q: {r['question']}\nA:", f" {c}") for c in r["mc1_targets"]["choices"]], list(r["mc1_targets"]["labels"]).index(1) |
| yield "truthfulqa_mc2", i, [(f"Q: {r['question']}\nA:", f" {c}") for c in r["mc2_targets"]["choices"]], list(r["mc2_targets"]["labels"]) |
| if lang == "welsh": |
| rows = pq.read_table(f"{D}/copa_cy/data/test.parquet").to_pylist()[:limit] |
| for i, r in enumerate(rows): |
| ctx = r["premise"].strip().rstrip(".") + (" oherwydd" if r["question"] == "cause" else " felly") |
| ch = [" " + c[0].lower() + c[1:] for c in (r["choice1"], r["choice2"])] |
| yield "copa", i, [(ctx, c) for c in ch], r["label"] |
| rows = pq.read_table(f"{D}/xnli_brit/{LANGS[lang]}/validation-00000-of-00001.parquet").to_pylist()[:limit] |
| for i, r in enumerate(rows): |
| p = r["premise"].strip().rstrip(".") |
| yield "xnli", i, [("", f"{p}, right? {w}, {r['hypothesis']}") for w in ("Yes", "Also", "No")], r["label"] |
|
|
| class Scorer: |
| def __init__(s, path): |
| s.tok = AutoTokenizer.from_pretrained(path); s.m = AutoModelForCausalLM.from_pretrained(path, dtype=torch.bfloat16).cuda().eval() |
| s.bos = [s.tok.bos_token_id] if s.tok.bos_token_id is not None else []; s.max = 2048 |
| def encode(s, ctx, cont): |
| |
| whole = s.tok(ctx + cont, add_special_tokens=False).input_ids; c = s.tok(ctx, add_special_tokens=False).input_ids if ctx else [] |
| if whole[:len(c)] != c: c = c[:-1] |
| return s.bos + c, whole[len(c):] |
| @torch.no_grad() |
| def loglik(s, pairs, bs=8): |
| enc = [s.encode(a, b) for a, b in pairs]; out = [None] * len(enc); order = sorted(range(len(enc)), key=lambda i: -len(enc[i][0]) - len(enc[i][1])) |
| for b in range(0, len(order), bs): |
| idx = order[b:b + bs]; seqs = [(enc[i][0] + enc[i][1])[-s.max:] for i in idx]; L = max(map(len, seqs)) |
| x = torch.full((len(seqs), L), s.tok.pad_token_id or 0, dtype=torch.long); att = torch.zeros_like(x) |
| for j, q in enumerate(seqs): x[j, :len(q)] = torch.tensor(q); att[j, :len(q)] = 1 |
| logits = s.m(input_ids=x.cuda(), attention_mask=att.cuda()).logits |
| for j, i in enumerate(idx): |
| n = len(enc[i][1]); q = seqs[j]; t = torch.tensor(q[-n:], device=logits.device) |
| lp = logits[j, len(q) - n - 1:len(q) - 1].float().log_softmax(-1) |
| out[i] = lp.gather(1, t[:, None]).sum().item() |
| del logits |
| return out |
|
|
| def evaluate(sc, lang, limit): |
| items = list(tasks(lang, limit)); flat = [p for _, _, ps, _ in items for p in ps]; ll = sc.loglik(flat); k = 0; res = {} |
| for task, _, ps, gold in items: |
| v = ll[k:k + len(ps)]; k += len(ps); r = res.setdefault(task, {"n": 0, "acc": 0.0, "acc_norm": 0.0}) |
| if task == "truthfulqa_mc2": |
| p = torch.tensor(v).exp(); r["acc"] += (p[torch.tensor(gold) == 1].sum() / p.sum()).item() |
| else: |
| norm = [x / max(1, len(c)) for x, (_, c) in zip(v, ps)] |
| r["acc"] += float(max(range(len(v)), key=v.__getitem__) == gold); r["acc_norm"] += float(max(range(len(v)), key=norm.__getitem__) == gold) |
| r["n"] += 1 |
| for r in res.values(): r["acc"] /= r["n"]; r["acc_norm"] /= r["n"] |
| arc = {"n": res["arc_easy"]["n"] + res["arc_challenge"]["n"]} |
| arc["acc_norm"] = (res["arc_easy"]["acc_norm"] * res["arc_easy"]["n"] + res["arc_challenge"]["acc_norm"] * res["arc_challenge"]["n"]) / arc["n"] |
| res["mean"] = {"value": (arc["acc_norm"] + res["piqa"]["acc_norm"] + res["truthfulqa_mc2"]["acc"] + res["xnli"]["acc"]) / 4, |
| "of": "ARC acc_norm (Easy+Challenge), PIQA acc_norm, TruthfulQA mc2, XNLI acc"} |
| return res |
|
|
| if __name__ == "__main__": |
| a = argparse.ArgumentParser(); a.add_argument("name"); a.add_argument("path"); a.add_argument("--limit", type=int, default=None); a.add_argument("--langs", default=",".join(LANGS)); x = a.parse_args() |
| sc = Scorer(x.path); out = {"model": x.name, "path": x.path, "revisions": json.load(open(f"{D}/revisions.json")), "limit": x.limit, "results": {}}; t = time.time() |
| for lang in x.langs.split(","): |
| r = evaluate(sc, lang, x.limit); out["results"][lang] = r |
| print(f"{x.name} {lang}: mean {100 * r['mean']['value']:.1f} | " + " ".join(f"{k} {100 * v['acc_norm' if k in ('arc_easy', 'arc_challenge', 'piqa') else 'acc']:.1f}" for k, v in r.items() if k != "mean") + f" | {time.time() - t:.0f}s", flush=True) |
| os.makedirs("results", exist_ok=True); json.dump(out, open(f"results/{x.name}.json", "w"), indent=1) |
|
|
