File size: 2,605 Bytes
e379257 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 | """Teacher-label report: accuracy / NLL / ECE of teacher distributions in labeled JSONL(s) vs gold.
Usage: python scripts/teacher_report.py data/synthetic/train.clef.jsonl data/synthetic/eval.clef.jsonl [--examples 3]"""
import argparse, json, math, sys
from collections import defaultdict
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from glint.data import load_jsonl, normalize_teacher
def ece(confs, hits, bins=15):
n, tot = len(confs), 0.0
for b in range(bins):
lo, hi = b / bins, (b + 1) / bins
idx = [i for i, c in enumerate(confs) if lo < c <= hi or (b == 0 and c == 0)]
if idx:
tot += len(idx) / n * abs(sum(hits[i] for i in idx) / len(idx) - sum(confs[i] for i in idx) / len(idx))
return tot
def main():
ap = argparse.ArgumentParser(); ap.add_argument("files", nargs="+"); ap.add_argument("--examples", type=int, default=3)
ap.add_argument("--json", default=None); args = ap.parse_args()
confs, hits, nll, per_q, wrong, rec_ok, nrec, ex_out = [], [], 0.0, defaultdict(lambda: [0, 0]), [], 0, 0, []
for fn in args.files:
for ex in load_jsonl(fn):
t, g = normalize_teacher(ex), ex.get("gold") or {}; ok_all = True; nrec += 1
for qid, gv in g.items():
d = t.get(qid)
if d is None:
continue
pred = max(d, key=d.get); h = int(pred == str(gv)); ok_all &= bool(h)
confs.append(d[pred]); hits.append(h); nll -= math.log(max(d.get(str(gv), 0.0), 1e-9))
per_q[qid][0] += h; per_q[qid][1] += 1
if not h:
wrong.append({"id": ex["id"], "q": qid, "gold": gv, "pred": pred, "p": {k: round(v, 4) for k, v in d.items()}})
rec_ok += int(ok_all)
if len(ex_out) < args.examples:
ex_out.append({"id": ex["id"], "gold": g, "teacher": {q: {k: round(v, 4) for k, v in d.items()} for q, d in t.items()}})
n = max(len(hits), 1)
rep = {"n_records": nrec, "n_questions": len(hits), "accuracy": sum(hits) / n, "record_exact": rec_ok / max(nrec, 1),
"nll": nll / n, "mean_conf": sum(confs) / n, "ece_15bin": ece(confs, hits),
"per_question": {q: v[0] / v[1] for q, v in per_q.items()}, "errors": wrong, "examples": ex_out}
print(json.dumps(rep, indent=1, ensure_ascii=False))
if args.json:
Path(args.json).write_text(json.dumps(rep, indent=1, ensure_ascii=False), encoding="utf-8")
if __name__ == "__main__":
main() |