| """Teacher-label report: accuracy / NLL / ECE of teacher distributions in labeled JSONL(s) vs gold. | |
| Usage: python scripts/teacher_report.py data/synthetic/train.clef.jsonl data/synthetic/eval.clef.jsonl [--examples 3]""" | |
| import argparse, json, math, sys | |
| from collections import defaultdict | |
| from pathlib import Path | |
| sys.path.insert(0, str(Path(__file__).resolve().parents[1])) | |
| from glint.data import load_jsonl, normalize_teacher | |
| def ece(confs, hits, bins=15): | |
| n, tot = len(confs), 0.0 | |
| for b in range(bins): | |
| lo, hi = b / bins, (b + 1) / bins | |
| idx = [i for i, c in enumerate(confs) if lo < c <= hi or (b == 0 and c == 0)] | |
| if idx: | |
| tot += len(idx) / n * abs(sum(hits[i] for i in idx) / len(idx) - sum(confs[i] for i in idx) / len(idx)) | |
| return tot | |
| def main(): | |
| ap = argparse.ArgumentParser(); ap.add_argument("files", nargs="+"); ap.add_argument("--examples", type=int, default=3) | |
| ap.add_argument("--json", default=None); args = ap.parse_args() | |
| confs, hits, nll, per_q, wrong, rec_ok, nrec, ex_out = [], [], 0.0, defaultdict(lambda: [0, 0]), [], 0, 0, [] | |
| for fn in args.files: | |
| for ex in load_jsonl(fn): | |
| t, g = normalize_teacher(ex), ex.get("gold") or {}; ok_all = True; nrec += 1 | |
| for qid, gv in g.items(): | |
| d = t.get(qid) | |
| if d is None: | |
| continue | |
| pred = max(d, key=d.get); h = int(pred == str(gv)); ok_all &= bool(h) | |
| confs.append(d[pred]); hits.append(h); nll -= math.log(max(d.get(str(gv), 0.0), 1e-9)) | |
| per_q[qid][0] += h; per_q[qid][1] += 1 | |
| if not h: | |
| wrong.append({"id": ex["id"], "q": qid, "gold": gv, "pred": pred, "p": {k: round(v, 4) for k, v in d.items()}}) | |
| rec_ok += int(ok_all) | |
| if len(ex_out) < args.examples: | |
| ex_out.append({"id": ex["id"], "gold": g, "teacher": {q: {k: round(v, 4) for k, v in d.items()} for q, d in t.items()}}) | |
| n = max(len(hits), 1) | |
| rep = {"n_records": nrec, "n_questions": len(hits), "accuracy": sum(hits) / n, "record_exact": rec_ok / max(nrec, 1), | |
| "nll": nll / n, "mean_conf": sum(confs) / n, "ece_15bin": ece(confs, hits), | |
| "per_question": {q: v[0] / v[1] for q, v in per_q.items()}, "errors": wrong, "examples": ex_out} | |
| print(json.dumps(rep, indent=1, ensure_ascii=False)) | |
| if args.json: | |
| Path(args.json).write_text(json.dumps(rep, indent=1, ensure_ascii=False), encoding="utf-8") | |
| if __name__ == "__main__": | |
| main() |