File size: 3,073 Bytes
e379257 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 | """Side-by-side comparison of student_report.py JSONs (e.g. q3 long prompt vs q3s short prompt).
usage: python scripts/compare_reports.py --report long=runs/q3-box-report.json --report short=runs/q3s-box-report.json
[--extra "short@long_prompt:style_eval=runs/q3s-box-on-long-report.json:style_eval"] --json out.json --md out.md
Rows: every set present in the first report; one column group per report (missing -> "-").
--extra adds a row LABEL taken from SET inside PATH (e.g. the short checkpoint evaluated on long prompts).
"""
import argparse, json
from pathlib import Path
METRICS = [("accuracy", ("accuracy",)), ("record_exact", ("record_exact",)), ("teacher_agreement", ("teacher_agreement",)),
("nll", ("nll",)), ("ece", ("ece_15bin",)), ("ece_ts", ("ece_ts_xfit",)),
("box_mean_iou", ("box", "overall", "mean_iou")), ("box_iou50", ("box", "overall", "iou50")),
("box_iou75", ("box", "overall", "iou75")), ("box_presence", ("box", "overall", "presence_acc")),
("box_conf_ece", ("box", "overall", "conf_ece_iou50")), ("sec_per_record", ("mean_sec",))]
def get(d, path):
for k in path:
if not isinstance(d, dict) or k not in d:
return None
d = d[k]
return d
def row(report_set):
a = (report_set or {}).get("all") or {}
return {m: get(a, p) for m, p in METRICS}
def fmt(v, m):
if v is None:
return "-"
return f"{v:.2f}" if m == "sec_per_record" else f"{v:.4f}"
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--report", action="append", required=True, help="name=path")
ap.add_argument("--extra", action="append", default=[], help="label=path:set")
ap.add_argument("--json"); ap.add_argument("--md")
a = ap.parse_args()
reps = [(n, json.loads(Path(p).read_text(encoding="utf-8"))) for n, p in (x.split("=", 1) for x in a.report)]
out = {"reports": [n for n, _ in reps], "rows": {}}
for s in reps[0][1]["sets"]:
out["rows"][s] = {n: row(r["sets"].get(s)) for n, r in reps}
for x in a.extra:
label, rest = x.split("=", 1); p, s = rest.rsplit(":", 1)
out["rows"][label] = {"extra": row(json.loads(Path(p).read_text(encoding="utf-8"))["sets"].get(s))}
lines = ["| set | metric | " + " | ".join(n for n, _ in reps) + " | delta |", "|---|---|" + "---|" * (len(reps) + 1)]
for s, cols in out["rows"].items():
for m, _ in METRICS:
vals = [cols.get(n, {}).get(m) for n, _ in reps] if "extra" not in cols else [cols["extra"].get(m)]
d = vals[-1] - vals[0] if len(vals) > 1 and None not in vals else None
lines.append(f"| {s} | {m} | " + " | ".join(fmt(v, m) for v in vals) + f" | {'-' if d is None else f'{d:+.4f}'} |")
md = "\n".join(lines)
print(md)
if a.json:
Path(a.json).write_text(json.dumps(out, indent=1), encoding="utf-8")
if a.md:
Path(a.md).write_text(md + "\n", encoding="utf-8")
if __name__ == "__main__":
main()
|