Buckets:
| """cwm3/eval/calibration_report.py — Brier scores + reliability diagrams complementing the ECE comparison of eval/operating_points.py | |
| (CockpitWM warning read-out vs the openpilot attention chain att@3s); CPU only; read-only on every existing run and table (new files and | |
| appended sections only). | |
| BOTH scores are temperature-calibrated on the clean-val gold rows (warning_gold.temperature_fit: NLL on a log-spaced grid [0.1, 20], 200 | |
| points; T_pinned = 1 marks a fit at a grid edge) and evaluated on the clean-test gold rows — exactly the reliability block of | |
| operating_points.py (lines 246–261). Gold rows, scores and masks are rebuilt by the same calls (LB.episode_labels -> WG.load_pack -> | |
| WG.merge_legacy(seed) -> WG.build_gold(legacy_first); DMS = the `att@3s` gold column; head = the w column of | |
| results/runs/<layout>/pred_layout_{regime}_{split}.npz at (regime, split, row)); operating_points is imported as a module (its main() is never | |
| run) for cboot / reliability_bins. Before anything is written, the recomputed T, ECE raw, ECE calibrated, the calibrated 10-bin curve and the | |
| 1000-resample ECE CI are asserted equal (1e-6) to the published reliability.csv of --ref-dir. | |
| Per score and seed on the clean-test rows: ECE raw / calibrated (warning_gold.ece, 15 equal-width bins, as the published CSV), Brier raw / | |
| calibrated (warning_gold.brier), the Brier of the constant clean-test prevalence as reference, device-cluster bootstrap CIs | |
| (operating_points.cboot, --n-boot resamples, seed 3 = the ECE seed of operating_points.py, so ECE and Brier share the resamples), the | |
| reliability curve (operating_points.reliability_bins, 10 equal-count bins, raw and calibrated), the paired Brier Δ (score − att@3s) on identical | |
| rows (both scores finite; cboot seed 2 = the paired-Δ seed of operating_points.py) and the p05 / p50 / p95 of the raw and calibrated scores. | |
| Secondary (reported separately, never replaces the primary temperature protocol): Platt scaling (slope AND bias on the logit, NLL on | |
| clean-val), because a temperature alone cannot move a score whose raw values sit above 0.5 below 0.5. | |
| Scores: the CockpitWM read-out of the published head-vs-DMS run (--w-runs / --col), the legacy chain att@3s and, when their layouts exist, the | |
| selected read-out of every DMS layer-2 item (results/warning_gold/dms_* / results/operating_points/dms_*). | |
| Outputs: results/operating_points/<item>_<stamp>/{calibration.csv, reliability_curves.csv, paired_brier.csv, ece_repro_check.csv, section.md, .log, | |
| run json}; paper/figs/reliability_warning.{pdf,png,txt} (two panels raw / calibrated, head vs att@3s, marker area = bin size; CockpitWM palette of | |
| BATON__/ICLR/cwm_macros.tex as fills, darker steps of the same hues as lines, fonts embedded); one section appended to | |
| results/tables/operating_points.md. | |
| python eval/calibration_report.py --ref-dir results/operating_points/final_ext_dwfc_20260912T034427Z | |
| """ | |
| from __future__ import annotations | |
| import os | |
| os.environ.setdefault("OMP_NUM_THREADS", "4") | |
| os.environ.setdefault("MKL_NUM_THREADS", "4") | |
| import argparse # noqa: E402 | |
| import hashlib # noqa: E402 | |
| import sys # noqa: E402 | |
| import time # noqa: E402 | |
| from pathlib import Path # noqa: E402 | |
| import numpy as np # noqa: E402 | |
| import pandas as pd # noqa: E402 | |
| from scipy.optimize import minimize # noqa: E402 | |
| CWM3 = Path(__file__).resolve().parents[1] | |
| sys.path.insert(0, str(CWM3 / "eval")) | |
| import operating_points as OP # noqa: E402 module import only (main() never called): cboot, reliability_bins, WG, LB, REGIMES | |
| WG, LB, REGIMES = OP.WG, OP.LB, OP.REGIMES | |
| OUT_ROOT = CWM3 / "results" / "operating_points" | |
| TABLE = CWM3 / "results" / "tables" / "operating_points.md" | |
| FIG_DIR = CWM3 / "paper" / "figs" | |
| RUNS = CWM3 / "results" / "runs" | |
| WGOLD = CWM3 / "results" / "warning_gold" | |
| HEAD_NAME = "CockpitWM read-out" | |
| ATT_NAME = "openpilot attention chain (att@3s)" | |
| SECTION_TITLE = "Calibration report — Brier and reliability (both scores temperature-calibrated on clean-val)" | |
| PUBLISHED = {"ref_dir": OUT_ROOT / "final_ext_dwfc_20260912T034427Z", | |
| "w_runs": [f"W_I1_all_minus_road2drv_s{s}_final_dwfc" for s in (0, 1, 2)], "seeds": [0, 1, 2], "col": "W_learned_curhgb_t15_w10_n979dwfc"} | |
| # DMS layer-2 items (baselines_dms): layouts results/runs/abl_dms_<x>_<kind>_s<seed>, selected read-out column dms_<x>_<kind>_sel_t15_w10_n979 | |
| L2_EXTRACTORS = [("l2cs_gaze", "L2CS-Net (Gaze360)"), ("videomae2", "VideoMAEv2 ViT-S features"), ("ov_gaze", "Intel OMZ head-pose / gaze / eye pipeline"), | |
| ("mp_face", "MediaPipe Face Landmarker"), ("ov_action", "Intel OMZ driver-action logits"), ("openface3", "OpenFace 3.0")] | |
| L2_KINDS = (("learned", "learned read-out"), ("ctx", "+ CockpitWM current-state context")) | |
| # figure palette: fills = cwm_macros.tex tints (cwPeachL = ours, cwGreyL = comparator); lines / marker edges = darker steps of the same two hues, | |
| # validated with the data-viz palette validator (light surface, all pairs: lightness band, chroma floor, CVD ΔE 19.8, normal ΔE 24.7, contrast >= 3:1) | |
| FILL = {"head": "#F5D8BF", "dms": "#BFDCE7"} | |
| LINE = {"head": "#B5601A", "dms": "#2A72A8"} | |
| MARK = {"head": "o", "dms": "s"} | |
| INK, INK2, HAIR = "#000000", "#404040", "#D9D9D9" | |
| EPS = 1e-6 | |
| def rel(p): | |
| p = Path(p) | |
| try: | |
| return str(p.relative_to(CWM3)) | |
| except ValueError: | |
| return str(p) | |
| def layouts_ok(w_run): | |
| return all((RUNS / w_run / f"pred_layout_{r}_{s}.npz").exists() for r in REGIMES for s in ("test", "val")) | |
| def layer2_items(): | |
| """[(item, display name, layouts, seeds, column)] for every DMS layer-2 item with an item dir and complete layouts.""" | |
| items = [] | |
| def has_dir(prefix): | |
| return bool(list(WGOLD.glob(prefix + "*")) or list(OUT_ROOT.glob(prefix + "_*"))) | |
| if has_dir("dms_r0") and layouts_ok("abl_dms_lmface_rule_s0"): | |
| items.append(("dms_r0", "R0 fixed gaze/eye rule (own face landmarks)", ["abl_dms_lmface_rule_s0"], [0], "dms_lmface_rule0")) | |
| for x, disp in L2_EXTRACTORS: | |
| for kind, suffix in L2_KINDS: | |
| item = f"dms_{x}_{kind}" | |
| if not has_dir(item): | |
| continue | |
| seeds = [s for s in (0, 1, 2) if layouts_ok(f"abl_dms_{x}_{kind}_s{s}")] | |
| if seeds: | |
| items.append((item, f"{disp}, {suffix}" if kind == "learned" else f"{disp} {suffix}", [f"abl_dms_{x}_{kind}_s{s}" for s in seeds], seeds, | |
| f"dms_{x}_{kind}_sel_t15_w10_n979")) | |
| return items | |
| def gold_for_seed(lab, packs, seed): | |
| """operating_points.py lines 128–131: legacy_first gold of one seed + the att@3s gold column.""" | |
| legacy = WG.merge_legacy(seed) | |
| assert legacy is not None, "legacy episode CSVs missing" | |
| G = WG.build_gold(lab, packs, legacy, "legacy_first") | |
| G["att"] = legacy.reindex(G.idx)["att@3s"].to_numpy(float) | |
| return G | |
| def layout_scores(G, packs, w_run, col): | |
| """operating_points.py lines 132–141: the layout's w column read at the gold rows (NaN where the split has no row).""" | |
| lay = {} | |
| for r in REGIMES: | |
| for s in ("test", "val"): | |
| z = np.load(RUNS / w_run / f"pred_layout_{r}_{s}.npz", allow_pickle=True) | |
| names = list(map(str, z["w_names"])) | |
| assert col in names, (w_run, col, names) | |
| assert len(z["uid"]) == len(packs[(r, s)]["uid"]) and np.array_equal(z["uid"], packs[(r, s)]["uid"]), f"{w_run} {r}/{s}: layout rows != targets rows" | |
| lay[(r, s)] = z["w"][:, names.index(col)].astype(np.float64) | |
| g = np.full(len(G), np.nan) | |
| for (r, s), grp in G.groupby(["regime", "split"]): | |
| g[grp.index.to_numpy()] = lay[(r, s)][grp.row.to_numpy()] | |
| return g | |
| def logit(p): | |
| q = np.clip(p, EPS, 1 - EPS) | |
| return np.log(q / (1 - q)) | |
| def platt_fit(p_val, y_val): | |
| """secondary: sigmoid(a * logit(p) + b) with (a, b) minimising the NLL on clean-val (L-BFGS-B from (1, 0)).""" | |
| z, y = logit(p_val), y_val.astype(float) | |
| def nll(w): | |
| q = 1 / (1 + np.exp(-(w[0] * z + w[1]))) | |
| return -np.mean(y * np.log(q + EPS) + (1 - y) * np.log(1 - q + EPS)) | |
| r = minimize(nll, np.array([1.0, 0.0]), method="L-BFGS-B") | |
| return float(r.x[0]), float(r.x[1]) | |
| def apply_platt(p, a, b): | |
| return 1 / (1 + np.exp(-(a * logit(p) + b))) | |
| def calibrate(G, gsc, n_boot): | |
| """operating_points.py lines 247–256 + Brier / quantiles / secondary Platt. -> (stats, {raw, cal, platt: bins}, {raw, cal, platt: full-length p}).""" | |
| yg, gt, gv, dong = G.y.to_numpy().astype(int), G.clean_test.to_numpy(), G.clean_val.to_numpy(), G.dongle.to_numpy() | |
| fv_, ft_ = gv & np.isfinite(gsc), gt & np.isfinite(gsc) | |
| T, pinned = WG.temperature_fit(gsc[fv_], yg[fv_].astype(float)) | |
| pa, pb = platt_fit(gsc[fv_], yg[fv_]) | |
| y, dg = yg[ft_], dong[ft_] | |
| P = {"raw": np.clip(gsc[ft_], 0, 1), "cal": WG.apply_temperature(gsc[ft_], T), "platt": apply_platt(gsc[ft_], pa, pb)} | |
| st = {"T": T, "T_pinned": int(pinned), "platt_a": pa, "platt_b": pb, "n_test": int(ft_.sum()), "n_pos": int(y.sum()), "n_dev": int(len(np.unique(dg))), | |
| "n_val": int(fv_.sum()), "n_val_pos": int(yg[fv_].sum()), "prevalence": float(y.mean()), "brier_const_prev": WG.brier(y, np.full(len(y), y.mean()))} | |
| for tag, p in P.items(): | |
| st[f"ece_{tag}"], st[f"brier_{tag}"] = WG.ece(y, p), WG.brier(y, p) | |
| for stat, fn in (("ece", WG.ece), ("brier", WG.brier)): | |
| b = OP.cboot(dg, lambda pk, p=p, fn=fn: fn(y[pk], p[pk]), n_boot, 3) | |
| st[f"{stat}_{tag}_ci_lo"], st[f"{stat}_{tag}_ci_hi"] = WG.pct(b) | |
| if stat == "ece" and tag == "cal": # first 1000 resamples = the operating_points.py CI (n_boot 1000, seed 3); NaN when fewer were drawn | |
| st["ece_cal_ci1000_lo"], st["ece_cal_ci1000_hi"] = WG.pct(b[:1000]) if n_boot >= 1000 else (np.nan, np.nan) | |
| st[f"p05_{tag}"], st[f"p50_{tag}"], st[f"p95_{tag}"] = (float(v) for v in np.percentile(p, [5, 50, 95])) | |
| bins = {tag: OP.reliability_bins(y, p) for tag, p in P.items()} | |
| full = {} | |
| for tag, p in P.items(): | |
| full[tag] = np.full(len(G), np.nan) | |
| full[tag][ft_] = p | |
| return st, bins, full | |
| def paired_delta(G, pa, pb, n_boot): | |
| """paired Brier Δ (a − b) on the identical clean-test rows where both probabilities exist; device-cluster bootstrap (seed 2).""" | |
| yg, gt, dong = G.y.to_numpy().astype(int), G.clean_test.to_numpy(), G.dongle.to_numpy() | |
| m = gt & np.isfinite(pa) & np.isfinite(pb) | |
| y = yg[m] | |
| d = (pa[m] - y) ** 2 - (pb[m] - y) ** 2 | |
| b = OP.cboot(dong[m], lambda pk: float(d[pk].mean()), n_boot, 2) | |
| lo, hi = WG.pct(b) | |
| return {"n": int(m.sum()), "n_pos": int(y.sum()), "n_dev": int(len(np.unique(dong[m]))), "delta": float(d.mean()), "ci_lo": lo, "ci_hi": hi, | |
| "frac_rows_a_better": float((d < 0).mean()), "verdict": "a better" if hi < 0 else ("b better" if lo > 0 else "CI ⊃ 0")} | |
| def sha256(p): | |
| h = hashlib.sha256() | |
| with open(p, "rb") as f: | |
| for ch in iter(lambda: f.read(1 << 20), b""): | |
| h.update(ch) | |
| return h.hexdigest() | |
| def ci_s(lo, hi, nd=3, sign=False): | |
| fmt = f"{{:+.{nd}f}}" if sign else f"{{:.{nd}f}}" | |
| return f"[{fmt.format(lo)}, {fmt.format(hi)}]" | |
| def t_s(r): | |
| """r = Series or itertuples row.""" | |
| T, pinned = (r["T"], r["T_pinned"]) if isinstance(r, pd.Series) else (getattr(r, "T"), getattr(r, "T_pinned")) | |
| return f"{T:.2f}" + (" (pinned)" if int(pinned) else "") | |
| def verdict_s(lo, hi): | |
| return "read-out better (CI excludes 0)" if hi < 0 else ("chain better (CI excludes 0)" if lo > 0 else "EQUAL (CI ⊃ 0)") | |
| # ----------------------------------------------------------------------------- figure | |
| def make_figure(C, B, P, out_pdf, csv_path, stamp): | |
| """two panels (raw | calibrated): 10 equal-count reliability bins of the head and the chain, marker area = bin size; seed 0 with markers, the | |
| chain's other seeds as hairlines; the read-out's other seeds are asserted identical to seed 0 (seed-invariant) or drawn as hairlines too. | |
| Identity: legend (panel a) + marker shape + direct labels; text in ink tokens. Fonts embedded (Type 42); PDF metadata carries the source.""" | |
| import matplotlib | |
| matplotlib.use("Agg") | |
| import matplotlib.pyplot as plt | |
| from matplotlib.lines import Line2D | |
| plt.rcParams.update({"font.family": "sans-serif", "font.sans-serif": ["Nimbus Sans", "Liberation Sans", "Helvetica", "Arial", "DejaVu Sans"], | |
| "mathtext.fontset": "custom", "mathtext.rm": "Nimbus Sans", "mathtext.it": "Nimbus Sans:italic", "mathtext.bf": "Nimbus Sans:bold", | |
| "pdf.fonttype": 42, "ps.fonttype": 42, "axes.edgecolor": HAIR, "axes.linewidth": 0.5, "text.color": INK, "axes.labelcolor": INK2, | |
| "xtick.color": INK2, "ytick.color": INK2, "figure.facecolor": "white", "savefig.facecolor": "white"}) | |
| main = C[C.item == "published"] | |
| seeds = sorted(main.seed.unique()) | |
| s0 = seeds[0] | |
| names = {"head": HEAD_NAME, "dms": ATT_NAME} | |
| def bins_of(sc, seed, kind): | |
| return B[(B.score == sc) & (B.seed == seed) & (B.kind == kind)].sort_values("bin") | |
| head_invariant = all(np.allclose(bins_of("head", s, k)[["p_mean", "y_rate", "n"]].to_numpy(), bins_of("head", s0, k)[["p_mean", "y_rate", "n"]].to_numpy()) | |
| for s in seeds[1:] for k in ("raw", "cal")) | |
| fig, axes = plt.subplots(1, 2, figsize=(5.5, 2.6), sharey=True) | |
| sidecar = [] | |
| for ax, kind, title in zip(axes, ("raw", "cal"), ("(a) raw scores", "(b) temperature fitted on clean-val (both scores)")): | |
| ax.plot([0, 1], [0, 1], color=HAIR, lw=0.8, zorder=1) | |
| ax.grid(True, color=HAIR, lw=0.4, zorder=0) | |
| for sc in ("head", "dms"): | |
| for seed in seeds: | |
| b = bins_of(sc, seed, kind) | |
| if seed == s0: | |
| ax.plot(b.p_mean, b.y_rate, color=LINE[sc], lw=1.6, solid_joinstyle="round", solid_capstyle="round", zorder=3) | |
| ax.scatter(b.p_mean, b.y_rate, s=1.6 * b.n, marker=MARK[sc], facecolor=FILL[sc], edgecolor=LINE[sc], linewidth=1.0, zorder=4) | |
| for r in b.itertuples(): | |
| sidecar.append(f"{kind}\t{names[sc]}\tseed {seed}\tbin {r.bin}\tp_mean {r.p_mean:.4f}\ty_rate {r.y_rate:.4f}\tn {int(r.n)}\tp_range [{r.p_lo:.4f}, {r.p_hi:.4f}]") | |
| elif sc == "dms" or not head_invariant: | |
| ax.plot(b.p_mean, b.y_rate, color=LINE[sc], lw=0.5, alpha=0.45, zorder=2) | |
| for r in b.itertuples(): | |
| sidecar.append(f"{kind}\t{names[sc]}\tseed {seed} (hairline)\tbin {r.bin}\tp_mean {r.p_mean:.4f}\ty_rate {r.y_rate:.4f}\tn {int(r.n)}") | |
| # direct labels at fixed free positions (both curves lie under the identity line in both panels) | |
| ax.text(0.03, 0.46, "CockpitWM read-out", fontsize=6, color=INK2, ha="left", va="center") | |
| ax.text(0.98, 0.34, "openpilot attention chain (att@3s)", fontsize=6, color=INK2, ha="right", va="center") | |
| ax.set_xlim(0, 1) | |
| ax.set_ylim(0, 1) | |
| ax.set_aspect("equal") | |
| ax.set_title(title, fontsize=7.5, loc="left", color=INK) | |
| ax.set_xlabel("mean predicted probability (10 equal-count bins)", fontsize=6.5) | |
| ax.tick_params(labelsize=6, length=2, width=0.5) | |
| for sp in ("top", "right"): | |
| ax.spines[sp].set_visible(False) | |
| axes[0].set_ylabel("observed positive rate (clean-test gold)", fontsize=6.5) | |
| handles = [Line2D([], [], color=LINE["head"], marker="o", markerfacecolor=FILL["head"], markeredgecolor=LINE["head"], lw=1.6, markersize=5.5, label="CockpitWM read-out (circles)"), | |
| Line2D([], [], color=LINE["dms"], marker="s", markerfacecolor=FILL["dms"], markeredgecolor=LINE["dms"], lw=1.6, markersize=5.5, label="openpilot attention chain, att@3s (squares)")] | |
| axes[0].legend(handles=handles, loc="upper left", fontsize=5.8, frameon=False, handlelength=2.2, borderaxespad=0.3, title="seed 0; marker area = bin size", title_fontsize=5.8, alignment="left") | |
| r_h, r_d = main[(main.score == "head") & (main.seed == s0)].iloc[0], main[(main.score == "dms") & (main.seed == s0)].iloc[0] | |
| axes[1].text(0.03, 0.97, f"read-out: T {r_h['T']:.2f}, ECE {r_h.ece_cal:.3f}, Brier {r_h.brier_cal:.3f}\n" | |
| f"chain: T {r_d['T']:.2f}{' (grid edge)' if int(r_d.T_pinned) else ''}, ECE {r_d.ece_cal:.3f}, Brier {r_d.brier_cal:.3f}\n" | |
| f"constant prevalence {r_h.prevalence:.3f}: Brier {r_h.brier_const_prev:.3f}", fontsize=5.8, color=INK2, ha="left", va="top", linespacing=1.4) | |
| fig.tight_layout(w_pad=1.0) | |
| src_sha = sha256(csv_path) | |
| meta = {"Title": "Reliability of the CockpitWM warning read-out vs the openpilot attention chain", "Subject": f"source {csv_path} sha256 {src_sha} generated {stamp} by eval/calibration_report.py"} | |
| fig.savefig(out_pdf, metadata=meta, bbox_inches="tight", pad_inches=0.03) | |
| fig.savefig(out_pdf.with_suffix(".png"), dpi=300, bbox_inches="tight", pad_inches=0.03) | |
| plt.close(fig) | |
| pr = P[(P.item == "published") & (P.seed == s0)].iloc[0] | |
| cap = (f"Reliability of the CockpitWM warning read-out (circles) and the openpilot attention chain att@3s (squares) on the clean-test gold " | |
| f"(read-out {int(r_h.n_test)} rows, chain {int(r_d.n_test)} rows, {int(r_h.n_pos)} positives, {int(r_h.n_dev)} devices), 10 equal-count bins, marker area proportional to the bin size, " | |
| f"identity line in grey. (a) Raw scores. (b) After a temperature fitted on the clean-val gold for BOTH scores (read-out T = {r_h['T']:.2f}, chain T = {r_d['T']:.2f}" | |
| f"{', the upper edge of the fitting grid: a temperature alone cannot move a score whose raw values lie above 0.5 below 0.5' if int(r_d.T_pinned) else ''}). " | |
| f"Calibrated ECE {r_h.ece_cal:.3f} vs {r_d.ece_cal:.3f}; calibrated Brier {r_h.brier_cal:.3f} vs {r_d.brier_cal:.3f} " | |
| f"(paired Δ read-out − chain on identical rows {pr.cal_delta:+.3f} [{pr.cal_ci_lo:+.3f}, {pr.cal_ci_hi:+.3f}]); constant clean-test prevalence would score {r_h.brier_const_prev:.3f}. " | |
| f"Seed 0 with markers; the chain's seeds 1–2 as hairlines{' (the read-out is seed-invariant)' if head_invariant else '; the read-out seeds 1–2 as hairlines'}. " | |
| f"ECE uses the 15 equal-width bins of warning_gold.ece; device-cluster bootstrap CIs. Source: {rel(csv_path)}.") | |
| with open(out_pdf.with_suffix(".txt"), "w") as f: | |
| f.write(f"# reliability_warning.pdf -- generated {stamp} by eval/calibration_report.py\n# source: {csv_path}\n# source sha256: {src_sha}\n" | |
| f"# palette: fills cwPeachL #F5D8BF (read-out) / cwGreyL #BFDCE7 (chain) from BATON__/ICLR/cwm_macros.tex; lines / edges #B5601A / #2A72A8 (darker steps of the same hues; validated)\n" | |
| f"# fonts: Nimbus Sans, embedded (pdf.fonttype 42); size 5.5 x 2.6 in (ICLR textwidth)\n\n[caption]\n{cap}\n\n[plotted bins]\n" + "\n".join(sidecar) + "\n") | |
| return cap, head_invariant | |
| # ----------------------------------------------------------------------------- table section | |
| def md_section(C, B, P, K, out, fig_pdf, a, head_invariant): | |
| def pair_of(item, score, seed): | |
| g = P[(P.item == item) & (P.score == score) & (P.seed == seed)] | |
| return g.iloc[0] if len(g) else None | |
| def cal_row(r, pr): | |
| pd_ = "— (reference)" if r.score == "dms" else (f"{pr.cal_delta:+.3f} {ci_s(pr.cal_ci_lo, pr.cal_ci_hi, sign=True)} (n {int(pr.cal_n)})" if pr is not None else "—") | |
| return (f"| {r.name} | {int(r.seed)} | {t_s(r)} | {r.ece_raw:.3f} | {r.ece_cal:.3f} {ci_s(r.ece_cal_ci_lo, r.ece_cal_ci_hi)} | {r.brier_raw:.3f} | " | |
| f"{r.brier_cal:.3f} {ci_s(r.brier_cal_ci_lo, r.brier_cal_ci_hi)} | {pd_} | {r.p05_raw:.2f} / {r.p50_raw:.2f} / {r.p95_raw:.2f} → {r.p05_cal:.2f} / {r.p50_cal:.2f} / {r.p95_cal:.2f} |") | |
| hdr = ["| score | seed | T (clean-val) | ECE raw | ECE cal [CI] | Brier raw | Brier cal [CI] | paired Δ Brier cal vs att@3s [CI] | p05 / p50 / p95 raw → calibrated |", | |
| "|---|---|---|---|---|---|---|---|---|"] | |
| main = C[C.item == "published"] | |
| s0 = a.seeds[0] | |
| r_h, r_d = main[(main.score == "head") & (main.seed == s0)].iloc[0], main[(main.score == "dms") & (main.seed == s0)].iloc[0] | |
| pr0 = pair_of("published", "head", s0) | |
| L = [f"## {SECTION_TITLE} — run {out.name}", "", | |
| f"Complement to the ECE rows of item `final_ext_dwfc` (run {Path(a.ref_dir).name}): the same gold rows, scores and clean-val / clean-test masks rebuilt by " | |
| f"`eval/calibration_report.py` through the unchanged `warning_gold.build_gold` (legacy_first anchors) and the layouts {', '.join(a.w_runs)} (column `{a.col}`, seeds {a.seeds}); " | |
| f"the {ATT_NAME} is the legacy `att@3s` gold column. **Both scores are temperature-calibrated on the clean-val gold rows** ({int(r_h.n_val)} rows / {int(r_h.n_val_pos)} positives; " | |
| f"`warning_gold.temperature_fit`, NLL on a log-spaced grid [0.1, 20], 200 points; 'pinned' = the fit sits at a grid edge) and evaluated on the clean-test gold rows " | |
| f"({int(r_h.n_test)} rows / {int(r_h.n_pos)} positives / {int(r_h.n_dev)} devices; the chain has {int(r_d.n_test)} finite rows). ECE = `warning_gold.ece` (15 equal-width bins, as the published CSV); " | |
| f"Brier = mean squared error of the probability (`warning_gold.brier`; the constant clean-test prevalence {r_h.prevalence:.3f} scores {r_h.brier_const_prev:.3f}); " | |
| f"CIs = device-cluster bootstrap ({a.n_boot} resamples; `operating_points.cboot` seed 3 for the per-score statistics, seed 2 for the paired Δ); the paired Δ Brier " | |
| f"(score − att@3s, calibrated probabilities) is computed on the identical clean-test rows where both scores exist (negative = the score is better); " | |
| f"p05 / p50 / p95 = quantiles of the raw and of the calibrated score on the clean-test rows (the chain's compressed distribution). " | |
| f"Reliability curve = 10 equal-count bins (`operating_points.reliability_bins`), raw and calibrated: figure `{rel(fig_pdf)}` (+ `.png`; caption in the `.txt` sidecar). " | |
| f"Raw rows: `results/operating_points/{out.name}/` (calibration.csv, reliability_curves.csv, paired_brier.csv, ece_repro_check.csv).", "", | |
| f"**ECE reproduction check** against `{rel(Path(a.ref_dir) / 'reliability.csv')}`: for every (score, seed) the recomputed T, ECE raw, ECE calibrated, the calibrated " | |
| f"10-bin curve (mean p, observed rate, n){f' and the 1000-resample ECE CI (the first 1000 of the {a.n_boot} resamples)' if a.n_boot >= 1000 else ''} equal the CSV — max |Δ| = {K.filter(like='absdiff_').to_numpy().max():.1e} " | |
| f"(asserted ≤ 1e-6 before any output was written; `ece_repro_check.csv`).", "", | |
| f"### A. {HEAD_NAME} (`{a.col}`) vs {ATT_NAME} — clean-test gold, per seed", ""] + hdr | |
| for sc in ("head", "dms"): | |
| for r in main[main.score == sc].sort_values("seed").itertuples(): | |
| L.append(cal_row(r, pair_of("published", "head", int(r.seed)) if sc == "head" else None)) | |
| L += ["", f"Paired Δ Brier ({HEAD_NAME} − att@3s) on identical rows; raw probabilities for reference:", "", | |
| "| seed | n rows / pos | Δ Brier raw [CI] | Δ Brier calibrated [CI] | share of rows where the read-out is closer | verdict (calibrated) |", "|---|---|---|---|---|---|"] | |
| for pr in P[P.item == "published"].sort_values("seed").itertuples(): | |
| L.append(f"| {int(pr.seed)} | {int(pr.cal_n)} / {int(pr.cal_n_pos)} | {pr.raw_delta:+.3f} {ci_s(pr.raw_ci_lo, pr.raw_ci_hi, sign=True)} | {pr.cal_delta:+.3f} {ci_s(pr.cal_ci_lo, pr.cal_ci_hi, sign=True)} | " | |
| f"{pr.cal_frac_rows_a_better:.2f} | {verdict_s(pr.cal_ci_lo, pr.cal_ci_hi)} |") | |
| L += ["", f"### B. Reliability bins of seed {s0} (mean predicted p → observed rate (n); 10 equal-count bins)", "", "| score | probabilities | bins |", "|---|---|---|"] | |
| for sc, name in (("head", HEAD_NAME), ("dms", ATT_NAME)): | |
| for kind, lab in (("raw", "raw"), ("cal", "temperature-calibrated")): | |
| b = B[(B.score == sc) & (B.seed == s0) & (B.kind == kind)].sort_values("bin") | |
| L.append(f"| {name} | {lab} | " + "; ".join(f"{x.p_mean:.2f}→{x.y_rate:.2f} ({int(x.n)})" for x in b.itertuples()) + " |") | |
| L += ["", "### C. Secondary — Platt scaling (slope a and bias b on the logit, NLL on the same clean-val rows); the primary protocol above stays temperature-only", "", | |
| "A temperature rescales the logit without a bias, so a score whose raw values lie almost entirely above 0.5 can never be mapped below 0.5; Platt scaling removes that " | |
| "restriction for both scores (reported for completeness, not used anywhere else).", "", | |
| "| score | seed | a / b | ECE Platt [CI] | Brier Platt [CI] | paired Δ Brier Platt vs att@3s [CI] | p05 / p50 / p95 Platt |", "|---|---|---|---|---|---|---|"] | |
| for sc in ("head", "dms"): | |
| for r in main[main.score == sc].sort_values("seed").itertuples(): | |
| pr = pair_of("published", "head", int(r.seed)) | |
| pd_ = "— (reference)" if sc == "dms" else f"{pr.platt_delta:+.3f} {ci_s(pr.platt_ci_lo, pr.platt_ci_hi, sign=True)} (n {int(pr.platt_n)})" | |
| L.append(f"| {r.name} | {int(r.seed)} | {r.platt_a:.2f} / {r.platt_b:+.2f} | {r.ece_platt:.3f} {ci_s(r.ece_platt_ci_lo, r.ece_platt_ci_hi)} | {r.brier_platt:.3f} {ci_s(r.brier_platt_ci_lo, r.brier_platt_ci_hi)} | " | |
| f"{pd_} | {r.p05_platt:.2f} / {r.p50_platt:.2f} / {r.p95_platt:.2f} |") | |
| l2 = C[C.item != "published"] | |
| if len(l2): | |
| L += ["", "### D. DMS layer-2 selected read-outs on the same gold (per seed; same temperature protocol; paired Δ vs att@3s of the same seed)", ""] + hdr | |
| for item in l2.item.unique(): | |
| for r in l2[l2.item == item].sort_values("seed").itertuples(): | |
| L.append(cal_row(r, pair_of(item, item, int(r.seed)))) | |
| verdicts = [verdict_s(p.cal_ci_lo, p.cal_ci_hi).split(" (")[0] for p in P[P.item == "published"].sort_values("seed").itertuples()] | |
| L += ["", f"**Reading.** After calibrating BOTH scores on clean-val, the {HEAD_NAME} has ECE {r_h.ece_cal:.3f} and Brier {r_h.brier_cal:.3f} {ci_s(r_h.brier_cal_ci_lo, r_h.brier_cal_ci_hi)} " | |
| f"(seed {s0}; {'the read-out is seed-invariant' if head_invariant else 'per-seed values above'}), the {ATT_NAME} ECE {r_d.ece_cal:.3f} and Brier {r_d.brier_cal:.3f} " | |
| f"{ci_s(r_d.brier_cal_ci_lo, r_d.brier_cal_ci_hi)}; the paired Δ Brier (read-out − chain, identical rows, calibrated) is {pr0.cal_delta:+.3f} {ci_s(pr0.cal_ci_lo, pr0.cal_ci_hi, sign=True)} " | |
| f"(seed {s0}; verdicts over seeds {a.seeds}: {', '.join(verdicts)}). The chain's temperature {r_d['T']:.2f}{' is pinned at the upper edge of the fitting grid' if int(r_d.T_pinned) else ''}: " | |
| f"its raw scores sit at {r_d.p05_raw:.2f} / {r_d.p50_raw:.2f} / {r_d.p95_raw:.2f} (p05 / p50 / p95) against an observed rate of {r_d.prevalence:.3f}, and a temperature can only compress " | |
| f"them towards 0.5 ({r_d.p05_cal:.2f}–{r_d.p95_cal:.2f} after calibration), so its calibrated Brier stays {'above' if r_d.brier_cal > r_d.brier_const_prev else 'at or below'} the constant-prevalence " | |
| f"reference ({r_d.brier_const_prev:.3f}) while the read-out's ({r_h.brier_cal:.3f}) is {'below' if r_h.brier_cal < r_h.brier_const_prev else 'not below'} it. " | |
| f"Under the secondary Platt scaling (slope + bias) the chain reaches ECE {r_d.ece_platt:.3f} / Brier {r_d.brier_platt:.3f} and the read-out ECE {r_h.ece_platt:.3f} / Brier {r_h.brier_platt:.3f} " | |
| f"(paired Δ {pr0.platt_delta:+.3f} {ci_s(pr0.platt_ci_lo, pr0.platt_ci_hi, sign=True)}: {verdict_s(pr0.platt_ci_lo, pr0.platt_ci_hi)}), i.e. most of the temperature-only ECE gap " | |
| f"reflects the chain's compressed score distribution, while the paired Brier Δ keeps its sign under both calibration protocols.", ""] | |
| return "\n".join(L) + "\n" | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--item", default="calibration_report") | |
| ap.add_argument("--ref-dir", default=str(PUBLISHED["ref_dir"]), help="published operating_points run whose reliability.csv is reproduced (ECE check)") | |
| ap.add_argument("--w-runs", nargs="+", default=PUBLISHED["w_runs"]) | |
| ap.add_argument("--seeds", nargs="+", type=int, default=PUBLISHED["seeds"]) | |
| ap.add_argument("--col", default=PUBLISHED["col"]) | |
| ap.add_argument("--n-boot", type=int, default=2000) | |
| ap.add_argument("--no-layer2", action="store_true") | |
| ap.add_argument("--fig-dir", default=str(FIG_DIR)) | |
| ap.add_argument("--no-table", action="store_true", help="do not append the section to results/tables/operating_points.md") | |
| a = ap.parse_args() | |
| assert len(a.w_runs) == len(a.seeds) | |
| t0 = time.time() | |
| stamp = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime()) | |
| out = OUT_ROOT / f"{a.item}_{stamp}" | |
| out.mkdir(parents=True, exist_ok=True) | |
| log_p = out / "calibration_report.log" | |
| def log(m): | |
| line = f"[{time.strftime('%H:%M:%S')}] {m}" | |
| print(line, flush=True) | |
| with open(log_p, "a") as f: | |
| f.write(line + "\n") | |
| REF = pd.read_csv(Path(a.ref_dir) / "reliability.csv") | |
| lab = LB.episode_labels() | |
| lab = lab[lab.label != "unsure"].reset_index(drop=True) | |
| packs = {(r, s): WG.load_pack(r, s) for r in REGIMES for s in ("test", "val")} | |
| rows_cal, rows_bins, rows_pair, rows_chk = [], [], [], [] | |
| golds, dms_full = {}, {} | |
| def gold(seed): | |
| if seed not in golds: | |
| golds[seed] = gold_for_seed(lab, packs, seed) | |
| return golds[seed] | |
| def add_bins(bins, score, name, seed, T, item): | |
| for tag, bl in bins.items(): | |
| for bn in bl: | |
| rows_bins.append({"item": item, "score": score, "name": name, "seed": seed, "kind": tag, "T": T, **bn}) | |
| def add_pairs(G, item, score, name, seed, full_a, full_b): | |
| row = {"item": item, "score": score, "name": name, "seed": seed, "vs": "att@3s"} | |
| for tag in ("raw", "cal", "platt"): | |
| row.update({f"{tag}_{k}": v for k, v in paired_delta(G, full_a[tag], full_b[tag], a.n_boot).items()}) | |
| rows_pair.append(row) | |
| return row | |
| # ---- published head-vs-DMS run: head + att@3s per seed, ECE reproduction check | |
| for w_run, seed in zip(a.w_runs, a.seeds): | |
| G = gold(seed) | |
| gh, att = layout_scores(G, packs, w_run, a.col), G["att"].to_numpy(float) | |
| gt = G.clean_test.to_numpy() | |
| log(f"seed {seed}: gold rows {len(G)}, clean_test {int(gt.sum())} / {int(G.y[gt].sum())} pos, clean_val {int(G.clean_val.sum())}; finite head {np.isfinite(gh[gt]).mean():.3f} finite att {np.isfinite(att[gt]).mean():.3f}") | |
| full = {} | |
| for key, name, gsc, layout, col in (("head", HEAD_NAME, gh, w_run, a.col), ("dms", ATT_NAME, att, "legacy att@3s gold column", "att@3s")): | |
| st, bins, full[key] = calibrate(G, gsc, a.n_boot) | |
| ref = REF[(REF.seed == seed) & (REF.score == key)] | |
| tot, rb = ref[ref.bin == -1].iloc[0], ref[ref.bin >= 0].sort_values("bin") | |
| mine = pd.DataFrame(bins["cal"]).sort_values("bin") | |
| diffs = {"T": abs(st["T"] - tot["T"]), "ece_raw": abs(st["ece_raw"] - tot.ece_raw), "ece_cal": abs(st["ece_cal"] - tot.ece_cal), "n": abs(st["n_test"] - tot.n), | |
| **({"ece_cal_ci_lo_1000": abs(st["ece_cal_ci1000_lo"] - tot.ece_cal_ci_lo), "ece_cal_ci_hi_1000": abs(st["ece_cal_ci1000_hi"] - tot.ece_cal_ci_hi)} if a.n_boot >= 1000 else {}), | |
| "bins_p_mean": float(np.abs(rb.p_mean.to_numpy() - mine.p_mean.to_numpy()).max()), "bins_y_rate": float(np.abs(rb.y_rate.to_numpy() - mine.y_rate.to_numpy()).max()), | |
| "bins_n": float(np.abs(rb.n.to_numpy() - mine.n.to_numpy()).max())} | |
| for k, v in diffs.items(): | |
| assert v <= 1e-6, f"ECE reproduction failed: {key} seed {seed} {k} |Δ| = {v}" | |
| rows_chk.append({"score": key, "seed": seed, "ref_dir": Path(a.ref_dir).name, "ref_T": tot["T"], "ref_ece_raw": tot.ece_raw, "ref_ece_cal": tot.ece_cal, | |
| "ref_ece_cal_ci_lo": tot.ece_cal_ci_lo, "ref_ece_cal_ci_hi": tot.ece_cal_ci_hi, **{f"absdiff_{k}": v for k, v in diffs.items()}}) | |
| rows_cal.append({"item": "published", "score": key, "name": name, "seed": seed, "layout": layout, "col": col, **st}) | |
| add_bins(bins, key, name, seed, st["T"], "published") | |
| log(f" {key:4s}: T {st['T']:.4f}{' (pinned)' if st['T_pinned'] else ''} ECE raw {st['ece_raw']:.4f} cal {st['ece_cal']:.4f} {ci_s(st['ece_cal_ci_lo'], st['ece_cal_ci_hi'])} | " | |
| f"Brier raw {st['brier_raw']:.4f} cal {st['brier_cal']:.4f} {ci_s(st['brier_cal_ci_lo'], st['brier_cal_ci_hi'])} (const-prev {st['brier_const_prev']:.4f}) | " | |
| f"p05/50/95 raw {st['p05_raw']:.2f}/{st['p50_raw']:.2f}/{st['p95_raw']:.2f} cal {st['p05_cal']:.2f}/{st['p50_cal']:.2f}/{st['p95_cal']:.2f} | " | |
| f"Platt a {st['platt_a']:.2f} b {st['platt_b']:+.2f}: ECE {st['ece_platt']:.4f} Brier {st['brier_platt']:.4f} | n {st['n_test']} -> CSV reproduced (max |Δ| {max(diffs.values()):.1e})") | |
| dms_full[seed] = full["dms"] | |
| row = add_pairs(G, "published", "head", HEAD_NAME, seed, full["head"], full["dms"]) | |
| log(f" paired Δ Brier head − att@3s (identical {row['cal_n']} rows): calibrated {row['cal_delta']:+.4f} {ci_s(row['cal_ci_lo'], row['cal_ci_hi'], 4, True)}; " | |
| f"raw {row['raw_delta']:+.4f} {ci_s(row['raw_ci_lo'], row['raw_ci_hi'], 4, True)}; Platt {row['platt_delta']:+.4f} {ci_s(row['platt_ci_lo'], row['platt_ci_hi'], 4, True)}") | |
| # ---- DMS layer-2 selected read-outs (cheap: one calibration per layout) | |
| if not a.no_layer2: | |
| for item, disp, runs, seeds, col in layer2_items(): | |
| for w_run, seed in zip(runs, seeds): | |
| G = gold(seed) | |
| if seed not in dms_full: | |
| dms_full[seed] = calibrate(G, G["att"].to_numpy(float), a.n_boot)[2] | |
| st, bins, full_l2 = calibrate(G, layout_scores(G, packs, w_run, col), a.n_boot) | |
| rows_cal.append({"item": item, "score": item, "name": disp, "seed": seed, "layout": w_run, "col": col, **st}) | |
| add_bins(bins, item, disp, seed, st["T"], item) | |
| row = add_pairs(G, item, item, disp, seed, full_l2, dms_full[seed]) | |
| log(f" L2 {item} s{seed}: T {st['T']:.2f}{' (pinned)' if st['T_pinned'] else ''} ECE {st['ece_raw']:.3f}->{st['ece_cal']:.3f} Brier {st['brier_raw']:.3f}->{st['brier_cal']:.3f} " | |
| f"{ci_s(st['brier_cal_ci_lo'], st['brier_cal_ci_hi'])} Δ vs att@3s {row['cal_delta']:+.3f} {ci_s(row['cal_ci_lo'], row['cal_ci_hi'], 3, True)} (n {row['cal_n']})") | |
| C, B, P, K = (pd.DataFrame(x) for x in (rows_cal, rows_bins, rows_pair, rows_chk)) | |
| C.to_csv(out / "calibration.csv", index=False) | |
| B.to_csv(out / "reliability_curves.csv", index=False) | |
| P.to_csv(out / "paired_brier.csv", index=False) | |
| K.to_csv(out / "ece_repro_check.csv", index=False) | |
| # ---- figure | |
| fig_dir = Path(a.fig_dir) | |
| fig_dir.mkdir(parents=True, exist_ok=True) | |
| fig_pdf = fig_dir / "reliability_warning.pdf" | |
| cap, head_invariant = make_figure(C, B, P, fig_pdf, out / "reliability_curves.csv", stamp) | |
| log(f"figure -> {fig_pdf} (+ .png, .txt); read-out seed-invariant: {head_invariant}") | |
| # ---- table section (append-only) | |
| md = md_section(C, B, P, K, out, fig_pdf, a, head_invariant) | |
| (out / "section.md").write_text(md) | |
| if not a.no_table: | |
| with open(TABLE, "a") as f: | |
| f.write("\n" + md) | |
| print(md) | |
| OP.write_run_json(out, "calibration_report", [Path(__file__), Path(OP.__file__), Path(WG.__file__)], | |
| [RUNS / w / f"pred_layout_{r}_{s}.npz" for w in a.w_runs for r in REGIMES for s in ("test", "val")] + [Path(a.ref_dir) / "reliability.csv"], | |
| seed=0, t_start=t0, input_listing=False, | |
| extra={"item": a.item, "ref_dir": str(a.ref_dir), "w_runs": a.w_runs, "seeds": a.seeds, "col": a.col, "n_boot": a.n_boot, "figure": str(fig_pdf), | |
| "ece_repro_max_absdiff": float(K.filter(like="absdiff_").to_numpy().max()), "head_seed_invariant": bool(head_invariant), | |
| "layer2_items": sorted(C.item.unique().tolist()), "dms_fields_used": []}) | |
| log(f"DONE in {time.time() - t0:.0f}s -> {out}; section {'appended to ' + str(TABLE) if not a.no_table else 'NOT appended (--no-table)'}") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 37.1 kB
- Xet hash:
- 72da37da59cb434a491a23ee96c2f2ddf1fee68f3540c8f40c1781bc6b2068e0
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.