Buckets:
| """cwm3/eval/warning_ecan_vs_original.py — PAIRED comparison of the ECAN warning read-out with the ORIGINAL read-out (pre-registration row | |
| 2026-09-17T16:32:09Z in docs/decisions.md; owner decision 1 + 2 of 16:24:02Z adopted ECAN as the default BATON configuration). CPU only; read-only on every | |
| existing run and table (new files + one appended section). | |
| The ECAN heads are the SAME recipe (train/train_warning_head.py, identical argv) fitted on the ECAN main runs `I1_all_minus_road2drv_s<k>_final_ecan`; the | |
| original heads are the frozen paper read-outs on `I1_all_minus_road2drv_s<k>_final`. Every Δ below is ECAN − ORIGINAL, per seed, on IDENTICAL rows and | |
| device-cluster resamples: | |
| 1. GOLD (this script, the evaluator's own calls: LB.episode_labels -> WG.load_pack -> WG.merge_legacy(seed) -> WG.build_gold(legacy_first), the same chain as | |
| operating_points.py / calibration_report.py): clean-test AUROC and PR-AUC pooled / manual / engaged for cur+dwell+face (paper read-out), current-only and | |
| the verified-mask arm (w_mask applied as in warning_gold.py), paired Δ with warning_gold.cluster_boot on shared resamples (n_boot, seed 0); the both / | |
| future heads are reported as descriptive rows (they carry ECAN-dependent predicted quantities; not pre-registered). Bit-identity of the two layouts is | |
| reported as max |Δ score| over EVERY test / val row of both regimes (an exactly-zero Δ is a valid outcome of the rule, not a skipped test). | |
| 2. TIMING (read from the alert_timing_eval.py run that holds the ECAN scores next to the original defaults): Recall_{>= δ s early}, δ = 0.5 / 1 / 2 s, | |
| K = 10 / 20 / 40 per h, families A (primary) / B / C1 / C2 / D1 / D2, pooled / manual / engaged, VAL-fitted thresholds primary (test-fitted secondary, | |
| descriptive) — paired.csv rows (a = ecan_cur_dwfc, b = cur_dwfc) with the run's own device-cluster CIs; alert-load and first-alert lead levels from metrics.csv. | |
| 3. LAYER 3 (read from operating_points.py items ours_l3_{dwfc,dwfcrmv,cur}_ecan with --pair-runs original=...): paired first-alert lead d_lead0_mean (pre-registered | |
| endpoint iii) and, descriptive, recall / precision / FAR at every load (dms_matched, low1 / 3 / 5 / 10, same_recall) against the original at its PUBLISHED thresholds. | |
| 4. CALIBRATION (this script): both heads temperature-calibrated on clean-val (warning_gold.temperature_fit), ECE raw / calibrated and Brier raw / calibrated on | |
| clean-test, paired Δ with operating_points.cboot (seed 3 = the ECE seed of operating_points.py); the reliability levels of the layer-3 runs are quoted. | |
| READING RULE (pre-registered): an endpoint is 'significantly worse' when the paired CI excludes 0 with Δ < 0 on >= ceil(2n/3) seeds; the warning section moves to | |
| ECAN iff NO pre-registered endpoint is significantly worse; otherwise it stays on the original configuration and the ECAN read-out goes to Supp; 'better' endpoints | |
| do not enter the decision. No openpilot DMS field is read (the att@3s chain enters only as the labelled comparator column already inside the gold frame). | |
| python eval/warning_ecan_vs_original.py [--seeds 0 1 2] [--n-boot 2000] [--timing-dir results/alert_timing/<stamp>] [--allow-missing] | |
| Outputs: results/warning_ecan_vs_original/<stamp>/{gold_pairs,calibration_pairs,timing_pairs,timing_levels,layer3_pairs,endpoints}.csv + section.md + run json; | |
| one section appended to results/tables/warning_ecan_vs_original.md (created with a title when absent; never overwritten). | |
| """ | |
| from __future__ import annotations | |
| import os | |
| os.environ.setdefault("OMP_NUM_THREADS", "4") | |
| os.environ.setdefault("MKL_NUM_THREADS", "4") | |
| import argparse # noqa: E402 | |
| import json # noqa: E402 | |
| import sys # noqa: E402 | |
| import time # noqa: E402 | |
| from pathlib import Path # noqa: E402 | |
| import numpy as np # noqa: E402 | |
| import pandas as pd # noqa: E402 | |
| CWM3 = Path(__file__).resolve().parents[1] | |
| sys.path.insert(0, str(CWM3 / "eval")) | |
| import calibration_report as CR # noqa: E402 module import only (main() never called): gold_for_seed, OP, WG, LB | |
| OP, WG, LB = CR.OP, CR.WG, CR.LB | |
| REGIMES = ("manual", "engaged") | |
| RUNS = CWM3 / "results" / "runs" | |
| OUT_ROOT = CWM3 / "results" / "warning_ecan_vs_original" | |
| TABLE = CWM3 / "results" / "tables" / "warning_ecan_vs_original.md" | |
| OP_ROOT = CWM3 / "results" / "operating_points" | |
| AT_ROOT = CWM3 / "results" / "alert_timing" | |
| PREREG = "2026-09-17T16:32:09Z" | |
| ORIG, ECAN = "I1_all_minus_road2drv_s{seed}_final", "I1_all_minus_road2drv_s{seed}_final_ecan" | |
| # (key, label, W layout suffix, column, pre-registered endpoint?) | |
| ARMS = [("dwfc", "cur+dwell+face (paper read-out)", "_dwfc", "W_learned_curhgb_t15_w10_n979dwfc", True), | |
| ("cur", "current-only", "", "W_learned_curhgb_t15_w10_n979", True), | |
| ("dwfcrmv", "cur+dwell+face, verified 3-route mask (rmv)", "_dwfcrmv", "W_learned_curhgb_t15_w10_n979dwfcrmv", True), | |
| ("both", "both = current + predicted future (descriptive; ECAN-dependent predicted features)", "", "W_learned_bothhgb_t15_w10_n979", False), | |
| ("fut", "predicted-future only (descriptive)", "", "W_learned_futhgb_t15_w10_n979", False)] | |
| TIMING_PAIRS = [("ecan_cur_dwfc", "cur_dwfc", True), ("ecan_cur", "cur_base", False), ("ecan_both", "both_base", False)] | |
| L3_ITEMS = [("dwfc", "ours_l3_dwfc_ecan", "ours_l3_dwfc_20260917T084250Z"), ("dwfcrmv", "ours_l3_dwfcrmv_ecan", "ours_l3_dwfcrmv_20260917T084259Z"), | |
| ("cur", "ours_l3_cur_ecan", "ours_l3_cur_20260917T084307Z")] | |
| LOAD_ORDER = ["dms_matched", "low10", "low5", "low3", "low1", "same_recall"] | |
| STRATA = ("pooled", "manual", "engaged") | |
| def f3(x, nd=3, sign=False): | |
| try: | |
| x = float(x) | |
| except (TypeError, ValueError): | |
| return "—" | |
| if not np.isfinite(x): | |
| return "—" | |
| return f"{x:+.{nd}f}" if sign else f"{x:.{nd}f}" | |
| def ci_s(lo, hi, nd=3): | |
| return f"[{f3(lo, nd, True)}, {f3(hi, nd, True)}]" | |
| def verdict1(d, lo, hi): | |
| """per-seed verdict of a paired Δ (ECAN − original): identical (Δ = 0 with a degenerate CI) / WORSE (CI < 0) / BETTER (CI > 0) / n.s.""" | |
| if not (np.isfinite(d) and np.isfinite(lo) and np.isfinite(hi)): | |
| return "—" | |
| if d == 0 and lo == 0 and hi == 0: | |
| return "identical" | |
| return "WORSE" if hi < 0 else ("BETTER" if lo > 0 else "n.s.") | |
| def need(n): | |
| return int(np.ceil(2 * n / 3)) | |
| def seed_rule(vs): | |
| """pre-registered multi-seed reading: significantly worse iff >= ceil(2n/3) per-seed verdicts are WORSE (analogue for better).""" | |
| vs = [v for v in vs if v != "—"] | |
| n = len(vs) | |
| if n == 0: | |
| return "—", 0, 0, 0 | |
| nw, nb = sum(v == "WORSE" for v in vs), sum(v == "BETTER" for v in vs) | |
| if all(v == "identical" for v in vs): | |
| return "identical (Δ = 0)", nw, nb, n | |
| if nw >= need(n): | |
| return f"SIGNIFICANTLY WORSE ({nw}/{n})", nw, nb, n | |
| if nb >= need(n): | |
| return f"better ({nb}/{n}; does not enter the decision)", nw, nb, n | |
| return f"not worse ({nw}/{n} worse, {nb}/{n} better)", nw, nb, n | |
| def load_col(w_run, col, packs): | |
| """one W layout column at every (regime, split) row, w_mask applied (NaN) as warning_gold.py / alert_timing_eval.py do -> {(regime, split): float64[n]}.""" | |
| out = {} | |
| for r in REGIMES: | |
| for s in ("test", "val"): | |
| p = RUNS / w_run / f"pred_layout_{r}_{s}.npz" | |
| assert p.exists(), p | |
| with np.load(p, allow_pickle=True) as z: | |
| names = list(map(str, z["w_names"])) | |
| assert col in names, (w_run, col, names) | |
| assert len(z["uid"]) == len(packs[(r, s)]["uid"]) and np.array_equal(z["uid"], packs[(r, s)]["uid"]), f"{w_run} {r}/{s}: layout rows != targets rows" | |
| v = z["w"][:, names.index(col)].astype(np.float64) | |
| if "w_mask" in z.files: | |
| v = np.where(z["w_mask"], v, np.nan) | |
| out[(r, s)] = v | |
| return out | |
| def at_gold(G, cols): | |
| g = np.full(len(G), np.nan) | |
| for (r, s), grp in G.groupby(["regime", "split"]): | |
| g[grp.index.to_numpy()] = cols[(r, s)][grp.row.to_numpy()] | |
| return g | |
| def strata(G): | |
| gt, reg = G.clean_test.to_numpy(), G.regime.to_numpy() | |
| return {"pooled": gt, "manual": gt & (reg == "manual"), "engaged": gt & (reg == "engaged")} | |
| def newest(pattern, root): | |
| ds = sorted(d for d in root.glob(pattern) if d.is_dir()) | |
| return ds[-1] if ds else None | |
| def find_timing_dir(explicit): | |
| if explicit: | |
| return Path(explicit) | |
| cands = [] | |
| for rj in sorted(AT_ROOT.glob("*/run.json")): | |
| try: | |
| txt = rj.read_text() | |
| except OSError: | |
| continue | |
| if "ecan_cur_dwfc" in txt: | |
| cands.append(rj.parent) | |
| return cands[-1] if cands else None | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--seeds", nargs="+", type=int, default=[0, 1, 2]) | |
| ap.add_argument("--n-boot", type=int, default=2000) | |
| ap.add_argument("--timing-dir", default="", help="alert_timing run dir holding ecan_cur_dwfc (default: newest such run)") | |
| ap.add_argument("--allow-missing", action="store_true", help="render what exists (debug); the final run must have every input") | |
| ap.add_argument("--no-table", action="store_true") | |
| ap.add_argument("--out-root", default=str(OUT_ROOT), help="run-dir root (default results/warning_ecan_vs_original; /tmp for smoke runs)") | |
| a = ap.parse_args() | |
| t0 = time.time() | |
| stamp = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime()) | |
| out = Path(a.out_root) / stamp | |
| out.mkdir(parents=True, exist_ok=True) | |
| log_p = out / "warning_ecan_vs_original.log" | |
| def log(m): | |
| line = f"[{time.strftime('%H:%M:%S')}] {m}" | |
| print(line, flush=True) | |
| with open(log_p, "a") as f: | |
| f.write(line + "\n") | |
| lab = LB.episode_labels() | |
| lab = lab[lab.label != "unsure"].reset_index(drop=True) | |
| packs = {(r, s): WG.load_pack(r, s) for r in REGIMES for s in ("test", "val")} | |
| golds = {} | |
| def gold(seed): | |
| if seed not in golds: | |
| golds[seed] = CR.gold_for_seed(lab, packs, seed) | |
| return golds[seed] | |
| inputs, missing = [], [] | |
| endpoints = [] # every pre-registered endpoint: (block, endpoint, per-seed verdicts, rule) | |
| # ------------------------------------------------------------------ 1. gold + 4. calibration (paired, this script) | |
| rows_g, rows_c, ident = [], [], {} | |
| for key, label, suf, col, prereg in ARMS: | |
| for seed in a.seeds: | |
| we, wo = f"W_{ECAN.format(seed=seed)}{suf}", f"W_{ORIG.format(seed=seed)}{suf}" | |
| if not all((RUNS / w / f"pred_layout_{r}_{s}.npz").exists() for w in (we, wo) for r in REGIMES for s in ("test", "val")): | |
| missing.append(f"layouts {we} / {wo}") | |
| log(f"MISSING layouts for arm {key} seed {seed}: {we} / {wo}") | |
| continue | |
| G = gold(seed) | |
| ce, co = load_col(we, col, packs), load_col(wo, col, packs) | |
| inputs += [RUNS / w / f"pred_layout_{r}_{s}.npz" for w in (we, wo) for r in REGIMES for s in ("test", "val")] | |
| maxabs, nan_mis, n_rows = 0.0, 0, 0 | |
| for k in ce: | |
| both = np.isfinite(ce[k]) & np.isfinite(co[k]) | |
| nan_mis += int((np.isfinite(ce[k]) != np.isfinite(co[k])).sum()) | |
| n_rows += len(ce[k]) | |
| if both.any(): | |
| maxabs = max(maxabs, float(np.abs(ce[k][both] - co[k][both]).max())) | |
| identical = (maxabs == 0.0) and (nan_mis == 0) | |
| ident[(key, seed)] = {"identical": identical, "max_abs_diff": maxabs, "nan_mismatch_rows": nan_mis, "n_rows": n_rows} | |
| log(f"arm {key} seed {seed}: {we} vs {wo} col {col}: rows {n_rows}, max |Δ| {maxabs:.3e}, NaN-pattern mismatches {nan_mis} -> {'IDENTICAL' if identical else 'different'}") | |
| ge, go = at_gold(G, ce), at_gold(G, co) | |
| yg, dong = G.y.to_numpy().astype(int), G.dongle.to_numpy() | |
| for st, m0 in strata(G).items(): | |
| m = m0 & np.isfinite(ge) & np.isfinite(go) | |
| y, dg = yg[m], dong[m] | |
| if y.min() == y.max(): | |
| continue | |
| bs = WG.cluster_boot(y, {"e": ge[m], "o": go[m]}, dg, a.n_boot, 0) | |
| for met, fn, kb in (("auroc", WG.fast_auroc, "auroc"), ("pr_auc", WG.fast_ap, "prauc")): | |
| ve, vo = float(fn(y, ge[m])), float(fn(y, go[m])) | |
| d = bs["e"][kb] - bs["o"][kb] | |
| lo, hi = WG.pct(d) | |
| v = verdict1(ve - vo, lo, hi) | |
| rows_g.append({"arm": key, "label": label, "prereg": prereg, "seed": seed, "stratum": st, "metric": met, "ecan": ve, "orig": vo, "delta": ve - vo, "ci_lo": lo, "ci_hi": hi, | |
| "frac_boot_d_gt0": float(np.mean(d[np.isfinite(d)] > 0)) if np.isfinite(d).any() else np.nan, "n": int(m.sum()), "n_pos": int(y.sum()), | |
| "n_dev": int(len(np.unique(dg))), "identical_layouts": identical, "max_abs_diff_all_rows": maxabs, "verdict": v, "n_boot": a.n_boot}) | |
| log(f" gold {st:8s} {met:6s}: ECAN {ve:.4f} orig {vo:.4f} Δ {ve - vo:+.4f} {ci_s(lo, hi, 4)} n {int(m.sum())}/{int(y.sum())} -> {v}") | |
| # calibration: temperature on clean-val (both), ECE / Brier on clean-test, paired Δ on identical device-cluster resamples (seed 3 = operating_points ECE seed) | |
| gt, gv = G.clean_test.to_numpy(), G.clean_val.to_numpy() | |
| fe_v, fo_v = gv & np.isfinite(ge), gv & np.isfinite(go) | |
| Te, pe_ = WG.temperature_fit(ge[fe_v], yg[fe_v].astype(float)) | |
| To, po_ = WG.temperature_fit(go[fo_v], yg[fo_v].astype(float)) | |
| m = gt & np.isfinite(ge) & np.isfinite(go) | |
| y, dg = yg[m], dong[m] | |
| P = {"raw": (np.clip(ge[m], 0, 1), np.clip(go[m], 0, 1)), "cal": (WG.apply_temperature(ge[m], Te), WG.apply_temperature(go[m], To))} | |
| for tag, (pe, po) in P.items(): | |
| for met, fn in (("ece", WG.ece), ("brier", WG.brier)): | |
| ve, vo = float(fn(y, pe)), float(fn(y, po)) | |
| b = OP.cboot(dg, lambda pk, pe=pe, po=po, fn=fn: float(fn(y[pk], pe[pk]) - fn(y[pk], po[pk])), a.n_boot, 3) | |
| lo, hi = WG.pct(b) | |
| d = ve - vo | |
| v = verdict1(d, lo, hi) | |
| v_low = {"WORSE": "BETTER", "BETTER": "WORSE"}.get(v, v) # lower ECE / Brier is better: Δ > 0 = worse | |
| rows_c.append({"arm": key, "label": label, "prereg": prereg, "seed": seed, "metric": f"{met}_{tag}", "ecan": ve, "orig": vo, "delta": d, "ci_lo": lo, "ci_hi": hi, | |
| "T_ecan": Te, "T_ecan_pinned": int(pe_), "T_orig": To, "T_orig_pinned": int(po_), "n": int(m.sum()), "n_pos": int(y.sum()), "n_dev": int(len(np.unique(dg))), | |
| "identical_layouts": identical, "verdict": v_low, "n_boot": a.n_boot}) | |
| log(f" calib {met}_{tag}: ECAN {ve:.4f} orig {vo:.4f} Δ {d:+.4f} {ci_s(lo, hi, 4)} (T {Te:.2f} / {To:.2f}) -> {v_low}") | |
| Gd, Cd = pd.DataFrame(rows_g), pd.DataFrame(rows_c) | |
| Gd.to_csv(out / "gold_pairs.csv", index=False) | |
| Cd.to_csv(out / "calibration_pairs.csv", index=False) | |
| for key, label, suf, col, prereg in ARMS: | |
| if not prereg or not len(Gd): | |
| continue | |
| for st in STRATA: | |
| for met in ("auroc", "pr_auc"): | |
| g = Gd[(Gd.arm == key) & (Gd.stratum == st) & (Gd.metric == met)].sort_values("seed") | |
| if len(g): | |
| rule, nw, nb, n = seed_rule(g.verdict.tolist()) | |
| endpoints.append({"block": "gold", "arm": key, "endpoint": f"gold {met} {st}", "seeds": n, "n_worse": nw, "n_better": nb, "rule": rule, "mean_delta": float(g.delta.mean())}) | |
| for met in ("ece_raw", "ece_cal"): | |
| g = Cd[(Cd.arm == key) & (Cd.metric == met)].sort_values("seed") | |
| if len(g): | |
| rule, nw, nb, n = seed_rule(g.verdict.tolist()) | |
| endpoints.append({"block": "calibration", "arm": key, "endpoint": f"paired {met} (pooled)", "seeds": n, "n_worse": nw, "n_better": nb, "rule": rule, "mean_delta": float(g.delta.mean())}) | |
| # ------------------------------------------------------------------ 2. timing (alert_timing_eval run with the ECAN scores) | |
| tdir = find_timing_dir(a.timing_dir) | |
| rows_t, rows_tl, t_summary = [], [], [] | |
| if tdir is None or not (tdir / "paired.csv").exists(): | |
| missing.append("alert_timing run with ecan_cur_dwfc") | |
| log("MISSING alert_timing run holding ecan_cur_dwfc") | |
| else: | |
| P, M = pd.read_csv(tdir / "paired.csv"), pd.read_csv(tdir / "metrics.csv") | |
| inputs += [tdir / "paired.csv", tdir / "metrics.csv"] | |
| log(f"timing run {tdir.name}: paired rows {len(P)}, scores {sorted(P.a.unique())}") | |
| for na, nb, prereg in TIMING_PAIRS: | |
| sub = P[(P.a == na) & (P.b == nb)] | |
| if not len(sub): | |
| log(f" timing pair {na} − {nb}: no rows") | |
| continue | |
| for (fit, fam, B, dl, reg), g in sub.groupby(["thr_fit", "family", "budget", "delta_s", "regime"]): | |
| g = g.sort_values("seed") | |
| vs = [verdict1(r.diff, r.ci_lo, r.ci_hi) for r in g.itertuples()] | |
| for r, v in zip(g.itertuples(), vs): | |
| rows_t.append({"pair": f"{na} − {nb}", "prereg": prereg, "thr_fit": fit, "family": fam, "budget": B, "delta_s": dl, "regime": reg, "seed": int(r.seed), "ecan": r.value_a, "orig": r.value_b, | |
| "delta": r.diff, "ci_lo": r.ci_lo, "ci_hi": r.ci_hi, "n_events": int(r.n_events), "verdict": v}) | |
| rule, nw, nbet, n = seed_rule(vs) | |
| t_summary.append({"pair": f"{na} − {nb}", "prereg": prereg, "thr_fit": fit, "family": fam, "budget": B, "delta_s": dl, "regime": reg, "seeds": n, "n_worse": nw, "n_better": nbet, "rule": rule, | |
| "mean_delta": float(g["diff"].mean()), "mean_ecan": float(g.value_a.mean()), "mean_orig": float(g.value_b.mean()), "n_events": int(g.n_events.iloc[0])}) | |
| if prereg and fit == "val": | |
| endpoints.append({"block": "timing", "arm": "dwfc", "endpoint": f"Recall>={dl:g}s early | family {fam} | {B:g}/h | {reg}", "seeds": n, "n_worse": nw, "n_better": nbet, "rule": rule, | |
| "mean_delta": float(g["diff"].mean())}) | |
| for sc in (na, nb): | |
| for (fit, fam, B, reg), g in M[(M.score == sc)].groupby(["thr_fit", "family", "budget", "regime"]): | |
| if "|" in str(reg): | |
| continue | |
| g = g.sort_values("seed") | |
| rows_tl.append({"pair": f"{na} − {nb}", "score": sc, "thr_fit": fit, "family": fam, "budget": B, "regime": reg, "alerts_per_h_test": g.alerts_per_h_test.tolist(), | |
| "fa_per_h_test": g.fa_per_h_test.tolist(), "recall_early_1s": g.recall_early_1s.tolist(), "detected": g.detected.tolist(), "lead0_mean": g.lead0_mean.tolist(), | |
| "lead_detected_median": g.lead_detected_median.tolist(), "n_events": int(g.n_events.iloc[0])}) | |
| Td, TL, TS = pd.DataFrame(rows_t), pd.DataFrame(rows_tl), pd.DataFrame(t_summary) | |
| Td.to_csv(out / "timing_pairs.csv", index=False) | |
| TL.to_csv(out / "timing_levels.csv", index=False) | |
| TS.to_csv(out / "timing_summary.csv", index=False) | |
| # ------------------------------------------------------------------ 3. layer 3 (operating_points --pair-runs original=...) | |
| rows_l3, l3_dirs, rows_rel = [], {}, [] | |
| for key, item_e, odir in L3_ITEMS: | |
| de = newest(f"{item_e}_2*", OP_ROOT) | |
| do = OP_ROOT / odir | |
| if de is None or not (de / "paired_public.csv").exists(): | |
| missing.append(f"operating_points item {item_e}") | |
| log(f"MISSING operating_points item {item_e}") | |
| continue | |
| l3_dirs[key] = (de.name, odir) | |
| PP = pd.read_csv(de / "paired_public.csv") | |
| inputs.append(de / "paired_public.csv") | |
| sub = PP[PP.comparator == "original"] | |
| log(f"layer 3 {item_e} ({de.name}): paired-vs-original rows {len(sub)} (loads {sorted(sub.load.unique())})") | |
| for (load, reg), g in sub.groupby(["load", "regime"]): | |
| g = g.sort_values("seed") | |
| for r in g.itertuples(): | |
| rec = {"arm": key, "item": item_e, "load": load, "regime": reg, "seed": int(r.seed), "n_common": int(r.n_common), "n_pos": int(r.n_pos), "thr_source": r.thr_source} | |
| for met, dn, lo, hi in (("recall", "d_recall", "d_recall_ci_lo", "d_recall_ci_hi"), ("precision", "d_precision", "d_precision_ci_lo", "d_precision_ci_hi"), | |
| ("far", "d_far", "d_far_ci_lo", "d_far_ci_hi"), ("lead0_mean", "d_lead0_mean", "d_lead0_ci_lo", "d_lead0_ci_hi")): | |
| d, l_, h_ = getattr(r, dn), getattr(r, lo), getattr(r, hi) | |
| v = verdict1(d, l_, h_) | |
| if met == "far": | |
| v = {"WORSE": "BETTER", "BETTER": "WORSE"}.get(v, v) # fewer false alerts = better | |
| lvl = ("lead_median_head", "lead_median_comp") if met == "lead0_mean" else (f"{met}_head", f"{met}_comp") # levels next to the paired Δ (lead levels = medians of detected) | |
| rec.update({f"{met}_head": getattr(r, lvl[0]), f"{met}_comp": getattr(r, lvl[1]), f"d_{met}": d, f"d_{met}_ci_lo": l_, f"d_{met}_ci_hi": h_, f"verdict_{met}": v}) | |
| rows_l3.append(rec) | |
| vs = [verdict1(r.d_lead0_mean, r.d_lead0_ci_lo, r.d_lead0_ci_hi) for r in g.itertuples()] | |
| rule, nw, nbet, n = seed_rule(vs) | |
| endpoints.append({"block": "lead", "arm": key, "endpoint": f"first-alert lead (lead0 mean) | load {load} | {reg}", "seeds": n, "n_worse": nw, "n_better": nbet, "rule": rule, | |
| "mean_delta": float(g.d_lead0_mean.mean())}) | |
| # reliability levels (ECE) of the two layer-3 runs | |
| for tag, d_ in (("ecan", de), ("orig", do)): | |
| rp = d_ / "reliability.csv" | |
| if rp.exists(): | |
| R = pd.read_csv(rp) | |
| R = R[(R["bin"] == -1) & (R.score == "head")] | |
| for r in R.itertuples(): | |
| rows_rel.append({"arm": key, "which": tag, "run_dir": d_.name, "seed": int(r.seed), "T": r.T, "ece_raw": r.ece_raw, "ece_cal": r.ece_cal, "ece_cal_ci_lo": r.ece_cal_ci_lo, "ece_cal_ci_hi": r.ece_cal_ci_hi, "n": int(r.n)}) | |
| L3, REL = pd.DataFrame(rows_l3), pd.DataFrame(rows_rel) | |
| L3.to_csv(out / "layer3_pairs.csv", index=False) | |
| REL.to_csv(out / "layer3_reliability_levels.csv", index=False) | |
| cal_e = newest("calibration_report_ecan_2*", OP_ROOT) | |
| cal_rows = [] | |
| for tag, d_ in (("ecan", cal_e), ("orig", OP_ROOT / "calibration_report_20260915T035901Z")): | |
| if d_ is not None and (d_ / "calibration.csv").exists() and (d_ / "paired_brier.csv").exists(): | |
| C = pd.read_csv(d_ / "calibration.csv") | |
| C = C[(C.item == "published") & (C.score == "head")] | |
| PB = pd.read_csv(d_ / "paired_brier.csv") | |
| PB = PB[(PB.item == "published") & (PB.score == "head")] | |
| for r in C.itertuples(): | |
| pb = PB[PB.seed == r.seed] | |
| cal_rows.append({"which": tag, "run_dir": d_.name, "seed": int(r.seed), "T": r.T, "ece_raw": r.ece_raw, "ece_cal": r.ece_cal, "ece_cal_ci_lo": r.ece_cal_ci_lo, "ece_cal_ci_hi": r.ece_cal_ci_hi, | |
| "brier_raw": r.brier_raw, "brier_cal": r.brier_cal, "brier_cal_ci_lo": r.brier_cal_ci_lo, "brier_cal_ci_hi": r.brier_cal_ci_hi, | |
| "paired_brier_cal_vs_att3s": float(pb.cal_delta.iloc[0]) if len(pb) else np.nan, "pb_ci_lo": float(pb.cal_ci_lo.iloc[0]) if len(pb) else np.nan, | |
| "pb_ci_hi": float(pb.cal_ci_hi.iloc[0]) if len(pb) else np.nan}) | |
| inputs += [d_ / "calibration.csv", d_ / "paired_brier.csv"] | |
| else: | |
| missing.append(f"calibration_report {tag}") | |
| CAL = pd.DataFrame(cal_rows) | |
| CAL.to_csv(out / "calibration_report_levels.csv", index=False) | |
| E = pd.DataFrame(endpoints) | |
| E.to_csv(out / "endpoints.csv", index=False) | |
| if missing and not a.allow_missing: | |
| raise SystemExit("MISSING inputs (use --allow-missing to render a partial table): " + "; ".join(missing)) | |
| # ------------------------------------------------------------------ verdict | |
| n_end = len(E) | |
| worse = E[E.rule.str.startswith("SIGNIFICANTLY WORSE")] if n_end else E | |
| n_ident = int(E.rule.str.startswith("identical").sum()) if n_end else 0 | |
| n_better = int(E.rule.str.startswith("better").sum()) if n_end else 0 | |
| decision = ("MOVE: no pre-registered endpoint is significantly worse (>= 2/3 seeds) -> the warning section moves to the ECAN configuration; both read-outs reported" | |
| if n_end and len(worse) == 0 and not missing else | |
| ("STAY: at least one pre-registered endpoint is significantly worse -> the warning section stays on the original configuration; the ECAN read-out is reported in Supp as a negative / neutral result" | |
| if len(worse) else "INCOMPLETE (missing inputs) — no decision")) | |
| log(f"VERDICT: {n_end} pre-registered endpoints; identical {n_ident}; significantly worse {len(worse)}; better {n_better}; missing inputs {len(missing)} -> {decision}") | |
| # ------------------------------------------------------------------ section | |
| ts = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) | |
| L = [f"## {ts} — ECAN warning read-out vs the ORIGINAL read-out, PAIRED (eval/warning_ecan_vs_original.py; pre-registration {PREREG}; run {stamp})", "", | |
| f"Every Δ = ECAN − ORIGINAL per seed on identical rows and device-cluster resamples. ECAN heads: the SAME `train/train_warning_head.py` argv fitted on `{ECAN}` " | |
| f"(layouts `W_{ECAN}<suffix>`); original heads: the frozen paper read-outs `W_{ORIG}<suffix>`. Gold = `warning_gold.build_gold` legacy_first anchors (clean-test / clean-val), " | |
| f"`warning_gold.cluster_boot` n_boot {a.n_boot} seed 0 (AUROC / PR-AUC), `operating_points.cboot` seed 3 (ECE / Brier). Timing cells from `results/alert_timing/{tdir.name if tdir else '—'}` " | |
| f"(alert_timing_eval.py, {', '.join(f'{k} <- {v[0]}' for k, v in l3_dirs.items()) if l3_dirs else 'layer 3: —'}); layer-3 pairs vs the original at its PUBLISHED thresholds " | |
| f"({', '.join(v[1] for v in l3_dirs.values())}). Reading rule (pre-registered): an endpoint is significantly worse when the paired CI excludes 0 with Δ < 0 on >= ceil(2n/3) seeds; " | |
| "the section moves to ECAN iff no pre-registered endpoint is significantly worse; better endpoints do not enter the decision. Every number below is a CSV cell of the run dir.", "", | |
| "### 0. Layout identity (max |Δ score| over EVERY test / val row of both regimes; the current-state heads read no predicted quantity, decisions row 2026-09-12)", "", | |
| "| arm | column | seed | rows | max abs Δ | NaN-pattern mismatches | identical |", "|---|---|---|---|---|---|---|"] | |
| for key, label, suf, col, prereg in ARMS: | |
| for seed in a.seeds: | |
| if (key, seed) in ident: | |
| i = ident[(key, seed)] | |
| L.append(f"| {label} | `{col}` | {seed} | {i['n_rows']} | {i['max_abs_diff']:.3e} | {i['nan_mismatch_rows']} | {'YES' if i['identical'] else 'no'} |") | |
| L += ["", "### 1. Gold clean-test AUROC / PR-AUC (pooled / manual / engaged), paired Δ ECAN − original", "", | |
| "| arm | metric | stratum | ECAN s0 / s1 / s2 | original s0 / s1 / s2 | Δ s0 [CI] | Δ s1 [CI] | Δ s2 [CI] | n / pos | rule |", "|---|---|---|---|---|---|---|---|---|---|"] | |
| for key, label, suf, col, prereg in ARMS: | |
| for met in ("auroc", "pr_auc"): | |
| for st in STRATA: | |
| g = Gd[(Gd.arm == key) & (Gd.metric == met) & (Gd.stratum == st)].sort_values("seed") if len(Gd) else Gd | |
| if not len(g): | |
| continue | |
| rule = seed_rule(g.verdict.tolist())[0] if prereg else "descriptive: " + seed_rule(g.verdict.tolist())[0] | |
| L.append(f"| {label} | {met} | {st} | {' / '.join(f3(x) for x in g.ecan)} | {' / '.join(f3(x) for x in g.orig)} | " | |
| + " | ".join(f"{f3(r.delta, 3, True)} {ci_s(r.ci_lo, r.ci_hi)}" for r in g.itertuples()) + " | " * (3 - len(g)) + f" {int(g.n.iloc[0])} / {int(g.n_pos.iloc[0])} | {rule} |") | |
| L += ["", "### 2. Event-level timing — Recall_{>= δ s early} at K = 10 / 20 / 40 per h, VAL-fitted thresholds (PRIMARY); pair `ecan_cur_dwfc − cur_dwfc` (cur+dwell+face on both models)", ""] | |
| if len(TS): | |
| A = TS[(TS.pair == "ecan_cur_dwfc − cur_dwfc") & (TS.thr_fit == "val") & (TS.family == "A")] | |
| L += ["**Family A (registered, primary) — every endpoint:**", "", "| K/h | δ s | regime | ECAN mean | orig mean | Δ s0 [CI] | Δ s1 [CI] | Δ s2 [CI] | events | rule |", "|---|---|---|---|---|---|---|---|---|---|"] | |
| for r in A.sort_values(["budget", "delta_s", "regime"]).itertuples(): | |
| g = Td[(Td.pair == r.pair) & (Td.thr_fit == "val") & (Td.family == "A") & (Td.budget == r.budget) & (Td.delta_s == r.delta_s) & (Td.regime == r.regime)].sort_values("seed") | |
| L.append(f"| {r.budget:g} | {r.delta_s:g} | {r.regime} | {f3(r.mean_ecan)} | {f3(r.mean_orig)} | " + " | ".join(f"{f3(x.delta, 3, True)} {ci_s(x.ci_lo, x.ci_hi)}" for x in g.itertuples()) | |
| + " | " * (3 - len(g)) + f" {r.n_events} | {r.rule} |") | |
| L += ["", "**All families, VAL-fitted (primary) and TEST-fitted (secondary, equal achieved test load) — endpoint counts per family × K (9 endpoints each = 3 δ × 3 regimes):**", "", | |
| "| pair | thr fit | family | K/h | endpoints | identical | significantly worse | better | not worse (n.s.) | mean Δ over endpoints |", "|---|---|---|---|---|---|---|---|---|---|"] | |
| for (pair, fit, fam, B), g in TS.groupby(["pair", "thr_fit", "family", "budget"], sort=False): | |
| ni = int(g.rule.str.startswith("identical").sum()); nw = int(g.rule.str.startswith("SIGNIFICANTLY").sum()); nb = int(g.rule.str.startswith("better").sum()) | |
| L.append(f"| {pair} | {fit} | {fam} | {B:g} | {len(g)} | {ni} | {nw} | {nb} | {len(g) - ni - nw - nb} | {f3(g.mean_delta.mean(), 4, True)} |") | |
| W_ = TS[TS.rule.str.startswith("SIGNIFICANTLY")] | |
| L += ["", f"Significantly worse timing endpoints (all pairs / fits): {len(W_)}" + ("" if not len(W_) else " — " + "; ".join(f"{r.pair} {r.thr_fit} {r.family} {r.budget:g}/h δ{r.delta_s:g} {r.regime} (Δ {f3(r.mean_delta, 3, True)})" for r in W_.itertuples()))] | |
| if len(TL): | |
| L += ["", "**Alert load and first-alert lead levels at the VAL-fitted thresholds (per seed s0 / s1 / s2; pooled; family A):**", "", | |
| "| score | K/h | alerts/h test | FA/h test | Recall >= 1 s early | detected | lead0 mean s (undetected = 0) | lead median s (detected) | events |", "|---|---|---|---|---|---|---|---|---|"] | |
| for r in TL[(TL.thr_fit == "val") & (TL.family == "A") & (TL.regime == "pooled") & (TL.score.isin(["cur_dwfc", "ecan_cur_dwfc", "cur_base", "ecan_cur", "both_base", "ecan_both"]))].drop_duplicates(["score", "budget"]).sort_values(["budget", "score"]).itertuples(): | |
| j = lambda xs, nd=3: " / ".join(f3(x, nd) for x in xs) # noqa: E731 | |
| L.append(f"| {r.score} | {r.budget:g} | {j(r.alerts_per_h_test, 1)} | {j(r.fa_per_h_test, 1)} | {j(r.recall_early_1s)} | {j(r.detected)} | {j(r.lead0_mean, 2)} | {j(r.lead_detected_median, 2)} | {r.n_events} |") | |
| else: | |
| L.append("timing run not available") | |
| L += ["", "### 3. Layer 3 — paired cells vs the ORIGINAL read-out at its published per-seed thresholds (operating_points.py --pair-runs original=...); first-alert lead = pre-registered endpoint (iii), the rest descriptive", ""] | |
| if len(L3): | |
| L += ["| arm | load | regime | Δ recall s0 / s1 / s2 [CI] | Δ precision (mean) | Δ FAR (mean) | Δ lead0 mean s s0 / s1 / s2 [CI] | lead rule | n common / pos |", "|---|---|---|---|---|---|---|---|---|"] | |
| for key, item_e, odir in L3_ITEMS: | |
| for load in LOAD_ORDER: | |
| for reg in ("pooled", "manual", "engaged"): | |
| g = L3[(L3.arm == key) & (L3.load == load) & (L3.regime == reg)].sort_values("seed") | |
| if not len(g): | |
| continue | |
| rule = seed_rule([verdict1(r.d_lead0_mean, r.d_lead0_mean_ci_lo, r.d_lead0_mean_ci_hi) for r in g.itertuples()])[0] | |
| L.append(f"| {key} | {load} | {reg} | " + "; ".join(f"{f3(r.d_recall, 3, True)} {ci_s(r.d_recall_ci_lo, r.d_recall_ci_hi)}" for r in g.itertuples()) | |
| + f" | {f3(g.d_precision.mean(), 3, True)} | {f3(g.d_far.mean(), 3, True)} | " + "; ".join(f"{f3(r.d_lead0_mean, 2, True)} {ci_s(r.d_lead0_mean_ci_lo, r.d_lead0_mean_ci_hi, 2)}" for r in g.itertuples()) | |
| + f" | {rule} | {int(g.n_common.iloc[0])} / {int(g.n_pos.iloc[0])} |") | |
| else: | |
| L.append("layer-3 runs not available") | |
| L += ["", "### 4. Calibration — both heads temperature-calibrated on clean-val; ECE (15 equal-width bins) and Brier on clean-test; paired Δ on identical resamples (lower is better; verdict already oriented)", ""] | |
| if len(Cd): | |
| L += ["| arm | metric | ECAN s0 / s1 / s2 | original s0 / s1 / s2 | Δ s0 [CI] | Δ s1 [CI] | Δ s2 [CI] | T ECAN / orig (s0) | rule |", "|---|---|---|---|---|---|---|---|---|"] | |
| for key, label, suf, col, prereg in ARMS: | |
| for met in ("ece_raw", "ece_cal", "brier_raw", "brier_cal"): | |
| g = Cd[(Cd.arm == key) & (Cd.metric == met)].sort_values("seed") | |
| if not len(g): | |
| continue | |
| rule = seed_rule(g.verdict.tolist())[0] | |
| L.append(f"| {label} | {met} | {' / '.join(f3(x) for x in g.ecan)} | {' / '.join(f3(x) for x in g.orig)} | " + " | ".join(f"{f3(r.delta, 4, True)} {ci_s(r.ci_lo, r.ci_hi, 4)}" for r in g.itertuples()) | |
| + " | " * (3 - len(g)) + f" {f3(g.T_ecan.iloc[0], 2)} / {f3(g.T_orig.iloc[0], 2)} | {rule if prereg and met.startswith('ece') else 'descriptive: ' + rule} |") | |
| if len(CAL): | |
| L += ["", "Published-protocol levels from `eval/calibration_report.py` (n_boot 2000): " + "; ".join( | |
| f"{r.which} s{r.seed} ({r.run_dir}): T {f3(r.T, 2)}, ECE {f3(r.ece_raw)} -> {f3(r.ece_cal)} {ci_s(r.ece_cal_ci_lo, r.ece_cal_ci_hi)}, Brier {f3(r.brier_raw)} -> {f3(r.brier_cal)} {ci_s(r.brier_cal_ci_lo, r.brier_cal_ci_hi)}, " | |
| f"paired Brier vs att@3s {f3(r.paired_brier_cal_vs_att3s, 3, True)} {ci_s(r.pb_ci_lo, r.pb_ci_hi)}" for r in CAL.sort_values(['which', 'seed']).itertuples())] | |
| if len(REL): | |
| L += ["", "Layer-3 reliability levels (operating_points.py, 1,000 resamples): " + "; ".join(f"{r.arm} {r.which} s{r.seed}: T {f3(r.T, 2)} ECE {f3(r.ece_raw)} -> {f3(r.ece_cal)} {ci_s(r.ece_cal_ci_lo, r.ece_cal_ci_hi)}" for r in REL.sort_values(['arm', 'which', 'seed']).itertuples())] | |
| L += ["", "### 5. VERDICT (pre-registered rule)", "", | |
| f"Pre-registered endpoints evaluated: **{n_end}** ({int((E.block == 'gold').sum()) if n_end else 0} gold, {int((E.block == 'timing').sum()) if n_end else 0} timing, {int((E.block == 'lead').sum()) if n_end else 0} first-alert lead, " | |
| f"{int((E.block == 'calibration').sum()) if n_end else 0} calibration); identical (Δ = 0 exactly): {n_ident}; significantly worse on >= 2/3 seeds: **{len(worse)}**; better on >= 2/3 seeds: {n_better} (not decision-relevant); missing inputs: {len(missing)}.", ""] | |
| if len(worse): | |
| L += ["Significantly worse endpoints: " + "; ".join(f"{r.endpoint} [{r.arm}] (mean Δ {f3(r.mean_delta, 3, True)}, {r.rule})" for r in worse.itertuples()), ""] | |
| if missing: | |
| L += ["Missing inputs: " + "; ".join(missing), ""] | |
| L += [f"**DECISION: {decision}**", ""] | |
| md = "\n".join(L) + "\n" | |
| (out / "section.md").write_text(md) | |
| if not a.no_table: | |
| if not TABLE.exists(): | |
| TABLE.write_text("# ECAN warning read-out vs the original read-out — paired comparison (append-only; eval/warning_ecan_vs_original.py)\n\n") | |
| with open(TABLE, "a") as f: | |
| f.write("\n" + md) | |
| print(md) | |
| CR.OP.write_run_json(out, "warning_ecan_vs_original", [Path(__file__), Path(CR.__file__), Path(OP.__file__), Path(WG.__file__)], sorted(set(inputs)), seed=0, t_start=t0, input_listing=False, | |
| extra={"prereg": PREREG, "seeds": a.seeds, "n_boot": a.n_boot, "timing_dir": str(tdir) if tdir else "", "layer3_dirs": l3_dirs, "calibration_ecan_dir": str(cal_e) if cal_e else "", | |
| "identity": {f"{k[0]}|s{k[1]}": v for k, v in ident.items()}, "n_endpoints": int(n_end), "n_identical": n_ident, "n_sig_worse": int(len(worse)), "n_better": n_better, | |
| "sig_worse": worse.endpoint.tolist() if len(worse) else [], "missing": missing, "decision": decision, "dms_fields_used": []}) | |
| log(f"DONE in {time.time() - t0:.0f}s -> {out}; section {'appended to ' + str(TABLE) if not a.no_table else 'NOT appended'}") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 37.9 kB
- Xet hash:
- 68c56126fbf8f948585556daf11d23af4eb8f4a084e292f8bdb0a213fffaeb8f
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.