Buckets:
| """cwm3/eval/warning_head_regime_report.py — per-regime strata report + between-column paired Δ of the learned warning heads scored in ONE warning_gold item | |
| (roadmap §B3 items 1a / 1b / 3; 2026-09-12). Reads only published rows: results/warning_gold/warning_gold.csv (the item's latest run_stamp; strata clean_test, | |
| clean_test|manual, clean_test|engaged incl. the paired rows vs the DMS att@3s comparator) and the per-episode scores CSV warn_gold_rows_<stamp>.csv of that run | |
| (for the between-column Δ on identical device-cluster resamples, warning_gold.cluster_boot — the same estimator the evaluator uses). | |
| Learned-head arms: the base head `cwm3_W` and the extended heads `abl_W_<tag>` (train_warning_head.py writes one arm per extended layout dir because | |
| warning_gold.py keys layouts by (arm, regime, seed, split)). Columns may be given as `arm::name` or as the bare `name` (resolved among the W arms). | |
| python eval/warning_head_regime_report.py --item cwm3_final_ext --cols W_learned_curhgb_t15_w10_n979 W_learned_curhgb_t15_w10_n979dw ... \ | |
| --pairs "dwell:W_learned_curhgb_t15_w10_n979dw-W_learned_curhgb_t15_w10_n979" [--ensemble-cols ...] [--out results/tables/warning_head_final_ext.md] | |
| Sections appended to --out (append-only): | |
| 1. clean-test AUROC per stratum for the requested learned columns (per seed + mean) and the DMS att@3s / W_direct rows of the same item | |
| 2. paired Δ vs DMS att@3s per stratum (warning_gold pairs; kill_checks.beats over the seeds) | |
| 3. between-column paired Δ per stratum for the requested pairs (rows CSV; identical resamples); kill rule = CI excl. 0 in >= 2/3 seeds in engaged OR pooled | |
| (as pre-registered for the dwell / face effects) -> verdict per pair. Current-state heads are seed-invariant by construction (the joint model's seed does | |
| not enter current-state features), which the report states when the three seeds coincide. | |
| 4. 3-seed score ensemble (mean of the per-episode scores over seeds) of --ensemble-cols: AUROC per stratum with CI + Δ vs the seed-0 column. | |
| No openpilot DMS field is read (the DMS chain enters only as the labelled comparator rows already in warning_gold.csv). | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import sys | |
| import time | |
| from pathlib import Path | |
| import numpy as np | |
| import pandas as pd | |
| CWM3 = Path(__file__).resolve().parents[1] | |
| EXPLORE = CWM3.parent | |
| CWM2 = EXPLORE / "cwm2" | |
| for p in (CWM2 / "eval", CWM3 / "common"): | |
| sys.path.insert(0, str(p)) | |
| import kill_checks as KC # noqa: E402 | |
| import warning_gold as WG # noqa: E402 | |
| from runmeta import write_run_json # noqa: E402 | |
| OUT_DIR = CWM3 / "results" / "warning_gold" | |
| STRATA = ("clean_test", "clean_test|manual", "clean_test|engaged") | |
| def fmt(x, nd=3): | |
| try: | |
| x = float(x) | |
| return "—" if not np.isfinite(x) else f"{x:.{nd}f}" | |
| except (TypeError, ValueError): | |
| return "—" | |
| def is_w_arm(arm): | |
| return str(arm) == "cwm3_W" or str(arm).startswith("abl_W") | |
| def stratum_mask(R, stratum): | |
| m = R.clean_test.astype(bool).to_numpy() | |
| if "|" in stratum: | |
| m &= (R.regime == stratum.split("|")[1]).to_numpy() | |
| return m | |
| def resolve(col, w_pairs, rows_cols): | |
| """'arm::name' or bare 'name' -> (arm, name, rows-CSV column).""" | |
| if "::" in col: | |
| arm, name = col.split("::", 1) | |
| else: | |
| hits = [(a_, n_) for a_, n_ in w_pairs if n_ == col] | |
| assert len(hits) == 1, f"column {col}: {len(hits)} matches among the W arms {sorted({a_ for a_, _ in w_pairs})}" | |
| arm, name = hits[0] | |
| key = f"{arm}::{name}" | |
| return arm, name, (key if key in rows_cols else None) | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--item", default="cwm3_final_ext") | |
| ap.add_argument("--stamp", default="", help="warning_gold run_stamp inside the item (default: latest)") | |
| ap.add_argument("--cols", nargs="*", default=[], help="learned columns to report (default: every *hgb* / *sel* column of the W arms)") | |
| ap.add_argument("--pairs", nargs="*", default=[], help="name:colA-colB (Δ = A − B on identical rows / resamples)") | |
| ap.add_argument("--ensemble-cols", nargs="*", default=[]) | |
| ap.add_argument("--n-boot", type=int, default=2000) | |
| ap.add_argument("--out", default=str(CWM3 / "results" / "tables" / "warning_head_final_ext.md")) | |
| ap.add_argument("--title", default="") | |
| a = ap.parse_args() | |
| t0 = time.time() | |
| W = pd.read_csv(OUT_DIR / "warning_gold.csv", low_memory=False) | |
| W = W[W.item == a.item] | |
| assert len(W), f"no rows for item {a.item}" | |
| stamp = a.stamp or W.run_stamp.max() | |
| W = W[W.run_stamp == stamp] | |
| seeds = sorted(int(s) for s in W.seed.unique() if str(s).isdigit()) | |
| rows_csv = OUT_DIR / f"warn_gold_rows_{stamp}.csv" | |
| R = pd.read_csv(rows_csv, low_memory=False) | |
| base = W[(W.metric == "auroc") & (W.calib == "raw") & (W.comparator.isna())] | |
| w_pairs = sorted({(arm, ev) for arm, ev in zip(base.arm, base.event_or_family) if is_w_arm(arm)}) | |
| cols = a.cols or [f"{arm}::{ev}" for arm, ev in w_pairs if ("hgb" in ev or "sel" in ev)] | |
| res_cols = [resolve(c, w_pairs, set(R.columns)) for c in cols] | |
| L = [f"## {a.title or 'Per-regime strata of the extended warning heads'} — item `{a.item}` (warning_gold run {stamp}; eval/warning_head_regime_report.py)", "", | |
| f"Seeds {seeds}; every AUROC / CI / paired-Δ-vs-DMS number is read from `results/warning_gold/warning_gold.csv`; between-column Δ recomputed from the per-episode scores " | |
| f"`{rows_csv.name}` with `warning_gold.cluster_boot` ({a.n_boot} device-cluster resamples, identical for A and B). Strata: clean_test (pooled) / |manual / |engaged. " | |
| f"Learned arms in the item: {sorted({arm for arm, _ in w_pairs})}.", ""] | |
| # ---- 1. AUROC per stratum | |
| L += ["### 1. Clean-test AUROC per stratum (raw; per seed, mean)", "", "| arm::column | " + " | ".join(f"{st.split('|')[-1] if '|' in st else 'pooled'} " + " ".join(f"s{s}" for s in seeds) + " | mean" for st in STRATA) + " | n / pos (pooled, manual, engaged) |", | |
| "|---|" + "---|---|" * len(STRATA) + "---|"] | |
| ref_rows = [("legacy_att", "att@3s"), ("cwm3_I", "W_direct_t15_w10_RV_6fam")] | |
| for arm, ev in [(arm, ev) for arm, ev, _ in res_cols] + ref_rows: | |
| cells, ns = [], [] | |
| for st in STRATA: | |
| g = base[(base.arm == arm) & (base.event_or_family == ev) & (base.stratum == st)] | |
| vals = {int(s): float(v) for s, v in zip(g.seed, g.value) if str(s).isdigit()} | |
| v = np.array([vals[s] for s in seeds if s in vals]) | |
| cells.append(" ".join(fmt(vals[s]) if s in vals else "—" for s in seeds) + f" | {fmt(v.mean()) if len(v) else '—'}") | |
| ns.append(f"{int(g.n.iloc[0])}/{int(g.n_pos.iloc[0])}" if len(g) else "—") | |
| L.append(f"| {arm}::{ev} | " + " | ".join(cells) + " | " + ", ".join(ns) + " |") | |
| # ---- 2. paired Δ vs DMS att@3s per stratum | |
| L += ["", "### 2. Paired Δ vs the DMS attention chain (legacy_att, att@3s) per stratum — warning_gold pairs, kill_checks.beats over the seeds", "", | |
| "| arm::column | stratum | " + " | ".join(f"s{s} Δ [CI]" for s in seeds) + " | mean Δ | verdict |", "|---|---|" + "---|" * len(seeds) + "---|---|"] | |
| verd_dms = {} | |
| for arm, ev, _ in res_cols: | |
| for st in STRATA: | |
| pr = KC.paired(W, arm, "legacy_att", "gold_W", ev, metric="auroc", stratum=st) | |
| pr = pr[pr.seed.astype(str).str.isdigit()] | |
| if not len(pr): | |
| continue | |
| va = KC.values(W, arm, "gold_W", metric="auroc", stratum=st, score=ev) | |
| vb = W[(W.arm == "legacy_att") & (W.stratum == st) & (W.metric == "auroc") & (W.calib == "raw") & (W.comparator.isna())] | |
| res = KC.beats(pr, va[va.seed.astype(str).str.isdigit()], vb[vb.seed.astype(str).str.isdigit()], min_seeds=3) | |
| verd_dms[(f"{arm}::{ev}", st)] = res | |
| cells = [] | |
| for s in seeds: | |
| x = pr[pr.seed.astype(int) == s] | |
| cells.append(f"{float(x.delta.iloc[0]):+.3f} [{float(x.delta_ci_lo.iloc[0]):+.3f},{float(x.delta_ci_hi.iloc[0]):+.3f}]" if len(x) else "—") | |
| L.append(f"| {arm}::{ev} | {st} | " + " | ".join(cells) + f" | {res.get('mean_delta', np.nan):+.3f} | {res['verdict']} ({res.get('ci_excl_zero', '?')}/{len(seeds)} CI excl. 0) |") | |
| # ---- 3. between-column paired Δ | |
| pair_rows, pair_verdicts = [], {} | |
| if a.pairs: | |
| L += ["", "### 3. Between-column paired Δ per stratum (identical rows and device-cluster resamples); pre-registered kill rule: CI excl. 0 in >= 2/3 seeds in engaged OR pooled", "", | |
| "| effect | A − B | stratum | " + " | ".join(f"s{s} Δ [CI]" for s in seeds) + " | mean Δ | seeds CI excl. 0 |", "|---|---|---|" + "---|" * len(seeds) + "---|---|"] | |
| for spec in a.pairs: | |
| name, ab = spec.split(":", 1) | |
| ca, cb = ab.split("-", 1) | |
| try: | |
| _, na, ka = resolve(ca, w_pairs, set(R.columns)) | |
| _, nb, kb = resolve(cb, w_pairs, set(R.columns)) | |
| except AssertionError as ex: | |
| L.append(f"| {name} | {ca} − {cb} | — | {ex} |") | |
| continue | |
| if ka is None or kb is None: | |
| L.append(f"| {name} | {ca} − {cb} | — | missing column(s) in the rows CSV |") | |
| continue | |
| excl_by_st = {} | |
| for st in STRATA: | |
| cells, deltas, los, his = [], [], [], [] | |
| for s in seeds: | |
| D = R[(R.seed == s) & stratum_mask(R, st) & R[ka].notna() & R[kb].notna()] | |
| if len(D) < 20 or D.y.nunique() < 2: | |
| cells.append("—") | |
| continue | |
| y = D.y.to_numpy().astype(int) | |
| bs = WG.cluster_boot(y, {"a": D[ka].to_numpy(float), "b": D[kb].to_numpy(float)}, D.dongle.to_numpy(), a.n_boot, 0) | |
| d = bs["a"]["auroc"] - bs["b"]["auroc"] | |
| lo, hi = WG.pct(d) | |
| dv = WG.fast_auroc(y, D[ka].to_numpy(float)) - WG.fast_auroc(y, D[kb].to_numpy(float)) | |
| deltas.append(dv); los.append(lo); his.append(hi) | |
| cells.append(f"{dv:+.3f} [{lo:+.3f},{hi:+.3f}]") | |
| pair_rows.append(dict(effect=name, col_a=ka, col_b=kb, stratum=st, seed=s, n=int(len(D)), n_pos=int(y.sum()), delta=dv, delta_ci_lo=lo, delta_ci_hi=hi, | |
| auroc_a=WG.fast_auroc(y, D[ka].to_numpy(float)), auroc_b=WG.fast_auroc(y, D[kb].to_numpy(float)), frac_boot_d_gt0=float(np.mean(d[np.isfinite(d)] > 0)))) | |
| excl = int(sum((lo > 0) or (hi < 0) for lo, hi in zip(los, his))) | |
| excl_by_st[st] = (excl, len(deltas), float(np.mean(deltas)) if deltas else np.nan) | |
| L.append(f"| {name} | {na} − {nb} | {st} | " + " | ".join(cells) + f" | {np.mean(deltas) if deltas else np.nan:+.3f} | {excl}/{len(deltas)} |") | |
| ok = any(excl_by_st.get(st, (0, 0, 0))[0] >= 2 and excl_by_st.get(st, (0, 0, 0))[1] >= 3 and excl_by_st[st][2] > 0 for st in ("clean_test", "clean_test|engaged")) | |
| seed_inv = all(len({round(r["delta"], 6) for r in pair_rows if r["effect"] == name and r["stratum"] == st}) <= 1 for st in STRATA if any(r["effect"] == name and r["stratum"] == st for r in pair_rows)) | |
| pair_verdicts[name] = {"effect": ok, "seed_invariant": seed_inv, "by_stratum": {k: {"n_excl": v[0], "n_seeds": v[1], "mean_delta": v[2]} for k, v in excl_by_st.items()}} | |
| L.append(f"| **{name} verdict** | | | " + " | ".join([""] * len(seeds)) + f" | | **{'EFFECT (kill rule met)' if ok else 'no effect by the pre-registered rule'}**" | |
| + ("; the three seeds coincide — current-state heads are seed-invariant, so the seed rule is one paired test" if seed_inv else "") + " |") | |
| # ---- 4. 3-seed ensembles | |
| ens_rows = [] | |
| if a.ensemble_cols: | |
| L += ["", "### 4. Three-seed score ensemble (mean of the per-episode scores over the seeds) — one row per column; Δ vs the seed-0 single head", "", | |
| "| column | stratum | ensemble AUROC [CI] | seed-0 AUROC | Δ ens − s0 [CI] | n / pos |", "|---|---|---|---|---|---|"] | |
| for c in a.ensemble_cols: | |
| try: | |
| _, nm, k = resolve(c, w_pairs, set(R.columns)) | |
| except AssertionError as ex: | |
| L.append(f"| {c} | — | {ex} | | | |") | |
| continue | |
| if k is None: | |
| L.append(f"| {c} | — | missing column | | | |") | |
| continue | |
| piv = R.pivot_table(index="idx", columns="seed", values=k, aggfunc="first") | |
| meta = R[R.seed == seeds[0]].set_index("idx") | |
| ens = piv.mean(1) | |
| for st in STRATA: | |
| m = stratum_mask(meta, st) | |
| idx = meta.index[m] | |
| e = ens.reindex(idx).to_numpy(float) | |
| s0 = meta.loc[idx, k].to_numpy(float) | |
| y = meta.loc[idx, "y"].to_numpy().astype(int) | |
| fin = np.isfinite(e) & np.isfinite(s0) | |
| if fin.sum() < 20 or y[fin].min() == y[fin].max(): | |
| continue | |
| bs = WG.cluster_boot(y[fin], {"e": e[fin], "s": s0[fin]}, meta.loc[idx, "dongle"].to_numpy()[fin], a.n_boot, 0) | |
| d = bs["e"]["auroc"] - bs["s"]["auroc"] | |
| ae, as0 = WG.fast_auroc(y[fin], e[fin]), WG.fast_auroc(y[fin], s0[fin]) | |
| ens_rows.append(dict(column=k, stratum=st, auroc_ens=ae, ci_lo=WG.pct(bs["e"]["auroc"])[0], ci_hi=WG.pct(bs["e"]["auroc"])[1], auroc_s0=as0, delta=ae - as0, delta_ci_lo=WG.pct(d)[0], delta_ci_hi=WG.pct(d)[1], n=int(fin.sum()), n_pos=int(y[fin].sum()))) | |
| r = ens_rows[-1] | |
| L.append(f"| {k} | {st} | {fmt(r['auroc_ens'])} [{fmt(r['ci_lo'], 2)},{fmt(r['ci_hi'], 2)}] | {fmt(r['auroc_s0'])} | {r['delta']:+.3f} [{r['delta_ci_lo']:+.3f},{r['delta_ci_hi']:+.3f}] | {r['n']} / {r['n_pos']} |") | |
| L.append("") | |
| md = "\n".join(L) + "\n" | |
| out = Path(a.out) | |
| if not out.exists(): | |
| out.write_text("# Extended warning heads on the final model — per-regime strata and paired effects (append-only; eval/warning_head_regime_report.py + eval/warning_head_eval.py)\n\n") | |
| with open(out, "a") as f: | |
| f.write(md) | |
| print(md) | |
| rep_dir = CWM3 / "results" / "warning_head_ext" | |
| rep_dir.mkdir(parents=True, exist_ok=True) | |
| tag = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime()) | |
| pd.DataFrame(pair_rows).to_csv(rep_dir / f"regime_pairs_{a.item}_{tag}.csv", index=False) | |
| pd.DataFrame(ens_rows).to_csv(rep_dir / f"regime_ensemble_{a.item}_{tag}.csv", index=False) | |
| write_run_json(rep_dir, "warning_head_regime_report", [Path(__file__), Path(WG.__file__)], [OUT_DIR / "warning_gold.csv", rows_csv], seed=0, t_start=t0, input_listing=False, | |
| extra={"item": a.item, "stamp": stamp, "cols": cols, "pairs": a.pairs, "verdicts_vs_dms": {f"{k[0]}|{k[1]}": v for k, v in verd_dms.items()}, "pair_verdicts": pair_verdicts, | |
| "pair_rows": pair_rows, "ensemble_rows": ens_rows, "dms_fields_used": []}) | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 15.3 kB
- Xet hash:
- bee4c4d988dd698d3e6190392de0de7869366bb499bbe2717daf90cdc1e1bd76
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.