Buckets:
| """cwm3/eval/report_warning_trunk_heads.py — EXPERIMENT B report (owner question 2026-09-18; pre-registration docs/decisions.md 2026-09-18T03:34:28Z): a Driver-WM-style | |
| 'warning head on the trunk' — a small MLP (41,153 params) trained on the DETACHED trunk features of (i) CockpitWM w/ Driver-WM-style Transformer U | |
| (I1_all_minus_road2drv_s<k>_final_udwm) and (ii) CockpitWM (ECAN, default), with the warranted-warning labels of the HGB read-out (train/train_trunk_warning_head.py). | |
| Reads ONLY script outputs (+ the paired gold bootstrap on identical resamples). Append-only outputs: results/tables/warning_trunk_heads.md + .json. | |
| Blocks: (1) head selection (CV AUROC, params) from results/warning_head/TRK_<run>_<stamp>/selected.json; (2) gold clean-test AUROC / PR-AUC pooled / manual / engaged | |
| per seed and paired Δ: (i) − (ii) PRIMARY, (i) − paper HGB current-state read-out, (ii) − paper; LR linear probes descriptive; (3) timing Recall_{>= δ s early} @ 10 / 20 / 40 | |
| per h from the alert_timing_eval.py run holding trk_udwm / trk_ecan (VAL-fitted primary). Reading rule as in Experiment A; NO adoption, no manuscript change. | |
| python eval/report_warning_trunk_heads.py [--seeds 0 1 2] [--n-boot 2000] [--timing-dir DIR] [--allow-missing] | |
| """ | |
| from __future__ import annotations | |
| import os | |
| os.environ.setdefault("OMP_NUM_THREADS", "4") | |
| import argparse # noqa: E402 | |
| import json # noqa: E402 | |
| import sys # noqa: E402 | |
| import time # noqa: E402 | |
| from pathlib import Path # noqa: E402 | |
| import pandas as pd # noqa: E402 | |
| CWM3 = Path(__file__).resolve().parents[1] | |
| sys.path.insert(0, str(CWM3 / "eval")) | |
| import alert_pair_common as C # noqa: E402 | |
| f3, ci_s = C.f3, C.ci_s | |
| PREREG = "2026-09-18T03:34:28Z" | |
| OUT_ROOT = CWM3 / "results" / "warning_trunk_heads" | |
| TABLE, TJSON = CWM3 / "results" / "tables" / "warning_trunk_heads.md", CWM3 / "results" / "tables" / "warning_trunk_heads.json" | |
| UDWM, ECAN, MAIN = "I1_all_minus_road2drv_s{seed}_final_udwm", "I1_all_minus_road2drv_s{seed}_final_ecan", "I1_all_minus_road2drv_s{seed}_final" | |
| MLP, LR = "W_trunk_mlp64_t15_w10_n979", "W_trunk_lr_t15_w10_n979" | |
| SPECS = {"trk_udwm": (f"W_TRK_{UDWM}", MLP), "trk_ecan": (f"W_TRK_{ECAN}", MLP), "paper": (f"W_{MAIN}_dwfc", "W_learned_curhgb_t15_w10_n979dwfc"), | |
| "paper_cur": (f"W_{MAIN}", "W_learned_curhgb_t15_w10_n979"), "trk_udwm_lr": (f"W_TRK_{UDWM}", LR), "trk_ecan_lr": (f"W_TRK_{ECAN}", LR)} | |
| LABELS = {"trk_udwm": "(i) trunk MLP head on CockpitWM w/ Driver-WM-style Transformer U (641 -> 64 -> 1, 41,153 params)", "trk_ecan": "(ii) trunk MLP head on CockpitWM (ECAN, default)", | |
| "paper": "PAPER HGB current-state read-out (cur+dwell+face)", "paper_cur": "paper HGB current-only read-out (descr.)", | |
| "trk_udwm_lr": "(i) linear probe (descr.)", "trk_ecan_lr": "(ii) linear probe (descr.)"} | |
| PAIRS = [("trk_udwm", "trk_ecan", "(i) − (ii): Driver-WM-style trunk vs ours (PRIMARY)", True), ("trk_udwm", "paper", "(i) − paper HGB current-state read-out", True), | |
| ("trk_ecan", "paper", "(ii) − paper HGB current-state read-out", True), ("trk_udwm_lr", "trk_ecan_lr", "linear probes (i) − (ii)", False), | |
| ("trk_udwm_lr", "paper", "linear probe (i) − paper", False), ("trk_ecan_lr", "paper", "linear probe (ii) − paper", False), ("trk_udwm", "paper_cur", "(i) − paper current-only", False), | |
| ("trk_ecan", "paper_cur", "(ii) − paper current-only", False)] | |
| TIMING_PAIRS = [("trk_udwm", "trk_ecan", True), ("trk_udwm", "cur_dwfc", True), ("trk_ecan", "cur_dwfc", True), ("trk_udwm_lr", "trk_ecan_lr", False), ("trk_udwm", "cur_base", False), ("trk_ecan", "cur_base", False)] | |
| LEVEL_SCORES = ["cur_dwfc", "trk_udwm", "trk_ecan", "trk_udwm_lr", "trk_ecan_lr", "cur_base", "surrogate_att3s", "persistence"] | |
| MARK = "trk_udwm=" | |
| def head_rows(seeds): | |
| rows = [] | |
| for key, run in (("trk_udwm", UDWM), ("trk_ecan", ECAN)): | |
| for seed in seeds: | |
| cands = sorted(d for d in (CWM3 / "results" / "warning_head").glob(f"TRK_{run.format(seed=seed)}_*") if (d / "FIT_DONE").exists()) | |
| if not cands: | |
| continue | |
| d = cands[-1] | |
| sel = json.loads((d / "selected.json").read_text()) | |
| gq = pd.read_csv(d / "gold_quick.csv") | |
| for kind in ("mlp", "lr"): | |
| h = sel["heads"][kind] | |
| g = gq[gq.model == kind].iloc[0] | |
| rows.append({"model": key, "seed": seed, "head": kind, "hp": json.dumps(h["hp"]), "cv_auroc": h["cv_auroc"], "n_params": h["n_params"], "n_inputs": h["n_inputs"], "gold_auroc": g.gold_test_auroc, | |
| "gold_auroc_ci": [g.ci_lo, g.ci_hi], "gold_prauc": g.gold_test_prauc, "clean_val_auroc": g.clean_val_auroc, "temperature": g.temperature, "dir": d.name}) | |
| return pd.DataFrame(rows) | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--seeds", nargs="+", type=int, default=[0, 1, 2]) | |
| ap.add_argument("--n-boot", type=int, default=2000) | |
| ap.add_argument("--timing-dir", default="") | |
| ap.add_argument("--allow-missing", action="store_true") | |
| ap.add_argument("--no-table", action="store_true") | |
| ap.add_argument("--out-root", default=str(OUT_ROOT)) | |
| a = ap.parse_args() | |
| t0 = time.time() | |
| stamp = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime()) | |
| out = Path(a.out_root) / stamp | |
| out.mkdir(parents=True, exist_ok=True) | |
| def log(m): | |
| line = f"[{time.strftime('%H:%M:%S')}] {m}" | |
| print(line, flush=True) | |
| with open(out / "report.log", "a") as f: | |
| f.write(line + "\n") | |
| missing = [] | |
| H = head_rows(a.seeds) | |
| H.to_csv(out / "heads.csv", index=False) | |
| if len(H) < 2 * len(a.seeds) * 2: | |
| missing.append(f"trunk head fits ({len(H)} rows of {4 * len(a.seeds)})") | |
| L, P, ident = C.gold_block(a.seeds, SPECS, PAIRS, a.n_boot, log) | |
| S = C.pair_summary(P) | |
| L.to_csv(out / "gold_levels.csv", index=False); P.to_csv(out / "gold_pairs.csv", index=False); S.to_csv(out / "gold_summary.csv", index=False) | |
| for key in SPECS: | |
| for seed in a.seeds: | |
| if not C.layouts_present(SPECS[key][0], seed): | |
| missing.append(f"layouts {key} s{seed}") | |
| tdir = C.find_timing_dir(MARK, a.timing_dir) | |
| if tdir is None or not (tdir / "paired.csv").exists(): | |
| missing.append("alert_timing run with the trk scores"); Td, TS, TL = pd.DataFrame(), pd.DataFrame(), pd.DataFrame() | |
| else: | |
| Td, TS, TL = C.timing_block(tdir, TIMING_PAIRS, log) | |
| Td.to_csv(out / "timing_pairs.csv", index=False); TS.to_csv(out / "timing_summary.csv", index=False); TL.to_csv(out / "timing_levels.csv", index=False) | |
| if missing and not a.allow_missing: | |
| raise SystemExit("MISSING inputs (use --allow-missing to render a partial table): " + "; ".join(missing)) | |
| Sp = S[S.prereg == True] if len(S) else S # noqa: E712 | |
| def count(df): | |
| n = len(df); return n, (int(df.rule.str.startswith("SIGNIFICANTLY").sum()) if n else 0), (int(df.rule.str.startswith("better").sum()) if n else 0) | |
| g12 = count(Sp[Sp.pair == "trk_udwm − trk_ecan"]) if len(Sp) else (0, 0, 0) | |
| g1p = count(Sp[Sp.pair == "trk_udwm − paper"]) if len(Sp) else (0, 0, 0) | |
| g2p = count(Sp[Sp.pair == "trk_ecan − paper"]) if len(Sp) else (0, 0, 0) | |
| t12 = count(TS[(TS.pair == "trk_udwm − trk_ecan") & (TS.thr_fit == "val")]) if len(TS) else (0, 0, 0) | |
| t1p = count(TS[(TS.pair == "trk_udwm − cur_dwfc") & (TS.thr_fit == "val")]) if len(TS) else (0, 0, 0) | |
| t2p = count(TS[(TS.pair == "trk_ecan − cur_dwfc") & (TS.thr_fit == "val")]) if len(TS) else (0, 0, 0) | |
| pooled = S[(S.stratum == "pooled") & (S.metric == "auroc")] if len(S) else S | |
| lv = lambda pair: (f"{f3(pooled[pooled.pair == pair].mean_a.iloc[0])} vs {f3(pooled[pooled.pair == pair].mean_b.iloc[0])} (mean Δ {f3(pooled[pooled.pair == pair].mean_delta.iloc[0], 3, True)}, {pooled[pooled.pair == pair].rule.iloc[0]})" # noqa: E731 | |
| if len(pooled) and (pooled.pair == pair).any() else "—") | |
| summary = (f"(i) − (ii) [Transformer-U trunk head − ECAN trunk head]: gold {g12[0]} endpoints, {g12[1]} significantly worse, {g12[2]} better (pooled AUROC {lv('trk_udwm − trk_ecan')}); " | |
| f"timing VAL-fit {t12[0]} endpoints, {t12[1]} worse, {t12[2]} better. (i) − paper HGB current-state read-out: gold {g1p[1]} worse / {g1p[2]} better of {g1p[0]} (pooled AUROC {lv('trk_udwm − paper')}); timing {t1p[1]} worse / {t1p[2]} better of {t1p[0]}. " | |
| f"(ii) − paper: gold {g2p[1]} worse / {g2p[2]} better of {g2p[0]} (pooled AUROC {lv('trk_ecan − paper')}); timing {t2p[1]} worse / {t2p[2]} better of {t2p[0]}. NO adoption, no manuscript change (pre-registered reading).") | |
| log("SUMMARY: " + summary) | |
| ts = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) | |
| md = [f"## {ts} — Experiment B: Driver-WM-style warning head on DETACHED trunk features — (i) CockpitWM w/ Driver-WM-style Transformer U vs (ii) CockpitWM (ECAN, default), both vs the paper HGB current-state read-out (eval/report_warning_trunk_heads.py; pre-registration {PREREG}; run {stamp})", "", | |
| "Head = train/train_trunk_warning_head.py: 0-imputation + StandardScaler + MLPClassifier(hidden (64,), alpha grid {1e-3, 1e-2, 1e-1}, early stopping, random_state 0) on [z_v 256 ; h_u 128 ; z_e 128 ; z_vf 128] + regime bit (641 inputs; 41,153 params <= 50 k), " | |
| "trained on the 979 clean TRAIN consensus labels with the read-out's device-grouped 5-fold CV selection; L2 logistic regression = descriptive linear probe. Gold = warning_gold.build_gold legacy_first (clean-test 512 / clean-val 107), " | |
| f"cluster_boot n_boot {a.n_boot} seed 0 on IDENTICAL resamples; timing from `results/alert_timing/{tdir.name if tdir else '—'}`. Reading rule: per-seed identical / WORSE / BETTER / n.s.; endpoint significantly worse / better iff >= ceil(2n/3) seeds agree. **NO adoption, no manuscript change.**", "", | |
| "### 1. Head selection (validation = device-grouped 5-fold CV AUROC on the 979 labels) and gold levels per fit", "", | |
| "| model | seed | head | selected hp | CV AUROC | params | gold clean-test AUROC [CI] | gold PR-AUC | clean-val AUROC | T |", "|---|---|---|---|---|---|---|---|---|---|"] | |
| for r in H.sort_values(["model", "head", "seed"]).itertuples() if len(H) else []: | |
| md.append(f"| {r.model} | {r.seed} | {r.head} | {r.hp} | {f3(r.cv_auroc)} | {r.n_params:,} | {f3(r.gold_auroc)} [{f3(r.gold_auroc_ci[0])}, {f3(r.gold_auroc_ci[1])}] | {f3(r.gold_prauc)} | {f3(r.clean_val_auroc)} | {f3(r.temperature, 2)} |") | |
| md += ["", "### 2. Gold clean-test AUROC / PR-AUC (pooled / manual / engaged) — levels per seed and paired Δ", ""] + C.md_gold_table(L, S, list(SPECS), LABELS) | |
| md += ["", "### 3. Event-level timing — Recall_{>= δ s early} at K = 10 / 20 / 40 per h (families A / B; VAL-fitted PRIMARY)", ""] + C.md_timing_tables(TS, TL, "trk_udwm − trk_ecan", LEVEL_SCORES) | |
| md += ["", "### 4. SUMMARY (pre-registered reading; nothing adopted)", "", summary, ""] | |
| if missing: | |
| md += ["Missing inputs: " + "; ".join(missing), ""] | |
| text = "\n".join(md) + "\n" | |
| (out / "section.md").write_text(text) | |
| rec = {"generated_utc": ts, "run": stamp, "prereg": PREREG, "seeds": a.seeds, "n_boot": a.n_boot, "timing_dir": str(tdir) if tdir else "", "heads": H.to_dict("records"), | |
| "identity": {f"{k[0]}|{k[1]}|s{k[2]}": v for k, v in ident.items()}, "gold_levels": L.to_dict("records"), "gold_pairs_summary": S.to_dict("records"), | |
| "timing_summary": TS.to_dict("records") if len(TS) else [], "timing_levels": TL.to_dict("records") if len(TL) else [], "summary": summary, "missing": missing} | |
| (out / "report.json").write_text(json.dumps(rec, indent=1, default=str)) | |
| if not a.no_table: | |
| C.append_table(TABLE, "Experiment B — Driver-WM-style warning head on detached trunk features; append-only (eval/report_warning_trunk_heads.py; pre-registration 2026-09-18T03:34:28Z)", text) | |
| C.append_json(TJSON, rec) | |
| print(text) | |
| C.OP.write_run_json(out, "report_warning_trunk_heads", [Path(__file__), Path(C.__file__), Path(C.EV.__file__)], [], seed=0, t_start=t0, input_listing=False, | |
| extra={"prereg": PREREG, "summary": summary, "missing": missing, "dms_fields_used": []}) | |
| log(f"DONE in {time.time() - t0:.0f}s -> {out}") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 12.8 kB
- Xet hash:
- 1e5bcd824f29931d1acb752f74aaf2314a346f13ad428e402a3b106787ea6b00
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.