AsianPlayer/TriDrive / code /eval /report_alert_nocan_ue.py
AsianPlayer's picture
download
raw
20.8 kB
"""cwm3/eval/report_alert_nocan_ue.py — EXPERIMENT A report (owner question 2026-09-18; pre-registration docs/decisions.md 2026-09-18T03:34:28Z): 'front video + driver
only, CAN missing' — the joint model with the vehicle branch masked as a missing modality (I1_ue_noV_s<k>_final: U + visual-only E) and the paper warning read-out
recipe without its CAN-derived features (--no-can). Reads ONLY script outputs; nothing is recomputed except the paired gold bootstrap (the evaluator's own
warning_gold functions on identical device-cluster resamples). CPU. Append-only outputs: results/tables/alert_noCAN_ue.md + results/tables/alert_noCAN_ue.json.
Blocks: (0) layout identity A1 vs A2 (the current-state block reads no predicted quantity: Δ = 0 expected); (1) gold clean-test AUROC / PR-AUC, pooled / manual /
engaged, per seed — A1 (PRIMARY: noV model + no-CAN cur+dwell+face) vs the PAPER read-out (cur+dwell+face with CAN), A2 (ECAN model + no-CAN) vs paper, A1 vs A2, and the
descriptive A3 heads (no-CAN current / future / both on the noV and the ECAN model); (2) event-level timing Recall_{>= δ s early} @ 10 / 20 / 40 per h from the
alert_timing_eval.py run holding the nocan scores (pair nocan_ue − cur_dwfc PRIMARY; VAL-fitted primary, test-fitted secondary); (3) layer-3 operating points at
dms_matched / low10 (+ the other loads, descriptive) from the operating_points.py item ours_l3_dwfc_nocan_ue paired against the paper read-out at its PUBLISHED
thresholds (paired_public.csv, comparator `original`); (4) MODEL cells: driver onset PR-AUC@3 s of the noV model vs the ECAN default and vs the visual-only main
(results/paired_compare.csv, tags finalc_s<k>_uenov_vs_allminusecan / _vs_allminus; verdict kill_checks.beats via eval/report_joint.verdict).
Reading rule (pre-registered): per-seed verdict identical / WORSE (CI < 0) / BETTER (CI > 0) / n.s.; endpoint significantly worse / better iff >= ceil(2n/3) seeds
agree; descriptive arms never enter a verdict; NO adoption, no manuscript change. No openpilot DMS field is read.
python eval/report_alert_nocan_ue.py [--seeds 0 1 2] [--n-boot 2000] [--timing-dir DIR] [--op-dir DIR] [--allow-missing]
"""
from __future__ import annotations
import os
os.environ.setdefault("OMP_NUM_THREADS", "4")
import argparse # noqa: E402
import json # noqa: E402
import sys # noqa: E402
import time # noqa: E402
from pathlib import Path # noqa: E402
import numpy as np # noqa: E402
import pandas as pd # noqa: E402
CWM3 = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(CWM3 / "eval"))
import alert_pair_common as C # noqa: E402
import report_joint as RJ # noqa: E402 (load / verdict on results/paired_compare.csv; nothing recomputed)
f3, ci_s = C.f3, C.ci_s
PREREG = "2026-09-18T03:34:28Z"
OUT_ROOT = CWM3 / "results" / "alert_nocan_ue"
TABLE, TJSON = CWM3 / "results" / "tables" / "alert_noCAN_ue.md", CWM3 / "results" / "tables" / "alert_noCAN_ue.json"
OP_ROOT = CWM3 / "results" / "operating_points"
NOV, ECAN, MAIN = "I1_ue_noV_s{seed}_final", "I1_all_minus_road2drv_s{seed}_final_ecan", "I1_all_minus_road2drv_s{seed}_final"
CUR = "W_learned_curhgb_t15_w10_n979"
SPECS = {"paper": (f"W_{MAIN}_dwfc", CUR + "dwfc"),
"a1_nov_nocan": (f"W_{NOV}_dwfc_nocan", CUR + "dwfcnocan"),
"a2_ecan_nocan": (f"W_{ECAN}_dwfc_nocan", CUR + "dwfcnocan"),
"a3_nov_cur": (f"W_{NOV}_nocan", CUR + "nocan"),
"a3_nov_fut": (f"W_{NOV}_nocan", "W_learned_futhgb_t15_w10_n979nocan"),
"a3_nov_both": (f"W_{NOV}_nocan", "W_learned_bothhgb_t15_w10_n979nocan"),
"a3_ecan_both": (f"W_{ECAN}_nocan", "W_learned_bothhgb_t15_w10_n979nocan"),
"a3_ecan_fut": (f"W_{ECAN}_nocan", "W_learned_futhgb_t15_w10_n979nocan")}
LABELS = {"paper": "PAPER read-out: cur+dwell+face incl. CAN (bit-identical on visual-only / ECAN / Transformer-U)",
"a1_nov_nocan": "A1 PRIMARY: noV model (U + visual E), cur+dwell+face, no CAN", "a2_ecan_nocan": "A2 attribution: ECAN default model, cur+dwell+face, no CAN",
"a3_nov_cur": "A3 descr.: noV, current-only, no CAN", "a3_nov_fut": "A3 descr.: noV, predicted-future only, no CAN (driver + vision-only P forecasts)",
"a3_nov_both": "A3 descr.: noV, current + predicted future, no CAN", "a3_ecan_both": "A3 descr.: ECAN model, current + predicted future, no CAN (CAN inside the trunk)",
"a3_ecan_fut": "A3 descr.: ECAN model, predicted-future only, no CAN"}
PAIRS = [("a1_nov_nocan", "paper", "A1 − paper (PRIMARY: how much of the warning performance depends on the CAN-derived features)", True),
("a2_ecan_nocan", "paper", "A2 − paper (attribution: same features, ECAN model)", True),
("a1_nov_nocan", "a2_ecan_nocan", "A1 − A2 (features vs model: identity expected)", True),
("a3_nov_both", "paper", "A3 noV both − paper", False), ("a3_nov_both", "a1_nov_nocan", "A3 noV both − A1 (does the noV model's predicted future add to the no-CAN current head?)", False),
("a3_nov_fut", "paper", "A3 noV future − paper", False), ("a3_nov_cur", "paper", "A3 noV current-only − paper", False),
("a3_ecan_both", "paper", "A3 ECAN both (no-CAN features, CAN-carrying trunk) − paper", False), ("a3_nov_both", "a3_ecan_both", "A3 noV both − ECAN both (model effect on the predicted-future head)", False)]
TIMING_PAIRS = [("nocan_ue", "cur_dwfc", True), ("nocan_ecan", "cur_dwfc", True), ("nocan_ue", "nocan_ecan", True), ("nocan_ue_both", "cur_dwfc", False), ("nocan_ue_both", "nocan_ue", False),
("nocan_ue_fut", "cur_dwfc", False), ("nocan_ecan_both", "cur_dwfc", False), ("nocan_ue_both", "nocan_ecan_both", False), ("nocan_ue", "cur_base", False)]
LEVEL_SCORES = ["cur_dwfc", "nocan_ue", "nocan_ecan", "nocan_ue_both", "nocan_ue_fut", "nocan_ecan_both", "cur_base", "surrogate_att3s", "persistence"]
MARK = "nocan_ue="
DRV_CELLS = ["drv_onset|hand_leave|3s", "drv_onset|hand_return|3s", "drv_onset|head_turn_onset|3s", "drv_onset|head_return|3s"]
PAPER_CELLS = {("manual", "drv_onset|hand_leave|3s"), ("manual", "drv_onset|hand_return|3s"), ("manual", "drv_onset|head_turn_onset|3s"), ("engaged", "drv_onset|head_turn_onset|3s")}
LOADS = ["dms_matched", "low10", "low5", "low3", "low1", "same_recall"]
def find_op_dir(explicit, item="ours_l3_dwfc_nocan_ue"):
if explicit:
return Path(explicit)
c = sorted(OP_ROOT.glob(f"{item}_*"))
return c[-1] if c else None
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--seeds", nargs="+", type=int, default=[0, 1, 2])
ap.add_argument("--n-boot", type=int, default=2000)
ap.add_argument("--timing-dir", default="")
ap.add_argument("--op-dir", default="")
ap.add_argument("--allow-missing", action="store_true")
ap.add_argument("--no-table", action="store_true")
ap.add_argument("--out-root", default=str(OUT_ROOT))
a = ap.parse_args()
t0 = time.time()
stamp = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime())
out = Path(a.out_root) / stamp
out.mkdir(parents=True, exist_ok=True)
def log(m):
line = f"[{time.strftime('%H:%M:%S')}] {m}"
print(line, flush=True)
with open(out / "report.log", "a") as f:
f.write(line + "\n")
missing = []
# ------------------------------------------------------------------ 1. gold (+ 0. identity)
L, P, ident = C.gold_block(a.seeds, SPECS, PAIRS, a.n_boot, log)
S = C.pair_summary(P)
L.to_csv(out / "gold_levels.csv", index=False); P.to_csv(out / "gold_pairs.csv", index=False); S.to_csv(out / "gold_summary.csv", index=False)
for key in SPECS:
for seed in a.seeds:
if not C.layouts_present(SPECS[key][0], seed):
missing.append(f"layouts {key} s{seed}")
# ------------------------------------------------------------------ 2. timing
tdir = C.find_timing_dir(MARK, a.timing_dir)
if tdir is None or not (tdir / "paired.csv").exists():
missing.append("alert_timing run with the nocan scores"); Td, TS, TL = pd.DataFrame(), pd.DataFrame(), pd.DataFrame()
else:
Td, TS, TL = C.timing_block(tdir, TIMING_PAIRS, log)
Td.to_csv(out / "timing_pairs.csv", index=False); TS.to_csv(out / "timing_summary.csv", index=False); TL.to_csv(out / "timing_levels.csv", index=False)
# ------------------------------------------------------------------ 3. layer-3 operating points (paired vs the paper read-out at its published thresholds)
odir = find_op_dir(a.op_dir)
op_rows = []
if odir is None or not (odir / "paired_public.csv").exists():
missing.append("operating_points item ours_l3_dwfc_nocan_ue"); OPd = pd.DataFrame()
else:
PP = pd.read_csv(odir / "paired_public.csv")
PP = PP[PP.comparator == "original"]
for (load, reg), g in PP.groupby(["load", "regime"]):
g = g.sort_values("seed")
for met, dcol, lo_c, hi_c in (("recall", "d_recall", "d_recall_ci_lo", "d_recall_ci_hi"), ("precision", "d_precision", "d_precision_ci_lo", "d_precision_ci_hi"),
("FAR", "d_far", "d_far_ci_lo", "d_far_ci_hi"), ("lead0_mean_s", "d_lead0_mean", "d_lead0_ci_lo", "d_lead0_ci_hi")):
vs = [C.verdict_signed(r[dcol], r[lo_c], r[hi_c], lower_better=(met == "FAR")) for _, r in g.iterrows()] # FAR: lower is better (sign flipped)
rule, nw, nb, n = C.seed_rule(vs)
head_c, comp_c = {"recall": ("recall_head", "recall_comp"), "precision": ("precision_head", "precision_comp"), "FAR": ("far_head", "far_comp"), "lead0_mean_s": ("lead_median_head", "lead_median_comp")}[met]
op_rows.append({"load": load, "regime": reg, "metric": met, "lower_better": met == "FAR", "prereg": load in ("dms_matched", "low10"), "seeds": n, "n_worse": nw, "n_better": nb, "rule": rule,
"deltas": g[dcol].round(4).tolist(), "ci": [[round(x, 4), round(y, 4)] for x, y in zip(g[lo_c], g[hi_c])], "head": g[head_c].round(4).tolist(), "comp": g[comp_c].round(4).tolist(),
"aph_manual": g.aph_comp_manual.round(2).tolist(), "aph_engaged": g.aph_comp_engaged.round(2).tolist(), "n_common": int(g.n_common.iloc[0]), "n_pos": int(g.n_pos.iloc[0])})
OPd = pd.DataFrame(op_rows); OPd.to_csv(out / "layer3_pairs.csv", index=False)
# ------------------------------------------------------------------ 4. model cells (driver onset PR-AUC@3 s)
cell_rows = []
for b_name, b_label in (("allminusecan", "CockpitWM (ECAN, default)"), ("allminus", "CockpitWM (visual-only environment input)")):
df = RJ.load("finalc", "uenov", b_name, a.seeds)
if not len(df):
missing.append(f"paired_compare tags finalc_s<k>_uenov_vs_{b_name}"); continue
for reg in C.REGIMES:
for cell in DRV_CELLS:
g = df[(df.regime == reg) & (df.cell == cell) & (df.metric == "pr_auc")].sort_values("seed")
if not len(g):
continue
v = RJ.verdict(g)
cell_rows.append({"b": b_name, "b_label": b_label, "regime": reg, "cell": cell, "paper_facing": (reg, cell) in PAPER_CELLS, "seeds": int(len(g)), "a_values": g.value_a.astype(float).round(4).tolist(),
"b_values": g.value_b.astype(float).round(4).tolist(), "deltas": g.delta.astype(float).round(4).tolist(),
"ci": [[round(float(x), 4), round(float(y), 4)] for x, y in zip(g.delta_ci_lo, g.delta_ci_hi)], "mean_delta": float(g.delta.astype(float).mean()),
"sd_delta": float(g.delta.astype(float).std(ddof=1)) if len(g) > 1 else np.nan, "verdict": v["verdict"], "n": int(g.n.iloc[0]), "n_pos": int(g.n_pos.iloc[0])})
veh = df[df.cell.str.startswith("veh_onset")]
log(f"vs {b_name}: {len(df)} rows; vehicle cells present: {len(veh)} (expected 0: valid_V = 0 -> NaN, skipped by the evaluator)")
CD = pd.DataFrame(cell_rows); CD.to_csv(out / "driver_cells.csv", index=False)
if missing and not a.allow_missing:
raise SystemExit("MISSING inputs (use --allow-missing to render a partial table): " + "; ".join(missing))
# ------------------------------------------------------------------ verdict summary (pre-registered endpoints only)
Sp = S[S.prereg == True] if len(S) else S # noqa: E712
prim = Sp[Sp.pair == "a1_nov_nocan − paper"] if len(Sp) else Sp
n_g = len(prim); n_gw = int(prim.rule.str.startswith("SIGNIFICANTLY").sum()) if n_g else 0; n_gb = int(prim.rule.str.startswith("better").sum()) if n_g else 0
n_gi = int(prim.rule.str.startswith("identical").sum()) if n_g else 0
TSp = TS[(TS.pair == "nocan_ue − cur_dwfc") & (TS.thr_fit == "val")] if len(TS) else TS
n_t = len(TSp); n_tw = int(TSp.rule.str.startswith("SIGNIFICANTLY").sum()) if n_t else 0; n_tb = int(TSp.rule.str.startswith("better").sum()) if n_t else 0
OPp = OPd[OPd.prereg == True] if len(OPd) else OPd # noqa: E712
n_o = len(OPp); n_ow = int(OPp.rule.str.startswith("SIGNIFICANTLY").sum()) if n_o else 0; n_ob = int(OPp.rule.str.startswith("better").sum()) if n_o else 0
ident_a1a2 = [ident[k] for k in ident if k[0] == "a1_nov_nocan" and k[1] == "a2_ecan_nocan"]
all_ident = bool(ident_a1a2) and all(i["identical"] for i in ident_a1a2)
cells_paper = CD[(CD.b == "allminusecan") & (CD.paper_facing)] if len(CD) else CD
summary = (f"A1 (noV model + no-CAN read-out) vs the PAPER read-out — gold: {n_g} pre-registered endpoints, {n_gw} significantly worse, {n_gb} better, {n_gi} identical; "
f"timing family A/B VAL-fit: {n_t} endpoints, {n_tw} significantly worse, {n_tb} better; layer-3 (dms_matched + low10): {n_o} cells, {n_ow} significantly worse, {n_ob} better. "
f"A1 = A2 layout identity on every row: {'YES (Δ = 0: the whole A1 effect is the FEATURE removal; the noV model changes nothing in a current-state read-out)' if all_ident else 'NO'}. "
f"Driver onset cells noV − ECAN default (paper-facing): " + "; ".join(f"{r.regime} {r.cell.split('|')[1]} {f3(r.mean_delta, 4, True)} ± {f3(r.sd_delta, 4)} {r.verdict}" for r in cells_paper.itertuples())
+ ". NO adoption, no manuscript change (pre-registered reading).")
log("SUMMARY: " + summary)
# ------------------------------------------------------------------ markdown
ts = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
md = [f"## {ts} — Experiment A: 'front video + driver only, CAN missing' (I1_ue_noV_s<k>_final = U + visual-only E, vehicle branch masked as a missing modality) with the paper read-out recipe minus its CAN-derived features (eval/report_alert_nocan_ue.py; pre-registration {PREREG}; run {stamp})", "",
f"Paper read-out = `W_{MAIN}_dwfc:{CUR}dwfc` (cur+dwell+face incl. the current CAN margins / vehicle anchor states / vEgo / regime; bit-identical under visual-only / ECAN / Transformer-U, rows 482 / 494). "
f"A1 = `W_{NOV}_dwfc_nocan:{CUR}dwfcnocan` (43 current-state features: driver anchor states, K0, dwell, face, vision-availability bits); A2 = the same argv on the ECAN default model; A3 = current / future / both no-CAN heads (descriptive). "
f"Gold = warning_gold.build_gold legacy_first (clean-test 512 / clean-val 107), cluster_boot n_boot {a.n_boot} seed 0 on IDENTICAL resamples; timing from `results/alert_timing/{tdir.name if tdir else '—'}`; layer 3 from `{odir.name if odir else '—'}` (comparator `original` = the paper read-out at its PUBLISHED per-seed thresholds); "
"driver cells from results/paired_compare.csv (eval/paired_compare_joint.py --only-onset, causal kin v2, device-cluster bootstrap n_boot 1000). Reading rule: per-seed identical / WORSE / BETTER / n.s.; endpoint significantly worse / better iff >= ceil(2n/3) seeds agree. "
"**NO adoption, no manuscript change.** Every number below is a CSV cell of the run dir.", "",
"### 0. Layout identity (max |Δ score| over EVERY test / val row of both regimes)", "", "| pair | seed | rows | max abs Δ | NaN-pattern mismatches | identical |", "|---|---|---|---|---|---|"]
for (ka, kb, seed), i in sorted(ident.items(), key=lambda kv: (kv[0][0], kv[0][1], kv[0][2])):
md.append(f"| {ka} − {kb} | {seed} | {i['n_rows']} | {i['max_abs_diff']:.3e} | {i['nan_mismatch_rows']} | {'YES' if i['identical'] else 'no'} |")
md += ["", "### 1. Gold clean-test AUROC / PR-AUC (pooled / manual / engaged) — levels per seed and paired Δ", ""] + C.md_gold_table(L, S, list(SPECS), LABELS)
md += ["", "### 2. Event-level timing — Recall_{>= δ s early} at K = 10 / 20 / 40 per h (families A / B; VAL-fitted PRIMARY, test-fitted secondary)", ""] + C.md_timing_tables(TS, TL, "nocan_ue − cur_dwfc", LEVEL_SCORES)
md += ["", "### 3. Layer-3 operating points — A1 vs the paper read-out at its PUBLISHED thresholds (dms_matched / low10 pre-registered; other loads descriptive). Δ = A1 − paper; for FAR LOWER is better and the verdict is sign-flipped accordingly (the 05:15:25Z section of this file labelled the FAR rows with the unflipped sign — superseded here; every other cell is identical)", "",
"| load | regime | metric | A1 s0 / s1 / s2 | paper s0 / s1 / s2 | Δ s0 / s1 / s2 | CIs | rule | n / pos |", "|---|---|---|---|---|---|---|---|---|"]
if len(OPd):
for load in LOADS:
for r in OPd[OPd.load == load].sort_values(["regime", "metric"]).itertuples():
rule = r.rule if r.prereg else "descriptive: " + r.rule
md.append(f"| {r.load} | {r.regime} | {r.metric} | {' / '.join(f3(x) for x in r.head)} | {' / '.join(f3(x) for x in r.comp)} | {' / '.join(f3(x, 3, True) for x in r.deltas)} | {'; '.join(ci_s(x, y) for x, y in r.ci)} | {rule} | {r.n_common} / {r.n_pos} |")
md += ["", "### 4. MODEL cells — driver onset PR-AUC@3 s of the noV model (A) vs the comparator (B), TEST split (vehicle cells undefined by construction: valid_V = 0)", "",
"| B | regime | cell | paper-facing | A s0 / s1 / s2 | B s0 / s1 / s2 | Δ s0 / s1 / s2 | CIs | mean Δ ± sd | verdict (kill_checks.beats) |", "|---|---|---|---|---|---|---|---|---|---|"]
for r in CD.itertuples() if len(CD) else []:
md.append(f"| {r.b_label} | {r.regime} | {r.cell} | {'yes' if r.paper_facing else 'descr. (owner item 3)'} | {' / '.join(f3(x, 4) for x in r.a_values)} | {' / '.join(f3(x, 4) for x in r.b_values)} | "
f"{' / '.join(f3(x, 4, True) for x in r.deltas)} | {'; '.join(ci_s(x, y, 4) for x, y in r.ci)} | {f3(r.mean_delta, 4, True)} ± {f3(r.sd_delta, 4)} | {r.verdict} |")
md += ["", "### 5. SUMMARY (pre-registered reading; nothing adopted)", "", summary, ""]
if missing:
md += ["Missing inputs: " + "; ".join(missing), ""]
text = "\n".join(md) + "\n"
(out / "section.md").write_text(text)
rec = {"generated_utc": ts, "run": stamp, "prereg": PREREG, "seeds": a.seeds, "n_boot": a.n_boot, "timing_dir": str(tdir) if tdir else "", "op_dir": str(odir) if odir else "",
"identity": {f"{k[0]}|{k[1]}|s{k[2]}": v for k, v in ident.items()}, "gold_levels": L.to_dict("records"), "gold_pairs_summary": S.to_dict("records"), "timing_summary": TS.to_dict("records") if len(TS) else [],
"timing_levels": TL.to_dict("records") if len(TL) else [], "layer3": OPd.to_dict("records") if len(OPd) else [], "driver_cells": CD.to_dict("records") if len(CD) else [], "summary": summary, "missing": missing}
(out / "report.json").write_text(json.dumps(rec, indent=1, default=str))
if not a.no_table:
C.append_table(TABLE, "Experiment A — alert read-out without CAN on the U + visual-E model (vehicle branch = missing modality); append-only (eval/report_alert_nocan_ue.py; pre-registration 2026-09-18T03:34:28Z)", text)
C.append_json(TJSON, rec)
print(text)
C.OP.write_run_json(out, "report_alert_nocan_ue", [Path(__file__), Path(C.__file__), Path(C.EV.__file__), Path(RJ.__file__)], [], seed=0, t_start=t0, input_listing=False,
extra={"prereg": PREREG, "summary": summary, "missing": missing, "dms_fields_used": []})
log(f"DONE in {time.time() - t0:.0f}s -> {out}")
if __name__ == "__main__":
main()

Xet Storage Details

Size:
20.8 kB
·
Xet hash:
a0f33f2f06fc2181ff99c6cd1746725c1a11e6300addcdd926851c1340190dd6

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.