Buckets:
| """cwm3/eval/report_robustness.py — results/tables/robustness_final.md from results/paired_compare.csv: test-time input perturbations of the frozen main model | |
| `I1_all_minus_road2drv_s<k>_final` (tags robust_s<k>_<pert>_vs_clean = onset cells, robustP_s<k>_<pert>_vs_clean = road-pressure P cells; A = perturbed, B = frozen, | |
| identical rows, device-cluster bootstrap) + the suspect-route-mask protocol from the existing finalc_/finalcm_ rows (level under the masked population — a different | |
| row set, so shown descriptively, not as a paired Δ). Verdict per (perturbation, cell) = cwm2/eval/kill_checks.beats over the 3 seeds read with the sign of | |
| "perturbed − clean": WORSE = the perturbation significantly hurts (CI excludes 0 on ≥ 2 seeds, same sign), NOT_WORSE = |Δ| < 0.01 on every seed with CIs containing 0, | |
| BEATS = significantly better, else INCONCLUSIVE. gaze/robustness agent 2026-09-12. python eval/report_robustness.py [--seeds "0 1 2"] [--out results/tables/robustness_final.md] | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import sys | |
| import time | |
| from pathlib import Path | |
| import numpy as np | |
| import pandas as pd | |
| CWM3 = Path(__file__).resolve().parents[1] | |
| sys.path.insert(0, str(CWM3.parent / "cwm2" / "eval")) | |
| import kill_checks as KC # noqa: E402 | |
| PERTS = [("kin_drop20", "driver kinematics dropout 20 % of history steps (valid bit 0)"), ("kin_drop50", "driver kinematics dropout 50 %"), | |
| ("can_noise", "CAN noise: vEgo + N(0, 0.5 m/s), steer + N(0, 2 deg), i.i.d. per step"), ("road_off", "road stream removed (vf256 = 0, vf_ok = 0)"), | |
| ("kh_trunc_old2s", "history truncation: oldest 2 s of Kh masked (last 2 s kept)"), ("kh_mask_recent2s", "most recent 2 s of Kh masked (stale driver state)")] | |
| MAIN = [("veh_onset|steer_phase_on|3s", "steer_phase"), ("veh_onset|gas_on|3s", "gas"), ("veh_onset|brake_on|3s", "brake"), ("veh_onset|veh_any_on|3s", "veh_any"), | |
| ("drv_onset|hand_leave|3s", "hand_leave"), ("drv_onset|hand_return|3s", "hand_return"), ("drv_onset|head_turn_onset|3s", "head_turn"), | |
| ("p_onset|ttc_drop|3s", "P ttc_drop"), ("p_onset|cut_in|3s", "P cut_in")] | |
| EXTRA = [("d_onset|steer_rate|3s", "D steer_rate"), ("d_onset|decel|3s", "D decel"), ("d_onset|ttc|3s", "D ttc"), ("d_onset|ttc_drop|3s", "D ttc_drop"), ("d_onset|lane|3s", "D lane"), | |
| ("d_onset|badas|3s", "D badas"), ("p_onset|ttc|3s", "P ttc"), ("p_onset|lead_braking|3s", "P lead_braking"), ("p_onset|lane_excursion|3s", "P lane_excursion"), ("p_onset|badas|3s", "P badas")] | |
| def beats(g): | |
| pr = pd.DataFrame({"seed": g.seed, "delta": g.delta.astype(float), "delta_ci_lo": g.delta_ci_lo.astype(float), "delta_ci_hi": g.delta_ci_hi.astype(float), "status": "secondary"}) | |
| return KC.beats(pr, pd.DataFrame({"value": g.value_a.astype(float)}), pd.DataFrame({"value": g.value_b.astype(float)}), min_seeds=3)["verdict"] | |
| def ms(v): | |
| v = np.asarray([x for x in v if x is not None and np.isfinite(x)], float) | |
| return "—" if not len(v) else (f"{v.mean():.3f} ± {v.std(ddof=1):.3f}" if len(v) > 1 else f"{v[0]:.3f}") | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--seeds", default="0 1 2") | |
| ap.add_argument("--out", default=str(CWM3 / "results" / "tables" / "robustness_final.md")) | |
| a = ap.parse_args() | |
| seeds = [int(s) for s in a.seeds.replace(",", " ").split()] | |
| raw = pd.read_csv(CWM3 / "results" / "paired_compare.csv", low_memory=False) | |
| d = raw[raw.tag.str.startswith(("robust_", "robustP_"))].copy() | |
| d["seed"] = d.tag.str.extract(r"_s(\d+)_")[0].astype(int) | |
| d["pert"] = d.tag.str.extract(r"_s\d+_(\w+)_vs_clean$")[0] | |
| d = d.sort_values("run_stamp").groupby(["tag", "regime", "cell", "metric"], as_index=False).tail(1) | |
| for c in ("value_a", "value_b", "delta", "delta_ci_lo", "delta_ci_hi", "n", "n_pos"): | |
| d[c] = pd.to_numeric(d[c], errors="coerce") | |
| d = d[d.metric == "pr_auc"] | |
| recs = {} | |
| for p, _ in PERTS: | |
| for s in seeds: | |
| pj = CWM3 / "results" / "runs" / f"I1_all_minus_road2drv_s{s}_final_pert_{p}" / "perturbation.json" | |
| if pj.exists(): | |
| recs[(p, s)] = json.loads(pj.read_text()) | |
| L = [f"# Robustness of the frozen main model `I1_all_minus_road2drv_s<k>_final` to test-time input perturbations (no retraining; seeds {seeds}; test; rendered {time.strftime('%Y-%m-%d %H:%M:%S UTC', time.gmtime())})", "", | |
| "Every perturbation is applied to the RAW test inputs before the frozen model's own prediction path (`eval/robustness_perturb.py` -> `train/train_cwm3_joint.predict_split` / " | |
| "`eval/write_pred_layout_joint.write_layout`, unchanged); targets and row validity are untouched, so every cell pairs the perturbed layout against the frozen layout on IDENTICAL rows " | |
| "(`eval/paired_compare_joint.py --only-onset --kin-targets causal_routes --kin-version v2`; P cells `eval/pressure_eval.py`; device-cluster bootstrap n_boot 1000, seed 0). " | |
| "Δ = perturbed − clean PR-AUC@3s; a negative Δ is a loss. Verdict (kill_checks.beats over the 3 seeds): **WORSE** = the perturbation significantly hurts; **NOT_WORSE** = |Δ| < 0.01 on every seed and no CI excludes 0; " | |
| "BEATS = significantly better; INCONCLUSIVE otherwise. Perturbation records (`perturbation.json`): " + | |
| "; ".join(f"{p}: " + ", ".join(f"s{s} " + ", ".join(f"{kk}={vv}" for kk, vv in list(r["regimes"].get("manual_test", {}).items())[:4]) for (pp, s), r in sorted(recs.items()) if pp == p) for p, _ in PERTS if any(pp == p for pp, _ in recs)), ""] | |
| # ---- main table: per perturbation x endpoint, per regime | |
| L += ["## 1. Δ PR-AUC@3s (perturbed − clean) on the main endpoints", ""] | |
| for reg in ("manual", "engaged"): | |
| L += [f"### {reg}", "", "| perturbation | " + " | ".join(f"{lab}" for _, lab in MAIN) + " |", "|---|" + "---|" * len(MAIN)] | |
| clean_row = ["clean level (mean ± sd)"] | |
| for cell, _ in MAIN: | |
| g = d[(d.regime == reg) & (d.cell == cell)] | |
| g = g.sort_values("run_stamp").groupby("seed", as_index=False).tail(1) | |
| clean_row.append(ms(g.value_b)) | |
| L.append("| " + " | ".join(clean_row) + " |") | |
| for p, desc in PERTS: | |
| cells = [] | |
| for cell, _ in MAIN: | |
| g = d[(d.regime == reg) & (d.cell == cell) & (d.pert == p)].sort_values("seed") | |
| if not len(g): | |
| cells.append("—") | |
| continue | |
| v = beats(g) if len(g) >= 3 else f"({len(g)} seeds)" | |
| cells.append(f"{g.delta.mean():+.3f} ± {g.delta.std(ddof=1):.3f} {v}" if len(g) > 1 else f"{g.delta.iloc[0]:+.3f} {v}") | |
| L.append(f"| {p} | " + " | ".join(cells) + " |") | |
| L.append("") | |
| # ---- per-seed detail | |
| L += ["## 2. Per-seed detail (A = perturbed, B = clean; Δ [95 % CI])", "", "| regime | perturbation | cell | " + " | ".join(f"A s{s}" for s in seeds) + " | " + " | ".join(f"Δ s{s} [CI]" for s in seeds) + " | n / pos (s0) | verdict |", | |
| "|---|---|---|" + "---|" * (2 * len(seeds)) + "---|---|"] | |
| for reg in ("manual", "engaged"): | |
| for p, _ in PERTS: | |
| for cell, lab in MAIN + EXTRA: | |
| g = d[(d.regime == reg) & (d.cell == cell) & (d.pert == p)].sort_values("seed") | |
| if not len(g): | |
| continue | |
| by = {int(s): r for s, r in zip(g.seed, g.itertuples())} | |
| A = [f"{by[s].value_a:.4f}" if s in by else "—" for s in seeds] | |
| D = [f"{by[s].delta:+.4f} [{by[s].delta_ci_lo:+.4f},{by[s].delta_ci_hi:+.4f}]" if s in by else "—" for s in seeds] | |
| n0 = f"{int(by[seeds[0]].n)} / {int(by[seeds[0]].n_pos)}" if seeds[0] in by else "—" | |
| L.append(f"| {reg} | {p} | {cell} | " + " | ".join(A) + " | " + " | ".join(D) + f" | {n0} | {beats(g) if len(g) >= 3 else '—'} |") | |
| L.append("") | |
| # ---- (v) suspect-route mask: existing rows | |
| f = raw[raw.tag.str.match(r"final(c|cm)_s\d+_allminus_vs_indep$") & (raw.metric == "pr_auc")].copy() | |
| if len(f): | |
| f["seed"] = f.tag.str.extract(r"_s(\d+)_")[0].astype(int) | |
| f["proto"] = np.where(f.tag.str.startswith("finalcm_"), "masked", "all rows") | |
| f = f.sort_values("run_stamp").groupby(["tag", "regime", "cell", "metric"], as_index=False).tail(1) | |
| for c in ("value_a", "n", "n_pos"): | |
| f[c] = pd.to_numeric(f[c], errors="coerce") | |
| L += ["## 3. (v) Suspect-route mask — the frozen model's level under the masked population (64 v2 driver-box-suspect routes censored from every population; existing `finalcm_*` rows) vs all rows (`finalc_*`)", "", | |
| "Different row sets (not a paired Δ): descriptive. Values = the main model (A of the finalc/finalcm allminus-vs-indep pairs), mean ± sd over seeds; n / pos of seed 0.", "", | |
| "| regime | cell | all rows | masked | n / pos all (s0) | n / pos masked (s0) |", "|---|---|---|---|---|---|"] | |
| for reg in ("manual", "engaged"): | |
| for cell, lab in MAIN: | |
| if cell.startswith("p_onset"): | |
| continue | |
| ga = f[(f.regime == reg) & (f.cell == cell) & (f.proto == "all rows")].sort_values("seed") | |
| gm = f[(f.regime == reg) & (f.cell == cell) & (f.proto == "masked")].sort_values("seed") | |
| if not len(ga) or not len(gm): | |
| continue | |
| L.append(f"| {reg} | {cell} | {ms(ga.value_a)} | {ms(gm.value_a)} | {int(ga.n.iloc[0])} / {int(ga.n_pos.iloc[0])} | {int(gm.n.iloc[0])} / {int(gm.n_pos.iloc[0])} |") | |
| L.append("") | |
| # ---- summary | |
| L += ["## 4. Summary (verdicts on the main endpoints, both regimes)", "", "| perturbation | description | WORSE cells | NOT_WORSE | INCONCLUSIVE | BEATS |", "|---|---|---|---|---|---|"] | |
| for p, desc in PERTS: | |
| vs = {"WORSE": [], "NOT_WORSE": [], "INCONCLUSIVE": [], "BEATS": []} | |
| for reg in ("manual", "engaged"): | |
| for cell, lab in MAIN: | |
| g = d[(d.regime == reg) & (d.cell == cell) & (d.pert == p)].sort_values("seed") | |
| if len(g) >= 3: | |
| vs.setdefault(beats(g), []).append(f"{reg} {lab} ({g.delta.mean():+.3f})") | |
| if any(vs.values()): | |
| L.append(f"| {p} | {desc} | {'; '.join(vs['WORSE']) or 'none'} | {len(vs['NOT_WORSE'])} | {len(vs['INCONCLUSIVE'])} | {'; '.join(vs['BEATS']) or 'none'} |") | |
| L.append("") | |
| Path(a.out).write_text("\n".join(L) + "\n") | |
| print(f"written {a.out} ({len(d)} paired rows over {d.pert.nunique() if len(d) else 0} perturbations)") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 10.7 kB
- Xet hash:
- 9b80622400a4d653eba0387e2eea6890dd06d222e9df4740a2b739d7d9bd1cd1
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.