Buckets:
| """cwm3/probes/report_state_sup.py — innovation 3 (b2) table: state-supervised E dynamics (`pooled_P_fs`, `pooled_P_fsonly`) vs the pooled_P baseline | |
| -> results/tables/state_supervised_E.md. Inputs (script outputs only): | |
| results/probes/future_vs_predicted_<tag>.csv (probes/future_vs_predicted.py --space v2 --fhat-runs ...; ridge, subset all; k >= 3 = non-overlapping windows) | |
| results/paired_compare.csv tags fs_s<k>_<arm>_vs_pooledP (eval/paired_compare.py D cells + f_cos; eval/pressure_eval.py P cells) | |
| results/runs/<run>/TRAIN_DONE best val chain NLL (selection objective, unchanged across arms) | |
| Recovered share (P − C) / (A − C) per (event, k): A = actual-future read-out, P = predicted F_hat, C = copy-current, all in the v2 future-PCA space. | |
| Kill rule (pre-stated): the recovered share on ttc / brake at k >= 3 rises from 16–27 % (v1 reference, latent_quality.md §7) to > 50 %; D and P onset cells NOT_WORSE | |
| vs pooled_P (kill_checks.beats, 3 seeds). python probes/report_state_sup.py --tag fs_v2 [--groups baseline:E0_pooled_P_s{k}_v2 fs:E0_pooled_P_fs_s{k}_v2 fsonly:E0_pooled_P_fsonly_s{k}_v2] | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import sys | |
| from pathlib import Path | |
| import numpy as np | |
| import pandas as pd | |
| CWM3 = Path(__file__).resolve().parents[1] | |
| sys.path.insert(0, str(CWM3.parent / "cwm2" / "eval")) | |
| import kill_checks as KC # noqa: E402 | |
| KEY_EVENTS = {"manual": ["P_ttc_active", "veh_brake_pressed", "P_badas_active", "D_decel_active", "D_steer_rate_active", "P_lane_excursion_active", "P_cut_in_active", "veh_gas_pressed"], | |
| "engaged": ["P_ttc_active", "P_badas_active", "P_lane_excursion_active", "P_cut_in_active", "D_steer_rate_active"]} | |
| KILL_EVENTS = ("P_ttc_active", "veh_brake_pressed") | |
| P_CHAINS = ("ttc", "ttc_drop", "cut_in", "lead_braking", "lane_excursion", "badas") | |
| D_FAMS = ("steer_rate", "decel", "ttc", "ttc_drop", "lane", "badas") | |
| def ms(v, nd=3): | |
| v = np.asarray([x for x in v if x is not None and np.isfinite(x)], float) | |
| return "—" if not len(v) else (f"{v.mean():.{nd}f} ± {v.std(ddof=1):.{nd}f}" if len(v) > 1 else f"{v[0]:.{nd}f}") | |
| def beats(g): | |
| if len(g) < 3: | |
| return "INSUFFICIENT" | |
| pr = pd.DataFrame({"seed": g.seed, "delta": g.delta.astype(float), "delta_ci_lo": g.delta_ci_lo.astype(float), "delta_ci_hi": g.delta_ci_hi.astype(float), "status": "secondary"}) | |
| return KC.beats(pr, pd.DataFrame({"value": g.value_a.astype(float)}), pd.DataFrame({"value": g.value_b.astype(float)}), min_seeds=3)["verdict"] | |
| def dcell(g): | |
| return "—" if not len(g) else " / ".join(f"{float(r.delta):+.3f} [{float(r.delta_ci_lo):+.3f},{float(r.delta_ci_hi):+.3f}]" for r in g.sort_values("seed").itertuples()) | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--tag", default="fs_v2") | |
| ap.add_argument("--groups", nargs="+", default=["baseline:E0_pooled_P_s{k}_v2", "fs:E0_pooled_P_fs_s{k}_v2", "fsonly:E0_pooled_P_fsonly_s{k}_v2"]) | |
| ap.add_argument("--seeds", nargs="+", type=int, default=[0, 1, 2]) | |
| ap.add_argument("--out", default=str(CWM3 / "results" / "tables" / "state_supervised_E.md")) | |
| a = ap.parse_args() | |
| groups = {g.split(":")[0]: [g.split(":")[1].format(k=s) for s in a.seeds] for g in a.groups} | |
| csvp = CWM3 / "results" / "probes" / f"future_vs_predicted_{a.tag}.csv" | |
| d = pd.read_csv(csvp, low_memory=False) | |
| d = d[d.stamp == d.stamp.max()] | |
| d = d[(d.probe == "ridge") & (d.subset == "all")].copy() | |
| for c in ("ap_actual", "ap_pred", "ap_copy", "d_pred_minus_copy"): | |
| d[c] = pd.to_numeric(d[c], errors="coerce") | |
| d["share"] = np.where((d.ap_actual - d.ap_copy).abs() > 0.01, (d.ap_pred - d.ap_copy) / (d.ap_actual - d.ap_copy), np.nan) | |
| d["k"] = d.k.astype(int) | |
| L = ["# State-supervised E dynamics (innovation 3 b2) — `pooled_P_fs` (F cosine 0.1 + state read-out, lambda FS 1) and `pooled_P_fsonly` (state read-out instead of the F cosine) vs the " | |
| "`pooled_P` baseline; 3 seeds; pack v2, stats v2, select_LF false", "", | |
| f"Protocol = `probes/future_vs_predicted.py --space v2` (latent_quality.md §7 in the pack-v2 future-PCA space): one frozen ridge read-out per (state event, horizon k) trained on the ACTUAL future " | |
| f"latent F(t+k) of the TRAIN probe rows, evaluated on TEST rows fed with (A) actual F(t+k), (P) each run's predicted F_hat(t+k) (`probes/dump_f_hat.py`, the run's own data path), (C) the current " | |
| f"clip projected with `future_pca_v2.npz`; aligned rows only; PR-AUC, device-cluster CI (300). Recovered share = (P − C) / (A − C); k >= 3 = non-overlapping windows. Source `results/probes/{csvp.name}` " | |
| f"(stamp {d.stamp.max()}). D / P onset cells: `eval/paired_compare.py` / `eval/pressure_eval.py` arm − pooled_P on identical rows (tags fs_s<k>_<arm>_vs_pooledP), verdict kill_checks.beats. " | |
| "**Kill rule (pre-stated)**: recovered share on ttc / brake at k >= 3 > 50 % (v1 reference 16–27 %) AND D / P onset cells NOT_WORSE.", ""] | |
| # ---- selection objective | |
| L += ["## 0. Validation selection objective (D + P chain NLL; unchanged by the arms) and training-loss parts at BEST", "", "| group | " + " | ".join(f"s{s}" for s in a.seeds) + " | mean |", "|---|" + "---|" * len(a.seeds) + "---|"] | |
| for gname, runs in groups.items(): | |
| vals = [] | |
| for r in runs: | |
| p = CWM3 / "results" / "runs" / r / "TRAIN_DONE" | |
| vals.append(json.loads(p.read_text())["best"] if p.exists() else np.nan) | |
| L.append(f"| {gname} | " + " | ".join(f"{v:.4f}" if np.isfinite(v) else "—" for v in vals) + f" | {np.nanmean(vals):.4f} |") | |
| # ---- recovered share table | |
| kill = {} | |
| for regime in ("manual", "engaged"): | |
| L += ["", f"## 1. {regime} — recovered share (P − C) / (A − C), ridge read-out, per group (mean ± sd over seeds); A / C levels from the baseline block", "", | |
| "| event | group | k=1 | k=2 | **k=3** | **k=4** | **k=5** | A (actual) k=3..5 | C (copy) k=3..5 |", "|---|---|---|---|---|---|---|---|---|"] | |
| for ev in KEY_EVENTS[regime]: | |
| for gname, runs in groups.items(): | |
| cells, present = [], False | |
| for k in range(1, 6): | |
| sub = d[(d.regime == regime) & (d.event == ev) & (d.k == k) & d.run.isin(runs)] | |
| present |= bool(len(sub)) | |
| cells.append(("**" if k >= 3 else "") + ms(sub.share, 2) + ("**" if k >= 3 else "")) | |
| if k >= 3 and ev in KILL_EVENTS and len(sub): | |
| kill.setdefault((gname, regime, ev), []).extend(sub.share.tolist()) | |
| if not present: | |
| continue | |
| ref = d[(d.regime == regime) & (d.event == ev) & (d.k >= 3) & d.run.isin(runs)] | |
| L.append(f"| {ev} | {gname} | " + " | ".join(cells) + f" | {ms(ref.ap_actual)} | {ms(ref.ap_copy)} |") | |
| # ---- D / P onset cells vs pooled_P | |
| pc = pd.read_csv(CWM3 / "results" / "paired_compare.csv", low_memory=False) | |
| pc = pc[pc.tag.str.startswith("fs_")].copy() | |
| if len(pc): | |
| pc["seed"] = pc.tag.str.extract(r"_s(\d+)_")[0].astype(int) | |
| pc["arm"] = pc.tag.str.replace(r"^fs_s\d+_", "", regex=True).str.replace("_vs_pooledP", "", regex=False) | |
| pc = pc.sort_values("run_stamp").groupby(["tag", "regime", "cell", "metric"], as_index=False).tail(1) | |
| for c in ("value_a", "value_b", "delta", "delta_ci_lo", "delta_ci_hi"): | |
| pc[c] = pd.to_numeric(pc[c], errors="coerce") | |
| worse = {} | |
| L += ["", "## 2. Onset cells — arm − pooled_P on identical rows (PR-AUC@3s; f_cos descriptive)", "", "| arm | regime | cell | arm | pooled_P | Δ (s0 / s1 / s2) | verdict |", "|---|---|---|---|---|---|---|"] | |
| for arm in [g for g in groups if g != "baseline"]: | |
| for regime in ("manual", "engaged"): | |
| cells = [f"d_onset|{f}|3s" for f in D_FAMS] + [f"p_onset|{c}|3s" for c in P_CHAINS] + [f"f_cos|+{i}s" for i in (1, 3, 5)] | |
| for cell in cells: | |
| g = pc[(pc.arm == arm) & (pc.regime == regime) & (pc.cell == cell)] if len(pc) else pc | |
| if not len(g): | |
| continue | |
| v = beats(g) | |
| if v == "WORSE" and not cell.startswith("f_cos"): | |
| worse.setdefault(arm, []).append(f"{regime}|{cell}") | |
| L.append(f"| {arm} | {regime} | {cell.replace('|', '/')} | {ms(g.value_a)} | {ms(g.value_b)} | {dcell(g)} | {v} |") | |
| # ---- supplementary: the run's OWN f_state read-out on F_hat (probes/own_readout_check.py) — the fair decoder for fsonly (its F_hat leaves the F space) | |
| ownp = CWM3 / "results" / "probes" / f"own_readout_{a.tag}.csv" | |
| if ownp.exists(): | |
| o = pd.read_csv(ownp, low_memory=False) | |
| o = o[o.stamp == o.stamp.max()].copy() | |
| for c in ("share_own", "share_frozen", "ap_own", "ap_actual", "ap_copy"): | |
| o[c] = pd.to_numeric(o[c], errors="coerce") | |
| o["group"] = o.run.map({r: g for g, runs in groups.items() for r in runs}) | |
| L += ["", "## 2b. Supplementary — the run's OWN trained `f_state` read-out applied to its F_hat (probes/own_readout_check.py): recovered share (AP_own − C) / (A − C) on the same rows, " | |
| "next to the frozen actual-F read-out share of §1. `fsonly` drops the F cosine, so its F_hat is not in the F space and only this decoder is fair for it; A / C are the protocol's frozen levels " | |
| "(no 'own read-out on actual F' exists, so the ceiling is the frozen one).", "", | |
| "| regime | event | group | own k=1 | k=2 | **k=3** | **k=4** | **k=5** | frozen k=3..5 (same runs) | AP own k=3..5 | A k=3..5 | C k=3..5 |", "|---|---|---|---|---|---|---|---|---|---|---|---|"] | |
| for regime in ("manual", "engaged"): | |
| for evn in [e for e in KEY_EVENTS[regime] if e in set(o.event)]: | |
| for gname in [g for g in groups if g != "baseline"]: | |
| g = o[(o.regime == regime) & (o.event == evn) & (o.group == gname)] | |
| if not len(g): | |
| continue | |
| cells = [("**" if k >= 3 else "") + ms(g[g.k == k].share_own, 2) + ("**" if k >= 3 else "") for k in range(1, 6)] | |
| r3 = g[g.k >= 3] | |
| L.append(f"| {regime} | {evn} | {gname} | " + " | ".join(cells) + f" | {ms(r3.share_frozen, 2)} | {ms(r3.ap_own)} | {ms(r3.ap_actual)} | {ms(r3.ap_copy)} |") | |
| # ---- verdicts | |
| L += ["", "## 3. Verdicts (kill rule)", ""] | |
| for gname in groups: | |
| parts = [] | |
| ok_share = True | |
| for regime in ("manual", "engaged"): | |
| for ev in KILL_EVENTS: | |
| v = kill.get((gname, regime, ev)) | |
| if v: | |
| m = float(np.nanmean(v)) | |
| parts.append(f"{regime} {ev} k>=3 share {m:.2f}") | |
| if regime == "manual": | |
| ok_share &= m > 0.50 | |
| if gname == "baseline": | |
| L.append(f"- **baseline pooled_P (v2 space)**: {'; '.join(parts)} — the reference level of this protocol (v1 reference 16–27 %).") | |
| continue | |
| has_pc = len(pc) and (pc.arm == gname).any() | |
| ok_cells = has_pc and not worse.get(gname) | |
| L.append(f"- **{gname}**: {'; '.join(parts)} -> share > 0.50 on manual ttc / brake: {'YES' if ok_share and parts else 'NO'}; onset cells WORSE: {worse.get(gname) or ('none' if has_pc else 'not evaluated')} " | |
| f"-> **{'ADOPT' if (ok_share and parts and ok_cells) else 'KILL (not adopted)'}**") | |
| L.append("") | |
| Path(a.out).write_text("\n".join(L) + "\n") | |
| print("\n".join(L[-len(groups) - 2:])) | |
| print(f"written {a.out}") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 11.9 kB
- Xet hash:
- b7a912325bc48096ccde7da8431c07329cc142691a7fdec412181f5461ca5c6e
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.