Buckets:
| """cwm3/eval/gaze_chain_eval.py — the gaze_off ONSET chain of the U0 `*_gz*` layouts (extra keys gaze_off_onset_p / gaze_off_pi, eval/write_pred_layout.py) | |
| scored on the s12 test grid (population rows) and on the gold clean-test rows (DMS comparator), paired device-cluster bootstrap. (gaze/robustness agent, 2026-09-12.) | |
| Population (first-passage contract, exactly P_drv of cwm2/eval/populations.py applied to results/targets_gaze_causal_routes_v2): for the event gaze_off_on | |
| (0 -> 1 of the causal gaze_off state) rows inactive at the anchor (S0_g = 0) with the onset mask EM_g[:, :15, 0] all true; y = first onset < 15 steps (3 s). | |
| gaze_off_off (1 -> 0) symmetrically on S0_g = 1. Scores: A = the layout's gaze_off_onset_p at 3 s; B = --b <layout> (identical rows) | --b-rule (the | |
| current-state RULE: |lm_yaw_dev|(t) = the continuous quantity behind the hysteresis, sign-flipped for the off event — the gaze analogue of the k0 heuristic) | | |
| --b-persistence (constant score -> PR-AUC = base rate; metric row "base_rate"). Rows appended to results/paired_compare.csv (cell gaze_onset|<event>|3s). | |
| --dms-gold: the 509-row clean human gold (cwm2/eval/warning_gold.py: episode_table / build_gold / merge_legacy — the ONLY place the DMS att@3s column is | |
| read; legacy_first anchoring, seed-matched legacy CSV), restricted to clean_test rows anchored in the TEST split whose anchor satisfies the population rule | |
| above; LABEL = OUR gaze_off_on onset within 3 s (not the human warrant); columns: model gaze_off_onset_p@3s (per row from the layout of the row's regime), | |
| DMS att@3s (comparator only), rule |lm_yaw_dev|(t), persistence. AUROC + PR-AUC with the same shared-resample device-cluster bootstrap; rows appended to | |
| results/gaze_chain/gaze_gold.csv (own CSV: the gold rows are not a grid population). No DMS quantity is stored anywhere. | |
| python eval/gaze_chain_eval.py --regime manual --a results/runs/U0_kin_can_noK_s0_kcv2_clip8_ln_gz --b-rule --tag gaze_s0_gz_vs_rule | |
| python eval/gaze_chain_eval.py --dms-gold --a results/runs/U0_kin_can_noK_s0_kcv2_clip8_ln_gz --seed 0 --tag gazegold_s0_gz | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import csv | |
| import json | |
| import sys | |
| import time | |
| from pathlib import Path | |
| import numpy as np | |
| CWM3 = Path(__file__).resolve().parents[1] | |
| EXPLORE = CWM3.parent | |
| CWM2 = EXPLORE / "cwm2" | |
| for p in (CWM2 / "eval", CWM2 / "data", EXPLORE / "s12_wm", CWM3 / "common", CWM3 / "eval"): | |
| sys.path.insert(0, str(p)) | |
| import importlib.util as _ilu # noqa: E402 | |
| _spec = _ilu.spec_from_file_location("cwm3_runmeta", CWM3 / "common" / "runmeta.py") # cwm3's fixed run-json writer (see paired_compare.py) | |
| _rm = _ilu.module_from_spec(_spec); _spec.loader.exec_module(_rm) | |
| write_run_json = _rm.write_run_json | |
| from paired_compare import boot_pair, COLS, OUT, layout_guard, guard_line # noqa: E402 | |
| from paired_compare_joint import boot_pair_auroc # noqa: E402 | |
| from pagecache import drop_all as _drop_cache # noqa: E402 | |
| S12R = EXPLORE / "s12_wm" / "results" | |
| GAZE = CWM3 / "results" / "targets_gaze_causal_routes_v2" | |
| GOLD_DIR = CWM3 / "results" / "gaze_chain" | |
| GOLD_CSV = GOLD_DIR / "gaze_gold.csv" | |
| GOLD_COLS = ["run_stamp", "tag", "seed", "regime", "n", "n_pos", "n_dev", "column", "comparator", "metric", "value", "ci_lo", "ci_hi", "value_b", "delta", "delta_ci_lo", "delta_ci_hi", | |
| "frac_boot_d_gt0", "n_boot", "label", "note"] | |
| TAU = 15 | |
| EVENTS = ("gaze_off_on", "gaze_off_off") | |
| def load_gaze(regime, split): | |
| d = np.load(GAZE / f"gaze_{regime}_{split}.npz", allow_pickle=True) | |
| tg = np.load(S12R / f"targets_{regime}_{split}.npz", allow_pickle=True) | |
| assert np.array_equal(d["uid"], tg["uid"]) and np.allclose(d["t"], tg["t"]), "gaze rows != targets rows" | |
| assert [str(e) for e in d["event_names"]] == list(EVENTS) | |
| ok = d["row_ok"].astype(bool) | |
| S = d["S_g"].astype(np.float32); S[~ok] = np.nan | |
| S0 = d["S0_g"].astype(np.float32); S0[~ok] = np.nan | |
| out = {"uid": tg["uid"], "t": tg["t"], "dongle": tg["dongle"], "M": tg["M"].astype(bool), "S": S, "S0": S0, "EM": d["EM_g"].astype(bool) & ok[:, None, None], | |
| "absdev0": d["absdev0"].astype(np.float32), "dwell0": d["dwell0"].astype(np.float32)} | |
| _drop_cache([GAZE / f"gaze_{regime}_{split}.npz", S12R / f"targets_{regime}_{split}.npz"]) | |
| return out | |
| def populations(g, tau=TAU): | |
| """{event: dict(sel, y, first, dongle, rule)} — P_drv rule: inactive at the anchor & EM[:, :tau] all; y = first onset < tau.""" | |
| pops = {} | |
| for j, ev in enumerate(EVENTS): | |
| z0, z = (g["S0"], g["S"]) if j == 0 else (1 - g["S0"], 1 - g["S"]) | |
| em = g["EM"][:, :, j] | |
| zp = np.concatenate([z0[:, None], z[:, :-1]], 1) | |
| onset = (np.nan_to_num(z, nan=0.0) > 0.5) & (np.nan_to_num(zp, nan=1.0) < 0.5) & em | |
| first = np.where(onset.any(1), onset.argmax(1), -1) | |
| inact0 = np.nan_to_num(zp[:, 0], nan=1.0) < 0.5 | |
| sel = np.flatnonzero(inact0 & em[:, :tau].all(1)) | |
| y = (first[sel] >= 0) & (first[sel] < tau) | |
| rule = np.nan_to_num(g["absdev0"][sel], nan=0.0) * (1.0 if j == 0 else -1.0) # |lm_yaw_dev|(t): closer to the on threshold -> higher; off event sign-flipped | |
| pops[ev] = {"sel": sel, "y": y, "first": first[sel], "dongle": g["dongle"][sel], "rule": rule.astype(np.float64)} | |
| return pops | |
| def gaze_scores(layout_dir, regime, split, N): | |
| z = np.load(Path(layout_dir) / f"pred_layout_{regime}_{split}.npz", allow_pickle=True) | |
| assert "gaze_off_onset_p" in z.files, f"{layout_dir}: no gaze_off_onset_p block (not a *_gz* layout)" | |
| assert z["gaze_off_onset_p"].shape[0] == N | |
| taus = [int(x) for x in z["gaze_off_taus"]] | |
| op = z["gaze_off_onset_p"].astype(np.float64)[:, taus.index(TAU)] | |
| # off event: P(first 1 -> 0 within tau | z0 = 1) = 1 - prod(1 - h_off) | |
| hoff = np.clip(z["gaze_off_h_off"].astype(np.float64), 1e-6, 1 - 1e-6) | |
| off_p = 1 - np.prod(1 - hoff[:, :TAU], 1) | |
| off_p[~np.isfinite(op)] = np.nan | |
| meta = json.loads(str(z["meta"])) | |
| return {"gaze_off_on": op, "gaze_off_off": off_p}, meta | |
| def row(stamp, tag, regime, split, cell, metric, na, nb, res, n, n_pos, n_boot): | |
| return {"run_stamp": stamp, "tag": tag, "regime": regime, "split": split, "cell": cell, "metric": metric, "a": na, "b": nb, | |
| "value_a": res["value_a"], "ci_a_lo": res["ci_a"][0], "ci_a_hi": res["ci_a"][1], "value_b": res["value_b"], "ci_b_lo": res["ci_b"][0], "ci_b_hi": res["ci_b"][1], | |
| "delta": res["delta"], "delta_ci_lo": res["ci_d"][0], "delta_ci_hi": res["ci_d"][1], "frac_boot_d_gt0": res["frac_gt0"], "n": n, "n_pos": n_pos, "n_dev": res["n_dev"], "n_boot": n_boot} | |
| def main_population(a, t0, stamp): | |
| guards = [layout_guard(a.a, a.regime, a.split)] + ([layout_guard(a.b, a.regime, a.split)] if a.b else []) | |
| for g_ in guards: | |
| print(guard_line(g_), flush=True) | |
| g = load_gaze(a.regime, a.split) | |
| N = len(g["t"]) | |
| pops = populations(g) | |
| sa_all, _ = gaze_scores(a.a, a.regime, a.split, N) | |
| na = Path(a.a).name | |
| sb_all, nb = None, ("persistence" if a.b_persistence else "rule_absyawdev_t" if a.b_rule else Path(a.b).name) | |
| if a.b: | |
| sb_all, _ = gaze_scores(a.b, a.regime, a.split, N) | |
| rows = [] | |
| for ev in EVENTS: | |
| pop = pops[ev] | |
| sel, y = pop["sel"], pop["y"] | |
| cell = f"gaze_onset|{ev}|3s" | |
| if a.b_persistence: | |
| rows.append({"run_stamp": stamp, "tag": a.tag, "regime": a.regime, "split": a.split, "cell": cell, "metric": "base_rate", "a": na, "b": "persistence", | |
| "value_a": float(y.mean()) if len(y) else "", "ci_a_lo": "", "ci_a_hi": "", "value_b": float(y.mean()) if len(y) else "", "ci_b_lo": "", "ci_b_hi": "", | |
| "delta": "", "delta_ci_lo": "", "delta_ci_hi": "", "frac_boot_d_gt0": "", "n": int(len(sel)), "n_pos": int(y.sum()), "n_dev": int(len(np.unique(pop["dongle"]))), "n_boot": 0}) | |
| print(f" {cell} base_rate {float(y.mean()) if len(y) else float('nan'):.4f} n={len(sel)} pos={int(y.sum())}", flush=True) | |
| continue | |
| sa = sa_all[ev][sel] | |
| sb = sb_all[ev][sel] if sb_all is not None else pop["rule"] | |
| keep = np.isfinite(sa) & np.isfinite(sb) | |
| if keep.sum() < a.min_rows or y[keep].sum() < a.min_pos or y[keep].sum() >= keep.sum(): | |
| print(f" {cell} skipped (n={int(keep.sum())}, pos={int(y[keep].sum())})", flush=True) | |
| continue | |
| res = boot_pair(y[keep], sa[keep], sb[keep], pop["dongle"][keep], a.n_boot, 0, "ap") | |
| rows.append(row(stamp, a.tag, a.regime, a.split, cell, "pr_auc", na, nb, res, int(keep.sum()), int(y[keep].sum()), a.n_boot)) | |
| print(f" {cell} pr_auc A={res['value_a']:.4f} B={res['value_b']:.4f} Δ={res['delta']:+.4f} [{res['ci_d'][0]:+.4f},{res['ci_d'][1]:+.4f}] P(Δ>0)={res['frac_gt0']:.2f} n={int(keep.sum())} pos={int(y[keep].sum())}", flush=True) | |
| if a.auroc: | |
| res2 = boot_pair_auroc(y[keep], sa[keep], sb[keep], pop["dongle"][keep], a.n_boot, 0) | |
| rows.append(row(stamp, a.tag, a.regime, a.split, cell, "auroc", na, nb, res2, int(keep.sum()), int(y[keep].sum()), a.n_boot)) | |
| print(f" {cell} auroc A={res2['value_a']:.4f} B={res2['value_b']:.4f} Δ={res2['delta']:+.4f} [{res2['ci_d'][0]:+.4f},{res2['ci_d'][1]:+.4f}]", flush=True) | |
| if a.no_append: | |
| print(f"no-append: {len(rows)} rows computed ({time.time() - t0:.0f}s)") | |
| return | |
| new = not OUT.exists() | |
| with open(OUT, "a", newline="") as f: | |
| w = csv.DictWriter(f, fieldnames=COLS) | |
| if new: | |
| w.writeheader() | |
| for r in rows: | |
| w.writerow(r) | |
| print(f"appended {len(rows)} rows to {OUT} ({time.time() - t0:.0f}s)") | |
| write_run_json(CWM3 / "results", "gaze_chain_eval", [Path(__file__), CWM3 / "eval" / "paired_compare.py"], | |
| [Path(a.a) / f"pred_layout_{a.regime}_{a.split}.npz", GAZE / f"gaze_{a.regime}_{a.split}.npz"] + ([Path(a.b) / f"pred_layout_{a.regime}_{a.split}.npz"] if a.b else []), | |
| seed=0, t_start=t0, input_listing=False, extra={"run_stamp": stamp, "tag": a.tag, "regime": a.regime, "a": a.a, "b": nb, "n_rows": len(rows), "dms_fields_used": [], "layout_guard": guards}) | |
| def main_gold(a, t0, stamp): | |
| """gold clean-test rows: our gaze_off_on label vs model / DMS att@3s / rule / persistence.""" | |
| import pandas as pd | |
| import warning_gold as WG # cwm2/eval — the declared reader of the legacy DMS comparator columns | |
| seed = int(a.seed) | |
| guards = [layout_guard(a.a, r, "test") for r in ("manual", "engaged")] | |
| for g_ in guards: | |
| print(guard_line(g_), flush=True) | |
| EP = WG.episode_table() | |
| lab = EP[EP.label != "unsure"].reset_index(drop=True) | |
| packs = {(r, s): WG.load_pack(r, s) for r in WG.REGIMES for s in ("test", "val")} | |
| legacy = WG.merge_legacy(seed) | |
| assert legacy is not None and "att@3s" in legacy, "legacy episode CSVs (att@3s) missing" | |
| G = WG.build_gold(lab, packs, legacy, "legacy_first") | |
| G = G[G.clean_test & (G.split == "test")].reset_index(drop=True) | |
| G["att3s"] = legacy.reindex(G.idx)["att@3s"].to_numpy(float) # DMS comparator column (never stored) | |
| print(f"gold clean_test rows anchored in test: {len(G)} (warrant positives {int(G.y.sum())}, devices {G.dongle.nunique()})", flush=True) | |
| # our label + scores per (regime) block | |
| y_g = np.full(len(G), np.nan) | |
| s_model, s_rule = np.full(len(G), np.nan), np.full(len(G), np.nan) | |
| inpop = np.zeros(len(G), bool) | |
| for regime, grp in G.groupby("regime"): | |
| g = load_gaze(regime, "test") | |
| pops = populations(g) | |
| pop = pops["gaze_off_on"] | |
| sel_set = np.zeros(len(g["t"]), bool) | |
| sel_set[pop["sel"]] = True | |
| ymap = np.full(len(g["t"]), np.nan) | |
| ymap[pop["sel"]] = pop["y"].astype(float) | |
| rows_ = grp.row.to_numpy() | |
| sa_all, _ = gaze_scores(a.a, regime, "test", len(g["t"])) | |
| ii = grp.index.to_numpy() | |
| inpop[ii] = sel_set[rows_] | |
| y_g[ii] = ymap[rows_] | |
| s_model[ii] = sa_all["gaze_off_on"][rows_] | |
| s_rule[ii] = np.nan_to_num(g["absdev0"][rows_], nan=0.0) | |
| keep = inpop & np.isfinite(y_g) & np.isfinite(s_model) & np.isfinite(G.att3s.to_numpy()) | |
| C = G[keep].reset_index(drop=True) | |
| y = y_g[keep].astype(bool) | |
| S = {"model_gaze_off_onset_p@3s": s_model[keep], "dms_att@3s": C.att3s.to_numpy(float), "rule_absyawdev_t": s_rule[keep], "persistence": np.zeros(int(keep.sum()))} | |
| dong = C.dongle.to_numpy() | |
| n, n_pos, n_dev = int(keep.sum()), int(y.sum()), int(len(np.unique(dong))) | |
| print(f"rows in the gaze_off_on population with every score: {n} (our onset positives {n_pos}, devices {n_dev}; dropped: anchor not inactive/valid {int((~inpop).sum())}, " | |
| f"model NaN {int((inpop & ~np.isfinite(s_model)).sum())}, att NaN {int((inpop & ~np.isfinite(G.att3s.to_numpy())).sum())})", flush=True) | |
| rows = [] | |
| na = Path(a.a).name | |
| label = "gaze_off_on within 3 s (ours; anchor inactive)" | |
| for strat, m in (("pooled", np.ones(n, bool)), ("manual", (C.regime == "manual").to_numpy()), ("engaged", (C.regime == "engaged").to_numpy())): | |
| if m.sum() < 20 or y[m].sum() < 5 or y[m].sum() >= m.sum(): | |
| print(f" {strat}: n={int(m.sum())} pos={int(y[m].sum())} — skipped", flush=True) | |
| continue | |
| for colname in ("model_gaze_off_onset_p@3s", "dms_att@3s", "rule_absyawdev_t"): | |
| for comp in ("dms_att@3s", "rule_absyawdev_t", "persistence"): | |
| if comp == colname: | |
| continue | |
| for metric in ("pr_auc", "auroc"): | |
| if comp == "persistence" and metric == "auroc": | |
| continue | |
| fn = (lambda yy, sa, sb, dd: boot_pair(yy, sa, sb, dd, a.n_boot, 0, "ap")) if metric == "pr_auc" else (lambda yy, sa, sb, dd: boot_pair_auroc(yy, sa, sb, dd, a.n_boot, 0)) | |
| res = fn(y[m], S[colname][m], S[comp][m], dong[m]) | |
| rows.append({"run_stamp": stamp, "tag": a.tag, "seed": seed, "regime": strat, "n": int(m.sum()), "n_pos": int(y[m].sum()), "n_dev": res["n_dev"], "column": f"{na}::{colname}" if colname.startswith("model") else colname, | |
| "comparator": comp, "metric": metric, "value": res["value_a"], "ci_lo": res["ci_a"][0], "ci_hi": res["ci_a"][1], "value_b": res["value_b"], "delta": res["delta"], | |
| "delta_ci_lo": res["ci_d"][0], "delta_ci_hi": res["ci_d"][1], "frac_boot_d_gt0": res["frac_gt0"], "n_boot": a.n_boot, "label": label, | |
| "note": "clean_test gold rows anchored in test (legacy_first anchors, seed-matched legacy CSV); DMS att@3s read via warning_gold.merge_legacy as a COMPARATOR only"}) | |
| print(f" {strat:8s} {colname:28s} vs {comp:18s} {metric:7s} A={res['value_a']:.4f} B={res['value_b']:.4f} Δ={res['delta']:+.4f} [{res['ci_d'][0]:+.4f},{res['ci_d'][1]:+.4f}] n={int(m.sum())} pos={int(y[m].sum())}", flush=True) | |
| if a.no_append: | |
| print(f"no-append: {len(rows)} rows computed ({time.time() - t0:.0f}s)") | |
| return | |
| GOLD_DIR.mkdir(parents=True, exist_ok=True) | |
| new = not GOLD_CSV.exists() | |
| with open(GOLD_CSV, "a", newline="") as f: | |
| w = csv.DictWriter(f, fieldnames=GOLD_COLS) | |
| if new: | |
| w.writeheader() | |
| for r in rows: | |
| w.writerow(r) | |
| print(f"appended {len(rows)} rows to {GOLD_CSV} ({time.time() - t0:.0f}s)") | |
| write_run_json(GOLD_DIR, "gaze_chain_eval_gold", [Path(__file__), CWM2 / "eval" / "warning_gold.py"], | |
| [Path(a.a) / f"pred_layout_{r}_test.npz" for r in ("manual", "engaged")] + [GAZE / f"gaze_{r}_test.npz" for r in ("manual", "engaged")] | |
| + [WG.LEGACY / f"episodes_{r}_s{seed}.csv" for r in WG.REGIMES], seed=seed, t_start=t0, input_listing=False, | |
| extra={"run_stamp": stamp, "tag": a.tag, "a": a.a, "n_rows": len(rows), "n_gold_rows_used": n, "n_pos_ours": n_pos, "dms_fields_used": [], | |
| "dms_comparator_columns": ["att@3s (legacy CSV via warning_gold.merge_legacy; comparator only, not stored)"], "layout_guard": guards}) | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--regime", default="", choices=["", "manual", "engaged"]) | |
| ap.add_argument("--a", required=True) | |
| ap.add_argument("--b", default="") | |
| ap.add_argument("--b-rule", action="store_true", help="B = the current-state rule |lm_yaw_dev|(t) (sign-flipped for the off event)") | |
| ap.add_argument("--b-persistence", action="store_true", help="only the base-rate (persistence) rows") | |
| ap.add_argument("--dms-gold", action="store_true", help="gold clean-test rows: model vs DMS att@3s / rule / persistence on OUR gaze_off_on label -> results/gaze_chain/gaze_gold.csv") | |
| ap.add_argument("--seed", type=int, default=0, help="--dms-gold: seed-matched legacy CSV") | |
| ap.add_argument("--split", default="test") | |
| ap.add_argument("--tag", default="") | |
| ap.add_argument("--n-boot", type=int, default=1000) | |
| ap.add_argument("--min-rows", type=int, default=1000) | |
| ap.add_argument("--min-pos", type=int, default=20) | |
| ap.add_argument("--auroc", action="store_true", help="also the AUROC rows (population mode)") | |
| ap.add_argument("--no-append", action="store_true") | |
| a = ap.parse_args() | |
| t0 = time.time() | |
| stamp = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime()) | |
| if a.dms_gold: | |
| return main_gold(a, t0, stamp) | |
| assert a.regime, "--regime required in population mode" | |
| assert sum([bool(a.b), a.b_rule, a.b_persistence]) == 1, "exactly one of --b / --b-rule / --b-persistence" | |
| return main_population(a, t0, stamp) | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 17.9 kB
- Xet hash:
- f4e367dabd4aeff44eebea5eebb7c2c2b3e8fb8573e26a4be68ddaf04e7ce70d
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.