Buckets:

AsianPlayer
/

TriDrive

Files
xet
AsianPlayer/TriDrive / code /eval /future_state_eval.py
AsianPlayer
16 days ago
download
raw
32.7 kB
"""cwm3/eval/future_state_eval.py — results/tables/future_state_eval.md (+ .json): F2′ future-state occupancy / recovery evaluation (R2 WP4 batch 1, 2026-09-16;
pre-registration docs/decisions.md 2026-09-16T06:46:35Z, dimensions (ii) (iii) (vi)).  CPU, numpy only; reads the FS runs' fs_{regime}_{split}.npz, the FROZEN FINAL
joint runs' pred_{regime}_{split}.npz (their own hands_on / head_turned hazards -> closed-form pi, first-passage law), the v2 causal kin / gaze targets (true future
states) and the F2′ TRAIN caches (results/runs/FS_cache: dwell features + targets, for the dwell-table baseline).
  python eval/future_state_eval.py [--name dwell] [--seeds 0,1,2] [--split test] [--n-boot 500] [--out results/tables/future_state_eval.md]
Per regime x state (hands_on, head_turned, gaze_off) x horizon (1 / 2 / 3 / 5 s = steps 5 / 10 / 15 / 25), rows = anchors with a causal box, a defined current
state and an observed true state at t + k dt:
  Brier and AUROC of P(state = 1 at t + k) for  FS = the F2′ head;  (a) persistence pi_k = z0;  (b) dwell table = TRAIN empirical P(s_{t+k} = 1 | z0, time-in-state
  bin) — the "dwell-timer extrapolation" baseline (no model);  (c) FINAL closed form = forward_marginal of the frozen model's own hands / head hazards (n/a for gaze_off).
  Δ = FS − baseline, shared-resample device-cluster bootstrap CI (n_boot, seed 0) per seed; verdict = kill_checks.beats over the 3 seeds (report_joint.verdict, lower better
  for Brier).  Calibration: ECE (10 equal-width bins) of pi at 3 s.  Segment classes at 3 s from the TRUE states: static (on-task, stays), onset (on-task -> off-task
  within 3 s), sustained (off-task, stays), recovery (off-task -> on-task within 3 s); Brier per class for FS / persistence / FINAL closed form.
  Recovery / remaining dwell (dimension iii): rows in the OFF-task state whose first-leave time is observed (event within 5 s or the full 5 s observed without a leave);
  MAE (s) of E[min(T, 5.2 s)] for FS rem_dwell_E_s, (b) the dwell-table conditional mean, (c) the FINAL first-passage law; calibration = ECE of P(recover within 3 s).
  First-leave time on ALL rows (any switch from the current state) likewise (FS first_leave_E_s vs (b) / (c)).
Nothing here reads an openpilot DMS field.
"""
from __future__ import annotations

import argparse
import json
import sys
import time
from pathlib import Path

import numpy as np
import pandas as pd

CWM3 = Path(__file__).resolve().parents[1]
for p in (CWM3 / "eval", CWM3 / "data", CWM3 / "train", CWM3.parent / "cwm2" / "eval"):
    sys.path.insert(0, str(p))
import report_joint as RJ                                            # noqa: E402
import kill_checks as KC                                             # noqa: E402

RUNS = CWM3 / "results" / "runs"
CACHE = RUNS / "FS_cache"
KIN = CWM3 / "results" / "targets_kin_causal_routes_v2"
GAZE = CWM3 / "results" / "targets_gaze_causal_routes_v2"
STATES = ("hands_on", "head_turned", "gaze_off")
OFF_VALUE = {"hands_on": 0.0, "head_turned": 1.0, "gaze_off": 1.0}
CHAIN_COL = {"hands_on": 4, "head_turned": 5}                        # columns of the joint pred pack hz_on / hz_off / z0 (CHAINS order)
DWELL_FEAT = {"hands_on": "hands_state_dwell_s", "head_turned": "head_state_dwell_s", "gaze_off": "gz_dwell0_s"}
HORIZONS = {"1s": 5, "2s": 10, "3s": 15, "5s": 25}
H, DT = 25, 0.2
DWELL_BINS = np.array([0.0, 0.5, 1.0, 2.0, 4.0, 8.0, np.inf])
FINAL_RUN = "I1_all_minus_road2drv_s{seed}_final"
REGIMES = ("manual", "engaged")


# ----------------------------------------------------------------------------- chain algebra (numpy)
def forward_marginal(h_on, h_off, z0):
    pi = np.empty_like(h_on, dtype=np.float64)
    prev = z0.astype(np.float64)
    for k in range(h_on.shape[1]):
        prev = prev * (1 - h_off[:, k]) + (1 - prev) * h_on[:, k]
        pi[:, k] = prev
    return pi


def first_leave_law_from_hazards(h_on, h_off, z0):
    """q [N, H+1]: q_k = prod_{j<k}(1 - h_leave_j) h_leave_k with h_leave = h_off where z0 = 1 else h_on; q_H = survival beyond H."""
    h = np.where(z0[:, None] > 0.5, h_off, h_on).astype(np.float64)
    surv = np.cumprod(1 - h, 1)
    prev = np.concatenate([np.ones((len(h), 1)), surv[:, :-1]], 1)
    return np.concatenate([prev * h, surv[:, -1:]], 1)


def expected_leave_s(p):
    return (p * (np.arange(1, p.shape[1] + 1) * DT)[None, :]).sum(1)


def leave_targets_np(z0, S, valid):
    """(event [N] bool, k [N] int): event at the first observed step differing from z0 within the fully observed prefix; else k = prefix length."""
    prefix = np.cumprod(valid.astype(np.int64), 1)
    diff = (S != z0[:, None]) & (prefix > 0)
    has = diff.any(1)
    first = np.where(has, diff.argmax(1) + 1, 0)
    c = prefix.sum(1)
    return has, np.where(has, first, c)


# ----------------------------------------------------------------------------- metrics + device bootstrap
def _pct(v):
    return float(np.nanpercentile(v, 2.5)), float(np.nanpercentile(v, 97.5))


def boot_mean_pair(xa, xb, dev, n_boot, seed=0):
    """paired device-cluster bootstrap of two per-row loss vectors (means): -> value_a, value_b, delta, ci_d."""
    uniq, inv = np.unique(dev, return_inverse=True)
    sa = np.bincount(inv, weights=xa, minlength=len(uniq))
    sb = np.bincount(inv, weights=xb, minlength=len(uniq))
    cnt = np.bincount(inv, minlength=len(uniq)).astype(np.float64)
    rng = np.random.default_rng(seed)
    d = np.empty(n_boot)
    for i in range(n_boot):
        w = np.bincount(rng.integers(0, len(uniq), len(uniq)), minlength=len(uniq)).astype(np.float64)
        n = (w * cnt).sum()
        d[i] = (w * sa).sum() / n - (w * sb).sum() / n
    return float(xa.mean()), float(xb.mean()), float(xa.mean() - xb.mean()), _pct(d)


class AurocBoot:
    """weighted AUROC with tie handling, scores pre-sorted once; per bootstrap O(N)."""

    def __init__(self, y, s):
        order = np.argsort(s, kind="mergesort")
        self.y = y[order].astype(np.float64)
        ss = s[order]
        self.starts = np.flatnonzero(np.concatenate([[True], ss[1:] != ss[:-1]]))
        self.order = order

    def value(self, w):
        w = w[self.order]
        wp, wn = w * self.y, w * (1 - self.y)
        gp, gn = np.add.reduceat(wp, self.starts), np.add.reduceat(wn, self.starts)
        cum_neg_before = np.concatenate([[0.0], np.cumsum(gn)[:-1]])
        P, N = wp.sum(), wn.sum()
        if P <= 0 or N <= 0:
            return np.nan
        return float((gp * (cum_neg_before + 0.5 * gn)).sum() / (P * N))


def boot_auroc_pair(y, sa, sb, dev, n_boot, seed=0):
    uniq, inv = np.unique(dev, return_inverse=True)
    A, B = AurocBoot(y, sa), AurocBoot(y, sb)
    ones = np.ones(len(y))
    va, vb = A.value(ones), B.value(ones)
    rng = np.random.default_rng(seed)
    d = np.full(n_boot, np.nan)
    for i in range(n_boot):
        w = np.bincount(rng.integers(0, len(uniq), len(uniq)), minlength=len(uniq)).astype(np.float64)[inv]
        d[i] = A.value(w) - B.value(w)
    return va, vb, va - vb, _pct(d)


def ece(p, y, n_bins=10):
    b = np.clip((p * n_bins).astype(int), 0, n_bins - 1)
    tot = 0.0
    for i in range(n_bins):
        m = b == i
        if m.any():
            tot += m.mean() * abs(p[m].mean() - y[m].mean())
    return float(tot)


def verdict_rows(rows, lower_better):
    """rows: list of dict(seed, value_a, value_b, delta, lo, hi) -> kill_checks verdict dict."""
    g = pd.DataFrame({"seed": [r["seed"] for r in rows], "delta": [r["delta"] for r in rows], "delta_ci_lo": [r["lo"] for r in rows], "delta_ci_hi": [r["hi"] for r in rows],
                      "value_a": [r["value_a"] for r in rows], "value_b": [r["value_b"] for r in rows]})
    return RJ.verdict(g, lower_better)


# ----------------------------------------------------------------------------- data
def load_truth(regime, split):
    with np.load(KIN / f"kin_causal_{regime}_{split}.npz", allow_pickle=True) as z:
        S = z["S_c"].astype(np.float32)
        S0 = z["S0_c"].astype(np.float32)
        ok = z["row_ok"].astype(bool)
        uid, t, dongle = z["uid"], z["t"], z["dongle"]
    with np.load(GAZE / f"gaze_{regime}_{split}.npz", allow_pickle=True) as z:
        assert np.array_equal(z["uid"], uid)
        gok = z["row_ok"].astype(bool)
        Sg = z["S_g"].astype(np.float32)
        Sg0 = z["S0_g"].astype(np.float32)
    S[~ok] = np.nan
    S0[~ok] = np.nan
    Sg[~gok] = np.nan
    Sg0[~gok] = np.nan
    return {"S": np.concatenate([S, Sg[:, :, None]], -1), "S0": np.concatenate([S0, Sg0[:, None]], 1), "uid": uid, "t": t, "dongle": dongle}


def load_fs(run, regime, split):
    with np.load(RUNS / run / f"fs_{regime}_{split}.npz", allow_pickle=True) as z:
        d = {k: z[k] for k in ("uid", "pi", "z0", "z0valid", "valid", "rem_dwell_E_s", "first_leave_E_s", "rem_within_3s", "leave_within_3s", "rem_dwell_p", "first_leave_p")}
        d["meta"] = json.loads(str(z["meta"]))
        assert [str(s) for s in z["state_names"]] == list(STATES)
    return d


def load_final_pi(final_run, regime, split):
    with np.load(RUNS / final_run / f"pred_{regime}_{split}.npz", allow_pickle=True) as z:
        hon, hoff, z0 = z["hz_on"].astype(np.float32), z["hz_off"].astype(np.float32), z["z0"].astype(np.float32)
    out = {}
    for s, c in CHAIN_COL.items():
        pi = forward_marginal(hon[:, :, c], hoff[:, :, c], z0[:, c])
        q = first_leave_law_from_hazards(hon[:, :, c], hoff[:, :, c], z0[:, c])
        out[s] = {"pi": pi.astype(np.float32), "E_leave": expected_leave_s(q).astype(np.float32), "p_leave_3s": q[:, :15].sum(1).astype(np.float32)}
    return out


def dwell_table(regimes):
    """TRAIN empirical tables per state: P(s_{t+k} = 1 | z0, dwell bin) [2, n_bins, H], the conditional mean of min(T, 5.2 s) and P(leave <= 3 s) [2, n_bins]."""
    tab = {}
    Xs, Ss, S0s = [], [], []
    for r in regimes:
        with np.load(CACHE / f"feats_{r}_train.npz", allow_pickle=True) as z:
            X, names = z["X"], [str(n) for n in z["names"]]
        zc = sorted(CACHE.glob(f"zU_*_{r}_train.npz"))
        assert zc, f"no z_U train cache for {r} (run train/train_future_state.py first)"
        with np.load(zc[0], allow_pickle=True) as z:
            S = np.concatenate([z["S"].astype(np.float32), z["Sg"].astype(np.float32)[:, :, None]], -1)
            S0 = np.concatenate([z["S0"], z["Sg0"][:, None]], 1).astype(np.float32)
        Xs.append(X); Ss.append(S); S0s.append(S0)
    X, S, S0 = np.concatenate(Xs), np.concatenate(Ss), np.concatenate(S0s)
    nb = len(DWELL_BINS) - 1
    for si, s in enumerate(STATES):
        d = X[:, names.index(DWELL_FEAT[s])]
        db = np.where(np.isfinite(d), np.clip(np.searchsorted(DWELL_BINS, d, side="right") - 1, 0, nb - 1), nb)     # bin nb = unknown dwell
        pi_t = np.full((2, nb + 1, H), np.nan, np.float32)
        E_t = np.full((2, nb + 1), np.nan, np.float32)
        p3_t = np.full((2, nb + 1), np.nan, np.float32)
        z0 = S0[:, si]
        ok0 = np.isfinite(z0)
        valid = np.isfinite(S[:, :, si])
        ev, k = leave_targets_np(np.nan_to_num(z0), np.nan_to_num(S[:, :, si]), valid)
        obs = ok0 & (ev | (k == H))
        tmin = np.minimum(np.where(ev, k, H + 1), H + 1) * DT
        for v in (0, 1):
            for b in range(nb + 1):
                m = ok0 & (z0 == v) & (db == b)
                if m.sum() < 50:
                    continue
                for kk in range(H):
                    mk = m & valid[:, kk]
                    if mk.sum() >= 50:
                        pi_t[v, b, kk] = S[mk, kk, si].mean()
                mo = m & obs
                if mo.sum() >= 50:
                    E_t[v, b] = tmin[mo].mean()
                    p3_t[v, b] = (ev[mo] & (k[mo] <= 15)).mean()
        tab[s] = {"pi": pi_t, "E_leave": E_t, "p_leave_3s": p3_t, "names": names}
    return tab


def apply_table(tab, s, z0, X, names):
    nb = len(DWELL_BINS) - 1
    d = X[:, names.index(DWELL_FEAT[s])]
    db = np.where(np.isfinite(d), np.clip(np.searchsorted(DWELL_BINS, d, side="right") - 1, 0, nb - 1), nb)
    v = np.nan_to_num(z0).astype(int).clip(0, 1)
    pi = tab[s]["pi"][v, db]                                      # [N,H]
    # fall back to the persistence value where the table has no entry
    pi = np.where(np.isfinite(pi), pi, v[:, None].astype(np.float32))
    E = tab[s]["E_leave"][v, db]
    p3 = tab[s]["p_leave_3s"][v, db]
    return pi.astype(np.float32), E.astype(np.float32), p3.astype(np.float32)


# ----------------------------------------------------------------------------- main evaluation
def evaluate(name, seeds, split, n_boot, regimes):
    t0 = time.time()
    tab = dwell_table(regimes)
    cells, seg_rows, rec_rows, cal_rows = {}, {}, {}, {}
    prov = []
    for seed in seeds:
        run, final_run = f"FS_{name}_s{seed}", FINAL_RUN.format(seed=seed)
        for regime in regimes:
            truth = load_truth(regime, split)
            fs = load_fs(run, regime, split)
            assert np.array_equal(fs["uid"], truth["uid"]), "fs rows are not the kin target rows"
            fin = load_final_pi(final_run, regime, split)
            with np.load(CACHE / f"feats_{regime}_{split}.npz", allow_pickle=True) as z:
                Xf, names = z["X"], [str(n) for n in z["names"]]
                assert np.array_equal(z["uid"], truth["uid"])
            prov.append({"seed": seed, "regime": regime, "fs_run": run, "fs_model_id": fs["meta"].get("model_id"), "fs_code": str(fs["meta"].get("code_sha256", ""))[:12],
                         "final_run": final_run, "n_rows": int(len(truth["t"]))})
            dev = truth["dongle"]
            for si, s in enumerate(STATES):
                z0 = truth["S0"][:, si]
                base_ok = fs["valid"].astype(bool) & np.isfinite(z0) & fs["z0valid"][:, si]
                pi_fs = fs["pi"][:, :, si].astype(np.float32)
                pi_pers = np.repeat(np.nan_to_num(z0)[:, None], H, 1).astype(np.float32)
                pi_tab, E_tab, p3_tab = apply_table(tab, s, z0, Xf, names)
                pi_fin = fin[s]["pi"] if s in fin else None
                baselines = {"persistence": pi_pers, "dwell_table": pi_tab}
                if pi_fin is not None:
                    baselines["final_closed_form"] = pi_fin
                for hn, k in HORIZONS.items():
                    y = truth["S"][:, k - 1, si]
                    m = base_ok & np.isfinite(y)
                    yb = y[m].astype(np.float64)
                    if yb.sum() < 20 or (1 - yb).sum() < 20:
                        continue
                    for bn, pb in baselines.items():
                        key = (regime, s, hn, bn)
                        a, b = pi_fs[m, k - 1].astype(np.float64), pb[m, k - 1].astype(np.float64)
                        va, vb, d, ci = boot_mean_pair((a - yb) ** 2, (b - yb) ** 2, dev[m], n_boot)
                        ua, ub, du, ciu = boot_auroc_pair(yb, a, b, dev[m], min(n_boot, 300))
                        cells.setdefault(key, []).append({"seed": seed, "brier": {"value_a": va, "value_b": vb, "delta": d, "lo": ci[0], "hi": ci[1]},
                                                          "auroc": {"value_a": ua, "value_b": ub, "delta": du, "lo": ciu[0], "hi": ciu[1]}, "n": int(m.sum()), "pos": int(yb.sum())})
                # calibration at 3 s
                y3 = truth["S"][:, 14, si]
                m3 = base_ok & np.isfinite(y3)
                cal = {"FS": ece(pi_fs[m3, 14], y3[m3]), "dwell_table": ece(pi_tab[m3, 14], y3[m3])}
                if pi_fin is not None:
                    cal["final_closed_form"] = ece(pi_fin[m3, 14], y3[m3])
                cal_rows.setdefault((regime, s), []).append(dict(cal, seed=seed, n=int(m3.sum())))
                # segment classes at 3 s (true states)
                off = OFF_VALUE[s]
                pre = truth["S"][:, :15, si]
                full = m3 & np.isfinite(pre).all(1)
                cur_off = np.nan_to_num(z0) == off
                any_off = (pre == off).any(1)
                any_on = (pre == 1 - off).any(1)
                classes = {"static": full & ~cur_off & ~any_off, "onset": full & ~cur_off & any_off, "sustained": full & cur_off & ~any_on, "recovery": full & cur_off & any_on}
                for cn, cm in classes.items():
                    if cm.sum() < 50:
                        continue
                    yy = y3[cm]
                    row = {"seed": seed, "n": int(cm.sum()), "share": float(cm.sum() / max(full.sum(), 1)), "FS": float(((pi_fs[cm, 14] - yy) ** 2).mean()),
                           "persistence": float(((pi_pers[cm, 14] - yy) ** 2).mean()), "dwell_table": float(((pi_tab[cm, 14] - yy) ** 2).mean())}
                    if pi_fin is not None:
                        row["final_closed_form"] = float(((pi_fin[cm, 14] - yy) ** 2).mean())
                    seg_rows.setdefault((regime, s, cn), []).append(row)
                # recovery time / remaining dwell (off-task rows with an observed first-leave) and first-leave time (all rows)
                valid = np.isfinite(truth["S"][:, :, si])
                ev, kk = leave_targets_np(np.nan_to_num(z0), np.nan_to_num(truth["S"][:, :, si]), valid)
                obs = base_ok & (ev | (kk == H))
                tmin = (np.minimum(np.where(ev, kk, H + 1), H + 1) * DT).astype(np.float64)
                for which, rows_m, E_fs, p3_fs in (("remaining_dwell_off", obs & cur_off, fs["rem_dwell_E_s"][:, si], fs["rem_within_3s"][:, si].astype(np.float32)),
                                                    ("first_leave_all", obs, fs["first_leave_E_s"][:, si], fs["leave_within_3s"][:, si].astype(np.float32))):
                    if rows_m.sum() < 100:
                        continue
                    y_ev3 = (ev[rows_m] & (kk[rows_m] <= 15)).astype(np.float64)
                    comp = {"dwell_table": (E_tab, p3_tab)}
                    if s in fin:
                        comp["final_closed_form"] = (fin[s]["E_leave"], fin[s]["p_leave_3s"])
                    rec = {"seed": seed, "n": int(rows_m.sum()), "event_share": float(ev[rows_m].mean()), "mae_FS": float(np.abs(E_fs[rows_m] - tmin[rows_m]).mean()),
                           "ece3_FS": ece(np.clip(p3_fs[rows_m], 0, 1), y_ev3), "obs_rate3": float(y_ev3.mean())}
                    for bn, (Eb, p3b) in comp.items():
                        Eb_ = np.where(np.isfinite(Eb), Eb, (H + 1) * DT)
                        va, vb, d, ci = boot_mean_pair(np.abs(E_fs[rows_m] - tmin[rows_m]), np.abs(Eb_[rows_m] - tmin[rows_m]), dev[rows_m], n_boot)
                        rec[f"mae_{bn}"] = vb
                        rec[f"delta_{bn}"] = {"value_a": va, "value_b": vb, "delta": d, "lo": ci[0], "hi": ci[1]}
                        rec[f"ece3_{bn}"] = ece(np.clip(np.nan_to_num(p3b[rows_m], nan=float(y_ev3.mean())), 0, 1), y_ev3)
                    rec_rows.setdefault((regime, s, which), []).append(rec)
            print(f"[{time.time() - t0:.0f}s] seed {seed} {regime}/{split} done", flush=True)
    return cells, seg_rows, rec_rows, cal_rows, prov


def render(name, seeds, split, n_boot, cells, seg_rows, rec_rows, cal_rows, prov, out):
    def ms(v):
        v = [float(x) for x in v]
        return f"{np.mean(v):.4f} ± {np.std(v, ddof=1):.4f}" if len(v) > 1 else f"{v[0]:.4f}"
    L = [f"# F2′ future-state head — occupancy / recovery evaluation (`FS_{name}_s<k>`, {split} split, seeds {seeds}, n_boot {n_boot})", "",
         "Detached horizon-conditioned head (models/future_state_head.py, ~0.034 M params) on the FROZEN FINAL joint model's driver state z_U + label-free dwell features "
         "(train/train_future_state.py FEATURES), trained on the TRAIN anchors of both regimes, selected on the validation occupancy BCE.  States: hands_on / head_turned "
         "(v2 causal kin targets) and gaze_off (our own landmark chain, validation pending).  Baselines: (a) persistence pi_k = z0; (b) dwell table = TRAIN empirical "
         "P(s_{t+k} | z0, time-in-state bin) (dwell-timer extrapolation, no model); (c) FINAL closed form = forward_marginal of the frozen model's own hazards (hands / head only).  "
         "Δ = FS − baseline, shared-resample device-cluster bootstrap per seed; verdict = kill_checks.beats over seeds (Brier lower better: BEATS = FS lower).  "
         "Nothing here changes the headline model; the adoption question of F2′ is answered downstream by the WP2 timing evaluator (alert_timing_eval) and the warning head.", "",
         "## Provenance", "", "| seed | regime | FS run | FS model_id (frozen joint) | FS code | FINAL run | rows |", "|---|---|---|---|---|---|---|"]
    L += [f"| {p['seed']} | {p['regime']} | {p['fs_run']} | {p['fs_model_id']} | {p['fs_code']} | {p['final_run']} | {p['n_rows']:,} |" for p in prov]
    js = {"name": name, "seeds": seeds, "split": split, "n_boot": n_boot, "cells": {}, "segments": {}, "recovery": {}, "calibration": {}, "provenance": prov}
    summary = {}
    for metric, lb in (("brier", True), ("auroc", False)):
        L += ["", f"## Occupancy {metric.upper()} of P(state = 1 at t + k) — FS vs baselines (Δ = FS − baseline{', lower better' if lb else ''})", "",
              "| regime | state | horizon | baseline | FS per seed | baseline per seed | Δ per seed [CI] | mean Δ ± sd | verdict | n / pos |", "|---|---|---|---|---|---|---|---|---|---|"]
        for (regime, s, hn, bn), rows in sorted(cells.items(), key=lambda kv: (kv[0][0], STATES.index(kv[0][1]), HORIZONS[kv[0][2]], kv[0][3])):
            rr = [dict(r[metric], seed=r["seed"]) for r in rows]
            v = verdict_rows(rr, lb)
            d = np.array([r["delta"] for r in rr])
            A = " / ".join(f"{r['value_a']:.4f}" for r in rr)
            B = " / ".join(f"{r['value_b']:.4f}" for r in rr)
            D = " / ".join(f"{r['delta']:+.4f} [{r['lo']:+.4f},{r['hi']:+.4f}]" for r in rr)
            L.append(f"| {regime} | {s} | {hn} | {bn} | {A} | {B} | {D} | {d.mean():+.4f} ± {d.std(ddof=1) if len(d) > 1 else 0:.4f} | {v['verdict']} | {rows[0]['n']:,} / {rows[0]['pos']:,} |")
            js["cells"][f"{regime}|{s}|{hn}|{bn}|{metric}"] = {"rows": rr, "verdict": v["verdict"], "mean_delta": float(d.mean())}
            summary.setdefault((metric, bn), []).append(v["verdict"])
    L += ["", "## Verdict counts (cells = regime x state x horizon)", "", "| metric | baseline | BEATS | NOT_WORSE | INCONCLUSIVE | WORSE | INSUFFICIENT |", "|---|---|---|---|---|---|---|"]
    for (metric, bn), vs in sorted(summary.items()):
        L.append(f"| {metric} | {bn} | " + " | ".join(str(vs.count(x)) for x in ("BEATS", "NOT_WORSE", "INCONCLUSIVE", "WORSE", "INSUFFICIENT")) + " |")
        js.setdefault("verdict_counts", {})[f"{metric}|{bn}"] = {x: vs.count(x) for x in ("BEATS", "NOT_WORSE", "INCONCLUSIVE", "WORSE", "INSUFFICIENT")}
    L += ["", "## Calibration at 3 s — ECE (10 bins) of P(state = 1 at t + 3 s)", "", "| regime | state | FS | dwell table | FINAL closed form | n |", "|---|---|---|---|---|---|"]
    for (regime, s), rows in sorted(cal_rows.items(), key=lambda kv: (kv[0][0], STATES.index(kv[0][1]))):
        L.append(f"| {regime} | {s} | {ms([r['FS'] for r in rows])} | {ms([r['dwell_table'] for r in rows])} | {ms([r['final_closed_form'] for r in rows]) if 'final_closed_form' in rows[0] else 'n/a'} | {rows[0]['n']:,} |")
        js["calibration"][f"{regime}|{s}"] = rows
    L += ["", "## Segment classes at 3 s (true states): Brier of P(state at t + 3 s) per class", "",
          "| regime | state | class | share of rows | FS | persistence | dwell table | FINAL closed form | n |", "|---|---|---|---|---|---|---|---|---|"]
    for (regime, s, cn), rows in sorted(seg_rows.items(), key=lambda kv: (kv[0][0], STATES.index(kv[0][1]), ("static", "onset", "sustained", "recovery").index(kv[0][2]))):
        L.append(f"| {regime} | {s} | {cn} | {np.mean([r['share'] for r in rows]):.3f} | {ms([r['FS'] for r in rows])} | {ms([r['persistence'] for r in rows])} | {ms([r['dwell_table'] for r in rows])} | "
                 f"{ms([r['final_closed_form'] for r in rows]) if 'final_closed_form' in rows[0] else 'n/a'} | {rows[0]['n']:,} |")
        js["segments"][f"{regime}|{s}|{cn}"] = rows
    L += ["", "## Recovery time (remaining dwell in the off-task state) and first-leave time — MAE (s) of E[min(T, 5.2 s)] and calibration of P(leave within 3 s)", "",
          "Rows: anchors whose first-leave time is observed (event within 5 s, or the full 5 s observed without a leave -> target 5.2 s).  Δ = FS − baseline MAE (lower better), device-cluster bootstrap.", "",
          "| regime | state | rows | n | event share | MAE FS | MAE dwell table | Δ vs table per seed [CI] | verdict | MAE FINAL closed form | Δ vs FINAL per seed [CI] | verdict | ECE3 FS / table / FINAL | observed P(leave ≤ 3 s) |",
          "|---|---|---|---|---|---|---|---|---|---|---|---|---|---|"]
    for (regime, s, which), rows in sorted(rec_rows.items(), key=lambda kv: (kv[0][0], STATES.index(kv[0][1]), kv[0][2])):
        vt = verdict_rows([dict(r["delta_dwell_table"], seed=r["seed"]) for r in rows], True)
        dt_ = " / ".join(f"{r['delta_dwell_table']['delta']:+.3f} [{r['delta_dwell_table']['lo']:+.3f},{r['delta_dwell_table']['hi']:+.3f}]" for r in rows)
        if "delta_final_closed_form" in rows[0]:
            vf = verdict_rows([dict(r["delta_final_closed_form"], seed=r["seed"]) for r in rows], True)
            df_ = " / ".join(f"{r['delta_final_closed_form']['delta']:+.3f} [{r['delta_final_closed_form']['lo']:+.3f},{r['delta_final_closed_form']['hi']:+.3f}]" for r in rows)
            mf, ef, vfv = ms([r["mae_final_closed_form"] for r in rows]), ms([r["ece3_final_closed_form"] for r in rows]), vf["verdict"]
        else:
            mf, df_, ef, vfv = "n/a", "n/a", "n/a", "n/a"
        L.append(f"| {regime} | {s} | {which} | {rows[0]['n']:,} | {np.mean([r['event_share'] for r in rows]):.3f} | {ms([r['mae_FS'] for r in rows])} | {ms([r['mae_dwell_table'] for r in rows])} | {dt_} | {vt['verdict']} | "
                 f"{mf} | {df_} | {vfv} | {ms([r['ece3_FS'] for r in rows])} / {ms([r['ece3_dwell_table'] for r in rows])} / {ef} | {np.mean([r['obs_rate3'] for r in rows]):.3f} |")
        js["recovery"][f"{regime}|{s}|{which}"] = {"rows": rows, "verdict_vs_table": vt["verdict"], "verdict_vs_final": vfv}
    out = Path(out); out.parent.mkdir(parents=True, exist_ok=True)
    out.write_text("\n".join(L) + "\n")
    out.with_suffix(".json").write_text(json.dumps(js, indent=1, default=str))
    return L


def attribution_section(arms, out, split):
    """--arms: append a '## Attribution arms' block to the md of the first arm from the per-arm json files (future_state_eval[_<arm>].json): per regime x state x
    horizon the Brier / AUROC of every arm next to the baselines (persistence, dwell table, FINAL closed form) and each arm's verdict vs the FINAL closed form and vs
    the dwell table; plus the verdict counts.  Arms: dwell (z_U + dwell features), zonly (z_U + regime only), featonly (dwell features + regime, z_U zeroed)."""
    js = {}
    for a in arms:
        pth = Path(out).with_suffix(".json") if a == arms[0] else Path(out).with_name(Path(out).stem + f"_{a}.json")
        if not pth.exists():
            print(f"attribution: {pth} missing -> skipped"); continue
        js[a] = json.loads(pth.read_text())
    if len(js) < 2:
        return
    L = ["", f"## Attribution arms ({split} split): what carries the gain — the trunk state, the dwell history, or both", "",
         "Arms trained with the identical recipe / selection / caches: `dwell` = z_U + 20 dwell features + regime (the F2′ head); `zonly` = z_U + regime only (the detached head + "
         "occupancy BCE on the trunk's own information); `featonly` = dwell features + regime with z_U zeroed (the 8 s dwell / switch history alone).  Baselines as above: "
         "persistence pi_k = z0, dwell table (TRAIN empirical P(s_{t+k} | z0, time-in-state bin)), FINAL closed form (the frozen model's own hazards through forward_marginal).  "
         "Verdicts = kill_checks.beats over the 3 seeds (Brier lower better).", ""]
    for metric in ("brier", "auroc"):
        L += [f"### {metric.upper()} of P(state = 1 at t + k), 3-seed mean", "",
              "| regime | state | horizon | persistence | dwell table | FINAL closed form | " + " | ".join(js.keys()) + " | " + " | ".join(f"{a} vs FINAL" for a in js) + " | " + " | ".join(f"{a} vs table" for a in js) + " |",
              "|---|---|---|---|---|---|" + "---|" * (3 * len(js))]
        keys = sorted({tuple(k.split("|")[:3]) for a in js for k in js[a]["cells"] if k.endswith(f"|{metric}")}, key=lambda t: (t[0], STATES.index(t[1]), HORIZONS[t[2]]))
        for regime, st, hn in keys:
            def cell(a, bn):
                return js[a]["cells"].get(f"{regime}|{st}|{hn}|{bn}|{metric}")
            ref = cell(arms[0], "persistence") or next((cell(a, "persistence") for a in js if cell(a, "persistence")), None)
            if ref is None:
                continue
            pers = np.mean([r["value_b"] for r in ref["rows"]])
            tab = cell(arms[0], "dwell_table"); tab = np.mean([r["value_b"] for r in tab["rows"]]) if tab else float("nan")
            fin = cell(arms[0], "final_closed_form"); fin = np.mean([r["value_b"] for r in fin["rows"]]) if fin else float("nan")
            vals = [np.mean([r["value_a"] for r in cell(a, "persistence")["rows"]]) if cell(a, "persistence") else float("nan") for a in js]
            vf = [(cell(a, "final_closed_form") or {}).get("verdict", "n/a") for a in js]
            vt = [(cell(a, "dwell_table") or {}).get("verdict", "n/a") for a in js]
            L.append(f"| {regime} | {st} | {hn} | {pers:.4f} | {tab:.4f} | {fin:.4f} | " + " | ".join(f"{v:.4f}" for v in vals) + " | " + " | ".join(vf) + " | " + " | ".join(vt) + " |")
        L.append("")
    L += ["### Verdict counts per arm (cells = regime x state x horizon; hands / head only for the FINAL closed form)", "",
          "| arm | metric | vs FINAL closed form: BEATS / NOT_WORSE / INCONCLUSIVE / WORSE | vs dwell table: BEATS / NOT_WORSE / INCONCLUSIVE / WORSE | vs persistence: BEATS / rest |", "|---|---|---|---|---|"]
    for a in js:
        vc = js[a].get("verdict_counts", {})
        for metric in ("brier", "auroc"):
            f = vc.get(f"{metric}|final_closed_form", {}); t = vc.get(f"{metric}|dwell_table", {}); pz = vc.get(f"{metric}|persistence", {})
            fmt = lambda d: " / ".join(str(d.get(x, 0)) for x in ("BEATS", "NOT_WORSE", "INCONCLUSIVE", "WORSE"))
            L.append(f"| {a} | {metric} | {fmt(f)} | {fmt(t)} | {pz.get('BEATS', 0)} / {sum(v for k, v in pz.items() if k != 'BEATS')} |")
    L += ["", "Reading: `zonly` reproduces the FINAL closed form (the head + occupancy loss add nothing on the trunk's own information); `featonly` beats the FINAL recursion on the hands "
          "chain and on engaged head_turned but is WORSE on manual head_turned (the trunk state carries the head chain in manual driving); `dwell` = both sources is best in every cell — "
          "the gain is complementary information (8 s dwell / switch history + gaze anchor state), not the loss or the head.", ""]
    with open(out, "a") as f:
        f.write("\n".join(L) + "\n")
    jp = Path(out).with_suffix(".json")
    j = json.loads(jp.read_text())
    j["attribution_arms"] = {a: js[a].get("verdict_counts", {}) for a in js}
    jp.write_text(json.dumps(j, indent=1, default=str))
    print("attribution section appended to", out)


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--name", default="dwell")
    ap.add_argument("--arms", default="", help="attribution only: comma list of finished arms (first = the main table), e.g. dwell,zonly,featonly -> appends a section to --out")
    ap.add_argument("--seeds", default="0,1,2")
    ap.add_argument("--split", default="test")
    ap.add_argument("--n-boot", type=int, default=500)
    ap.add_argument("--regimes", default="manual,engaged")
    ap.add_argument("--out", default=str(CWM3 / "results" / "tables" / "future_state_eval.md"))
    a = ap.parse_args()
    if a.arms:
        attribution_section(a.arms.split(","), a.out, a.split)
        return
    seeds = [int(s) for s in a.seeds.split(",") if (RUNS / f"FS_{a.name}_s{s}" / "PRED_DONE").exists()]
    assert seeds, "no finished FS run"
    regimes = a.regimes.split(",")
    cells, seg, rec, cal, prov = evaluate(a.name, seeds, a.split, a.n_boot, regimes)
    L = render(a.name, seeds, a.split, a.n_boot, cells, seg, rec, cal, prov, a.out)
    print("\n".join(L[:8]))
    print(f"written {a.out}")


if __name__ == "__main__":
    main()
Xet Storage Details

Size:: 32.7 kB
Xet hash:: 241aaf727983fa249b95999acddb39d1b0e212ad04f4f9c7bf59ceef586e8c3e
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.