Buckets:
| """cwm3/eval/warning_oracle_diagnostic.py — ORACLE-FEATURE DIAGNOSTIC of the learned warning head (plan R2 WP2.2; pre-registration row | |
| 2026-09-16T06:46:35Z in docs/decisions.md). CPU only, scikit-learn. | |
| python eval/warning_oracle_diagnostic.py --runs I1_all_minus_road2drv_s0_final I1_all_minus_road2drv_s1_final I1_all_minus_road2drv_s2_final [--n-boot 2000] | |
| THIS IS A DIAGNOSTIC UNDER ONE FEATURE SET AND ONE READ-OUT, NOT AN UPPER BOUND: it asks how much the fixed HGB read-out (train/train_warning_head.py | |
| GRID['hgb'], device-grouped 5-fold CV selection, monotone constraints of the known features, 979 clean TRAIN consensus labels) gains on the clean-test | |
| gold when it is handed (b) the TRUE future driver state (from the causal targets, i.e. label-free look-ahead) or (c) the joint model's PREDICTED future | |
| (the existing 144 predicted-future features), on top of the same current-state features cur+dwell+face. Levels (every level = same labels, same CV, | |
| same gold rows, same device-cluster resamples -> paired Δ): | |
| base cur+dwell+face the 77 current-state features of the `_dwfc` head (feature_mask 'current' incl. the dw_* / fc_* blocks) | |
| (a) true-future ONSET scores alone union hand_leave ∪ head_turn_onset within 1 / 3 / 5 s, gaze_off_on within 1 / 3 / 5 s, onset count in 5 s | |
| (raw-score AUROC of each score reported too — union@3 s is the 0.614 number of the 2026-09-16 exploration; plus one HGB on the 7 scores) | |
| (b) base + TRUE future occupancy / recovery future hands_off / head_turned / gaze_off / unmet share over 1 / 3 / 5 s, time to next hands-on return, | |
| time to recovery (first future step with no unmet state), time to next unmet-state entry, remaining unmet time (or_* features; 5.2 s = none within 5 s) | |
| (b+) (b) + the onset scores of (a) | |
| (c) base + the model's PREDICTED future (the 144 `is_future` features) — in-sample label-row packs; (c-oof) the same with the out-of-fold packs | |
| Reading fixed in the pre-registration: (b) gains and (c) does not -> strengthen the predictor; neither gains -> revisit the target definition / read-out. | |
| True-future arrays: results/targets_kin_causal_routes_v2/kin_causal_{regime}_{split}.npz (S_c [N,25,2] hands_on / head_turned at +0.2..+5.0 s, NaN invalid; | |
| S0_c; E_c / EM_c first-passage onsets hand_return / hand_leave / head_turn_onset) and results/targets_gaze_causal_routes_v2/gaze_{regime}_{split}.npz | |
| (S_g, S0_g, E_g / EM_g gaze_off_on / gaze_off_off). A component that is invalid at a step counts as "met" for the unmet union (under-detection disclosed via | |
| the finite shares printed in the table). Gold = the unchanged warning_gold.build_gold rows (legacy_first anchors, seed-matched surrogate rows) restricted to | |
| clean_test (512 rows / 63 positives expected); strata pooled / manual / engaged. No openpilot DMS field is read. | |
| Outputs (append-only, new files): results/tables/warning_oracle_diagnostic.md (section appended), results/warning_oracle/<stamp>/{levels.csv,deltas.csv, | |
| raw_scores.csv,run.json}. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import sys | |
| import time | |
| from pathlib import Path | |
| import numpy as np | |
| import pandas as pd | |
| CWM3 = Path(__file__).resolve().parents[1] | |
| EXPLORE = CWM3.parent | |
| CWM2 = EXPLORE / "cwm2" | |
| for p in (CWM3 / "train", CWM3 / "data", CWM3 / "eval", CWM3 / "common", CWM2 / "eval", CWM2 / "data", CWM2, CWM2 / "baselines" / "legacy_cockpitwm"): | |
| sys.path.insert(0, str(p)) | |
| import train_warning_head as TW # noqa: E402 (features_from_pred, feature_names, feature_mask, make_model, cv_auroc, GRID, LB, WG, ds) | |
| import warning_gold as WG # noqa: E402 | |
| LB, ds = TW.LB, TW.ds | |
| REGIMES = ("manual", "engaged") | |
| KINC = CWM3 / "results" / "targets_kin_causal_routes_v2" | |
| GAZE = CWM3 / "results" / "targets_gaze_causal_routes_v2" | |
| STEPS = {"1s": 5, "3s": 15, "5s": 25} | |
| CAP_S = 5.2 # "no such step within the 5 s horizon" code for the time-to-* features | |
| ONSET_NAMES = [f"or_onset_hl_ht_{k}" for k in STEPS] + [f"or_onset_gaze_{k}" for k in STEPS] + ["or_n_onsets_5s"] | |
| OCC_NAMES = [f"or_hands_off_share_{k}" for k in STEPS] + [f"or_head_turned_share_{k}" for k in STEPS] + [f"or_gaze_off_share_{k}" for k in STEPS] + \ | |
| [f"or_unmet_share_{k}" for k in STEPS] + ["or_t_hands_return_s", "or_t_recovery_s", "or_t_unmet_entry_s", "or_remaining_unmet_s"] | |
| def _share(x, k): | |
| """nanmean of x[:, :k] (NaN where no finite step).""" | |
| v = x[:, :k] | |
| fin = np.isfinite(v) | |
| with np.errstate(invalid="ignore", divide="ignore"): | |
| return np.where(fin.any(1), np.nansum(np.where(fin, v, 0.0), 1) / np.maximum(fin.sum(1), 1), np.nan).astype(np.float32) | |
| def _first_time(cond, valid): | |
| """time (s) of the first future step k (0.2 s grid, +0.2..+5.0) where cond holds among valid steps; CAP_S when none; NaN when no valid step.""" | |
| n, T = cond.shape | |
| c = cond & valid | |
| idx = np.where(c.any(1), c.argmax(1), -1) | |
| out = np.where(idx >= 0, 0.2 * (idx + 1), CAP_S).astype(np.float32) | |
| out[~valid.any(1)] = np.nan | |
| return out | |
| def true_future_features(regime, split, rows): | |
| """or_* oracle features at `rows` of (regime, split) from the causal kin / gaze targets -> (X_onset [n,7], X_occ [n,16], names_onset, names_occ, raw dict).""" | |
| rows = np.asarray(rows, np.int64) | |
| with np.load(KINC / f"kin_causal_{regime}_{split}.npz", allow_pickle=True) as z: | |
| assert [str(e) for e in z["event_names"]] == ["hand_return", "hand_leave", "head_turn_onset"] and [str(s) for s in z["state_names"]] == ["hands_on", "head_turned"] | |
| S, S0, E, EM = z["S_c"][rows].astype(np.float32), z["S0_c"][rows].astype(np.float32), z["E_c"][rows].astype(bool), z["EM_c"][rows].astype(bool) | |
| with np.load(GAZE / f"gaze_{regime}_{split}.npz", allow_pickle=True) as z: | |
| assert [str(e) for e in z["event_names"]] == ["gaze_off_on", "gaze_off_off"] | |
| Sg, S0g, Eg, EMg = z["S_g"][rows].astype(np.float32), z["S0_g"][rows].astype(np.float32), z["E_g"][rows].astype(bool), z["EM_g"][rows].astype(bool) | |
| hands_on, head, gaze = S[:, :, 0], S[:, :, 1], Sg | |
| v_h, v_d, v_g = np.isfinite(hands_on), np.isfinite(head), np.isfinite(gaze) | |
| hands_off = np.where(v_h, 1.0 - hands_on, np.nan) | |
| # unmet union: invalid component counts as met; the union is valid where any component is valid | |
| unmet_any_valid = v_h | v_d | v_g | |
| unmet = ((v_h & (hands_on < 0.5)) | (v_d & (head > 0.5)) | (v_g & (gaze > 0.5))) | |
| unmet_f = np.where(unmet_any_valid, unmet.astype(np.float32), np.nan) | |
| # onset scores (first-passage events with their masks) | |
| hl_ht = (E[:, :, 1] & EM[:, :, 1]) | (E[:, :, 2] & EM[:, :, 2]) | |
| m_hl_ht = EM[:, :, 1] | EM[:, :, 2] | |
| gz = Eg[:, :, 0] & EMg[:, :, 0] | |
| m_gz = EMg[:, :, 0] | |
| on = [] | |
| for k in STEPS.values(): | |
| on.append(np.where(m_hl_ht[:, :k].any(1), hl_ht[:, :k].any(1).astype(np.float32), np.nan)) | |
| for k in STEPS.values(): | |
| on.append(np.where(m_gz[:, :k].any(1), gz[:, :k].any(1).astype(np.float32), np.nan)) | |
| n_on = (E[:, :, 1] & EM[:, :, 1]).sum(1) + (E[:, :, 2] & EM[:, :, 2]).sum(1) + gz.sum(1) | |
| on.append(np.where(m_hl_ht.any(1) | m_gz.any(1), n_on.astype(np.float32), np.nan)) | |
| X_on = np.stack(on, 1).astype(np.float32) | |
| # occupancy / recovery | |
| occ = [_share(hands_off, k) for k in STEPS.values()] + [_share(head, k) for k in STEPS.values()] + [_share(gaze, k) for k in STEPS.values()] + [_share(unmet_f, k) for k in STEPS.values()] | |
| cur_on = S0[:, 0] | |
| t_ret = _first_time(hands_on > 0.5, v_h) | |
| t_ret = np.where(np.isfinite(cur_on) & (cur_on > 0.5), 0.0, t_ret).astype(np.float32) # already on -> 0 | |
| unmet0_valid = np.isfinite(S0).any(1) | np.isfinite(S0g) | |
| unmet0 = ((np.isfinite(S0[:, 0]) & (S0[:, 0] < 0.5)) | (np.isfinite(S0[:, 1]) & (S0[:, 1] > 0.5)) | (np.isfinite(S0g) & (S0g > 0.5))) | |
| t_rec = _first_time(unmet_f < 0.5, unmet_any_valid) | |
| t_rec = np.where(unmet0_valid & ~unmet0, 0.0, t_rec).astype(np.float32) # currently met -> 0 | |
| t_entry = _first_time(unmet_f > 0.5, unmet_any_valid) | |
| t_entry = np.where(unmet0_valid & unmet0, 0.0, t_entry).astype(np.float32) # currently unmet -> 0 | |
| remaining = np.where(unmet0_valid & unmet0, t_rec, 0.0).astype(np.float32) | |
| remaining[~unmet0_valid] = np.nan | |
| occ += [t_ret, t_rec, t_entry, remaining] | |
| X_occ = np.stack(occ, 1).astype(np.float32) | |
| assert X_on.shape[1] == len(ONSET_NAMES) and X_occ.shape[1] == len(OCC_NAMES) | |
| return X_on, X_occ | |
| def select_and_fit(X, y, groups, names, seed=0): | |
| """train_warning_head GRID['hgb'] under the device-grouped 5-fold CV (same estimator as the trainer) -> (fitted model, hp, cv_auroc).""" | |
| best = None | |
| for hp0 in TW.GRID["hgb"]: | |
| hp = dict(hp0, demand="P", names=names) | |
| m, _ = TW.cv_auroc("hgb", hp, X, y, groups, 5, seed) | |
| if best is None or m > best[0]: | |
| best = (m, hp) | |
| return TW.make_model("hgb", best[1], seed).fit(X, y), {k: v for k, v in best[1].items() if k != "names"}, best[0] | |
| def fmt(x, nd=3): | |
| return "—" if x is None or not np.isfinite(x) else f"{x:.{nd}f}" | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--runs", nargs="+", default=[f"I1_all_minus_road2drv_s{k}_final" for k in (0, 1, 2)]) | |
| ap.add_argument("--n-boot", type=int, default=2000) | |
| ap.add_argument("--seed", type=int, default=0, help="HGB / CV seed (the trainer's --seed default)") | |
| ap.add_argument("--out-table", default=str(CWM3 / "results" / "tables" / "warning_oracle_diagnostic.md")) | |
| a = ap.parse_args() | |
| t0 = time.time() | |
| stamp = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime()) | |
| out = CWM3 / "results" / "warning_oracle" / stamp | |
| out.mkdir(parents=True, exist_ok=True) | |
| TW.EXTRA_OPTS["sets"] = ("dwell", "face") | |
| tab = LB.train_label_table(sources=("consensus",)) | |
| assert len(tab) == 979, len(tab) | |
| EP = LB.episode_labels() | |
| lab = EP[EP.label != "unsure"].reset_index(drop=True) | |
| packs = {(r, s): WG.load_pack(r, s) for r in REGIMES for s in ("test", "val")} | |
| level_rows, delta_rows, raw_rows, meta_runs = [], [], [], {} | |
| for run in a.runs: | |
| rd = CWM3 / "results" / "runs" / run | |
| cfg = json.loads((rd / "config.json").read_text()) | |
| seed_run = int(cfg["args"]["seed"]) | |
| ds.set_kin_source(cfg["cfg"].get("kin_source", "cwm2") or "cwm2", cfg["cfg"].get("kin_version", "v1") or "v1") | |
| ds.set_pack_version(cfg["cfg"].get("pack_version", "v1") or "v1") | |
| names_all = TW.feature_names("P") | |
| m_cur, m_fut = TW.feature_mask(names_all, "current"), TW.feature_mask(names_all, "future") | |
| names_cur, names_fut = [n for n, m in zip(names_all, m_cur) if m], [n for n, m in zip(names_all, m_fut) if m] | |
| # ---- training table (979 rows): current / predicted-future (in-sample and OOF) / true-future | |
| Xc, Xf, Xf_oof, Xon, Xoc, ys, gs = [], [], [], [], [], [], [] | |
| for regime in REGIMES: | |
| sub = tab[tab.regime == regime] | |
| rows = sub.row.to_numpy().astype(np.int64) | |
| for kind, store in (("", Xf), ("_oof", Xf_oof)): | |
| pr = np.load(rd / f"pred_{regime}_train_labelrows{kind}.npz", allow_pickle=True) | |
| posmap = {int(r): i for i, r in enumerate(pr["rows"].astype(np.int64))} | |
| pos = np.array([posmap[int(r)] for r in rows], np.int64) | |
| X = TW.features_from_pred(pr, regime, "train", rows, pos, "P") | |
| store.append(X[:, m_fut]) | |
| if kind == "": | |
| Xc.append(X[:, m_cur]) | |
| xo, xq = true_future_features(regime, "train", rows) | |
| Xon.append(xo); Xoc.append(xq) | |
| ys.append(sub.y.to_numpy().astype(int)); gs.append(sub.dongle.to_numpy()) | |
| Xc, Xf, Xf_oof, Xon, Xoc = map(np.concatenate, (Xc, Xf, Xf_oof, Xon, Xoc)) | |
| y, groups = np.concatenate(ys), np.concatenate(gs) | |
| # ---- gold rows (seed-matched anchors), features at the anchors | |
| legacy = WG.merge_legacy(seed_run) | |
| G = WG.build_gold(lab, packs, legacy, "legacy_first") | |
| assert not set(G[G.clean_test].idx) & set(tab.idx) | |
| Gc, Gf, Gon, Goc = (np.full((len(G), k), np.nan, np.float32) for k in (Xc.shape[1], Xf.shape[1], Xon.shape[1], Xoc.shape[1])) | |
| for (r, s), g in G.groupby(["regime", "split"]): | |
| pr = np.load(rd / f"pred_{r}_{s}.npz", allow_pickle=True) | |
| rows = g.row.to_numpy().astype(np.int64) | |
| X = TW.features_from_pred(pr, r, s, rows, None, "P") | |
| Gc[g.index.to_numpy()], Gf[g.index.to_numpy()] = X[:, m_cur], X[:, m_fut] | |
| xo, xq = true_future_features(r, s, rows) | |
| Gon[g.index.to_numpy()], Goc[g.index.to_numpy()] = xo, xq | |
| gt = G.clean_test.to_numpy() | |
| yg, dong = G.y.to_numpy().astype(int), G.dongle.to_numpy() | |
| reg = G.regime.to_numpy() | |
| print(f"[{run}] train {len(y)} / {int(y.sum())} pos; gold clean_test {int(gt.sum())} / {int(yg[gt].sum())} pos / {len(np.unique(dong[gt]))} dev; " | |
| f"oracle finite share (gold): onset {np.isfinite(Gon[gt]).mean():.3f} occ {np.isfinite(Goc[gt]).mean():.3f}; train: onset {np.isfinite(Xon).mean():.3f} occ {np.isfinite(Xoc).mean():.3f}", flush=True) | |
| # ---- levels | |
| levels = {"base: cur+dwell+face": (Xc, Gc, names_cur), | |
| "(a) true-future onset scores alone": (Xon, Gon, ONSET_NAMES), | |
| "(b) cur+dwell+face + TRUE future occupancy/recovery": (np.concatenate([Xc, Xoc], 1), np.concatenate([Gc, Goc], 1), names_cur + OCC_NAMES), | |
| "(b+) (b) + true-future onset scores": (np.concatenate([Xc, Xoc, Xon], 1), np.concatenate([Gc, Goc, Gon], 1), names_cur + OCC_NAMES + ONSET_NAMES), | |
| "(c) cur+dwell+face + PREDICTED future (in-sample packs)": (np.concatenate([Xc, Xf], 1), np.concatenate([Gc, Gf], 1), names_cur + names_fut), | |
| "(c-oof) cur+dwell+face + PREDICTED future (OOF packs)": (np.concatenate([Xc, Xf_oof], 1), np.concatenate([Gc, Gf], 1), names_cur + names_fut)} | |
| S = {} | |
| for lv, (X, Xg, names) in levels.items(): | |
| mdl, hp, cvm = select_and_fit(X, y, groups, names, a.seed) | |
| S[lv] = mdl.predict_proba(Xg)[:, 1] | |
| print(f" {lv}: {X.shape[1]} features, cv AUROC {cvm:.4f}, hp {hp}", flush=True) | |
| meta_runs.setdefault(run, {})[lv] = {"n_features": int(X.shape[1]), "cv_auroc": cvm, "hp": hp} | |
| # raw true-future scores (no fit) | |
| for j, n in enumerate(ONSET_NAMES): | |
| v = Gon[:, j] | |
| fin = gt & np.isfinite(v) | |
| if yg[fin].min() != yg[fin].max(): | |
| b = WG.cluster_boot(yg[fin], {"s": v[fin]}, dong[fin], a.n_boot, 0)["s"]["auroc"] | |
| raw_rows.append({"run": run, "seed": seed_run, "score": n, "auroc": WG.fast_auroc(yg[fin], v[fin]), "ci_lo": WG.pct(b)[0], "ci_hi": WG.pct(b)[1], "n": int(fin.sum()), "n_pos": int(yg[fin].sum()), | |
| "pos_rate_score": float(np.nanmean(v[fin]))}) | |
| # ---- strata: shared resamples, paired deltas vs base | |
| for stratum, m in (("pooled", gt), ("manual", gt & (reg == "manual")), ("engaged", gt & (reg == "engaged"))): | |
| if m.sum() < 20 or yg[m].min() == yg[m].max(): | |
| continue | |
| Sm = {k: v[m] for k, v in S.items()} | |
| bs = WG.cluster_boot(yg[m], Sm, dong[m], a.n_boot, 0) | |
| base = "base: cur+dwell+face" | |
| for lv, sc in Sm.items(): | |
| au, ap_ = WG.fast_auroc(yg[m], sc), WG.fast_ap(yg[m], sc) | |
| level_rows.append({"run": run, "seed": seed_run, "stratum": stratum, "level": lv, "auroc": au, "auroc_ci_lo": WG.pct(bs[lv]["auroc"])[0], "auroc_ci_hi": WG.pct(bs[lv]["auroc"])[1], | |
| "prauc": ap_, "prauc_ci_lo": WG.pct(bs[lv]["prauc"])[0], "prauc_ci_hi": WG.pct(bs[lv]["prauc"])[1], "n": int(m.sum()), "n_pos": int(yg[m].sum()), "n_dev": len(np.unique(dong[m]))}) | |
| if lv != base: | |
| for met in ("auroc", "prauc"): | |
| d = bs[lv][met] - bs[base][met] | |
| fn = WG.fast_auroc if met == "auroc" else WG.fast_ap | |
| delta_rows.append({"run": run, "seed": seed_run, "stratum": stratum, "level": lv, "metric": met, "value": fn(yg[m], sc), "value_base": fn(yg[m], Sm[base]), | |
| "delta": fn(yg[m], sc) - fn(yg[m], Sm[base]), "delta_ci_lo": WG.pct(d)[0], "delta_ci_hi": WG.pct(d)[1], "frac_boot_d_gt0": float(np.mean(d[np.isfinite(d)] > 0)), | |
| "n": int(m.sum()), "n_pos": int(yg[m].sum())}) | |
| L = pd.DataFrame(level_rows); D = pd.DataFrame(delta_rows); R = pd.DataFrame(raw_rows) | |
| L.to_csv(out / "levels.csv", index=False); D.to_csv(out / "deltas.csv", index=False); R.to_csv(out / "raw_scores.csv", index=False) | |
| seeds = sorted(int(s) for s in L.seed.unique()) | |
| # ---- table | |
| T = [f"## Oracle-feature diagnostic of the warning read-out — runs {', '.join(a.runs)} (seeds {seeds}); {stamp}; eval/warning_oracle_diagnostic.py", "", | |
| "**This is a diagnostic under ONE feature set and ONE read-out (the trainer's HGB grid + device-grouped 5-fold CV on the 979 clean TRAIN consensus labels), NOT an upper bound on what any " | |
| "predictor or read-out could achieve.** Levels: base = cur+dwell+face (77 current-state features); (a) TRUE-future onset scores alone (hand_leave ∪ head_turn_onset and gaze_off_on within " | |
| "1 / 3 / 5 s, onset count); (b) base + TRUE future occupancy / recovery (hands_off / head_turned / gaze_off / unmet shares over 1 / 3 / 5 s, time to next hands-on return, time to recovery, " | |
| "time to next unmet-state entry, remaining unmet time; 5.2 s = none within the 5 s horizon); (b+) = (b) + the onset scores; (c) base + the model's PREDICTED future (the 144 `is_future` " | |
| "features; in-sample label-row packs) and (c-oof) the same with the out-of-fold packs. True-future arrays = the causal kin / gaze targets (`targets_kin_causal_routes_v2`, " | |
| "`targets_gaze_causal_routes_v2`) at +0.2..+5.0 s; an invalid component counts as met. Gold = clean_test rows of the unchanged `warning_gold.build_gold` (legacy_first, seed-matched); " | |
| f"device-cluster bootstrap {a.n_boot} resamples shared by every level -> paired Δ vs base on identical rows. Pre-registered reading: (b) gains and (c) does not -> strengthen the predictor; " | |
| "neither gains -> revisit the target definition / read-out.", "", | |
| "### 1. Clean-test AUROC per level and stratum (per seed; mean ± sd over seeds)", "", | |
| "| level | stratum | " + " | ".join(f"s{s} AUROC [CI]" for s in seeds) + " | mean ± sd | PR-AUC mean | n / pos |", "|---|---|" + "---|" * len(seeds) + "---|---|---|"] | |
| for lv in L.level.unique(): | |
| for st in ("pooled", "manual", "engaged"): | |
| g = L[(L.level == lv) & (L.stratum == st)].sort_values("seed") | |
| if not len(g): | |
| continue | |
| cells = [f"{fmt(r.auroc)} [{fmt(r.auroc_ci_lo, 2)},{fmt(r.auroc_ci_hi, 2)}]" for r in g.itertuples()] | |
| T.append(f"| {lv} | {st} | " + " | ".join(cells) + f" | {g.auroc.mean():.3f} ± {g.auroc.std(ddof=1) if len(g) > 1 else 0:.3f} | {g.prauc.mean():.3f} | {int(g.n.iloc[0])} / {int(g.n_pos.iloc[0])} |") | |
| T += ["", "### 2. Paired Δ vs base (cur+dwell+face) — AUROC, identical rows and device-cluster resamples; per seed [CI], mean ± sd, seeds with CI excl. 0", "", | |
| "| level | stratum | " + " | ".join(f"s{s} Δ [CI]" for s in seeds) + " | mean Δ ± sd | CI excl. 0 | verdict (paired CI primary; seed count = stability) |", "|---|---|" + "---|" * len(seeds) + "---|---|---|"] | |
| verdicts = {} | |
| for lv in D.level.unique(): | |
| for st in ("pooled", "manual", "engaged"): | |
| g = D[(D.level == lv) & (D.stratum == st) & (D.metric == "auroc")].sort_values("seed") | |
| if not len(g): | |
| continue | |
| excl = int(((g.delta_ci_lo > 0) | (g.delta_ci_hi < 0)).sum()) | |
| pos_excl = int((g.delta_ci_lo > 0).sum()) | |
| cells = [f"{r.delta:+.3f} [{r.delta_ci_lo:+.3f},{r.delta_ci_hi:+.3f}]" for r in g.itertuples()] | |
| v = "GAIN (every seed's paired CI > 0)" if pos_excl == len(g) else ("gain in some seeds" if pos_excl > 0 else ("LOSS (CI < 0 in some seed)" if int((g.delta_ci_hi < 0).sum()) > 0 else "no paired evidence (CIs ⊃ 0)")) | |
| verdicts[(lv, st)] = v | |
| T.append(f"| {lv} | {st} | " + " | ".join(cells) + f" | {g.delta.mean():+.3f} ± {g.delta.std(ddof=1) if len(g) > 1 else 0:.3f} | {excl}/{len(g)} | {v} |") | |
| T += ["", "### 3. Raw-score AUROC of the TRUE-future onset scores on the clean-test gold (no fit; reference — union@3 s is the 0.614 number of the 2026-09-16 exploration)", "", | |
| "| score | " + " | ".join(f"s{s} AUROC [CI] (n)" for s in seeds) + " |", "|---|" + "---|" * len(seeds)] | |
| for n in ONSET_NAMES: | |
| g = R[R.score == n].sort_values("seed") | |
| T.append(f"| {n} | " + " | ".join(f"{fmt(r.auroc)} [{fmt(r.ci_lo, 2)},{fmt(r.ci_hi, 2)}] ({r.n})" for r in g.itertuples()) + " |") | |
| rd_b = [verdicts.get(("(b) cur+dwell+face + TRUE future occupancy/recovery", "pooled"), "—"), verdicts.get(("(c) cur+dwell+face + PREDICTED future (in-sample packs)", "pooled"), "—"), | |
| verdicts.get(("(c-oof) cur+dwell+face + PREDICTED future (OOF packs)", "pooled"), "—")] | |
| T += ["", f"### 4. Reading (pooled stratum, pre-registered rule): (b) → **{rd_b[0]}**; (c) in-sample → **{rd_b[1]}**; (c-oof) → **{rd_b[2]}**. " | |
| "Per the pre-registration: (b) gains and (c) does not → strengthen the predictor; neither gains → revisit the target definition / read-out. Numbers above are the only source for any wording.", ""] | |
| md = "\n".join(T) + "\n" | |
| op = Path(a.out_table) | |
| if not op.exists(): | |
| op.write_text("# Oracle-feature diagnostic of the warning read-out (append-only; eval/warning_oracle_diagnostic.py)\n\n") | |
| with open(op, "a") as f: | |
| f.write(md) | |
| print(md) | |
| (out / "run.json").write_text(json.dumps({"stamp": stamp, "runs": a.runs, "n_boot": a.n_boot, "seed": a.seed, "levels": meta_runs, "verdicts": {f"{k[0]}|{k[1]}": v for k, v in verdicts.items()}, | |
| "onset_names": ONSET_NAMES, "occ_names": OCC_NAMES, "cap_s": CAP_S, "dms_fields_used": [], "wall_s": time.time() - t0}, indent=1)) | |
| print(f"DONE in {time.time() - t0:.0f}s -> {out}") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 22.7 kB
- Xet hash:
- d85852f52050e843f4feb2f56f6edf212249759c42f3d0fd38649df45de2cdf4
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.