Buckets:
| """cwm3/eval/operating_points.py — equal-alert-load operating points of the learned warning head vs the DMS attention chain (innovation 1, roadmap §二 1; CPU). | |
| Per seed and regime the head's threshold is swept (bisection on alerts / hour with the 60 s per-route cooldown of train_warning_head.alerts_per_hour, exposure = | |
| rows x 0.2 s of the full TEST split) to (i) the DMS att@3s alert load — the legacy chain at ITS clean-val-recall-0.5 threshold (≈ 55.6 / h manual, 49.3 / h engaged | |
| in warning_head_oof.md) — and (ii) a low-load point (--low-load alerts / h, default 10). At each matched load, for the head AND for the DMS chain re-thresholded to the | |
| same load: gold clean-test recall / precision (device-cluster bootstrap CI with the threshold FIXED), lead time = t0 − first alert in [t0 − lookback, t_anchor] over | |
| the gold positives anchored on the TEST split (median, IQR, detected share), the PAIRED lead-time Δ head − DMS at equal load (all positives with undetected -> lead 0; | |
| and the both-detected subset), reliability (10 equal-count bins after the clean-val temperature) + ECE (raw / calibrated). Inputs are published outputs only: | |
| results/runs/<w_run>/pred_layout_{regime}_{test,val}.npz (the head's scores), the legacy DMS marginals (cwm2/baselines/legacy_cockpitwm/results/ | |
| marginals_{regime}_s<seed>.npz -> compose.legacy_compose att_only, exactly as train_warning_head's DMS rows) and the UNCHANGED cwm2/eval/warning_gold.py gold | |
| (build_gold / temperature_fit / ece). Pre-stated kill rule: paired lead-time Δ CI ⊃ 0 -> "EQUAL". The DMS chain enters only as the labelled legacy comparator (as in | |
| warning_gold.py); no DMS field is read by this script. Outputs (append-only): results/operating_points/<item>_<stamp>/*.csv + a section appended to | |
| results/tables/operating_points.md. | |
| python eval/operating_points.py --item oof --w-runs W_I1_all_s0_oof W_I1_all_s1_oof W_I1_all_s2_oof --seeds 0 1 2 --col W_learned_curhgb_t15_w10_n979oof | |
| 2026-09-17 additive option (`--pair-runs`, `--pair-surrogate`; nothing above changes when they are absent — same CSVs, same section): PAIRED ours − comparator cells at | |
| every load against other read-out layouts (the public DMS read-outs of baselines_dms/, `results/runs/abl_dms_*_full_s<seed>`) and, with --pair-surrogate, against the | |
| att@3s surrogate. Spec `LABEL=RUN_PATTERN:COL[:OPDIR]` ({seed} in RUN_PATTERN). The comparator is put at the SAME operating point as the head: with OPDIR (an existing | |
| results/operating_points/<item>_<stamp> directory) its published per-seed / per-load thresholds are re-used verbatim (score=head rows of operating_points.csv; the | |
| load targets are asserted equal and the alerts / hour are recomputed and asserted equal, so the comparator sits exactly on its frozen-table row); without OPDIR it is | |
| thresholded by the identical bisection rule as the head (full TEST split, 60 s cooldown; same_recall by the same recall bisection). Cells on the common finite gold | |
| rows: recall, precision, FAR = share of gold clean-test NEGATIVE rows alerted at the fixed threshold (not a natural rate; the per-hour load is matched by construction), | |
| first-alert lead over the common test-anchored positives (undetected -> 0), every Δ = head − comparator with the device-cluster bootstrap on IDENTICAL resamples | |
| (seed 0 for recall / precision / FAR, seed 2 for lead, as the existing cells). Per-seed verdict: CI ⊃ 0 -> EQUAL; multi-seed: >= ceil(2n/3) seed CIs exclude 0, same | |
| sign, |mean Δ| > sd over seeds (report_dms.verdict_multi rule). A comparator whose layouts are missing for a seed is skipped for that seed (logged, listed in the run | |
| json). Extra outputs: `paired_public.csv` (+ `paired_public_skipped.csv`) and a "Paired ours − comparator" subsection. | |
| """ | |
| from __future__ import annotations | |
| import os | |
| os.environ.setdefault("OMP_NUM_THREADS", "4") | |
| os.environ.setdefault("MKL_NUM_THREADS", "4") | |
| import argparse # noqa: E402 | |
| import json # noqa: E402 | |
| import sys # noqa: E402 | |
| import time # noqa: E402 | |
| from pathlib import Path # noqa: E402 | |
| import numpy as np # noqa: E402 | |
| import pandas as pd # noqa: E402 | |
| CWM3 = Path(__file__).resolve().parents[1] | |
| EXPLORE = CWM3.parent | |
| CWM2 = EXPLORE / "cwm2" | |
| for p in (CWM3 / "train", CWM3 / "data", CWM3 / "eval", CWM3 / "common", CWM2 / "eval", CWM2 / "data", CWM2, CWM2 / "baselines" / "legacy_cockpitwm"): | |
| sys.path.insert(0, str(p)) | |
| import train_warning_head as TW # noqa: E402 (alerts_per_hour, threshold_at_recall, lead_times — unchanged) | |
| import warning_gold as WG # noqa: E402 (build_gold, temperature_fit, apply_temperature, ece, pct — unchanged) | |
| from compose import legacy_compose # noqa: E402 | |
| from runmeta import write_run_json # noqa: E402 | |
| LB = TW.LB | |
| REGIMES = ("manual", "engaged") | |
| TAU = 15 | |
| OUT_ROOT = CWM3 / "results" / "operating_points" | |
| TABLE = CWM3 / "results" / "tables" / "operating_points.md" | |
| DMS_DIR = CWM2 / "baselines" / "legacy_cockpitwm" / "results" | |
| def thr_for_rate(uid, t, s, rate, it=24): | |
| """bisection on the threshold so that alerts / hour (60 s cooldown) matches `rate` (alerts/h is non-increasing in the threshold).""" | |
| fin = np.isfinite(s) | |
| lo, hi = float(np.nanmin(s[fin])), float(np.nanmax(s[fin])) | |
| for _ in range(it): | |
| mid = 0.5 * (lo + hi) | |
| if TW.alerts_per_hour(uid, t, s, mid)[2] > rate: | |
| lo = mid | |
| else: | |
| hi = mid | |
| return 0.5 * (lo + hi) | |
| def cboot(groups, fn, n_boot, seed=0): | |
| """device-cluster bootstrap of a statistic fn(pick_indices) -> value.""" | |
| ug = np.unique(groups) | |
| gidx = {g: np.flatnonzero(groups == g) for g in ug} | |
| rng = np.random.default_rng(seed) | |
| out = np.full(n_boot, np.nan) | |
| for b in range(n_boot): | |
| pick = np.concatenate([gidx[g] for g in rng.choice(ug, len(ug), replace=True)]) | |
| out[b] = fn(pick) | |
| return out | |
| def rec_prec(y, hit): | |
| tp = int((hit & (y == 1)).sum()) | |
| return tp / max(int((y == 1).sum()), 1), (tp / max(int(hit.sum()), 1) if hit.sum() else np.nan) | |
| def reliability_bins(y, p, n_bins=10): | |
| order = np.argsort(p, kind="mergesort") | |
| return [{"bin": i, "n": int(len(e)), "p_mean": float(p[e].mean()), "y_rate": float(y[e].mean()), "p_lo": float(p[e].min()), "p_hi": float(p[e].max())} | |
| for i, e in enumerate(np.array_split(order, n_bins)) if len(e)] | |
| def f3(x, nd=3): | |
| return "—" if x is None or not np.isfinite(x) else f"{x:.{nd}f}" | |
| def ci(v, nd=3): | |
| lo, hi = WG.pct(np.asarray(v, float)) | |
| return f"[{f3(lo, nd)}, {f3(hi, nd)}]" | |
| # ----------------------------------------------------------------------------- 2026-09-17 additive: paired ours − comparator helpers (used only with --pair-runs / --pair-surrogate) | |
| def far_rate(y, hit): | |
| """FAR on the gold rows = share of gold NEGATIVE rows (y == 0) that receive an alert at the fixed threshold (NaN when there is no negative).""" | |
| neg = (y == 0) | |
| return float((hit & neg).sum()) / int(neg.sum()) if neg.any() else np.nan | |
| def parse_pair_spec(spec): | |
| """'LABEL=RUN_PATTERN:COL[:OPDIR]' -> (label, run_pattern, col, opdir or None); RUN_PATTERN holds '{seed}'.""" | |
| label, rest = spec.split("=", 1) | |
| parts = rest.split(":") | |
| assert len(parts) in (2, 3), spec | |
| run_pat, col = parts[0], parts[1] | |
| opdir = Path(parts[2]) if len(parts) == 3 and parts[2] else None | |
| if opdir is not None and not opdir.is_absolute(): | |
| opdir = CWM3 / opdir | |
| assert "{seed}" in run_pat, f"RUN_PATTERN must contain {{seed}}: {spec}" | |
| return label.strip(), run_pat, col, opdir | |
| def read_pub_thresholds(opdir, run_c, seed): | |
| """published per-load thresholds of a comparator from an existing operating_points.csv (score == head rows of that run / seed) -> {load: row dict}.""" | |
| df = pd.read_csv(Path(opdir) / "operating_points.csv") | |
| df = df[(df.w_run == run_c) & (df.seed == seed) & (df.score == "head")] | |
| out = {} | |
| for lname, g in df.groupby("load"): | |
| r = g.iloc[0] | |
| out[str(lname)] = {"thr": {"manual": float(r.thr_manual), "engaged": float(r.thr_engaged)}, "target": {"manual": float(r.load_target_manual), "engaged": float(r.load_target_engaged)}, | |
| "aph": {"manual": float(r.alerts_per_hour_manual), "engaged": float(r.alerts_per_hour_engaged)}} | |
| return out | |
| def multi_verdict(d, lo, hi, pos_word, neg_word): | |
| """pre-stated multi-seed rule (report_dms.verdict_multi): >= ceil(2n/3) seed CIs exclude 0, same sign, |mean Δ| > sd over seeds (single seed: its CI).""" | |
| d, lo, hi = (np.asarray(x, float) for x in (d, lo, hi)) | |
| k = np.isfinite(d) & np.isfinite(lo) & np.isfinite(hi) | |
| d, lo, hi = d[k], lo[k], hi[k] | |
| n = len(d) | |
| if n == 0: | |
| return "—" | |
| need = int(np.ceil(2 * n / 3)) | |
| excl = int(((lo > 0) | (hi < 0)).sum()) | |
| same = bool(np.all(d > 0) or np.all(d < 0)) | |
| sd = float(d.std(ddof=1)) if n > 1 else 0.0 | |
| ok = excl >= need and same and (n == 1 or abs(float(d.mean())) > sd) | |
| if ok and d.mean() > 0: | |
| return f"{pos_word} ({excl}/{n})" | |
| if ok and d.mean() < 0: | |
| return f"{neg_word} ({excl}/{n})" | |
| return f"n.s. ({excl}/{n}{'' if same else ', signs differ'})" | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--item", default="oof") | |
| ap.add_argument("--w-runs", nargs="+", required=True, help="layout dirs under results/runs holding the head's w columns (e.g. W_I1_all_s0_oof)") | |
| ap.add_argument("--seeds", nargs="+", type=int, required=True, help="seed of each w-run (selects the legacy DMS comparator files)") | |
| ap.add_argument("--col", default="W_learned_curhgb_t15_w10_n979oof") | |
| ap.add_argument("--label", default="HGB(current-state)") | |
| ap.add_argument("--low-load", type=float, default=10.0) | |
| ap.add_argument("--loads", default="dms_matched,low10", help="comma list of operating points: dms_matched (alerts/h of the DMS chain at its clean-val-recall threshold), " | |
| "low<N> (N alerts/h), same_recall (2026-09-12: the head's threshold per regime set by bisection on the gold clean-test rows " | |
| "to the DMS chain's recall in that regime at its own threshold; the DMS side keeps its own threshold)") | |
| ap.add_argument("--n-boot", type=int, default=1000) | |
| ap.add_argument("--val-recall", type=float, default=0.5) | |
| ap.add_argument("--lookback", type=float, default=10.0) | |
| ap.add_argument("--dms-marginals", default=str(DMS_DIR)) | |
| ap.add_argument("--pair-runs", nargs="*", default=[], help="2026-09-17 additive: paired ours − comparator cells at every load; spec LABEL=RUN_PATTERN:COL[:OPDIR] " | |
| "({seed} in RUN_PATTERN; OPDIR = existing results/operating_points/<item>_<stamp> whose operating_points.csv " | |
| "gives the comparator's published thresholds, else the same bisection rule as the head). See the module docstring.") | |
| ap.add_argument("--out-root", default=str(OUT_ROOT), help="2026-09-17 additive: output root (default results/operating_points; /tmp for smoke runs)") | |
| ap.add_argument("--table-out", default=str(TABLE), help="2026-09-17 additive: table to append the section to (default results/tables/operating_points.md)") | |
| ap.add_argument("--pair-surrogate", action="store_true", help="2026-09-17 additive: the same paired cells against the DMS att@3s surrogate at its equal-load threshold (label att@3s)") | |
| a = ap.parse_args() | |
| assert len(a.w_runs) == len(a.seeds) | |
| t0 = time.time() | |
| stamp = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime()) | |
| out = Path(a.out_root) / f"{a.item}_{stamp}" | |
| out.mkdir(parents=True, exist_ok=True) | |
| log_p = out / "operating_points.log" | |
| def log(m): | |
| line = f"[{time.strftime('%H:%M:%S')}] {m}" | |
| print(line, flush=True) | |
| with open(log_p, "a") as f: | |
| f.write(line + "\n") | |
| lab = LB.episode_labels() | |
| lab = lab[lab.label != "unsure"].reset_index(drop=True) | |
| packs = {(r, s): WG.load_pack(r, s) for r in REGIMES for s in ("test", "val")} | |
| rows_op, rows_lead, rows_rel, rows_pair, rows_ref = [], [], [], [], [] | |
| rows_pub, rows_pubskip, pair_inputs = [], [], [] # 2026-09-17 additive (--pair-runs / --pair-surrogate) | |
| pair_specs = [parse_pair_spec(s) for s in a.pair_runs] | |
| for w_run, seed in zip(a.w_runs, a.seeds): | |
| legacy = WG.merge_legacy(seed) | |
| assert legacy is not None, "legacy episode CSVs missing" | |
| G = WG.build_gold(lab, packs, legacy, "legacy_first") | |
| G["att"] = legacy.reindex(G.idx)["att@3s"].to_numpy(float) | |
| lay = {(r, s): np.load(CWM3 / "results" / "runs" / w_run / f"pred_layout_{r}_{s}.npz", allow_pickle=True) for r in REGIMES for s in ("test", "val")} | |
| head = {} | |
| for (r, s), z in lay.items(): | |
| names = list(map(str, z["w_names"])) | |
| assert a.col in names, (a.col, names) | |
| assert len(z["uid"]) == len(packs[(r, s)]["uid"]) and np.array_equal(z["uid"], packs[(r, s)]["uid"]), f"{w_run} {r}/{s}: layout rows != targets rows" | |
| head[(r, s)] = (z["uid"], z["t"].astype(float), z["w"][:, names.index(a.col)].astype(np.float64)) | |
| gh = np.full(len(G), np.nan) | |
| for (r, s), g in G.groupby(["regime", "split"]): | |
| gh[g.index.to_numpy()] = head[(r, s)][2][g.row.to_numpy()] | |
| # ---- 2026-09-17 additive: comparator layouts for the paired ours − comparator cells (same row contract as the head's layouts) | |
| comps = {} | |
| for label, run_pat, col_c, opdir in pair_specs: | |
| run_c = run_pat.format(seed=seed) | |
| files = {(r, s): CWM3 / "results" / "runs" / run_c / f"pred_layout_{r}_{s}.npz" for r in REGIMES for s in ("test", "val")} | |
| if not all(f.exists() for f in files.values()): | |
| rows_pubskip.append({"item": a.item, "seed": seed, "comparator": label, "comp_run": run_c, "reason": "layouts incomplete: " + ", ".join(f"{r}/{s}" for (r, s), f in files.items() if not f.exists())}) | |
| log(f" pair {label}: {run_c} layouts incomplete for seed {seed} -> skipped") | |
| continue | |
| routes_c, gc = {}, np.full(len(G), np.nan) | |
| for (r, s), f in files.items(): | |
| z = np.load(f, allow_pickle=True) | |
| names = list(map(str, z["w_names"])) | |
| assert col_c in names, (label, col_c, names) | |
| assert len(z["uid"]) == len(packs[(r, s)]["uid"]) and np.array_equal(z["uid"], packs[(r, s)]["uid"]), f"{run_c} {r}/{s}: layout rows != targets rows" | |
| sc = z["w"][:, names.index(col_c)].astype(np.float64) | |
| if s == "test": | |
| tt = z["t"].astype(float) | |
| assert np.array_equal(tt, head[(r, s)][1]), f"{run_c} {r}/{s}: layout t != head t" | |
| routes_c[(r, s)] = (head[(r, s)][0], head[(r, s)][1], sc) | |
| g = G[(G.regime == r) & (G.split == s)] | |
| gc[g.index.to_numpy()] = sc[g.row.to_numpy()] | |
| del z | |
| pair_inputs.append(f) | |
| thr_tab = read_pub_thresholds(opdir, run_c, seed) if opdir is not None else {} | |
| comps[label] = {"run": run_c, "col": col_c, "routes": routes_c, "gsc": gc, "thr_tab": thr_tab, "opdir": (str(opdir) if opdir is not None else "")} | |
| log(f" pair {label}: {run_c} col {col_c}; finite on gold clean-test {np.isfinite(gc[G.clean_test.to_numpy()]).mean():.3f}; published thresholds for loads {sorted(thr_tab) if thr_tab else 'none (bisection)'}") | |
| dms = {} | |
| for r in REGIMES: | |
| z = np.load(Path(a.dms_marginals) / f"marginals_{r}_s{seed}.npz", allow_pickle=True) | |
| dms[(r, "test")] = (z["uid"], z["t"].astype(float), legacy_compose(z["pA"].astype(np.float32), None, TAU)["att_only"].astype(np.float64)) | |
| yg, gt, gv = G.y.to_numpy().astype(int), G.clean_test.to_numpy(), G.clean_val.to_numpy() | |
| dong, att, greg, gsplit = G.dongle.to_numpy(), G["att"].to_numpy(float), G.regime.to_numpy(), G.split.to_numpy() | |
| log(f"{w_run} (seed {seed}): gold clean_test {int(gt.sum())} / {int(yg[gt].sum())} pos ({len(np.unique(dong[gt]))} devices); clean_val {int(gv.sum())} / {int(yg[gv].sum())}; " | |
| f"head col {a.col}; finite head {np.isfinite(gh[gt]).mean():.3f}, finite DMS {np.isfinite(att[gt]).mean():.3f}") | |
| # ---- DMS reference: its own threshold at clean-val recall, alert loads per regime (the targets of the equal-load sweep) | |
| fv = gv & np.isfinite(att) | |
| thr_dms = TW.threshold_at_recall(yg[fv], att[fv], a.val_recall) | |
| load_dms = {r: TW.alerts_per_hour(*dms[(r, "test")], thr_dms) for r in REGIMES} | |
| ft = gt & np.isfinite(att) | |
| rc0, pr0 = rec_prec(yg[ft], att[ft] >= thr_dms) | |
| rows_ref.append({"item": a.item, "w_run": w_run, "seed": seed, "thr_dms": thr_dms, "alerts_per_hour_manual": load_dms["manual"][2], "alerts_per_hour_engaged": load_dms["engaged"][2], | |
| "hours_manual": load_dms["manual"][1], "hours_engaged": load_dms["engaged"][1], "gold_recall": rc0, "gold_precision": pr0, "n_gold": int(ft.sum())}) | |
| log(f" DMS att@3s own threshold {thr_dms:.4f}: alerts/h manual {load_dms['manual'][2]:.1f} engaged {load_dms['engaged'][2]:.1f}; gold recall {rc0:.2f} precision {pr0:.2f}") | |
| loads = {} | |
| for tok in [x.strip() for x in a.loads.split(",") if x.strip()]: | |
| if tok == "dms_matched": | |
| loads[tok] = {r: load_dms[r][2] for r in REGIMES} | |
| elif tok.startswith("low"): | |
| lv = float(tok[3:]) if tok[3:] else a.low_load | |
| loads[f"low{int(lv)}"] = {r: lv for r in REGIMES} | |
| elif tok == "same_recall": # target = the DMS chain's gold recall per regime at ITS threshold (thr_dms) | |
| loads[tok] = {r: rec_prec(yg[ft & (greg == r)], att[ft & (greg == r)] >= thr_dms)[0] for r in REGIMES} | |
| else: | |
| raise ValueError(tok) | |
| scores = {"head": (head, gh), "dms": (dms, att)} | |
| lead_store = {} | |
| thr_hist = {} # 2026-09-17 additive: (load, score) -> per-regime thresholds (read by the paired cells) | |
| def thr_for_recall(gsc, target, r): | |
| """threshold on the head's gold clean-test scores of regime r reaching the target recall (bisection on the recall, non-increasing in the threshold).""" | |
| m = gt & (greg == r) & np.isfinite(gsc) | |
| yy, ss = yg[m], gsc[m] | |
| lo, hi = float(np.nanmin(ss)), float(np.nanmax(ss)) | |
| for _ in range(40): | |
| mid = 0.5 * (lo + hi) | |
| if rec_prec(yy, ss >= mid)[0] >= target: | |
| lo = mid | |
| else: | |
| hi = mid | |
| return lo | |
| for lname, L in loads.items(): | |
| for sname, (routes, gsc) in scores.items(): | |
| if lname == "same_recall": | |
| thr = {r: (thr_dms if sname == "dms" else thr_for_recall(gsc, L[r], r)) for r in REGIMES} | |
| else: | |
| thr = {r: thr_for_rate(*routes[(r, "test")], L[r]) for r in REGIMES} | |
| ach = {r: TW.alerts_per_hour(*routes[(r, "test")], thr[r]) for r in REGIMES} | |
| thr_hist[(lname, sname)] = thr | |
| thr_row = np.array([thr[r] for r in greg], float) | |
| for reg in REGIMES + ("pooled",): | |
| m = gt & np.isfinite(gsc) & ((greg == reg) if reg != "pooled" else np.ones(len(G), bool)) | |
| y, hit, dg = yg[m], gsc[m] >= thr_row[m], dong[m] | |
| rc, pr = rec_prec(y, hit) | |
| b_rc = cboot(dg, lambda pk: rec_prec(y[pk], hit[pk])[0], a.n_boot, 0) | |
| b_pr = cboot(dg, lambda pk: rec_prec(y[pk], hit[pk])[1], a.n_boot, 0) | |
| # lead time over the gold positives anchored on the TEST split (route scores available for both the head and the DMS chain) | |
| mp = m & (yg == 1) & (gsplit == "test") | |
| idx_p, leads = [], [] | |
| for r in REGIMES: | |
| mr = mp & (greg == r) | |
| if mr.any(): | |
| idx_p.append(np.flatnonzero(mr)) | |
| leads.append(TW.lead_times(G[mr], {(r, "test"): routes[(r, "test")]}, thr[r], a.lookback)) | |
| idx_p = np.concatenate(idx_p) if idx_p else np.array([], int) | |
| leads = np.concatenate(leads) if leads else np.array([]) | |
| det = np.isfinite(leads) | |
| lead0 = np.where(det, leads, 0.0) | |
| lead_store[(lname, sname, reg)] = (idx_p, leads) | |
| q = np.nanpercentile(leads[det], [25, 50, 75]) if det.any() else [np.nan] * 3 | |
| b_med = cboot(dong[idx_p], lambda pk: (np.nanmedian(leads[pk]) if np.isfinite(leads[pk]).any() else np.nan), a.n_boot, 1) if len(idx_p) else np.array([np.nan]) | |
| b_det = cboot(dong[idx_p], lambda pk: float(det[pk].mean()), a.n_boot, 1) if len(idx_p) else np.array([np.nan]) | |
| rows_op.append({"item": a.item, "w_run": w_run, "seed": seed, "load": lname, "load_target_manual": L["manual"], "load_target_engaged": L["engaged"], "score": sname, | |
| "regime": reg, "thr_manual": thr["manual"], "thr_engaged": thr["engaged"], "alerts_per_hour_manual": ach["manual"][2], "alerts_per_hour_engaged": ach["engaged"][2], | |
| "n_gold": int(m.sum()), "n_pos": int((y == 1).sum()), "n_alerts_gold": int(hit.sum()), "recall": rc, "recall_ci_lo": WG.pct(b_rc)[0], "recall_ci_hi": WG.pct(b_rc)[1], | |
| "precision": pr, "precision_ci_lo": WG.pct(b_pr)[0], "precision_ci_hi": WG.pct(b_pr)[1], "n_pos_lead": int(len(idx_p)), "lead_detected_frac": float(det.mean()) if len(det) else np.nan, | |
| "lead_detected_ci_lo": WG.pct(b_det)[0], "lead_detected_ci_hi": WG.pct(b_det)[1], "lead_q25": q[0], "lead_median": q[1], "lead_q75": q[2], | |
| "lead_median_ci_lo": WG.pct(b_med)[0], "lead_median_ci_hi": WG.pct(b_med)[1], "lead0_mean": float(lead0.mean()) if len(lead0) else np.nan, "n_dev": int(len(np.unique(dg)))}) | |
| for i, l in zip(idx_p, leads): | |
| rows_lead.append({"item": a.item, "seed": seed, "load": lname, "score": sname, "regime_cell": reg, "idx": G.idx.iloc[i], "regime": greg[i], "dongle": dong[i], "lead_s": l}) | |
| log(f" [{lname}] {sname:4s} {reg:8s}: thr m {thr['manual']:.4f} e {thr['engaged']:.4f} -> alerts/h {ach['manual'][2]:.1f} / {ach['engaged'][2]:.1f}; recall {rc:.3f} {ci(b_rc, 2)} " | |
| f"precision {f3(pr)} {ci(b_pr, 2)} (n {int(m.sum())} / {int((y == 1).sum())} pos); lead median {f3(q[1], 1)} s IQR [{f3(q[0], 1)}, {f3(q[2], 1)}] detected {det.mean() if len(det) else float('nan'):.2f} of {len(idx_p)}") | |
| # ---- paired lead-time Δ head − DMS at the same load | |
| for reg in REGIMES + ("pooled",): | |
| ih, lh = lead_store[(lname, "head", reg)] | |
| idd, ld = lead_store[(lname, "dms", reg)] | |
| common = np.intersect1d(ih, idd) | |
| if len(common) < 5: | |
| continue | |
| ph, pdm = {i: l for i, l in zip(ih, lh)}, {i: l for i, l in zip(idd, ld)} | |
| lh_c, ld_c = np.array([ph[i] for i in common]), np.array([pdm[i] for i in common]) | |
| d0 = np.nan_to_num(lh_c, nan=0.0) - np.nan_to_num(ld_c, nan=0.0) | |
| both = np.isfinite(lh_c) & np.isfinite(ld_c) | |
| dg = dong[common] | |
| b_d0 = cboot(dg, lambda pk: float(d0[pk].mean()), a.n_boot, 2) | |
| b_med = cboot(dg, lambda pk: (float(np.median((lh_c - ld_c)[pk][both[pk]])) if both[pk].any() else np.nan), a.n_boot, 2) | |
| lo, hi = WG.pct(b_d0) | |
| verdict = "EQUAL (CI ⊃ 0)" if lo <= 0 <= hi else ("HEAD EARLIER" if lo > 0 else "DMS EARLIER") | |
| rows_pair.append({"item": a.item, "w_run": w_run, "seed": seed, "load": lname, "regime": reg, "n_pos": int(len(common)), "n_both_detected": int(both.sum()), | |
| "det_head": float(np.isfinite(lh_c).mean()), "det_dms": float(np.isfinite(ld_c).mean()), "delta_lead0_mean": float(d0.mean()), "delta_lead0_ci_lo": lo, "delta_lead0_ci_hi": hi, | |
| "delta_median_both": float(np.median((lh_c - ld_c)[both])) if both.any() else np.nan, "delta_median_both_ci_lo": WG.pct(b_med)[0], "delta_median_both_ci_hi": WG.pct(b_med)[1], | |
| "frac_head_earlier": float((d0 > 0).mean()), "frac_dms_earlier": float((d0 < 0).mean()), "verdict": verdict, "n_dev": int(len(np.unique(dg)))}) | |
| log(f" [{lname}] paired Δ lead (head − DMS, undetected -> 0) {reg}: {d0.mean():+.2f} s {ci(b_d0, 2)} n={len(common)} both-detected {int(both.sum())} median Δ {f3(rows_pair[-1]['delta_median_both'], 2)} -> {verdict}") | |
| # ---- 2026-09-17 additive: paired ours − comparator cells (recall / precision / FAR / lead) at this load, common finite gold rows, identical resamples | |
| targets = dict(comps) | |
| if a.pair_surrogate: | |
| targets["att@3s"] = {"run": "legacy att@3s", "col": "att@3s", "routes": dms, "gsc": att, "thr_tab": {}, "opdir": "", "thr_fixed": thr_hist[(lname, "dms")]} | |
| thr_h = thr_hist[(lname, "head")] | |
| thr_row_h = np.array([thr_h[r] for r in greg], float) | |
| for label, C in targets.items(): | |
| gc, routes_c = C["gsc"], C["routes"] | |
| if "thr_fixed" in C: | |
| thr_c, thr_src = C["thr_fixed"], "equal-load threshold of the surrogate (main table)" | |
| elif lname in C["thr_tab"]: | |
| pub = C["thr_tab"][lname] | |
| for r in REGIMES: | |
| assert abs(pub["target"][r] - L[r]) < 1e-6, (label, lname, r, pub["target"][r], L[r]) | |
| thr_c, thr_src = pub["thr"], f"published ({Path(C['opdir']).name})" | |
| elif lname == "same_recall": | |
| thr_c, thr_src = {r: thr_for_recall(gc, L[r], r) for r in REGIMES}, "bisection (same rule as the head)" | |
| else: | |
| thr_c, thr_src = {r: thr_for_rate(*routes_c[(r, "test")], L[r]) for r in REGIMES}, "bisection (same rule as the head)" | |
| ach_c = {r: TW.alerts_per_hour(*routes_c[(r, "test")], thr_c[r]) for r in REGIMES} | |
| if lname in C["thr_tab"] and "thr_fixed" not in C: | |
| for r in REGIMES: | |
| assert abs(ach_c[r][2] - C["thr_tab"][lname]["aph"][r]) < 1e-6, (label, lname, r, ach_c[r][2], C["thr_tab"][lname]["aph"][r]) | |
| thr_row_c = np.array([thr_c[r] for r in greg], float) | |
| for reg in REGIMES + ("pooled",): | |
| m = gt & np.isfinite(gh) & np.isfinite(gc) & ((greg == reg) if reg != "pooled" else np.ones(len(G), bool)) | |
| y, dg = yg[m], dong[m] | |
| hh, hc = gh[m] >= thr_row_h[m], gc[m] >= thr_row_c[m] | |
| rc_h, pr_h = rec_prec(y, hh) | |
| rc_c, pr_c = rec_prec(y, hc) | |
| far_h, far_c = far_rate(y, hh), far_rate(y, hc) | |
| b_drc = cboot(dg, lambda pk: rec_prec(y[pk], hh[pk])[0] - rec_prec(y[pk], hc[pk])[0], a.n_boot, 0) | |
| b_dpr = cboot(dg, lambda pk: rec_prec(y[pk], hh[pk])[1] - rec_prec(y[pk], hc[pk])[1], a.n_boot, 0) | |
| b_dfar = cboot(dg, lambda pk: far_rate(y[pk], hh[pk]) - far_rate(y[pk], hc[pk]), a.n_boot, 0) | |
| # lead over the common test-anchored positives: the head's leads from lead_store, the comparator's at ITS threshold on ITS routes | |
| ih, lh = lead_store[(lname, "head", reg)] | |
| keep = np.isfinite(gc[ih]) if len(ih) else np.zeros(0, bool) | |
| ih_c, lh_c = ih[keep], lh[keep] | |
| lc = np.full(len(ih_c), np.nan) | |
| for r in REGIMES: | |
| mr = greg[ih_c] == r | |
| if mr.any(): | |
| lc[mr] = TW.lead_times(G.iloc[ih_c[mr]], {(r, "test"): routes_c[(r, "test")]}, thr_c[r], a.lookback) | |
| d0 = np.nan_to_num(lh_c, nan=0.0) - np.nan_to_num(lc, nan=0.0) | |
| both = np.isfinite(lh_c) & np.isfinite(lc) | |
| dgp = dong[ih_c] | |
| if len(ih_c) >= 5: | |
| b_dl = cboot(dgp, lambda pk: float(d0[pk].mean()), a.n_boot, 2) | |
| b_dlm = cboot(dgp, lambda pk: (float(np.median((lh_c - lc)[pk][both[pk]])) if both[pk].any() else np.nan), a.n_boot, 2) | |
| else: | |
| b_dl, b_dlm = np.array([np.nan]), np.array([np.nan]) | |
| lo_r, hi_r = WG.pct(b_drc) | |
| lo_l, hi_l = WG.pct(b_dl) | |
| v_rc = "—" if not np.isfinite(lo_r) else ("EQUAL (CI ⊃ 0)" if lo_r <= 0 <= hi_r else ("OURS HIGHER" if lo_r > 0 else "COMPARATOR HIGHER")) | |
| v_ld = "—" if not np.isfinite(lo_l) else ("EQUAL (CI ⊃ 0)" if lo_l <= 0 <= hi_l else ("OURS EARLIER" if lo_l > 0 else "COMPARATOR EARLIER")) | |
| rows_pub.append({"item": a.item, "w_run": w_run, "seed": seed, "load": lname, "load_target_manual": L["manual"], "load_target_engaged": L["engaged"], "comparator": label, | |
| "comp_run": C["run"], "comp_col": C["col"], "thr_source": thr_src, "regime": reg, | |
| "thr_head_manual": thr_h["manual"], "thr_head_engaged": thr_h["engaged"], "thr_comp_manual": thr_c["manual"], "thr_comp_engaged": thr_c["engaged"], | |
| "aph_comp_manual": ach_c["manual"][2], "aph_comp_engaged": ach_c["engaged"][2], | |
| "n_common": int(m.sum()), "n_pos": int((y == 1).sum()), "n_neg": int((y == 0).sum()), "n_dev": int(len(np.unique(dg))), | |
| "n_alerts_gold_head": int(hh.sum()), "n_alerts_gold_comp": int(hc.sum()), | |
| "recall_head": rc_h, "recall_comp": rc_c, "d_recall": rc_h - rc_c, "d_recall_ci_lo": lo_r, "d_recall_ci_hi": hi_r, | |
| "precision_head": pr_h, "precision_comp": pr_c, "d_precision": pr_h - pr_c, "d_precision_ci_lo": WG.pct(b_dpr)[0], "d_precision_ci_hi": WG.pct(b_dpr)[1], | |
| "far_head": far_h, "far_comp": far_c, "d_far": far_h - far_c, "d_far_ci_lo": WG.pct(b_dfar)[0], "d_far_ci_hi": WG.pct(b_dfar)[1], | |
| "n_pos_lead": int(len(ih_c)), "n_both_detected": int(both.sum()), "det_head": float(np.isfinite(lh_c).mean()) if len(ih_c) else np.nan, | |
| "det_comp": float(np.isfinite(lc).mean()) if len(ih_c) else np.nan, | |
| "lead_median_head": float(np.nanmedian(lh_c)) if np.isfinite(lh_c).any() else np.nan, "lead_median_comp": float(np.nanmedian(lc)) if np.isfinite(lc).any() else np.nan, | |
| "d_lead0_mean": float(d0.mean()) if len(ih_c) else np.nan, "d_lead0_ci_lo": lo_l, "d_lead0_ci_hi": hi_l, | |
| "d_lead_median_both": float(np.median((lh_c - lc)[both])) if both.any() else np.nan, "d_lead_median_both_ci_lo": WG.pct(b_dlm)[0], "d_lead_median_both_ci_hi": WG.pct(b_dlm)[1], | |
| "frac_head_earlier": float((d0 > 0).mean()) if len(ih_c) else np.nan, "frac_comp_earlier": float((d0 < 0).mean()) if len(ih_c) else np.nan, | |
| "verdict_recall": v_rc, "verdict_lead": v_ld}) | |
| log(f" [{lname}] pair vs {label:16s} {reg:8s}: comp thr m {thr_c['manual']:.4f} e {thr_c['engaged']:.4f} ({thr_src}) -> alerts/h {ach_c['manual'][2]:.1f} / {ach_c['engaged'][2]:.1f}; " | |
| f"n {int(m.sum())} / {int((y == 1).sum())} pos; recall ours {rc_h:.3f} comp {rc_c:.3f} Δ {rc_h - rc_c:+.3f} {ci(b_drc, 2)} -> {v_rc}; precision Δ {f3(pr_h - pr_c)} {ci(b_dpr, 2)}; " | |
| f"FAR ours {far_h:.3f} comp {far_c:.3f} Δ {far_h - far_c:+.3f} {ci(b_dfar, 3)}; lead Δ {f3(float(d0.mean()) if len(ih_c) else np.nan, 2)} s {ci(b_dl, 2)} n={len(ih_c)} both {int(both.sum())} -> {v_ld}") | |
| # ---- reliability + ECE (threshold-free; temperature on clean-val) | |
| for sname, gsc in (("head", gh), ("dms", att)): | |
| fv_, ft_ = gv & np.isfinite(gsc), gt & np.isfinite(gsc) | |
| T, pinned = WG.temperature_fit(gsc[fv_], yg[fv_].astype(float)) | |
| p_raw, p_cal = np.clip(gsc[ft_], 0, 1), WG.apply_temperature(gsc[ft_], T) | |
| e_raw, e_cal = WG.ece(yg[ft_], p_raw), WG.ece(yg[ft_], p_cal) | |
| b_e = cboot(dong[ft_], lambda pk: WG.ece(yg[ft_][pk], p_cal[pk]), a.n_boot, 3) | |
| for bn in reliability_bins(yg[ft_], p_cal): | |
| rows_rel.append({"item": a.item, "w_run": w_run, "seed": seed, "score": sname, "T": T, **bn}) | |
| rows_rel.append({"item": a.item, "w_run": w_run, "seed": seed, "score": sname, "T": T, "bin": -1, "n": int(ft_.sum()), "p_mean": float(p_cal.mean()), "y_rate": float(yg[ft_].mean()), | |
| "p_lo": np.nan, "p_hi": np.nan, "ece_raw": e_raw, "ece_cal": e_cal, "ece_cal_ci_lo": WG.pct(b_e)[0], "ece_cal_ci_hi": WG.pct(b_e)[1], "T_pinned": pinned}) | |
| log(f" reliability {sname}: T {T:.2f}{' (pinned)' if pinned else ''} ECE raw {e_raw:.3f} calibrated {e_cal:.3f} {ci(b_e)} on {int(ft_.sum())} clean-test rows") | |
| OP, LD, RL, PR, RF = (pd.DataFrame(x) for x in (rows_op, rows_lead, rows_rel, rows_pair, rows_ref)) | |
| OP.to_csv(out / "operating_points.csv", index=False) | |
| LD.to_csv(out / "lead_times.csv", index=False) | |
| RL.to_csv(out / "reliability.csv", index=False) | |
| PR.to_csv(out / "paired_lead.csv", index=False) | |
| RF.to_csv(out / "dms_reference.csv", index=False) | |
| PB = pd.DataFrame(rows_pub) | |
| if len(rows_pub) or len(rows_pubskip): # 2026-09-17 additive outputs (only when --pair-runs / --pair-surrogate were given) | |
| PB.to_csv(out / "paired_public.csv", index=False) | |
| pd.DataFrame(rows_pubskip, columns=["item", "seed", "comparator", "comp_run", "reason"]).to_csv(out / "paired_public_skipped.csv", index=False) | |
| # ---- table section (append-only) | |
| seeds = sorted(set(a.seeds)) | |
| L = [f"## item `{a.item}` — {a.label} (`{a.col}`) vs DMS att@3s at equal alert load — run {stamp}", "", | |
| f"Layouts {', '.join(a.w_runs)} (seeds {seeds}); DMS = legacy att@3s (`compose.legacy_compose` att_only on `marginals_<regime>_s<seed>.npz`, test split) and the `att@3s` gold column; " | |
| f"gold = clean human test set via the unchanged `warning_gold.build_gold` (legacy_first anchors); alerts / hour on the full TEST split with the 60 s per-route cooldown of " | |
| f"`train_warning_head.alerts_per_hour`; thresholds per regime by bisection to the target load (DMS-matched = the DMS chain at its own clean-val-recall-{a.val_recall:g} threshold; low = {a.low_load:g} / h); " | |
| f"recall / precision on the clean-test gold rows at the FIXED thresholds with a device-cluster bootstrap ({a.n_boot} resamples); lead time = t0 − first alert in [t0 − {a.lookback:g} s, t_anchor] over " | |
| f"the gold positives anchored on the test split; paired Δ = head − DMS on the same positives (undetected -> lead 0; both-detected median as secondary); pre-stated kill rule: Δ CI ⊃ 0 -> EQUAL. " | |
| f"Raw rows: `results/operating_points/{out.name}/`.", ""] | |
| L += ["### DMS reference (its own threshold at clean-val recall)", "", "| seed | thr | alerts/h manual | alerts/h engaged | gold recall | gold precision |", "|---|---|---|---|---|---|"] | |
| for r in RF.itertuples(): | |
| L.append(f"| {r.seed} | {r.thr_dms:.4f} | {r.alerts_per_hour_manual:.1f} | {r.alerts_per_hour_engaged:.1f} | {r.gold_recall:.3f} | {r.gold_precision:.3f} |") | |
| for lname in OP.load.unique(): | |
| L += ["", f"### Load `{lname}` — recall / precision / lead time at the matched threshold (mean over seeds; per-seed CIs in the CSV)" | |
| + (" — same_recall: the head is thresholded per regime to the DMS chain's gold recall at its own threshold (so precision and alerts/h are compared at equal recall)" if lname == "same_recall" else ""), "", | |
| "| regime | score | alerts/h m / e (achieved) | n gold / pos | recall [CI, seed 0] | precision [CI, seed 0] | lead median s [IQR] | lead median CI (seed 0) | detected | n pos (test-anchored) |", "|---|---|---|---|---|---|---|---|---|---|"] | |
| for reg in REGIMES + ("pooled",): | |
| for sname in ("head", "dms"): | |
| g = OP[(OP.load == lname) & (OP.regime == reg) & (OP.score == sname)].sort_values("seed") | |
| if not len(g): | |
| continue | |
| s0 = g.iloc[0] | |
| L.append(f"| {reg} | {a.label if sname == 'head' else 'DMS att@3s'} | {g.alerts_per_hour_manual.mean():.1f} / {g.alerts_per_hour_engaged.mean():.1f} | {int(s0.n_gold)} / {int(s0.n_pos)} | " | |
| f"{g.recall.mean():.3f} [{s0.recall_ci_lo:.2f}, {s0.recall_ci_hi:.2f}] | {f3(g.precision.mean())} [{f3(s0.precision_ci_lo, 2)}, {f3(s0.precision_ci_hi, 2)}] | " | |
| f"{f3(g.lead_median.mean(), 1)} [{f3(g.lead_q25.mean(), 1)}, {f3(g.lead_q75.mean(), 1)}] | [{f3(s0.lead_median_ci_lo, 1)}, {f3(s0.lead_median_ci_hi, 1)}] | {f3(g.lead_detected_frac.mean(), 2)} | {int(s0.n_pos_lead)} |") | |
| L += ["", f"Paired lead-time Δ ({a.label} − DMS) at load `{lname}`:", "", "| regime | seed | n pos | both detected | det head / DMS | Δ mean (undetected -> 0) [CI] | Δ median both-detected [CI] | head earlier / DMS earlier | verdict |", "|---|---|---|---|---|---|---|---|---|"] | |
| for r in PR[PR.load == lname].sort_values(["regime", "seed"]).itertuples(): | |
| L.append(f"| {r.regime} | {r.seed} | {r.n_pos} | {r.n_both_detected} | {r.det_head:.2f} / {r.det_dms:.2f} | {r.delta_lead0_mean:+.2f} [{r.delta_lead0_ci_lo:+.2f}, {r.delta_lead0_ci_hi:+.2f}] | " | |
| f"{f3(r.delta_median_both, 2)} [{f3(r.delta_median_both_ci_lo, 2)}, {f3(r.delta_median_both_ci_hi, 2)}] | {r.frac_head_earlier:.2f} / {r.frac_dms_earlier:.2f} | {r.verdict} |") | |
| L += ["", "### Reliability (clean-test gold, temperature fitted on clean-val; 10 equal-count bins) and ECE", "", "| score | seed | T | ECE raw | ECE calibrated [CI] | bins: mean p -> observed rate (n) |", "|---|---|---|---|---|---|"] | |
| for (sname, seed), g in RL.groupby(["score", "seed"]): | |
| tot = g[g.bin == -1].iloc[0] | |
| bins = g[g.bin >= 0].sort_values("bin") | |
| L.append(f"| {a.label if sname == 'head' else 'DMS att@3s'} | {seed} | {tot['T']:.2f} | {tot.ece_raw:.3f} | {tot.ece_cal:.3f} [{tot.ece_cal_ci_lo:.3f}, {tot.ece_cal_ci_hi:.3f}] | " | |
| + "; ".join(f"{b.p_mean:.2f}→{b.y_rate:.2f} ({int(b.n)})" for b in bins.itertuples()) + " |") | |
| if len(rows_pub): # 2026-09-17 additive subsection: paired ours − comparator cells | |
| L += ["", f"### Paired ours − comparator cells (2026-09-17 option `--pair-runs` / `--pair-surrogate`; `paired_public.csv`)", "", | |
| f"Comparators at the SAME load per seed and regime (published thresholds re-used where an OPDIR was given — load targets and alerts/h asserted equal — else the head's bisection rule); " | |
| f"cells on the common finite gold clean-test rows; FAR = share of gold NEGATIVE rows alerted (not a natural rate); lead over the common test-anchored positives (undetected -> 0); " | |
| f"every Δ = {a.label} − comparator, device-cluster bootstrap on identical resamples ({a.n_boot}); multi-seed verdict = >= ceil(2n/3) seed CIs exclude 0, same sign, |mean Δ| > sd. " | |
| f"Values = mean over seeds, CI = seed {seeds[0]}; k/n = seed CIs excluding 0.", ""] | |
| if len(rows_pubskip): | |
| L += ["Skipped (layouts incomplete): " + "; ".join(f"{r['comparator']} seed {r['seed']} ({r['reason']})" for r in rows_pubskip), ""] | |
| for lname in PB.load.unique(): | |
| L += [f"Load `{lname}`:", "", "| regime | comparator | alerts/h comp m / e | n common / pos | recall ours / comp | Δ recall [CI] | verdict | Δ precision [CI] | FAR ours / comp | Δ FAR [CI] | verdict | lead med ours / comp | Δ lead s [CI] | verdict |", | |
| "|---|---|---|---|---|---|---|---|---|---|---|---|---|---|"] | |
| for reg in REGIMES + ("pooled",): | |
| for label in PB.comparator.unique(): | |
| g = PB[(PB.load == lname) & (PB.regime == reg) & (PB.comparator == label)].sort_values("seed") | |
| if not len(g): | |
| continue | |
| s0 = g.iloc[0] | |
| L.append(f"| {reg} | {label} | {g.aph_comp_manual.mean():.1f} / {g.aph_comp_engaged.mean():.1f} | {int(s0.n_common)} / {int(s0.n_pos)} | {g.recall_head.mean():.3f} / {g.recall_comp.mean():.3f} | " | |
| f"{g.d_recall.mean():+.3f} [{f3(s0.d_recall_ci_lo)}, {f3(s0.d_recall_ci_hi)}] | {multi_verdict(g.d_recall, g.d_recall_ci_lo, g.d_recall_ci_hi, 'OURS HIGHER', 'COMP HIGHER')} | " | |
| f"{f3(g.d_precision.mean())} [{f3(s0.d_precision_ci_lo)}, {f3(s0.d_precision_ci_hi)}] | {g.far_head.mean():.3f} / {g.far_comp.mean():.3f} | " | |
| f"{g.d_far.mean():+.3f} [{f3(s0.d_far_ci_lo)}, {f3(s0.d_far_ci_hi)}] | {multi_verdict(g.d_far, g.d_far_ci_lo, g.d_far_ci_hi, 'OURS MORE FA', 'OURS FEWER FA')} | " | |
| f"{f3(g.lead_median_head.mean(), 1)} / {f3(g.lead_median_comp.mean(), 1)} | {f3(g.d_lead0_mean.mean(), 2)} [{f3(s0.d_lead0_ci_lo, 2)}, {f3(s0.d_lead0_ci_hi, 2)}] | " | |
| f"{multi_verdict(g.d_lead0_mean, g.d_lead0_ci_lo, g.d_lead0_ci_hi, 'OURS EARLIER', 'COMP EARLIER')} |") | |
| L.append("") | |
| pooled = PR[(PR.load == "dms_matched") & (PR.regime == "pooled")] | |
| verdict = "EQUAL" if all(v.startswith("EQUAL") for v in pooled.verdict) else "MIXED: " + ", ".join(f"s{r.seed} {r.verdict}" for r in pooled.itertuples()) | |
| L += ["", f"**Verdict (pre-stated rule, DMS-matched load, pooled regimes, {len(pooled)} seed(s))**: paired lead-time Δ = " | |
| + ", ".join(f"{r.delta_lead0_mean:+.2f} [{r.delta_lead0_ci_lo:+.2f}, {r.delta_lead0_ci_hi:+.2f}]" for r in pooled.itertuples()) + f" s -> **{verdict}**.", ""] | |
| md = "\n".join(L) + "\n" | |
| table = Path(a.table_out) | |
| if not table.exists(): | |
| table.write_text("# Equal-alert-load operating points — learned warning head vs the DMS attention chain (eval/operating_points.py; sections appended per run)\n\n") | |
| with open(table, "a") as f: | |
| f.write(md) | |
| print(md) | |
| write_run_json(out, "operating_points", [Path(__file__), Path(TW.__file__), Path(WG.__file__)], | |
| [CWM3 / "results" / "runs" / w / f"pred_layout_{r}_{s}.npz" for w in a.w_runs for r in REGIMES for s in ("test", "val")] + sorted(set(pair_inputs)), seed=0, t_start=t0, input_listing=False, | |
| extra={"item": a.item, "w_runs": a.w_runs, "seeds": a.seeds, "col": a.col, "low_load": a.low_load, "verdict": verdict, "dms_reference": rows_ref, "paired": rows_pair, "dms_fields_used": [], | |
| **({"pair_runs": a.pair_runs, "pair_surrogate": a.pair_surrogate, "pair_skipped": rows_pubskip, "n_paired_rows": len(rows_pub)} if (a.pair_runs or a.pair_surrogate) else {})}) | |
| log(f"DONE in {time.time() - t0:.0f}s -> {out}; table section appended to {table}") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 44.4 kB
- Xet hash:
- 134c36cd00a6522cab35d8321ea8d90c5f2858776f6e94d382ea290c8d3acac0
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.