Buckets:
| """cwm3/eval/warning_udwm_vs_original.py — PAIRED comparison of the Transformer-U warning read-out with the ORIGINAL (visual-only) read-out | |
| (demo / read-out parity lane; pre-registration row 2026-09-17T20:21:53Z in docs/decisions.md; DESCRIPTIVE ONLY — no manuscript change, nothing is selected). | |
| CPU only; read-only on every existing run and table (new files + one appended section). Additive: the helpers of eval/warning_ecan_vs_original.py are | |
| imported unchanged (that script's main() is never called). | |
| The Transformer-U heads are the SAME recipe (train/train_warning_head.py, identical argv) fitted on the joint runs `I1_all_minus_road2drv_s<k>_final_udwm` | |
| (driver branch = Driver-WM-style Transformer encoder, visual-only environment input); the original heads are the frozen paper read-outs on | |
| `I1_all_minus_road2drv_s<k>_final`. Every Δ below is UDWM − ORIGINAL, per seed, on IDENTICAL rows and device-cluster resamples: | |
| 1. GOLD (this script; the evaluator's own chain LB.episode_labels -> WG.load_pack -> WG.merge_legacy(seed) -> WG.build_gold(legacy_first) via | |
| calibration_report.gold_for_seed): clean-test AUROC and PR-AUC pooled / manual / engaged for cur+dwell+face (paper read-out; the demo lane) and | |
| current-only (pre-registered), both / future heads descriptive; paired Δ with warning_gold.cluster_boot on shared resamples (n_boot, seed 0). | |
| Bit-identity of the two layouts is reported as max |Δ score| over EVERY test / val row of both regimes (an exactly-zero Δ is a valid outcome). | |
| 2. TIMING (read from the alert_timing_eval.py run that holds the udwm scores next to the original defaults): Recall_{>= δ s early}, δ = 0.5 / 1 / 2 s, | |
| K = 10 / 20 / 40 per h, families A (primary) / B / C1 / C2 / D1 / D2, pooled / manual / engaged, VAL-fitted thresholds primary (test-fitted secondary) | |
| — paired.csv rows (a = udwm_cur_dwfc, b = cur_dwfc) with the run's own device-cluster CIs; alert-load and first-alert lead levels from metrics.csv. | |
| READING RULE (pre-registered): per-seed verdict identical / WORSE (CI < 0) / BETTER (CI > 0) / n.s.; an endpoint is 'significantly worse' / 'better' iff | |
| >= ceil(2n/3) seeds agree with the CI excluding 0. No openpilot DMS field is read (the att@3s chain enters only as the labelled comparator column already | |
| inside the gold frame). | |
| python eval/warning_udwm_vs_original.py [--seeds 0 1 2] [--n-boot 2000] [--timing-dir results/alert_timing/<stamp>] [--allow-missing] | |
| Outputs: results/warning_udwm_vs_original/<stamp>/{gold_pairs,timing_pairs,timing_levels,timing_summary,endpoints}.csv + section.md + run json; | |
| one section appended to results/tables/warning_udwm_vs_original.md (created with a title when absent; never overwritten). | |
| """ | |
| from __future__ import annotations | |
| import os | |
| os.environ.setdefault("OMP_NUM_THREADS", "4") | |
| os.environ.setdefault("MKL_NUM_THREADS", "4") | |
| import argparse # noqa: E402 | |
| import json # noqa: E402 | |
| import time # noqa: E402 | |
| import sys # noqa: E402 | |
| from pathlib import Path # noqa: E402 | |
| import numpy as np # noqa: E402 | |
| import pandas as pd # noqa: E402 | |
| CWM3 = Path(__file__).resolve().parents[1] | |
| sys.path.insert(0, str(CWM3 / "eval")) | |
| import warning_ecan_vs_original as EV # noqa: E402 helpers only (f3, ci_s, verdict1, seed_rule, load_col, at_gold, strata); its main() is never called | |
| CR, OP, WG, LB = EV.CR, EV.OP, EV.WG, EV.LB | |
| f3, ci_s, verdict1, seed_rule, load_col, at_gold, strata = EV.f3, EV.ci_s, EV.verdict1, EV.seed_rule, EV.load_col, EV.at_gold, EV.strata | |
| REGIMES = ("manual", "engaged") | |
| RUNS = CWM3 / "results" / "runs" | |
| OUT_ROOT = CWM3 / "results" / "warning_udwm_vs_original" | |
| TABLE = CWM3 / "results" / "tables" / "warning_udwm_vs_original.md" | |
| AT_ROOT = CWM3 / "results" / "alert_timing" | |
| PREREG = "2026-09-17T20:21:53Z" | |
| ORIG, UDWM = "I1_all_minus_road2drv_s{seed}_final", "I1_all_minus_road2drv_s{seed}_final_udwm" | |
| # (key, label, W layout suffix, column, pre-registered endpoint?) | |
| ARMS = [("dwfc", "cur+dwell+face (paper read-out recipe; the demo alert lane)", "_dwfc", "W_learned_curhgb_t15_w10_n979dwfc", True), | |
| ("cur", "current-only", "", "W_learned_curhgb_t15_w10_n979", True), | |
| ("both", "both = current + predicted future (descriptive; Transformer-U-dependent predicted features)", "", "W_learned_bothhgb_t15_w10_n979", False), | |
| ("fut", "predicted-future only (descriptive)", "", "W_learned_futhgb_t15_w10_n979", False)] | |
| TIMING_PAIRS = [("udwm_cur_dwfc", "cur_dwfc", True), ("udwm_cur", "cur_base", False), ("udwm_both", "both_base", False)] | |
| STRATA = ("pooled", "manual", "engaged") | |
| MARK = "udwm_cur_dwfc" | |
| def find_timing_dir(explicit): | |
| if explicit: | |
| return Path(explicit) | |
| cands = [] | |
| for rj in sorted(AT_ROOT.glob("*/run.json")): | |
| try: | |
| txt = rj.read_text() | |
| except OSError: | |
| continue | |
| if MARK in txt: | |
| cands.append(rj.parent) | |
| return cands[-1] if cands else None | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--seeds", nargs="+", type=int, default=[0, 1, 2]) | |
| ap.add_argument("--n-boot", type=int, default=2000) | |
| ap.add_argument("--timing-dir", default="", help=f"alert_timing run dir holding {MARK} (default: newest such run)") | |
| ap.add_argument("--allow-missing", action="store_true", help="render what exists (debug); the final run must have every input") | |
| ap.add_argument("--no-table", action="store_true") | |
| ap.add_argument("--out-root", default=str(OUT_ROOT), help="run-dir root (default results/warning_udwm_vs_original; /tmp for smoke runs)") | |
| a = ap.parse_args() | |
| t0 = time.time() | |
| stamp = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime()) | |
| out = Path(a.out_root) / stamp | |
| out.mkdir(parents=True, exist_ok=True) | |
| log_p = out / "warning_udwm_vs_original.log" | |
| def log(m): | |
| line = f"[{time.strftime('%H:%M:%S')}] {m}" | |
| print(line, flush=True) | |
| with open(log_p, "a") as f: | |
| f.write(line + "\n") | |
| lab = LB.episode_labels() | |
| lab = lab[lab.label != "unsure"].reset_index(drop=True) | |
| packs = {(r, s): WG.load_pack(r, s) for r in REGIMES for s in ("test", "val")} | |
| golds = {} | |
| def gold(seed): | |
| if seed not in golds: | |
| golds[seed] = CR.gold_for_seed(lab, packs, seed) | |
| return golds[seed] | |
| inputs, missing, endpoints = [], [], [] | |
| # ------------------------------------------------------------------ 1. gold (paired, this script) + layout identity | |
| rows_g, ident = [], {} | |
| for key, label, suf, col, prereg in ARMS: | |
| for seed in a.seeds: | |
| wu, wo = f"W_{UDWM.format(seed=seed)}{suf}", f"W_{ORIG.format(seed=seed)}{suf}" | |
| if not all((RUNS / w / f"pred_layout_{r}_{s}.npz").exists() for w in (wu, wo) for r in REGIMES for s in ("test", "val")): | |
| missing.append(f"layouts {wu} / {wo}") | |
| log(f"MISSING layouts for arm {key} seed {seed}: {wu} / {wo}") | |
| continue | |
| G = gold(seed) | |
| cu, co = load_col(wu, col, packs), load_col(wo, col, packs) | |
| inputs += [RUNS / w / f"pred_layout_{r}_{s}.npz" for w in (wu, wo) for r in REGIMES for s in ("test", "val")] | |
| maxabs, nan_mis, n_rows = 0.0, 0, 0 | |
| for k in cu: | |
| both = np.isfinite(cu[k]) & np.isfinite(co[k]) | |
| nan_mis += int((np.isfinite(cu[k]) != np.isfinite(co[k])).sum()) | |
| n_rows += len(cu[k]) | |
| if both.any(): | |
| maxabs = max(maxabs, float(np.abs(cu[k][both] - co[k][both]).max())) | |
| identical = (maxabs == 0.0) and (nan_mis == 0) | |
| ident[(key, seed)] = {"identical": identical, "max_abs_diff": maxabs, "nan_mismatch_rows": nan_mis, "n_rows": n_rows} | |
| log(f"arm {key} seed {seed}: {wu} vs {wo} col {col}: rows {n_rows}, max |Δ| {maxabs:.3e}, NaN-pattern mismatches {nan_mis} -> {'IDENTICAL' if identical else 'different'}") | |
| gu, go = at_gold(G, cu), at_gold(G, co) | |
| yg, dong = G.y.to_numpy().astype(int), G.dongle.to_numpy() | |
| for st, m0 in strata(G).items(): | |
| m = m0 & np.isfinite(gu) & np.isfinite(go) | |
| y, dg = yg[m], dong[m] | |
| if y.min() == y.max(): | |
| continue | |
| bs = WG.cluster_boot(y, {"u": gu[m], "o": go[m]}, dg, a.n_boot, 0) | |
| for met, fn, kb in (("auroc", WG.fast_auroc, "auroc"), ("pr_auc", WG.fast_ap, "prauc")): | |
| vu, vo = float(fn(y, gu[m])), float(fn(y, go[m])) | |
| d = bs["u"][kb] - bs["o"][kb] | |
| lo, hi = WG.pct(d) | |
| v = verdict1(vu - vo, lo, hi) | |
| rows_g.append({"arm": key, "label": label, "prereg": prereg, "seed": seed, "stratum": st, "metric": met, "udwm": vu, "orig": vo, "delta": vu - vo, "ci_lo": lo, "ci_hi": hi, | |
| "frac_boot_d_gt0": float(np.mean(d[np.isfinite(d)] > 0)) if np.isfinite(d).any() else np.nan, "n": int(m.sum()), "n_pos": int(y.sum()), | |
| "n_dev": int(len(np.unique(dg))), "identical_layouts": identical, "max_abs_diff_all_rows": maxabs, "verdict": v, "n_boot": a.n_boot}) | |
| log(f" gold {st:8s} {met:6s}: UDWM {vu:.4f} orig {vo:.4f} Δ {vu - vo:+.4f} {ci_s(lo, hi, 4)} n {int(m.sum())}/{int(y.sum())} -> {v}") | |
| Gd = pd.DataFrame(rows_g) | |
| Gd.to_csv(out / "gold_pairs.csv", index=False) | |
| for key, label, suf, col, prereg in ARMS: | |
| if not len(Gd): | |
| continue | |
| for st in STRATA: | |
| for met in ("auroc", "pr_auc"): | |
| g = Gd[(Gd.arm == key) & (Gd.stratum == st) & (Gd.metric == met)].sort_values("seed") | |
| if len(g): | |
| rule, nw, nb, n = seed_rule(g.verdict.tolist()) | |
| endpoints.append({"block": "gold", "arm": key, "prereg": prereg, "endpoint": f"gold {met} {st}", "seeds": n, "n_worse": nw, "n_better": nb, "rule": rule, | |
| "mean_delta": float(g.delta.mean()), "mean_udwm": float(g.udwm.mean()), "mean_orig": float(g.orig.mean())}) | |
| # ------------------------------------------------------------------ 2. timing (alert_timing_eval run with the udwm scores) | |
| tdir = find_timing_dir(a.timing_dir) | |
| rows_t, rows_tl, t_summary = [], [], [] | |
| if tdir is None or not (tdir / "paired.csv").exists(): | |
| missing.append(f"alert_timing run with {MARK}") | |
| log(f"MISSING alert_timing run holding {MARK}") | |
| else: | |
| P, M = pd.read_csv(tdir / "paired.csv"), pd.read_csv(tdir / "metrics.csv") | |
| inputs += [tdir / "paired.csv", tdir / "metrics.csv"] | |
| log(f"timing run {tdir.name}: paired rows {len(P)}, scores {sorted(P.a.unique())}") | |
| for na, nb, prereg in TIMING_PAIRS: | |
| sub = P[(P.a == na) & (P.b == nb)] | |
| if not len(sub): | |
| log(f" timing pair {na} − {nb}: no rows") | |
| continue | |
| for (fit, fam, B, dl, reg), g in sub.groupby(["thr_fit", "family", "budget", "delta_s", "regime"]): | |
| g = g.sort_values("seed") | |
| vs = [verdict1(r.diff, r.ci_lo, r.ci_hi) for r in g.itertuples()] | |
| for r, v in zip(g.itertuples(), vs): | |
| rows_t.append({"pair": f"{na} − {nb}", "prereg": prereg, "thr_fit": fit, "family": fam, "budget": B, "delta_s": dl, "regime": reg, "seed": int(r.seed), "udwm": r.value_a, "orig": r.value_b, | |
| "delta": r.diff, "ci_lo": r.ci_lo, "ci_hi": r.ci_hi, "n_events": int(r.n_events), "verdict": v}) | |
| rule, nw, nbet, n = seed_rule(vs) | |
| t_summary.append({"pair": f"{na} − {nb}", "prereg": prereg, "thr_fit": fit, "family": fam, "budget": B, "delta_s": dl, "regime": reg, "seeds": n, "n_worse": nw, "n_better": nbet, "rule": rule, | |
| "mean_delta": float(g["diff"].mean()), "mean_udwm": float(g.value_a.mean()), "mean_orig": float(g.value_b.mean()), "n_events": int(g.n_events.iloc[0])}) | |
| if prereg and fit == "val": | |
| endpoints.append({"block": "timing", "arm": "dwfc", "prereg": True, "endpoint": f"Recall>={dl:g}s early | family {fam} | {B:g}/h | {reg}", "seeds": n, "n_worse": nw, "n_better": nbet, | |
| "rule": rule, "mean_delta": float(g["diff"].mean()), "mean_udwm": float(g.value_a.mean()), "mean_orig": float(g.value_b.mean())}) | |
| for sc in (na, nb): | |
| for (fit, fam, B, reg), g in M[(M.score == sc)].groupby(["thr_fit", "family", "budget", "regime"]): | |
| if "|" in str(reg): | |
| continue | |
| g = g.sort_values("seed") | |
| rows_tl.append({"pair": f"{na} − {nb}", "score": sc, "thr_fit": fit, "family": fam, "budget": B, "regime": reg, "alerts_per_h_test": g.alerts_per_h_test.tolist(), | |
| "fa_per_h_test": g.fa_per_h_test.tolist(), "recall_early_1s": g.recall_early_1s.tolist(), "detected": g.detected.tolist(), "lead0_mean": g.lead0_mean.tolist(), | |
| "lead_detected_median": g.lead_detected_median.tolist(), "n_events": int(g.n_events.iloc[0])}) | |
| Td, TL, TS = pd.DataFrame(rows_t), pd.DataFrame(rows_tl), pd.DataFrame(t_summary) | |
| Td.to_csv(out / "timing_pairs.csv", index=False) | |
| TL.to_csv(out / "timing_levels.csv", index=False) | |
| TS.to_csv(out / "timing_summary.csv", index=False) | |
| E = pd.DataFrame(endpoints) | |
| E.to_csv(out / "endpoints.csv", index=False) | |
| if missing and not a.allow_missing: | |
| raise SystemExit("MISSING inputs (use --allow-missing to render a partial table): " + "; ".join(missing)) | |
| # ------------------------------------------------------------------ summary (descriptive; no decision is attached to this lane) | |
| Ep = E[E.prereg == True] if len(E) else E # noqa: E712 | |
| n_end = len(Ep) | |
| n_ident = int(Ep.rule.str.startswith("identical").sum()) if n_end else 0 | |
| n_worse = int(Ep.rule.str.startswith("SIGNIFICANTLY").sum()) if n_end else 0 | |
| n_better = int(Ep.rule.str.startswith("better").sum()) if n_end else 0 | |
| summary = (f"{n_end} pre-registered endpoints ({int((Ep.block == 'gold').sum()) if n_end else 0} gold, {int((Ep.block == 'timing').sum()) if n_end else 0} timing); identical (Δ = 0 exactly) {n_ident}; " | |
| f"significantly worse on >= 2/3 seeds {n_worse}; better on >= 2/3 seeds {n_better}; not worse / n.s. {n_end - n_ident - n_worse - n_better}; missing inputs {len(missing)}. " | |
| "DESCRIPTIVE ONLY (demo / read-out parity lane): no manuscript table or claim changes, nothing is selected.") | |
| log("SUMMARY: " + summary) | |
| # ------------------------------------------------------------------ section | |
| ts = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) | |
| L = [f"## {ts} — Transformer-U warning read-out vs the ORIGINAL (visual-only) read-out, PAIRED (eval/warning_udwm_vs_original.py; pre-registration {PREREG}; run {stamp})", "", | |
| f"Every Δ = UDWM − ORIGINAL per seed on identical rows and device-cluster resamples. Transformer-U heads: the SAME `train/train_warning_head.py` argv fitted on `{UDWM}` " | |
| f"(layouts `W_{UDWM}<suffix>`; driver branch = Driver-WM-style Transformer encoder, visual-only environment input); original heads: the frozen paper read-outs `W_{ORIG}<suffix>`. " | |
| f"Gold = `warning_gold.build_gold` legacy_first anchors (clean-test / clean-val), `warning_gold.cluster_boot` n_boot {a.n_boot} seed 0 (AUROC / PR-AUC). Timing cells from `results/alert_timing/{tdir.name if tdir else '—'}` " | |
| "(alert_timing_eval.py, udwm scores next to the original defaults in one run). Reading rule (pre-registered): per-seed verdict identical / WORSE (CI < 0) / BETTER (CI > 0) / n.s.; " | |
| "an endpoint is significantly worse / better iff >= ceil(2n/3) seeds agree with the CI excluding 0. **Descriptive lane (demo / read-out parity): no manuscript change, nothing selected.** " | |
| "Every number below is a CSV cell of the run dir.", "", | |
| "### 0. Layout identity (max |Δ score| over EVERY test / val row of both regimes; the current-state heads read no predicted quantity — z0 / z0_P / z0_R / valid_V are byte-identical between the two packs)", "", | |
| "| arm | column | seed | rows | max abs Δ | NaN-pattern mismatches | identical |", "|---|---|---|---|---|---|---|"] | |
| for key, label, suf, col, prereg in ARMS: | |
| for seed in a.seeds: | |
| if (key, seed) in ident: | |
| i = ident[(key, seed)] | |
| L.append(f"| {label} | `{col}` | {seed} | {i['n_rows']} | {i['max_abs_diff']:.3e} | {i['nan_mismatch_rows']} | {'YES' if i['identical'] else 'no'} |") | |
| L += ["", "### 1. Gold clean-test AUROC / PR-AUC (pooled / manual / engaged), paired Δ UDWM − original", "", | |
| "| arm | metric | stratum | UDWM s0 / s1 / s2 | original s0 / s1 / s2 | Δ s0 [CI] | Δ s1 [CI] | Δ s2 [CI] | n / pos | rule |", "|---|---|---|---|---|---|---|---|---|---|"] | |
| for key, label, suf, col, prereg in ARMS: | |
| for met in ("auroc", "pr_auc"): | |
| for st in STRATA: | |
| g = Gd[(Gd.arm == key) & (Gd.metric == met) & (Gd.stratum == st)].sort_values("seed") if len(Gd) else Gd | |
| if not len(g): | |
| continue | |
| rule = seed_rule(g.verdict.tolist())[0] if prereg else "descriptive: " + seed_rule(g.verdict.tolist())[0] | |
| L.append(f"| {label} | {met} | {st} | {' / '.join(f3(x) for x in g.udwm)} | {' / '.join(f3(x) for x in g.orig)} | " | |
| + " | ".join(f"{f3(r.delta, 3, True)} {ci_s(r.ci_lo, r.ci_hi)}" for r in g.itertuples()) + " | " * (3 - len(g)) + f" {int(g.n.iloc[0])} / {int(g.n_pos.iloc[0])} | {rule} |") | |
| L += ["", "### 2. Event-level timing — Recall_{>= δ s early} at K = 10 / 20 / 40 per h, VAL-fitted thresholds (PRIMARY); pair `udwm_cur_dwfc − cur_dwfc` (cur+dwell+face on both models)", ""] | |
| if len(TS): | |
| A = TS[(TS.pair == "udwm_cur_dwfc − cur_dwfc") & (TS.thr_fit == "val") & (TS.family == "A")] | |
| L += ["**Family A (registered, primary) — every endpoint:**", "", "| K/h | δ s | regime | UDWM mean | orig mean | Δ s0 [CI] | Δ s1 [CI] | Δ s2 [CI] | events | rule |", "|---|---|---|---|---|---|---|---|---|---|"] | |
| for r in A.sort_values(["budget", "delta_s", "regime"]).itertuples(): | |
| g = Td[(Td.pair == r.pair) & (Td.thr_fit == "val") & (Td.family == "A") & (Td.budget == r.budget) & (Td.delta_s == r.delta_s) & (Td.regime == r.regime)].sort_values("seed") | |
| L.append(f"| {r.budget:g} | {r.delta_s:g} | {r.regime} | {f3(r.mean_udwm)} | {f3(r.mean_orig)} | " + " | ".join(f"{f3(x.delta, 3, True)} {ci_s(x.ci_lo, x.ci_hi)}" for x in g.itertuples()) | |
| + " | " * (3 - len(g)) + f" {r.n_events} | {r.rule} |") | |
| L += ["", "**All families, VAL-fitted (primary) and TEST-fitted (secondary, equal achieved test load) — endpoint counts per family × K (9 endpoints each = 3 δ × 3 regimes):**", "", | |
| "| pair | thr fit | family | K/h | endpoints | identical | significantly worse | better | not worse (n.s.) | mean Δ over endpoints |", "|---|---|---|---|---|---|---|---|---|---|"] | |
| for (pair, fit, fam, B), g in TS.groupby(["pair", "thr_fit", "family", "budget"], sort=False): | |
| ni = int(g.rule.str.startswith("identical").sum()); nw = int(g.rule.str.startswith("SIGNIFICANTLY").sum()); nb = int(g.rule.str.startswith("better").sum()) | |
| L.append(f"| {pair} | {fit} | {fam} | {B:g} | {len(g)} | {ni} | {nw} | {nb} | {len(g) - ni - nw - nb} | {f3(g.mean_delta.mean(), 4, True)} |") | |
| W_ = TS[TS.rule.str.startswith("SIGNIFICANTLY")] | |
| B_ = TS[TS.rule.str.startswith("better")] | |
| L += ["", f"Significantly worse timing endpoints (all pairs / fits): {len(W_)}" + ("" if not len(W_) else " — " + "; ".join(f"{r.pair} {r.thr_fit} {r.family} {r.budget:g}/h δ{r.delta_s:g} {r.regime} (Δ {f3(r.mean_delta, 3, True)})" for r in W_.itertuples())), | |
| f"Better timing endpoints (all pairs / fits): {len(B_)}" + ("" if not len(B_) else " — " + "; ".join(f"{r.pair} {r.thr_fit} {r.family} {r.budget:g}/h δ{r.delta_s:g} {r.regime} (Δ {f3(r.mean_delta, 3, True)})" for r in B_.itertuples()))] | |
| if len(TL): | |
| L += ["", "**Alert load and first-alert lead levels at the VAL-fitted thresholds (per seed s0 / s1 / s2; pooled; family A):**", "", | |
| "| score | K/h | alerts/h test | FA/h test | Recall >= 1 s early | detected | lead0 mean s (undetected = 0) | lead median s (detected) | events |", "|---|---|---|---|---|---|---|---|---|"] | |
| for r in TL[(TL.thr_fit == "val") & (TL.family == "A") & (TL.regime == "pooled") & (TL.score.isin(["cur_dwfc", "udwm_cur_dwfc", "cur_base", "udwm_cur", "both_base", "udwm_both"]))].drop_duplicates(["score", "budget"]).sort_values(["budget", "score"]).itertuples(): | |
| j = lambda xs, nd=3: " / ".join(f3(x, nd) for x in xs) # noqa: E731 | |
| L.append(f"| {r.score} | {r.budget:g} | {j(r.alerts_per_h_test, 1)} | {j(r.fa_per_h_test, 1)} | {j(r.recall_early_1s)} | {j(r.detected)} | {j(r.lead0_mean, 2)} | {j(r.lead_detected_median, 2)} | {r.n_events} |") | |
| else: | |
| L.append("timing run not available") | |
| L += ["", "### 3. SUMMARY (pre-registered reading rule; descriptive — no decision attached)", "", summary, ""] | |
| if n_worse: | |
| L += ["Significantly worse endpoints: " + "; ".join(f"{r.endpoint} [{r.arm}] (mean Δ {f3(r.mean_delta, 3, True)}, {r.rule})" for r in Ep[Ep.rule.str.startswith('SIGNIFICANTLY')].itertuples()), ""] | |
| if n_better: | |
| L += ["Better endpoints: " + "; ".join(f"{r.endpoint} [{r.arm}] (mean Δ {f3(r.mean_delta, 3, True)}, {r.rule})" for r in Ep[Ep.rule.str.startswith('better')].itertuples()), ""] | |
| if missing: | |
| L += ["Missing inputs: " + "; ".join(missing), ""] | |
| md = "\n".join(L) + "\n" | |
| (out / "section.md").write_text(md) | |
| if not a.no_table: | |
| if not TABLE.exists(): | |
| TABLE.write_text("# Transformer-U warning read-out vs the original (visual-only) read-out — paired comparison (append-only; eval/warning_udwm_vs_original.py; demo / read-out parity lane)\n\n") | |
| with open(TABLE, "a") as f: | |
| f.write("\n" + md) | |
| print(md) | |
| OP.write_run_json(out, "warning_udwm_vs_original", [Path(__file__), Path(EV.__file__), Path(CR.__file__), Path(OP.__file__), Path(WG.__file__)], sorted(set(inputs)), seed=0, t_start=t0, input_listing=False, | |
| extra={"prereg": PREREG, "seeds": a.seeds, "n_boot": a.n_boot, "timing_dir": str(tdir) if tdir else "", "identity": {f"{k[0]}|s{k[1]}": v for k, v in ident.items()}, | |
| "n_endpoints_prereg": int(n_end), "n_identical": n_ident, "n_sig_worse": n_worse, "n_better": n_better, "summary": summary, "missing": missing, "dms_fields_used": []}) | |
| log(f"DONE in {time.time() - t0:.0f}s -> {out}; section {'appended to ' + str(TABLE) if not a.no_table else 'NOT appended'}") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 23.5 kB
- Xet hash:
- 8e60dfea2f03f7bc2b4e76ce595cbfdeecd2e53caaf0553874df9b2d69adfdf4
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.