Buckets:
| """cwm3/eval/report_holdout.py — results/tables/stage_holdout.md: the BATON-HOLDOUT confirmation batch (WP5; pre-registration docs/decisions.md 2026-09-17T11:45:58Z), | |
| composed from results/paired_compare.csv rows with split `holdout` (written by eval/pc_holdout.py over the *_holdout layouts of eval/predict_holdout.py) and the benchmark | |
| TEST rows of results/tables/stage_final.md / stage_final_ecan.md (tags finalc_* / finalP_*). Nothing is recomputed here; every number comes from the CSV / the holdout | |
| artefact files. Verdict rule = the benchmark's (report_joint.verdict -> kill_checks.beats: >= 2/3 seeds CI excluding 0, same sign, |mean Δ| > seed sd). | |
| python eval/report_holdout.py [--seeds 0,1,2] [--out results/tables/stage_holdout.md] | |
| Candidates (both pre-specified, both reported; NO selection between them is made on the holdout): | |
| (i) frozen main `I1_all_minus_road2drv_s<k>_final` vs its indep comparator `I1_indep_s<k>_final` tags holdc_s<k>_allminus_vs_indep (+ holdP_) | |
| (ii) ECAN main `I1_all_minus_road2drv_s<k>_final_ecan` vs `I1_indep_s<k>_final_ecan` tags holdc_s<k>_allminusecan_vs_indepecan (+ holdP_) | |
| (ii-a) ECAN main vs frozen main — the benchmark adoption comparison (a) of stage_final_ecan.md tags holdc_s<k>_allminusecan_vs_allminus (+ holdP_) | |
| Replication per cell = same sign of the mean Δ AND the same significance class (significant = BEATS / WORSE; null = NOT_WORSE / INCONCLUSIVE) as the benchmark TEST verdict. | |
| The table is append-only: an existing file receives a new dated section. | |
| (iii) F1 kpf joint model `I1_all_minus_road2drv_s<k>_final_ecan_kpf` vs the frozen ECAN main `I1_all_minus_road2drv_s<k>_final_ecan` tags holdc_s<k>_allminusecankpf_vs_allminusecan (+ holdP_) | |
| (2026-09-18, pre-registration docs/decisions.md 2026-09-18T03:41:14Z; rendered with `--candidates iii --out results/holdout/candidate_iii/stage_holdout_candidate_iii.md | |
| --json-out .../stage_holdout_candidate_iii.json`; its READING RULE block (primary head_turn_onset cells BEATS, no own onset WORSE / margin breach, no hazard-NLL | |
| margin breach) is candidate-specific and additive — the (i) / (ii) blocks render exactly as before). | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import os | |
| import sys | |
| import time | |
| from pathlib import Path | |
| import numpy as np | |
| import pandas as pd | |
| CWM3 = Path(__file__).resolve().parents[1] | |
| sys.path.insert(0, str(CWM3 / "eval")) | |
| import report_joint as RJ # noqa: E402 | |
| import report_final_face as RF # noqa: E402 (margin_table / margin_ok of the F1 acceptance rule; 2026-09-18 candidate (iii)) | |
| H = CWM3 / "results" / "holdout" | |
| S12R = CWM3.parent / "s12_wm" / "results" | |
| RUNS = CWM3 / "results" / "runs" | |
| GRID_DT = 0.2 | |
| REGS = ("manual", "engaged") | |
| ROAD_D = ("ttc", "ttc_drop", "lane", "badas") | |
| P_CHAINS = ("ttc", "ttc_drop", "cut_in", "lead_braking", "lane_excursion", "badas") | |
| DRIVER_REPORTED = {("manual", "drv_onset|hand_leave|3s"), ("manual", "drv_onset|hand_return|3s"), ("manual", "drv_onset|head_turn_onset|3s"), ("engaged", "drv_onset|head_turn_onset|3s")} | |
| DRIVER_NOT_REPORTED = {("engaged", "drv_onset|hand_leave|3s"), ("engaged", "drv_onset|hand_return|3s")} | |
| GROUPS = [ | |
| ("driver (4 reported cells + 2 engaged hand cells computed, NOT reported)", [f"drv_onset|{e}|3s" for e in ("hand_leave", "hand_return", "head_turn_onset")], "pr_auc"), | |
| ("vehicle onsets", [f"veh_onset|{e}|3s" for e in ("gas_on", "brake_on", "steer_phase_on", "veh_any_on")], "pr_auc"), | |
| ("road D cells (ttc / ttc_drop / lane / badas = ROAD; decel / steer_rate = vehicle-dynamics D families)", [f"d_onset|{f}|3s" for f in ("ttc", "ttc_drop", "lane", "badas", "decel", "steer_rate")], "pr_auc"), | |
| ("road P cells (eval/pressure_eval.py)", [f"p_onset|{c}|3s" for c in P_CHAINS], "pr_auc"), | |
| ("hazard-NLL costs (lower better)", [f"hazard_nll|{e}" for e in ("hand_leave", "hand_return", "head_turn_onset", "gas_on", "brake_on", "steer_phase_on")] + [f"hazard_nll|D_{f}" for f in ("ttc", "ttc_drop", "lane", "badas", "decel", "steer_rate")], "nll"), | |
| ] | |
| CANDS = { | |
| "i": dict(title="Candidate (i) — frozen main `I1_all_minus_road2drv_s<k>_final` (A) vs indep `I1_indep_s<k>_final` (B)", | |
| hold=("holdc", "allminus", "indep"), holdP=("holdP", "allminus", "indep"), test=("finalc", "allminus", "indep"), testP=None, | |
| key=[(r, f"veh_onset|{e}|3s", "pr_auc") for r in REGS for e in ("steer_phase_on", "brake_on")], | |
| key_label="vehicle steer_phase_on / brake_on cells, both regimes (the pre-registered reading for the main model)", | |
| bench="results/tables/stage_final.md §1 B2 main comparison"), | |
| "ii": dict(title="Candidate (ii) — ECAN main `I1_all_minus_road2drv_s<k>_final_ecan` (A) vs ECAN indep `I1_indep_s<k>_final_ecan` (B)", | |
| hold=("holdc", "allminusecan", "indepecan"), holdP=("holdP", "allminusecan", "indepecan"), test=("finalc", "allminusecan", "indepecan"), testP=("finalP", "allminusecan", "indepecan"), | |
| key=[(r, f"veh_onset|{e}|3s", "pr_auc") for r in REGS for e in ("steer_phase_on", "brake_on")], | |
| key_label="vehicle steer_phase_on / brake_on cells, both regimes (the Table 3 ECAN main column main_ecan − indep_ecan)", | |
| bench="results/tables/stage_final_ecan_table3.md §4 main_ecan − indep_ecan (the paper's Table 3 ECAN main column)"), | |
| "ii-a": dict(title="Candidate (ii) adoption comparison (a) — ECAN main (A) vs frozen main `I1_all_minus_road2drv_s<k>_final` (B)", | |
| hold=("holdc", "allminusecan", "allminus"), holdP=("holdP", "allminusecan", "allminus"), test=("finalc", "allminusecan", "allminus"), testP=("finalP", "allminusecan", "allminus"), | |
| key=[(r, f"d_onset|{f}|3s", "pr_auc") for r in REGS for f in ROAD_D] + [(r, f"p_onset|{c}|3s", "pr_auc") for r in REGS for c in P_CHAINS], | |
| key_label="the ECAN road cells: D onset ttc / ttc_drop / lane / badas and the six P cells, both regimes (the pre-registered reading for the ECAN candidate)", | |
| bench="results/tables/stage_final_ecan.md §1 (a) + §7 P cells"), | |
| } | |
| SIG = {"BEATS", "WORSE"} | |
| # ---- candidate (iii) (2026-09-18, pre-registration 2026-09-18T03:41:14Z): the F1 kpf joint model vs the frozen ECAN main | |
| PREREG_III = "2026-09-18T03:41:14Z" | |
| HT = "drv_onset|head_turn_onset|3s" | |
| VEH4 = [f"veh_onset|{e}|3s" for e in ("gas_on", "brake_on", "steer_phase_on", "veh_any_on")] | |
| NLL_VEH_CELLS = [f"hazard_nll|{e}" for e in ("gas_on", "brake_on", "steer_phase_on")] # paired_compare_joint's names (report_final_face.NLL_VEH uses gas/brake/steer_phase -> no rows) | |
| CANDS["iii"] = dict(title="Candidate (iii) — F1 kpf joint model `I1_all_minus_road2drv_s<k>_final_ecan_kpf` (A) vs the frozen ECAN main `I1_all_minus_road2drv_s<k>_final_ecan` (B)", | |
| hold=("holdc", "allminusecankpf", "allminusecan"), holdP=("holdP", "allminusecankpf", "allminusecan"), test=("finalc", "allminusecankpf", "allminusecan"), testP=("finalP", "allminusecankpf", "allminusecan"), | |
| key=[(r, HT, "pr_auc") for r in REGS], | |
| key_label="the two validation-selected cells manual / engaged head_turn_onset PR-AUC@3s (replication descriptor; the decisive pre-registered READING RULE is the block below)", | |
| bench="results/tables/kp_residual_f1.md §2 (a) TEST confirmation, tags finalc_/finalP_s<k>_allminusecankpf_vs_allminusecan", rule="kpf_iii") | |
| def header_iii(stamp): | |
| return [f"# Stage HOLDOUT — candidate (iii): the F1 kpf joint model on the BATON-holdout (60 leaves / 60 unseen devices), frozen models evaluated ONCE ({stamp})", "", | |
| f"Pre-registration: `docs/decisions.md` row {PREREG_III} (candidate, hold-out input construction, tags, READING RULE — written before any hold-out number of this candidate existed); " | |
| "owner decision row 2026-09-18T03:35:56Z (option A); F1 outcome row 2026-09-18T03:31:07Z (validation rule MET, test confirms; `results/tables/kp_residual_f1.md`); hold-out batch rows 2026-09-17T11:45:58Z / 15:49:42Z / 16:25:16Z; runbook `docs/holdout_runbook.md` §4. ", | |
| "Forecaster hold-out inputs: `data/build_kp_residual_store.py --holdout` (the same builder on the hold-out grid; pose `results/holdout/pose_causal_routes`, population driver-side prior; hold-out kin-v2 anchors) -> `results/holdout/kp_residual/`; " | |
| "`eval/predict_kpf_holdout.py --run KPF_mb_full_s<k>` (FROZEN forecaster checkpoint, frozen TRAIN `scale_fallback`, `train_kp_residual.predict` unchanged) -> `results/runs/KPF_mb_full_s<k>_holdout/kpf_{r}_holdout.npz`. " | |
| "Joint predictions: `eval/predict_holdout.py --run I1_all_minus_road2drv_s<k>_final_ecan_kpf --u-kpf-dir results/runs/KPF_mb_full_s<k>_holdout` -> NEW `results/runs/<run>_holdout/` (frozen checkpoint symlinked; the B layouts are the candidate-(ii) hold-out layouts of the frozen ECAN main, re-used unchanged). " | |
| "Pairings: `eval/pc_holdout.py joint|pressure` = eval/paired_compare_joint.py / eval/pressure_eval.py with the holdout target dirs, split `holdout`, causal v2 driver populations, device-cluster bootstrap n_boot 1000, seeds 0/1/2; " | |
| "verdict = kill_checks.beats (>= 2/3 seeds CI excluding 0, same sign, |mean Δ| > seed sd). Every number below comes from `results/paired_compare.csv` (split `holdout` for the holdout, split `test` for the benchmark) or the listed artefact files.", "", | |
| "**Evaluated once, no selection**: the frozen kpf joint runs were predicted once on the hold-out, each cell paired once; no model, threshold, forecaster or population rule was changed after the hold-out numbers were seen; " | |
| "nothing was refit on hold-out data (forecaster normalisation = the frozen TRAIN constant + the per-anchor causal ref / scale of the frozen forward). The reading rule was fixed in the pre-registration row and is applied mechanically below; adoption follows only if it is met.", "", | |
| "## Holdout population", ""] + population_lines() + [""] | |
| def status_iii(): | |
| return ["- Inputs: the forecaster's hold-out channel exists for 56 / 60 leaves with anchors (the 4 leaves without a valid anchor contribute no rows, as for every candidate); the KPF store was built by the same builder (`--holdout`, additive; the benchmark `results/kp_residual/` untouched); the frozen forecasters and the frozen joint runs were not modified (mtimes recorded in `results/holdout/candidate_iii/frozen_mtimes_before.txt`).", | |
| "- Evaluated ONCE: each frozen kpf joint run predicted once on the holdout, each cell paired once; no selection, no threshold change, no re-run; the B layouts are those of candidate (ii) (2026-09-17T16:25:16Z), not re-predicted.", | |
| "- Engaged hand cells are computed and shown but marked *(not reported)* per the pre-registration (v2 label-check story, Supp E); `d_onset|badas` has n = 0 on the holdout (badas_risk 0 / 60 routes) and is excluded from the own-onset check as pre-registered.", | |
| "- This file is written to `results/holdout/candidate_iii/` only; `results/tables/stage_holdout.md` / `stage_holdout*.json` are not touched (manuscript round 5 running; the coordinator merges after it closes)."] | |
| def rule_block_iii(dh, verd, dt, seeds): | |
| """the pre-registered READING RULE of candidate (iii) (docs/decisions.md 2026-09-18T03:41:14Z), applied mechanically to the hold-out verdicts: | |
| (a) manual AND engaged head_turn_onset PR-AUC@3s BEATS; (b) no own onset cell (report_joint.OWN, badas excluded: n = 0) WORSE and none beyond the −0.005 mean-Δ margin; | |
| (c) no hazard-NLL cell (hand_leave / hand_return / head_turn_onset / gas_on / brake_on / steer_phase_on, both regimes) beyond +0.010 nats (mean; no seed CI entirely above).""" | |
| mt, checks = RF.margin_table(dh, verd, "Reading-rule cells — own onset cells and hazard-NLL costs with the pre-registered margins (HOLD-OUT; A = kpf joint model, B = frozen ECAN main)", RF.DRV + VEH4, RF.NLL_DRV + NLL_VEH_CELLS) | |
| own = {k: v for k, v in verd.items() if k[1] in RJ.OWN and k[2] == "pr_auc" and k[1] != "d_onset|badas|3s"} | |
| worse = [k for k, v in own.items() if v["verdict"] == "WORSE"] | |
| insuff = [k for k, v in own.items() if v["verdict"] == "INSUFFICIENT"] | |
| core = [(r, c, "pr_auc") for r in REGS for c in RF.DRV + RF.VEH] | |
| missing_core = [k for k in core if k not in verd] | |
| broke = [k for k, c in checks.items() if k[2] == "pr_auc" and k[1] in RJ.OWN and not c["ok"]] | |
| nll_broke = [k for k, c in checks.items() if k[2] == "nll" and not c["ok"]] | |
| nll_missing = [(r, c) for r in REGS for c in RF.NLL_DRV + NLL_VEH_CELLS if (r, c, "nll") not in checks] | |
| primary = {r: verd.get((r, HT, "pr_auc"), {}).get("verdict", "NO ROWS") for r in REGS} | |
| primary_ok = all(v == "BEATS" for v in primary.values()) | |
| complete = not insuff and not missing_core and not nll_missing | |
| met = primary_ok and not worse and not broke and not nll_broke and complete | |
| status = "INSUFFICIENT (missing / < 3-seed cells)" if not complete else ("MET" if met else "NOT MET") | |
| verd_t = RJ.render(dt, sorted(dt.seed.unique()), CANDS["iii"]["test"][1], CANDS["iii"]["test"][2], "x")[1] if len(dt) else {} | |
| L = [f"### Pre-registered READING RULE (docs/decisions.md {PREREG_III}) — **{status}**", "", | |
| f"- (a) primary, validation-selected cells must be BEATS on the hold-out: manual head_turn_onset **{primary[REGS[0]]}**, engaged head_turn_onset **{primary[REGS[1]]}** -> {'satisfied' if primary_ok else 'NOT satisfied'}", | |
| f"- (b) no own onset cell WORSE (report_joint.OWN both regimes, badas excluded): WORSE = {RF.fmt(worse)}; own onset cells beyond the −0.005 mean-Δ margin: {RF.fmt(broke)} -> {'satisfied' if not worse and not broke else 'NOT satisfied'}", | |
| f"- (c) no hazard-NLL margin (+0.010 nats, mean and every seed CI) broken: {RF.fmt(nll_broke)} -> {'satisfied' if not nll_broke else 'NOT satisfied'}", | |
| f"- completeness: INSUFFICIENT own cells {RF.fmt(insuff)}; missing core cells {RF.fmt(missing_core)}; missing hazard-NLL cells {nll_missing or 'none'}", | |
| f"- **rule {status}** -> {'adoption of the kpf joint model as CockpitWM (ECAN, default) follows per owner decision 2026-09-18T03:35:56Z' if met else 'NO adoption: the kpf joint model stays a disclosed WP4 candidate (Supp); the frozen ECAN main remains CockpitWM'}", ""] | |
| L += mt | |
| L += ["### Secondary cells (reported, not part of the rule) — hold-out vs TEST", "", | |
| "| regime | cell | holdout mean Δ ± sd | holdout verdict | TEST mean Δ ± sd | TEST verdict | replicates |", "|---|---|---|---|---|---|---|"] | |
| sec = {} | |
| for r, cell in [("manual", "drv_onset|hand_return|3s")] + [(r, c) for r in REGS for c in VEH4] + [(r, HT) for r in REGS] + [(r, f"hazard_nll|{e}") for r in REGS for e in ("head_turn_onset", "hand_leave", "hand_return")]: | |
| metric = "nll" if cell.startswith("hazard_nll") else "pr_auc" | |
| h = cell_stats(dh, r, cell, metric); t = cell_stats(dt, r, cell, metric) if len(dt) else None | |
| if h is None: | |
| continue | |
| rep = replicates(h, t) | |
| tcol = f"{t['mean']:+.4f} ± {t['sd']:.4f}" if t else "—" | |
| L.append(f"| {r} | {cell.replace('|', '/')} | {h['mean']:+.4f} ± {h['sd']:.4f} | {h['verdict']} | {tcol} | {t['verdict'] if t else '—'} | {rep} |") | |
| sec[f"{r}|{cell}|{metric}"] = {"holdout": {k: v for k, v in h.items() if k != "by"}, "test": ({k: v for k, v in t.items() if k != "by"} if t else None), "replicates": rep} | |
| L.append("") | |
| js = {"status": status, "met": bool(met), "primary": primary, "own_worse": RF.fmt(worse), "own_beyond_margin": RF.fmt(broke), "nll_beyond_margin": RF.fmt(nll_broke), | |
| "insufficient_own": RF.fmt(insuff), "missing_core": RF.fmt(missing_core), "missing_nll": [f"{r}|{c}" for r, c in nll_missing], | |
| "margin_checks": {f"{r}|{c}|{m}": c_ for (r, c, m), c_ in checks.items()}, "secondary": sec, "test_verdicts": {f"{r}|{c}|{m}": v["verdict"] for (r, c, m), v in verd_t.items()}} | |
| return L, js | |
| def load_pair(stage, a, b, seeds): | |
| df = RJ.load(stage, a, b, seeds) | |
| return df | |
| def cell_stats(df, regime, cell, metric): | |
| g = df[(df.regime == regime) & (df.cell == cell) & (df.metric == metric)].sort_values("seed") | |
| if not len(g): | |
| return None | |
| lb = cell.startswith(RJ.LOWER_BETTER) | |
| res = RJ.verdict(g, lb) | |
| d = g.delta.astype(float).to_numpy() | |
| by = {int(s): r for s, r in zip(g.seed, g.itertuples())} | |
| return {"verdict": res["verdict"], "mean": float(d.mean()), "sd": float(d.std(ddof=1)) if len(d) > 1 else 0.0, "n_seeds": int(len(d)), "by": by, | |
| "n": int(g.n.iloc[0]) if str(g.n.iloc[0]).strip() not in ("", "nan") else None, "n_pos": int(g.n_pos.iloc[0]) if str(g.n_pos.iloc[0]).strip() not in ("", "nan") else None, | |
| "n_dev": int(g.n_dev.iloc[0]) if str(g.n_dev.iloc[0]).strip() not in ("", "nan") else None, "value_a": [float(by[s].value_a) for s in sorted(by)], "value_b": [float(by[s].value_b) for s in sorted(by)]} | |
| def replicates(h, t): | |
| if h is None or t is None: | |
| return "—" | |
| if h["verdict"] == "INSUFFICIENT" or t["verdict"] == "INSUFFICIENT": | |
| return "— (insufficient seeds)" | |
| same_sign = np.sign(h["mean"]) == np.sign(t["mean"]) | |
| same_class = (h["verdict"] in SIG) == (t["verdict"] in SIG) | |
| return "yes" if (same_sign and same_class) else "no" | |
| def provenance_holdout(run_names, df): | |
| lines = ["| run dir (new, holdout) | source frozen run | checkpoint | regime | layout mtime (UTC) | source sha256 | model_id / code | rows vs layout |", "|---|---|---|---|---|---|---|---|"] | |
| for r in run_names: | |
| rd = RUNS / r | |
| if not rd.exists(): | |
| lines.append(f"| {r} | MISSING | | | | | | |") | |
| continue | |
| pd_ = json.loads((rd / "PRED_DONE").read_text()) if (rd / "PRED_DONE").exists() else {} | |
| best = (rd / "BEST").read_text().strip() if (rd / "BEST").exists() else "?" | |
| ck = f"{best} -> {os.readlink(rd / best)}" if best != "base" and (rd / best).is_symlink() else best | |
| for reg in REGS: | |
| lay = rd / f"pred_layout_{reg}_holdout.npz" | |
| if not lay.exists(): | |
| lines.append(f"| {r} | {pd_.get('source_run', '?')} | {ck} | {reg} | layout MISSING | | | |") | |
| continue | |
| mt = os.path.getmtime(lay) | |
| meta = json.loads(str(np.load(lay, allow_pickle=True)["meta"])) | |
| rows = df[((df.a.astype(str) == r) | (df.b.astype(str) == r)) & (df.regime == reg)] | |
| oldest = str(rows.run_stamp.min()) if len(rows) else "" | |
| st = ("STALE ROWS" if oldest and oldest < time.strftime("%Y%m%dT%H%M%SZ", time.gmtime(mt)) else "rows postdate layout") if oldest else "no rows" | |
| lines.append(f"| {r} | {pd_.get('source_run', '?')} | {ck} | {reg} | {time.strftime('%Y-%m-%d %H:%M:%SZ', time.gmtime(mt))} | {str(meta.get('inputs_digest', ''))[:16]} | {meta.get('model_id', '')} / code {str(meta.get('code_sha256', ''))[:12]} | {st} |") | |
| return lines | |
| def population_lines(): | |
| L = [] | |
| draw = json.loads((H / "draw_v1.json").read_text()) | |
| tb = json.loads((H / "targets_build_holdout.json").read_text()) | |
| per_route = tb.get("per_route", {}) | |
| no_anchor = [u for u, v in per_route.items() if not v] | |
| L += [f"- draw (`results/holdout/draw_v1.json`, seed {draw['seed']}, {draw['utc']}): **{draw['n_chosen']} leaves / {draw['n_chosen_devices']} devices / {draw['n_cars']} car dirs**, " | |
| f"{draw['n_chosen_engaged']} engaged + {draw['n_chosen_manual_only']} manual-only strata; video {draw['video_hours']} h (engaged {draw['engaged_hours']} h, manual {draw['manual_hours']} h); " | |
| f"eligible population {draw['n_eligible']} leaves / {draw['n_eligible_devices']} devices; vehicle-disjoint from the benchmark ({len(draw['excluded_devices_vin_in_benchmark'])} device excluded by VIN); no benchmark device."] | |
| for r in REGS: | |
| with np.load(S12R / f"targets_{r}_holdout.npz", allow_pickle=True) as z: | |
| n, nd, nu = len(z["t"]), len(np.unique(z["dongle"])), len(np.unique(z["uid"])) | |
| m = int(z["M"].any(1).sum()) if "M" in z else n | |
| L.append(f"- anchors `{r}`: **{n:,} rows / {nu} leaves / {nd} devices / {n * GRID_DT / 3600:.2f} anchor-h** (rows with any valid step {m:,}); `s12_wm/results/targets_{r}_holdout.npz`") | |
| L.append(f"- leaves without any valid anchor: {len(no_anchor)} of {tb.get('n_leaves', 60)} (`results/holdout/targets_build_holdout.json`); leaves contributing anchors {tb.get('n_leaves', 60) - len(no_anchor)}") | |
| al = H / "pack_v2" / "alignment_holdout.json" | |
| if al.exists(): | |
| a = json.loads(al.read_text()) | |
| L.append("- pack v2 (frozen PCA / future PCA): " + "; ".join(f"`{k}` row_ok {v['row_ok']:,}/{v['N']:,} ({v['row_ok_frac']:.4f}), routes_ok {v['routes_ok']}" for k, v in a["streams"].items()) + f"; s5 routes {a['routes']}") | |
| return L | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--seeds", default="0,1,2") | |
| ap.add_argument("--out", default=str(CWM3 / "results" / "tables" / "stage_holdout.md")) | |
| ap.add_argument("--candidates", default="i,ii,ii-a", help="which blocks to render (comma list of i, ii, ii-a); an existing --out receives the rendering as a NEW dated section") | |
| ap.add_argument("--section-note", default="", help="one-line note placed under the section title (e.g. the decisions row that unblocked candidate (ii))") | |
| ap.add_argument("--json-out", default="", help="explicit json path (2026-09-18 candidate (iii): results/holdout/candidate_iii/stage_holdout_candidate_iii.json); default = the dated / sibling json as before") | |
| a = ap.parse_args() | |
| seeds = [int(s) for s in a.seeds.split(",")] | |
| cands = [c.strip() for c in a.candidates.split(",") if c.strip()] | |
| assert all(c in CANDS for c in cands), cands | |
| stamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) | |
| L = header_iii(stamp) if cands == ["iii"] else [f"# Stage HOLDOUT — BATON-holdout confirmation batch (60 leaves / 60 unseen devices), frozen models evaluated ONCE ({stamp})", "", | |
| "Pre-registration: `docs/decisions.md` row 2026-09-17T11:45:58Z (candidates, cells, protocol, reading) and the WP5 rows 2026-09-16T06:46:35Z / 07:2x Z (draw, fetch); runbook `docs/holdout_runbook.md` §4. " | |
| "Predictions: `eval/predict_holdout.py --run <frozen run>` -> NEW dirs `results/runs/<run>_holdout/` (frozen checkpoint symlinked, holdout artefacts of `results/holdout/` substituted for the benchmark packs; the frozen run dirs are untouched). " | |
| "Pairings: `eval/pc_holdout.py joint|pressure` = eval/paired_compare_joint.py / eval/pressure_eval.py with the holdout target dirs, split `holdout`, causal v2 driver populations, device-cluster bootstrap n_boot 1000, seeds 0/1/2; " | |
| "verdict = kill_checks.beats (>= 2/3 seeds CI excluding 0, same sign, |mean Δ| > seed sd). Every number below comes from `results/paired_compare.csv` (split `holdout` for the holdout, split `test` for the benchmark) or the listed artefact files.", "", | |
| "**No selection was made on the holdout**: both pre-specified candidates are reported regardless of direction; no model, threshold or population rule was changed after the holdout numbers were seen; each frozen run was predicted once and each cell paired once (the benchmark evaluators' code, unchanged). The owner's adoption decision between (i) and (ii) is taken on the benchmark Table 3 ECAN; the holdout is reported for whichever is adopted PLUS the other as a disclosed second candidate (Supp E).", "", | |
| "## Holdout population", ""] + population_lines() + [""] | |
| if a.section_note: | |
| L += [a.section_note, ""] | |
| js = {"stamp": stamp, "seeds": seeds, "candidates": {}, "rendered": cands} | |
| all_hold = [] | |
| for cid, C in CANDS.items(): | |
| if cid not in cands: | |
| continue | |
| dh = load_pair(*C["hold"], seeds); dhP = load_pair(*C["holdP"], seeds) | |
| dt = load_pair(*C["test"], seeds); dtP = load_pair(*C["testP"], seeds) if C["testP"] else pd.DataFrame() | |
| dh_all = pd.concat([dh, dhP], ignore_index=True) if len(dhP) else dh | |
| dt_all = pd.concat([dt, dtP], ignore_index=True) if len(dtP) else dt | |
| L += [f"## {C['title']}", ""] | |
| if not len(dh_all): | |
| L += [f"**NOT RUN on the holdout** — no `split = holdout` rows for tags `{C['hold'][0]}_s<k>_{C['hold'][1]}_vs_{C['hold'][2]}` (see the status notes at the end).", ""] | |
| js["candidates"][cid] = {"status": "NOT RUN"} | |
| continue | |
| all_hold.append(dh_all) | |
| runs = sorted(set(dh_all.a.astype(str)) | set(dh_all.b.astype(str))) | |
| L += ["### Provenance of the holdout layouts", ""] + provenance_holdout(runs, dh_all) + [""] | |
| L += [f"### Confirmation table — holdout Δ = A − B per seed [device-cluster 95 % CI], mean ± sd over seeds, holdout verdict, side by side with the benchmark TEST verdict ({C['bench']}); replicates = same sign of mean Δ and same significance class (BEATS/WORSE vs NOT_WORSE/INCONCLUSIVE)", "", | |
| "| group | regime | cell | metric | " + " | ".join(f"holdout Δ s{s} [CI]" for s in seeds) + " | holdout mean Δ ± sd (n, pos, dev) | holdout verdict | TEST mean Δ ± sd | TEST verdict | replicates |", | |
| "|---|---|---|---|" + "---|" * len(seeds) + "---|---|---|---|---|"] | |
| cells_js = {} | |
| for gname, cells, metric in GROUPS: | |
| for cell in cells: | |
| for r in REGS: | |
| h = cell_stats(dh_all, r, cell, metric) | |
| if h is None: | |
| continue | |
| t = cell_stats(dt_all, r, cell, metric) | |
| rep = replicates(h, t) | |
| tag = " *(not reported)*" if (r, cell) in DRIVER_NOT_REPORTED else (" **(reported)**" if (r, cell) in DRIVER_REPORTED else "") | |
| road = " [ROAD]" if (cell.startswith("d_onset|") and cell.split("|")[1] in ROAD_D) or cell.startswith("p_onset|") else "" | |
| D = [f"{h['by'][s].delta:+.4f} [{h['by'][s].delta_ci_lo:+.4f},{h['by'][s].delta_ci_hi:+.4f}]" if s in h["by"] else "—" for s in seeds] | |
| tcol = f"{t['mean']:+.4f} ± {t['sd']:.4f}" if t else "— (not in benchmark table)" | |
| L.append(f"| {gname.split(' (')[0]} | {r} | {cell.replace('|', '/')}{tag}{road} | {metric} | " + " | ".join(D) + | |
| f" | {h['mean']:+.4f} ± {h['sd']:.4f} (n={h['n']:,}, pos={h['n_pos'] if h['n_pos'] is not None else '—'}, dev={h['n_dev']}) | {h['verdict']} | {tcol} | {t['verdict'] if t else '—'} | {rep} |") | |
| cells_js[f"{r}|{cell}|{metric}"] = {"holdout": {k: v for k, v in h.items() if k != "by"}, "test": ({k: v for k, v in t.items() if k != "by"} if t else None), "replicates": rep, | |
| "reported": (r, cell) in DRIVER_REPORTED or not cell.startswith("drv_onset|")} | |
| # ---- key reading | |
| key_rows = [] | |
| for (r, cell, metric) in C["key"]: | |
| k = f"{r}|{cell}|{metric}" | |
| if k in cells_js: | |
| key_rows.append((k, cells_js[k]["replicates"], cells_js[k]["holdout"]["verdict"], cells_js[k]["test"]["verdict"] if cells_js[k]["test"] else "—")) | |
| n_yes = sum(1 for _, rep, _, _ in key_rows if rep == "yes"); n_tot = sum(1 for _, rep, _, _ in key_rows if rep in ("yes", "no")) | |
| reading = "CONFIRMED" if n_tot and n_yes == n_tot else ("NOT CONFIRMED" if n_tot and n_yes == 0 else (f"PARTIAL ({n_yes}/{n_tot})" if n_tot else "NO BENCHMARK ROWS")) | |
| L += ["", f"### Pre-registered reading for this block — {C['key_label']}: **{reading}** ({n_yes}/{n_tot} key cells replicate sign + significance class)", ""] | |
| L += [f"- {k}: holdout {hv} vs TEST {tv} -> {rep}" for k, rep, hv, tv in key_rows] + [""] | |
| # ---- B2-style status on the holdout (own endpoints WORSE / cross-leg BEATS) for the interaction blocks | |
| body, verd = RJ.render(dh, sorted(dh.seed.unique()), C["hold"][1], C["hold"][2], "Full holdout cell table (every paired cell, as in the benchmark stage tables)") | |
| st, worse, bc, bo = RJ.claim(verd, "B2") | |
| L += [f"- B2-style status on the holdout (no own endpoint WORSE and >= 1 cross-leg BEATS; descriptive here): **{st}** — WORSE own {[f'{r}|{c}' for r, c, _ in worse] or 'none'}; BEATS cross {[f'{r}|{c}|{m}' for r, c, m in bc] or 'none'}; BEATS own {[f'{r}|{c}' for r, c, _ in bo] or 'none'}", ""] | |
| L += body + [""] | |
| if len(dhP): | |
| bodyP, _ = RJ.render(dhP, sorted(dhP.seed.unique()), C["holdP"][1], C["holdP"][2], "Holdout P cells (road-pressure onset PR-AUC@3s)") | |
| L += bodyP + [""] | |
| rule_js = None | |
| if C.get("rule") == "kpf_iii": # 2026-09-18 candidate (iii): the pre-registered reading rule, applied mechanically | |
| rl, rule_js = rule_block_iii(dh, verd, dt, seeds) | |
| L += rl | |
| js["candidates"][cid] = {"status": "RUN", "reading": reading, "key_cells": key_rows, "B2_style": st, "cells": cells_js, "tags": [f"{C['hold'][0]}_s<k>_{C['hold'][1]}_vs_{C['hold'][2]}", f"{C['holdP'][0]}_s<k>_{C['holdP'][1]}_vs_{C['holdP'][2]}"], | |
| **({"rule": rule_js, "prereg": PREREG_III} if rule_js is not None else {})} | |
| # ---- read-out confirmation (automatic-event timing families) + notes: rendered with candidate (i) (first section); a (ii)-only section carries its own status note | |
| if "i" not in cands and cands == ["iii"]: | |
| L += ["## Status", ""] + status_iii() + [""] | |
| out = Path(a.out); out.parent.mkdir(parents=True, exist_ok=True) | |
| text = "\n".join(L) + "\n" | |
| if out.exists() and out.stat().st_size > 0: | |
| with open(out, "a") as f: | |
| f.write("\n\n---\n\n" + text) | |
| print(f"appended to {out}") | |
| else: | |
| out.write_text(text) | |
| print(f"written {out}") | |
| jp = Path(a.json_out) if a.json_out else out.with_name(out.stem + f"_{time.strftime('%Y%m%dT%H%M%SZ', time.gmtime())}.json") | |
| jp.write_text(json.dumps(js, indent=1, default=str)) | |
| print(f"written {jp}") | |
| for cid, c in js["candidates"].items(): | |
| print(f"{cid}: {c.get('status')} {c.get('reading', '')} rule={c.get('rule', {}).get('status') if c.get('rule') else '—'}") | |
| return | |
| if "i" not in cands: | |
| L += ["## Status", "", | |
| "- Candidate (ii) inputs: the F5b side streams `results/holdout/targets_road_dyn/vfcan256_{manual,engaged}_holdout.npz` were built by `data/build_road_dynamics_targets.py --holdout` (additive path override; coordinator decision docs/decisions.md 2026-09-17T15:51:24Z) = holdout pack_v2 vf256 PCA components 0..215 + the 40 CAN anchor features standardised with the FROZEN TRAIN `results/targets_road_dyn/ecan_stats.npz` (sha256 5aa42063…ffdd53, loaded and asserted, never recomputed); deterministic input construction, no fitting, no selection; the benchmark `results/targets_road_dyn/` is untouched.", | |
| "- Evaluated ONCE: each ECAN frozen run predicted once on the holdout, each cell paired once; no model, threshold or population change after seeing the numbers; no selection between (i) and (ii) on the holdout.", ""] | |
| out = Path(a.out); out.parent.mkdir(parents=True, exist_ok=True) | |
| text = "\n".join(L) + "\n" | |
| if out.exists() and out.stat().st_size > 0: | |
| with open(out, "a") as f: | |
| f.write("\n\n---\n\n" + text) | |
| print(f"appended to {out}") | |
| else: | |
| out.write_text(text) | |
| print(f"written {out}") | |
| jp = out.with_name(out.stem + f"_{time.strftime('%Y%m%dT%H%M%SZ', time.gmtime())}.json") | |
| jp.write_text(json.dumps(js, indent=1, default=str)) | |
| print(f"written {jp}") | |
| for cid, c in js["candidates"].items(): | |
| print(f"{cid}: {c.get('status')} {c.get('reading', '')}") | |
| return | |
| gaze = H / "targets_gaze_causal_routes_v2" | |
| w_dirs = [f"W_I1_all_minus_road2drv_s{s}_final_{sfx}" for s in seeds for sfx in ("dwfc", "dwfcboth", "merged")] | |
| w_have = [w for w in w_dirs if (RUNS / w / "pred_layout_manual_holdout.npz").exists()] | |
| L += ["## Warning read-out confirmation (automatic-event timing families A/B/C1/C2/D, K in {10, 20, 40}/h) — status", "", | |
| f"- NOT RUN on the holdout in this batch: `eval/alert_timing_eval.py` needs, per regime, the gaze state chain `gaze_{{r}}_holdout.npz` ({'present' if gaze.exists() else 'ABSENT: `results/holdout/targets_gaze_causal_routes_v2` does not exist'}), " | |
| f"the warning-head score layouts `W_<run>_dwfc / _dwfcboth / _merged/pred_layout_{{r}}_holdout.npz` ({len(w_have)}/{len(w_dirs)} present) and the openpilot-derived surrogate marginals for the holdout (absent), and it fits thresholds on `val` and evaluates on a hard-coded `test` split; " | |
| "none of these holdout inputs exists and nothing was substituted or re-tuned. The read-out therefore remains confirmed only on the benchmark (results/tables/alert_timing.md); the holdout confirms the model cells above.", ""] | |
| L += ["## Notes", "", | |
| "- `eval/predict_holdout.py` received two additive fixes before the first run (2026-09-17): the `holdout` split is added to `cwm3_dataset.SPLITS` for the process (load_split asserted split in train/val/test) and the F5b side-stream dir `ROAD_DYN` is redirected / checked (`vfcan256_{r}_holdout.npz`).", | |
| "- `--check-only` for the ECAN runs lists `results/holdout/targets_road_dyn/vfcan256_{manual,engaged}_holdout.npz` as missing (the holdout pack was built without the F5b side stream; `data/build_road_dynamics_targets.py` was never run for the holdout split) — candidate (ii) blocks are NOT RUN until that side stream exists; nothing was rebuilt in this batch.", | |
| "- Engaged hand cells are computed and shown but marked *(not reported)* per the pre-registration (v2 label-check story, Supp E).", ""] | |
| out = Path(a.out); out.parent.mkdir(parents=True, exist_ok=True) | |
| text = "\n".join(L) + "\n" | |
| if out.exists() and out.stat().st_size > 0: | |
| with open(out, "a") as f: | |
| f.write("\n\n---\n\n" + text) | |
| print(f"appended to {out}") | |
| else: | |
| out.write_text(text) | |
| print(f"written {out}") | |
| jp = out.with_suffix(".json") | |
| if jp.exists(): | |
| jp = out.with_name(out.stem + f"_{time.strftime('%Y%m%dT%H%M%SZ', time.gmtime())}.json") | |
| jp.write_text(json.dumps(js, indent=1, default=str)) | |
| print(f"written {jp}") | |
| for cid, c in js["candidates"].items(): | |
| print(f"{cid}: {c.get('status')} {c.get('reading', '')}") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 35 kB
- Xet hash:
- 3a62db0ad5a2fd088351a52b90fde6a68e9feced0c79d057ae2d62436eef644e
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.