Buckets:
| """cwm3/eval/report_final_ecan_table3.py — results/tables/stage_final_ecan_table3.md: the ECAN-version of the paper's Table 3 (owner decision docs/decisions.md | |
| 2026-09-17T06:47:06Z item 1: "ECAN-version Table 3 with all paired columns (vs indep_ecan, none_ecan, naive_ecan, k0)"), from results/paired_compare.csv only. | |
| Composes eval/report_joint.py blocks (load / verdict / claim / provenance); nothing is recomputed here; every number comes from the CSV rows written by | |
| eval/run_joint_evals_final_ecan_table3.sh (joint cells: eval/pc_joint_locked.py = paired_compare_joint; P cells: eval/pressure_eval.py). | |
| python eval/report_final_ecan_table3.py [--seeds 0,1,2] [--out results/tables/stage_final_ecan_table3.md] | |
| Model = the FINAL joint recipe on the ECAN E base (E0_pooled_P_can_s<k>_v2: vf256 PCA 0..215 + 40 CAN road anchor features in the 256-d side vector): | |
| main_ecan = I1_all_minus_road2drv_s<k>_final_ecan indep_ecan = I1_indep_s<k>_final_ecan (assembled bases, no training) | |
| none_ecan = I1_none_s<k>_final_ecan (fine-tuning, no interaction) naive_ecan = I1_naive_s<k>_final_ecan (concatenated trunks -> shared MLP, default init) | |
| k0 = the current-state heuristic from the causal K0 (paired_compare_joint --b-k0; driver cells only; persistence = base-rate rows) | |
| Comparisons (tags): finalc_s<k>_allminusecan_vs_{indepecan,noneecan,naiveecan,k0}, finalc_s<k>_{noneecan,naiveecan}_vs_indepecan, | |
| finalcv_s<k>_allminusecan_vs_indepecan (masked protocol, human-VERIFIED 3-route driver-box mask), finalP_s<k>_<a>_vs_<b> (P cells). | |
| Cells: vehicle onset PR-AUC@3 s (brake_on / gas_on / steer_phase_on / veh_any_on), driver onset PR-AUC@3 s (hand_leave / hand_return / head_turn_onset; the ENGAGED | |
| hand cells are marked "(not reported in the paper)" — owner decision 2026-09-17T06:47:06Z item 3), D road cells (six demand families), P road cells (six pressure | |
| chains), hazard-NLL cost cells (lower better; acceptable-degradation margin +0.01 nats on the mean Δ and on every seed), cross-leg C1 / C1_given_k / C2 pooled. | |
| Verdict = kill_checks.beats (>= 2/3 seeds CI excluding 0, same sign, |mean Δ| > seed sd); the count of seeds whose CI excludes 0 on the side of the mean is shown as k/3. | |
| Also: the parameter count of the ECAN model per module from the run's params.json (models/cwm3.py param_report) and how the E input change alters the 3.50 M total. | |
| Writes results/tables/stage_final_ecan_table3.json alongside. Nothing here edits the manuscript. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import sys | |
| from pathlib import Path | |
| import numpy as np | |
| import pandas as pd | |
| CWM3 = Path(__file__).resolve().parents[1] | |
| sys.path.insert(0, str(CWM3 / "eval")) | |
| import report_joint as RJ # noqa: E402 | |
| RUNS = CWM3 / "results" / "runs" | |
| MARGIN = 0.01 | |
| VEH = ["veh_onset|brake_on|3s", "veh_onset|gas_on|3s", "veh_onset|steer_phase_on|3s", "veh_onset|veh_any_on|3s"] | |
| DRV = ["drv_onset|hand_leave|3s", "drv_onset|hand_return|3s", "drv_onset|head_turn_onset|3s"] | |
| NOT_IN_PAPER = {("engaged", "drv_onset|hand_leave|3s"), ("engaged", "drv_onset|hand_return|3s")} | |
| PAPER_DRV = [("manual", c) for c in DRV] + [("engaged", "drv_onset|head_turn_onset|3s")] | |
| DCELLS = [f"d_onset|{f}|3s" for f in ("steer_rate", "decel", "ttc", "ttc_drop", "lane", "badas")] | |
| PCELLS = [f"p_onset|{f}|3s" for f in ("ttc", "ttc_drop", "cut_in", "lead_braking", "lane_excursion", "badas")] | |
| COST = [f"hazard_nll|{c}" for c in ("brake_on", "gas_on", "steer_phase_on", "hand_leave", "hand_return", "head_turn_onset", | |
| "D_steer_rate", "D_decel", "D_ttc", "D_ttc_drop", "D_lane", "D_badas")] | |
| CROSS = [("C1_given_k", "auroc"), ("C1_given_k", "pr_auc"), ("C1", "auroc"), ("C1", "pr_auc"), ("C2|pooled_warnable", "pr_auc")] | |
| # (key, stage, a, b, title, claim) | |
| COMPS = [("main_vs_indep", "finalc", "allminusecan", "indepecan", "main_ecan − indep_ecan (interaction effect on the ECAN bases; the paper's Table 3 main column)", "B2"), | |
| ("main_vs_none", "finalc", "allminusecan", "noneecan", "main_ecan − none_ecan (fine-tuning control: same recipe, no interaction modules)", "B2"), | |
| ("main_vs_naive", "finalc", "allminusecan", "naiveecan", "main_ecan − naive_ecan (naive fusion control: pretrained trunks + default-init fusion MLP)", "B2"), | |
| ("none_vs_indep", "finalc", "noneecan", "indepecan", "none_ecan − indep_ecan (what fine-tuning alone buys)", "B2"), | |
| ("naive_vs_indep", "finalc", "naiveecan", "indepecan", "naive_ecan − indep_ecan (what naive fusion buys)", "B2"), | |
| ("main_vs_k0", "finalc", "allminusecan", "k0", "main_ecan − k0 heuristic (driver cells only; claim B1; persistence = base rate)", "B1"), | |
| ("masked_main_vs_indep", "finalcv", "allminusecan", "indepecan", "masked protocol (human-verified 3-route driver-box mask): main_ecan − indep_ecan", "B2")] | |
| PCOMPS = [("main_vs_indep", "allminusecan", "indepecan"), ("main_vs_none", "allminusecan", "noneecan"), ("main_vs_naive", "allminusecan", "naiveecan"), | |
| ("none_vs_indep", "noneecan", "indepecan"), ("naive_vs_indep", "naiveecan", "indepecan")] | |
| SUMMARY_COLS = [("main_vs_indep", "Δ main − indep"), ("main_vs_none", "Δ main − none"), ("main_vs_naive", "Δ main − naive"), ("main_vs_k0", "Δ main − k0"), | |
| ("none_vs_indep", "Δ none − indep"), ("naive_vs_indep", "Δ naive − indep"), ("masked_main_vs_indep", "Δ main − indep (masked)")] | |
| def cellname(c): | |
| return c.replace("|", "/") | |
| def excl_count(g, lower_better): | |
| d = g.delta.astype(float).to_numpy(); lo = g.delta_ci_lo.astype(float).to_numpy(); hi = g.delta_ci_hi.astype(float).to_numpy() | |
| if lower_better: | |
| d, lo, hi = -d, -hi, -lo | |
| m = float(np.nanmean(d)) | |
| return int(((lo > 0) if m > 0 else (hi < 0)).sum()) if m != 0 else 0 | |
| def stat(df, regime, cell, metric, seeds): | |
| """one cell of one comparison -> dict or None (no rows).""" | |
| g = df[(df.regime == regime) & (df.cell == cell) & (df.metric == metric)].sort_values("seed") | |
| if not len(g): | |
| return None | |
| lb = cell.startswith(RJ.LOWER_BETTER) | |
| res = RJ.verdict(g, lb) | |
| d = g.delta.astype(float).to_numpy() | |
| by = {int(s): r for s, r in zip(g.seed, g.itertuples())} | |
| return {"A": [f"{by[s].value_a:.4f}" if s in by else "—" for s in seeds], | |
| "B": [f"{by[s].value_b:.4f}" if s in by else "—" for s in seeds], | |
| "D": [f"{by[s].delta:+.4f} [{by[s].delta_ci_lo:+.4f},{by[s].delta_ci_hi:+.4f}]" if s in by else "—" for s in seeds], | |
| "mean": float(d.mean()), "sd": float(d.std(ddof=1)) if len(d) > 1 else 0.0, "n": int(len(d)), "excl": excl_count(g, lb), | |
| "verdict": res["verdict"], "lower_better": lb, "max_delta": float(np.nanmax(d)), "b_name": str(g.b.iloc[0]), | |
| "n_pos": int(g.n_pos.iloc[0]) if str(g.n_pos.iloc[0]).strip() not in ("", "nan") else None, "deltas": d.tolist(), "a_vals": g.value_a.astype(float).tolist()} | |
| def note_for(regime, cell, s): | |
| notes = [] | |
| if (regime, cell) in NOT_IN_PAPER: | |
| notes.append("not reported in the paper") | |
| if cell.startswith("hazard_nll") and s is not None: | |
| notes.append(f"within +{MARGIN:.2f} margin" if (s["mean"] <= MARGIN and s["max_delta"] <= MARGIN) else f"EXCEEDS +{MARGIN:.2f} margin (mean {s['mean']:+.4f}, max seed {s['max_delta']:+.4f})") | |
| return "; ".join(notes) | |
| def compact(s): | |
| if s is None: | |
| return "—" | |
| return f"{s['mean']:+.4f} ± {s['sd']:.4f} {s['verdict']} ({s['excl']}/{s['n']})" | |
| def detail_table(df, dfP, seeds, title, a_name, b_name): | |
| """full detail of one comparison over the Table-3 cells (+ D / P road cells, cost cells, cross-leg).""" | |
| L = [f"### {title}: A = {a_name}, B = {b_name}", "", | |
| "| regime | cell | metric | " + " | ".join(f"A s{s}" for s in seeds) + " | " + " | ".join(f"B s{s}" for s in seeds) + " | " + " | ".join(f"Δ s{s} [CI]" for s in seeds) + | |
| " | mean Δ ± sd | CI≠0 | verdict | note |", "|---|---|---|" + "---|" * (3 * len(seeds)) + "---|---|---|---|"] | |
| verd, stats = {}, {} | |
| for regime in ("manual", "engaged"): | |
| groups = [("vehicle", VEH, "pr_auc", df), ("driver", DRV, "pr_auc", df), ("D road", DCELLS, "pr_auc", df), ("P road", PCELLS, "pr_auc", dfP), ("cost", COST, "nll", df)] | |
| for gname, cells, metric, src in groups: | |
| for cell in cells: | |
| s = stat(src, regime, cell, metric, seeds) if src is not None and len(src) else None | |
| if s is None: | |
| continue | |
| key = (regime, cell, metric) | |
| stats[key] = s | |
| verd[key] = {"verdict": s["verdict"], "mean_delta": s["mean"]} | |
| L.append(f"| {regime} | {cellname(cell)} | {metric}{' (lower better)' if s['lower_better'] else ''} | " + " | ".join(s["A"]) + " | " + " | ".join(s["B"]) + " | " + " | ".join(s["D"]) + | |
| f" | {s['mean']:+.4f} ± {s['sd']:.4f} (n={s['n']}, pos={s['n_pos'] if s['n_pos'] is not None else '—'}) | {s['excl']}/{s['n']} | {s['verdict']} | {note_for(regime, cell, s)} |") | |
| for cell, metric in CROSS: | |
| s = stat(df, regime, cell, metric, seeds) if len(df) else None | |
| if s is None: | |
| continue | |
| key = (regime, cell, metric) | |
| stats[key] = s | |
| verd[key] = {"verdict": s["verdict"], "mean_delta": s["mean"]} | |
| L.append(f"| {regime} | {cellname(cell)} (cross-leg) | {metric} | " + " | ".join(s["A"]) + " | " + " | ".join(s["B"]) + " | " + " | ".join(s["D"]) + | |
| f" | {s['mean']:+.4f} ± {s['sd']:.4f} (n={s['n']}, pos={s['n_pos'] if s['n_pos'] is not None else '—'}) | {s['excl']}/{s['n']} | {s['verdict']} | |") | |
| return L, verd, stats | |
| def fmt(ks): | |
| return [f"{r}|{c}" + (f"|{m}" if m not in ("pr_auc", "nll") else "") for (r, c, m) in ks] or "none" | |
| def claim_lines(verd, kind, a_name, b_name, seeds): | |
| """B2 (project rule, RJ.OWN incl. the engaged hand cells) + the same rule restricted to the paper's cells; B1 for k0.""" | |
| if kind == "B1": | |
| st, worse, beats, _ = RJ.claim(verd, "B1") | |
| drv = {k: v for k, v in verd.items() if k[1] in RJ.DRV and k[2] == "pr_auc"} | |
| n_beats = sum(v["verdict"] == "BEATS" for v in drv.values()) | |
| n_paper = sum(v["verdict"] == "BEATS" for k, v in drv.items() if (k[0], k[1]) in PAPER_DRV) | |
| n_worse = sum(v["verdict"] == "WORSE" for v in drv.values()) | |
| return [f"**B1 verdict ({a_name} vs k0, report_joint rule: hand_leave AND head_turn_onset BEATS k0 in both regimes): {st}** — {n_beats}/6 driver cells BEATS k0 " | |
| f"({n_paper}/4 among the cells reported in the paper: manual hand_leave / hand_return / head_turn, engaged head_turn), {n_worse}/6 WORSE; " | |
| f"INCONCLUSIVE: {fmt([k for k, v in drv.items() if v['verdict'] == 'INCONCLUSIVE'])}."], {"status": st, "beats_6": n_beats, "beats_paper_4": n_paper, "worse": n_worse} | |
| st, worse_own, beats_cross, beats_own = RJ.claim(verd, "B2") | |
| # the rule on the paper's cells only (engaged hand cells excluded) | |
| own_p = {k: v for k, v in verd.items() if k[1] in RJ.OWN and k[2] == "pr_auc" and (k[0], k[1]) not in NOT_IN_PAPER} | |
| worse_p = [k for k, v in own_p.items() if v["verdict"] == "WORSE"] | |
| beats_p = [k for k, v in own_p.items() if v["verdict"] == "BEATS"] | |
| insuff = any(v["verdict"] == "INSUFFICIENT" for v in own_p.values()) | |
| st_p = "INSUFFICIENT (< 3 seeds)" if insuff else ("PASS" if (not worse_p and beats_cross) else "FAIL") | |
| cost = {k: v for k, v in verd.items() if k[1].startswith("hazard_nll")} | |
| cost_worse = [k for k, v in cost.items() if v["verdict"] == "WORSE"] | |
| cost_over = [k for k, v in cost.items() if v["mean_delta"] > MARGIN] | |
| n_own = len([k for k in verd if k[1] in RJ.OWN and k[2] == "pr_auc"]) | |
| return [f"**B2 verdict ({a_name} vs {b_name}; project rule over all {n_own} own onset cells incl. the engaged hand cells): {st}** — own WORSE {fmt(worse_own)}; own BEATS {fmt(beats_own)}; cross-leg BEATS {fmt(beats_cross)}.", | |
| f"- the same rule on the paper's cells only (engaged hand_leave / hand_return excluded): **{st_p}** — own WORSE {fmt(worse_p)}; own BEATS {fmt(beats_p)}.", | |
| f"- hazard-NLL cost cells (lower better): WORSE {fmt(cost_worse)}; mean Δ above the +{MARGIN:.2f} acceptable-degradation margin: {fmt(cost_over)}."], \ | |
| {"status": st, "status_paper_cells": st_p, "own_worse": fmt(worse_own), "own_beats": fmt(beats_own), "cross_beats": fmt(beats_cross), | |
| "own_worse_paper_cells": fmt(worse_p), "cost_worse": fmt(cost_worse), "cost_over_margin": fmt(cost_over)} | |
| def summary_table(stats, seeds): | |
| """Table-3 shape: rows = cells per regime, columns = main_ecan level per seed + one compact Δ column per comparison.""" | |
| L = ["| regime | endpoint | main_ecan s0 / s1 / s2 | " + " | ".join(c for _, c in SUMMARY_COLS) + " | note |", "|---|---|---|" + "---|" * len(SUMMARY_COLS) + "---|"] | |
| for regime in ("manual", "engaged"): | |
| for gname, cells, metric in (("vehicle", VEH, "pr_auc"), ("driver", DRV, "pr_auc"), ("D road", DCELLS, "pr_auc"), ("P road", PCELLS, "pr_auc"), ("cost (nats, lower better)", COST, "nll")): | |
| L.append(f"| **{regime}** | *{gname}* | | " + " | ".join("" for _ in SUMMARY_COLS) + " | |") | |
| for cell in cells: | |
| key = (regime, cell, metric) | |
| lead = stats.get("main_vs_indep", {}).get(key) or next((stats[k][key] for k, _ in SUMMARY_COLS if k in stats and key in stats[k] and k.startswith("main")), None) | |
| level = " / ".join(lead["A"]) if lead else "—" | |
| s_main = stats.get("main_vs_indep", {}).get(key) | |
| L.append(f"| {regime} | {cellname(cell)} | {level} | " + " | ".join(compact(stats.get(k, {}).get(key)) for k, _ in SUMMARY_COLS) + f" | {note_for(regime, cell, s_main)} |") | |
| for cell, metric in CROSS: | |
| key = (regime, cell, metric) | |
| lead = stats.get("main_vs_indep", {}).get(key) | |
| L.append(f"| {regime} | {cellname(cell)} ({metric}, cross-leg) | {' / '.join(lead['A']) if lead else '—'} | " + " | ".join(compact(stats.get(k, {}).get(key)) for k, _ in SUMMARY_COLS) + " | |") | |
| return L | |
| def params_section(seeds): | |
| """per-module parameter count of the ECAN model from the run's params.json (models/cwm3.py param_report); identity across seeds and vs the frozen main asserted.""" | |
| def load(run): | |
| p = RUNS / run / "params.json" | |
| return json.loads(p.read_text()) if p.exists() else None | |
| mains = {s: load(f"I1_all_minus_road2drv_s{s}_final_ecan") for s in seeds} | |
| ref = mains[seeds[0]] | |
| if ref is None: | |
| return ["(params.json of I1_all_minus_road2drv_s0_final_ecan missing)"], {} | |
| same_seeds = all(m == ref for m in mains.values() if m is not None) | |
| frozen = load("I1_all_minus_road2drv_s0_final") | |
| same_frozen = frozen == ref | |
| others = {n: load(f"I1_{v}_s{seeds[0]}_final_ecan") for n, v in (("indep_ecan", "indep"), ("none_ecan", "none"), ("naive_ecan", "naive"))} | |
| V, U, E = ref["V"], ref["U"], ref["E"] | |
| v_trunk = V["total"] - V["head"] | |
| u_trunk = U["total"] - U["head"] | |
| e_trunk = E["total"] - E["head"] - E["f_head"] - E.get("p_head", 0) | |
| rows = [("V trunk (TCN + mode embedding)", v_trunk, "CAN history [B,40,6] → z_v 256"), | |
| ("V head (direct 25-step decoder: 3 MDNs + gas / brake / steer_phase / resp chains)", V["head"], "on [z_v ; mode]"), | |
| ("U trunk (input LN + GRU + mode embedding)", u_trunk, "pose kinematics + CAN [B,40,18] → h_u 128"), | |
| ("U head (direct decoder: hands_on / head_turned chains; dims 0:22 replaced by the K head)", U["head"], "on [h_u ; mode]"), | |
| ("E trunk (token in-proj, slot / region / Δt embeddings, 2 causal encoder layers, norm, side in-proj `vf`, mode, missing, readout)", e_trunk, | |
| "16 pooled V-JEPA2 clip tokens + the 256-d side vector (ECAN: vf256 PCA 0..215 + 40 CAN anchor features) → [z_e ; z_vf] 256"), | |
| ("E head (direct decoder: six demand chains D)", E["head"], "on [z_e ; z_vf ; mode]"), | |
| ("E f_head (future latent F, 5 × 256)", E["f_head"], "on [z_e ; z_vf ; mode]"), | |
| ("E base's own trunk-supervised P head (loaded with the base; OFF the joint forward path — the joint's detached P head below supplies out[P])", E.get("p_head", 0), "not on the forward path; receives no gradient"), | |
| ("connection env→vehicle (Env2Vehicle, zero-init)", ref["env2veh"], "[z_e ; z_vf] 256 → residual on z_v"), | |
| ("connection driver→vehicle (Driver2Vehicle, zero-init, mode gate)", ref["drv2veh"], "h_u 128 → residual on z_v"), | |
| ("connection road→driver (Road2Driver + RegionTokenEmbed; NOT in all_minus_road2drv)", ref["road2drv"] + ref["tok_embed"], "0 in this model"), | |
| ("naive fusion MLP (control only)", ref["fusion"], "0 in this model"), | |
| ("K head (detached, Gaussian ΔK̂ on zU)", ref["k_head"], "on zU.detach()"), | |
| ("P head (detached, six road-pressure chains on zE)", ref["p_head"], "on zE.detach()"), | |
| ("R head (detached, hard-response chain on zV)", ref["r_head"], "on zV.detach()")] | |
| total = ref["total"] | |
| onpath = total - E.get("p_head", 0) | |
| L = ["| module | params | input → output |", "|---|---|---|"] + [f"| {n} | {p:,} | {d} |" for n, p, d in rows] + [ | |
| f"| **total, params.json (`param_report`, every registered parameter; all trainable in the joint stage at lr × 0.1 for the bases, lr for the new modules)** | **{total:,}** | |", | |
| f"| **on the forward path (total − the E base's off-path P head {E.get('p_head', 0):,})** | **{onpath:,}** | = {onpath / 1e6:.2f} M |", | |
| f"| new modules in the joint stage (`new_total`: env2veh + drv2veh + K + P + R heads) | {ref['new_total']:,} | interaction_total {ref['interaction_total']:,} |"] | |
| L += ["", f"- params.json identical across the three ECAN seeds: **{same_seeds}**; identical to the frozen main `I1_all_minus_road2drv_s0_final`: **{same_frozen}** " | |
| f"(frozen total {frozen['total']:,})." if frozen else f"- params.json identical across the three ECAN seeds: **{same_seeds}** (frozen main params.json missing).", | |
| f"- **How the E input change alters the 3.50 M total: it does not.** The 40 CAN anchor features enter through the E branch's existing 256-d side vector " | |
| f"(`side: vfcan256` = vf256 PCA components 0..215 + 40 standardised CAN features replacing PCA components 216..255; train/stages.yaml '# --- R2 B2 road variants ---'), so the " | |
| f"side in-proj `vf` (LN + Linear 256→128 = {E['vf']:,}) and every other module keep their shapes; no model or joint-trainer code changed. " | |
| f"The paper's 3.50 M is the derived figure 2,926,133 + 329,088 + 247,680 = 3,502,901 (paper/figs/make_figs.py:663, from docs/model_spec.md §6 counts of the E `pooled` base without P head and the U base without input LN); " | |
| f"the FINAL bases' params.json gives {onpath:,} on the forward path (+24 = the U input LayerNorm of `kin_can_noK_clip8_ln`; the E base's own P head {E.get('p_head', 0):,} is loaded but not on the joint forward path), i.e. {onpath / 1e6:.2f} M for the frozen main AND for the ECAN model alike; " | |
| f"params.json total {total:,} counts that off-path head too."] | |
| comp = {n: (o["total"], o["new_total"], o["interaction_total"]) for n, o in others.items() if o} | |
| if comp: | |
| L += ["", "| comparator (same E base) | total (params.json) | new_total | interaction / fusion |", "|---|---|---|---|"] + \ | |
| [f"| {n} | {t:,} | {nw:,} | {it:,} |" for n, (t, nw, it) in comp.items()] + \ | |
| ["", "(indep_ecan / none_ecan: no interaction modules — total = main − 576,768; naive_ecan: the 821,760-parameter fusion MLP instead of the two 576,768-parameter connections.)"] | |
| js = {"total": total, "on_path": onpath, "new_total": ref["new_total"], "interaction_total": ref["interaction_total"], "same_across_seeds": same_seeds, "same_as_frozen_main": same_frozen, | |
| "modules": {n: p for n, p, _ in rows}, "comparators": comp, "paper_derived_3_50M": 3502901} | |
| return L, js | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--seeds", default="0,1,2") | |
| ap.add_argument("--out", default=str(CWM3 / "results" / "tables" / "stage_final_ecan_table3.md")) | |
| a = ap.parse_args() | |
| seeds = [int(s) for s in a.seeds.split(",")] | |
| dfs, blocks, allstats, claims, missing = [], [], {}, {}, [] | |
| pdf = {k: RJ.load("finalP", an, bn, seeds) for k, an, bn in PCOMPS} | |
| for key, stage, an, bn, title, kind in COMPS: | |
| df = RJ.load(stage, an, bn, seeds) | |
| dfP = pdf.get(key) | |
| if not len(df): | |
| missing.append(f"{stage}_s<k>_{an}_vs_{bn}") | |
| blocks.append([f"### {title}: A = {an}, B = {bn} — no rows yet (tags `{stage}_s<k>_{an}_vs_{bn}`)", ""]) | |
| continue | |
| dfs.append(df) | |
| if dfP is not None and len(dfP): | |
| dfs.append(dfP) | |
| body, verd, stats = detail_table(df, dfP, seeds, title, an, bn) | |
| allstats[key] = stats | |
| cl, js = claim_lines(verd, kind, an, bn, seeds) | |
| js["seeds_with_rows"] = sorted(int(s) for s in df.seed.unique()) | |
| if dfP is not None: | |
| js["P_seeds_with_rows"] = sorted(int(s) for s in dfP.seed.unique()) if len(dfP) else [] | |
| claims[key] = js | |
| blocks.append(body + [""] + cl + [""]) | |
| for k, an, bn in PCOMPS: | |
| if not len(pdf[k]): | |
| missing.append(f"finalP_s<k>_{an}_vs_{bn}") | |
| allrows = pd.concat(dfs, ignore_index=True) if dfs else pd.DataFrame(columns=["a", "b", "regime", "run_stamp"]) | |
| prov, stale = RJ.provenance(allrows) if len(allrows) else ([], []) | |
| void = f" — **VOID: stale rows for {stale}**" if stale else "" | |
| plines, pjs = params_section(seeds) | |
| n_runs_done = sum((RUNS / f"I1_{v}_s{s}_final_ecan" / "PRED_DONE").exists() for v in ("all_minus_road2drv", "indep", "none", "naive") for s in seeds) | |
| head = [f"# Stage FINAL — ECAN-version Table 3: the FINAL joint recipe on the ECAN E base (`E0_pooled_P_can_s<k>_v2`: vf256 PCA 0..215 + 40 CAN road anchor features), main vs indep / none / naive / k0, 3 seeds, test split, both regimes{void}", "", | |
| "Owner decision docs/decisions.md 2026-09-17T06:47:06Z item 1 (\"do the work first, decide adoption after\"). Runs: `I1_all_minus_road2drv_s<k>_final_ecan` (main_ecan), `I1_indep_s<k>_final_ecan` (indep_ecan; assembled bases, no training), " | |
| "`I1_none_s<k>_final_ecan` (none_ecan; fine-tuning control) and `I1_naive_s<k>_final_ecan` (naive_ecan; naive fusion control) — the ECAN main command line of train/queue_r2_e_joint_ecan.txt with ONLY `--variant` swapped (train/queue_r2_ecan_controls.txt); " | |
| f"{n_runs_done}/12 run dirs complete. Paired device-cluster bootstrap on identical rows (eval/paired_compare_joint.py through eval/pc_joint_locked.py; n_boot 1000; seeds 0/1/2; driver populations from the v2 causal kin targets; " | |
| "P cells eval/pressure_eval.py; masked protocol = the human-VERIFIED 3-route driver-box mask `results/targets_kin_causal_routes_v2/driver_box_verified_mask.json`; k0 = paired_compare_joint `--b-k0`). " | |
| "Verdict = kill_checks.beats (≥ 2/3 seeds CI excluding 0, same sign, |mean Δ| > seed sd; NOT_WORSE = every CI covers 0 and every |Δ| < 0.01); the `CI≠0` column counts the seeds whose CI excludes 0 on the side of the mean (k/3). " | |
| "Hazard-NLL cost cells: Δ in nats (A − B, lower better); the acceptable-degradation margin is +0.01 nats on the mean Δ and on every seed. " | |
| "**Engaged hand_leave / hand_return cells are reported here but marked '(not reported in the paper)' (owner decision item 3: the paper's driver cells are manual hand_leave / hand_return / head_turn and engaged head_turn).** " | |
| "Every number comes from `results/paired_compare.csv`; nothing is recomputed here; the manuscript is not edited by this report.", ""] | |
| if missing: | |
| head += [f"**Missing comparisons (no rows yet): {missing}**", ""] | |
| L = head + ["## 0. Verdict summary", ""] | |
| for key, stage, an, bn, title, kind in COMPS: | |
| c = claims.get(key) | |
| if c is None: | |
| L.append(f"- {key} ({an} vs {bn}): no rows yet") | |
| elif kind == "B1": | |
| L.append(f"- {key} ({an} vs k0): **B1 {c['status']}**, {c['beats_6']}/6 driver cells BEATS k0 ({c['beats_paper_4']}/4 paper cells), {c['worse']} WORSE") | |
| else: | |
| L.append(f"- {key} ({an} vs {bn}): **B2 {c['status']}** (paper cells only: {c['status_paper_cells']}); own WORSE {c['own_worse']}; own BEATS {c['own_beats']}; cross BEATS {c['cross_beats']}; cost WORSE {c['cost_worse']}; cost over +0.01 margin {c['cost_over_margin']}") | |
| L += ["", "## 1. Table 3 (ECAN version) — compact: main_ecan level per seed and mean Δ ± seed sd, verdict, seeds with CI ≠ 0 (k/3), per comparison", ""] + summary_table(allstats, seeds) + \ | |
| ["", "## 2. Parameter count of the ECAN model (per module, from the run's params.json = models/cwm3.py `param_report`)", ""] + plines + \ | |
| ["", "## 3. Provenance of the A / B layouts (guard 2): layout mtime, source hash, model_id / code hash, completeness; rows older than the layout are STALE", "", | |
| "| run | regime | layout mtime (UTC) | source sha256 | model_id / code | status |", "|---|---|---|---|---|---|", *prov, "", | |
| *([f"**STALE ROWS DETECTED for {stale} — every verdict is VOID; recompute (eval/run_joint_evals_final_ecan_table3.sh).**", ""] if stale else []), | |
| "## 4. Per-comparison detail (A / B levels per seed, Δ with the device-cluster bootstrap CI per seed, mean ± sd, CI≠0 count, verdict)", ""] | |
| for b in blocks: | |
| L += b | |
| out = Path(a.out) | |
| out.parent.mkdir(parents=True, exist_ok=True) | |
| out.write_text("\n".join(L) + "\n") | |
| js = {"claims": claims, "missing": missing, "stale": stale, "seeds": seeds, "params": pjs, "runs_complete": n_runs_done, | |
| "cells": {k: {f"{r}|{c}|{m}": {"mean_delta": s["mean"], "sd": s["sd"], "n": s["n"], "ci_excl": s["excl"], "verdict": s["verdict"], "deltas": s["deltas"], "A": s["a_vals"]} | |
| for (r, c, m), s in st.items()} for k, st in allstats.items()}} | |
| (out.parent / "stage_final_ecan_table3.json").write_text(json.dumps(js, indent=1, default=str)) | |
| print("\n".join(L[:3 + len(COMPS) + 3])) | |
| print(f"\nmissing: {missing}\nwritten {out} and {out.parent / 'stage_final_ecan_table3.json'}") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 26.7 kB
- Xet hash:
- a5f6f5f9cff242e43628413432d525b1107d6e824681f51896f92c4e7bd13800
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.