Buckets:
| """cwm3/eval/report_final_ecan.py — results/tables/stage_final_ecan.md: the F5b E-CAN variants of the joint model (WP4 batch 1, road-side agent, 2026-09-16; rule | |
| pre-registered in docs/decisions.md 2026-09-16T14:10:00Z BEFORE training) from results/paired_compare.csv. Composes eval/report_joint.py blocks (load / render / | |
| verdict / claim / provenance); nothing is recomputed here; every number comes from the CSV rows written by eval/run_joint_evals_final_ecan.sh. | |
| python eval/report_final_ecan.py [--seeds 0,1,2] [--out results/tables/stage_final_ecan.md] | |
| Blocks (tags): finalc_s<k>_allminusecan_vs_allminus (a) sensor + vision E base (E0_pooled_P_can: vf256 PCA 0..215 + 40 CAN anchor features) vs the FROZEN main | |
| finalc_s<k>_allminusecanonly_vs_allminus (a2) the sensor-only control (E0_pooled_P_canonly: no vision tokens, CAN features only) vs the FROZEN main | |
| finalc_s<k>_allminusecan_vs_allminusecanonly (b) vision on top of the sensors | |
| finalc_s<k>_indepecan_vs_indep, finalc_s<k>_indepecanonly_vs_indep the E-base swap alone (assembled bases, no joint training) | |
| finalc_s<k>_allminusecan_vs_indepecan interaction effect on the ecan bases (B2-style, descriptive) | |
| finalcm_* masked protocol (64 v2 driver-box-suspect routes censored) — sensitivity | |
| finalP_s<k>_allminusecan{,only}_vs_allminus, finalP_s<k>_allminusecan_vs_allminusecanonly P cells (eval/pressure_eval.py) | |
| PRE-REGISTERED RULE: `pooled_P_can` becomes the CANDIDATE E base for the frozen combination iff in (a) NO own endpoint (report_joint.OWN: veh / drv / D onset PR-AUC@3s, | |
| both regimes) is WORSE (kill_checks.beats) AND >= 1 ROAD cell BEATS (D onset ttc / ttc_drop / lane / badas in (a), or any P cell of finalP (a)); otherwise NEGATIVE RESULT. | |
| Attribution is reported separately: INFORMATION = (a2) sensor-only control vs the frozen main; VISION ON TOP = (b) can vs canonly. Adoption is decided on validation- | |
| selected models and confirmed once on the WP5 holdout; nothing here edits the manuscript. Also writes results/tables/stage_final_ecan_decision.json. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import sys | |
| from pathlib import Path | |
| import pandas as pd | |
| CWM3 = Path(__file__).resolve().parents[1] | |
| sys.path.insert(0, str(CWM3 / "eval")) | |
| import report_joint as RJ # noqa: E402 | |
| ROAD_D = {f"d_onset|{f}|3s" for f in ("ttc", "ttc_drop", "lane", "badas")} | |
| def block(stage, a, b, claim, title, seeds): | |
| df = RJ.load(stage, a, b, seeds) | |
| if not len(df): | |
| return df, [f"### {title}: A = {a}, B = {b} — no rows yet (tags `{stage}_s<k>_{a}_vs_{b}`)", ""], {}, ("NO ROWS", [], [], []) | |
| body, verdicts = RJ.render(df, sorted(df.seed.unique()), a, b, title) | |
| return df, body, verdicts, RJ.claim(verdicts, claim) | |
| def fmt(ks): | |
| return [f"{r}|{c}" + (f"|{m}" if m != "pr_auc" else "") for (r, c, m) in ks] or "none" | |
| def own_table(verdicts, extra=()): | |
| lines = ["| regime | endpoint | mean Δ | verdict |", "|---|---|---|---|"] | |
| for (r, c, m), v in sorted(verdicts.items()): | |
| if (c in RJ.OWN or c in extra) and m == "pr_auc": | |
| lines.append(f"| {r} | {c.replace('|', '/')} | {v.get('mean_delta', float('nan')):+.4f} | {v['verdict']} |") | |
| return lines | |
| def road_beats(vA, vP): | |
| """road cells that BEAT in (a): D road families of the joint comparison + every P cell of the pressure comparison.""" | |
| out = [k for k, v in vA.items() if k[1] in ROAD_D and k[2] == "pr_auc" and v["verdict"] == "BEATS"] | |
| out += [k for k, v in vP.items() if k[1].startswith("p_onset|") and k[2] == "pr_auc" and v["verdict"] == "BEATS"] | |
| return out | |
| def road_worse(vA, vP): | |
| out = [k for k, v in vA.items() if k[1] in ROAD_D and k[2] == "pr_auc" and v["verdict"] == "WORSE"] | |
| out += [k for k, v in vP.items() if k[1].startswith("p_onset|") and k[2] == "pr_auc" and v["verdict"] == "WORSE"] | |
| return out | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--seeds", default="0,1,2") | |
| ap.add_argument("--out", default=str(CWM3 / "results" / "tables" / "stage_final_ecan.md")) | |
| a = ap.parse_args() | |
| seeds = [int(s) for s in a.seeds.split(",")] | |
| dA, bodyA, vA, (stA, worseA, bcA, boA) = block("finalc", "allminusecan", "allminus", "B2", "(a) sensor + vision E base vs the FROZEN main model (v2 causal kin targets)", seeds) | |
| dA2, bodyA2, vA2, (stA2, worseA2, bcA2, boA2) = block("finalc", "allminusecanonly", "allminus", "B2", "(a2) sensor-only control (no vision tokens) vs the FROZEN main model", seeds) | |
| dB, bodyB, vB, (stB, worseB, bcB, boB) = block("finalc", "allminusecan", "allminusecanonly", "B2", "(b) vision on top of the sensors: all_minus_road2drv_ecan vs all_minus_road2drv_ecanonly", seeds) | |
| dC, bodyC, vC, (stC, worseC, bcC, boC) = block("finalc", "indepecan", "indep", "B2", "E-base swap alone: indep_ecan vs indep (assembled bases, no training; descriptive)", seeds) | |
| dC2, bodyC2, vC2, (stC2, worseC2, bcC2, boC2) = block("finalc", "indepecanonly", "indep", "B2", "E-base swap alone: indep_ecanonly vs indep (descriptive)", seeds) | |
| dD, bodyD, vD, (stD, worseD, bcD, boD) = block("finalc", "allminusecan", "indepecan", "B2", "Interaction effect on the ecan bases: all_minus_road2drv_ecan vs indep_ecan (B2-style)", seeds) | |
| dAm, bodyAm, vAm, (stAm, worseAm, bcAm, boAm) = block("finalcm", "allminusecan", "allminus", "B2", "Masked protocol (64 v2 suspect routes censored): (a)", seeds) | |
| dA2m, bodyA2m, vA2m, (stA2m, worseA2m, bcA2m, boA2m) = block("finalcm", "allminusecanonly", "allminus", "B2", "Masked protocol: (a2)", seeds) | |
| dP1, bodyP1, vP1, _ = block("finalP", "allminusecan", "allminus", "B3", "P cells (road-pressure onset PR-AUC@3s, eval/pressure_eval.py): (a) vs FROZEN main", seeds) | |
| dP2, bodyP2, vP2, _ = block("finalP", "allminusecanonly", "allminus", "B3", "P cells: (a2) sensor-only vs FROZEN main", seeds) | |
| dP3, bodyP3, vP3, _ = block("finalP", "allminusecan", "allminusecanonly", "B3", "P cells: (b) can vs canonly", seeds) | |
| dfs = [dA, dA2, dB, dC, dC2, dD, dAm, dA2m, dP1, dP2, dP3] | |
| # ---- the pre-registered rule | |
| rb, rw = road_beats(vA, vP1), road_worse(vA, vP1) | |
| rb2, rw2 = road_beats(vA2, vP2), road_worse(vA2, vP2) | |
| rbB, rwB = road_beats(vB, vP3), road_worse(vB, vP3) | |
| insuff = any(v["verdict"] == "INSUFFICIENT" for (r, c, m), v in vA.items() if c in RJ.OWN and m == "pr_auc") | |
| n_seeds = int(dA.seed.nunique()) if len(dA) else 0 | |
| n_seeds_P = int(dP1.seed.nunique()) if len(dP1) else 0 | |
| if not len(dA): | |
| decision, dtxt = "NOT DECIDABLE", "no finalc_s<k>_allminusecan_vs_allminus rows yet" | |
| elif n_seeds < 3 or insuff or n_seeds_P < 3: | |
| decision, dtxt = "INSUFFICIENT", f"{n_seeds}/3 joint seeds and {n_seeds_P}/3 P-cell seeds paired (rule needs 3)" | |
| elif not worseA and rb: | |
| decision, dtxt = "CANDIDATE E BASE: pooled_P_can (sensor + vision CAN side stream)", f"no own endpoint WORSE vs the frozen main; road cells BEATS: {fmt(rb)}" | |
| else: | |
| decision, dtxt = "NEGATIVE RESULT (frozen main E base kept)", (f"own endpoint(s) WORSE vs the frozen main: {fmt(worseA)}; " if worseA else "no own endpoint WORSE; ") + \ | |
| (f"road cells BEATS: {fmt(rb)}" if rb else "no road cell BEATS (rule needs >= 1)") | |
| attribution = {"information_alone (a2: canonly vs main)": {"own_worse": fmt(worseA2), "own_beats": fmt(boA2), "road_beats": fmt(rb2), "road_worse": fmt(rw2), "B2_style": stA2}, | |
| "vision_on_top (b: can vs canonly)": {"own_worse": fmt(worseB), "own_beats": fmt(boB), "road_beats": fmt(rbB), "road_worse": fmt(rwB), "B2_style": stB}, | |
| "swap_alone (indep_ecan vs indep)": {"own_worse": fmt(worseC), "own_beats": fmt(boC), "B2_style": stC}, | |
| "swap_alone (indep_ecanonly vs indep)": {"own_worse": fmt(worseC2), "own_beats": fmt(boC2), "B2_style": stC2}} | |
| allrows = pd.concat([d for d in dfs if len(d)], ignore_index=True) if any(len(d) for d in dfs) else pd.DataFrame(columns=["a", "b", "regime", "run_stamp"]) | |
| prov, stale = RJ.provenance(allrows) if len(allrows) else ([], []) | |
| void = f" — **VOID: stale rows for {stale}**" if stale else "" | |
| L = ["# Stage FINAL — E-CAN variants (F5b): the frozen joint recipe with the E base swapped to `E0_pooled_P_can_s<k>_v2` (vf256 PCA 0..215 + 40 CAN road anchor features) and to the sensor-only control `E0_pooled_P_canonly_s<k>_v2` (no vision tokens), 3 seeds, test split, both regimes", "", | |
| "Runs `I1_{indep,all_minus_road2drv}_s<k>_final_ecan` / `_ecanonly` = the FINAL command line with only `--base-e` / `--base-dir-e` swapped (V0 `can4s_stats2`, U0 `kin_can_noK_clip8_ln`, seeds, epochs and the selection rule unchanged). " | |
| "The frozen main `I1_all_minus_road2drv_s<k>_final` and `I1_indep_s<k>_final` are read as B layouts only. Paired device-cluster bootstrap on identical rows (eval/paired_compare_joint.py, n_boot 1000; verdict = kill_checks.beats: ≥ 2/3 seeds CI excl. 0, same sign, |mean Δ| > seed sd); " | |
| "driver populations from the v2 causal kin targets (`finalc_*`), masked protocol `finalcm_*`; P cells by eval/pressure_eval.py (`finalP_*`). Every number comes from `results/paired_compare.csv`. " | |
| "The 40 CAN anchor features (exact list `ECAN_NAMES` in data/build_road_dynamics_targets.py): the 10 current margins M0 of targets_pressure (ttc_s, headway_s, dRel_m, vRel_mps, aLead_mps2, lat_margin_m, lane_width_m, badas, lead_valid, lane_valid) + 30 summaries of the 8 s CAN road history " | |
| "(mean / slope / min / max of lead_dRel, lead_vRel, lead_vLead, lane_center_offset, vEgo; lane_width mean; lead / lane valid fractions; lead target switches; minima of ttc / headway / lateral margin / lead acceleration; approaching and excursion fractions); canspec whitelist columns only (+ badas_risk of the R block, exactly as M0).", "", | |
| "**Pre-registered rule (docs/decisions.md 2026-09-16T14:10:00Z, before training):** `pooled_P_can` becomes the CANDIDATE E base iff in (a) NO own endpoint is WORSE vs the FROZEN main AND ≥ 1 road cell BEATS (D onset ttc / ttc_drop / lane / badas, or any P cell); otherwise a negative result. " | |
| "Attribution is reported separately: INFORMATION alone = (a2) sensor-only control vs the frozen main; VISION on top = (b) can vs canonly. Adoption is a validation decision confirmed once on the WP5 holdout.", "", | |
| f"## VERDICT: **{decision}**{void}", "", f"- {dtxt}", f"- own endpoints WORSE vs frozen main (a): {fmt(worseA)}", f"- own endpoints BEATS vs frozen main (a): {fmt(boA)}", | |
| f"- road cells BEATS (a): {fmt(rb)}; road cells WORSE (a): {fmt(rw)}", f"- cross-leg endpoints BEATS vs frozen main (a): {fmt(bcA)}", | |
| f"- INFORMATION alone (a2, sensor-only control vs frozen main): own WORSE {fmt(worseA2)}; own BEATS {fmt(boA2)}; road BEATS {fmt(rb2)}; road WORSE {fmt(rw2)}", | |
| f"- VISION on top (b, can vs canonly): own WORSE {fmt(worseB)}; own BEATS {fmt(boB)}; road BEATS {fmt(rbB)}; road WORSE {fmt(rwB)}", | |
| f"- masked-protocol sensitivity (a): own WORSE {fmt(worseAm)}; own BEATS {fmt(boAm)}", "", | |
| "## Provenance of the A / B layouts (guard 2): layout mtime, source hash, model_id / code hash, completeness; rows older than the layout are STALE", "", | |
| "| run | regime | layout mtime (UTC) | source sha256 | model_id / code | status |", "|---|---|---|---|---|---|", *prov, "", | |
| *([f"**STALE ROWS DETECTED for {stale} — every verdict is VOID; recompute (eval/run_joint_evals_final_ecan.sh).**", ""] if stale else []), | |
| "## 1. (a) Sensor + vision E base vs the FROZEN main model — the pre-registered comparison", ""] + bodyA + ["", "### Own + road-D endpoints of (a) (the rule's inputs)", ""] + own_table(vA) + \ | |
| ["", f"B2-style status of (a) for reference: **{stA}**", "", | |
| "## 2. (a2) Sensor-only control vs the FROZEN main model — the CAN information alone", ""] + bodyA2 + ["", f"B2-style status: **{stA2}** — own WORSE {fmt(worseA2)}; own BEATS {fmt(boA2)}", "", | |
| "## 3. (b) Vision on top of the sensors — can vs canonly", ""] + bodyB + ["", f"B2-style status: **{stB}** — own WORSE {fmt(worseB)}; own BEATS {fmt(boB)}", "", | |
| "## 4. Attribution — the E-base swap alone (no joint training)", ""] + bodyC + [""] + bodyC2 + ["", | |
| "## 5. Interaction effect on the ecan bases", ""] + bodyD + ["", f"B2-style verdict: **{stD}** — own WORSE {fmt(worseD)}; cross BEATS {fmt(bcD)}; own BEATS {fmt(boD)}", "", | |
| "## 6. Masked protocol (sensitivity to the 64 suspect routes)", ""] + bodyAm + [""] + bodyA2m + ["", | |
| "## 7. P cells (road-pressure chains)", ""] + bodyP1 + [""] + bodyP2 + [""] + bodyP3 | |
| out = Path(a.out) | |
| out.parent.mkdir(parents=True, exist_ok=True) | |
| out.write_text("\n".join(L) + "\n") | |
| dec = {"decision": decision, "detail": dtxt, "worse_own_vs_frozen_main": fmt(worseA), "beats_own_vs_frozen_main": fmt(boA), "beats_cross_vs_frozen_main": fmt(bcA), | |
| "road_beats_a": fmt(rb), "road_worse_a": fmt(rw), "attribution": attribution, "masked_worse_own": fmt(worseAm), "B2_style_a": stA, "stale": stale, "seeds": seeds, | |
| "n_seeds_paired_a": n_seeds, "n_seeds_P": n_seeds_P, | |
| "own_endpoints_a": {f"{r}|{c}": {"mean_delta": v.get("mean_delta", float("nan")), "verdict": v["verdict"]} for (r, c, m), v in vA.items() if c in RJ.OWN and m == "pr_auc"}, | |
| "p_cells_a": {f"{r}|{c}": {"mean_delta": v.get("mean_delta", float("nan")), "verdict": v["verdict"]} for (r, c, m), v in vP1.items() if m == "pr_auc"}, | |
| "own_endpoints_a2": {f"{r}|{c}": {"mean_delta": v.get("mean_delta", float("nan")), "verdict": v["verdict"]} for (r, c, m), v in vA2.items() if c in RJ.OWN and m == "pr_auc"}, | |
| "p_cells_a2": {f"{r}|{c}": {"mean_delta": v.get("mean_delta", float("nan")), "verdict": v["verdict"]} for (r, c, m), v in vP2.items() if m == "pr_auc"}} | |
| (out.parent / "stage_final_ecan_decision.json").write_text(json.dumps(dec, indent=1, default=str)) | |
| print("\n".join(L[:12])) | |
| print(f"\nVERDICT {decision}: {dtxt}\nwritten {out} and {out.parent / 'stage_final_ecan_decision.json'}") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 14.4 kB
- Xet hash:
- 4466bc1b6dc830f4fb00473dba3c7102fb08687fac152c104f75991686ba70e4
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.