Buckets:
| """cwm3/eval/report_joint.py — stage tables for the joint model (results/tables/stage_I1.md, stage_F.md) from results/paired_compare.csv. | |
| python report_joint.py --stage I1 --a all --b indep [--extra-b none naive] [--seeds 0,1,2] [--out results/tables/stage_I1.md] | |
| python report_joint.py --stage F1 --a chunked --b direct_fresh --extra-a ... --out results/tables/stage_F.md | |
| Reads the rows written by eval/paired_compare_joint.py with tag "<stage>_s<seed>_<a>_vs_<b>" (both regimes, test split) and renders, per cell: | |
| A / B values per seed, Δ = A − B with the device-cluster bootstrap CI per seed, the 3-seed mean ± sd of Δ, and the kill_checks.beats verdict | |
| (BEATS / WORSE / NOT_WORSE / INCONCLUSIVE; INSUFFICIENT with < 3 seeds). Claim logic (plan §2): | |
| B2 (I1): own endpoints = veh_onset gas/brake/steer_phase@3s, drv_onset hand_return/hand_leave/head_turn_onset@3s, d_onset six families@3s, both | |
| regimes — PASS iff no own endpoint is WORSE AND ≥ 1 cross-leg endpoint (C1, C2|<f>, C2|pooled_warnable) is BEATS (pr_auc or auroc). | |
| B3 (F1): PASS iff no endpoint (own + cross-leg) is WORSE AND ≥ 1 endpoint is BEATS. | |
| Numbers come only from the CSV; nothing is recomputed here. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import sys | |
| from pathlib import Path | |
| import numpy as np | |
| import pandas as pd | |
| CWM3 = Path(__file__).resolve().parents[1] | |
| CWM2 = CWM3.parent / "cwm2" | |
| sys.path.insert(0, str(CWM2 / "eval")) | |
| import kill_checks as KC # noqa: E402 | |
| CSV = CWM3 / "results" / "paired_compare.csv" | |
| RUNS = CWM3 / "results" / "runs" | |
| def provenance(df): | |
| """Guard 2 (2026-09-11): for every A / B run dir in the rendered rows, the CURRENT layout files' mtime, source hash (meta.inputs_digest) and model_id, | |
| plus whether the rows were computed AFTER that layout was written (a row older than the layout was computed from a previous layout -> STALE).""" | |
| import json, os, time | |
| lines, stale = [], [] | |
| runs = sorted(set(df.a.astype(str)) | set(df.b.astype(str))) | |
| for r in runs: | |
| rd = RUNS / r | |
| if not rd.exists(): | |
| lines.append(f"| {r} | — | (not a run dir: reference / heuristic) | | | |") | |
| continue | |
| flags = ", ".join(m for m, p in (("TRAIN_DONE", rd / "TRAIN_DONE"), ("PRED_DONE", rd / "PRED_DONE")) if not p.exists()) or "complete" | |
| for reg in ("manual", "engaged"): | |
| lay = rd / f"pred_layout_{reg}_test.npz" | |
| if not lay.exists(): | |
| lines.append(f"| {r} | {reg} | layout MISSING | | | {flags} |") | |
| continue | |
| mt = os.path.getmtime(lay) | |
| meta = json.loads(str(np.load(lay, allow_pickle=True)["meta"])) | |
| rows = df[((df.a.astype(str) == r) | (df.b.astype(str) == r)) & (df.regime == reg)] | |
| oldest = str(rows.run_stamp.min()) if len(rows) else "" | |
| layout_stamp = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime(mt)) | |
| st = "STALE ROWS (computed before this layout)" if oldest and oldest < layout_stamp else "rows postdate layout" | |
| if st.startswith("STALE"): | |
| stale.append(f"{r}/{reg}") | |
| lines.append(f"| {r} | {reg} | {time.strftime('%Y-%m-%d %H:%M:%SZ', time.gmtime(mt))} | {str(meta.get('inputs_digest', ''))[:16]} | {meta.get('model_id', '')} / code {str(meta.get('code_sha256', ''))[:12]} | {flags}; oldest row {oldest}; {st} |") | |
| return lines, stale | |
| OWN = {"veh_onset|gas_on|3s", "veh_onset|brake_on|3s", "veh_onset|steer_phase_on|3s", "drv_onset|hand_return|3s", "drv_onset|hand_leave|3s", | |
| "drv_onset|head_turn_onset|3s"} | {f"d_onset|{f}|3s" for f in ("steer_rate", "decel", "ttc", "ttc_drop", "lane", "badas")} | |
| CROSS = {"C1", "C1_given_k", "C2|pooled_warnable"} | {f"C2|{f}" for f in ("steer_rate", "decel", "ttc", "ttc_drop", "lane", "badas")} | |
| LOWER_BETTER = ("hazard_nll", "y_mae", "y_crps", "nr_brier", "cooccur_brier") | |
| def load(stage, a_name, b_name, seeds): | |
| df = pd.read_csv(CSV) | |
| tags = [f"{stage}_s{s}_{a_name}_vs_{b_name}" for s in seeds] | |
| df = df[df.tag.isin(tags)].copy() | |
| df["seed"] = df.tag.str.extract(r"_s(\d+)_")[0].astype(int) | |
| # latest run_stamp per (tag, regime, cell, metric) | |
| df = df.sort_values("run_stamp").groupby(["tag", "regime", "split", "cell", "metric"], as_index=False).tail(1) | |
| return df | |
| def verdict(g, lower_better=False): | |
| pr = pd.DataFrame({"seed": g.seed, "delta": g.delta.astype(float) * (-1 if lower_better else 1), | |
| "delta_ci_lo": (g.delta_ci_lo if not lower_better else -g.delta_ci_hi).astype(float), | |
| "delta_ci_hi": (g.delta_ci_hi if not lower_better else -g.delta_ci_lo).astype(float), "status": "secondary"}) | |
| va, vb = pd.DataFrame({"value": g.value_a.astype(float)}), pd.DataFrame({"value": g.value_b.astype(float)}) | |
| return KC.beats(pr, va, vb, min_seeds=3) | |
| def render(df, seeds, a_name, b_name, title): | |
| lines = [f"### {title}: A = {a_name}, B = {b_name}", "", | |
| "| regime | cell | metric | " + " | ".join(f"A s{s}" for s in seeds) + " | " + " | ".join(f"Δ s{s} [CI]" for s in seeds) + " | mean Δ ± sd | verdict |", | |
| "|---|---|---|" + "---|" * (2 * len(seeds)) + "---|---|"] | |
| verdicts = {} | |
| order = sorted(df.cell.unique(), key=lambda c: (0 if c in OWN else 1 if c in CROSS else 2, c)) | |
| for regime in ("manual", "engaged"): | |
| for cell in order: | |
| for metric in sorted(df[(df.regime == regime) & (df.cell == cell)].metric.unique()): | |
| g = df[(df.regime == regime) & (df.cell == cell) & (df.metric == metric)].sort_values("seed") | |
| if not len(g): | |
| continue | |
| lb = cell.startswith(LOWER_BETTER) | |
| res = verdict(g, lb) | |
| verdicts[(regime, cell, metric)] = res | |
| by = {int(s): r for s, r in zip(g.seed, g.itertuples())} | |
| A = [f"{by[s].value_a:.4f}" if s in by else "—" for s in seeds] | |
| D = [f"{by[s].delta:+.4f} [{by[s].delta_ci_lo:+.4f},{by[s].delta_ci_hi:+.4f}]" if s in by else "—" for s in seeds] | |
| d = g.delta.astype(float).to_numpy() | |
| lines.append(f"| {regime} | {cell.replace('|', '/')} | {metric}{' (lower better)' if lb else ''} | " + " | ".join(A) + " | " + " | ".join(D) + | |
| f" | {d.mean():+.4f} ± {d.std(ddof=1) if len(d) > 1 else 0.0:.4f} (n={len(d)}, pos={int(g.n_pos.iloc[0]) if str(g.n_pos.iloc[0]).strip() not in ('', 'nan') else '—'}) | {res['verdict']} |") | |
| return lines, verdicts | |
| DRV = {"drv_onset|hand_leave|3s", "drv_onset|hand_return|3s", "drv_onset|head_turn_onset|3s"} | |
| def claim(verdicts, kind): | |
| if kind in ("K4", "B1", "K4x"): | |
| drv = {k: v for k, v in verdicts.items() if k[1] in DRV and k[2] == "pr_auc"} | |
| worse = [k for k, v in drv.items() if v["verdict"] == "WORSE"] | |
| beats = [k for k, v in drv.items() if v["verdict"] == "BEATS"] | |
| insuff = any(v["verdict"] == "INSUFFICIENT" for v in drv.values()) | |
| if kind == "K4x": | |
| return "DESCRIPTIVE (cross-target definition mismatch; no verdict)", worse, beats, [] | |
| if kind == "K4": | |
| ok = not worse and len(beats) >= 1 | |
| else: # B1: hand_leave and head_turn_onset @3s beat the reference in both regimes | |
| need = {(r, c) for r in ("manual", "engaged") for c in ("drv_onset|hand_leave|3s", "drv_onset|head_turn_onset|3s")} | |
| ok = all((r, c, "pr_auc") in drv and drv[(r, c, "pr_auc")]["verdict"] == "BEATS" for r, c in need) | |
| return ("INSUFFICIENT (< 3 seeds)" if insuff else ("PASS" if ok else "FAIL")), worse, beats, [] | |
| own = {k: v for k, v in verdicts.items() if k[1] in OWN and k[2] == "pr_auc"} | |
| cross = {k: v for k, v in verdicts.items() if k[1] in CROSS} | |
| worse_own = [k for k, v in own.items() if v["verdict"] == "WORSE"] | |
| beats_cross = [k for k, v in cross.items() if v["verdict"] == "BEATS"] | |
| beats_own = [k for k, v in own.items() if v["verdict"] == "BEATS"] | |
| insuff = any(v["verdict"] == "INSUFFICIENT" for v in list(own.values()) + list(cross.values())) | |
| if kind == "B2": | |
| ok = not worse_own and len(beats_cross) >= 1 | |
| else: | |
| worse_any = [k for k, v in verdicts.items() if v["verdict"] == "WORSE"] | |
| ok = not worse_any and (len(beats_cross) + len(beats_own)) >= 1 | |
| worse_own = worse_any | |
| status = "INSUFFICIENT (< 3 seeds)" if insuff else ("PASS" if ok else "FAIL") | |
| return status, worse_own, beats_cross, beats_own | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--stage", required=True) | |
| ap.add_argument("--a", required=True) | |
| ap.add_argument("--b", required=True) | |
| ap.add_argument("--extra", nargs="*", default=[], help="additional a:b pairs rendered as secondary tables") | |
| ap.add_argument("--seeds", default="0,1,2") | |
| ap.add_argument("--claim", default="B2", choices=["B2", "B3", "K4", "B1", "K4x"], | |
| help="K4 = input causality (causal vs old inputs on causal targets: not worse on any driver endpoint, better on >= 1); " | |
| "B1 = causal model vs persistence / k0 (BEATS hand_leave and head_turn@3s in both regimes); K4x = cross-target rows (definition mismatch, descriptive only)") | |
| ap.add_argument("--out", default="") | |
| ap.add_argument("--append", action="store_true", help="2026-09-16 (WP3 R2 controls): APPEND the rendered sections to --out under a dated header instead of overwriting it") | |
| a = ap.parse_args() | |
| seeds = [int(s) for s in a.seeds.split(",")] | |
| df = load(a.stage, a.a, a.b, seeds) | |
| assert len(df), f"no paired_compare rows for {a.stage} {a.a} vs {a.b} seeds {seeds}" | |
| have = sorted(df.seed.unique()) | |
| caveat = {"K4x": "**Caveat: A and B are trained AND scored on different event-target definitions (causal-ROI vs whole-route-median driver states, which disagree on hand events with F1 ≈ 0.3); " | |
| "the sign of these rows follows whichever definition the scoring uses — descriptive only, not an input-causality result.**", | |
| "K4": "Both models are scored on the CAUSAL targets; A = causal-ROI inputs, B = old route-median inputs (input-isolation control) — the K4 input effect.", | |
| "B1": "A = causal-input model on its own (causal) targets; B = the label-free reference (k0 heuristic from the causal K0 / persistence = base rate).", | |
| }.get(a.claim, "") | |
| prov, stale = provenance(pd.concat([df] + [load(a.stage, *pair.split(":"), seeds) for pair in a.extra], ignore_index=True) if a.extra else df) | |
| lines = [f"# Stage {a.stage}: {a.a} vs {a.b} — paired device-cluster bootstrap (eval/paired_compare_joint.py, n_boot 1000), test split, both regimes", "", | |
| *([caveat, ""] if caveat else []), | |
| "## Provenance of the A / B layouts (guard 2, 2026-09-11): layout mtime, source hash (sha256 of the prediction pack the layout was built from), model_id / code hash, completeness; " | |
| "a row whose run_stamp precedes the current layout was computed from a PREVIOUS layout of that dir and is STALE", "", | |
| "| run | regime | layout mtime (UTC) | source sha256 | model_id / code | status |", "|---|---|---|---|---|---|", *prov, "", | |
| *([f"**STALE ROWS DETECTED for {stale} — every verdict below is VOID; recompute the pairs with eval/paired_compare_joint.py (its layout guard refuses unfinished / mismatched layouts).**", ""] if stale else []), | |
| f"Seeds with rows: {have} (kill rule needs 3: ≥ 2/3 seeds CI excluding 0, same sign, |mean Δ| > seed sd). Source: `results/paired_compare.csv` " | |
| f"tags `{a.stage}_s<k>_{a.a}_vs_{a.b}`. Cells: own endpoints (onset PR-AUC@3s), cross-leg endpoints (C1 / C2, PR-AUC + AUROC), hazard NLL, " | |
| f"continuous emissions, nr / cooccur Brier. Driver-leg cells use the route-median (non-causal) ROI inputs (diagnostic, K4 pending).", ""] | |
| body, verdicts = render(df, have, a.a, a.b, "Main comparison") | |
| lines += body | |
| status, worse, beats_cross, beats_own = claim(verdicts, a.claim) | |
| if stale: | |
| status = f"VOID (stale rows: {stale})" | |
| label = {"K4": "K4 (input causality) verdict", "B1": "B1 (vs persistence / k0 on causal targets) verdict", "K4x": "Cross-target rows"}.get(a.claim, f"Claim {a.claim} verdict") | |
| lines += ["", f"## {label}: **{status}**", "", | |
| f"- own endpoints WORSE: {[f'{r}|{c}' for r, c, _ in worse] or 'none'}", | |
| f"- cross-leg endpoints BEATS: {[f'{r}|{c}|{m}' for r, c, m in beats_cross] or 'none'}", | |
| f"- own endpoints BEATS: {[f'{r}|{c}' for r, c, _ in beats_own] or 'none'}", | |
| f"- INCONCLUSIVE own endpoints (mean Δ): " + (", ".join(f"{r}|{c} {v['mean_delta']:+.4f}" for (r, c, m), v in verdicts.items() if c in OWN and m == 'pr_auc' and v['verdict'] == 'INCONCLUSIVE') or "none"), ""] | |
| for pair in a.extra: | |
| an, bn = pair.split(":") | |
| d2 = load(a.stage, an, bn, seeds) | |
| if not len(d2): | |
| lines += [f"### Secondary: {an} vs {bn} — no rows yet", ""] | |
| continue | |
| body2, v2 = render(d2, sorted(d2.seed.unique()), an, bn, "Secondary comparison") | |
| st2, w2, bc2, bo2 = claim(v2, a.claim) | |
| lines += [""] + body2 + ["", f"Secondary {a.claim}-style verdict ({an} vs {bn}): **{st2}** — WORSE {[f'{r}|{c}' for r, c, _ in w2] or 'none'}; BEATS cross {[f'{r}|{c}|{m}' for r, c, m in bc2] or 'none'}; BEATS own {[f'{r}|{c}' for r, c, _ in bo2] or 'none'}", ""] | |
| # ---- K5 shifted-twin pair-AUROC of the driver endpoints (eval/shift_twin.py, read-only; tags <stage>_s<k>_<a>_vs_<b>) | |
| tw_csv = CWM3 / "results" / "shift_twin.csv" | |
| if tw_csv.exists(): | |
| tw = pd.read_csv(tw_csv) | |
| pairs = [(a.a, a.b)] + [tuple(x.split(":")) for x in a.extra] | |
| for an, bn in pairs: | |
| t = tw[tw.tag.isin([f"{a.stage}_s{s}_{an}_vs_{bn}" for s in seeds])].copy() | |
| if not len(t): | |
| continue | |
| t["seed"] = t.tag.str.extract(r"_s(\d+)_")[0].astype(int) | |
| t = t.sort_values("run_stamp").groupby(["tag", "regime", "event", "statistic"], as_index=False).tail(1) | |
| hs = sorted(t.seed.unique()) | |
| lines += ["", f"### K5 shifted-twin pair-AUROC@3s (driver endpoints), A = {an}, B = {bn} — same-route twins (Δt 30–60 s, |ΔvEgo| < 2 m/s, label flips) " | |
| "share camera / identity; a gain that survives here is driver-state discrimination, a PR-AUC change that appears only against random negatives rides on identity", "", | |
| "| regime | event | statistic | " + " | ".join(f"A s{s}" for s in hs) + " | " + " | ".join(f"Δ s{s} [CI]" for s in hs) + " | mean Δ ± sd | verdict |", | |
| "|---|---|---|" + "---|" * (2 * len(hs)) + "---|---|"] | |
| for (reg, ev, stt), g in t.groupby(["regime", "event", "statistic"]): | |
| g = g.sort_values("seed") | |
| by = {int(r.seed): r for r in g.itertuples()} | |
| A = [f"{by[s].value_a:.4f}" if s in by else "—" for s in hs] | |
| D = [(f"{by[s].delta:+.4f} [{by[s].delta_ci_lo:+.4f},{by[s].delta_ci_hi:+.4f}]" if s in by and np.isfinite(by[s].delta_ci_lo) else f"{by[s].delta:+.4f}" if s in by else "—") for s in hs] | |
| d = g.delta.astype(float).to_numpy() | |
| res = verdict(g.assign(seed=g.seed), False) if g.delta_ci_lo.notna().all() else {"verdict": "—"} | |
| lines.append(f"| {reg} | {ev} | {stt} | " + " | ".join(A) + " | " + " | ".join(D) + f" | {d.mean():+.4f} ± {d.std(ddof=1) if len(d) > 1 else 0.0:.4f} (n={len(d)}) | {res['verdict']} |") | |
| out = Path(a.out) if a.out else CWM3 / "results" / "tables" / f"stage_{a.stage}.md" | |
| out.parent.mkdir(parents=True, exist_ok=True) | |
| if a.append and out.exists(): | |
| import time as _t | |
| stamp = _t.strftime("%Y-%m-%dT%H:%M:%SZ", _t.gmtime()) | |
| with open(out, "a") as f: | |
| f.write("\n\n---\n\n" + f"# Appended {stamp}: {a.stage} {a.a} vs {a.b}" + (f" (+ {' '.join(a.extra)})" if a.extra else "") + "\n\n" + "\n".join(lines) + "\n") | |
| print("\n".join(lines)) | |
| print(f"\nappended to {out}") | |
| return | |
| out.write_text("\n".join(lines) + "\n") | |
| print("\n".join(lines)) | |
| print(f"\nwritten {out}") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 16.4 kB
- Xet hash:
- 4f8424eca720acb1650b932ea1bc9632a7e670e8a919df197755f179780426a5
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.