Buckets:
| """cwm3/eval/report_branch_arch.py — GPU-baselines experiment B table: branch-architecture ablation (results/tables/branch_arch_ablation.md). | |
| Reads results/paired_compare.csv tags `Barch_s<k>_<a>_vs_<b>` (U: utcn / utfm vs ugru, K4 tools on the causal-v2 driver populations; V: vgru / vtfm vs vtcn; | |
| E: eL1 / eL4 vs eL2, D cells + F cosine) and `BarchP_s<k>_eL<n>_vs_eL2` (P onsets, eval/pressure_eval.py); latest row per (tag, regime, cell, metric). | |
| Verdict per cell = cwm2/eval/kill_checks.beats (>= 2/3 seeds CI excl. 0, same sign, |mean Δ| > seed sd); lower-is-better cells (y_mae, y_crps, hazard_nll, | |
| nr_brier, cooccur_brier) are sign-flipped first, so BEATS always reads "the alternative architecture is better", WORSE "worse". | |
| Per comparison: parameter counts (total / encoder; from run_train_*.json `params`), epochs / best epoch / wall, and the pre-registered verdict line: | |
| "architecture-insensitive" iff no OWN cell (the branch's own onset cells at 3 s; V also y_mae / y_crps; E also the P onsets) is BEATS or WORSE. | |
| Numbers come only from the CSV / run json files; nothing is recomputed. python eval/report_branch_arch.py [--seeds "0 1 2"] [--out results/tables/branch_arch_ablation.md] | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import glob | |
| import json | |
| import sys | |
| import time | |
| from pathlib import Path | |
| import numpy as np | |
| import pandas as pd | |
| CWM3 = Path(__file__).resolve().parents[1] | |
| sys.path.insert(0, str(CWM3.parent / "cwm2" / "eval")) | |
| sys.path.insert(0, str(CWM3 / "common")) | |
| import kill_checks as KC # noqa: E402 | |
| from runmeta import write_run_json # noqa: E402 | |
| CSV = CWM3 / "results" / "paired_compare.csv" | |
| RUNS = CWM3 / "results" / "runs" | |
| LOWER_BETTER = ("hazard_nll", "y_mae", "y_crps", "nr_brier", "cooccur_brier") | |
| U_OWN = {f"drv_onset|{e}|3s" for e in ("hand_leave", "hand_return", "head_turn_onset")} | |
| V_OWN = {f"veh_onset|{e}|3s" for e in ("gas_on", "brake_on", "steer_phase_on")} | {f"y_mae|{c}" for c in ("steeringAngleDeg", "vEgo", "aEgo")} | {f"y_crps|{c}" for c in ("steeringAngleDeg", "vEgo", "aEgo")} | |
| E_OWN = {f"d_onset|{f}|3s" for f in ("steer_rate", "decel", "ttc", "ttc_drop", "lane", "badas")} | {f"p_onset|{c}|3s" for c in ("ttc", "ttc_drop", "cut_in", "lead_braking", "lane_excursion", "badas")} | |
| COMPARISONS = [ | |
| {"branch": "U", "title": "U driver trunk: causal TCN (param-matched) vs GRU (survivor `kin_can_noK_clip8_ln`)", "tags": ["Barch_s{k}_utcn_vs_ugru"], | |
| "a": "U0_kin_can_noK_s{k}_kcv2_clip8_ln_tcn", "b": "U0_kin_can_noK_s{k}_kcv2_clip8_ln", "own": U_OWN}, | |
| {"branch": "U", "title": "U driver trunk: 2-layer causal Transformer (param-matched) vs GRU (survivor)", "tags": ["Barch_s{k}_utfm_vs_ugru"], | |
| "a": "U0_kin_can_noK_s{k}_kcv2_clip8_ln_tfm", "b": "U0_kin_can_noK_s{k}_kcv2_clip8_ln", "own": U_OWN}, | |
| {"branch": "V", "title": "V vehicle trunk: GRU (param-matched) vs TCN (survivor `can4s_stats2`)", "tags": ["Barch_s{k}_vgru_vs_vtcn"], | |
| "a": "V0_can4s_s{k}_stats2_gru", "b": "V0_can4s_s{k}_stats2", "own": V_OWN}, | |
| {"branch": "V", "title": "V vehicle trunk: 2-layer causal Transformer (param-matched) vs TCN (survivor)", "tags": ["Barch_s{k}_vtfm_vs_vtcn"], | |
| "a": "V0_can4s_s{k}_stats2_tfm", "b": "V0_can4s_s{k}_stats2", "own": V_OWN}, | |
| {"branch": "E", "title": "E environment encoder: 1 layer vs 2 layers (survivor `pooled_P`)", "tags": ["Barch_s{k}_eL1_vs_eL2", "BarchP_s{k}_eL1_vs_eL2"], | |
| "a": "E0_pooled_P_L1_s{k}_v2", "b": "E0_pooled_P_s{k}_v2", "own": E_OWN}, | |
| {"branch": "E", "title": "E environment encoder: 4 layers vs 2 layers (survivor)", "tags": ["Barch_s{k}_eL4_vs_eL2", "BarchP_s{k}_eL4_vs_eL2"], | |
| "a": "E0_pooled_P_L4_s{k}_v2", "b": "E0_pooled_P_s{k}_v2", "own": E_OWN}, | |
| ] | |
| def beats(g, lower): | |
| sgn = -1.0 if lower else 1.0 | |
| lo, hi = (g.delta_ci_hi, g.delta_ci_lo) if lower else (g.delta_ci_lo, g.delta_ci_hi) | |
| pr = pd.DataFrame({"seed": g.seed.values, "delta": sgn * g.delta.astype(float).values, "delta_ci_lo": sgn * lo.astype(float).values, "delta_ci_hi": sgn * hi.astype(float).values, "status": "secondary"}) | |
| return KC.beats(pr, pd.DataFrame({"value": g.value_a.astype(float).values}), pd.DataFrame({"value": g.value_b.astype(float).values}), min_seeds=3)["verdict"] | |
| def run_info(name): | |
| rd = RUNS / name | |
| fs = sorted(glob.glob(str(rd / "run_train_*.json"))) | |
| if not fs: | |
| return None | |
| d = json.load(open(fs[-1])) | |
| par = d.get("params", {}) or {} | |
| enc = next((par[k] for k in ("enc", "gru", "tcn") if k in par), None) | |
| return {"total": par.get("total"), "encoder": enc, "epochs": d.get("epochs_run"), "best_epoch": d.get("best_epoch"), "wall_s": d.get("wall_s"), | |
| "arch": (d.get("cfg", {}) or {}).get("arch") or (f"{(d.get('cfg', {}) or {}).get('model', {}).get('n_layers')} layers" if (d.get("cfg", {}) or {}).get("branch") == "E" else "reference")} | |
| def fmt(v): | |
| return "—" if v is None or (isinstance(v, float) and not np.isfinite(v)) else (f"{v:.4f}" if isinstance(v, float) else str(v)) | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--seeds", default="0 1 2") | |
| ap.add_argument("--out", default=str(CWM3 / "results" / "tables" / "branch_arch_ablation.md")) | |
| a = ap.parse_args() | |
| t0 = time.time() | |
| seeds = [int(s) for s in a.seeds.replace(",", " ").split()] | |
| df = pd.read_csv(CSV, low_memory=False) | |
| df = df[df.tag.astype(str).str.startswith(("Barch_", "BarchP_"))].copy() | |
| df["seed"] = df.tag.str.extract(r"_s(\d+)_")[0].astype(int) | |
| df = df.sort_values("run_stamp").groupby(["tag", "regime", "cell", "metric"], as_index=False).tail(1) | |
| for c in ("value_a", "value_b", "delta", "delta_ci_lo", "delta_ci_hi"): | |
| df[c] = pd.to_numeric(df[c], errors="coerce") | |
| stamp = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime()) | |
| L = ["# Branch-architecture ablation (GPU-baselines experiment B) — parameter-matched trunk swaps on the survivor recipes, 3 seeds, test split, both regimes", | |
| "", f"Generated {stamp} by `eval/report_branch_arch.py` from `results/paired_compare.csv` (paired device-cluster bootstrap on identical rows, n_boot 1000: " | |
| "U cells = `eval/paired_compare_joint.py --kin-targets causal_routes --kin-version v2` (the K4 tools), V / E cells = `eval/paired_compare.py`, E P-onsets = `eval/pressure_eval.py`) " | |
| "and the run dirs' `run_train_*.json`. Alternatives: `models/branches_alt.py` (encoder width solved to the reference encoder's parameter count ±10 %; input LN, mode embedding, " | |
| "flat head and their seeded init are the reference's); E depth = `pooled_P` with 1 / 4 encoder layers (`train/stages.yaml` variants `pooled_P_L1` / `pooled_P_L4`). " | |
| "Δ = A (alternative) − B (survivor); lower-is-better cells (y_mae / y_crps / hazard NLL / Brier) are sign-flipped for the verdict so BEATS = alternative better, WORSE = worse " | |
| "(`kill_checks.beats`: ≥ 2/3 seeds CI excl. 0, same sign, |mean Δ| > seed sd). Pre-registered reading per comparison: **architecture-insensitive** iff no own cell is BEATS or WORSE.", ""] | |
| summary = {} | |
| for cmp in COMPARISONS: | |
| tags = [t.format(k=k) for t in cmp["tags"] for k in seeds] | |
| x = df[df.tag.isin(tags)] | |
| infos = {k: (run_info(cmp["a"].format(k=k)), run_info(cmp["b"].format(k=k))) for k in seeds} | |
| L += [f"## {cmp['title']}", ""] | |
| L += ["| seed | A run | A arch | A params total / encoder | A epochs (best) | A wall s | B run | B params total / encoder | B epochs (best) | B wall s |", "|---|---|---|---|---|---|---|---|---|---|"] | |
| for k in seeds: | |
| ia, ib = infos[k] | |
| fa = lambda i, key: "—" if i is None or i.get(key) is None else (f"{i[key]:,}" if isinstance(i[key], int) else str(i[key])) | |
| L.append(f"| {k} | {cmp['a'].format(k=k)} | {fa(ia, 'arch')} | {fa(ia, 'total')} / {fa(ia, 'encoder')} | {fa(ia, 'epochs')} ({fa(ia, 'best_epoch')}) | {fa(ia, 'wall_s')} | {cmp['b'].format(k=k)} | {fa(ib, 'total')} / {fa(ib, 'encoder')} | {fa(ib, 'epochs')} ({fa(ib, 'best_epoch')}) | {fa(ib, 'wall_s')} |") | |
| L.append("") | |
| if not len(x): | |
| L += ["_(no paired rows yet)_", ""] | |
| summary[cmp["title"]] = {"verdict": "NOT_RUN"} | |
| continue | |
| L += ["| regime | cell | metric | A s0 / s1 / s2 | B s0 / s1 / s2 | Δ s0 [CI] | Δ s1 [CI] | Δ s2 [CI] | mean Δ ± sd | own | verdict |", "|---|---|---|---|---|---|---|---|---|---|---|"] | |
| verdicts, own_v = {}, [] | |
| order = lambda c: (0 if c.split("|")[0] in ("drv_onset", "veh_onset", "d_onset", "p_onset") else 1 if c.startswith(("y_mae", "y_crps")) else 2 if c.startswith("f_cos") else 3, c) | |
| for reg in ("manual", "engaged"): | |
| xr = x[x.regime == reg] | |
| for cell in sorted(xr.cell.unique(), key=order): | |
| for metric in sorted(xr[xr.cell == cell].metric.unique()): | |
| g = xr[(xr.cell == cell) & (xr.metric == metric)].sort_values("seed") | |
| if metric == "base_rate": | |
| continue | |
| lower = cell.startswith(LOWER_BETTER) | |
| v = beats(g, lower) if len(g) >= 3 else "INSUFFICIENT" | |
| own = cell in cmp["own"] | |
| if own and len(g) >= 3: | |
| own_v.append(v) | |
| d = g.delta.astype(float).values | |
| per = {int(s): r for s, r in zip(g.seed, g.itertuples())} | |
| cells_d = " | ".join(f"{per[s].delta:+.4f} [{per[s].delta_ci_lo:+.4f},{per[s].delta_ci_hi:+.4f}]" if s in per else "—" for s in seeds) | |
| va = " / ".join(f"{per[s].value_a:.4f}" if s in per else "—" for s in seeds) | |
| vb = " / ".join(f"{per[s].value_b:.4f}" if s in per else "—" for s in seeds) | |
| ms = f"{d.mean():+.4f} ± {d.std(ddof=1):.4f}" if len(d) > 1 else f"{d.mean():+.4f}" | |
| L.append(f"| {reg} | {cell} | {metric} | {va} | {vb} | {cells_d} | {ms} | {'own' if own else ''} | {v}{' (lower better)' if lower else ''} |") | |
| verdicts[f"{reg}|{cell}|{metric}"] = {"verdict": v, "own": own, "mean_delta": float(d.mean()), "deltas": d.tolist(), "n_seeds": int(len(g))} | |
| n_beats, n_worse = sum(v == "BEATS" for v in own_v), sum(v == "WORSE" for v in own_v) | |
| insuff = sum(v == "INSUFFICIENT" for v in own_v) + (0 if own_v else 1) | |
| if not own_v: | |
| line = "verdict: NOT_RUN (no own cells with 3 seeds yet)" | |
| elif n_beats == 0 and n_worse == 0: | |
| line = f"verdict: **architecture-insensitive** (0 own cells BEATS / 0 WORSE over {len(own_v)} own cells; {sum(v == 'INCONCLUSIVE' for v in own_v)} INCONCLUSIVE, {sum(v == 'NOT_WORSE' for v in own_v)} NOT_WORSE)" | |
| else: | |
| line = f"verdict: **{n_beats} own cell(s) BEATS, {n_worse} WORSE** over {len(own_v)} own cells — " + ("the alternative is not a free swap" if n_worse else "the alternative improves the branch") | |
| L += ["", line, ""] | |
| summary[cmp["title"]] = {"verdict_line": line, "own_beats": n_beats, "own_worse": n_worse, "n_own": len(own_v), "cells": verdicts} | |
| L += ["## Reading", "", "The ablation asks whether the survivor recipes' conclusions depend on the sequence encoder family; a comparison is reported as architecture-insensitive when " | |
| "no own cell moves beyond seed noise in either direction. Parameter counts are the trainer's `params_by_module` totals; encoder = `enc` (alternative) vs `gru` / `tcn` (reference).", ""] | |
| Path(a.out).write_text("\n".join(L)) | |
| Path(a.out).with_suffix(".json").write_text(json.dumps({"stamp": stamp, "seeds": seeds, "summary": summary}, indent=1, default=str)) | |
| write_run_json(CWM3 / "results" / "tables", "report_branch_arch", [Path(__file__), CWM3.parent / "cwm2" / "eval" / "kill_checks.py"], [CSV], seed=0, t_start=t0, input_listing=False, | |
| extra={"out": a.out, "seeds": seeds, "summary": {k: {kk: vv for kk, vv in v.items() if kk != "cells"} for k, v in summary.items()}, "dms_fields_used": []}) | |
| print(f"written {a.out}") | |
| for k, v in summary.items(): | |
| print(f" {k}: {v.get('verdict_line', v.get('verdict'))}") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 12.3 kB
- Xet hash:
- 5b426265d6627eb3e285acbcb1350225ecf3d393d2d87519c072169c8e731b6f
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.