Buckets:
| """cwm3/eval/summarize.py — experiment tables from the script-written result files (numbers only from scripts). | |
| Sources: results/eval_grid/metrics.csv (eval_grid rows: drv_onset / veh_onset / d_onset / y_mae / y_crps / future_latent / hazard_nll), | |
| results/exp1_kin_repro.csv (Exp1 comparator vs cwm2 gru_kin_can), results/cwm3_quick_metrics.csv (trainer quick metrics), | |
| cwm2/results/grid/metrics.csv (reference rows: can_only_direct veh_onset, current_signal_S0 d_onset, item grid4_v2). | |
| Writes results/summary_<utc>.md (append-only, timestamped) and prints it. | |
| python summarize.py [--items cwm3_exp1,cwm3_exp2,cwm3_exp3] | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import csv | |
| import re | |
| import time | |
| from collections import defaultdict | |
| from pathlib import Path | |
| import numpy as np | |
| CWM3 = Path(__file__).resolve().parents[1] | |
| CWM2 = CWM3.parent / "cwm2" | |
| EG = CWM3 / "results" / "eval_grid" / "metrics.csv" | |
| EXP1 = CWM3 / "results" / "exp1_kin_repro.csv" | |
| QUICK = CWM3 / "results" / "cwm3_quick_metrics.csv" | |
| REF = CWM2 / "results" / "grid" / "metrics.csv" | |
| def rows_of(p): | |
| return list(csv.DictReader(open(p))) if p.exists() else [] | |
| def f(x, nd=3): | |
| try: | |
| return f"{float(x):.{nd}f}" | |
| except (TypeError, ValueError): | |
| return "" if x in ("", None) else str(x) | |
| def cell(r): | |
| ci = f" [{f(r['ci_lo'])}, {f(r['ci_hi'])}]" if r.get("ci_lo") else "" | |
| return f"{f(r['value'])}{ci}" | |
| def table_exp1(): | |
| out = ["### Exp1 — U0 vs cwm2 gru_kin_can (onset PR-AUC, kin_predictability populations, cluster-bootstrap 95 % CI, n_boot 300)", "", | |
| "| variant | seed | regime | event | tau | cwm3 U0 | ref gru_kin_can mean ± sd (3 seeds) | Δ | within ±0.02 | n / n_pos |", "|---|---|---|---|---|---|---|---|---|---|"] | |
| rows = [r for r in rows_of(EXP1) if "_max" not in r["run_dir"] and "_synthetic" not in r["run_dir"]] # smoke runs excluded | |
| latest = {} | |
| for r in rows: | |
| latest[(r["variant"], r["seed"], r["regime"], r["event"], r["tau"])] = r # last write wins (append-only file) | |
| for k in sorted(latest, key=lambda k: (k[0], int(k[1]), k[2], k[3], k[4])): | |
| r = latest[k] | |
| out.append(f"| {r['variant']} | {r['seed']} | {r['regime']} | {r['event']} | {r['tau']} | {cell(r)} | {f(r['ref_mean'])} ± {f(r['ref_sd'])} | " | |
| f"{float(r['delta_vs_ref_mean']):+.3f} | {r['within_0.02']} | {r['n']} / {r['n_pos']} |") | |
| return out | |
| def _tag(r): | |
| return r["model_id"].split(":")[0] if r.get("model_id") and r["arm"].startswith("cwm3") else (r["model_id"].split(":")[0] if r["arm"] == "can_only_direct" else r["arm"]) | |
| def table_grid(item, endpoint, title, ref_arms=(), taus=("3.0",)): | |
| """one table per regime; rows keyed by (run tag, seed) so several cwm3 variants of one arm never collide; latest row wins.""" | |
| rows = [r for r in rows_of(EG) if r["item"] == item and r["endpoint"] == endpoint and r["comparator"] == "" and r["metric"] in ("pr_auc", "mean") | |
| and (endpoint not in ("drv_onset", "veh_onset", "d_onset") or r["tau_s"] in taus)] | |
| out = [f"### {title} (item {item}, eval_grid rows, device-cluster bootstrap 95 % CI)", ""] | |
| if not rows: | |
| return out + ["(no rows yet)"] | |
| for regime in ("manual", "engaged"): | |
| rr = [r for r in rows if r["regime"] == regime] | |
| if not rr: | |
| continue | |
| evs = sorted({r["event_or_family"] for r in rr}) | |
| idx = {} | |
| for r in rr: | |
| idx[(_tag(r), r["seed"], r["event_or_family"])] = r | |
| keys = sorted({(t, sd) for t, sd, _ in idx}, key=lambda k: (not k[0].startswith(("U0", "V0", "E0")), k)) | |
| out += [f"**{regime}**", "", "| run / arm | seed | " + " | ".join(evs) + " | status |", "|---|---|" + "---|" * len(evs) + "---|"] | |
| for t, sd in keys: | |
| cells, status = [], set() | |
| for ev in evs: | |
| r = idx.get((t, sd, ev)) | |
| cells.append(cell(r) if r else "") | |
| if r: | |
| status.add(r["status"]) | |
| out.append(f"| {t} | {sd} | " + " | ".join(cells) + f" | {','.join(sorted(status))} |") | |
| pops = {} | |
| for (t, sd, ev), r in idx.items(): | |
| pops[ev] = (r["n"], r["n_pos"]) | |
| out += ["", "n / n_pos per cell: " + "; ".join(f"{ev}: {n}/{p}" for ev, (n, p) in sorted(pops.items())), ""] | |
| if ref_arms: | |
| ref = [r for r in rows_of(REF) if r["item"] == "grid4_v2" and r["endpoint"] == endpoint and r["comparator"] == "" and r["metric"] == "pr_auc" | |
| and r["arm"] in ref_arms and r["tau_s"] in taus] | |
| if ref: | |
| out += [f"reference rows (cwm2 grid4_v2, {', '.join(ref_arms)}; other row subset):"] | |
| agg = defaultdict(list) | |
| for r in ref: | |
| agg[(r["regime"], r["arm"], r["event_or_family"])].append(float(r["value"])) | |
| for (reg, arm, ev), v in sorted(agg.items()): | |
| out.append(f"- {reg} {arm} {ev}: {np.mean(v):.3f}" + (f" ± {np.std(v):.3f} (seeds {len(v)})" if len(v) > 1 else "")) | |
| return out | |
| def table_paired(): | |
| rows = rows_of(CWM3 / "results" / "paired_compare.csv") | |
| out = ["### Paired comparisons on identical rows (eval/paired_compare.py, shared-resample device-cluster bootstrap)", ""] | |
| if not rows: | |
| return out + ["(none)"] | |
| latest = {} | |
| for r in rows: | |
| latest[(r["tag"].split("_donset")[0], r["regime"], r["a"], r["b"], r["cell"])] = r | |
| out += ["| tag | regime | A | B | cell | A | B | A − B [95 % CI] | P(Δ>0) | n |", "|---|---|---|---|---|---|---|---|---|---|"] | |
| for k in sorted(latest): | |
| r = latest[k] | |
| out.append(f"| {k[0]} | {r['regime']} | {r['a']} | {r['b']} | {r['cell'].replace('|', ' / ')} | {f(r['value_a'], 4)} | {f(r['value_b'], 4)} | " | |
| f"{float(r['delta']):+.4f} [{float(r['delta_ci_lo']):+.4f}, {float(r['delta_ci_hi']):+.4f}] | {f(r['frac_boot_d_gt0'], 2)} | {r['n']} |") | |
| return out | |
| def table_quick(): | |
| rows = rows_of(QUICK) | |
| out = ["### Trainer quick metrics (eval_s13.transition_metrics on all predicted test rows; no CI)", ""] | |
| if not rows: | |
| return out + ["(none)"] | |
| keep = [r for r in rows if r["split"] == "test" and r["metric"] == "onset_prauc@3s"] | |
| latest = {} | |
| for r in keep: | |
| latest[(r["stage"], r["variant"], r["seed"], r["regime"], r["chain"])] = r | |
| out.append("| stage | variant | seed | regime | chain | PR-AUC@3s | base rate | n | best epoch | val objective |") | |
| out.append("|---|---|---|---|---|---|---|---|---|---|") | |
| for k in sorted(latest, key=lambda k: (k[0], k[1], int(k[2]), k[3], k[4])): | |
| r = latest[k] | |
| out.append(f"| {r['stage']} | {r['variant']} | {r['seed']} | {r['regime']} | {r['chain']} | {f(r['value'])} | {f(r['base_rate'])} | {r['n']} | " | |
| f"{r['best_epoch']} | {f(r['val_objective'], 4)} |") | |
| extra = [r for r in rows if r["split"] == "test" and r["metric"] == "mean" and r["chain"] in ("y_mae", "y_crps", "f_cos", "k_mae")] | |
| if extra: | |
| out += ["", "| stage | variant | seed | regime | quantity | mean (per channel / horizon) |", "|---|---|---|---|---|---|"] | |
| lat = {} | |
| for r in extra: | |
| lat[(r["stage"], r["variant"], r["seed"], r["regime"], r["chain"])] = r | |
| for k in sorted(lat, key=lambda k: (k[0], k[1], int(k[2]), k[3], k[4])): | |
| r = lat[k] | |
| out.append(f"| {r['stage']} | {r['variant']} | {r['seed']} | {r['regime']} | {r['chain']} | {r['value']} |") | |
| return out | |
| def _agg(vals): | |
| v = np.asarray(vals, float) | |
| return f"{v.mean():.3f} ± {v.std():.3f} (n={len(v)})" if len(v) > 1 else (f"{v[0]:.3f} (n=1)" if len(v) else "") | |
| def table_seed_aggregates(): | |
| out = ["### 3-seed aggregates (mean ± sd over seeds; every number from the script-written CSVs)", ""] | |
| # Exp1: kin_repro | |
| rows = [r for r in rows_of(EXP1) if "_max" not in r["run_dir"] and "_synthetic" not in r["run_dir"] and r["tau"] == "3s"] | |
| latest = {} | |
| for r in rows: | |
| latest[(r["variant"], r["seed"], r["regime"], r["event"])] = r | |
| agg = defaultdict(list) | |
| ref = {} | |
| for (v, sd, reg, ev), r in latest.items(): | |
| agg[(v, reg, ev)].append(float(r["value"])) | |
| ref[(reg, ev)] = (float(r["ref_mean"]), float(r["ref_sd"])) | |
| if agg: | |
| out += ["**Exp1 — U0 onset PR-AUC@3s vs cwm2 gru_kin_can (kin_predictability populations)**", "", | |
| "| variant | regime | event | cwm3 mean ± sd | ref mean ± sd | Δ of means | seeds within ±0.02 |", "|---|---|---|---|---|---|---|"] | |
| for (v, reg, ev) in sorted(agg): | |
| vals = agg[(v, reg, ev)] | |
| rm, rs = ref[(reg, ev)] | |
| within = sum(abs(x - rm) <= 0.02 for x in vals) | |
| out.append(f"| {v} | {reg} | {ev} | {_agg(vals)} | {rm:.3f} ± {rs:.3f} | {np.mean(vals) - rm:+.3f} | {within}/{len(vals)} |") | |
| out.append("") | |
| # eval_grid aggregates per (item, endpoint, regime, run-variant, event) | |
| eg = rows_of(EG) | |
| for item, endpoint, title in (("cwm3_exp1", "drv_onset", "Exp1 — drv_onset PR-AUC@3s (eval_grid)"), ("cwm3_exp2", "veh_onset", "Exp2 — veh_onset PR-AUC@3s (eval_grid, can_only_direct row subset)"), | |
| ("cwm3_exp3", "d_onset", "Exp3 — d_onset PR-AUC@3s (eval_grid, can_only_direct row subset)"), ("cwm3_exp3", "future_latent", "Exp3 — F cosine (eval_grid)")): | |
| rr = [r for r in eg if r["item"] == item and r["endpoint"] == endpoint and r["comparator"] == "" and r["metric"] in ("pr_auc", "mean") | |
| and (endpoint == "future_latent" or r["tau_s"] == "3.0")] | |
| if not rr: | |
| continue | |
| latest = {} | |
| for r in rr: | |
| latest[(r["regime"], _tag(r), r["seed"], r["event_or_family"])] = r | |
| agg = defaultdict(list) | |
| for (reg, tag, sd, ev), r in latest.items(): | |
| var = tag.rsplit("_s", 1)[0] if tag.startswith(("U0", "V0", "E0", "can_only_direct")) else tag | |
| if var.startswith("can_only_direct"): | |
| var = "can_only_direct" | |
| if r["value"] != "": | |
| agg[(reg, var, ev)].append(float(r["value"])) | |
| evs = sorted({k[2] for k in agg}) | |
| variants = sorted({(k[0], k[1]) for k in agg}, key=lambda k: (k[0], not k[1].startswith(("U0", "V0", "E0")), k[1])) | |
| out += [f"**{title}**", "", "| regime | variant | " + " | ".join(evs) + " |", "|---|---|" + "---|" * len(evs)] | |
| for reg, var in variants: | |
| out.append(f"| {reg} | {var} | " + " | ".join(_agg(agg.get((reg, var, ev), [])) for ev in evs) + " |") | |
| out.append("") | |
| # paired verdicts per tag family across seeds | |
| pc = rows_of(CWM3 / "results" / "paired_compare.csv") | |
| if pc: | |
| latest = {} | |
| for r in pc: | |
| latest[(r["tag"], r["regime"], r["a"], r["b"], r["cell"])] = r | |
| fam = defaultdict(list) | |
| for (tag, reg, a_, b_, cell), r in latest.items(): | |
| base = re.sub(r"_s\d+", "", tag).replace("_donset", "") # exp3_s0_donset / exp3_s1 -> exp3 ; exp1_s0_noK -> exp1_noK | |
| key = (base, reg, a_.rsplit("_s", 1)[0] + " − " + b_.rsplit("_s", 1)[0], cell) | |
| fam[key].append((float(r["delta"]), float(r["delta_ci_lo"]), float(r["delta_ci_hi"]))) | |
| out += ["**Paired deltas across seeds (A − B on identical rows; kill rule: ≥ 2/3 seeds same sign with CI excluding 0)**", "", | |
| "| comparison | regime | cell | per-seed Δ [CI] | seeds CI excl. 0, same sign | verdict |", "|---|---|---|---|---|---|"] | |
| for key in sorted(fam): | |
| ds = fam[key] | |
| pos = sum(1 for d, lo, hi in ds if lo > 0) | |
| neg = sum(1 for d, lo, hi in ds if hi < 0) | |
| verdict = ("A > B" if pos >= 2 and pos >= len(ds) * 2 / 3 else ("A < B" if neg >= 2 and neg >= len(ds) * 2 / 3 else "no call")) | |
| if len(ds) < 2: | |
| verdict = "1 seed" | |
| out.append(f"| {key[2]} | {key[1]} | {key[3].replace('|', ' / ')} | " + "; ".join(f"{d:+.3f} [{lo:+.3f}, {hi:+.3f}]" for d, lo, hi in ds) + f" | +{pos} / −{neg} of {len(ds)} | {verdict} |") | |
| out.append("") | |
| return out | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--items", default="cwm3_exp1,cwm3_exp2,cwm3_exp3") | |
| a = ap.parse_args() | |
| items = a.items.split(",") | |
| lines = [f"# cwm3 experiment summary — {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}", ""] | |
| lines += table_exp1() + [""] | |
| if "cwm3_exp1" in items: | |
| lines += table_grid("cwm3_exp1", "drv_onset", "Exp1 — driver-event onset PR-AUC@3s (eval_grid drv_onset, with persistence / k0 baselines)") + [""] | |
| lines += table_grid("cwm3_exp1", "hazard_nll", "Exp1 — hazard NLL (mean over inactive steps up to the first onset)") + [""] | |
| if "cwm3_exp2" in items: | |
| lines += table_grid("cwm3_exp2", "veh_onset", "Exp2 — vehicle onset PR-AUC@3s (gas_on / brake_on / steer_phase_on)", ref_arms=("can_only_direct",)) + [""] | |
| lines += table_grid("cwm3_exp2", "y_mae", "Exp2 — y_mae (per-row mean absolute error of the MDN mean)") + [""] | |
| lines += table_grid("cwm3_exp2", "y_crps", "Exp2 — y_crps (32-sample energy form)") + [""] | |
| if "cwm3_exp3" in items: | |
| lines += table_grid("cwm3_exp3", "d_onset", "Exp3 — demand-family onset PR-AUC@3s vs current_signal_S0", ref_arms=("current_signal_S0", "can_only_direct")) + [""] | |
| lines += table_grid("cwm3_exp3", "future_latent", "Exp3 — F cosine (+1..+5 s)") + [""] | |
| lines += table_paired() + [""] | |
| lines += table_seed_aggregates() + [""] | |
| lines += table_quick() | |
| text = "\n".join(lines) | |
| stamp = time.strftime('%Y%m%dT%H%M%SZ', time.gmtime()) | |
| out = CWM3 / "results" / f"summary_{stamp}.md" | |
| out.write_text(text) | |
| (CWM3 / "results" / "tables").mkdir(exist_ok=True) | |
| (CWM3 / "results" / "tables" / f"exp1_exp2_exp3_{stamp}.md").write_text(text) | |
| print(text) | |
| print(f"\nwritten {out}") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 14 kB
- Xet hash:
- 175aab6984e0dbaeb6d1417a92ac2df03c239f6d4842bb95dd3afbf636f100ce
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.