Buckets:
| """cwm3/eval/report_latent_aux_c4.py — results/tables/latent_aux_c4.md (+ .json): R2 WP4 batch 2 **C4** low-dimensional multi-horizon latent auxiliary objective on the | |
| ECAN default (pre-registration docs/decisions.md 2026-09-17T19:44:28Z; plan <local-notes>/plans/elegant-dazzling-stallman.md 'C4') from results/paired_compare.csv. Composes | |
| eval/report_joint.py blocks (load / render / verdict / provenance); nothing is recomputed here; every paired number comes from the CSV rows written by | |
| eval/run_joint_evals_c4.sh; parameters / epochs / GPU minutes come from the run dirs' params.json / TRAIN_DONE / history.jsonl and the queue logs. | |
| python eval/report_latent_aux_c4.py [--seeds 0,1,2] [--out results/tables/latent_aux_c4.md] | |
| Arms: lat01 / lat03 = I1_all_minus_road2drv_lat{01,03}_s<k>_final_ecan (lam 0.1 / 0.3); lat0 = the lam = 0 RE-TRAINING control (module not built); frozen ECAN main = | |
| I1_all_minus_road2drv_s<k>_final_ecan. Blocks (tags): finalcv_s<k>_allminusecanlat{01,03,0}_vs_allminusecan (VALIDATION — selection + adoption), finalc_* (TEST — | |
| confirmation), *_vs_allminusecanlat0 (vs the re-training control), finalc_*_vs_k0 (B1), finalP_* (P road cells). | |
| PRE-REGISTERED READING (written before any run finished): lam* = argmax over {0.1, 0.3} of the mean VALIDATION Δ onset PR-AUC@3 s over the four paper driver cells | |
| (manual hand_leave / hand_return / head_turn_onset, engaged head_turn_onset) vs the frozen ECAN main; lam* is an ADOPTION CANDIDATE iff on VALIDATION no own onset cell | |
| (report_joint.OWN, both regimes) is WORSE or breaks the −0.005 margin (mean Δ AND no seed CI entirely beyond it), no hazard-NLL cell breaks +0.01 nats, AND ≥ 1 paper | |
| driver onset cell BEATS; a candidate Δ inside the lam = 0 re-run's own Δ band (min..max over seeds of Δ(lat0 − frozen) on that cell) is attributed to re-training | |
| noise. Otherwise NEGATIVE RESULT. The test block is the confirmation. No manuscript edit here. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import re | |
| import sys | |
| import time | |
| from pathlib import Path | |
| import numpy as np | |
| import pandas as pd | |
| CWM3 = Path(__file__).resolve().parents[1] | |
| sys.path.insert(0, str(CWM3 / "eval")) | |
| import report_joint as RJ # noqa: E402 | |
| RUNS = CWM3 / "results" / "runs" | |
| ARMS = {"lat01": ("allminusecanlat01", 0.1), "lat03": ("allminusecanlat03", 0.3), "lat0": ("allminusecanlat0", 0.0)} | |
| MAIN = "allminusecan" | |
| PAPER_DRV = [("manual", "drv_onset|hand_leave|3s"), ("manual", "drv_onset|hand_return|3s"), ("manual", "drv_onset|head_turn_onset|3s"), ("engaged", "drv_onset|head_turn_onset|3s")] | |
| EV = ("hand_leave", "hand_return", "head_turn_onset") | |
| VEH = ("gas_on", "brake_on", "steer_phase_on") | |
| ONSET_CELLS = [f"veh_onset|{e}|3s" for e in VEH] + [f"drv_onset|{e}|3s" for e in EV] | |
| NLL_CELLS = [f"hazard_nll|{e}" for e in ("gas", "brake", "steer_phase")] + [f"hazard_nll|{e}" for e in EV] | |
| MARGIN_ONSET, MARGIN_NLL = -0.005, 0.010 | |
| def run_name(arm, seed): | |
| return f"I1_all_minus_road2drv_{arm}_s{seed}_final_ecan" if arm != "main" else f"I1_all_minus_road2drv_s{seed}_final_ecan" | |
| def block(stage, a, b, title, seeds): | |
| df = RJ.load(stage, a, b, seeds) | |
| if not len(df): | |
| return df, [f"### {title}: A = {a}, B = {b} — no rows yet (tags `{stage}_s<k>_{a}_vs_{b}`)", ""], {} | |
| body, verdicts = RJ.render(df, sorted(df.seed.unique()), a, b, title) | |
| return df, body + [""], verdicts | |
| def margin_ok(g, lower_better): | |
| d = g.delta.astype(float).to_numpy() | |
| lo, hi = g.delta_ci_lo.astype(float).to_numpy(), g.delta_ci_hi.astype(float).to_numpy() | |
| if lower_better: | |
| return bool(np.nanmean(d) <= MARGIN_NLL and not np.any(lo > MARGIN_NLL)), float(np.nanmean(d)) | |
| return bool(np.nanmean(d) >= MARGIN_ONSET and not np.any(hi < MARGIN_ONSET)), float(np.nanmean(d)) | |
| def margin_table(df, verdicts, title): | |
| lines = [f"### {title}", "", "| regime | cell | metric | A (variant) per seed | B per seed | Δ per seed [CI] | mean Δ ± sd | verdict | within margin |", "|---|---|---|---|---|---|---|---|---|"] | |
| checks = {} | |
| for r in ("manual", "engaged"): | |
| for cell, metric, lb, margin in [(c, "pr_auc", False, MARGIN_ONSET) for c in ONSET_CELLS] + [(c, "nll", True, MARGIN_NLL) for c in NLL_CELLS]: | |
| g = df[(df.regime == r) & (df.cell == cell) & (df.metric == metric)].sort_values("seed") | |
| if not len(g): | |
| lines.append(f"| {r} | {cell.replace('|', '/')} | {metric} | — | — | no rows | | | |") | |
| continue | |
| v = verdicts.get((r, cell, metric), {"verdict": "—"}) | |
| ok, md = margin_ok(g, lb) | |
| d = g.delta.astype(float).to_numpy() | |
| checks[(r, cell, metric)] = {"ok": ok, "mean_delta": md, "sd": float(d.std(ddof=1)) if len(d) > 1 else 0.0, "verdict": v["verdict"], "n_seeds": int(len(d)), | |
| "deltas": [float(x) for x in d], "ci_lo": [float(x) for x in g.delta_ci_lo], "ci_hi": [float(x) for x in g.delta_ci_hi], | |
| "value_a": [float(x) for x in g.value_a], "value_b": [float(x) for x in g.value_b]} | |
| A = " / ".join(f"{float(x):.4f}" for x in g.value_a) | |
| B = " / ".join(f"{float(x):.4f}" for x in g.value_b) | |
| D = " / ".join(f"{float(x.delta):+.4f} [{float(x.delta_ci_lo):+.4f},{float(x.delta_ci_hi):+.4f}]" for x in g.itertuples()) | |
| note = "" if (r == "manual" or cell.startswith("veh") or "head_turn" in cell) else " (not reported in the paper)" | |
| lines.append(f"| {r} | {cell.replace('|', '/')}{note} | {metric}{' (lower better)' if lb else ''} | {A} | {B} | {D} | {md:+.4f} ± {checks[(r, cell, metric)]['sd']:.4f} | {v['verdict']} | " | |
| f"{'yes' if ok else '**NO**'} (margin {margin:+.3f}) |") | |
| return lines + [""], checks | |
| def paper_drv_mean(checks): | |
| vals = [checks[(r, c, "pr_auc")]["mean_delta"] for r, c in PAPER_DRV if (r, c, "pr_auc") in checks] | |
| return (float(np.mean(vals)) if len(vals) == 4 else float("nan")), len(vals) | |
| def acceptance(verdicts, checks, noise): | |
| """pre-registered rule on VALIDATION -> (status, reasons dict)""" | |
| own = {k: v for k, v in verdicts.items() if k[1] in RJ.OWN and k[2] == "pr_auc"} | |
| worse = [f"{r}|{c}" for (r, c, m), v in own.items() if v["verdict"] == "WORSE"] | |
| broke_onset = [f"{r}|{c}" for (r, c, m), ch in checks.items() if m == "pr_auc" and not ch["ok"]] | |
| broke_nll = [f"{r}|{c}" for (r, c, m), ch in checks.items() if m == "nll" and not ch["ok"]] | |
| beats = [f"{r}|{c}" for r, c in PAPER_DRV if verdicts.get((r, c, "pr_auc"), {}).get("verdict") == "BEATS"] | |
| insuff = len(own) < 12 or any(v["verdict"] == "INSUFFICIENT" for v in own.values()) | |
| inside_noise = [] | |
| for k in beats: | |
| r, c = k.split("|", 1) | |
| ch, nb = checks.get((r, c, "pr_auc")), noise.get((r, c)) | |
| if ch and nb and nb[0] <= ch["mean_delta"] <= nb[1]: | |
| inside_noise.append(k) | |
| ok = not worse and not broke_onset and not broke_nll and len([b for b in beats if b not in inside_noise]) >= 1 | |
| status = "INSUFFICIENT (rows missing)" if insuff else ("ADOPTION CANDIDATE (validation)" if ok else "NEGATIVE (validation)") | |
| return status, {"own_worse": worse, "onset_margin_broken": broke_onset, "nll_margin_broken": broke_nll, "paper_driver_beats": beats, "beats_inside_lat0_noise_band": inside_noise} | |
| def run_facts(arm, seeds): | |
| rows = [] | |
| for s in seeds: | |
| d = RUNS / run_name(arm, s) | |
| f = {"run": d.name, "complete": (d / "PRED_DONE").exists() and (d / "TRAIN_DONE").exists()} | |
| if (d / "params.json").exists(): | |
| p = json.loads((d / "params.json").read_text()) | |
| f.update(params_total=p.get("total"), params_new=p.get("new_total"), params_latent=p.get("latent_aux", 0), latent_ema_buffers=p.get("latent_aux_ema_buffers", 0)) | |
| if (d / "TRAIN_DONE").exists(): | |
| td = json.loads((d / "TRAIN_DONE").read_text()) | |
| f.update(best_epoch=td.get("best_epoch"), epochs_run=td.get("epochs_run"), val_objective=td.get("best")) | |
| if (d / "history.jsonl").exists(): | |
| h = [json.loads(l) for l in (d / "history.jsonl").read_text().splitlines() if l.strip()] | |
| best = [x for x in h if x.get("epoch") == f.get("best_epoch")] | |
| if best: | |
| vp = best[0].get("val_parts", {}) | |
| f.update({k: round(float(v), 4) for k, v in vp.items() if k.startswith("lat_") and k != "lat_n"}) | |
| rows.append(f) | |
| return rows | |
| def gpu_minutes(): | |
| """wall seconds of every C4 job from the queue logs ('pass N: end <run>: exit C in S s'); the last record per run wins.""" | |
| out = {} | |
| for lg in sorted(RUNS.glob("r2_c4_*_*.log")): | |
| for line in lg.read_text().splitlines(): | |
| m = re.search(r"end (I1_all_minus_road2drv_lat\S+): exit (\d+) in (\d+) s", line) | |
| if m: | |
| out.setdefault(m.group(1), []).append((int(m.group(2)), int(m.group(3)))) | |
| return out | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--seeds", default="0,1,2") | |
| ap.add_argument("--out", default=str(CWM3 / "results" / "tables" / "latent_aux_c4.md")) | |
| a = ap.parse_args() | |
| seeds = [int(s) for s in a.seeds.split(",")] | |
| stamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) | |
| L = [f"# C4 — task-oriented low-dimensional multi-horizon latent auxiliary objective on the ECAN default (R2 WP4 batch 2; pre-registration docs/decisions.md 2026-09-17T19:44:28Z; rendered {stamp})", "", | |
| "Arms: `lat01` / `lat03` = `I1_all_minus_road2drv_lat{01,03}_s<k>_final_ecan` (λ = 0.1 / 0.3: FINAL ECAN recipe + `latent_aux`, models/latent_aux.py — predictor on the post-interaction " | |
| "trunks → 16-d latent per horizon 1 / 2 / 3 s, EMA target encoder of the 13-d physically / state-supervised future vector, VICReg variance + linear read-back, no SimNorm); " | |
| "`lat0` = λ = 0 RE-TRAINING control (module not built: the ECAN main path bit for bit); frozen ECAN main = `I1_all_minus_road2drv_s<k>_final_ecan`. Paired device-cluster bootstrap on " | |
| "identical rows (eval/paired_compare_joint.py via pc_joint_locked.py, n_boot 1000, v2 causal kin targets; P cells eval/pressure_eval.py); verdict = kill_checks.beats over 3 seeds; " | |
| "acceptable-degradation margins onset PR-AUC −0.005 / hazard NLL +0.01 nats (mean Δ and no seed CI entirely beyond). Every paired number comes from `results/paired_compare.csv`; " | |
| "nothing is recomputed; the manuscript is not edited here. The stop-gradient sits on the target side only — the auxiliary loss updates the trunks (pre-registration).", ""] | |
| J = {"rendered": stamp, "pre_registration": "2026-09-17T19:44:28Z", "arms": {}, "selection": {}, "runs": {}, "gpu": {}} | |
| dfs = [] | |
| # ---- the lam = 0 re-run noise band (validation and test) | |
| noise = {} | |
| for split, stage in (("val", "finalcv"), ("test", "finalc")): | |
| df0 = RJ.load(stage, ARMS["lat0"][0], MAIN, seeds) | |
| nb = {} | |
| for r, c in PAPER_DRV + [(rg, cl) for rg in ("manual", "engaged") for cl in ONSET_CELLS if (rg, cl) not in PAPER_DRV]: | |
| g = df0[(df0.regime == r) & (df0.cell == c) & (df0.metric == "pr_auc")] | |
| if len(g): | |
| d = g.delta.astype(float) | |
| nb[(r, c)] = (float(d.min()), float(d.max())) | |
| noise[split] = nb | |
| # ---- per arm: validation (selection / adoption), test (confirmation) | |
| sel = {} | |
| for arm, (nm, lam) in ARMS.items(): | |
| J["arms"][arm] = {"lam": lam, "a_name": nm} | |
| for split, stage in (("val", "finalcv"), ("test", "finalc")): | |
| df, body, verd = block(stage, nm, MAIN, f"{arm} (λ = {lam}) vs the FROZEN ECAN main — {split.upper()} split, all cells", seeds) | |
| dfs.append(df) | |
| L += [f"## {arm} (λ = {lam}) — {split}", ""] | |
| if len(df): | |
| mt, checks = margin_table(df, verd, f"{arm} vs frozen ECAN main — reported cells with margins ({split})") | |
| L += mt | |
| pm, n4 = paper_drv_mean(checks) | |
| status, why = acceptance(verd, checks, noise.get(split, {})) | |
| L += [f"- mean Δ onset PR-AUC over the four paper driver cells ({n4}/4 present): **{pm:+.4f}**; rule on this split: **{status}**; {json.dumps(why)}", ""] | |
| J["arms"][arm][split] = {"paper_driver_mean_delta": pm, "n_paper_cells": n4, "status": status, "reasons": why, | |
| "cells": {f"{r}|{c}|{m}": ch for (r, c, m), ch in checks.items()}, | |
| "own_verdicts": {f"{r}|{c}|{m}": v["verdict"] for (r, c, m), v in verd.items() if c in RJ.OWN or c.startswith("hazard_nll")}} | |
| if split == "val" and arm != "lat0": | |
| sel[arm] = pm | |
| L += ["<details><summary>all cells</summary>", ""] + body + ["</details>", ""] | |
| else: | |
| L += body | |
| # vs the lam = 0 control | |
| if arm != "lat0": | |
| for split, stage in (("val", "finalcv"), ("test", "finalc")): | |
| df, body, verd = block(stage, nm, ARMS["lat0"][0], f"{arm} vs the λ = 0 re-training control — {split}", seeds) | |
| dfs.append(df) | |
| if len(df): | |
| mt, checks = margin_table(df, verd, f"{arm} vs lat0 (re-training control) — reported cells ({split})") | |
| L += mt | |
| J["arms"][arm][f"vs_lat0_{split}"] = {f"{r}|{c}|{m}": ch for (r, c, m), ch in checks.items()} | |
| else: | |
| L += body | |
| # P cells | |
| dfP, bodyP, verdP = block("finalP", nm, MAIN, f"{arm} vs frozen ECAN main — P road cells (test)", seeds) | |
| dfs.append(dfP) | |
| L += bodyP | |
| if len(dfP): | |
| J["arms"][arm]["P_cells"] = {f"{r}|{c}": v["verdict"] for (r, c, m), v in verdP.items()} | |
| if arm != "lat0": | |
| dfk, bodyk, verdk = block("finalc", nm, "k0", f"{arm} driver cells vs the k0 heuristic (B1 on the variant, test)", seeds) | |
| dfs.append(dfk) | |
| L += bodyk | |
| if len(dfk): | |
| J["arms"][arm]["B1"] = {"verdict": RJ.claim(verdk, "B1")[0], "cells": {f"{r}|{c}": v["verdict"] for (r, c, m), v in verdk.items() if m == "pr_auc"}} | |
| # ---- selection | |
| L.insert(3, "") | |
| if sel: | |
| lam_star = max(sel, key=lambda k: sel[k]) | |
| st_val = J["arms"][lam_star].get("val", {}).get("status", "INSUFFICIENT") | |
| st_test = J["arms"][lam_star].get("test", {}).get("status", "—") | |
| J["selection"] = {"lam_star": lam_star, "validation_paper_driver_mean_delta": sel, "validation_status": st_val, "test_confirmation_status": st_test, | |
| "lat0_noise_band_val": {f"{r}|{c}": v for (r, c), v in noise.get("val", {}).items()}, "lat0_noise_band_test": {f"{r}|{c}": v for (r, c), v in noise.get("test", {}).items()}} | |
| L.insert(4, f"## 0. Selection (VALIDATION) and verdict\n\n- mean validation Δ over the four paper driver cells vs the frozen ECAN main: " + | |
| ", ".join(f"{k} {v:+.4f}" for k, v in sel.items()) + f" → **λ\\* = {lam_star}** (λ = {ARMS[lam_star][1]}).\n" | |
| f"- pre-registered rule on λ\\* (validation): **{st_val}**; test confirmation: **{st_test}**.\n" | |
| f"- λ = 0 re-training noise band (Δ lat0 − frozen, min..max over seeds, validation): " + | |
| ("; ".join(f"{r}/{c.split('|')[1]} [{lo:+.4f}, {hi:+.4f}]" for (r, c), (lo, hi) in noise.get("val", {}).items() if (r, c) in PAPER_DRV) or "no rows") + "\n") | |
| print(f"SELECT lam*={lam_star} val={sel} VERDICT validation={st_val} test={st_test}") | |
| else: | |
| L.insert(4, "## 0. Selection — no validation rows yet\n") | |
| # ---- runs / params / GPU | |
| L += ["## Runs, parameters, GPU minutes", "", "| arm | run | complete | params total | new | latent (trainable) | EMA buffers | best ep / epochs | val objective | lat_pred / lat_rec / lat_var (val @ best) | wall s (exit) |", "|---|---|---|---|---|---|---|---|---|---|---|"] | |
| gm = gpu_minutes() | |
| tot_s = 0 | |
| for arm in ("lat01", "lat03", "lat0"): | |
| for f in run_facts(arm, seeds): | |
| w = gm.get(f["run"], []) | |
| wall = " + ".join(f"{s} ({c})" for c, s in w) if w else "—" | |
| tot_s += sum(s for c, s in w if c == 0) | |
| L.append(f"| {arm} | {f['run']} | {'yes' if f['complete'] else 'NO'} | {f.get('params_total', '—')} | {f.get('params_new', '—')} | {f.get('params_latent', '—')} | {f.get('latent_ema_buffers', '—')} | " | |
| f"{f.get('best_epoch', '—')} / {f.get('epochs_run', '—')} | {f.get('val_objective', float('nan')) if isinstance(f.get('val_objective'), float) else '—'} | " | |
| f"{f.get('lat_pred', '—')} / {f.get('lat_rec', '—')} / {f.get('lat_var', '—')} | {wall} |") | |
| J["runs"][f["run"]] = dict(f, wall=w) | |
| J["gpu"] = {"successful_wall_s_total": tot_s, "successful_wall_min_total": round(tot_s / 60, 1), "note": "wall seconds of exit-0 queue jobs (training + prediction + layouts); failed attempts listed with their exit code"} | |
| L += ["", f"GPU wall of the successful jobs: {tot_s / 60:.1f} min (queue logs results/runs/r2_c4_{{A,B,C}}_*.log; failed attempts shown with their exit code).", ""] | |
| # ---- provenance | |
| dfs = [d for d in dfs if len(d)] | |
| if dfs: | |
| prov, stale = RJ.provenance(pd.concat(dfs)) | |
| L += ["## Provenance (layouts vs rows)", "", "| run | regime | layout written | inputs digest | model / code | status |", "|---|---|---|---|---|---|"] + prov + [""] | |
| J["stale_rows"] = stale | |
| Path(a.out).write_text("\n".join(L) + "\n") | |
| Path(a.out).with_suffix(".json").write_text(json.dumps(J, indent=1, default=str)) | |
| print(f"written {a.out} (+ .json)") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 17.9 kB
- Xet hash:
- ee98ccaf3557bc15c274ff8f6cdf29c99bf80a3dec5a399e2827e2a70b0a4a6f
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.