Buckets:
| """cwm3/eval/report_exp3_v2.py — Exp3 on pack v2 (E inputs / F targets on the frame/20 axis): pooled vs tok8s vs spat256 (3 seeds) and pooled v2 vs v1. | |
| Inputs (script outputs only): results/paired_compare.csv tags exp3_v2_s<k>_{spat256_vs_pooled,tok8s_vs_pooled,spat256_vs_tok8s,pooledv2_vs_pooledv1} | |
| (eval/paired_compare.py: d_onset PR-AUC@3s per family, f_cos per horizon, identical rows, device-cluster bootstrap n_boot 1000) and | |
| results/eval_grid/metrics.csv item cwm3_exp3_v2 (unchanged cwm2 eval_grid: d_onset vs current_signal_S0 / can_only_direct on the same rows, | |
| future_latent cosine). Verdicts per cell: kill_checks.beats (>= 2/3 seeds CI excl. 0, same sign, |mean| > seed sd). Rule for the v2 E base: | |
| prefer the variant that BEATS pooled on >= 1 demand family or F horizon with no WORSE cell; otherwise pooled (the simplest). | |
| python report_exp3_v2.py [--out results/tables/exp3_v2.md] | |
| Exp3-v2b (2026-09-11): --prefix exp3_v2b_ --pairs regionpca256:pooled pooledlfoff:pooledv2 --item cwm3_exp3_v2b --descriptive f_cos --out results/tables/exp3_v2b.md | |
| (--descriptive: cell prefixes excluded from the E-base decision rule and reported as descriptive — F cosine under select_LF false) | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import sys | |
| from pathlib import Path | |
| import numpy as np | |
| import pandas as pd | |
| CWM3 = Path(__file__).resolve().parents[1] | |
| CWM2 = CWM3.parent / "cwm2" | |
| sys.path.insert(0, str(CWM2 / "eval")) | |
| import kill_checks as KC # noqa: E402 | |
| PC = CWM3 / "results" / "paired_compare.csv" | |
| MET = CWM3 / "results" / "eval_grid" / "metrics.csv" | |
| PAIRS = [("spat256", "pooled"), ("tok8s", "pooled"), ("spat256", "tok8s"), ("pooledv2", "pooledv1")] | |
| FAMS = ("steer_rate", "decel", "ttc", "ttc_drop", "lane", "badas") | |
| def verdict(g, lower_better=False): | |
| pr = pd.DataFrame({"seed": g.seed, "delta": g.delta.astype(float) * (-1 if lower_better else 1), | |
| "delta_ci_lo": (g.delta_ci_lo if not lower_better else -g.delta_ci_hi).astype(float), | |
| "delta_ci_hi": (g.delta_ci_hi if not lower_better else -g.delta_ci_lo).astype(float), "status": "secondary"}) | |
| return KC.beats(pr, pd.DataFrame({"value": g.value_a.astype(float)}), pd.DataFrame({"value": g.value_b.astype(float)}), min_seeds=3) | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--out", default=str(CWM3 / "results" / "tables" / "exp3_v2.md")) | |
| ap.add_argument("--prefix", default="exp3_v2_", help="paired_compare tag prefix (tags <prefix>s<k>_<a>_vs_<b>)") | |
| ap.add_argument("--pairs", nargs="*", default=[f"{a_}:{b_}" for a_, b_ in PAIRS], help="a:b pairs to render (first b is the base)") | |
| ap.add_argument("--item", default="cwm3_exp3_v2", help="eval_grid item for the absolute-levels table") | |
| ap.add_argument("--base", default="pooled", help="the incumbent E base the candidates are judged against") | |
| ap.add_argument("--descriptive", nargs="*", default=[], help="cell prefixes reported but excluded from the decision rule (e.g. f_cos)") | |
| a = ap.parse_args() | |
| pairs = [tuple(x.split(":")) for x in a.pairs] | |
| df = pd.read_csv(PC) | |
| df = df[df.tag.str.startswith(a.prefix)].copy() | |
| df["seed"] = df.tag.str.extract(r"_s(\d+)_")[0].astype(int) | |
| df["pair"] = df.tag.str.replace("^" + a.prefix + r"s\d+_", "", regex=True) | |
| df = df.sort_values("run_stamp").groupby(["tag", "regime", "cell", "metric"], as_index=False).tail(1) | |
| v2b = a.prefix != "exp3_v2_" | |
| lines = (["# Exp3 on pack v2 — E0 pooled vs tok8s vs spat256 (3 seeds) and pooled v2 vs v1", "", | |
| "E inputs (vf256) and F targets rebuilt on the frame/20 time axis (results/pack_v2); the region token bank is unchanged (exactly causal). "] | |
| if not v2b else | |
| ["# Exp3-v2b — E0 `regionpca256` vs E0 `pooled` under the new recipe (select_LF false, pack v2, stats v2; 3 seeds)", "", | |
| "`regionpca256` (`E0_regionpca256_s<k>_rp256_v2`): the `pooled` architecture (16 pooled256 bank tokens → the same 2-layer causal Transformer) with the per-anchor " | |
| "side vector vf256 [256] REPLACED by the regionpca256 road stream [512] = PCA-256 of the flattened 16 × 1408 region grid of the last window ⊕ its 8 s mean " | |
| "(results/pack_v2/regionpca256_*, frame/20 axis), through the same LN → Linear(512→128) → GELU path gated by `ok`; nothing tiled or split. " | |
| "`pooled` here = `E0_pooled_s<k>_v2s2` (variant `pooled_lfoff`: identical to pooled except the selection objective = Σ chain NLL without 0.1·L_F). " | |
| "Both: pack v2, `stats_version 2` (float64 standardisation statistics). `pooledlfoff_vs_pooledv2` = the recipe/stats change alone (vs the exp3_v2 pooled runs, " | |
| "stats v1, select_LF true). The E0 branch emits the six demand (D) chains and F only — no road-pressure (P) chains, so no P cells exist. " | |
| "F cosine is DESCRIPTIVE only (P2 verdict: it mostly measures the present's similarity to the future) and is excluded from the decision rule. "]) | |
| lines += ["Paired device-cluster bootstrap (eval/paired_compare.py, n_boot 1000) on identical rows; verdict = kill_checks.beats. Source: `results/paired_compare.csv` " | |
| f"tags `{a.prefix}*`; absolute levels vs current_signal_S0 / can_only_direct from `results/eval_grid/metrics.csv` item `{a.item}`.", ""] | |
| summary = {} | |
| for an, bn in pairs: | |
| d = df[df.pair == f"{an}_vs_{bn}"] | |
| if not len(d): | |
| lines += [f"### {an} vs {bn}: no rows yet", ""] | |
| continue | |
| seeds = sorted(d.seed.unique()) | |
| lines += [f"### A = {an}, B = {bn} (Δ = A − B)", "", "| regime | cell | metric | " + " | ".join(f"A s{s}" for s in seeds) + " | " + " | ".join(f"Δ s{s} [CI]" for s in seeds) + " | mean Δ ± sd | verdict |", | |
| "|---|---|---|" + "---|" * (2 * len(seeds)) + "---|---|"] | |
| for regime in ("manual", "engaged"): | |
| for cell in [f"d_onset|{f}|3s" for f in FAMS] + [f"f_cos|+{i}s" for i in range(1, 6)]: | |
| g = d[(d.regime == regime) & (d.cell == cell)].sort_values("seed") | |
| if not len(g): | |
| continue | |
| lb = False | |
| res = verdict(g, lb) | |
| by = {int(s): r for s, r in zip(g.seed, g.itertuples())} | |
| A = [f"{by[s].value_a:.4f}" if s in by else "—" for s in seeds] | |
| D = [f"{by[s].delta:+.4f} [{by[s].delta_ci_lo:+.4f},{by[s].delta_ci_hi:+.4f}]" if s in by else "—" for s in seeds] | |
| dl = g.delta.astype(float).to_numpy() | |
| lines.append(f"| {regime} | {cell.replace('|', '/')} | {g.metric.iloc[0]} | " + " | ".join(A) + " | " + " | ".join(D) + | |
| f" | {dl.mean():+.4f} ± {dl.std(ddof=1) if len(dl) > 1 else 0.0:.4f} | {res['verdict']} |") | |
| summary[(an, bn, regime, cell)] = res["verdict"] | |
| lines.append("") | |
| # ---- absolute levels from eval_grid (item cwm3_exp3_v2): mean over seeds per arm / family / regime | |
| if MET.exists(): | |
| m = pd.read_csv(MET, low_memory=False) | |
| m = m[(m["item"] == a.item) & (m.comparator.isna()) & (m.metric.isin(["pr_auc", "cos"]))] | |
| if len(m): | |
| m["value"] = pd.to_numeric(m.value, errors="coerce") | |
| m["arm_id"] = m.model_id.fillna(m.arm).astype(str).str.replace(r"_s\d.*$", "", regex=True).str.replace(r":.*$", "", regex=True) | |
| arms = sorted(m.arm_id.unique()) | |
| lines += [f"### Absolute levels (eval_grid item {a.item}; mean over seeds; d_onset PR-AUC@3s on the can_only_direct row subset, F cosine)", "", | |
| "| regime | endpoint | event | " + " | ".join(arms) + " |"] | |
| lines.append("|---|---|---|" + "---|" * len(arms)) | |
| for (regime, ep, ev), g in m.groupby(["regime", "endpoint", "event_or_family"]): | |
| vals = g.groupby("arm_id").value.mean() | |
| lines.append(f"| {regime} | {ep} | {ev} | " + " | ".join(f"{vals[x]:.3f}" if x in vals else "—" for x in arms) + " |") | |
| lines.append("") | |
| # ---- E base decision | |
| def wins(an, bn, decision_only=True): | |
| v = {k: x for k, x in summary.items() if k[0] == an and k[1] == bn and (not decision_only or not k[3].startswith(tuple(a.descriptive)))} | |
| return [k for k, x in v.items() if x == "BEATS"], [k for k, x in v.items() if x == "WORSE"] | |
| lines += ["## Decision rule for the v2 E base", ""] | |
| if a.descriptive: | |
| lines.append(f"- descriptive cells (excluded from the rule): prefixes {a.descriptive}") | |
| choice = a.base | |
| for cand, bn in pairs: | |
| if bn != a.base: | |
| continue | |
| b, w = wins(cand, a.base) | |
| lines.append(f"- {cand} vs {a.base}: BEATS on {[f'{k[2]}/{k[3]}' for k in b] or 'none'}; WORSE on {[f'{k[2]}/{k[3]}' for k in w] or 'none'}") | |
| if a.descriptive: | |
| bd, wd = wins(cand, a.base, decision_only=False) | |
| desc = [k for k in bd + wd if k[3].startswith(tuple(a.descriptive))] | |
| lines.append(f" - descriptive: " + (", ".join(f"{k[2]}/{k[3]} {summary[k]}" for k in desc) or "no BEATS / WORSE cell")) | |
| if b and not w and choice == a.base: | |
| choice = cand | |
| for cand, bn in pairs: | |
| if bn == a.base: | |
| continue | |
| b, w = wins(cand, bn, decision_only=False) | |
| lines.append(f"- {cand} vs {bn} (identical rows; informational): BEATS on {[f'{k[2]}/{k[3]}' for k in b] or 'none'}; WORSE on {[f'{k[2]}/{k[3]}' for k in w] or 'none'}") | |
| suffix = "_v2s2" if (v2b and choice == a.base) else ("_rp256_v2" if v2b else "_v2") | |
| lines += ["", f"**{'Exp3-v2b' if v2b else 'v2'} E base for I1: `E0_{choice}_s<k>{suffix}`** (rule: a variant that beats {a.base} on ≥ 1 cell with no WORSE cell replaces {a.base}; otherwise {a.base}).", ""] | |
| Path(a.out).write_text("\n".join(lines) + "\n") | |
| print("\n".join(lines[-8:])) | |
| print(f"written {a.out}; choice={choice}") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 10 kB
- Xet hash:
- 9b8d3cd34efaa108a88b45702274da520653c89ef2965466a815e14a615f8687
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.