Buckets:
| """cwm3/aide/eval/pooled_paired.py — pooled paired estimate for the two main-text AIDE comparisons (owner-approved item 6, 2026-09-15). | |
| For each comparison the per-clip paired differences (A − B on identical test clips) are computed per seed-matched seed, AVERAGED OVER | |
| SEEDS FIRST (one pooled per-clip difference per clip), and the mean of that pooled per-clip difference is clip-bootstrapped (10,000 | |
| resamples, seed 0, percentile 2.5 / 97.5) → one pooled Δ [CI] per cell. The per-seed rule of arch_attribution.md / baton_pretrain.md | |
| (CI excl. 0 on ≥ 2/3 or ≥ 4/5 seeds, same sign) is NOT replaced: it stays the stability description; the pooled estimate is an | |
| additional, single-interval summary of the same clips. | |
| Comparisons (run names as in arch_attribution.md §2 / kv2_program.md §4): | |
| (1) Driver-WM architecture on CockpitWM tokens − CockpitWM, seed-matched seeds 42 / 123 / 456 | |
| (driverwm_kp_s<k>_v2 − cwm3d_kp_s<k>_v2) | |
| (2) CockpitWM + BATON pretraining − CockpitWM, seeds 42 / 123 / 456 / 789 / 1011 | |
| (cwm3d_kp_batonpre_trunkp123_s<k>_v2 − cwm3d_kp_s<k>_v2) | |
| Reads ONLY pred_test.npz of the existing runs (no retraining, no selection); ground truth via aide.eval.harness.load_split("test"); | |
| cells as report_arch_attribution.per_clip_cells (All-MPJPE, d-n, HM-MPJPE, PCK@0.05, HM h=5). Also prints the per-seed means (from the | |
| same per-clip values) so that the pooled Δ can be read next to the per-seed rule. | |
| python aide/eval/pooled_paired.py [--n-boot 10000] [--out aide/results/tables/arch_attribution.md] | |
| Output policy: APPENDS one dated section "## Pooled paired estimate, <date>" to --out (append-only; nothing above is edited) and writes | |
| aide/results/tables/pooled_paired.json. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import sys | |
| import time | |
| from pathlib import Path | |
| import numpy as np | |
| ROOT = Path(__file__).resolve().parents[2] | |
| sys.path.insert(0, str(ROOT)) | |
| from aide.data import protocol as P # noqa: E402 | |
| from aide.eval import metrics as M # noqa: E402 | |
| from aide.eval.harness import align, load_split # noqa: E402 | |
| from aide.eval.report_arch_attribution import load_pred, per_clip_cells # noqa: E402 | |
| OUT = P.RESULTS / "tables" / "arch_attribution.md" | |
| JSON_OUT = P.RESULTS / "tables" / "pooled_paired.json" | |
| CELLS = ("All-MPJPE", "d-n", "HM-MPJPE", "PCK@0.05", "HM h=5") | |
| COMPARISONS = [ | |
| ("Driver-WM architecture on CockpitWM tokens − CockpitWM", "driverwm_kp_s{k}_v2", "cwm3d_kp_s{k}_v2", (42, 123, 456)), | |
| ("CockpitWM + BATON pretraining − CockpitWM", "cwm3d_kp_batonpre_trunkp123_s{k}_v2", "cwm3d_kp_s{k}_v2", (42, 123, 456, 789, 1011)), | |
| ] | |
| def fmt(x: float, nd: int) -> str: | |
| return f"{x:+.{nd}f}" | |
| def main() -> int: | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--n-boot", type=int, default=10000) | |
| ap.add_argument("--seed", type=int, default=0) | |
| ap.add_argument("--out", type=Path, default=OUT) | |
| a = ap.parse_args() | |
| d = load_split("test") | |
| gt = d["S10"][:, P.T_OBS:P.T_OBS + P.T_PRED] | |
| hm, thr, _ = M.hm_mask(gt) | |
| stamp = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime()) | |
| n = len(gt) | |
| _idx: dict[int, np.ndarray] = {} | |
| def resample(m: int) -> np.ndarray: | |
| """One resample matrix per clip count (609 for All / d-n / PCK, 60 for the HM cells), drawn once from default_rng(seed) | |
| and shared by every cell of that size and every comparison (paired over clips).""" | |
| if m not in _idx: | |
| _idx[m] = np.random.default_rng(a.seed).integers(0, m, size=(a.n_boot, m)) | |
| return _idx[m] | |
| res = {"generated": stamp, "n_boot": a.n_boot, "seed": a.seed, "n_clips": int(n), "hm_threshold_px": float(thr), "comparisons": []} | |
| L = [f"## Pooled paired estimate, {time.strftime('%Y-%m-%d', time.gmtime())} — run pooled_paired_{stamp}", "", | |
| f"Generated {stamp} by `aide/eval/pooled_paired.py` (owner-approved item 6, 2026-09-15) from `pred_test.npz` of the existing runs only " | |
| f"(no retraining; ground truth `harness.load_split('test')`, n = {n}; HM = top 60 clips, threshold {thr:.2f} px; cells as §2 of this file). " | |
| f"Pooled Δ = the per-clip paired differences A − B are computed per seed-matched seed, **averaged over the seeds first** (one pooled per-clip " | |
| f"difference per clip) and the mean of that pooled per-clip difference is clip-bootstrapped ({a.n_boot:,} resamples, `default_rng({a.seed})`, " | |
| f"percentile 2.5 / 97.5; one shared resample matrix per clip count — 609 for All / d-n / PCK, 60 for the HM cells). Negative = A better for errors, positive = A better for PCK. " | |
| f"The per-seed rule of §2 / §3 (CI excl. 0 on ≥ 2/3 or ≥ 4/5 seeds, same sign) is unchanged and remains the stability description; " | |
| f"the pooled estimate is a single-interval summary of the same clips and is not an adoption criterion. 'per-seed mean ± sd' = mean and " | |
| f"sample sd (ddof 1) of the per-seed mean differences from the same per-clip values.", ""] | |
| for label, pa, pb, seeds in COMPARISONS: | |
| per_seed = {} | |
| for k in seeds: | |
| A, B = load_pred(pa.format(k=k), d), load_pred(pb.format(k=k), d) | |
| if A is None or B is None: | |
| raise FileNotFoundError(f"missing pred_test.npz for seed {k}: {pa.format(k=k)} / {pb.format(k=k)}") | |
| ca, cb = per_clip_cells(A, gt, hm), per_clip_cells(B, gt, hm) | |
| per_seed[k] = {c: ca[c][0] - cb[c][0] for c in CELLS} | |
| entry = {"label": label, "a": pa, "b": pb, "seeds": list(seeds), "cells": {}} | |
| L += [f"### {label} (seed-matched seeds {' / '.join(str(s) for s in seeds)}; A = `{pa}`, B = `{pb}`)", "", | |
| "| cell | pooled Δ (mean over seeds of the per-clip Δ, then clip bootstrap) [CI] | per-seed mean Δ | per-seed mean ± sd | n seeds |", | |
| "|---|---|---|---|---|"] | |
| for c in CELLS: | |
| nd = 3 if c == "d-n" else 2 | |
| pooled = np.mean([per_seed[k][c] for k in seeds], axis=0) # [n clips] | |
| mean = float(pooled.mean()) | |
| b = pooled[resample(len(pooled))].mean(1) | |
| lo, hi = float(np.quantile(b, 0.025)), float(np.quantile(b, 0.975)) | |
| ps = [float(per_seed[k][c].mean()) for k in seeds] | |
| sd = float(np.std(ps, ddof=1)) | |
| entry["cells"][c] = {"pooled_mean": mean, "ci_lo": lo, "ci_hi": hi, "per_seed_means": ps, "per_seed_sd": sd, | |
| "ci_excludes_zero": bool(lo > 0 or hi < 0)} | |
| L.append(f"| {c} | {fmt(mean, nd)} [{fmt(lo, nd)}, {fmt(hi, nd)}] | {' / '.join(fmt(v, nd) for v in ps)} | " | |
| f"{fmt(float(np.mean(ps)), nd)} ± {sd:.{nd}f} | {len(seeds)} |") | |
| L.append("") | |
| res["comparisons"].append(entry) | |
| L += ["Provenance: `aide/results/tables/pooled_paired.json` (this run); the per-seed intervals and verdicts stay in §2.1 (Driver-WM architecture " | |
| "on CockpitWM tokens − CockpitWM) of this file and in `kv2_program.md` §4 / `baton_pretrain.md` §6 (CockpitWM + BATON pretraining − CockpitWM, 5 / 3 seeds).", ""] | |
| text = "\n".join(L) | |
| with open(a.out, "a", encoding="utf-8") as f: | |
| f.write("\n" + text + "\n") | |
| JSON_OUT.write_text(json.dumps(res, indent=1), encoding="utf-8") | |
| print(text) | |
| print(f"appended to {a.out}; json {JSON_OUT}") | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |
Xet Storage Details
- Size:
- 7.41 kB
- Xet hash:
- a49270332ba2323d2738730c121528d84f53dadbeb4b4d3592c0fc933615dbc7
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.