Buckets:
| """cwm3/aide/eval/report_f7_pretrain_distil.py — F7 lane (docs/decisions.md pre-registration row 2026-09-17, 'F7 AIDE-only full-corpus | |
| pretraining + ensemble distillation') -> aide/results/tables/f7_pretrain_distil.md (+ .json). | |
| Reads ONLY run outputs under aide/results/runs (pred_test.npz -> paired per-clip bootstraps recomputed with aide/eval/metrics.paired_diff, 10k | |
| resamples, seed 0, exactly as pretrain_variants.md / arch_attribution.md; metrics_test.json / metrics_val.json -> levels written once per run by | |
| the harness; run json -> params / GPU-min / init / distil report; history.json of the pretraining runs). Nothing is typed by hand. | |
| Helpers (paired, verdict, delta_table, level_row, ...) are imported from aide/eval/report_arch_attribution.py. | |
| Arms (3 seeds 42/123/456 each; every fine-tune = the adopted recipe verbatim, selection on VAL All-MPJPE only, test evaluated once): | |
| (a) aidepre cwm3d_kp_aidepre_trunkp123_s<k>_v2 trunk init from cwm3d_kp_aidepre_pretrain_s123 (AIDE-only multi-stride) | |
| (a2) aidepre_fb cwm3d_kp_aidepre_fb_trunkp123_s<k>_v2 trunk init from cwm3d_kp_aidepre_fb_pretrain_s123 (BATON ckpt -> AIDE multi-stride) | |
| (b) distil5 cwm3d_kp_distil5_batonpre_trunkp123_s<k>_v2 adopted recipe + BATON trunk + distillation from the 5 adopted seeds | |
| (bc) mb_distil5 motionbert_kp_distil5_batonpre_trunkp123_s<k>_v2 same-recipe MotionBERT architecture, same distillation (its 5 pretrained seeds) | |
| (c) aidepre_fb_distil5 cwm3d_kp_aidepre_fb_distil5_trunkp123_s<k>_v2 (a2) init + distillation | |
| Comparators: adopted = cwm3d_kp_batonpre_trunkp123_s<k>_v2 (paper row), scratch = cwm3d_kp_s<k>_v2, mb_pre = motionbert_kp_batonpre_trunkp123_s<k>_v2. | |
| Reading rule (pre-registered): REPORTABLE iff Δ All-MPJPE vs the adopted arm has CI excl. 0 on >= 2/3 seeds with the same (improving) sign on | |
| every seed (verdict BEATS) AND the mean Δ HM h=5 does not exceed +1.0 px. SOTA criterion (plan): mean All <= 47.5 over 5 seeds AND paired | |
| BEATS vs the same-recipe MotionBERT architecture (here: vs mb_pre and vs mb_distil5); with 3 seeds the All <= 47.5 part is reported as | |
| 'pending the 5-seed extension'. The paper's adopted row changes only by owner decision. | |
| python aide/eval/report_f7_pretrain_distil.py [--n-boot 10000] | |
| Output policy: the .md is APPENDED as a dated block if it exists (append-only tables); the .json is rewritten from run outputs. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import sys | |
| import time | |
| from pathlib import Path | |
| import numpy as np | |
| ROOT = Path(__file__).resolve().parents[2] | |
| sys.path.insert(0, str(ROOT)) | |
| from aide.data import protocol as P # noqa: E402 | |
| from aide.eval import metrics as M # noqa: E402 | |
| from aide.eval.harness import load_split # noqa: E402 | |
| from aide.eval.report_arch_attribution import (LEVEL_COLS, MAIN_CELLS, SEEDS3, TOL_NOT_WORSE, delta_table, level_row, load_pred, metrics_test, # noqa: E402 | |
| ms, paired, per_clip_cells, run_json, run_meta) | |
| RUNS = P.RESULTS / "runs" | |
| OUT = P.RESULTS / "tables" / "f7_pretrain_distil.md" | |
| ADOPTED = "cwm3d_kp_batonpre_trunkp123_s{k}_v2" | |
| SCRATCH = "cwm3d_kp_s{k}_v2" | |
| MB_PRE = "motionbert_kp_batonpre_trunkp123_s{k}_v2" | |
| SEEDS5 = (42, 123, 456, 789, 1011) | |
| PRETRAIN = {"aidepre": "cwm3d_kp_aidepre_pretrain_s123", "aidepre_fb": "cwm3d_kp_aidepre_fb_pretrain_s123"} | |
| ARMS = { # tag -> (label, run template, comparators [(label, template)]) | |
| "aidepre": ("(a) AIDE-only multi-stride pretraining -> trunk init, adopted recipe", "cwm3d_kp_aidepre_trunkp123_s{k}_v2", [("adopted", ADOPTED), ("scratch", SCRATCH), ("mb_pre", MB_PRE)]), | |
| "aidepre_fb": ("(a2) BATON checkpoint -> AIDE multi-stride pretraining -> trunk init", "cwm3d_kp_aidepre_fb_trunkp123_s{k}_v2", [("adopted", ADOPTED), ("scratch", SCRATCH), ("mb_pre", MB_PRE)]), | |
| "distil5": ("(b) ensemble distillation (5 adopted seeds -> single 5.8 M model), adopted recipe", "cwm3d_kp_distil5_batonpre_trunkp123_s{k}_v2", [("adopted", ADOPTED), ("scratch", SCRATCH), ("mb_pre", MB_PRE)]), | |
| "mb_distil5": ("(b-control) MotionBERT architecture, same recipe + same distillation (its 5 pretrained seeds)", "motionbert_kp_distil5_batonpre_trunkp123_s{k}_v2", [("mb_pre", MB_PRE), ("adopted", ADOPTED)]), | |
| "aidepre_fb_distil5": ("(c) (a2) init + ensemble distillation", "cwm3d_kp_aidepre_fb_distil5_trunkp123_s{k}_v2", [("adopted", ADOPTED), ("scratch", SCRATCH), ("mb_pre", MB_PRE)]), | |
| } | |
| CROSS = [("distil5", "mb_distil5"), ("aidepre_fb_distil5", "mb_distil5")] # CockpitWM arm − MotionBERT control under the same treatment | |
| HM5_TOL = 1.0 | |
| SOTA_ALL = 47.5 | |
| def exists(name): | |
| return (RUNS / name / "pred_test.npz").exists() and metrics_test(name) is not None | |
| def reading(res_adopted): | |
| """pre-registered reading from the delta_table result vs the adopted arm.""" | |
| if res_adopted is None: | |
| return "INSUFFICIENT" | |
| v_all = res_adopted["All-MPJPE"]["verdict"] | |
| d_h5 = res_adopted["HM h=5"]["mean"] | |
| if v_all == "BEATS" and d_h5 <= HM5_TOL: | |
| return f"REPORTABLE (All BEATS; mean Δ HM h=5 {d_h5:+.2f} <= +{HM5_TOL} px)" | |
| if v_all == "BEATS": | |
| return f"NOT reportable (All BEATS but mean Δ HM h=5 {d_h5:+.2f} > +{HM5_TOL} px)" | |
| return f"NOT reportable (All verdict {v_all})" | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--n-boot", type=int, default=10000) | |
| ap.add_argument("--out", default=str(OUT)) | |
| a = ap.parse_args() | |
| t0 = time.time() | |
| d = load_split("test") | |
| gt = d["S10"][:, P.T_OBS:] | |
| hm, thr, _ = M.hm_mask(gt) | |
| stamp = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime()) | |
| L = ["# F7 — AIDE-only multi-stride pretraining + ensemble distillation (pre-registered 2026-09-17; lane isolated from the BATON hold-out devices) — AIDE test, n = 609, paired per-clip bootstrap (10k)", "", | |
| f"Generated {stamp} by `aide/eval/report_f7_pretrain_distil.py` from pred_test.npz / metrics_test.json / metrics_val.json / run json / history.json only (locked protocol: driver `first`, grid `floor`, " | |
| f"HM top {int(hm.sum())} (threshold {thr:.2f} px), PCK @ {P.pck_threshold_px(0.05):.0f} px; every fine-tune selected on VAL All-MPJPE only (EMA weights); every pretraining selected on AIDE VAL only; test evaluated once per run by the harness). " | |
| "Δ = paired per-clip difference (negative = better for errors, positive = better for PCK), CI = 10k clip-bootstrap percentile interval. " | |
| f"**Verdict rule: BEATS / WORSE = Δ CI excl. 0 on >= 2/3 seeds with the same sign on every seed; NOT_WORSE = all CIs cover 0 and |Δ| < {TOL_NOT_WORSE}; else INCONCLUSIVE. " | |
| f"Pre-registered reading: REPORTABLE iff All-MPJPE BEATS the adopted arm AND mean Δ HM h=5 <= +{HM5_TOL} px. SOTA criterion (plan): mean All <= {SOTA_ALL} over 5 seeds AND BEATS the same-recipe MotionBERT architecture.** The paper's adopted row (`cwm3d_kp_batonpre_trunkp123_s<k>_v2`) changes only by owner decision.", ""] | |
| J = {"generated_utc": stamp, "n_boot": a.n_boot, "arms": {}, "pretraining": {}, "cross": {}, "gpu_minutes": {}, "reference_ensemble": None} | |
| gpu = {} | |
| def note_gpu(name): | |
| r = run_json(name) | |
| if r and r.get("gpu_minutes") is not None: | |
| gpu[name] = float(r["gpu_minutes"]) | |
| # §1 pretraining runs | |
| L += ["## 1. Pretraining runs (AIDE train split only; selection on AIDE val All-MPJPE, EMA weights)", "", "| run | init | strides | windows/clip | epochs run (best) | stop | AIDE val All at best | ms/step | GPU-min | params |", "|---|---|---|---|---|---|---|---|---|---|"] | |
| for tag, pre in PRETRAIN.items(): | |
| r = run_json(pre) | |
| if r is None: | |
| L.append(f"| `{pre}` | — | — | — | not run | — | — | — | — | — |"); continue | |
| note_gpu(pre) | |
| cfg, dat, ini = r.get("config", {}), r.get("data", {}), r.get("init") or {} | |
| L.append(f"| `{pre}` | {ini.get('ckpt', 'seed init (torch.manual_seed({}))'.format(cfg.get('seed')))} ({ini.get('n_loaded', 0)} tensors) | {','.join(dat.get('strides', []))} | {dat.get('windows_per_clip')} | " | |
| f"{r.get('epochs_run')} ({r.get('best_epoch')}) | {r.get('stop_reason')} | {r.get('best_val_mpjpe_all'):.3f} | {r.get('ms_per_step_mean'):.1f} | {r.get('gpu_minutes')} | {r.get('n_params'):,} |") | |
| J["pretraining"][pre] = {"best_epoch": r.get("best_epoch"), "epochs_run": r.get("epochs_run"), "stop": r.get("stop_reason"), "best_val_all": r.get("best_val_mpjpe_all"), | |
| "gpu_minutes": r.get("gpu_minutes"), "n_params": r.get("n_params"), "curve": [(h["epoch"], round(h["val_mpjpe_all"], 3)) for h in r.get("history", [])], | |
| "isolation": r.get("isolation"), "init": ini} | |
| hist = r.get("history", []) | |
| if hist: | |
| L.append(""); L.append(f"`{pre}` AIDE val All per epoch: " + ", ".join(f"{h['epoch']}:{h['val_mpjpe_all']:.2f}" for h in hist)); L.append("") | |
| # reference ensemble of the 5 adopted seeds (not a single model) | |
| ens = [load_pred(ADOPTED.format(k=k), d) for k in SEEDS5] | |
| if all(e is not None for e in ens): | |
| cells = per_clip_cells(np.mean(ens, 0), gt, hm) | |
| J["reference_ensemble"] = {c: float(v[0].mean()) for c, v in cells.items()} | |
| L += ["", "Reference (teacher of arms (b)/(c); an ensemble, not a single model): mean of the 5 adopted seeds' saved test coordinates -> " | |
| + ", ".join(f"{c} {v:.2f}" for c, v in J["reference_ensemble"].items()), ""] | |
| # §2 arms | |
| L += ["## 2. Fine-tuned arms — per-seed levels (test, harness CIs) and paired Δ", ""] | |
| for tag, (label, tmpl, comps) in ARMS.items(): | |
| L += [f"### 2.{list(ARMS).index(tag) + 1} {tag} — {label}", "", "| run | val All | " + " | ".join(c for _, c, _ in LEVEL_COLS) + " | best ep / epochs / stop | GPU-min | params | init | distil |", "|---|---|" + "---|" * len(LEVEL_COLS) + "---|---|---|---|---|"] | |
| present = [] | |
| for k in SEEDS3: | |
| name = tmpl.format(k=k) | |
| if not exists(name): | |
| L.append(f"| `{name}` | not run | " + " | ".join("—" for _ in LEVEL_COLS) + " | — | — | — | — | — |"); continue | |
| present.append(k); note_gpu(name) | |
| r = run_json(name) or {} | |
| be, er, st, gm, npar = run_meta(name) | |
| ini = (r.get("init") or {}) | |
| dis = r.get("distil") or {} | |
| L.append(f"| `{name}` | {(r.get('val') or {}).get('mpjpe_all', float('nan')):.3f} | {level_row(name)} | {be} / {er} / {st} | {gm} | {npar:,} | " | |
| f"{Path(ini.get('ckpt', '—')).parent.name if ini else '—'}:{ini.get('parts', '')} ({ini.get('n_loaded', '')}) | " | |
| f"{dis.get('n_teachers', 0)} teachers w {dis.get('w_distil', 0)}/{dis.get('w_distil_modes', 0)}/{dis.get('w_distil_logb', 0)} |") | |
| J["arms"][tag] = {"label": label, "template": tmpl, "seeds_present": present, "levels": {}, "paired": {}, "params": None, "reading": None} | |
| if present: | |
| J["arms"][tag]["params"] = run_meta(tmpl.format(k=present[0]))[4] | |
| J["arms"][tag]["levels"] = {str(k): {c: metrics_test(tmpl.format(k=k))[c]["value"] for c, _, _ in LEVEL_COLS} for k in present} | |
| allv = [metrics_test(tmpl.format(k=k))["mpjpe_all"]["value"] for k in present] | |
| L += ["", f"mean over {len(present)} seeds: All {ms(allv, 2)}; HM h=5 {ms([metrics_test(tmpl.format(k=k))['hm_h5']['value'] for k in present], 2)}", ""] | |
| res_adopted = None | |
| for clabel, ctmpl in comps: | |
| per_seed = {} | |
| for k in present: | |
| pr = paired(tmpl.format(k=k), ctmpl.format(k=k), d, gt, hm, a.n_boot) | |
| if pr is not None: | |
| per_seed[k] = pr | |
| if not per_seed: | |
| L += [f"Δ vs {clabel} (`{ctmpl}`): no pairing available", ""]; continue | |
| tl, res = delta_table(per_seed, tag, clabel, cells=list(MAIN_CELLS), n_req=2) | |
| L += [f"**Δ vs {clabel}** (`{ctmpl}`; Δ = {tag} − {clabel}; {len(per_seed)} seeds, >= 2/3 rule):", ""] + tl + [""] | |
| J["arms"][tag]["paired"][clabel] = res | |
| if clabel == "adopted": | |
| res_adopted = res | |
| rd = reading(res_adopted) if tag != "mb_distil5" else ("(control arm; reading vs mb_pre) " + reading(J["arms"][tag]["paired"].get("mb_pre"))) | |
| J["arms"][tag]["reading"] = rd | |
| mean_all = float(np.mean(allv)) if present else None | |
| sota = None | |
| if present: | |
| mb_res = J["arms"][tag]["paired"].get("mb_pre") | |
| sota = {"mean_all": mean_all, "n_seeds": len(present), "all_le_47.5": bool(mean_all <= SOTA_ALL), "beats_mb_pre": (mb_res or {}).get("All-MPJPE", {}).get("verdict"), | |
| "status": ("5-seed extension pending" if len(present) < 5 else "5 seeds")} | |
| J["arms"][tag]["sota"] = sota | |
| L += [f"**Reading ({tag}): {rd}.**" + (f" SOTA check: mean All {mean_all:.2f} ({'<=' if mean_all <= SOTA_ALL else '>'} {SOTA_ALL}, {len(present)} seeds, {sota['status']}); vs same-recipe MotionBERT architecture (mb_pre) All verdict {sota['beats_mb_pre']}." if sota else ""), ""] | |
| # §3 cross-architecture fairness under the same treatment | |
| L += ["## 3. CockpitWM arm − MotionBERT control under the SAME treatment (paired, 3 seeds)", ""] | |
| for ta, tb in CROSS: | |
| per_seed = {} | |
| for k in SEEDS3: | |
| na, nb = ARMS[ta][1].format(k=k), ARMS[tb][1].format(k=k) | |
| if exists(na) and exists(nb): | |
| per_seed[k] = paired(na, nb, d, gt, hm, a.n_boot) | |
| if not per_seed: | |
| L += [f"{ta} − {tb}: no pairing available", ""]; continue | |
| tl, res = delta_table(per_seed, ta, tb, cells=list(MAIN_CELLS), n_req=2) | |
| L += [f"**{ta} − {tb}** (negative All = CockpitWM better):", ""] + tl + [""] | |
| J["cross"][f"{ta}-{tb}"] = res | |
| # §4 GPU minutes + outcome | |
| J["gpu_minutes"] = {"per_run": gpu, "total": round(sum(gpu.values()), 2), "n_runs": len(gpu)} | |
| L += ["## 4. GPU minutes (run json `gpu_minutes`, lane runs only)", "", f"{len(gpu)} runs, total {sum(gpu.values()):.1f} GPU-min: " + ", ".join(f"`{k}` {v}" for k, v in gpu.items()), "", | |
| "## 5. Outcome (per arm)", "", "| arm | seeds | params | mean All | reading | SOTA check |", "|---|---|---|---|---|---|"] | |
| for tag, r in J["arms"].items(): | |
| s_ = r.get("sota") or {} | |
| L.append(f"| {tag} | {len(r['seeds_present'])} | {r['params'] if r['params'] is None else format(r['params'], ',')} | {s_.get('mean_all', float('nan')):.2f} | {r['reading']} | " | |
| f"{'—' if not s_ else ('All <= 47.5: ' + str(s_['all_le_47.5']) + ' (' + s_['status'] + '); vs MotionBERT arch: ' + str(s_['beats_mb_pre']))} |") | |
| L += ["", f"Report time {time.time() - t0:.0f} s.", ""] | |
| out = Path(a.out) | |
| block = "\n".join(L) | |
| if out.exists(): | |
| out.write_text(out.read_text().rstrip("\n") + "\n\n---\n\n" + block) | |
| else: | |
| out.write_text(block) | |
| Path(str(out).replace(".md", ".json")).write_text(json.dumps(J, indent=1, default=float)) | |
| print(f"wrote {out} (+ .json); {len(gpu)} runs, {sum(gpu.values()):.1f} GPU-min") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 15.3 kB
- Xet hash:
- a5adb3f7aaa2be6bc816a1a3422e0cc1a6a4f8f0840ce56080381d03581c2b00
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.