Buckets:
| """cwm3/aide/eval/report_motionbert_faithful.py — faithful MotionBERT baseline (released weights, fine-tuned) vs CockpitWM on AIDE | |
| (pre-registration docs/decisions.md 2026-09-16T21:28:48Z) -> aide/results/tables/motionbert_faithful.md (+ .json). | |
| Reads ONLY run outputs: pred_test.npz (paired per-clip bootstraps recomputed here with aide/eval/metrics.paired_diff, 10k resamples, seed 0 — | |
| the procedure of arch_attribution.md / pretrain_variants.md), metrics_test.json (levels + clip-bootstrap CIs written once per run by the | |
| harness), the run json files (best epoch / epochs / stop / GPU minutes / peak VRAM / weight-load report) and weights_report.json. Nothing is | |
| typed by hand except the pre-registration text. | |
| Arms (aide/train/run_motionbert_faithful.sh): (a) mb_official_res_s<k>_v2 released weights + anchored residual read-out; | |
| (b) mb_official_abs_s<k>_v2 released weights + absolute read-out (their formulation); (c) mb_official_rand_s<k>_v2 random init + residual. | |
| References (seed-matched, existing runs, untouched): CockpitWM = cwm3d_kp_s<k>_v2 (locked seeds 42 / 123 / 456; 48.63 +- 0.15), | |
| CockpitWM + BATON pretraining = cwm3d_kp_batonpre_trunkp123_s<k>_v2 (adopted arm of pretrain_variants.md; 48.05 +- 0.06 over 5 seeds), | |
| architecture-family control = motionbert_kp_s<k>_v2 (descriptive). Verdict rule (as everywhere): BEATS / WORSE = Δ CI excl. 0 on >= ceil(2n/3) | |
| seeds (3/3 -> 2; the pre-registration text says 3/3 or >= 4/5 -> reported as the strict count too) with the same sign on every seed; | |
| NOT_WORSE = all CIs cover 0 and |Δ| < 0.25; else INCONCLUSIVE. Placement rule (pre-registered): every row enters Supp D regardless of direction; | |
| the main-text sentence depends on (a) vs CockpitWM + pretraining (see the verdict section). | |
| python aide/eval/report_motionbert_faithful.py [--n-boot 10000] [--out aide/results/tables/motionbert_faithful.md] | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import glob | |
| import json | |
| import math | |
| import re | |
| import sys | |
| import time | |
| from pathlib import Path | |
| import numpy as np | |
| ROOT = Path(__file__).resolve().parents[2] | |
| sys.path.insert(0, str(ROOT)) | |
| from aide.data import protocol as P # noqa: E402 | |
| from aide.eval import metrics as M # noqa: E402 | |
| from aide.eval.harness import align, load_split # noqa: E402 | |
| from aide.eval.report_arch_attribution import fmt_ci, ms, run_json, verdict # noqa: E402 | |
| from common.runmeta import write_run_json # noqa: E402 | |
| RUNS = P.RESULTS / "runs" | |
| OUT = P.RESULTS / "tables" / "motionbert_faithful.md" | |
| SEEDS3 = (42, 123, 456) | |
| SEEDS5 = (42, 123, 456, 789, 1011) | |
| ARMS = {"a": ("mb_official_res", "(a) MotionBERT released weights, anchored residual read-out"), | |
| "b": ("mb_official_abs", "(b) MotionBERT released weights, absolute read-out (their formulation)"), | |
| "c": ("mb_official_rand", "(c) same architecture, random init, residual read-out")} | |
| # POST-HOC (not pre-registered; launched after the pre-registered set, run_motionbert_faithful.sh MODE=posthoc_noema): (b) with the harness EMA disabled | |
| POSTHOC = {"b_noema": ("mb_official_abs_noema", "POST-HOC (b) released weights, absolute read-out, EMA disabled (--ema 0; selection on raw weights)")} | |
| REFS = {"K": ("cwm3d_kp", "CockpitWM (cwm3d_kp)"), | |
| "Kpre": ("cwm3d_kp_batonpre_trunkp123", "CockpitWM + BATON pretraining (cwm3d_kp_batonpre_trunkp123)"), | |
| "MBkp": ("motionbert_kp", "MotionBERT architecture-family control (motionbert_kp, d 128 / 3 blocks, no released weights)")} | |
| COLS = [("mpjpe_all", "All", 2), ("dn_mpjpe", "d-n", 3), ("mpjpe_hm", "HM", 2), ("dn_mpjpe_hm", "HM d-n", 3), ("pck05", "PCK@0.05", 2), ("hm_h5", "HM h=5", 2)] | |
| CELLS = ("All-MPJPE", "d-n", "HM-MPJPE", "HM d-n", "PCK@0.05", "HM h=5") | |
| # ----------------------------------------------------------------------------- io | |
| def metrics_test(name): | |
| p = RUNS / name / "metrics_test.json" | |
| return json.load(open(p))["metrics"] if p.exists() else None | |
| def metrics_val(name): | |
| p = RUNS / name / "metrics_val.json" | |
| return json.load(open(p))["metrics"] if p.exists() else None | |
| def load_pred(name, d): | |
| p = RUNS / name / "pred_test.npz" | |
| if not p.exists(): | |
| return None | |
| z = np.load(p, allow_pickle=False) | |
| return np.asarray(z["pred"], dtype=np.float64)[align(z, d["clip_id"])] | |
| def per_clip_cells(pred, gt, hm): | |
| err = M.joint_errors(pred, gt) | |
| clip = err.mean((1, 2)) | |
| return {"All-MPJPE": (clip, False), "d-n": (100.0 * clip / P.DIAG, False), "HM-MPJPE": (err[hm].mean((1, 2)), False), | |
| "HM d-n": (100.0 * err[hm].mean((1, 2)) / P.DIAG, False), | |
| "PCK@0.05": (100.0 * (err <= P.pck_threshold_px(0.05)).mean((1, 2)), True), "HM h=5": (err[hm][:, 4].mean(1), False)} | |
| def paired(a_name, b_name, d, gt, hm, n_boot): | |
| pa, pb = load_pred(a_name, d), load_pred(b_name, d) | |
| if pa is None or pb is None: | |
| return None | |
| ca, cb = per_clip_cells(pa, gt, hm), per_clip_cells(pb, gt, hm) | |
| return {c: {"a": float(ca[c][0].mean()), "b": float(cb[c][0].mean()), **M.paired_diff(ca[c][0], cb[c][0], n_boot, 0), "higher_better": ca[c][1]} for c in ca} | |
| def finished(name): | |
| return run_json(name) is not None and (RUNS / name / "pred_test.npz").exists() and metrics_test(name) is not None | |
| def seeds_done(stem, seeds=SEEDS5): | |
| return [s for s in seeds if finished(f"{stem}_s{s}_v2")] | |
| def level_row(name): | |
| m = metrics_test(name) | |
| if m is None: | |
| return "—" | |
| return " | ".join(f"{m[c]['value']:.{nd}f} [{m[c]['ci_lo']:.{nd}f}, {m[c]['ci_hi']:.{nd}f}]" for c, _, nd in COLS) | |
| def mean_sd_row(names): | |
| ms_ = [metrics_test(n) for n in names] | |
| ms_ = [m for m in ms_ if m is not None] | |
| if not ms_: | |
| return "—" | |
| return " | ".join(ms([m[c]["value"] for m in ms_], nd) for c, _, nd in COLS) | |
| def train_info(name): | |
| """peak VRAM / guard / gate wait: train_info.json (runs after 2026-09-16T23:55Z) or the trainer's own log line (earlier runs); None if absent.""" | |
| p = RUNS / name / "train_info.json" | |
| if p.exists(): | |
| return json.load(open(p)) | |
| lg = RUNS / f"{name}_train.log" | |
| if lg.exists(): | |
| m = re.search(r"peak ([0-9.]+) GiB \(model only ([0-9.]+) GiB, guard ([0-9.]+) GiB\)", lg.read_text()) | |
| if m: | |
| return {"peak_gpu_gib": float(m.group(1)), "peak_model_gib": float(m.group(2)), "vram_guard_gib": float(m.group(3))} | |
| return {} | |
| def run_meta(name): | |
| r = run_json(name) or {} | |
| mv = metrics_val(name) | |
| ti = train_info(name) | |
| return {"best_epoch": r.get("best_epoch", "—"), "epochs_run": r.get("epochs_run", "—"), "stop": r.get("stop_reason", "—"), "gpu_min": r.get("gpu_minutes"), | |
| "peak_gib": ti.get("peak_model_gib"), "peak_total_gib": ti.get("peak_gpu_gib"), "guard_gib": ti.get("vram_guard_gib"), "n_params": r.get("n_params"), | |
| "best_val": r.get("best_val_mpjpe_all"), "val_all": (mv["mpjpe_all"]["value"] if mv else None), "init": r.get("init") or {}} | |
| def delta_table(per_seed, label_a, label_b, n_req=None): | |
| seeds = list(per_seed) | |
| L = ["| cell | " + " | ".join(f"{label_a} s{k}" for k in seeds) + " | " + " | ".join(f"{label_b} s{k}" for k in seeds) + " | " | |
| + " | ".join(f"Δ s{k} [CI]" for k in seeds) + " | mean Δ ± sd | CI excl. 0 | verdict |", "|---|" + "---|" * (3 * len(seeds)) + "---|---|---|"] | |
| res = {} | |
| for c in CELLS: | |
| rows = [per_seed[k][c] for k in seeds] | |
| v = verdict(rows, rows[0]["higher_better"], n_req) | |
| dm = np.array([r["mean"] for r in rows]) | |
| excl = int(sum(1 for r in rows if r["ci_lo"] > 0 or r["ci_hi"] < 0)) | |
| nd = 3 if "d-n" in c else 2 | |
| L.append(f"| {c} | " + " | ".join(f"{r['a']:.{nd}f}" for r in rows) + " | " + " | ".join(f"{r['b']:.{nd}f}" for r in rows) + " | " | |
| + " | ".join(fmt_ci(r, nd) for r in rows) + f" | {dm.mean():+.{nd}f} ± {(dm.std(ddof=1) if len(dm) > 1 else float('nan')):.{nd}f} | {excl}/{len(rows)} | {v} |") | |
| res[c] = {"verdict": v, "mean": float(dm.mean()), "sd": float(dm.std(ddof=1)) if len(dm) > 1 else None, "n_ci_excl_0": excl, "n_seeds": len(rows), | |
| "per_seed": {str(k): {kk: vv for kk, vv in per_seed[k][c].items()} for k in seeds}} | |
| return L, res | |
| # ----------------------------------------------------------------------------- main | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--n-boot", type=int, default=10000) | |
| ap.add_argument("--out", default=str(OUT)) | |
| a = ap.parse_args() | |
| t0 = time.time() | |
| d = load_split("test") | |
| gt = d["S10"][:, P.T_OBS:] | |
| hm, thr, _ = M.hm_mask(gt) | |
| stamp = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime()) | |
| J = {"generated_utc": stamp, "n_boot": a.n_boot, "hm_threshold": thr, "arms": {}, "paired": {}, "weights": {}} | |
| L = ["# Faithful MotionBERT baseline — released MB_release weights fine-tuned on AIDE (two read-outs) + random-init control vs CockpitWM — AIDE test, n = 609, paired per-clip bootstrap (10k)", "", | |
| f"Generated {stamp} by `aide/eval/report_motionbert_faithful.py` from pred_test.npz / metrics_test.json / metrics_val.json / run json / weights_report.json files only " | |
| f"(locked protocol: driver `first`, grid `floor`, HM top {P.hm_top(len(gt))} (threshold {thr:.2f} px), PCK @ {P.pck_threshold_px(0.05):.0f} px; every run selected on VAL All-MPJPE of the EMA " | |
| f"weights only, test evaluated once by the harness). Δ = paired per-clip difference (negative = better for errors, positive = better for PCK), CI = {a.n_boot // 1000}k clip-bootstrap percentile interval. " | |
| "**Verdict rule (as `baton_pretrain.md` / `arch_attribution.md`): BEATS / WORSE = Δ CI excl. 0 on ≥ 2/3 of the seeds (n = 3: 2; n = 5: 4) with the same sign on every seed; NOT_WORSE = all CIs cover 0 and |Δ| < 0.25; else INCONCLUSIVE. " | |
| "The pre-registration row additionally names the strict count ('3/3 or ≥ 4/5'); the column `CI excl. 0` reports it.**", "", | |
| "Pre-registration: docs/decisions.md 2026-09-16 (2026-09-16T21:28:48Z, 'faithful MotionBERT baseline'). Owner's rule: '我必须要用原来的权重, 然后再在我们的这个数据上面做微调 然后进行比较 这样才算复现'. " | |
| "Model: official DSTformer (vendored `aide/models/motionbert_official/DSTformer.py`, Apache-2.0; dim_feat 512, depth 5, 8 heads, mlp_ratio 2, maxlen 243, dim_rep 512) loaded from the released pretraining checkpoint MB_release; " | |
| "input = HALPE-136 keypoints in MotionBERT's `--pixel` normalisation (xy' = (xy − (W, H)/2) / (min(W, H)/2), confidence as channel 3); 136 joint tokens, each with the pretrained spatial position embedding of its H36M-17 joint " | |
| "(body via the repo's `halpe2h36m`, the other 120 keypoints from the nearest pretrained body joint inside the pre-registered candidate sets — `aide/models/motionbert_official/joint_map.json`); " | |
| "5 observed frames + 5 placeholder copies of the last observed frame → per-frame 512-d representation of the placeholders → Linear(512 → 2). " | |
| "(a) residual: pred = last xy + 200·head, head zero-init; (b) absolute: pred = denormalise(head) px, default init; (c) official random init, residual. " | |
| "Fine-tuning of ALL weights: AdamW, backbone lr 1e-4, read-out lr 1e-3, weight decay 0.01, batch 32, 10 warm-up epochs + cosine, EMA 0.999, ≤ 60 epochs, early stopping patience 10 on val All-MPJPE, " | |
| "harness augmentation (sub-window grids, noise 2(1 − conf) px, joint dropout 0.1, translate ±20 px, scale ±5 %), plain Euclidean loss; seeds 42 / 123 / 456 (+ 789 / 1011 for (a) if the 9 runs took ≤ 60 GPU-min). " | |
| "Placement (fixed before the runs): all rows enter Supp D regardless of direction; main text per the rule in the pre-registration row.", ""] | |
| # ---- 0. weights | |
| L += ["## 0. Released weights (public artefact; no clip or AIDE data left the machine)", ""] | |
| wr = None | |
| for arm in ("a", "b"): | |
| for s in SEEDS5: | |
| p = RUNS / f"{ARMS[arm][0]}_s{s}_v2" / "weights_report.json" | |
| if p.exists(): | |
| wr = json.load(open(p)); break | |
| if wr: | |
| break | |
| if wr: | |
| J["weights"] = wr | |
| L += [f"- checkpoint `{wr.get('checkpoint')}`: `{wr.get('url')}` → `{wr.get('path')}`; **sha256 `{wr.get('sha256')}`** (expected `{wr.get('sha256_expected')}`), {wr.get('bytes'):,} bytes.", | |
| f"- strict load: {wr.get('n_loaded_exact')} / {wr.get('n_ckpt_tensors')} checkpoint tensors loaded bit-exactly ({wr.get('n_params_loaded'):,} parameters; groups {wr.get('loaded_groups')}); " | |
| f"dropped {wr.get('dropped')} (the pretrained 3-D lifting head, 512 → 3); shape mismatches {wr.get('shape_mismatch')}; backbone tensors missing from the checkpoint {wr.get('missing_in_ckpt')}.", | |
| f"- new / re-initialised: {wr.get('new_parameters')} ({wr.get('n_params_new_head')} parameters, the 2-D read-out); copied: {wr.get('copied_parameters')} " | |
| f"(17 pretrained rows → 136 rows = {wr.get('n_params_pos_embed_136'):,} parameters, no random values). Model total {wr.get('n_params_total'):,} parameters. " | |
| f"Probe tensor `{wr.get('probe_tensor')}` sha256 `{str(wr.get('probe_sha256'))[:16]}…` (checked bit-identical in `tests/test_motionbert_official.py`). Joint map sha256 `{str(wr.get('joint_map_sha256'))[:16]}…`.", ""] | |
| else: | |
| L += ["- no finished run yet: the weights report is written by every (a) / (b) run (`weights_report.json`).", ""] | |
| jm_path = ROOT / "aide/models/motionbert_official/joint_map.json" | |
| if jm_path.exists(): | |
| jm = json.load(open(jm_path)) | |
| L += [f"- joint map (`{jm_path.relative_to(ROOT)}`, built from {jm.get('n_train_frames'):,} TRAIN protocol frames): tokens per pretrained row {jm.get('tokens_per_pretrained_row')}; unused pretrained rows {jm.get('unused_pretrained_rows')}.", ""] | |
| J["joint_map_summary"] = {"tokens_per_pretrained_row": jm.get("tokens_per_pretrained_row"), "unused": jm.get("unused_pretrained_rows")} | |
| # ---- 1. levels | |
| L += ["## 1. Test levels per run (harness values with clip-bootstrap CIs) and seed means ± sd", "", | |
| "| arm | run | seed | " + " | ".join(c for _, c, _ in COLS) + " | best val All (EMA) | best ep / epochs / stop | GPU min | peak VRAM GiB (model only; + guard) | params |", "|---|---|---|" + "---|" * len(COLS) + "---|---|---|---|---|"] | |
| for key, (stem, label) in ARMS.items(): | |
| done = seeds_done(stem) | |
| J["arms"][key] = {"stem": stem, "label": label, "seeds_done": done, "runs": {}} | |
| for s in done: | |
| n = f"{stem}_s{s}_v2" | |
| rm = run_meta(n) | |
| J["arms"][key]["runs"][str(s)] = {"metrics": {c: metrics_test(n)[c]["value"] for c, _, _ in COLS}, **{k: v for k, v in rm.items() if k != "init"}} | |
| L.append(f"| {label} | `{n}` | {s} | {level_row(n)} | {rm['best_val']:.3f} | {rm['best_epoch']} / {rm['epochs_run']} / {rm['stop']} | {rm['gpu_min']} | {rm['peak_gib']} (+{rm['guard_gib']}) | {rm['n_params']:,} |") | |
| if done: | |
| L.append(f"| **{label} — mean ± sd** | n = {len(done)} | {'/'.join(map(str, done))} | {mean_sd_row([f'{stem}_s{s}_v2' for s in done])} | {ms([run_meta(f'{stem}_s{s}_v2')['best_val'] for s in done], 3)} | — | {ms([run_meta(f'{stem}_s{s}_v2')['gpu_min'] for s in done], 2)} | — | — |") | |
| J["arms"][key]["mean_sd"] = {c: {"mean": float(np.mean([metrics_test(f"{stem}_s{s}_v2")[c]["value"] for s in done])), | |
| "sd": (float(np.std([metrics_test(f"{stem}_s{s}_v2")[c]["value"] for s in done], ddof=1)) if len(done) > 1 else None)} for c, _, _ in COLS} | |
| else: | |
| L.append(f"| {label} | — | — | " + " | ".join("—" for _ in COLS) + " | — | not run yet | — | — | — |") | |
| for key, (stem, label) in POSTHOC.items(): | |
| done = seeds_done(stem) | |
| J["arms"][key] = {"stem": stem, "label": label, "seeds_done": done, "posthoc": True, "runs": {}} | |
| for s in done: | |
| n = f"{stem}_s{s}_v2" | |
| rm = run_meta(n) | |
| J["arms"][key]["runs"][str(s)] = {"metrics": {c: metrics_test(n)[c]["value"] for c, _, _ in COLS}, **{k: v for k, v in rm.items() if k != "init"}} | |
| L.append(f"| {label} | `{n}` | {s} | {level_row(n)} | {rm['best_val']:.3f} | {rm['best_epoch']} / {rm['epochs_run']} / {rm['stop']} | {rm['gpu_min']} | {rm['peak_gib']} (+{rm['guard_gib']}) | {rm['n_params']:,} |") | |
| if done: | |
| L.append(f"| **{label} — mean ± sd** | n = {len(done)} | {'/'.join(map(str, done))} | {mean_sd_row([f'{stem}_s{s}_v2' for s in done])} | {ms([run_meta(f'{stem}_s{s}_v2')['best_val'] for s in done], 3)} | — | {ms([run_meta(f'{stem}_s{s}_v2')['gpu_min'] for s in done], 2)} | — | — |") | |
| J["arms"][key]["mean_sd"] = {c: {"mean": float(np.mean([metrics_test(f"{stem}_s{s}_v2")[c]["value"] for s in done])), | |
| "sd": (float(np.std([metrics_test(f"{stem}_s{s}_v2")[c]["value"] for s in done], ddof=1)) if len(done) > 1 else None)} for c, _, _ in COLS} | |
| for key, (stem, label) in REFS.items(): | |
| done = seeds_done(stem) | |
| for s in done: | |
| n = f"{stem}_s{s}_v2" | |
| rm = run_meta(n) | |
| L.append(f"| reference: {label} | `{n}` | {s} | {level_row(n)} | {rm['best_val']:.3f} | {rm['best_epoch']} / {rm['epochs_run']} / {rm['stop']} | {rm['gpu_min']} | — | {rm['n_params']:,} |") | |
| if done: | |
| L.append(f"| **reference: {label} — mean ± sd** | n = {len(done)} | {'/'.join(map(str, done))} | {mean_sd_row([f'{stem}_s{s}_v2' for s in done])} | — | — | — | — | — |") | |
| if len(done) >= 3: | |
| L.append(f"| **reference: {label} — locked seeds 42 / 123 / 456 mean ± sd** | n = 3 | 42/123/456 | {mean_sd_row([f'{stem}_s{s}_v2' for s in SEEDS3])} | — | — | — | — | — |") | |
| zv = metrics_test("zero_velocity") | |
| if zv: | |
| L.append(f"| reference: zero-velocity (analytic, run `zero_velocity`) | `zero_velocity` | — | {level_row('zero_velocity')} | — | — | 0 | — | 0 |") | |
| L.append("") | |
| tot_gpu = sum(float(run_meta(f"{stem}_s{s}_v2")["gpu_min"] or 0) for stem, _ in ARMS.values() for s in seeds_done(stem)) | |
| L += [f"Total GPU minutes of the finished PRE-REGISTERED faithful-MotionBERT runs: **{tot_gpu:.1f}** (budget rule for (a) seeds 789 / 1011: the 9 pre-registered runs ≤ 60 min).", ""] | |
| J["total_gpu_minutes"] = tot_gpu | |
| # ---- 2. paired comparisons | |
| comparisons = [("a_vs_K", "a", "K", "(a) − CockpitWM", "MB(a)", "K"), | |
| ("a_vs_Kpre", "a", "Kpre", "(a) − CockpitWM + BATON pretraining", "MB(a)", "K+pre"), | |
| ("b_vs_K", "b", "K", "(b) − CockpitWM", "MB(b)", "K"), | |
| ("b_vs_Kpre", "b", "Kpre", "(b) − CockpitWM + BATON pretraining", "MB(b)", "K+pre"), | |
| ("c_vs_K", "c", "K", "(c) − CockpitWM", "MB(c)", "K"), | |
| ("a_minus_c", "a", "c", "(a) − (c): value of the released weights", "MB(a)", "MB(c)"), | |
| ("a_minus_b", "a", "b", "(a) − (b): value of anchoring (residual vs absolute read-out)", "MB(a)", "MB(b)"), | |
| ("a_vs_MBkp", "a", "MBkp", "(a) − architecture-family control motionbert_kp (descriptive)", "MB(a)", "MB-kp")] | |
| L += ["## 2. Paired per-clip Δ (seed-matched; errors: negative = the first arm is better; PCK: positive = better)", ""] | |
| for key, x, y, title, la, lb in comparisons: | |
| sx = (ARMS | POSTHOC | REFS)[x][0] | |
| sy = (ARMS | POSTHOC | REFS)[y][0] | |
| seeds = [s for s in SEEDS5 if finished(f"{sx}_s{s}_v2") and finished(f"{sy}_s{s}_v2")] | |
| L += [f"### 2.{comparisons.index((key, x, y, title, la, lb)) + 1} {title}", ""] | |
| if not seeds: | |
| L += ["_pending — no seed with both runs finished._", ""] | |
| J["paired"][key] = {"status": "PENDING", "seeds": []} | |
| continue | |
| per_seed = {s: paired(f"{sx}_s{s}_v2", f"{sy}_s{s}_v2", d, gt, hm, a.n_boot) for s in seeds} | |
| rows, res = delta_table(per_seed, la, lb) | |
| L += rows + [""] | |
| r_all = res["All-MPJPE"] | |
| L += [f"**{title}: Δ All-MPJPE {r_all['mean']:+.2f} ± {(r_all['sd'] if r_all['sd'] is not None else float('nan')):.2f} px over {len(seeds)} seeds " | |
| f"({'/'.join(map(str, seeds))}); CI excl. 0 on {r_all['n_ci_excl_0']}/{len(seeds)} seeds → {r_all['verdict']}; " | |
| f"HM {res['HM-MPJPE']['verdict']} ({res['HM-MPJPE']['mean']:+.2f}), PCK@0.05 {res['PCK@0.05']['verdict']} ({res['PCK@0.05']['mean']:+.2f}), HM h=5 {res['HM h=5']['verdict']} ({res['HM h=5']['mean']:+.2f}).**", ""] | |
| J["paired"][key] = {"status": "DONE", "title": title, "seeds": seeds, "cells": res} | |
| # ---- 2b. POST-HOC sensitivity block (labelled; never feeds the pre-registered verdicts) | |
| posthoc_cmp = [("bnoema_minus_b", "b_noema", "b", "POST-HOC (b, EMA off) − (b, pre-registered): effect of the EMA lag under the 60-epoch cap", "MB(b,noEMA)", "MB(b)"), | |
| ("a_minus_bnoema", "a", "b_noema", "POST-HOC (a) − (b, EMA off): value of anchoring with the EMA confound removed", "MB(a)", "MB(b,noEMA)"), | |
| ("bnoema_vs_K", "b_noema", "K", "POST-HOC (b, EMA off) − CockpitWM", "MB(b,noEMA)", "K")] | |
| if any(seeds_done(POSTHOC[k][0]) for k in POSTHOC): | |
| L += ["## 2b. POST-HOC sensitivity (NOT pre-registered; launched 2026-09-17 after the pre-registered set; does not enter §3)", "", | |
| "_Motivation (recorded before these runs started): under the pre-registered budget the harness EMA (0.999, ≈ 17-epoch time constant) is still dominated by the default-init absolute " | |
| "read-out at the 60-epoch cap ((b) val All 250 → 59 px over the run, best epoch 59/60 on every seed), so the pre-registered (b) level and the (a) − (b) contrast are confounded by the EMA lag. " | |
| "These runs are (b) with `--ema 0` (selection and prediction on the raw weights), everything else identical._", ""] | |
| for key, x, y, title, la, lb in posthoc_cmp: | |
| sx, sy = (ARMS | POSTHOC | REFS)[x][0], (ARMS | POSTHOC | REFS)[y][0] | |
| seeds = [s for s in SEEDS5 if finished(f"{sx}_s{s}_v2") and finished(f"{sy}_s{s}_v2")] | |
| L += [f"### {title}", ""] | |
| if not seeds: | |
| L += ["_pending._", ""] | |
| J["paired"][key] = {"status": "PENDING", "posthoc": True, "seeds": []} | |
| continue | |
| per_seed = {s: paired(f"{sx}_s{s}_v2", f"{sy}_s{s}_v2", d, gt, hm, a.n_boot) for s in seeds} | |
| rows, res = delta_table(per_seed, la, lb) | |
| r_all = res["All-MPJPE"] | |
| L += rows + ["", f"**{title}: Δ All-MPJPE {r_all['mean']:+.2f} ± {(r_all['sd'] if r_all['sd'] is not None else float('nan')):.2f} px over {len(seeds)} seeds; CI excl. 0 on " | |
| f"{r_all['n_ci_excl_0']}/{len(seeds)} → {r_all['verdict']} (descriptive; post-hoc).**", ""] | |
| J["paired"][key] = {"status": "DONE", "posthoc": True, "title": title, "seeds": seeds, "cells": res} | |
| # ---- 3. verdicts (pre-registered placement rule) | |
| L += ["## 3. Pre-registered reading", "", | |
| "_Two counts are reported for every verdict: the project's standard rule (CI excl. 0 on ≥ 2/3 seeds, same sign; the `verdict` column) and the STRICT count written in this experiment's " | |
| "pre-registration row ('CI excl. 0 on ≥ 4/5 or 3/3 seeds'; the `CI excl. 0` column). The placement reading below follows the strict count of the pre-registration row; the standard-rule verdict is " | |
| "stated next to it so the owner can see where the two disagree._", ""] | |
| pa, pk = J["paired"].get("a_vs_Kpre", {}), J["paired"].get("a_vs_K", {}) | |
| if pa.get("status") == "DONE": | |
| c_all = pa["cells"]["All-MPJPE"] | |
| v_std, mean = c_all["verdict"], c_all["mean"] | |
| rows_all = [c_all["per_seed"][k] for k in c_all["per_seed"]] | |
| v = verdict(rows_all, False, n_req=len(rows_all)) # STRICT: every seed's CI must exclude 0 | |
| J["reading_rules"] = {"standard_2of3": v_std, "strict_all_seeds": v, "n_ci_excl_0": c_all["n_ci_excl_0"], "n_seeds": c_all["n_seeds"]} | |
| if v == "WORSE": | |
| reading = ("(a) is significantly WORSE than CockpitWM + BATON pretraining on All-MPJPE → the attribution wording of the paper stands (the lead is not explained by a stronger released " | |
| "backbone); the block enters Supp D.") | |
| elif v == "BEATS": | |
| reading = ("(a) is significantly BETTER than CockpitWM + BATON pretraining on All-MPJPE → the paper must report it as such, keep the CockpitWM row as its own method, and state plainly that a " | |
| "fine-tuned MotionBERT backbone is the more accurate keypoint forecaster under our anchored formulation.") | |
| else: | |
| reading = ("(a) is NOT significantly worse than CockpitWM + BATON pretraining on All-MPJPE (verdict {v}) → the main-text attribution sentence must say that a fine-tuned released MotionBERT under " | |
| "our anchored formulation reaches comparable error; the gain is attributed to the formulation and pretraining, not to the backbone.").format(v=v) | |
| L += [f"- **(a) vs CockpitWM + BATON pretraining: Δ All {mean:+.2f} ± {(c_all['sd'] or float('nan')):.2f} px; CI excl. 0 on {c_all['n_ci_excl_0']}/{c_all['n_seeds']} seeds → strict-count verdict {v} " | |
| f"(standard ≥ 2/3 rule: {v_std}).** {reading}", ""] | |
| if v != v_std: | |
| L += [f"- _The two rules disagree here: under the project's standard rule (a) is {v_std} than CockpitWM + BATON pretraining; under the strict count of this pre-registration row it is {v}. " | |
| "Both are recorded; the wording decision is the owner's._", ""] | |
| J["reading"] = {"a_vs_Kpre_all": v, "a_vs_Kpre_all_standard_rule": v_std, "mean": mean, "text": reading} | |
| else: | |
| L += ["- (a) vs CockpitWM + BATON pretraining: pending.", ""] | |
| if pk.get("status") == "DONE": | |
| L += [f"- (a) vs CockpitWM (no pretraining): Δ All {pk['cells']['All-MPJPE']['mean']:+.2f} px, {pk['cells']['All-MPJPE']['verdict']}.", ""] | |
| for key, txt in (("a_minus_c", "value of the released weights ((a) − (c))"), ("a_minus_b", "value of anchoring ((a) − (b))"), ("b_vs_K", "their formulation (b) vs CockpitWM")): | |
| r = J["paired"].get(key, {}) | |
| if r.get("status") == "DONE": | |
| L += [f"- {txt}: Δ All {r['cells']['All-MPJPE']['mean']:+.2f} ± {(r['cells']['All-MPJPE']['sd'] or float('nan')):.2f} px, {r['cells']['All-MPJPE']['verdict']}; HM {r['cells']['HM-MPJPE']['mean']:+.2f} ({r['cells']['HM-MPJPE']['verdict']}).", ""] | |
| L += ["_Caveats fixed by the pre-registration: (c) shares the 60-epoch / patience-10 budget of the fine-tunes, so it measures the released weights' value under that budget (its best epoch / stop reason " | |
| "are listed in §1); the model has 42.5 M parameters against CockpitWM's 5.77 M — the comparison is deliberately not parameter-matched (it asks whether the released backbone, fine-tuned, forecasts " | |
| "better, not whether it is efficient)._", ""] | |
| out = Path(a.out) | |
| out.write_text("\n".join(L)) | |
| out.with_suffix(".json").write_text(json.dumps(J, indent=1, default=str)) | |
| write_run_json(out.parent, "report_motionbert_faithful", [__file__, P.__file__, M.__file__], [P.CACHE_NPZ], seed=0, t_start=t0, input_listing=False, | |
| extra={"out": str(out), "n_boot": a.n_boot, "arms_done": {k: v["seeds_done"] for k, v in J["arms"].items()}, "reading": J.get("reading")}) | |
| print(f"wrote {out} (+ .json) in {time.time() - t0:.1f}s; arms done {[(k, v['seeds_done']) for k, v in J['arms'].items()]}; reading {J.get('reading', {}).get('a_vs_Kpre_all')}") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 27.9 kB
- Xet hash:
- 7b90a473062b9a6dda91dd3aa1070ff703cea298af4f10dfd8cee1a76f7543bb
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.