Buckets:
| """cwm3/aide/eval/report_arch_attribution.py — AIDE architecture-attribution fair experiments (docs/decisions.md 2026-09-14, row | |
| "AIDE 架构归因公平实验"; rules fixed before the runs) -> aide/results/tables/arch_attribution.md (+ .json). | |
| Reads ONLY run outputs: pred_test.npz (paired per-clip bootstraps recomputed here with aide/eval/metrics.paired_diff, 10k resamples, seed 0 — | |
| the same procedure as baton_pretrain.md), metrics_test.json (level values + clip-bootstrap CIs written once per run by the harness), the run json | |
| files (best epoch / epochs / stop / GPU minutes / init report / params) and the pretraining history. Nothing is typed by hand. | |
| (1) matched BATON in-cabin pose pretraining for the keypoint baselines: A = <model>_batonpre_trunkp123_s<k>_v2 (trunk-only init from | |
| <model>_batonpre_pretrain_s123) vs B = the EXISTING seed-init runs <model>_s<k>_v2, seeds 42 / 123 / 456; Δ = A − B per clip; verdict as | |
| baton_pretrain.md: BEATS / WORSE = Δ CI excl. 0 on >= 2/3 seeds with the same sign on every seed; NOT_WORSE = all CIs cover 0 and |Δ| < 0.25; | |
| else INCONCLUSIVE. The gain is shown next to K's trunk-only gain (cwm3d_kp_batonpre_trunkp123 vs cwm3d_kp, recomputed from the same files); | |
| descriptive only, no cross-architecture test. | |
| (2) Driver-WM-kp (driverwm_kp_s<k>_v2) vs K (cwm3d_kp_s<k>_v2) and vs MotionBERT-kp (motionbert_kp_s<k>_v2), seed-matched paired Δ. Reading | |
| (decisions row): (a) every seed's Δ All (Driver-WM-kp − K) has a CI containing 0 and |Δ| < 1 px -> the lead belongs to the keypoint | |
| formulation; (b) K significantly better (Δ > 0 with CI excl. 0 on >= 2/3 seeds) -> supports an architecture advantage under the keypoint | |
| formulation; (c) Driver-WM-kp better (Δ < 0 with CI excl. 0 on >= 2/3 seeds) -> reported as is; anything else -> none of the three readings. | |
| (3) MotionBERT-kp at 5 seeds (42 / 123 / 456 + 789 / 1011 = K's 5-seed set): per-seed paired Δ = K − MotionBERT-kp; verdict by the beats rule | |
| generalised to n seeds (CI excl. 0 on >= ceil(2n/3) seeds, same sign on all; NOT_WORSE / INCONCLUSIVE as above), reported for the 5 seeds | |
| and for the locked 3; 5-seed mean ± sd of both models. | |
| python aide/eval/report_arch_attribution.py [--n-boot 10000] [--out aide/results/tables/arch_attribution.md] | |
| Output policy: if --out already exists the new content is APPENDED as a dated block (append-only tables); existing sections are never edited. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import glob | |
| import json | |
| import math | |
| import sys | |
| import time | |
| from pathlib import Path | |
| import numpy as np | |
| ROOT = Path(__file__).resolve().parents[2] | |
| sys.path.insert(0, str(ROOT)) | |
| from aide.data import protocol as P # noqa: E402 | |
| from aide.eval import metrics as M # noqa: E402 | |
| from aide.eval.harness import align, load_split # noqa: E402 | |
| from common.runmeta import write_run_json # noqa: E402 | |
| RUNS = P.RESULTS / "runs" | |
| OUT = P.RESULTS / "tables" / "arch_attribution.md" | |
| SEEDS3, SEEDS5 = (42, 123, 456), (42, 123, 456, 789, 1011) | |
| TOL_NOT_WORSE = 0.25 | |
| LEVEL_COLS = [("mpjpe_all", "All", 2), ("dn_mpjpe", "d-n", 3), ("mpjpe_hm", "HM", 2), ("pck05", "PCK@0.05", 2), ("hm_h5", "HM h=5", 2)] | |
| MAIN_CELLS = ("All-MPJPE", "d-n", "HM-MPJPE", "PCK@0.05", "HM h=5") | |
| # ----------------------------------------------------------------------------- io | |
| def run_json(name): | |
| fs = sorted(glob.glob(str(RUNS / name / f"run_{name}_2*.json"))) | |
| return json.load(open(fs[-1])) if fs else None | |
| def metrics_test(name): | |
| p = RUNS / name / "metrics_test.json" | |
| return json.load(open(p))["metrics"] if p.exists() else None | |
| def load_pred(name, d): | |
| p = RUNS / name / "pred_test.npz" | |
| if not p.exists(): | |
| return None | |
| z = np.load(p, allow_pickle=False) | |
| return np.asarray(z["pred"], dtype=np.float64)[align(z, d["clip_id"])] | |
| def per_clip_cells(pred, gt, hm, full=False): | |
| """-> {cell: (per-clip values, higher_better)} — baton_pretrain.md's cells (+ d-n).""" | |
| err = M.joint_errors(pred, gt) | |
| clip = err.mean((1, 2)) | |
| cells = {"All-MPJPE": (clip, False), "d-n": (100.0 * clip / P.DIAG, False), "HM-MPJPE": (err[hm].mean((1, 2)), False), | |
| "PCK@0.05": (100.0 * (err <= P.pck_threshold_px(0.05)).mean((1, 2)), True), "HM h=5": (err[hm][:, 4].mean(1), False)} | |
| if full: | |
| for k, idx in P.PARTS.items(): | |
| cells[f"All {k}"] = (err[:, :, idx].mean((1, 2)), False) | |
| for h in range(P.T_PRED): | |
| cells[f"All h={h + 1}"] = (err[:, h].mean(1), False) | |
| return cells | |
| def paired(a_name, b_name, d, gt, hm, n_boot, full=False): | |
| pa, pb = load_pred(a_name, d), load_pred(b_name, d) | |
| if pa is None or pb is None: | |
| return None | |
| ca, cb = per_clip_cells(pa, gt, hm, full), per_clip_cells(pb, gt, hm, full) | |
| return {c: {"a": float(ca[c][0].mean()), "b": float(cb[c][0].mean()), **M.paired_diff(ca[c][0], cb[c][0], n_boot, 0), "higher_better": ca[c][1]} for c in ca} | |
| def verdict(rows, higher_better, n_req=None): | |
| """baton_pretrain.md rule; n_req = number of seeds whose CI must exclude 0 (default ceil(2n/3), i.e. 2 of 3, 4 of 5).""" | |
| n = len(rows) | |
| if n == 0: | |
| return "INSUFFICIENT" | |
| n_req = math.ceil(2 * n / 3) if n_req is None else n_req | |
| d = np.array([r["mean"] for r in rows]); lo = np.array([r["ci_lo"] for r in rows]); hi = np.array([r["ci_hi"] for r in rows]) | |
| improving = (d > 0) if higher_better else (d < 0) | |
| excl = (lo > 0) | (hi < 0) | |
| same = np.all(np.sign(d) == np.sign(d.mean())) and d.mean() != 0 | |
| if excl.sum() >= n_req and same and improving.all(): | |
| return "BEATS" | |
| if excl.sum() >= n_req and same and (~improving).all(): | |
| return "WORSE" | |
| if np.all((lo <= 0) & (hi >= 0)) and np.all(np.abs(d) < TOL_NOT_WORSE): | |
| return "NOT_WORSE" | |
| return "INCONCLUSIVE" | |
| def fmt_ci(r, nd=2): | |
| return f"{r['mean']:+.{nd}f} [{r['ci_lo']:+.{nd}f},{r['ci_hi']:+.{nd}f}]" | |
| def ms(vals, nd): | |
| v = np.asarray([x for x in vals if x is not None], float) | |
| if not len(v): | |
| return "—" | |
| return f"{v.mean():.{nd}f} ± {v.std(ddof=1):.{nd}f}" if len(v) > 1 else f"{v[0]:.{nd}f}" | |
| def level_row(name, cols=LEVEL_COLS): | |
| m = metrics_test(name) | |
| if m is None: | |
| return None | |
| return " | ".join(f"{m[c]['value']:.{nd}f} [{m[c]['ci_lo']:.{nd}f}, {m[c]['ci_hi']:.{nd}f}]" for c, _, nd in cols) | |
| def run_meta(name): | |
| r = run_json(name) or {} | |
| return r.get("best_epoch", "—"), r.get("epochs_run", "—"), r.get("stop_reason", "—"), r.get("gpu_minutes"), r.get("n_params") | |
| def delta_table(per_seed, label_a, label_b, cells=None, n_req=None): | |
| seeds = list(per_seed) | |
| cells = cells or list(next(iter(per_seed.values()))) | |
| L = ["| cell | " + " | ".join(f"{label_a} s{k}" for k in seeds) + " | " + " | ".join(f"{label_b} s{k}" for k in seeds) + " | " | |
| + " | ".join(f"Δ s{k} [CI]" for k in seeds) + " | mean Δ ± sd | verdict |", "|---|" + "---|" * (3 * len(seeds)) + "---|---|"] | |
| res = {} | |
| for c in cells: | |
| rows = [per_seed[k][c] for k in seeds] | |
| v = verdict(rows, rows[0]["higher_better"], n_req) | |
| dm = np.array([r["mean"] for r in rows]) | |
| nd = 3 if c == "d-n" else 2 | |
| L.append(f"| {c} | " + " | ".join(f"{r['a']:.{nd}f}" for r in rows) + " | " + " | ".join(f"{r['b']:.{nd}f}" for r in rows) + " | " | |
| + " | ".join(fmt_ci(r, nd) for r in rows) + f" | {dm.mean():+.{nd}f} ± {(dm.std(ddof=1) if len(dm) > 1 else float('nan')):.{nd}f} | {v} |") | |
| res[c] = {"verdict": v, "mean": float(dm.mean()), "sd": float(dm.std(ddof=1)) if len(dm) > 1 else None, "per_seed": {str(k): per_seed[k][c] for k in seeds}} | |
| return L, res | |
| # ----------------------------------------------------------------------------- main | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--n-boot", type=int, default=10000) | |
| ap.add_argument("--out", default=str(OUT)) | |
| a = ap.parse_args() | |
| t0 = time.time() | |
| d = load_split("test") | |
| gt = d["S10"][:, P.T_OBS:] | |
| hm, thr, _ = M.hm_mask(gt) | |
| stamp = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime()) | |
| gpu = {} # run -> GPU minutes (from run json) | |
| summary = {} | |
| def note_gpu(name): | |
| r = run_json(name) | |
| if r is not None and r.get("gpu_minutes") is not None: | |
| gpu[name] = float(r["gpu_minutes"]) | |
| L = ["# AIDE architecture attribution — fair experiments (owner-approved 2026-09-14; rules fixed before the runs) — AIDE test, n = 609, paired per-clip bootstrap (10k)", "", | |
| f"Generated {stamp} by `aide/eval/report_arch_attribution.py` from pred_test.npz / metrics_test.json / run json / pretraining history files only (locked protocol: driver `first`, " | |
| f"grid `floor`, HM top {int(hm.sum())} (threshold {thr:.2f} px), PCK @ {P.pck_threshold_px(0.05):.0f} px; every run selected on val All-MPJPE only, test evaluated once by the harness). " | |
| "Δ = paired per-clip difference (negative = better for errors, positive = better for PCK), CI = 10k clip-bootstrap percentile interval. " | |
| f"**Verdict rule (as `baton_pretrain.md`): BEATS / WORSE = Δ CI excl. 0 on ≥ 2/3 of the seeds (n = 3: 2; n = 5: 4) with the same sign on every seed; NOT_WORSE = all CIs cover 0 and |Δ| < {TOL_NOT_WORSE}; else INCONCLUSIVE.**", "", | |
| "Pre-specified readings (docs/decisions.md 2026-09-14 row \"AIDE 架构归因公平实验\"): (1) Δ All-MPJPE BEATS / INCONCLUSIVE per baseline, gains listed next to K's −0.62 ± 0.17 (descriptive, no cross-architecture test); " | |
| "(2) Driver-WM-kp vs K per seed: CI ∋ 0 and |Δ| < 1 px on every seed → \"领先属于关键点表述\" (the lead belongs to the keypoint formulation); K significantly better on ≥ 2/3 seeds → " | |
| "\"支持架构在关键点表述下的优势\" (supports an architecture advantage under the keypoint formulation); Driver-WM-kp better → \"如实报告\" (reported as is); " | |
| "(3) K − MotionBERT-kp per seed over the 5-seed set, beats rule. **论文措辞按结果改：现有\"领先来自关键点锚定表述\"只在 (2)(3) 支持时才可改写为架构优势。**", ""] | |
| # ------------------------------------------------------------------ (1) matched pretraining | |
| L += ["## 1. Matched BATON in-cabin pose pretraining for the keypoint baselines (trunk-only init, pretraining seed 123) vs their EXISTING seed-init runs", ""] | |
| sec1 = {} | |
| k_pair = {k: paired(f"cwm3d_kp_batonpre_trunkp123_s{k}_v2", f"cwm3d_kp_s{k}_v2", d, gt, hm, a.n_boot) for k in SEEDS3} | |
| k_pair = {k: v for k, v in k_pair.items() if v} | |
| for model, label in (("motionbert_kp", "MotionBERT-kp"), ("stgcn_kp", "ST-GCN-kp")): | |
| pre = f"{model}_batonpre_pretrain_s123" | |
| pr = run_json(pre) | |
| L += [f"### 1.{1 if model == 'motionbert_kp' else 2} {label}", ""] | |
| if pr is None: | |
| L += [f"_(pretraining run `{pre}` not finished — no numbers)_", ""] | |
| sec1[model] = {"status": "PRETRAIN_MISSING"} | |
| continue | |
| note_gpu(pre) | |
| hist = pr.get("history", []) | |
| zs = pr.get("zeroshot") or {} | |
| L += [f"- Pretraining `{pre}` (`aide/train/pretrain_cwm3d_baton.py --model {model} --seed 123`: same corpus sha256 {pr['corpus']['corpus_sha256'][:16]}, {pr['corpus']['n_windows'] - pr['corpus']['n_windows_val']:,} fit windows, " | |
| f"same task / augmentation / optimiser / schedule / EMA / held-out-route selection as `cwm3d_kp_batonpre_pretrain_s123`; active loss terms {pr.get('loss_terms_active')} — the tensor-output baselines have no " | |
| f"Laplace / bone / jerk terms, exactly as in their AIDE recipe): {pr['n_params']:,} params, bs {pr['config']['bs']}, lr {pr['config']['lr']}, {pr['epochs_run']} epochs run, best epoch {pr['best_epoch']} " | |
| f"(held-out-route All-MPJPE {pr['best_corpus_val_all']:.2f} px), stop = {pr['stop_reason']}; **{pr['ms_per_step_mean']:.1f} ms/step, {pr['gpu_minutes']:.1f} GPU-min**.", | |
| "", "| epoch | ms/step | samples/s | train loss | corpus held-out All | corpus held-out HM | AIDE val zero-shot All | AIDE val zero-shot HM |", "|---|---|---|---|---|---|---|---|"] | |
| for h in hist: | |
| L.append(f"| {h['epoch']} | {h['ms_per_step']:.1f} | {h['samples_per_s']:.0f} | {h.get('train_total', float('nan')):.3f} | {h['corpus_val_all']:.2f} | {h['corpus_val_hm']:.2f} | {h['aide_val_zeroshot_all']:.3f} | {h['aide_val_zeroshot_hm']:.2f} |") | |
| if zs: | |
| L += ["", f"- Zero-shot run `{zs['name']}` (pretrained model on AIDE without any AIDE training, test once): test All {zs['test'].get('mpjpe_all', float('nan')):.3f} / HM {zs['test'].get('mpjpe_hm', float('nan')):.2f} / " | |
| f"PCK@0.05 {zs['test'].get('pck05', float('nan')):.2f} / HM h=5 {zs['test'].get('hm_h5', float('nan')):.2f}; Δ vs zero-velocity All {zs['test'].get('diff_vs_ref_all', float('nan')):+.2f}."] | |
| per_seed = {k: paired(f"{model}_batonpre_trunkp123_s{k}_v2", f"{model}_s{k}_v2", d, gt, hm, a.n_boot, full=True) for k in SEEDS3} | |
| per_seed = {k: v for k, v in per_seed.items() if v} | |
| if not per_seed: | |
| L += ["", f"_(no finished fine-tune pairs `{model}_batonpre_trunkp123_s<k>_v2` vs `{model}_s<k>_v2` yet)_", ""] | |
| sec1[model] = {"status": "FINETUNE_MISSING"} | |
| continue | |
| L += ["", "| seed | A run (trunk init) | A best epoch / epochs / stop / GPU min | init (tensors loaded / kept at seed init; kept keys) | B run (existing seed init, not retrained) | B best epoch / epochs / GPU min |", "|---|---|---|---|---|---|"] | |
| for k in per_seed: | |
| an, bn = f"{model}_batonpre_trunkp123_s{k}_v2", f"{model}_s{k}_v2" | |
| ra, rb = run_json(an) or {}, run_json(bn) or {} | |
| ini = ra.get("init") or {} | |
| note_gpu(an) | |
| L.append(f"| {k} | {an} | {ra.get('best_epoch', '—')} / {ra.get('epochs_run', '—')} / {ra.get('stop_reason', '—')} / {ra.get('gpu_minutes', '—')} | " | |
| f"{ini.get('parts', '—')}: {ini.get('n_loaded', '—')} / {ini.get('n_missing_or_kept', '—')} (ckpt epoch {ini.get('ckpt_epoch', '—')}; kept {', '.join(ini.get('kept_keys', [])[:6])}) | " | |
| f"{bn} | {rb.get('best_epoch', '—')} / {rb.get('epochs_run', '—')} / {rb.get('gpu_minutes', '—')} |") | |
| rows, res = delta_table(per_seed, "A", "B") | |
| L += [""] + rows | |
| v_all = res["All-MPJPE"]["verdict"] | |
| L += ["", f"**Reading (1) for {label}: Δ All-MPJPE = {v_all} (mean {res['All-MPJPE']['mean']:+.2f} ± {res['All-MPJPE']['sd']:.2f} px over {len(per_seed)} seeds); " | |
| f"HM-MPJPE {res['HM-MPJPE']['verdict']} ({res['HM-MPJPE']['mean']:+.2f}), PCK@0.05 {res['PCK@0.05']['verdict']} ({res['PCK@0.05']['mean']:+.2f}), HM h=5 {res['HM h=5']['verdict']} ({res['HM h=5']['mean']:+.2f}).**", ""] | |
| sec1[model] = {"status": "DONE", "n_seeds": len(per_seed), "verdict_all": v_all, "cells": {c: {kk: vv for kk, vv in r.items() if kk != "per_seed"} for c, r in res.items()}, "per_seed_all": res["All-MPJPE"]["per_seed"]} | |
| if k_pair: | |
| rows_k, res_k = delta_table(k_pair, "A", "B", cells=list(MAIN_CELLS)) | |
| L += ["### 1.3 Reference: K (CWM3-D-kp) trunk-only init from the same pretraining seed (`cwm3d_kp_batonpre_trunkp123_s<k>_v2` vs `cwm3d_kp_s<k>_v2`, recomputed from the same files; `baton_pretrain.md` §6)", ""] + rows_k | |
| L += ["", "### 1.4 Gain from matched pretraining, side by side (mean Δ All-MPJPE ± sd over seeds; DESCRIPTIVE — no cross-architecture test is performed)", "", | |
| "| model | params | Δ All-MPJPE (A − B) per seed | mean ± sd | verdict | Δ HM-MPJPE mean ± sd | Δ HM h=5 mean ± sd |", "|---|---|---|---|---|---|---|"] | |
| for model, label in (("cwm3d", "K = CWM3-D-kp"), ("motionbert_kp", "MotionBERT-kp"), ("stgcn_kp", "ST-GCN-kp")): | |
| if model == "cwm3d": | |
| res, rjs = res_k, run_json("cwm3d_kp_s42_v2") | |
| else: | |
| if sec1.get(model, {}).get("status") != "DONE": | |
| L.append(f"| {label} | — | (not finished) | — | — | — | — |") | |
| continue | |
| res = {c: {**sec1[model]["cells"][c], "per_seed": None} for c in sec1[model]["cells"]} | |
| res["All-MPJPE"]["per_seed"] = sec1[model]["per_seed_all"] | |
| rjs = run_json(f"{model}_s42_v2") | |
| ps = res["All-MPJPE"]["per_seed"] | |
| L.append(f"| {label} | {(rjs or {}).get('n_params', 0):,} | " + " / ".join(f"{ps[str(k)]['mean']:+.2f}" for k in SEEDS3 if str(k) in ps) + f" | {res['All-MPJPE']['mean']:+.2f} ± {res['All-MPJPE']['sd']:.2f} | {res['All-MPJPE']['verdict']} | " | |
| f"{res['HM-MPJPE']['mean']:+.2f} ± {res['HM-MPJPE']['sd']:.2f} | {res['HM h=5']['mean']:+.2f} ± {res['HM h=5']['sd']:.2f} |") | |
| summary["k_trunk_gain_all"] = {"mean": res_k["All-MPJPE"]["mean"], "sd": res_k["All-MPJPE"]["sd"], "verdict": res_k["All-MPJPE"]["verdict"]} | |
| L.append("") | |
| summary["exp1"] = sec1 | |
| # ------------------------------------------------------------------ (2) Driver-WM-kp | |
| L += ["## 2. Driver-WM-kp (Driver-WM reproduction with K's keypoint part tokens in the internal stream, exterior stream as in the reproduction, shared recipe; `aide/docs/driverwm_kp_port.md`) vs K and vs MotionBERT-kp, seed-matched", ""] | |
| dwm = {k: f"driverwm_kp_s{k}_v2" for k in SEEDS3} | |
| have2 = [k for k in SEEDS3 if metrics_test(dwm[k]) is not None and run_json(dwm[k]) is not None] | |
| sec2 = {"status": "MISSING"} | |
| if not have2: | |
| L += ["_(no finished `driverwm_kp_s<k>_v2` run yet)_", ""] | |
| else: | |
| L += ["| seed | run | params | best epoch / epochs / stop | GPU min | " + " | ".join(f"{lab} [CI]" for _, lab, _ in LEVEL_COLS) + " |", "|---|---|---|---|---|" + "---|" * len(LEVEL_COLS)] | |
| for k in have2: | |
| for name in (dwm[k], f"cwm3d_kp_s{k}_v2", f"motionbert_kp_s{k}_v2"): | |
| be, ep, st, gm, npar = run_meta(name) | |
| if name == dwm[k]: | |
| note_gpu(name) | |
| L.append(f"| {k} | {name} | {npar:,} | {be} / {ep} / {st} | {gm} | {level_row(name)} |") | |
| pk = {k: paired(dwm[k], f"cwm3d_kp_s{k}_v2", d, gt, hm, a.n_boot) for k in have2} | |
| pm = {k: paired(dwm[k], f"motionbert_kp_s{k}_v2", d, gt, hm, a.n_boot) for k in have2} | |
| pk = {k: v for k, v in pk.items() if v}; pm = {k: v for k, v in pm.items() if v} | |
| rows, res_k2 = delta_table(pk, "DWM-kp", "K") | |
| L += ["", "### 2.1 Paired Δ = Driver-WM-kp − K (cwm3d_kp_s<k>_v2), per seed", ""] + rows | |
| rows, res_m2 = delta_table(pm, "DWM-kp", "MB") | |
| L += ["", "### 2.2 Paired Δ = Driver-WM-kp − MotionBERT-kp (motionbert_kp_s<k>_v2), per seed", ""] + rows | |
| # pre-specified reading on Δ All vs K | |
| r_all = [res_k2["All-MPJPE"]["per_seed"][str(k)] for k in pk] | |
| d_ = np.array([r["mean"] for r in r_all]); lo = np.array([r["ci_lo"] for r in r_all]); hi = np.array([r["ci_hi"] for r in r_all]) | |
| cover = (lo <= 0) & (hi >= 0) | |
| n_req = math.ceil(2 * len(r_all) / 3) | |
| k_sig_better = int(((lo > 0)).sum()) >= n_req # Δ (DWM − K) > 0 with CI excl. 0 -> K better | |
| dwm_sig_better = int(((hi < 0)).sum()) >= n_req | |
| reading_a = bool(np.all(cover & (np.abs(d_) < 1.0))) | |
| if reading_a: | |
| reading = "(a) 领先属于关键点表述 — every seed's Δ All CI contains 0 and |Δ| < 1 px: the lead over the published Driver-WM belongs to the keypoint formulation, not to the CWM3-D architecture" | |
| elif k_sig_better: | |
| reading = "(b) 支持架构在关键点表述下的优势 — K is significantly better than Driver-WM-kp on ≥ 2/3 seeds (Δ All CI excl. 0, K lower)" | |
| elif dwm_sig_better: | |
| reading = "(c) Driver-WM-kp 更好，如实报告 — Driver-WM-kp is significantly better than K on ≥ 2/3 seeds (Δ All CI excl. 0)" | |
| else: | |
| reading = "none of the three pre-specified readings obtained (mixed: neither all-seeds-within-1-px-with-CI∋0 nor a ≥ 2/3-seed significant direction)" | |
| L += ["", f"**Reading (2): Δ All (Driver-WM-kp − K) per seed = " + "; ".join(f"s{k} {fmt_ci(res_k2['All-MPJPE']['per_seed'][str(k)])}" for k in pk) | |
| + f" (CI ∋ 0 on {int(cover.sum())}/{len(r_all)} seeds; |Δ| < 1 px on {int((np.abs(d_) < 1.0).sum())}/{len(r_all)}; K significantly better on {int((lo > 0).sum())}/{len(r_all)}; Driver-WM-kp significantly better on {int((hi < 0).sum())}/{len(r_all)}) → {reading}. " | |
| f"Verdicts (beats rule, sign from Driver-WM-kp's side): All {res_k2['All-MPJPE']['verdict']}, HM {res_k2['HM-MPJPE']['verdict']}, PCK {res_k2['PCK@0.05']['verdict']}, HM h=5 {res_k2['HM h=5']['verdict']}; vs MotionBERT-kp: All {res_m2['All-MPJPE']['verdict']}, HM {res_m2['HM-MPJPE']['verdict']}.**", ""] | |
| sec2 = {"status": "DONE", "n_seeds": len(pk), "reading": reading, "reading_key": "a" if reading_a else "b" if k_sig_better else "c" if dwm_sig_better else "none", | |
| "vs_k": {c: {kk: vv for kk, vv in r.items()} for c, r in res_k2.items()}, "vs_mb": {c: {kk: vv for kk, vv in r.items()} for c, r in res_m2.items()}} | |
| summary["exp2"] = sec2 | |
| # ------------------------------------------------------------------ (3) MotionBERT-kp 5 seeds | |
| L += ["## 3. MotionBERT-kp at 5 seeds (unchanged hardening-D recipe; seeds 789 / 1011 new) — paired K − MotionBERT-kp per seed", ""] | |
| mb = {k: f"motionbert_kp_s{k}_v2" for k in SEEDS5} | |
| kk_ = {k: f"cwm3d_kp_s{k}_v2" for k in SEEDS5} | |
| have3 = [k for k in SEEDS5 if metrics_test(mb[k]) is not None and metrics_test(kk_[k]) is not None] | |
| cols5 = LEVEL_COLS[:3] + [("dn_mpjpe_hm", "HM d-n", 3)] + LEVEL_COLS[3:4] + [("pck10", "PCK@0.10", 2)] + LEVEL_COLS[4:] | |
| L += ["| seed | status | model | run | best epoch / epochs / stop | GPU min | val All | " + " | ".join(lab for _, lab, _ in cols5) + " |", "|---|---|---|---|---|---|---|" + "---|" * len(cols5)] | |
| vals = {"K": {c: [] for c, _, _ in cols5}, "MB": {c: [] for c, _, _ in cols5}} | |
| for k in SEEDS5: | |
| for tag, name in (("K", kk_[k]), ("MB", mb[k])): | |
| mt = metrics_test(name) | |
| if mt is None: | |
| L.append(f"| {k} | {'locked' if k in SEEDS3 else 'NEW'} | {tag} | {name} | (not finished) | | | " + " | ".join("—" for _ in cols5) + " |") | |
| continue | |
| be, ep, st, gm, _ = run_meta(name) | |
| if tag == "MB" and k not in SEEDS3: | |
| note_gpu(name) | |
| mv = json.load(open(RUNS / name / "metrics_val.json"))["metrics"] if (RUNS / name / "metrics_val.json").exists() else {} | |
| L.append(f"| {k} | {'locked' if k in SEEDS3 else 'NEW'} | {tag} | {name} | {be} / {ep} / {st} | {gm} | {mv.get('mpjpe_all', {}).get('value', float('nan')):.3f} | " | |
| + " | ".join(f"{mt[c]['value']:.{nd}f}" for c, _, nd in cols5) + " |") | |
| if k in have3: | |
| for c, _, _ in cols5: | |
| vals[tag][c].append(mt[c]["value"]) | |
| L += ["", f"### 3.1 Seed means (n = {len(have3)} seeds with both runs finished: {have3})", "", "| model | n | " + " | ".join(lab for _, lab, _ in cols5) + " |", "|---|---|" + "---|" * len(cols5)] | |
| for tag, label in (("K", "CWM3-D-kp (K)"), ("MB", "MotionBERT-kp")): | |
| L.append(f"| {label} | {len(have3)} | " + " | ".join(ms(vals[tag][c], nd) for c, _, nd in cols5) + " |") | |
| loc = [k for k in have3 if k in SEEDS3] | |
| if len(loc) != len(have3): | |
| L.append(f"| {label}, locked 3 seeds only | {len(loc)} | " + " | ".join(ms([metrics_test(kk_[k] if tag == 'K' else mb[k])[c]['value'] for k in loc], nd) for c, _, nd in cols5) + " |") | |
| sec3 = {"status": "MISSING"} | |
| p5 = {k: paired(kk_[k], mb[k], d, gt, hm, a.n_boot) for k in have3} | |
| p5 = {k: v for k, v in p5.items() if v} | |
| if p5: | |
| rows5, res5 = delta_table(p5, "K", "MB") | |
| L += ["", f"### 3.2 Paired Δ = K − MotionBERT-kp per seed ({len(p5)} seeds; verdict threshold = CI excl. 0 on ≥ {math.ceil(2 * len(p5) / 3)}/{len(p5)} seeds, same sign on all)", ""] + rows5 | |
| loc = {k: v for k, v in p5.items() if k in SEEDS3} | |
| res3 = None | |
| if loc and len(loc) != len(p5): | |
| rows3, res3 = delta_table(loc, "K", "MB") | |
| L += ["", "### 3.3 The same on the locked 3 seeds only (42 / 123 / 456; CI excl. 0 on ≥ 2/3)", ""] + rows3 | |
| v_all5 = res5["All-MPJPE"]["verdict"] | |
| r_all = [res5["All-MPJPE"]["per_seed"][str(k)] for k in p5] | |
| n_k_better = int(sum(r["ci_hi"] < 0 for r in r_all)); n_mb_better = int(sum(r["ci_lo"] > 0 for r in r_all)) | |
| if v_all5 == "BEATS": | |
| reading3 = "K BEATS MotionBERT-kp on All-MPJPE (supports an architecture advantage under the keypoint formulation)" | |
| elif v_all5 == "WORSE": | |
| reading3 = "MotionBERT-kp BEATS K on All-MPJPE (the architecture advantage is NOT supported; reported as is)" | |
| else: | |
| reading3 = f"{v_all5}: no seed-consistent significant difference between K and MotionBERT-kp on All-MPJPE — the architecture advantage is NOT supported by (3)" | |
| L += ["", f"**Reading (3) over {len(p5)} seeds: Δ All (K − MotionBERT-kp) = " + "; ".join(f"s{k} {fmt_ci(res5['All-MPJPE']['per_seed'][str(k)])}" for k in p5) | |
| + f"; mean {res5['All-MPJPE']['mean']:+.2f} ± {res5['All-MPJPE']['sd']:.2f} px; K significantly better on {n_k_better}/{len(p5)} seeds, MotionBERT-kp significantly better on {n_mb_better}/{len(p5)} → {reading3}. " | |
| f"HM-MPJPE {res5['HM-MPJPE']['verdict']} ({res5['HM-MPJPE']['mean']:+.2f} ± {res5['HM-MPJPE']['sd']:.2f}), PCK@0.05 {res5['PCK@0.05']['verdict']} ({res5['PCK@0.05']['mean']:+.2f}), HM h=5 {res5['HM h=5']['verdict']} ({res5['HM h=5']['mean']:+.2f}).**", ""] | |
| sec3 = {"status": "DONE", "n_seeds": len(p5), "seeds": list(p5), "verdict_all_5": v_all5, "reading": reading3, "cells": {c: {kk: vv for kk, vv in r.items()} for c, r in res5.items()}, | |
| "locked3": None if res3 is None else {c: {"verdict": r["verdict"], "mean": r["mean"], "sd": r["sd"]} for c, r in res3.items()}, | |
| "means": {tag: {c: {"mean": float(np.mean(v)), "sd": float(np.std(v, ddof=1)) if len(v) > 1 else None, "n": len(v)} for c, v in vals[tag].items() if v} for tag in vals}} | |
| summary["exp3"] = sec3 | |
| # ------------------------------------------------------------------ outcome + GPU minutes | |
| L += ["## 4. Outcome (decisions-row wording)", ""] | |
| e1 = "; ".join(f"{m}: {'Δ All ' + v['verdict_all'] if v.get('status') == 'DONE' else v.get('status')}" for m, v in sec1.items()) | |
| L.append(f"- (1) matched pretraining — {e1}; K's trunk-only gain (same pretraining seed): {summary.get('k_trunk_gain_all', {}).get('mean', float('nan')):+.2f} ± {summary.get('k_trunk_gain_all', {}).get('sd', float('nan')):.2f} " | |
| f"({summary.get('k_trunk_gain_all', {}).get('verdict', '—')}). Descriptive comparison of gain magnitudes only.") | |
| L.append(f"- (2) Driver-WM-kp — {sec2.get('reading', sec2.get('status'))}.") | |
| L.append(f"- (3) MotionBERT-kp 5 seeds — {sec3.get('reading', sec3.get('status'))}.") | |
| arch_supported = (sec2.get("reading_key") == "b") and (sec3.get("verdict_all_5") == "BEATS") | |
| L.append(f"- **Paper wording: the existing claim \"the lead comes from the keypoint-anchored formulation\" may be rewritten as an ARCHITECTURE advantage only if (2) AND (3) support it → " | |
| f"{'SUPPORTED by both (2) and (3): may be rewritten' if arch_supported else 'NOT supported (' + ('(2) reading ' + str(sec2.get('reading_key')) + ', (3) ' + str(sec3.get('verdict_all_5'))) + '): the formulation wording stays'}.**") | |
| summary["architecture_advantage_supported"] = bool(arch_supported) | |
| L += ["", "## 5. Runs and GPU minutes (from the run json files; existing runs were not retrained)", "", "| run | GPU min |", "|---|---|"] | |
| for name, g in gpu.items(): | |
| L.append(f"| {name} | {g:.2f} |") | |
| L += [f"| **total (new runs)** | **{sum(gpu.values()):.1f}** |", "", | |
| "## Provenance", "", "- New run dirs only (`aide/results/runs/{motionbert_kp,stgcn_kp}_batonpre_pretrain_s123`, `*_batonpre_zeroshot_s123_v2`, `*_batonpre_trunkp123_s<k>_v2`, `motionbert_kp_s{789,1011}_v2`, `driverwm_kp_s<k>_v2`); " | |
| "the comparison runs `cwm3d_kp_s*_v2`, `motionbert_kp_s{42,123,456}_v2`, `stgcn_kp_s{42,123,456}_v2`, `cwm3d_kp_batonpre_trunkp123_s*_v2` are the existing ones.", | |
| "- Pretraining corpus: BATON train-split, train-only-device routes (no val / test device or route); AIDE side: TRAIN statistics only. No AIDE test clip entered any training or selection.", | |
| "- Launch gate before every job (free VRAM ≥ need + 4 GB, MemAvailable ≥ 28 GB, < 4 trainers, OMP/MKL 4): `aide/train/run_arch_attribution.sh`; pids in `aide/results/arch_attribution/pids.txt`.", ""] | |
| out = Path(a.out) | |
| out.parent.mkdir(parents=True, exist_ok=True) | |
| text = "\n".join(L) | |
| if out.exists(): # append-only: never edit an existing table | |
| with open(out, "a") as f: | |
| f.write(f"\n\n---\n\n<!-- appended regeneration {stamp} -->\n\n" + text) | |
| mode = "appended" | |
| else: | |
| out.write_text(text) | |
| mode = "written" | |
| js = out.with_suffix(".json") | |
| rec = {"stamp": stamp, "n_boot": a.n_boot, "summary": summary, "gpu_minutes": gpu} | |
| if js.exists(): | |
| old = json.load(open(js)) | |
| old = old if isinstance(old, list) else [old] | |
| js.write_text(json.dumps(old + [rec], indent=1, default=str)) | |
| else: | |
| js.write_text(json.dumps([rec], indent=1, default=str)) | |
| write_run_json(P.RESULTS / "tables", "report_arch_attribution", [Path(__file__), M.__file__, P.__file__], [P.CACHE_NPZ], seed=0, t_start=t0, input_listing=False, | |
| extra={"out": str(out), "mode": mode, "summary": {k: (v if k != "exp1" else {m: {kk: vv for kk, vv in s.items() if kk not in ("cells", "per_seed_all")} for m, s in v.items()}) for k, v in summary.items()}, | |
| "gpu_minutes": gpu, "dms_fields_used": []}) | |
| print(text) | |
| print(f"\n[{mode}] {out}") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 30.4 kB
- Xet hash:
- 47c7e6da14f10245e5c01804a89ae0e424fbb9706b29f7c938f04d2e040de600
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.