Buckets:
| """Parameter-free / analytic baselines on the AIDE protocol (observed frames 0-4 -> future 5-9). | |
| python aide/eval/baselines_analytic.py [--variant protocol|alt] [--n-boot 10000] | |
| Predictors (obs [N,5,136,3] -> pred [N,5,136,2]; NOTHING beyond frame index 4 is read): | |
| zero_velocity copy last observed pose | |
| const_vel_a{1,.5,.25} last + a*h*(last - prev), h = 1..5 | |
| anti_velocity_0.1 last - 0.1*(last - prev) (same for all horizons) | |
| mean_obs mean of the 5 observed poses | |
| per_part_copy face + hands: copy last; body: mean of observed | |
| zv_shrink_val (1-a)*last + a*mean_obs with a chosen on the VAL split (grid 0..1 step .01, | |
| criterion val All-MPJPE) — the honest analytic bar; TEST numbers reported. | |
| mean_pose_shrink_a{0.1,0.2,0.3} (1-a)*last + a*mu, mu = TRAIN-split mean pose of protocol frame 4 (per joint), | |
| same for every horizon (roadmap B0: the "shrink towards the population mean" family that | |
| reproduces the published Driver-WM / static-pooling operating points) | |
| zv_shrink_h_val horizon-dependent shrink towards mean_obs, alpha_h = a*h/5, a fitted on VAL All (grid 0..1) | |
| Each predictor writes results/runs/<name>/pred_test.npz, is scored by eval/harness.py (rows in | |
| results/tables/results.csv) and results/tables/analytic_baselines.md is regenerated from the | |
| harness output (no hand-typed numbers; paper rows come from protocol.PAPER / eval/paper_rows.py). | |
| --reuse: predictors whose results/runs/<name>/metrics_test.json already exists are loaded, not recomputed. | |
| Also written (roadmap B0): results/tables/tradeoff_all_vs_hm.md + .json (the "All vs HM h=5 trade-off" table: | |
| analytic predictors, our models' harness numbers, published rows) and results/tables/pareto_all_vs_hm.png | |
| (shrink curves evaluated on TEST for description only; every reported row keeps its val-fitted parameter). | |
| """ | |
| from __future__ import annotations | |
| import os | |
| # ops rule (docs/decisions.md 2026-09-10, host CPU saturation): cap BLAS/OMP threads before numpy import | |
| for _v in ("OMP_NUM_THREADS", "MKL_NUM_THREADS", "OPENBLAS_NUM_THREADS", "NUMEXPR_NUM_THREADS"): | |
| os.environ.setdefault(_v, "4") | |
| import argparse | |
| import json | |
| import sys | |
| import time | |
| from pathlib import Path | |
| import numpy as np | |
| ROOT = Path(__file__).resolve().parents[2] | |
| sys.path.insert(0, str(ROOT)) | |
| from aide.data import protocol as P # noqa: E402 | |
| from aide.eval import metrics as M # noqa: E402 | |
| from aide.eval.harness import load_split, run_harness, summary_line # noqa: E402 | |
| from aide.eval.paper_rows import GROUP, PAPER_HM5, PAPER_ROWS # noqa: E402 | |
| from common.runmeta import write_run_json # noqa: E402 | |
| # ----------------------------------------------------------------------------- predictors | |
| def _obs_xy(obs): | |
| obs = np.asarray(obs, dtype=np.float64) | |
| assert obs.shape[1] == P.T_OBS, "predictors take exactly the T_OBS observed frames" | |
| return obs[..., :2] | |
| def zero_velocity(obs): | |
| xy = _obs_xy(obs) | |
| return np.repeat(xy[:, -1:], P.T_PRED, axis=1) | |
| def const_velocity(obs, alpha=1.0): | |
| xy = _obs_xy(obs) | |
| v = xy[:, -1] - xy[:, -2] | |
| h = np.arange(1, P.T_PRED + 1, dtype=np.float64)[None, :, None, None] | |
| return xy[:, -1:] + alpha * h * v[:, None] | |
| def anti_velocity(obs, beta=0.1): | |
| xy = _obs_xy(obs) | |
| p = xy[:, -1] - beta * (xy[:, -1] - xy[:, -2]) | |
| return np.repeat(p[:, None], P.T_PRED, axis=1) | |
| def mean_obs(obs): | |
| xy = _obs_xy(obs) | |
| return np.repeat(xy.mean(1, keepdims=True), P.T_PRED, axis=1) | |
| def per_part_copy(obs): | |
| zv, mo = zero_velocity(obs), mean_obs(obs) | |
| out = zv.copy() | |
| out[:, :, P.PARTS["body"]] = mo[:, :, P.PARTS["body"]] | |
| return out | |
| def zv_shrink(obs, alpha): | |
| return (1.0 - alpha) * zero_velocity(obs) + alpha * mean_obs(obs) | |
| def mean_pose_shrink(obs, alpha, mu): | |
| """Shrink the last observed pose towards a FIXED population mean pose mu [J,2] (train split, frame 4).""" | |
| zv = zero_velocity(obs) | |
| return (1.0 - alpha) * zv + alpha * np.asarray(mu, dtype=np.float64)[None, None] | |
| def zv_shrink_h(obs, a): | |
| """Horizon-dependent shrink towards the observed mean: alpha_h = a * h / T_PRED, h = 1..T_PRED.""" | |
| xy = _obs_xy(obs) | |
| last, mo = xy[:, -1:], xy.mean(1, keepdims=True) | |
| alpha = a * np.arange(1, P.T_PRED + 1, dtype=np.float64)[None, :, None, None] / P.T_PRED | |
| return (1.0 - alpha) * last + alpha * mo | |
| def train_mean_pose(S10_train): | |
| """mu [J,2] = mean over TRAIN clips of the last observed protocol frame (index T_OBS-1 = 4).""" | |
| return np.asarray(S10_train, dtype=np.float64)[:, P.T_OBS - 1, :, :2].mean(0) | |
| PREDICTORS = { | |
| "zero_velocity": zero_velocity, | |
| "const_vel_a1.0": lambda o: const_velocity(o, 1.0), | |
| "const_vel_a0.5": lambda o: const_velocity(o, 0.5), | |
| "const_vel_a0.25": lambda o: const_velocity(o, 0.25), | |
| "anti_velocity_0.1": anti_velocity, | |
| "mean_obs": mean_obs, | |
| "per_part_copy": per_part_copy, | |
| } | |
| def fit_shrink_alpha(S10_val, grid=None, predictor=zv_shrink): | |
| """Grid-search alpha on the val split (criterion: All-MPJPE) for predictor(obs, alpha). Returns (alpha, table).""" | |
| grid = np.round(np.linspace(0, 1, 101), 2) if grid is None else grid | |
| obs, fut = S10_val[:, :P.T_OBS], S10_val[:, P.T_OBS:] | |
| rows = [] | |
| for a in grid: | |
| err = M.joint_errors(predictor(obs, a), fut) | |
| m, _, _ = M.hm_mask(fut) | |
| rows.append({"alpha": float(a), "val_mpjpe_all": M.mpjpe(err), "val_mpjpe_hm": M.mpjpe(err[m]), | |
| "val_hm_h5": float(err[m][:, -1].mean())}) | |
| best = min(rows, key=lambda r: r["val_mpjpe_all"]) | |
| best_hm = min(rows, key=lambda r: r["val_mpjpe_hm"]) | |
| return best["alpha"], {"criterion": "val_mpjpe_all", "chosen_alpha": best["alpha"], | |
| "alpha_min_val_hm": best_hm["alpha"], "grid": rows} | |
| # ----------------------------------------------------------------------------- shrink curves / trade-off (roadmap B0) | |
| def quick_metrics(pred, S10): | |
| """Bootstrap-free All / d-n / HM / HM d-n / HM h=5 / h=5 of a prediction on a split (descriptive curves only).""" | |
| fut = S10[:, P.T_OBS:] | |
| err = M.joint_errors(pred, fut) | |
| m, _, _ = M.hm_mask(fut) | |
| return {"mpjpe_all": M.mpjpe(err), "dn_mpjpe": M.dn_mpjpe(err), "mpjpe_hm": M.mpjpe(err[m]), "dn_mpjpe_hm": M.dn_mpjpe(err[m]), | |
| "hm_h5": float(err[m][:, -1].mean()), "mpjpe_h5": float(err[:, -1].mean()), "pck05": M.pck(err, 0.05)} | |
| def shrink_curves(split_d, mu, grid=None): | |
| """Descriptive curves on one split: mean-pose shrink (towards the TRAIN mean pose) and ZV-shrink (towards mean_obs) | |
| for alpha in grid. Returned as {family: [{alpha, metrics...}]}.""" | |
| grid = np.round(np.linspace(0, 1, 21), 2) if grid is None else grid | |
| obs = split_d["S10"][:, :P.T_OBS] | |
| out = {"mean_pose_shrink": [], "zv_shrink": []} | |
| for a in grid: | |
| out["mean_pose_shrink"].append({"alpha": float(a), **quick_metrics(mean_pose_shrink(obs, a, mu), split_d["S10"])}) | |
| out["zv_shrink"].append({"alpha": float(a), **quick_metrics(zv_shrink(obs, a), split_d["S10"])}) | |
| return out | |
| OUR_MODELS = [ # label, runs (3 seeds where available) -- harness numbers loaded from each run's latest metrics_test.json | |
| ("SiMLPe-kp (ours, shared recipe)", [f"simlpe_kp_s{s}_v2" for s in (42, 123, 456)]), | |
| ("MotionBERT-kp (ours, shared recipe)", [f"motionbert_kp_s{s}_v2" for s in (42, 123, 456)]), | |
| ("ST-GCN-kp (ours, shared recipe)", [f"stgcn_kp_s{s}_v2" for s in (42, 123, 456)]), | |
| ("CWM3-D-kp (5.8 M)", [f"cwm3d_kp_s{s}_v2" for s in (42, 123, 456)]), | |
| ("CWM3-D vis+sem (6.3 M)", [f"cwm3d_vis_sem_s{s}_v2" for s in (42, 123, 456)]), | |
| ("CWM3-D vis+sem, motion-weighted (Stage C ii)", [f"cwm3d_vis_sem_mw_s{s}_v2" for s in (42, 123, 456)]), | |
| ("CWM3-D vis+sem, motion weight 5 + horizon weights (B1 i)", [f"cwm3d_vis_sem_mw5hw_s{s}_v2" for s in (42, 123, 456)]), | |
| ("CWM3-D vis+sem, motion oversampling k=2 (B1 ii)", [f"cwm3d_vis_sem_mos2_s{s}_v2" for s in (42, 123, 456)]), | |
| ("CWM3-D vis+sem, late-horizon refinement (B3)", [f"cwm3d_vis_sem_rl_s{s}_v2" for s in (42, 123, 456)]), | |
| ("CWM3-D vis+sem 3-seed ENSEMBLE (not a single model)", ["cwm3d_vis_sem_ens3_v2"]), | |
| ("Driver-WM reproduction (ours, C1)", [f"driverwm_repro_s{s}_v2" for s in (42, 123, 456)]), | |
| ] | |
| TRADEOFF_KEYS = ("mpjpe_all", "dn_mpjpe", "mpjpe_hm", "dn_mpjpe_hm", "hm_h5", "mpjpe_h5", "pck05") | |
| def load_run_metrics(run): | |
| d = P.RESULTS / "runs" / run | |
| rj = d / f"run_{run}_latest.json" | |
| if rj.exists(): | |
| j = json.loads(rj.read_text()) | |
| if j.get("test_metrics_json") and Path(j["test_metrics_json"]).exists(): | |
| return json.loads(Path(j["test_metrics_json"]).read_text()) | |
| c = sorted(d.glob("metrics_test*.json")) if d.exists() else [] | |
| return json.loads(c[-1].read_text()) if c else None | |
| def our_rows(): | |
| rows = [] | |
| for label, runs in OUR_MODELS: | |
| ms = [m for m in (load_run_metrics(r) for r in runs) if m] | |
| if not ms: | |
| continue | |
| vals = {k: [m["metrics"][k]["value"] for m in ms if k in m["metrics"]] for k in TRADEOFF_KEYS} | |
| rows.append({"label": label, "n": len(ms), "runs": [r for r in runs if load_run_metrics(r)], | |
| **{k: float(np.mean(v)) if v else float("nan") for k, v in vals.items()}, | |
| **{f"{k}_sd": float(np.std(v, ddof=1)) if len(v) > 1 else float("nan") for k, v in vals.items()}}) | |
| return rows | |
| def tradeoff_table(analytic, ours, curves_test, curves_val, fits): | |
| f = lambda x, nd=2: "–" if x is None or (isinstance(x, float) and not np.isfinite(x)) else f"{x:.{nd}f}" # noqa: E731 | |
| L = ["# All-MPJPE vs High-Motion h=5 trade-off (AIDE test, n=609; roadmap supplement B, item B0)", "", | |
| f"Generated {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())} by aide/eval/baselines_analytic.py. Analytic rows and our models: " | |
| "harness output (metrics_test.json; group rows = mean over seeds). Published rows: typed verbatim from Driver-WM " | |
| "(aide/eval/paper_rows.py; HM h=5 quoted only for the main model and zero-velocity). Shrink parameters of the reported " | |
| "analytic rows are fixed a priori (a = 0.1 / 0.2 / 0.3) or fitted on VAL; the curves in the figure are evaluated on test " | |
| "for description only and are not selected on.", "", | |
| "| row | kind | All | d-n | HM | HM d-n | HM h=5 | h=5 (All) | PCK@0.05 |", "|---|---|---|---|---|---|---|---|---|"] | |
| for r in analytic: | |
| m = r["metrics"] | |
| g = lambda k: m[k]["value"] if k in m else float("nan") # noqa: E731 | |
| L.append(f"| {r['run']} | analytic (0 params) | {f(g('mpjpe_all'))} | {f(g('dn_mpjpe'), 3)} | {f(g('mpjpe_hm'))} | {f(g('dn_mpjpe_hm'), 3)} | " | |
| f"{f(g('hm_h5'))} | {f(g('mpjpe_h5'))} | {f(g('pck05'))} |") | |
| for r in ours: | |
| L.append(f"| {r['label']} | ours, n={r['n']} | {f(r['mpjpe_all'])} | {f(r['dn_mpjpe'], 3)} | {f(r['mpjpe_hm'])} | {f(r['dn_mpjpe_hm'], 3)} | " | |
| f"{f(r['hm_h5'])} | {f(r['mpjpe_h5'])} | {f(r['pck05'])} |") | |
| for name, v in PAPER_ROWS.items(): | |
| L.append(f"| paper {name} | published ({GROUP[name]}) | {f(v['mpjpe_all'])} | {f(v['dn_mpjpe'])} | {f(v['mpjpe_hm'])} | {f(v['dn_mpjpe_hm'])} | " | |
| f"{f(PAPER_HM5.get(name))} | – | {f(v['pck05'])} |") | |
| # shrink curve at the published operating points (descriptive) | |
| L += ["", "## Mean-pose shrink curve (descriptive; test and val; mu = TRAIN mean pose of frame 4)", "", | |
| "| a | test All | test HM | test HM h=5 | val All | val HM | val HM h=5 |", "|---|---|---|---|---|---|---|"] | |
| for ct, cv in zip(curves_test["mean_pose_shrink"], curves_val["mean_pose_shrink"]): | |
| L.append(f"| {ct['alpha']:.2f} | {ct['mpjpe_all']:.2f} | {ct['mpjpe_hm']:.2f} | {ct['hm_h5']:.2f} | {cv['mpjpe_all']:.2f} | {cv['mpjpe_hm']:.2f} | {cv['hm_h5']:.2f} |") | |
| L += ["", "## ZV-shrink curve (towards the observed mean; descriptive)", "", | |
| "| α | test All | test HM | test HM h=5 | val All | val HM | val HM h=5 |", "|---|---|---|---|---|---|---|"] | |
| for ct, cv in zip(curves_test["zv_shrink"], curves_val["zv_shrink"]): | |
| L.append(f"| {ct['alpha']:.2f} | {ct['mpjpe_all']:.2f} | {ct['mpjpe_hm']:.2f} | {ct['hm_h5']:.2f} | {cv['mpjpe_all']:.2f} | {cv['mpjpe_hm']:.2f} | {cv['hm_h5']:.2f} |") | |
| L += ["", f"Val-fitted parameters: zv_shrink α = {fits['zv_shrink']['chosen_alpha']:.2f}; zv_shrink_h a = {fits['zv_shrink_h']['chosen_alpha']:.2f} " | |
| f"(α_h = a·h/5). Figure: `pareto_all_vs_hm.png`.", ""] | |
| return "\n".join(L) + "\n" | |
| def _codes(analytic, ours): | |
| """Short codes for the crowded lower-left cluster (published rows keep their names).""" | |
| codes = {} | |
| for r in analytic: | |
| codes[r["run"]] = {"zero_velocity": "Z", "zv_shrink_val": "Zs", "zv_shrink_h_val": "Zh", "mean_obs": "Mo", | |
| "mean_pose_shrink_a0.1": "P1", "mean_pose_shrink_a0.2": "P2", "mean_pose_shrink_a0.3": "P3"}.get(r["run"], r["run"]) | |
| short = {"SiMLPe-kp": "S", "MotionBERT-kp": "M", "ST-GCN-kp": "G", "CWM3-D-kp": "K", "CWM3-D vis+sem": "V", | |
| "CWM3-D vis+sem, motion-weighted": "Vmw", "CWM3-D vis+sem, motion weight 5 + horizon weights": "Vmw5", | |
| "CWM3-D vis+sem, motion oversampling k=2": "Vos", "CWM3-D vis+sem, late-horizon refinement": "Vrl", | |
| "CWM3-D vis+sem 3-seed ENSEMBLE": "Ve"} | |
| for r in ours: | |
| codes[r["label"]] = short.get(r["label"].split(" (")[0], r["label"][:6]) | |
| return codes | |
| def pareto_figure(analytic, ours, curves_test, out_png): | |
| """Two panels: (a) All vs HM-MPJPE for every row that reports both; (b) All vs HM h=5 for rows that report it. | |
| Lower-left is better in both. Palette: dataviz reference categorical hues (validated), text in ink tokens.""" | |
| import matplotlib | |
| matplotlib.use("Agg") | |
| import matplotlib.pyplot as plt | |
| analytic = [r for r in analytic if not r["run"].startswith("const_vel")] # off-scale outlier; stays in the table | |
| codes = _codes(analytic, ours) | |
| key = "; ".join(f"{c} = {n}" for n, c in codes.items() if c != n) | |
| ink, muted, grid = "#0b0b0b", "#52514e", "#e5e5e2" # dataviz reference palette (text tokens) | |
| c_pub, c_ours, c_ana, c_curve1, c_curve2 = "#9a9a96", "#2a78d6", "#eb6834", "#199e70", "#4a3aa7" # neutral grey (reference rows) + series slots 1, 2, 3 (dark step, 3:1 contrast), 7 -- validated with scripts/validate_palette.js: all checks pass | |
| fig, axes = plt.subplots(1, 2, figsize=(13, 5.4), dpi=150) | |
| for ax in axes: | |
| ax.grid(True, color=grid, linewidth=0.6) | |
| ax.set_axisbelow(True) | |
| for sp in ("top", "right"): | |
| ax.spines[sp].set_visible(False) | |
| for sp in ("left", "bottom"): | |
| ax.spines[sp].set_color(muted) | |
| ax.tick_params(colors=muted, labelsize=8) | |
| ax = axes[0] | |
| ax.set_title("(a) All-MPJPE vs High-Motion MPJPE (test, lower-left is better)", color=ink, fontsize=10, loc="left") | |
| ax.set_xlabel("All-MPJPE (px)", color=ink, fontsize=9) | |
| ax.set_ylabel("HM-MPJPE (px)", color=ink, fontsize=9) | |
| cm = curves_test["mean_pose_shrink"] | |
| cz = curves_test["zv_shrink"] | |
| ax.plot([c["mpjpe_all"] for c in cm], [c["mpjpe_hm"] for c in cm], color=c_curve1, linewidth=2, label="mean-pose shrink curve a ∈ [0, 1] (test, descriptive)") | |
| ax.plot([c["mpjpe_all"] for c in cz], [c["mpjpe_hm"] for c in cz], color=c_curve2, linewidth=2, linestyle="--", label="ZV-shrink curve α ∈ [0, 1] (test, descriptive)") | |
| for name, v in PAPER_ROWS.items(): | |
| if v["mpjpe_all"] is None: | |
| continue | |
| ax.scatter(v["mpjpe_all"], v["mpjpe_hm"], s=34, color=c_pub, edgecolor="white", linewidth=1, zorder=3) | |
| ax.annotate(name, (v["mpjpe_all"], v["mpjpe_hm"]), xytext=(4, 3), textcoords="offset points", fontsize=6.5, color=muted) | |
| for r in analytic: | |
| m = r["metrics"] | |
| ax.scatter(m["mpjpe_all"]["value"], m["mpjpe_hm"]["value"], s=40, marker="^", color=c_ana, edgecolor="white", linewidth=1, zorder=4) | |
| ax.annotate(codes[r["run"]], (m["mpjpe_all"]["value"], m["mpjpe_hm"]["value"]), xytext=(3, -8), textcoords="offset points", fontsize=6.5, color=ink) | |
| for r in ours: | |
| if "reproduction" in r["label"]: | |
| continue | |
| ax.scatter(r["mpjpe_all"], r["mpjpe_hm"], s=46, marker="s", color=c_ours, edgecolor="white", linewidth=1, zorder=5) | |
| ax.annotate(codes[r["label"]], (r["mpjpe_all"], r["mpjpe_hm"]), xytext=(3, 3), textcoords="offset points", fontsize=6.5, color=ink) | |
| ax.scatter([], [], s=34, color=c_pub, label="published Driver-WM table rows (18)") | |
| ax.scatter([], [], s=40, marker="^", color=c_ana, label="analytic predictors (ours, 0 params)") | |
| ax.scatter([], [], s=46, marker="s", color=c_ours, label="our models (test, seed means)") | |
| ax.legend(fontsize=7, frameon=False, loc="upper left", labelcolor=ink) | |
| ax = axes[1] | |
| ax.set_title("(b) All-MPJPE vs High-Motion h=5 (rows that report it)", color=ink, fontsize=10, loc="left") | |
| ax.set_xlabel("All-MPJPE (px)", color=ink, fontsize=9) | |
| ax.set_ylabel("HM MPJPE at h = 5 (px)", color=ink, fontsize=9) | |
| ax.plot([c["mpjpe_all"] for c in cm], [c["hm_h5"] for c in cm], color=c_curve1, linewidth=2) | |
| ax.plot([c["mpjpe_all"] for c in cz], [c["hm_h5"] for c in cz], color=c_curve2, linewidth=2, linestyle="--") | |
| for name, h5 in PAPER_HM5.items(): | |
| v = PAPER_ROWS[name] | |
| ax.scatter(v["mpjpe_all"], h5, s=34, color=c_pub, edgecolor="white", linewidth=1, zorder=3) | |
| ax.annotate(f"paper {name} ({h5})", (v["mpjpe_all"], h5), xytext=(4, 3), textcoords="offset points", fontsize=6.5, color=muted) | |
| for r in analytic: | |
| m = r["metrics"] | |
| ax.scatter(m["mpjpe_all"]["value"], m["hm_h5"]["value"], s=40, marker="^", color=c_ana, edgecolor="white", linewidth=1, zorder=4) | |
| ax.annotate(codes[r["run"]], (m["mpjpe_all"]["value"], m["hm_h5"]["value"]), xytext=(3, -8), textcoords="offset points", fontsize=6.5, color=ink) | |
| for r in ours: | |
| if "reproduction" in r["label"]: | |
| continue | |
| ax.scatter(r["mpjpe_all"], r["hm_h5"], s=46, marker="s", color=c_ours, edgecolor="white", linewidth=1, zorder=5) | |
| ax.annotate(codes[r["label"]], (r["mpjpe_all"], r["hm_h5"]), xytext=(3, 3), textcoords="offset points", fontsize=6.5, color=ink) | |
| ax.axhline(PAPER_HM5["Driver-WM main"], color=c_pub, linewidth=1, linestyle=":") | |
| ax.set_ylim(150, 185) | |
| ax.set_xlim(44, 110) | |
| axes[0].set_xlim(44, 115) | |
| fig.text(0.01, 0.03, "Codes: " + key, fontsize=6.5, color=ink, wrap=True) | |
| fig.text(0.01, 0.005, "Curves: parameter-free shrink families evaluated on the test split for description only (no selection). " | |
| "Reported analytic rows use a priori or VAL-fitted parameters. HM subset = 60 test clips (CI half-width ≈ ±51 px at h=5).", | |
| fontsize=6.5, color=muted) | |
| fig.tight_layout(rect=(0, 0.06, 1, 1)) | |
| fig.savefig(out_png) | |
| plt.close(fig) | |
| # ----------------------------------------------------------------------------- driver | |
| def write_pred(run_dir: Path, split: str, pred, clip_ids): | |
| run_dir.mkdir(parents=True, exist_ok=True) | |
| p = run_dir / f"pred_{split}.npz" | |
| np.savez(p, pred=pred.astype(np.float32), clip_id=np.asarray(clip_ids).astype(str), split=np.array(split)) | |
| return p | |
| def md_table(recs, variant): | |
| paper = P.PAPER | |
| cols = [("mpjpe_all", "All-MPJPE"), ("dn_mpjpe", "d-nMPJPE"), ("mpjpe_hm", "HM-MPJPE"), ("dn_mpjpe_hm", "HM d-nMPJPE"), | |
| ("pck05", "PCK@0.05"), ("pck10", "PCK@0.10"), ("mpjpe_h1", "h=1"), ("mpjpe_h5", "h=5"), | |
| ("hm_h5", "HM h=5"), ("diff_vs_ref_all", "Δ vs ZV (All)"), ("diff_vs_ref_hm", "Δ vs ZV (HM)")] | |
| lines = [f"# AIDE analytic baselines — official test split (n={recs[0]['n']}, variant={variant})", "", | |
| f"Protocol: driver rule `{P.DRIVER_RULE}`, grid `{P.SAMPLE_GRID}` ({P.sample_idx(45).tolist()}), " | |
| f"HM = top {P.hm_top(recs[0]['n'])} clips by mean displacement over {P.HM_N_TRANSITIONS} future " | |
| f"transitions (threshold {recs[0]['hm_threshold']:.2f} px), PCK threshold t·max(W,H) " | |
| f"({P.pck_threshold_px(0.05):.0f} px @0.05). 95% CIs = clip-level bootstrap " | |
| f"({recs[0]['protocol']['n_boot']} resamples). Δ vs ZV = paired per-clip difference (negative = better).", | |
| f"Generated {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())} by aide/eval/baselines_analytic.py; " | |
| "all numbers come from eval/harness.py output (metrics_test.json of each run).", "", | |
| "| run | " + " | ".join(c[1] for c in cols) + " |", "|---|" + "---|" * len(cols)] | |
| for r in recs: | |
| m = r["metrics"] | |
| cells = [] | |
| for k, _ in cols: | |
| if k not in m: | |
| cells.append("–") | |
| elif k.startswith("diff"): | |
| cells.append(f"{m[k]['value']:+.2f} [{m[k]['ci_lo']:+.2f}, {m[k]['ci_hi']:+.2f}]") | |
| elif k in ("mpjpe_all", "mpjpe_hm"): | |
| cells.append(f"{m[k]['value']:.2f} [{m[k]['ci_lo']:.2f}, {m[k]['ci_hi']:.2f}]") | |
| elif k in ("dn_mpjpe", "dn_mpjpe_hm"): | |
| cells.append(f"{m[k]['value']:.3f}") | |
| else: | |
| cells.append(f"{m[k]['value']:.2f}") | |
| lines.append(f"| {r['run']} | " + " | ".join(cells) + " |") | |
| zp, dw, sp = paper["zero_velocity"], paper["driver_wm"], paper["static_pooling"] | |
| lines += ["", "Published (Driver-WM Table 1, typed from the paper into protocol.PAPER):", "", | |
| "| source | All-MPJPE | d-nMPJPE | HM-MPJPE | HM d-nMPJPE | PCK@0.05 | HM h=5 |", "|---|---|---|---|---|---|---|", | |
| f"| paper zero-velocity | {zp['mpjpe_all']} | {zp['dn_mpjpe']} | {zp['mpjpe_hm']} | {PAPER_ROWS['Zero-Velocity']['dn_mpjpe_hm']} | {zp['pck05']} | – |", | |
| f"| paper Driver-WM | {dw['mpjpe_all']} | {dw['dn_mpjpe']} | {dw['mpjpe_hm']} | {PAPER_ROWS['Driver-WM main']['dn_mpjpe_hm']} | {dw['pck05']} | {dw['hm_h5']} |", | |
| f"| paper static pooling (best HM) | – | – | {sp['mpjpe_hm']} | {PAPER_ROWS['Static pooling']['dn_mpjpe_hm']} | – | – |", | |
| "", "Roadmap B0 rows: `mean_pose_shrink_a{0.1,0.2,0.3}` shrink the last observed pose towards the TRAIN-split mean pose " | |
| "(protocol frame 4, per joint; a fixed a priori); `zv_shrink_h_val` uses α_h = a·h/5 towards the observed mean with a " | |
| "fitted on val All. Trade-off table and Pareto figure: `tradeoff_all_vs_hm.md`, `pareto_all_vs_hm.png`."] | |
| return "\n".join(lines) + "\n" | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--variant", default="protocol", choices=("protocol", "alt")) | |
| ap.add_argument("--n-boot", type=int, default=10000) | |
| ap.add_argument("--only", default="", help="comma list of predictor names (debug)") | |
| ap.add_argument("--reuse", action="store_true", help="load existing metrics_test.json of a predictor instead of recomputing") | |
| ap.add_argument("--no-figure", action="store_true") | |
| a = ap.parse_args() | |
| t0 = time.time() | |
| suffix = "" if a.variant == "protocol" else f"_{a.variant}" | |
| test, val, train = load_split("test", a.variant), load_split("val", a.variant), load_split("train", a.variant) | |
| obs_test = test["S10"][:, :P.T_OBS] | |
| runs_dir = P.RESULTS / "runs" | |
| recs, extra = [], {"variant": a.variant} | |
| preds = dict(PREDICTORS) | |
| alpha, fit = fit_shrink_alpha(val["S10"]) | |
| extra["zv_shrink_fit"] = fit | |
| print(f"zv_shrink: alpha chosen on val = {alpha:.2f} (val All {min(r['val_mpjpe_all'] for r in fit['grid']):.3f}); " | |
| f"alpha minimising val HM would be {fit['alpha_min_val_hm']:.2f}") | |
| preds[f"zv_shrink_val"] = lambda o, _a=alpha: zv_shrink(o, _a) | |
| # roadmap B0: shrink towards the TRAIN mean pose (a fixed a priori) and horizon-dependent ZV-shrink (a fitted on val) | |
| mu = train_mean_pose(train["S10"]) | |
| extra["train_mean_pose_n"] = int(len(train["S10"])) | |
| for a_ in (0.1, 0.2, 0.3): | |
| preds[f"mean_pose_shrink_a{a_}"] = lambda o, _a=a_, _mu=mu: mean_pose_shrink(o, _a, _mu) | |
| alpha_h, fit_h = fit_shrink_alpha(val["S10"], predictor=zv_shrink_h) | |
| extra["zv_shrink_h_fit"] = fit_h | |
| print(f"zv_shrink_h: a chosen on val = {alpha_h:.2f} (alpha_h = a*h/5; val All {min(r['val_mpjpe_all'] for r in fit_h['grid']):.3f})") | |
| preds["zv_shrink_h_val"] = lambda o, _a=alpha_h: zv_shrink_h(o, _a) | |
| param_note = {"zv_shrink_val": f"analytic;alpha={alpha:.2f}(val-fitted)", "zv_shrink_h_val": f"analytic;a={alpha_h:.2f}(val-fitted);alpha_h=a*h/5", | |
| **{f"mean_pose_shrink_a{a_}": f"analytic;a={a_}(a priori);mu=train_mean_frame4" for a_ in (0.1, 0.2, 0.3)}} | |
| if a.only: | |
| preds = {k: v for k, v in preds.items() if k in a.only.split(",")} | |
| val_quick = {} | |
| for name, fn in preds.items(): | |
| run = f"{name}{suffix}" | |
| rd = runs_dir / run | |
| notes = param_note.get(name, "analytic") | |
| existing = rd / ("metrics_test.json" if a.variant == "protocol" else f"metrics_test_{a.variant}.json") | |
| val_quick[run] = quick_metrics(fn(val["S10"][:, :P.T_OBS]), val["S10"]) # bootstrap-free val numbers (descriptive) | |
| if a.reuse and existing.exists(): | |
| rec = json.loads(existing.read_text()) | |
| rec["reused"] = True | |
| print(f"{run}: reused {existing}", flush=True) | |
| recs.append(rec) | |
| continue | |
| pt = write_pred(rd, "test", fn(obs_test), test["clip_id"]) | |
| rec = run_harness(pt, run, name, seed=None, split="test", notes=notes, n_boot=a.n_boot, variant=a.variant) | |
| print(summary_line(rec), flush=True) | |
| recs.append(rec) | |
| if name in ("zero_velocity", "zv_shrink_val", "zv_shrink_h_val") or name.startswith("mean_pose_shrink"): # val numbers on record | |
| pv = write_pred(rd, "val", fn(val["S10"][:, :P.T_OBS]), val["clip_id"]) | |
| rv = run_harness(pv, run, name, seed=None, split="val", notes=notes, n_boot=a.n_boot, variant=a.variant) | |
| print(summary_line(rv), flush=True) | |
| if name == "zv_shrink_val": | |
| (rd / "alpha_fit_val.json").write_text(json.dumps(fit, indent=1)) | |
| if name == "zv_shrink_h_val": | |
| (rd / "alpha_fit_val.json").write_text(json.dumps(fit_h, indent=1)) | |
| extra["val_quick"] = val_quick | |
| md = md_table(recs, a.variant) | |
| out_md = P.RESULTS / "tables" / f"analytic_baselines{suffix}.md" | |
| out_md.write_text(md) | |
| print(md) | |
| extra["runs"] = {r["run"]: {k: r["metrics"][k]["value"] for k in ("mpjpe_all", "dn_mpjpe", "mpjpe_hm", "dn_mpjpe_hm", "pck05", "hm_h5") if k in r["metrics"]} | |
| for r in recs} | |
| if a.variant == "protocol": | |
| # roadmap B0: trade-off table + Pareto figure (analytic rows from this run's harness output, our models from their | |
| # metrics_test.json, published rows from paper_rows.py; curves descriptive) | |
| curves_test, curves_val = shrink_curves(test, mu), shrink_curves(val, mu) | |
| ours = our_rows() | |
| fits = {"zv_shrink": fit, "zv_shrink_h": fit_h} | |
| analytic_rows = [r for r in recs if r["run"] in ("zero_velocity", "zv_shrink_val", "zv_shrink_h_val", "mean_pose_shrink_a0.1", | |
| "mean_pose_shrink_a0.2", "mean_pose_shrink_a0.3", "mean_obs", "const_vel_a0.25")] | |
| tmd = tradeoff_table(analytic_rows, ours, curves_test, curves_val, fits) | |
| (P.RESULTS / "tables" / "tradeoff_all_vs_hm.md").write_text(tmd) | |
| print(tmd) | |
| tj = {"analytic": {r["run"]: {k: r["metrics"][k]["value"] for k in TRADEOFF_KEYS if k in r["metrics"]} for r in analytic_rows}, | |
| "ours": ours, "curves_test": curves_test, "curves_val": curves_val, "val_quick": val_quick, | |
| "fits": {"zv_shrink_alpha": alpha, "zv_shrink_h_a": alpha_h}, "paper_hm5": PAPER_HM5} | |
| (P.RESULTS / "tables" / "tradeoff_all_vs_hm.json").write_text(json.dumps(tj, indent=1, default=float)) | |
| extra["tradeoff"] = {"analytic": tj["analytic"], "fits": tj["fits"]} | |
| if not a.no_figure: | |
| png = P.RESULTS / "tables" / "pareto_all_vs_hm.png" | |
| pareto_figure(analytic_rows, ours, curves_test, png) | |
| print("figure ->", png) | |
| write_run_json(P.RESULTS / "tables", f"analytic_baselines{suffix}", [__file__, P.__file__, M.__file__], | |
| [P.CACHE_NPZ], seed=None, t_start=t0, extra=extra, input_listing=False) | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 28.5 kB
- Xet hash:
- e4646c6cb03e7bce40144a55206c1074bdd2bb5cb3f13874ecb72814b36d0223
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.