Buckets:
| """cwm3/aide/eval/report_hm_h5_arms.py — pre-registered HM h=5 arms (docs/decisions.md row "PRE-REG (2026-09-17T07:00:22Z) — three | |
| owner-requested experiments", part (A)) -> aide/results/tables/hm_h5_arms.md (+ .json). | |
| Reads ONLY run outputs: pred_test.npz (every per-clip cell recomputed here; paired per-clip bootstraps with aide/eval/metrics.paired_diff, 10k | |
| resamples, seed 0 — the procedure of baton_pretrain.md / arch_attribution.md / pretrain_variants.md), metrics_test.json / metrics_val.json (the | |
| harness' one-shot levels; cross-checked against the recomputed values), run json (argv, best epoch / epochs / stop / GPU minutes / init report / | |
| learned gains `hgain` / `hand_weight`). Nothing is typed by hand. | |
| Arms (3 seeds 42 / 123 / 456, adopted recipe verbatim + the arm flag, same pretraining init, val-only selection, test once): | |
| a1 hw2 = --hand-weight 2; a2 hw4 = --hand-weight 4; b hgain = --horizon-gain learned; c hgainp = --horizon-gain parts; d hw2hgain = a1 + b. | |
| Comparator = the paper's adopted row cwm3d_kp_batonpre_trunkp123_s<k>_v2 (existing runs, not retrained), seed-matched. | |
| Cells: All-MPJPE, d-n, HM-MPJPE, HM h=5, HM hands h=5 (the 42 hand joints 94-135 at h=5 on the official HM subset — the `hm_subsets.py` | |
| PART_JOINTS["hands"] machinery; this is the "hands h=5" of the pre-registration's (D) 184.63 vs 234.6), hands h=5 on all clips, hands h=5 on the | |
| part-specific hands-HM subset (top 10 % by the hand-joint motion score, hm_subsets section B), PCK@0.05. | |
| Reading rule (pre-registered verbatim): "an arm is 'reportable as an improvement' iff HM h=5 improves by >= 3 px with the paired per-clip CI | |
| excluding zero on >= 2/3 seeds AND All-MPJPE degrades by <= 0.2 px (CI not excluding a degradation > 0.2)". Operationalisation (fixed here, | |
| before any number was read): (i) seed-mean paired Δ HM h=5 <= -3.0 px AND Δ HM h=5 CI entirely < 0 on >= 2 of 3 seeds; (ii) seed-mean paired | |
| Δ All <= +0.2 px AND on every seed the Δ All CI lower bound <= +0.2 (no seed's CI lies entirely above +0.2). A stricter per-seed variant | |
| (>= 2/3 seeds each with Δ HM h=5 <= -3 and CI < 0) is printed next to it. All arms are reported regardless (Supp A). | |
| python aide/eval/report_hm_h5_arms.py [--n-boot 10000] [--out aide/results/tables/hm_h5_arms.md] | |
| Output policy: if --out already exists the new content is APPENDED as a dated block (append-only tables); existing sections are never edited. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import glob | |
| import json | |
| import math | |
| import sys | |
| import time | |
| from pathlib import Path | |
| import numpy as np | |
| ROOT = Path(__file__).resolve().parents[2] | |
| sys.path.insert(0, str(ROOT)) | |
| from aide.data import protocol as P # noqa: E402 | |
| from aide.eval import metrics as M # noqa: E402 | |
| from aide.eval.harness import align, load_split # noqa: E402 | |
| from aide.eval.report_arch_attribution import fmt_ci, ms, verdict # noqa: E402 | |
| from common.runmeta import write_run_json # noqa: E402 | |
| RUNS = P.RESULTS / "runs" | |
| OUT = P.RESULTS / "tables" / "hm_h5_arms.md" | |
| PREREG = "docs/decisions.md 2026-09-17T07:00:22Z (PRE-REG — three owner-requested experiments, part (A))" | |
| SEEDS = (42, 123, 456) | |
| N_REQ = 2 # >= 2/3 seeds | |
| HM5_GAIN_PX = 3.0 | |
| ALL_DEGRADE_PX = 0.2 | |
| ADOPTED = "cwm3d_kp_batonpre_trunkp123_s{k}_v2" | |
| NAME = "cwm3d_kp_hm5_{tag}_batonpre_trunkp123_s{k}_v2" | |
| ARMS = [("a1", "hw2", "hand-joint loss weight w_hand = 2 (`--hand-weight 2`)"), | |
| ("a2", "hw4", "hand-joint loss weight w_hand = 4 (`--hand-weight 4`)"), | |
| ("b", "hgain", "learned per-horizon residual gain g_h, 5 scalars (`--horizon-gain learned`)"), | |
| ("c", "hgainp", "hand-specific per-horizon gain, 5 × 2 scalars hands / other joints (`--horizon-gain parts`)"), | |
| ("d", "hw2hgain", "(a1) + (b): `--hand-weight 2 --horizon-gain learned`")] | |
| HANDS = np.arange(94, 136) # == report_hm_subsets.PART_JOINTS["hands"] == heads.HAND_JOINTS | |
| CELLS = [("All-MPJPE", "mpjpe_all", 2, False), ("d-n", "dn_mpjpe", 3, False), ("HM-MPJPE", "mpjpe_hm", 2, False), ("HM h=5", "hm_h5", 2, False), | |
| ("HM hands h=5", None, 2, False), ("hands h=5 (all clips)", None, 2, False), ("hands-HM hands h=5", None, 2, False), ("PCK@0.05", "pck05", 2, True)] | |
| RULE_CELLS = ("All-MPJPE", "HM h=5", "HM hands h=5") | |
| # ----------------------------------------------------------------------------- io | |
| def run_json(name): | |
| fs = sorted(glob.glob(str(RUNS / name / f"run_{name}_2*.json"))) | |
| return json.load(open(fs[-1])) if fs else None | |
| def metrics(name, split="test"): | |
| p = RUNS / name / f"metrics_{split}.json" | |
| return json.load(open(p))["metrics"] if p.exists() else None | |
| def n_test_evals(name): | |
| return len(list((RUNS / name).glob("metrics_test*.json"))) if (RUNS / name).exists() else 0 | |
| def load_pred(name, d): | |
| p = RUNS / name / "pred_test.npz" | |
| if not p.exists(): | |
| return None | |
| z = np.load(p, allow_pickle=False) | |
| pred = np.asarray(z["pred"], dtype=np.float64)[align(z, d["clip_id"])] | |
| assert pred.shape == (len(d["clip_id"]), P.T_PRED, P.J, 2) and np.isfinite(pred).all(), name | |
| return pred | |
| def finished(name): | |
| return run_json(name) is not None and metrics(name) is not None and (RUNS / name / "pred_test.npz").exists() | |
| def per_clip_cells(pred, gt, hm, hands_hm): | |
| """-> {cell: per-clip values} (the HM cells are over the 60 HM clips, hands-HM over the 60 hand-motion clips).""" | |
| err = M.joint_errors(pred, gt) # [N,H,J] | |
| clip = err.mean((1, 2)) | |
| return {"All-MPJPE": clip, "d-n": 100.0 * clip / P.DIAG, "HM-MPJPE": err[hm].mean((1, 2)), "HM h=5": err[hm][:, 4].mean(1), | |
| "HM hands h=5": err[hm][:, 4][:, HANDS].mean(1), "hands h=5 (all clips)": err[:, 4][:, HANDS].mean(1), | |
| "hands-HM hands h=5": err[hands_hm][:, 4][:, HANDS].mean(1), "PCK@0.05": 100.0 * (err <= P.pck_threshold_px(0.05)).mean((1, 2))} | |
| def reading_rule(res): | |
| """res: {cell: {"per_seed": {seed: {mean, ci_lo, ci_hi}}}} -> dict with the pre-registered decision and its ingredients.""" | |
| hm5 = [res["HM h=5"]["per_seed"][str(k)] for k in SEEDS if str(k) in res["HM h=5"]["per_seed"]] | |
| al = [res["All-MPJPE"]["per_seed"][str(k)] for k in SEEDS if str(k) in res["All-MPJPE"]["per_seed"]] | |
| mean_hm5 = float(np.mean([r["mean"] for r in hm5])) | |
| n_hm5_excl_neg = int(sum(r["ci_hi"] < 0 for r in hm5)) | |
| n_hm5_strict = int(sum((r["ci_hi"] < 0) and (r["mean"] <= -HM5_GAIN_PX) for r in hm5)) | |
| mean_all = float(np.mean([r["mean"] for r in al])) | |
| all_ci_ok = bool(all(r["ci_lo"] <= ALL_DEGRADE_PX for r in al)) | |
| cond_hm = (mean_hm5 <= -HM5_GAIN_PX) and (n_hm5_excl_neg >= N_REQ) | |
| cond_all = (mean_all <= ALL_DEGRADE_PX) and all_ci_ok | |
| return {"n_seeds": len(hm5), "mean_delta_hm_h5": mean_hm5, "n_seeds_hm_h5_ci_below_zero": n_hm5_excl_neg, "cond_hm_h5": bool(cond_hm), | |
| "n_seeds_hm_h5_strict(<=-3 & CI<0)": n_hm5_strict, "cond_hm_h5_strict_per_seed": bool(n_hm5_strict >= N_REQ), | |
| "mean_delta_all": mean_all, "all_ci_lo_max": float(max(r["ci_lo"] for r in al)), "cond_all": bool(cond_all), | |
| "reportable": bool(cond_hm and cond_all and len(hm5) == len(SEEDS)), | |
| "reportable_strict": bool(n_hm5_strict >= N_REQ and cond_all and len(hm5) == len(SEEDS))} | |
| # ----------------------------------------------------------------------------- main | |
| def main(argv=None): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--n-boot", type=int, default=10000) | |
| ap.add_argument("--out", default=str(OUT)) | |
| a = ap.parse_args(argv) | |
| t0 = time.time() | |
| d = load_split("test") | |
| gt = d["S10"][:, P.T_OBS:] | |
| hm, thr, _ = M.hm_mask(gt) | |
| score_hands = M.motion_score(gt[:, :, HANDS]) | |
| order = np.argsort(-score_hands, kind="stable") | |
| hands_hm = np.zeros(len(score_hands), bool); hands_hm[order[:P.hm_top(len(score_hands))]] = True | |
| thr_hands = float(score_hands[order[P.hm_top(len(score_hands)) - 1]]) | |
| stamp = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime()) | |
| N = len(gt) | |
| cache = {} | |
| def cells_of(name): | |
| if name not in cache: | |
| pr = load_pred(name, d) | |
| cache[name] = None if pr is None else per_clip_cells(pr, gt, hm, hands_hm) | |
| return cache[name] | |
| L = [f"# Pre-registered HM h=5 arms — CockpitWM (`cwm3d`) + BATON pretraining, adopted recipe verbatim — AIDE test, n = {N}, paired per-clip bootstrap ({a.n_boot // 1000}k)", "", | |
| f"Generated {stamp} by `aide/eval/report_hm_h5_arms.py` from pred_test.npz / metrics_test.json / metrics_val.json / run json files only " | |
| f"(locked protocol: driver `first`, grid `floor`, HM top {int(hm.sum())} (threshold {thr:.2f} px), PCK @ {P.pck_threshold_px(0.05):.0f} px; hands-HM subset = top {int(hands_hm.sum())} clips by the " | |
| f"hand-joint motion score (threshold {thr_hands:.2f} px, `hm_subsets` section B); every run selected on val All-MPJPE only (EMA weights); test evaluated once per run by the harness). " | |
| f"Pre-registration: {PREREG}. Δ = paired per-clip difference arm − adopted (negative = better for errors, positive = better for PCK), CI = {a.n_boot // 1000}k clip-bootstrap percentile " | |
| f"interval (seed 0). Verdict rule: BEATS / WORSE = CI excl. 0 on ≥ {N_REQ}/3 seeds with the same sign on every seed; NOT_WORSE = all CIs cover 0 and |Δ| < 0.25; else INCONCLUSIVE.", "", | |
| f"**Pre-registered reading rule (verbatim): \"an arm is 'reportable as an improvement' iff HM h=5 improves by ≥ 3 px with the paired per-clip CI excluding zero on ≥ 2/3 seeds AND " | |
| f"All-MPJPE degrades by ≤ 0.2 px (CI not excluding a degradation > 0.2); all arms are reported in Supp A regardless (All, d-n, HM, HM h=5, hands h=5, PCK); the paper's adopted " | |
| f"row (48.05) does NOT change unless the owner decides after seeing the table.\"** Operationalisation (fixed in this script before any number was read): (i) seed-mean Δ HM h=5 ≤ −{HM5_GAIN_PX:.0f} px " | |
| f"AND the Δ HM h=5 CI lies entirely below 0 on ≥ {N_REQ}/3 seeds; (ii) seed-mean Δ All ≤ +{ALL_DEGRADE_PX} px AND no seed's Δ All CI lies entirely above +{ALL_DEGRADE_PX}. " | |
| f"A stricter per-seed variant (≥ {N_REQ}/3 seeds each with Δ HM h=5 ≤ −{HM5_GAIN_PX:.0f} and CI < 0) is shown next to it. \"hands h=5\" = HM hands h=5 (42 hand joints 94–135 at h = 5 on the " | |
| f"official HM subset; the quantity behind the pre-registration's (D) 184.63 vs 234.6); two further hand cells (all clips; hands-HM subset) are descriptive.", ""] | |
| # ---- §1 runs | |
| L += ["## 1. Runs (adopted comparator + 15 arm fine-tunes; recipe = the adopted run's argv + the arm flag)", "", | |
| "| arm | seed | run | status | best epoch / epochs / stop / GPU min | init (loaded / kept; ckpt epoch; sha) | hand w | learned gains g_h (EMA weights) | val All (selection) | test evals |", | |
| "|---|---|---|---|---|---|---|---|---|---|"] | |
| gpu, runs_info = {}, {} | |
| def run_line(arm, k, name): | |
| r = run_json(name) | |
| if r is None or not finished(name): | |
| L.append(f"| {arm} | {k} | {name} | **not finished** | | | | | | |") | |
| return None | |
| ini, cfg = r.get("init") or {}, r.get("config") or {} | |
| hg = r.get("hgain") | |
| if hg is None: | |
| hgs = "—" | |
| elif hg.get("mode") == "learned": | |
| hgs = "g = " + "/".join(f"{x:.3f}" for x in hg["g_h"]) | |
| else: | |
| hgs = "hands " + "/".join(f"{x:.3f}" for x in hg["g_h_hands"]) + "; other " + "/".join(f"{x:.3f}" for x in hg["g_h_other"]) | |
| mv = metrics(name, "val") or {} | |
| L.append(f"| {arm} | {k} | {name} | done | {r.get('best_epoch')} / {r.get('epochs_run')} / {r.get('stop_reason')} / {r.get('gpu_minutes')} | " | |
| f"{ini.get('parts', '—')}: {ini.get('n_loaded', '—')} / {ini.get('n_missing_or_kept', '—')} (ep {ini.get('ckpt_epoch', '—')}; {ini.get('ckpt_sha256', '—')}) | " | |
| f"{cfg.get('hand_weight', 1.0)} | {hgs} | {mv.get('mpjpe_all', {}).get('value', float('nan')):.3f} | {n_test_evals(name)} |") | |
| return {"best_epoch": r.get("best_epoch"), "epochs_run": r.get("epochs_run"), "stop": r.get("stop_reason"), "gpu_minutes": r.get("gpu_minutes"), | |
| "init": {kk: ini.get(kk) for kk in ("parts", "n_loaded", "n_missing_or_kept", "ckpt_epoch", "ckpt_sha256")}, "hand_weight": cfg.get("hand_weight", 1.0), | |
| "horizon_gain": cfg.get("horizon_gain", "none"), "hgain": hg, "selection": r.get("selection"), "val_all": mv.get("mpjpe_all", {}).get("value"), | |
| "n_test_evals": n_test_evals(name), "command": r.get("command"), "n_params": r.get("n_params")} | |
| for k in SEEDS: | |
| runs_info[ADOPTED.format(k=k)] = run_line("adopted", k, ADOPTED.format(k=k)) | |
| for arm, tag, _ in ARMS: | |
| for k in SEEDS: | |
| name = NAME.format(tag=tag, k=k) | |
| info = run_line(arm, k, name) | |
| runs_info[name] = info | |
| if info and info.get("gpu_minutes") is not None: | |
| gpu[name] = float(info["gpu_minutes"]) | |
| L.append("") | |
| # ---- §2 levels | |
| L += ["## 2. Levels — seed mean ± sd (sample sd) of the per-run test values recomputed from pred_test.npz; [CI] = 10k clip-bootstrap of the seed-mean per-clip value", "", | |
| "| arm | definition | n seeds | " + " | ".join(c for c, _, _, _ in CELLS) + " |", "|---|---|---|" + "---|" * len(CELLS)] | |
| levels = {} | |
| def level_block(label, definition, names): | |
| have = [n for n in names if finished(n) and cells_of(n) is not None] | |
| if not have: | |
| L.append(f"| {label} | {definition} | 0 | " + " | ".join("—" for _ in CELLS) + " |") | |
| return None | |
| rec = {"n_seeds": len(have), "runs": have, "cells": {}} | |
| row = [] | |
| for c, _, nd, _ in CELLS: | |
| vals = [float(cells_of(n)[c].mean()) for n in have] | |
| seed_mean_clip = np.mean([cells_of(n)[c] for n in have], axis=0) | |
| lo, hi = M.bootstrap_ci(seed_mean_clip, a.n_boot, 0) | |
| rec["cells"][c] = {"mean": float(np.mean(vals)), "sd": float(np.std(vals, ddof=1)) if len(vals) > 1 else None, "per_seed": vals, "ci95": [lo, hi]} | |
| row.append(f"{ms(vals, nd)} [{lo:.{nd}f}, {hi:.{nd}f}]") | |
| L.append(f"| {label} | {definition} | {len(have)} | " + " | ".join(row) + " |") | |
| return rec | |
| levels["adopted"] = level_block("adopted row", "CockpitWM + BATON pretraining (`cwm3d_kp_batonpre_trunkp123_s<k>_v2`, existing runs)", [ADOPTED.format(k=k) for k in SEEDS]) | |
| for arm, tag, definition in ARMS: | |
| levels[arm] = level_block(f"({arm}) {tag}", definition, [NAME.format(tag=tag, k=k) for k in SEEDS]) | |
| L.append("") | |
| # ---- §2b harness cross-check | |
| checks = [] | |
| for name, info in runs_info.items(): | |
| if not info: | |
| continue | |
| mt, cc = metrics(name), cells_of(name) | |
| for c, key, _, _ in CELLS: | |
| if key is None: | |
| continue | |
| mine, theirs = float(cc[c].mean()), float(mt[key]["value"]) | |
| checks.append({"run": name, "cell": c, "recomputed": mine, "harness": theirs, "ok": bool(abs(mine - theirs) < 1e-6)}) | |
| n_bad = sum(1 for c in checks if not c["ok"]) | |
| L += [f"Integrity: {len(checks)} recomputed cells cross-checked against the harness' metrics_test.json values (|diff| < 1e-6): **{len(checks) - n_bad} ok, {n_bad} mismatches**" | |
| + ("" if n_bad == 0 else " — " + "; ".join(f"{c['run']} {c['cell']} {c['recomputed']:.4f} vs {c['harness']:.4f}" for c in checks if not c["ok"])) + ". " | |
| f"Selection field of every finished run = `{sorted(set(i['selection'] for i in runs_info.values() if i))}`; test evaluations per run = {sorted(set(i['n_test_evals'] for i in runs_info.values() if i))} (1 = evaluated once).", ""] | |
| # ---- §3 paired deltas | |
| L += ["## 3. Paired per-clip Δ = arm − adopted row, seed-matched (10k clip bootstrap, seed 0)", ""] | |
| deltas = {} | |
| for arm, tag, definition in ARMS: | |
| per_seed = {} | |
| for k in SEEDS: | |
| an, bn = NAME.format(tag=tag, k=k), ADOPTED.format(k=k) | |
| if finished(an) and finished(bn) and cells_of(an) is not None and cells_of(bn) is not None: | |
| ca, cb = cells_of(an), cells_of(bn) | |
| per_seed[k] = {c: {"a": float(ca[c].mean()), "b": float(cb[c].mean()), **M.paired_diff(ca[c], cb[c], a.n_boot, 0), "higher_better": hb} for c, _, _, hb in CELLS} | |
| L += [f"### 3.{ARMS.index((arm, tag, definition)) + 1} ({arm}) {tag} — {definition}", ""] | |
| if not per_seed: | |
| L += ["_(no finished seed yet)_", ""] | |
| deltas[arm] = None | |
| continue | |
| seeds = list(per_seed) | |
| note = "" if len(seeds) == len(SEEDS) else f" **PARTIAL: {len(seeds)}/3 seeds finished ({seeds}).**" | |
| L += [f"Seeds paired: {seeds}.{note}", "", | |
| "| cell | " + " | ".join(f"arm s{k}" for k in seeds) + " | " + " | ".join(f"adopted s{k}" for k in seeds) + " | " + " | ".join(f"Δ s{k} [CI]" for k in seeds) + " | mean Δ ± sd | verdict |", | |
| "|---|" + "---|" * (3 * len(seeds)) + "---|---|"] | |
| res = {} | |
| for c, _, nd, hb in CELLS: | |
| rows = [per_seed[k][c] for k in seeds] | |
| v = verdict(rows, hb, N_REQ) | |
| dm = np.array([r["mean"] for r in rows]) | |
| L.append(f"| {c} | " + " | ".join(f"{r['a']:.{nd}f}" for r in rows) + " | " + " | ".join(f"{r['b']:.{nd}f}" for r in rows) + " | " | |
| + " | ".join(fmt_ci(r, nd) for r in rows) + f" | {dm.mean():+.{nd}f} ± {(dm.std(ddof=1) if len(dm) > 1 else float('nan')):.{nd}f} | {v} |") | |
| res[c] = {"verdict": v, "mean": float(dm.mean()), "sd": float(dm.std(ddof=1)) if len(dm) > 1 else None, | |
| "per_seed": {str(k): {kk: vv for kk, vv in per_seed[k][c].items()} for k in seeds}} | |
| deltas[arm] = res | |
| L.append("") | |
| # ---- §4 reading rule | |
| L += ["## 4. Pre-registered reading rule applied (arm is 'reportable as an improvement' iff (i) AND (ii))", "", | |
| f"| arm | seeds | mean Δ HM h=5 (≤ −{HM5_GAIN_PX:.0f}?) | seeds with Δ HM h=5 CI < 0 (≥ {N_REQ}?) | (i) | strict per-seed (≥ {N_REQ} seeds Δ ≤ −3 & CI < 0) | mean Δ All (≤ +{ALL_DEGRADE_PX}?) | max Δ All CI lower bound (≤ +{ALL_DEGRADE_PX}?) | (ii) | **reportable** | reportable (strict) | Δ HM hands h=5 mean ± sd / verdict |", | |
| "|---|---|---|---|---|---|---|---|---|---|---|---|"] | |
| rule = {} | |
| for arm, tag, _ in ARMS: | |
| res = deltas.get(arm) | |
| if not res: | |
| L.append(f"| ({arm}) {tag} | 0 | — | — | — | — | — | — | — | **NOT EVALUABLE** | — | — |") | |
| rule[arm] = None | |
| continue | |
| rr = reading_rule(res) | |
| rule[arm] = rr | |
| hh = res["HM hands h=5"] | |
| L.append(f"| ({arm}) {tag} | {rr['n_seeds']} | {rr['mean_delta_hm_h5']:+.2f} | {rr['n_seeds_hm_h5_ci_below_zero']}/{rr['n_seeds']} | {'yes' if rr['cond_hm_h5'] else 'no'} | " | |
| f"{rr['n_seeds_hm_h5_strict(<=-3 & CI<0)']}/{rr['n_seeds']} → {'yes' if rr['cond_hm_h5_strict_per_seed'] else 'no'} | {rr['mean_delta_all']:+.3f} | {rr['all_ci_lo_max']:+.3f} | " | |
| f"{'yes' if rr['cond_all'] else 'no'} | **{'YES' if rr['reportable'] else 'NO'}**{'' if rr['n_seeds'] == 3 else ' (partial)'} | {'YES' if rr['reportable_strict'] else 'NO'} | " | |
| f"{hh['mean']:+.2f} ± {(hh['sd'] if hh['sd'] is not None else float('nan')):.2f} / {hh['verdict']} |") | |
| reportable = [f"({arm}) {tag}" for arm, tag, _ in ARMS if rule.get(arm) and rule[arm]["reportable"]] | |
| reportable_strict = [f"({arm}) {tag}" for arm, tag, _ in ARMS if rule.get(arm) and rule[arm]["reportable_strict"]] | |
| n_eval = sum(1 for arm, _, _ in ARMS if rule.get(arm) and rule[arm]["n_seeds"] == 3) | |
| L += ["", f"**Reportable arms under the pre-registered rule ({n_eval}/5 arms fully evaluated): {', '.join(reportable) if reportable else 'NONE'}" | |
| f" (strict per-seed variant: {', '.join(reportable_strict) if reportable_strict else 'NONE'}). The paper's adopted row (48.05) is unchanged; every arm is reported in Supp A regardless.**", ""] | |
| # ---- §5 GPU minutes | |
| L += ["## 5. Runs and GPU minutes (new runs only, from the run json files; the adopted comparator runs were not retrained)", "", "| run | GPU min |", "|---|---|"] | |
| for name, g in gpu.items(): | |
| L.append(f"| {name} | {g:.2f} |") | |
| L += [f"| **total (15 new runs: {len(gpu)} finished)** | **{sum(gpu.values()):.2f}** |", ""] | |
| # ---- provenance | |
| L += ["## Provenance", "", | |
| f"- Pre-registration: {PREREG}; arms, seeds, selection rule, reading rule and cells were fixed there before any run was launched.", | |
| "- Implementation (additive, defaults = old behaviour, `aide/tests/test_hm_h5_arms.py`): `--hand-weight` (aide/models/losses.py joint_weights / compute_losses jw: the 42 hand keypoints " | |
| "PARTS lhand + rhand weighted in the Euclidean term only, other joints 1, not renormalised; NLL / bone / jerk and all metrics unchanged), `--horizon-gain learned|parts` " | |
| "(aide/models/heads.py HorizonGain: pred = last + g_h ⊙ (pred_mix − last), g = 1 + δ, δ zero-init; `parts` = separate gains for the 42 hand joints), `--horizon-gain-parts` alias.", | |
| f"- Recipe: the adopted run's argv verbatim (`--lr 3e-4 --wd 0.05 --warmup 10 --epochs 200 --bs 32 --ema 0.999 --clip 1.0 --patience 40 --aug-window 1 --noise-px 2.0 --joint-drop 0.1 " | |
| f"--translate 20 --scale 0.05 --flip-p 0 --max-minutes 25`, all four loss terms), init `aide/results/runs/cwm3d_kp_batonpre_pretrain_s123/ckpt_best.pt` trunk only (the gain δ is not in the " | |
| f"pretraining checkpoint and starts at 0); queue `aide/train/run_hm_h5_arms.sh` (gate before every trainer: free VRAM ≥ 7 GB, MemAvailable ≥ 28 GB, < 4 trainers, threads 4; pids in " | |
| f"`aide/results/hm_h5_arms/pids.txt`).", | |
| "- New run dirs only: `cwm3d_kp_hm5_<tag>_batonpre_trunkp123_s<k>_v2`; comparator runs `cwm3d_kp_batonpre_trunkp123_s<k>_v2` are the existing ones. No AIDE test clip entered training or selection.", ""] | |
| out = Path(a.out) | |
| out.parent.mkdir(parents=True, exist_ok=True) | |
| text = "\n".join(L) | |
| if out.exists(): | |
| with open(out, "a") as f: | |
| f.write(f"\n\n---\n\n<!-- appended regeneration {stamp} -->\n\n" + text) | |
| mode = "appended" | |
| else: | |
| out.write_text(text) | |
| mode = "written" | |
| summary = {"stamp": stamp, "n_boot": a.n_boot, "prereg": PREREG, "hm_threshold": thr, "hands_hm_threshold": thr_hands, "runs": runs_info, "levels": levels, | |
| "deltas": deltas, "reading_rule": rule, "reportable": reportable, "reportable_strict": reportable_strict, "gpu_minutes": gpu, "gpu_minutes_total": float(sum(gpu.values())), | |
| "integrity": {"n_checks": len(checks), "n_mismatch": n_bad}} | |
| js = out.with_suffix(".json") | |
| if js.exists(): | |
| old = json.load(open(js)) | |
| old = old if isinstance(old, list) else [old] | |
| js.write_text(json.dumps(old + [summary], indent=1, default=str)) | |
| else: | |
| js.write_text(json.dumps([summary], indent=1, default=str)) | |
| write_run_json(P.RESULTS / "tables", "report_hm_h5_arms", [Path(__file__), M.__file__, P.__file__], [P.CACHE_NPZ], seed=0, t_start=t0, input_listing=False, | |
| extra={"out": str(out), "mode": mode, "reportable": reportable, "reportable_strict": reportable_strict, "gpu_minutes_total": float(sum(gpu.values())), | |
| "reading_rule": rule, "delta_summary": {arm: ({c: {kk: vv for kk, vv in r.items() if kk != "per_seed"} for c, r in res.items()} if res else None) for arm, res in deltas.items()}, | |
| "integrity": summary["integrity"]}) | |
| print(text) | |
| print(f"\n[{mode}] {out}") | |
| return 0 | |
| if __name__ == "__main__": | |
| sys.exit(main()) | |
Xet Storage Details
- Size:
- 24 kB
- Xet hash:
- 093b8fe3eee77fe5ff016e7019a12c7db98207d6beed99363c44147a3189becb
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.