Buckets:

AsianPlayer
/

TriDrive

Files
xet
AsianPlayer/TriDrive / code /aide /eval /report_master_aide.py
AsianPlayer
16 days ago
download
raw
38.1 kB
#!/usr/bin/env python
"""cwm3/aide/eval/report_master_aide.py — ONE master table for the AIDE benchmark in the Driver-WM column format
    Model | All MPJPE↓ | All d-nMPJPE↓ | HM MPJPE↓ | HM d-nMPJPE↓ | PCK@0.05↑ | DBR F1↑ | DER F1↑ | TCR F1↑ | VCR F1↑
Every number of every OUR row is RECOMPUTED here from the saved predictions (`aide/results/runs/<run>/pred_test.npz`, plus the
saved semantic logits when present) with the functions of `aide/eval/metrics.py` on the `load_split("test")` ground truth, and
cross-checked against the harness record `metrics_test.json` of the same run (tolerance --tol, default 1e-6; any mismatch is
printed verbatim and written to the outputs — nothing is "fixed").  Group rows are mean ± sample sd (ddof = 1) over seeds.
Ensembles are re-averaged from their members' saved predictions exactly as `aide/eval/ensemble.py` does (float64 mean →
float32) and compared with the saved ensemble prediction; they are labelled "ensemble, not a single model".  The 18 published
rows are copied VERBATIM from `aide/results/tables/stage_bc.md` §1b (typed from arXiv 2605.05092v2; cross-checked numerically
against `aide/eval/paper_rows.py`) — no recomputation is possible for them.
Read-only on aide/results/runs/*.  Outputs (new files only): aide/results/tables/master_aide.md and master_aide.json.
No training, no GPU, no bootstraps (the CIs / paired tests live in the source tables named per group).
    python aide/eval/report_master_aide.py [--tol 1e-6] [--out aide/results/tables]
"""
from __future__ import annotations

import os

for _v in ("OMP_NUM_THREADS", "MKL_NUM_THREADS", "OPENBLAS_NUM_THREADS", "NUMEXPR_NUM_THREADS"):
    os.environ.setdefault(_v, "4")

import argparse  # noqa: E402
import json  # noqa: E402
import sys  # noqa: E402
import time  # noqa: E402
from pathlib import Path  # noqa: E402

import numpy as np  # noqa: E402

ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT))
from aide.data import protocol as P  # noqa: E402
from aide.eval import metrics as M  # noqa: E402
from aide.eval.harness import align, load_split  # noqa: E402
from aide.eval.paper_rows import GROUP as PAPER_GROUP, PAPER_ROWS  # noqa: E402

RUNS = P.RESULTS / "runs"
TABLES = P.RESULTS / "tables"
SEEDS3 = [42, 123, 456]
SEEDS5 = [42, 123, 456, 789, 1011]
GEO = ["mpjpe_all", "dn_mpjpe", "mpjpe_hm", "dn_mpjpe_hm", "pck05"]
F1 = [f"f1_{t}" for t in P.TASKS]
KEYS = GEO + F1
ND = {"mpjpe_all": 2, "dn_mpjpe": 3, "mpjpe_hm": 2, "dn_mpjpe_hm": 3, "pck05": 2, **{k: 2 for k in F1}}
LOWER = {"mpjpe_all", "dn_mpjpe", "mpjpe_hm", "dn_mpjpe_hm"}
HEADER = ["Model", "All MPJPE↓", "All d-nMPJPE↓", "HM MPJPE↓", "HM d-nMPJPE↓", "PCK@0.05↑", "DBR F1↑", "DER F1↑", "TCR F1↑", "VCR F1↑"]

SEL_GEO = "val All-MPJPE of the EMA weights (reading (a) for F1); test evaluated once"
SEL_F1 = "DUAL SELECTION: checkpoint with the best val mean macro-F1 (`__f1sel`, reading (b)); its own geometry; test evaluated once"
PRE_TRUNK = "BATON in-cabin pose pretraining, trunk-only init (tokenizer / blocks / ln_f) from `cwm3d_kp_batonpre_pretrain_s123`; head at seed init"


def rr(fmt: str, seeds, suffix: str = "") -> list[str]:
    return [fmt.format(s=s) + suffix for s in seeds]


# label, block (analytic | single | ensemble), runs, reading of the saved semantic logits (None | "a" | "b"), selection, pretraining, source
GROUPS = [
    dict(key="zv", label="Zero-velocity (copy the last observed pose)", block="analytic", runs=["zero_velocity"], reading=None,
         selection="none (analytic rule, no parameters)", pretraining="none", source="analytic_baselines.md; stage_a.md §1; stage_bc.md §1"),
    dict(key="zv_shrink", label="ZV-shrink α=0.44 (val-fitted)", block="analytic", runs=["zv_shrink_val"], reading=None,
         selection="α chosen on val All-MPJPE (grid; `alpha_fit_val.json`)", pretraining="none", source="analytic_baselines.md; stage_a.md §1; stage_bc.md §1"),
    dict(key="zv_shrink_h", label="ZV-shrink per horizon α_h=0.76·h/5 (val-fitted)", block="analytic", runs=["zv_shrink_h_val"], reading=None,
         selection="a chosen on val All-MPJPE (grid; `alpha_fit_val.json`)", pretraining="none", source="analytic_baselines.md (roadmap B0 row)"),
    # ---- ours, keypoint only
    dict(key="K3", label="CockpitWM-K (n=3, locked seeds 42/123/456)", block="single", runs=rr("cwm3d_kp_s{s}_v2", SEEDS3), reading=None,
         selection=SEL_GEO, pretraining="none", source="stage_a.md §1; stage_bc.md §1 'CWM3-D-kp (Stage A)'"),
    dict(key="K5", label="CockpitWM-K (n=5, seeds 42/123/456/789/1011)", block="single", runs=rr("cwm3d_kp_s{s}_v2", SEEDS5), reading=None,
         selection=SEL_GEO, pretraining="none", source="stage_a_5seed.md §2; arch_attribution.md §3.1"),
    dict(key="K5_pre", label="CockpitWM-K + BATON trunk (n=5)", block="single", runs=rr("cwm3d_kp_batonpre_trunkp123_s{s}_v2", SEEDS5), reading=None,
         selection=SEL_GEO, pretraining=PRE_TRUNK + " (pre-registered replication, BEATS on Δ All; adopted)",
         source="baton_pretrain.md §6 (seeds 42/123/456); kv2_program.md last block §3.1 'K + pre' (seeds 789/1011)"),
    # ---- ours, keypoints + vision + semantic heads
    dict(key="KS3", label="CockpitWM-KS (n=3, locked seeds 42/123/456)", block="single", runs=rr("cwm3d_vis_sem_s{s}_v2", SEEDS3), reading="a",
         selection=SEL_GEO + "; the B2 main recipe has no dual selection → no reading-(b) checkpoint", pretraining="none",
         source="stage_bc.md §1 'CWM3-D vis+sem (B2 main)'"),
    dict(key="KS5", label="CockpitWM-KS (n=5, seeds 42/123/456/789/1011)", block="single", runs=rr("cwm3d_vis_sem_s{s}_v2", SEEDS5), reading="a",
         selection=SEL_GEO + "; no reading-(b) checkpoint", pretraining="none", source="stage_bc_5seed.md §2"),
    dict(key="KS3_pre", label="CockpitWM-KS + BATON trunk (n=3)", block="single", runs=rr("cwm3d_vis_sem_batonpre_trunkp123_s{s}_v2", SEEDS3), reading="a",
         selection=SEL_GEO + "; no reading-(b) checkpoint", pretraining=PRE_TRUNK + " (pre-registered; Δ All INCONCLUSIVE → not adopted)",
         source="baton_pretrain.md §7"),
    dict(key="KSvg_a", label="CockpitWM-KS + view-gate attention heads, (a) geometry-selected checkpoint (n=3)", block="single",
         runs=rr("cwm3d_vis_sema_vg_s{s}_v2", SEEDS3), reading="a", selection=SEL_GEO + " (dual-select run: this is the geometry checkpoint)",
         pretraining="none", source="stage_bc.md §1 'DER (i) attn heads + learned view gate — (a) geometry ckpt'"),
    dict(key="KSvg_b", label="CockpitWM-KS + view-gate attention heads, (b) val-F1-selected checkpoint (n=3)", block="single",
         runs=rr("cwm3d_vis_sema_vg_s{s}_v2", SEEDS3, "__f1sel"), reading="b", selection=SEL_F1, pretraining="none",
         source="stage_bc.md §1 'DER (i) attn heads + learned view gate — (b) DUAL SELECTION val-F1 ckpt'"),
    dict(key="KSvg_pre_a", label="CockpitWM-KS + view-gate heads + BATON trunk, (a) geometry-selected checkpoint (n=3)", block="single",
         runs=rr("cwm3d_vis_sema_vg_batonpre_trunkp123_s{s}_v2", SEEDS3), reading="a", selection=SEL_GEO + " (dual-select run: geometry checkpoint)",
         pretraining=PRE_TRUNK + " (pre-registered; Δ All BEATS → adopted as a reported row)", source="baton_pretrain.md §8"),
    dict(key="KSvg_pre_b", label="CockpitWM-KS + view-gate heads + BATON trunk, (b) val-F1-selected checkpoint (n=3)", block="single",
         runs=rr("cwm3d_vis_sema_vg_batonpre_trunkp123_s{s}_v2", SEEDS3, "__f1sel"), reading="b", selection=SEL_F1,
         pretraining=PRE_TRUNK, source="baton_pretrain.md §8 (the `__f1sel` companions; written as usual, reported here)"),
    # ---- K-v2 programme (ablation family, not adopted)
    dict(key="Kv2_ns", label="CockpitWM-K-v2 nospatial (n=5; K-v2 programme, not adopted)", block="single", runs=rr("cwm3d_v2_nospatial_s{s}_v2", SEEDS5),
         reading=None, selection=SEL_GEO, pretraining="none", source="kv2_program.md last block §2.1"),
    dict(key="Kv2_ns_pre", label="CockpitWM-K-v2 nospatial + BATON trunk (n=5; not adopted)", block="single",
         runs=rr("cwm3d_v2_nospatial_batonpre_trunkp123_s{s}_v2", SEEDS5), reading=None, selection=SEL_GEO,
         pretraining="BATON in-cabin pose pretraining of the K-v2 architecture itself, trunk-only init from `cwm3d_v2_nospatial_batonpre_pretrain_s123`",
         source="kv2_program.md last block §2.1"),
    # ---- keypoint baselines, shared recipe
    dict(key="MB5", label="MotionBERT-kp, default recipe (n=5)", block="single", runs=rr("motionbert_kp_s{s}_v2", SEEDS5), reading=None,
         selection=SEL_GEO, pretraining="none", source="stage_bc.md §1 (seeds 42/123/456); arch_attribution.md §3.1 (5 seeds)"),
    dict(key="MB5_pre", label="MotionBERT-kp + BATON trunk (n=5)", block="single", runs=rr("motionbert_kp_batonpre_trunkp123_s{s}_v2", SEEDS5), reading=None,
         selection=SEL_GEO, pretraining="BATON in-cabin pose pretraining of MotionBERT-kp itself (`motionbert_kp_batonpre_pretrain_s123`), trunk-only init",
         source="arch_attribution.md §1.1 (seeds 42/123/456); kv2_program.md last block §3.1 (seeds 789/1011)"),
    dict(key="MBdo5", label="MotionBERT-kp, dropout 0.1 (n=5; MB-best of the K-v2 val grid)", block="single", runs=rr("motionbert_kp_do0.1_s{s}_v2", SEEDS5),
         reading=None, selection=SEL_GEO + "; recipe chosen on the val grid of kv2_program.md §3", pretraining="none", source="kv2_program.md last block §3 / §3.1"),
    dict(key="MBdo5_pre", label="MotionBERT-kp, dropout 0.1 + BATON trunk (n=5)", block="single",
         runs=rr("motionbert_kp_do0.1_batonpre_trunkp123_s{s}_v2", SEEDS5), reading=None, selection=SEL_GEO,
         pretraining="BATON in-cabin pose pretraining of MotionBERT-kp itself (`motionbert_kp_batonpre_pretrain_s123`), trunk-only init",
         source="kv2_program.md last block §3.1"),
    dict(key="STG3", label="ST-GCN-kp (n=3)", block="single", runs=rr("stgcn_kp_s{s}_v2", SEEDS3), reading=None, selection=SEL_GEO, pretraining="none",
         source="stage_bc.md §1 'ST-GCN-kp (shared recipe, hardening D)'"),
    dict(key="STG3_pre", label="ST-GCN-kp + BATON trunk (n=3)", block="single", runs=rr("stgcn_kp_batonpre_trunkp123_s{s}_v2", SEEDS3), reading=None,
         selection=SEL_GEO, pretraining="BATON in-cabin pose pretraining of ST-GCN-kp itself (`stgcn_kp_batonpre_pretrain_s123`), trunk-only init",
         source="arch_attribution.md §1.2"),
    dict(key="SiM3", label="SiMLPe-kp (n=3)", block="single", runs=rr("simlpe_kp_s{s}_v2", SEEDS3), reading=None, selection=SEL_GEO, pretraining="none",
         source="stage_a.md §1; stage_bc.md §1"),
    dict(key="DWMkp3", label="Driver-WM-kp (Driver-WM architecture on the keypoint part tokens, n=3)", block="single", runs=rr("driverwm_kp_s{s}_v2", SEEDS3),
         reading=None, selection=SEL_GEO, pretraining="none", source="arch_attribution.md §2 (per-seed levels; no seed-mean row exists there)"),
    dict(key="DWMrepro3", label="Driver-WM reproduction (our port of the published architecture, n=3)", block="single", runs=rr("driverwm_repro_s{s}_v2", SEEDS3),
         reading="a", selection="val All-MPJPE (repro recipe: no EMA); F1 at that checkpoint (reading (a)); no reading-(b) checkpoint", pretraining="none",
         source="stage_bc.md §1 'Driver-WM reproduction (C1)'"),
    # ---- ensembles (not single models)
    dict(key="ensA3", label="A3 logit ensemble of the three (b) view-gate seeds — ensemble, not a single model", block="ensemble",
         runs=["cwm3d_vis_sema_vg_f1sel_ens3_v2"], reading="b",
         selection="no selection: mean of the members' saved coordinates AND logits (`aide/eval/ensemble.py`); members = the (b) checkpoints",
         pretraining="none", source="stage_bc.md §1 'Hardening A3: logit ENSEMBLE of DER (i) (b) seeds'"),
    dict(key="ensC4", label="Stage C (iv) 3-seed ensemble of CockpitWM-KS — ensemble, not a single model", block="ensemble", runs=["cwm3d_vis_sem_ens3_v2"],
         reading="a", selection="no selection: mean of the members' saved coordinates and logits; members = the (a) checkpoints", pretraining="none",
         source="stage_bc.md §1 'Stage C (iv) 3-seed ENSEMBLE of B2 main'"),
    dict(key="ensB4", label="B4 ensemble of the 3 motion-weighted seeds — ensemble, not a single model", block="ensemble", runs=["cwm3d_vis_sem_mw_ens3_v2"],
         reading="a", selection="no selection: mean of the members' saved coordinates and logits; members = the (a) checkpoints", pretraining="none",
         source="stage_bc.md §1 'Hardening B4 ENSEMBLE of the 3 motion-weighted seeds'"),
    dict(key="ens9", label="B4 9-run ensemble (mw + hw + dct × 3 seeds) — ensemble, not a single model", block="ensemble", runs=["cwm3d_vis_sem_9ens_v2"],
         reading="a", selection="no selection: mean of the members' saved coordinates and logits; members = the (a) checkpoints", pretraining="none",
         source="stage_bc.md §1 'Hardening B4 ENSEMBLE of 9 runs mw + hw + dct'"),
]


# ------------------------------------------------------------------------------------------------------------------ io
def run_json(name: str) -> dict:
    fs = sorted((RUNS / name).glob("run_*_latest.json"))
    return json.loads(fs[0].read_text()) if fs else {}


def n_params(name: str):
    r = run_json(name)
    if r.get("n_params"):
        return int(r["n_params"])
    if name.endswith("__f1sel"):
        return n_params(name[: -len("__f1sel")])
    return None


def harness_record(name: str) -> dict:
    p = RUNS / name / "metrics_test.json"
    if not p.exists():
        raise FileNotFoundError(p)
    return json.loads(p.read_text())


def load_pred(name: str, clip_ids: np.ndarray):
    z = np.load(RUNS / name / "pred_test.npz", allow_pickle=False)
    order = align(z, clip_ids)
    pred = np.asarray(z["pred"], dtype=np.float64)[order]
    logits = {t: np.asarray(z[f"logits_{t}"], dtype=np.float64)[order] for t in P.TASKS if f"logits_{t}" in z.files}
    return pred, logits


# ------------------------------------------------------------------------------------------------------------- metrics
def recompute(pred: np.ndarray, logits: dict, gt: np.ndarray, hm: np.ndarray, labels: dict) -> dict:
    """The master-table cells from a prediction, with the functions of aide/eval/metrics.py (exactly the harness' code path)."""
    err = M.joint_errors(pred, gt)                               # [N, 5, 136]
    out = {"mpjpe_all": M.mpjpe(err), "dn_mpjpe": M.dn_mpjpe(err),
           "mpjpe_hm": M.mpjpe(err[hm]), "dn_mpjpe_hm": M.dn_mpjpe(err[hm]),
           "pck05": M.pck(err, 0.05, "max_side")}
    for t, lg in logits.items():
        out[f"f1_{t}"] = M.macro_f1(labels[t], np.asarray(lg).argmax(-1))
    return out


def check_run(name: str, d: dict, gt, hm, thr, labels, tol: float):
    pred, logits = load_pred(name, d["clip_id"])
    assert pred.shape == (len(d["clip_id"]), P.T_PRED, P.J, 2), pred.shape
    assert np.isfinite(pred).all(), f"{name}: non-finite predictions"
    rec = harness_record(name)
    ours = recompute(pred, logits, gt, hm, labels)
    hv = {k: rec["metrics"][k]["value"] for k in ours if k in rec["metrics"]}
    missing_in_harness = [k for k in ours if k not in rec["metrics"]]
    extra_in_harness = [k for k in F1 if k in rec["metrics"] and k not in ours]
    diffs = {k: abs(ours[k] - hv[k]) for k in hv}
    mism = [f"{name}: {k} recomputed {ours[k]!r} vs metrics_test.json {hv[k]!r} (|Δ| = {diffs[k]:.3e} > {tol:g})" for k in hv if diffs[k] > tol]
    if missing_in_harness:
        mism.append(f"{name}: recomputed {missing_in_harness} but metrics_test.json has no such key")
    if extra_in_harness:
        mism.append(f"{name}: metrics_test.json has {extra_in_harness} but pred_test.npz carries no logits")
    if abs(rec["hm_threshold"] - thr) > 1e-6:
        mism.append(f"{name}: hm_threshold recomputed {thr!r} vs metrics_test.json {rec['hm_threshold']!r}")
    if rec.get("n") != len(d["clip_id"]) or rec.get("split") != "test" or rec.get("variant", "protocol") != "protocol":
        mism.append(f"{name}: harness record is not a protocol test evaluation of n={len(d['clip_id'])} (n={rec.get('n')}, split={rec.get('split')}, variant={rec.get('variant')})")
    return {"run": name, "recomputed": ours, "harness": hv, "max_abs_diff": (max(diffs.values()) if diffs else None),
            "harness_utc": rec.get("utc"), "harness_pred": rec.get("pred"), "mismatches": mism, "has_logits": bool(logits),
            "pred": pred, "logits": logits}


def reaverage_ensemble(name: str, members: list[str]):
    """Exactly aide/eval/ensemble.py: rows aligned to the FIRST member's clip order, float64 mean, stored as float32."""
    zs = [np.load(RUNS / m / "pred_test.npz", allow_pickle=False) for m in members]
    ids = zs[0]["clip_id"].astype(str)
    arrs, logits = [], {}
    for z in zs:
        order = {c: i for i, c in enumerate(z["clip_id"].astype(str))}
        idx = [order[c] for c in ids]
        arrs.append(np.asarray(z["pred"], np.float64)[idx])
        for t in P.TASKS:
            if f"logits_{t}" in z.files:
                logits.setdefault(t, []).append(np.asarray(z[f"logits_{t}"], np.float64)[idx])
    pred = np.mean(arrs, 0).astype(np.float32)
    lg = {t: np.mean(v, 0).astype(np.float32) for t, v in logits.items()}
    saved = np.load(RUNS / name / "pred_test.npz", allow_pickle=False)
    so = {c: i for i, c in enumerate(saved["clip_id"].astype(str))}
    sidx = [so[c] for c in ids]
    d_pred = float(np.abs(pred - np.asarray(saved["pred"], np.float32)[sidx]).max())
    d_lg = {t: float(np.abs(lg[t] - np.asarray(saved[f"logits_{t}"], np.float32)[sidx]).max()) for t in lg if f"logits_{t}" in saved.files}
    missing = [t for t in lg if f"logits_{t}" not in saved.files] + [t for t in P.TASKS if f"logits_{t}" in saved.files and t not in lg]
    return {"members": members, "n_members": len(members), "max_abs_diff_pred_px": d_pred, "max_abs_diff_logits": d_lg, "logit_key_mismatch": missing}


# ------------------------------------------------------------------------------------------------------------ format
def fmt(x: float, nd: int) -> str:
    return f"{x:.{nd}f}"


def ms(vals, nd: int) -> str:
    v = np.asarray(vals, dtype=np.float64)
    return f"{v.mean():.{nd}f} ± {v.std(ddof=1):.{nd}f}" if len(v) > 1 else f"{v[0]:.{nd}f}"


def agg(vals: list[float]) -> dict:
    v = np.asarray(vals, dtype=np.float64)
    return {"mean": float(v.mean()), "sd": (float(v.std(ddof=1)) if len(v) > 1 else None), "n": int(len(v)), "per_seed": [float(x) for x in v]}


def published_rows() -> list[dict]:
    """The 18 rows of stage_bc.md §1b, cell strings VERBATIM; numeric cross-check against paper_rows.PAPER_ROWS."""
    ls = (TABLES / "stage_bc.md").read_text(encoding="utf-8").splitlines()
    i0 = next(i for i, l_ in enumerate(ls) if l_.startswith("## 1b."))
    tab = []
    j = i0 + 1
    while j < len(ls) and not ls[j].lstrip().startswith("|"):
        j += 1
    while j < len(ls) and ls[j].lstrip().startswith("|"):
        tab.append(ls[j])
        j += 1
    header = [c.strip() for c in tab[0].strip().strip("|").split("|")]
    rows = []
    for l_ in tab[2:]:
        cells = [c.strip() for c in l_.strip().strip("|").split("|")]
        r = dict(zip(header, cells))
        rows.append(r)
    assert len(rows) == 18, f"stage_bc.md §1b: expected 18 published rows, found {len(rows)}"
    cmap = {"All": "mpjpe_all", "d-n": "dn_mpjpe", "HM": "mpjpe_hm", "HM d-n": "dn_mpjpe_hm", "PCK@0.05": "pck05",
            "F1 DBR": "f1_dbr", "F1 DER": "f1_der", "F1 TCR": "f1_tcr", "F1 VCR": "f1_vcr"}
    problems = []
    for r in rows:
        pr = PAPER_ROWS[r["row"]]
        for c, k in cmap.items():
            s = r[c]
            v = None if s in ("–", "—", "-", "") else float(s)
            if (v is None) != (pr[k] is None) or (v is not None and abs(v - pr[k]) > 1e-9):
                problems.append(f"§1b row {r['row']!r} column {c}: table {s!r} vs paper_rows.py {pr[k]!r}")
        if r["group"] != PAPER_GROUP[r["row"]]:
            problems.append(f"§1b row {r['row']!r}: group {r['group']!r} vs paper_rows.py {PAPER_GROUP[r['row']]!r}")
    return rows, header, i0 + 1, problems


# ------------------------------------------------------------------------------------------------------------- main
def main(argv=None) -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument("--tol", type=float, default=1e-6)
    ap.add_argument("--out", type=Path, default=TABLES)
    a = ap.parse_args(argv)
    t0 = time.time()
    stamp = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime())

    d = load_split("test")
    N = len(d["clip_id"])
    gt = d["S10"][:, P.T_OBS:]                                   # [N, 5, 136, 3]; only the future block is ever touched here
    hm, thr, _ = M.hm_mask(gt)
    labels = {t: d[f"y_{t}"] for t in P.TASKS}
    assert N == P.EXPECTED_SPLIT_SIZES["test"] and int(hm.sum()) == P.hm_top(N) == 60, (N, int(hm.sum()))
    print(f"[master] test n = {N}; HM top {int(hm.sum())} (threshold {thr:.4f} px); PCK@0.05 threshold {P.pck_threshold_px(0.05):.1f} px; diag {P.DIAG:.3f}")

    # ---- every run of every group: recompute + check
    checks: dict[str, dict] = {}
    all_mism: list[str] = []
    missing_groups: list[dict] = []
    groups_out: dict[str, dict] = {}
    for g in GROUPS:
        present = [r for r in g["runs"] if (RUNS / r / "pred_test.npz").exists() and (RUNS / r / "metrics_test.json").exists()]
        absent = [r for r in g["runs"] if r not in present]
        if absent:
            missing_groups.append({"key": g["key"], "label": g["label"], "missing_runs": absent})
        if not present:
            continue
        for r in present:
            if r not in checks:
                checks[r] = check_run(r, d, gt, hm, thr, labels, a.tol)
                all_mism += checks[r]["mismatches"]
                print(f"[check] {r:55s} max|Δ| vs harness = {checks[r]['max_abs_diff']:.2e}" + ("" if not checks[r]["mismatches"] else "  MISMATCH"))
        vals = {k: [checks[r]["recomputed"][k] for r in present if k in checks[r]["recomputed"]] for k in KEYS}
        vals = {k: v for k, v in vals.items() if len(v) == len(present) and v}
        seeds = [run_json(r).get("seed") for r in present]
        pj = [n_params(r) for r in present]
        pj = sorted({p for p in pj if p is not None})
        groups_out[g["key"]] = {
            "label": g["label"], "block": g["block"], "runs": present, "missing_runs": absent, "n": len(present),
            "seeds": seeds, "params": (pj[0] if len(pj) == 1 else (pj if pj else (0 if g["block"] == "analytic" else None))),
            "reading": g["reading"], "selection": g["selection"], "pretraining": g["pretraining"], "source": g["source"],
            "cells": {k: agg(v) for k, v in vals.items()},
            "per_run": {r: {"recomputed": checks[r]["recomputed"], "harness": checks[r]["harness"], "max_abs_diff": checks[r]["max_abs_diff"],
                            "harness_utc": checks[r]["harness_utc"], "mismatches": checks[r]["mismatches"]} for r in present},
        }
        if g["block"] == "ensemble":
            members = run_json(present[0]).get("members")
            if not members:
                groups_out[g["key"]]["reaverage"] = {"error": "run json has no 'members' list"}
                all_mism.append(f"{present[0]}: run json has no 'members' list — ensemble could not be re-averaged")
            else:
                ra = reaverage_ensemble(present[0], members)
                # metrics of the RE-AVERAGED prediction (float32 as ensemble.py stores it) must equal the saved-prediction metrics
                zs0 = np.load(RUNS / members[0] / "pred_test.npz", allow_pickle=False)
                ids = zs0["clip_id"].astype(str)
                pos = {c: i for i, c in enumerate(ids)}
                order = np.array([pos[c] for c in d["clip_id"].astype(str)])
                arrs, lgs = [], {}
                for m_ in members:
                    z = np.load(RUNS / m_ / "pred_test.npz", allow_pickle=False)
                    o = {c: i for i, c in enumerate(z["clip_id"].astype(str))}
                    idx = [o[c] for c in ids]
                    arrs.append(np.asarray(z["pred"], np.float64)[idx])
                    for t in P.TASKS:
                        if f"logits_{t}" in z.files:
                            lgs.setdefault(t, []).append(np.asarray(z[f"logits_{t}"], np.float64)[idx])
                pred_ra = np.mean(arrs, 0).astype(np.float32).astype(np.float64)[order]
                lg_ra = {t: np.mean(v, 0).astype(np.float32).astype(np.float64)[order] for t, v in lgs.items()}
                rec_ra = recompute(pred_ra, lg_ra, gt, hm, labels)
                ra["metrics_from_reaveraged_members"] = rec_ra
                ra["max_abs_diff_metrics_vs_saved_pred"] = max(abs(rec_ra[k] - checks[present[0]]["recomputed"][k]) for k in rec_ra)
                if ra["max_abs_diff_metrics_vs_saved_pred"] > a.tol or ra["logit_key_mismatch"]:
                    all_mism.append(f"{present[0]}: re-averaged members differ from the saved ensemble prediction (metrics max|Δ| {ra['max_abs_diff_metrics_vs_saved_pred']:.3e}; logit keys {ra['logit_key_mismatch']})")
                groups_out[g["key"]]["reaverage"] = ra
                print(f"[ensemble] {present[0]}: {len(members)} members re-averaged; max|Δ| pred {ra['max_abs_diff_pred_px']:.2e} px, logits {ra['max_abs_diff_logits']}, metrics {ra['max_abs_diff_metrics_vs_saved_pred']:.2e}")

    pub_rows, pub_header, pub_line, pub_problems = published_rows()
    all_mism += [f"published §1b vs paper_rows.py: {p}" for p in pub_problems]
    n_checked = len(checks)
    max_diff = max(c["max_abs_diff"] for c in checks.values() if c["max_abs_diff"] is not None)

    # ---- best per column among SINGLE models (mean values at the displayed rounding; ties → no mark)
    def best_marks(keys_ok, table_reading):
        marks = {}
        for k in KEYS:
            cand = []
            for key, g in groups_out.items():
                if g["block"] != "single" or k not in g["cells"]:
                    continue
                if k in F1 and g["reading"] != table_reading:
                    continue
                cand.append((round(g["cells"][k]["mean"], ND[k]), key))
            if not cand:
                continue
            target = (min if k in LOWER else max)(v for v, _ in cand)
            w = [key for v, key in cand if v == target]
            if len(w) == 1:
                marks[k] = w[0]
        return marks

    marks_b = best_marks(KEYS, "b")
    marks_a = {k: v for k, v in best_marks(KEYS, "a").items() if k in F1}     # §2 marks F1 only; the geometry bests are those of §1

    def cell(g, k, key, marks, table_reading):
        if k not in g["cells"]:
            return "—"
        if k in F1 and g["reading"] != table_reading:
            return "—"
        s = ms(g["cells"][k]["per_seed"], ND[k])
        return f"**{s}**" if marks.get(k) == key else s

    def table(table_reading: str, marks: dict, with_published: bool, only_semantic: bool) -> list[str]:
        L = ["| " + " | ".join(HEADER) + " |", "|" + "---|" * len(HEADER)]
        grp = lambda text: "| *" + text + "* |" + " |" * (len(HEADER) - 1)  # noqa: E731
        if with_published:
            L.append(grp("Published — single seed, typed from the Driver-WM paper (arXiv 2605.05092v2) as in stage_bc.md §1b; no recomputation possible; the paper's own checkpoint selection"))
            for r in pub_rows:
                L.append("| " + " | ".join([r["row"] + f" ({r['group']})", r["All"], r["d-n"], r["HM"], r["HM d-n"], r["PCK@0.05"], r["F1 DBR"], r["F1 DER"], r["F1 TCR"], r["F1 VCR"]]) + " |")
        blocks = [("analytic", "Analytic rules, our harness (no parameters; single deterministic prediction)"),
                  ("single", f"Single models, our harness — mean ± sd over seeds (ddof 1); F1 = reading ({table_reading}); bold = best per column among these rows"
                             + ("" if table_reading == "b" else " (F1 columns only; the geometry bests are marked in §1)")),
                  ("ensemble", "Ensembles — not single models (mean of the members' saved coordinates and logits)")]
        for blk, title in blocks:
            rows = [(key, g) for key, g in groups_out.items() if g["block"] == blk and (not only_semantic or (g["reading"] == table_reading and any(k in g["cells"] for k in F1)))]
            if not rows:
                continue
            L.append(grp(title))
            for key, g in rows:
                L.append("| " + " | ".join([g["label"]] + [cell(g, k, key, marks, table_reading) for k in KEYS]) + " |")
        return L

    # ---- markdown
    L = [f"# AIDE master table — Driver-WM column format (official test split, n = {N})", "",
         f"Generated {stamp} by `aide/eval/report_master_aide.py`.  Every value of every OUR row is RECOMPUTED from the run's saved `pred_test.npz` "
         f"(+ saved semantic logits) with `aide/eval/metrics.py` on the `load_split(\"test\")` ground truth, then cross-checked against the run's harness record "
         f"`metrics_test.json` (tolerance {a.tol:g}); nothing is copied from a CSV or an earlier table.  Locked protocol (`aide/data/protocol.py`, Gate 0): driver `{P.DRIVER_RULE}`, "
         f"grid `{P.SAMPLE_GRID}`, {P.T_OBS} observed → {P.T_PRED} future protocol frames, HALPE-136 pixels of a {P.W}×{P.H} image.  Group rows: mean ± sd over seeds (sample sd, ddof 1); "
         f"n in the label.  '—' = the model has no output for that column.  **Recompute-vs-harness check: {n_checked} runs, max |Δ| = {max_diff:.2e}; "
         f"{len(all_mism)} mismatch(es)**" + (":" if all_mism else ".") , ""]
    L += [f"- MISMATCH: {m}" for m in all_mism]
    if all_mism:
        L.append("")
    L += ["## 1. Master table — semantic F1 under reading (b): checkpoint selected on validation macro-F1 (dual selection, `__f1sel`); '—' in an F1 cell of a semantic model = that recipe has no reading-(b) checkpoint (see §2)", ""]
    L += table("b", marks_b, with_published=True, only_semantic=False)
    L += ["", "## 2. Same columns, semantic F1 under reading (a): F1 of the checkpoint selected on validation All-MPJPE (the geometry checkpoint; protocol S5) — semantic models only", ""]
    L += table("a", marks_a, with_published=False, only_semantic=True)
    L += ["", "For reference under reading (a): published Driver-WM main (single seed, the paper's own selection) F1 DBR / DER / TCR / VCR = "
          + " / ".join(next(r for r in pub_rows if r["row"] == "Driver-WM main")[c] for c in ["F1 DBR", "F1 DER", "F1 TCR", "F1 VCR"]) + " (stage_bc.md §1b).", ""]

    # ---- provenance
    L += ["## 3. Provenance per group (runs, seeds, params, selection, pretraining, source table, recompute check)", "",
          "| group | runs (aide/results/runs/) | n | seeds | params | selection rule | pretraining | source table | recompute vs harness: max abs diff |",
          "|---|---|---|---|---|---|---|---|---|"]
    for key, g in groups_out.items():
        md = max(g["per_run"][r]["max_abs_diff"] for r in g["runs"])
        params = f"{g['params']:,}" if isinstance(g["params"], int) else ("—" if g["params"] is None else str(g["params"]))
        if g["block"] == "ensemble":
            params = "— (members: " + ", ".join(f"{n_params(m):,}" for m in g.get("reaverage", {}).get("members", []) if n_params(m)) + ")" if g.get("reaverage", {}).get("members") else "—"
        seeds = "/".join(str(s) for s in g["seeds"] if s is not None) or "—"
        L.append(f"| {g['label']} | " + ", ".join(f"`{r}`" for r in g["runs"]) + f" | {g['n']} | {seeds} | {params} | {g['selection']} | {g['pretraining']} | {g['source']} | {md:.2e} |")
    L += ["", "### 3.1 Per-run recomputed values (test) and the harness check", "",
          "| run | All | d-n | HM | HM d-n | PCK@0.05 | F1 DBR | F1 DER | F1 TCR | F1 VCR | max abs diff vs metrics_test.json | harness UTC |",
          "|---|---|---|---|---|---|---|---|---|---|---|---|"]
    for r, c in checks.items():
        v = c["recomputed"]
        L.append(f"| `{r}` | " + " | ".join(fmt(v[k], ND[k]) if k in v else "—" for k in KEYS) + f" | {c['max_abs_diff']:.2e} | {c['harness_utc']} |")
    L += ["", "### 3.2 Ensembles re-averaged from their members exactly as `aide/eval/ensemble.py` (float64 mean of the members' saved coordinates / logits, stored float32)", "",
          "| ensemble | members | max abs diff re-averaged vs saved pred (px) | max abs diff logits | metrics(re-averaged) vs metrics(saved) max abs diff |", "|---|---|---|---|---|"]
    for key, g in groups_out.items():
        if g["block"] == "ensemble" and "reaverage" in g and "members" in g["reaverage"]:
            ra = g["reaverage"]
            L.append(f"| `{g['runs'][0]}` | " + ", ".join(f"`{m}`" for m in ra["members"]) + f" | {ra['max_abs_diff_pred_px']:.2e} | "
                     + ", ".join(f"{t} {v:.2e}" for t, v in ra["max_abs_diff_logits"].items()) + f" | {ra['max_abs_diff_metrics_vs_saved_pred']:.2e} |")
    L += ["", f"Published rows: stage_bc.md §1b (table header at line {pub_line}), {len(pub_rows)} rows copied verbatim; numeric cross-check against `aide/eval/paper_rows.py`: "
          + ("all 18 rows identical." if not pub_problems else "; ".join(pub_problems))]
    if missing_groups:
        L += ["", "Groups with missing runs (reported, not silently dropped): " + "; ".join(f"{m['label']}: {m['missing_runs']}" for m in missing_groups)]

    # ---- definitions
    L += ["", "## 4. Definitions (aide/docs/protocol.md; aide/eval/metrics.py)", "",
          f"- All MPJPE = mean over all {N} test clips × {P.T_PRED} horizons × {P.J} joints of the Euclidean error (px), all joints regardless of confidence.",
          f"- All d-nMPJPE = 100 · MPJPE / diag({P.W}, {P.H}) = 100 · MPJPE / {P.DIAG:.3f}.",
          f"- HM subset = the top ⌊0.1 · N⌋ = {int(hm.sum())} test clips by mean GT displacement over the {P.HM_N_TRANSITIONS} future frame-to-frame transitions and all joints "
          f"(threshold {thr:.2f} px; the paper's 72.05); HM MPJPE / HM d-nMPJPE = the same two metrics on that subset.",
          f"- PCK@0.05 = percentage of (clip, horizon, joint) errors ≤ 0.05 · max(W, H) = {P.pck_threshold_px(0.05):.0f} px (Gate-0 normaliser).",
          "- Semantic macro-F1 (%) per task (DBR 7 / DER 5 / TCR 3 / VCR 5 classes) = sklearn `f1_score(average='macro')` of argmax logits vs the clip label, observed window only. "
          "Reading (a) = F1 of the checkpoint selected on validation All-MPJPE (the geometry checkpoint; protocol S5); reading (b) = F1 of the checkpoint selected on validation mean macro-F1 "
          "(dual selection, protocol S8, run `<name>__f1sel`, which carries its own geometry). Selection never touches the test split.",
          "- Seeds: locked headline seeds 42 / 123 / 456; the 5-seed sets add 789 / 1011 (protocol C22 / C23, trained with the locked recipe verbatim and reported whatever their value).",
          "", "## 5. How to read", "",
          "- Published rows are single-seed numbers typed from the paper: they cannot be paired with our clips and carry no uncertainty; our rows are seed means ± sd of one test evaluation per run.",
          "- Only the paired per-clip tests in `arch_attribution.md`, `baton_pretrain.md` and `kv2_program.md` (10k clip bootstrap, BEATS / WORSE / NOT_WORSE / INCONCLUSIVE rule) carry significance; "
          "a bold cell here is a descriptive per-column best among the single-model rows of this table, not a test.",
          "- Ensembles average the saved predictions of several trained runs and are never a single model; they are listed separately and excluded from the bold marks.",
          "- The two F1 readings are two checkpoints of the same training run (dual-select recipes) or one checkpoint (recipes without dual selection, which therefore have only reading (a)); "
          "the geometry shown next to an F1 value is always that of the checkpoint whose F1 is shown.",
          "- Analytic rows have no parameters; ZV-shrink's α (and the per-horizon a) were fitted on the validation split only."]

    out_md = a.out / "master_aide.md"
    out_js = a.out / "master_aide.json"
    a.out.mkdir(parents=True, exist_ok=True)
    out_md.write_text("\n".join(L) + "\n", encoding="utf-8")
    js = {"generated_utc": stamp, "script": str(Path(__file__).resolve().relative_to(ROOT)), "tolerance": a.tol, "n_test": N,
          "protocol": {"driver_rule": P.DRIVER_RULE, "sample_grid": P.SAMPLE_GRID, "hm_top": int(hm.sum()), "hm_threshold_px": thr,
                       "hm_n_transitions": P.HM_N_TRANSITIONS, "pck05_threshold_px": P.pck_threshold_px(0.05), "diag_px": P.DIAG},
          "check": {"n_runs": n_checked, "max_abs_diff": max_diff, "mismatches": all_mism},
          "columns": KEYS, "header": HEADER, "readings": {"a": "checkpoint selected on val All-MPJPE (geometry)", "b": "checkpoint selected on val mean macro-F1 (dual selection, __f1sel)"},
          "best_marks": {"reading_b_table": marks_b, "reading_a_table": marks_a},
          "groups": groups_out,
          "published": {"source": f"stage_bc.md §1b (line {pub_line}); typed from arXiv 2605.05092v2 in aide/eval/paper_rows.py", "header": pub_header, "rows": pub_rows,
                        "cross_check_problems": pub_problems},
          "missing_groups": missing_groups, "wall_s": round(time.time() - t0, 1)}
    out_js.write_text(json.dumps(js, indent=1, default=lambda o: o.tolist() if hasattr(o, "tolist") else str(o)), encoding="utf-8")
    print(f"[master] {n_checked} runs checked, max |Δ| vs harness {max_diff:.2e}, {len(all_mism)} mismatch(es)")
    for m in all_mism:
        print("  MISMATCH:", m)
    print(f"[master] wrote {out_md}\n[master] wrote {out_js}  ({time.time() - t0:.1f} s)")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())
Xet Storage Details

Size:: 38.1 kB
Xet hash:: c2588c3e8091980671258046a5d742b985f415fb7ad0adcbef13343eb9a337bc
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.