Buckets:
| """cwm3/aide/eval/report_kv2_program.py — K-v2 programme report (docs/decisions.md 2026-09-14 row "K-v2 programme — S0 pre-registration", | |
| protocol C23) -> aide/results/tables/kv2_program.md (+ .json), append-only dated blocks. | |
| Reads ONLY run outputs (pred_val.npz / pred_test.npz, metrics_val.json / metrics_test.json, run json), the gate decisions written by | |
| aide/eval/kv2_gate.py (aide/results/kv2/*.json) and the pre-registration row itself (docs/decisions.md, quoted verbatim). Paired per-clip | |
| bootstraps are recomputed here with aide/eval/metrics.paired_diff (10k, seed 0) exactly as arch_attribution.md; verdicts use | |
| report_arch_attribution.verdict (BEATS / WORSE = CI excl. 0 on >= ceil(2n/3) seeds, same sign on all; NOT_WORSE = CIs cover 0 and |Δ| < 0.25; | |
| else INCONCLUSIVE). Confirmatory cells (P1, P2, A1, A2) require all 5 seeds; anything else is "pending". Runs gracefully with partial results. | |
| Sections: §0 pre-registration + earliest run + timestamp check; §1 S1 / S1b (val; test recorded only); §2 S2 5-seed tables (test) with paired | |
| Δ vs K / K+pre / MB-best / MB-best+pre; §3 MB fairness arm; §4 P1 / P2 / A1 / A2 + consequence table + `**ADOPTION A1: ADOPTED|NOT ADOPTED**`; | |
| §5 GPU minutes; §6 provenance. | |
| python aide/eval/report_kv2_program.py [--n-boot 10000] [--out-root aide/results] | |
| """ | |
| from __future__ import annotations | |
| import os | |
| os.environ.setdefault("OMP_NUM_THREADS", "4") | |
| import argparse # noqa: E402 | |
| import glob # noqa: E402 | |
| import json # noqa: E402 | |
| import math # noqa: E402 | |
| import re # noqa: E402 | |
| import sys # noqa: E402 | |
| import time # noqa: E402 | |
| from pathlib import Path # noqa: E402 | |
| import numpy as np # noqa: E402 | |
| ROOT = Path(__file__).resolve().parents[2] | |
| sys.path.insert(0, str(ROOT)) | |
| from aide.data import protocol as P # noqa: E402 | |
| from aide.eval import kv2_gate as G # noqa: E402 | |
| from aide.eval import metrics as M # noqa: E402 | |
| from aide.eval.harness import align, load_split # noqa: E402 | |
| from aide.eval.report_arch_attribution import TOL_NOT_WORSE, delta_table, fmt_ci, ms, per_clip_cells, verdict # noqa: E402 | |
| from common.runmeta import write_run_json # noqa: E402 | |
| DECISIONS = ROOT / "docs" / "decisions.md" | |
| ROW_MARK = "K-v2 programme — S0 pre-registration" | |
| SEEDS5 = G.SEEDS5 | |
| LEVEL_COLS = [("mpjpe_all", "All", 2), ("dn_mpjpe", "d-n", 3), ("mpjpe_hm", "HM", 2), ("pck05", "PCK@0.05", 2), ("hm_h5", "HM h=5", 2)] | |
| MAIN_CELLS = ("All-MPJPE", "d-n", "HM-MPJPE", "PCK@0.05", "HM h=5") | |
| CONSEQUENCES = [("P1 ∧ P2", "an architecture advantage under the keypoint formulation may be written FOR K-v2 (K's own attribution sentence unchanged)"), | |
| ("P1 ∧ ¬P2", "\"advantage without pretraining that the BATON trunk equalises\"; attribution sentence kept"), | |
| ("¬P1 ∧ P2", "\"advantage conditional on the pretrained trunk\"; attribution sentence kept"), | |
| ("neither", "wording unchanged verbatim; K-v2 = ablation family + gen_tables.py NEGATIVES entry; any WORSE reported as is")] | |
| # ----------------------------------------------------------------------------- io (out-root aware) | |
| class IO: | |
| def __init__(self, out_root: Path): | |
| self.root = Path(out_root) | |
| self.runs, self.tables, self.kv2 = self.root / "runs", self.root / "tables", self.root / "kv2" | |
| def run_json(self, name): | |
| return G.run_json(name, self.runs) | |
| def metrics(self, name, split): | |
| p = self.runs / name / f"metrics_{split}.json" | |
| return json.load(open(p))["metrics"] if p.exists() else None | |
| def pred(self, name, split, d): | |
| p = self.runs / name / f"pred_{split}.npz" | |
| if not p.exists(): | |
| return None | |
| z = np.load(p, allow_pickle=False) | |
| return np.asarray(z["pred"], dtype=np.float64)[align(z, d["clip_id"])] | |
| def haz(self, name, split, d): | |
| """Per-clip hazard probability summary (max / mean over all trailing axes of `haz_P`), None if the run does not save it.""" | |
| p = self.runs / name / f"pred_{split}.npz" | |
| if not p.exists(): | |
| return None | |
| z = np.load(p, allow_pickle=False) | |
| if "haz_P" not in z.files: | |
| return None | |
| h = np.asarray(z["haz_P"], dtype=np.float64)[align(z, d["clip_id"])].reshape(len(d["clip_id"]), -1) | |
| return {"max": h.max(1), "mean": h.mean(1)} | |
| def finished(self, name): | |
| return G.finished(name, self.runs) | |
| def decision(self, stem): | |
| p = self.kv2 / f"{stem}.json" | |
| return json.load(open(p)) if p.exists() else None | |
| def meta(self, name): | |
| r = self.run_json(name) or {} | |
| return r.get("best_epoch", "—"), r.get("epochs_run", "—"), r.get("stop_reason", "—"), r.get("gpu_minutes"), r.get("n_params"), r.get("t_start_utc") | |
| def paired(io: IO, a_name, b_name, split, d, gt, hm, n_boot, full=False): | |
| pa, pb = io.pred(a_name, split, d), io.pred(b_name, split, d) | |
| if pa is None or pb is None: | |
| return None | |
| ca, cb = per_clip_cells(pa, gt, hm, full), per_clip_cells(pb, gt, hm, full) | |
| return {c: {"a": float(ca[c][0].mean()), "b": float(cb[c][0].mean()), **M.paired_diff(ca[c][0], cb[c][0], n_boot, 0), "higher_better": ca[c][1]} for c in ca} | |
| def rank_auc(score, label): | |
| """Mann–Whitney AUC of `score` for label == True (ties -> average ranks); nan if one class is empty.""" | |
| score, label = np.asarray(score, float), np.asarray(label, bool) | |
| n1, n0 = int(label.sum()), int((~label).sum()) | |
| if n1 == 0 or n0 == 0: | |
| return float("nan") | |
| order = np.argsort(score, kind="mergesort") | |
| ranks = np.empty(len(score), float) | |
| s = score[order] | |
| i = 0 | |
| while i < len(s): | |
| j = i | |
| while j + 1 < len(s) and s[j + 1] == s[i]: | |
| j += 1 | |
| ranks[order[i:j + 1]] = 0.5 * (i + j) + 1.0 | |
| i = j + 1 | |
| return float((ranks[label].sum() - n1 * (n1 + 1) / 2) / (n1 * n0)) | |
| def level_row(io: IO, name, split="test", cols=LEVEL_COLS): | |
| m = io.metrics(name, split) | |
| if m is None: | |
| return " | ".join("—" for _ in cols) | |
| return " | ".join(f"{m[c]['value']:.{nd}f} [{m[c]['ci_lo']:.{nd}f}, {m[c]['ci_hi']:.{nd}f}]" if c in m else "—" for c, _, nd in cols) | |
| def val_all(io: IO, name, dv, gtv): | |
| pr = io.pred(name, "val", dv) | |
| return None if pr is None else float(M.joint_errors(pr, gtv).mean()) | |
| def test_all(io: IO, name): | |
| m = io.metrics(name, "test") | |
| return None if m is None else m["mpjpe_all"]["value"] | |
| def f(x, nd=3, plus=False): | |
| if x is None or (isinstance(x, float) and math.isnan(x)): | |
| return "pending" | |
| return f"{x:+.{nd}f}" if plus else f"{x:.{nd}f}" | |
| # ----------------------------------------------------------------------------- report | |
| def main(argv=None): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--n-boot", type=int, default=10000) | |
| ap.add_argument("--out-root", default=str(P.RESULTS), help="root holding runs/, tables/, kv2/ (tests use a tmp root)") | |
| ap.add_argument("--decisions", default=str(DECISIONS)) | |
| a = ap.parse_args(argv) | |
| t0 = time.time() | |
| io = IO(a.out_root) | |
| out = io.tables / "kv2_program.md" | |
| stamp = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime()) | |
| dv = load_split("val"); gtv = dv["S10"][:, P.T_OBS:]; hmv, thrv, _ = M.hm_mask(gtv) | |
| dt = load_split("test"); gtt = dt["S10"][:, P.T_OBS:]; hmt, thrt, _ = M.hm_mask(gtt) | |
| summary, gpu, pend = {}, {}, [] | |
| def note_gpu(name): | |
| r = io.run_json(name) | |
| if r is not None and r.get("gpu_minutes") is not None: | |
| gpu[name] = float(r["gpu_minutes"]) | |
| # ---------------------------------------------------------------- §0 pre-registration | |
| row = next((l.rstrip("\n") for l in open(a.decisions, encoding="utf-8") if ROW_MARK in l), None) if Path(a.decisions).exists() else None | |
| m_utc = re.search(r"S0 UTC (\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z)", row or "") | |
| s0_utc = m_utc.group(1) if m_utc else ((io.kv2 / "S0_UTC.txt").read_text().strip() if (io.kv2 / "S0_UTC.txt").exists() else None) | |
| programme_runs = [] | |
| for cfg in G.KV2_CONFIGS: | |
| for s in SEEDS5: | |
| programme_runs += [G.kv2_run(cfg, s), G.kv2_run(cfg, s, pre=True)] | |
| programme_runs += sorted({G.kv2_pretrain_run(c) for c in G.KV2_CONFIGS}) + [G.k_run(s, pre=True) for s in (789, 1011)] | |
| for cfg in G.MB_GRID: | |
| for s in SEEDS5: | |
| if cfg == "default" and s not in (789, 1011): | |
| continue # the existing default MB runs are not programme runs | |
| programme_runs += ([G.mb_run(cfg, s)] if cfg != "default" else []) + [G.mb_run(cfg, s, pre=True)] | |
| if cfg != "default" and G.mb_changes_dl(cfg): | |
| programme_runs.append(G.mb_pretrain_run(cfg)) | |
| programme_runs = list(dict.fromkeys(programme_runs)) | |
| started = {n: (io.run_json(n) or {}).get("t_start_utc") for n in programme_runs if io.run_json(n) is not None} | |
| earliest = min(started.values()) if started else None | |
| violations = sorted(n for n, t in started.items() if s0_utc and t and t <= s0_utc) | |
| for n in started: | |
| note_gpu(n) | |
| L = [f"# K-v2 programme — CockpitWM-K improvement, pre-registered 2026-09-14 (protocol C23) — val n = {len(dv['clip_id'])} (selection), test n = {len(dt['clip_id'])} (one evaluation per run), paired per-clip bootstrap ({a.n_boot})", "", | |
| f"Generated {stamp} by `aide/eval/report_kv2_program.py` from pred_{{val,test}}.npz / metrics_{{val,test}}.json / run json / `aide/results/kv2/*.json` only (locked protocol: driver `first`, grid `floor`, " | |
| f"val HM top {int(hmv.sum())} (threshold {thrv:.2f} px, NEVER a ranking statistic), test HM top {int(hmt.sum())} (threshold {thrt:.2f} px), PCK @ {P.pck_threshold_px(0.05):.0f} px; every run selected on val All-MPJPE of the EMA weights only, test evaluated once by the harness). " | |
| f"Δ = paired per-clip difference (negative = better for errors, positive = better for PCK), CI = clip-bootstrap percentile interval. **Verdict rule (as `arch_attribution.md`): BEATS / WORSE = Δ CI excl. 0 on ≥ ceil(2n/3) seeds (5 seeds: 4) with the same sign on every seed; NOT_WORSE = all CIs cover 0 and |Δ| < {TOL_NOT_WORSE}; else INCONCLUSIVE.** Confirmatory cells need all 5 seeds; otherwise \"pending\".", "", | |
| "## 0. Pre-registration (verbatim decisions row) and timestamp check", ""] | |
| L += [f"- S0 UTC: **{s0_utc or 'NOT FOUND'}**; earliest programme run `t_start_utc`: **{earliest or 'no programme run finished yet'}** ({len(started)} programme runs finished of {len(programme_runs)} possible names); " | |
| f"runs started at or before S0: {('**VIOLATION** ' + ', '.join(violations)) if violations else 'none'}.", ""] | |
| L += ["> " + (row if row else f"_(pre-registration row containing \"{ROW_MARK}\" not found in {a.decisions})_"), ""] | |
| summary["s0"] = {"s0_utc": s0_utc, "earliest_run_utc": earliest, "n_finished": len(started), "violations": violations} | |
| # ---------------------------------------------------------------- §1 S1 / S1b | |
| L += ["## 1. S1 val screen (seed 42; statistic = paired per-clip Δ val All vs `cwm3d_kp_s42_v2`; SURVIVE iff mean Δ ≤ −0.30 px; ≤ 2 survivors, ties 0.10 → fewer components; HM guard KILL iff Δ val HM > +3.0 with CI excl. 0 — the val HM column is SECONDARY)", ""] | |
| s1 = io.decision("s1_decision") | |
| live1 = None | |
| if s1 is not None and not s1.get("complete"): | |
| s1 = None # an incomplete decision json is superseded by a live recomputation (still not a decision) | |
| if s1 is None: | |
| try: | |
| live1 = G.stage_s1(io.runs, io.kv2, a.n_boot, write=False) | |
| except FileNotFoundError as e: | |
| live1 = {"rows": [], "survivors": [], "complete": False, "error": str(e)} | |
| src1 = "no complete `s1_decision.json` yet — LIVE recomputation (not a decision)" | |
| s1_use = live1 | |
| else: | |
| src1 = f"`aide/results/kv2/s1_decision.json` written {s1['utc']} (complete = {s1['complete']})" | |
| s1_use = s1 | |
| L += [f"- Decision source: {src1}. Rule: {G.RULE_S1}.", "", | |
| "| cfg | run | flags | params | best ep / epochs / stop / GPU min | val All | Δ val All [CI] | Δ val HM [CI] (secondary) | guard | S1 status | test All (recorded, not used) |", "|---|---|---|---|---|---|---|---|---|---|---|"] | |
| kb = G.k_run(42) | |
| L.append(f"| K (baseline) | {kb} | — | {(io.run_json(kb) or {}).get('n_params', '—')} | {' / '.join(str(x) for x in io.meta(kb)[:4])} | {f(val_all(io, kb, dv, gtv))} | 0 | 0 | — | — | {f(test_all(io, kb))} |") | |
| for r in s1_use.get("rows", []): | |
| cfg, name = r["cfg"], r["run"] | |
| fl = " ".join(G.KV2_CONFIGS[cfg][0]) or "(full)" | |
| if r.get("status") == "PENDING": | |
| pend.append(name) | |
| L.append(f"| {cfg} | {name} | {fl} | — | pending | pending | pending | pending | — | PENDING | — |") | |
| continue | |
| be, ep, st, gm, npar, _ = io.meta(name) | |
| note_gpu(name) | |
| L.append(f"| {cfg} | {name} | {fl} | {npar if npar is not None else '—'} | {be} / {ep} / {st} / {gm} | {r['val_all']:.3f} | {fmt_ci(r['d_all'], 3)} | {fmt_ci(r['d_hm'], 2)} | " | |
| f"{'KILL' if r['guard_kill'] else 'ok'} | {r['status']} | {f(test_all(io, name))} |") | |
| surv = list(s1_use.get("survivors", [])) | |
| L += ["", f"**S1 survivors ({'decision' if s1 else 'provisional, live'}): {', '.join(surv) if surv else 'none'}{' — S1 incomplete: ' + str(sum(r.get('status') == 'PENDING' for r in s1_use.get('rows', []))) + ' run(s) pending' if not s1_use.get('complete') else ''}.**", ""] | |
| summary["s1"] = {"source": "decision" if s1 else "live", "complete": bool(s1_use.get("complete")), "survivors": surv, | |
| "rows": {r["cfg"]: {k: r.get(k) for k in ("status", "val_all", "d_all", "d_hm", "guard_kill")} for r in s1_use.get("rows", [])}} | |
| # S1b | |
| L += ["### 1.1 S1b (survivors at seed 123 vs `cwm3d_kp_s123_v2`; CONTINUE iff Δ val(123) < 0 and 2-seed mean ≤ −0.25)", "", | |
| "| cfg | seed-123 run | val All | Δ val All s42 [CI] | Δ val All s123 [CI] | 2-seed mean | decision | source |", "|---|---|---|---|---|---|---|---|"] | |
| cont = [] | |
| s1b_all = {} | |
| for cfg in surv: | |
| dec = io.decision(f"s1b_{cfg}_decision") | |
| src = "decision json " + dec["utc"] if dec else "live" | |
| if dec is None: | |
| dec = G.stage_s1b(cfg, io.runs, io.kv2, a.n_boot, write=False) | |
| rows = {r.get("seed"): r for r in dec["rows"]} | |
| r42, r123 = rows.get(42, {}), rows.get(123, {}) | |
| name123 = G.kv2_run(cfg, 123) | |
| if r123.get("status") != "DONE": | |
| pend.append(name123) | |
| d42 = r42.get("d_all"); d123 = r123.get("d_all") | |
| two = 0.5 * (d42["mean"] + d123["mean"]) if d42 and d123 else None | |
| L.append(f"| {cfg} | {name123} | {f(r123.get('val_all'))} | {fmt_ci(d42, 3) if d42 else 'pending'} | {fmt_ci(d123, 3) if d123 else 'pending'} | {f(two, 3, True)} | {dec['decision']} | {src} |") | |
| if dec["decision"] == "CONTINUE": | |
| cont.append(cfg) | |
| s1b_all[cfg] = {"decision": dec["decision"], "reason": dec.get("reason"), "two_seed_mean": two} | |
| if r123.get("status") == "DONE": | |
| note_gpu(name123) | |
| if not surv: | |
| L.append("| — | — | — | — | — | — | (no S1 survivors yet) | — |") | |
| L += ["", f"**S1b: continuing to 5 seeds: {', '.join(cont) if cont else 'none yet'}.**", ""] | |
| summary["s1b"] = s1b_all | |
| # ---------------------------------------------------------------- §3 data needed by §2: MB-best | |
| s3 = io.decision("s3_decision") | |
| live3 = None | |
| if s3 is not None and not s3.get("complete"): | |
| s3 = None # incomplete decision json -> live recomputation (provisional, not a decision) | |
| if s3 is None: | |
| live3 = G.stage_s3(io.runs, io.kv2, a.n_boot, write=False) | |
| s3_use, src3 = live3, "no complete `s3_decision.json` yet — LIVE recomputation (not a decision)" | |
| else: | |
| s3_use, src3 = s3, f"`aide/results/kv2/s3_decision.json` written {s3['utc']} (complete = {s3['complete']})" | |
| mb_best = s3_use.get("mb_best") | |
| mb_best_final = bool(s3 and s3.get("complete") and mb_best) | |
| mb_cfg = mb_best if mb_best else "default" | |
| # ---------------------------------------------------------------- §2 S2 | |
| L += ["## 2. S2 — 5 seeds (42 / 123 / 456 / 789 / 1011), TEST (one evaluation per run), K-v2 survivors without / with BATON trunk-only init", ""] | |
| s2_cfgs = [c for c in G.KV2_CONFIGS if c in cont or any(io.finished(G.kv2_run(c, s)) or io.finished(G.kv2_run(c, s, pre=True)) for s in SEEDS5 if s != 42)] | |
| for pre_run in sorted({G.kv2_pretrain_run(c) for c in s2_cfgs}): | |
| pr = io.run_json(pre_run) | |
| cfgs_of = [c for c in s2_cfgs if G.kv2_pretrain_run(c) == pre_run] | |
| L.append(f"- Pretraining `{pre_run}` (architecture-matched trunk for {', '.join(cfgs_of)}): " + (f"finished ({pr.get('epochs_run')} epochs, best epoch {pr.get('best_epoch')}, held-out All {pr.get('best_corpus_val_all', float('nan')):.2f}, " | |
| f"{pr.get('gpu_minutes')} GPU-min, corpus sha256 {(pr.get('corpus') or {}).get('corpus_sha256', '—')[:16]}, command `{' '.join(pr.get('command', '').split()[1:])}`)" if pr else "not run / not finished") + ".") | |
| if pr: | |
| note_gpu(pre_run) | |
| for c in s2_cfgs: | |
| d_k, d_own = io.decision(f"pretrain_check_{c}"), io.decision(f"pretrain_check_{c}_own") | |
| L.append(f"- pretrain-check `{c}`: vs K checkpoint → {(d_k or {}).get('decision', 'no json')} (loaded {(d_k or {}).get('n_loaded', '—')} / {(d_k or {}).get('n_trunk_tensors', '—')} trunk tensors, shape-skipped {(d_k or {}).get('n_skipped_shape', '—')}, missing {(d_k or {}).get('n_missing_trunk', '—')}); " | |
| f"vs own pretraining → {('trunk_complete = ' + str(d_own.get('trunk_complete')) + f" (loaded {d_own.get('n_loaded')} / {d_own.get('n_trunk_tensors')}, shape-skipped {d_own.get('n_skipped_shape')}, missing {d_own.get('n_missing_trunk')})") if d_own else 'no json'}.") | |
| L.append("") | |
| s2 = {} | |
| if not s2_cfgs: | |
| L += ["_(no K-v2 config has reached S2 yet)_", ""] | |
| for cfg in s2_cfgs: | |
| s1b_dec = s1b_all.get(cfg, {}).get("decision", "no S1b decision") | |
| L += [f"### 2.{s2_cfgs.index(cfg) + 1} `{cfg}` ({' '.join(G.KV2_CONFIGS[cfg][0]) or 'full'}) — S1b: **{s1b_dec}**" | |
| + (" → S2 NOT RUN by rule (rows below marked pending were never launched; the seed-123 run is the S1b run, shown for the record)" if s1b_dec == "KILLED" else ""), "", | |
| "| seed | arm | run | params | best ep / epochs / stop | GPU min | val All | " + " | ".join(f"{lab} [CI]" for _, lab, _ in LEVEL_COLS) + " |", "|---|---|---|---|---|---|---|" + "---|" * len(LEVEL_COLS)] | |
| for s in SEEDS5: | |
| for arm, name in (("K-v2", G.kv2_run(cfg, s)), ("K-v2 + pre", G.kv2_run(cfg, s, pre=True))): | |
| if not io.finished(name): | |
| L.append(f"| {s} | {arm} | {name} | — | pending | — | — | " + " | ".join("—" for _ in LEVEL_COLS) + " |") | |
| continue | |
| be, ep, st, gm, npar, _ = io.meta(name) | |
| note_gpu(name) | |
| L.append(f"| {s} | {arm} | {name} | {npar} | {be} / {ep} / {st} | {gm} | {f(val_all(io, name, dv, gtv))} | {level_row(io, name)} |") | |
| s2[cfg] = {} | |
| for lab, a_fn, b_fn in (("K-v2 − K", lambda s: G.kv2_run(cfg, s), lambda s: G.k_run(s)), | |
| ("K-v2+pre − K+pre", lambda s: G.kv2_run(cfg, s, pre=True), lambda s: G.k_run(s, pre=True)), | |
| (f"K-v2 − MB-best[{mb_cfg}]", lambda s: G.kv2_run(cfg, s), lambda s: G.mb_run(mb_cfg, s)), | |
| (f"K-v2+pre − MB-best[{mb_cfg}]+pre", lambda s: G.kv2_run(cfg, s, pre=True), lambda s: G.mb_run(mb_cfg, s, pre=True))): | |
| per_seed = {s: paired(io, a_fn(s), b_fn(s), "test", dt, gtt, hmt, a.n_boot, full=True) for s in SEEDS5} | |
| per_seed = {s: v for s, v in per_seed.items() if v} | |
| L += ["", f"#### Δ = {lab}, per seed (n = {len(per_seed)} seed pairs finished; n_req = ceil(2n/3) = {math.ceil(2 * len(per_seed) / 3) if per_seed else '—'})", ""] | |
| if not per_seed: | |
| L.append("_(pending — no finished seed pair)_") | |
| s2[cfg][lab] = {"n": 0} | |
| continue | |
| rows, res = delta_table(per_seed, "A", "B") | |
| L += rows | |
| s2[cfg][lab] = {"n": len(per_seed), "seeds": list(per_seed), "cells": {c: {k: v for k, v in r.items() if k != "per_seed"} for c, r in res.items()}, | |
| "per_seed_all": res["All-MPJPE"]["per_seed"]} | |
| # secondary: hazard AUC vs HM membership | |
| auc_rows = [] | |
| for s in SEEDS5: | |
| for split, d_, hm_ in (("val", dv, hmv), ("test", dt, hmt)): | |
| h = io.haz(G.kv2_run(cfg, s), split, d_) | |
| if h is not None: | |
| auc_rows.append(f"| {s} | {split} | {rank_auc(h['max'], hm_):.3f} | {rank_auc(h['mean'], hm_):.3f} |") | |
| if auc_rows: | |
| L += ["", "#### Secondary: AUC of the hazard probability haz_P (max / mean over parts × horizons) for HM membership (descriptive)", "", "| seed | split | AUC(max haz_P) | AUC(mean haz_P) |", "|---|---|---|---|"] + auc_rows | |
| L.append("") | |
| summary["s2"] = s2 | |
| # ---------------------------------------------------------------- §3 MB fairness arm | |
| L += ["## 3. S3 — MotionBERT-kp fairness arm (same val budget: 9 new grid points + existing default = 10; MB-best = argmin val All, ties 0.10 → fewer params)", "", | |
| f"- Decision source: {src3}. Rule: {G.RULE_S3}.", "", | |
| "| cfg | run | model | flags | params | best ep / epochs / stop / GPU min | val All | Δ val All vs default [CI] | test All (recorded, not used) | S3 |", "|---|---|---|---|---|---|---|---|---|---|"] | |
| for r in s3_use.get("rows", []): | |
| name = r["run"] | |
| fl = " ".join(r.get("flags", [])) or "(default)" | |
| if r.get("status") == "PENDING": | |
| pend.append(name) | |
| L.append(f"| {r['cfg']} | {name} | {r['model']} | {fl} | — | pending | pending | pending | — | PENDING |") | |
| continue | |
| be, ep, st, gm, npar, _ = io.meta(name) | |
| if r["cfg"] != "default": | |
| note_gpu(name) | |
| dvd = r.get("d_all_vs_default") | |
| L.append(f"| {r['cfg']} | {name} | {r['model']} | {fl} | {npar} | {be} / {ep} / {st} / {gm} | {r['val_all']:.3f} | {fmt_ci(dvd, 3) if dvd else '0'} | {f(test_all(io, name))} | {'**MB-best**' if r['cfg'] == mb_best else ''} |") | |
| L += ["", f"**MB-best = {mb_best or 'pending'}** ({'final' if mb_best_final else 'provisional — S3 incomplete or no decision json'}; model {s3_use.get('mb_best_model')}, flags {s3_use.get('mb_best_flags')}; " | |
| f"changes d/L: {s3_use.get('mb_best_changes_dL')} → {'new pretraining `' + str(s3_use.get('mb_best_pretrain_run')) + '` required' if s3_use.get('mb_best_changes_dL') else 'existing `motionbert_kp_batonpre_pretrain_s123` checkpoint reused'}).", ""] | |
| # MB-best at 5 seeds (and +pre), K+pre completion | |
| L += ["### 3.1 MB-best and MB-best + BATON trunk-only init at 5 seeds; K + pre completion (test levels)", "", | |
| "| seed | arm | run | params | best ep / epochs / stop | GPU min | val All | " + " | ".join(f"{lab} [CI]" for _, lab, _ in LEVEL_COLS) + " |", "|---|---|---|---|---|---|---|" + "---|" * len(LEVEL_COLS)] | |
| for s in SEEDS5: | |
| arms = [("MB-best", G.mb_run(mb_cfg, s)), ("MB-best + pre", G.mb_run(mb_cfg, s, pre=True)), ("K + pre", G.k_run(s, pre=True))] | |
| if mb_cfg != "default": | |
| arms = [("MB default", G.mb_run("default", s)), ("MB default + pre", G.mb_run("default", s, pre=True))] + arms | |
| for arm, name in arms: | |
| if not io.finished(name): | |
| L.append(f"| {s} | {arm} | {name} | — | pending | — | — | " + " | ".join("—" for _ in LEVEL_COLS) + " |") | |
| continue | |
| be, ep, st, gm, npar, _ = io.meta(name) | |
| if name in programme_runs: | |
| note_gpu(name) | |
| L.append(f"| {s} | {arm} | {name} | {npar} | {be} / {ep} / {st} | {gm} | {f(val_all(io, name, dv, gtv))} | {level_row(io, name)} |") | |
| s3b = {} | |
| for lab, a_fn, b_fn in ((f"MB-best[{mb_cfg}]+pre − MB-best[{mb_cfg}]", lambda s: G.mb_run(mb_cfg, s, pre=True), lambda s: G.mb_run(mb_cfg, s)), | |
| ("K+pre − K", lambda s: G.k_run(s, pre=True), lambda s: G.k_run(s)), | |
| (f"K − MB-best[{mb_cfg}]", lambda s: G.k_run(s), lambda s: G.mb_run(mb_cfg, s)), | |
| (f"K+pre − MB-best[{mb_cfg}]+pre", lambda s: G.k_run(s, pre=True), lambda s: G.mb_run(mb_cfg, s, pre=True))): | |
| per_seed = {s: paired(io, a_fn(s), b_fn(s), "test", dt, gtt, hmt, a.n_boot) for s in SEEDS5} | |
| per_seed = {s: v for s, v in per_seed.items() if v} | |
| L += ["", f"#### Δ = {lab}, per seed (descriptive; n = {len(per_seed)})", ""] | |
| if not per_seed: | |
| L.append("_(pending)_") | |
| s3b[lab] = {"n": 0} | |
| continue | |
| rows, res = delta_table(per_seed, "A", "B", cells=list(MAIN_CELLS)) | |
| L += rows | |
| s3b[lab] = {"n": len(per_seed), "cells": {c: {k: v for k, v in r.items() if k != "per_seed"} for c, r in res.items()}} | |
| L.append("") | |
| summary["s3"] = {"source": "decision" if s3 else "live", "complete": bool(s3_use.get("complete")), "mb_best": mb_best, "final": mb_best_final, | |
| "changes_dL": s3_use.get("mb_best_changes_dL"), "pairs": s3b} | |
| # ---------------------------------------------------------------- §4 verdicts | |
| L += ["## 4. Primary endpoints and adoption (test All-MPJPE, 5 seeds, both sides in the same pretraining state; BEATS rule, 4/5)", ""] | |
| # K-v2(best): lowest mean VAL All over finished seeds among CONTINUE survivors (ties 0.10 -> fewer components) | |
| cand = [] | |
| for cfg in cont: | |
| vals = [val_all(io, G.kv2_run(cfg, s), dv, gtv) for s in SEEDS5] | |
| vals = [v for v in vals if v is not None] | |
| if vals: | |
| cand.append({"cfg": cfg, "val_all": float(np.mean(vals)), "n_params": G.KV2_CONFIGS[cfg][1], "n_seeds": len(vals)}) | |
| best_cfg = G.s3_select(cand, tie=G.S1_TIE) # same argmin + tie helper; "n_params" carries the component count here | |
| L += [f"- K-v2(best) = **{best_cfg or 'pending'}** (lowest mean VAL All over finished seeds among S1b-continued survivors; ties 0.10 → fewer components): " | |
| + ("; ".join(f"{c['cfg']} val All {c['val_all']:.3f} over {c['n_seeds']} seeds, {c['n_params']} components" for c in cand) or "no continued survivor with finished seeds") + ".", ""] | |
| def endpoint(label, a_fn, b_fn, confirmatory=True): | |
| per_seed = {s: paired(io, a_fn(s), b_fn(s), "test", dt, gtt, hmt, a.n_boot) for s in SEEDS5} | |
| per_seed = {s: v for s, v in per_seed.items() if v} | |
| n = len(per_seed) | |
| if n == 0: | |
| return {"label": label, "n": 0, "verdict": "PENDING", "final": False} | |
| rows = [per_seed[s]["All-MPJPE"] for s in per_seed] | |
| v = verdict(rows, False) | |
| final = n == len(SEEDS5) | |
| d = np.array([r["mean"] for r in rows]) | |
| return {"label": label, "n": n, "seeds": list(per_seed), "verdict": v if final else f"PENDING ({n}/5 seeds; interim {v})", "final": final, | |
| "interim_verdict": v, "mean": float(d.mean()), "sd": float(d.std(ddof=1)) if n > 1 else None, "per_seed": {str(s): per_seed[s]["All-MPJPE"] for s in per_seed}, | |
| "secondary": {c: {"verdict": verdict([per_seed[s][c] for s in per_seed], per_seed[next(iter(per_seed))][c]["higher_better"]), "mean": float(np.mean([per_seed[s][c]["mean"] for s in per_seed]))} for c in MAIN_CELLS if c != "All-MPJPE"}} | |
| eps = {} | |
| if best_cfg: | |
| eps["P1"] = endpoint(f"P1 = K-v2({best_cfg}) − MB-best[{mb_cfg}]", lambda s: G.kv2_run(best_cfg, s), lambda s: G.mb_run(mb_cfg, s)) | |
| eps["P2"] = endpoint(f"P2 = K-v2({best_cfg})+pre − MB-best[{mb_cfg}]+pre", lambda s: G.kv2_run(best_cfg, s, pre=True), lambda s: G.mb_run(mb_cfg, s, pre=True)) | |
| eps["A1"] = endpoint(f"A1 = K-v2({best_cfg}) − K", lambda s: G.kv2_run(best_cfg, s), lambda s: G.k_run(s)) | |
| eps["A2"] = endpoint(f"A2 = K-v2({best_cfg})+pre − K+pre", lambda s: G.kv2_run(best_cfg, s, pre=True), lambda s: G.k_run(s, pre=True)) | |
| if not mb_best_final: | |
| for k in ("P1", "P2"): | |
| eps[k]["final"] = False | |
| eps[k]["verdict"] = f"PENDING (MB-best provisional; interim {eps[k].get('interim_verdict', '—')})" | |
| L += ["| endpoint | n seeds | Δ All per seed [CI] | mean Δ ± sd | verdict | secondary (HM / PCK@0.05 / HM h=5 / d-n verdicts) |", "|---|---|---|---|---|---|"] | |
| for k in ("P1", "P2", "A1", "A2"): | |
| e = eps.get(k) | |
| if e is None: | |
| L.append(f"| {k} | 0 | pending (no K-v2(best) yet) | — | PENDING | — |") | |
| continue | |
| sec = "; ".join(f"{c} {v['verdict']} ({v['mean']:+.2f})" for c, v in e.get("secondary", {}).items()) or "—" | |
| L.append(f"| {e['label']} | {e['n']} | " + ("; ".join(f"s{s} {fmt_ci(r)}" for s, r in e.get("per_seed", {}).items()) or "pending") + f" | {f(e.get('mean'), 2, True)}{(' ± ' + format(e['sd'], '.2f')) if e.get('sd') is not None else ''} | **{e['verdict']}** | {sec} |") | |
| p1 = eps.get("P1", {}).get("final") and eps["P1"]["verdict"] == "BEATS" | |
| p2 = eps.get("P2", {}).get("final") and eps["P2"]["verdict"] == "BEATS" | |
| a1 = eps.get("A1", {}).get("final") and eps["A1"]["verdict"] == "BEATS" | |
| a2 = eps.get("A2", {}).get("final") and eps["A2"]["verdict"] == "BEATS" | |
| all_final = all(eps.get(k, {}).get("final") for k in ("P1", "P2")) | |
| realised = ("P1 ∧ P2" if p1 and p2 else "P1 ∧ ¬P2" if p1 else "¬P1 ∧ P2" if p2 else "neither") if all_final else None | |
| L += ["", "### 4.1 Paper-wording consequence table (pre-registered; the realised row is marked)", "", "| case | wording consequence | realised |", "|---|---|---|"] | |
| for case, text in CONSEQUENCES: | |
| L.append(f"| {case} | {text} | {'**← REALISED**' if realised == case else ('pending' if realised is None else '')} |") | |
| L += ["", f"**ADOPTION A1: {'ADOPTED' if a1 else 'NOT ADOPTED'}**" + ("" if eps.get("A1", {}).get("final") else f" (pending — {eps.get('A1', {}).get('verdict', 'no K-v2(best) yet')}; NOT ADOPTED is the conservative default until 5 seeds are final)"), | |
| f"**ADOPTION A2: {'ADOPTED' if a2 else 'NOT ADOPTED'}**" + ("" if eps.get("A2", {}).get("final") else f" (pending — {eps.get('A2', {}).get('verdict', 'no K-v2(best) yet')})"), | |
| f"- Consequence-table row: **{realised or 'pending (P1 / P2 not final)'}**. Any WORSE verdict above is reported as is.", ""] | |
| summary["s4"] = {"kv2_best": best_cfg, "mb_best": mb_cfg, "endpoints": {k: {kk: vv for kk, vv in e.items() if kk != "per_seed"} for k, e in eps.items()}, | |
| "adoption_a1": bool(a1), "adoption_a2": bool(a2), "realised_case": realised} | |
| # ---------------------------------------------------------------- §5 GPU | |
| L += ["## 5. GPU minutes of the programme's new runs (from run json; the GPU was shared with a DMS feature-extraction job and other lanes, so minutes are wall-clock upper bounds)", "", "| run | GPU min |", "|---|---|"] | |
| for name, g in sorted(gpu.items()): | |
| L.append(f"| {name} | {g:.2f} |") | |
| L += [f"| **total ({len(gpu)} runs)** | **{sum(gpu.values()):.1f}** |", ""] | |
| # ---------------------------------------------------------------- §6 provenance | |
| decs = sorted(p.name for p in io.kv2.glob("*.json")) if io.kv2.exists() else [] | |
| L += ["## 6. Provenance", "", | |
| f"- Inputs: `pred_val.npz` / `pred_test.npz` / `metrics_val.json` / `metrics_test.json` / `run_<name>_<utc>.json` under `{io.runs}`; gate decisions `{io.kv2}`: {', '.join(decs) if decs else 'none yet'}; pre-registration row from `{a.decisions}` (S0 UTC {s0_utc}).", | |
| "- Comparison runs `cwm3d_kp_s<k>_v2`, `cwm3d_kp_batonpre_trunkp123_s{42,123,456}_v2`, `motionbert_kp_s<k>_v2`, `motionbert_kp_batonpre_trunkp123_s{42,123,456}_v2` are the EXISTING runs (never retrained); every programme run is a new run dir.", | |
| "- Selection: val All-MPJPE of the EMA weights inside the trainer; S1 / S1b / S3 decisions from pred_val.npz only (`aide/eval/kv2_gate.py`); one test evaluation per run by the harness; no test number was used for any decision.", | |
| "- Launcher `aide/train/run_kv2_program.sh` (gate: free VRAM ≥ need + 4 GB, MemAvailable ≥ 28 GB, < 4 trainers, OMP/MKL 4; finished runs skipped); pids `aide/results/kv2/pids.txt`.", | |
| f"- Pending runs referenced above: {len(set(pend))}." + (" This block is append-only; earlier blocks are never edited." if out.exists() else "")] | |
| shas = {} | |
| for n in started: | |
| shas.setdefault((io.run_json(n) or {}).get("code_sha256", "—"), []).append(n) | |
| L += ["- Trainer code hashes (`code_sha256` of each finished programme run's run json; distinct values = code changed between launches, e.g. the train_kp.py:630 guard fix of 2026-09-14T23:11:53Z, behaviour-neutral for dict-output models — see `aide/results/kv2/NOTES.md`): " | |
| + ("; ".join(f"`{k[:16]}` ({len(v)} runs: {', '.join(sorted(v))})" for k, v in sorted(shas.items(), key=lambda kv: -len(kv[1]))) or "none yet") + ".", ""] | |
| summary["s6"] = {"code_sha256_groups": {k: sorted(v) for k, v in shas.items()}} | |
| out.parent.mkdir(parents=True, exist_ok=True) | |
| text = "\n".join(L) | |
| if out.exists(): | |
| with open(out, "a", encoding="utf-8") as fh: | |
| fh.write(f"\n\n---\n\n<!-- appended regeneration {stamp} -->\n\n" + text) | |
| mode = "appended" | |
| else: | |
| out.write_text(text, encoding="utf-8") | |
| mode = "written" | |
| js = out.with_suffix(".json") | |
| rec = {"stamp": stamp, "n_boot": a.n_boot, "summary": summary, "gpu_minutes": gpu, "pending": sorted(set(pend))} | |
| old = json.load(open(js)) if js.exists() else [] | |
| old = old if isinstance(old, list) else [old] | |
| js.write_text(json.dumps(old + [rec], indent=1, default=str)) | |
| write_run_json(io.tables, "report_kv2_program", [Path(__file__), G.__file__, M.__file__, P.__file__], [P.CACHE_NPZ], seed=0, t_start=t0, input_listing=False, | |
| extra={"out": str(out), "mode": mode, "adoption_a1": bool(a1), "adoption_a2": bool(a2), "realised_case": realised, "kv2_best": best_cfg, "mb_best": mb_cfg, | |
| "s1_survivors": surv, "s1b_continue": cont, "gpu_minutes": gpu, "dms_fields_used": []}) | |
| print(text) | |
| print(f"\n[{mode}] {out}") | |
| return 0 | |
| if __name__ == "__main__": | |
| sys.exit(main()) | |
Xet Storage Details
- Size:
- 34.9 kB
- Xet hash:
- 3461764aeac85eea184885288d8a581e57ea0f0c3ad0d3b263204097a00fcfdd
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.