Buckets:
| #!/usr/bin/env python | |
| """Driver-WM quantile sweep (pre-registered decisions.md 2026-09-17T07:00:22Z, item B). | |
| Compiles, from the frozen `aide/results/tables/hm_subsets.json` (generated 20260915T031542Z by | |
| `aide/eval/report_hm_subsets.py`), a compact table of horizon-averaged and h=5 MPJPE for the | |
| motion-quantile subsets top 5/10/20/30/50/100 % of the AIDE official test split, for zero-velocity, | |
| CockpitWM, CockpitWM + BATON pretraining, CockpitWM (+ semantic heads), the Driver-WM reproduction | |
| (ours, their formulation) and the Driver-WM architecture on CockpitWM tokens, plus the published | |
| Driver-WM values where they exist, and the seed-matched paired Δ (CockpitWM − Driver-WM | |
| architecture on CockpitWM tokens) at h=5 with seed counts. Every cell is re-read from the json and | |
| cross-checked against the strings printed in `hm_subsets.md` (§1.2, §1.4, §1.5); mismatches are | |
| counted in the header. No training, no GPU, no existing file modified. | |
| """ | |
| from __future__ import annotations | |
| import argparse, json, re, sys, time | |
| from pathlib import Path | |
| import numpy as np | |
| ROOT = Path(__file__).resolve().parents[2] | |
| TAB = ROOT / "aide" / "results" / "tables" | |
| QS = [("q005", "5 %"), ("q010", "10 %"), ("q020", "20 %"), ("q030", "30 %"), ("q050", "50 %"), ("q100", "100 %")] | |
| ROWS = [("zv", "zero-velocity"), ("K", "CockpitWM"), ("Kpre", "CockpitWM + BATON pretraining"), | |
| ("KS", "CockpitWM (+ semantic heads)"), ("DWMrepro", "Driver-WM reproduction (ours, their formulation)"), | |
| ("DWMkp", "Driver-WM architecture on CockpitWM tokens")] | |
| MD14 = {"DWMkp": "Driver-WM arch. on CockpitWM tokens", "DWMrepro": "Driver-WM reproduction (ours)"} | |
| # Published Driver-WM numbers (arXiv 2605.05092v2), as transcribed and audited in driverwm_v2_audit.md. | |
| PUB = { | |
| "seed42": {"label": "Driver-WM published, single run (v2 Table 1 / Table 7 seed 42; text / Tables 10–11 for h=5)", | |
| "avg": {"q010": "138.03", "q100": "71.47"}, "h5": {"q010": "155.82"}, | |
| "src": "driverwm_v2_audit.md:23 (71.47 / 138.03), :56 and :58 (155.82)"}, | |
| "seeds3": {"label": "Driver-WM published, 3-seed mean ± sd (v2 Table 8, seeds 42/123/456)", | |
| "avg": {"q010": "138.34 ± 1.2", "q100": "71.66 ± 0.3"}, "h5": {}, | |
| "src": "driverwm_v2_audit.md:235"}, | |
| } | |
| NP = "not published" | |
| def fmt_pm(m, sd, n): | |
| return f"{m:.2f}" if n == 1 or sd is None else f"{m:.2f} ± {sd:.2f}" | |
| def sd1(x): | |
| x = np.asarray(x, float) | |
| return float(np.std(x, ddof=1)) if len(x) > 1 else None | |
| def level(summ, q, key): | |
| s = summ[q][key] | |
| n = s["n_runs"] | |
| avg = (s["mpjpe"]["mean"], s["mpjpe"]["sd"] if n > 1 else None, n) | |
| h5 = s["h"][4] | |
| return avg, (h5["mean"], h5["sd"] if n > 1 else None, n), n | |
| def delta_h5(summ, q, model, comp): | |
| """Seed-matched paired Δ (model − comp) at h=5 and horizon-averaged, from the per-seed blocks.""" | |
| d = summ[q][model]["delta"][comp] | |
| ps = d["per_seed"] | |
| h5 = [p["h"][4] for p in ps] | |
| m5 = [x["mean"] for x in h5] | |
| k5 = sum(1 for x in h5 if (x["ci_lo"] > 0) or (x["ci_hi"] < 0)) | |
| # cross-check with the stored aggregates when present | |
| if "h_n_ci_excl0" in d: | |
| assert int(d["h_n_ci_excl0"][4]) == k5, (q, model, comp, d["h_n_ci_excl0"][4], k5) | |
| ma = [p["mean"] for p in ps] | |
| ka = int(d["n_ci_excl0"]) | |
| assert ka == sum(1 for p in ps if (p["ci_lo"] > 0) or (p["ci_hi"] < 0)) | |
| return dict(n=len(ps), h5_mean=float(np.mean(m5)), h5_sd=sd1(m5), h5_k=k5, | |
| h5_per_seed=[(p["run"].split("_s")[-1].split("_")[0], x["mean"], x["ci_lo"], x["ci_hi"]) for p, x in zip(ps, h5)], | |
| avg_mean=float(np.mean(ma)), avg_sd=sd1(ma), avg_k=ka, verdict=d["verdict"]) | |
| def parse_md(md_text): | |
| """Return dicts of the strings printed in hm_subsets.md §1.2 (avg levels), §1.5 (h=5 seed means), §1.4 (Δ).""" | |
| lines = md_text.splitlines() | |
| sec = {} | |
| cur = None | |
| for ln in lines: | |
| m = re.match(r"^### (1\.\d)", ln) | |
| if m: | |
| cur = m.group(1) | |
| if ln.startswith("|") and cur: | |
| sec.setdefault(cur, []).append([c.strip() for c in ln.strip().strip("|").split("|")]) | |
| lv = {} | |
| for row in sec.get("1.2", []): | |
| if len(row) == 8 and row[0] not in ("model", "---"): | |
| lv[row[0]] = [re.match(r"^([-+]?\d+\.\d+(?: ± \d+\.\d+)?)", c).group(1) for c in row[2:]] | |
| h5 = {} # §1.5 holds two tables of identical shape (levels, then Δ vs zero-velocity): keep the FIRST (levels) | |
| for row in sec.get("1.5", []): | |
| if len(row) == 8 and row[1] == "h=5" and row[0] not in h5: | |
| h5[row[0]] = [re.match(r"^([-+]?\d+\.\d+)", c).group(1) for c in row[2:]] | |
| dl = {} | |
| for row in sec.get("1.4", []): | |
| if len(row) == 10 and row[0].endswith("%"): | |
| dl[(row[0], row[2], row[3])] = dict(avg=row[4], k_avg=row[5], h5=row[6], k_h5=row[7], verdict=row[8]) | |
| return lv, h5, dl | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--hm-json", default=str(TAB / "hm_subsets.json")) | |
| ap.add_argument("--hm-md", default=str(TAB / "hm_subsets.md")) | |
| ap.add_argument("--release-json", default=str(TAB / "driverwm_release_check_20260917T07Z.json")) | |
| ap.add_argument("--out", default=str(TAB / "driverwm_quantile_sweep")) | |
| args = ap.parse_args() | |
| t0 = time.time() | |
| hm = json.load(open(args.hm_json)) | |
| summ = hm["summary"] | |
| labels = hm["labels"] | |
| subsets = {s["key"]: s for s in hm["subsets"] if s["section"] == "A"} | |
| md_lv, md_h5, md_dl = parse_md(Path(args.hm_md).read_text()) | |
| rel = json.load(open(args.release_json)) | |
| mism = [] | |
| out = {"generated_utc": time.strftime("%Y%m%dT%H%M%SZ", time.gmtime()), "script": "aide/eval/report_driverwm_quantile_sweep.py", | |
| "source": {"hm_subsets_json_generated_utc": hm["generated_utc"], "hm_subsets_prereg_utc": hm["preregistration_utc"], | |
| "prereg_this_table": "docs/decisions.md row 2026-09-17T07:00:22Z (B)"}, | |
| "subsets": {q: {k: subsets[q][k] for k in ("q", "n", "threshold", "mean_score")} for q, _ in QS}, | |
| "levels_avg": {}, "levels_h5": {}, "published": PUB, "delta_h5": {}, "release_check": rel} | |
| # ---- levels ---- | |
| L = [] | |
| L.append("# Driver-WM quantile sweep — CockpitWM, our two Driver-WM ports and the published Driver-WM on the top-q motion subsets (AIDE official test split, n = 609)\n") | |
| L.append(f"Generated {out['generated_utc']} by `aide/eval/report_driverwm_quantile_sweep.py` from `aide/results/tables/hm_subsets.json` (generated {hm['generated_utc']}, pre-registered {hm['preregistration_utc']}); this table itself is item **(B)** of the pre-registration row `docs/decisions.md` 2026-09-17T07:00:22Z. No new run, no GPU, no existing file modified. Subsets: " + | |
| "; ".join(f"top {lab} = n {subsets[q]['n']} (motion-score threshold {subsets[q]['threshold']:.2f} px)" for q, lab in QS) + | |
| ". The top-10 % subset IS the official High-Motion (HM) subset of Driver-WM (same 60 clips, threshold 72.05 px); top 100 % IS the full test set (their 'All'). Levels are mean ± sd over seeds (ddof 1) of one test evaluation per run; Δ = paired per-clip difference on the same clips, seed-matched (42/123/456), with the 10k clip-bootstrap 95 % CI per seed; k/n = seeds whose CI excludes 0. __INTEGRITY__\n") | |
| L.append("## 0. Which comparisons are with our ports and which with published numbers\n") | |
| L.append("- **Every paired Δ and every non-'published' cell below is against OUR ports**, trained by us under the locked protocol (`aide/data/protocol.py`; 5 observed → 5 future frames, 136 joints, 1920×1080 px): (i) **Driver-WM reproduction (ours, their formulation)** — absolute sigmoid-normalised coordinates with an MSE loss on frozen VLM features, as described in their §3.4; it does NOT reproduce their numbers (All 130.86 ± 3.11 vs published 71.47), so margins against it say nothing about Driver-WM as published; (ii) **Driver-WM architecture on CockpitWM tokens** — their temporal encoder trained on our keypoint part tokens under our anchored-residual formulation (the meaningful architecture comparator).") | |
| L.append("- **Published Driver-WM numbers exist for exactly three cells of this grid**: All-MPJPE 71.47 (top 100 %, horizon-averaged), HM-MPJPE 138.03 (top 10 %, horizon-averaged) and HM h=5 155.82 (top 10 %) — all from ONE run (seed 42) of their Table 1 / text, without CI; their Table 8 adds 3-seed means for All and HM (71.66 ± 0.3 / 138.34 ± 1.2). No published value exists for 5 / 20 / 30 / 50 % or for h=5 outside the HM subset (Fig. 3(b) gives approximate h=1–4 HM readings only; `driverwm_v2_audit.md`). Published cells are single-run levels: they can be placed next to ours but cannot be paired or bootstrapped.") | |
| L.append("- **Consequence for the owner's hypothesis** ('Driver-WM is better only at the top-10 % subset'): on the PUBLISHED model it is untestable — one published h=5 point at 10 % and no code/weights (§4). What CAN be read is how the two ports behave across the sweep (§3).\n") | |
| L.append("## 1. Horizon-averaged MPJPE (px) per quantile — mean ± sd over seeds\n") | |
| L.append("| model | seeds | " + " | ".join(f"top {lab} (n {subsets[q]['n']})" for q, lab in QS) + " |") | |
| L.append("|---|---|" + "---|" * len(QS)) | |
| for key, lab in ROWS: | |
| cells, n = [], None | |
| for q, _ in QS: | |
| avg, _h5, n = level(summ, q, key) | |
| s = fmt_pm(*avg) | |
| cells.append(s) | |
| ref = md_lv.get(labels[key]) | |
| qi = [x for x, _ in QS].index(q) | |
| if ref is None or ref[qi] != s: | |
| mism.append(("1.2", labels[key], q, s, ref[qi] if ref else None)) | |
| out["levels_avg"].setdefault(key, {})[q] = {"mean": avg[0], "sd": avg[1], "n": n} | |
| L.append(f"| {lab} | {n} | " + " | ".join(cells) + " |") | |
| for pk, p in PUB.items(): | |
| L.append(f"| {p['label']} | {'1' if pk == 'seed42' else '3'} | " + " | ".join(p["avg"].get(q, NP) for q, _ in QS) + " |") | |
| L.append("") | |
| L.append("## 2. h=5 MPJPE (px) per quantile — mean ± sd over seeds (seed means = §1.5 of hm_subsets.md)\n") | |
| L.append("| model | seeds | " + " | ".join(f"top {lab}" for q, lab in QS) + " |") | |
| L.append("|---|---|" + "---|" * len(QS)) | |
| for key, lab in ROWS: | |
| cells, n = [], None | |
| for q, _ in QS: | |
| _avg, h5, n = level(summ, q, key) | |
| s = fmt_pm(*h5) | |
| cells.append(s) | |
| ref = md_h5.get(labels[key]) | |
| qi = [x for x, _ in QS].index(q) | |
| if ref is None or ref[qi] != f"{h5[0]:.2f}": | |
| mism.append(("1.5", labels[key], q, f"{h5[0]:.2f}", ref[qi] if ref else None)) | |
| out["levels_h5"].setdefault(key, {})[q] = {"mean": h5[0], "sd": h5[1], "n": n} | |
| L.append(f"| {lab} | {n} | " + " | ".join(cells) + " |") | |
| for pk, p in PUB.items(): | |
| L.append(f"| {p['label']} | {'1' if pk == 'seed42' else '3'} | " + " | ".join(p["h5"].get(q, NP) for q, _ in QS) + " |") | |
| L.append("") | |
| # ---- paired Δ ---- | |
| L.append("## 3. Paired Δ at h=5 — CockpitWM − Driver-WM architecture on CockpitWM tokens (seed-matched 42/123/456; negative = CockpitWM better)\n") | |
| L.append("| top q | n clips | Δ h=5 mean ± sd over seeds | k/n seeds CI excl. 0 (h=5) | per-seed Δ h=5 [95 % CI] (s42; s123; s456) | Δ horizon-avg mean ± sd | k/n (avg) | verdict (avg, existing rule) |") | |
| L.append("|---|---|---|---|---|---|---|---|") | |
| for q, lab in QS: | |
| d = delta_h5(summ, q, "K", "DWMkp") | |
| s5, k5 = f"{d['h5_mean']:+.2f} ± {d['h5_sd']:.2f}", f"{d['h5_k']}/{d['n']}" | |
| sa, ka = f"{d['avg_mean']:+.2f} ± {d['avg_sd']:.2f}", f"{d['avg_k']}/{d['n']}" | |
| ref = md_dl.get((lab, "CockpitWM", MD14["DWMkp"])) | |
| for what, mine, theirs in (("Δh5", s5, ref and ref["h5"]), ("k_h5", k5, ref and ref["k_h5"]), ("Δavg", sa, ref and ref["avg"]), ("k_avg", ka, ref and ref["k_avg"]), ("verdict", d["verdict"], ref and ref["verdict"])): | |
| if theirs != mine: | |
| mism.append(("1.4", f"K vs DWMkp {what}", q, mine, theirs)) | |
| per = "; ".join(f"{m:+.2f} [{lo:+.2f}, {hi:+.2f}]" for _s, m, lo, hi in d["h5_per_seed"]) | |
| L.append(f"| {lab} | {subsets[q]['n']} | {s5} | {k5} | {per} | {sa} | {ka} | {d['verdict']} |") | |
| out["delta_h5"].setdefault("K_vs_DWMkp", {})[q] = d | |
| L.append("") | |
| L.append("Secondary (same rule; h=5; mean ± sd over seeds, k/n) — the other CockpitWM variants against the same port, and all three against the reproduction port:\n") | |
| L.append("| model − comparator | " + " | ".join(f"top {lab}" for q, lab in QS) + " |") | |
| L.append("|---|" + "---|" * len(QS)) | |
| for model, comp in (("Kpre", "DWMkp"), ("KS", "DWMkp"), ("K", "DWMrepro"), ("Kpre", "DWMrepro"), ("KS", "DWMrepro")): | |
| cells = [] | |
| for q, lab in QS: | |
| d = delta_h5(summ, q, model, comp) | |
| s5, k5 = f"{d['h5_mean']:+.2f} ± {d['h5_sd']:.2f}", f"{d['h5_k']}/{d['n']}" | |
| ref = md_dl.get((lab, labels[model], MD14[comp])) | |
| if ref is None or ref["h5"] != s5 or ref["k_h5"] != k5: | |
| mism.append(("1.4", f"{model} vs {comp}", q, f"{s5} {k5}", ref and (ref["h5"], ref["k_h5"]))) | |
| cells.append(f"{s5} ({k5})") | |
| out["delta_h5"].setdefault(f"{model}_vs_{comp}", {})[q] = d | |
| L.append(f"| {labels[model]} − {labels[comp]} | " + " | ".join(cells) + " |") | |
| L.append("") | |
| # reading | |
| dK = {q: delta_h5(summ, q, "K", "DWMkp") for q, _ in QS} | |
| dR = {q: delta_h5(summ, q, "K", "DWMrepro") for q, _ in QS} | |
| L.append("**Reading (descriptive; the pre-registration fixes no acceptance rule for B — 'decision on entering the paper after the owner sees the table'):** ") | |
| L.append(f"(i) Against the **architecture port** at h=5, CockpitWM's paired advantage is smallest on the high-motion tail and grows toward the full set: Δ h=5 = {', '.join('%+.2f (%d/%d) at %s' % (dK[q]['h5_mean'], dK[q]['h5_k'], dK[q]['n'], lab) for q, lab in QS)}; only the full set reaches 3/3 seeds, the 10 % point is the one quantile where the port is nominally lower (+0.54, 0/3, sd 1.96) — INCONCLUSIVE under the existing rule at every quantile except 100 %, and the level difference at 10 % ({level(summ,'q010','K')[1][0]:.2f} vs {level(summ,'q010','DWMkp')[1][0]:.2f}) is far inside the CI half-width of an n = 60 subset. ") | |
| L.append(f"(ii) Against the **reproduction port** the gap is huge everywhere (3/3 at every quantile) but also smallest on the tail: Δ h=5 {dR['q005']['h5_mean']:+.2f} at 5 % → {dR['q100']['h5_mean']:+.2f} at 100 %; i.e. the absolute-coordinate formulation loses least where the driver actually moves, which is the pattern the owner's hypothesis predicts, yet it never becomes better than CockpitWM. ") | |
| L.append(f"(iii) Against the **published** single-run numbers only the 10 % column is comparable: CockpitWM {level(summ,'q010','K')[0][0]:.2f} ± {level(summ,'q010','K')[0][1]:.2f} vs 138.03 horizon-averaged (ours lower), and {level(summ,'q010','K')[1][0]:.2f} ± {level(summ,'q010','K')[1][1]:.2f} vs 155.82 at h=5 (theirs lower by {level(summ,'q010','K')[1][0]-155.82:.2f} px; CockpitWM + BATON pretraining {level(summ,'q010','Kpre')[1][0]:.2f}); whether that h=5 edge persists at 5 % or vanishes at 20–100 % cannot be known without their model.\n") | |
| # ---- release check ---- | |
| L.append(f"## 4. Release check of Driver-WM code / weights ({rel['checked_utc']}; {rel['method']})\n") | |
| L.append(f"**Verdict: {rel['verdict']}.** Previous record: {rel['previous_record']}.\n") | |
| L.append("Official channels:\n") | |
| for it in rel["official"]: | |
| L.append(f"- **{it['item']}** — <{it['url']}> — {it['observed']}") | |
| L.append("\nThird-party artefacts found (NOT official, NOT downloaded, no licence):\n") | |
| for it in rel["third_party_not_official"]: | |
| L.append(f"- **{it['item']}** — <{it['url']}> — {it['observed']}") | |
| L.append(f"\n{rel['consequence']}.\n") | |
| L.append("## 5. Provenance\n") | |
| L.append("- Runs (all pre-existing, `aide/results/runs/`): CockpitWM `cwm3d_kp_s{42,123,456,789,1011}_v2`; + BATON pretraining `cwm3d_kp_batonpre_trunkp123_s*_v2` (5 seeds); (+ semantic heads) `cwm3d_vis_sem_s{42,123,456}_v2`; Driver-WM architecture on CockpitWM tokens `driverwm_kp_s{42,123,456}_v2`; Driver-WM reproduction `driverwm_repro_s{42,123,456}_v2`; zero-velocity recomputed from the observed frames. All numbers are read from `hm_subsets.json` (levels: `summary[q][model].mpjpe` / `.h[4]`; Δ: `summary[q][model].delta[comp].per_seed[*]` and `.h[4]`), which `report_hm_subsets.py` verified against every run's `metrics_test.json` and `master_aide.json` (594 cells, max abs diff 1.99e-13).") | |
| L.append(f"- Published cells: {PUB['seed42']['src']}; {PUB['seeds3']['src']}; local PDF `/home/USER/project/related_works/DriverWM-V2.pdf` (arXiv 2605.05092v2, 26 Jun 2026).") | |
| L.append(f"- Release-check evidence: `aide/results/tables/driverwm_release_check_20260917T07Z.json` (URLs, dates, observations as fetched). Wall time {time.time()-t0:.1f} s, CPU only.") | |
| integ = f"**Integrity check against `hm_subsets.md`: {len(mism)} mismatch(es)** over all §1.2 / §1.5 / §1.4 strings reproduced here" + ("" if not mism else " — " + "; ".join(str(m) for m in mism[:10])) | |
| md = "\n".join(L).replace("__INTEGRITY__", integ) + "\n" | |
| out["integrity"] = {"n_mismatches": len(mism), "mismatches": mism} | |
| Path(args.out + ".md").write_text(md) | |
| Path(args.out + ".json").write_text(json.dumps(out, indent=1, default=float)) | |
| print(f"wrote {args.out}.md and .json; mismatches {len(mism)}") | |
| for m in mism: | |
| print(" MISMATCH", m) | |
| return 0 if not mism else 1 | |
| if __name__ == "__main__": | |
| sys.exit(main()) | |
Xet Storage Details
- Size:
- 17.8 kB
- Xet hash:
- a8b6275f957328f63aa356165004bb24c6fffca8171c5525c733352ee2fc101c
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.