Buckets:
| """cwm3/weak/report_dms_teacher_pretrain.py — assemble results/tables/dms_teacher_pretrain.md from script outputs only. | |
| Sections: (1) pretraining runs (results/runs/U0_dmspre_trunk_s<k>: best epoch, val loss, per-channel anchor / future error vs the constant | |
| train-mean predictor); (2) fine-tuned U0 vs scratch (results/cwm3_quick_metrics.csv rows of both run families, onset PR-AUC@3s per event | |
| and regime; paired device-cluster bootstrap deltas from results/paired_compare.csv tags dmspre_s<k>); (3) acceptance verdict (plan §5.2: | |
| engaged hand_leave / head_turn_onset +0.02 on >= 2/3 seeds, manual not below -1 sd); (4) K5 shifted-twin rows from results/shift_twin.csv; | |
| (5) exp1_kin_repro rows of the fine-tuned runs vs the gru_kin_can reference. Missing pieces are listed, never invented. | |
| """ | |
| from __future__ import annotations | |
| import csv | |
| import json | |
| import time | |
| from collections import defaultdict | |
| from pathlib import Path | |
| import numpy as np | |
| CWM3 = Path(__file__).resolve().parents[1] | |
| RUNS = CWM3 / "results" / "runs" | |
| TABLES = CWM3 / "results" / "tables" | |
| QUICK = CWM3 / "results" / "cwm3_quick_metrics.csv" | |
| PAIRED = CWM3 / "results" / "paired_compare.csv" | |
| TWIN = CWM3 / "results" / "shift_twin.csv" | |
| REPRO = CWM3 / "results" / "exp1_kin_repro.csv" | |
| SEEDS = (0, 1, 2) | |
| EVENTS = ("hand_return", "hand_leave", "head_turn_onset") | |
| GATE_EVENTS = ("hand_leave", "head_turn_onset") | |
| GATE_DELTA = 0.02 | |
| def f4(x): | |
| try: | |
| return f"{float(x):.4f}" | |
| except (TypeError, ValueError): | |
| return str(x) | |
| def main(): | |
| stamp = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime()) | |
| L = ["# DMS weak-teacher pretraining of the U trunk — results", "", | |
| f"Generated by `weak/report_dms_teacher_pretrain.py` at {stamp}; every number below is read from run outputs (nothing typed by hand).", | |
| "Pretraining: `weak/dms_teacher_pretrain.py` (the only other DMS-derived-data reader; targets `results/targets_dms_teacher/`). " | |
| "Fine-tune: `train/train_u0_from_init.py` -> unchanged `train/train_cwm3.py` (U0 kin_can_noK recipe, `--no-lock`) from the pretrained trunk; " | |
| "scratch baseline: `results/runs/U0_kin_can_noK_s<k>`. Inputs of both: Kh (route-median ROI, non-causal — K4 pending), X, Δt, mode; no DMS field.", ""] | |
| missing = [] | |
| # ---- 1. pretraining | |
| L += ["## 1. Pretraining runs (val, best epoch; error = MAE in raw units for the 4 continuous channels, BCE for the 2 soft probabilities)", "", | |
| "| seed | best epoch | epochs run | val loss | channel | anchor err (model) | anchor err (const mean) | future err (model) | future err (const mean) |", "|---|---|---|---|---|---|---|---|---|"] | |
| for s in SEEDS: | |
| d = RUNS / f"U0_dmspre_trunk_s{s}" | |
| vm = d / "val_metrics.json" | |
| if not vm.exists(): | |
| missing.append(f"pretrain s{s}") | |
| continue | |
| v = json.loads(vm.read_text()) | |
| for j, ch in enumerate(v["channels"]): | |
| L.append(f"| {s} | {v['best_epoch']} | {v['epochs_run']} | {v['best_val_loss']:.4f} | {ch} | {v['anchor_err'][j]:.4f} | {v['const_anchor_err'][j]:.4f} | " | |
| f"{v['future_err'][j]:.4f} | {v['const_future_err'][j]:.4f} |") | |
| # ---- 2. quick metrics of both families | |
| q = defaultdict(dict) | |
| if QUICK.exists(): | |
| for r in csv.DictReader(open(QUICK)): | |
| if r["stage"] != "U0" or r["variant"] != "kin_can_noK" or r["split"] != "test" or r["metric"] != "onset_prauc@3s": | |
| continue | |
| fam = "dmspre" if "_dmspre_s" in r["run_dir"] else ("scratch" if r["run_dir"].rstrip("/").endswith(f"U0_kin_can_noK_s{r['seed']}") else None) | |
| if fam is None or "smoke" in r["run_dir"] or "max" in Path(r["run_dir"]).name: | |
| continue | |
| q[(fam, int(r["seed"]), r["regime"], r["chain"])] = float(r["value"]) | |
| L += ["", "## 2. Fine-tuned U0 (pretrained trunk) vs scratch — test onset PR-AUC@3s (`results/cwm3_quick_metrics.csv`)", "", | |
| "| regime | event | " + " | ".join(f"s{s} pre / scratch / Δ" for s in SEEDS) + " | mean Δ |", "|---|---|" + "---|" * (len(SEEDS) + 1)] | |
| deltas = {} | |
| for reg in ("engaged", "manual"): | |
| for ev in EVENTS: | |
| cells, ds_ = [], [] | |
| for s in SEEDS: | |
| a_, b_ = q.get(("dmspre", s, reg, ev)), q.get(("scratch", s, reg, ev)) | |
| if a_ is None or b_ is None: | |
| cells.append("— / — / —" if a_ is None and b_ is None else f"{f4(a_) if a_ is not None else '—'} / {f4(b_) if b_ is not None else '—'} / —") | |
| else: | |
| cells.append(f"{a_:.4f} / {b_:.4f} / {a_ - b_:+.4f}") | |
| ds_.append(a_ - b_) | |
| deltas[(reg, ev)] = ds_ | |
| L.append(f"| {reg} | {ev} | " + " | ".join(cells) + f" | {np.mean(ds_):+.4f} |" if ds_ else f"| {reg} | {ev} | " + " | ".join(cells) + " | — |") | |
| # ---- paired bootstrap | |
| pr = [] | |
| if PAIRED.exists(): | |
| for r in csv.DictReader(open(PAIRED)): | |
| if r["tag"].startswith("dmspre_s") and r["cell"].startswith("drv_onset|"): | |
| pr.append(r) | |
| L += ["", "### Paired device-cluster bootstrap (`eval/paired_compare.py`, rows both scored, n_boot 1000): A = pretrained-init, B = scratch", "", | |
| "| tag | regime | cell | A | B | Δ | Δ CI 95 % | P(Δ>0) | n | n_pos | n_dev |", "|---|---|---|---|---|---|---|---|---|---|---|"] | |
| gate = defaultdict(list) | |
| for r in sorted(pr, key=lambda r: (r["tag"], r["regime"], r["cell"])): | |
| L.append(f"| {r['tag']} | {r['regime']} | {r['cell']} | {f4(r['value_a'])} | {f4(r['value_b'])} | {float(r['delta']):+.4f} | " | |
| f"[{float(r['delta_ci_lo']):+.4f}, {float(r['delta_ci_hi']):+.4f}] | {f4(r['frac_boot_d_gt0'])} | {r['n']} | {r['n_pos']} | {r['n_dev']} |") | |
| ev = r["cell"].split("|")[1] | |
| gate[(r["regime"], ev)].append((r["tag"], float(r["delta"]), float(r["delta_ci_lo"]), float(r["delta_ci_hi"]))) | |
| if not pr: | |
| L.append("| (no paired rows yet) | | | | | | | | | | |") | |
| # ---- 3. verdict | |
| L += ["", "## 3. Acceptance gate (plan §5.2): engaged hand_leave and head_turn_onset Δ >= +0.02 on >= 2 of 3 seeds; manual not degraded beyond −1 sd", ""] | |
| verdict_lines, passes = [], {} | |
| for ev in GATE_EVENTS: | |
| rows = gate.get(("engaged", ev), []) | |
| n_pass = sum(1 for _, d, lo, hi in rows if d >= GATE_DELTA) | |
| n_ci = sum(1 for _, d, lo, hi in rows if lo > 0) | |
| passes[ev] = (n_pass, len(rows), n_ci) | |
| verdict_lines.append(f"- engaged `{ev}`: seeds with Δ >= +{GATE_DELTA}: **{n_pass}/{len(rows)}** (CI excluding 0 on {n_ci}); deltas " + | |
| ", ".join(f"{t} {d:+.4f} [{lo:+.4f},{hi:+.4f}]" for t, d, lo, hi in rows)) | |
| man = [d for (reg, ev), ds_ in deltas.items() if reg == "manual" for d in ds_] | |
| verdict_lines.append(f"- manual (all events, quick metrics): mean Δ {np.mean(man):+.4f}, min Δ {np.min(man):+.4f} over {len(man)} (seed, event) cells" if man else "- manual: no cells yet") | |
| ok = all(v[1] >= 2 and v[0] >= 2 for v in passes.values()) and bool(passes) | |
| verdict = "PASS (both engaged gate events +0.02 on >= 2/3 seeds)" if ok else ("FAIL / not met" if all(v[1] >= 3 for v in passes.values()) and passes else "INCOMPLETE (seeds / baselines missing)") | |
| L += verdict_lines + ["", f"**Verdict: {verdict}.**", ""] | |
| # ---- 4. K5 | |
| L += ["## 4. K5 shifted-twin identity-leak check (`eval/shift_twin.py`)", "", | |
| "| tag | regime | event | statistic | A | B | Δ | Δ CI | n | pairs / pos | n_dev | note |", "|---|---|---|---|---|---|---|---|---|---|---|---|"] | |
| tw = list(csv.DictReader(open(TWIN))) if TWIN.exists() else [] | |
| for r in tw: | |
| if not r["tag"].startswith("dmspre_s"): | |
| continue | |
| ci = f"[{float(r['delta_ci_lo']):+.4f}, {float(r['delta_ci_hi']):+.4f}]" if r["delta_ci_lo"] not in ("", "nan") else "—" | |
| L.append(f"| {r['tag']} | {r['regime']} | {r['event']} | {r['statistic']} | {f4(r['value_a'])} | {f4(r['value_b'])} | {float(r['delta']):+.4f} | {ci} | {r['n']} | " | |
| f"{r['n_pairs_or_pos']} | {r['n_dev']} | {r['note']} |") | |
| if not tw: | |
| L.append("| (no rows yet) | | | | | | | | | | | |") | |
| L += ["", "Reading: `twin_pair_auroc` = P(score(positive) > score(same-route twin negative, Δt 30–60 s, matched vEgo)); identity-only models give 0.5. " | |
| "`identity_probe_acc` = device-id accuracy of a linear probe on the frozen 128-d latent (flag if A − B > 0.05). `latent_probe_prauc[_centred]` = " | |
| "linear probe latent -> 3 s onset label with / without per-device mean-centring; a gain that vanishes under centring rides on identity.", ""] | |
| # ---- 5. repro rows | |
| L += ["## 5. `eval/kin_repro.py` rows of the fine-tuned runs (reference gru_kin_can, ±0.02 rule)", "", | |
| "| run | regime | event | tau | value | CI | ref mean | Δ vs ref | within 0.02 |", "|---|---|---|---|---|---|---|---|---|"] | |
| if REPRO.exists(): | |
| seen = set() | |
| for r in csv.DictReader(open(REPRO)): | |
| if "_dmspre_s" in r["run_dir"] and "smoke" not in r["run_dir"] and r["tau"] == "3s": | |
| key = (r["run_dir"], r["regime"], r["event"]) | |
| seen.add(key) | |
| L.append(f"| {Path(r['run_dir']).name} | {r['regime']} | {r['event']} | {r['tau']} | {f4(r['value'])} | [{f4(r['ci_lo'])}, {f4(r['ci_hi'])}] | {f4(r['ref_mean'])} | " | |
| f"{float(r['delta_vs_ref_mean']):+.4f} | {r['within_0.02']} |") | |
| if missing: | |
| L += ["", "Missing: " + ", ".join(missing)] | |
| L += ["", f"Files: pretraining `results/runs/U0_dmspre_trunk_s<k>/` (trunk.pt, run_dms_teacher_pretrain_*.json with `dms_fields_used`); fine-tunes " | |
| "`results/runs/U0_kin_can_noK_dmspre_s<k>/` (init_from.json, run_train_U0_kin_can_noK_s<k>_*.json with `dms_fields_used: []`); " | |
| "K5 `results/shift_twin/`, `results/shift_twin.csv`; chain log `results/runs/dmspre_chain_*.log`.", ""] | |
| TABLES.mkdir(parents=True, exist_ok=True) | |
| (TABLES / "dms_teacher_pretrain.md").write_text("\n".join(L)) | |
| print("\n".join(L)) | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 10.2 kB
- Xet hash:
- 0592b31619458d975a1612f4f9b4be979ace135a86eeab775669f9713f94aeed
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.