Buckets:

AsianPlayer
/

TriDrive

Files

xet

AsianPlayer/TriDrive / code /eval /report_c1_transfer.py

AsianPlayer

16 days ago

download

raw

13.8 kB

"""cwm3/eval/report_c1_transfer.py — results/tables/c1_transfer.md for claim C1 (minimal AIDE -> BATON transfer, roadmap E10).
Reads only script outputs: the AIDE pretraining runs (results/runs/U0_aidepre_trunk_s<k>/val_metrics.json, TRAIN_DONE), the fine-tune runs'
init_from.json, results/paired_compare.csv (tags C1_s<k>_aidepre_vs_scratch, C1_s<k>_aidepre_vs_k0; A = fine-tuned-from-AIDE-trunk
U0_kin_can_noK_s<k>_kcv2_clip8_ln_aidepre, B = the scratch survivor U0_kin_can_noK_s<k>_kcv2_clip8_ln, identical causal-v2 rows) and
results/shift_twin.csv (tags C1_s<k>, K5).  Verdicts = cwm2/eval/kill_checks.beats.
Pre-registered acceptance (roadmap §四 E10 / dms_teacher_pretrain.md §3): engaged hand_leave AND head_turn_onset onset PR-AUC@3s
Δ >= +0.02 on >= 2/3 seeds AND kill_checks.beats == BEATS on both cells, and no manual driver cell WORSE -> "transfer";
one of the two cells -> "partial transfer"; otherwise the pre-registered wording "no transfer".
  python eval/report_c1_transfer.py [--seeds 0,1,2] [--out results/tables/c1_transfer.md]
"""
from __future__ import annotations

import argparse
import json
import sys
import time
from pathlib import Path

import numpy as np
import pandas as pd

CWM3 = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(CWM3.parent / "cwm2" / "eval"))
sys.path.insert(0, str(CWM3 / "eval"))
import kill_checks as KC                                            # noqa: E402
from kill_report_cwm3 import beats_rows, tag_rows, fmt_d, fmt_v, ms, latest_rows   # noqa: E402

RUNS = CWM3 / "results" / "runs"
PAIRED = CWM3 / "results" / "paired_compare.csv"
TWIN = CWM3 / "results" / "shift_twin.csv"
DRV = ("hand_leave", "hand_return", "head_turn_onset")
GATE_EV = ("hand_leave", "head_turn_onset")


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--seeds", default="0,1,2")
    ap.add_argument("--a-prefix", default="U0_kin_can_noK_s{k}_kcv2_clip8_ln_aidepre")
    ap.add_argument("--b-prefix", default="U0_kin_can_noK_s{k}_kcv2_clip8_ln")
    ap.add_argument("--trunk-prefix", default="U0_aidepre_trunk_s{k}")
    ap.add_argument("--tag-scratch", default="C1_s{k}_aidepre_vs_scratch")
    ap.add_argument("--tag-k0", default="C1_s{k}_aidepre_vs_k0")
    ap.add_argument("--twin-tag", default="C1_s{k}")
    ap.add_argument("--out", default=str(CWM3 / "results" / "tables" / "c1_transfer.md"))
    a = ap.parse_args()
    seeds = [int(s) for s in a.seeds.split(",")]
    stamp = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime())
    paired = pd.read_csv(PAIRED, low_memory=False)
    twin = pd.read_csv(TWIN, low_memory=False) if TWIN.exists() else pd.DataFrame()
    L = [f"# C1 — minimal AIDE -> BATON transfer of the driver trunk (E10), rendered {stamp}", "",
         "Generated by `eval/report_c1_transfer.py`; every number is read from run outputs / `results/paired_compare.csv` / `results/shift_twin.csv` (nothing typed by hand). "
         "Chain: `aide/features/build_aide_kin11.py` (11 BATON channels from the AIDE in-cabin RTMW keypoints; deviations documented in its header) -> "
         "`train/pretrain_u_trunk_aide.py` (the survivor DriverBranch trunk in_ln + GRU + mode, self-supervised 2 s -> next 1 s of the 11 channels, Laplace NLL, AIDE train / val) -> "
         "`train/train_u0_from_init.py --kin-source causal_routes --kin-version v2 --stats-version 2 --variant kin_can_noK_clip8_ln` (unchanged `train_cwm3.py` recipe from the "
         "pretrained trunk; head at the seed-k init) vs the scratch survivor `U0_kin_can_noK_s<k>_kcv2_clip8_ln` on identical causal-v2 rows (`eval/paired_compare_joint.py`).", ""]
    # 1. pretraining
    L += ["## 1. AIDE self-supervised pretraining (val split; MAE in the pack's crop-normalised units, mean over the 5 horizons and 11 channels)", "",
          "| seed | best epoch | val Laplace NLL | MAE model | MAE copy-K0 | MAE constant | MAE model / copy-K0 | trunk sha256 |", "|---|---|---|---|---|---|---|---|"]
    for k in seeds:
        d = RUNS / a.trunk_prefix.format(k=k)
        if not (d / "val_metrics.json").exists():
            L.append(f"| {k} | — | (not run) | | | | | |")
            continue
        v = json.loads((d / "val_metrics.json").read_text())
        rj = sorted(d.glob("run_pretrain_u_trunk_aide_s*_latest.json"))
        sha = ""
        for ft in (RUNS / a.a_prefix.format(k=k) / "init_from.json",):
            if ft.exists():
                sha = json.loads(ft.read_text()).get("trunk_sha256", "")[:16]
        m, c, z = float(np.mean(v["mae_model"])), float(np.mean(v["mae_copy_k0"])), float(np.mean(v["mae_const"]))
        L.append(f"| {k} | {v['best_epoch']} | {v['val_loss']:.4f} | {m:.4f} | {c:.4f} | {z:.4f} | {m / c:.3f} | {sha} |")
    pre = RUNS / a.trunk_prefix.format(k=seeds[0]) / "val_metrics.json"
    if pre.exists():
        v = json.loads(pre.read_text())
        L += ["", "Per-horizon MAE (seed 0, model / copy-K0): " + ", ".join(f"+{h:.1f} s {mm:.4f} / {cc:.4f}" for h, mm, cc in zip(v["horizons_s"], np.mean(v["mae_model"], 1), np.mean(v["mae_copy_k0"], 1))), ""]
    # 2. fine-tune: paired vs scratch
    L += ["## 2. Fine-tuned U0 (AIDE trunk) vs scratch survivor — test onset PR-AUC@3s, paired device-cluster bootstrap on identical causal-v2 rows (A = AIDE-init, B = scratch)", "",
          "| regime | event | A s0/s1/s2 | B s0/s1/s2 | Δ s0 [CI] | Δ s1 [CI] | Δ s2 [CI] | mean Δ ± sd | seeds Δ ≥ +0.02 | verdict |", "|---|---|---|---|---|---|---|---|---|---|"]
    verd, gate = {}, {}
    for reg in ("engaged", "manual"):
        for ev in DRV:
            g = tag_rows(paired, a.tag_scratch, seeds, cell=f"drv_onset|{ev}|3s", regime=reg)
            v = beats_rows(g)
            verd[(reg, ev)] = v
            if len(g):
                d = pd.to_numeric(g.delta).to_numpy()
                lo, hi = pd.to_numeric(g.delta_ci_lo).to_numpy(), pd.to_numeric(g.delta_ci_hi).to_numpy()
                n02 = int((d >= 0.02).sum())
                gate[(reg, ev)] = {"n_ge_002": n02, "verdict": v["verdict"], "deltas": d.tolist()}
                by = {int(k_): f"{fmt_d(x)} [{fmt_d(l)},{fmt_d(h)}]" for k_, x, l, h in zip(g.seed, d, lo, hi)}
                cis = [by.get(k_, "—") for k_ in seeds]
                va_, vb_ = {int(k_): x for k_, x in zip(g.seed, pd.to_numeric(g.value_a))}, {int(k_): x for k_, x in zip(g.seed, pd.to_numeric(g.value_b))}
                L.append(f"| {reg} | {ev} | {' / '.join(fmt_v(va_.get(k_, np.nan)) for k_ in seeds)} | {' / '.join(fmt_v(vb_.get(k_, np.nan)) for k_ in seeds)} | {' | '.join(cis)} | "
                         f"{ms(d)} | {n02}/{len(d)} | {v['verdict']} |")
            else:
                L.append(f"| {reg} | {ev} | — | — | — | — | — | — | — | INSUFFICIENT (no paired rows) |")
    # other cells (hazard NLL, K MAE if present)
    other = paired[paired.tag.isin([a.tag_scratch.format(k=k) for k in seeds]) & ~paired.cell.str.startswith("drv_onset")]
    if len(other):
        inv = {"BEATS": "BETTER (lower)", "WORSE": "WORSE (higher)"}                   # hazard NLL / MAE / Brier cells: lower is better
        def _v(g):
            v = beats_rows(g.assign(seed=g.tag.str.extract(r'_s(\d+)_')[0].astype(int)).sort_values("seed"))["verdict"]
            return inv.get(v, v)
        L += ["", "Other paired cells (A − B, mean over seeds; lower is better for NLL / MAE cells, labels inverted accordingly): " + "; ".join(
            f"{reg} {cell} {ms(pd.to_numeric(g.delta))} {_v(g)}" for (reg, cell), g in latest_rows(other, ["tag", "regime", "cell", "metric"]).groupby(["regime", "cell"]))]
    # 3. vs k0
    L += ["", "## 3. Fine-tuned U0 (AIDE trunk) vs the k0 current-state heuristic (causal v2 K0), same rows", "", "| regime | event | A (AIDE-init) | k0 | mean Δ ± sd | verdict |", "|---|---|---|---|---|---|"]
    for reg in ("engaged", "manual"):
        for ev in DRV:
            g = tag_rows(paired, a.tag_k0, seeds, cell=f"drv_onset|{ev}|3s", regime=reg)
            g = g[g.b.astype(str).str.startswith("k0")] if len(g) else g
            v = beats_rows(g)
            L.append(f"| {reg} | {ev} | {ms(pd.to_numeric(g.value_a)) if len(g) else '—'} | {ms(pd.to_numeric(g.value_b)) if len(g) else '—'} | {ms(pd.to_numeric(g.delta)) if len(g) else '—'} | {v['verdict']} |")
    # 4. acceptance
    L += ["", "## 4. Pre-registered acceptance (engaged hand_leave AND head_turn_onset: Δ ≥ +0.02 on ≥ 2/3 seeds AND kill_checks.beats BEATS; no manual driver cell WORSE)", ""]
    ok_cells = []
    for ev in GATE_EV:
        gv = gate.get(("engaged", ev))
        if gv is None:
            L.append(f"- engaged `{ev}`: no paired rows")
            continue
        ok = gv["n_ge_002"] >= 2 and gv["verdict"] == "BEATS"
        ok_cells.append(ok)
        L.append(f"- engaged `{ev}`: seeds with Δ ≥ +0.02: **{gv['n_ge_002']}/{len(gv['deltas'])}**; beats verdict **{gv['verdict']}**; deltas {', '.join(fmt_d(x) for x in gv['deltas'])} -> {'met' if ok else 'not met'}")
    manual_worse = [ev for ev in DRV if verd.get(("manual", ev), {}).get("verdict") == "WORSE"]
    L.append(f"- manual driver cells WORSE: {', '.join(manual_worse) or 'none'}")
    if len(ok_cells) < 2 or any(v.get("verdict") == "INSUFFICIENT" for v in verd.values()):
        status = "INSUFFICIENT (rows missing)"
    elif all(ok_cells) and not manual_worse:
        status = "PASS — transfer"
    elif any(ok_cells) and not manual_worse:
        status = "PARTIAL — one of the two engaged cells; report as \"partial transfer\""
    else:
        status = "FAIL — no transfer (pre-registered wording)"
    L += ["", f"**Verdict: {status}.**", ""]
    # 5. K5 twins
    L += ["## 5. K5 shifted-twin / identity check (`eval/shift_twin_c1.py`, causal-v2 populations; A = AIDE-init, B = scratch)", "",
          "| regime | event | statistic | A | B | Δ s0/s1/s2 | mean Δ | verdict / flag | note |", "|---|---|---|---|---|---|---|---|---|"]
    if len(twin):
        tags = [a.twin_tag.format(k=k) for k in seeds]
        x = twin[twin.tag.isin(tags)].copy()
        if len(x):
            x["seed"] = x.tag.map({t: k for t, k in zip(tags, seeds)})
            x = latest_rows(x, ["tag", "regime", "event", "statistic"])
            for (reg, ev, st), g in x.groupby(["regime", "event", "statistic"], sort=False):
                g = g.sort_values("seed")
                d = pd.to_numeric(g.delta)
                if st == "twin_pair_auroc@3s":
                    flag = beats_rows(g)["verdict"]
                elif st == "identity_probe_acc":
                    flag = "FLAG (A−B > 0.05)" if d.mean() > 0.05 else "ok"
                elif st == "latent_probe_prauc_centred":
                    raw = x[(x.regime == reg) & (x.event == ev) & (x.statistic == "latent_probe_prauc")]
                    gain_raw, gain_c = float(pd.to_numeric(raw.delta).mean()) if len(raw) else np.nan, float(d.mean())
                    flag = "gain rides on identity" if (np.isfinite(gain_raw) and gain_raw > 0.01 and gain_c < 0.5 * gain_raw) else "ok"
                else:
                    flag = "—"
                L.append(f"| {reg} | {ev} | {st} | {ms(pd.to_numeric(g.value_a))} | {ms(pd.to_numeric(g.value_b))} | {' / '.join(fmt_d(v) for v in d)} | {fmt_d(d.mean())} | {flag} | {str(g.note.iloc[0])[:60]} |")
        else:
            L.append("| — | — | (no twin rows for these tags) | | | | | | |")
    L += ["", "Reading: `twin_pair_auroc` = P(score(positive) > score(same-route twin negative, Δt 30–60 s, matched vEgo)); identity-only models give 0.5; a gain that appears on random negatives but not on twins rides on identity. "
          "`identity_probe_acc` = device-id accuracy of a linear probe on the frozen 128-d GRU state (flag if A − B > 0.05). `latent_probe_prauc[_centred]` = linear probe latent -> 3 s onset with / without per-device mean-centring.", ""]
    # 6. provenance
    L += ["## 6. Provenance", ""]
    for k in seeds:
        d = RUNS / a.a_prefix.format(k=k)
        if (d / "init_from.json").exists():
            i = json.loads((d / "init_from.json").read_text())
            rj = sorted(d.glob("run_train_U0_*_latest.json"))
            z = json.loads(rj[-1].read_text()) if rj else {}
            L.append(f"- `{d.name}`: trunk `{Path(i['trunk']).parent.name}/trunk.pt` sha256 {i['trunk_sha256'][:16]} (epoch {i['trunk_epoch']}, source {i.get('trunk_source', '')}); loaded {len(i['loaded_keys'])} tensors "
                     f"({', '.join(sorted({k_.split('.')[0] for k_ in i['loaded_keys']}))}), head kept at seed-{k} init; flags {i.get('forwarded_flags', {})}; "
                     f"best epoch {z.get('best_epoch', '?')}, val {z.get('val_objective', float('nan')):.4f}, kin_source {z.get('kin_source', '?')} {z.get('kin_version', '?')}, stats v{z.get('stats_version', '?')}, dms_fields_used {z.get('dms_fields_used', '?')}")
        else:
            L.append(f"- `{d.name}`: not run")
    L += ["", "Deviations of the AIDE channel build from the BATON build (from `aide/results/cache/aide_kin11_summary.json`): "]
    sm = CWM3 / "aide" / "results" / "cache" / "aide_kin11_summary.json"
    if sm.exists():
        s = json.loads(sm.read_text())
        L += [f"  - {k}: {v}" for k, v in s["deviations"].items()]
        L += [f"  - coverage: {s['n_clips']} clips, valid steps {s['valid_frac_of_steps']:.3f} (first second {s['valid_frac_first_second']:.3f}), clips without a valid step {s['clips_with_no_valid_step']}; "
              f"hands_on prevalence {s['hands_on_prev_valid']:.3f}, head_turned {s['head_turned_prev_valid']:.3f} (descriptive, clip-median references)."]
    Path(a.out).write_text("\n".join(L) + "\n")
    print("\n".join(l for l in L if l.startswith(("**Verdict", "| engaged", "- engaged"))))
    print(f"written {a.out}")


if __name__ == "__main__":
    main()

Xet Storage Details

Size:: 13.8 kB
Xet hash:: 9e443d8b3763f001785a132d9b2f1da9f1ef0a6117d7c1f0349222c85309cd34

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.

	"""cwm3/eval/report_c1_transfer.py — results/tables/c1_transfer.md for claim C1 (minimal AIDE -> BATON transfer, roadmap E10).

	Reads only script outputs: the AIDE pretraining runs (results/runs/U0_aidepre_trunk_s<k>/val_metrics.json, TRAIN_DONE), the fine-tune runs'
	init_from.json, results/paired_compare.csv (tags C1_s<k>_aidepre_vs_scratch, C1_s<k>_aidepre_vs_k0; A = fine-tuned-from-AIDE-trunk
	U0_kin_can_noK_s<k>_kcv2_clip8_ln_aidepre, B = the scratch survivor U0_kin_can_noK_s<k>_kcv2_clip8_ln, identical causal-v2 rows) and
	results/shift_twin.csv (tags C1_s<k>, K5). Verdicts = cwm2/eval/kill_checks.beats.
	Pre-registered acceptance (roadmap §四 E10 / dms_teacher_pretrain.md §3): engaged hand_leave AND head_turn_onset onset PR-AUC@3s
	Δ >= +0.02 on >= 2/3 seeds AND kill_checks.beats == BEATS on both cells, and no manual driver cell WORSE -> "transfer";
	one of the two cells -> "partial transfer"; otherwise the pre-registered wording "no transfer".
	python eval/report_c1_transfer.py [--seeds 0,1,2] [--out results/tables/c1_transfer.md]
	"""
	from __future__ import annotations

	import argparse
	import json
	import sys
	import time
	from pathlib import Path

	import numpy as np
	import pandas as pd

	CWM3 = Path(__file__).resolve().parents[1]
	sys.path.insert(0, str(CWM3.parent / "cwm2" / "eval"))
	sys.path.insert(0, str(CWM3 / "eval"))
	import kill_checks as KC # noqa: E402
	from kill_report_cwm3 import beats_rows, tag_rows, fmt_d, fmt_v, ms, latest_rows # noqa: E402

	RUNS = CWM3 / "results" / "runs"
	PAIRED = CWM3 / "results" / "paired_compare.csv"
	TWIN = CWM3 / "results" / "shift_twin.csv"
	DRV = ("hand_leave", "hand_return", "head_turn_onset")
	GATE_EV = ("hand_leave", "head_turn_onset")


	def main():
	ap = argparse.ArgumentParser()
	ap.add_argument("--seeds", default="0,1,2")
	ap.add_argument("--a-prefix", default="U0_kin_can_noK_s{k}_kcv2_clip8_ln_aidepre")
	ap.add_argument("--b-prefix", default="U0_kin_can_noK_s{k}_kcv2_clip8_ln")
	ap.add_argument("--trunk-prefix", default="U0_aidepre_trunk_s{k}")
	ap.add_argument("--tag-scratch", default="C1_s{k}_aidepre_vs_scratch")
	ap.add_argument("--tag-k0", default="C1_s{k}_aidepre_vs_k0")
	ap.add_argument("--twin-tag", default="C1_s{k}")
	ap.add_argument("--out", default=str(CWM3 / "results" / "tables" / "c1_transfer.md"))
	a = ap.parse_args()
	seeds = [int(s) for s in a.seeds.split(",")]
	stamp = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime())
	paired = pd.read_csv(PAIRED, low_memory=False)
	twin = pd.read_csv(TWIN, low_memory=False) if TWIN.exists() else pd.DataFrame()
	L = [f"# C1 — minimal AIDE -> BATON transfer of the driver trunk (E10), rendered {stamp}", "",
	"Generated by `eval/report_c1_transfer.py`; every number is read from run outputs / `results/paired_compare.csv` / `results/shift_twin.csv` (nothing typed by hand). "
	"Chain: `aide/features/build_aide_kin11.py` (11 BATON channels from the AIDE in-cabin RTMW keypoints; deviations documented in its header) -> "
	"`train/pretrain_u_trunk_aide.py` (the survivor DriverBranch trunk in_ln + GRU + mode, self-supervised 2 s -> next 1 s of the 11 channels, Laplace NLL, AIDE train / val) -> "
	"`train/train_u0_from_init.py --kin-source causal_routes --kin-version v2 --stats-version 2 --variant kin_can_noK_clip8_ln` (unchanged `train_cwm3.py` recipe from the "
	"pretrained trunk; head at the seed-k init) vs the scratch survivor `U0_kin_can_noK_s<k>_kcv2_clip8_ln` on identical causal-v2 rows (`eval/paired_compare_joint.py`).", ""]
	# 1. pretraining
	L += ["## 1. AIDE self-supervised pretraining (val split; MAE in the pack's crop-normalised units, mean over the 5 horizons and 11 channels)", "",
	"\| seed \| best epoch \| val Laplace NLL \| MAE model \| MAE copy-K0 \| MAE constant \| MAE model / copy-K0 \| trunk sha256 \|", "\|---\|---\|---\|---\|---\|---\|---\|---\|"]
	for k in seeds:
	d = RUNS / a.trunk_prefix.format(k=k)
	if not (d / "val_metrics.json").exists():
	L.append(f"\| {k} \| — \| (not run) \| \| \| \| \| \|")
	continue
	v = json.loads((d / "val_metrics.json").read_text())
	rj = sorted(d.glob("run_pretrain_u_trunk_aide_s*_latest.json"))
	sha = ""
	for ft in (RUNS / a.a_prefix.format(k=k) / "init_from.json",):
	if ft.exists():
	sha = json.loads(ft.read_text()).get("trunk_sha256", "")[:16]
	m, c, z = float(np.mean(v["mae_model"])), float(np.mean(v["mae_copy_k0"])), float(np.mean(v["mae_const"]))
	L.append(f"\| {k} \| {v['best_epoch']} \| {v['val_loss']:.4f} \| {m:.4f} \| {c:.4f} \| {z:.4f} \| {m / c:.3f} \| {sha} \|")
	pre = RUNS / a.trunk_prefix.format(k=seeds[0]) / "val_metrics.json"
	if pre.exists():
	v = json.loads(pre.read_text())
	L += ["", "Per-horizon MAE (seed 0, model / copy-K0): " + ", ".join(f"+{h:.1f} s {mm:.4f} / {cc:.4f}" for h, mm, cc in zip(v["horizons_s"], np.mean(v["mae_model"], 1), np.mean(v["mae_copy_k0"], 1))), ""]
	# 2. fine-tune: paired vs scratch
	L += ["## 2. Fine-tuned U0 (AIDE trunk) vs scratch survivor — test onset PR-AUC@3s, paired device-cluster bootstrap on identical causal-v2 rows (A = AIDE-init, B = scratch)", "",
	"\| regime \| event \| A s0/s1/s2 \| B s0/s1/s2 \| Δ s0 [CI] \| Δ s1 [CI] \| Δ s2 [CI] \| mean Δ ± sd \| seeds Δ ≥ +0.02 \| verdict \|", "\|---\|---\|---\|---\|---\|---\|---\|---\|---\|---\|"]
	verd, gate = {}, {}
	for reg in ("engaged", "manual"):
	for ev in DRV:
	g = tag_rows(paired, a.tag_scratch, seeds, cell=f"drv_onset\|{ev}\|3s", regime=reg)
	v = beats_rows(g)
	verd[(reg, ev)] = v
	if len(g):
	d = pd.to_numeric(g.delta).to_numpy()
	lo, hi = pd.to_numeric(g.delta_ci_lo).to_numpy(), pd.to_numeric(g.delta_ci_hi).to_numpy()
	n02 = int((d >= 0.02).sum())
	gate[(reg, ev)] = {"n_ge_002": n02, "verdict": v["verdict"], "deltas": d.tolist()}
	by = {int(k_): f"{fmt_d(x)} [{fmt_d(l)},{fmt_d(h)}]" for k_, x, l, h in zip(g.seed, d, lo, hi)}
	cis = [by.get(k_, "—") for k_ in seeds]
	va_, vb_ = {int(k_): x for k_, x in zip(g.seed, pd.to_numeric(g.value_a))}, {int(k_): x for k_, x in zip(g.seed, pd.to_numeric(g.value_b))}
	L.append(f"\| {reg} \| {ev} \| {' / '.join(fmt_v(va_.get(k_, np.nan)) for k_ in seeds)} \| {' / '.join(fmt_v(vb_.get(k_, np.nan)) for k_ in seeds)} \| {' \| '.join(cis)} \| "
	f"{ms(d)} \| {n02}/{len(d)} \| {v['verdict']} \|")
	else:
	L.append(f"\| {reg} \| {ev} \| — \| — \| — \| — \| — \| — \| — \| INSUFFICIENT (no paired rows) \|")
	# other cells (hazard NLL, K MAE if present)
	other = paired[paired.tag.isin([a.tag_scratch.format(k=k) for k in seeds]) & ~paired.cell.str.startswith("drv_onset")]
	if len(other):
	inv = {"BEATS": "BETTER (lower)", "WORSE": "WORSE (higher)"} # hazard NLL / MAE / Brier cells: lower is better
	def _v(g):
	v = beats_rows(g.assign(seed=g.tag.str.extract(r'_s(\d+)_')[0].astype(int)).sort_values("seed"))["verdict"]
	return inv.get(v, v)
	L += ["", "Other paired cells (A − B, mean over seeds; lower is better for NLL / MAE cells, labels inverted accordingly): " + "; ".join(
	f"{reg} {cell} {ms(pd.to_numeric(g.delta))} {_v(g)}" for (reg, cell), g in latest_rows(other, ["tag", "regime", "cell", "metric"]).groupby(["regime", "cell"]))]
	# 3. vs k0
	L += ["", "## 3. Fine-tuned U0 (AIDE trunk) vs the k0 current-state heuristic (causal v2 K0), same rows", "", "\| regime \| event \| A (AIDE-init) \| k0 \| mean Δ ± sd \| verdict \|", "\|---\|---\|---\|---\|---\|---\|"]
	for reg in ("engaged", "manual"):
	for ev in DRV:
	g = tag_rows(paired, a.tag_k0, seeds, cell=f"drv_onset\|{ev}\|3s", regime=reg)
	g = g[g.b.astype(str).str.startswith("k0")] if len(g) else g
	v = beats_rows(g)
	L.append(f"\| {reg} \| {ev} \| {ms(pd.to_numeric(g.value_a)) if len(g) else '—'} \| {ms(pd.to_numeric(g.value_b)) if len(g) else '—'} \| {ms(pd.to_numeric(g.delta)) if len(g) else '—'} \| {v['verdict']} \|")
	# 4. acceptance
	L += ["", "## 4. Pre-registered acceptance (engaged hand_leave AND head_turn_onset: Δ ≥ +0.02 on ≥ 2/3 seeds AND kill_checks.beats BEATS; no manual driver cell WORSE)", ""]
	ok_cells = []
	for ev in GATE_EV:
	gv = gate.get(("engaged", ev))
	if gv is None:
	L.append(f"- engaged `{ev}`: no paired rows")
	continue
	ok = gv["n_ge_002"] >= 2 and gv["verdict"] == "BEATS"
	ok_cells.append(ok)
	L.append(f"- engaged `{ev}`: seeds with Δ ≥ +0.02: {gv['n_ge_002']}/{len(gv['deltas'])}; beats verdict {gv['verdict']}; deltas {', '.join(fmt_d(x) for x in gv['deltas'])} -> {'met' if ok else 'not met'}")
	manual_worse = [ev for ev in DRV if verd.get(("manual", ev), {}).get("verdict") == "WORSE"]
	L.append(f"- manual driver cells WORSE: {', '.join(manual_worse) or 'none'}")
	if len(ok_cells) < 2 or any(v.get("verdict") == "INSUFFICIENT" for v in verd.values()):
	status = "INSUFFICIENT (rows missing)"
	elif all(ok_cells) and not manual_worse:
	status = "PASS — transfer"
	elif any(ok_cells) and not manual_worse:
	status = "PARTIAL — one of the two engaged cells; report as \"partial transfer\""
	else:
	status = "FAIL — no transfer (pre-registered wording)"
	L += ["", f"Verdict: {status}.", ""]
	# 5. K5 twins
	L += ["## 5. K5 shifted-twin / identity check (`eval/shift_twin_c1.py`, causal-v2 populations; A = AIDE-init, B = scratch)", "",
	"\| regime \| event \| statistic \| A \| B \| Δ s0/s1/s2 \| mean Δ \| verdict / flag \| note \|", "\|---\|---\|---\|---\|---\|---\|---\|---\|---\|"]
	if len(twin):
	tags = [a.twin_tag.format(k=k) for k in seeds]
	x = twin[twin.tag.isin(tags)].copy()
	if len(x):
	x["seed"] = x.tag.map({t: k for t, k in zip(tags, seeds)})
	x = latest_rows(x, ["tag", "regime", "event", "statistic"])
	for (reg, ev, st), g in x.groupby(["regime", "event", "statistic"], sort=False):
	g = g.sort_values("seed")
	d = pd.to_numeric(g.delta)
	if st == "twin_pair_auroc@3s":
	flag = beats_rows(g)["verdict"]
	elif st == "identity_probe_acc":
	flag = "FLAG (A−B > 0.05)" if d.mean() > 0.05 else "ok"
	elif st == "latent_probe_prauc_centred":
	raw = x[(x.regime == reg) & (x.event == ev) & (x.statistic == "latent_probe_prauc")]
	gain_raw, gain_c = float(pd.to_numeric(raw.delta).mean()) if len(raw) else np.nan, float(d.mean())
	flag = "gain rides on identity" if (np.isfinite(gain_raw) and gain_raw > 0.01 and gain_c < 0.5 * gain_raw) else "ok"
	else:
	flag = "—"
	L.append(f"\| {reg} \| {ev} \| {st} \| {ms(pd.to_numeric(g.value_a))} \| {ms(pd.to_numeric(g.value_b))} \| {' / '.join(fmt_d(v) for v in d)} \| {fmt_d(d.mean())} \| {flag} \| {str(g.note.iloc[0])[:60]} \|")
	else:
	L.append("\| — \| — \| (no twin rows for these tags) \| \| \| \| \| \| \|")
	L += ["", "Reading: `twin_pair_auroc` = P(score(positive) > score(same-route twin negative, Δt 30–60 s, matched vEgo)); identity-only models give 0.5; a gain that appears on random negatives but not on twins rides on identity. "
	"`identity_probe_acc` = device-id accuracy of a linear probe on the frozen 128-d GRU state (flag if A − B > 0.05). `latent_probe_prauc[_centred]` = linear probe latent -> 3 s onset with / without per-device mean-centring.", ""]
	# 6. provenance
	L += ["## 6. Provenance", ""]
	for k in seeds:
	d = RUNS / a.a_prefix.format(k=k)
	if (d / "init_from.json").exists():
	i = json.loads((d / "init_from.json").read_text())
	rj = sorted(d.glob("run_train_U0_*_latest.json"))
	z = json.loads(rj[-1].read_text()) if rj else {}
	L.append(f"- `{d.name}`: trunk `{Path(i['trunk']).parent.name}/trunk.pt` sha256 {i['trunk_sha256'][:16]} (epoch {i['trunk_epoch']}, source {i.get('trunk_source', '')}); loaded {len(i['loaded_keys'])} tensors "
	f"({', '.join(sorted({k_.split('.')[0] for k_ in i['loaded_keys']}))}), head kept at seed-{k} init; flags {i.get('forwarded_flags', {})}; "
	f"best epoch {z.get('best_epoch', '?')}, val {z.get('val_objective', float('nan')):.4f}, kin_source {z.get('kin_source', '?')} {z.get('kin_version', '?')}, stats v{z.get('stats_version', '?')}, dms_fields_used {z.get('dms_fields_used', '?')}")
	else:
	L.append(f"- `{d.name}`: not run")
	L += ["", "Deviations of the AIDE channel build from the BATON build (from `aide/results/cache/aide_kin11_summary.json`): "]
	sm = CWM3 / "aide" / "results" / "cache" / "aide_kin11_summary.json"
	if sm.exists():
	s = json.loads(sm.read_text())
	L += [f" - {k}: {v}" for k, v in s["deviations"].items()]
	L += [f" - coverage: {s['n_clips']} clips, valid steps {s['valid_frac_of_steps']:.3f} (first second {s['valid_frac_first_second']:.3f}), clips without a valid step {s['clips_with_no_valid_step']}; "
	f"hands_on prevalence {s['hands_on_prev_valid']:.3f}, head_turned {s['head_turned_prev_valid']:.3f} (descriptive, clip-median references)."]
	Path(a.out).write_text("\n".join(L) + "\n")
	print("\n".join(l for l in L if l.startswith(("**Verdict", "\| engaged", "- engaged"))))
	print(f"written {a.out}")


	if __name__ == "__main__":
	main()