Buckets:

AsianPlayer
/

TriDrive

Files

xet

AsianPlayer/TriDrive / code /eval /report_exp3_P.py

AsianPlayer

16 days ago

download

raw

8.43 kB

"""cwm3/eval/report_exp3_P.py — Exp3-P: the E0 `*_P` variants (pooled_P, regionpca256_P; six road-pressure chains on the E trunk) -> results/tables/exp3_P.md.
Inputs (script outputs only, results/paired_compare.csv tags exp3P_s<k>_*):
  p_onset cells (eval/pressure_eval.py): pooledP_vs_margin, rp256P_vs_margin, rp256P_vs_pooledP, *_vs_persistence (base_rate rows)
  d_onset / f_cos cells (eval/paired_compare.py): pooledP_vs_pooled (vs E0_pooled_s<k>_v2s2: no-loss check), rp256P_vs_pooledP
Verdict per cell: kill_checks.beats (>= 2/3 seeds CI excl. 0, same sign, |mean| > seed sd).  python report_exp3_P.py [--out results/tables/exp3_P.md]
"""
from __future__ import annotations

import argparse
import sys
from pathlib import Path

import numpy as np
import pandas as pd

CWM3 = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(CWM3.parent / "cwm2" / "eval"))
import kill_checks as KC                                            # noqa: E402

PC = CWM3 / "results" / "paired_compare.csv"
P_CHAINS = ("ttc", "ttc_drop", "cut_in", "lead_braking", "lane_excursion", "badas")
FAMS = ("steer_rate", "decel", "ttc", "ttc_drop", "lane", "badas")


def ms(v):
    v = np.asarray([x for x in v if x is not None and np.isfinite(x)], float)
    return "—" if not len(v) else (f"{v.mean():.3f} ± {v.std(ddof=1):.3f}" if len(v) > 1 else f"{v[0]:.3f}")


def beats(g):
    if not len(g):
        return "—"
    pr = pd.DataFrame({"seed": g.seed, "delta": g.delta.astype(float), "delta_ci_lo": g.delta_ci_lo.astype(float), "delta_ci_hi": g.delta_ci_hi.astype(float), "status": "secondary"})
    return KC.beats(pr, pd.DataFrame({"value": g.value_a.astype(float)}), pd.DataFrame({"value": g.value_b.astype(float)}), min_seeds=3)["verdict"]


def dcell(g):
    if not len(g):
        return "—"
    return " / ".join(f"{float(r.delta):+.3f} [{float(r.delta_ci_lo):+.3f},{float(r.delta_ci_hi):+.3f}]" for r in g.sort_values("seed").itertuples())


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--out", default=str(CWM3 / "results" / "tables" / "exp3_P.md"))
    a = ap.parse_args()
    d = pd.read_csv(PC, low_memory=False)
    d = d[d.tag.str.startswith("exp3P_")].copy()
    d["seed"] = d.tag.str.extract(r"_s(\d+)_")[0].astype(int)
    d["pair"] = d.tag.str.replace(r"^exp3P_s\d+_", "", regex=True)
    d = d.sort_values("run_stamp").groupby(["tag", "regime", "cell", "metric"], as_index=False).tail(1)
    for c in ("value_a", "value_b", "delta", "delta_ci_lo", "delta_ci_hi"):
        d[c] = pd.to_numeric(d[c], errors="coerce")
    L = ["# Exp3-P — road-pressure chains on the E trunk: E0 `pooled_P` and `regionpca256_P` (3 seeds; pack v2, stats v2, select_LF false, λ_P = 1, D chains kept)", "",
         "P chains ttc / ttc_drop / cut_in / lead_braking / lane_excursion / badas (`results/targets_pressure`, P0 / Pvalid), emitted as a second flat head on the E trunk "
         "(`EnvBranch(p_head=True)`); selection objective = Σ D-chain + P-chain NLL (no L_F). Layout: extra keys `p_names` (= `P_<chain>`) / `p_onset_p` / `p_h_on` / `p_pi` / `p_z0` "
         "(the joint layout's convention, consumed by the warning head); the 14 contract event columns are unchanged. "
         "(a) P-onset PR-AUC@3s per chain (rows inactive & valid at t, Pvalid & Moving & M over the first 15 steps) vs the current-margin RULE score S0 (threshold-normalised margin at t; "
         "> 1 = fired) and vs persistence (= base rate); (b) the D cells vs `E0_pooled_s<k>_v2s2` (no-loss check); (c) `regionpca256_P` − `pooled_P` on the P cells (does the spatial road "
         "stream reach lane_excursion / cut_in onsets?). Paired device-cluster bootstrap on identical rows (n_boot 1000); verdict = kill_checks.beats. Source: `results/paired_compare.csv` tags `exp3P_*`.", ""]
    verd = {}
    # ---- (a) P onsets
    L += ["## (a) P-onset PR-AUC@3s — levels (mean ± sd over seeds) and paired Δ vs the current-margin rule", "",
          "| regime | chain | pooled_P | regionpca256_P | margin rule S0 | persistence | pooled_P − margin (s0 / s1 / s2) | verdict | rp256_P − margin | verdict |", "|---|---|---|---|---|---|---|---|---|---|"]
    for reg in ("manual", "engaged"):
        for c in P_CHAINS:
            cell = f"p_onset|{c}|3s"
            g1 = d[(d.pair == "pooledP_vs_margin") & (d.regime == reg) & (d.cell == cell) & (d.metric == "pr_auc")]
            g2 = d[(d.pair == "rp256P_vs_margin") & (d.regime == reg) & (d.cell == cell) & (d.metric == "pr_auc")]
            gb = d[(d.pair == "pooledP_vs_persistence") & (d.regime == reg) & (d.cell == cell) & (d.metric == "base_rate")]
            v1, v2 = beats(g1), beats(g2)
            verd[("pooledP_vs_margin", reg, c)], verd[("rp256P_vs_margin", reg, c)] = v1, v2
            L.append(f"| {reg} | {c} | {ms(g1.value_a)} | {ms(g2.value_a)} | {ms(g1.value_b)} | {ms(gb.value_a)} | {dcell(g1)} | {v1} | {dcell(g2)} | {v2} |")
    L.append("")
    # ---- (c) spatial test on the P cells
    L += ["## (c) regionpca256_P − pooled_P on the P cells (identical rows)", "", "| regime | chain | Δ s0 / s1 / s2 | mean ± sd | verdict |", "|---|---|---|---|---|"]
    for reg in ("manual", "engaged"):
        for c in P_CHAINS:
            g = d[(d.pair == "rp256P_vs_pooledP") & (d.regime == reg) & (d.cell == f"p_onset|{c}|3s") & (d.metric == "pr_auc")]
            v = beats(g); verd[("rp256P_vs_pooledP", reg, c)] = v
            L.append(f"| {reg} | {c} | {dcell(g)} | {ms(g.delta)} | {v} |")
    L.append("")
    # ---- (b) D cells + F cosine
    for pair, title in (("pooledP_vs_pooled", "(b) pooled_P − pooled (E0_pooled_s<k>_v2s2): demand-onset cells (no-loss check) and F cosine (descriptive)"),
                        ("rp256P_vs_pooledP", "(b') regionpca256_P − pooled_P: demand-onset cells and F cosine (descriptive)")):
        L += [f"## {title}", "", "| regime | cell | A | B | Δ s0 / s1 / s2 | mean ± sd | verdict |", "|---|---|---|---|---|---|---|"]
        for reg in ("manual", "engaged"):
            for cell in [f"d_onset|{f}|3s" for f in FAMS] + [f"f_cos|+{i}s" for i in range(1, 6)]:
                g = d[(d.pair == pair) & (d.regime == reg) & (d.cell == cell)]
                if not len(g):
                    continue
                v = beats(g); verd[(pair, reg, cell)] = v
                L.append(f"| {reg} | {cell.replace('|', '/')} | {ms(g.value_a)} | {ms(g.value_b)} | {dcell(g)} | {ms(g.delta)} | {v} |")
        L.append("")
    # ---- verdict summary
    def lst(pair, pred, cells=None):
        return [f"{r}|{c}" for (p_, r, c), v in verd.items() if p_ == pair and pred(v) and (cells is None or c in cells)]
    L += ["## Verdicts", ""]
    L.append(f"- **(a) pooled_P vs the current-margin rule**: BEATS {lst('pooledP_vs_margin', lambda v: v == 'BEATS') or 'none'}; WORSE {lst('pooledP_vs_margin', lambda v: v == 'WORSE') or 'none'}; "
             f"INCONCLUSIVE {lst('pooledP_vs_margin', lambda v: v == 'INCONCLUSIVE') or 'none'}.")
    L.append(f"- **(a') regionpca256_P vs the current-margin rule**: BEATS {lst('rp256P_vs_margin', lambda v: v == 'BEATS') or 'none'}; WORSE {lst('rp256P_vs_margin', lambda v: v == 'WORSE') or 'none'}.")
    dw = lst("pooledP_vs_pooled", lambda v: v == "WORSE", [f"d_onset|{f}|3s" for f in FAMS]); db = lst("pooledP_vs_pooled", lambda v: v == "BEATS", [f"d_onset|{f}|3s" for f in FAMS])
    L.append(f"- **(b) adding the P head costs the D chains nothing?** D cells WORSE {dw or 'none'}; BEATS {db or 'none'} → {'PASS (no D cell WORSE)' if not dw else 'FAIL'}.")
    sp = {c: [verd.get(('rp256P_vs_pooledP', r, c), '—') for r in ('manual', 'engaged')] for c in ("lane_excursion", "cut_in")}
    pcells = [f"p_onset|{c}|3s" for c in P_CHAINS] + list(P_CHAINS)
    L.append(f"- **(c) spatial road stream on the P cells (regionpca256_P − pooled_P)**: lane_excursion manual / engaged {sp['lane_excursion']}, cut_in {sp['cut_in']}; "
             f"P cells BEATS {lst('rp256P_vs_pooledP', lambda v: v == 'BEATS', pcells) or 'none'}, WORSE {lst('rp256P_vs_pooledP', lambda v: v == 'WORSE', pcells) or 'none'}; "
             f"its D / F cells: WORSE {lst('rp256P_vs_pooledP', lambda v: v == 'WORSE', [f'd_onset|{f}|3s' for f in FAMS] + [f'f_cos|+{i}s' for i in range(1, 6)]) or 'none'} (as in exp3_v2b).")
    Path(a.out).write_text("\n".join(L) + "\n")
    print("\n".join(L[-6:]))
    print(f"written {a.out}")


if __name__ == "__main__":
    main()

Xet Storage Details

Size:: 8.43 kB
Xet hash:: 85bc310fdbcc511ac1d36cf788b60fe9134e63d1a3c1799cc00dcf1ba6191551

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.

	"""cwm3/eval/report_exp3_P.py — Exp3-P: the E0 `*_P` variants (pooled_P, regionpca256_P; six road-pressure chains on the E trunk) -> results/tables/exp3_P.md.

	Inputs (script outputs only, results/paired_compare.csv tags exp3P_s<k>_*):
	p_onset cells (eval/pressure_eval.py): pooledP_vs_margin, rp256P_vs_margin, rp256P_vs_pooledP, *_vs_persistence (base_rate rows)
	d_onset / f_cos cells (eval/paired_compare.py): pooledP_vs_pooled (vs E0_pooled_s<k>_v2s2: no-loss check), rp256P_vs_pooledP
	Verdict per cell: kill_checks.beats (>= 2/3 seeds CI excl. 0, same sign, \|mean\| > seed sd). python report_exp3_P.py [--out results/tables/exp3_P.md]
	"""
	from __future__ import annotations

	import argparse
	import sys
	from pathlib import Path

	import numpy as np
	import pandas as pd

	CWM3 = Path(__file__).resolve().parents[1]
	sys.path.insert(0, str(CWM3.parent / "cwm2" / "eval"))
	import kill_checks as KC # noqa: E402

	PC = CWM3 / "results" / "paired_compare.csv"
	P_CHAINS = ("ttc", "ttc_drop", "cut_in", "lead_braking", "lane_excursion", "badas")
	FAMS = ("steer_rate", "decel", "ttc", "ttc_drop", "lane", "badas")


	def ms(v):
	v = np.asarray([x for x in v if x is not None and np.isfinite(x)], float)
	return "—" if not len(v) else (f"{v.mean():.3f} ± {v.std(ddof=1):.3f}" if len(v) > 1 else f"{v[0]:.3f}")


	def beats(g):
	if not len(g):
	return "—"
	pr = pd.DataFrame({"seed": g.seed, "delta": g.delta.astype(float), "delta_ci_lo": g.delta_ci_lo.astype(float), "delta_ci_hi": g.delta_ci_hi.astype(float), "status": "secondary"})
	return KC.beats(pr, pd.DataFrame({"value": g.value_a.astype(float)}), pd.DataFrame({"value": g.value_b.astype(float)}), min_seeds=3)["verdict"]


	def dcell(g):
	if not len(g):
	return "—"
	return " / ".join(f"{float(r.delta):+.3f} [{float(r.delta_ci_lo):+.3f},{float(r.delta_ci_hi):+.3f}]" for r in g.sort_values("seed").itertuples())


	def main():
	ap = argparse.ArgumentParser()
	ap.add_argument("--out", default=str(CWM3 / "results" / "tables" / "exp3_P.md"))
	a = ap.parse_args()
	d = pd.read_csv(PC, low_memory=False)
	d = d[d.tag.str.startswith("exp3P_")].copy()
	d["seed"] = d.tag.str.extract(r"_s(\d+)_")[0].astype(int)
	d["pair"] = d.tag.str.replace(r"^exp3P_s\d+_", "", regex=True)
	d = d.sort_values("run_stamp").groupby(["tag", "regime", "cell", "metric"], as_index=False).tail(1)
	for c in ("value_a", "value_b", "delta", "delta_ci_lo", "delta_ci_hi"):
	d[c] = pd.to_numeric(d[c], errors="coerce")
	L = ["# Exp3-P — road-pressure chains on the E trunk: E0 `pooled_P` and `regionpca256_P` (3 seeds; pack v2, stats v2, select_LF false, λ_P = 1, D chains kept)", "",
	"P chains ttc / ttc_drop / cut_in / lead_braking / lane_excursion / badas (`results/targets_pressure`, P0 / Pvalid), emitted as a second flat head on the E trunk "
	"(`EnvBranch(p_head=True)`); selection objective = Σ D-chain + P-chain NLL (no L_F). Layout: extra keys `p_names` (= `P_<chain>`) / `p_onset_p` / `p_h_on` / `p_pi` / `p_z0` "
	"(the joint layout's convention, consumed by the warning head); the 14 contract event columns are unchanged. "
	"(a) P-onset PR-AUC@3s per chain (rows inactive & valid at t, Pvalid & Moving & M over the first 15 steps) vs the current-margin RULE score S0 (threshold-normalised margin at t; "
	"> 1 = fired) and vs persistence (= base rate); (b) the D cells vs `E0_pooled_s<k>_v2s2` (no-loss check); (c) `regionpca256_P` − `pooled_P` on the P cells (does the spatial road "
	"stream reach lane_excursion / cut_in onsets?). Paired device-cluster bootstrap on identical rows (n_boot 1000); verdict = kill_checks.beats. Source: `results/paired_compare.csv` tags `exp3P_*`.", ""]
	verd = {}
	# ---- (a) P onsets
	L += ["## (a) P-onset PR-AUC@3s — levels (mean ± sd over seeds) and paired Δ vs the current-margin rule", "",
	"\| regime \| chain \| pooled_P \| regionpca256_P \| margin rule S0 \| persistence \| pooled_P − margin (s0 / s1 / s2) \| verdict \| rp256_P − margin \| verdict \|", "\|---\|---\|---\|---\|---\|---\|---\|---\|---\|---\|"]
	for reg in ("manual", "engaged"):
	for c in P_CHAINS:
	cell = f"p_onset\|{c}\|3s"
	g1 = d[(d.pair == "pooledP_vs_margin") & (d.regime == reg) & (d.cell == cell) & (d.metric == "pr_auc")]
	g2 = d[(d.pair == "rp256P_vs_margin") & (d.regime == reg) & (d.cell == cell) & (d.metric == "pr_auc")]
	gb = d[(d.pair == "pooledP_vs_persistence") & (d.regime == reg) & (d.cell == cell) & (d.metric == "base_rate")]
	v1, v2 = beats(g1), beats(g2)
	verd[("pooledP_vs_margin", reg, c)], verd[("rp256P_vs_margin", reg, c)] = v1, v2
	L.append(f"\| {reg} \| {c} \| {ms(g1.value_a)} \| {ms(g2.value_a)} \| {ms(g1.value_b)} \| {ms(gb.value_a)} \| {dcell(g1)} \| {v1} \| {dcell(g2)} \| {v2} \|")
	L.append("")
	# ---- (c) spatial test on the P cells
	L += ["## (c) regionpca256_P − pooled_P on the P cells (identical rows)", "", "\| regime \| chain \| Δ s0 / s1 / s2 \| mean ± sd \| verdict \|", "\|---\|---\|---\|---\|---\|"]
	for reg in ("manual", "engaged"):
	for c in P_CHAINS:
	g = d[(d.pair == "rp256P_vs_pooledP") & (d.regime == reg) & (d.cell == f"p_onset\|{c}\|3s") & (d.metric == "pr_auc")]
	v = beats(g); verd[("rp256P_vs_pooledP", reg, c)] = v
	L.append(f"\| {reg} \| {c} \| {dcell(g)} \| {ms(g.delta)} \| {v} \|")
	L.append("")
	# ---- (b) D cells + F cosine
	for pair, title in (("pooledP_vs_pooled", "(b) pooled_P − pooled (E0_pooled_s<k>_v2s2): demand-onset cells (no-loss check) and F cosine (descriptive)"),
	("rp256P_vs_pooledP", "(b') regionpca256_P − pooled_P: demand-onset cells and F cosine (descriptive)")):
	L += [f"## {title}", "", "\| regime \| cell \| A \| B \| Δ s0 / s1 / s2 \| mean ± sd \| verdict \|", "\|---\|---\|---\|---\|---\|---\|---\|"]
	for reg in ("manual", "engaged"):
	for cell in [f"d_onset\|{f}\|3s" for f in FAMS] + [f"f_cos\|+{i}s" for i in range(1, 6)]:
	g = d[(d.pair == pair) & (d.regime == reg) & (d.cell == cell)]
	if not len(g):
	continue
	v = beats(g); verd[(pair, reg, cell)] = v
	L.append(f"\| {reg} \| {cell.replace('\|', '/')} \| {ms(g.value_a)} \| {ms(g.value_b)} \| {dcell(g)} \| {ms(g.delta)} \| {v} \|")
	L.append("")
	# ---- verdict summary
	def lst(pair, pred, cells=None):
	return [f"{r}\|{c}" for (p_, r, c), v in verd.items() if p_ == pair and pred(v) and (cells is None or c in cells)]
	L += ["## Verdicts", ""]
	L.append(f"- (a) pooled_P vs the current-margin rule: BEATS {lst('pooledP_vs_margin', lambda v: v == 'BEATS') or 'none'}; WORSE {lst('pooledP_vs_margin', lambda v: v == 'WORSE') or 'none'}; "
	f"INCONCLUSIVE {lst('pooledP_vs_margin', lambda v: v == 'INCONCLUSIVE') or 'none'}.")
	L.append(f"- (a') regionpca256_P vs the current-margin rule: BEATS {lst('rp256P_vs_margin', lambda v: v == 'BEATS') or 'none'}; WORSE {lst('rp256P_vs_margin', lambda v: v == 'WORSE') or 'none'}.")
	dw = lst("pooledP_vs_pooled", lambda v: v == "WORSE", [f"d_onset\|{f}\|3s" for f in FAMS]); db = lst("pooledP_vs_pooled", lambda v: v == "BEATS", [f"d_onset\|{f}\|3s" for f in FAMS])
	L.append(f"- (b) adding the P head costs the D chains nothing? D cells WORSE {dw or 'none'}; BEATS {db or 'none'} → {'PASS (no D cell WORSE)' if not dw else 'FAIL'}.")
	sp = {c: [verd.get(('rp256P_vs_pooledP', r, c), '—') for r in ('manual', 'engaged')] for c in ("lane_excursion", "cut_in")}
	pcells = [f"p_onset\|{c}\|3s" for c in P_CHAINS] + list(P_CHAINS)
	L.append(f"- (c) spatial road stream on the P cells (regionpca256_P − pooled_P): lane_excursion manual / engaged {sp['lane_excursion']}, cut_in {sp['cut_in']}; "
	f"P cells BEATS {lst('rp256P_vs_pooledP', lambda v: v == 'BEATS', pcells) or 'none'}, WORSE {lst('rp256P_vs_pooledP', lambda v: v == 'WORSE', pcells) or 'none'}; "
	f"its D / F cells: WORSE {lst('rp256P_vs_pooledP', lambda v: v == 'WORSE', [f'd_onset\|{f}\|3s' for f in FAMS] + [f'f_cos\|+{i}s' for i in range(1, 6)]) or 'none'} (as in exp3_v2b).")
	Path(a.out).write_text("\n".join(L) + "\n")
	print("\n".join(L[-6:]))
	print(f"written {a.out}")


	if __name__ == "__main__":
	main()