Buckets:
| #!/usr/bin/env python3 | |
| """cwm3/baselines_dms/report_dms.py — render the DMS-baseline comparison tables from the UNCHANGED evaluators' CSV outputs (append-only sections). | |
| Inputs (produced by the existing tools, never recomputed here): | |
| results/warning_gold/<item_dir>/warning_gold.csv cwm2/eval/warning_gold.py (--item <item>): AUROC / PR-AUC per stratum with device-cluster CIs, | |
| paired Δ vs the comparators on identical resamples (column `comparator`) | |
| results/warning_gold/<item_dir>/paired_vs_head.csv eval_dms.paired_vs_head: AUROC / PR-AUC Δ vs the frozen CockpitWM read-out (agg row keyed on AUROC) | |
| results/warning_gold/<item_dir>/paired_vs_head_prauc.csv eval_dms.paired_vs_head_prauc: the same per-seed PR-AUC Δ + an agg row keyed on PR-AUC (2026-09-15) | |
| results/operating_points/<item>_<stamp>/{operating_points,paired_lead,reliability}.csv eval/operating_points.py (--item <item>) | |
| Outputs (sections appended; existing sections never modified): | |
| results/tables/dms_warning_comparison.md one section per gold item: baseline columns vs the openpilot att@3s chain on the SAME rows | |
| (common-valid subset of that item; n printed) + paired Δ [CI] with the kill_checks.beats wording | |
| --consolidated: ONE section, one row per baseline arm (selected read-out) x stratum, AUROC and PR-AUC side by side | |
| results/tables/dms_layer2_summary.md --summary: compact owner-facing table (AUROC / PR-AUC per stratum for every baseline, att@3s and the head, with the | |
| parameter and pre-training columns copied from results/tables/dms_fairness_audit.md) | |
| results/tables/dms_operating_points.md one section per operating-point item: recall / precision at dms_matched and low10, paired lead verdict, ECE | |
| --verify re-reads the CSVs and checks that every number printed in the newest section of each table is present verbatim. | |
| --metrics auroc,pr_auc (default auroc = the historical layout, byte-identical): a gold section then prints, per column x stratum x seed and the mean row, | |
| n / pos, prevalence (pos / n = the PR-AUC chance level), and for EACH metric the baseline value [CI], the att@3s value on the same rows, Δ vs att@3s [CI] | |
| with the verdict, and (when the head was passed to the gold run) the head value, Δ vs head [CI] and its verdict. | |
| Wording: BEATS -> "significantly better", WORSE -> "significantly worse", else "no significant difference" (single seed: "CI excludes 0" / "CI includes 0"; | |
| several seeds (2026-09-14 layer-2 launch): one row per seed + a `mean` row from warning_gold's `agg` rows, verdict by the pre-stated rule >= ceil(2n/3) seed CIs | |
| exclude 0, same sign, |mean Δ| > sd over seeds — the kill_checks.beats reading). When the frozen CockpitWM read-out was passed to the gold run (eval_dms --head), | |
| its AUROC on the same rows and the paired Δ vs it (eval_dms.paired_vs_head, --paired-head-csv) are two extra columns. --table-dir redirects the tables (dry runs). | |
| python baselines_dms/report_dms.py --gold-item dms_r0 --gold-dir results/warning_gold/dms_r0_20260914 --op-item dms_r0 \ | |
| --baseline "R0 fixed gaze/eye rule (own HALPE face pack)" --arm abl_dms_lmface_rule --status "adapted; zero-cost fixed rule; no video" | |
| python baselines_dms/report_dms.py --gold-item dms_videomae2_learned --gold-dir <dir> --arm abl_dms_videomae2_learned --baseline "..." \ | |
| --paired-head-csv <dir>/paired_vs_head.csv --metrics auroc,pr_auc --table-dir /tmp/dry --verify | |
| python baselines_dms/report_dms.py --consolidated --verify # newest gold dir per item of CONSOLIDATED -> one section in dms_warning_comparison.md | |
| python baselines_dms/report_dms.py --summary --verify # -> results/tables/dms_layer2_summary.md | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import math | |
| import re | |
| import sys | |
| import time | |
| from pathlib import Path | |
| import pandas as pd | |
| CWM3 = Path(__file__).resolve().parents[1] | |
| TABLES = CWM3 / "results" / "tables" | |
| GOLD_ROOT = CWM3 / "results" / "warning_gold" | |
| GOLD_TABLE = TABLES / "dms_warning_comparison.md" | |
| OP_TABLE = TABLES / "dms_operating_points.md" | |
| SUMMARY_TABLE = TABLES / "dms_layer2_summary.md" | |
| FAIRNESS_TABLE = TABLES / "dms_fairness_audit.md" | |
| STRATA = [("clean_test", "pooled"), ("clean_test|manual", "manual"), ("clean_test|engaged", "engaged")] | |
| HEAD_ARM, HEAD_COL = "abl_W_dwfc", "W_learned_curhgb_t15_w10_n979dwfc" | |
| # metric key in warning_gold.csv -> (printed label, column stem in paired_vs_head*.csv, sd field of the agg row in paired_vs_head*.csv) | |
| METRICS = {"auroc": ("AUROC", "auroc", "sd_seed_delta"), "pr_auc": ("PR-AUC", "prauc", "sd_seed_delta_prauc")} | |
| # layer-2 consolidated read-out: (item, arm, column selector — exact name or a substring such as "_sel_" (the CV-selected kind), label as in the per-item section headers) | |
| CONSOLIDATED = [ | |
| ("dms_r0", "abl_dms_lmface_rule", "dms_lmface_rule0", "R0 fixed gaze/eye rule (own HALPE face pack)"), | |
| ("dms_l2cs_gaze_learned", "abl_dms_l2cs_gaze_learned", "_sel_", "L2CS-Net (Gaze360), learned read-out"), | |
| ("dms_l2cs_gaze_ctx", "abl_dms_l2cs_gaze_ctx", "_sel_", "L2CS-Net (Gaze360) + CockpitWM current-state context"), | |
| ("dms_videomae2_learned", "abl_dms_videomae2_learned", "_sel_", "VideoMAEv2 ViT-S frozen features, learned read-out"), | |
| ("dms_videomae2_ctx", "abl_dms_videomae2_ctx", "_sel_", "VideoMAEv2 ViT-S frozen features + CockpitWM current-state context"), | |
| ("dms_ov_gaze_learned", "abl_dms_ov_gaze_learned", "_sel_", "Intel OMZ head-pose / gaze / eye pipeline, learned read-out"), | |
| ("dms_ov_gaze_ctx", "abl_dms_ov_gaze_ctx", "_sel_", "Intel OMZ head-pose / gaze / eye pipeline + CockpitWM current-state context"), | |
| ("dms_mp_face_learned", "abl_dms_mp_face_learned", "_sel_", "MediaPipe Face Landmarker, learned read-out"), | |
| ("dms_mp_face_ctx", "abl_dms_mp_face_ctx", "_sel_", "MediaPipe Face Landmarker + CockpitWM current-state context"), | |
| ("dms_ov_action_learned", "abl_dms_ov_action_learned", "_sel_", "Intel OMZ driver-action-recognition logits, learned read-out"), | |
| ("dms_ov_action_ctx", "abl_dms_ov_action_ctx", "_sel_", "Intel OMZ driver-action-recognition logits + CockpitWM current-state context"), | |
| ("dms_openface3_learned", "abl_dms_openface3_learned", "_sel_", "OpenFace 3.0, learned read-out"), | |
| ("dms_openface3_ctx", "abl_dms_openface3_ctx", "_sel_", "OpenFace 3.0 + CockpitWM current-state context"), | |
| ] | |
| # --summary columns copied from results/tables/dms_fairness_audit.md (section 2026-09-14T18:47:22Z; "row k" = table row k there). Never estimated: "not documented" is kept. | |
| FROZEN_PARAMS = { | |
| "att": "on-device openpilot DMS network: not documented; legacy hazard chain (ours): count not documented (row 1)", | |
| "head": "3,502,901 trainable (derived, row 9) + uncounted frozen perception (YOLO11x-pose, RTMW-x-l, V-JEPA2: counts not documented)", | |
| "r0": "0 trainable (fixed rule); frozen RTMW-x-l + YOLO11x-pose landmarks: counts not documented (row 2)", | |
| "l2cs_gaze": "23,936,181 frozen (local state-dict count) + RetinaFace mobilenet0.25 433,343 (row 7)", | |
| "videomae2": "22,153,286 frozen (local count, row 8)", | |
| "ov_gaze": "Σ 9.452 M frozen (5 OMZ model cards, row 3)", | |
| "mp_face": "not documented (no count on the 3 cards; bundle 3,758,596 B fp16, row 5)", | |
| "ov_action": "Σ 7.068 M frozen (encoder 2.863 M + decoder 4.205 M, row 4)", | |
| "openface3": "42,953,497 frozen as run (RetinaFace 433,343 + STAR-68 17,131,618 + MTL 25,388,536; paper 29.4 M, not reconciled; row 6)", | |
| } | |
| PRETRAIN = { | |
| "att": "comma fleet uploads (corpus size / years not documented); in-cabin driving video: yes", | |
| "head": "BATON train split only (in-cabin, target domain); frozen perception models' training data: not documented", | |
| "r0": "none (fixed rule); hysteresis thresholds = BATON-train percentiles; landmarks from the frozen pose stack (training data not documented)", | |
| "l2cs_gaze": "ImageNet-pretrained ResNet-50 fine-tuned on Gaze360 (ICCV 2019; 238 subjects, 200,000 images, Ladybug rig); in-cabin: no", | |
| "videomae2": "K710 distillation (Kinetics-400/600/700, 658k train clips) from a teacher pre-trained on UnlabeledHybrid 1.348 M clips (CVPR 2023); in-cabin: none documented", | |
| "ov_gaze": "Intel-internal; cards state no training set (validation sets only: WIDER FACE, Biwi, internal 300- / 60-person sets, MRL Eye); 2018–2019; in-cabin: not documented", | |
| "mp_face": "smartphone AR-app images (BlazeFace; Face Mesh V2, 2022-09) + synthetic (z, Blendshape V2, 2022-11); sizes not documented; in-cabin: no", | |
| "ov_action": "not documented (9-class driver-action taxonomy; OMZ 2023.0 IR); in-cabin: by intended use, corpus not documented", | |
| "openface3": "300W 4,437; WFLW 10,000; AffectNet 1,000,000; DISFA 27 h; BP4D 192,000 frames; MPIIGaze 213,659; Gaze360 200,000; in-cabin: no", | |
| } | |
| READOUT_NOTE = "learned read-out (LR / HGB / MLP, CV-selected; 979 labels): trainable count pending in the audit" | |
| CTX_NOTE = "; + CockpitWM current-state context as input (the head's 3,502,901 + uncounted frozen perception)" | |
| def f3(x) -> str: | |
| return "--" if pd.isna(x) else f"{float(x):.3f}" | |
| def sd3(x) -> str: | |
| return "--" if pd.isna(x) else f"{float(x):+.3f}" | |
| def verdict_single(lo, hi) -> str: | |
| if pd.isna(lo) or pd.isna(hi): | |
| return "--" | |
| if lo > 0: | |
| return "significantly better (CI excludes 0)" | |
| if hi < 0: | |
| return "significantly worse (CI excludes 0)" | |
| return "no significant difference (CI includes 0)" | |
| def _note_field(notes, key, cast=float): | |
| m = re.search(rf"{key}=([^;]+)", str(notes) if not pd.isna(notes) else "") | |
| try: | |
| return cast(m.group(1)) if m else None | |
| except ValueError: | |
| return None | |
| def _int_or_none(x): | |
| return None if x is None or pd.isna(x) else int(x) | |
| def verdict_multi(mean, sd, n_excl, same_sign, n_seeds) -> str: | |
| """pre-stated multi-seed rule (plan §1b / kill_checks.beats): >= ceil(2n/3) seed CIs exclude 0, all deltas same sign, |mean| > sd over seeds.""" | |
| if pd.isna(mean) or n_seeds is None or n_seeds < 2: | |
| return "--" | |
| need = math.ceil(2 * n_seeds / 3) | |
| ok = (n_excl is not None and n_excl >= need) and bool(same_sign) and (sd is None or pd.isna(sd) or abs(mean) > sd) | |
| if ok and mean > 0: | |
| return f"significantly better ({n_excl}/{n_seeds} seed CIs exclude 0, same sign, |mean Δ| > sd)" | |
| if ok and mean < 0: | |
| return f"significantly worse ({n_excl}/{n_seeds} seed CIs exclude 0, same sign, |mean Δ| > sd)" | |
| return f"no significant difference ({n_excl if n_excl is not None else '?'}/{n_seeds} seed CIs exclude 0{'' if same_sign else ', signs differ'})" | |
| def verdict_word(v: str) -> str: | |
| """'significantly better (...)' -> 'better'; 'significantly worse' -> 'worse'; 'no significant difference' -> 'n.s.'; '--' stays.""" | |
| return "better" if v.startswith("significantly better") else "worse" if v.startswith("significantly worse") else "n.s." if v.startswith("no significant") else v | |
| def _first(df): | |
| return df.iloc[0] if df is not None and not df.empty else None | |
| # ----------------------------------------------------------------------------------------------------------------------------------------------- gold item loading | |
| class GoldItem: | |
| """One gold run dir (warning_gold.csv + optional paired_vs_head*.csv) filtered to `item`, calib raw, seed as str.""" | |
| def __init__(self, gold_dir: Path, item: str, arm: str, paired_csv: Path | None = None): | |
| self.dir, self.item, self.arm = Path(gold_dir), item, arm | |
| raw = pd.read_csv(self.dir / "warning_gold.csv") | |
| raw = raw[(raw["item"] == item) & (raw["calib"].astype(str) == "raw")] | |
| self.raw = raw.assign(seed=raw.seed.astype(str)) | |
| self.cols = sorted(self.raw[self.raw.arm == arm]["event_or_family"].dropna().unique()) | |
| self.seeds = sorted(s for s in self.raw[self.raw.arm == arm].seed.unique() if s != "agg") | |
| self.has_head = bool((self.raw.arm == HEAD_ARM).any()) | |
| self.PH = self.PHP = None | |
| self.paired_csv = self.prauc_csv = None | |
| if paired_csv and Path(paired_csv).exists(): | |
| self.paired_csv = Path(paired_csv) | |
| PH = pd.read_csv(self.paired_csv) | |
| self.PH = PH[(PH["item"] == item) & (PH.arm == arm)].assign(seed=lambda d: d.seed.astype(str)) | |
| pp = self.paired_csv.with_name("paired_vs_head_prauc.csv") | |
| if pp.exists(): | |
| self.prauc_csv = pp | |
| PHP = pd.read_csv(pp) | |
| self.PHP = PHP[(PHP["item"] == item) & (PHP.arm == arm)].assign(seed=lambda d: d.seed.astype(str)) | |
| else: # derive the PR-AUC-keyed aggregate in memory with the writer's own function (identical rows) | |
| sys.path.insert(0, str(Path(__file__).resolve().parent)) | |
| from eval_dms import prauc_agg_rows | |
| self.PHP = prauc_agg_rows(self.PH) | |
| def m(self, metric): | |
| return self.raw[self.raw.metric == metric] | |
| def base(self, metric, c, st, seed): | |
| d = self.m(metric) | |
| return _first(d[(d.arm == self.arm) & (d.event_or_family == c) & (d.stratum == st) & (d.seed == seed) & d["delta"].isna()]) | |
| def att(self, metric, st, seed): | |
| d = self.m(metric) | |
| return _first(d[(d.arm == "legacy_att") & (d.stratum == st) & (d.seed == seed) & d["delta"].isna()]) | |
| def d_att(self, metric, c, st, seed): | |
| d = self.m(metric) | |
| return _first(d[(d.arm == self.arm) & (d.event_or_family == c) & (d.stratum == st) & (d.seed == seed) & d["delta"].notna() | |
| & (d["comparator"].astype(str).str.contains("legacy_att|att@3s", regex=True))]) | |
| def head(self, metric, st, seed): | |
| d = self.m(metric) | |
| return _first(d[(d.arm == HEAD_ARM) & (d.event_or_family == HEAD_COL) & (d.stratum == st) & (d.seed == seed) & d["delta"].isna()]) | |
| def d_head(self, metric, c, st, seed): | |
| P = self.PH if metric == "auroc" else self.PHP | |
| return _first(P[(P.column == c) & (P.stratum == st) & (P.seed == seed)]) if P is not None else None | |
| def cells(self, metric, c, st, seed): | |
| """-> dict(base, att, d_att, r_att, head, d_head, r_head) of printed strings for one metric; None if the baseline row is missing.""" | |
| b = self.base(metric, c, st, seed) | |
| if b is None: | |
| return None | |
| a, p = self.att(metric, st, seed), self.d_att(metric, c, st, seed) | |
| out = {"row": b, "base": f"{f3(b['value'])} [{f3(b['ci_lo'])}, {f3(b['ci_hi'])}]", | |
| "att": f"{f3(a['value'])} [{f3(a['ci_lo'])}, {f3(a['ci_hi'])}]" if a is not None else "--", | |
| "d_att": f"{sd3(p['delta'])} [{sd3(p['delta_ci_lo'])}, {sd3(p['delta_ci_hi'])}]" if p is not None else "--"} | |
| if p is None: | |
| out["r_att"] = "--" | |
| elif seed == "agg": | |
| out["r_att"] = verdict_multi(p["delta"], _note_field(p["notes"], "sd_seed_delta"), _note_field(p["notes"], "n_seeds_ci_excl_0", int), | |
| _note_field(p["notes"], "same_sign_all_seeds", int), len(self.seeds)) | |
| else: | |
| out["r_att"] = verdict_single(p["delta_ci_lo"], p["delta_ci_hi"]) | |
| out.update(head="--", d_head="--", r_head="--") | |
| if self.has_head: | |
| h = self.head(metric, st, seed) | |
| out["head"] = f"{f3(h['value'])} [{f3(h['ci_lo'])}, {f3(h['ci_hi'])}]" if h is not None else "--" | |
| q = self.d_head(metric, c, st, seed) | |
| if q is not None: | |
| stem, sd_key = METRICS[metric][1], METRICS[metric][2] | |
| out["d_head"] = f"{sd3(q[f'delta_{stem}'])} [{sd3(q[f'delta_{stem}_ci_lo'])}, {sd3(q[f'delta_{stem}_ci_hi'])}] (n {int(q['n'])})" | |
| out["r_head"] = (verdict_multi(q[f"delta_{stem}"], q.get(sd_key), _int_or_none(q.get("n_seeds_ci_excl_0")), _int_or_none(q.get("same_sign_all_seeds")), | |
| _int_or_none(q.get("n_seeds")) or len(self.seeds)) | |
| if seed == "agg" else verdict_single(q[f"delta_{stem}_ci_lo"], q[f"delta_{stem}_ci_hi"])) | |
| return out | |
| def sources(self): | |
| return [self.dir / "warning_gold.csv"] + ([self.paired_csv] if self.paired_csv else []) + ([self.prauc_csv] if self.prauc_csv else []) | |
| def newest_gold_dir(item: str, root: Path = GOLD_ROOT) -> Path: | |
| cands = sorted(d for d in root.glob(f"{item}_*") if d.is_dir() and (d / "warning_gold.csv").exists()) | |
| assert cands, f"no gold dir {root}/{item}_* with warning_gold.csv" | |
| return cands[-1] | |
| def select_column(G: GoldItem, selector: str) -> str: | |
| hits = [c for c in G.cols if c == selector] or [c for c in G.cols if selector in c] | |
| assert len(hits) == 1, f"{G.item}: selector {selector!r} matches {hits} among {G.cols}" | |
| return hits[0] | |
| def _rel(p: Path) -> str: | |
| return str(p.relative_to(CWM3)) if p.is_relative_to(CWM3) else str(p) | |
| # ----------------------------------------------------------------------------------------------------------------------------------------------- sections | |
| def gold_section(gold_dir: Path, item: str, arm: str, baseline: str, status: str, paired_csv: Path | None = None, metrics=("auroc",)) -> str: | |
| G = GoldItem(gold_dir, item, arm, paired_csv) | |
| metrics = tuple(metrics) | |
| multi = len(metrics) > 1 # single metric (default auroc) = the historical layout, byte-identical | |
| cols, seeds, has_head, PH = G.cols, G.seeds, G.has_head, G.PH | |
| gold_rel = _rel(Path(gold_dir)) | |
| lines = [f"## item `{item}` — {baseline} (arm `{arm}`) vs the openpilot DMS attention chain{' and the frozen CockpitWM current-state read-out' if has_head else ''} — {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}", "", | |
| f"Status: {status}. Evaluator: `cwm2/eval/warning_gold.py` (unchanged), n_boot as recorded in the CSV; rows = the common-valid subset of this item " | |
| f"(every column finite), so n may be smaller than the 509 of the published head-vs-DMS comparison and the DMS AUROC below is recomputed on the same rows. " | |
| f"Δ = baseline − comparator on identical device-cluster resamples. Seeds {', '.join(seeds)}" | |
| + ("; the `mean` row is warning_gold's aggregate (mean Δ, CI of the seed-median Δ) with the pre-stated verdict rule (>= 2/3 seed CIs exclude 0, same sign, |mean Δ| > sd)." if len(seeds) > 1 else ".") | |
| + (f" Head = `{HEAD_COL}` of the frozen `W_I1_all_minus_road2drv_s<k>_final_dwfc` layouts (read-only) passed to the same gold run; Δ vs head from `paired_vs_head.csv` " | |
| f"(eval_dms.paired_vs_head: warning_gold.cluster_boot on the item's common-valid rows, baseline seed k vs head seed k)." if has_head else "") | |
| + (" Metrics: " + " and ".join(METRICS[m][0] for m in metrics) + "; prevalence = pos / n on the printed rows (the PR-AUC of a chance scorer); PR-AUC Δ vs att@3s from the evaluator's own `pr_auc` rows " | |
| "(same resamples as the AUROC Δ); PR-AUC Δ vs head per seed from `paired_vs_head.csv` (same cluster_boot call as its AUROC Δ), mean row from `paired_vs_head_prauc.csv` " | |
| "(aggregate rule applied to the PR-AUC Δ: CI of the seed-median PR-AUC Δ, sd / n_excl / same-sign from the PR-AUC seeds)." if multi else "") | |
| + f" Source: `{gold_rel}/warning_gold.csv`" + (f", `{gold_rel}/paired_vs_head.csv`" if PH is not None else "") + (f", `{_rel(G.prauc_csv)}`" if multi and G.prauc_csv else "") + ".", ""] | |
| if not multi: | |
| hdr = ("| column | stratum | seed | n / pos | baseline AUROC [CI] | att@3s AUROC (same rows) | Δ vs att@3s [CI] | reading vs att@3s |" | |
| + (" head AUROC (same rows) | Δ vs head [CI] | reading vs head |" if has_head else "")) | |
| sep = "|---|---|---|---|---|---|---|---|" + ("---|---|---|" if has_head else "") | |
| else: | |
| hdr, sep = "| column | stratum | seed | n / pos | prevalence |", "|---|---|---|---|---|" | |
| for m in metrics: | |
| L = METRICS[m][0] | |
| hdr += f" baseline {L} [CI] | att@3s {L} (same rows) | Δ {L} vs att@3s [CI] | reading vs att@3s |" + (f" head {L} (same rows) | Δ {L} vs head [CI] | reading vs head |" if has_head else "") | |
| sep += "---|---|---|---|" + ("---|---|---|" if has_head else "") | |
| lines += [hdr, sep] | |
| for c in cols: | |
| for st, lab in STRATA: | |
| for seed in seeds + (["agg"] if len(seeds) > 1 else []): | |
| first = G.cells(metrics[0], c, st, seed) | |
| if first is None: | |
| continue | |
| b = first["row"] | |
| n = f"{int(b['n'])} / {int(b['n_pos'])}" if "n_pos" in b and not pd.isna(b["n_pos"]) else str(int(b["n"])) | |
| row = f"| `{c}` | {lab} | {'mean' if seed == 'agg' else seed} | {n} |" + (f" {f3(b['base_rate'])} |" if multi else "") | |
| for m in metrics: | |
| k = first if m == metrics[0] else (G.cells(m, c, st, seed) or {}) | |
| row += f" {k.get('base', '--')} | {k.get('att', '--')} | {k.get('d_att', '--')} | {k.get('r_att', '--')} |" | |
| if has_head: | |
| row += f" {k.get('head', '--')} | {k.get('d_head', '--')} | {k.get('r_head', '--')} |" | |
| lines.append(row) | |
| lines.append("") | |
| return "\n".join(lines) + "\n" | |
| def load_consolidated(gold_root: Path = GOLD_ROOT, manifest=CONSOLIDATED): | |
| """-> list of (label, GoldItem, selected column) for every manifest entry whose newest gold dir exists.""" | |
| out = [] | |
| for item, arm, selector, label in manifest: | |
| gd = newest_gold_dir(item, gold_root) | |
| G = GoldItem(gd, item, arm, gd / "paired_vs_head.csv") | |
| out.append((label, G, select_column(G, selector))) | |
| return out | |
| def consolidated_section(entries, metrics=("auroc", "pr_auc")) -> str: | |
| stamp = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()) | |
| srcs = sorted({_rel(p) for _, G, _ in entries for p in G.sources()}) | |
| lines = [f"## Layer 2 consolidated — AUROC and PR-AUC, selected read-out per baseline (tier-A caches) — {stamp}", "", | |
| "One row per baseline arm x stratum: the CV-selected read-out (`*_sel_*` = the (kind, hp) with the best GroupKFold-by-dongle CV AUROC, fit_readout.py; R0 = the fixed rule " | |
| "`dms_lmface_rule0`), `mean` over the seeds listed (warning_gold's `agg` rows: seed-mean value, CI of the seed-median run; Δ = mean Δ, CI of the seed-median Δ; verdict by the " | |
| "pre-stated rule >= 2/3 seed CIs exclude 0, same sign, |mean Δ| > sd; a single-seed item gets the single-seed reading). Rows = each item's common-valid subset " | |
| "(n / pos printed; prevalence = pos / n is the PR-AUC of a chance scorer, so read PR-AUC against it). att@3s = the openpilot attention chain on the same rows; " | |
| f"head = `{HEAD_COL}` of the frozen `W_I1_all_minus_road2drv_s<k>_final_dwfc` layouts. Δ AUROC / Δ PR-AUC vs att@3s: the evaluator's own paired rows (`cwm2/eval/warning_gold.py`, " | |
| "unchanged, identical device-cluster resamples). Δ vs head: `paired_vs_head.csv` (AUROC; eval_dms.paired_vs_head, warning_gold.cluster_boot on the same rows, baseline seed k vs head seed k) " | |
| "and `paired_vs_head_prauc.csv` (PR-AUC: the same per-seed resamples, aggregate rule applied to the PR-AUC Δ). `--` = not available in that gold run (the R0 run of 2026-09-14 " | |
| "had no head layouts). Items whose common-valid set is smaller (MediaPipe: fewer detected faces) are compared with att@3s / head on THEIR rows — read n / pos per row. " | |
| "Nothing recomputed here; every number is copied from the CSVs. Sources: " + ", ".join(f"`{s}`" for s in srcs) + ".", "", | |
| "| baseline | column | stratum | seeds | n / pos (prevalence) |" | |
| + "".join(f" {METRICS[m][0]} [CI] | att@3s {METRICS[m][0]} | Δ {METRICS[m][0]} vs att@3s [CI] | verdict vs att@3s | head {METRICS[m][0]} | Δ {METRICS[m][0]} vs head [CI] | verdict vs head |" for m in metrics), | |
| "|---|---|---|---|---|" + "---|---|---|---|---|---|---|" * len(metrics)] | |
| for label, G, c in entries: | |
| seed = "agg" if len(G.seeds) > 1 else G.seeds[0] | |
| seeds_txt = f"{','.join(G.seeds)} (mean)" if len(G.seeds) > 1 else f"{G.seeds[0]} (single deterministic run)" | |
| for st, lab in STRATA: | |
| first = G.cells(metrics[0], c, st, seed) | |
| if first is None: | |
| continue | |
| b = first["row"] | |
| row = f"| {label} | `{c}` | {lab} | {seeds_txt} | {int(b['n'])} / {int(b['n_pos'])} ({f3(b['base_rate'])}) |" | |
| for m in metrics: | |
| k = first if m == metrics[0] else (G.cells(m, c, st, seed) or {}) | |
| row += f" {k.get('base', '--')} | {k.get('att', '--')} | {k.get('d_att', '--')} | {k.get('r_att', '--')} | {k.get('head', '--')} | {k.get('d_head', '--')} | {k.get('r_head', '--')} |" | |
| lines.append(row) | |
| lines.append("") | |
| return "\n".join(lines) + "\n" | |
| def summary_section(entries, metrics=("auroc", "pr_auc")) -> str: | |
| """owner-facing table: one row per comparator (att@3s, head, every baseline arm), AUROC / PR-AUC per stratum (mean over seeds [CI of the seed-median run]), pooled verdicts, | |
| frozen-parameter and pre-training columns from dms_fairness_audit.md. att@3s / head values are read from the items that share the modal common-valid row set (identical rows, | |
| hence identical values — a range is printed if they ever differ); items on a smaller row set (R0, MediaPipe) are flagged in their own row (their att@3s / head: consolidated section).""" | |
| from collections import Counter | |
| stamp = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()) | |
| fmt = lambda r: f"{f3(r['value'])} [{f3(r['ci_lo'])}, {f3(r['ci_hi'])}]" if r is not None else "--" | |
| seed_of = lambda G: "agg" if len(G.seeds) > 1 else G.seeds[0] | |
| def pooled_n(G): | |
| r = G.att("auroc", "clean_test", seed_of(G)) | |
| return int(r["n"]) if r is not None else -1 | |
| tierA = [e for e in entries if e[1].has_head] | |
| n_mode = Counter(pooled_n(G) for _, G, _ in tierA).most_common(1)[0][0] | |
| refs = [e for e in tierA if pooled_n(e[1]) == n_mode] | |
| ref = refs[0][1] | |
| def ref_cell(getter, metric, st): | |
| vals = [v for v in (getter(G, metric, st, seed_of(G)) for _, G, _ in refs) if v is not None] | |
| if not vals: | |
| return "--" | |
| strs = sorted({fmt(v) for v in vals}) | |
| return strs[0] if len(strs) == 1 else f"{f3(min(float(v['value']) for v in vals))}–{f3(max(float(v['value']) for v in vals))} (differs across items)" | |
| prev = {lab: (int(r["n_pos"]), int(r["n"]), f3(r["base_rate"])) for st, lab in STRATA for r in [ref.att("auroc", st, seed_of(ref))] if r is not None} | |
| off = [lab for lab, G, _ in entries if (lab, G, _) not in refs] | |
| lines = [f"## Layer 2 summary — AUROC and PR-AUC per stratum, every baseline vs the openpilot attention chain and the frozen CockpitWM current-state read-out — {stamp}", "", | |
| "Values = mean over seeds 0,1,2 [CI of the seed-median run] from `cwm2/eval/warning_gold.py` (unchanged) on each item's common-valid rows. Shared row set of the reference items " | |
| f"({len(refs)} of {len(entries)} arms): " + "; ".join(f"{lab} n {n} / pos {p} → prevalence {pr}" for lab, (p, n, pr) in prev.items()) | |
| + "; the att@3s and head rows below are read on that set (identical on every reference item). Arms on a smaller common-valid set are flagged \"own rows\" in their row " | |
| "(their att@3s / head values on those rows, and every per-stratum Δ [CI], are in the \"Layer 2 consolidated\" section of `results/tables/dms_warning_comparison.md`). " | |
| "Prevalence = pos / n = the PR-AUC of a chance scorer (AUROC chance = 0.5). Verdicts (pooled stratum, pre-stated rule >= 2/3 seed CIs exclude 0, same sign, |mean Δ| > sd; " | |
| "R0 single-seed: CI excludes / includes 0): better = significantly better, worse = significantly worse, n.s. = no significant difference; printed as AUROC / PR-AUC. " | |
| "Parameter and pre-training columns are copied from `results/tables/dms_fairness_audit.md` (rows 1–9; \"not documented\" is never replaced by an estimate; the learned read-outs' " | |
| "trainable counts are still pending there).", "", | |
| "| comparator | parameters (frozen perception / trainable) | pre-training / training data | " | |
| + " | ".join(f"{METRICS[m][0]} {lab}" for m in metrics for _, lab in STRATA) + " | pooled verdict vs att@3s (AUROC / PR-AUC) | pooled verdict vs head (AUROC / PR-AUC) |", | |
| "|---|---|---|" + "---|" * (len(metrics) * len(STRATA)) + "---|---|"] | |
| lines.append(f"| openpilot attention chain `att@3s` (the logged monitor fields) | {FROZEN_PARAMS['att']} | {PRETRAIN['att']} | " | |
| + " | ".join(ref_cell(lambda G, m, st, s: G.att(m, st, s), m, st) for m in metrics for st, _ in STRATA) + " | — | — |") | |
| lines.append(f"| CockpitWM current-state read-out `{HEAD_COL}` (frozen head) | {FROZEN_PARAMS['head']} | {PRETRAIN['head']} | " | |
| + " | ".join(ref_cell(lambda G, m, st, s: G.head(m, st, s), m, st) for m in metrics for st, _ in STRATA) + " | — | — |") | |
| for label, G, c in entries: | |
| key = "r0" if G.item == "dms_r0" else G.item.replace("dms_", "").rsplit("_", 1)[0] | |
| ctx = G.item.endswith("_ctx") | |
| params = FROZEN_PARAMS[key] + ("" if key == "r0" else "; " + READOUT_NOTE) + (CTX_NOTE if ctx else "") | |
| seed = seed_of(G) | |
| cells = {(m, st): G.cells(m, c, st, seed) for m in metrics for st, _ in STRATA} | |
| vals = " | ".join((cells[(m, st)] or {}).get("base", "--") for m in metrics for st, _ in STRATA) | |
| pooled = [cells[(m, "clean_test")] or {} for m in metrics] | |
| v_att = " / ".join(verdict_word(k.get("r_att", "--")) for k in pooled) | |
| v_head = " / ".join(verdict_word(k.get("r_head", "--")) for k in pooled) if G.has_head else "— (head not in this gold run)" | |
| extra = "" | |
| if label in off: | |
| rows = [(lab, cells[(metrics[0], st)]) for st, lab in STRATA] | |
| extra = " (own rows: " + "; ".join(f"{lab} n {int(k['row']['n'])} / pos {int(k['row']['n_pos'])}, prevalence {f3(k['row']['base_rate'])}" for lab, k in rows if k) + ")" | |
| lines.append(f"| {label}{extra} (`{c}`) | {params} | {PRETRAIN[key]} | {vals} | {v_att} | {v_head} |") | |
| lines.append("") | |
| return "\n".join(lines) + "\n" | |
| def op_section(op_dir: Path, item: str, baseline: str, status: str) -> str: | |
| op = pd.read_csv(op_dir / "operating_points.csv") | |
| pl = pd.read_csv(op_dir / "paired_lead.csv") | |
| rel = pd.read_csv(op_dir / "reliability.csv") | |
| lines = [f"## item `{item}` — {baseline} — {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}", "", | |
| f"Status: {status}. Evaluator: `eval/operating_points.py` (unchanged): thresholds bisected on the full test split to the load, 60 s per-route cooldown; " | |
| f"recall / precision on the human gold at that fixed threshold (shares of the enriched gold set, not natural rates). Source: `{_rel(op_dir)}/`.", "", | |
| "| load | regime | score | alerts/h achieved | n gold / pos | recall [CI] | precision [CI] | lead median s |", "|---|---|---|---|---|---|---|---|"] | |
| for _, r in op.iterrows(): | |
| aph = r["alerts_per_hour_manual"] if r["regime"] == "manual" else (r["alerts_per_hour_engaged"] if r["regime"] == "engaged" else float("nan")) | |
| lead = r["lead_median_s"] if "lead_median_s" in r else float("nan") | |
| lines.append(f"| {r['load']} | {r['regime']} | {r['score']} | {f3(aph)} | {int(r['n_gold'])} / {int(r['n_pos'])} | {f3(r['recall'])} [{f3(r['recall_ci_lo'])}, {f3(r['recall_ci_hi'])}] | " | |
| f"{f3(r['precision'])} [{f3(r['precision_ci_lo'])}, {f3(r['precision_ci_hi'])}] | {f3(lead)} |") | |
| lines += ["", "Paired lead time (baseline − DMS, undetected → 0) and verdict by the pre-stated rule:", "", | |
| "| load | regime | n both detected | Δ lead mean s [CI] | verdict |", "|---|---|---|---|---|"] | |
| for _, r in pl.iterrows(): | |
| lines.append(f"| {r['load']} | {r['regime']} | {int(r['n_both_detected'])} | {sd3(r['delta_lead0_mean'])} [{sd3(r['delta_lead0_ci_lo'])}, {sd3(r['delta_lead0_ci_hi'])}] | {str(r['verdict']).strip()} |") | |
| e = rel.dropna(subset=["ece_cal"]).drop_duplicates(subset=["score"]) | |
| if not e.empty: | |
| lines += ["", "Calibration (temperature fitted on clean-val; 10 equal-count bins):", "", "| score | T | ECE raw | ECE calibrated [CI] |", "|---|---|---|---|"] | |
| for _, r in e.iterrows(): | |
| lines.append(f"| {r['score']} | {f3(r['T'])} | {f3(r['ece_raw'])} | {f3(r['ece_cal'])} [{f3(r['ece_cal_ci_lo'])}, {f3(r['ece_cal_ci_hi'])}] |") | |
| lines.append("") | |
| return "\n".join(lines) + "\n" | |
| def append(table: Path, section: str, header: str) -> None: | |
| if not table.exists(): | |
| table.write_text(header + "\n") | |
| with open(table, "a", encoding="utf-8") as fh: | |
| fh.write("\n" + section) | |
| def _num_forms(blob: str) -> set[str]: | |
| """every number in the source text (floats AND bare integers — results_schema.fmt writes 1.0 as `1`) at 3 dp, signed and unsigned.""" | |
| S = set() | |
| for m in re.finditer(r"(?<![\w.])[-+]?\d+(?:\.\d+)?(?![\w.])", blob): | |
| try: | |
| v = float(m.group()) | |
| except ValueError: | |
| continue | |
| S.add(f"{v:.3f}") | |
| S.add(f"{v:+.3f}") | |
| return S | |
| def verify(table: Path, csvs: list[Path]) -> int: | |
| """every 3-dp numeric token printed in the NEWEST section of `table` must round-match a number in one of the source files (same rule as before; set-based, one pass over the sources).""" | |
| text = table.read_text(encoding="utf-8") | |
| last = text.split("\n## ")[-1] | |
| S = set() | |
| for p in csvs: | |
| if Path(p).exists(): | |
| S |= _num_forms(Path(p).read_text(encoding="utf-8")) | |
| nums = sorted({m for m in re.findall(r"(?<![\w.])[-+]?\d+\.\d{3}(?![\w.])", last)}) | |
| missing = [tok for tok in nums if not (f"{float(tok):.3f}" in S or tok in S)] | |
| print(f"verify {table.name}: {len(nums)} numeric tokens in the newest section, {len(missing)} not reproducible from the CSVs {missing[:10]}") | |
| return len(missing) | |
| def main(argv=None) -> int: | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--gold-item") | |
| ap.add_argument("--gold-dir") | |
| ap.add_argument("--op-item") | |
| ap.add_argument("--op-dir", help="results/operating_points/<item>_<stamp>; default = newest dir starting with <op-item>_") | |
| ap.add_argument("--arm", default="", help="required with --gold-item / --op-item") | |
| ap.add_argument("--baseline", default="", help="required with --gold-item / --op-item") | |
| ap.add_argument("--status", default="adapted") | |
| ap.add_argument("--verify", action="store_true") | |
| ap.add_argument("--paired-head-csv", default="", help="eval_dms.paired_vs_head output (Δ vs the frozen CockpitWM read-out) rendered as extra columns") | |
| ap.add_argument("--metrics", default="auroc", help="comma list of warning_gold metrics to print in a gold section: auroc (default, historical layout) or auroc,pr_auc") | |
| ap.add_argument("--consolidated", action="store_true", help="append the ONE 'Layer 2 consolidated' section (newest gold dir per CONSOLIDATED item, selected read-out, AUROC + PR-AUC)") | |
| ap.add_argument("--summary", action="store_true", help="write / append the owner-facing results/tables/dms_layer2_summary.md") | |
| ap.add_argument("--gold-root", default="", help="root of the gold run dirs for --consolidated / --summary (default results/warning_gold)") | |
| ap.add_argument("--table-dir", default="", help="directory of dms_warning_comparison.md / dms_operating_points.md / dms_layer2_summary.md (default results/tables; dry runs point elsewhere)") | |
| a = ap.parse_args(argv) | |
| rc = 0 | |
| metrics = tuple(m.strip() for m in a.metrics.split(",") if m.strip()) | |
| assert all(m in METRICS for m in metrics), f"--metrics must be from {list(METRICS)}" | |
| tdir = Path(a.table_dir).resolve() if a.table_dir else TABLES | |
| tdir.mkdir(parents=True, exist_ok=True) | |
| gold_table, op_table, summary_table = tdir / GOLD_TABLE.name, tdir / OP_TABLE.name, tdir / SUMMARY_TABLE.name | |
| gold_root = Path(a.gold_root).resolve() if a.gold_root else GOLD_ROOT | |
| gold_header = "# DMS baselines vs the openpilot attention chain on the human gold — sections appended per item (baselines_dms/report_dms.py; numbers copied from the evaluators' CSVs)\n" | |
| if a.gold_item or a.op_item: | |
| assert a.arm and a.baseline, "--arm and --baseline are required with --gold-item / --op-item" | |
| if a.gold_item: | |
| gd = (Path(a.gold_dir).resolve() if a.gold_dir else (CWM3 / "results" / "warning_gold" / a.gold_item)) | |
| pc = Path(a.paired_head_csv).resolve() if a.paired_head_csv else None | |
| sec = gold_section(gd, a.gold_item, a.arm, a.baseline, a.status, pc, metrics) | |
| append(gold_table, sec, gold_header) | |
| print(sec) | |
| if a.verify: | |
| rc += verify(gold_table, [gd / "warning_gold.csv"] + ([pc, pc.with_name("paired_vs_head_prauc.csv")] if pc else [])) | |
| if a.op_item: | |
| if a.op_dir: | |
| od = Path(a.op_dir).resolve() | |
| else: | |
| cands = sorted((CWM3 / "results" / "operating_points").glob(f"{a.op_item}_*")) | |
| assert cands, f"no operating_points dir for item {a.op_item}" | |
| od = cands[-1] | |
| sec = op_section(od, a.op_item, a.baseline, a.status) | |
| append(op_table, sec, "# DMS baselines — operating points at equal alert load (baselines_dms/report_dms.py; numbers copied from eval/operating_points.py CSVs)\n") | |
| print(sec) | |
| if a.verify: | |
| rc += verify(op_table, [od / "operating_points.csv", od / "paired_lead.csv", od / "reliability.csv"]) | |
| if a.consolidated or a.summary: | |
| entries = load_consolidated(gold_root) | |
| srcs = sorted({p for _, G, _ in entries for p in G.sources()}) | |
| if a.consolidated: | |
| sec = consolidated_section(entries, ("auroc", "pr_auc")) | |
| append(gold_table, sec, gold_header) | |
| print(sec) | |
| if a.verify: | |
| rc += verify(gold_table, srcs) | |
| if a.summary: | |
| sec = summary_section(entries, ("auroc", "pr_auc")) | |
| append(summary_table, sec, "# DMS layer 2 — owner summary (baselines_dms/report_dms.py --summary; append-only; numbers copied from the evaluators' CSVs, parameter / pre-training columns from dms_fairness_audit.md)\n") | |
| print(sec) | |
| if a.verify: | |
| rc += verify(summary_table, srcs + [FAIRNESS_TABLE]) | |
| return 1 if rc else 0 | |
| if __name__ == "__main__": | |
| sys.exit(main()) | |
Xet Storage Details
- Size:
- 39 kB
- Xet hash:
- 6cc56fb3c73e82894c3859e4d66fad6f1e9906df36b3b5be592c5f61ce87910e
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.