Buckets:
| """cwm3/eval/report_v_stats2.py — results/tables/v0_stats2.md: V base stats2 vs stats1 (roadmap §三 6) from results/paired_compare.csv tags V0s2_s<k>_stats2_vs_stats1. | |
| python eval/report_v_stats2.py [--seeds 0,1,2] [--out results/tables/v0_stats2.md] | |
| Rule: manual veh_onset brake_on / gas_on / steer_phase_on @3s must not be WORSE (kill_checks.beats) — else the Exp2 arms are re-rendered with stats2. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import sys | |
| from pathlib import Path | |
| CWM3 = Path(__file__).resolve().parents[1] | |
| sys.path.insert(0, str(CWM3 / "eval")) | |
| import report_joint as RJ # noqa: E402 | |
| RULE_CELLS = [("manual", f"veh_onset|{c}|3s") for c in ("brake_on", "gas_on", "steer_phase_on")] | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--seeds", default="0,1,2") | |
| ap.add_argument("--out", default=str(CWM3 / "results" / "tables" / "v0_stats2.md")) | |
| a = ap.parse_args() | |
| seeds = [int(s) for s in a.seeds.split(",")] | |
| df = RJ.load("V0s2", "stats2", "stats1", seeds) | |
| assert len(df), "no V0s2_* rows" | |
| body, verdicts = RJ.render(df, sorted(df.seed.unique()), "V0_can4s_s<k>_stats2", "V0_can4s_s<k>", "V base: stats2 (float64 statistics) vs stats1 (pre-fix float32), identical rows") | |
| prov, stale = RJ.provenance(df) | |
| rv = {c: verdicts.get((r, c, "pr_auc"), {}).get("verdict", "MISSING") for r, c in RULE_CELLS} | |
| ok = all(v not in ("WORSE", "MISSING", "INSUFFICIENT") for v in rv.values()) and not stale | |
| worse_any = [f"{r}|{c}|{m}" for (r, c, m), v in verdicts.items() if v["verdict"] == "WORSE"] | |
| beats_any = [f"{r}|{c}|{m}" for (r, c, m), v in verdicts.items() if v["verdict"] == "BEATS"] | |
| lines = ["# V base — `V0/can4s_stats2` (float64 statistics, stats_version 2) vs `V0/can4s` (pre-fix float32 statistics), 3 seeds, test split, both regimes", "", | |
| "The `can4s` recipe verbatim; only the standardisation statistics differ (train/batch.compute_stats v2; results/tables/stats_fix.md). Paired device-cluster bootstrap on identical rows " | |
| "(eval/paired_compare.py, n_boot 1000); verdict = kill_checks.beats. Source: `results/paired_compare.csv` tags `V0s2_s<k>_stats2_vs_stats1`.", "", | |
| "## Provenance of the A / B layouts", "", "| run | regime | layout mtime (UTC) | source sha256 | model_id / code | status |", "|---|---|---|---|---|---|", *prov, "", | |
| *([f"**STALE ROWS for {stale} — VOID**", ""] if stale else []), *body, "", | |
| "## Verdict (roadmap §三 6: manual brake / gas / steer_phase onset PR-AUC@3s must not be WORSE)", "", | |
| *[f"- manual {c.split('|')[1]}: **{v}**" for (r, c), v in zip(RULE_CELLS, rv.values())], | |
| f"- any cell WORSE: {worse_any or 'none'}; any cell BEATS: {beats_any or 'none'}", "", | |
| f"**Rule verdict: {'PASS — stats2 V base adopted as-is' if ok else 'FAIL — stats2 is used anyway (canonical SD units) and the Exp2 arms must be re-rendered with stats2 (roadmap §三 6)'}**", ""] | |
| Path(a.out).write_text("\n".join(lines) + "\n") | |
| (Path(a.out).parent / "v0_stats2_decision.json").write_text(json.dumps({"pass": ok, "rule_cells": rv, "worse": worse_any, "beats": beats_any, "stale": stale}, indent=1)) | |
| print("\n".join(lines)); print(f"written {a.out}") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 3.38 kB
- Xet hash:
- e8a73cdea80d134fa3c893ad56462063b7d6f3685d33340923199e6f0542b5a0
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.