Buckets:
| """cwm3/eval/warning_head_pair_summary.py — compact dated summary of ONE paired warning-head effect (plan R2 WP2.1): levels per seed, paired Δ [CI] per seed, | |
| mean ± sd over seeds, and the verdict under the pre-registered rules (paired device-cluster CI = primary evidence; the '≥ 2/3 seeds CI excl. 0 in engaged OR pooled' | |
| count = stability description; both reported). Reads ONLY published outputs: results/warning_gold/warning_gold.csv (levels of the item's latest run) and the | |
| between-column pair CSV written by eval/warning_head_regime_report.py (results/warning_head_ext/regime_pairs_<item>_<tag>.csv; identical rows / resamples). | |
| python eval/warning_head_pair_summary.py --item cwm3_final_dwfcboth --effect both_minus_cur_dwfc --label "both+dwell+face − cur+dwell+face (in-sample)" [--out results/tables/warning_head_final_ext.md] | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import time | |
| from pathlib import Path | |
| import numpy as np | |
| import pandas as pd | |
| CWM3 = Path(__file__).resolve().parents[1] | |
| STRATA = ("clean_test", "clean_test|manual", "clean_test|engaged") | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--item", required=True) | |
| ap.add_argument("--effect", required=True, help="effect name given to warning_head_regime_report.py --pairs") | |
| ap.add_argument("--label", default="") | |
| ap.add_argument("--out", default=str(CWM3 / "results" / "tables" / "warning_head_final_ext.md")) | |
| a = ap.parse_args() | |
| pair_files = sorted((CWM3 / "results" / "warning_head_ext").glob(f"regime_pairs_{a.item}_*.csv")) | |
| assert pair_files, f"no regime_pairs CSV for item {a.item}" | |
| P = pd.read_csv(pair_files[-1]) | |
| P = P[P.effect == a.effect] | |
| assert len(P), f"effect {a.effect} not in {pair_files[-1].name}" | |
| W = pd.read_csv(CWM3 / "results" / "warning_gold" / "warning_gold.csv", low_memory=False) | |
| W = W[(W.item == a.item)] | |
| W = W[W.run_stamp == W.run_stamp.max()] | |
| seeds = sorted(int(s) for s in P.seed.unique()) | |
| col_a, col_b = P.col_a.iloc[0], P.col_b.iloc[0] | |
| stamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) | |
| L = [f"## {stamp} — paired effect `{a.effect}`: {a.label or col_a + ' − ' + col_b} — item `{a.item}` (eval/warning_head_pair_summary.py; pair CSV {pair_files[-1].name}; warning_gold run {W.run_stamp.max()})", "", | |
| f"A = `{col_a}`, B = `{col_b}`. Levels = clean-test AUROC (raw) from `warning_gold.csv`; Δ = A − B on identical rows and device-cluster resamples (regime report §3). " | |
| "Verdict rule (plan R2 判据): the paired CI is the primary evidence — **EFFECT** only if the paired Δ CI excludes 0 in the same direction in every seed; the seed count " | |
| "'≥ 2/3 seeds CI excl. 0 in engaged OR pooled' (the regime report's kill rule) is reported as a stability description, not as the criterion.", "", | |
| "| stratum | " + " | ".join(f"s{s} A / B" for s in seeds) + " | A mean ± sd | B mean ± sd | " + " | ".join(f"s{s} Δ [CI]" for s in seeds) + " | Δ mean ± sd | seeds CI excl. 0 | verdict |", | |
| "|---|" + "---|" * len(seeds) + "---|---|" + "---|" * len(seeds) + "---|---|---|"] | |
| verdicts = {} | |
| for st in STRATA: | |
| g = P[P.stratum == st].sort_values("seed") | |
| if not len(g): | |
| continue | |
| lv = [] | |
| for s in seeds: | |
| x = g[g.seed == s] | |
| lv.append(f"{float(x.auroc_a.iloc[0]):.3f} / {float(x.auroc_b.iloc[0]):.3f}" if len(x) else "—") | |
| d = g.delta.to_numpy(float) | |
| excl_pos = int((g.delta_ci_lo > 0).sum()) | |
| excl_neg = int((g.delta_ci_hi < 0).sum()) | |
| excl = excl_pos + excl_neg | |
| if excl_pos == len(g): | |
| v = "EFFECT (+)" | |
| elif excl_neg == len(g): | |
| v = "EFFECT (−)" | |
| elif excl == 0: | |
| v = "no paired evidence (every CI ⊃ 0)" | |
| else: | |
| v = f"unstable ({excl_pos} seeds CI > 0, {excl_neg} seeds CI < 0)" | |
| verdicts[st] = v | |
| L.append(f"| {st} | " + " | ".join(lv) + f" | {g.auroc_a.mean():.3f} ± {g.auroc_a.std(ddof=1) if len(g) > 1 else 0:.3f} | {g.auroc_b.mean():.3f} ± {g.auroc_b.std(ddof=1) if len(g) > 1 else 0:.3f} | " | |
| + " | ".join(f"{x.delta:+.3f} [{x.delta_ci_lo:+.3f},{x.delta_ci_hi:+.3f}]" for x in g.itertuples()) + f" | {d.mean():+.3f} ± {d.std(ddof=1) if len(d) > 1 else 0:.3f} | {excl}/{len(g)} | {v} |") | |
| kill = any(int((P[(P.stratum == st)].delta_ci_lo > 0).sum()) >= 2 and P[(P.stratum == st)].delta.mean() > 0 for st in ("clean_test", "clean_test|engaged")) | |
| L += ["", f"Pre-registered kill rule of the regime report (CI excl. 0 in ≥ 2/3 seeds, positive, in engaged OR pooled): **{'met' if kill else 'NOT met'}**. " | |
| f"Primary (paired-CI) verdict per stratum: pooled **{verdicts.get('clean_test', '—')}**, manual **{verdicts.get('clean_test|manual', '—')}**, engaged **{verdicts.get('clean_test|engaged', '—')}**.", ""] | |
| md = "\n".join(L) + "\n" | |
| with open(a.out, "a") as f: | |
| f.write(md) | |
| print(md) | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 5.06 kB
- Xet hash:
- 3cd3c67f28fdf1e5527c792956d8cab8226a97678a0d6839dac7892b70eb325b
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.