Buckets:
| """cwm3/eval/plot_reliability.py — reliability diagram of the equal-alert-load study (eval/operating_points.py reliability.csv -> PNG in the same directory). | |
| Form: calibration curve (mean predicted probability per equal-count bin vs observed positive rate), one line per score (head = categorical slot 1 blue, | |
| DMS att@3s = slot 2 orange; validated with the data-viz palette validator, light surface), the identity diagonal as a recessive hairline, 2 px lines, 8 px | |
| markers with a 2 px surface ring, hairline gridlines, legend present (2 series) carrying the ECE, text in text tokens (never the series colour). One panel per | |
| seed (small multiples; the head's current-state features are seed-independent, the DMS chain is not). Bin rows come from reliability.csv (bin >= 0); the | |
| ECE rows are bin == -1. python eval/plot_reliability.py --dir results/operating_points/oof_<stamp> [--label "HGB(current-state), OOF"] | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| from pathlib import Path | |
| import matplotlib | |
| matplotlib.use("Agg") | |
| import matplotlib.pyplot as plt # noqa: E402 | |
| import pandas as pd # noqa: E402 | |
| SURFACE, TEXT1, TEXT2, GRID = "#fcfcfb", "#0b0b0b", "#52514e", "#e6e5e1" | |
| SERIES = {"head": "#2a78d6", "dms": "#eb6834"} # categorical slots 1 / 2 (references/palette.md), fixed order | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--dir", required=True) | |
| ap.add_argument("--label", default="HGB(current-state), out-of-fold") | |
| a = ap.parse_args() | |
| d = Path(a.dir) | |
| R = pd.read_csv(d / "reliability.csv") | |
| seeds = sorted(R.seed.unique()) | |
| fig, axes = plt.subplots(1, len(seeds), figsize=(4.2 * len(seeds), 4.2), sharey=True, facecolor=SURFACE) | |
| axes = [axes] if len(seeds) == 1 else list(axes) | |
| names = {"head": a.label, "dms": "DMS att@3s"} | |
| for ax, seed in zip(axes, seeds): | |
| ax.set_facecolor(SURFACE) | |
| ax.plot([0, 1], [0, 1], color=GRID, lw=1, zorder=1) | |
| for s in ("head", "dms"): | |
| g = R[(R.seed == seed) & (R.score == s)] | |
| b, tot = g[g.bin >= 0].sort_values("bin"), g[g.bin == -1].iloc[0] | |
| ax.plot(b.p_mean, b.y_rate, color=SERIES[s], lw=2, solid_joinstyle="round", solid_capstyle="round", zorder=2, | |
| label=f"{names[s]} — ECE {tot.ece_cal:.3f} (T {tot['T']:.2f})") | |
| ax.scatter(b.p_mean, b.y_rate, s=64, color=SERIES[s], edgecolor=SURFACE, linewidth=2, zorder=3) | |
| ax.set_xlim(0, 1) | |
| ax.set_ylim(0, 1) | |
| ax.set_xlabel("mean calibrated probability (10 equal-count bins)", color=TEXT2, fontsize=9) | |
| ax.set_title(f"seed {seed}", color=TEXT1, fontsize=10, loc="left") | |
| ax.grid(True, color=GRID, lw=1) | |
| for sp in ("top", "right"): | |
| ax.spines[sp].set_visible(False) | |
| for sp in ("left", "bottom"): | |
| ax.spines[sp].set_color(GRID) | |
| ax.tick_params(colors=TEXT2, labelsize=8) | |
| ax.legend(loc="upper left", frameon=False, fontsize=8, labelcolor=TEXT1) | |
| axes[0].set_ylabel("observed positive rate (clean-test gold)", color=TEXT2, fontsize=9) | |
| fig.suptitle("Reliability on the clean human gold — temperature fitted on clean-val", color=TEXT1, fontsize=11, x=0.01, ha="left") | |
| fig.tight_layout() | |
| out = d / "reliability.png" | |
| fig.savefig(out, dpi=150, facecolor=SURFACE) | |
| print(f"written {out}") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 3.39 kB
- Xet hash:
- c4bdbbff3c310e2162f260400fa82290fe32cf0bf2433aa0dffd1d1bb6ab2571
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.