PULSE-code / figures /f5 /plot_f5.py
velvet-pine-22's picture
PULSE-code: benchmark code for the PULSE dataset (camera-ready release)
dffbc1f verified
Raw History Blame Contribute Delete
7.35 kB
#!/usr/bin/env python
"""F5 data-fraction curve and the fraction x condition table, from the aggregated table.
python scripts/aggregate.py --table 5 # -> tables/tab5.json (+ tab5*.tex, numbers.tex)
python figures/f5/plot_f5.py --table 5 --tables tables --out figures/out
Reads ``<tables>/tab<N>.json`` written by the aggregator (rows ``scratch_fXXX``
and ``s9pre_fXXX``, each pre-trained row paired with the scratch row of the
same fraction on the common test set). Writes ``f5_data_fraction_tab<N>.{pdf,png}``
(mean +- SD over the 5 fold means; each fold = mean over seeds) and
``<tables>/tab<N>_fraction.tex`` (Fraction | Scratch | S9 pre-trained | Delta |
positive folds | p (n=5) | p (n=15)).
"""
from __future__ import annotations
import argparse
import json
import re
import sys
from pathlib import Path
REPO = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(REPO))
import matplotlib # noqa: E402
matplotlib.use("Agg")
import matplotlib.pyplot as plt # noqa: E402
from pulse.stats import format_p, format_pm # noqa: E402
SCRATCH, PRE = "#2a78d6", "#eb6834" # categorical slots 1, 2
INK, INK2, MUTED, GRID, AXIS = "#0b0b0b", "#52514e", "#898781", "#e1e0d9", "#c3c2b7"
ROW_RE = re.compile(r"^(scratch|s9pre)_f(\d{3})$")
def style():
plt.rcParams.update({
"pdf.fonttype": 42, "ps.fonttype": 42, "font.size": 9, "axes.titlesize": 9,
"axes.labelsize": 9, "legend.fontsize": 8, "xtick.labelsize": 8, "ytick.labelsize": 8,
"axes.edgecolor": AXIS, "axes.labelcolor": INK2, "xtick.color": MUTED, "ytick.color": MUTED,
"text.color": INK, "axes.grid": True, "grid.color": GRID, "grid.linewidth": 0.6,
"axes.spines.top": False, "axes.spines.right": False, "legend.frameon": False,
})
def collect(summary: dict, metric: str):
by = {}
for key, r in summary["rows"].items():
m = ROW_RE.match(key)
if not m:
continue
cond, frac = m.group(1), int(m.group(2))
st = r["common"][metric]
if not st["n_runs"]:
continue # partial table: row without runs
by.setdefault(frac, {})[cond] = {"mean": st["mean"], "sd": st["sd"], "row": key,
"label": r["label"], "vs": r.get("vs_reference", {}).get(metric),
"n_folds": st["n_folds"], "n_runs": st["n_runs"]}
fr = sorted(f for f, d in by.items() if {"scratch", "s9pre"} <= set(d))
if not fr:
raise SystemExit("no fraction has both a scratch and an s9pre row")
return fr, by
def _z(x):
"""SD for drawing: NaN (a single fold) -> 0."""
return x if x is not None and x == x else 0.0
def plot(summary, metric, label, fr, by, out: Path, tag: str):
fig, ax = plt.subplots(figsize=(4.2, 2.8))
# evenly spaced fractions (5 % and 10 % would collide on a linear axis)
pos = {f: i for i, f in enumerate(fr)}
for cond, color, name, dx in (("scratch", SCRATCH, "Random init", -0.06), ("s9pre", PRE, "S9 pre-trained", 0.06)):
xs = [pos[f] + dx for f in fr]
ys = [by[f][cond]["mean"] for f in fr]
es = [_z(by[f][cond]["sd"]) for f in fr]
ax.errorbar(xs, ys, yerr=es, color=color, lw=1.6, marker="o", ms=5, mec="white", mew=1.0,
capsize=2.5, elinewidth=0.9, label=name)
for f in fr:
c = by[f]["s9pre"]["vs"]
if not c:
continue
top = max(by[f][k]["mean"] + _z(by[f][k]["sd"]) for k in ("scratch", "s9pre"))
p15 = (c.get("t15") or {}).get("p")
ax.text(pos[f], top + 0.004, f"{c['delta_mean']:+.3f}\n{c['positive_folds']}/{len(c['folds'])} folds",
ha="center", va="bottom", fontsize=6.5, color=INK2 if (p15 or 1) >= 0.05 else INK)
ax.set_xticks([pos[f] for f in fr])
ax.set_xticklabels([f"{f}%" for f in fr])
ax.set_xlabel("share of labelled T2 training segments")
ax.set_ylabel(label)
lo = min(by[f][k]["mean"] - _z(by[f][k]["sd"]) for f in fr for k in ("scratch", "s9pre"))
hi = max(by[f][k]["mean"] + _z(by[f][k]["sd"]) for f in fr for k in ("scratch", "s9pre"))
ax.set_ylim(lo - 0.01, hi + 0.06)
ax.set_xlim(-0.5, len(fr) - 0.5)
seeds = summary["seeds"]
ax.set_title(f"DAF-v2 (EMG+IMU), {len(summary['folds'])}-fold CV x {len(seeds)} seed"
f"{'s' if len(seeds) > 1 else ''}" + ("" if summary["complete"] else " [INCOMPLETE]"),
loc="left")
ax.legend(loc="lower right")
fig.tight_layout()
for ext in ("pdf", "png"):
fig.savefig(out / f"f5_data_fraction_tab{tag}.{ext}", dpi=200)
plt.close(fig)
def table(summary, metric, label, fr, by, digits=3) -> str:
lines = [f"% Generated by figures/f5/plot_f5.py from tab{summary['table']}.json -- do not edit.",
f"% {summary['title']}; metric {metric} on the common test set; mean +- SD (ddof="
f"{summary['ddof']}) over fold means; Delta = S9 pre-trained - scratch (paired by fold "
f"and by fold x seed).",
f"% git {summary['provenance'].get('git_sha')} data {summary['provenance'].get('data_tag')}"]
if not summary["complete"]:
lines.append(f"% INCOMPLETE: {len(summary['missing_runs'])} run(s) missing")
lines += [r"\begin{tabular}{lcccccc}", r"\toprule",
rf"Train fraction & Scratch & S9 pre-trained & $\Delta$ {label} & pos. folds & $p_{{5}}$ & $p_{{15}}$ \\",
r"\midrule"]
for f in fr:
s, p = by[f]["scratch"], by[f]["s9pre"]
c = p["vs"] or {}
t15 = c.get("t15")
lines.append(" & ".join([
rf"{f}\%", format_pm(s["mean"], s["sd"], digits), format_pm(p["mean"], p["sd"], digits),
format_pm(c["delta_mean"], c["delta_sd"], digits) if c else "--",
f"{c['positive_folds']}/{len(c['folds'])}" if c else "--",
format_p(c["t5"]["p"]) if c else "--", format_p(t15["p"]) if t15 else "--"]) + r" \\")
lines += [r"\bottomrule", r"\end{tabular}", ""]
return "\n".join(lines)
def main(argv=None) -> int:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--table", default="5", help="5 (sliding, primary) or 5seg")
ap.add_argument("--tables", default="tables", help="directory with tab<N>.json")
ap.add_argument("--metric", default=None, help="default: the table's first metric")
ap.add_argument("--out", default="figures/out")
args = ap.parse_args(argv)
src = Path(args.tables) / f"tab{args.table}.json"
summary = json.loads(src.read_text(encoding="utf-8"))
if summary.get("task") != "f5":
raise SystemExit(f"{src} is not an F5 table")
metric = args.metric or summary["metrics"][0]["key"]
label = next(m["label"] for m in summary["metrics"] if m["key"] == metric)
fr, by = collect(summary, metric)
out = Path(args.out)
out.mkdir(parents=True, exist_ok=True)
style()
plot(summary, metric, label, fr, by, out, args.table)
tex = Path(args.tables) / f"tab{args.table}_fraction.tex"
tex.write_text(table(summary, metric, label, fr, by), encoding="utf-8", newline="\n")
print(f"wrote {out / f'f5_data_fraction_tab{args.table}.pdf'} and {tex}")
return 0
if __name__ == "__main__":
sys.exit(main())