File size: 7,348 Bytes
dffbc1f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 | #!/usr/bin/env python
"""F5 data-fraction curve and the fraction x condition table, from the aggregated table.
python scripts/aggregate.py --table 5 # -> tables/tab5.json (+ tab5*.tex, numbers.tex)
python figures/f5/plot_f5.py --table 5 --tables tables --out figures/out
Reads ``<tables>/tab<N>.json`` written by the aggregator (rows ``scratch_fXXX``
and ``s9pre_fXXX``, each pre-trained row paired with the scratch row of the
same fraction on the common test set). Writes ``f5_data_fraction_tab<N>.{pdf,png}``
(mean +- SD over the 5 fold means; each fold = mean over seeds) and
``<tables>/tab<N>_fraction.tex`` (Fraction | Scratch | S9 pre-trained | Delta |
positive folds | p (n=5) | p (n=15)).
"""
from __future__ import annotations
import argparse
import json
import re
import sys
from pathlib import Path
REPO = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(REPO))
import matplotlib # noqa: E402
matplotlib.use("Agg")
import matplotlib.pyplot as plt # noqa: E402
from pulse.stats import format_p, format_pm # noqa: E402
SCRATCH, PRE = "#2a78d6", "#eb6834" # categorical slots 1, 2
INK, INK2, MUTED, GRID, AXIS = "#0b0b0b", "#52514e", "#898781", "#e1e0d9", "#c3c2b7"
ROW_RE = re.compile(r"^(scratch|s9pre)_f(\d{3})$")
def style():
plt.rcParams.update({
"pdf.fonttype": 42, "ps.fonttype": 42, "font.size": 9, "axes.titlesize": 9,
"axes.labelsize": 9, "legend.fontsize": 8, "xtick.labelsize": 8, "ytick.labelsize": 8,
"axes.edgecolor": AXIS, "axes.labelcolor": INK2, "xtick.color": MUTED, "ytick.color": MUTED,
"text.color": INK, "axes.grid": True, "grid.color": GRID, "grid.linewidth": 0.6,
"axes.spines.top": False, "axes.spines.right": False, "legend.frameon": False,
})
def collect(summary: dict, metric: str):
by = {}
for key, r in summary["rows"].items():
m = ROW_RE.match(key)
if not m:
continue
cond, frac = m.group(1), int(m.group(2))
st = r["common"][metric]
if not st["n_runs"]:
continue # partial table: row without runs
by.setdefault(frac, {})[cond] = {"mean": st["mean"], "sd": st["sd"], "row": key,
"label": r["label"], "vs": r.get("vs_reference", {}).get(metric),
"n_folds": st["n_folds"], "n_runs": st["n_runs"]}
fr = sorted(f for f, d in by.items() if {"scratch", "s9pre"} <= set(d))
if not fr:
raise SystemExit("no fraction has both a scratch and an s9pre row")
return fr, by
def _z(x):
"""SD for drawing: NaN (a single fold) -> 0."""
return x if x is not None and x == x else 0.0
def plot(summary, metric, label, fr, by, out: Path, tag: str):
fig, ax = plt.subplots(figsize=(4.2, 2.8))
# evenly spaced fractions (5 % and 10 % would collide on a linear axis)
pos = {f: i for i, f in enumerate(fr)}
for cond, color, name, dx in (("scratch", SCRATCH, "Random init", -0.06), ("s9pre", PRE, "S9 pre-trained", 0.06)):
xs = [pos[f] + dx for f in fr]
ys = [by[f][cond]["mean"] for f in fr]
es = [_z(by[f][cond]["sd"]) for f in fr]
ax.errorbar(xs, ys, yerr=es, color=color, lw=1.6, marker="o", ms=5, mec="white", mew=1.0,
capsize=2.5, elinewidth=0.9, label=name)
for f in fr:
c = by[f]["s9pre"]["vs"]
if not c:
continue
top = max(by[f][k]["mean"] + _z(by[f][k]["sd"]) for k in ("scratch", "s9pre"))
p15 = (c.get("t15") or {}).get("p")
ax.text(pos[f], top + 0.004, f"{c['delta_mean']:+.3f}\n{c['positive_folds']}/{len(c['folds'])} folds",
ha="center", va="bottom", fontsize=6.5, color=INK2 if (p15 or 1) >= 0.05 else INK)
ax.set_xticks([pos[f] for f in fr])
ax.set_xticklabels([f"{f}%" for f in fr])
ax.set_xlabel("share of labelled T2 training segments")
ax.set_ylabel(label)
lo = min(by[f][k]["mean"] - _z(by[f][k]["sd"]) for f in fr for k in ("scratch", "s9pre"))
hi = max(by[f][k]["mean"] + _z(by[f][k]["sd"]) for f in fr for k in ("scratch", "s9pre"))
ax.set_ylim(lo - 0.01, hi + 0.06)
ax.set_xlim(-0.5, len(fr) - 0.5)
seeds = summary["seeds"]
ax.set_title(f"DAF-v2 (EMG+IMU), {len(summary['folds'])}-fold CV x {len(seeds)} seed"
f"{'s' if len(seeds) > 1 else ''}" + ("" if summary["complete"] else " [INCOMPLETE]"),
loc="left")
ax.legend(loc="lower right")
fig.tight_layout()
for ext in ("pdf", "png"):
fig.savefig(out / f"f5_data_fraction_tab{tag}.{ext}", dpi=200)
plt.close(fig)
def table(summary, metric, label, fr, by, digits=3) -> str:
lines = [f"% Generated by figures/f5/plot_f5.py from tab{summary['table']}.json -- do not edit.",
f"% {summary['title']}; metric {metric} on the common test set; mean +- SD (ddof="
f"{summary['ddof']}) over fold means; Delta = S9 pre-trained - scratch (paired by fold "
f"and by fold x seed).",
f"% git {summary['provenance'].get('git_sha')} data {summary['provenance'].get('data_tag')}"]
if not summary["complete"]:
lines.append(f"% INCOMPLETE: {len(summary['missing_runs'])} run(s) missing")
lines += [r"\begin{tabular}{lcccccc}", r"\toprule",
rf"Train fraction & Scratch & S9 pre-trained & $\Delta$ {label} & pos. folds & $p_{{5}}$ & $p_{{15}}$ \\",
r"\midrule"]
for f in fr:
s, p = by[f]["scratch"], by[f]["s9pre"]
c = p["vs"] or {}
t15 = c.get("t15")
lines.append(" & ".join([
rf"{f}\%", format_pm(s["mean"], s["sd"], digits), format_pm(p["mean"], p["sd"], digits),
format_pm(c["delta_mean"], c["delta_sd"], digits) if c else "--",
f"{c['positive_folds']}/{len(c['folds'])}" if c else "--",
format_p(c["t5"]["p"]) if c else "--", format_p(t15["p"]) if t15 else "--"]) + r" \\")
lines += [r"\bottomrule", r"\end{tabular}", ""]
return "\n".join(lines)
def main(argv=None) -> int:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--table", default="5", help="5 (sliding, primary) or 5seg")
ap.add_argument("--tables", default="tables", help="directory with tab<N>.json")
ap.add_argument("--metric", default=None, help="default: the table's first metric")
ap.add_argument("--out", default="figures/out")
args = ap.parse_args(argv)
src = Path(args.tables) / f"tab{args.table}.json"
summary = json.loads(src.read_text(encoding="utf-8"))
if summary.get("task") != "f5":
raise SystemExit(f"{src} is not an F5 table")
metric = args.metric or summary["metrics"][0]["key"]
label = next(m["label"] for m in summary["metrics"] if m["key"] == metric)
fr, by = collect(summary, metric)
out = Path(args.out)
out.mkdir(parents=True, exist_ok=True)
style()
plot(summary, metric, label, fr, by, out, args.table)
tex = Path(args.tables) / f"tab{args.table}_fraction.tex"
tex.write_text(table(summary, metric, label, fr, by), encoding="utf-8", newline="\n")
print(f"wrote {out / f'f5_data_fraction_tab{args.table}.pdf'} and {tex}")
return 0
if __name__ == "__main__":
sys.exit(main())
|