#!/usr/bin/env python """F5 data-fraction curve and the fraction x condition table, from the aggregated table. python scripts/aggregate.py --table 5 # -> tables/tab5.json (+ tab5*.tex, numbers.tex) python figures/f5/plot_f5.py --table 5 --tables tables --out figures/out Reads ``/tab.json`` written by the aggregator (rows ``scratch_fXXX`` and ``s9pre_fXXX``, each pre-trained row paired with the scratch row of the same fraction on the common test set). Writes ``f5_data_fraction_tab.{pdf,png}`` (mean +- SD over the 5 fold means; each fold = mean over seeds) and ``/tab_fraction.tex`` (Fraction | Scratch | S9 pre-trained | Delta | positive folds | p (n=5) | p (n=15)). """ from __future__ import annotations import argparse import json import re import sys from pathlib import Path REPO = Path(__file__).resolve().parents[2] sys.path.insert(0, str(REPO)) import matplotlib # noqa: E402 matplotlib.use("Agg") import matplotlib.pyplot as plt # noqa: E402 from pulse.stats import format_p, format_pm # noqa: E402 SCRATCH, PRE = "#2a78d6", "#eb6834" # categorical slots 1, 2 INK, INK2, MUTED, GRID, AXIS = "#0b0b0b", "#52514e", "#898781", "#e1e0d9", "#c3c2b7" ROW_RE = re.compile(r"^(scratch|s9pre)_f(\d{3})$") def style(): plt.rcParams.update({ "pdf.fonttype": 42, "ps.fonttype": 42, "font.size": 9, "axes.titlesize": 9, "axes.labelsize": 9, "legend.fontsize": 8, "xtick.labelsize": 8, "ytick.labelsize": 8, "axes.edgecolor": AXIS, "axes.labelcolor": INK2, "xtick.color": MUTED, "ytick.color": MUTED, "text.color": INK, "axes.grid": True, "grid.color": GRID, "grid.linewidth": 0.6, "axes.spines.top": False, "axes.spines.right": False, "legend.frameon": False, }) def collect(summary: dict, metric: str): by = {} for key, r in summary["rows"].items(): m = ROW_RE.match(key) if not m: continue cond, frac = m.group(1), int(m.group(2)) st = r["common"][metric] if not st["n_runs"]: continue # partial table: row without runs by.setdefault(frac, {})[cond] = {"mean": st["mean"], "sd": st["sd"], "row": key, "label": r["label"], "vs": r.get("vs_reference", {}).get(metric), "n_folds": st["n_folds"], "n_runs": st["n_runs"]} fr = sorted(f for f, d in by.items() if {"scratch", "s9pre"} <= set(d)) if not fr: raise SystemExit("no fraction has both a scratch and an s9pre row") return fr, by def _z(x): """SD for drawing: NaN (a single fold) -> 0.""" return x if x is not None and x == x else 0.0 def plot(summary, metric, label, fr, by, out: Path, tag: str): fig, ax = plt.subplots(figsize=(4.2, 2.8)) # evenly spaced fractions (5 % and 10 % would collide on a linear axis) pos = {f: i for i, f in enumerate(fr)} for cond, color, name, dx in (("scratch", SCRATCH, "Random init", -0.06), ("s9pre", PRE, "S9 pre-trained", 0.06)): xs = [pos[f] + dx for f in fr] ys = [by[f][cond]["mean"] for f in fr] es = [_z(by[f][cond]["sd"]) for f in fr] ax.errorbar(xs, ys, yerr=es, color=color, lw=1.6, marker="o", ms=5, mec="white", mew=1.0, capsize=2.5, elinewidth=0.9, label=name) for f in fr: c = by[f]["s9pre"]["vs"] if not c: continue top = max(by[f][k]["mean"] + _z(by[f][k]["sd"]) for k in ("scratch", "s9pre")) p15 = (c.get("t15") or {}).get("p") ax.text(pos[f], top + 0.004, f"{c['delta_mean']:+.3f}\n{c['positive_folds']}/{len(c['folds'])} folds", ha="center", va="bottom", fontsize=6.5, color=INK2 if (p15 or 1) >= 0.05 else INK) ax.set_xticks([pos[f] for f in fr]) ax.set_xticklabels([f"{f}%" for f in fr]) ax.set_xlabel("share of labelled T2 training segments") ax.set_ylabel(label) lo = min(by[f][k]["mean"] - _z(by[f][k]["sd"]) for f in fr for k in ("scratch", "s9pre")) hi = max(by[f][k]["mean"] + _z(by[f][k]["sd"]) for f in fr for k in ("scratch", "s9pre")) ax.set_ylim(lo - 0.01, hi + 0.06) ax.set_xlim(-0.5, len(fr) - 0.5) seeds = summary["seeds"] ax.set_title(f"DAF-v2 (EMG+IMU), {len(summary['folds'])}-fold CV x {len(seeds)} seed" f"{'s' if len(seeds) > 1 else ''}" + ("" if summary["complete"] else " [INCOMPLETE]"), loc="left") ax.legend(loc="lower right") fig.tight_layout() for ext in ("pdf", "png"): fig.savefig(out / f"f5_data_fraction_tab{tag}.{ext}", dpi=200) plt.close(fig) def table(summary, metric, label, fr, by, digits=3) -> str: lines = [f"% Generated by figures/f5/plot_f5.py from tab{summary['table']}.json -- do not edit.", f"% {summary['title']}; metric {metric} on the common test set; mean +- SD (ddof=" f"{summary['ddof']}) over fold means; Delta = S9 pre-trained - scratch (paired by fold " f"and by fold x seed).", f"% git {summary['provenance'].get('git_sha')} data {summary['provenance'].get('data_tag')}"] if not summary["complete"]: lines.append(f"% INCOMPLETE: {len(summary['missing_runs'])} run(s) missing") lines += [r"\begin{tabular}{lcccccc}", r"\toprule", rf"Train fraction & Scratch & S9 pre-trained & $\Delta$ {label} & pos. folds & $p_{{5}}$ & $p_{{15}}$ \\", r"\midrule"] for f in fr: s, p = by[f]["scratch"], by[f]["s9pre"] c = p["vs"] or {} t15 = c.get("t15") lines.append(" & ".join([ rf"{f}\%", format_pm(s["mean"], s["sd"], digits), format_pm(p["mean"], p["sd"], digits), format_pm(c["delta_mean"], c["delta_sd"], digits) if c else "--", f"{c['positive_folds']}/{len(c['folds'])}" if c else "--", format_p(c["t5"]["p"]) if c else "--", format_p(t15["p"]) if t15 else "--"]) + r" \\") lines += [r"\bottomrule", r"\end{tabular}", ""] return "\n".join(lines) def main(argv=None) -> int: ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--table", default="5", help="5 (sliding, primary) or 5seg") ap.add_argument("--tables", default="tables", help="directory with tab.json") ap.add_argument("--metric", default=None, help="default: the table's first metric") ap.add_argument("--out", default="figures/out") args = ap.parse_args(argv) src = Path(args.tables) / f"tab{args.table}.json" summary = json.loads(src.read_text(encoding="utf-8")) if summary.get("task") != "f5": raise SystemExit(f"{src} is not an F5 table") metric = args.metric or summary["metrics"][0]["key"] label = next(m["label"] for m in summary["metrics"] if m["key"] == metric) fr, by = collect(summary, metric) out = Path(args.out) out.mkdir(parents=True, exist_ok=True) style() plot(summary, metric, label, fr, by, out, args.table) tex = Path(args.tables) / f"tab{args.table}_fraction.tex" tex.write_text(table(summary, metric, label, fr, by), encoding="utf-8", newline="\n") print(f"wrote {out / f'f5_data_fraction_tab{args.table}.pdf'} and {tex}") return 0 if __name__ == "__main__": sys.exit(main())