File size: 7,348 Bytes
dffbc1f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
#!/usr/bin/env python
"""F5 data-fraction curve and the fraction x condition table, from the aggregated table.

    python scripts/aggregate.py --table 5            # -> tables/tab5.json (+ tab5*.tex, numbers.tex)
    python figures/f5/plot_f5.py --table 5 --tables tables --out figures/out

Reads ``<tables>/tab<N>.json`` written by the aggregator (rows ``scratch_fXXX``
and ``s9pre_fXXX``, each pre-trained row paired with the scratch row of the
same fraction on the common test set).  Writes ``f5_data_fraction_tab<N>.{pdf,png}``
(mean +- SD over the 5 fold means; each fold = mean over seeds) and
``<tables>/tab<N>_fraction.tex`` (Fraction | Scratch | S9 pre-trained | Delta |
positive folds | p (n=5) | p (n=15)).
"""
from __future__ import annotations

import argparse
import json
import re
import sys
from pathlib import Path

REPO = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(REPO))

import matplotlib  # noqa: E402

matplotlib.use("Agg")
import matplotlib.pyplot as plt  # noqa: E402

from pulse.stats import format_p, format_pm  # noqa: E402

SCRATCH, PRE = "#2a78d6", "#eb6834"        # categorical slots 1, 2
INK, INK2, MUTED, GRID, AXIS = "#0b0b0b", "#52514e", "#898781", "#e1e0d9", "#c3c2b7"
ROW_RE = re.compile(r"^(scratch|s9pre)_f(\d{3})$")


def style():
    plt.rcParams.update({
        "pdf.fonttype": 42, "ps.fonttype": 42, "font.size": 9, "axes.titlesize": 9,
        "axes.labelsize": 9, "legend.fontsize": 8, "xtick.labelsize": 8, "ytick.labelsize": 8,
        "axes.edgecolor": AXIS, "axes.labelcolor": INK2, "xtick.color": MUTED, "ytick.color": MUTED,
        "text.color": INK, "axes.grid": True, "grid.color": GRID, "grid.linewidth": 0.6,
        "axes.spines.top": False, "axes.spines.right": False, "legend.frameon": False,
    })


def collect(summary: dict, metric: str):
    by = {}
    for key, r in summary["rows"].items():
        m = ROW_RE.match(key)
        if not m:
            continue
        cond, frac = m.group(1), int(m.group(2))
        st = r["common"][metric]
        if not st["n_runs"]:
            continue                      # partial table: row without runs
        by.setdefault(frac, {})[cond] = {"mean": st["mean"], "sd": st["sd"], "row": key,
                                         "label": r["label"], "vs": r.get("vs_reference", {}).get(metric),
                                         "n_folds": st["n_folds"], "n_runs": st["n_runs"]}
    fr = sorted(f for f, d in by.items() if {"scratch", "s9pre"} <= set(d))
    if not fr:
        raise SystemExit("no fraction has both a scratch and an s9pre row")
    return fr, by


def _z(x):
    """SD for drawing: NaN (a single fold) -> 0."""
    return x if x is not None and x == x else 0.0


def plot(summary, metric, label, fr, by, out: Path, tag: str):
    fig, ax = plt.subplots(figsize=(4.2, 2.8))
    # evenly spaced fractions (5 % and 10 % would collide on a linear axis)
    pos = {f: i for i, f in enumerate(fr)}
    for cond, color, name, dx in (("scratch", SCRATCH, "Random init", -0.06), ("s9pre", PRE, "S9 pre-trained", 0.06)):
        xs = [pos[f] + dx for f in fr]
        ys = [by[f][cond]["mean"] for f in fr]
        es = [_z(by[f][cond]["sd"]) for f in fr]
        ax.errorbar(xs, ys, yerr=es, color=color, lw=1.6, marker="o", ms=5, mec="white", mew=1.0,
                    capsize=2.5, elinewidth=0.9, label=name)
    for f in fr:
        c = by[f]["s9pre"]["vs"]
        if not c:
            continue
        top = max(by[f][k]["mean"] + _z(by[f][k]["sd"]) for k in ("scratch", "s9pre"))
        p15 = (c.get("t15") or {}).get("p")
        ax.text(pos[f], top + 0.004, f"{c['delta_mean']:+.3f}\n{c['positive_folds']}/{len(c['folds'])} folds",
                ha="center", va="bottom", fontsize=6.5, color=INK2 if (p15 or 1) >= 0.05 else INK)
    ax.set_xticks([pos[f] for f in fr])
    ax.set_xticklabels([f"{f}%" for f in fr])
    ax.set_xlabel("share of labelled T2 training segments")
    ax.set_ylabel(label)
    lo = min(by[f][k]["mean"] - _z(by[f][k]["sd"]) for f in fr for k in ("scratch", "s9pre"))
    hi = max(by[f][k]["mean"] + _z(by[f][k]["sd"]) for f in fr for k in ("scratch", "s9pre"))
    ax.set_ylim(lo - 0.01, hi + 0.06)
    ax.set_xlim(-0.5, len(fr) - 0.5)
    seeds = summary["seeds"]
    ax.set_title(f"DAF-v2 (EMG+IMU), {len(summary['folds'])}-fold CV x {len(seeds)} seed"
                 f"{'s' if len(seeds) > 1 else ''}" + ("" if summary["complete"] else "  [INCOMPLETE]"),
                 loc="left")
    ax.legend(loc="lower right")
    fig.tight_layout()
    for ext in ("pdf", "png"):
        fig.savefig(out / f"f5_data_fraction_tab{tag}.{ext}", dpi=200)
    plt.close(fig)


def table(summary, metric, label, fr, by, digits=3) -> str:
    lines = [f"% Generated by figures/f5/plot_f5.py from tab{summary['table']}.json -- do not edit.",
             f"% {summary['title']}; metric {metric} on the common test set; mean +- SD (ddof="
             f"{summary['ddof']}) over fold means; Delta = S9 pre-trained - scratch (paired by fold "
             f"and by fold x seed).",
             f"% git {summary['provenance'].get('git_sha')}  data {summary['provenance'].get('data_tag')}"]
    if not summary["complete"]:
        lines.append(f"% INCOMPLETE: {len(summary['missing_runs'])} run(s) missing")
    lines += [r"\begin{tabular}{lcccccc}", r"\toprule",
              rf"Train fraction & Scratch & S9 pre-trained & $\Delta$ {label} & pos. folds & $p_{{5}}$ & $p_{{15}}$ \\",
              r"\midrule"]
    for f in fr:
        s, p = by[f]["scratch"], by[f]["s9pre"]
        c = p["vs"] or {}
        t15 = c.get("t15")
        lines.append(" & ".join([
            rf"{f}\%", format_pm(s["mean"], s["sd"], digits), format_pm(p["mean"], p["sd"], digits),
            format_pm(c["delta_mean"], c["delta_sd"], digits) if c else "--",
            f"{c['positive_folds']}/{len(c['folds'])}" if c else "--",
            format_p(c["t5"]["p"]) if c else "--", format_p(t15["p"]) if t15 else "--"]) + r" \\")
    lines += [r"\bottomrule", r"\end{tabular}", ""]
    return "\n".join(lines)


def main(argv=None) -> int:
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--table", default="5", help="5 (sliding, primary) or 5seg")
    ap.add_argument("--tables", default="tables", help="directory with tab<N>.json")
    ap.add_argument("--metric", default=None, help="default: the table's first metric")
    ap.add_argument("--out", default="figures/out")
    args = ap.parse_args(argv)
    src = Path(args.tables) / f"tab{args.table}.json"
    summary = json.loads(src.read_text(encoding="utf-8"))
    if summary.get("task") != "f5":
        raise SystemExit(f"{src} is not an F5 table")
    metric = args.metric or summary["metrics"][0]["key"]
    label = next(m["label"] for m in summary["metrics"] if m["key"] == metric)
    fr, by = collect(summary, metric)
    out = Path(args.out)
    out.mkdir(parents=True, exist_ok=True)
    style()
    plot(summary, metric, label, fr, by, out, args.table)
    tex = Path(args.tables) / f"tab{args.table}_fraction.tex"
    tex.write_text(table(summary, metric, label, fr, by), encoding="utf-8", newline="\n")
    print(f"wrote {out / f'f5_data_fraction_tab{args.table}.pdf'} and {tex}")
    return 0


if __name__ == "__main__":
    sys.exit(main())