"""Plot validation AUC against the number of labelled training events, and write the same numbers as Markdown and LaTeX tables. Reads the sweeps of ``scripts/label_efficiency.py`` (``//n/ _metrics.json``, one series per ````) and the probes of ``scripts/pretrain.sh`` (``/probe/_n.json``, two series per run: k-NN and linear): uv run --extra plots python scripts/plot_label_efficiency.py \\ --sweeps outputs/label_efficiency --probes outputs/delphes_jepa \\ --output plots/label_efficiency writes ``plots/label_efficiency.{pdf,png,md,tex}``. ``--metric`` chooses another metric (``accuracy``, ``f1``, ...). """ from __future__ import annotations import argparse import json import re from pathlib import Path #: Categorical colours in fixed order (validated for colour-vision #: deficiency on a light background) and marker shapes, so that series are #: not told apart by colour alone. COLOURS = ( "#2a78d6", "#eb6834", "#1baf7a", "#eda100", "#e87ba4", "#008300", "#6250d6", "#e34948", ) MARKERS = ("o", "s", "^", "D", "v", "P", "X", "*") TEXT, MUTED, GRID = "#2b2b29", "#6b6a64", "#e4e3de" def _sweeps(root: Path, split: str, metric: str) -> dict[str, dict[int, float]]: series: dict[str, dict[int, float]] = {} for path in sorted(root.glob(f"*/n*/{split}_metrics.json")): events = int(path.parent.name.removeprefix("n")) value = json.loads(path.read_text())["metrics"][metric] series.setdefault(path.parent.parent.name, {})[events] = value return series def _probes(run: Path, split: str, metric: str) -> dict[str, dict[int, float]]: series: dict[str, dict[int, float]] = {} for path in sorted((run / "probe").glob(f"{split}_n*.json")): match = re.fullmatch(rf"{split}_n(\d+)\.json", path.name) if match is None: continue summary = json.loads(path.read_text()) for probe, label in (("knn", "k-NN"), ("linear", "linear")): name = f"{run.name} ({label} probe)" series.setdefault(name, {})[int(match[1])] = summary[probe][metric] return series def _tables(series: dict[str, dict[int, float]], metric: str) -> tuple[str, str]: counts = sorted({count for values in series.values() for count in values}) def cell(values: dict[int, float], count: int) -> str: return f"{100 * values[count]:.2f}" if count in values else "--" header = ["model", *(f"N = {count}" for count in counts)] markdown = [ "| " + " | ".join(header) + " |", "|---" * len(header) + "|", *( "| " + " | ".join([name, *(cell(v, c) for c in counts)]) + " |" for name, v in series.items() ), ] latex = [ r"\begin{tabular}{l" + "r" * len(counts) + "}", r"\hline", " & ".join(["Model", *(f"$N={count}$" for count in counts)]) + r" \\", r"\hline", *( " & ".join([name.replace("_", r"\_"), *(cell(v, c) for c in counts)]) + r" \\" for name, v in series.items() ), r"\hline", r"\end{tabular}", f"% {metric} (%) against labelled training events N", ] return "\n".join(markdown) + "\n", "\n".join(latex) + "\n" def _plot(series: dict[str, dict[int, float]], metric: str, output: Path) -> None: import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt from matplotlib.ticker import FuncFormatter, LogLocator if len(series) > len(COLOURS): raise SystemExit(f"at most {len(COLOURS)} series; plot fewer per figure") figure, axes = plt.subplots(figsize=(6.4, 4.2), layout="constrained") figure.set_facecolor("#fcfcfb") axes.set_facecolor("#fcfcfb") for index, (name, values) in enumerate(series.items()): counts = sorted(values) percent = [100 * values[count] for count in counts] axes.plot( counts, percent, color=COLOURS[index], marker=MARKERS[index], markersize=7, linewidth=2, markeredgecolor="#fcfcfb", markeredgewidth=1.5, label=name, ) # A direct label at the last point when there are few series. if len(series) <= 4: axes.annotate( name, (counts[-1], percent[-1]), xytext=(6, 0), textcoords="offset points", va="center", fontsize=8, color=TEXT, ) axes.set_xscale("log") axes.xaxis.set_major_locator(LogLocator(base=10)) axes.xaxis.set_major_formatter(FuncFormatter(lambda x, _: f"{x:,.0f}")) if len(series) <= 4: # room on the right for the direct labels low, high = axes.get_xlim() axes.set_xlim(low, high * 4) axes.set_xlabel("labelled training events", color=TEXT) label = "AUC" if metric == "auc" else metric axes.set_ylabel(f"validation {label} (%)", color=TEXT) axes.grid(True, which="major", color=GRID, linewidth=0.8) axes.tick_params(colors=MUTED) for spine in ("top", "right"): axes.spines[spine].set_visible(False) for spine in ("left", "bottom"): axes.spines[spine].set_color(GRID) if len(series) > 1: axes.legend(frameon=False, fontsize=8, labelcolor=TEXT) for suffix in (".pdf", ".png"): figure.savefig(output.with_suffix(suffix), dpi=200) plt.close(figure) def main() -> None: parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) parser.add_argument("--sweeps", nargs="*", type=Path, default=[]) parser.add_argument("--probes", nargs="*", type=Path, default=[]) parser.add_argument("--split", default="validation") parser.add_argument("--metric", default="auc") parser.add_argument("--output", type=Path, default=Path("plots/label_efficiency")) args = parser.parse_args() series: dict[str, dict[int, float]] = {} found = [_sweeps(root, args.split, args.metric) for root in args.sweeps] found += [_probes(run, args.split, args.metric) for run in args.probes] for part in found: for name, values in part.items(): merged = series.setdefault(name, {}) if merged.keys() & values.keys(): raise SystemExit(f"two results for {name} at the same N; plot apart") merged.update(values) if not series: raise SystemExit("no results found") args.output.parent.mkdir(parents=True, exist_ok=True) markdown, latex = _tables(series, args.metric) args.output.with_suffix(".md").write_text(markdown) args.output.with_suffix(".tex").write_text(latex) _plot(series, args.metric, args.output) print(markdown, end="") if __name__ == "__main__": main()