Claude Opus 5.5
feat: evaluation subsample, Slurm wrappers for the study and label-efficiency plots
e71731f Download scripts/plot_label_efficiency.py from HWresearch/GNN4Colliders: direct link, hf CLI and curl.
- Browser
- Download file 6.91 kB
-
https://huggingface.co/HWresearch/GNN4Colliders/resolve/main/scripts/plot_label_efficiency.py
- Command line
-
hf download hf://HWresearch/GNN4Colliders/scripts/plot_label_efficiency.py
-
curl -L -o plot_label_efficiency.py https://huggingface.co/HWresearch/GNN4Colliders/resolve/main/scripts/plot_label_efficiency.py
6.91 kB
| """Plot validation AUC against the number of labelled training events, and | |
| write the same numbers as Markdown and LaTeX tables. | |
| Reads the sweeps of ``scripts/label_efficiency.py`` (``<out>/<name>/n<N>/ | |
| <split>_metrics.json``, one series per ``<name>``) and the probes of | |
| ``scripts/pretrain.sh`` (``<run>/probe/<split>_n<N>.json``, two series per | |
| run: k-NN and linear): | |
| uv run --extra plots python scripts/plot_label_efficiency.py \\ | |
| --sweeps outputs/label_efficiency --probes outputs/delphes_jepa \\ | |
| --output plots/label_efficiency | |
| writes ``plots/label_efficiency.{pdf,png,md,tex}``. ``--metric`` chooses | |
| another metric (``accuracy``, ``f1``, ...). | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import re | |
| from pathlib import Path | |
| #: Categorical colours in fixed order (validated for colour-vision | |
| #: deficiency on a light background) and marker shapes, so that series are | |
| #: not told apart by colour alone. | |
| COLOURS = ( | |
| "#2a78d6", | |
| "#eb6834", | |
| "#1baf7a", | |
| "#eda100", | |
| "#e87ba4", | |
| "#008300", | |
| "#6250d6", | |
| "#e34948", | |
| ) | |
| MARKERS = ("o", "s", "^", "D", "v", "P", "X", "*") | |
| TEXT, MUTED, GRID = "#2b2b29", "#6b6a64", "#e4e3de" | |
| def _sweeps(root: Path, split: str, metric: str) -> dict[str, dict[int, float]]: | |
| series: dict[str, dict[int, float]] = {} | |
| for path in sorted(root.glob(f"*/n*/{split}_metrics.json")): | |
| events = int(path.parent.name.removeprefix("n")) | |
| value = json.loads(path.read_text())["metrics"][metric] | |
| series.setdefault(path.parent.parent.name, {})[events] = value | |
| return series | |
| def _probes(run: Path, split: str, metric: str) -> dict[str, dict[int, float]]: | |
| series: dict[str, dict[int, float]] = {} | |
| for path in sorted((run / "probe").glob(f"{split}_n*.json")): | |
| match = re.fullmatch(rf"{split}_n(\d+)\.json", path.name) | |
| if match is None: | |
| continue | |
| summary = json.loads(path.read_text()) | |
| for probe, label in (("knn", "k-NN"), ("linear", "linear")): | |
| name = f"{run.name} ({label} probe)" | |
| series.setdefault(name, {})[int(match[1])] = summary[probe][metric] | |
| return series | |
| def _tables(series: dict[str, dict[int, float]], metric: str) -> tuple[str, str]: | |
| counts = sorted({count for values in series.values() for count in values}) | |
| def cell(values: dict[int, float], count: int) -> str: | |
| return f"{100 * values[count]:.2f}" if count in values else "--" | |
| header = ["model", *(f"N = {count}" for count in counts)] | |
| markdown = [ | |
| "| " + " | ".join(header) + " |", | |
| "|---" * len(header) + "|", | |
| *( | |
| "| " + " | ".join([name, *(cell(v, c) for c in counts)]) + " |" | |
| for name, v in series.items() | |
| ), | |
| ] | |
| latex = [ | |
| r"\begin{tabular}{l" + "r" * len(counts) + "}", | |
| r"\hline", | |
| " & ".join(["Model", *(f"$N={count}$" for count in counts)]) + r" \\", | |
| r"\hline", | |
| *( | |
| " & ".join([name.replace("_", r"\_"), *(cell(v, c) for c in counts)]) | |
| + r" \\" | |
| for name, v in series.items() | |
| ), | |
| r"\hline", | |
| r"\end{tabular}", | |
| f"% {metric} (%) against labelled training events N", | |
| ] | |
| return "\n".join(markdown) + "\n", "\n".join(latex) + "\n" | |
| def _plot(series: dict[str, dict[int, float]], metric: str, output: Path) -> None: | |
| import matplotlib | |
| matplotlib.use("Agg") | |
| import matplotlib.pyplot as plt | |
| from matplotlib.ticker import FuncFormatter, LogLocator | |
| if len(series) > len(COLOURS): | |
| raise SystemExit(f"at most {len(COLOURS)} series; plot fewer per figure") | |
| figure, axes = plt.subplots(figsize=(6.4, 4.2), layout="constrained") | |
| figure.set_facecolor("#fcfcfb") | |
| axes.set_facecolor("#fcfcfb") | |
| for index, (name, values) in enumerate(series.items()): | |
| counts = sorted(values) | |
| percent = [100 * values[count] for count in counts] | |
| axes.plot( | |
| counts, | |
| percent, | |
| color=COLOURS[index], | |
| marker=MARKERS[index], | |
| markersize=7, | |
| linewidth=2, | |
| markeredgecolor="#fcfcfb", | |
| markeredgewidth=1.5, | |
| label=name, | |
| ) | |
| # A direct label at the last point when there are few series. | |
| if len(series) <= 4: | |
| axes.annotate( | |
| name, | |
| (counts[-1], percent[-1]), | |
| xytext=(6, 0), | |
| textcoords="offset points", | |
| va="center", | |
| fontsize=8, | |
| color=TEXT, | |
| ) | |
| axes.set_xscale("log") | |
| axes.xaxis.set_major_locator(LogLocator(base=10)) | |
| axes.xaxis.set_major_formatter(FuncFormatter(lambda x, _: f"{x:,.0f}")) | |
| if len(series) <= 4: # room on the right for the direct labels | |
| low, high = axes.get_xlim() | |
| axes.set_xlim(low, high * 4) | |
| axes.set_xlabel("labelled training events", color=TEXT) | |
| label = "AUC" if metric == "auc" else metric | |
| axes.set_ylabel(f"validation {label} (%)", color=TEXT) | |
| axes.grid(True, which="major", color=GRID, linewidth=0.8) | |
| axes.tick_params(colors=MUTED) | |
| for spine in ("top", "right"): | |
| axes.spines[spine].set_visible(False) | |
| for spine in ("left", "bottom"): | |
| axes.spines[spine].set_color(GRID) | |
| if len(series) > 1: | |
| axes.legend(frameon=False, fontsize=8, labelcolor=TEXT) | |
| for suffix in (".pdf", ".png"): | |
| figure.savefig(output.with_suffix(suffix), dpi=200) | |
| plt.close(figure) | |
| def main() -> None: | |
| parser = argparse.ArgumentParser( | |
| description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter | |
| ) | |
| parser.add_argument("--sweeps", nargs="*", type=Path, default=[]) | |
| parser.add_argument("--probes", nargs="*", type=Path, default=[]) | |
| parser.add_argument("--split", default="validation") | |
| parser.add_argument("--metric", default="auc") | |
| parser.add_argument("--output", type=Path, default=Path("plots/label_efficiency")) | |
| args = parser.parse_args() | |
| series: dict[str, dict[int, float]] = {} | |
| found = [_sweeps(root, args.split, args.metric) for root in args.sweeps] | |
| found += [_probes(run, args.split, args.metric) for run in args.probes] | |
| for part in found: | |
| for name, values in part.items(): | |
| merged = series.setdefault(name, {}) | |
| if merged.keys() & values.keys(): | |
| raise SystemExit(f"two results for {name} at the same N; plot apart") | |
| merged.update(values) | |
| if not series: | |
| raise SystemExit("no results found") | |
| args.output.parent.mkdir(parents=True, exist_ok=True) | |
| markdown, latex = _tables(series, args.metric) | |
| args.output.with_suffix(".md").write_text(markdown) | |
| args.output.with_suffix(".tex").write_text(latex) | |
| _plot(series, args.metric, args.output) | |
| print(markdown, end="") | |
| if __name__ == "__main__": | |
| main() | |