GNN4Colliders / scripts /label_efficiency.py
ho22joshua's picture
Claude Opus 5.5
fix: warm up CUDA graphs again after resuming; test split and all events in label_efficiency
9a04b7b
Raw History Blame Contribute Delete
4.59 kB
"""Validation (or test) AUC against the number of labelled training events.
For every config and every ``--events`` count N (``all``: every training
event, with the classes' own proportions), runs the model with
``data.train_events=N`` in ``<out>/<name>/n<N>``: supervised models are
trained, then evaluated on ``--split``; in-context models (TabPFN, TabICL,
PFN) are only evaluated, with the N events as context. ``--eval-override``
applies to the evaluation only (``data.eval_events=null`` reports on the
whole split while training validates on a subsample). Runs whose metrics
exist are skipped, so an interrupted sweep restarts where it stopped.
Prints a table of AUC (%) and evaluation seconds. The event cache must
exist (``gnn4colliders prepare``).
uv run python scripts/label_efficiency.py configs/gamgam_tabpfn.yaml \\
configs/gamgam.yaml --events 100 1000 10000 50000
Meta-trained in-context models (the PFN; TabPFN or TabICL after
meta-training) need ``--override checkpoint=<path>``. Overrides apply to
every config; ``name=...`` keeps variants apart (run one config at a time
then, as names must differ):
uv run python scripts/label_efficiency.py configs/gamgam_tabpfn.yaml \\
--override name=gamgam_tabpfn_multiclass \\
model.tabpfn_checkpoint=tabpfn-v3.5-20260909_multiclass.safetensors
"""
from __future__ import annotations
import argparse
import json
from pathlib import Path
from gnn4colliders.cli import main as gnn4colliders
from gnn4colliders.config import load_config
from gnn4colliders.models import IN_CONTEXT_MODELS
def _run(
config: Path,
events: str,
out: Path,
overrides: list[str],
split: str,
eval_overrides: list[str],
) -> dict:
name = load_config(config, overrides).name
run_dir = out / name / f"n{events}"
metrics = run_dir / f"{split}_metrics.json"
if not metrics.exists():
count = "null" if events == "all" else events
args = [*overrides, f"data.train_events={count}", f"run_dir={run_dir}"]
resolved = load_config(config, args)
if resolved.model.name == "pfn" and resolved.checkpoint is None:
# The run directory is new: the meta-trained weights are elsewhere.
raise SystemExit("the PFN needs --override checkpoint=<meta-trained>.pt")
if resolved.model.name not in IN_CONTEXT_MODELS:
if gnn4colliders(["train", str(config), *args]) != 0:
raise SystemExit(f"training {name} with {events} events failed")
evaluate = ["evaluate", str(config), *args, f"split={split}", *eval_overrides]
if gnn4colliders(evaluate) != 0:
raise SystemExit(f"evaluating {name} with {events} events failed")
return {"name": name, "events": events, **json.loads(metrics.read_text())}
def main() -> None:
parser = argparse.ArgumentParser(
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
)
parser.add_argument("configs", nargs="+", type=Path)
parser.add_argument("--events", nargs="+", required=True, help="counts or all")
parser.add_argument("--override", nargs="*", default=[], help="key=value")
parser.add_argument("--split", default="validation", help="validation or test")
parser.add_argument("--eval-override", nargs="*", default=[], help="key=value")
parser.add_argument("--out", type=Path, default=Path("outputs/label_efficiency"))
args = parser.parse_args()
if not all(events == "all" or events.isdigit() for events in args.events):
parser.error("--events takes counts or all")
names = [load_config(config, args.override).name for config in args.configs]
if len(set(names)) < len(names):
raise SystemExit(f"configs share a run name ({names}); override name per run")
runs = [
_run(config, events, args.out, args.override, args.split, args.eval_override)
for config in args.configs
for events in args.events
]
print("| model | " + " | ".join(f"N = {n}" for n in args.events) + " |")
print("|---" * (len(args.events) + 1) + "|")
for name in dict.fromkeys(run["name"] for run in runs):
cells = []
for events in args.events:
run = next(r for r in runs if r["name"] == name and r["events"] == events)
auc = 100 * run["metrics"]["auc"]
seconds = run.get("seconds")
total = f", {sum(seconds.values()):.0f} s" if seconds else ""
cells.append(f"{auc:.2f}{total}")
print(f"| {name} | " + " | ".join(cells) + " |")
if __name__ == "__main__":
main()