"""Validation (or test) AUC against the number of labelled training events. For every config and every ``--events`` count N (``all``: every training event, with the classes' own proportions), runs the model with ``data.train_events=N`` in ``//n``: supervised models are trained, then evaluated on ``--split``; in-context models (TabPFN, TabICL, PFN) are only evaluated, with the N events as context. ``--eval-override`` applies to the evaluation only (``data.eval_events=null`` reports on the whole split while training validates on a subsample). Runs whose metrics exist are skipped, so an interrupted sweep restarts where it stopped. Prints a table of AUC (%) and evaluation seconds. The event cache must exist (``gnn4colliders prepare``). uv run python scripts/label_efficiency.py configs/gamgam_tabpfn.yaml \\ configs/gamgam.yaml --events 100 1000 10000 50000 Meta-trained in-context models (the PFN; TabPFN or TabICL after meta-training) need ``--override checkpoint=``. Overrides apply to every config; ``name=...`` keeps variants apart (run one config at a time then, as names must differ): uv run python scripts/label_efficiency.py configs/gamgam_tabpfn.yaml \\ --override name=gamgam_tabpfn_multiclass \\ model.tabpfn_checkpoint=tabpfn-v3.5-20260909_multiclass.safetensors """ from __future__ import annotations import argparse import json from pathlib import Path from gnn4colliders.cli import main as gnn4colliders from gnn4colliders.config import load_config from gnn4colliders.models import IN_CONTEXT_MODELS def _run( config: Path, events: str, out: Path, overrides: list[str], split: str, eval_overrides: list[str], ) -> dict: name = load_config(config, overrides).name run_dir = out / name / f"n{events}" metrics = run_dir / f"{split}_metrics.json" if not metrics.exists(): count = "null" if events == "all" else events args = [*overrides, f"data.train_events={count}", f"run_dir={run_dir}"] resolved = load_config(config, args) if resolved.model.name == "pfn" and resolved.checkpoint is None: # The run directory is new: the meta-trained weights are elsewhere. raise SystemExit("the PFN needs --override checkpoint=.pt") if resolved.model.name not in IN_CONTEXT_MODELS: if gnn4colliders(["train", str(config), *args]) != 0: raise SystemExit(f"training {name} with {events} events failed") evaluate = ["evaluate", str(config), *args, f"split={split}", *eval_overrides] if gnn4colliders(evaluate) != 0: raise SystemExit(f"evaluating {name} with {events} events failed") return {"name": name, "events": events, **json.loads(metrics.read_text())} def main() -> None: parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) parser.add_argument("configs", nargs="+", type=Path) parser.add_argument("--events", nargs="+", required=True, help="counts or all") parser.add_argument("--override", nargs="*", default=[], help="key=value") parser.add_argument("--split", default="validation", help="validation or test") parser.add_argument("--eval-override", nargs="*", default=[], help="key=value") parser.add_argument("--out", type=Path, default=Path("outputs/label_efficiency")) args = parser.parse_args() if not all(events == "all" or events.isdigit() for events in args.events): parser.error("--events takes counts or all") names = [load_config(config, args.override).name for config in args.configs] if len(set(names)) < len(names): raise SystemExit(f"configs share a run name ({names}); override name per run") runs = [ _run(config, events, args.out, args.override, args.split, args.eval_override) for config in args.configs for events in args.events ] print("| model | " + " | ".join(f"N = {n}" for n in args.events) + " |") print("|---" * (len(args.events) + 1) + "|") for name in dict.fromkeys(run["name"] for run in runs): cells = [] for events in args.events: run = next(r for r in runs if r["name"] == name and r["events"] == events) auc = 100 * run["metrics"]["auc"] seconds = run.get("seconds") total = f", {sum(seconds.values()):.0f} s" if seconds else "" cells.append(f"{auc:.2f}{total}") print(f"| {name} | " + " | ".join(cells) + " |") if __name__ == "__main__": main()