GNN4Colliders / scripts /label_efficiency.py
ho22joshua's picture
feat: tabular in-context models, LoRA meta-training, pretraining objectives and probes
8dff74c
Raw History Blame
3.85 kB
"""Validation AUC against the number of labelled training events.
For every config and every ``--events`` count N, runs the model with
``data.train_events=N`` in ``<out>/<name>/n<N>``: supervised models are
trained, then evaluated; in-context models (TabPFN, TabICL, PFN) are only evaluated,
with the N events as context. Runs whose metrics exist are skipped, so an
interrupted sweep restarts where it stopped. Prints a table of AUC (%) and
evaluation seconds. The event cache must exist (``gnn4colliders prepare``).
uv run python scripts/label_efficiency.py configs/gamgam_tabpfn.yaml \\
configs/gamgam.yaml --events 100 1000 10000 50000
Meta-trained in-context models (the PFN; TabPFN or TabICL after
meta-training) need ``--override checkpoint=<path>``. Overrides apply to
every config; ``name=...`` keeps variants apart (run one config at a time
then, as names must differ):
uv run python scripts/label_efficiency.py configs/gamgam_tabpfn.yaml \\
--override name=gamgam_tabpfn_multiclass \\
model.tabpfn_checkpoint=tabpfn-v3.5-20260909_multiclass.safetensors
"""
from __future__ import annotations
import argparse
import json
from pathlib import Path
from gnn4colliders.cli import main as gnn4colliders
from gnn4colliders.config import load_config
from gnn4colliders.models import IN_CONTEXT_MODELS
def _run(config: Path, events: int, out: Path, overrides: list[str]) -> dict:
name = load_config(config, overrides).name
run_dir = out / name / f"n{events}"
metrics = run_dir / "validation_metrics.json"
if not metrics.exists():
args = [*overrides, f"data.train_events={events}", f"run_dir={run_dir}"]
resolved = load_config(config, args)
if resolved.model.name == "pfn" and resolved.checkpoint is None:
# The run directory is new: the meta-trained weights are elsewhere.
raise SystemExit("the PFN needs --override checkpoint=<meta-trained>.pt")
if resolved.model.name not in IN_CONTEXT_MODELS:
if gnn4colliders(["train", str(config), *args]) != 0:
raise SystemExit(f"training {name} with {events} events failed")
if gnn4colliders(["evaluate", str(config), *args, "split=validation"]) != 0:
raise SystemExit(f"evaluating {name} with {events} events failed")
return {"name": name, "events": events, **json.loads(metrics.read_text())}
def main() -> None:
parser = argparse.ArgumentParser(
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
)
parser.add_argument("configs", nargs="+", type=Path)
parser.add_argument("--events", nargs="+", type=int, required=True)
parser.add_argument("--override", nargs="*", default=[], help="key=value")
parser.add_argument("--out", type=Path, default=Path("outputs/label_efficiency"))
args = parser.parse_args()
names = [load_config(config, args.override).name for config in args.configs]
if len(set(names)) < len(names):
raise SystemExit(f"configs share a run name ({names}); override name per run")
runs = [
_run(config, events, args.out, args.override)
for config in args.configs
for events in args.events
]
print("| model | " + " | ".join(f"N = {n}" for n in args.events) + " |")
print("|---" * (len(args.events) + 1) + "|")
for name in dict.fromkeys(run["name"] for run in runs):
cells = []
for events in args.events:
run = next(r for r in runs if r["name"] == name and r["events"] == events)
auc = 100 * run["metrics"]["auc"]
seconds = run.get("seconds")
total = f", {sum(seconds.values()):.0f} s" if seconds else ""
cells.append(f"{auc:.2f}{total}")
print(f"| {name} | " + " | ".join(cells) + " |")
if __name__ == "__main__":
main()