Claude Opus 5.5
fix: warm up CUDA graphs again after resuming; test split and all events in label_efficiency
9a04b7b Download scripts/label_efficiency.py from HWresearch/GNN4Colliders: direct link, hf CLI and curl.
- Browser
- Download file 4.59 kB
-
https://huggingface.co/HWresearch/GNN4Colliders/resolve/main/scripts/label_efficiency.py
- Command line
-
hf download hf://HWresearch/GNN4Colliders/scripts/label_efficiency.py
-
curl -L -o label_efficiency.py https://huggingface.co/HWresearch/GNN4Colliders/resolve/main/scripts/label_efficiency.py
4.59 kB
| """Validation (or test) AUC against the number of labelled training events. | |
| For every config and every ``--events`` count N (``all``: every training | |
| event, with the classes' own proportions), runs the model with | |
| ``data.train_events=N`` in ``<out>/<name>/n<N>``: supervised models are | |
| trained, then evaluated on ``--split``; in-context models (TabPFN, TabICL, | |
| PFN) are only evaluated, with the N events as context. ``--eval-override`` | |
| applies to the evaluation only (``data.eval_events=null`` reports on the | |
| whole split while training validates on a subsample). Runs whose metrics | |
| exist are skipped, so an interrupted sweep restarts where it stopped. | |
| Prints a table of AUC (%) and evaluation seconds. The event cache must | |
| exist (``gnn4colliders prepare``). | |
| uv run python scripts/label_efficiency.py configs/gamgam_tabpfn.yaml \\ | |
| configs/gamgam.yaml --events 100 1000 10000 50000 | |
| Meta-trained in-context models (the PFN; TabPFN or TabICL after | |
| meta-training) need ``--override checkpoint=<path>``. Overrides apply to | |
| every config; ``name=...`` keeps variants apart (run one config at a time | |
| then, as names must differ): | |
| uv run python scripts/label_efficiency.py configs/gamgam_tabpfn.yaml \\ | |
| --override name=gamgam_tabpfn_multiclass \\ | |
| model.tabpfn_checkpoint=tabpfn-v3.5-20260909_multiclass.safetensors | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| from pathlib import Path | |
| from gnn4colliders.cli import main as gnn4colliders | |
| from gnn4colliders.config import load_config | |
| from gnn4colliders.models import IN_CONTEXT_MODELS | |
| def _run( | |
| config: Path, | |
| events: str, | |
| out: Path, | |
| overrides: list[str], | |
| split: str, | |
| eval_overrides: list[str], | |
| ) -> dict: | |
| name = load_config(config, overrides).name | |
| run_dir = out / name / f"n{events}" | |
| metrics = run_dir / f"{split}_metrics.json" | |
| if not metrics.exists(): | |
| count = "null" if events == "all" else events | |
| args = [*overrides, f"data.train_events={count}", f"run_dir={run_dir}"] | |
| resolved = load_config(config, args) | |
| if resolved.model.name == "pfn" and resolved.checkpoint is None: | |
| # The run directory is new: the meta-trained weights are elsewhere. | |
| raise SystemExit("the PFN needs --override checkpoint=<meta-trained>.pt") | |
| if resolved.model.name not in IN_CONTEXT_MODELS: | |
| if gnn4colliders(["train", str(config), *args]) != 0: | |
| raise SystemExit(f"training {name} with {events} events failed") | |
| evaluate = ["evaluate", str(config), *args, f"split={split}", *eval_overrides] | |
| if gnn4colliders(evaluate) != 0: | |
| raise SystemExit(f"evaluating {name} with {events} events failed") | |
| return {"name": name, "events": events, **json.loads(metrics.read_text())} | |
| def main() -> None: | |
| parser = argparse.ArgumentParser( | |
| description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter | |
| ) | |
| parser.add_argument("configs", nargs="+", type=Path) | |
| parser.add_argument("--events", nargs="+", required=True, help="counts or all") | |
| parser.add_argument("--override", nargs="*", default=[], help="key=value") | |
| parser.add_argument("--split", default="validation", help="validation or test") | |
| parser.add_argument("--eval-override", nargs="*", default=[], help="key=value") | |
| parser.add_argument("--out", type=Path, default=Path("outputs/label_efficiency")) | |
| args = parser.parse_args() | |
| if not all(events == "all" or events.isdigit() for events in args.events): | |
| parser.error("--events takes counts or all") | |
| names = [load_config(config, args.override).name for config in args.configs] | |
| if len(set(names)) < len(names): | |
| raise SystemExit(f"configs share a run name ({names}); override name per run") | |
| runs = [ | |
| _run(config, events, args.out, args.override, args.split, args.eval_override) | |
| for config in args.configs | |
| for events in args.events | |
| ] | |
| print("| model | " + " | ".join(f"N = {n}" for n in args.events) + " |") | |
| print("|---" * (len(args.events) + 1) + "|") | |
| for name in dict.fromkeys(run["name"] for run in runs): | |
| cells = [] | |
| for events in args.events: | |
| run = next(r for r in runs if r["name"] == name and r["events"] == events) | |
| auc = 100 * run["metrics"]["auc"] | |
| seconds = run.get("seconds") | |
| total = f", {sum(seconds.values()):.0f} s" if seconds else "" | |
| cells.append(f"{auc:.2f}{total}") | |
| print(f"| {name} | " + " | ".join(cells) + " |") | |
| if __name__ == "__main__": | |
| main() | |