Download scripts/label_efficiency.py from HWresearch/GNN4Colliders: direct link, hf CLI and curl.
- Browser
- Download file 3.85 kB
-
https://huggingface.co/HWresearch/GNN4Colliders/resolve/e71731f4373a94de06af1589654ada8cd48dc088/scripts/label_efficiency.py
- Command line
-
hf download hf://HWresearch/GNN4Colliders@e71731f4373a94de06af1589654ada8cd48dc088/scripts/label_efficiency.py
-
curl -L -o label_efficiency.py https://huggingface.co/HWresearch/GNN4Colliders/resolve/e71731f4373a94de06af1589654ada8cd48dc088/scripts/label_efficiency.py
3.85 kB
| """Validation AUC against the number of labelled training events. | |
| For every config and every ``--events`` count N, runs the model with | |
| ``data.train_events=N`` in ``<out>/<name>/n<N>``: supervised models are | |
| trained, then evaluated; in-context models (TabPFN, TabICL, PFN) are only evaluated, | |
| with the N events as context. Runs whose metrics exist are skipped, so an | |
| interrupted sweep restarts where it stopped. Prints a table of AUC (%) and | |
| evaluation seconds. The event cache must exist (``gnn4colliders prepare``). | |
| uv run python scripts/label_efficiency.py configs/gamgam_tabpfn.yaml \\ | |
| configs/gamgam.yaml --events 100 1000 10000 50000 | |
| Meta-trained in-context models (the PFN; TabPFN or TabICL after | |
| meta-training) need ``--override checkpoint=<path>``. Overrides apply to | |
| every config; ``name=...`` keeps variants apart (run one config at a time | |
| then, as names must differ): | |
| uv run python scripts/label_efficiency.py configs/gamgam_tabpfn.yaml \\ | |
| --override name=gamgam_tabpfn_multiclass \\ | |
| model.tabpfn_checkpoint=tabpfn-v3.5-20260909_multiclass.safetensors | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| from pathlib import Path | |
| from gnn4colliders.cli import main as gnn4colliders | |
| from gnn4colliders.config import load_config | |
| from gnn4colliders.models import IN_CONTEXT_MODELS | |
| def _run(config: Path, events: int, out: Path, overrides: list[str]) -> dict: | |
| name = load_config(config, overrides).name | |
| run_dir = out / name / f"n{events}" | |
| metrics = run_dir / "validation_metrics.json" | |
| if not metrics.exists(): | |
| args = [*overrides, f"data.train_events={events}", f"run_dir={run_dir}"] | |
| resolved = load_config(config, args) | |
| if resolved.model.name == "pfn" and resolved.checkpoint is None: | |
| # The run directory is new: the meta-trained weights are elsewhere. | |
| raise SystemExit("the PFN needs --override checkpoint=<meta-trained>.pt") | |
| if resolved.model.name not in IN_CONTEXT_MODELS: | |
| if gnn4colliders(["train", str(config), *args]) != 0: | |
| raise SystemExit(f"training {name} with {events} events failed") | |
| if gnn4colliders(["evaluate", str(config), *args, "split=validation"]) != 0: | |
| raise SystemExit(f"evaluating {name} with {events} events failed") | |
| return {"name": name, "events": events, **json.loads(metrics.read_text())} | |
| def main() -> None: | |
| parser = argparse.ArgumentParser( | |
| description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter | |
| ) | |
| parser.add_argument("configs", nargs="+", type=Path) | |
| parser.add_argument("--events", nargs="+", type=int, required=True) | |
| parser.add_argument("--override", nargs="*", default=[], help="key=value") | |
| parser.add_argument("--out", type=Path, default=Path("outputs/label_efficiency")) | |
| args = parser.parse_args() | |
| names = [load_config(config, args.override).name for config in args.configs] | |
| if len(set(names)) < len(names): | |
| raise SystemExit(f"configs share a run name ({names}); override name per run") | |
| runs = [ | |
| _run(config, events, args.out, args.override) | |
| for config in args.configs | |
| for events in args.events | |
| ] | |
| print("| model | " + " | ".join(f"N = {n}" for n in args.events) + " |") | |
| print("|---" * (len(args.events) + 1) + "|") | |
| for name in dict.fromkeys(run["name"] for run in runs): | |
| cells = [] | |
| for events in args.events: | |
| run = next(r for r in runs if r["name"] == name and r["events"] == events) | |
| auc = 100 * run["metrics"]["auc"] | |
| seconds = run.get("seconds") | |
| total = f", {sum(seconds.values()):.0f} s" if seconds else "" | |
| cells.append(f"{auc:.2f}{total}") | |
| print(f"| {name} | " + " | ".join(cells) + " |") | |
| if __name__ == "__main__": | |
| main() | |