File size: 4,594 Bytes
9a04b7b
8dff74c
9a04b7b
 
8dff74c
9a04b7b
 
 
 
 
 
 
8dff74c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9a04b7b
 
 
 
 
 
 
 
8dff74c
 
9a04b7b
8dff74c
9a04b7b
 
8dff74c
 
 
 
 
 
 
9a04b7b
 
8dff74c
 
 
 
 
 
 
 
 
9a04b7b
8dff74c
9a04b7b
 
8dff74c
 
9a04b7b
 
8dff74c
 
 
 
9a04b7b
8dff74c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
"""Validation (or test) AUC against the number of labelled training events.

For every config and every ``--events`` count N (``all``: every training
event, with the classes' own proportions), runs the model with
``data.train_events=N`` in ``<out>/<name>/n<N>``: supervised models are
trained, then evaluated on ``--split``; in-context models (TabPFN, TabICL,
PFN) are only evaluated, with the N events as context. ``--eval-override``
applies to the evaluation only (``data.eval_events=null`` reports on the
whole split while training validates on a subsample). Runs whose metrics
exist are skipped, so an interrupted sweep restarts where it stopped.
Prints a table of AUC (%) and evaluation seconds. The event cache must
exist (``gnn4colliders prepare``).

    uv run python scripts/label_efficiency.py configs/gamgam_tabpfn.yaml \\
        configs/gamgam.yaml --events 100 1000 10000 50000

Meta-trained in-context models (the PFN; TabPFN or TabICL after
meta-training) need ``--override checkpoint=<path>``. Overrides apply to
every config; ``name=...`` keeps variants apart (run one config at a time
then, as names must differ):

    uv run python scripts/label_efficiency.py configs/gamgam_tabpfn.yaml \\
        --override name=gamgam_tabpfn_multiclass \\
        model.tabpfn_checkpoint=tabpfn-v3.5-20260909_multiclass.safetensors
"""

from __future__ import annotations

import argparse
import json
from pathlib import Path

from gnn4colliders.cli import main as gnn4colliders
from gnn4colliders.config import load_config
from gnn4colliders.models import IN_CONTEXT_MODELS


def _run(
    config: Path,
    events: str,
    out: Path,
    overrides: list[str],
    split: str,
    eval_overrides: list[str],
) -> dict:
    name = load_config(config, overrides).name
    run_dir = out / name / f"n{events}"
    metrics = run_dir / f"{split}_metrics.json"
    if not metrics.exists():
        count = "null" if events == "all" else events
        args = [*overrides, f"data.train_events={count}", f"run_dir={run_dir}"]
        resolved = load_config(config, args)
        if resolved.model.name == "pfn" and resolved.checkpoint is None:
            # The run directory is new: the meta-trained weights are elsewhere.
            raise SystemExit("the PFN needs --override checkpoint=<meta-trained>.pt")
        if resolved.model.name not in IN_CONTEXT_MODELS:
            if gnn4colliders(["train", str(config), *args]) != 0:
                raise SystemExit(f"training {name} with {events} events failed")
        evaluate = ["evaluate", str(config), *args, f"split={split}", *eval_overrides]
        if gnn4colliders(evaluate) != 0:
            raise SystemExit(f"evaluating {name} with {events} events failed")
    return {"name": name, "events": events, **json.loads(metrics.read_text())}


def main() -> None:
    parser = argparse.ArgumentParser(
        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
    )
    parser.add_argument("configs", nargs="+", type=Path)
    parser.add_argument("--events", nargs="+", required=True, help="counts or all")
    parser.add_argument("--override", nargs="*", default=[], help="key=value")
    parser.add_argument("--split", default="validation", help="validation or test")
    parser.add_argument("--eval-override", nargs="*", default=[], help="key=value")
    parser.add_argument("--out", type=Path, default=Path("outputs/label_efficiency"))
    args = parser.parse_args()
    if not all(events == "all" or events.isdigit() for events in args.events):
        parser.error("--events takes counts or all")
    names = [load_config(config, args.override).name for config in args.configs]
    if len(set(names)) < len(names):
        raise SystemExit(f"configs share a run name ({names}); override name per run")
    runs = [
        _run(config, events, args.out, args.override, args.split, args.eval_override)
        for config in args.configs
        for events in args.events
    ]
    print("| model | " + " | ".join(f"N = {n}" for n in args.events) + " |")
    print("|---" * (len(args.events) + 1) + "|")
    for name in dict.fromkeys(run["name"] for run in runs):
        cells = []
        for events in args.events:
            run = next(r for r in runs if r["name"] == name and r["events"] == events)
            auc = 100 * run["metrics"]["auc"]
            seconds = run.get("seconds")
            total = f", {sum(seconds.values()):.0f} s" if seconds else ""
            cells.append(f"{auc:.2f}{total}")
        print(f"| {name} | " + " | ".join(cells) + " |")


if __name__ == "__main__":
    main()