ho22joshua Claude Opus 5.5 commited on
Commit
e71731f
·
1 Parent(s): 8dff74c

feat: evaluation subsample, Slurm wrappers for the study and label-efficiency plots

Browse files

- data.eval_events: evaluate (validation in training, evaluate, predict,
probe) only that many seeded random validation or test events.
- scripts/label_efficiency.sh, meta_train.sh, pretrain.sh: one-GPU Slurm
wrappers for the sweep, meta-training then in-context sweep, and
pretraining then probes.
- scripts/plot_label_efficiency.py: AUC against labelled events from sweeps
and probes, as PDF/PNG and Markdown/LaTeX tables; optional extra `plots`.

Wrappers smoke-tested outside Slurm on fixtures; tests pass on CPU and MPS.
No change to existing models' results.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

docs/usage.md CHANGED
@@ -15,6 +15,7 @@ uv sync --dev # includes pytest, ruff and the ONNX packages
15
  uv sync --dev --extra tabpfn # also TabPFN (configs/gamgam_tabpfn.yaml)
16
  uv sync --dev --extra tabicl # also TabICL (configs/gamgam_tabicl.yaml)
17
  uv sync --dev --extra tabdpt # also TabDPT (configs/gamgam_tabdpt.yaml)
 
18
  uv run pytest # about 2 minutes; GPU tests run when CUDA is available
19
  uv run pytest --device mps # the fixture runs on an Apple GPU
20
  ```
@@ -49,6 +50,10 @@ the encoder of `model.pretrained` (JEPA, contrastive or supervised; random
49
  without) on `data.train_events` training events and reports their metrics
50
  on `split`, to compare pretraining without fine-tuning.
51
 
 
 
 
 
52
  `evaluate`, `predict` and `export` use `checkpoint` (default `best.pt`, else
53
  `last.pt`) on `split` (default `test`); `output` overrides the output file.
54
  Predictions hold the logits, scores, predicted class, label, weight, fold,
@@ -221,6 +226,23 @@ done
221
  uv run python scripts/summarize_ensemble.py outputs/gamgam/seed* # mean +- std
222
  ```
223
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
224
  Under `torchrun` with more than one process, training is data parallel:
225
  `data.batch_size` is per process, training events are sharded over ranks
226
  (padded to equal length) and metrics are computed over all ranks' events.
 
15
  uv sync --dev --extra tabpfn # also TabPFN (configs/gamgam_tabpfn.yaml)
16
  uv sync --dev --extra tabicl # also TabICL (configs/gamgam_tabicl.yaml)
17
  uv sync --dev --extra tabdpt # also TabDPT (configs/gamgam_tabdpt.yaml)
18
+ uv sync --dev --extra plots # matplotlib, for scripts/plot_label_efficiency.py
19
  uv run pytest # about 2 minutes; GPU tests run when CUDA is available
20
  uv run pytest --device mps # the fixture runs on an Apple GPU
21
  ```
 
50
  without) on `data.train_events` training events and reports their metrics
51
  on `split`, to compare pretraining without fine-tuning.
52
 
53
+ `data.eval_events=20000` evaluates (and validates during training,
54
+ predicts and probes) only that many random validation or test events,
55
+ seeded by `data.seed`: AUCs to about 0.003, in minutes, for quick runs.
56
+
57
  `evaluate`, `predict` and `export` use `checkpoint` (default `best.pt`, else
58
  `last.pt`) on `split` (default `test`); `output` overrides the output file.
59
  Predictions hold the logits, scores, predicted class, label, weight, fold,
 
226
  uv run python scripts/summarize_ensemble.py outputs/gamgam/seed* # mean +- std
227
  ```
228
 
229
+ The study of in-context models and pretraining has one wrapper per kind of
230
+ job, each on one GPU (the header of each script lists its variables):
231
+
232
+ ```bash
233
+ # Label-efficiency sweep of released in-context models and supervised references:
234
+ sbatch -A <account> --export=ALL,CONFIGS="configs/gamgam_tabpfn.yaml configs/gamgam_tabicl.yaml configs/gamgam.yaml" \
235
+ scripts/label_efficiency.sh
236
+ # Meta-training, then GamGam in context with the meta-trained model:
237
+ sbatch -A <account> --export=ALL,CONFIG=configs/delphes_tabicl.yaml,EVAL_CONFIG=configs/gamgam_tabicl.yaml \
238
+ scripts/meta_train.sh task.prior_classes=[1,6,7,8,9,10,11]
239
+ # Label-free pretraining, then k-NN and linear probes on GamGam:
240
+ sbatch -A <account> --export=ALL,CONFIG=configs/delphes_contrastive.yaml scripts/pretrain.sh
241
+ # Curves and tables of every finished run:
242
+ uv run --extra plots python scripts/plot_label_efficiency.py --sweeps outputs/label_efficiency \
243
+ --probes outputs/delphes_jepa outputs/delphes_contrastive outputs/delphes_masked
244
+ ```
245
+
246
  Under `torchrun` with more than one process, training is data parallel:
247
  `data.batch_size` is per process, training events are sharded over ranks
248
  (padded to equal length) and metrics are computed over all ranks' events.
pyproject.toml CHANGED
@@ -31,6 +31,9 @@ tabicl = [
31
  tabdpt = [
32
  "tabdpt>=1.3.1,<2",
33
  ]
 
 
 
34
 
35
  [project.scripts]
36
  gnn4colliders = "gnn4colliders.cli:main"
 
31
  tabdpt = [
32
  "tabdpt>=1.3.1,<2",
33
  ]
34
+ plots = [
35
+ "matplotlib>=3.9,<4",
36
+ ]
37
 
38
  [project.scripts]
39
  gnn4colliders = "gnn4colliders.cli:main"
scripts/label_efficiency.sh ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ # Label-efficiency sweep (scripts/label_efficiency.py) on one GPU:
3
+ # sbatch -A <account> --export=ALL,CONFIGS="configs/gamgam_tabpfn.yaml configs/gamgam.yaml" \
4
+ # scripts/label_efficiency.sh [key=value ...]
5
+ # EVENTS (default "100 1000 10000 50000") are the labelled training events;
6
+ # key=value overrides apply to every config. Restartable: finished runs are
7
+ # skipped. TabPFN-3.5 needs its weights in TABPFN_MODEL_CACHE_DIR
8
+ # (docs/usage.md). Prepare the event cache first.
9
+ #SBATCH -N 1
10
+ #SBATCH -C gpu
11
+ #SBATCH -q shared
12
+ #SBATCH -G 1
13
+ #SBATCH -c 32
14
+ #SBATCH -t 06:00:00
15
+ #SBATCH -o logs/label_efficiency_%j.out
16
+
17
+ set -euo pipefail
18
+ cd "${SLURM_SUBMIT_DIR:-$(pwd)}"
19
+ read -r -a configs <<< "${CONFIGS:-configs/gamgam_tabpfn.yaml}"
20
+ read -r -a events <<< "${EVENTS:-100 1000 10000 50000}"
21
+ export OMP_NUM_THREADS="${OMP_NUM_THREADS:-8}"
22
+ export UV_CACHE_DIR="${UV_CACHE_DIR:-${SCRATCH:-${HOME}/.cache}/uv}"
23
+ export HF_HOME="${HF_HOME:-${SCRATCH:-${HOME}/.cache}/huggingface}"
24
+ export TABPFN_MODEL_CACHE_DIR="${TABPFN_MODEL_CACHE_DIR:-${SCRATCH:-${HOME}/.cache}/tabpfn}"
25
+ mkdir -p logs
26
+ overrides=()
27
+ if (($#)); then overrides=(--override "$@"); fi
28
+ uv run --all-extras python scripts/label_efficiency.py "${configs[@]}" \
29
+ --events "${events[@]}" --out "${OUT:-outputs/label_efficiency}" ${overrides[@]+"${overrides[@]}"}
scripts/meta_train.sh ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ # Meta-train an in-context model on one GPU, then classify GamGam in context
3
+ # with its best checkpoint for each number of labelled events:
4
+ # sbatch -A <account> --export=ALL,CONFIG=configs/delphes_tabicl.yaml,EVAL_CONFIG=configs/gamgam_tabicl.yaml \
5
+ # scripts/meta_train.sh [key=value ...]
6
+ # CONFIG is configs/delphes_{pfn,tabpfn,tabicl}.yaml and EVAL_CONFIG the
7
+ # matching configs/gamgam_*.yaml (none: meta-training only). key=value
8
+ # overrides apply to both (e.g. task.prior_classes=[1,6,7,8,9,10,11], which
9
+ # leaves the GamGam processes out; model.lora_rank=0). Prepare the Delphes
10
+ # and GamGam event caches first.
11
+ #SBATCH -N 1
12
+ #SBATCH -C gpu
13
+ #SBATCH -q shared
14
+ #SBATCH -G 1
15
+ #SBATCH -c 32
16
+ #SBATCH -t 12:00:00
17
+ #SBATCH -o logs/meta_train_%j.out
18
+
19
+ set -euo pipefail
20
+ cd "${SLURM_SUBMIT_DIR:-$(pwd)}"
21
+ CONFIG="${CONFIG:-configs/delphes_pfn.yaml}"
22
+ name="$(basename "${CONFIG}" .yaml)"
23
+ RUN="${RUN:-outputs/${name}}"
24
+ export OMP_NUM_THREADS="${OMP_NUM_THREADS:-8}"
25
+ export UV_CACHE_DIR="${UV_CACHE_DIR:-${SCRATCH:-${HOME}/.cache}/uv}"
26
+ export HF_HOME="${HF_HOME:-${SCRATCH:-${HOME}/.cache}/huggingface}"
27
+ export TABPFN_MODEL_CACHE_DIR="${TABPFN_MODEL_CACHE_DIR:-${SCRATCH:-${HOME}/.cache}/tabpfn}"
28
+ mkdir -p logs
29
+ uv run --all-extras gnn4colliders train "${CONFIG}" run_dir="${RUN}" "$@"
30
+ if [[ -n "${EVAL_CONFIG:-}" ]]; then
31
+ read -r -a events <<< "${EVENTS:-100 1000 10000 50000}"
32
+ evaluated="$(basename "${EVAL_CONFIG}" .yaml)_from_${name}"
33
+ uv run --all-extras python scripts/label_efficiency.py "${EVAL_CONFIG}" \
34
+ --events "${events[@]}" --out "${OUT:-outputs/label_efficiency}" \
35
+ --override name="${evaluated}" checkpoint="${RUN}/best.pt" "$@"
36
+ fi
scripts/plot_label_efficiency.py ADDED
@@ -0,0 +1,184 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Plot validation AUC against the number of labelled training events, and
2
+ write the same numbers as Markdown and LaTeX tables.
3
+
4
+ Reads the sweeps of ``scripts/label_efficiency.py`` (``<out>/<name>/n<N>/
5
+ <split>_metrics.json``, one series per ``<name>``) and the probes of
6
+ ``scripts/pretrain.sh`` (``<run>/probe/<split>_n<N>.json``, two series per
7
+ run: k-NN and linear):
8
+
9
+ uv run --extra plots python scripts/plot_label_efficiency.py \\
10
+ --sweeps outputs/label_efficiency --probes outputs/delphes_jepa \\
11
+ --output plots/label_efficiency
12
+
13
+ writes ``plots/label_efficiency.{pdf,png,md,tex}``. ``--metric`` chooses
14
+ another metric (``accuracy``, ``f1``, ...).
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ import argparse
20
+ import json
21
+ import re
22
+ from pathlib import Path
23
+
24
+ #: Categorical colours in fixed order (validated for colour-vision
25
+ #: deficiency on a light background) and marker shapes, so that series are
26
+ #: not told apart by colour alone.
27
+ COLOURS = (
28
+ "#2a78d6",
29
+ "#eb6834",
30
+ "#1baf7a",
31
+ "#eda100",
32
+ "#e87ba4",
33
+ "#008300",
34
+ "#6250d6",
35
+ "#e34948",
36
+ )
37
+ MARKERS = ("o", "s", "^", "D", "v", "P", "X", "*")
38
+ TEXT, MUTED, GRID = "#2b2b29", "#6b6a64", "#e4e3de"
39
+
40
+
41
+ def _sweeps(root: Path, split: str, metric: str) -> dict[str, dict[int, float]]:
42
+ series: dict[str, dict[int, float]] = {}
43
+ for path in sorted(root.glob(f"*/n*/{split}_metrics.json")):
44
+ events = int(path.parent.name.removeprefix("n"))
45
+ value = json.loads(path.read_text())["metrics"][metric]
46
+ series.setdefault(path.parent.parent.name, {})[events] = value
47
+ return series
48
+
49
+
50
+ def _probes(run: Path, split: str, metric: str) -> dict[str, dict[int, float]]:
51
+ series: dict[str, dict[int, float]] = {}
52
+ for path in sorted((run / "probe").glob(f"{split}_n*.json")):
53
+ match = re.fullmatch(rf"{split}_n(\d+)\.json", path.name)
54
+ if match is None:
55
+ continue
56
+ summary = json.loads(path.read_text())
57
+ for probe, label in (("knn", "k-NN"), ("linear", "linear")):
58
+ name = f"{run.name} ({label} probe)"
59
+ series.setdefault(name, {})[int(match[1])] = summary[probe][metric]
60
+ return series
61
+
62
+
63
+ def _tables(series: dict[str, dict[int, float]], metric: str) -> tuple[str, str]:
64
+ counts = sorted({count for values in series.values() for count in values})
65
+
66
+ def cell(values: dict[int, float], count: int) -> str:
67
+ return f"{100 * values[count]:.2f}" if count in values else "--"
68
+
69
+ header = ["model", *(f"N = {count}" for count in counts)]
70
+ markdown = [
71
+ "| " + " | ".join(header) + " |",
72
+ "|---" * len(header) + "|",
73
+ *(
74
+ "| " + " | ".join([name, *(cell(v, c) for c in counts)]) + " |"
75
+ for name, v in series.items()
76
+ ),
77
+ ]
78
+ latex = [
79
+ r"\begin{tabular}{l" + "r" * len(counts) + "}",
80
+ r"\hline",
81
+ " & ".join(["Model", *(f"$N={count}$" for count in counts)]) + r" \\",
82
+ r"\hline",
83
+ *(
84
+ " & ".join([name.replace("_", r"\_"), *(cell(v, c) for c in counts)])
85
+ + r" \\"
86
+ for name, v in series.items()
87
+ ),
88
+ r"\hline",
89
+ r"\end{tabular}",
90
+ f"% {metric} (%) against labelled training events N",
91
+ ]
92
+ return "\n".join(markdown) + "\n", "\n".join(latex) + "\n"
93
+
94
+
95
+ def _plot(series: dict[str, dict[int, float]], metric: str, output: Path) -> None:
96
+ import matplotlib
97
+
98
+ matplotlib.use("Agg")
99
+ import matplotlib.pyplot as plt
100
+ from matplotlib.ticker import FuncFormatter, LogLocator
101
+
102
+ if len(series) > len(COLOURS):
103
+ raise SystemExit(f"at most {len(COLOURS)} series; plot fewer per figure")
104
+ figure, axes = plt.subplots(figsize=(6.4, 4.2), layout="constrained")
105
+ figure.set_facecolor("#fcfcfb")
106
+ axes.set_facecolor("#fcfcfb")
107
+ for index, (name, values) in enumerate(series.items()):
108
+ counts = sorted(values)
109
+ percent = [100 * values[count] for count in counts]
110
+ axes.plot(
111
+ counts,
112
+ percent,
113
+ color=COLOURS[index],
114
+ marker=MARKERS[index],
115
+ markersize=7,
116
+ linewidth=2,
117
+ markeredgecolor="#fcfcfb",
118
+ markeredgewidth=1.5,
119
+ label=name,
120
+ )
121
+ # A direct label at the last point when there are few series.
122
+ if len(series) <= 4:
123
+ axes.annotate(
124
+ name,
125
+ (counts[-1], percent[-1]),
126
+ xytext=(6, 0),
127
+ textcoords="offset points",
128
+ va="center",
129
+ fontsize=8,
130
+ color=TEXT,
131
+ )
132
+ axes.set_xscale("log")
133
+ axes.xaxis.set_major_locator(LogLocator(base=10))
134
+ axes.xaxis.set_major_formatter(FuncFormatter(lambda x, _: f"{x:,.0f}"))
135
+ if len(series) <= 4: # room on the right for the direct labels
136
+ low, high = axes.get_xlim()
137
+ axes.set_xlim(low, high * 4)
138
+ axes.set_xlabel("labelled training events", color=TEXT)
139
+ label = "AUC" if metric == "auc" else metric
140
+ axes.set_ylabel(f"validation {label} (%)", color=TEXT)
141
+ axes.grid(True, which="major", color=GRID, linewidth=0.8)
142
+ axes.tick_params(colors=MUTED)
143
+ for spine in ("top", "right"):
144
+ axes.spines[spine].set_visible(False)
145
+ for spine in ("left", "bottom"):
146
+ axes.spines[spine].set_color(GRID)
147
+ if len(series) > 1:
148
+ axes.legend(frameon=False, fontsize=8, labelcolor=TEXT)
149
+ for suffix in (".pdf", ".png"):
150
+ figure.savefig(output.with_suffix(suffix), dpi=200)
151
+ plt.close(figure)
152
+
153
+
154
+ def main() -> None:
155
+ parser = argparse.ArgumentParser(
156
+ description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
157
+ )
158
+ parser.add_argument("--sweeps", nargs="*", type=Path, default=[])
159
+ parser.add_argument("--probes", nargs="*", type=Path, default=[])
160
+ parser.add_argument("--split", default="validation")
161
+ parser.add_argument("--metric", default="auc")
162
+ parser.add_argument("--output", type=Path, default=Path("plots/label_efficiency"))
163
+ args = parser.parse_args()
164
+ series: dict[str, dict[int, float]] = {}
165
+ found = [_sweeps(root, args.split, args.metric) for root in args.sweeps]
166
+ found += [_probes(run, args.split, args.metric) for run in args.probes]
167
+ for part in found:
168
+ for name, values in part.items():
169
+ merged = series.setdefault(name, {})
170
+ if merged.keys() & values.keys():
171
+ raise SystemExit(f"two results for {name} at the same N; plot apart")
172
+ merged.update(values)
173
+ if not series:
174
+ raise SystemExit("no results found")
175
+ args.output.parent.mkdir(parents=True, exist_ok=True)
176
+ markdown, latex = _tables(series, args.metric)
177
+ args.output.with_suffix(".md").write_text(markdown)
178
+ args.output.with_suffix(".tex").write_text(latex)
179
+ _plot(series, args.metric, args.output)
180
+ print(markdown, end="")
181
+
182
+
183
+ if __name__ == "__main__":
184
+ main()
scripts/pretrain.sh ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ # Label-free pretraining of an object encoder on one GPU, then k-NN and
3
+ # linear probes of its last epoch on GamGam for each number of labelled
4
+ # events:
5
+ # sbatch -A <account> --export=ALL,CONFIG=configs/delphes_contrastive.yaml scripts/pretrain.sh [key=value ...]
6
+ # CONFIG is configs/delphes_{jepa,contrastive,masked}.yaml (or
7
+ # delphes_jepa_edge.yaml with PROBE_CONFIG=configs/gamgam_finetune.yaml);
8
+ # PROBE_CONFIG (default configs/gamgam_jepa_finetune.yaml) must have the
9
+ # encoder's sizes. The probes evaluate EVAL_EVENTS (default 20000) random
10
+ # validation events. key=value overrides apply to the pretraining. For
11
+ # several GPUs, pretrain with scripts/train_ddp.sh instead. Prepare the
12
+ # Delphes and GamGam event caches first.
13
+ #SBATCH -N 1
14
+ #SBATCH -C gpu
15
+ #SBATCH -q shared
16
+ #SBATCH -G 1
17
+ #SBATCH -c 32
18
+ #SBATCH -t 12:00:00
19
+ #SBATCH -o logs/pretrain_%j.out
20
+
21
+ set -euo pipefail
22
+ cd "${SLURM_SUBMIT_DIR:-$(pwd)}"
23
+ CONFIG="${CONFIG:-configs/delphes_jepa.yaml}"
24
+ PROBE_CONFIG="${PROBE_CONFIG:-configs/gamgam_jepa_finetune.yaml}"
25
+ name="$(basename "${CONFIG}" .yaml)"
26
+ RUN="${RUN:-outputs/${name}}"
27
+ read -r -a events <<< "${EVENTS:-100 1000 10000}"
28
+ export OMP_NUM_THREADS="${OMP_NUM_THREADS:-8}"
29
+ export UV_CACHE_DIR="${UV_CACHE_DIR:-${SCRATCH:-${HOME}/.cache}/uv}"
30
+ export HF_HOME="${HF_HOME:-${SCRATCH:-${HOME}/.cache}/huggingface}"
31
+ mkdir -p logs
32
+ uv run gnn4colliders train "${CONFIG}" run_dir="${RUN}" "$@"
33
+ for count in "${events[@]}"; do
34
+ uv run gnn4colliders probe "${PROBE_CONFIG}" split=validation \
35
+ model.pretrained="${RUN}/last.pt" data.train_events="${count}" \
36
+ data.eval_events="${EVAL_EVENTS:-20000}" \
37
+ output="${RUN}/probe/validation_n${count}.json"
38
+ done
src/gnn4colliders/cli.py CHANGED
@@ -85,6 +85,8 @@ def make_loaders(
85
  indices = events.split_indices(folds[split])
86
  if split == "train" and data.train_events is not None:
87
  indices = events.balanced_subset(indices, data.train_events, data.seed)
 
 
88
  if config.task.name == "pfn":
89
  from .pfn import TaskLoader
90
 
@@ -429,6 +431,8 @@ def run_probe(config: Config) -> Path:
429
  queries = events.split_indices(folds[config.split])
430
  # The probes are not scored on the events they are fitted to.
431
  queries = queries[~torch.isin(queries, train)]
 
 
432
  device = resolve_device(config.train.device)
433
  start = time.perf_counter()
434
  # All the dataset's classes, whatever the config's task (a pretraining
 
85
  indices = events.split_indices(folds[split])
86
  if split == "train" and data.train_events is not None:
87
  indices = events.balanced_subset(indices, data.train_events, data.seed)
88
+ if split != "train" and data.eval_events is not None:
89
+ indices = events.random_subset(indices, data.eval_events, data.seed)
90
  if config.task.name == "pfn":
91
  from .pfn import TaskLoader
92
 
 
431
  queries = events.split_indices(folds[config.split])
432
  # The probes are not scored on the events they are fitted to.
433
  queries = queries[~torch.isin(queries, train)]
434
+ if config.split != "train" and data.eval_events is not None:
435
+ queries = events.random_subset(queries, data.eval_events, data.seed)
436
  device = resolve_device(config.train.device)
437
  start = time.perf_counter()
438
  # All the dataset's classes, whatever the config's task (a pretraining
src/gnn4colliders/config.py CHANGED
@@ -44,6 +44,10 @@ class DataConfig:
44
  #: Train on (and give in-context models as context) only this many
45
  #: training events, an equal number per class drawn with ``seed``.
46
  train_events: int | None = None
 
 
 
 
47
  #: Events per batch and process.
48
  batch_size: int = 1024
49
  shuffle: bool = True
 
44
  #: Train on (and give in-context models as context) only this many
45
  #: training events, an equal number per class drawn with ``seed``.
46
  train_events: int | None = None
47
+ #: Evaluate (validation in training, evaluate, predict, probe) only this
48
+ #: many random validation or test events, drawn with ``seed``: 20000
49
+ #: give AUCs to about 0.003, for quick runs.
50
+ eval_events: int | None = None
51
  #: Events per batch and process.
52
  batch_size: int = 1024
53
  shuffle: bool = True
src/gnn4colliders/data.py CHANGED
@@ -118,6 +118,14 @@ class PackedEvents:
118
  chosen.append(members[order[: count // len(classes)]])
119
  return torch.cat(chosen).sort().values
120
 
 
 
 
 
 
 
 
 
121
 
122
  def pack_events(
123
  objects: Sequence[torch.Tensor],
 
118
  chosen.append(members[order[: count // len(classes)]])
119
  return torch.cat(chosen).sort().values
120
 
121
+ @staticmethod
122
+ def random_subset(indices: torch.Tensor, count: int, seed: int) -> torch.Tensor:
123
+ """Up to ``count`` random events of ``indices`` (keeping the class
124
+ proportions on average), in stored order."""
125
+ generator = torch.Generator().manual_seed(seed)
126
+ order = torch.randperm(len(indices), generator=generator)
127
+ return indices[order[:count]].sort().values
128
+
129
 
130
  def pack_events(
131
  objects: Sequence[torch.Tensor],
tests/test_tabular.py CHANGED
@@ -15,7 +15,7 @@ from torch.nn.utils import parametrize
15
 
16
  from gnn4colliders.cli import main
17
  from gnn4colliders.config import ModelConfig, save_config
18
- from gnn4colliders.data import pack_events
19
  from gnn4colliders.sequences import collate_sequence_batch
20
  from gnn4colliders.tabular import (
21
  EventInputs,
@@ -112,6 +112,15 @@ def test_lora_starts_at_the_frozen_weight_and_trains_only_its_update():
112
  torch.testing.assert_close(network(x), expected)
113
 
114
 
 
 
 
 
 
 
 
 
 
115
  def test_balanced_subsets_are_equal_per_class_and_nested():
116
  labels = torch.tensor([0] * 50 + [1] * 30 + [2] * 5)
117
  events = pack_events([torch.zeros(1, 7)] * len(labels), labels)
 
15
 
16
  from gnn4colliders.cli import main
17
  from gnn4colliders.config import ModelConfig, save_config
18
+ from gnn4colliders.data import PackedEvents, pack_events
19
  from gnn4colliders.sequences import collate_sequence_batch
20
  from gnn4colliders.tabular import (
21
  EventInputs,
 
112
  torch.testing.assert_close(network(x), expected)
113
 
114
 
115
+ def test_random_evaluation_subsets_are_seeded_and_sorted():
116
+ indices = torch.arange(100) * 3
117
+ subset = PackedEvents.random_subset(indices, 20, seed=1)
118
+ assert len(subset) == 20 and set(subset.tolist()) <= set(indices.tolist())
119
+ assert torch.equal(subset, subset.sort().values)
120
+ assert torch.equal(subset, PackedEvents.random_subset(indices, 20, seed=1))
121
+ assert len(PackedEvents.random_subset(indices, 500, seed=1)) == 100
122
+
123
+
124
  def test_balanced_subsets_are_equal_per_class_and_nested():
125
  labels = torch.tensor([0] * 50 + [1] * 30 + [2] * 5)
126
  events = pack_events([torch.zeros(1, 7)] * len(labels), labels)
tests/test_workflows.py CHANGED
@@ -55,6 +55,10 @@ def test_gamgam_train_evaluate_predict_export_and_resume(gamgam, tmp_path):
55
  assert all(math.isfinite(value) for value in summary["metrics"].values())
56
  assert set(summary["seconds"]) == {"setup", "inference"}
57
 
 
 
 
 
58
  assert main(["predict", path, "split=validation"]) == 0
59
  predictions = np.load(run / "validation_predictions.npz")
60
  assert predictions["scores"].shape == (int((events.fold == 8).sum()), 5)
 
55
  assert all(math.isfinite(value) for value in summary["metrics"].values())
56
  assert set(summary["seconds"]) == {"setup", "inference"}
57
 
58
+ subset = tmp_path / "subset.json"
59
+ assert main(["evaluate", path, "data.eval_events=10", f"output={subset}"]) == 0
60
+ assert json.loads(subset.read_text())["num_samples"] == 10
61
+
62
  assert main(["predict", path, "split=validation"]) == 0
63
  predictions = np.load(run / "validation_predictions.npz")
64
  assert predictions["scores"].shape == (int((events.fold == 8).sum()), 5)
uv.lock CHANGED
@@ -409,6 +409,9 @@ onnx = [
409
  { name = "onnx" },
410
  { name = "onnxruntime" },
411
  ]
 
 
 
412
  tabdpt = [
413
  { name = "tabdpt" },
414
  ]
@@ -431,6 +434,7 @@ dev = [
431
  requires-dist = [
432
  { name = "awkward", specifier = ">=2.0" },
433
  { name = "huggingface-hub", specifier = ">=0.36,<1" },
 
434
  { name = "numpy", specifier = ">=2.1" },
435
  { name = "onnx", marker = "extra == 'onnx'", specifier = ">=1.16,<2" },
436
  { name = "onnxruntime", marker = "extra == 'onnx'", specifier = ">=1.18,<2" },
@@ -442,7 +446,7 @@ requires-dist = [
442
  { name = "torch", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'", specifier = "==2.14.1", index = "https://download.pytorch.org/whl/cu130" },
443
  { name = "uproot", specifier = ">=5.0" },
444
  ]
445
- provides-extras = ["onnx", "tabpfn", "tabicl", "tabdpt"]
446
 
447
  [package.metadata.requires-dev]
448
  dev = [
 
409
  { name = "onnx" },
410
  { name = "onnxruntime" },
411
  ]
412
+ plots = [
413
+ { name = "matplotlib" },
414
+ ]
415
  tabdpt = [
416
  { name = "tabdpt" },
417
  ]
 
434
  requires-dist = [
435
  { name = "awkward", specifier = ">=2.0" },
436
  { name = "huggingface-hub", specifier = ">=0.36,<1" },
437
+ { name = "matplotlib", marker = "extra == 'plots'", specifier = ">=3.9,<4" },
438
  { name = "numpy", specifier = ">=2.1" },
439
  { name = "onnx", marker = "extra == 'onnx'", specifier = ">=1.16,<2" },
440
  { name = "onnxruntime", marker = "extra == 'onnx'", specifier = ">=1.18,<2" },
 
446
  { name = "torch", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'", specifier = "==2.14.1", index = "https://download.pytorch.org/whl/cu130" },
447
  { name = "uproot", specifier = ">=5.0" },
448
  ]
449
+ provides-extras = ["onnx", "tabpfn", "tabicl", "tabdpt", "plots"]
450
 
451
  [package.metadata.requires-dev]
452
  dev = [