GNN4Colliders / tests /test_workflows.py
ho22joshua's picture
Claude Opus 5.5
feat: evaluation subsample, Slurm wrappers for the study and label-efficiency plots
e71731f
Raw History Blame Contribute Delete
7.44 kB
"""End-to-end command-line runs on the 64-event ROOT fixtures."""
import json
import math
import multiprocessing as mp
import os
import socket
import sys
import numpy as np
import pytest
import torch
import uproot
from conftest import GAMGAM_FIXTURES, PRETRAINED, TTH_FIXTURE, fixture_config
from gnn4colliders.cli import main
from gnn4colliders.config import save_config
from gnn4colliders.data import load_events
from gnn4colliders.inference import write_root_scores
from gnn4colliders.train import load_checkpoint
def _config_file(config, tmp_path):
return str(save_config(config, tmp_path / "experiment.yaml"))
def _history(run):
return [
json.loads(line) for line in (run / "history.jsonl").read_text().splitlines()
]
def test_gamgam_train_evaluate_predict_export_and_resume(gamgam, tmp_path):
path, run = _config_file(gamgam, tmp_path), gamgam.path
assert main(["train", path, "train.max_epochs=3"]) == 0
history = _history(run)
assert [row["epoch"] for row in history] == [0, 1, 2]
for row in history:
assert row["learning_rate"][0] == pytest.approx(
1e-4 * 0.99 ** (row["epoch"] + 1)
)
assert {"accuracy", "auc", "f1", "precision", "recall"} <= set(
row["validation"]
)
best = load_checkpoint(run / "best.pt")
best_auc = max(row["validation"]["auc"] for row in history)
assert best["best_value"] == pytest.approx(best_auc)
assert history[best["epoch"]]["validation"]["auc"] == pytest.approx(best_auc)
assert load_checkpoint(run / "last.pt")["epoch"] == 2
assert main(["evaluate", path]) == 0
summary = json.loads((run / "test_metrics.json").read_text())
events = load_events(gamgam.data.cache)
assert summary["num_samples"] == int((events.fold == 9).sum())
assert all(math.isfinite(value) for value in summary["metrics"].values())
assert set(summary["seconds"]) == {"setup", "inference"}
subset = tmp_path / "subset.json"
assert main(["evaluate", path, "data.eval_events=10", f"output={subset}"]) == 0
assert json.loads(subset.read_text())["num_samples"] == 10
assert main(["predict", path, "split=validation"]) == 0
predictions = np.load(run / "validation_predictions.npz")
assert predictions["scores"].shape == (int((events.fold == 8).sum()), 5)
np.testing.assert_allclose(predictions["scores"].sum(axis=1), 1, rtol=1e-5)
assert (predictions["fold"] == 8).all()
first = str(predictions["source_file"][0])
entries = predictions["entry"][predictions["source_file"] == first]
tree = uproot.open(first)["mini"].arrays(["eventNumber"], library="np")
assert (tree["eventNumber"][entries] % 10 == 8).all()
pytest.importorskip("onnxruntime")
assert main(["export", path]) == 0
assert (run / "model.onnx").is_file() and (run / "model.json").is_file()
assert (
main(["train", path, "train.max_epochs=4", f"train.resume={run}/last.pt"]) == 0
)
assert [row["epoch"] for row in _history(run)] == [0, 1, 2, 3]
def test_root_scores_are_aligned_by_entry(tmp_path):
entries = np.array([3, 0, 5])
scores = np.array([[0.1, 0.9], [0.8, 0.2], [0.5, 0.5]])
output = write_root_scores(
scores,
entries,
input_path=GAMGAM_FIXTURES[0],
output_path=tmp_path / "scores.root",
tree_name="mini",
)
tree = uproot.open(output)["mini"].arrays(library="np")
assert tree["selection_pass"].tolist()[:6] == [1, 0, 0, 1, 0, 1]
assert tree["score_class_1"][3] == pytest.approx(0.9)
assert np.isnan(tree["score_class_0"][1])
# Jagged branches are copied unchanged.
original = uproot.open(GAMGAM_FIXTURES[0])["mini"]["jet_pt"].array()
copied = uproot.open(output)["mini"]["jet_pt"].array()
assert copied.tolist() == original.tolist()
def test_fine_tuning_from_the_pretrained_model(tmp_path):
config = fixture_config(
"gamgam_finetune",
GAMGAM_FIXTURES,
list(range(5)),
tmp_path,
"train.max_epochs=1",
f"model.pretrained={PRETRAINED}/multiclass_pretrained_model_12/model_epoch_71.pt",
)
path = _config_file(config, tmp_path)
assert main(["prepare", path]) == 0
assert main(["train", path]) == 0
assert main(["evaluate", path]) == 0
state = load_checkpoint(config.path / "best.pt")["model_state_dict"]
assert state["classifier.weight"].shape == (5, 64)
@pytest.mark.parametrize("name", ["gamgam_transformer", "gamgam_deep_sets"])
def test_sequence_models_train_on_the_same_cache(name, gamgam, tmp_path):
config = fixture_config(
name,
GAMGAM_FIXTURES,
list(range(5)),
tmp_path,
f"data.cache={gamgam.data.cache}",
"train.max_epochs=1",
)
path = _config_file(config, tmp_path)
assert main(["train", path]) == 0
assert main(["evaluate", path]) == 0
pytest.importorskip("onnxruntime")
assert main(["export", path]) == 0
assert json.loads((config.path / "model.json").read_text())["inputs"] == [
"tokens",
"token_mask",
]
def test_binary_delphes_task(tmp_path):
# The same file twice with labels 0 and 1 exercises the binary metrics.
config = fixture_config(
"tth_cp_even_odd",
[TTH_FIXTURE, TTH_FIXTURE],
[0, 1],
tmp_path,
"train.max_epochs=1",
)
path = _config_file(config, tmp_path)
assert main(["prepare", path]) == 0
assert main(["train", path]) == 0
assert math.isfinite(_history(config.path)[0]["validation"]["auc"])
def _port() -> int:
with socket.socket() as sock:
sock.bind(("127.0.0.1", 0))
return sock.getsockname()[1]
def _rank(rank: int, port: int, argv: list[str], queue) -> None:
os.environ.update(
RANK=str(rank),
LOCAL_RANK=str(rank),
WORLD_SIZE="2",
MASTER_ADDR="127.0.0.1",
MASTER_PORT=str(port),
)
torch.set_num_threads(1)
sys.argv = ["gnn4colliders", *argv]
queue.put((rank, main(argv)))
def test_training_runs_on_two_ddp_ranks(gamgam, tmp_path):
config = _config_file(gamgam, tmp_path)
# Gloo ranks on the CPU: MPS has no collectives.
argv = ["train", config, "train.max_epochs=2", "train.device=cpu"]
context = mp.get_context("spawn")
queue = context.Queue()
port = _port()
processes = [
context.Process(target=_rank, args=(rank, port, argv, queue))
for rank in range(2)
]
try:
for process in processes:
process.start()
for process in processes:
process.join(300)
assert [process.exitcode for process in processes] == [0, 0]
assert sorted(queue.get(timeout=5) for _ in processes) == [(0, 0), (1, 0)]
finally:
# A regression can deadlock a rank in a collective; never leave it.
for process in processes:
if process.is_alive():
process.kill()
process.join(5)
history = _history(gamgam.path)
events = load_events(gamgam.data.cache)
assert len(history) == 2
assert load_checkpoint(gamgam.path / "last.pt")["epoch"] == 1
# Validation covers every event once; training pads ranks to equal size.
assert history[0]["validation"]["events"] == int((events.fold == 8).sum())
train_events = int((events.fold < 8).sum())
assert history[0]["train"]["events"] == train_events + train_events % 2