"""End-to-end command-line runs on the 64-event ROOT fixtures.""" import json import math import multiprocessing as mp import os import socket import sys import numpy as np import pytest import torch import uproot from conftest import GAMGAM_FIXTURES, PRETRAINED, TTH_FIXTURE, fixture_config from gnn4colliders.cli import main from gnn4colliders.config import save_config from gnn4colliders.data import load_events from gnn4colliders.inference import write_root_scores from gnn4colliders.train import load_checkpoint def _config_file(config, tmp_path): return str(save_config(config, tmp_path / "experiment.yaml")) def _history(run): return [ json.loads(line) for line in (run / "history.jsonl").read_text().splitlines() ] def test_gamgam_train_evaluate_predict_export_and_resume(gamgam, tmp_path): path, run = _config_file(gamgam, tmp_path), gamgam.path assert main(["train", path, "train.max_epochs=3"]) == 0 history = _history(run) assert [row["epoch"] for row in history] == [0, 1, 2] for row in history: assert row["learning_rate"][0] == pytest.approx( 1e-4 * 0.99 ** (row["epoch"] + 1) ) assert {"accuracy", "auc", "f1", "precision", "recall"} <= set( row["validation"] ) best = load_checkpoint(run / "best.pt") best_auc = max(row["validation"]["auc"] for row in history) assert best["best_value"] == pytest.approx(best_auc) assert history[best["epoch"]]["validation"]["auc"] == pytest.approx(best_auc) assert load_checkpoint(run / "last.pt")["epoch"] == 2 assert main(["evaluate", path]) == 0 summary = json.loads((run / "test_metrics.json").read_text()) events = load_events(gamgam.data.cache) assert summary["num_samples"] == int((events.fold == 9).sum()) assert all(math.isfinite(value) for value in summary["metrics"].values()) assert set(summary["seconds"]) == {"setup", "inference"} subset = tmp_path / "subset.json" assert main(["evaluate", path, "data.eval_events=10", f"output={subset}"]) == 0 assert json.loads(subset.read_text())["num_samples"] == 10 assert main(["predict", path, "split=validation"]) == 0 predictions = np.load(run / "validation_predictions.npz") assert predictions["scores"].shape == (int((events.fold == 8).sum()), 5) np.testing.assert_allclose(predictions["scores"].sum(axis=1), 1, rtol=1e-5) assert (predictions["fold"] == 8).all() first = str(predictions["source_file"][0]) entries = predictions["entry"][predictions["source_file"] == first] tree = uproot.open(first)["mini"].arrays(["eventNumber"], library="np") assert (tree["eventNumber"][entries] % 10 == 8).all() pytest.importorskip("onnxruntime") assert main(["export", path]) == 0 assert (run / "model.onnx").is_file() and (run / "model.json").is_file() assert ( main(["train", path, "train.max_epochs=4", f"train.resume={run}/last.pt"]) == 0 ) assert [row["epoch"] for row in _history(run)] == [0, 1, 2, 3] def test_root_scores_are_aligned_by_entry(tmp_path): entries = np.array([3, 0, 5]) scores = np.array([[0.1, 0.9], [0.8, 0.2], [0.5, 0.5]]) output = write_root_scores( scores, entries, input_path=GAMGAM_FIXTURES[0], output_path=tmp_path / "scores.root", tree_name="mini", ) tree = uproot.open(output)["mini"].arrays(library="np") assert tree["selection_pass"].tolist()[:6] == [1, 0, 0, 1, 0, 1] assert tree["score_class_1"][3] == pytest.approx(0.9) assert np.isnan(tree["score_class_0"][1]) # Jagged branches are copied unchanged. original = uproot.open(GAMGAM_FIXTURES[0])["mini"]["jet_pt"].array() copied = uproot.open(output)["mini"]["jet_pt"].array() assert copied.tolist() == original.tolist() def test_fine_tuning_from_the_pretrained_model(tmp_path): config = fixture_config( "gamgam_finetune", GAMGAM_FIXTURES, list(range(5)), tmp_path, "train.max_epochs=1", f"model.pretrained={PRETRAINED}/multiclass_pretrained_model_12/model_epoch_71.pt", ) path = _config_file(config, tmp_path) assert main(["prepare", path]) == 0 assert main(["train", path]) == 0 assert main(["evaluate", path]) == 0 state = load_checkpoint(config.path / "best.pt")["model_state_dict"] assert state["classifier.weight"].shape == (5, 64) @pytest.mark.parametrize("name", ["gamgam_transformer", "gamgam_deep_sets"]) def test_sequence_models_train_on_the_same_cache(name, gamgam, tmp_path): config = fixture_config( name, GAMGAM_FIXTURES, list(range(5)), tmp_path, f"data.cache={gamgam.data.cache}", "train.max_epochs=1", ) path = _config_file(config, tmp_path) assert main(["train", path]) == 0 assert main(["evaluate", path]) == 0 pytest.importorskip("onnxruntime") assert main(["export", path]) == 0 assert json.loads((config.path / "model.json").read_text())["inputs"] == [ "tokens", "token_mask", ] def test_binary_delphes_task(tmp_path): # The same file twice with labels 0 and 1 exercises the binary metrics. config = fixture_config( "tth_cp_even_odd", [TTH_FIXTURE, TTH_FIXTURE], [0, 1], tmp_path, "train.max_epochs=1", ) path = _config_file(config, tmp_path) assert main(["prepare", path]) == 0 assert main(["train", path]) == 0 assert math.isfinite(_history(config.path)[0]["validation"]["auc"]) def _port() -> int: with socket.socket() as sock: sock.bind(("127.0.0.1", 0)) return sock.getsockname()[1] def _rank(rank: int, port: int, argv: list[str], queue) -> None: os.environ.update( RANK=str(rank), LOCAL_RANK=str(rank), WORLD_SIZE="2", MASTER_ADDR="127.0.0.1", MASTER_PORT=str(port), ) torch.set_num_threads(1) sys.argv = ["gnn4colliders", *argv] queue.put((rank, main(argv))) def test_training_runs_on_two_ddp_ranks(gamgam, tmp_path): config = _config_file(gamgam, tmp_path) # Gloo ranks on the CPU: MPS has no collectives. argv = ["train", config, "train.max_epochs=2", "train.device=cpu"] context = mp.get_context("spawn") queue = context.Queue() port = _port() processes = [ context.Process(target=_rank, args=(rank, port, argv, queue)) for rank in range(2) ] try: for process in processes: process.start() for process in processes: process.join(300) assert [process.exitcode for process in processes] == [0, 0] assert sorted(queue.get(timeout=5) for _ in processes) == [(0, 0), (1, 0)] finally: # A regression can deadlock a rank in a collective; never leave it. for process in processes: if process.is_alive(): process.kill() process.join(5) history = _history(gamgam.path) events = load_events(gamgam.data.cache) assert len(history) == 2 assert load_checkpoint(gamgam.path / "last.pt")["epoch"] == 1 # Validation covers every event once; training pads ranks to equal size. assert history[0]["validation"]["events"] == int((events.fold == 8).sum()) train_events = int((events.fold < 8).sum()) assert history[0]["train"]["events"] == train_events + train_events % 2