Claude Opus 5.5
feat: evaluation subsample, Slurm wrappers for the study and label-efficiency plots
e71731f Download tests/test_workflows.py from HWresearch/GNN4Colliders: direct link, hf CLI and curl.
- Browser
- Download file 7.44 kB
-
https://huggingface.co/HWresearch/GNN4Colliders/resolve/main/tests/test_workflows.py
- Command line
-
hf download hf://HWresearch/GNN4Colliders/tests/test_workflows.py
-
curl -L -o test_workflows.py https://huggingface.co/HWresearch/GNN4Colliders/resolve/main/tests/test_workflows.py
7.44 kB
| """End-to-end command-line runs on the 64-event ROOT fixtures.""" | |
| import json | |
| import math | |
| import multiprocessing as mp | |
| import os | |
| import socket | |
| import sys | |
| import numpy as np | |
| import pytest | |
| import torch | |
| import uproot | |
| from conftest import GAMGAM_FIXTURES, PRETRAINED, TTH_FIXTURE, fixture_config | |
| from gnn4colliders.cli import main | |
| from gnn4colliders.config import save_config | |
| from gnn4colliders.data import load_events | |
| from gnn4colliders.inference import write_root_scores | |
| from gnn4colliders.train import load_checkpoint | |
| def _config_file(config, tmp_path): | |
| return str(save_config(config, tmp_path / "experiment.yaml")) | |
| def _history(run): | |
| return [ | |
| json.loads(line) for line in (run / "history.jsonl").read_text().splitlines() | |
| ] | |
| def test_gamgam_train_evaluate_predict_export_and_resume(gamgam, tmp_path): | |
| path, run = _config_file(gamgam, tmp_path), gamgam.path | |
| assert main(["train", path, "train.max_epochs=3"]) == 0 | |
| history = _history(run) | |
| assert [row["epoch"] for row in history] == [0, 1, 2] | |
| for row in history: | |
| assert row["learning_rate"][0] == pytest.approx( | |
| 1e-4 * 0.99 ** (row["epoch"] + 1) | |
| ) | |
| assert {"accuracy", "auc", "f1", "precision", "recall"} <= set( | |
| row["validation"] | |
| ) | |
| best = load_checkpoint(run / "best.pt") | |
| best_auc = max(row["validation"]["auc"] for row in history) | |
| assert best["best_value"] == pytest.approx(best_auc) | |
| assert history[best["epoch"]]["validation"]["auc"] == pytest.approx(best_auc) | |
| assert load_checkpoint(run / "last.pt")["epoch"] == 2 | |
| assert main(["evaluate", path]) == 0 | |
| summary = json.loads((run / "test_metrics.json").read_text()) | |
| events = load_events(gamgam.data.cache) | |
| assert summary["num_samples"] == int((events.fold == 9).sum()) | |
| assert all(math.isfinite(value) for value in summary["metrics"].values()) | |
| assert set(summary["seconds"]) == {"setup", "inference"} | |
| subset = tmp_path / "subset.json" | |
| assert main(["evaluate", path, "data.eval_events=10", f"output={subset}"]) == 0 | |
| assert json.loads(subset.read_text())["num_samples"] == 10 | |
| assert main(["predict", path, "split=validation"]) == 0 | |
| predictions = np.load(run / "validation_predictions.npz") | |
| assert predictions["scores"].shape == (int((events.fold == 8).sum()), 5) | |
| np.testing.assert_allclose(predictions["scores"].sum(axis=1), 1, rtol=1e-5) | |
| assert (predictions["fold"] == 8).all() | |
| first = str(predictions["source_file"][0]) | |
| entries = predictions["entry"][predictions["source_file"] == first] | |
| tree = uproot.open(first)["mini"].arrays(["eventNumber"], library="np") | |
| assert (tree["eventNumber"][entries] % 10 == 8).all() | |
| pytest.importorskip("onnxruntime") | |
| assert main(["export", path]) == 0 | |
| assert (run / "model.onnx").is_file() and (run / "model.json").is_file() | |
| assert ( | |
| main(["train", path, "train.max_epochs=4", f"train.resume={run}/last.pt"]) == 0 | |
| ) | |
| assert [row["epoch"] for row in _history(run)] == [0, 1, 2, 3] | |
| def test_root_scores_are_aligned_by_entry(tmp_path): | |
| entries = np.array([3, 0, 5]) | |
| scores = np.array([[0.1, 0.9], [0.8, 0.2], [0.5, 0.5]]) | |
| output = write_root_scores( | |
| scores, | |
| entries, | |
| input_path=GAMGAM_FIXTURES[0], | |
| output_path=tmp_path / "scores.root", | |
| tree_name="mini", | |
| ) | |
| tree = uproot.open(output)["mini"].arrays(library="np") | |
| assert tree["selection_pass"].tolist()[:6] == [1, 0, 0, 1, 0, 1] | |
| assert tree["score_class_1"][3] == pytest.approx(0.9) | |
| assert np.isnan(tree["score_class_0"][1]) | |
| # Jagged branches are copied unchanged. | |
| original = uproot.open(GAMGAM_FIXTURES[0])["mini"]["jet_pt"].array() | |
| copied = uproot.open(output)["mini"]["jet_pt"].array() | |
| assert copied.tolist() == original.tolist() | |
| def test_fine_tuning_from_the_pretrained_model(tmp_path): | |
| config = fixture_config( | |
| "gamgam_finetune", | |
| GAMGAM_FIXTURES, | |
| list(range(5)), | |
| tmp_path, | |
| "train.max_epochs=1", | |
| f"model.pretrained={PRETRAINED}/multiclass_pretrained_model_12/model_epoch_71.pt", | |
| ) | |
| path = _config_file(config, tmp_path) | |
| assert main(["prepare", path]) == 0 | |
| assert main(["train", path]) == 0 | |
| assert main(["evaluate", path]) == 0 | |
| state = load_checkpoint(config.path / "best.pt")["model_state_dict"] | |
| assert state["classifier.weight"].shape == (5, 64) | |
| def test_sequence_models_train_on_the_same_cache(name, gamgam, tmp_path): | |
| config = fixture_config( | |
| name, | |
| GAMGAM_FIXTURES, | |
| list(range(5)), | |
| tmp_path, | |
| f"data.cache={gamgam.data.cache}", | |
| "train.max_epochs=1", | |
| ) | |
| path = _config_file(config, tmp_path) | |
| assert main(["train", path]) == 0 | |
| assert main(["evaluate", path]) == 0 | |
| pytest.importorskip("onnxruntime") | |
| assert main(["export", path]) == 0 | |
| assert json.loads((config.path / "model.json").read_text())["inputs"] == [ | |
| "tokens", | |
| "token_mask", | |
| ] | |
| def test_binary_delphes_task(tmp_path): | |
| # The same file twice with labels 0 and 1 exercises the binary metrics. | |
| config = fixture_config( | |
| "tth_cp_even_odd", | |
| [TTH_FIXTURE, TTH_FIXTURE], | |
| [0, 1], | |
| tmp_path, | |
| "train.max_epochs=1", | |
| ) | |
| path = _config_file(config, tmp_path) | |
| assert main(["prepare", path]) == 0 | |
| assert main(["train", path]) == 0 | |
| assert math.isfinite(_history(config.path)[0]["validation"]["auc"]) | |
| def _port() -> int: | |
| with socket.socket() as sock: | |
| sock.bind(("127.0.0.1", 0)) | |
| return sock.getsockname()[1] | |
| def _rank(rank: int, port: int, argv: list[str], queue) -> None: | |
| os.environ.update( | |
| RANK=str(rank), | |
| LOCAL_RANK=str(rank), | |
| WORLD_SIZE="2", | |
| MASTER_ADDR="127.0.0.1", | |
| MASTER_PORT=str(port), | |
| ) | |
| torch.set_num_threads(1) | |
| sys.argv = ["gnn4colliders", *argv] | |
| queue.put((rank, main(argv))) | |
| def test_training_runs_on_two_ddp_ranks(gamgam, tmp_path): | |
| config = _config_file(gamgam, tmp_path) | |
| # Gloo ranks on the CPU: MPS has no collectives. | |
| argv = ["train", config, "train.max_epochs=2", "train.device=cpu"] | |
| context = mp.get_context("spawn") | |
| queue = context.Queue() | |
| port = _port() | |
| processes = [ | |
| context.Process(target=_rank, args=(rank, port, argv, queue)) | |
| for rank in range(2) | |
| ] | |
| try: | |
| for process in processes: | |
| process.start() | |
| for process in processes: | |
| process.join(300) | |
| assert [process.exitcode for process in processes] == [0, 0] | |
| assert sorted(queue.get(timeout=5) for _ in processes) == [(0, 0), (1, 0)] | |
| finally: | |
| # A regression can deadlock a rank in a collective; never leave it. | |
| for process in processes: | |
| if process.is_alive(): | |
| process.kill() | |
| process.join(5) | |
| history = _history(gamgam.path) | |
| events = load_events(gamgam.data.cache) | |
| assert len(history) == 2 | |
| assert load_checkpoint(gamgam.path / "last.pt")["epoch"] == 1 | |
| # Validation covers every event once; training pads ranks to equal size. | |
| assert history[0]["validation"]["events"] == int((events.fold == 8).sum()) | |
| train_events = int((events.fold < 8).sum()) | |
| assert history[0]["train"]["events"] == train_events + train_events % 2 | |