import copy import math import pytest import torch from conftest import random_events from gnn4colliders.graphs import collate_graph_batch from gnn4colliders.loader import EventLoader from gnn4colliders.models import EdgeNetwork from gnn4colliders.tasks import MulticlassClassificationTask from gnn4colliders.train import ( EarlyStopping, Trainer, load_checkpoint, restore, save_checkpoint, ) cuda = pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA unavailable") def _trainer(seed=0, **options): torch.manual_seed(seed) model = EdgeNetwork(7, 3, 0, 8, 3, 2, 1) optimizer = torch.optim.Adam(model.parameters(), lr=1e-2) scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.5) return Trainer( model, MulticlassClassificationTask(), optimizer, scheduler, **options ) def _loader(events, **options): return EventLoader(events, collate_graph_batch, 8, **options) def test_training_reduces_the_loss_and_follows_the_schedule(): events = random_events() trainer = _trainer() seen = [] history = trainer.fit( _loader(events, shuffle=True), _loader(events), max_epochs=4, on_epoch_end=lambda train, validation: seen.append( (trainer.epoch, trainer.optimizer.param_groups[0]["lr"]) ), ) assert seen == [(0, 5e-3), (1, 2.5e-3), (2, 1.25e-3), (3, 6.25e-4)] assert history[-1][0].loss < history[0][0].loss assert history[0][1].num_samples == 40 assert {"accuracy", "auc", "f1"} <= set(history[0][1].metrics) def test_training_loss_early_stopping_needs_no_validation(): trainer = _trainer(early_stopping=EarlyStopping("train_loss", 1, min_delta=10.0)) history = trainer.fit(_loader(random_events()), max_epochs=5) # The first epoch sets the best; the second cannot beat it by min_delta. assert len(history) == 2 and trainer.epoch == 1 def test_non_finite_loss_is_reported_at_the_end_of_the_epoch(): events = random_events() events.object_features[0, 0] = float("nan") with pytest.raises(FloatingPointError, match="batch 0"): _trainer().train_epoch(_loader(events)) def test_resumed_training_continues_identically(tmp_path): events = random_events() straight = _trainer() straight.fit(_loader(events, shuffle=True), max_epochs=3) first = _trainer() first.fit(_loader(events, shuffle=True), max_epochs=1) save_checkpoint(tmp_path / "last.pt", first) resumed = _trainer(seed=1) restore(resumed, load_checkpoint(tmp_path / "last.pt")) resumed.fit(_loader(events, shuffle=True), max_epochs=3) assert resumed.epoch == 2 for left, right in zip(straight.model.parameters(), resumed.model.parameters()): torch.testing.assert_close(left, right) def test_bfloat16_autocast_keeps_float32_logits_and_weights(): events = random_events(seed=6) batch = next(iter(_loader(events))) fast = _trainer(precision="bfloat16") logits = fast.forward(batch) assert logits.dtype == torch.float32 assert all(p.dtype == torch.float32 for p in fast.model.parameters()) torch.testing.assert_close(logits, _trainer().forward(batch), rtol=5e-2, atol=5e-2) with pytest.raises(ValueError, match="precision"): _trainer(precision="float16") @pytest.mark.gpu @cuda def test_cuda_graph_steps_match_eager_gradients(): from gnn4colliders.cuda_graphs import CudaGraphTrainStep batches = [b.to("cuda") for b in _loader(random_events(64, seed=5), shuffle=False)] task = MulticlassClassificationTask() torch.manual_seed(0) eager = EdgeNetwork(7, 3, 0, 8, 3, 2, 1).cuda() graphed = copy.deepcopy(eager) # A zero learning rate keeps parameters fixed so gradients are comparable. optimizer = torch.optim.Adam(graphed.parameters(), lr=0.0, capturable=True) for batch in batches[:3]: optimizer.zero_grad(set_to_none=True) task.loss(graphed(batch), batch.labels, batch.weight).backward() optimizer.step() step = CudaGraphTrainStep(graphed, task, optimizer) for batch in batches[3:]: eager.zero_grad(set_to_none=True) expected = task.loss(eager(batch), batch.labels, batch.weight) expected.backward() loss, logits = step(batch) torch.testing.assert_close(loss, expected.detach(), rtol=1e-5, atol=1e-6) assert logits.shape == (8, 3) for left, right in zip(eager.parameters(), graphed.parameters()): torch.testing.assert_close(right.grad, left.grad, rtol=1e-4, atol=1e-6) @pytest.mark.gpu @cuda def test_compiled_graph_steps_with_bfloat16_track_eager_training(): loader = EventLoader(random_events(96, seed=7), collate_graph_batch, 16) def run(**options): trainer = _trainer(device="cuda", **options) return [trainer.train_epoch(loader).loss for _ in range(3)] eager = run() fast = run(compile_model=True, cuda_graphs=True, precision="bfloat16") assert fast == pytest.approx(eager, rel=5e-2) @pytest.mark.gpu @cuda def test_cuda_graph_steps_follow_the_learning_rate_schedule(tmp_path): loader = EventLoader(random_events(96, seed=8), collate_graph_batch, 16) trainer = _trainer(device="cuda", cuda_graphs=True) trainer.train_epoch(loader) # warmup steps, then captured replays # A zero rate must reach the already captured steps. torch.optim.lr_scheduler.ExponentialLR(trainer.optimizer, gamma=0.0).step() before = [p.detach().clone() for p in trainer.model.parameters()] trainer.train_epoch(loader) for old, new in zip(before, trainer.model.parameters()): assert torch.equal(old, new) # Checkpoints stay loadable by eager training on any device. groups = load_checkpoint(save_checkpoint(tmp_path / "c.pt", trainer))[ "optimizer_state_dict" ]["param_groups"] assert all(type(g["lr"]) is float and not g["capturable"] for g in groups) @pytest.mark.gpu @cuda def test_resumed_compiled_graph_training_warms_up_again(tmp_path): loader = EventLoader(random_events(96, seed=9), collate_graph_batch, 16) options = dict(device="cuda", compile_model=True, cuda_graphs=True) first = _trainer(**options) first.fit(loader, max_epochs=1) # As in a new process, the resumed trainer compiles again: in eager # steps, not during capture. torch._dynamo.reset() resumed = _trainer(**options) restore(resumed, load_checkpoint(save_checkpoint(tmp_path / "last.pt", first))) resumed.fit(loader, max_epochs=2) assert resumed.epoch == 1 def test_cosine_schedule_warms_up_then_decays(tmp_path): from conftest import GAMGAM_FIXTURES, fixture_config from gnn4colliders.cli import make_trainer from gnn4colliders.distributed import DistributedContext config = fixture_config( "gamgam", GAMGAM_FIXTURES, list(range(len(GAMGAM_FIXTURES))), tmp_path, "train.lr_schedule=cosine", "train.warmup_epochs=2", "train.max_epochs=6", "train.learning_rate=1.0", ) trainer = make_trainer( config, EdgeNetwork(7, 3, 0, 8, 3, 2, 1), DistributedContext() ) rates = [] for _ in range(6): rates.append(trainer.optimizer.param_groups[0]["lr"]) trainer.scheduler.step() # Zero would be at epoch 6, after the last. cosine = [0.5 * (1 + math.cos(math.pi * e / 4)) for e in range(4)] assert rates == pytest.approx([1 / 3, 2 / 3, *cosine])