File size: 7,472 Bytes
0cef53c 71aefe8 0cef53c 9a04b7b 71aefe8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 | import copy
import math
import pytest
import torch
from conftest import random_events
from gnn4colliders.graphs import collate_graph_batch
from gnn4colliders.loader import EventLoader
from gnn4colliders.models import EdgeNetwork
from gnn4colliders.tasks import MulticlassClassificationTask
from gnn4colliders.train import (
EarlyStopping,
Trainer,
load_checkpoint,
restore,
save_checkpoint,
)
cuda = pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA unavailable")
def _trainer(seed=0, **options):
torch.manual_seed(seed)
model = EdgeNetwork(7, 3, 0, 8, 3, 2, 1)
optimizer = torch.optim.Adam(model.parameters(), lr=1e-2)
scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.5)
return Trainer(
model, MulticlassClassificationTask(), optimizer, scheduler, **options
)
def _loader(events, **options):
return EventLoader(events, collate_graph_batch, 8, **options)
def test_training_reduces_the_loss_and_follows_the_schedule():
events = random_events()
trainer = _trainer()
seen = []
history = trainer.fit(
_loader(events, shuffle=True),
_loader(events),
max_epochs=4,
on_epoch_end=lambda train, validation: seen.append(
(trainer.epoch, trainer.optimizer.param_groups[0]["lr"])
),
)
assert seen == [(0, 5e-3), (1, 2.5e-3), (2, 1.25e-3), (3, 6.25e-4)]
assert history[-1][0].loss < history[0][0].loss
assert history[0][1].num_samples == 40
assert {"accuracy", "auc", "f1"} <= set(history[0][1].metrics)
def test_training_loss_early_stopping_needs_no_validation():
trainer = _trainer(early_stopping=EarlyStopping("train_loss", 1, min_delta=10.0))
history = trainer.fit(_loader(random_events()), max_epochs=5)
# The first epoch sets the best; the second cannot beat it by min_delta.
assert len(history) == 2 and trainer.epoch == 1
def test_non_finite_loss_is_reported_at_the_end_of_the_epoch():
events = random_events()
events.object_features[0, 0] = float("nan")
with pytest.raises(FloatingPointError, match="batch 0"):
_trainer().train_epoch(_loader(events))
def test_resumed_training_continues_identically(tmp_path):
events = random_events()
straight = _trainer()
straight.fit(_loader(events, shuffle=True), max_epochs=3)
first = _trainer()
first.fit(_loader(events, shuffle=True), max_epochs=1)
save_checkpoint(tmp_path / "last.pt", first)
resumed = _trainer(seed=1)
restore(resumed, load_checkpoint(tmp_path / "last.pt"))
resumed.fit(_loader(events, shuffle=True), max_epochs=3)
assert resumed.epoch == 2
for left, right in zip(straight.model.parameters(), resumed.model.parameters()):
torch.testing.assert_close(left, right)
def test_bfloat16_autocast_keeps_float32_logits_and_weights():
events = random_events(seed=6)
batch = next(iter(_loader(events)))
fast = _trainer(precision="bfloat16")
logits = fast.forward(batch)
assert logits.dtype == torch.float32
assert all(p.dtype == torch.float32 for p in fast.model.parameters())
torch.testing.assert_close(logits, _trainer().forward(batch), rtol=5e-2, atol=5e-2)
with pytest.raises(ValueError, match="precision"):
_trainer(precision="float16")
@pytest.mark.gpu
@cuda
def test_cuda_graph_steps_match_eager_gradients():
from gnn4colliders.cuda_graphs import CudaGraphTrainStep
batches = [b.to("cuda") for b in _loader(random_events(64, seed=5), shuffle=False)]
task = MulticlassClassificationTask()
torch.manual_seed(0)
eager = EdgeNetwork(7, 3, 0, 8, 3, 2, 1).cuda()
graphed = copy.deepcopy(eager)
# A zero learning rate keeps parameters fixed so gradients are comparable.
optimizer = torch.optim.Adam(graphed.parameters(), lr=0.0, capturable=True)
for batch in batches[:3]:
optimizer.zero_grad(set_to_none=True)
task.loss(graphed(batch), batch.labels, batch.weight).backward()
optimizer.step()
step = CudaGraphTrainStep(graphed, task, optimizer)
for batch in batches[3:]:
eager.zero_grad(set_to_none=True)
expected = task.loss(eager(batch), batch.labels, batch.weight)
expected.backward()
loss, logits = step(batch)
torch.testing.assert_close(loss, expected.detach(), rtol=1e-5, atol=1e-6)
assert logits.shape == (8, 3)
for left, right in zip(eager.parameters(), graphed.parameters()):
torch.testing.assert_close(right.grad, left.grad, rtol=1e-4, atol=1e-6)
@pytest.mark.gpu
@cuda
def test_compiled_graph_steps_with_bfloat16_track_eager_training():
loader = EventLoader(random_events(96, seed=7), collate_graph_batch, 16)
def run(**options):
trainer = _trainer(device="cuda", **options)
return [trainer.train_epoch(loader).loss for _ in range(3)]
eager = run()
fast = run(compile_model=True, cuda_graphs=True, precision="bfloat16")
assert fast == pytest.approx(eager, rel=5e-2)
@pytest.mark.gpu
@cuda
def test_cuda_graph_steps_follow_the_learning_rate_schedule(tmp_path):
loader = EventLoader(random_events(96, seed=8), collate_graph_batch, 16)
trainer = _trainer(device="cuda", cuda_graphs=True)
trainer.train_epoch(loader) # warmup steps, then captured replays
# A zero rate must reach the already captured steps.
torch.optim.lr_scheduler.ExponentialLR(trainer.optimizer, gamma=0.0).step()
before = [p.detach().clone() for p in trainer.model.parameters()]
trainer.train_epoch(loader)
for old, new in zip(before, trainer.model.parameters()):
assert torch.equal(old, new)
# Checkpoints stay loadable by eager training on any device.
groups = load_checkpoint(save_checkpoint(tmp_path / "c.pt", trainer))[
"optimizer_state_dict"
]["param_groups"]
assert all(type(g["lr"]) is float and not g["capturable"] for g in groups)
@pytest.mark.gpu
@cuda
def test_resumed_compiled_graph_training_warms_up_again(tmp_path):
loader = EventLoader(random_events(96, seed=9), collate_graph_batch, 16)
options = dict(device="cuda", compile_model=True, cuda_graphs=True)
first = _trainer(**options)
first.fit(loader, max_epochs=1)
# As in a new process, the resumed trainer compiles again: in eager
# steps, not during capture.
torch._dynamo.reset()
resumed = _trainer(**options)
restore(resumed, load_checkpoint(save_checkpoint(tmp_path / "last.pt", first)))
resumed.fit(loader, max_epochs=2)
assert resumed.epoch == 1
def test_cosine_schedule_warms_up_then_decays(tmp_path):
from conftest import GAMGAM_FIXTURES, fixture_config
from gnn4colliders.cli import make_trainer
from gnn4colliders.distributed import DistributedContext
config = fixture_config(
"gamgam",
GAMGAM_FIXTURES,
list(range(len(GAMGAM_FIXTURES))),
tmp_path,
"train.lr_schedule=cosine",
"train.warmup_epochs=2",
"train.max_epochs=6",
"train.learning_rate=1.0",
)
trainer = make_trainer(
config, EdgeNetwork(7, 3, 0, 8, 3, 2, 1), DistributedContext()
)
rates = []
for _ in range(6):
rates.append(trainer.optimizer.param_groups[0]["lr"])
trainer.scheduler.step()
# Zero would be at epoch 6, after the last.
cosine = [0.5 * (1 + math.cos(math.pi * e / 4)) for e in range(4)]
assert rates == pytest.approx([1 / 3, 2 / 3, *cosine])
|