GNN4Colliders / tests /test_train.py
ho22joshua's picture
Claude Opus 5.5
feat: TabPFN-3.5 options for the PFN, a cosine schedule and faster meta-training
71aefe8
Raw History Blame Contribute Delete
7.47 kB
import copy
import math
import pytest
import torch
from conftest import random_events
from gnn4colliders.graphs import collate_graph_batch
from gnn4colliders.loader import EventLoader
from gnn4colliders.models import EdgeNetwork
from gnn4colliders.tasks import MulticlassClassificationTask
from gnn4colliders.train import (
EarlyStopping,
Trainer,
load_checkpoint,
restore,
save_checkpoint,
)
cuda = pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA unavailable")
def _trainer(seed=0, **options):
torch.manual_seed(seed)
model = EdgeNetwork(7, 3, 0, 8, 3, 2, 1)
optimizer = torch.optim.Adam(model.parameters(), lr=1e-2)
scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.5)
return Trainer(
model, MulticlassClassificationTask(), optimizer, scheduler, **options
)
def _loader(events, **options):
return EventLoader(events, collate_graph_batch, 8, **options)
def test_training_reduces_the_loss_and_follows_the_schedule():
events = random_events()
trainer = _trainer()
seen = []
history = trainer.fit(
_loader(events, shuffle=True),
_loader(events),
max_epochs=4,
on_epoch_end=lambda train, validation: seen.append(
(trainer.epoch, trainer.optimizer.param_groups[0]["lr"])
),
)
assert seen == [(0, 5e-3), (1, 2.5e-3), (2, 1.25e-3), (3, 6.25e-4)]
assert history[-1][0].loss < history[0][0].loss
assert history[0][1].num_samples == 40
assert {"accuracy", "auc", "f1"} <= set(history[0][1].metrics)
def test_training_loss_early_stopping_needs_no_validation():
trainer = _trainer(early_stopping=EarlyStopping("train_loss", 1, min_delta=10.0))
history = trainer.fit(_loader(random_events()), max_epochs=5)
# The first epoch sets the best; the second cannot beat it by min_delta.
assert len(history) == 2 and trainer.epoch == 1
def test_non_finite_loss_is_reported_at_the_end_of_the_epoch():
events = random_events()
events.object_features[0, 0] = float("nan")
with pytest.raises(FloatingPointError, match="batch 0"):
_trainer().train_epoch(_loader(events))
def test_resumed_training_continues_identically(tmp_path):
events = random_events()
straight = _trainer()
straight.fit(_loader(events, shuffle=True), max_epochs=3)
first = _trainer()
first.fit(_loader(events, shuffle=True), max_epochs=1)
save_checkpoint(tmp_path / "last.pt", first)
resumed = _trainer(seed=1)
restore(resumed, load_checkpoint(tmp_path / "last.pt"))
resumed.fit(_loader(events, shuffle=True), max_epochs=3)
assert resumed.epoch == 2
for left, right in zip(straight.model.parameters(), resumed.model.parameters()):
torch.testing.assert_close(left, right)
def test_bfloat16_autocast_keeps_float32_logits_and_weights():
events = random_events(seed=6)
batch = next(iter(_loader(events)))
fast = _trainer(precision="bfloat16")
logits = fast.forward(batch)
assert logits.dtype == torch.float32
assert all(p.dtype == torch.float32 for p in fast.model.parameters())
torch.testing.assert_close(logits, _trainer().forward(batch), rtol=5e-2, atol=5e-2)
with pytest.raises(ValueError, match="precision"):
_trainer(precision="float16")
@pytest.mark.gpu
@cuda
def test_cuda_graph_steps_match_eager_gradients():
from gnn4colliders.cuda_graphs import CudaGraphTrainStep
batches = [b.to("cuda") for b in _loader(random_events(64, seed=5), shuffle=False)]
task = MulticlassClassificationTask()
torch.manual_seed(0)
eager = EdgeNetwork(7, 3, 0, 8, 3, 2, 1).cuda()
graphed = copy.deepcopy(eager)
# A zero learning rate keeps parameters fixed so gradients are comparable.
optimizer = torch.optim.Adam(graphed.parameters(), lr=0.0, capturable=True)
for batch in batches[:3]:
optimizer.zero_grad(set_to_none=True)
task.loss(graphed(batch), batch.labels, batch.weight).backward()
optimizer.step()
step = CudaGraphTrainStep(graphed, task, optimizer)
for batch in batches[3:]:
eager.zero_grad(set_to_none=True)
expected = task.loss(eager(batch), batch.labels, batch.weight)
expected.backward()
loss, logits = step(batch)
torch.testing.assert_close(loss, expected.detach(), rtol=1e-5, atol=1e-6)
assert logits.shape == (8, 3)
for left, right in zip(eager.parameters(), graphed.parameters()):
torch.testing.assert_close(right.grad, left.grad, rtol=1e-4, atol=1e-6)
@pytest.mark.gpu
@cuda
def test_compiled_graph_steps_with_bfloat16_track_eager_training():
loader = EventLoader(random_events(96, seed=7), collate_graph_batch, 16)
def run(**options):
trainer = _trainer(device="cuda", **options)
return [trainer.train_epoch(loader).loss for _ in range(3)]
eager = run()
fast = run(compile_model=True, cuda_graphs=True, precision="bfloat16")
assert fast == pytest.approx(eager, rel=5e-2)
@pytest.mark.gpu
@cuda
def test_cuda_graph_steps_follow_the_learning_rate_schedule(tmp_path):
loader = EventLoader(random_events(96, seed=8), collate_graph_batch, 16)
trainer = _trainer(device="cuda", cuda_graphs=True)
trainer.train_epoch(loader) # warmup steps, then captured replays
# A zero rate must reach the already captured steps.
torch.optim.lr_scheduler.ExponentialLR(trainer.optimizer, gamma=0.0).step()
before = [p.detach().clone() for p in trainer.model.parameters()]
trainer.train_epoch(loader)
for old, new in zip(before, trainer.model.parameters()):
assert torch.equal(old, new)
# Checkpoints stay loadable by eager training on any device.
groups = load_checkpoint(save_checkpoint(tmp_path / "c.pt", trainer))[
"optimizer_state_dict"
]["param_groups"]
assert all(type(g["lr"]) is float and not g["capturable"] for g in groups)
@pytest.mark.gpu
@cuda
def test_resumed_compiled_graph_training_warms_up_again(tmp_path):
loader = EventLoader(random_events(96, seed=9), collate_graph_batch, 16)
options = dict(device="cuda", compile_model=True, cuda_graphs=True)
first = _trainer(**options)
first.fit(loader, max_epochs=1)
# As in a new process, the resumed trainer compiles again: in eager
# steps, not during capture.
torch._dynamo.reset()
resumed = _trainer(**options)
restore(resumed, load_checkpoint(save_checkpoint(tmp_path / "last.pt", first)))
resumed.fit(loader, max_epochs=2)
assert resumed.epoch == 1
def test_cosine_schedule_warms_up_then_decays(tmp_path):
from conftest import GAMGAM_FIXTURES, fixture_config
from gnn4colliders.cli import make_trainer
from gnn4colliders.distributed import DistributedContext
config = fixture_config(
"gamgam",
GAMGAM_FIXTURES,
list(range(len(GAMGAM_FIXTURES))),
tmp_path,
"train.lr_schedule=cosine",
"train.warmup_epochs=2",
"train.max_epochs=6",
"train.learning_rate=1.0",
)
trainer = make_trainer(
config, EdgeNetwork(7, 3, 0, 8, 3, 2, 1), DistributedContext()
)
rates = []
for _ in range(6):
rates.append(trainer.optimizer.param_groups[0]["lr"])
trainer.scheduler.step()
# Zero would be at epoch 6, after the last.
cosine = [0.5 * (1 + math.cos(math.pi * e / 4)) for e in range(4)]
assert rates == pytest.approx([1 / 3, 2 / 3, *cosine])