Claude Opus 5.5
feat: TabPFN-3.5 options for the PFN, a cosine schedule and faster meta-training
71aefe8 Download tests/test_train.py from HWresearch/GNN4Colliders: direct link, hf CLI and curl.
- Browser
- Download file 7.47 kB
-
https://huggingface.co/HWresearch/GNN4Colliders/resolve/main/tests/test_train.py
- Command line
-
hf download hf://HWresearch/GNN4Colliders/tests/test_train.py
-
curl -L -o test_train.py https://huggingface.co/HWresearch/GNN4Colliders/resolve/main/tests/test_train.py
7.47 kB
| import copy | |
| import math | |
| import pytest | |
| import torch | |
| from conftest import random_events | |
| from gnn4colliders.graphs import collate_graph_batch | |
| from gnn4colliders.loader import EventLoader | |
| from gnn4colliders.models import EdgeNetwork | |
| from gnn4colliders.tasks import MulticlassClassificationTask | |
| from gnn4colliders.train import ( | |
| EarlyStopping, | |
| Trainer, | |
| load_checkpoint, | |
| restore, | |
| save_checkpoint, | |
| ) | |
| cuda = pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA unavailable") | |
| def _trainer(seed=0, **options): | |
| torch.manual_seed(seed) | |
| model = EdgeNetwork(7, 3, 0, 8, 3, 2, 1) | |
| optimizer = torch.optim.Adam(model.parameters(), lr=1e-2) | |
| scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.5) | |
| return Trainer( | |
| model, MulticlassClassificationTask(), optimizer, scheduler, **options | |
| ) | |
| def _loader(events, **options): | |
| return EventLoader(events, collate_graph_batch, 8, **options) | |
| def test_training_reduces_the_loss_and_follows_the_schedule(): | |
| events = random_events() | |
| trainer = _trainer() | |
| seen = [] | |
| history = trainer.fit( | |
| _loader(events, shuffle=True), | |
| _loader(events), | |
| max_epochs=4, | |
| on_epoch_end=lambda train, validation: seen.append( | |
| (trainer.epoch, trainer.optimizer.param_groups[0]["lr"]) | |
| ), | |
| ) | |
| assert seen == [(0, 5e-3), (1, 2.5e-3), (2, 1.25e-3), (3, 6.25e-4)] | |
| assert history[-1][0].loss < history[0][0].loss | |
| assert history[0][1].num_samples == 40 | |
| assert {"accuracy", "auc", "f1"} <= set(history[0][1].metrics) | |
| def test_training_loss_early_stopping_needs_no_validation(): | |
| trainer = _trainer(early_stopping=EarlyStopping("train_loss", 1, min_delta=10.0)) | |
| history = trainer.fit(_loader(random_events()), max_epochs=5) | |
| # The first epoch sets the best; the second cannot beat it by min_delta. | |
| assert len(history) == 2 and trainer.epoch == 1 | |
| def test_non_finite_loss_is_reported_at_the_end_of_the_epoch(): | |
| events = random_events() | |
| events.object_features[0, 0] = float("nan") | |
| with pytest.raises(FloatingPointError, match="batch 0"): | |
| _trainer().train_epoch(_loader(events)) | |
| def test_resumed_training_continues_identically(tmp_path): | |
| events = random_events() | |
| straight = _trainer() | |
| straight.fit(_loader(events, shuffle=True), max_epochs=3) | |
| first = _trainer() | |
| first.fit(_loader(events, shuffle=True), max_epochs=1) | |
| save_checkpoint(tmp_path / "last.pt", first) | |
| resumed = _trainer(seed=1) | |
| restore(resumed, load_checkpoint(tmp_path / "last.pt")) | |
| resumed.fit(_loader(events, shuffle=True), max_epochs=3) | |
| assert resumed.epoch == 2 | |
| for left, right in zip(straight.model.parameters(), resumed.model.parameters()): | |
| torch.testing.assert_close(left, right) | |
| def test_bfloat16_autocast_keeps_float32_logits_and_weights(): | |
| events = random_events(seed=6) | |
| batch = next(iter(_loader(events))) | |
| fast = _trainer(precision="bfloat16") | |
| logits = fast.forward(batch) | |
| assert logits.dtype == torch.float32 | |
| assert all(p.dtype == torch.float32 for p in fast.model.parameters()) | |
| torch.testing.assert_close(logits, _trainer().forward(batch), rtol=5e-2, atol=5e-2) | |
| with pytest.raises(ValueError, match="precision"): | |
| _trainer(precision="float16") | |
| def test_cuda_graph_steps_match_eager_gradients(): | |
| from gnn4colliders.cuda_graphs import CudaGraphTrainStep | |
| batches = [b.to("cuda") for b in _loader(random_events(64, seed=5), shuffle=False)] | |
| task = MulticlassClassificationTask() | |
| torch.manual_seed(0) | |
| eager = EdgeNetwork(7, 3, 0, 8, 3, 2, 1).cuda() | |
| graphed = copy.deepcopy(eager) | |
| # A zero learning rate keeps parameters fixed so gradients are comparable. | |
| optimizer = torch.optim.Adam(graphed.parameters(), lr=0.0, capturable=True) | |
| for batch in batches[:3]: | |
| optimizer.zero_grad(set_to_none=True) | |
| task.loss(graphed(batch), batch.labels, batch.weight).backward() | |
| optimizer.step() | |
| step = CudaGraphTrainStep(graphed, task, optimizer) | |
| for batch in batches[3:]: | |
| eager.zero_grad(set_to_none=True) | |
| expected = task.loss(eager(batch), batch.labels, batch.weight) | |
| expected.backward() | |
| loss, logits = step(batch) | |
| torch.testing.assert_close(loss, expected.detach(), rtol=1e-5, atol=1e-6) | |
| assert logits.shape == (8, 3) | |
| for left, right in zip(eager.parameters(), graphed.parameters()): | |
| torch.testing.assert_close(right.grad, left.grad, rtol=1e-4, atol=1e-6) | |
| def test_compiled_graph_steps_with_bfloat16_track_eager_training(): | |
| loader = EventLoader(random_events(96, seed=7), collate_graph_batch, 16) | |
| def run(**options): | |
| trainer = _trainer(device="cuda", **options) | |
| return [trainer.train_epoch(loader).loss for _ in range(3)] | |
| eager = run() | |
| fast = run(compile_model=True, cuda_graphs=True, precision="bfloat16") | |
| assert fast == pytest.approx(eager, rel=5e-2) | |
| def test_cuda_graph_steps_follow_the_learning_rate_schedule(tmp_path): | |
| loader = EventLoader(random_events(96, seed=8), collate_graph_batch, 16) | |
| trainer = _trainer(device="cuda", cuda_graphs=True) | |
| trainer.train_epoch(loader) # warmup steps, then captured replays | |
| # A zero rate must reach the already captured steps. | |
| torch.optim.lr_scheduler.ExponentialLR(trainer.optimizer, gamma=0.0).step() | |
| before = [p.detach().clone() for p in trainer.model.parameters()] | |
| trainer.train_epoch(loader) | |
| for old, new in zip(before, trainer.model.parameters()): | |
| assert torch.equal(old, new) | |
| # Checkpoints stay loadable by eager training on any device. | |
| groups = load_checkpoint(save_checkpoint(tmp_path / "c.pt", trainer))[ | |
| "optimizer_state_dict" | |
| ]["param_groups"] | |
| assert all(type(g["lr"]) is float and not g["capturable"] for g in groups) | |
| def test_resumed_compiled_graph_training_warms_up_again(tmp_path): | |
| loader = EventLoader(random_events(96, seed=9), collate_graph_batch, 16) | |
| options = dict(device="cuda", compile_model=True, cuda_graphs=True) | |
| first = _trainer(**options) | |
| first.fit(loader, max_epochs=1) | |
| # As in a new process, the resumed trainer compiles again: in eager | |
| # steps, not during capture. | |
| torch._dynamo.reset() | |
| resumed = _trainer(**options) | |
| restore(resumed, load_checkpoint(save_checkpoint(tmp_path / "last.pt", first))) | |
| resumed.fit(loader, max_epochs=2) | |
| assert resumed.epoch == 1 | |
| def test_cosine_schedule_warms_up_then_decays(tmp_path): | |
| from conftest import GAMGAM_FIXTURES, fixture_config | |
| from gnn4colliders.cli import make_trainer | |
| from gnn4colliders.distributed import DistributedContext | |
| config = fixture_config( | |
| "gamgam", | |
| GAMGAM_FIXTURES, | |
| list(range(len(GAMGAM_FIXTURES))), | |
| tmp_path, | |
| "train.lr_schedule=cosine", | |
| "train.warmup_epochs=2", | |
| "train.max_epochs=6", | |
| "train.learning_rate=1.0", | |
| ) | |
| trainer = make_trainer( | |
| config, EdgeNetwork(7, 3, 0, 8, 3, 2, 1), DistributedContext() | |
| ) | |
| rates = [] | |
| for _ in range(6): | |
| rates.append(trainer.optimizer.param_groups[0]["lr"]) | |
| trainer.scheduler.step() | |
| # Zero would be at epoch 6, after the last. | |
| cosine = [0.5 * (1 + math.cos(math.pi * e / 4)) for e in range(4)] | |
| assert rates == pytest.approx([1 / 3, 2 / 3, *cosine]) | |