Claude Opus 5.5
feat: evaluation subsample, Slurm wrappers for the study and label-efficiency plots
e71731f Download tests/test_tabular.py from HWresearch/GNN4Colliders: direct link, hf CLI and curl.
- Browser
- Download file 15.1 kB
-
https://huggingface.co/HWresearch/GNN4Colliders/resolve/main/tests/test_tabular.py
- Command line
-
hf download hf://HWresearch/GNN4Colliders/tests/test_tabular.py
-
curl -L -o test_tabular.py https://huggingface.co/HWresearch/GNN4Colliders/resolve/main/tests/test_tabular.py
15.1 kB
| """Fixed-width event tables, balanced label subsets, LoRA, TabICL in | |
| context, and TabPFN in context and meta-trained (when installed, with its | |
| weights).""" | |
| import json | |
| import math | |
| import os | |
| import sys | |
| import pytest | |
| import torch | |
| from conftest import GAMGAM_FIXTURES, TTH_FIXTURE, fixture_config | |
| from torch import nn | |
| from torch.nn.utils import parametrize | |
| from gnn4colliders.cli import main | |
| from gnn4colliders.config import ModelConfig, save_config | |
| from gnn4colliders.data import PackedEvents, pack_events | |
| from gnn4colliders.sequences import collate_sequence_batch | |
| from gnn4colliders.tabular import ( | |
| EventInputs, | |
| RowLayout, | |
| add_lora, | |
| merge_lora, | |
| sequence_table, | |
| table_features, | |
| ) | |
| from gnn4colliders.train import load_checkpoint | |
| def _objects(types): | |
| rows = torch.arange(len(types) * 7, dtype=torch.float32).view(-1, 7) | |
| rows[:, 6] = torch.tensor(types, dtype=torch.float32) | |
| return rows | |
| def test_table_keeps_the_leading_objects_of_each_type(): | |
| first, second = _objects([0, 0, 0, 3, 3, 4]), _objects([1, 4]) | |
| events = pack_events([first, second], [0, 1]) | |
| layout = RowLayout((2, 1, 1, 1, 1)) | |
| table = table_features(events, torch.tensor([0, 1]), layout) | |
| assert table.shape == (2, layout.width) == (2, 6 * 6 + 5) | |
| slots = table[:, :36].view(2, 6, 6) | |
| # Slots: jet, jet, electron, muon, photon, MET. | |
| torch.testing.assert_close(slots[0, :2], first[:2, :6]) | |
| torch.testing.assert_close(slots[0, 4:], first[[3, 5], :6]) | |
| assert slots[0, 2:4].isnan().all() | |
| torch.testing.assert_close(slots[1, 2], second[0, :6]) | |
| torch.testing.assert_close(slots[1, 5], second[1, :6]) | |
| assert slots[1, [0, 1, 3, 4]].isnan().all() | |
| assert table[:, 36:].tolist() == [[3, 0, 0, 2, 1], [0, 1, 0, 0, 1]] | |
| def test_sequence_batches_give_the_same_table(): | |
| events = pack_events( | |
| [_objects([0, 0, 0, 3, 3, 4]), _objects([1, 4]), _objects([2, 2, 2, 0])], | |
| [0, 1, 2], | |
| ) | |
| indices = torch.tensor([2, 0, 1]) | |
| layout = RowLayout((2, 1, 1, 1, 1), relative_phi=True, pair_objects=3) | |
| torch.testing.assert_close( | |
| sequence_table(collate_sequence_batch(events, indices), layout), | |
| table_features(events, indices, layout), | |
| equal_nan=True, | |
| ) | |
| def _object(pt, eta, phi, node_type): | |
| return [pt, eta, phi, pt * math.cosh(eta), 0.0, 0.0, node_type] | |
| def test_relative_phi_and_the_masses_and_delta_r_of_leading_pairs(): | |
| # Two back-to-back photons (mass 2, Delta R pi) and a softer jet; an | |
| # event with one photon has no pair. | |
| pair = torch.tensor( | |
| [_object(1.0, 0.0, 0.5, 3), _object(1.0, 0.0, 0.5 - math.pi, 3)] | |
| + [_object(0.5, 1.0, 2.0, 0)] | |
| ) | |
| single = torch.tensor([_object(1.0, 0.3, 1.0, 3)]) | |
| events = pack_events([pair, single], [0, 1]) | |
| layout = RowLayout((1, 0, 0, 2, 0), relative_phi=True, pair_objects=2) | |
| table = table_features(events, torch.tensor([0, 1]), layout) | |
| assert table.shape == (2, layout.width) == (2, 3 * 6 + 5 + 2) | |
| slots = table[:, :18].view(2, 3, 6) | |
| # Azimuths from the leading photon's. | |
| torch.testing.assert_close(slots[0, :, 2], torch.tensor([1.5, 0.0, -math.pi])) | |
| assert slots[1, 1, 2] == 0 | |
| torch.testing.assert_close(table[0, -2:], torch.tensor([2.0, math.pi])) | |
| assert table[1, -2:].isnan().all() | |
| def test_lora_starts_at_the_frozen_weight_and_trains_only_its_update(): | |
| network = nn.Sequential(nn.Linear(6, 4)) | |
| weight, x = network[0].weight.detach().clone(), torch.randn(5, 6) | |
| expected = network(x) | |
| add_lora(network, ["0.weight"], rank=2) | |
| torch.testing.assert_close(network(x), expected) | |
| trainable = [name for name, p in network.named_parameters() if p.requires_grad] | |
| assert sorted(trainable) == [ | |
| "0.parametrizations.weight.0.down", | |
| "0.parametrizations.weight.0.up", | |
| ] | |
| # Code that reads the weight directly sees the update too. | |
| with torch.no_grad(): | |
| network[0].parametrizations.weight[0].up.fill_(1.0) | |
| lora = network[0].parametrizations.weight[0] | |
| torch.testing.assert_close(network[0].weight, weight + lora.up @ lora.down) | |
| # Merged for inference, the network computes the same. | |
| expected = network(x) | |
| merge_lora(network) | |
| assert not parametrize.is_parametrized(network[0]) | |
| torch.testing.assert_close(network(x), expected) | |
| def test_random_evaluation_subsets_are_seeded_and_sorted(): | |
| indices = torch.arange(100) * 3 | |
| subset = PackedEvents.random_subset(indices, 20, seed=1) | |
| assert len(subset) == 20 and set(subset.tolist()) <= set(indices.tolist()) | |
| assert torch.equal(subset, subset.sort().values) | |
| assert torch.equal(subset, PackedEvents.random_subset(indices, 20, seed=1)) | |
| assert len(PackedEvents.random_subset(indices, 500, seed=1)) == 100 | |
| def test_balanced_subsets_are_equal_per_class_and_nested(): | |
| labels = torch.tensor([0] * 50 + [1] * 30 + [2] * 5) | |
| events = pack_events([torch.zeros(1, 7)] * len(labels), labels) | |
| indices = torch.arange(len(labels)) | |
| small = events.balanced_subset(indices, 12, seed=3) | |
| assert torch.bincount(labels[small]).tolist() == [4, 4, 4] | |
| large = events.balanced_subset(indices, 30, seed=3) | |
| assert torch.bincount(labels[large]).tolist() == [10, 10, 5] # 5 exist | |
| assert set(small.tolist()) <= set(large.tolist()) | |
| assert torch.equal(small, events.balanced_subset(indices, 12, seed=3)) | |
| assert not torch.equal(small, events.balanced_subset(indices, 12, seed=4)) | |
| def test_tabpfn_classifies_gamgam_in_context(tmp_path): | |
| pytest.importorskip("tabpfn") | |
| config = fixture_config( | |
| "gamgam_tabpfn", | |
| GAMGAM_FIXTURES, | |
| list(range(5)), | |
| tmp_path, | |
| "data.train_events=100", | |
| # Version 2 weights download without a login. | |
| "model.tabpfn_version=v2", | |
| "model.tabpfn_estimators=1", | |
| "split=validation", | |
| ) | |
| path = str(save_config(config, tmp_path / "tabpfn.yaml")) | |
| assert main(["prepare", path]) == 0 | |
| assert main(["evaluate", path]) == 0 | |
| metrics = json.loads((config.path / "validation_metrics.json").read_text()) | |
| assert not math.isnan(metrics["metrics"]["auc"]) | |
| assert metrics["checkpoint"] == "TabPFN v2 (default checkpoint)" | |
| def _tabpfn_3_5_weights() -> None: | |
| pytest.importorskip("tabpfn") | |
| from tabpfn.model_loading import get_cache_dir | |
| if not (get_cache_dir() / "tabpfn-v3.5-20260909.safetensors").exists(): | |
| pytest.skip("TabPFN-3.5 weights are not in the TabPFN cache") | |
| def test_meta_train_tabpfn_with_lora_then_classify_gamgam(tmp_path): | |
| _tabpfn_3_5_weights() | |
| meta = fixture_config( | |
| "delphes_tabpfn", | |
| [TTH_FIXTURE, TTH_FIXTURE], | |
| [0, 1], | |
| tmp_path / "meta", | |
| "train.max_epochs=1", | |
| "task.context_events=[4,16]", | |
| "task.query_events=4", | |
| "task.steps_per_epoch=2", | |
| "task.validation_steps=1", | |
| "model.relative_phi=true", | |
| "model.pair_objects=4", | |
| ) | |
| path = str(save_config(meta, tmp_path / "meta.yaml")) | |
| assert main(["prepare", path]) == 0 | |
| assert main(["train", path]) == 0 | |
| # Of the weights, only the LoRA updates are saved: 24 blocks of 4 | |
| # projections and 2 MLP layers. | |
| state = load_checkpoint(meta.path / "best.pt")["model_state_dict"] | |
| weights = [key for key in state if key.endswith(".weight")] | |
| updates = [key for key in state if key.endswith((".down", ".up"))] | |
| assert not weights and len(updates) == 24 * 6 * 2 | |
| gamgam = fixture_config( | |
| "gamgam_tabpfn", | |
| GAMGAM_FIXTURES, | |
| list(range(5)), | |
| tmp_path / "gamgam", | |
| f"checkpoint={meta.path}/best.pt", | |
| "data.train_events=40", | |
| "split=validation", | |
| "model.relative_phi=true", | |
| "model.pair_objects=4", | |
| ) | |
| path = str(save_config(gamgam, tmp_path / "gamgam.yaml")) | |
| assert main(["prepare", path]) == 0 | |
| assert main(["evaluate", path]) == 0 | |
| metrics = json.loads((gamgam.path / "validation_metrics.json").read_text()) | |
| assert metrics["checkpoint"] == f"{meta.path}/best.pt" | |
| assert math.isfinite(metrics["metrics"]["auc"]) | |
| def test_tabicl_classifies_gamgam_in_context_with_physics_features(tmp_path): | |
| pytest.importorskip("tabicl") | |
| config = fixture_config( | |
| "gamgam_tabicl", | |
| GAMGAM_FIXTURES, | |
| list(range(5)), | |
| tmp_path, | |
| "data.train_events=100", | |
| "model.tabicl_estimators=1", | |
| "model.relative_phi=true", | |
| "model.pair_objects=4", | |
| "split=validation", | |
| ) | |
| path = str(save_config(config, tmp_path / "tabicl.yaml")) | |
| assert main(["prepare", path]) == 0 | |
| assert main(["evaluate", path]) == 0 | |
| metrics = json.loads((config.path / "validation_metrics.json").read_text()) | |
| assert metrics["checkpoint"] == "TabICL (tabicl-classifier-v2-20260212.ckpt)" | |
| assert math.isfinite(metrics["metrics"]["auc"]) | |
| #: faiss and torch crash together on macOS unless OpenMP has one thread. | |
| RETRIEVAL = sys.platform != "darwin" or os.environ.get("OMP_NUM_THREADS") == "1" | |
| def test_tabdpt_classifies_gamgam_in_context(context_size, tmp_path): | |
| pytest.importorskip("tabdpt") | |
| config = fixture_config( | |
| "gamgam_tabdpt", | |
| GAMGAM_FIXTURES, | |
| list(range(5)), | |
| tmp_path, | |
| "data.train_events=100", | |
| "model.tabdpt_estimators=1", | |
| f"model.tabdpt_context_size={context_size or 'null'}", | |
| "split=validation", | |
| ) | |
| path = str(save_config(config, tmp_path / "tabdpt.yaml")) | |
| assert main(["prepare", path]) == 0 | |
| assert main(["evaluate", path]) == 0 | |
| metrics = json.loads((config.path / "validation_metrics.json").read_text()) | |
| assert ( | |
| metrics["checkpoint"] == f"TabDPT (context {context_size or 'all'} per query)" | |
| ) | |
| assert math.isfinite(metrics["metrics"]["auc"]) | |
| def test_meta_train_tabicl_with_lora_then_classify_gamgam(tmp_path): | |
| pytest.importorskip("tabicl") | |
| meta = fixture_config( | |
| "delphes_tabicl", | |
| [TTH_FIXTURE, TTH_FIXTURE], | |
| [0, 1], | |
| tmp_path / "meta", | |
| "train.max_epochs=1", | |
| "task.context_events=[4,16]", | |
| "task.query_events=4", | |
| "task.steps_per_epoch=2", | |
| "task.validation_steps=1", | |
| "model.pair_objects=4", | |
| ) | |
| path = str(save_config(meta, tmp_path / "meta.yaml")) | |
| assert main(["prepare", path]) == 0 | |
| assert main(["train", path]) == 0 | |
| # Only the LoRA updates are saved: 12 blocks of 4 weights. | |
| state = load_checkpoint(meta.path / "best.pt")["model_state_dict"] | |
| assert len(state) == 12 * 4 * 2 | |
| assert all(key.endswith((".down", ".up")) for key in state) | |
| gamgam = fixture_config( | |
| "gamgam_tabicl", | |
| GAMGAM_FIXTURES, | |
| list(range(5)), | |
| tmp_path / "gamgam", | |
| f"checkpoint={meta.path}/best.pt", | |
| "data.train_events=40", | |
| "split=validation", | |
| "model.pair_objects=4", | |
| ) | |
| path = str(save_config(gamgam, tmp_path / "gamgam.yaml")) | |
| assert main(["prepare", path]) == 0 | |
| assert main(["evaluate", path]) == 0 | |
| metrics = json.loads((gamgam.path / "validation_metrics.json").read_text()) | |
| assert metrics["checkpoint"] == f"{meta.path}/best.pt" | |
| assert math.isfinite(metrics["metrics"]["auc"]) | |
| #: The encoder of the hybrid tests, as pretrained by JEPA on the fixtures. | |
| ENCODER = { | |
| "hid_size": 16, | |
| "n_layers": 1, | |
| "n_heads": 2, | |
| "ffn_size": 64, | |
| "pair_bias": True, | |
| } | |
| def test_event_inputs_are_rows_embeddings_or_both(): | |
| events = pack_events([_objects([0, 0, 3, 3, 4]), _objects([1, 3, 4])], [0, 1]) | |
| batch = collate_sequence_batch(events, torch.tensor([0, 1])) | |
| width = RowLayout((6, 2, 2, 3, 1)).width | |
| for inputs, expected in (("rows", width), ("embeddings", 16), ("both", width + 16)): | |
| config = ModelConfig(name="tabicl", tabular_inputs=inputs, **ENCODER) | |
| model = EventInputs.of(config, num_features=7, num_node_types=5) | |
| assert model(batch).shape == (2, expected) | |
| with pytest.raises(ValueError, match="tabular_inputs"): | |
| EventInputs.of( | |
| ModelConfig(tabular_inputs="objects"), num_features=7, num_node_types=5 | |
| ) | |
| def jepa_encoder(tmp_path): | |
| small = [f"model.{key}={value}" for key, value in ENCODER.items()] | |
| config = fixture_config( | |
| "delphes_jepa", | |
| [TTH_FIXTURE, TTH_FIXTURE], | |
| [0, 1], | |
| tmp_path / "jepa", | |
| "train.max_epochs=1", | |
| *small, | |
| ) | |
| path = str(save_config(config, tmp_path / "jepa.yaml")) | |
| assert main(["prepare", path]) == 0 | |
| assert main(["train", path]) == 0 | |
| return f"{config.path}/last.pt" | |
| def _hybrid(name, tmp_path, jepa_encoder, *overrides): | |
| small = [f"model.{key}={value}" for key, value in ENCODER.items()] | |
| config = fixture_config( | |
| name, | |
| GAMGAM_FIXTURES if name.startswith("gamgam") else [TTH_FIXTURE, TTH_FIXTURE], | |
| list(range(5)) if name.startswith("gamgam") else [0, 1], | |
| tmp_path / name, | |
| "model.tabular_inputs=both", | |
| f"model.pretrained={jepa_encoder}", | |
| *small, | |
| *overrides, | |
| ) | |
| path = str(save_config(config, tmp_path / f"{name}.yaml")) | |
| assert main(["prepare", path]) == 0 | |
| return config, path | |
| def test_tabicl_classifies_gamgam_on_rows_and_jepa_embeddings(tmp_path, jepa_encoder): | |
| pytest.importorskip("tabicl") | |
| config, path = _hybrid( | |
| "gamgam_tabicl", | |
| tmp_path, | |
| jepa_encoder, | |
| "data.train_events=100", | |
| "model.tabicl_estimators=1", | |
| "split=validation", | |
| ) | |
| assert main(["evaluate", path]) == 0 | |
| metrics = json.loads((config.path / "validation_metrics.json").read_text()) | |
| assert math.isfinite(metrics["metrics"]["auc"]) | |
| def test_meta_train_tabicl_with_its_encoder(tmp_path, jepa_encoder, freeze): | |
| pytest.importorskip("tabicl") | |
| meta, path = _hybrid( | |
| "delphes_tabicl", | |
| tmp_path, | |
| jepa_encoder, | |
| "train.max_epochs=1", | |
| "task.context_events=[4,16]", | |
| "task.query_events=4", | |
| "task.steps_per_epoch=2", | |
| "task.validation_steps=1", | |
| f"model.freeze_backbone={str(freeze).lower()}", | |
| ) | |
| assert main(["train", path]) == 0 | |
| state = load_checkpoint(meta.path / "best.pt")["model_state_dict"] | |
| encoder = [key for key in state if key.startswith("inputs.encoder.")] | |
| # A trained encoder is saved; a frozen one comes from model.pretrained. | |
| assert bool(encoder) != freeze | |
| gamgam, path = _hybrid( | |
| "gamgam_tabicl", | |
| tmp_path, | |
| jepa_encoder, | |
| f"checkpoint={meta.path}/best.pt", | |
| "data.train_events=40", | |
| "split=validation", | |
| f"model.freeze_backbone={str(freeze).lower()}", | |
| ) | |
| assert main(["evaluate", path]) == 0 | |
| metrics = json.loads((gamgam.path / "validation_metrics.json").read_text()) | |
| assert math.isfinite(metrics["metrics"]["auc"]) | |