GNN4Colliders / tests /test_tabular.py
ho22joshua's picture
Claude Opus 5.5
feat: evaluation subsample, Slurm wrappers for the study and label-efficiency plots
e71731f
Raw History Blame Contribute Delete
15.1 kB
"""Fixed-width event tables, balanced label subsets, LoRA, TabICL in
context, and TabPFN in context and meta-trained (when installed, with its
weights)."""
import json
import math
import os
import sys
import pytest
import torch
from conftest import GAMGAM_FIXTURES, TTH_FIXTURE, fixture_config
from torch import nn
from torch.nn.utils import parametrize
from gnn4colliders.cli import main
from gnn4colliders.config import ModelConfig, save_config
from gnn4colliders.data import PackedEvents, pack_events
from gnn4colliders.sequences import collate_sequence_batch
from gnn4colliders.tabular import (
EventInputs,
RowLayout,
add_lora,
merge_lora,
sequence_table,
table_features,
)
from gnn4colliders.train import load_checkpoint
def _objects(types):
rows = torch.arange(len(types) * 7, dtype=torch.float32).view(-1, 7)
rows[:, 6] = torch.tensor(types, dtype=torch.float32)
return rows
def test_table_keeps_the_leading_objects_of_each_type():
first, second = _objects([0, 0, 0, 3, 3, 4]), _objects([1, 4])
events = pack_events([first, second], [0, 1])
layout = RowLayout((2, 1, 1, 1, 1))
table = table_features(events, torch.tensor([0, 1]), layout)
assert table.shape == (2, layout.width) == (2, 6 * 6 + 5)
slots = table[:, :36].view(2, 6, 6)
# Slots: jet, jet, electron, muon, photon, MET.
torch.testing.assert_close(slots[0, :2], first[:2, :6])
torch.testing.assert_close(slots[0, 4:], first[[3, 5], :6])
assert slots[0, 2:4].isnan().all()
torch.testing.assert_close(slots[1, 2], second[0, :6])
torch.testing.assert_close(slots[1, 5], second[1, :6])
assert slots[1, [0, 1, 3, 4]].isnan().all()
assert table[:, 36:].tolist() == [[3, 0, 0, 2, 1], [0, 1, 0, 0, 1]]
def test_sequence_batches_give_the_same_table():
events = pack_events(
[_objects([0, 0, 0, 3, 3, 4]), _objects([1, 4]), _objects([2, 2, 2, 0])],
[0, 1, 2],
)
indices = torch.tensor([2, 0, 1])
layout = RowLayout((2, 1, 1, 1, 1), relative_phi=True, pair_objects=3)
torch.testing.assert_close(
sequence_table(collate_sequence_batch(events, indices), layout),
table_features(events, indices, layout),
equal_nan=True,
)
def _object(pt, eta, phi, node_type):
return [pt, eta, phi, pt * math.cosh(eta), 0.0, 0.0, node_type]
def test_relative_phi_and_the_masses_and_delta_r_of_leading_pairs():
# Two back-to-back photons (mass 2, Delta R pi) and a softer jet; an
# event with one photon has no pair.
pair = torch.tensor(
[_object(1.0, 0.0, 0.5, 3), _object(1.0, 0.0, 0.5 - math.pi, 3)]
+ [_object(0.5, 1.0, 2.0, 0)]
)
single = torch.tensor([_object(1.0, 0.3, 1.0, 3)])
events = pack_events([pair, single], [0, 1])
layout = RowLayout((1, 0, 0, 2, 0), relative_phi=True, pair_objects=2)
table = table_features(events, torch.tensor([0, 1]), layout)
assert table.shape == (2, layout.width) == (2, 3 * 6 + 5 + 2)
slots = table[:, :18].view(2, 3, 6)
# Azimuths from the leading photon's.
torch.testing.assert_close(slots[0, :, 2], torch.tensor([1.5, 0.0, -math.pi]))
assert slots[1, 1, 2] == 0
torch.testing.assert_close(table[0, -2:], torch.tensor([2.0, math.pi]))
assert table[1, -2:].isnan().all()
def test_lora_starts_at_the_frozen_weight_and_trains_only_its_update():
network = nn.Sequential(nn.Linear(6, 4))
weight, x = network[0].weight.detach().clone(), torch.randn(5, 6)
expected = network(x)
add_lora(network, ["0.weight"], rank=2)
torch.testing.assert_close(network(x), expected)
trainable = [name for name, p in network.named_parameters() if p.requires_grad]
assert sorted(trainable) == [
"0.parametrizations.weight.0.down",
"0.parametrizations.weight.0.up",
]
# Code that reads the weight directly sees the update too.
with torch.no_grad():
network[0].parametrizations.weight[0].up.fill_(1.0)
lora = network[0].parametrizations.weight[0]
torch.testing.assert_close(network[0].weight, weight + lora.up @ lora.down)
# Merged for inference, the network computes the same.
expected = network(x)
merge_lora(network)
assert not parametrize.is_parametrized(network[0])
torch.testing.assert_close(network(x), expected)
def test_random_evaluation_subsets_are_seeded_and_sorted():
indices = torch.arange(100) * 3
subset = PackedEvents.random_subset(indices, 20, seed=1)
assert len(subset) == 20 and set(subset.tolist()) <= set(indices.tolist())
assert torch.equal(subset, subset.sort().values)
assert torch.equal(subset, PackedEvents.random_subset(indices, 20, seed=1))
assert len(PackedEvents.random_subset(indices, 500, seed=1)) == 100
def test_balanced_subsets_are_equal_per_class_and_nested():
labels = torch.tensor([0] * 50 + [1] * 30 + [2] * 5)
events = pack_events([torch.zeros(1, 7)] * len(labels), labels)
indices = torch.arange(len(labels))
small = events.balanced_subset(indices, 12, seed=3)
assert torch.bincount(labels[small]).tolist() == [4, 4, 4]
large = events.balanced_subset(indices, 30, seed=3)
assert torch.bincount(labels[large]).tolist() == [10, 10, 5] # 5 exist
assert set(small.tolist()) <= set(large.tolist())
assert torch.equal(small, events.balanced_subset(indices, 12, seed=3))
assert not torch.equal(small, events.balanced_subset(indices, 12, seed=4))
def test_tabpfn_classifies_gamgam_in_context(tmp_path):
pytest.importorskip("tabpfn")
config = fixture_config(
"gamgam_tabpfn",
GAMGAM_FIXTURES,
list(range(5)),
tmp_path,
"data.train_events=100",
# Version 2 weights download without a login.
"model.tabpfn_version=v2",
"model.tabpfn_estimators=1",
"split=validation",
)
path = str(save_config(config, tmp_path / "tabpfn.yaml"))
assert main(["prepare", path]) == 0
assert main(["evaluate", path]) == 0
metrics = json.loads((config.path / "validation_metrics.json").read_text())
assert not math.isnan(metrics["metrics"]["auc"])
assert metrics["checkpoint"] == "TabPFN v2 (default checkpoint)"
def _tabpfn_3_5_weights() -> None:
pytest.importorskip("tabpfn")
from tabpfn.model_loading import get_cache_dir
if not (get_cache_dir() / "tabpfn-v3.5-20260909.safetensors").exists():
pytest.skip("TabPFN-3.5 weights are not in the TabPFN cache")
def test_meta_train_tabpfn_with_lora_then_classify_gamgam(tmp_path):
_tabpfn_3_5_weights()
meta = fixture_config(
"delphes_tabpfn",
[TTH_FIXTURE, TTH_FIXTURE],
[0, 1],
tmp_path / "meta",
"train.max_epochs=1",
"task.context_events=[4,16]",
"task.query_events=4",
"task.steps_per_epoch=2",
"task.validation_steps=1",
"model.relative_phi=true",
"model.pair_objects=4",
)
path = str(save_config(meta, tmp_path / "meta.yaml"))
assert main(["prepare", path]) == 0
assert main(["train", path]) == 0
# Of the weights, only the LoRA updates are saved: 24 blocks of 4
# projections and 2 MLP layers.
state = load_checkpoint(meta.path / "best.pt")["model_state_dict"]
weights = [key for key in state if key.endswith(".weight")]
updates = [key for key in state if key.endswith((".down", ".up"))]
assert not weights and len(updates) == 24 * 6 * 2
gamgam = fixture_config(
"gamgam_tabpfn",
GAMGAM_FIXTURES,
list(range(5)),
tmp_path / "gamgam",
f"checkpoint={meta.path}/best.pt",
"data.train_events=40",
"split=validation",
"model.relative_phi=true",
"model.pair_objects=4",
)
path = str(save_config(gamgam, tmp_path / "gamgam.yaml"))
assert main(["prepare", path]) == 0
assert main(["evaluate", path]) == 0
metrics = json.loads((gamgam.path / "validation_metrics.json").read_text())
assert metrics["checkpoint"] == f"{meta.path}/best.pt"
assert math.isfinite(metrics["metrics"]["auc"])
def test_tabicl_classifies_gamgam_in_context_with_physics_features(tmp_path):
pytest.importorskip("tabicl")
config = fixture_config(
"gamgam_tabicl",
GAMGAM_FIXTURES,
list(range(5)),
tmp_path,
"data.train_events=100",
"model.tabicl_estimators=1",
"model.relative_phi=true",
"model.pair_objects=4",
"split=validation",
)
path = str(save_config(config, tmp_path / "tabicl.yaml"))
assert main(["prepare", path]) == 0
assert main(["evaluate", path]) == 0
metrics = json.loads((config.path / "validation_metrics.json").read_text())
assert metrics["checkpoint"] == "TabICL (tabicl-classifier-v2-20260212.ckpt)"
assert math.isfinite(metrics["metrics"]["auc"])
#: faiss and torch crash together on macOS unless OpenMP has one thread.
RETRIEVAL = sys.platform != "darwin" or os.environ.get("OMP_NUM_THREADS") == "1"
@pytest.mark.parametrize(
"context_size",
[
None,
pytest.param(
50,
marks=pytest.mark.skipif(
not RETRIEVAL, reason="retrieval on macOS needs OMP_NUM_THREADS=1"
),
),
],
)
def test_tabdpt_classifies_gamgam_in_context(context_size, tmp_path):
pytest.importorskip("tabdpt")
config = fixture_config(
"gamgam_tabdpt",
GAMGAM_FIXTURES,
list(range(5)),
tmp_path,
"data.train_events=100",
"model.tabdpt_estimators=1",
f"model.tabdpt_context_size={context_size or 'null'}",
"split=validation",
)
path = str(save_config(config, tmp_path / "tabdpt.yaml"))
assert main(["prepare", path]) == 0
assert main(["evaluate", path]) == 0
metrics = json.loads((config.path / "validation_metrics.json").read_text())
assert (
metrics["checkpoint"] == f"TabDPT (context {context_size or 'all'} per query)"
)
assert math.isfinite(metrics["metrics"]["auc"])
def test_meta_train_tabicl_with_lora_then_classify_gamgam(tmp_path):
pytest.importorskip("tabicl")
meta = fixture_config(
"delphes_tabicl",
[TTH_FIXTURE, TTH_FIXTURE],
[0, 1],
tmp_path / "meta",
"train.max_epochs=1",
"task.context_events=[4,16]",
"task.query_events=4",
"task.steps_per_epoch=2",
"task.validation_steps=1",
"model.pair_objects=4",
)
path = str(save_config(meta, tmp_path / "meta.yaml"))
assert main(["prepare", path]) == 0
assert main(["train", path]) == 0
# Only the LoRA updates are saved: 12 blocks of 4 weights.
state = load_checkpoint(meta.path / "best.pt")["model_state_dict"]
assert len(state) == 12 * 4 * 2
assert all(key.endswith((".down", ".up")) for key in state)
gamgam = fixture_config(
"gamgam_tabicl",
GAMGAM_FIXTURES,
list(range(5)),
tmp_path / "gamgam",
f"checkpoint={meta.path}/best.pt",
"data.train_events=40",
"split=validation",
"model.pair_objects=4",
)
path = str(save_config(gamgam, tmp_path / "gamgam.yaml"))
assert main(["prepare", path]) == 0
assert main(["evaluate", path]) == 0
metrics = json.loads((gamgam.path / "validation_metrics.json").read_text())
assert metrics["checkpoint"] == f"{meta.path}/best.pt"
assert math.isfinite(metrics["metrics"]["auc"])
#: The encoder of the hybrid tests, as pretrained by JEPA on the fixtures.
ENCODER = {
"hid_size": 16,
"n_layers": 1,
"n_heads": 2,
"ffn_size": 64,
"pair_bias": True,
}
def test_event_inputs_are_rows_embeddings_or_both():
events = pack_events([_objects([0, 0, 3, 3, 4]), _objects([1, 3, 4])], [0, 1])
batch = collate_sequence_batch(events, torch.tensor([0, 1]))
width = RowLayout((6, 2, 2, 3, 1)).width
for inputs, expected in (("rows", width), ("embeddings", 16), ("both", width + 16)):
config = ModelConfig(name="tabicl", tabular_inputs=inputs, **ENCODER)
model = EventInputs.of(config, num_features=7, num_node_types=5)
assert model(batch).shape == (2, expected)
with pytest.raises(ValueError, match="tabular_inputs"):
EventInputs.of(
ModelConfig(tabular_inputs="objects"), num_features=7, num_node_types=5
)
@pytest.fixture
def jepa_encoder(tmp_path):
small = [f"model.{key}={value}" for key, value in ENCODER.items()]
config = fixture_config(
"delphes_jepa",
[TTH_FIXTURE, TTH_FIXTURE],
[0, 1],
tmp_path / "jepa",
"train.max_epochs=1",
*small,
)
path = str(save_config(config, tmp_path / "jepa.yaml"))
assert main(["prepare", path]) == 0
assert main(["train", path]) == 0
return f"{config.path}/last.pt"
def _hybrid(name, tmp_path, jepa_encoder, *overrides):
small = [f"model.{key}={value}" for key, value in ENCODER.items()]
config = fixture_config(
name,
GAMGAM_FIXTURES if name.startswith("gamgam") else [TTH_FIXTURE, TTH_FIXTURE],
list(range(5)) if name.startswith("gamgam") else [0, 1],
tmp_path / name,
"model.tabular_inputs=both",
f"model.pretrained={jepa_encoder}",
*small,
*overrides,
)
path = str(save_config(config, tmp_path / f"{name}.yaml"))
assert main(["prepare", path]) == 0
return config, path
def test_tabicl_classifies_gamgam_on_rows_and_jepa_embeddings(tmp_path, jepa_encoder):
pytest.importorskip("tabicl")
config, path = _hybrid(
"gamgam_tabicl",
tmp_path,
jepa_encoder,
"data.train_events=100",
"model.tabicl_estimators=1",
"split=validation",
)
assert main(["evaluate", path]) == 0
metrics = json.loads((config.path / "validation_metrics.json").read_text())
assert math.isfinite(metrics["metrics"]["auc"])
@pytest.mark.parametrize("freeze", [False, True])
def test_meta_train_tabicl_with_its_encoder(tmp_path, jepa_encoder, freeze):
pytest.importorskip("tabicl")
meta, path = _hybrid(
"delphes_tabicl",
tmp_path,
jepa_encoder,
"train.max_epochs=1",
"task.context_events=[4,16]",
"task.query_events=4",
"task.steps_per_epoch=2",
"task.validation_steps=1",
f"model.freeze_backbone={str(freeze).lower()}",
)
assert main(["train", path]) == 0
state = load_checkpoint(meta.path / "best.pt")["model_state_dict"]
encoder = [key for key in state if key.startswith("inputs.encoder.")]
# A trained encoder is saved; a frozen one comes from model.pretrained.
assert bool(encoder) != freeze
gamgam, path = _hybrid(
"gamgam_tabicl",
tmp_path,
jepa_encoder,
f"checkpoint={meta.path}/best.pt",
"data.train_events=40",
"split=validation",
f"model.freeze_backbone={str(freeze).lower()}",
)
assert main(["evaluate", path]) == 0
metrics = json.loads((gamgam.path / "validation_metrics.json").read_text())
assert math.isfinite(metrics["metrics"]["auc"])