"""k-NN and linear probes of frozen encoders: the classifiers, the encoder weights of each kind of checkpoint, and the probe command.""" import json import math import runpy import sys import pytest import torch from conftest import GAMGAM_FIXTURES, ROOT, TTH_FIXTURE, fixture_config from gnn4colliders.cli import main from gnn4colliders.config import ModelConfig, save_config from gnn4colliders.data import load_events from gnn4colliders.models import encoder_state, pretrained_encoder from gnn4colliders.probe import ( knn_logits, layer_embeddings, linear_cka, linear_logits, ) SMALL = {"hid_size": 16, "n_layers": 1, "n_heads": 2, "pair_bias": True} BACKBONES = { "transformer": SMALL, "edge_network": {"hid_size": 16, "n_layers": 2, "n_proc_steps": 2, "n_heads": 2}, } def _clusters(count=60, seed=0): """Two well separated classes of 8-dimensional embeddings.""" generator = torch.Generator().manual_seed(seed) labels = torch.arange(count) % 2 centres = torch.tensor([[3.0] + [0.0] * 7, [0.0, 3.0] + [0.0] * 6]) return centres[labels] + 0.3 * torch.randn(count, 8, generator=generator), labels def test_knn_and_linear_probes_separate_clusters(): train, labels = _clusters(seed=0) queries, expected = _clusters(seed=1) for logits in ( knn_logits(train, labels, queries, num_classes=2, neighbours=5), linear_logits(train, labels, queries, num_classes=2, l2=1e-4), ): assert logits.shape == (60, 2) assert torch.equal(logits.argmax(dim=1), expected) def test_encoder_weights_of_each_kind_of_checkpoint(tmp_path): weight = torch.ones(2) checkpoints = { "jepa": {"context_encoder.norm.weight": weight, "target_encoder.x": weight}, "transformer": {"encoder.norm.weight": weight, "classifier.weight": weight}, "edge_network": {"node_encoder.0.weight": weight, "classifier.weight": weight}, "fine_tuned": {"backbone.node_encoder.0.weight": weight, "head.weight": weight}, "tabular": {"inputs.encoder.norm.weight": weight, "network.x.up": weight}, } expected = { "jepa": ["norm.weight"], "transformer": ["norm.weight"], "edge_network": ["node_encoder.0.weight"], "fine_tuned": ["node_encoder.0.weight"], "tabular": ["norm.weight"], } for name, state in checkpoints.items(): path = tmp_path / f"{name}.pt" torch.save({"model_state_dict": state}, path) assert list(encoder_state(str(path))) == expected[name] @pytest.mark.parametrize( ("backbone", "pretrain_config", "probe_config"), [ ("transformer", "delphes_jepa", "gamgam_jepa_finetune"), ("edge_network", "delphes_jepa_edge", "gamgam_finetune"), ], ) def test_probe_random_and_jepa_encoders_on_gamgam( backbone, pretrain_config, probe_config, tmp_path ): small = [f"model.{key}={value}" for key, value in BACKBONES[backbone].items()] pretrain = fixture_config( pretrain_config, [TTH_FIXTURE, TTH_FIXTURE], [0, 1], tmp_path / "jepa", "train.max_epochs=1", *small, ) path = str(save_config(pretrain, tmp_path / "jepa.yaml")) assert main(["prepare", path]) == 0 assert main(["train", path]) == 0 for name, encoder in (("random", "null"), ("jepa", f"{pretrain.path}/last.pt")): config = fixture_config( probe_config, GAMGAM_FIXTURES, list(range(5)), tmp_path / name, f"model.pretrained={encoder}", "data.train_events=100", "model.probe_neighbours=5", "split=validation", *small, ) path = str(save_config(config, tmp_path / f"{name}.yaml")) assert main(["prepare", path]) == 0 assert main(["probe", path]) == 0 summary = json.loads((config.path / "validation_probe.json").read_text()) assert summary["train_events"] == 100 for probe in ("knn", "linear"): assert math.isfinite(summary[probe]["auc"]) def test_probe_a_supervised_transformer(tmp_path): small = [f"model.{key}={value}" for key, value in SMALL.items()] trained = fixture_config( "gamgam_transformer", GAMGAM_FIXTURES, list(range(5)), tmp_path / "supervised", "train.max_epochs=1", *small, ) path = str(save_config(trained, tmp_path / "supervised.yaml")) assert main(["prepare", path]) == 0 assert main(["train", path]) == 0 config = fixture_config( "gamgam_transformer", GAMGAM_FIXTURES, list(range(5)), tmp_path / "probe", f"model.pretrained={trained.path}/last.pt", "data.train_events=50", *small, ) path = str(save_config(config, tmp_path / "probe.yaml")) assert main(["prepare", path]) == 0 assert main(["probe", path]) == 0 summary = json.loads((config.path / "test_probe.json").read_text()) assert summary["encoder"] == f"{trained.path}/last.pt" # On the training split, the probes' own events are not queries. assert main(["probe", path, "split=train"]) == 0 summary = json.loads((config.path / "train_probe.json").read_text()) events = load_events(config.data.cache) train_fold = torch.isin(events.fold, torch.tensor(config.data.train_folds)) assert summary["num_samples"] == int(train_fold.sum()) - 50 def test_a_random_encoder_is_rebuilt_identically_without_touching_global_state(): config = ModelConfig(name="transformer", **SMALL) state = torch.random.get_rng_state() first, second = ( pretrained_encoder(config, num_features=7, num_node_types=5) for _ in range(2) ) assert torch.equal(torch.random.get_rng_state(), state) for a, b in zip(first.parameters(), second.parameters()): assert torch.equal(a, b) def test_linear_cka_is_one_for_rotations_and_scales_and_small_for_noise(): generator = torch.Generator().manual_seed(0) x = torch.randn(500, 8, generator=generator) rotation, _ = torch.linalg.qr(torch.randn(8, 8, generator=generator)) assert linear_cka(x, x) == pytest.approx(1.0) assert linear_cka(x, 3.0 * x @ rotation + 1.0) == pytest.approx(1.0) assert linear_cka(x, torch.randn(500, 8, generator=generator)) < 0.1 # A representation that drops half the dimensions is in between. assert 0.3 < linear_cka(x, x[:, :4]) < 0.9 def test_linear_cka_of_a_constant_representation_is_undefined(): x = torch.randn(100, 8, generator=torch.Generator().manual_seed(0)) # 0.1 is not exact in binary: centering leaves rounding, not zeros. constant = torch.full((100, 8), 0.1) assert math.isnan(linear_cka(x, constant)) assert math.isnan(linear_cka(constant, constant)) @pytest.mark.parametrize("backbone", BACKBONES) def test_layer_embeddings_end_with_the_event_embedding(backbone): from conftest import random_events from gnn4colliders.jepa import event_embedding from gnn4colliders.sequences import collate_sequence_batch events = random_events(count=30) events.object_features[:, 6] = torch.randint( 0, 5, (events.object_features.shape[0],) ).float() config = ModelConfig(name=backbone, **BACKBONES[backbone]) encoder = pretrained_encoder( config, num_features=7, num_node_types=5, backbone=backbone ).eval() indices = torch.arange(30) layers = layer_embeddings( encoder, events, indices, batch_size=8, device=torch.device("cpu") ) # Transformer: 1 block and the final norm; edge network: encoder, 2 steps, # global state. assert len(layers) == {"transformer": 2, "edge_network": 4}[backbone] assert all(layer.shape == (30, 16) for layer in layers) batch = collate_sequence_batch(events, indices) with torch.no_grad(): expected = event_embedding(encoder, batch.tokens, batch.token_mask) torch.testing.assert_close(layers[-1], expected) def test_compare_encoders_script(tmp_path, monkeypatch, capsys): small = [f"model.{key}={value}" for key, value in SMALL.items()] pretrain = fixture_config( "delphes_jepa", [TTH_FIXTURE, TTH_FIXTURE], [0, 1], tmp_path / "jepa", "train.max_epochs=1", "model.ffn_size=64", *small, ) path = str(save_config(pretrain, tmp_path / "jepa.yaml")) assert main(["prepare", path]) == 0 assert main(["train", path]) == 0 config = fixture_config( "gamgam_jepa_finetune", GAMGAM_FIXTURES, list(range(5)), tmp_path / "gamgam", "model.ffn_size=64", *small, ) path = str(save_config(config, tmp_path / "gamgam.yaml")) assert main(["prepare", path]) == 0 output = tmp_path / "cka.json" encoders = ["random=random", f"jepa={pretrain.path}/last.pt"] argv = [path, "--encoders", *encoders, "--events", "30", "--output", str(output)] monkeypatch.setattr(sys, "argv", ["compare_encoders", *argv]) runpy.run_path(str(ROOT / "scripts/compare_encoders.py"), run_name="__main__") summary = json.loads(output.read_text()) assert summary["events"] == 30 embeddings = summary["embeddings"] assert embeddings["random"][0] == pytest.approx(1.0) assert embeddings["jepa"][1] == pytest.approx(1.0) # The two encoders differ, and CKA is symmetric. assert 0 < embeddings["jepa"][0] < 1 - 1e-6 assert embeddings["jepa"][0] == pytest.approx(embeddings["random"][1]) layerwise = summary["layerwise"]["jepa"] assert len(layerwise) == 2 and all(0 < value <= 1 for value in layerwise) assert layerwise[-1] == pytest.approx(embeddings["jepa"][0]) assert "Layer by layer against random" in capsys.readouterr().out for encoders in (["random=random", "random=random"], ["last.pt"]): argv = [path, "--encoders", *encoders, "--output", str(output)] monkeypatch.setattr(sys, "argv", ["compare_encoders", *argv]) with pytest.raises(SystemExit) as error: runpy.run_path( str(ROOT / "scripts/compare_encoders.py"), run_name="__main__" ) assert error.value.code == 2