Download tests/test_probe.py from HWresearch/GNN4Colliders: direct link, hf CLI and curl.
- Browser
- Download file 10.2 kB
-
https://huggingface.co/HWresearch/GNN4Colliders/resolve/main/tests/test_probe.py
- Command line
-
hf download hf://HWresearch/GNN4Colliders/tests/test_probe.py
-
curl -L -o test_probe.py https://huggingface.co/HWresearch/GNN4Colliders/resolve/main/tests/test_probe.py
10.2 kB
| """k-NN and linear probes of frozen encoders: the classifiers, the encoder | |
| weights of each kind of checkpoint, and the probe command.""" | |
| import json | |
| import math | |
| import runpy | |
| import sys | |
| import pytest | |
| import torch | |
| from conftest import GAMGAM_FIXTURES, ROOT, TTH_FIXTURE, fixture_config | |
| from gnn4colliders.cli import main | |
| from gnn4colliders.config import ModelConfig, save_config | |
| from gnn4colliders.data import load_events | |
| from gnn4colliders.models import encoder_state, pretrained_encoder | |
| from gnn4colliders.probe import ( | |
| knn_logits, | |
| layer_embeddings, | |
| linear_cka, | |
| linear_logits, | |
| ) | |
| SMALL = {"hid_size": 16, "n_layers": 1, "n_heads": 2, "pair_bias": True} | |
| BACKBONES = { | |
| "transformer": SMALL, | |
| "edge_network": {"hid_size": 16, "n_layers": 2, "n_proc_steps": 2, "n_heads": 2}, | |
| } | |
| def _clusters(count=60, seed=0): | |
| """Two well separated classes of 8-dimensional embeddings.""" | |
| generator = torch.Generator().manual_seed(seed) | |
| labels = torch.arange(count) % 2 | |
| centres = torch.tensor([[3.0] + [0.0] * 7, [0.0, 3.0] + [0.0] * 6]) | |
| return centres[labels] + 0.3 * torch.randn(count, 8, generator=generator), labels | |
| def test_knn_and_linear_probes_separate_clusters(): | |
| train, labels = _clusters(seed=0) | |
| queries, expected = _clusters(seed=1) | |
| for logits in ( | |
| knn_logits(train, labels, queries, num_classes=2, neighbours=5), | |
| linear_logits(train, labels, queries, num_classes=2, l2=1e-4), | |
| ): | |
| assert logits.shape == (60, 2) | |
| assert torch.equal(logits.argmax(dim=1), expected) | |
| def test_encoder_weights_of_each_kind_of_checkpoint(tmp_path): | |
| weight = torch.ones(2) | |
| checkpoints = { | |
| "jepa": {"context_encoder.norm.weight": weight, "target_encoder.x": weight}, | |
| "transformer": {"encoder.norm.weight": weight, "classifier.weight": weight}, | |
| "edge_network": {"node_encoder.0.weight": weight, "classifier.weight": weight}, | |
| "fine_tuned": {"backbone.node_encoder.0.weight": weight, "head.weight": weight}, | |
| "tabular": {"inputs.encoder.norm.weight": weight, "network.x.up": weight}, | |
| } | |
| expected = { | |
| "jepa": ["norm.weight"], | |
| "transformer": ["norm.weight"], | |
| "edge_network": ["node_encoder.0.weight"], | |
| "fine_tuned": ["node_encoder.0.weight"], | |
| "tabular": ["norm.weight"], | |
| } | |
| for name, state in checkpoints.items(): | |
| path = tmp_path / f"{name}.pt" | |
| torch.save({"model_state_dict": state}, path) | |
| assert list(encoder_state(str(path))) == expected[name] | |
| def test_probe_random_and_jepa_encoders_on_gamgam( | |
| backbone, pretrain_config, probe_config, tmp_path | |
| ): | |
| small = [f"model.{key}={value}" for key, value in BACKBONES[backbone].items()] | |
| pretrain = fixture_config( | |
| pretrain_config, | |
| [TTH_FIXTURE, TTH_FIXTURE], | |
| [0, 1], | |
| tmp_path / "jepa", | |
| "train.max_epochs=1", | |
| *small, | |
| ) | |
| path = str(save_config(pretrain, tmp_path / "jepa.yaml")) | |
| assert main(["prepare", path]) == 0 | |
| assert main(["train", path]) == 0 | |
| for name, encoder in (("random", "null"), ("jepa", f"{pretrain.path}/last.pt")): | |
| config = fixture_config( | |
| probe_config, | |
| GAMGAM_FIXTURES, | |
| list(range(5)), | |
| tmp_path / name, | |
| f"model.pretrained={encoder}", | |
| "data.train_events=100", | |
| "model.probe_neighbours=5", | |
| "split=validation", | |
| *small, | |
| ) | |
| path = str(save_config(config, tmp_path / f"{name}.yaml")) | |
| assert main(["prepare", path]) == 0 | |
| assert main(["probe", path]) == 0 | |
| summary = json.loads((config.path / "validation_probe.json").read_text()) | |
| assert summary["train_events"] == 100 | |
| for probe in ("knn", "linear"): | |
| assert math.isfinite(summary[probe]["auc"]) | |
| def test_probe_a_supervised_transformer(tmp_path): | |
| small = [f"model.{key}={value}" for key, value in SMALL.items()] | |
| trained = fixture_config( | |
| "gamgam_transformer", | |
| GAMGAM_FIXTURES, | |
| list(range(5)), | |
| tmp_path / "supervised", | |
| "train.max_epochs=1", | |
| *small, | |
| ) | |
| path = str(save_config(trained, tmp_path / "supervised.yaml")) | |
| assert main(["prepare", path]) == 0 | |
| assert main(["train", path]) == 0 | |
| config = fixture_config( | |
| "gamgam_transformer", | |
| GAMGAM_FIXTURES, | |
| list(range(5)), | |
| tmp_path / "probe", | |
| f"model.pretrained={trained.path}/last.pt", | |
| "data.train_events=50", | |
| *small, | |
| ) | |
| path = str(save_config(config, tmp_path / "probe.yaml")) | |
| assert main(["prepare", path]) == 0 | |
| assert main(["probe", path]) == 0 | |
| summary = json.loads((config.path / "test_probe.json").read_text()) | |
| assert summary["encoder"] == f"{trained.path}/last.pt" | |
| # On the training split, the probes' own events are not queries. | |
| assert main(["probe", path, "split=train"]) == 0 | |
| summary = json.loads((config.path / "train_probe.json").read_text()) | |
| events = load_events(config.data.cache) | |
| train_fold = torch.isin(events.fold, torch.tensor(config.data.train_folds)) | |
| assert summary["num_samples"] == int(train_fold.sum()) - 50 | |
| def test_a_random_encoder_is_rebuilt_identically_without_touching_global_state(): | |
| config = ModelConfig(name="transformer", **SMALL) | |
| state = torch.random.get_rng_state() | |
| first, second = ( | |
| pretrained_encoder(config, num_features=7, num_node_types=5) for _ in range(2) | |
| ) | |
| assert torch.equal(torch.random.get_rng_state(), state) | |
| for a, b in zip(first.parameters(), second.parameters()): | |
| assert torch.equal(a, b) | |
| def test_linear_cka_is_one_for_rotations_and_scales_and_small_for_noise(): | |
| generator = torch.Generator().manual_seed(0) | |
| x = torch.randn(500, 8, generator=generator) | |
| rotation, _ = torch.linalg.qr(torch.randn(8, 8, generator=generator)) | |
| assert linear_cka(x, x) == pytest.approx(1.0) | |
| assert linear_cka(x, 3.0 * x @ rotation + 1.0) == pytest.approx(1.0) | |
| assert linear_cka(x, torch.randn(500, 8, generator=generator)) < 0.1 | |
| # A representation that drops half the dimensions is in between. | |
| assert 0.3 < linear_cka(x, x[:, :4]) < 0.9 | |
| def test_linear_cka_of_a_constant_representation_is_undefined(): | |
| x = torch.randn(100, 8, generator=torch.Generator().manual_seed(0)) | |
| # 0.1 is not exact in binary: centering leaves rounding, not zeros. | |
| constant = torch.full((100, 8), 0.1) | |
| assert math.isnan(linear_cka(x, constant)) | |
| assert math.isnan(linear_cka(constant, constant)) | |
| def test_layer_embeddings_end_with_the_event_embedding(backbone): | |
| from conftest import random_events | |
| from gnn4colliders.jepa import event_embedding | |
| from gnn4colliders.sequences import collate_sequence_batch | |
| events = random_events(count=30) | |
| events.object_features[:, 6] = torch.randint( | |
| 0, 5, (events.object_features.shape[0],) | |
| ).float() | |
| config = ModelConfig(name=backbone, **BACKBONES[backbone]) | |
| encoder = pretrained_encoder( | |
| config, num_features=7, num_node_types=5, backbone=backbone | |
| ).eval() | |
| indices = torch.arange(30) | |
| layers = layer_embeddings( | |
| encoder, events, indices, batch_size=8, device=torch.device("cpu") | |
| ) | |
| # Transformer: 1 block and the final norm; edge network: encoder, 2 steps, | |
| # global state. | |
| assert len(layers) == {"transformer": 2, "edge_network": 4}[backbone] | |
| assert all(layer.shape == (30, 16) for layer in layers) | |
| batch = collate_sequence_batch(events, indices) | |
| with torch.no_grad(): | |
| expected = event_embedding(encoder, batch.tokens, batch.token_mask) | |
| torch.testing.assert_close(layers[-1], expected) | |
| def test_compare_encoders_script(tmp_path, monkeypatch, capsys): | |
| small = [f"model.{key}={value}" for key, value in SMALL.items()] | |
| pretrain = fixture_config( | |
| "delphes_jepa", | |
| [TTH_FIXTURE, TTH_FIXTURE], | |
| [0, 1], | |
| tmp_path / "jepa", | |
| "train.max_epochs=1", | |
| "model.ffn_size=64", | |
| *small, | |
| ) | |
| path = str(save_config(pretrain, tmp_path / "jepa.yaml")) | |
| assert main(["prepare", path]) == 0 | |
| assert main(["train", path]) == 0 | |
| config = fixture_config( | |
| "gamgam_jepa_finetune", | |
| GAMGAM_FIXTURES, | |
| list(range(5)), | |
| tmp_path / "gamgam", | |
| "model.ffn_size=64", | |
| *small, | |
| ) | |
| path = str(save_config(config, tmp_path / "gamgam.yaml")) | |
| assert main(["prepare", path]) == 0 | |
| output = tmp_path / "cka.json" | |
| encoders = ["random=random", f"jepa={pretrain.path}/last.pt"] | |
| argv = [path, "--encoders", *encoders, "--events", "30", "--output", str(output)] | |
| monkeypatch.setattr(sys, "argv", ["compare_encoders", *argv]) | |
| runpy.run_path(str(ROOT / "scripts/compare_encoders.py"), run_name="__main__") | |
| summary = json.loads(output.read_text()) | |
| assert summary["events"] == 30 | |
| embeddings = summary["embeddings"] | |
| assert embeddings["random"][0] == pytest.approx(1.0) | |
| assert embeddings["jepa"][1] == pytest.approx(1.0) | |
| # The two encoders differ, and CKA is symmetric. | |
| assert 0 < embeddings["jepa"][0] < 1 - 1e-6 | |
| assert embeddings["jepa"][0] == pytest.approx(embeddings["random"][1]) | |
| layerwise = summary["layerwise"]["jepa"] | |
| assert len(layerwise) == 2 and all(0 < value <= 1 for value in layerwise) | |
| assert layerwise[-1] == pytest.approx(embeddings["jepa"][0]) | |
| assert "Layer by layer against random" in capsys.readouterr().out | |
| for encoders in (["random=random", "random=random"], ["last.pt"]): | |
| argv = [path, "--encoders", *encoders, "--output", str(output)] | |
| monkeypatch.setattr(sys, "argv", ["compare_encoders", *argv]) | |
| with pytest.raises(SystemExit) as error: | |
| runpy.run_path( | |
| str(ROOT / "scripts/compare_encoders.py"), run_name="__main__" | |
| ) | |
| assert error.value.code == 2 | |