GNN4Colliders / tests /test_probe.py
ho22joshua's picture
Claude Opus 5.5
fix: CKA of collapsed encoders, encoder labels and seeding
ecb17a5
Raw History Blame Contribute Delete
10.2 kB
"""k-NN and linear probes of frozen encoders: the classifiers, the encoder
weights of each kind of checkpoint, and the probe command."""
import json
import math
import runpy
import sys
import pytest
import torch
from conftest import GAMGAM_FIXTURES, ROOT, TTH_FIXTURE, fixture_config
from gnn4colliders.cli import main
from gnn4colliders.config import ModelConfig, save_config
from gnn4colliders.data import load_events
from gnn4colliders.models import encoder_state, pretrained_encoder
from gnn4colliders.probe import (
knn_logits,
layer_embeddings,
linear_cka,
linear_logits,
)
SMALL = {"hid_size": 16, "n_layers": 1, "n_heads": 2, "pair_bias": True}
BACKBONES = {
"transformer": SMALL,
"edge_network": {"hid_size": 16, "n_layers": 2, "n_proc_steps": 2, "n_heads": 2},
}
def _clusters(count=60, seed=0):
"""Two well separated classes of 8-dimensional embeddings."""
generator = torch.Generator().manual_seed(seed)
labels = torch.arange(count) % 2
centres = torch.tensor([[3.0] + [0.0] * 7, [0.0, 3.0] + [0.0] * 6])
return centres[labels] + 0.3 * torch.randn(count, 8, generator=generator), labels
def test_knn_and_linear_probes_separate_clusters():
train, labels = _clusters(seed=0)
queries, expected = _clusters(seed=1)
for logits in (
knn_logits(train, labels, queries, num_classes=2, neighbours=5),
linear_logits(train, labels, queries, num_classes=2, l2=1e-4),
):
assert logits.shape == (60, 2)
assert torch.equal(logits.argmax(dim=1), expected)
def test_encoder_weights_of_each_kind_of_checkpoint(tmp_path):
weight = torch.ones(2)
checkpoints = {
"jepa": {"context_encoder.norm.weight": weight, "target_encoder.x": weight},
"transformer": {"encoder.norm.weight": weight, "classifier.weight": weight},
"edge_network": {"node_encoder.0.weight": weight, "classifier.weight": weight},
"fine_tuned": {"backbone.node_encoder.0.weight": weight, "head.weight": weight},
"tabular": {"inputs.encoder.norm.weight": weight, "network.x.up": weight},
}
expected = {
"jepa": ["norm.weight"],
"transformer": ["norm.weight"],
"edge_network": ["node_encoder.0.weight"],
"fine_tuned": ["node_encoder.0.weight"],
"tabular": ["norm.weight"],
}
for name, state in checkpoints.items():
path = tmp_path / f"{name}.pt"
torch.save({"model_state_dict": state}, path)
assert list(encoder_state(str(path))) == expected[name]
@pytest.mark.parametrize(
("backbone", "pretrain_config", "probe_config"),
[
("transformer", "delphes_jepa", "gamgam_jepa_finetune"),
("edge_network", "delphes_jepa_edge", "gamgam_finetune"),
],
)
def test_probe_random_and_jepa_encoders_on_gamgam(
backbone, pretrain_config, probe_config, tmp_path
):
small = [f"model.{key}={value}" for key, value in BACKBONES[backbone].items()]
pretrain = fixture_config(
pretrain_config,
[TTH_FIXTURE, TTH_FIXTURE],
[0, 1],
tmp_path / "jepa",
"train.max_epochs=1",
*small,
)
path = str(save_config(pretrain, tmp_path / "jepa.yaml"))
assert main(["prepare", path]) == 0
assert main(["train", path]) == 0
for name, encoder in (("random", "null"), ("jepa", f"{pretrain.path}/last.pt")):
config = fixture_config(
probe_config,
GAMGAM_FIXTURES,
list(range(5)),
tmp_path / name,
f"model.pretrained={encoder}",
"data.train_events=100",
"model.probe_neighbours=5",
"split=validation",
*small,
)
path = str(save_config(config, tmp_path / f"{name}.yaml"))
assert main(["prepare", path]) == 0
assert main(["probe", path]) == 0
summary = json.loads((config.path / "validation_probe.json").read_text())
assert summary["train_events"] == 100
for probe in ("knn", "linear"):
assert math.isfinite(summary[probe]["auc"])
def test_probe_a_supervised_transformer(tmp_path):
small = [f"model.{key}={value}" for key, value in SMALL.items()]
trained = fixture_config(
"gamgam_transformer",
GAMGAM_FIXTURES,
list(range(5)),
tmp_path / "supervised",
"train.max_epochs=1",
*small,
)
path = str(save_config(trained, tmp_path / "supervised.yaml"))
assert main(["prepare", path]) == 0
assert main(["train", path]) == 0
config = fixture_config(
"gamgam_transformer",
GAMGAM_FIXTURES,
list(range(5)),
tmp_path / "probe",
f"model.pretrained={trained.path}/last.pt",
"data.train_events=50",
*small,
)
path = str(save_config(config, tmp_path / "probe.yaml"))
assert main(["prepare", path]) == 0
assert main(["probe", path]) == 0
summary = json.loads((config.path / "test_probe.json").read_text())
assert summary["encoder"] == f"{trained.path}/last.pt"
# On the training split, the probes' own events are not queries.
assert main(["probe", path, "split=train"]) == 0
summary = json.loads((config.path / "train_probe.json").read_text())
events = load_events(config.data.cache)
train_fold = torch.isin(events.fold, torch.tensor(config.data.train_folds))
assert summary["num_samples"] == int(train_fold.sum()) - 50
def test_a_random_encoder_is_rebuilt_identically_without_touching_global_state():
config = ModelConfig(name="transformer", **SMALL)
state = torch.random.get_rng_state()
first, second = (
pretrained_encoder(config, num_features=7, num_node_types=5) for _ in range(2)
)
assert torch.equal(torch.random.get_rng_state(), state)
for a, b in zip(first.parameters(), second.parameters()):
assert torch.equal(a, b)
def test_linear_cka_is_one_for_rotations_and_scales_and_small_for_noise():
generator = torch.Generator().manual_seed(0)
x = torch.randn(500, 8, generator=generator)
rotation, _ = torch.linalg.qr(torch.randn(8, 8, generator=generator))
assert linear_cka(x, x) == pytest.approx(1.0)
assert linear_cka(x, 3.0 * x @ rotation + 1.0) == pytest.approx(1.0)
assert linear_cka(x, torch.randn(500, 8, generator=generator)) < 0.1
# A representation that drops half the dimensions is in between.
assert 0.3 < linear_cka(x, x[:, :4]) < 0.9
def test_linear_cka_of_a_constant_representation_is_undefined():
x = torch.randn(100, 8, generator=torch.Generator().manual_seed(0))
# 0.1 is not exact in binary: centering leaves rounding, not zeros.
constant = torch.full((100, 8), 0.1)
assert math.isnan(linear_cka(x, constant))
assert math.isnan(linear_cka(constant, constant))
@pytest.mark.parametrize("backbone", BACKBONES)
def test_layer_embeddings_end_with_the_event_embedding(backbone):
from conftest import random_events
from gnn4colliders.jepa import event_embedding
from gnn4colliders.sequences import collate_sequence_batch
events = random_events(count=30)
events.object_features[:, 6] = torch.randint(
0, 5, (events.object_features.shape[0],)
).float()
config = ModelConfig(name=backbone, **BACKBONES[backbone])
encoder = pretrained_encoder(
config, num_features=7, num_node_types=5, backbone=backbone
).eval()
indices = torch.arange(30)
layers = layer_embeddings(
encoder, events, indices, batch_size=8, device=torch.device("cpu")
)
# Transformer: 1 block and the final norm; edge network: encoder, 2 steps,
# global state.
assert len(layers) == {"transformer": 2, "edge_network": 4}[backbone]
assert all(layer.shape == (30, 16) for layer in layers)
batch = collate_sequence_batch(events, indices)
with torch.no_grad():
expected = event_embedding(encoder, batch.tokens, batch.token_mask)
torch.testing.assert_close(layers[-1], expected)
def test_compare_encoders_script(tmp_path, monkeypatch, capsys):
small = [f"model.{key}={value}" for key, value in SMALL.items()]
pretrain = fixture_config(
"delphes_jepa",
[TTH_FIXTURE, TTH_FIXTURE],
[0, 1],
tmp_path / "jepa",
"train.max_epochs=1",
"model.ffn_size=64",
*small,
)
path = str(save_config(pretrain, tmp_path / "jepa.yaml"))
assert main(["prepare", path]) == 0
assert main(["train", path]) == 0
config = fixture_config(
"gamgam_jepa_finetune",
GAMGAM_FIXTURES,
list(range(5)),
tmp_path / "gamgam",
"model.ffn_size=64",
*small,
)
path = str(save_config(config, tmp_path / "gamgam.yaml"))
assert main(["prepare", path]) == 0
output = tmp_path / "cka.json"
encoders = ["random=random", f"jepa={pretrain.path}/last.pt"]
argv = [path, "--encoders", *encoders, "--events", "30", "--output", str(output)]
monkeypatch.setattr(sys, "argv", ["compare_encoders", *argv])
runpy.run_path(str(ROOT / "scripts/compare_encoders.py"), run_name="__main__")
summary = json.loads(output.read_text())
assert summary["events"] == 30
embeddings = summary["embeddings"]
assert embeddings["random"][0] == pytest.approx(1.0)
assert embeddings["jepa"][1] == pytest.approx(1.0)
# The two encoders differ, and CKA is symmetric.
assert 0 < embeddings["jepa"][0] < 1 - 1e-6
assert embeddings["jepa"][0] == pytest.approx(embeddings["random"][1])
layerwise = summary["layerwise"]["jepa"]
assert len(layerwise) == 2 and all(0 < value <= 1 for value in layerwise)
assert layerwise[-1] == pytest.approx(embeddings["jepa"][0])
assert "Layer by layer against random" in capsys.readouterr().out
for encoders in (["random=random", "random=random"], ["last.pt"]):
argv = [path, "--encoders", *encoders, "--output", str(output)]
monkeypatch.setattr(sys, "argv", ["compare_encoders", *argv])
with pytest.raises(SystemExit) as error:
runpy.run_path(
str(ROOT / "scripts/compare_encoders.py"), run_name="__main__"
)
assert error.value.code == 2