GNN4Colliders / scripts /compare_encoders.py
ho22joshua's picture
Claude Opus 5.5
fix: CKA of collapsed encoders, encoder labels and seeding
ecb17a5
Raw History Blame Contribute Delete
5.43 kB
"""Linear CKA between object encoders, as in Ho et al. (2026): of their event
embeddings, and layer by layer against the first encoder.
All encoders have the architecture of the config (``model.backbone`` of a
pretraining config, or the transformer or edge network of a classifier's):
uv run python scripts/compare_encoders.py configs/gamgam_jepa_finetune.yaml \\
--encoders random=random jepa=outputs/delphes_jepa/last.pt \\
contrastive=outputs/delphes_contrastive/last.pt \\
masked=outputs/delphes_masked/last.pt --events 5000
``random`` is a randomly initialized encoder (the reference). The events are
``--events`` random events of ``--split``, drawn with ``data.seed``. Writes
``--output`` (JSON; undefined values, of an encoder that gives every event
the same embedding, are null) and prints Markdown tables.
"""
from __future__ import annotations
import argparse
import dataclasses
import json
import math
from pathlib import Path
from gnn4colliders.config import load_config
from gnn4colliders.data import load_events
from gnn4colliders.distributed import resolve_device
from gnn4colliders.probe import layer_embeddings, linear_cka, load_encoder
def _table(rows: list[str], columns: list[str], values: list[list[float]]) -> str:
lines = [
"| | " + " | ".join(columns) + " |",
"|---" * (len(columns) + 1) + "|",
]
for row, numbers in zip(rows, values):
lines.append(f"| {row} | " + " | ".join(f"{v:.3f}" for v in numbers) + " |")
return "\n".join(lines)
def _encoders(parser: argparse.ArgumentParser, entries: list[str]) -> dict[str, str]:
"""``label=path`` entries as {label: path}, labels distinct."""
encoders: dict[str, str] = {}
for entry in entries:
label, separator, path = entry.partition("=")
if not separator or not label or not path:
parser.error(
f"--encoders takes label=path (or label=random), not {entry!r}"
)
if label in encoders:
parser.error(f"--encoders label {label!r} is given twice")
encoders[label] = path
return encoders
def _finite(values: list[float]) -> list[float | None]:
return [None if math.isnan(value) else value for value in values]
def main() -> None:
parser = argparse.ArgumentParser(
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
)
parser.add_argument("config", type=Path)
parser.add_argument("--encoders", nargs="+", required=True, help="label=path")
parser.add_argument("--events", type=int, default=5000)
parser.add_argument("--split", default="validation")
parser.add_argument("--override", nargs="*", default=[], help="key=value")
parser.add_argument("--output", type=Path, default=Path("outputs/cka.json"))
args = parser.parse_args()
encoders = _encoders(parser, args.encoders)
config = load_config(args.config, args.override)
events = load_events(config.data.cache)
folds = {
"train": config.data.train_folds,
"validation": config.data.validation_folds,
"test": config.data.test_folds,
}[args.split]
indices = events.split_indices(folds)
indices = events.random_subset(indices, args.events, config.data.seed)
device = resolve_device(config.train.device)
layers = {}
for label, path in encoders.items():
model = dataclasses.replace(
config.model, pretrained=None if path == "random" else path
)
encoder_config = dataclasses.replace(config, model=model)
encoder = load_encoder(encoder_config, events).to(device)
layers[label] = layer_embeddings(
encoder,
events,
indices,
batch_size=config.data.batch_size,
device=device,
)
labels = list(layers)
final = [[linear_cka(layers[a][-1], layers[b][-1]) for b in labels] for a in labels]
reference = labels[0]
layerwise = {
label: [linear_cka(x, y) for x, y in zip(layers[reference], layers[label])]
for label in labels[1:]
}
depth = len(layers[reference])
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(
json.dumps(
{
"events": len(indices),
"split": args.split,
"encoders": encoders,
"embeddings": {
label: _finite(row) for label, row in zip(labels, final)
},
"layerwise_against": reference,
"layerwise": {label: _finite(row) for label, row in layerwise.items()},
},
indent=2,
allow_nan=False,
)
+ "\n"
)
print(f"Linear CKA of the event embeddings ({len(indices)} {args.split} events):\n")
print(_table(labels, labels, final))
# An encoder's CKA with itself is undefined only if it does not vary.
undefined = [
label for index, label in enumerate(labels) if math.isnan(final[index][index])
]
if undefined:
print(f"\nnan: {', '.join(undefined)} give every event the same embedding")
if layerwise:
print(f"\nLayer by layer against {reference} (last: the event embedding):\n")
columns = [f"layer {index}" for index in range(depth)]
print(_table(list(layerwise), columns, list(layerwise.values())))
if __name__ == "__main__":
main()