File size: 7,435 Bytes
0cef53c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8dff74c
0cef53c
e71731f
 
 
 
0cef53c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
df3eb75
 
0cef53c
df3eb75
0cef53c
 
 
 
 
 
 
 
 
f43b1e5
 
 
 
 
 
0cef53c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8dff74c
 
 
0cef53c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
"""End-to-end command-line runs on the 64-event ROOT fixtures."""

import json
import math
import multiprocessing as mp
import os
import socket
import sys

import numpy as np
import pytest
import torch
import uproot
from conftest import GAMGAM_FIXTURES, PRETRAINED, TTH_FIXTURE, fixture_config

from gnn4colliders.cli import main
from gnn4colliders.config import save_config
from gnn4colliders.data import load_events
from gnn4colliders.inference import write_root_scores
from gnn4colliders.train import load_checkpoint


def _config_file(config, tmp_path):
    return str(save_config(config, tmp_path / "experiment.yaml"))


def _history(run):
    return [
        json.loads(line) for line in (run / "history.jsonl").read_text().splitlines()
    ]


def test_gamgam_train_evaluate_predict_export_and_resume(gamgam, tmp_path):
    path, run = _config_file(gamgam, tmp_path), gamgam.path
    assert main(["train", path, "train.max_epochs=3"]) == 0
    history = _history(run)
    assert [row["epoch"] for row in history] == [0, 1, 2]
    for row in history:
        assert row["learning_rate"][0] == pytest.approx(
            1e-4 * 0.99 ** (row["epoch"] + 1)
        )
        assert {"accuracy", "auc", "f1", "precision", "recall"} <= set(
            row["validation"]
        )
    best = load_checkpoint(run / "best.pt")
    best_auc = max(row["validation"]["auc"] for row in history)
    assert best["best_value"] == pytest.approx(best_auc)
    assert history[best["epoch"]]["validation"]["auc"] == pytest.approx(best_auc)
    assert load_checkpoint(run / "last.pt")["epoch"] == 2

    assert main(["evaluate", path]) == 0
    summary = json.loads((run / "test_metrics.json").read_text())
    events = load_events(gamgam.data.cache)
    assert summary["num_samples"] == int((events.fold == 9).sum())
    assert all(math.isfinite(value) for value in summary["metrics"].values())
    assert set(summary["seconds"]) == {"setup", "inference"}

    subset = tmp_path / "subset.json"
    assert main(["evaluate", path, "data.eval_events=10", f"output={subset}"]) == 0
    assert json.loads(subset.read_text())["num_samples"] == 10

    assert main(["predict", path, "split=validation"]) == 0
    predictions = np.load(run / "validation_predictions.npz")
    assert predictions["scores"].shape == (int((events.fold == 8).sum()), 5)
    np.testing.assert_allclose(predictions["scores"].sum(axis=1), 1, rtol=1e-5)
    assert (predictions["fold"] == 8).all()
    first = str(predictions["source_file"][0])
    entries = predictions["entry"][predictions["source_file"] == first]
    tree = uproot.open(first)["mini"].arrays(["eventNumber"], library="np")
    assert (tree["eventNumber"][entries] % 10 == 8).all()

    pytest.importorskip("onnxruntime")
    assert main(["export", path]) == 0
    assert (run / "model.onnx").is_file() and (run / "model.json").is_file()

    assert (
        main(["train", path, "train.max_epochs=4", f"train.resume={run}/last.pt"]) == 0
    )
    assert [row["epoch"] for row in _history(run)] == [0, 1, 2, 3]


def test_root_scores_are_aligned_by_entry(tmp_path):
    entries = np.array([3, 0, 5])
    scores = np.array([[0.1, 0.9], [0.8, 0.2], [0.5, 0.5]])
    output = write_root_scores(
        scores,
        entries,
        input_path=GAMGAM_FIXTURES[0],
        output_path=tmp_path / "scores.root",
        tree_name="mini",
    )
    tree = uproot.open(output)["mini"].arrays(library="np")
    assert tree["selection_pass"].tolist()[:6] == [1, 0, 0, 1, 0, 1]
    assert tree["score_class_1"][3] == pytest.approx(0.9)
    assert np.isnan(tree["score_class_0"][1])
    # Jagged branches are copied unchanged.
    original = uproot.open(GAMGAM_FIXTURES[0])["mini"]["jet_pt"].array()
    copied = uproot.open(output)["mini"]["jet_pt"].array()
    assert copied.tolist() == original.tolist()


def test_fine_tuning_from_the_pretrained_model(tmp_path):
    config = fixture_config(
        "gamgam_finetune",
        GAMGAM_FIXTURES,
        list(range(5)),
        tmp_path,
        "train.max_epochs=1",
        f"model.pretrained={PRETRAINED}/multiclass_pretrained_model_12/model_epoch_71.pt",
    )
    path = _config_file(config, tmp_path)
    assert main(["prepare", path]) == 0
    assert main(["train", path]) == 0
    assert main(["evaluate", path]) == 0
    state = load_checkpoint(config.path / "best.pt")["model_state_dict"]
    assert state["classifier.weight"].shape == (5, 64)


@pytest.mark.parametrize("name", ["gamgam_transformer", "gamgam_deep_sets"])
def test_sequence_models_train_on_the_same_cache(name, gamgam, tmp_path):
    config = fixture_config(
        name,
        GAMGAM_FIXTURES,
        list(range(5)),
        tmp_path,
        f"data.cache={gamgam.data.cache}",
        "train.max_epochs=1",
    )
    path = _config_file(config, tmp_path)
    assert main(["train", path]) == 0
    assert main(["evaluate", path]) == 0
    pytest.importorskip("onnxruntime")
    assert main(["export", path]) == 0
    assert json.loads((config.path / "model.json").read_text())["inputs"] == [
        "tokens",
        "token_mask",
    ]


def test_binary_delphes_task(tmp_path):
    # The same file twice with labels 0 and 1 exercises the binary metrics.
    config = fixture_config(
        "tth_cp_even_odd",
        [TTH_FIXTURE, TTH_FIXTURE],
        [0, 1],
        tmp_path,
        "train.max_epochs=1",
    )
    path = _config_file(config, tmp_path)
    assert main(["prepare", path]) == 0
    assert main(["train", path]) == 0
    assert math.isfinite(_history(config.path)[0]["validation"]["auc"])


def _port() -> int:
    with socket.socket() as sock:
        sock.bind(("127.0.0.1", 0))
        return sock.getsockname()[1]


def _rank(rank: int, port: int, argv: list[str], queue) -> None:
    os.environ.update(
        RANK=str(rank),
        LOCAL_RANK=str(rank),
        WORLD_SIZE="2",
        MASTER_ADDR="127.0.0.1",
        MASTER_PORT=str(port),
    )
    torch.set_num_threads(1)
    sys.argv = ["gnn4colliders", *argv]
    queue.put((rank, main(argv)))


def test_training_runs_on_two_ddp_ranks(gamgam, tmp_path):
    config = _config_file(gamgam, tmp_path)
    # Gloo ranks on the CPU: MPS has no collectives.
    argv = ["train", config, "train.max_epochs=2", "train.device=cpu"]
    context = mp.get_context("spawn")
    queue = context.Queue()
    port = _port()
    processes = [
        context.Process(target=_rank, args=(rank, port, argv, queue))
        for rank in range(2)
    ]
    try:
        for process in processes:
            process.start()
        for process in processes:
            process.join(300)
        assert [process.exitcode for process in processes] == [0, 0]
        assert sorted(queue.get(timeout=5) for _ in processes) == [(0, 0), (1, 0)]
    finally:
        # A regression can deadlock a rank in a collective; never leave it.
        for process in processes:
            if process.is_alive():
                process.kill()
                process.join(5)
    history = _history(gamgam.path)
    events = load_events(gamgam.data.cache)
    assert len(history) == 2
    assert load_checkpoint(gamgam.path / "last.pt")["epoch"] == 1
    # Validation covers every event once; training pads ranks to equal size.
    assert history[0]["validation"]["events"] == int((events.fold == 8).sum())
    train_events = int((events.fold < 8).sum())
    assert history[0]["train"]["events"] == train_events + train_events % 2