GNN4Colliders / scripts /summarize_ensemble.py
ho22joshua's picture
chore: move test fixtures into tests/, flatten scripts/, drop paper source
2f04f23
Raw History Blame
2.02 kB
"""Summarize an ensemble of training runs as mean +- standard deviation.
Each run directory must contain ``history.jsonl`` and ``test_metrics.json``
(written by ``scripts/train.sh``). The standard deviation
is over runs, as in Table 3 of Ho et al. (2026).
"""
from __future__ import annotations
import argparse
import json
import statistics
from pathlib import Path
METRICS = ("accuracy", "auc", "f1", "precision", "recall")
def _run(path: Path) -> dict[str, float]:
history = [
json.loads(line) for line in (path / "history.jsonl").read_text().splitlines()
]
test = json.loads((path / "test_metrics.json").read_text())
best = max(history, key=lambda row: row["validation"]["auc"])
seconds = [row["seconds"] for row in history]
return {
**{name: 100 * test["metrics"][name] for name in METRICS},
"epochs": len(history),
"best_epoch": best["epoch"],
"seconds_per_epoch": statistics.median(seconds),
"training_hours": sum(seconds) / 3600,
}
def _format(values: list[float]) -> str:
mean = statistics.fmean(values)
spread = statistics.stdev(values) if len(values) > 1 else 0.0
return f"{mean:.2f} +- {spread:.2f}"
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("runs", nargs="+", type=Path)
args = parser.parse_args()
runs = {path.name: _run(path) for path in args.runs}
keys = list(next(iter(runs.values())))
width = max(len(name) for name in runs)
print(" ".join([f"{'run':{width}}", *(f"{key:>17}" for key in keys)]))
for name, values in runs.items():
print(" ".join([f"{name:{width}}", *(f"{values[key]:17.3f}" for key in keys)]))
print(
" ".join(
[
f"{'mean':{width}}",
*(
f"{_format([run[key] for run in runs.values()]):>17}"
for key in keys
),
]
)
)
if __name__ == "__main__":
main()