"""Summarize an ensemble of training runs as mean +- standard deviation. Each run directory must contain ``history.jsonl`` and ``test_metrics.json`` (written by ``scripts/train.sh``). The standard deviation is over runs, as in Table 3 of Ho et al. (2026). """ from __future__ import annotations import argparse import json import statistics from pathlib import Path METRICS = ("accuracy", "auc", "f1", "precision", "recall") def _run(path: Path) -> dict[str, float]: history = [ json.loads(line) for line in (path / "history.jsonl").read_text().splitlines() ] test = json.loads((path / "test_metrics.json").read_text()) best = max(history, key=lambda row: row["validation"]["auc"]) seconds = [row["seconds"] for row in history] return { **{name: 100 * test["metrics"][name] for name in METRICS}, "epochs": len(history), "best_epoch": best["epoch"], "seconds_per_epoch": statistics.median(seconds), "training_hours": sum(seconds) / 3600, } def _format(values: list[float]) -> str: mean = statistics.fmean(values) spread = statistics.stdev(values) if len(values) > 1 else 0.0 return f"{mean:.2f} +- {spread:.2f}" def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("runs", nargs="+", type=Path) args = parser.parse_args() runs = {path.name: _run(path) for path in args.runs} keys = list(next(iter(runs.values()))) width = max(len(name) for name in runs) print(" ".join([f"{'run':{width}}", *(f"{key:>17}" for key in keys)])) for name, values in runs.items(): print(" ".join([f"{name:{width}}", *(f"{values[key]:17.3f}" for key in keys)])) print( " ".join( [ f"{'mean':{width}}", *( f"{_format([run[key] for run in runs.values()]):>17}" for key in keys ), ] ) ) if __name__ == "__main__": main()