File size: 2,015 Bytes
6285343 2f04f23 6285343 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 | """Summarize an ensemble of training runs as mean +- standard deviation.
Each run directory must contain ``history.jsonl`` and ``test_metrics.json``
(written by ``scripts/train.sh``). The standard deviation
is over runs, as in Table 3 of Ho et al. (2026).
"""
from __future__ import annotations
import argparse
import json
import statistics
from pathlib import Path
METRICS = ("accuracy", "auc", "f1", "precision", "recall")
def _run(path: Path) -> dict[str, float]:
history = [
json.loads(line) for line in (path / "history.jsonl").read_text().splitlines()
]
test = json.loads((path / "test_metrics.json").read_text())
best = max(history, key=lambda row: row["validation"]["auc"])
seconds = [row["seconds"] for row in history]
return {
**{name: 100 * test["metrics"][name] for name in METRICS},
"epochs": len(history),
"best_epoch": best["epoch"],
"seconds_per_epoch": statistics.median(seconds),
"training_hours": sum(seconds) / 3600,
}
def _format(values: list[float]) -> str:
mean = statistics.fmean(values)
spread = statistics.stdev(values) if len(values) > 1 else 0.0
return f"{mean:.2f} +- {spread:.2f}"
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("runs", nargs="+", type=Path)
args = parser.parse_args()
runs = {path.name: _run(path) for path in args.runs}
keys = list(next(iter(runs.values())))
width = max(len(name) for name in runs)
print(" ".join([f"{'run':{width}}", *(f"{key:>17}" for key in keys)]))
for name, values in runs.items():
print(" ".join([f"{name:{width}}", *(f"{values[key]:17.3f}" for key in keys)]))
print(
" ".join(
[
f"{'mean':{width}}",
*(
f"{_format([run[key] for run in runs.values()]):>17}"
for key in keys
),
]
)
)
if __name__ == "__main__":
main()
|