File size: 2,015 Bytes
6285343
 
 
2f04f23
6285343
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
"""Summarize an ensemble of training runs as mean +- standard deviation.

Each run directory must contain ``history.jsonl`` and ``test_metrics.json``
(written by ``scripts/train.sh``). The standard deviation
is over runs, as in Table 3 of Ho et al. (2026).
"""

from __future__ import annotations

import argparse
import json
import statistics
from pathlib import Path

METRICS = ("accuracy", "auc", "f1", "precision", "recall")


def _run(path: Path) -> dict[str, float]:
    history = [
        json.loads(line) for line in (path / "history.jsonl").read_text().splitlines()
    ]
    test = json.loads((path / "test_metrics.json").read_text())
    best = max(history, key=lambda row: row["validation"]["auc"])
    seconds = [row["seconds"] for row in history]
    return {
        **{name: 100 * test["metrics"][name] for name in METRICS},
        "epochs": len(history),
        "best_epoch": best["epoch"],
        "seconds_per_epoch": statistics.median(seconds),
        "training_hours": sum(seconds) / 3600,
    }


def _format(values: list[float]) -> str:
    mean = statistics.fmean(values)
    spread = statistics.stdev(values) if len(values) > 1 else 0.0
    return f"{mean:.2f} +- {spread:.2f}"


def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("runs", nargs="+", type=Path)
    args = parser.parse_args()
    runs = {path.name: _run(path) for path in args.runs}
    keys = list(next(iter(runs.values())))
    width = max(len(name) for name in runs)
    print(" ".join([f"{'run':{width}}", *(f"{key:>17}" for key in keys)]))
    for name, values in runs.items():
        print(" ".join([f"{name:{width}}", *(f"{values[key]:17.3f}" for key in keys)]))
    print(
        " ".join(
            [
                f"{'mean':{width}}",
                *(
                    f"{_format([run[key] for run in runs.values()]):>17}"
                    for key in keys
                ),
            ]
        )
    )


if __name__ == "__main__":
    main()