File size: 1,566 Bytes
04ae9bb
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
"""Collect run metrics into a comparison table."""

from __future__ import annotations

from pathlib import Path
from typing import Any, Dict, List

import pandas as pd

from .utils import load_json


def collect_runs(root: str | Path = "runs") -> pd.DataFrame:
    root = Path(root)
    rows: List[Dict[str, Any]] = []
    for path in sorted(root.rglob("metrics.json")):
        if any(part.startswith("fold_") for part in path.parts):
            continue
        try:
            m = load_json(path)
        except Exception:
            continue
        if "smape_plus_1" not in m:
            continue
        run_dir = path.parent
        rows.append(
            {
                "run_dir": str(run_dir),
                "name": run_dir.parent.name if run_dir.parent.name != "baselines" else run_dir.parent.parent.name + "/" + run_dir.parent.name,
                "smape_plus_1": float(m.get("smape_plus_1")),
                "mae": float(m.get("mae", float("nan"))),
                "n_rows": int(m.get("n_rows", 0)),
            }
        )
    if not rows:
        return pd.DataFrame(columns=["run_dir", "name", "smape_plus_1", "mae", "n_rows"])
    df = pd.DataFrame(rows).sort_values("smape_plus_1").reset_index(drop=True)
    return df


def write_comparison(root: str | Path = "runs", out: str | Path = "runs/comparison.csv") -> pd.DataFrame:
    df = collect_runs(root)
    out = Path(out)
    out.parent.mkdir(parents=True, exist_ok=True)
    df.to_csv(out, index=False)
    print(df.to_string(index=False))
    print(f"wrote {out}")
    return df