fryabcfry's picture
Publish AMP-PD checkpoints, portable loaders and verified metrics (part 3)
04ae9bb verified
Raw History Blame Contribute Delete
1.57 kB
"""Collect run metrics into a comparison table."""
from __future__ import annotations
from pathlib import Path
from typing import Any, Dict, List
import pandas as pd
from .utils import load_json
def collect_runs(root: str | Path = "runs") -> pd.DataFrame:
root = Path(root)
rows: List[Dict[str, Any]] = []
for path in sorted(root.rglob("metrics.json")):
if any(part.startswith("fold_") for part in path.parts):
continue
try:
m = load_json(path)
except Exception:
continue
if "smape_plus_1" not in m:
continue
run_dir = path.parent
rows.append(
{
"run_dir": str(run_dir),
"name": run_dir.parent.name if run_dir.parent.name != "baselines" else run_dir.parent.parent.name + "/" + run_dir.parent.name,
"smape_plus_1": float(m.get("smape_plus_1")),
"mae": float(m.get("mae", float("nan"))),
"n_rows": int(m.get("n_rows", 0)),
}
)
if not rows:
return pd.DataFrame(columns=["run_dir", "name", "smape_plus_1", "mae", "n_rows"])
df = pd.DataFrame(rows).sort_values("smape_plus_1").reset_index(drop=True)
return df
def write_comparison(root: str | Path = "runs", out: str | Path = "runs/comparison.csv") -> pd.DataFrame:
df = collect_runs(root)
out = Path(out)
out.parent.mkdir(parents=True, exist_ok=True)
df.to_csv(out, index=False)
print(df.to_string(index=False))
print(f"wrote {out}")
return df