fryabcfry's picture
Publish AMP-PD checkpoints, portable loaders and verified metrics (part 3)
04ae9bb verified
Raw History Blame Contribute Delete
5.63 kB
"""SMAPE+1 and related metrics."""
from __future__ import annotations
from typing import Any, Dict, Iterable, List, Optional
import numpy as np
import pandas as pd
def smape_plus_1(y_true, y_pred) -> float:
y_true = np.asarray(y_true, dtype=np.float64)
y_pred = np.asarray(y_pred, dtype=np.float64)
denom = (np.abs(y_true) + np.abs(y_pred)) / 2.0 + 1.0
return float(100.0 * np.mean(np.abs(y_true - y_pred) / denom))
def mae(y_true, y_pred) -> float:
return float(np.mean(np.abs(np.asarray(y_true) - np.asarray(y_pred))))
def rmse(y_true, y_pred) -> float:
return float(np.sqrt(np.mean((np.asarray(y_true) - np.asarray(y_pred)) ** 2)))
def _metric_value(name: str, y_true, y_pred) -> float:
if name == "smape_plus_1":
return smape_plus_1(y_true, y_pred)
if name == "mae":
return mae(y_true, y_pred)
if name == "rmse":
return rmse(y_true, y_pred)
raise ValueError(f"Unknown metric: {name}")
def aggregate_metrics(
df: pd.DataFrame,
metrics: Iterable[str] = ("smape_plus_1", "mae"),
y_true_col: str = "y_true",
y_pred_col: str = "y_pred",
per_target: bool = True,
per_horizon: bool = True,
per_cohort: bool = True,
per_fold: bool = True,
bootstrap: int = 0,
seed: int = 42,
) -> Dict[str, Any]:
metrics = list(metrics)
out: Dict[str, Any] = {"n_rows": int(len(df))}
for name in metrics:
out[name] = _metric_value(name, df[y_true_col], df[y_pred_col])
if per_target:
out["per_target"] = {
str(t): {m: _metric_value(m, g[y_true_col], g[y_pred_col]) for m in metrics}
for t, g in df.groupby("target", sort=True)
}
if per_horizon:
out["per_horizon"] = {
str(h): {m: _metric_value(m, g[y_true_col], g[y_pred_col]) for m in metrics}
for h, g in df.groupby("horizon", sort=True)
}
if per_cohort and "cohort" in df.columns:
out["per_cohort"] = {
str(c): {m: _metric_value(m, g[y_true_col], g[y_pred_col]) for m in metrics}
for c, g in df.groupby("cohort", sort=True)
}
if per_fold and "fold" in df.columns:
out["per_fold"] = {
str(f): {m: _metric_value(m, g[y_true_col], g[y_pred_col]) for m in metrics}
for f, g in df.groupby("fold", sort=True)
}
if bootstrap and bootstrap > 0:
out["bootstrap"] = {}
for name in metrics:
lo, hi = bootstrap_ci(df, name, n_boot=int(bootstrap), seed=seed, y_true_col=y_true_col, y_pred_col=y_pred_col)
out["bootstrap"][name] = {"lower": lo, "upper": hi}
return out
def bootstrap_ci(
df: pd.DataFrame,
metric: str,
n_boot: int = 200,
seed: int = 42,
y_true_col: str = "y_true",
y_pred_col: str = "y_pred",
) -> tuple[float, float]:
"""Patient-level bootstrap if uid exists, else row-level."""
rng = np.random.default_rng(seed)
if "uid" in df.columns and df["uid"].nunique() > 1:
uids = df["uid"].drop_duplicates().to_numpy()
values = []
indexed = df.set_index("uid")
for _ in range(n_boot):
sample = rng.choice(uids, size=len(uids), replace=True)
sub = indexed.loc[sample].reset_index()
values.append(_metric_value(metric, sub[y_true_col], sub[y_pred_col]))
else:
n = len(df)
values = []
for _ in range(n_boot):
idx = rng.integers(0, n, size=n)
sub = df.iloc[idx]
values.append(_metric_value(metric, sub[y_true_col], sub[y_pred_col]))
lo, hi = np.percentile(values, [2.5, 97.5])
return float(lo), float(hi)
def metrics_table(metrics: Dict[str, Any]) -> pd.DataFrame:
rows = []
for scope in ["overall"]:
if scope == "overall":
rows.append({"scope": "overall", "name": "all", "smape_plus_1": metrics.get("smape_plus_1"), "mae": metrics.get("mae")})
for key in ["per_target", "per_horizon", "per_cohort", "per_fold"]:
for name, vals in metrics.get(key, {}).items():
rows.append(
{
"scope": key.replace("per_", ""),
"name": str(name),
"smape_plus_1": vals.get("smape_plus_1"),
"mae": vals.get("mae"),
}
)
return pd.DataFrame(rows)
def parse_target_col(col: str) -> tuple[str, int]:
base = col[2:] if col.startswith("y_") else col
target, h = base.rsplit("_", 1)
return target, int(h)
def prediction_frame(
y_true: np.ndarray,
y_pred: np.ndarray,
mask: np.ndarray,
meta: pd.DataFrame,
target_cols: List[str],
) -> pd.DataFrame:
rows = []
for i in range(len(meta)):
m = meta.iloc[i]
for j, col in enumerate(target_cols):
if mask[i, j] <= 0:
continue
target, horizon = parse_target_col(col)
rows.append(
{
"uid": m.get("uid"),
"patient_id": m.get("patient_id"),
"visit_id": m.get("visit_id"),
"visit_month": int(m.get("visit_month")),
"cohort": m.get("cohort"),
"is_supplemental": int(m.get("is_supplemental", 0)),
"fold": int(m.get("fold", -1)) if "fold" in meta.columns else -1,
"target": target,
"horizon": int(horizon),
"y_true": float(y_true[i, j]),
"y_pred": float(y_pred[i, j]),
}
)
return pd.DataFrame(rows)