Download code/ampd/evaluation/metrics.py from fryabcfry/cs4486-amp-pd-checkpoints: direct link, hf CLI and curl.
- Browser
- Download file 5.63 kB
-
https://huggingface.co/fryabcfry/cs4486-amp-pd-checkpoints/resolve/main/code/ampd/evaluation/metrics.py
- Command line
-
hf download hf://fryabcfry/cs4486-amp-pd-checkpoints/code/ampd/evaluation/metrics.py
-
curl -L -o metrics.py https://huggingface.co/fryabcfry/cs4486-amp-pd-checkpoints/resolve/main/code/ampd/evaluation/metrics.py
5.63 kB
| """SMAPE+1 and related metrics.""" | |
| from __future__ import annotations | |
| from typing import Any, Dict, Iterable, List, Optional | |
| import numpy as np | |
| import pandas as pd | |
| def smape_plus_1(y_true, y_pred) -> float: | |
| y_true = np.asarray(y_true, dtype=np.float64) | |
| y_pred = np.asarray(y_pred, dtype=np.float64) | |
| denom = (np.abs(y_true) + np.abs(y_pred)) / 2.0 + 1.0 | |
| return float(100.0 * np.mean(np.abs(y_true - y_pred) / denom)) | |
| def mae(y_true, y_pred) -> float: | |
| return float(np.mean(np.abs(np.asarray(y_true) - np.asarray(y_pred)))) | |
| def rmse(y_true, y_pred) -> float: | |
| return float(np.sqrt(np.mean((np.asarray(y_true) - np.asarray(y_pred)) ** 2))) | |
| def _metric_value(name: str, y_true, y_pred) -> float: | |
| if name == "smape_plus_1": | |
| return smape_plus_1(y_true, y_pred) | |
| if name == "mae": | |
| return mae(y_true, y_pred) | |
| if name == "rmse": | |
| return rmse(y_true, y_pred) | |
| raise ValueError(f"Unknown metric: {name}") | |
| def aggregate_metrics( | |
| df: pd.DataFrame, | |
| metrics: Iterable[str] = ("smape_plus_1", "mae"), | |
| y_true_col: str = "y_true", | |
| y_pred_col: str = "y_pred", | |
| per_target: bool = True, | |
| per_horizon: bool = True, | |
| per_cohort: bool = True, | |
| per_fold: bool = True, | |
| bootstrap: int = 0, | |
| seed: int = 42, | |
| ) -> Dict[str, Any]: | |
| metrics = list(metrics) | |
| out: Dict[str, Any] = {"n_rows": int(len(df))} | |
| for name in metrics: | |
| out[name] = _metric_value(name, df[y_true_col], df[y_pred_col]) | |
| if per_target: | |
| out["per_target"] = { | |
| str(t): {m: _metric_value(m, g[y_true_col], g[y_pred_col]) for m in metrics} | |
| for t, g in df.groupby("target", sort=True) | |
| } | |
| if per_horizon: | |
| out["per_horizon"] = { | |
| str(h): {m: _metric_value(m, g[y_true_col], g[y_pred_col]) for m in metrics} | |
| for h, g in df.groupby("horizon", sort=True) | |
| } | |
| if per_cohort and "cohort" in df.columns: | |
| out["per_cohort"] = { | |
| str(c): {m: _metric_value(m, g[y_true_col], g[y_pred_col]) for m in metrics} | |
| for c, g in df.groupby("cohort", sort=True) | |
| } | |
| if per_fold and "fold" in df.columns: | |
| out["per_fold"] = { | |
| str(f): {m: _metric_value(m, g[y_true_col], g[y_pred_col]) for m in metrics} | |
| for f, g in df.groupby("fold", sort=True) | |
| } | |
| if bootstrap and bootstrap > 0: | |
| out["bootstrap"] = {} | |
| for name in metrics: | |
| lo, hi = bootstrap_ci(df, name, n_boot=int(bootstrap), seed=seed, y_true_col=y_true_col, y_pred_col=y_pred_col) | |
| out["bootstrap"][name] = {"lower": lo, "upper": hi} | |
| return out | |
| def bootstrap_ci( | |
| df: pd.DataFrame, | |
| metric: str, | |
| n_boot: int = 200, | |
| seed: int = 42, | |
| y_true_col: str = "y_true", | |
| y_pred_col: str = "y_pred", | |
| ) -> tuple[float, float]: | |
| """Patient-level bootstrap if uid exists, else row-level.""" | |
| rng = np.random.default_rng(seed) | |
| if "uid" in df.columns and df["uid"].nunique() > 1: | |
| uids = df["uid"].drop_duplicates().to_numpy() | |
| values = [] | |
| indexed = df.set_index("uid") | |
| for _ in range(n_boot): | |
| sample = rng.choice(uids, size=len(uids), replace=True) | |
| sub = indexed.loc[sample].reset_index() | |
| values.append(_metric_value(metric, sub[y_true_col], sub[y_pred_col])) | |
| else: | |
| n = len(df) | |
| values = [] | |
| for _ in range(n_boot): | |
| idx = rng.integers(0, n, size=n) | |
| sub = df.iloc[idx] | |
| values.append(_metric_value(metric, sub[y_true_col], sub[y_pred_col])) | |
| lo, hi = np.percentile(values, [2.5, 97.5]) | |
| return float(lo), float(hi) | |
| def metrics_table(metrics: Dict[str, Any]) -> pd.DataFrame: | |
| rows = [] | |
| for scope in ["overall"]: | |
| if scope == "overall": | |
| rows.append({"scope": "overall", "name": "all", "smape_plus_1": metrics.get("smape_plus_1"), "mae": metrics.get("mae")}) | |
| for key in ["per_target", "per_horizon", "per_cohort", "per_fold"]: | |
| for name, vals in metrics.get(key, {}).items(): | |
| rows.append( | |
| { | |
| "scope": key.replace("per_", ""), | |
| "name": str(name), | |
| "smape_plus_1": vals.get("smape_plus_1"), | |
| "mae": vals.get("mae"), | |
| } | |
| ) | |
| return pd.DataFrame(rows) | |
| def parse_target_col(col: str) -> tuple[str, int]: | |
| base = col[2:] if col.startswith("y_") else col | |
| target, h = base.rsplit("_", 1) | |
| return target, int(h) | |
| def prediction_frame( | |
| y_true: np.ndarray, | |
| y_pred: np.ndarray, | |
| mask: np.ndarray, | |
| meta: pd.DataFrame, | |
| target_cols: List[str], | |
| ) -> pd.DataFrame: | |
| rows = [] | |
| for i in range(len(meta)): | |
| m = meta.iloc[i] | |
| for j, col in enumerate(target_cols): | |
| if mask[i, j] <= 0: | |
| continue | |
| target, horizon = parse_target_col(col) | |
| rows.append( | |
| { | |
| "uid": m.get("uid"), | |
| "patient_id": m.get("patient_id"), | |
| "visit_id": m.get("visit_id"), | |
| "visit_month": int(m.get("visit_month")), | |
| "cohort": m.get("cohort"), | |
| "is_supplemental": int(m.get("is_supplemental", 0)), | |
| "fold": int(m.get("fold", -1)) if "fold" in meta.columns else -1, | |
| "target": target, | |
| "horizon": int(horizon), | |
| "y_true": float(y_true[i, j]), | |
| "y_pred": float(y_pred[i, j]), | |
| } | |
| ) | |
| return pd.DataFrame(rows) | |