fryabcfry's picture
Publish AMP-PD checkpoints, portable loaders and verified metrics (part 3)
04ae9bb verified
Raw History Blame Contribute Delete
7.3 kB
"""Non-neural baselines: median and Ridge, using the same folds/metric."""
from __future__ import annotations
from pathlib import Path
from typing import Any, Dict, Iterable, List, Tuple
import numpy as np
import pandas as pd
from sklearn.linear_model import Ridge
from .data.clinical import DataBundle, build_all_data
from .data.preprocess import FeaturePreprocessor
from .data.splits import fold_ids, make_folds, save_folds
from .evaluation.metrics import aggregate_metrics, metrics_table, parse_target_col
from .evaluation.postprocess import updrs4_fallback
from .utils import ensure_dir, now_stamp, save_json, set_seed
def _fold_uids(folds: pd.DataFrame, fold: int, scheme: str) -> Tuple[set, set]:
val = set(folds.loc[folds["fold"] == fold, "uid"])
if scheme == "single":
train = set(folds.loc[folds["fold"] != 0, "uid"])
else:
train = set(folds.loc[(folds["fold"] != fold) & (folds["fold"] >= 0), "uid"])
train |= set(folds.loc[folds["fold"] == -2, "uid"])
return train, val
def _rows_from_predictions(
val_df: pd.DataFrame,
preds: np.ndarray,
target_cols: List[str],
mask_cols: List[str],
fold: int,
) -> pd.DataFrame:
rows = []
for i in range(len(val_df)):
row = val_df.iloc[i]
for j, col in enumerate(target_cols):
if row[mask_cols[j]] <= 0:
continue
target, horizon = parse_target_col(col)
rows.append(
{
"uid": row["uid"],
"patient_id": row["patient_id"],
"visit_id": row["visit_id"],
"visit_month": int(row["visit_month"]),
"cohort": row["cohort"],
"is_supplemental": int(row["is_supplemental"]),
"fold": int(fold),
"target": target,
"horizon": int(horizon),
"y_true": float(row[col]),
"y_pred": float(preds[i, j]),
}
)
return pd.DataFrame(rows)
def _summarize(oof: pd.DataFrame, out_dir: Path, cfg: Dict[str, Any]) -> Dict[str, Any]:
metrics = aggregate_metrics(
oof,
metrics=tuple(cfg["evaluation"].get("metrics", ["smape_plus_1", "mae"])),
per_target=True,
per_horizon=True,
per_cohort=True,
per_fold=True,
bootstrap=int(cfg["evaluation"].get("bootstrap", 200)),
seed=int(cfg["experiment"].get("seed", 42)),
)
save_json(metrics, out_dir / "metrics.json")
oof.to_csv(out_dir / "oof.csv", index=False)
table = metrics_table(metrics)
table.to_csv(out_dir / "metrics_table.csv", index=False)
print(table.to_string(index=False))
return metrics
def run_median_baseline(
bundle: DataBundle,
folds: pd.DataFrame,
cfg: Dict[str, Any],
out_dir: Path,
) -> Dict[str, Any]:
out_dir = ensure_dir(out_dir)
oof = []
for fold in fold_ids(folds):
train_uids, val_uids = _fold_uids(folds, fold, cfg["cv"]["scheme"])
train = bundle.samples_all[bundle.samples_all["uid"].isin(train_uids)]
val = bundle.samples_eval[bundle.samples_eval["uid"].isin(val_uids)].reset_index(drop=True)
preds = np.zeros((len(val), len(bundle.target_cols)), dtype=np.float64)
for j, (tcol, mcol) in enumerate(zip(bundle.target_cols, bundle.mask_cols)):
vals = train.loc[train[mcol] == 1, tcol].dropna()
med = float(vals.median()) if len(vals) else 0.0
preds[:, j] = med
oof.append(_rows_from_predictions(val, preds, bundle.target_cols, bundle.mask_cols, fold))
oof_df = pd.concat(oof, ignore_index=True)
return _summarize(oof_df, out_dir, cfg)
def _feature_indices(bundle: DataBundle, feature_set: str) -> List[int]:
if feature_set == "all":
return list(range(len(bundle.feature_names)))
if feature_set == "process":
return [i for i, n in enumerate(bundle.feature_names) if not n.startswith(("prot_", "pep_"))]
if feature_set == "protein":
return [i for i, n in enumerate(bundle.feature_names) if n.startswith("prot_")]
if feature_set == "peptide":
return [i for i, n in enumerate(bundle.feature_names) if n.startswith("pep_")]
raise ValueError(f"Unknown feature_set: {feature_set}")
def run_ridge_baseline(
bundle: DataBundle,
folds: pd.DataFrame,
cfg: Dict[str, Any],
out_dir: Path,
feature_set: str = "process",
alpha: float = 100.0,
) -> Dict[str, Any]:
out_dir = ensure_dir(out_dir)
idx = _feature_indices(bundle, feature_set)
if not idx:
raise ValueError(f"No features for feature_set={feature_set}")
oof = []
for fold in fold_ids(folds):
train_uids, val_uids = _fold_uids(folds, fold, cfg["cv"]["scheme"])
tr_mask = bundle.samples_all["uid"].isin(train_uids).to_numpy()
va_mask = bundle.samples_eval["uid"].isin(val_uids).to_numpy()
tr = bundle.samples_all.loc[tr_mask].reset_index(drop=True)
va = bundle.samples_eval.loc[va_mask].reset_index(drop=True)
Xtr_raw = bundle.X_all[tr_mask][:, idx]
Xva_raw = bundle.X_eval[va_mask][:, idx]
pre = FeaturePreprocessor(impute=cfg["data"].get("impute", "median"), standardize=True, pca=0)
Xtr = pre.fit_transform(Xtr_raw)
Xva = pre.transform(Xva_raw)
preds = np.zeros((len(va), len(bundle.target_cols)), dtype=np.float64)
for j, (tcol, mcol) in enumerate(zip(bundle.target_cols, bundle.mask_cols)):
y = tr[tcol].to_numpy(dtype=np.float64)
m = tr[mcol].to_numpy(dtype=np.float64)
valid = (m > 0) & np.isfinite(y)
if valid.sum() < 2:
preds[:, j] = float(np.nanmedian(y[valid])) if valid.sum() else 0.0
continue
model = Ridge(alpha=float(alpha))
model.fit(Xtr[valid], y[valid])
preds[:, j] = model.predict(Xva)
rows = _rows_from_predictions(va, preds, bundle.target_cols, bundle.mask_cols, fold)
fallback = cfg["evaluation"].get("postprocess", {}).get("updrs_4", "none")
rows = updrs4_fallback(rows, tr, bundle.target_cols, bundle.mask_cols, fallback)
oof.append(rows)
oof_df = pd.concat(oof, ignore_index=True)
return _summarize(oof_df, out_dir, cfg)
def run_baselines(cfg: Dict[str, Any], kind: str = "median") -> Dict[str, Any]:
set_seed(int(cfg["experiment"].get("seed", 42)))
bundle = build_all_data(cfg)
folds = make_folds(bundle.patients, cfg["cv"])
root = Path(cfg["experiment"].get("output_dir", "runs")) / "baselines" / kind / now_stamp()
root = ensure_dir(root)
save_folds(folds, root / "folds.csv")
if kind == "median":
metrics = run_median_baseline(bundle, folds, cfg, root)
elif kind in {"process", "protein", "peptide", "all"}:
alpha = float(cfg["evaluation"].get("ridge_alpha", 100.0))
metrics = run_ridge_baseline(bundle, folds, cfg, root, feature_set=kind, alpha=alpha)
else:
raise ValueError(f"Unknown baseline kind: {kind}")
save_json({"kind": kind, "metrics": metrics}, root / "baseline_summary.json")
print(f"baseline artifacts: {root}")
return {"run_dir": str(root), "metrics": metrics}