fryabcfry's picture
Publish AMP-PD checkpoints, portable loaders and verified metrics (part 3)
04ae9bb verified
Raw History Blame Contribute Delete
8.45 kB
"""End-to-end experiment runner: folds, training, automatic evaluation."""
from __future__ import annotations
import platform
import sys
from pathlib import Path
from typing import Any, Dict, List, Optional
import numpy as np
import pandas as pd
import torch
from .config import save_config
from .data.clinical import DataBundle, build_all_data
from .data.dataset import make_loader
from .data.preprocess import FeaturePreprocessor, TargetScaler, make_preprocessor
from .data.splits import fold_ids, fold_summary, make_folds, save_folds
from .evaluation.evaluator import evaluate_predictions, predict_loader, save_predictions
from .evaluation.postprocess import updrs4_fallback
from .evaluation.metrics import metrics_table, prediction_frame
from .models import build_model
from .training.losses import build_loss
from .training.trainer import Trainer
from .utils import count_parameters, ensure_dir, get_device, now_stamp, save_json, set_seed
def _env_text(device: torch.device) -> str:
lines = [
f"python={sys.version}",
f"platform={platform.platform()}",
f"torch={torch.__version__}",
f"cuda_available={torch.cuda.is_available()}",
f"device={device}",
]
if torch.cuda.is_available():
lines.append(f"cuda_device={torch.cuda.get_device_name(0)}")
lines.append(f"cuda_version={torch.version.cuda}")
return "\n".join(lines) + "\n"
class ExperimentRunner:
def __init__(self, cfg: Dict[str, Any]):
self.cfg = cfg
self.device = get_device(cfg["experiment"].get("device", "auto"))
self.bundle: DataBundle = build_all_data(cfg)
self.folds = make_folds(self.bundle.patients, cfg["cv"])
stamp = now_stamp()
self.run_dir = ensure_dir(
Path(cfg["experiment"].get("output_dir", "runs"))
/ cfg["experiment"].get("name", "ampd_exp")
/ stamp
)
save_config(cfg, self.run_dir / "config.yaml")
save_folds(self.folds, self.run_dir / "folds.csv")
fold_summary(self.folds).to_csv(self.run_dir / "fold_summary.csv", index=False)
(self.run_dir / "environment.txt").write_text(_env_text(self.device), encoding="utf-8")
set_seed(int(cfg["experiment"].get("seed", 42)))
print(f"run_dir: {self.run_dir}")
print(f"device: {self.device} | patients: {len(self.bundle.patients)} | features: {len(self.bundle.feature_names)}")
def _uids_for_fold(self, fold: int) -> tuple[set[str], set[str]]:
scheme = self.cfg["cv"]["scheme"]
folds = self.folds
val = set(folds.loc[folds["fold"] == fold, "uid"])
if scheme == "single":
train = set(folds.loc[folds["fold"] != 0, "uid"])
else:
train = set(folds.loc[(folds["fold"] != fold) & (folds["fold"] >= 0), "uid"])
train |= set(folds.loc[folds["fold"].isin([-1, -2]), "uid"])
return train, val
def _subset(self, fold: int):
train_uids, val_uids = self._uids_for_fold(fold)
tr = self.bundle.samples_all["uid"].isin(train_uids).to_numpy()
va = self.bundle.samples_eval["uid"].isin(val_uids).to_numpy()
Xtr_raw = self.bundle.X_all[tr]
Xva_raw = self.bundle.X_eval[va]
str_df = self.bundle.samples_all.loc[tr].reset_index(drop=True).copy()
sva_df = self.bundle.samples_eval.loc[va].reset_index(drop=True).copy()
if len(str_df) == 0:
raise RuntimeError(f"Fold {fold} has no training samples")
if len(sva_df) == 0:
raise RuntimeError(f"Fold {fold} has no validation samples")
str_df["fold"] = fold
sva_df["fold"] = fold
return Xtr_raw, Xva_raw, str_df, sva_df
def _run_fold(self, fold: int) -> pd.DataFrame:
Xtr_raw, Xva_raw, str_df, sva_df = self._subset(fold)
fold_dir = ensure_dir(self.run_dir / f"fold_{fold}")
pre = make_preprocessor(self.cfg["data"])
pre.seed = int(self.cfg["experiment"].get("seed", 42))
Xtr = pre.fit_transform(Xtr_raw)
Xva = pre.transform(Xva_raw)
pre.save(fold_dir / "preprocessor.pkl")
target_scaler = None
if self.cfg["data"].get("target_transform", "none") == "standardize" and self.cfg["model"]["output_type"] == "regression":
target_scaler = TargetScaler().fit(
np.nan_to_num(str_df[self.bundle.target_cols].to_numpy(dtype=np.float64), nan=0.0),
str_df[self.bundle.mask_cols].to_numpy(dtype=np.float64),
)
target_scaler.save(fold_dir / "target_scaler.pkl")
n_classes = int(self.cfg["model"].get("params", {}).get("n_classes", 87))
output_type = self.cfg["model"]["output_type"]
train_loader = make_loader(
Xtr, str_df, self.bundle.target_cols, self.bundle.mask_cols,
batch_size=int(self.cfg["trainer"]["batch_size"]),
shuffle=True,
num_workers=int(self.cfg["trainer"].get("num_workers", 0)),
n_classes=n_classes if output_type == "distribution" else None,
)
val_loader = make_loader(
Xva, sva_df, self.bundle.target_cols, self.bundle.mask_cols,
batch_size=int(self.cfg["trainer"]["batch_size"]),
shuffle=False,
num_workers=0,
n_classes=n_classes if output_type == "distribution" else None,
)
model = build_model(self.cfg["model"], input_dim=Xtr.shape[1], output_dim=len(self.bundle.target_cols)).to(self.device)
print(f"fold {fold}: train {len(str_df)} | val {len(sva_df)} | model params {count_parameters(model):,}")
loss_fn = build_loss(self.cfg["loss"])
extra = {
"config": self.cfg,
"model_name": self.cfg["model"]["name"],
"input_dim": int(Xtr.shape[1]),
"output_dim": int(len(self.bundle.target_cols)),
"output_type": output_type,
"n_classes": int(n_classes),
"feature_names": list(self.bundle.feature_names),
"target_cols": list(self.bundle.target_cols),
"fold": int(fold),
}
trainer = Trainer(
self.cfg,
model,
train_loader,
val_loader,
self.device,
fold_dir,
loss_fn,
n_classes=n_classes,
target_scaler=target_scaler,
extra=extra,
)
result = trainer.fit()
print(f"fold {fold}: best {result.best_metrics}")
P, Y, M = trainer.predict(val_loader)
pred_df = prediction_frame(Y, P, M, sva_df, self.bundle.target_cols)
fallback = self.cfg["evaluation"].get("postprocess", {}).get("updrs_4", "none")
pred_df = updrs4_fallback(pred_df, str_df, self.bundle.target_cols, self.bundle.mask_cols, fallback)
save_predictions(pred_df, fold_dir / "predictions.csv")
fold_metrics = evaluate_predictions(
pred_df,
metrics=tuple(self.cfg["evaluation"].get("metrics", ["smape_plus_1", "mae"])),
bootstrap=0,
seed=int(self.cfg["experiment"].get("seed", 42)),
)
save_json(fold_metrics, fold_dir / "metrics.json")
return pred_df
def run(self) -> Dict[str, Any]:
scheme = self.cfg["cv"]["scheme"]
fids = [0] if scheme == "single" else fold_ids(self.folds)
all_preds: List[pd.DataFrame] = []
for fold in fids:
all_preds.append(self._run_fold(fold))
oof = pd.concat(all_preds, ignore_index=True)
save_predictions(oof, self.run_dir / "oof.csv")
metrics = evaluate_predictions(
oof,
metrics=tuple(self.cfg["evaluation"].get("metrics", ["smape_plus_1", "mae"])),
bootstrap=int(self.cfg["evaluation"].get("bootstrap", 200)),
seed=int(self.cfg["experiment"].get("seed", 42)),
)
save_json(metrics, self.run_dir / "metrics.json")
table = metrics_table(metrics)
table.to_csv(self.run_dir / "metrics_table.csv", index=False)
with open(self.run_dir / "summary.md", "w", encoding="utf-8") as f:
f.write(f"# {self.cfg['experiment'].get('name')}\n\n")
f.write(f"run_dir: {self.run_dir}\n\n")
f.write(table.to_string(index=False))
f.write("\n")
print(table.to_string(index=False))
return {"run_dir": str(self.run_dir), "metrics": metrics}