File size: 8,446 Bytes
04ae9bb | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 | """End-to-end experiment runner: folds, training, automatic evaluation."""
from __future__ import annotations
import platform
import sys
from pathlib import Path
from typing import Any, Dict, List, Optional
import numpy as np
import pandas as pd
import torch
from .config import save_config
from .data.clinical import DataBundle, build_all_data
from .data.dataset import make_loader
from .data.preprocess import FeaturePreprocessor, TargetScaler, make_preprocessor
from .data.splits import fold_ids, fold_summary, make_folds, save_folds
from .evaluation.evaluator import evaluate_predictions, predict_loader, save_predictions
from .evaluation.postprocess import updrs4_fallback
from .evaluation.metrics import metrics_table, prediction_frame
from .models import build_model
from .training.losses import build_loss
from .training.trainer import Trainer
from .utils import count_parameters, ensure_dir, get_device, now_stamp, save_json, set_seed
def _env_text(device: torch.device) -> str:
lines = [
f"python={sys.version}",
f"platform={platform.platform()}",
f"torch={torch.__version__}",
f"cuda_available={torch.cuda.is_available()}",
f"device={device}",
]
if torch.cuda.is_available():
lines.append(f"cuda_device={torch.cuda.get_device_name(0)}")
lines.append(f"cuda_version={torch.version.cuda}")
return "\n".join(lines) + "\n"
class ExperimentRunner:
def __init__(self, cfg: Dict[str, Any]):
self.cfg = cfg
self.device = get_device(cfg["experiment"].get("device", "auto"))
self.bundle: DataBundle = build_all_data(cfg)
self.folds = make_folds(self.bundle.patients, cfg["cv"])
stamp = now_stamp()
self.run_dir = ensure_dir(
Path(cfg["experiment"].get("output_dir", "runs"))
/ cfg["experiment"].get("name", "ampd_exp")
/ stamp
)
save_config(cfg, self.run_dir / "config.yaml")
save_folds(self.folds, self.run_dir / "folds.csv")
fold_summary(self.folds).to_csv(self.run_dir / "fold_summary.csv", index=False)
(self.run_dir / "environment.txt").write_text(_env_text(self.device), encoding="utf-8")
set_seed(int(cfg["experiment"].get("seed", 42)))
print(f"run_dir: {self.run_dir}")
print(f"device: {self.device} | patients: {len(self.bundle.patients)} | features: {len(self.bundle.feature_names)}")
def _uids_for_fold(self, fold: int) -> tuple[set[str], set[str]]:
scheme = self.cfg["cv"]["scheme"]
folds = self.folds
val = set(folds.loc[folds["fold"] == fold, "uid"])
if scheme == "single":
train = set(folds.loc[folds["fold"] != 0, "uid"])
else:
train = set(folds.loc[(folds["fold"] != fold) & (folds["fold"] >= 0), "uid"])
train |= set(folds.loc[folds["fold"].isin([-1, -2]), "uid"])
return train, val
def _subset(self, fold: int):
train_uids, val_uids = self._uids_for_fold(fold)
tr = self.bundle.samples_all["uid"].isin(train_uids).to_numpy()
va = self.bundle.samples_eval["uid"].isin(val_uids).to_numpy()
Xtr_raw = self.bundle.X_all[tr]
Xva_raw = self.bundle.X_eval[va]
str_df = self.bundle.samples_all.loc[tr].reset_index(drop=True).copy()
sva_df = self.bundle.samples_eval.loc[va].reset_index(drop=True).copy()
if len(str_df) == 0:
raise RuntimeError(f"Fold {fold} has no training samples")
if len(sva_df) == 0:
raise RuntimeError(f"Fold {fold} has no validation samples")
str_df["fold"] = fold
sva_df["fold"] = fold
return Xtr_raw, Xva_raw, str_df, sva_df
def _run_fold(self, fold: int) -> pd.DataFrame:
Xtr_raw, Xva_raw, str_df, sva_df = self._subset(fold)
fold_dir = ensure_dir(self.run_dir / f"fold_{fold}")
pre = make_preprocessor(self.cfg["data"])
pre.seed = int(self.cfg["experiment"].get("seed", 42))
Xtr = pre.fit_transform(Xtr_raw)
Xva = pre.transform(Xva_raw)
pre.save(fold_dir / "preprocessor.pkl")
target_scaler = None
if self.cfg["data"].get("target_transform", "none") == "standardize" and self.cfg["model"]["output_type"] == "regression":
target_scaler = TargetScaler().fit(
np.nan_to_num(str_df[self.bundle.target_cols].to_numpy(dtype=np.float64), nan=0.0),
str_df[self.bundle.mask_cols].to_numpy(dtype=np.float64),
)
target_scaler.save(fold_dir / "target_scaler.pkl")
n_classes = int(self.cfg["model"].get("params", {}).get("n_classes", 87))
output_type = self.cfg["model"]["output_type"]
train_loader = make_loader(
Xtr, str_df, self.bundle.target_cols, self.bundle.mask_cols,
batch_size=int(self.cfg["trainer"]["batch_size"]),
shuffle=True,
num_workers=int(self.cfg["trainer"].get("num_workers", 0)),
n_classes=n_classes if output_type == "distribution" else None,
)
val_loader = make_loader(
Xva, sva_df, self.bundle.target_cols, self.bundle.mask_cols,
batch_size=int(self.cfg["trainer"]["batch_size"]),
shuffle=False,
num_workers=0,
n_classes=n_classes if output_type == "distribution" else None,
)
model = build_model(self.cfg["model"], input_dim=Xtr.shape[1], output_dim=len(self.bundle.target_cols)).to(self.device)
print(f"fold {fold}: train {len(str_df)} | val {len(sva_df)} | model params {count_parameters(model):,}")
loss_fn = build_loss(self.cfg["loss"])
extra = {
"config": self.cfg,
"model_name": self.cfg["model"]["name"],
"input_dim": int(Xtr.shape[1]),
"output_dim": int(len(self.bundle.target_cols)),
"output_type": output_type,
"n_classes": int(n_classes),
"feature_names": list(self.bundle.feature_names),
"target_cols": list(self.bundle.target_cols),
"fold": int(fold),
}
trainer = Trainer(
self.cfg,
model,
train_loader,
val_loader,
self.device,
fold_dir,
loss_fn,
n_classes=n_classes,
target_scaler=target_scaler,
extra=extra,
)
result = trainer.fit()
print(f"fold {fold}: best {result.best_metrics}")
P, Y, M = trainer.predict(val_loader)
pred_df = prediction_frame(Y, P, M, sva_df, self.bundle.target_cols)
fallback = self.cfg["evaluation"].get("postprocess", {}).get("updrs_4", "none")
pred_df = updrs4_fallback(pred_df, str_df, self.bundle.target_cols, self.bundle.mask_cols, fallback)
save_predictions(pred_df, fold_dir / "predictions.csv")
fold_metrics = evaluate_predictions(
pred_df,
metrics=tuple(self.cfg["evaluation"].get("metrics", ["smape_plus_1", "mae"])),
bootstrap=0,
seed=int(self.cfg["experiment"].get("seed", 42)),
)
save_json(fold_metrics, fold_dir / "metrics.json")
return pred_df
def run(self) -> Dict[str, Any]:
scheme = self.cfg["cv"]["scheme"]
fids = [0] if scheme == "single" else fold_ids(self.folds)
all_preds: List[pd.DataFrame] = []
for fold in fids:
all_preds.append(self._run_fold(fold))
oof = pd.concat(all_preds, ignore_index=True)
save_predictions(oof, self.run_dir / "oof.csv")
metrics = evaluate_predictions(
oof,
metrics=tuple(self.cfg["evaluation"].get("metrics", ["smape_plus_1", "mae"])),
bootstrap=int(self.cfg["evaluation"].get("bootstrap", 200)),
seed=int(self.cfg["experiment"].get("seed", 42)),
)
save_json(metrics, self.run_dir / "metrics.json")
table = metrics_table(metrics)
table.to_csv(self.run_dir / "metrics_table.csv", index=False)
with open(self.run_dir / "summary.md", "w", encoding="utf-8") as f:
f.write(f"# {self.cfg['experiment'].get('name')}\n\n")
f.write(f"run_dir: {self.run_dir}\n\n")
f.write(table.to_string(index=False))
f.write("\n")
print(table.to_string(index=False))
return {"run_dir": str(self.run_dir), "metrics": metrics}
|