"""End-to-end experiment runner: folds, training, automatic evaluation.""" from __future__ import annotations import platform import sys from pathlib import Path from typing import Any, Dict, List, Optional import numpy as np import pandas as pd import torch from .config import save_config from .data.clinical import DataBundle, build_all_data from .data.dataset import make_loader from .data.preprocess import FeaturePreprocessor, TargetScaler, make_preprocessor from .data.splits import fold_ids, fold_summary, make_folds, save_folds from .evaluation.evaluator import evaluate_predictions, predict_loader, save_predictions from .evaluation.postprocess import updrs4_fallback from .evaluation.metrics import metrics_table, prediction_frame from .models import build_model from .training.losses import build_loss from .training.trainer import Trainer from .utils import count_parameters, ensure_dir, get_device, now_stamp, save_json, set_seed def _env_text(device: torch.device) -> str: lines = [ f"python={sys.version}", f"platform={platform.platform()}", f"torch={torch.__version__}", f"cuda_available={torch.cuda.is_available()}", f"device={device}", ] if torch.cuda.is_available(): lines.append(f"cuda_device={torch.cuda.get_device_name(0)}") lines.append(f"cuda_version={torch.version.cuda}") return "\n".join(lines) + "\n" class ExperimentRunner: def __init__(self, cfg: Dict[str, Any]): self.cfg = cfg self.device = get_device(cfg["experiment"].get("device", "auto")) self.bundle: DataBundle = build_all_data(cfg) self.folds = make_folds(self.bundle.patients, cfg["cv"]) stamp = now_stamp() self.run_dir = ensure_dir( Path(cfg["experiment"].get("output_dir", "runs")) / cfg["experiment"].get("name", "ampd_exp") / stamp ) save_config(cfg, self.run_dir / "config.yaml") save_folds(self.folds, self.run_dir / "folds.csv") fold_summary(self.folds).to_csv(self.run_dir / "fold_summary.csv", index=False) (self.run_dir / "environment.txt").write_text(_env_text(self.device), encoding="utf-8") set_seed(int(cfg["experiment"].get("seed", 42))) print(f"run_dir: {self.run_dir}") print(f"device: {self.device} | patients: {len(self.bundle.patients)} | features: {len(self.bundle.feature_names)}") def _uids_for_fold(self, fold: int) -> tuple[set[str], set[str]]: scheme = self.cfg["cv"]["scheme"] folds = self.folds val = set(folds.loc[folds["fold"] == fold, "uid"]) if scheme == "single": train = set(folds.loc[folds["fold"] != 0, "uid"]) else: train = set(folds.loc[(folds["fold"] != fold) & (folds["fold"] >= 0), "uid"]) train |= set(folds.loc[folds["fold"].isin([-1, -2]), "uid"]) return train, val def _subset(self, fold: int): train_uids, val_uids = self._uids_for_fold(fold) tr = self.bundle.samples_all["uid"].isin(train_uids).to_numpy() va = self.bundle.samples_eval["uid"].isin(val_uids).to_numpy() Xtr_raw = self.bundle.X_all[tr] Xva_raw = self.bundle.X_eval[va] str_df = self.bundle.samples_all.loc[tr].reset_index(drop=True).copy() sva_df = self.bundle.samples_eval.loc[va].reset_index(drop=True).copy() if len(str_df) == 0: raise RuntimeError(f"Fold {fold} has no training samples") if len(sva_df) == 0: raise RuntimeError(f"Fold {fold} has no validation samples") str_df["fold"] = fold sva_df["fold"] = fold return Xtr_raw, Xva_raw, str_df, sva_df def _run_fold(self, fold: int) -> pd.DataFrame: Xtr_raw, Xva_raw, str_df, sva_df = self._subset(fold) fold_dir = ensure_dir(self.run_dir / f"fold_{fold}") pre = make_preprocessor(self.cfg["data"]) pre.seed = int(self.cfg["experiment"].get("seed", 42)) Xtr = pre.fit_transform(Xtr_raw) Xva = pre.transform(Xva_raw) pre.save(fold_dir / "preprocessor.pkl") target_scaler = None if self.cfg["data"].get("target_transform", "none") == "standardize" and self.cfg["model"]["output_type"] == "regression": target_scaler = TargetScaler().fit( np.nan_to_num(str_df[self.bundle.target_cols].to_numpy(dtype=np.float64), nan=0.0), str_df[self.bundle.mask_cols].to_numpy(dtype=np.float64), ) target_scaler.save(fold_dir / "target_scaler.pkl") n_classes = int(self.cfg["model"].get("params", {}).get("n_classes", 87)) output_type = self.cfg["model"]["output_type"] train_loader = make_loader( Xtr, str_df, self.bundle.target_cols, self.bundle.mask_cols, batch_size=int(self.cfg["trainer"]["batch_size"]), shuffle=True, num_workers=int(self.cfg["trainer"].get("num_workers", 0)), n_classes=n_classes if output_type == "distribution" else None, ) val_loader = make_loader( Xva, sva_df, self.bundle.target_cols, self.bundle.mask_cols, batch_size=int(self.cfg["trainer"]["batch_size"]), shuffle=False, num_workers=0, n_classes=n_classes if output_type == "distribution" else None, ) model = build_model(self.cfg["model"], input_dim=Xtr.shape[1], output_dim=len(self.bundle.target_cols)).to(self.device) print(f"fold {fold}: train {len(str_df)} | val {len(sva_df)} | model params {count_parameters(model):,}") loss_fn = build_loss(self.cfg["loss"]) extra = { "config": self.cfg, "model_name": self.cfg["model"]["name"], "input_dim": int(Xtr.shape[1]), "output_dim": int(len(self.bundle.target_cols)), "output_type": output_type, "n_classes": int(n_classes), "feature_names": list(self.bundle.feature_names), "target_cols": list(self.bundle.target_cols), "fold": int(fold), } trainer = Trainer( self.cfg, model, train_loader, val_loader, self.device, fold_dir, loss_fn, n_classes=n_classes, target_scaler=target_scaler, extra=extra, ) result = trainer.fit() print(f"fold {fold}: best {result.best_metrics}") P, Y, M = trainer.predict(val_loader) pred_df = prediction_frame(Y, P, M, sva_df, self.bundle.target_cols) fallback = self.cfg["evaluation"].get("postprocess", {}).get("updrs_4", "none") pred_df = updrs4_fallback(pred_df, str_df, self.bundle.target_cols, self.bundle.mask_cols, fallback) save_predictions(pred_df, fold_dir / "predictions.csv") fold_metrics = evaluate_predictions( pred_df, metrics=tuple(self.cfg["evaluation"].get("metrics", ["smape_plus_1", "mae"])), bootstrap=0, seed=int(self.cfg["experiment"].get("seed", 42)), ) save_json(fold_metrics, fold_dir / "metrics.json") return pred_df def run(self) -> Dict[str, Any]: scheme = self.cfg["cv"]["scheme"] fids = [0] if scheme == "single" else fold_ids(self.folds) all_preds: List[pd.DataFrame] = [] for fold in fids: all_preds.append(self._run_fold(fold)) oof = pd.concat(all_preds, ignore_index=True) save_predictions(oof, self.run_dir / "oof.csv") metrics = evaluate_predictions( oof, metrics=tuple(self.cfg["evaluation"].get("metrics", ["smape_plus_1", "mae"])), bootstrap=int(self.cfg["evaluation"].get("bootstrap", 200)), seed=int(self.cfg["experiment"].get("seed", 42)), ) save_json(metrics, self.run_dir / "metrics.json") table = metrics_table(metrics) table.to_csv(self.run_dir / "metrics_table.csv", index=False) with open(self.run_dir / "summary.md", "w", encoding="utf-8") as f: f.write(f"# {self.cfg['experiment'].get('name')}\n\n") f.write(f"run_dir: {self.run_dir}\n\n") f.write(table.to_string(index=False)) f.write("\n") print(table.to_string(index=False)) return {"run_dir": str(self.run_dir), "metrics": metrics}