Download code/ampd/runner.py from fryabcfry/cs4486-amp-pd-checkpoints: direct link, hf CLI and curl.
- Browser
- Download file 8.45 kB
-
https://huggingface.co/fryabcfry/cs4486-amp-pd-checkpoints/resolve/main/code/ampd/runner.py
- Command line
-
hf download hf://fryabcfry/cs4486-amp-pd-checkpoints/code/ampd/runner.py
-
curl -L -o runner.py https://huggingface.co/fryabcfry/cs4486-amp-pd-checkpoints/resolve/main/code/ampd/runner.py
8.45 kB
| """End-to-end experiment runner: folds, training, automatic evaluation.""" | |
| from __future__ import annotations | |
| import platform | |
| import sys | |
| from pathlib import Path | |
| from typing import Any, Dict, List, Optional | |
| import numpy as np | |
| import pandas as pd | |
| import torch | |
| from .config import save_config | |
| from .data.clinical import DataBundle, build_all_data | |
| from .data.dataset import make_loader | |
| from .data.preprocess import FeaturePreprocessor, TargetScaler, make_preprocessor | |
| from .data.splits import fold_ids, fold_summary, make_folds, save_folds | |
| from .evaluation.evaluator import evaluate_predictions, predict_loader, save_predictions | |
| from .evaluation.postprocess import updrs4_fallback | |
| from .evaluation.metrics import metrics_table, prediction_frame | |
| from .models import build_model | |
| from .training.losses import build_loss | |
| from .training.trainer import Trainer | |
| from .utils import count_parameters, ensure_dir, get_device, now_stamp, save_json, set_seed | |
| def _env_text(device: torch.device) -> str: | |
| lines = [ | |
| f"python={sys.version}", | |
| f"platform={platform.platform()}", | |
| f"torch={torch.__version__}", | |
| f"cuda_available={torch.cuda.is_available()}", | |
| f"device={device}", | |
| ] | |
| if torch.cuda.is_available(): | |
| lines.append(f"cuda_device={torch.cuda.get_device_name(0)}") | |
| lines.append(f"cuda_version={torch.version.cuda}") | |
| return "\n".join(lines) + "\n" | |
| class ExperimentRunner: | |
| def __init__(self, cfg: Dict[str, Any]): | |
| self.cfg = cfg | |
| self.device = get_device(cfg["experiment"].get("device", "auto")) | |
| self.bundle: DataBundle = build_all_data(cfg) | |
| self.folds = make_folds(self.bundle.patients, cfg["cv"]) | |
| stamp = now_stamp() | |
| self.run_dir = ensure_dir( | |
| Path(cfg["experiment"].get("output_dir", "runs")) | |
| / cfg["experiment"].get("name", "ampd_exp") | |
| / stamp | |
| ) | |
| save_config(cfg, self.run_dir / "config.yaml") | |
| save_folds(self.folds, self.run_dir / "folds.csv") | |
| fold_summary(self.folds).to_csv(self.run_dir / "fold_summary.csv", index=False) | |
| (self.run_dir / "environment.txt").write_text(_env_text(self.device), encoding="utf-8") | |
| set_seed(int(cfg["experiment"].get("seed", 42))) | |
| print(f"run_dir: {self.run_dir}") | |
| print(f"device: {self.device} | patients: {len(self.bundle.patients)} | features: {len(self.bundle.feature_names)}") | |
| def _uids_for_fold(self, fold: int) -> tuple[set[str], set[str]]: | |
| scheme = self.cfg["cv"]["scheme"] | |
| folds = self.folds | |
| val = set(folds.loc[folds["fold"] == fold, "uid"]) | |
| if scheme == "single": | |
| train = set(folds.loc[folds["fold"] != 0, "uid"]) | |
| else: | |
| train = set(folds.loc[(folds["fold"] != fold) & (folds["fold"] >= 0), "uid"]) | |
| train |= set(folds.loc[folds["fold"].isin([-1, -2]), "uid"]) | |
| return train, val | |
| def _subset(self, fold: int): | |
| train_uids, val_uids = self._uids_for_fold(fold) | |
| tr = self.bundle.samples_all["uid"].isin(train_uids).to_numpy() | |
| va = self.bundle.samples_eval["uid"].isin(val_uids).to_numpy() | |
| Xtr_raw = self.bundle.X_all[tr] | |
| Xva_raw = self.bundle.X_eval[va] | |
| str_df = self.bundle.samples_all.loc[tr].reset_index(drop=True).copy() | |
| sva_df = self.bundle.samples_eval.loc[va].reset_index(drop=True).copy() | |
| if len(str_df) == 0: | |
| raise RuntimeError(f"Fold {fold} has no training samples") | |
| if len(sva_df) == 0: | |
| raise RuntimeError(f"Fold {fold} has no validation samples") | |
| str_df["fold"] = fold | |
| sva_df["fold"] = fold | |
| return Xtr_raw, Xva_raw, str_df, sva_df | |
| def _run_fold(self, fold: int) -> pd.DataFrame: | |
| Xtr_raw, Xva_raw, str_df, sva_df = self._subset(fold) | |
| fold_dir = ensure_dir(self.run_dir / f"fold_{fold}") | |
| pre = make_preprocessor(self.cfg["data"]) | |
| pre.seed = int(self.cfg["experiment"].get("seed", 42)) | |
| Xtr = pre.fit_transform(Xtr_raw) | |
| Xva = pre.transform(Xva_raw) | |
| pre.save(fold_dir / "preprocessor.pkl") | |
| target_scaler = None | |
| if self.cfg["data"].get("target_transform", "none") == "standardize" and self.cfg["model"]["output_type"] == "regression": | |
| target_scaler = TargetScaler().fit( | |
| np.nan_to_num(str_df[self.bundle.target_cols].to_numpy(dtype=np.float64), nan=0.0), | |
| str_df[self.bundle.mask_cols].to_numpy(dtype=np.float64), | |
| ) | |
| target_scaler.save(fold_dir / "target_scaler.pkl") | |
| n_classes = int(self.cfg["model"].get("params", {}).get("n_classes", 87)) | |
| output_type = self.cfg["model"]["output_type"] | |
| train_loader = make_loader( | |
| Xtr, str_df, self.bundle.target_cols, self.bundle.mask_cols, | |
| batch_size=int(self.cfg["trainer"]["batch_size"]), | |
| shuffle=True, | |
| num_workers=int(self.cfg["trainer"].get("num_workers", 0)), | |
| n_classes=n_classes if output_type == "distribution" else None, | |
| ) | |
| val_loader = make_loader( | |
| Xva, sva_df, self.bundle.target_cols, self.bundle.mask_cols, | |
| batch_size=int(self.cfg["trainer"]["batch_size"]), | |
| shuffle=False, | |
| num_workers=0, | |
| n_classes=n_classes if output_type == "distribution" else None, | |
| ) | |
| model = build_model(self.cfg["model"], input_dim=Xtr.shape[1], output_dim=len(self.bundle.target_cols)).to(self.device) | |
| print(f"fold {fold}: train {len(str_df)} | val {len(sva_df)} | model params {count_parameters(model):,}") | |
| loss_fn = build_loss(self.cfg["loss"]) | |
| extra = { | |
| "config": self.cfg, | |
| "model_name": self.cfg["model"]["name"], | |
| "input_dim": int(Xtr.shape[1]), | |
| "output_dim": int(len(self.bundle.target_cols)), | |
| "output_type": output_type, | |
| "n_classes": int(n_classes), | |
| "feature_names": list(self.bundle.feature_names), | |
| "target_cols": list(self.bundle.target_cols), | |
| "fold": int(fold), | |
| } | |
| trainer = Trainer( | |
| self.cfg, | |
| model, | |
| train_loader, | |
| val_loader, | |
| self.device, | |
| fold_dir, | |
| loss_fn, | |
| n_classes=n_classes, | |
| target_scaler=target_scaler, | |
| extra=extra, | |
| ) | |
| result = trainer.fit() | |
| print(f"fold {fold}: best {result.best_metrics}") | |
| P, Y, M = trainer.predict(val_loader) | |
| pred_df = prediction_frame(Y, P, M, sva_df, self.bundle.target_cols) | |
| fallback = self.cfg["evaluation"].get("postprocess", {}).get("updrs_4", "none") | |
| pred_df = updrs4_fallback(pred_df, str_df, self.bundle.target_cols, self.bundle.mask_cols, fallback) | |
| save_predictions(pred_df, fold_dir / "predictions.csv") | |
| fold_metrics = evaluate_predictions( | |
| pred_df, | |
| metrics=tuple(self.cfg["evaluation"].get("metrics", ["smape_plus_1", "mae"])), | |
| bootstrap=0, | |
| seed=int(self.cfg["experiment"].get("seed", 42)), | |
| ) | |
| save_json(fold_metrics, fold_dir / "metrics.json") | |
| return pred_df | |
| def run(self) -> Dict[str, Any]: | |
| scheme = self.cfg["cv"]["scheme"] | |
| fids = [0] if scheme == "single" else fold_ids(self.folds) | |
| all_preds: List[pd.DataFrame] = [] | |
| for fold in fids: | |
| all_preds.append(self._run_fold(fold)) | |
| oof = pd.concat(all_preds, ignore_index=True) | |
| save_predictions(oof, self.run_dir / "oof.csv") | |
| metrics = evaluate_predictions( | |
| oof, | |
| metrics=tuple(self.cfg["evaluation"].get("metrics", ["smape_plus_1", "mae"])), | |
| bootstrap=int(self.cfg["evaluation"].get("bootstrap", 200)), | |
| seed=int(self.cfg["experiment"].get("seed", 42)), | |
| ) | |
| save_json(metrics, self.run_dir / "metrics.json") | |
| table = metrics_table(metrics) | |
| table.to_csv(self.run_dir / "metrics_table.csv", index=False) | |
| with open(self.run_dir / "summary.md", "w", encoding="utf-8") as f: | |
| f.write(f"# {self.cfg['experiment'].get('name')}\n\n") | |
| f.write(f"run_dir: {self.run_dir}\n\n") | |
| f.write(table.to_string(index=False)) | |
| f.write("\n") | |
| print(table.to_string(index=False)) | |
| return {"run_dir": str(self.run_dir), "metrics": metrics} | |