Download code/ampd/evaluation/evaluator.py from fryabcfry/cs4486-amp-pd-checkpoints: direct link, hf CLI and curl.
- Browser
- Download file 2.53 kB
-
https://huggingface.co/fryabcfry/cs4486-amp-pd-checkpoints/resolve/main/code/ampd/evaluation/evaluator.py
- Command line
-
hf download hf://fryabcfry/cs4486-amp-pd-checkpoints/code/ampd/evaluation/evaluator.py
-
curl -L -o evaluator.py https://huggingface.co/fryabcfry/cs4486-amp-pd-checkpoints/resolve/main/code/ampd/evaluation/evaluator.py
2.53 kB
| """Prediction, evaluation, and submission helpers.""" | |
| from __future__ import annotations | |
| from pathlib import Path | |
| from typing import Any, Dict, Optional | |
| import numpy as np | |
| import pandas as pd | |
| import torch | |
| from ..utils import save_json | |
| from .decision import point_from_output | |
| from .metrics import aggregate_metrics, prediction_frame | |
| def predict_loader( | |
| model: torch.nn.Module, | |
| loader, | |
| device: torch.device, | |
| output_type: str = "regression", | |
| n_classes: int = 87, | |
| target_scaler=None, | |
| ) -> tuple[np.ndarray, np.ndarray, np.ndarray]: | |
| model.eval() | |
| preds = [] | |
| targets = [] | |
| masks = [] | |
| for batch in loader: | |
| x = batch["x"].to(device, non_blocking=True) | |
| pred = model(x) | |
| point = point_from_output(pred, output_type, n_classes) | |
| if target_scaler is not None and output_type == "regression": | |
| point = torch.from_numpy(target_scaler.inverse(point.cpu().numpy())).to(device).float() | |
| preds.append(point.cpu()) | |
| targets.append(batch["y"]) | |
| masks.append(batch["mask"]) | |
| return ( | |
| torch.cat(preds).numpy(), | |
| torch.cat(targets).numpy(), | |
| torch.cat(masks).numpy(), | |
| ) | |
| def evaluate_predictions( | |
| pred_df: pd.DataFrame, | |
| metrics=("smape_plus_1", "mae"), | |
| bootstrap: int = 0, | |
| seed: int = 42, | |
| ) -> Dict[str, Any]: | |
| return aggregate_metrics( | |
| pred_df, | |
| metrics=metrics, | |
| per_target=True, | |
| per_horizon=True, | |
| per_cohort=True, | |
| per_fold=True, | |
| bootstrap=bootstrap, | |
| seed=seed, | |
| ) | |
| def save_predictions(pred_df: pd.DataFrame, path: str | Path) -> None: | |
| p = Path(path) | |
| p.parent.mkdir(parents=True, exist_ok=True) | |
| pred_df.to_csv(p, index=False) | |
| def make_submission( | |
| pred_df: pd.DataFrame, | |
| rating_col: str = "y_pred", | |
| group_key_col: str = "group_key", | |
| clip_min: float = 0.0, | |
| clip_max: float = 100.0, | |
| ) -> pd.DataFrame: | |
| df = pred_df.copy() | |
| df["prediction_id"] = ( | |
| df["visit_id"].astype(str) | |
| + "_" | |
| + df["target"].astype(str) | |
| + "_plus_" | |
| + df["horizon"].astype(int).astype(str) | |
| + "_months" | |
| ) | |
| df["rating"] = df[rating_col].clip(clip_min, clip_max) | |
| if group_key_col not in df.columns: | |
| df[group_key_col] = df["visit_month"] | |
| return df[["prediction_id", "rating", group_key_col]].rename(columns={group_key_col: "group_key"}) | |
| def save_metrics(metrics: Dict[str, Any], path: str | Path) -> None: | |
| save_json(metrics, path) | |