File size: 2,532 Bytes
04ae9bb | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 | """Prediction, evaluation, and submission helpers."""
from __future__ import annotations
from pathlib import Path
from typing import Any, Dict, Optional
import numpy as np
import pandas as pd
import torch
from ..utils import save_json
from .decision import point_from_output
from .metrics import aggregate_metrics, prediction_frame
@torch.no_grad()
def predict_loader(
model: torch.nn.Module,
loader,
device: torch.device,
output_type: str = "regression",
n_classes: int = 87,
target_scaler=None,
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
model.eval()
preds = []
targets = []
masks = []
for batch in loader:
x = batch["x"].to(device, non_blocking=True)
pred = model(x)
point = point_from_output(pred, output_type, n_classes)
if target_scaler is not None and output_type == "regression":
point = torch.from_numpy(target_scaler.inverse(point.cpu().numpy())).to(device).float()
preds.append(point.cpu())
targets.append(batch["y"])
masks.append(batch["mask"])
return (
torch.cat(preds).numpy(),
torch.cat(targets).numpy(),
torch.cat(masks).numpy(),
)
def evaluate_predictions(
pred_df: pd.DataFrame,
metrics=("smape_plus_1", "mae"),
bootstrap: int = 0,
seed: int = 42,
) -> Dict[str, Any]:
return aggregate_metrics(
pred_df,
metrics=metrics,
per_target=True,
per_horizon=True,
per_cohort=True,
per_fold=True,
bootstrap=bootstrap,
seed=seed,
)
def save_predictions(pred_df: pd.DataFrame, path: str | Path) -> None:
p = Path(path)
p.parent.mkdir(parents=True, exist_ok=True)
pred_df.to_csv(p, index=False)
def make_submission(
pred_df: pd.DataFrame,
rating_col: str = "y_pred",
group_key_col: str = "group_key",
clip_min: float = 0.0,
clip_max: float = 100.0,
) -> pd.DataFrame:
df = pred_df.copy()
df["prediction_id"] = (
df["visit_id"].astype(str)
+ "_"
+ df["target"].astype(str)
+ "_plus_"
+ df["horizon"].astype(int).astype(str)
+ "_months"
)
df["rating"] = df[rating_col].clip(clip_min, clip_max)
if group_key_col not in df.columns:
df[group_key_col] = df["visit_month"]
return df[["prediction_id", "rating", group_key_col]].rename(columns={group_key_col: "group_key"})
def save_metrics(metrics: Dict[str, Any], path: str | Path) -> None:
save_json(metrics, path)
|