fryabcfry's picture
Publish AMP-PD checkpoints, portable loaders and verified metrics (part 3)
04ae9bb verified
Raw History Blame Contribute Delete
2.53 kB
"""Prediction, evaluation, and submission helpers."""
from __future__ import annotations
from pathlib import Path
from typing import Any, Dict, Optional
import numpy as np
import pandas as pd
import torch
from ..utils import save_json
from .decision import point_from_output
from .metrics import aggregate_metrics, prediction_frame
@torch.no_grad()
def predict_loader(
model: torch.nn.Module,
loader,
device: torch.device,
output_type: str = "regression",
n_classes: int = 87,
target_scaler=None,
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
model.eval()
preds = []
targets = []
masks = []
for batch in loader:
x = batch["x"].to(device, non_blocking=True)
pred = model(x)
point = point_from_output(pred, output_type, n_classes)
if target_scaler is not None and output_type == "regression":
point = torch.from_numpy(target_scaler.inverse(point.cpu().numpy())).to(device).float()
preds.append(point.cpu())
targets.append(batch["y"])
masks.append(batch["mask"])
return (
torch.cat(preds).numpy(),
torch.cat(targets).numpy(),
torch.cat(masks).numpy(),
)
def evaluate_predictions(
pred_df: pd.DataFrame,
metrics=("smape_plus_1", "mae"),
bootstrap: int = 0,
seed: int = 42,
) -> Dict[str, Any]:
return aggregate_metrics(
pred_df,
metrics=metrics,
per_target=True,
per_horizon=True,
per_cohort=True,
per_fold=True,
bootstrap=bootstrap,
seed=seed,
)
def save_predictions(pred_df: pd.DataFrame, path: str | Path) -> None:
p = Path(path)
p.parent.mkdir(parents=True, exist_ok=True)
pred_df.to_csv(p, index=False)
def make_submission(
pred_df: pd.DataFrame,
rating_col: str = "y_pred",
group_key_col: str = "group_key",
clip_min: float = 0.0,
clip_max: float = 100.0,
) -> pd.DataFrame:
df = pred_df.copy()
df["prediction_id"] = (
df["visit_id"].astype(str)
+ "_"
+ df["target"].astype(str)
+ "_plus_"
+ df["horizon"].astype(int).astype(str)
+ "_months"
)
df["rating"] = df[rating_col].clip(clip_min, clip_max)
if group_key_col not in df.columns:
df[group_key_col] = df["visit_month"]
return df[["prediction_id", "rating", group_key_col]].rename(columns={group_key_col: "group_key"})
def save_metrics(metrics: Dict[str, Any], path: str | Path) -> None:
save_json(metrics, path)