Forcast / src /pipeline /preprocessing /normalizer.py
Harshilforworks's picture
Upload 40 files
3d2a487 verified
Raw History Blame Contribute Delete
2.66 kB
"""Preprocessing β€” StandardScaler fit/transform + joblib persistence."""
from __future__ import annotations
from pathlib import Path
from typing import Any
import joblib
import mlflow
import numpy as np
import pandas as pd
from sklearn.preprocessing import StandardScaler
from src.pipeline.ingestion.loader import FEATURE_COLUMNS
def fit_transform(
df: pd.DataFrame,
train_mask: np.ndarray,
cfg: dict[str, Any],
) -> tuple[np.ndarray, np.ndarray, StandardScaler]:
"""Fit a StandardScaler on the training rows only, then transform all rows.
Args:
df: Full DataFrame (all splits).
train_mask: Boolean mask identifying training rows.
cfg: Project config dict.
Returns:
features_scaled: float32 array [N, 20]
labels: float32 array [N] (raw FloodProbability, already 0-1)
scaler: Fitted StandardScaler (saved to disk + MLflow artifact).
"""
label_col = cfg["data"]["label_column"]
feature_cols = FEATURE_COLUMNS
X = df[feature_cols].values.astype(np.float32)
y = df[label_col].values.astype(np.float32)
scaler = StandardScaler()
scaler.fit(X[train_mask])
X_scaled = scaler.transform(X).astype(np.float32)
# ── Persist scaler ───────────────────────────────────────────────────────
scaler_path = Path(cfg["paths"]["scaler"])
scaler_path.parent.mkdir(parents=True, exist_ok=True)
joblib.dump(scaler, scaler_path)
# ── MLflow logging ───────────────────────────────────────────────────────
try:
mlflow.log_params({
"preprocessing.scaler": "StandardScaler",
"preprocessing.n_features": len(feature_cols),
"preprocessing.train_rows": int(train_mask.sum()),
})
mlflow.log_artifact(str(scaler_path), artifact_path="preprocessing")
except Exception:
pass
print(f"[preprocessing] Scaler fitted on {int(train_mask.sum()):,} training rows.")
print(f"[preprocessing] Scaler saved β†’ '{scaler_path}'")
return X_scaled, y, scaler
def load_scaler(cfg: dict[str, Any]) -> StandardScaler:
"""Load a previously saved scaler from disk."""
path = Path(cfg["paths"]["scaler"])
if not path.exists():
raise FileNotFoundError(f"Scaler not found at '{path}'. Run training first.")
return joblib.load(path)