Spaces:
Sleeping
Sleeping
File size: 2,662 Bytes
3d2a487 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 | """Preprocessing β StandardScaler fit/transform + joblib persistence."""
from __future__ import annotations
from pathlib import Path
from typing import Any
import joblib
import mlflow
import numpy as np
import pandas as pd
from sklearn.preprocessing import StandardScaler
from src.pipeline.ingestion.loader import FEATURE_COLUMNS
def fit_transform(
df: pd.DataFrame,
train_mask: np.ndarray,
cfg: dict[str, Any],
) -> tuple[np.ndarray, np.ndarray, StandardScaler]:
"""Fit a StandardScaler on the training rows only, then transform all rows.
Args:
df: Full DataFrame (all splits).
train_mask: Boolean mask identifying training rows.
cfg: Project config dict.
Returns:
features_scaled: float32 array [N, 20]
labels: float32 array [N] (raw FloodProbability, already 0-1)
scaler: Fitted StandardScaler (saved to disk + MLflow artifact).
"""
label_col = cfg["data"]["label_column"]
feature_cols = FEATURE_COLUMNS
X = df[feature_cols].values.astype(np.float32)
y = df[label_col].values.astype(np.float32)
scaler = StandardScaler()
scaler.fit(X[train_mask])
X_scaled = scaler.transform(X).astype(np.float32)
# ββ Persist scaler βββββββββββββββββββββββββββββββββββββββββββββββββββββββ
scaler_path = Path(cfg["paths"]["scaler"])
scaler_path.parent.mkdir(parents=True, exist_ok=True)
joblib.dump(scaler, scaler_path)
# ββ MLflow logging βββββββββββββββββββββββββββββββββββββββββββββββββββββββ
try:
mlflow.log_params({
"preprocessing.scaler": "StandardScaler",
"preprocessing.n_features": len(feature_cols),
"preprocessing.train_rows": int(train_mask.sum()),
})
mlflow.log_artifact(str(scaler_path), artifact_path="preprocessing")
except Exception:
pass
print(f"[preprocessing] Scaler fitted on {int(train_mask.sum()):,} training rows.")
print(f"[preprocessing] Scaler saved β '{scaler_path}'")
return X_scaled, y, scaler
def load_scaler(cfg: dict[str, Any]) -> StandardScaler:
"""Load a previously saved scaler from disk."""
path = Path(cfg["paths"]["scaler"])
if not path.exists():
raise FileNotFoundError(f"Scaler not found at '{path}'. Run training first.")
return joblib.load(path)
|