Spaces:
Sleeping
Sleeping
Download src/pipeline/preprocessing/normalizer.py from Harshilforworks/Forcast: direct link, hf CLI and curl.
- Browser
- Download file 2.66 kB
-
https://huggingface.co/spaces/Harshilforworks/Forcast/resolve/main/src/pipeline/preprocessing/normalizer.py
- Command line
-
hf download hf://spaces/Harshilforworks/Forcast/src/pipeline/preprocessing/normalizer.py
-
curl -L -o normalizer.py https://huggingface.co/spaces/Harshilforworks/Forcast/resolve/main/src/pipeline/preprocessing/normalizer.py
2.66 kB
| """Preprocessing β StandardScaler fit/transform + joblib persistence.""" | |
| from __future__ import annotations | |
| from pathlib import Path | |
| from typing import Any | |
| import joblib | |
| import mlflow | |
| import numpy as np | |
| import pandas as pd | |
| from sklearn.preprocessing import StandardScaler | |
| from src.pipeline.ingestion.loader import FEATURE_COLUMNS | |
| def fit_transform( | |
| df: pd.DataFrame, | |
| train_mask: np.ndarray, | |
| cfg: dict[str, Any], | |
| ) -> tuple[np.ndarray, np.ndarray, StandardScaler]: | |
| """Fit a StandardScaler on the training rows only, then transform all rows. | |
| Args: | |
| df: Full DataFrame (all splits). | |
| train_mask: Boolean mask identifying training rows. | |
| cfg: Project config dict. | |
| Returns: | |
| features_scaled: float32 array [N, 20] | |
| labels: float32 array [N] (raw FloodProbability, already 0-1) | |
| scaler: Fitted StandardScaler (saved to disk + MLflow artifact). | |
| """ | |
| label_col = cfg["data"]["label_column"] | |
| feature_cols = FEATURE_COLUMNS | |
| X = df[feature_cols].values.astype(np.float32) | |
| y = df[label_col].values.astype(np.float32) | |
| scaler = StandardScaler() | |
| scaler.fit(X[train_mask]) | |
| X_scaled = scaler.transform(X).astype(np.float32) | |
| # ββ Persist scaler βββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| scaler_path = Path(cfg["paths"]["scaler"]) | |
| scaler_path.parent.mkdir(parents=True, exist_ok=True) | |
| joblib.dump(scaler, scaler_path) | |
| # ββ MLflow logging βββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| try: | |
| mlflow.log_params({ | |
| "preprocessing.scaler": "StandardScaler", | |
| "preprocessing.n_features": len(feature_cols), | |
| "preprocessing.train_rows": int(train_mask.sum()), | |
| }) | |
| mlflow.log_artifact(str(scaler_path), artifact_path="preprocessing") | |
| except Exception: | |
| pass | |
| print(f"[preprocessing] Scaler fitted on {int(train_mask.sum()):,} training rows.") | |
| print(f"[preprocessing] Scaler saved β '{scaler_path}'") | |
| return X_scaled, y, scaler | |
| def load_scaler(cfg: dict[str, Any]) -> StandardScaler: | |
| """Load a previously saved scaler from disk.""" | |
| path = Path(cfg["paths"]["scaler"]) | |
| if not path.exists(): | |
| raise FileNotFoundError(f"Scaler not found at '{path}'. Run training first.") | |
| return joblib.load(path) | |