Tabular Regression
Scikit-learn
Joblib
space-weather
geomagnetic-storms
kp-index
solar-wind
scikit-learn
Instructions to use aliza-ayaz/tormenta with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Scikit-learn
How to use aliza-ayaz/tormenta with Scikit-learn:
from huggingface_hub import hf_hub_download import joblib model = joblib.load( hf_hub_download("aliza-ayaz/tormenta", "sklearn_model.joblib") ) # only load pickle files from sources you trust # read more about it here https://skops.readthedocs.io/en/stable/persistence.html - Notebooks
- Google Colab
- Kaggle
Download src/train.py from aliza-ayaz/tormenta: direct link, hf CLI and curl.
- Browser
- Download file 20.7 kB
-
https://huggingface.co/aliza-ayaz/tormenta/resolve/main/src/train.py
- Command line
-
hf download hf://aliza-ayaz/tormenta/src/train.py
-
curl -L -o train.py https://huggingface.co/aliza-ayaz/tormenta/resolve/main/src/train.py
20.7 kB
| """Training pipeline: features β time split β baselines + 4 model families β tuning β | |
| validation β single test evaluation β weighting ablation β production fit β save. | |
| Run from the notebook (03_modeling.ipynb) step by step, or end-to-end: | |
| python -m src.train # full run, including tuning (long-running) | |
| python -m src.train --fast # smoke test (~2 min) | |
| """ | |
| import argparse | |
| import json | |
| import time | |
| from datetime import datetime, timezone | |
| import joblib | |
| import numpy as np | |
| import pandas as pd | |
| import sklearn | |
| from scipy.stats import loguniform | |
| from sklearn.base import clone | |
| from sklearn.ensemble import HistGradientBoostingRegressor, RandomForestRegressor | |
| from sklearn.impute import SimpleImputer | |
| from sklearn.linear_model import LinearRegression | |
| from sklearn.model_selection import RandomizedSearchCV, TimeSeriesSplit, cross_val_score | |
| from sklearn.neighbors import KNeighborsRegressor | |
| from sklearn.pipeline import Pipeline | |
| from sklearn.preprocessing import StandardScaler | |
| from sklearn.svm import SVR | |
| from src import config | |
| from src.evaluate import metrics, metrics_table | |
| from src.features import build_features, monotonic_cst | |
| from src.preprocessing import load_processed | |
| # ββ Data βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def load_xy(): | |
| """Feature matrix, target and persistence baseline on definitive-label hours.""" | |
| df = load_processed() | |
| f107_daily = df["f107_obs"].resample("D").first() | |
| X = build_features(df[config.SW_FEATURES], f107_daily) | |
| y = df[config.TARGET] | |
| # Persistence: Kp of the previous 3-h block (uses past Kp β baseline only, never a feature) | |
| prev_block = df.index.floor(f"{config.KP_BLOCK_HOURS}h") - pd.Timedelta(hours=config.KP_BLOCK_HOURS) | |
| persistence = pd.Series(y.reindex(prev_block).to_numpy(), index=df.index) | |
| keep = y.notna() & df["kp_definitive"] | |
| return X[keep], y[keep], persistence[keep].fillna(y[keep].median()) | |
| def split_masks(index: pd.DatetimeIndex) -> dict: | |
| year = index.year | |
| return { | |
| "train": year <= config.TRAIN_YEARS[1], | |
| "val": (year >= config.VAL_YEARS[0]) & (year <= config.VAL_YEARS[1]), | |
| "test": year >= config.TEST_START_YEAR, | |
| } | |
| # ββ Models βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def _scaled(model): | |
| return Pipeline([("impute", SimpleImputer(strategy="median")), ("scale", StandardScaler()), ("model", model)]) | |
| def model_specs() -> dict: | |
| """Estimator, search space and row strides (tune on / fit on) per model family.""" | |
| rs = config.RANDOM_STATE | |
| return { | |
| "Linear Regression": dict(est=_scaled(LinearRegression()), params=None, | |
| tune_stride=1, fit_stride=1, n_jobs=1), | |
| "KNN": dict(est=_scaled(KNeighborsRegressor()), | |
| params={"model__n_neighbors": [10, 20, 40, 80], "model__weights": ["uniform", "distance"]}, | |
| n_iter=8, tune_stride=config.SUBSAMPLE_STRIDE, fit_stride=config.SUBSAMPLE_STRIDE, | |
| n_jobs=config.N_JOBS), | |
| "SVR": dict(est=_scaled(SVR(cache_size=500)), | |
| params={"model__C": loguniform(0.3, 10), "model__gamma": loguniform(3e-3, 1e-1), | |
| "model__epsilon": [0.1, 0.3, 0.5]}, | |
| n_iter=6, tune_stride=config.SUBSAMPLE_STRIDE, fit_stride=config.SUBSAMPLE_STRIDE, | |
| n_jobs=config.N_JOBS), | |
| "Random Forest": dict( | |
| est=Pipeline([("impute", SimpleImputer(strategy="median")), | |
| ("model", RandomForestRegressor(n_estimators=100, n_jobs=1, random_state=rs))]), | |
| params={"model__max_depth": [12, 18, 24], "model__min_samples_leaf": [5, 10, 20], | |
| "model__max_features": [0.3, 0.5]}, | |
| n_iter=5, tune_stride=config.RF_TUNE_STRIDE, fit_stride=1, n_jobs=config.N_JOBS), | |
| "HistGB": dict( | |
| est=Pipeline([("model", HistGradientBoostingRegressor(early_stopping=False, random_state=rs))]), | |
| params={"model__learning_rate": loguniform(0.03, 0.2), "model__max_iter": [200, 400, 700], | |
| "model__max_leaf_nodes": [15, 31, 63], "model__min_samples_leaf": [20, 50, 100], | |
| "model__l2_regularization": loguniform(1e-3, 10)}, | |
| n_iter=config.HISTGB_N_ITER, tune_stride=1, fit_stride=1, n_jobs=2), # HistGB is multithreaded | |
| } | |
| def _constrain(est, columns, monotonic=None): | |
| """Apply (or clear) the physics monotonic constraints on a HistGB pipeline.""" | |
| use = config.USE_MONOTONIC if monotonic is None else monotonic | |
| return est.set_params(model__monotonic_cst=monotonic_cst(columns) if use else None) | |
| def _rows(mask, stride, fast): | |
| idx = np.flatnonzero(mask) | |
| return idx[:: stride * (6 if fast else 1)] | |
| def tune_models(X, y, masks, fast=False, log=print, names=None) -> dict: | |
| """TimeSeriesSplit CV search within the training years; refit best params on train rows.""" | |
| cv = TimeSeriesSplit(n_splits=3 if fast else config.CV_SPLITS) | |
| out = {} | |
| for name, spec in model_specs().items(): | |
| if names is not None and name not in names: | |
| continue | |
| t0 = time.time() | |
| rows = _rows(masks["train"], spec["tune_stride"], fast) | |
| Xt, yt = X.iloc[rows], y.iloc[rows] | |
| base = clone(spec["est"]) | |
| if name == "HistGB": | |
| _constrain(base, X.columns) | |
| if spec["params"] is None: | |
| scores = cross_val_score(base, Xt, yt, cv=cv, scoring="neg_root_mean_squared_error") | |
| best, cv_rmse = {}, -scores.mean() | |
| else: | |
| search = RandomizedSearchCV(base, spec["params"], n_iter=2 if fast else spec["n_iter"], | |
| cv=cv, scoring="neg_root_mean_squared_error", n_jobs=spec["n_jobs"], | |
| random_state=config.RANDOM_STATE, refit=False) | |
| search.fit(Xt, yt) | |
| best, cv_rmse = search.best_params_, -search.best_score_ | |
| est = fit_model(name, best, X, y, masks["train"], fast) | |
| out[name] = {"estimator": est, "best_params": best, "cv_rmse": cv_rmse, "seconds": time.time() - t0} | |
| log(f"{name:18s} CV RMSE {cv_rmse:.3f} ({time.time() - t0:5.0f} s) {best}") | |
| return out | |
| def fit_model(name, params, X, y, mask, fast=False, sample_weight=None, monotonic=None): | |
| """Fit one family with given params on the rows of `mask` (at that family's fit stride).""" | |
| spec = model_specs()[name] | |
| rows = _rows(mask, spec["fit_stride"], fast) | |
| est = clone(spec["est"]).set_params(**params) | |
| if name == "Random Forest": | |
| est.set_params(model__n_jobs=config.N_JOBS) | |
| if name == "HistGB": | |
| _constrain(est, X.columns, monotonic) | |
| kw = {} if sample_weight is None else {"model__sample_weight": sample_weight.iloc[rows].to_numpy()} | |
| return est.fit(X.iloc[rows], y.iloc[rows], **kw) | |
| # ββ Evaluation steps βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def validation_table(tuned, X, y, masks, persistence) -> pd.DataFrame: | |
| """Val metrics + CV RMSE + train RMSE (strided sample) for the overfitting check.""" | |
| val, tr = masks["val"], _rows(masks["train"], config.SUBSAMPLE_STRIDE, False) | |
| preds = {"Persistence (baseline)": persistence[val]} | |
| preds.update({n: r["estimator"].predict(X[val]) for n, r in tuned.items()}) | |
| preds = with_ensemble(preds) | |
| table = metrics_table(y[val], preds) | |
| table["cv_rmse"] = pd.Series({n: r["cv_rmse"] for n, r in tuned.items()}) | |
| table["train_rmse"] = pd.Series({n: metrics(y.iloc[tr], r["estimator"].predict(X.iloc[tr]))["rmse"] | |
| for n, r in tuned.items()}) | |
| return table | |
| def test_evaluation(tuned, X, y, masks, persistence, fast=False, log=print): | |
| """Refit every family on train+val with its tuned params, evaluate ONCE on test.""" | |
| trval, test = masks["train"] | masks["val"], masks["test"] | |
| preds, models = {"Persistence (baseline)": persistence[test].to_numpy()}, {} | |
| for name, r in tuned.items(): | |
| t0 = time.time() | |
| models[name] = fit_model(name, r["best_params"], X, y, trval, fast) | |
| preds[name] = models[name].predict(X[test]) | |
| log(f"{name:18s} refit + test in {time.time() - t0:4.0f} s") | |
| preds = with_ensemble(preds) | |
| return metrics_table(y[test], preds), preds, models | |
| def with_ensemble(preds: dict) -> dict: | |
| """Add an equal-weight average of the ensemble members (no weights are fitted, so no leakage).""" | |
| members = [m for m in config.ENSEMBLE_MEMBERS if m in preds] | |
| if len(members) == len(config.ENSEMBLE_MEMBERS): | |
| preds = dict(preds) | |
| preds["Ensemble (HistGB+RF+SVR)"] = np.mean([np.asarray(preds[m]) for m in members], axis=0) | |
| return preds | |
| def kp_history_ablation(best_params, X, y, masks, kp_feats, fast=False): | |
| """ABLATION ONLY: HistGB on solar wind vs solar wind + past Kp; trained on train, scored on val.""" | |
| train, val = masks["train"], masks["val"] | |
| Xk = pd.concat([X, kp_feats], axis=1) | |
| preds = {"HistGB (solar wind only)": fit_model("HistGB", best_params, X, y, train, fast).predict(X[val]), | |
| "HistGB + past Kp (ablation)": fit_model("HistGB", best_params, Xk, y, train, fast).predict(Xk[val])} | |
| return metrics_table(y[val], preds), preds | |
| def weighting_ablation(best_params, X, y, masks, fast=False): | |
| """HistGB unweighted vs storm-weighted (w = 1 + Kp/3), both trained on train+val.""" | |
| trval, test = masks["train"] | masks["val"], masks["test"] | |
| w = config.SAMPLE_WEIGHT(y) | |
| preds = {"HistGB (unweighted)": fit_model("HistGB", best_params, X, y, trval, fast).predict(X[test]), | |
| "HistGB (weighted 1+Kp/3)": fit_model("HistGB", best_params, X, y, trval, fast, | |
| sample_weight=w).predict(X[test])} | |
| return metrics_table(y[test], preds), preds | |
| # ββ Final round (v3): variants, diagnostics, interval calibration βββββββββββββββ | |
| def compare_variant(best_params, X, y, masks, cols, monotonic, fast=False) -> dict: | |
| """HistGB with FIXED params on a column subset: CV RMSE (train years) + validation metrics.""" | |
| Xc = X[cols] | |
| rows = _rows(masks["train"], 1, fast) | |
| est = _constrain(clone(model_specs()["HistGB"]["est"]).set_params(**best_params), cols, monotonic) | |
| cv = TimeSeriesSplit(n_splits=3 if fast else config.CV_SPLITS) | |
| cv_rmse = -cross_val_score(est, Xc.iloc[rows], y.iloc[rows], cv=cv, n_jobs=1, | |
| scoring="neg_root_mean_squared_error").mean() | |
| model = fit_model("HistGB", best_params, Xc, y, masks["train"], fast, monotonic=monotonic) | |
| return {"cv_rmse": cv_rmse, **metrics(y[masks["val"]], model.predict(Xc[masks["val"]]))} | |
| def learning_curve(best_params, X, y, masks, strides=(32, 16, 8, 4, 2, 1), monotonic=False, fast=False): | |
| """Train on every k-th training hour (less data); score train and validation RMSE.""" | |
| out = {} | |
| for k in strides: | |
| rows = _rows(masks["train"], k, fast) | |
| est = fit_model("HistGB", best_params, X.iloc[rows], y.iloc[rows], np.ones(len(rows), bool), | |
| monotonic=monotonic) | |
| out[len(rows)] = {"train_rmse": metrics(y.iloc[rows], est.predict(X.iloc[rows]))["rmse"], | |
| "val_rmse": metrics(y[masks["val"]], est.predict(X[masks["val"]]))["rmse"]} | |
| return pd.DataFrame(out).T.rename_axis("training hours") | |
| def complexity_curve(best_params, X, y, masks, leaves=(7, 15, 31, 63, 127, 255), monotonic=False, fast=False): | |
| """Vary tree size with everything else fixed: does a more complex model help validation?""" | |
| tr = _rows(masks["train"], config.SUBSAMPLE_STRIDE, False) | |
| out = {} | |
| for n in leaves: | |
| est = fit_model("HistGB", {**best_params, "model__max_leaf_nodes": n}, X, y, masks["train"], fast, | |
| monotonic=monotonic) | |
| out[n] = {"train_rmse": metrics(y.iloc[tr], est.predict(X.iloc[tr]))["rmse"], | |
| "val_rmse": metrics(y[masks["val"]], est.predict(X[masks["val"]]))["rmse"]} | |
| return pd.DataFrame(out).T.rename_axis("max_leaf_nodes") | |
| def conformal_margin(best_params, X, y, masks, fast=False, log=print) -> tuple[float, pd.DataFrame]: | |
| """Conformalized quantile regression (CQR) margin for the q10-q90 band. | |
| Conformity score = how far y falls outside the band (negative if inside). Scores come only from | |
| predictions on data the quantile models did NOT train on: out-of-fold TimeSeriesSplit folds of | |
| the training years (skipping the first, tiny-training fold) plus the validation years (models | |
| trained on the training years). The margin is the 80% conformal quantile of those scores and is | |
| added to both sides of the band. Test and gate-holdout data are never used. | |
| """ | |
| params = {k.removeprefix("model__"): v for k, v in best_params.items()} | |
| cst = monotonic_cst(X.columns) if config.USE_MONOTONIC else None | |
| def band(fit_idx, pred_idx): | |
| return [HistGradientBoostingRegressor(loss="quantile", quantile=q, early_stopping=False, | |
| random_state=config.RANDOM_STATE, monotonic_cst=cst, **params) | |
| .fit(X.iloc[fit_idx], y.iloc[fit_idx]).predict(X.iloc[pred_idx]) for q in config.QUANTILES] | |
| train_idx = _rows(masks["train"], 1, fast) | |
| folds = list(TimeSeriesSplit(n_splits=3 if fast else config.CV_SPLITS).split(train_idx)) | |
| pieces = [(train_idx[a], train_idx[b]) for a, b in folds[config.CONFORMAL_SKIP_FOLDS:]] | |
| pieces.append((train_idx, _rows(masks["val"], 1, fast))) | |
| scores, rows = [], {} | |
| for fit_idx, pred_idx in pieces: | |
| t0 = time.time() | |
| lo, hi = band(fit_idx, pred_idx) | |
| yt = y.iloc[pred_idx].to_numpy() | |
| s = np.maximum(lo - yt, yt - hi) | |
| scores.append(s) | |
| idx = y.index[pred_idx] | |
| label = f"{idx.min():%Y-%m} to {idx.max():%Y-%m}" | |
| rows[label] = {"raw_coverage": float((s <= 0).mean()), "n": len(s)} | |
| log(f"calibration block {label}: raw coverage {(s <= 0).mean():.1%} ({time.time() - t0:.0f} s)") | |
| all_scores = np.concatenate(scores) | |
| level = config.QUANTILES[1] - config.QUANTILES[0] | |
| margin = float(np.quantile(all_scores, min(1.0, np.ceil((len(all_scores) + 1) * level) / len(all_scores)))) | |
| table = pd.DataFrame(rows).T | |
| table["calibrated_coverage"] = [float((s <= margin).mean()) for s in scores] | |
| return margin, table | |
| # ββ Production model βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def fit_bundle(best_params, X, y, mask, fast=False, sample_weight=None, margin=0.0) -> dict: | |
| """Point + q10/q90 HistGB trained on the rows of `mask` (+ conformal margin), packaged for predict.py.""" | |
| params = {k.removeprefix("model__"): v for k, v in best_params.items()} | |
| rows = _rows(mask, 1, fast) | |
| Xf, yf = X.iloc[rows], y.iloc[rows] | |
| w = None if sample_weight is None else sample_weight.iloc[rows].to_numpy() | |
| rs = config.RANDOM_STATE | |
| cst = monotonic_cst(X.columns) if config.USE_MONOTONIC else None | |
| point = HistGradientBoostingRegressor(early_stopping=False, random_state=rs, monotonic_cst=cst, | |
| **params).fit(Xf, yf, sample_weight=w) | |
| q_lo, q_hi = (HistGradientBoostingRegressor(loss="quantile", quantile=q, early_stopping=False, | |
| random_state=rs, monotonic_cst=cst, **params).fit(Xf, yf) | |
| for q in config.QUANTILES) | |
| return { | |
| "point": point, "q_lo": q_lo, "q_hi": q_hi, | |
| "features": list(X.columns), "params": params, "weighted": sample_weight is not None, | |
| "monotonic": cst is not None, "interval_margin": float(margin), | |
| "trained_until": str(y.index[rows].max()), "sklearn_version": sklearn.__version__, | |
| "created_utc": datetime.now(timezone.utc).isoformat(timespec="seconds"), | |
| } | |
| def _holdout_mask(y): | |
| return y.index >= y.index.max() - pd.DateOffset(months=config.GATE_HOLDOUT_MONTHS) | |
| def gate_metrics(bundle, X, y) -> dict: | |
| """Score a bundle on the current gate holdout (the most recent 6 months of definitive labels).""" | |
| from src.predict import predict_with_interval | |
| hold = _holdout_mask(y) | |
| band = predict_with_interval(bundle, X[hold]) | |
| gate = metrics(y[hold], band["kp_pred"]) | |
| gate["coverage_80"] = float(((y[hold] >= band["kp_lo"]) & (y[hold] <= band["kp_hi"])).mean()) | |
| gate["holdout"] = f"{y.index[hold].min()} β {y.index[hold].max()}" | |
| return gate | |
| def fit_production(best_params, X, y, fast=False, sample_weight=None, margin=0.0): | |
| """Bundle trained up to the gate-holdout start (no refit-on-all); gate metrics on the holdout.""" | |
| bundle = fit_bundle(best_params, X, y, ~_holdout_mask(y), fast, sample_weight, margin) | |
| return bundle, gate_metrics(bundle, X, y) | |
| def gate_decision(candidate_gate: dict, X, y) -> tuple[bool, dict | None]: | |
| """Promote if candidate RMSE β€ incumbent RMSE Γ tolerance, both scored on THIS run's holdout. | |
| The incumbent must not have been trained on the holdout; its stored trained_until is checked. | |
| """ | |
| if not config.CURRENT_MODEL.exists(): | |
| return True, None | |
| from src.predict import load_bundle | |
| incumbent = load_bundle() | |
| hold_start = y.index[_holdout_mask(y)].min() | |
| assert pd.Timestamp(incumbent["trained_until"]) < hold_start, "incumbent was trained on the holdout" | |
| inc = gate_metrics(incumbent, X, y) | |
| return candidate_gate["rmse"] <= inc["rmse"] * config.GATE_TOLERANCE, inc | |
| def save_bundle(bundle, test_metrics: dict, gate_metrics: dict, promote: bool = True, | |
| incumbent_gate: dict | None = None, keep_rejected: bool = True, note: str | None = None) -> str: | |
| """Versioned model file (+ current.joblib if promoted) and an entry in metrics_history.json. | |
| keep_rejected=False (weekly retrain) logs a rejected candidate without writing its model file. | |
| """ | |
| config.MODELS_DIR.mkdir(parents=True, exist_ok=True) | |
| version = f"tormenta_v{datetime.now(timezone.utc):%Y%m%d_%H%M}" | |
| bundle["version"] = version | |
| if promote or keep_rejected: | |
| joblib.dump(bundle, config.MODELS_DIR / f"{version}.joblib", compress=3) | |
| if promote: | |
| joblib.dump(bundle, config.CURRENT_MODEL, compress=3) | |
| history = json.loads(config.METRICS_HISTORY.read_text()) if config.METRICS_HISTORY.exists() else [] | |
| clean = lambda d: {k: (float(v) if isinstance(v, (int, float, np.floating)) else v) for k, v in d.items()} | |
| history.append({"version": version, "created_utc": bundle["created_utc"], | |
| "trained_until": bundle["trained_until"], "params": bundle["params"], | |
| "weighted": bundle["weighted"], "n_features": len(bundle["features"]), | |
| "test": clean(test_metrics), "gate": clean(gate_metrics), | |
| "incumbent_gate": clean(incumbent_gate) if incumbent_gate else None, | |
| "promoted": promote, **({"note": note} if note else {})}) | |
| config.METRICS_HISTORY.write_text(json.dumps(history, indent=2, default=str)) | |
| return version | |
| def run(fast=False): | |
| """End-to-end run (same steps as the notebook).""" | |
| X, y, persistence = load_xy() | |
| masks = split_masks(X.index) | |
| print(f"{X.shape[1]} features | train {masks['train'].sum():,} | val {masks['val'].sum():,} | " | |
| f"test {masks['test'].sum():,} hours") | |
| tuned = tune_models(X, y, masks, fast) | |
| print(validation_table(tuned, X, y, masks, persistence).round(3)) | |
| test_table, _, _ = test_evaluation(tuned, X, y, masks, persistence, fast) | |
| print(test_table.round(3)) | |
| best = tuned["HistGB"]["best_params"] | |
| margin, _ = conformal_margin(best, X, y, masks, fast) | |
| bundle, gate = fit_production(best, X, y, fast, margin=margin) | |
| promote, incumbent = gate_decision(gate, X, y) | |
| version = save_bundle(bundle, test_table.loc["HistGB"].to_dict(), gate, promote, incumbent) | |
| print(f"Saved {version} (promoted={promote}); gate: {gate}; incumbent: {incumbent}") | |
| if __name__ == "__main__": | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--fast", action="store_true", help="tiny search + heavy subsampling (smoke test)") | |
| run(ap.parse_args().fast) | |