tormenta / src /train.py
aliza-ayaz's picture
Daily forecast: origin 2026-11-13
0482c4f verified
Raw History Blame Contribute Delete
20.7 kB
"""Training pipeline: features β†’ time split β†’ baselines + 4 model families β†’ tuning β†’
validation β†’ single test evaluation β†’ weighting ablation β†’ production fit β†’ save.
Run from the notebook (03_modeling.ipynb) step by step, or end-to-end:
python -m src.train # full run, including tuning (long-running)
python -m src.train --fast # smoke test (~2 min)
"""
import argparse
import json
import time
from datetime import datetime, timezone
import joblib
import numpy as np
import pandas as pd
import sklearn
from scipy.stats import loguniform
from sklearn.base import clone
from sklearn.ensemble import HistGradientBoostingRegressor, RandomForestRegressor
from sklearn.impute import SimpleImputer
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import RandomizedSearchCV, TimeSeriesSplit, cross_val_score
from sklearn.neighbors import KNeighborsRegressor
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.svm import SVR
from src import config
from src.evaluate import metrics, metrics_table
from src.features import build_features, monotonic_cst
from src.preprocessing import load_processed
# ── Data ─────────────────────────────────────────────────────────────────────
def load_xy():
"""Feature matrix, target and persistence baseline on definitive-label hours."""
df = load_processed()
f107_daily = df["f107_obs"].resample("D").first()
X = build_features(df[config.SW_FEATURES], f107_daily)
y = df[config.TARGET]
# Persistence: Kp of the previous 3-h block (uses past Kp β€” baseline only, never a feature)
prev_block = df.index.floor(f"{config.KP_BLOCK_HOURS}h") - pd.Timedelta(hours=config.KP_BLOCK_HOURS)
persistence = pd.Series(y.reindex(prev_block).to_numpy(), index=df.index)
keep = y.notna() & df["kp_definitive"]
return X[keep], y[keep], persistence[keep].fillna(y[keep].median())
def split_masks(index: pd.DatetimeIndex) -> dict:
year = index.year
return {
"train": year <= config.TRAIN_YEARS[1],
"val": (year >= config.VAL_YEARS[0]) & (year <= config.VAL_YEARS[1]),
"test": year >= config.TEST_START_YEAR,
}
# ── Models ───────────────────────────────────────────────────────────────────
def _scaled(model):
return Pipeline([("impute", SimpleImputer(strategy="median")), ("scale", StandardScaler()), ("model", model)])
def model_specs() -> dict:
"""Estimator, search space and row strides (tune on / fit on) per model family."""
rs = config.RANDOM_STATE
return {
"Linear Regression": dict(est=_scaled(LinearRegression()), params=None,
tune_stride=1, fit_stride=1, n_jobs=1),
"KNN": dict(est=_scaled(KNeighborsRegressor()),
params={"model__n_neighbors": [10, 20, 40, 80], "model__weights": ["uniform", "distance"]},
n_iter=8, tune_stride=config.SUBSAMPLE_STRIDE, fit_stride=config.SUBSAMPLE_STRIDE,
n_jobs=config.N_JOBS),
"SVR": dict(est=_scaled(SVR(cache_size=500)),
params={"model__C": loguniform(0.3, 10), "model__gamma": loguniform(3e-3, 1e-1),
"model__epsilon": [0.1, 0.3, 0.5]},
n_iter=6, tune_stride=config.SUBSAMPLE_STRIDE, fit_stride=config.SUBSAMPLE_STRIDE,
n_jobs=config.N_JOBS),
"Random Forest": dict(
est=Pipeline([("impute", SimpleImputer(strategy="median")),
("model", RandomForestRegressor(n_estimators=100, n_jobs=1, random_state=rs))]),
params={"model__max_depth": [12, 18, 24], "model__min_samples_leaf": [5, 10, 20],
"model__max_features": [0.3, 0.5]},
n_iter=5, tune_stride=config.RF_TUNE_STRIDE, fit_stride=1, n_jobs=config.N_JOBS),
"HistGB": dict(
est=Pipeline([("model", HistGradientBoostingRegressor(early_stopping=False, random_state=rs))]),
params={"model__learning_rate": loguniform(0.03, 0.2), "model__max_iter": [200, 400, 700],
"model__max_leaf_nodes": [15, 31, 63], "model__min_samples_leaf": [20, 50, 100],
"model__l2_regularization": loguniform(1e-3, 10)},
n_iter=config.HISTGB_N_ITER, tune_stride=1, fit_stride=1, n_jobs=2), # HistGB is multithreaded
}
def _constrain(est, columns, monotonic=None):
"""Apply (or clear) the physics monotonic constraints on a HistGB pipeline."""
use = config.USE_MONOTONIC if monotonic is None else monotonic
return est.set_params(model__monotonic_cst=monotonic_cst(columns) if use else None)
def _rows(mask, stride, fast):
idx = np.flatnonzero(mask)
return idx[:: stride * (6 if fast else 1)]
def tune_models(X, y, masks, fast=False, log=print, names=None) -> dict:
"""TimeSeriesSplit CV search within the training years; refit best params on train rows."""
cv = TimeSeriesSplit(n_splits=3 if fast else config.CV_SPLITS)
out = {}
for name, spec in model_specs().items():
if names is not None and name not in names:
continue
t0 = time.time()
rows = _rows(masks["train"], spec["tune_stride"], fast)
Xt, yt = X.iloc[rows], y.iloc[rows]
base = clone(spec["est"])
if name == "HistGB":
_constrain(base, X.columns)
if spec["params"] is None:
scores = cross_val_score(base, Xt, yt, cv=cv, scoring="neg_root_mean_squared_error")
best, cv_rmse = {}, -scores.mean()
else:
search = RandomizedSearchCV(base, spec["params"], n_iter=2 if fast else spec["n_iter"],
cv=cv, scoring="neg_root_mean_squared_error", n_jobs=spec["n_jobs"],
random_state=config.RANDOM_STATE, refit=False)
search.fit(Xt, yt)
best, cv_rmse = search.best_params_, -search.best_score_
est = fit_model(name, best, X, y, masks["train"], fast)
out[name] = {"estimator": est, "best_params": best, "cv_rmse": cv_rmse, "seconds": time.time() - t0}
log(f"{name:18s} CV RMSE {cv_rmse:.3f} ({time.time() - t0:5.0f} s) {best}")
return out
def fit_model(name, params, X, y, mask, fast=False, sample_weight=None, monotonic=None):
"""Fit one family with given params on the rows of `mask` (at that family's fit stride)."""
spec = model_specs()[name]
rows = _rows(mask, spec["fit_stride"], fast)
est = clone(spec["est"]).set_params(**params)
if name == "Random Forest":
est.set_params(model__n_jobs=config.N_JOBS)
if name == "HistGB":
_constrain(est, X.columns, monotonic)
kw = {} if sample_weight is None else {"model__sample_weight": sample_weight.iloc[rows].to_numpy()}
return est.fit(X.iloc[rows], y.iloc[rows], **kw)
# ── Evaluation steps ─────────────────────────────────────────────────────────
def validation_table(tuned, X, y, masks, persistence) -> pd.DataFrame:
"""Val metrics + CV RMSE + train RMSE (strided sample) for the overfitting check."""
val, tr = masks["val"], _rows(masks["train"], config.SUBSAMPLE_STRIDE, False)
preds = {"Persistence (baseline)": persistence[val]}
preds.update({n: r["estimator"].predict(X[val]) for n, r in tuned.items()})
preds = with_ensemble(preds)
table = metrics_table(y[val], preds)
table["cv_rmse"] = pd.Series({n: r["cv_rmse"] for n, r in tuned.items()})
table["train_rmse"] = pd.Series({n: metrics(y.iloc[tr], r["estimator"].predict(X.iloc[tr]))["rmse"]
for n, r in tuned.items()})
return table
def test_evaluation(tuned, X, y, masks, persistence, fast=False, log=print):
"""Refit every family on train+val with its tuned params, evaluate ONCE on test."""
trval, test = masks["train"] | masks["val"], masks["test"]
preds, models = {"Persistence (baseline)": persistence[test].to_numpy()}, {}
for name, r in tuned.items():
t0 = time.time()
models[name] = fit_model(name, r["best_params"], X, y, trval, fast)
preds[name] = models[name].predict(X[test])
log(f"{name:18s} refit + test in {time.time() - t0:4.0f} s")
preds = with_ensemble(preds)
return metrics_table(y[test], preds), preds, models
def with_ensemble(preds: dict) -> dict:
"""Add an equal-weight average of the ensemble members (no weights are fitted, so no leakage)."""
members = [m for m in config.ENSEMBLE_MEMBERS if m in preds]
if len(members) == len(config.ENSEMBLE_MEMBERS):
preds = dict(preds)
preds["Ensemble (HistGB+RF+SVR)"] = np.mean([np.asarray(preds[m]) for m in members], axis=0)
return preds
def kp_history_ablation(best_params, X, y, masks, kp_feats, fast=False):
"""ABLATION ONLY: HistGB on solar wind vs solar wind + past Kp; trained on train, scored on val."""
train, val = masks["train"], masks["val"]
Xk = pd.concat([X, kp_feats], axis=1)
preds = {"HistGB (solar wind only)": fit_model("HistGB", best_params, X, y, train, fast).predict(X[val]),
"HistGB + past Kp (ablation)": fit_model("HistGB", best_params, Xk, y, train, fast).predict(Xk[val])}
return metrics_table(y[val], preds), preds
def weighting_ablation(best_params, X, y, masks, fast=False):
"""HistGB unweighted vs storm-weighted (w = 1 + Kp/3), both trained on train+val."""
trval, test = masks["train"] | masks["val"], masks["test"]
w = config.SAMPLE_WEIGHT(y)
preds = {"HistGB (unweighted)": fit_model("HistGB", best_params, X, y, trval, fast).predict(X[test]),
"HistGB (weighted 1+Kp/3)": fit_model("HistGB", best_params, X, y, trval, fast,
sample_weight=w).predict(X[test])}
return metrics_table(y[test], preds), preds
# ── Final round (v3): variants, diagnostics, interval calibration ───────────────
def compare_variant(best_params, X, y, masks, cols, monotonic, fast=False) -> dict:
"""HistGB with FIXED params on a column subset: CV RMSE (train years) + validation metrics."""
Xc = X[cols]
rows = _rows(masks["train"], 1, fast)
est = _constrain(clone(model_specs()["HistGB"]["est"]).set_params(**best_params), cols, monotonic)
cv = TimeSeriesSplit(n_splits=3 if fast else config.CV_SPLITS)
cv_rmse = -cross_val_score(est, Xc.iloc[rows], y.iloc[rows], cv=cv, n_jobs=1,
scoring="neg_root_mean_squared_error").mean()
model = fit_model("HistGB", best_params, Xc, y, masks["train"], fast, monotonic=monotonic)
return {"cv_rmse": cv_rmse, **metrics(y[masks["val"]], model.predict(Xc[masks["val"]]))}
def learning_curve(best_params, X, y, masks, strides=(32, 16, 8, 4, 2, 1), monotonic=False, fast=False):
"""Train on every k-th training hour (less data); score train and validation RMSE."""
out = {}
for k in strides:
rows = _rows(masks["train"], k, fast)
est = fit_model("HistGB", best_params, X.iloc[rows], y.iloc[rows], np.ones(len(rows), bool),
monotonic=monotonic)
out[len(rows)] = {"train_rmse": metrics(y.iloc[rows], est.predict(X.iloc[rows]))["rmse"],
"val_rmse": metrics(y[masks["val"]], est.predict(X[masks["val"]]))["rmse"]}
return pd.DataFrame(out).T.rename_axis("training hours")
def complexity_curve(best_params, X, y, masks, leaves=(7, 15, 31, 63, 127, 255), monotonic=False, fast=False):
"""Vary tree size with everything else fixed: does a more complex model help validation?"""
tr = _rows(masks["train"], config.SUBSAMPLE_STRIDE, False)
out = {}
for n in leaves:
est = fit_model("HistGB", {**best_params, "model__max_leaf_nodes": n}, X, y, masks["train"], fast,
monotonic=monotonic)
out[n] = {"train_rmse": metrics(y.iloc[tr], est.predict(X.iloc[tr]))["rmse"],
"val_rmse": metrics(y[masks["val"]], est.predict(X[masks["val"]]))["rmse"]}
return pd.DataFrame(out).T.rename_axis("max_leaf_nodes")
def conformal_margin(best_params, X, y, masks, fast=False, log=print) -> tuple[float, pd.DataFrame]:
"""Conformalized quantile regression (CQR) margin for the q10-q90 band.
Conformity score = how far y falls outside the band (negative if inside). Scores come only from
predictions on data the quantile models did NOT train on: out-of-fold TimeSeriesSplit folds of
the training years (skipping the first, tiny-training fold) plus the validation years (models
trained on the training years). The margin is the 80% conformal quantile of those scores and is
added to both sides of the band. Test and gate-holdout data are never used.
"""
params = {k.removeprefix("model__"): v for k, v in best_params.items()}
cst = monotonic_cst(X.columns) if config.USE_MONOTONIC else None
def band(fit_idx, pred_idx):
return [HistGradientBoostingRegressor(loss="quantile", quantile=q, early_stopping=False,
random_state=config.RANDOM_STATE, monotonic_cst=cst, **params)
.fit(X.iloc[fit_idx], y.iloc[fit_idx]).predict(X.iloc[pred_idx]) for q in config.QUANTILES]
train_idx = _rows(masks["train"], 1, fast)
folds = list(TimeSeriesSplit(n_splits=3 if fast else config.CV_SPLITS).split(train_idx))
pieces = [(train_idx[a], train_idx[b]) for a, b in folds[config.CONFORMAL_SKIP_FOLDS:]]
pieces.append((train_idx, _rows(masks["val"], 1, fast)))
scores, rows = [], {}
for fit_idx, pred_idx in pieces:
t0 = time.time()
lo, hi = band(fit_idx, pred_idx)
yt = y.iloc[pred_idx].to_numpy()
s = np.maximum(lo - yt, yt - hi)
scores.append(s)
idx = y.index[pred_idx]
label = f"{idx.min():%Y-%m} to {idx.max():%Y-%m}"
rows[label] = {"raw_coverage": float((s <= 0).mean()), "n": len(s)}
log(f"calibration block {label}: raw coverage {(s <= 0).mean():.1%} ({time.time() - t0:.0f} s)")
all_scores = np.concatenate(scores)
level = config.QUANTILES[1] - config.QUANTILES[0]
margin = float(np.quantile(all_scores, min(1.0, np.ceil((len(all_scores) + 1) * level) / len(all_scores))))
table = pd.DataFrame(rows).T
table["calibrated_coverage"] = [float((s <= margin).mean()) for s in scores]
return margin, table
# ── Production model ─────────────────────────────────────────────────────────
def fit_bundle(best_params, X, y, mask, fast=False, sample_weight=None, margin=0.0) -> dict:
"""Point + q10/q90 HistGB trained on the rows of `mask` (+ conformal margin), packaged for predict.py."""
params = {k.removeprefix("model__"): v for k, v in best_params.items()}
rows = _rows(mask, 1, fast)
Xf, yf = X.iloc[rows], y.iloc[rows]
w = None if sample_weight is None else sample_weight.iloc[rows].to_numpy()
rs = config.RANDOM_STATE
cst = monotonic_cst(X.columns) if config.USE_MONOTONIC else None
point = HistGradientBoostingRegressor(early_stopping=False, random_state=rs, monotonic_cst=cst,
**params).fit(Xf, yf, sample_weight=w)
q_lo, q_hi = (HistGradientBoostingRegressor(loss="quantile", quantile=q, early_stopping=False,
random_state=rs, monotonic_cst=cst, **params).fit(Xf, yf)
for q in config.QUANTILES)
return {
"point": point, "q_lo": q_lo, "q_hi": q_hi,
"features": list(X.columns), "params": params, "weighted": sample_weight is not None,
"monotonic": cst is not None, "interval_margin": float(margin),
"trained_until": str(y.index[rows].max()), "sklearn_version": sklearn.__version__,
"created_utc": datetime.now(timezone.utc).isoformat(timespec="seconds"),
}
def _holdout_mask(y):
return y.index >= y.index.max() - pd.DateOffset(months=config.GATE_HOLDOUT_MONTHS)
def gate_metrics(bundle, X, y) -> dict:
"""Score a bundle on the current gate holdout (the most recent 6 months of definitive labels)."""
from src.predict import predict_with_interval
hold = _holdout_mask(y)
band = predict_with_interval(bundle, X[hold])
gate = metrics(y[hold], band["kp_pred"])
gate["coverage_80"] = float(((y[hold] >= band["kp_lo"]) & (y[hold] <= band["kp_hi"])).mean())
gate["holdout"] = f"{y.index[hold].min()} β†’ {y.index[hold].max()}"
return gate
def fit_production(best_params, X, y, fast=False, sample_weight=None, margin=0.0):
"""Bundle trained up to the gate-holdout start (no refit-on-all); gate metrics on the holdout."""
bundle = fit_bundle(best_params, X, y, ~_holdout_mask(y), fast, sample_weight, margin)
return bundle, gate_metrics(bundle, X, y)
def gate_decision(candidate_gate: dict, X, y) -> tuple[bool, dict | None]:
"""Promote if candidate RMSE ≀ incumbent RMSE Γ— tolerance, both scored on THIS run's holdout.
The incumbent must not have been trained on the holdout; its stored trained_until is checked.
"""
if not config.CURRENT_MODEL.exists():
return True, None
from src.predict import load_bundle
incumbent = load_bundle()
hold_start = y.index[_holdout_mask(y)].min()
assert pd.Timestamp(incumbent["trained_until"]) < hold_start, "incumbent was trained on the holdout"
inc = gate_metrics(incumbent, X, y)
return candidate_gate["rmse"] <= inc["rmse"] * config.GATE_TOLERANCE, inc
def save_bundle(bundle, test_metrics: dict, gate_metrics: dict, promote: bool = True,
incumbent_gate: dict | None = None, keep_rejected: bool = True, note: str | None = None) -> str:
"""Versioned model file (+ current.joblib if promoted) and an entry in metrics_history.json.
keep_rejected=False (weekly retrain) logs a rejected candidate without writing its model file.
"""
config.MODELS_DIR.mkdir(parents=True, exist_ok=True)
version = f"tormenta_v{datetime.now(timezone.utc):%Y%m%d_%H%M}"
bundle["version"] = version
if promote or keep_rejected:
joblib.dump(bundle, config.MODELS_DIR / f"{version}.joblib", compress=3)
if promote:
joblib.dump(bundle, config.CURRENT_MODEL, compress=3)
history = json.loads(config.METRICS_HISTORY.read_text()) if config.METRICS_HISTORY.exists() else []
clean = lambda d: {k: (float(v) if isinstance(v, (int, float, np.floating)) else v) for k, v in d.items()}
history.append({"version": version, "created_utc": bundle["created_utc"],
"trained_until": bundle["trained_until"], "params": bundle["params"],
"weighted": bundle["weighted"], "n_features": len(bundle["features"]),
"test": clean(test_metrics), "gate": clean(gate_metrics),
"incumbent_gate": clean(incumbent_gate) if incumbent_gate else None,
"promoted": promote, **({"note": note} if note else {})})
config.METRICS_HISTORY.write_text(json.dumps(history, indent=2, default=str))
return version
def run(fast=False):
"""End-to-end run (same steps as the notebook)."""
X, y, persistence = load_xy()
masks = split_masks(X.index)
print(f"{X.shape[1]} features | train {masks['train'].sum():,} | val {masks['val'].sum():,} | "
f"test {masks['test'].sum():,} hours")
tuned = tune_models(X, y, masks, fast)
print(validation_table(tuned, X, y, masks, persistence).round(3))
test_table, _, _ = test_evaluation(tuned, X, y, masks, persistence, fast)
print(test_table.round(3))
best = tuned["HistGB"]["best_params"]
margin, _ = conformal_margin(best, X, y, masks, fast)
bundle, gate = fit_production(best, X, y, fast, margin=margin)
promote, incumbent = gate_decision(gate, X, y)
version = save_bundle(bundle, test_table.loc["HistGB"].to_dict(), gate, promote, incumbent)
print(f"Saved {version} (promoted={promote}); gate: {gate}; incumbent: {incumbent}")
if __name__ == "__main__":
ap = argparse.ArgumentParser()
ap.add_argument("--fast", action="store_true", help="tiny search + heavy subsampling (smoke test)")
run(ap.parse_args().fast)