Konuktor92's picture
Add full training code for model v1 (same files as PR #1)
7e6579d verified
Raw History Blame Contribute Delete
18.2 kB
"""Train and evaluate the expected-greenness model exactly as locked in configs/model_v1.yaml.
python scripts/30_train.py --git-commit <hash> [--tune-minutes 25]
Writes runs/v1/: metrics.json, expected_oof.parquet, model files for the Space, feature importance.
Nothing here looks at the test years before the final evaluation step.
"""
from __future__ import annotations
import argparse
import json
import time
from pathlib import Path
import numpy as np
import pandas as pd
import xgboost as xgb
from sklearn.linear_model import Ridge
from sklearn.model_selection import GroupKFold
CUBE = Path("/root/trendex-ml/data/cube")
RUN = Path("/root/trendex-ml/runs/v1")
Y0, NY = 2000, 26
YEARS = np.arange(2001, 2026)
MONTHS = np.arange(5, 10)
STATIC = ["elev_mean", "elev_std", "slope_mean", "steep_share", "aspect_sin", "aspect_cos", "tpi",
"grass_frac", "barren_frac", "lc_type1"]
WEATHER = ["precip_m", "precip_3m", "precip_winter", "lst_day_m", "lst_night_m", "lst_day_prev",
"snow_spring", "snow_m", "soil_m", "swe_apr"]
FEATURES = STATIC + ["month", "clim_ndvi"] + WEATHER
ABLATION = {"precip": ["precip_m", "precip_3m", "precip_winter"],
"+lst": ["precip_m", "precip_3m", "precip_winter", "lst_day_m", "lst_night_m", "lst_day_prev"],
"+snow": ["precip_m", "precip_3m", "precip_winter", "lst_day_m", "lst_night_m", "lst_day_prev",
"snow_spring", "snow_m"],
"+gldas (full)": WEATHER}
SEEDS = [0, 1, 2, 3, 4]
FIXED = {"tree_method": "hist", "device": "cuda", "objective": "reg:squarederror"}
def log(msg: str) -> None:
print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True)
# ----------------------------------------------------------------------------- data
def load():
cells = pd.read_parquet(CUBE / "cells.parquet")
sel = cells[cells["in_oblast"] & cells["mask_reason"].isna()].reset_index(drop=True)
cols = ["cell_id", "month", "ndvi", "ndvi_ok", "precip", "lst_day", "lst_night", "snow", "soil_moist", "swe", "year"]
m = pd.read_parquet(CUBE / "monthly", columns=cols)
m["year"] = m["year"].astype(int)
m = m[m["cell_id"].isin(sel["cell_id"])]
pos = pd.Series(np.arange(len(sel)), index=sel["cell_id"].values)
ci, yi, mi = pos[m["cell_id"].values].values, m["year"].values - Y0, m["month"].values - 1
def cube(col):
a = np.full((len(sel), NY, 12), np.nan, np.float32)
a[ci, yi, mi] = m[col].values.astype("float32")
return a
arr = {k: cube(k) for k in ["ndvi", "precip", "lst_day", "lst_night", "snow", "soil_moist", "swe"]}
ok = np.zeros((len(sel), NY, 12), bool)
ok[ci, yi, mi] = m["ndvi_ok"].values
arr["ndvi"][~ok] = np.nan
return sel, arr
def build_features(sel: pd.DataFrame, arr: dict) -> dict:
"""All features as arrays shaped (cells, 25 years, 5 months). Weather anomalies vs full-period mean."""
yy, mm = YEARS - Y0, MONTHS - 1
take = lambda a, dm=0: a[:, yy][:, :, mm - dm] # noqa: E731
anom_m = lambda x: x - np.nanmean(x, axis=1, keepdims=True) # noqa: E731 per cell-month
anom_y = lambda x: x - np.nanmean(x, axis=1, keepdims=True) # noqa: E731 per cell (annual)
rep = lambda x: np.repeat(x[:, :, None], len(MONTHS), axis=2) # noqa: E731
p, ld, ln, sn, so, sw = (arr[k] for k in ["precip", "lst_day", "lst_night", "snow", "soil_moist", "swe"])
with np.errstate(all="ignore"):
raw_m = {"precip_m": take(p), "precip_3m": take(p) + take(p, 1) + take(p, 2),
"lst_day_m": take(ld), "lst_night_m": take(ln), "lst_day_prev": take(ld, 1),
"snow_m": take(sn), "soil_m": take(so)}
raw_y = {"precip_winter": p[:, yy - 1, 9:12].sum(-1) + p[:, yy, 0:4].sum(-1),
"snow_spring": np.nanmean(sn[:, yy, 2:5], axis=-1), "swe_apr": sw[:, yy, 3]}
f = {k: anom_m(v) for k, v in raw_m.items()} | {k: rep(anom_y(v)) for k, v in raw_y.items()}
# weather climatology for inference: raw value - this mean = the anomaly the model sees
wc = [pd.DataFrame({"cell_id": sel["cell_id"].values, "month": int(mo)} |
{f"{k}_mean": np.nanmean(v[:, :, j], axis=1) for k, v in raw_m.items()} |
{f"{k}_mean": np.nanmean(v, axis=1) for k, v in raw_y.items()})
for j, mo in enumerate(MONTHS)]
pd.concat(wc, ignore_index=True).to_parquet(RUN / "weather_climatology.parquet", index=False)
shape = (len(sel), len(YEARS), len(MONTHS))
for k in STATIC:
f[k] = np.broadcast_to(sel[k].values.astype("float32")[:, None, None], shape)
f["month"] = np.broadcast_to(MONTHS.astype("float32")[None, None, :], shape)
f["ndvi"] = take(arr["ndvi"])
f["year"] = np.broadcast_to(YEARS[None, :, None], shape)
f["cell"] = np.broadcast_to(np.arange(len(sel))[:, None, None], shape)
sb = (sel["block_row"].values // 3) * 1000 + sel["block_col"].values // 3
f["superblock"] = np.broadcast_to(sb[:, None, None], shape)
return f
def clim_fold(ndvi: np.ndarray, train_years: np.ndarray) -> np.ndarray:
"""Cell-month NDVI climatology from training years only; leave-own-year-out for training rows."""
tm = np.isin(YEARS, train_years)
s = np.nansum(np.where(tm[None, :, None], ndvi, np.nan), axis=1, keepdims=True)
n = np.sum(np.isfinite(ndvi) & tm[None, :, None], axis=1, keepdims=True).astype("float32")
own = np.where(np.isfinite(ndvi) & tm[None, :, None], ndvi, 0.0)
own_n = (np.isfinite(ndvi) & tm[None, :, None]).astype("float32")
with np.errstate(all="ignore"):
return ((s - own) / (n - own_n)).astype("float32")
def table(f: dict, train_years: np.ndarray, features: list[str]) -> pd.DataFrame:
clim = clim_fold(f["ndvi"], train_years)
d = {k: f[k].ravel() for k in ["cell", "year", "superblock"]}
for k in features:
d[k] = (clim if k == "clim_ndvi" else f[k]).ravel()
d["clim_ndvi"] = clim.ravel()
d["target"] = (f["ndvi"] - clim).ravel()
d["ndvi"] = f["ndvi"].ravel()
df = pd.DataFrame(d)
return df[np.isfinite(df["target"])].reset_index(drop=True)
# ----------------------------------------------------------------------------- models
def fit_xgb(Xtr, ytr, params, n_estimators, seed, Xva=None, yva=None):
m = xgb.XGBRegressor(**FIXED, **params, n_estimators=n_estimators, random_state=seed,
early_stopping_rounds=50 if Xva is not None else None)
m.fit(Xtr, ytr, eval_set=[(Xva, yva)] if Xva is not None else None, verbose=False)
return m
class RidgeModel:
def __init__(self, alpha: float):
self.alpha = alpha
def fit(self, X: pd.DataFrame, y):
self.cols = list(X.columns)
self.med = X.median()
Z = X.fillna(self.med)
self.mu, self.sd = Z.mean(), Z.std().replace(0, 1)
self.r = Ridge(alpha=self.alpha).fit(((Z - self.mu) / self.sd).values, y)
return self
def predict(self, X: pd.DataFrame):
Z = X[self.cols].fillna(self.med)
return self.r.predict(((Z - self.mu) / self.sd).values)
def to_json(self) -> dict:
return {"kind": "ridge", "alpha": self.alpha, "features": self.cols,
"fill": self.med.to_dict(), "mean": self.mu.to_dict(), "std": self.sd.to_dict(),
"coef": self.r.coef_.tolist(), "intercept": float(self.r.intercept_)}
def split(f, train_years, test_years, features):
t = table(f, train_years, features)
tr = t[np.isin(t["year"], train_years)].reset_index(drop=True)
te = t[np.isin(t["year"], test_years)].reset_index(drop=True)
return tr, te
# ----------------------------------------------------------------------------- metrics
def boot_skill(sse_a: np.ndarray, sse_b: np.ndarray, rng, n=2000) -> dict:
"""Skill 1 - SSE_a/SSE_b with a bootstrap over independent units (years or superblocks)."""
point = 1 - sse_a.sum() / sse_b.sum()
idx = rng.integers(0, len(sse_a), size=(n, len(sse_a)))
dist = 1 - sse_a[idx].sum(1) / sse_b[idx].sum(1)
return {"point": float(point), "ci_lo": float(np.quantile(dist, 0.025)), "ci_hi": float(np.quantile(dist, 0.975)),
"units": int(len(sse_a))}
def tercile_bacc(y_true, y_pred, cuts) -> float:
t = np.digitize(y_true, cuts)
p = np.digitize(y_pred, cuts)
return float(np.mean([np.mean(p[t == k] == k) for k in range(3) if np.any(t == k)]))
def score(te: pd.DataFrame, preds: dict, unit: str, cuts, rng) -> dict:
y = te["target"].values
out = {"n_rows": int(len(y))}
sse = {}
for name, p in preds.items():
err = y - p
out[name] = {"rmse": float(np.sqrt(np.mean(err ** 2))), "mae": float(np.mean(np.abs(err))),
"anomaly_corr": float(np.corrcoef(y, p)[0, 1]) if np.std(p) > 0 else None,
"tercile_bacc": tercile_bacc(y, p, cuts)}
sse[name] = pd.Series(err ** 2).groupby(te[unit].values).sum().values
out["skill_xgb_vs_ridge"] = boot_skill(sse["xgb"], sse["ridge"], rng)
out["skill_xgb_vs_clim"] = boot_skill(sse["xgb"], sse["clim"], rng)
out["skill_ridge_vs_clim"] = boot_skill(sse["ridge"], sse["clim"], rng)
return out
# ----------------------------------------------------------------------------- main
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("--git-commit", required=True)
ap.add_argument("--tune-minutes", type=float, default=25)
ap.add_argument("--tune-trials", type=int, default=40)
a = ap.parse_args()
RUN.mkdir(parents=True, exist_ok=True)
rng = np.random.default_rng(0)
metrics: dict = {"protocol": "configs/model_v1.yaml", "git_commit": a.git_commit, "features": FEATURES}
sel, arr = load()
f = build_features(sel, arr)
log(f"cells {len(sel)}, feature arrays {f['ndvi'].shape}")
metrics["data"] = {"cells": len(sel), "years": [int(YEARS[0]), int(YEARS[-1])], "months": MONTHS.tolist()}
# --- 1. tuning on the inner split only (2001-2015 -> 2016-2019)
inner_tr, inner_va = split(f, np.arange(2001, 2016), np.arange(2016, 2020), FEATURES)
Xi, yi_, Xv, yv = inner_tr[FEATURES], inner_tr["target"], inner_va[FEATURES], inner_va["target"]
alphas = [0.1, 1, 10, 100, 1000]
ridge_rmse = {al: float(np.sqrt(np.mean((yv - RidgeModel(al).fit(Xi, yi_).predict(Xv)) ** 2))) for al in alphas}
alpha = min(ridge_rmse, key=ridge_rmse.get)
log(f"ridge alpha {alpha} inner rmse {ridge_rmse[alpha]:.4f}")
import optuna
optuna.logging.set_verbosity(optuna.logging.WARNING)
def objective(trial):
params = {"max_depth": trial.suggest_int("max_depth", 4, 10),
"learning_rate": trial.suggest_float("learning_rate", 0.02, 0.2, log=True),
"min_child_weight": trial.suggest_float("min_child_weight", 1, 200, log=True),
"subsample": trial.suggest_float("subsample", 0.5, 1.0),
"colsample_bytree": trial.suggest_float("colsample_bytree", 0.5, 1.0),
"reg_lambda": trial.suggest_float("reg_lambda", 1e-3, 30, log=True),
"reg_alpha": trial.suggest_float("reg_alpha", 1e-3, 10, log=True)}
m = fit_xgb(Xi, yi_, params, 3000, 0, Xv, yv)
trial.set_user_attr("n_estimators", int(m.best_iteration) + 1)
return float(np.sqrt(np.mean((yv - m.predict(Xv)) ** 2)))
study = optuna.create_study(direction="minimize", sampler=optuna.samplers.TPESampler(seed=0))
study.optimize(objective, n_trials=a.tune_trials, timeout=a.tune_minutes * 60)
best, n_est = study.best_params, study.best_trial.user_attrs["n_estimators"]
log(f"tuning: {len(study.trials)} trials, best inner rmse {study.best_value:.4f}, n_estimators {n_est}")
metrics["tuning"] = {"trials": len(study.trials), "best_params": best, "n_estimators": n_est,
"inner_rmse_xgb": study.best_value, "ridge_alpha": alpha, "inner_rmse_ridge": ridge_rmse}
cuts = np.quantile(yi_, [1 / 3, 2 / 3])
# conformal residual quantiles per month from the inner split (model never saw 2016-2019)
m_inner = fit_xgb(Xi, yi_, best, n_est, 0)
res_inner = yv.values - m_inner.predict(Xv)
conf_inner = {int(mo): [float(np.quantile(res_inner[inner_va["month"] == mo], q)) for q in (0.1, 0.9)]
for mo in MONTHS}
# --- 2. temporal test (train 2001-2019, test 2020-2025), 5 seeds
tr, te = split(f, np.arange(2001, 2020), np.arange(2020, 2026), FEATURES)
ridge = RidgeModel(alpha).fit(tr[FEATURES], tr["target"])
p_ridge = ridge.predict(te[FEATURES])
seed_scores, p_xgb_seeds = [], []
for s in SEEDS:
m = fit_xgb(tr[FEATURES], tr["target"], best, n_est, s)
p = m.predict(te[FEATURES])
p_xgb_seeds.append(p)
seed_scores.append(float(1 - np.mean((te["target"] - p) ** 2) / np.mean((te["target"] - p_ridge) ** 2)))
p_xgb = np.mean(p_xgb_seeds, axis=0)
temporal = score(te, {"xgb": p_xgb, "ridge": p_ridge, "clim": np.zeros(len(te))}, "year", cuts, rng)
temporal["seed_skill_vs_ridge"] = {"values": seed_scores, "mean": float(np.mean(seed_scores)),
"sd": float(np.std(seed_scores))}
lo, hi = np.array([conf_inner[int(mo)] for mo in te["month"]]).T
res = te["target"].values - p_xgb
temporal["conformal_coverage_p10_p90"] = float(np.mean((res >= lo) & (res <= hi)))
metrics["temporal_test"] = temporal
log(f"temporal: skill vs ridge {temporal['skill_xgb_vs_ridge']}")
# ablation on the temporal split (one seed)
abl = {}
for name, wcols in ABLATION.items():
cols = STATIC + ["month", "clim_ndvi"] + wcols
m = fit_xgb(tr[cols], tr["target"], best, n_est, 0)
abl[name] = float(1 - np.mean((te["target"] - m.predict(te[cols])) ** 2) / np.mean(te["target"] ** 2))
metrics["ablation_skill_vs_clim_temporal"] = abl
log(f"ablation {abl}")
# --- 3. spatial test: GroupKFold over 33x33-cell superblocks, all years
full = table(f, YEARS, FEATURES)
preds = {"xgb": np.zeros(len(full)), "ridge": np.zeros(len(full))}
for k, (itr, ite) in enumerate(GroupKFold(5).split(full, groups=full["superblock"])):
Xtr, ytr, Xte = full.loc[itr, FEATURES], full.loc[itr, "target"], full.loc[ite, FEATURES]
preds["xgb"][ite] = fit_xgb(Xtr, ytr, best, n_est, 0).predict(Xte)
preds["ridge"][ite] = RidgeModel(alpha).fit(Xtr, ytr).predict(Xte)
preds["clim"] = np.zeros(len(full))
spatial = score(full, preds, "superblock", cuts, rng)
metrics["spatial_test"] = spatial
log(f"spatial: skill vs ridge {spatial['skill_xgb_vs_ridge']}")
# --- 4. gate (pre-registered)
passed = temporal["skill_xgb_vs_ridge"]["ci_lo"] > 0 and spatial["skill_xgb_vs_ridge"]["ci_lo"] > 0
chosen = "xgb" if passed else "ridge"
metrics["gate"] = {"rule": "CI lower bound of skill_xgb_vs_ridge > 0 on temporal AND spatial tests",
"passed": bool(passed), "shipped_model": chosen}
log(f"GATE passed={passed} -> shipping {chosen}")
# --- 5. cross-fitted out-of-fold predictions (5 contiguous year blocks) with the chosen model
blocks = [np.arange(s, s + 5) for s in (2001, 2006, 2011, 2016, 2021)]
oof = []
for blk in blocks:
trn = np.setdiff1d(YEARS, blk)
trb, teb = split(f, trn, blk, FEATURES)
model = fit_xgb(trb[FEATURES], trb["target"], best, n_est, 0) if chosen == "xgb" else \
RidgeModel(alpha).fit(trb[FEATURES], trb["target"])
teb = teb.assign(pred_anom=model.predict(teb[FEATURES]))
oof.append(teb[["cell", "year", "month", "ndvi", "clim_ndvi", "target", "pred_anom"]])
oof = pd.concat(oof, ignore_index=True)
res_all = oof["target"] - oof["pred_anom"]
conf_prod = {int(mo): [float(np.quantile(res_all[oof["month"] == mo], q)) for q in (0.1, 0.9)] for mo in MONTHS}
lo = oof["month"].map(lambda mo: conf_prod[int(mo)][0])
hi = oof["month"].map(lambda mo: conf_prod[int(mo)][1])
oof["cell_id"] = sel["cell_id"].values[oof["cell"].values]
oof["expected"] = oof["clim_ndvi"] + oof["pred_anom"]
oof["p10"], oof["p90"] = oof["expected"] + lo, oof["expected"] + hi
oof["residual"] = res_all
oof[["cell_id", "year", "month", "ndvi", "expected", "p10", "p90", "residual"]].rename(
columns={"ndvi": "observed"}).to_parquet(RUN / "expected_oof.parquet", index=False)
metrics["conformal_p10_p90_by_month"] = conf_prod
log(f"oof rows {len(oof)}")
# --- 6. production model: refit on all years; climatology tables for inference
full_all = full
if chosen == "xgb":
prod = fit_xgb(full_all[FEATURES], full_all["target"], best, n_est, 0)
prod.save_model(RUN / "model_a.ubj")
contrib = prod.get_booster().predict(xgb.DMatrix(full_all[FEATURES].sample(200_000, random_state=0)),
pred_contribs=True)
imp = pd.Series(np.abs(contrib[:, :-1]).mean(0), index=FEATURES).sort_values(ascending=False)
else:
prod = RidgeModel(alpha).fit(full_all[FEATURES], full_all["target"])
(RUN / "ridge_a.json").write_text(json.dumps(prod.to_json()))
imp = pd.Series(np.abs(prod.r.coef_), index=FEATURES).sort_values(ascending=False)
metrics["feature_importance_mean_abs"] = imp.round(6).to_dict()
yy = YEARS - Y0
clim_tab = []
for k, mo in enumerate(MONTHS):
rec = {"cell_id": sel["cell_id"].values, "month": int(mo),
"clim_ndvi": np.nanmean(arr["ndvi"][:, yy, mo - 1], axis=1)}
clim_tab.append(pd.DataFrame(rec))
pd.concat(clim_tab).to_parquet(RUN / "climatology.parquet", index=False)
(RUN / "feature_schema.json").write_text(json.dumps({
"features": FEATURES, "target": "ndvi_anomaly = ndvi - clim_ndvi", "months": MONTHS.tolist(),
"weather_anomaly_baseline": "cell-month mean 2001-2025 (see climatology.parquet / weather_climatology.parquet)",
"shipped_model": chosen}, indent=2))
(RUN / "conformal.json").write_text(json.dumps(conf_prod, indent=2))
(RUN / "metrics.json").write_text(json.dumps(metrics, indent=2, default=float))
log("TRAIN_DONE")
if __name__ == "__main__":
main()