Download training_code/scripts/30_train.py from TrendeX90/expected-greenness: direct link, hf CLI and curl.
- Browser
- Download file 18.2 kB
-
https://huggingface.co/TrendeX90/expected-greenness/resolve/refs%2Fpr%2F1/training_code/scripts/30_train.py
- Command line
-
hf download hf://TrendeX90/expected-greenness@refs/pr/1/training_code/scripts/30_train.py
-
curl -L -o 30_train.py https://huggingface.co/TrendeX90/expected-greenness/resolve/refs%2Fpr%2F1/training_code/scripts/30_train.py
18.2 kB
| """Train and evaluate the expected-greenness model exactly as locked in configs/model_v1.yaml. | |
| python scripts/30_train.py --git-commit <hash> [--tune-minutes 25] | |
| Writes runs/v1/: metrics.json, expected_oof.parquet, model files for the Space, feature importance. | |
| Nothing here looks at the test years before the final evaluation step. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import time | |
| from pathlib import Path | |
| import numpy as np | |
| import pandas as pd | |
| import xgboost as xgb | |
| from sklearn.linear_model import Ridge | |
| from sklearn.model_selection import GroupKFold | |
| CUBE = Path("/root/trendex-ml/data/cube") | |
| RUN = Path("/root/trendex-ml/runs/v1") | |
| Y0, NY = 2000, 26 | |
| YEARS = np.arange(2001, 2026) | |
| MONTHS = np.arange(5, 10) | |
| STATIC = ["elev_mean", "elev_std", "slope_mean", "steep_share", "aspect_sin", "aspect_cos", "tpi", | |
| "grass_frac", "barren_frac", "lc_type1"] | |
| WEATHER = ["precip_m", "precip_3m", "precip_winter", "lst_day_m", "lst_night_m", "lst_day_prev", | |
| "snow_spring", "snow_m", "soil_m", "swe_apr"] | |
| FEATURES = STATIC + ["month", "clim_ndvi"] + WEATHER | |
| ABLATION = {"precip": ["precip_m", "precip_3m", "precip_winter"], | |
| "+lst": ["precip_m", "precip_3m", "precip_winter", "lst_day_m", "lst_night_m", "lst_day_prev"], | |
| "+snow": ["precip_m", "precip_3m", "precip_winter", "lst_day_m", "lst_night_m", "lst_day_prev", | |
| "snow_spring", "snow_m"], | |
| "+gldas (full)": WEATHER} | |
| SEEDS = [0, 1, 2, 3, 4] | |
| FIXED = {"tree_method": "hist", "device": "cuda", "objective": "reg:squarederror"} | |
| def log(msg: str) -> None: | |
| print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True) | |
| # ----------------------------------------------------------------------------- data | |
| def load(): | |
| cells = pd.read_parquet(CUBE / "cells.parquet") | |
| sel = cells[cells["in_oblast"] & cells["mask_reason"].isna()].reset_index(drop=True) | |
| cols = ["cell_id", "month", "ndvi", "ndvi_ok", "precip", "lst_day", "lst_night", "snow", "soil_moist", "swe", "year"] | |
| m = pd.read_parquet(CUBE / "monthly", columns=cols) | |
| m["year"] = m["year"].astype(int) | |
| m = m[m["cell_id"].isin(sel["cell_id"])] | |
| pos = pd.Series(np.arange(len(sel)), index=sel["cell_id"].values) | |
| ci, yi, mi = pos[m["cell_id"].values].values, m["year"].values - Y0, m["month"].values - 1 | |
| def cube(col): | |
| a = np.full((len(sel), NY, 12), np.nan, np.float32) | |
| a[ci, yi, mi] = m[col].values.astype("float32") | |
| return a | |
| arr = {k: cube(k) for k in ["ndvi", "precip", "lst_day", "lst_night", "snow", "soil_moist", "swe"]} | |
| ok = np.zeros((len(sel), NY, 12), bool) | |
| ok[ci, yi, mi] = m["ndvi_ok"].values | |
| arr["ndvi"][~ok] = np.nan | |
| return sel, arr | |
| def build_features(sel: pd.DataFrame, arr: dict) -> dict: | |
| """All features as arrays shaped (cells, 25 years, 5 months). Weather anomalies vs full-period mean.""" | |
| yy, mm = YEARS - Y0, MONTHS - 1 | |
| take = lambda a, dm=0: a[:, yy][:, :, mm - dm] # noqa: E731 | |
| anom_m = lambda x: x - np.nanmean(x, axis=1, keepdims=True) # noqa: E731 per cell-month | |
| anom_y = lambda x: x - np.nanmean(x, axis=1, keepdims=True) # noqa: E731 per cell (annual) | |
| rep = lambda x: np.repeat(x[:, :, None], len(MONTHS), axis=2) # noqa: E731 | |
| p, ld, ln, sn, so, sw = (arr[k] for k in ["precip", "lst_day", "lst_night", "snow", "soil_moist", "swe"]) | |
| with np.errstate(all="ignore"): | |
| raw_m = {"precip_m": take(p), "precip_3m": take(p) + take(p, 1) + take(p, 2), | |
| "lst_day_m": take(ld), "lst_night_m": take(ln), "lst_day_prev": take(ld, 1), | |
| "snow_m": take(sn), "soil_m": take(so)} | |
| raw_y = {"precip_winter": p[:, yy - 1, 9:12].sum(-1) + p[:, yy, 0:4].sum(-1), | |
| "snow_spring": np.nanmean(sn[:, yy, 2:5], axis=-1), "swe_apr": sw[:, yy, 3]} | |
| f = {k: anom_m(v) for k, v in raw_m.items()} | {k: rep(anom_y(v)) for k, v in raw_y.items()} | |
| # weather climatology for inference: raw value - this mean = the anomaly the model sees | |
| wc = [pd.DataFrame({"cell_id": sel["cell_id"].values, "month": int(mo)} | | |
| {f"{k}_mean": np.nanmean(v[:, :, j], axis=1) for k, v in raw_m.items()} | | |
| {f"{k}_mean": np.nanmean(v, axis=1) for k, v in raw_y.items()}) | |
| for j, mo in enumerate(MONTHS)] | |
| pd.concat(wc, ignore_index=True).to_parquet(RUN / "weather_climatology.parquet", index=False) | |
| shape = (len(sel), len(YEARS), len(MONTHS)) | |
| for k in STATIC: | |
| f[k] = np.broadcast_to(sel[k].values.astype("float32")[:, None, None], shape) | |
| f["month"] = np.broadcast_to(MONTHS.astype("float32")[None, None, :], shape) | |
| f["ndvi"] = take(arr["ndvi"]) | |
| f["year"] = np.broadcast_to(YEARS[None, :, None], shape) | |
| f["cell"] = np.broadcast_to(np.arange(len(sel))[:, None, None], shape) | |
| sb = (sel["block_row"].values // 3) * 1000 + sel["block_col"].values // 3 | |
| f["superblock"] = np.broadcast_to(sb[:, None, None], shape) | |
| return f | |
| def clim_fold(ndvi: np.ndarray, train_years: np.ndarray) -> np.ndarray: | |
| """Cell-month NDVI climatology from training years only; leave-own-year-out for training rows.""" | |
| tm = np.isin(YEARS, train_years) | |
| s = np.nansum(np.where(tm[None, :, None], ndvi, np.nan), axis=1, keepdims=True) | |
| n = np.sum(np.isfinite(ndvi) & tm[None, :, None], axis=1, keepdims=True).astype("float32") | |
| own = np.where(np.isfinite(ndvi) & tm[None, :, None], ndvi, 0.0) | |
| own_n = (np.isfinite(ndvi) & tm[None, :, None]).astype("float32") | |
| with np.errstate(all="ignore"): | |
| return ((s - own) / (n - own_n)).astype("float32") | |
| def table(f: dict, train_years: np.ndarray, features: list[str]) -> pd.DataFrame: | |
| clim = clim_fold(f["ndvi"], train_years) | |
| d = {k: f[k].ravel() for k in ["cell", "year", "superblock"]} | |
| for k in features: | |
| d[k] = (clim if k == "clim_ndvi" else f[k]).ravel() | |
| d["clim_ndvi"] = clim.ravel() | |
| d["target"] = (f["ndvi"] - clim).ravel() | |
| d["ndvi"] = f["ndvi"].ravel() | |
| df = pd.DataFrame(d) | |
| return df[np.isfinite(df["target"])].reset_index(drop=True) | |
| # ----------------------------------------------------------------------------- models | |
| def fit_xgb(Xtr, ytr, params, n_estimators, seed, Xva=None, yva=None): | |
| m = xgb.XGBRegressor(**FIXED, **params, n_estimators=n_estimators, random_state=seed, | |
| early_stopping_rounds=50 if Xva is not None else None) | |
| m.fit(Xtr, ytr, eval_set=[(Xva, yva)] if Xva is not None else None, verbose=False) | |
| return m | |
| class RidgeModel: | |
| def __init__(self, alpha: float): | |
| self.alpha = alpha | |
| def fit(self, X: pd.DataFrame, y): | |
| self.cols = list(X.columns) | |
| self.med = X.median() | |
| Z = X.fillna(self.med) | |
| self.mu, self.sd = Z.mean(), Z.std().replace(0, 1) | |
| self.r = Ridge(alpha=self.alpha).fit(((Z - self.mu) / self.sd).values, y) | |
| return self | |
| def predict(self, X: pd.DataFrame): | |
| Z = X[self.cols].fillna(self.med) | |
| return self.r.predict(((Z - self.mu) / self.sd).values) | |
| def to_json(self) -> dict: | |
| return {"kind": "ridge", "alpha": self.alpha, "features": self.cols, | |
| "fill": self.med.to_dict(), "mean": self.mu.to_dict(), "std": self.sd.to_dict(), | |
| "coef": self.r.coef_.tolist(), "intercept": float(self.r.intercept_)} | |
| def split(f, train_years, test_years, features): | |
| t = table(f, train_years, features) | |
| tr = t[np.isin(t["year"], train_years)].reset_index(drop=True) | |
| te = t[np.isin(t["year"], test_years)].reset_index(drop=True) | |
| return tr, te | |
| # ----------------------------------------------------------------------------- metrics | |
| def boot_skill(sse_a: np.ndarray, sse_b: np.ndarray, rng, n=2000) -> dict: | |
| """Skill 1 - SSE_a/SSE_b with a bootstrap over independent units (years or superblocks).""" | |
| point = 1 - sse_a.sum() / sse_b.sum() | |
| idx = rng.integers(0, len(sse_a), size=(n, len(sse_a))) | |
| dist = 1 - sse_a[idx].sum(1) / sse_b[idx].sum(1) | |
| return {"point": float(point), "ci_lo": float(np.quantile(dist, 0.025)), "ci_hi": float(np.quantile(dist, 0.975)), | |
| "units": int(len(sse_a))} | |
| def tercile_bacc(y_true, y_pred, cuts) -> float: | |
| t = np.digitize(y_true, cuts) | |
| p = np.digitize(y_pred, cuts) | |
| return float(np.mean([np.mean(p[t == k] == k) for k in range(3) if np.any(t == k)])) | |
| def score(te: pd.DataFrame, preds: dict, unit: str, cuts, rng) -> dict: | |
| y = te["target"].values | |
| out = {"n_rows": int(len(y))} | |
| sse = {} | |
| for name, p in preds.items(): | |
| err = y - p | |
| out[name] = {"rmse": float(np.sqrt(np.mean(err ** 2))), "mae": float(np.mean(np.abs(err))), | |
| "anomaly_corr": float(np.corrcoef(y, p)[0, 1]) if np.std(p) > 0 else None, | |
| "tercile_bacc": tercile_bacc(y, p, cuts)} | |
| sse[name] = pd.Series(err ** 2).groupby(te[unit].values).sum().values | |
| out["skill_xgb_vs_ridge"] = boot_skill(sse["xgb"], sse["ridge"], rng) | |
| out["skill_xgb_vs_clim"] = boot_skill(sse["xgb"], sse["clim"], rng) | |
| out["skill_ridge_vs_clim"] = boot_skill(sse["ridge"], sse["clim"], rng) | |
| return out | |
| # ----------------------------------------------------------------------------- main | |
| def main() -> None: | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--git-commit", required=True) | |
| ap.add_argument("--tune-minutes", type=float, default=25) | |
| ap.add_argument("--tune-trials", type=int, default=40) | |
| a = ap.parse_args() | |
| RUN.mkdir(parents=True, exist_ok=True) | |
| rng = np.random.default_rng(0) | |
| metrics: dict = {"protocol": "configs/model_v1.yaml", "git_commit": a.git_commit, "features": FEATURES} | |
| sel, arr = load() | |
| f = build_features(sel, arr) | |
| log(f"cells {len(sel)}, feature arrays {f['ndvi'].shape}") | |
| metrics["data"] = {"cells": len(sel), "years": [int(YEARS[0]), int(YEARS[-1])], "months": MONTHS.tolist()} | |
| # --- 1. tuning on the inner split only (2001-2015 -> 2016-2019) | |
| inner_tr, inner_va = split(f, np.arange(2001, 2016), np.arange(2016, 2020), FEATURES) | |
| Xi, yi_, Xv, yv = inner_tr[FEATURES], inner_tr["target"], inner_va[FEATURES], inner_va["target"] | |
| alphas = [0.1, 1, 10, 100, 1000] | |
| ridge_rmse = {al: float(np.sqrt(np.mean((yv - RidgeModel(al).fit(Xi, yi_).predict(Xv)) ** 2))) for al in alphas} | |
| alpha = min(ridge_rmse, key=ridge_rmse.get) | |
| log(f"ridge alpha {alpha} inner rmse {ridge_rmse[alpha]:.4f}") | |
| import optuna | |
| optuna.logging.set_verbosity(optuna.logging.WARNING) | |
| def objective(trial): | |
| params = {"max_depth": trial.suggest_int("max_depth", 4, 10), | |
| "learning_rate": trial.suggest_float("learning_rate", 0.02, 0.2, log=True), | |
| "min_child_weight": trial.suggest_float("min_child_weight", 1, 200, log=True), | |
| "subsample": trial.suggest_float("subsample", 0.5, 1.0), | |
| "colsample_bytree": trial.suggest_float("colsample_bytree", 0.5, 1.0), | |
| "reg_lambda": trial.suggest_float("reg_lambda", 1e-3, 30, log=True), | |
| "reg_alpha": trial.suggest_float("reg_alpha", 1e-3, 10, log=True)} | |
| m = fit_xgb(Xi, yi_, params, 3000, 0, Xv, yv) | |
| trial.set_user_attr("n_estimators", int(m.best_iteration) + 1) | |
| return float(np.sqrt(np.mean((yv - m.predict(Xv)) ** 2))) | |
| study = optuna.create_study(direction="minimize", sampler=optuna.samplers.TPESampler(seed=0)) | |
| study.optimize(objective, n_trials=a.tune_trials, timeout=a.tune_minutes * 60) | |
| best, n_est = study.best_params, study.best_trial.user_attrs["n_estimators"] | |
| log(f"tuning: {len(study.trials)} trials, best inner rmse {study.best_value:.4f}, n_estimators {n_est}") | |
| metrics["tuning"] = {"trials": len(study.trials), "best_params": best, "n_estimators": n_est, | |
| "inner_rmse_xgb": study.best_value, "ridge_alpha": alpha, "inner_rmse_ridge": ridge_rmse} | |
| cuts = np.quantile(yi_, [1 / 3, 2 / 3]) | |
| # conformal residual quantiles per month from the inner split (model never saw 2016-2019) | |
| m_inner = fit_xgb(Xi, yi_, best, n_est, 0) | |
| res_inner = yv.values - m_inner.predict(Xv) | |
| conf_inner = {int(mo): [float(np.quantile(res_inner[inner_va["month"] == mo], q)) for q in (0.1, 0.9)] | |
| for mo in MONTHS} | |
| # --- 2. temporal test (train 2001-2019, test 2020-2025), 5 seeds | |
| tr, te = split(f, np.arange(2001, 2020), np.arange(2020, 2026), FEATURES) | |
| ridge = RidgeModel(alpha).fit(tr[FEATURES], tr["target"]) | |
| p_ridge = ridge.predict(te[FEATURES]) | |
| seed_scores, p_xgb_seeds = [], [] | |
| for s in SEEDS: | |
| m = fit_xgb(tr[FEATURES], tr["target"], best, n_est, s) | |
| p = m.predict(te[FEATURES]) | |
| p_xgb_seeds.append(p) | |
| seed_scores.append(float(1 - np.mean((te["target"] - p) ** 2) / np.mean((te["target"] - p_ridge) ** 2))) | |
| p_xgb = np.mean(p_xgb_seeds, axis=0) | |
| temporal = score(te, {"xgb": p_xgb, "ridge": p_ridge, "clim": np.zeros(len(te))}, "year", cuts, rng) | |
| temporal["seed_skill_vs_ridge"] = {"values": seed_scores, "mean": float(np.mean(seed_scores)), | |
| "sd": float(np.std(seed_scores))} | |
| lo, hi = np.array([conf_inner[int(mo)] for mo in te["month"]]).T | |
| res = te["target"].values - p_xgb | |
| temporal["conformal_coverage_p10_p90"] = float(np.mean((res >= lo) & (res <= hi))) | |
| metrics["temporal_test"] = temporal | |
| log(f"temporal: skill vs ridge {temporal['skill_xgb_vs_ridge']}") | |
| # ablation on the temporal split (one seed) | |
| abl = {} | |
| for name, wcols in ABLATION.items(): | |
| cols = STATIC + ["month", "clim_ndvi"] + wcols | |
| m = fit_xgb(tr[cols], tr["target"], best, n_est, 0) | |
| abl[name] = float(1 - np.mean((te["target"] - m.predict(te[cols])) ** 2) / np.mean(te["target"] ** 2)) | |
| metrics["ablation_skill_vs_clim_temporal"] = abl | |
| log(f"ablation {abl}") | |
| # --- 3. spatial test: GroupKFold over 33x33-cell superblocks, all years | |
| full = table(f, YEARS, FEATURES) | |
| preds = {"xgb": np.zeros(len(full)), "ridge": np.zeros(len(full))} | |
| for k, (itr, ite) in enumerate(GroupKFold(5).split(full, groups=full["superblock"])): | |
| Xtr, ytr, Xte = full.loc[itr, FEATURES], full.loc[itr, "target"], full.loc[ite, FEATURES] | |
| preds["xgb"][ite] = fit_xgb(Xtr, ytr, best, n_est, 0).predict(Xte) | |
| preds["ridge"][ite] = RidgeModel(alpha).fit(Xtr, ytr).predict(Xte) | |
| preds["clim"] = np.zeros(len(full)) | |
| spatial = score(full, preds, "superblock", cuts, rng) | |
| metrics["spatial_test"] = spatial | |
| log(f"spatial: skill vs ridge {spatial['skill_xgb_vs_ridge']}") | |
| # --- 4. gate (pre-registered) | |
| passed = temporal["skill_xgb_vs_ridge"]["ci_lo"] > 0 and spatial["skill_xgb_vs_ridge"]["ci_lo"] > 0 | |
| chosen = "xgb" if passed else "ridge" | |
| metrics["gate"] = {"rule": "CI lower bound of skill_xgb_vs_ridge > 0 on temporal AND spatial tests", | |
| "passed": bool(passed), "shipped_model": chosen} | |
| log(f"GATE passed={passed} -> shipping {chosen}") | |
| # --- 5. cross-fitted out-of-fold predictions (5 contiguous year blocks) with the chosen model | |
| blocks = [np.arange(s, s + 5) for s in (2001, 2006, 2011, 2016, 2021)] | |
| oof = [] | |
| for blk in blocks: | |
| trn = np.setdiff1d(YEARS, blk) | |
| trb, teb = split(f, trn, blk, FEATURES) | |
| model = fit_xgb(trb[FEATURES], trb["target"], best, n_est, 0) if chosen == "xgb" else \ | |
| RidgeModel(alpha).fit(trb[FEATURES], trb["target"]) | |
| teb = teb.assign(pred_anom=model.predict(teb[FEATURES])) | |
| oof.append(teb[["cell", "year", "month", "ndvi", "clim_ndvi", "target", "pred_anom"]]) | |
| oof = pd.concat(oof, ignore_index=True) | |
| res_all = oof["target"] - oof["pred_anom"] | |
| conf_prod = {int(mo): [float(np.quantile(res_all[oof["month"] == mo], q)) for q in (0.1, 0.9)] for mo in MONTHS} | |
| lo = oof["month"].map(lambda mo: conf_prod[int(mo)][0]) | |
| hi = oof["month"].map(lambda mo: conf_prod[int(mo)][1]) | |
| oof["cell_id"] = sel["cell_id"].values[oof["cell"].values] | |
| oof["expected"] = oof["clim_ndvi"] + oof["pred_anom"] | |
| oof["p10"], oof["p90"] = oof["expected"] + lo, oof["expected"] + hi | |
| oof["residual"] = res_all | |
| oof[["cell_id", "year", "month", "ndvi", "expected", "p10", "p90", "residual"]].rename( | |
| columns={"ndvi": "observed"}).to_parquet(RUN / "expected_oof.parquet", index=False) | |
| metrics["conformal_p10_p90_by_month"] = conf_prod | |
| log(f"oof rows {len(oof)}") | |
| # --- 6. production model: refit on all years; climatology tables for inference | |
| full_all = full | |
| if chosen == "xgb": | |
| prod = fit_xgb(full_all[FEATURES], full_all["target"], best, n_est, 0) | |
| prod.save_model(RUN / "model_a.ubj") | |
| contrib = prod.get_booster().predict(xgb.DMatrix(full_all[FEATURES].sample(200_000, random_state=0)), | |
| pred_contribs=True) | |
| imp = pd.Series(np.abs(contrib[:, :-1]).mean(0), index=FEATURES).sort_values(ascending=False) | |
| else: | |
| prod = RidgeModel(alpha).fit(full_all[FEATURES], full_all["target"]) | |
| (RUN / "ridge_a.json").write_text(json.dumps(prod.to_json())) | |
| imp = pd.Series(np.abs(prod.r.coef_), index=FEATURES).sort_values(ascending=False) | |
| metrics["feature_importance_mean_abs"] = imp.round(6).to_dict() | |
| yy = YEARS - Y0 | |
| clim_tab = [] | |
| for k, mo in enumerate(MONTHS): | |
| rec = {"cell_id": sel["cell_id"].values, "month": int(mo), | |
| "clim_ndvi": np.nanmean(arr["ndvi"][:, yy, mo - 1], axis=1)} | |
| clim_tab.append(pd.DataFrame(rec)) | |
| pd.concat(clim_tab).to_parquet(RUN / "climatology.parquet", index=False) | |
| (RUN / "feature_schema.json").write_text(json.dumps({ | |
| "features": FEATURES, "target": "ndvi_anomaly = ndvi - clim_ndvi", "months": MONTHS.tolist(), | |
| "weather_anomaly_baseline": "cell-month mean 2001-2025 (see climatology.parquet / weather_climatology.parquet)", | |
| "shipped_model": chosen}, indent=2)) | |
| (RUN / "conformal.json").write_text(json.dumps(conf_prod, indent=2)) | |
| (RUN / "metrics.json").write_text(json.dumps(metrics, indent=2, default=float)) | |
| log("TRAIN_DONE") | |
| if __name__ == "__main__": | |
| main() | |