Konuktor92's picture
Add full training code for model v1 (same files as PR #1)
7e6579d verified
Raw History Blame Contribute Delete
8.81 kB
"""Products built from the out-of-fold predictions (runs/v1 -> runs/v1/products).
Residuals for trends use ONE reference per cell-month (the all-years climatology), not the per-block
training climatology used for evaluation. With per-block climatologies a real trend would be inflated
(each held-out block's reference comes from the other years). pred_anom (the weather effect) still comes
from models that never saw that block, so no target information leaks into it.
Outputs (cell level, plus block and district aggregates):
expected_oof.parquet cell_id, year, month, observed, expected, p10, p90, residual
residual_trend*.parquet Sen slope / CI / Mann-Kendall (Hamed-Rao) / BH q / class of summer residuals
season_vs_expected*.parquet summer mean residual per year -> below / within / above usual for its weather
aqua_terra_blocks.parquet per-block summer NDVI trend from Terra and Aqua (2003-2025)
products_summary.json headline shares, placebo, Aqua-Terra agreement
"""
from __future__ import annotations
import json
import sys
import time
from pathlib import Path
import numpy as np
import pandas as pd
sys.path.insert(0, "/root/trendex-ml/src")
from trendex_ml.stats.trend import bh_fdr, classify, mk_vectorized, trend_many # noqa: E402
CUBE = Path("/root/trendex-ml/data/cube")
RUN = Path("/root/trendex-ml/runs/v1")
OUT = RUN / "products"
YEARS = np.arange(2001, 2026)
BLOCKS = [np.arange(s, s + 5) for s in (2001, 2006, 2011, 2016, 2021)]
SUMMER = [6, 7, 8]
ALPHA, MIN_YEARS, N_PLACEBO = 0.05, 18, 200
def log(msg):
print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True)
def trend_table(wide: pd.DataFrame, key: str) -> pd.DataFrame:
r = trend_many(wide.values, wide.columns.values.astype(float), method="hamed_rao", min_n=MIN_YEARS)
df = pd.DataFrame(r)
df.insert(0, key, wide.index.values)
df["q"] = bh_fdr(df["p"].values)
df["class"] = classify(df["slope"].values, df["q"].values, df["n"].values, ALPHA, MIN_YEARS)
df["slope_per_decade"] = df["slope"] * 10
return df
def main() -> None:
OUT.mkdir(parents=True, exist_ok=True)
cells = pd.read_parquet(CUBE / "cells.parquet")
oof = pd.read_parquet(RUN / "expected_oof.parquet")
conf = {int(k): v for k, v in json.loads((RUN / "conformal.json").read_text()).items()}
# recover pred_anom = expected - clim_fold (clim of the other 20 years), then re-reference to clim_all
ndvi = pd.read_parquet(CUBE / "monthly", columns=["cell_id", "month", "ndvi", "ndvi_ok", "ndvi_aqua", "year"])
ndvi["year"] = ndvi["year"].astype(int)
ndvi = ndvi[ndvi["cell_id"].isin(oof["cell_id"].unique()) & ndvi["year"].between(2001, 2025)]
ok = ndvi[ndvi["ndvi_ok"] & ndvi["month"].between(5, 9)]
clim_all = ok.groupby(["cell_id", "month"])["ndvi"].mean().rename("clim_all")
parts = []
for blk in BLOCKS:
cf = ok[~ok["year"].isin(blk)].groupby(["cell_id", "month"])["ndvi"].mean().rename("clim_fold").reset_index()
sub = oof[oof["year"].isin(blk)].merge(cf, on=["cell_id", "month"], how="left")
parts.append(sub)
oof = pd.concat(parts, ignore_index=True).merge(clim_all.reset_index(), on=["cell_id", "month"], how="left")
pred_anom = oof["expected"] - oof["clim_fold"]
oof["expected"] = oof["clim_all"] + pred_anom
lo = oof["month"].map(lambda m: conf[int(m)][0])
hi = oof["month"].map(lambda m: conf[int(m)][1])
oof["p10"], oof["p90"] = oof["expected"] + lo, oof["expected"] + hi
oof["residual"] = oof["observed"] - oof["expected"]
oof = oof[["cell_id", "year", "month", "observed", "expected", "p10", "p90", "residual"]]
oof.to_parquet(OUT / "expected_oof.parquet", index=False)
log(f"expected_oof: {len(oof)} rows")
# summer residual series
summer = oof[oof["month"].isin(SUMMER)].groupby(["cell_id", "year"])["residual"].mean().rename("res").reset_index()
summer = summer.merge(cells[["cell_id", "block_id", "district"]], on="cell_id")
wide = summer.pivot(index="cell_id", columns="year", values="res").reindex(columns=YEARS)
# residual trends: cell, block, district
rt = trend_table(wide, "cell_id")
rt.to_parquet(OUT / "residual_trend.parquet", index=False)
blk_w = summer.groupby(["block_id", "year"])["res"].mean().unstack().reindex(columns=YEARS)
trend_table(blk_w, "block_id").to_parquet(OUT / "residual_trend_block.parquet", index=False)
dis_w = summer.groupby(["district", "year"])["res"].mean().unstack().reindex(columns=YEARS)
rtd = trend_table(dis_w, "district")
rtd.to_parquet(OUT / "residual_trend_district.parquet", index=False)
shares = rt["class"].value_counts(normalize=True).round(4).to_dict()
log(f"residual trend classes (cells): {shares}")
# placebo: shuffle years within each cell; how many cells would be flagged by chance?
rng = np.random.default_rng(0)
W = wide.values.astype("float32")
flagged_raw, flagged_q = [], []
for _ in range(N_PLACEBO):
perm = np.argsort(rng.random(W.shape), axis=1)
Wp = np.take_along_axis(W, perm, axis=1)
_, _, _, p = mk_vectorized(Wp)
flagged_raw.append(float(np.nanmean(p < ALPHA)))
flagged_q.append(float(np.nanmean(bh_fdr(p) < ALPHA)))
_, _, _, p_obs = mk_vectorized(W)
placebo = {"permutations": N_PLACEBO, "test": "original MK (mk_vectorized), same for observed and placebo",
"observed_share_p<0.05": float(np.nanmean(p_obs < ALPHA)),
"observed_share_q<0.05": float(np.nanmean(bh_fdr(p_obs) < ALPHA)),
"placebo_share_p<0.05_mean": float(np.mean(flagged_raw)),
"placebo_share_p<0.05_p95": float(np.quantile(flagged_raw, 0.95)),
"placebo_share_q<0.05_mean": float(np.mean(flagged_q)),
"placebo_share_q<0.05_p95": float(np.quantile(flagged_q, 0.95))}
log(f"placebo {placebo}")
# season vs expected: summer mean residual vs the empirical P10/P90 of all cell-years
q10, q90 = summer["res"].quantile([0.1, 0.9])
summer["season_class"] = np.select([summer["res"] < q10, summer["res"] > q90], ["below", "above"], "within")
summer.rename(columns={"res": "summer_residual"}).to_parquet(OUT / "season_vs_expected.parquet", index=False)
sb = summer.groupby(["block_id", "year"])["res"].mean().rename("summer_residual").reset_index()
bq10, bq90 = sb["summer_residual"].quantile([0.1, 0.9])
sb["season_class"] = np.select([sb["summer_residual"] < bq10, sb["summer_residual"] > bq90], ["below", "above"], "within")
sb.to_parquet(OUT / "season_vs_expected_block.parquet", index=False)
# Aqua vs Terra: per-block summer NDVI trend 2003-2025, both satellites (raw NDVI, not residuals)
s = ndvi[ndvi["month"].isin(SUMMER) & ndvi["year"].between(2003, 2025)].merge(cells[["cell_id", "block_id"]], on="cell_id")
s.loc[~s["ndvi_ok"], "ndvi"] = np.nan
yrs = np.arange(2003, 2026)
terra = s.groupby(["block_id", "year"])["ndvi"].mean().unstack().reindex(columns=yrs)
aqua = s.groupby(["block_id", "year"])["ndvi_aqua"].mean().unstack().reindex(columns=yrs).reindex(terra.index)
tt, ta = trend_table(terra, "block_id"), trend_table(aqua, "block_id")
at = tt[["block_id", "slope", "q", "class"]].merge(ta[["block_id", "slope", "q", "class"]], on="block_id",
suffixes=("_terra", "_aqua"))
at.to_parquet(OUT / "aqua_terra_blocks.parquet", index=False)
sig_t = at["class_terra"].isin(["growth", "decline"])
agreement = {"blocks": int(len(at)),
"slope_sign_agreement": float(np.mean(np.sign(at["slope_terra"]) == np.sign(at["slope_aqua"]))),
"terra_significant_blocks": int(sig_t.sum()),
"same_class_where_terra_significant": float(np.mean(at.loc[sig_t, "class_terra"] == at.loc[sig_t, "class_aqua"])) if sig_t.any() else None,
"terra_class_shares": at["class_terra"].value_counts(normalize=True).round(4).to_dict(),
"aqua_class_shares": at["class_aqua"].value_counts(normalize=True).round(4).to_dict()}
log(f"aqua vs terra {agreement}")
summary = {"residual_trend_cell_class_shares": shares,
"residual_trend_district": rtd[["district", "slope_per_decade", "ci_lo", "ci_hi", "p", "q", "class"]]
.round(5).to_dict(orient="records"),
"season_band_summer_residual_q10_q90": [float(q10), float(q90)],
"placebo": placebo, "aqua_vs_terra_blocks": agreement,
"wording": "change not explained by precipitation, temperature, snow and soil moisture (not a cause)"}
(OUT / "products_summary.json").write_text(json.dumps(summary, indent=2, default=float))
log("PRODUCTS_DONE")
if __name__ == "__main__":
main()