"""Products built from the out-of-fold predictions (runs/v1 -> runs/v1/products). Residuals for trends use ONE reference per cell-month (the all-years climatology), not the per-block training climatology used for evaluation. With per-block climatologies a real trend would be inflated (each held-out block's reference comes from the other years). pred_anom (the weather effect) still comes from models that never saw that block, so no target information leaks into it. Outputs (cell level, plus block and district aggregates): expected_oof.parquet cell_id, year, month, observed, expected, p10, p90, residual residual_trend*.parquet Sen slope / CI / Mann-Kendall (Hamed-Rao) / BH q / class of summer residuals season_vs_expected*.parquet summer mean residual per year -> below / within / above usual for its weather aqua_terra_blocks.parquet per-block summer NDVI trend from Terra and Aqua (2003-2025) products_summary.json headline shares, placebo, Aqua-Terra agreement """ from __future__ import annotations import json import sys import time from pathlib import Path import numpy as np import pandas as pd sys.path.insert(0, "/root/trendex-ml/src") from trendex_ml.stats.trend import bh_fdr, classify, mk_vectorized, trend_many # noqa: E402 CUBE = Path("/root/trendex-ml/data/cube") RUN = Path("/root/trendex-ml/runs/v1") OUT = RUN / "products" YEARS = np.arange(2001, 2026) BLOCKS = [np.arange(s, s + 5) for s in (2001, 2006, 2011, 2016, 2021)] SUMMER = [6, 7, 8] ALPHA, MIN_YEARS, N_PLACEBO = 0.05, 18, 200 def log(msg): print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True) def trend_table(wide: pd.DataFrame, key: str) -> pd.DataFrame: r = trend_many(wide.values, wide.columns.values.astype(float), method="hamed_rao", min_n=MIN_YEARS) df = pd.DataFrame(r) df.insert(0, key, wide.index.values) df["q"] = bh_fdr(df["p"].values) df["class"] = classify(df["slope"].values, df["q"].values, df["n"].values, ALPHA, MIN_YEARS) df["slope_per_decade"] = df["slope"] * 10 return df def main() -> None: OUT.mkdir(parents=True, exist_ok=True) cells = pd.read_parquet(CUBE / "cells.parquet") oof = pd.read_parquet(RUN / "expected_oof.parquet") conf = {int(k): v for k, v in json.loads((RUN / "conformal.json").read_text()).items()} # recover pred_anom = expected - clim_fold (clim of the other 20 years), then re-reference to clim_all ndvi = pd.read_parquet(CUBE / "monthly", columns=["cell_id", "month", "ndvi", "ndvi_ok", "ndvi_aqua", "year"]) ndvi["year"] = ndvi["year"].astype(int) ndvi = ndvi[ndvi["cell_id"].isin(oof["cell_id"].unique()) & ndvi["year"].between(2001, 2025)] ok = ndvi[ndvi["ndvi_ok"] & ndvi["month"].between(5, 9)] clim_all = ok.groupby(["cell_id", "month"])["ndvi"].mean().rename("clim_all") parts = [] for blk in BLOCKS: cf = ok[~ok["year"].isin(blk)].groupby(["cell_id", "month"])["ndvi"].mean().rename("clim_fold").reset_index() sub = oof[oof["year"].isin(blk)].merge(cf, on=["cell_id", "month"], how="left") parts.append(sub) oof = pd.concat(parts, ignore_index=True).merge(clim_all.reset_index(), on=["cell_id", "month"], how="left") pred_anom = oof["expected"] - oof["clim_fold"] oof["expected"] = oof["clim_all"] + pred_anom lo = oof["month"].map(lambda m: conf[int(m)][0]) hi = oof["month"].map(lambda m: conf[int(m)][1]) oof["p10"], oof["p90"] = oof["expected"] + lo, oof["expected"] + hi oof["residual"] = oof["observed"] - oof["expected"] oof = oof[["cell_id", "year", "month", "observed", "expected", "p10", "p90", "residual"]] oof.to_parquet(OUT / "expected_oof.parquet", index=False) log(f"expected_oof: {len(oof)} rows") # summer residual series summer = oof[oof["month"].isin(SUMMER)].groupby(["cell_id", "year"])["residual"].mean().rename("res").reset_index() summer = summer.merge(cells[["cell_id", "block_id", "district"]], on="cell_id") wide = summer.pivot(index="cell_id", columns="year", values="res").reindex(columns=YEARS) # residual trends: cell, block, district rt = trend_table(wide, "cell_id") rt.to_parquet(OUT / "residual_trend.parquet", index=False) blk_w = summer.groupby(["block_id", "year"])["res"].mean().unstack().reindex(columns=YEARS) trend_table(blk_w, "block_id").to_parquet(OUT / "residual_trend_block.parquet", index=False) dis_w = summer.groupby(["district", "year"])["res"].mean().unstack().reindex(columns=YEARS) rtd = trend_table(dis_w, "district") rtd.to_parquet(OUT / "residual_trend_district.parquet", index=False) shares = rt["class"].value_counts(normalize=True).round(4).to_dict() log(f"residual trend classes (cells): {shares}") # placebo: shuffle years within each cell; how many cells would be flagged by chance? rng = np.random.default_rng(0) W = wide.values.astype("float32") flagged_raw, flagged_q = [], [] for _ in range(N_PLACEBO): perm = np.argsort(rng.random(W.shape), axis=1) Wp = np.take_along_axis(W, perm, axis=1) _, _, _, p = mk_vectorized(Wp) flagged_raw.append(float(np.nanmean(p < ALPHA))) flagged_q.append(float(np.nanmean(bh_fdr(p) < ALPHA))) _, _, _, p_obs = mk_vectorized(W) placebo = {"permutations": N_PLACEBO, "test": "original MK (mk_vectorized), same for observed and placebo", "observed_share_p<0.05": float(np.nanmean(p_obs < ALPHA)), "observed_share_q<0.05": float(np.nanmean(bh_fdr(p_obs) < ALPHA)), "placebo_share_p<0.05_mean": float(np.mean(flagged_raw)), "placebo_share_p<0.05_p95": float(np.quantile(flagged_raw, 0.95)), "placebo_share_q<0.05_mean": float(np.mean(flagged_q)), "placebo_share_q<0.05_p95": float(np.quantile(flagged_q, 0.95))} log(f"placebo {placebo}") # season vs expected: summer mean residual vs the empirical P10/P90 of all cell-years q10, q90 = summer["res"].quantile([0.1, 0.9]) summer["season_class"] = np.select([summer["res"] < q10, summer["res"] > q90], ["below", "above"], "within") summer.rename(columns={"res": "summer_residual"}).to_parquet(OUT / "season_vs_expected.parquet", index=False) sb = summer.groupby(["block_id", "year"])["res"].mean().rename("summer_residual").reset_index() bq10, bq90 = sb["summer_residual"].quantile([0.1, 0.9]) sb["season_class"] = np.select([sb["summer_residual"] < bq10, sb["summer_residual"] > bq90], ["below", "above"], "within") sb.to_parquet(OUT / "season_vs_expected_block.parquet", index=False) # Aqua vs Terra: per-block summer NDVI trend 2003-2025, both satellites (raw NDVI, not residuals) s = ndvi[ndvi["month"].isin(SUMMER) & ndvi["year"].between(2003, 2025)].merge(cells[["cell_id", "block_id"]], on="cell_id") s.loc[~s["ndvi_ok"], "ndvi"] = np.nan yrs = np.arange(2003, 2026) terra = s.groupby(["block_id", "year"])["ndvi"].mean().unstack().reindex(columns=yrs) aqua = s.groupby(["block_id", "year"])["ndvi_aqua"].mean().unstack().reindex(columns=yrs).reindex(terra.index) tt, ta = trend_table(terra, "block_id"), trend_table(aqua, "block_id") at = tt[["block_id", "slope", "q", "class"]].merge(ta[["block_id", "slope", "q", "class"]], on="block_id", suffixes=("_terra", "_aqua")) at.to_parquet(OUT / "aqua_terra_blocks.parquet", index=False) sig_t = at["class_terra"].isin(["growth", "decline"]) agreement = {"blocks": int(len(at)), "slope_sign_agreement": float(np.mean(np.sign(at["slope_terra"]) == np.sign(at["slope_aqua"]))), "terra_significant_blocks": int(sig_t.sum()), "same_class_where_terra_significant": float(np.mean(at.loc[sig_t, "class_terra"] == at.loc[sig_t, "class_aqua"])) if sig_t.any() else None, "terra_class_shares": at["class_terra"].value_counts(normalize=True).round(4).to_dict(), "aqua_class_shares": at["class_aqua"].value_counts(normalize=True).round(4).to_dict()} log(f"aqua vs terra {agreement}") summary = {"residual_trend_cell_class_shares": shares, "residual_trend_district": rtd[["district", "slope_per_decade", "ci_lo", "ci_hi", "p", "q", "class"]] .round(5).to_dict(orient="records"), "season_band_summer_residual_q10_q90": [float(q10), float(q90)], "placebo": placebo, "aqua_vs_terra_blocks": agreement, "wording": "change not explained by precipitation, temperature, snow and soil moisture (not a cause)"} (OUT / "products_summary.json").write_text(json.dumps(summary, indent=2, default=float)) log("PRODUCTS_DONE") if __name__ == "__main__": main()