Download training_code/scripts/40_products.py from TrendeX90/expected-greenness: direct link, hf CLI and curl.
- Browser
- Download file 8.81 kB
-
https://huggingface.co/TrendeX90/expected-greenness/resolve/refs%2Fpr%2F1/training_code/scripts/40_products.py
- Command line
-
hf download hf://TrendeX90/expected-greenness@refs/pr/1/training_code/scripts/40_products.py
-
curl -L -o 40_products.py https://huggingface.co/TrendeX90/expected-greenness/resolve/refs%2Fpr%2F1/training_code/scripts/40_products.py
8.81 kB
| """Products built from the out-of-fold predictions (runs/v1 -> runs/v1/products). | |
| Residuals for trends use ONE reference per cell-month (the all-years climatology), not the per-block | |
| training climatology used for evaluation. With per-block climatologies a real trend would be inflated | |
| (each held-out block's reference comes from the other years). pred_anom (the weather effect) still comes | |
| from models that never saw that block, so no target information leaks into it. | |
| Outputs (cell level, plus block and district aggregates): | |
| expected_oof.parquet cell_id, year, month, observed, expected, p10, p90, residual | |
| residual_trend*.parquet Sen slope / CI / Mann-Kendall (Hamed-Rao) / BH q / class of summer residuals | |
| season_vs_expected*.parquet summer mean residual per year -> below / within / above usual for its weather | |
| aqua_terra_blocks.parquet per-block summer NDVI trend from Terra and Aqua (2003-2025) | |
| products_summary.json headline shares, placebo, Aqua-Terra agreement | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import sys | |
| import time | |
| from pathlib import Path | |
| import numpy as np | |
| import pandas as pd | |
| sys.path.insert(0, "/root/trendex-ml/src") | |
| from trendex_ml.stats.trend import bh_fdr, classify, mk_vectorized, trend_many # noqa: E402 | |
| CUBE = Path("/root/trendex-ml/data/cube") | |
| RUN = Path("/root/trendex-ml/runs/v1") | |
| OUT = RUN / "products" | |
| YEARS = np.arange(2001, 2026) | |
| BLOCKS = [np.arange(s, s + 5) for s in (2001, 2006, 2011, 2016, 2021)] | |
| SUMMER = [6, 7, 8] | |
| ALPHA, MIN_YEARS, N_PLACEBO = 0.05, 18, 200 | |
| def log(msg): | |
| print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True) | |
| def trend_table(wide: pd.DataFrame, key: str) -> pd.DataFrame: | |
| r = trend_many(wide.values, wide.columns.values.astype(float), method="hamed_rao", min_n=MIN_YEARS) | |
| df = pd.DataFrame(r) | |
| df.insert(0, key, wide.index.values) | |
| df["q"] = bh_fdr(df["p"].values) | |
| df["class"] = classify(df["slope"].values, df["q"].values, df["n"].values, ALPHA, MIN_YEARS) | |
| df["slope_per_decade"] = df["slope"] * 10 | |
| return df | |
| def main() -> None: | |
| OUT.mkdir(parents=True, exist_ok=True) | |
| cells = pd.read_parquet(CUBE / "cells.parquet") | |
| oof = pd.read_parquet(RUN / "expected_oof.parquet") | |
| conf = {int(k): v for k, v in json.loads((RUN / "conformal.json").read_text()).items()} | |
| # recover pred_anom = expected - clim_fold (clim of the other 20 years), then re-reference to clim_all | |
| ndvi = pd.read_parquet(CUBE / "monthly", columns=["cell_id", "month", "ndvi", "ndvi_ok", "ndvi_aqua", "year"]) | |
| ndvi["year"] = ndvi["year"].astype(int) | |
| ndvi = ndvi[ndvi["cell_id"].isin(oof["cell_id"].unique()) & ndvi["year"].between(2001, 2025)] | |
| ok = ndvi[ndvi["ndvi_ok"] & ndvi["month"].between(5, 9)] | |
| clim_all = ok.groupby(["cell_id", "month"])["ndvi"].mean().rename("clim_all") | |
| parts = [] | |
| for blk in BLOCKS: | |
| cf = ok[~ok["year"].isin(blk)].groupby(["cell_id", "month"])["ndvi"].mean().rename("clim_fold").reset_index() | |
| sub = oof[oof["year"].isin(blk)].merge(cf, on=["cell_id", "month"], how="left") | |
| parts.append(sub) | |
| oof = pd.concat(parts, ignore_index=True).merge(clim_all.reset_index(), on=["cell_id", "month"], how="left") | |
| pred_anom = oof["expected"] - oof["clim_fold"] | |
| oof["expected"] = oof["clim_all"] + pred_anom | |
| lo = oof["month"].map(lambda m: conf[int(m)][0]) | |
| hi = oof["month"].map(lambda m: conf[int(m)][1]) | |
| oof["p10"], oof["p90"] = oof["expected"] + lo, oof["expected"] + hi | |
| oof["residual"] = oof["observed"] - oof["expected"] | |
| oof = oof[["cell_id", "year", "month", "observed", "expected", "p10", "p90", "residual"]] | |
| oof.to_parquet(OUT / "expected_oof.parquet", index=False) | |
| log(f"expected_oof: {len(oof)} rows") | |
| # summer residual series | |
| summer = oof[oof["month"].isin(SUMMER)].groupby(["cell_id", "year"])["residual"].mean().rename("res").reset_index() | |
| summer = summer.merge(cells[["cell_id", "block_id", "district"]], on="cell_id") | |
| wide = summer.pivot(index="cell_id", columns="year", values="res").reindex(columns=YEARS) | |
| # residual trends: cell, block, district | |
| rt = trend_table(wide, "cell_id") | |
| rt.to_parquet(OUT / "residual_trend.parquet", index=False) | |
| blk_w = summer.groupby(["block_id", "year"])["res"].mean().unstack().reindex(columns=YEARS) | |
| trend_table(blk_w, "block_id").to_parquet(OUT / "residual_trend_block.parquet", index=False) | |
| dis_w = summer.groupby(["district", "year"])["res"].mean().unstack().reindex(columns=YEARS) | |
| rtd = trend_table(dis_w, "district") | |
| rtd.to_parquet(OUT / "residual_trend_district.parquet", index=False) | |
| shares = rt["class"].value_counts(normalize=True).round(4).to_dict() | |
| log(f"residual trend classes (cells): {shares}") | |
| # placebo: shuffle years within each cell; how many cells would be flagged by chance? | |
| rng = np.random.default_rng(0) | |
| W = wide.values.astype("float32") | |
| flagged_raw, flagged_q = [], [] | |
| for _ in range(N_PLACEBO): | |
| perm = np.argsort(rng.random(W.shape), axis=1) | |
| Wp = np.take_along_axis(W, perm, axis=1) | |
| _, _, _, p = mk_vectorized(Wp) | |
| flagged_raw.append(float(np.nanmean(p < ALPHA))) | |
| flagged_q.append(float(np.nanmean(bh_fdr(p) < ALPHA))) | |
| _, _, _, p_obs = mk_vectorized(W) | |
| placebo = {"permutations": N_PLACEBO, "test": "original MK (mk_vectorized), same for observed and placebo", | |
| "observed_share_p<0.05": float(np.nanmean(p_obs < ALPHA)), | |
| "observed_share_q<0.05": float(np.nanmean(bh_fdr(p_obs) < ALPHA)), | |
| "placebo_share_p<0.05_mean": float(np.mean(flagged_raw)), | |
| "placebo_share_p<0.05_p95": float(np.quantile(flagged_raw, 0.95)), | |
| "placebo_share_q<0.05_mean": float(np.mean(flagged_q)), | |
| "placebo_share_q<0.05_p95": float(np.quantile(flagged_q, 0.95))} | |
| log(f"placebo {placebo}") | |
| # season vs expected: summer mean residual vs the empirical P10/P90 of all cell-years | |
| q10, q90 = summer["res"].quantile([0.1, 0.9]) | |
| summer["season_class"] = np.select([summer["res"] < q10, summer["res"] > q90], ["below", "above"], "within") | |
| summer.rename(columns={"res": "summer_residual"}).to_parquet(OUT / "season_vs_expected.parquet", index=False) | |
| sb = summer.groupby(["block_id", "year"])["res"].mean().rename("summer_residual").reset_index() | |
| bq10, bq90 = sb["summer_residual"].quantile([0.1, 0.9]) | |
| sb["season_class"] = np.select([sb["summer_residual"] < bq10, sb["summer_residual"] > bq90], ["below", "above"], "within") | |
| sb.to_parquet(OUT / "season_vs_expected_block.parquet", index=False) | |
| # Aqua vs Terra: per-block summer NDVI trend 2003-2025, both satellites (raw NDVI, not residuals) | |
| s = ndvi[ndvi["month"].isin(SUMMER) & ndvi["year"].between(2003, 2025)].merge(cells[["cell_id", "block_id"]], on="cell_id") | |
| s.loc[~s["ndvi_ok"], "ndvi"] = np.nan | |
| yrs = np.arange(2003, 2026) | |
| terra = s.groupby(["block_id", "year"])["ndvi"].mean().unstack().reindex(columns=yrs) | |
| aqua = s.groupby(["block_id", "year"])["ndvi_aqua"].mean().unstack().reindex(columns=yrs).reindex(terra.index) | |
| tt, ta = trend_table(terra, "block_id"), trend_table(aqua, "block_id") | |
| at = tt[["block_id", "slope", "q", "class"]].merge(ta[["block_id", "slope", "q", "class"]], on="block_id", | |
| suffixes=("_terra", "_aqua")) | |
| at.to_parquet(OUT / "aqua_terra_blocks.parquet", index=False) | |
| sig_t = at["class_terra"].isin(["growth", "decline"]) | |
| agreement = {"blocks": int(len(at)), | |
| "slope_sign_agreement": float(np.mean(np.sign(at["slope_terra"]) == np.sign(at["slope_aqua"]))), | |
| "terra_significant_blocks": int(sig_t.sum()), | |
| "same_class_where_terra_significant": float(np.mean(at.loc[sig_t, "class_terra"] == at.loc[sig_t, "class_aqua"])) if sig_t.any() else None, | |
| "terra_class_shares": at["class_terra"].value_counts(normalize=True).round(4).to_dict(), | |
| "aqua_class_shares": at["class_aqua"].value_counts(normalize=True).round(4).to_dict()} | |
| log(f"aqua vs terra {agreement}") | |
| summary = {"residual_trend_cell_class_shares": shares, | |
| "residual_trend_district": rtd[["district", "slope_per_decade", "ci_lo", "ci_hi", "p", "q", "class"]] | |
| .round(5).to_dict(orient="records"), | |
| "season_band_summer_residual_q10_q90": [float(q10), float(q90)], | |
| "placebo": placebo, "aqua_vs_terra_blocks": agreement, | |
| "wording": "change not explained by precipitation, temperature, snow and soil moisture (not a cause)"} | |
| (OUT / "products_summary.json").write_text(json.dumps(summary, indent=2, default=float)) | |
| log("PRODUCTS_DONE") | |
| if __name__ == "__main__": | |
| main() | |