File size: 8,805 Bytes
d7c810d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
"""Products built from the out-of-fold predictions (runs/v1 -> runs/v1/products).

Residuals for trends use ONE reference per cell-month (the all-years climatology), not the per-block
training climatology used for evaluation. With per-block climatologies a real trend would be inflated
(each held-out block's reference comes from the other years). pred_anom (the weather effect) still comes
from models that never saw that block, so no target information leaks into it.

Outputs (cell level, plus block and district aggregates):
  expected_oof.parquet          cell_id, year, month, observed, expected, p10, p90, residual
  residual_trend*.parquet       Sen slope / CI / Mann-Kendall (Hamed-Rao) / BH q / class of summer residuals
  season_vs_expected*.parquet   summer mean residual per year -> below / within / above usual for its weather
  aqua_terra_blocks.parquet     per-block summer NDVI trend from Terra and Aqua (2003-2025)
  products_summary.json         headline shares, placebo, Aqua-Terra agreement
"""
from __future__ import annotations

import json
import sys
import time
from pathlib import Path

import numpy as np
import pandas as pd

sys.path.insert(0, "/root/trendex-ml/src")
from trendex_ml.stats.trend import bh_fdr, classify, mk_vectorized, trend_many  # noqa: E402

CUBE = Path("/root/trendex-ml/data/cube")
RUN = Path("/root/trendex-ml/runs/v1")
OUT = RUN / "products"
YEARS = np.arange(2001, 2026)
BLOCKS = [np.arange(s, s + 5) for s in (2001, 2006, 2011, 2016, 2021)]
SUMMER = [6, 7, 8]
ALPHA, MIN_YEARS, N_PLACEBO = 0.05, 18, 200


def log(msg):
    print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True)


def trend_table(wide: pd.DataFrame, key: str) -> pd.DataFrame:
    r = trend_many(wide.values, wide.columns.values.astype(float), method="hamed_rao", min_n=MIN_YEARS)
    df = pd.DataFrame(r)
    df.insert(0, key, wide.index.values)
    df["q"] = bh_fdr(df["p"].values)
    df["class"] = classify(df["slope"].values, df["q"].values, df["n"].values, ALPHA, MIN_YEARS)
    df["slope_per_decade"] = df["slope"] * 10
    return df


def main() -> None:
    OUT.mkdir(parents=True, exist_ok=True)
    cells = pd.read_parquet(CUBE / "cells.parquet")
    oof = pd.read_parquet(RUN / "expected_oof.parquet")
    conf = {int(k): v for k, v in json.loads((RUN / "conformal.json").read_text()).items()}

    # recover pred_anom = expected - clim_fold (clim of the other 20 years), then re-reference to clim_all
    ndvi = pd.read_parquet(CUBE / "monthly", columns=["cell_id", "month", "ndvi", "ndvi_ok", "ndvi_aqua", "year"])
    ndvi["year"] = ndvi["year"].astype(int)
    ndvi = ndvi[ndvi["cell_id"].isin(oof["cell_id"].unique()) & ndvi["year"].between(2001, 2025)]
    ok = ndvi[ndvi["ndvi_ok"] & ndvi["month"].between(5, 9)]
    clim_all = ok.groupby(["cell_id", "month"])["ndvi"].mean().rename("clim_all")
    parts = []
    for blk in BLOCKS:
        cf = ok[~ok["year"].isin(blk)].groupby(["cell_id", "month"])["ndvi"].mean().rename("clim_fold").reset_index()
        sub = oof[oof["year"].isin(blk)].merge(cf, on=["cell_id", "month"], how="left")
        parts.append(sub)
    oof = pd.concat(parts, ignore_index=True).merge(clim_all.reset_index(), on=["cell_id", "month"], how="left")
    pred_anom = oof["expected"] - oof["clim_fold"]
    oof["expected"] = oof["clim_all"] + pred_anom
    lo = oof["month"].map(lambda m: conf[int(m)][0])
    hi = oof["month"].map(lambda m: conf[int(m)][1])
    oof["p10"], oof["p90"] = oof["expected"] + lo, oof["expected"] + hi
    oof["residual"] = oof["observed"] - oof["expected"]
    oof = oof[["cell_id", "year", "month", "observed", "expected", "p10", "p90", "residual"]]
    oof.to_parquet(OUT / "expected_oof.parquet", index=False)
    log(f"expected_oof: {len(oof)} rows")

    # summer residual series
    summer = oof[oof["month"].isin(SUMMER)].groupby(["cell_id", "year"])["residual"].mean().rename("res").reset_index()
    summer = summer.merge(cells[["cell_id", "block_id", "district"]], on="cell_id")
    wide = summer.pivot(index="cell_id", columns="year", values="res").reindex(columns=YEARS)

    # residual trends: cell, block, district
    rt = trend_table(wide, "cell_id")
    rt.to_parquet(OUT / "residual_trend.parquet", index=False)
    blk_w = summer.groupby(["block_id", "year"])["res"].mean().unstack().reindex(columns=YEARS)
    trend_table(blk_w, "block_id").to_parquet(OUT / "residual_trend_block.parquet", index=False)
    dis_w = summer.groupby(["district", "year"])["res"].mean().unstack().reindex(columns=YEARS)
    rtd = trend_table(dis_w, "district")
    rtd.to_parquet(OUT / "residual_trend_district.parquet", index=False)
    shares = rt["class"].value_counts(normalize=True).round(4).to_dict()
    log(f"residual trend classes (cells): {shares}")

    # placebo: shuffle years within each cell; how many cells would be flagged by chance?
    rng = np.random.default_rng(0)
    W = wide.values.astype("float32")
    flagged_raw, flagged_q = [], []
    for _ in range(N_PLACEBO):
        perm = np.argsort(rng.random(W.shape), axis=1)
        Wp = np.take_along_axis(W, perm, axis=1)
        _, _, _, p = mk_vectorized(Wp)
        flagged_raw.append(float(np.nanmean(p < ALPHA)))
        flagged_q.append(float(np.nanmean(bh_fdr(p) < ALPHA)))
    _, _, _, p_obs = mk_vectorized(W)
    placebo = {"permutations": N_PLACEBO, "test": "original MK (mk_vectorized), same for observed and placebo",
               "observed_share_p<0.05": float(np.nanmean(p_obs < ALPHA)),
               "observed_share_q<0.05": float(np.nanmean(bh_fdr(p_obs) < ALPHA)),
               "placebo_share_p<0.05_mean": float(np.mean(flagged_raw)),
               "placebo_share_p<0.05_p95": float(np.quantile(flagged_raw, 0.95)),
               "placebo_share_q<0.05_mean": float(np.mean(flagged_q)),
               "placebo_share_q<0.05_p95": float(np.quantile(flagged_q, 0.95))}
    log(f"placebo {placebo}")

    # season vs expected: summer mean residual vs the empirical P10/P90 of all cell-years
    q10, q90 = summer["res"].quantile([0.1, 0.9])
    summer["season_class"] = np.select([summer["res"] < q10, summer["res"] > q90], ["below", "above"], "within")
    summer.rename(columns={"res": "summer_residual"}).to_parquet(OUT / "season_vs_expected.parquet", index=False)
    sb = summer.groupby(["block_id", "year"])["res"].mean().rename("summer_residual").reset_index()
    bq10, bq90 = sb["summer_residual"].quantile([0.1, 0.9])
    sb["season_class"] = np.select([sb["summer_residual"] < bq10, sb["summer_residual"] > bq90], ["below", "above"], "within")
    sb.to_parquet(OUT / "season_vs_expected_block.parquet", index=False)

    # Aqua vs Terra: per-block summer NDVI trend 2003-2025, both satellites (raw NDVI, not residuals)
    s = ndvi[ndvi["month"].isin(SUMMER) & ndvi["year"].between(2003, 2025)].merge(cells[["cell_id", "block_id"]], on="cell_id")
    s.loc[~s["ndvi_ok"], "ndvi"] = np.nan
    yrs = np.arange(2003, 2026)
    terra = s.groupby(["block_id", "year"])["ndvi"].mean().unstack().reindex(columns=yrs)
    aqua = s.groupby(["block_id", "year"])["ndvi_aqua"].mean().unstack().reindex(columns=yrs).reindex(terra.index)
    tt, ta = trend_table(terra, "block_id"), trend_table(aqua, "block_id")
    at = tt[["block_id", "slope", "q", "class"]].merge(ta[["block_id", "slope", "q", "class"]], on="block_id",
                                                       suffixes=("_terra", "_aqua"))
    at.to_parquet(OUT / "aqua_terra_blocks.parquet", index=False)
    sig_t = at["class_terra"].isin(["growth", "decline"])
    agreement = {"blocks": int(len(at)),
                 "slope_sign_agreement": float(np.mean(np.sign(at["slope_terra"]) == np.sign(at["slope_aqua"]))),
                 "terra_significant_blocks": int(sig_t.sum()),
                 "same_class_where_terra_significant": float(np.mean(at.loc[sig_t, "class_terra"] == at.loc[sig_t, "class_aqua"])) if sig_t.any() else None,
                 "terra_class_shares": at["class_terra"].value_counts(normalize=True).round(4).to_dict(),
                 "aqua_class_shares": at["class_aqua"].value_counts(normalize=True).round(4).to_dict()}
    log(f"aqua vs terra {agreement}")

    summary = {"residual_trend_cell_class_shares": shares,
               "residual_trend_district": rtd[["district", "slope_per_decade", "ci_lo", "ci_hi", "p", "q", "class"]]
               .round(5).to_dict(orient="records"),
               "season_band_summer_residual_q10_q90": [float(q10), float(q90)],
               "placebo": placebo, "aqua_vs_terra_blocks": agreement,
               "wording": "change not explained by precipitation, temperature, snow and soil moisture (not a cause)"}
    (OUT / "products_summary.json").write_text(json.dumps(summary, indent=2, default=float))
    log("PRODUCTS_DONE")


if __name__ == "__main__":
    main()