File size: 24,585 Bytes
7e6579d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
"""Build the shared TrendeX data cube (F1) from the downloaded NASA rasters.

    python scripts/20_build_cube.py                 # everything
    python scripts/20_build_cube.py --only static   # cells.parquet only (terrain, land cover, districts)

Master grid = the global MODIS sinusoidal 1 km grid, cut to the request area (aoi_request.geojson).
NDVI is never resampled (same grid -> nearest = copy). Other layers: 500 m / 30 m -> average or mode,
0.1 / 0.25 deg weather -> bilinear at cell centres.

Each MODIS/NASADEM layer is read from AppEEARS GeoTIFFs when that AppEEARS task finished, otherwise
from raw LP DAAC files fetched with earthaccess (HDF4 tiles / NASADEM zips). manifest.json records which.

Outputs in data/cube/:
  cells.parquet                 one row per 1 km cell inside the request area
  monthly/year=YYYY/*.parquet   one row per cell x month
  manifest.json                 inputs, sources, counts, scale checks, notes
"""
from __future__ import annotations

import argparse
import calendar
import json
import re
import time
from concurrent.futures import ProcessPoolExecutor
from pathlib import Path

import geopandas as gpd
import numpy as np
import pandas as pd
import pyarrow as pa
import pyarrow.parquet as pq
import rasterio
import xarray as xr
from pyproj import Transformer
from rasterio.warp import Resampling, reproject
from scipy.ndimage import uniform_filter
from shapely import contains_xy

ROOT = Path("/root/trendex-ml/data")
RAW, AOI_DIR, OUT = ROOT / "raw", ROOT / "aoi", ROOT / "cube"
PAT = re.compile(r"^(?P<prod>[A-Z0-9_]+\.\d{3})_(?P<layer>.+)_doy(?P<doy>\d{7})_aid(?P<aid>\d{4})\.tif$")
BLOCK = 11            # 11 x 11 cells ~ 10.2 km blocks (F5, spatial CV)
STEEP_DEG = 35.0      # "too steep" default; to be agreed with the formula owner (F1)
TPI_WINDOW = 5        # cells (~5 km) for the topographic position index
WORKERS = 32

SINU = "+proj=sinu +lon_0=0 +x_0=0 +y_0=0 +R=6371007.181 +units=m +no_defs"
SINU_CRS = rasterio.crs.CRS.from_proj4(SINU)
TILE = 1111950.5196666666                      # MODIS tile size, m
X0, Y0 = -20015109.355798, 10007554.677899     # upper-left corner of the global MODIS grid
PIX1K = TILE / 1200

# Raw HDF4 tiles (LP DAAC fallback): layer key -> (SDS name, pixels per tile side)
HDF_SDS = {
    "MOD13A3": {"NDVI": ("1 km monthly NDVI", 1200), "pixel_reliability": ("1 km monthly pixel reliability", 1200)},
    "MYD13A3": {"NDVI": ("1 km monthly NDVI", 1200), "pixel_reliability": ("1 km monthly pixel reliability", 1200)},
    "MOD11A2": {k: (k, 1200) for k in ("LST_Day_1km", "LST_Night_1km", "QC_Day", "QC_Night")},
    "MOD10A2": {"Maximum_Snow_Extent": ("Maximum_Snow_Extent", 2400)},
    "MCD12Q1": {"LC_Type1": ("LC_Type1", 2400), "LW": ("LW", 2400)},
}

GRID: dict = {}       # filled in main(), read by worker processes after fork
SOURCES_USED: dict = {}


# ----------------------------------------------------------------------------- sources
def appeears_done(product: str) -> bool:
    marker = RAW / "appeears_done.json"
    return marker.exists() and json.loads(marker.read_text()).get(product) == "done"


def appeears_index(product: str) -> pd.DataFrame:
    rows = []
    for p in sorted((RAW / "appeears" / product).glob("*.tif")):
        m = PAT.match(p.name)
        if not m:
            continue
        year, doy = int(m["doy"][:4]), int(m["doy"][4:])
        date = pd.Timestamp(year=year, month=1, day=1) + pd.Timedelta(days=doy - 1)
        rows.append({"path": str(p), "layer": m["layer"], "date": date})
    if not rows:
        raise SystemExit(f"AppEEARS marked {product} done but no GeoTIFFs matched {PAT.pattern}")
    return pd.DataFrame(rows)


def sources(product: str, layer: str) -> dict:
    """date -> list of raster sources for one layer (AppEEARS if finished, else raw LP DAAC files)."""
    if appeears_done(product):
        idx = appeears_index(product)
        idx = idx[idx["layer"].str.endswith(layer)]
        SOURCES_USED[product] = "AppEEARS"
        return {d: g["path"].tolist() for d, g in idx.groupby("date")}
    folder = RAW / "earthaccess" / product
    if product == "NASADEM":
        tiles = sorted(folder.glob("*.zip"))
        if not tiles:
            raise SystemExit("No NASADEM data from AppEEARS (not finished) or LP DAAC")
        SOURCES_USED[product] = "LP DAAC (earthaccess)"
        return {pd.Timestamp("2000-02-11"): [f"/vsizip/{z}/{z.stem.split('_')[-1]}.hgt" for z in tiles]}
    hdfs = sorted(folder.glob("*.hdf"))
    if not hdfs:
        raise SystemExit(f"No {product} data from AppEEARS (not finished) or LP DAAC ({folder})")
    SOURCES_USED[product] = "LP DAAC (earthaccess)"
    sds, npix = HDF_SDS[product][layer]
    by_date: dict = {}
    for p in hdfs:
        m = re.search(r"\.A(\d{4})(\d{3})\.", p.name)
        d = pd.Timestamp(year=int(m[1]), month=1, day=1) + pd.Timedelta(days=int(m[2]) - 1)
        by_date.setdefault(d, []).append(("hdf", str(p), sds, npix))
    return dict(sorted(by_date.items()))


def flat(d: dict) -> list:
    return [x for v in d.values() for x in v]


def _open(src):
    """-> (raw array, nodata, transform, crs) for a GDAL path or an ('hdf', path, sds, npix) tuple."""
    if isinstance(src, str):
        with rasterio.open(src) as s:
            return s.read(1), s.nodata, s.transform, s.crs
    _, path, sds, npix = src
    from pyhdf.SD import SD, SDC
    m = re.search(r"\.h(\d{2})v(\d{2})\.", Path(path).name)
    h, v = int(m[1]), int(m[2])
    f = SD(path, SDC.READ)
    try:
        ds = f.select(sds)
        raw, attrs = ds.get(), ds.attributes()
    finally:
        f.end()
    pix = TILE / npix
    return raw, attrs.get("_FillValue"), rasterio.transform.from_origin(X0 + h * TILE, Y0 - v * TILE, pix, pix), SINU_CRS


# ----------------------------------------------------------------------------- grid helpers
def build_master_grid() -> dict:
    minx, miny, maxx, maxy = gpd.read_file(AOI_DIR / "aoi_request.geojson").to_crs(SINU).total_bounds
    c0, c1 = int(np.floor((minx - X0) / PIX1K)), int(np.ceil((maxx - X0) / PIX1K))
    r0, r1 = int(np.floor((Y0 - maxy) / PIX1K)), int(np.ceil((Y0 - miny) / PIX1K))
    left, top = X0 + c0 * PIX1K, Y0 - r0 * PIX1K
    return {"crs": SINU_CRS, "transform": rasterio.transform.from_origin(left, top, PIX1K, PIX1K),
            "shape": (r1 - r0, c1 - c0), "res": (PIX1K, PIX1K), "left": left, "top": top,
            "global_row0": r0, "global_col0": c0}


def _nodata_to_nan(raw: np.ndarray, nodata) -> np.ndarray:
    a = raw.astype("float32")
    if nodata is not None:
        a[raw == nodata] = np.nan
    return a


def to_master(srcs: list, resampling: Resampling, fn=None) -> np.ndarray:
    """Mosaic all pieces (AppEEARS area pieces or MODIS tiles) of one layer/date onto the master grid.

    fn(raw_array, nodata) -> float32 array with NaN for invalid pixels. Default: nodata -> NaN.
    """
    out = np.full(GRID["shape"], np.nan, dtype="float32")
    for src in srcs:
        raw, nodata, transform, crs = _open(src)
        a = fn(raw, nodata) if fn else _nodata_to_nan(raw, nodata)
        tmp = np.full(GRID["shape"], np.nan, dtype="float32")
        reproject(a.astype("float32"), tmp, src_transform=transform, src_crs=crs,
                  dst_transform=GRID["transform"], dst_crs=GRID["crs"],
                  resampling=resampling, src_nodata=np.nan, dst_nodata=np.nan)
        out = np.where(np.isnan(out), tmp, out)
    return out


def autoscale(a: np.ndarray, scale: float, raw_if_above: float) -> tuple[np.ndarray, bool]:
    """Raw HDF4 is never scaled; AppEEARS output may or may not be. Decide from the data and record it."""
    m = np.nanmax(a) if np.isfinite(a).any() else 0
    return (a * scale, True) if m > raw_if_above else (a, False)


# ----------------------------------------------------------------------------- per-date workers
def ndvi_date(args) -> tuple:
    date, ndvi_src, rel_src = args
    ndvi, scaled = autoscale(to_master(ndvi_src, Resampling.nearest), 1e-4, 1.5)
    rel = to_master(rel_src, Resampling.nearest) if rel_src else np.full(GRID["shape"], np.nan, "float32")
    return date, ndvi.astype("float32"), rel.astype("float32"), scaled


def _lst(raw, nodata):
    a = raw.astype("float32")
    a[(raw == 0) | ((nodata is not None) & (raw == nodata))] = np.nan
    return a


def lst_date(args) -> tuple:
    date, day_src, night_src, qcd_src, qcn_src = args
    out = []
    for vals, qcs in ((day_src, qcd_src), (night_src, qcn_src)):
        t, _ = autoscale(to_master(vals, Resampling.nearest, fn=_lst), 0.02, 1000)
        qc = to_master(qcs, Resampling.nearest, fn=lambda r, nd: r.astype("float32"))
        good = np.isfinite(qc) & ((np.nan_to_num(qc).astype("uint8") & 3) <= 1)  # bits 0-1: 00 good, 01 other
        out.append(np.where(good, t - 273.15, np.nan).astype("float32"))
    return date, out[0], out[1]


def _snow(raw, nodata):  # MOD10A2 Maximum_Snow_Extent: 200 snow, 25 no snow; everything else invalid
    a = np.full(raw.shape, np.nan, "float32")
    a[raw == 200] = 1.0
    a[raw == 25] = 0.0
    return a


def snow_date(args) -> tuple:
    date, srcs = args
    return date, to_master(srcs, Resampling.average, fn=_snow)


# ----------------------------------------------------------------------------- static layers
def static_layers(rc: tuple[np.ndarray, np.ndarray]) -> dict:
    r, c = rc
    out = {}
    lc_p, lw_p = flat(sources("MCD12Q1", "LC_Type1")), flat(sources("MCD12Q1", "LW"))
    out["lc_type1"] = to_master(lc_p, Resampling.mode,
                                fn=lambda raw, nd: np.where(raw == 255, np.nan, raw).astype("float32"))[r, c]
    for name, cls in {"grass_frac": 10, "ice_frac": 15, "urban_frac": 13, "barren_frac": 16}.items():
        out[name] = to_master(lc_p, Resampling.average,
                              fn=lambda raw, nd, k=cls: np.where(raw == 255, np.nan, raw == k).astype("float32"))[r, c]
    out["water_frac"] = to_master(lw_p, Resampling.average,
                                  fn=lambda raw, nd: np.where(raw == 255, np.nan, raw == 1).astype("float32"))[r, c]

    acc = {k: np.full(GRID["shape"], np.nan, "float32") for k in ("z", "z2", "slope", "steep", "asin", "acos")}
    for src in flat(sources("NASADEM", "NASADEM_HGT")):  # slope on the native 30 m grid first, then average
        raw, nodata, tr, crs = _open(src)
        z = _nodata_to_nan(raw, nodata if nodata is not None else -32768)
        lat = tr.f + tr.e * (np.arange(z.shape[0]) + 0.5)
        dy = abs(tr.e) * 111_320.0
        dx = tr.a * 111_320.0 * np.cos(np.deg2rad(lat))[:, None]
        gy_rows, gx_cols = np.gradient(z)
        gx, gy = gx_cols / dx, -gy_rows / dy                     # east, north
        slope = np.degrees(np.arctan(np.hypot(gx, gy))).astype("float32")
        aspect = np.arctan2(-gx, -gy)                            # downslope direction, clockwise from north
        layers = {"z": z, "z2": z * z, "slope": slope,
                  "steep": np.where(np.isnan(slope), np.nan, slope > STEEP_DEG).astype("float32"),
                  "asin": np.sin(aspect).astype("float32"), "acos": np.cos(aspect).astype("float32")}
        for k, arr in layers.items():
            tmp = np.full(GRID["shape"], np.nan, "float32")
            reproject(arr.astype("float32"), tmp, src_transform=tr, src_crs=crs,
                      dst_transform=GRID["transform"], dst_crs=GRID["crs"],
                      resampling=Resampling.average, src_nodata=np.nan, dst_nodata=np.nan)
            acc[k] = np.where(np.isnan(acc[k]), tmp, acc[k])
    elev = acc["z"]
    out["elev_mean"] = elev[r, c]
    out["elev_std"] = np.sqrt(np.clip(acc["z2"] - elev ** 2, 0, None))[r, c]
    out["slope_mean"], out["steep_share"] = acc["slope"][r, c], acc["steep"][r, c]
    out["aspect_sin"], out["aspect_cos"] = acc["asin"][r, c], acc["acos"][r, c]
    valid = np.isfinite(elev)
    num = uniform_filter(np.where(valid, elev, 0), TPI_WINDOW)
    den = uniform_filter(valid.astype("float32"), TPI_WINDOW)
    out["tpi"] = (elev - num / np.maximum(den, 1e-6))[r, c]
    return out


def mask_reason(df: pd.DataFrame) -> pd.Series:
    reason = pd.Series(pd.NA, index=df.index, dtype="string")
    rules = [("water", (df["water_frac"] > 0.5) | (df["lc_type1"] == 17)),
             ("ice", df["lc_type1"] == 15),
             ("urban", df["lc_type1"] == 13),
             ("too_steep", df["slope_mean"] > STEEP_DEG)]
    for name, cond in rules:  # first matching rule wins
        reason = reason.mask(reason.isna() & cond.fillna(False), name)
    return reason


# ----------------------------------------------------------------------------- weather (GES DISC)
def weather(cells: pd.DataFrame) -> pd.DataFrame:
    lon = xr.DataArray(cells["lon"].values, dims="cell")
    lat = xr.DataArray(cells["lat"].values, dims="cell")
    pad = 0.5
    lo0, lo1 = float(lon.min()) - pad, float(lon.max()) + pad
    la0, la1 = float(lat.min()) - pad, float(lat.max()) + pad
    ids = cells["cell_id"].values

    frames = []
    for p in sorted((RAW / "gesdisc" / "GPM_3IMERGM").glob("*.HDF5")):
        ym = re.search(r"\.(\d{8})-S", p.name).group(1)
        y, m = int(ym[:4]), int(ym[4:6])
        with xr.open_dataset(p, group="Grid", engine="h5netcdf") as ds:
            da = ds["precipitation"].squeeze(drop=True).sel(lon=slice(lo0, lo1), lat=slice(la0, la1)).load()
        mm_month = da.interp(lon=lon, lat=lat) * 24 * calendar.monthrange(y, m)[1]   # mm/hr -> mm/month
        frames.append(pd.DataFrame({"cell_id": ids, "year": y, "month": m,
                                    "precip": mm_month.values.astype("float32")}))
    precip = pd.concat(frames, ignore_index=True)

    frames = []
    for p in sorted((RAW / "gesdisc" / "GLDAS_NOAH025_M").glob("*.nc4")):
        ym = re.search(r"\.A(\d{6})\.", p.name).group(1)
        y, m = int(ym[:4]), int(ym[4:])
        with xr.open_dataset(p) as ds:
            sub = ds[["SoilMoi0_10cm_inst", "SWE_inst"]].squeeze(drop=True).sel(
                lon=slice(lo0, lo1), lat=slice(la0, la1)).load()
        pts = sub.interp(lon=lon, lat=lat)
        frames.append(pd.DataFrame({"cell_id": ids, "year": y, "month": m,
                                    "soil_moist": pts["SoilMoi0_10cm_inst"].values.astype("float32"),
                                    "swe": pts["SWE_inst"].values.astype("float32")}))
    gldas = pd.concat(frames, ignore_index=True)
    return precip.merge(gldas, on=["cell_id", "year", "month"], how="outer")


# ----------------------------------------------------------------------------- main
def monthly_mean(dated: list[tuple], r, c) -> dict:
    """Average 8-day composites into calendar months (by composite start date)."""
    groups: dict = {}
    for date, arr in dated:
        groups.setdefault((date.year, date.month), []).append(arr[r, c])
    with np.errstate(all="ignore"):
        return {k: np.nanmean(np.stack(v), axis=0).astype("float32") for k, v in groups.items()}


def build_cells() -> tuple[pd.DataFrame, np.ndarray, np.ndarray]:
    nrow, ncol = GRID["shape"]
    rr, cc = np.meshgrid(np.arange(nrow), np.arange(ncol), indexing="ij")
    x = GRID["left"] + (cc + 0.5) * PIX1K
    y = GRID["top"] - (rr + 0.5) * PIX1K
    lon, lat = Transformer.from_crs(GRID["crs"], 4326, always_xy=True).transform(x, y)
    aoi = gpd.read_file(AOI_DIR / "aoi_request.geojson").to_crs(4326)
    region = np.full(x.shape, "", dtype=object)
    for _, f in aoi.iterrows():
        region[(region == "") & contains_xy(f.geometry, lon, lat)] = f["name"]
    keep = region != ""
    r, c = rr[keep], cc[keep]
    cells = pd.DataFrame({
        # global MODIS row/col -> stable id that does not depend on our crop
        "cell_id": ((r + GRID["global_row0"]) * 43200 + (c + GRID["global_col0"])).astype("int64"),
        "row": r, "col": c, "x": x[keep], "y": y[keep], "lon": lon[keep], "lat": lat[keep],
        "region": region[keep].astype(str),
        "block_id": [f"{a}-{b}" for a, b in zip(r // BLOCK, c // BLOCK)],
        "block_row": (r // BLOCK).astype("int16"), "block_col": (c // BLOCK).astype("int16")})
    dist = gpd.read_file(AOI_DIR / "naryn_districts.geojson").to_crs(4326)
    cells["district"] = pd.Series(pd.NA, index=cells.index, dtype="string")
    for _, d in dist.iterrows():
        cells.loc[contains_xy(d.geometry, cells["lon"].values, cells["lat"].values), "district"] = d["shapeName"]
    cells["in_oblast"] = cells["district"].notna()
    adm1 = gpd.read_file(AOI_DIR / "geoBoundaries-KGZ-ADM1.geojson").to_crs(4326)
    gb = adm1[adm1["shapeName"].str.contains("Naryn")].geometry.iloc[0]
    cells["in_adm1_naryn"] = contains_xy(gb, cells["lon"].values, cells["lat"].values)
    cells["boundary_conflict"] = cells["in_oblast"] != cells["in_adm1_naryn"]
    return cells, r, c


def write_monthly(cells, ndvi, rel, aqua, lst_day, lst_night, snow, wx) -> list[int]:
    """One parquet file per year under monthly/year=YYYY/ (hive layout: pd.read_parquet(dir) adds `year`)."""
    nan = np.full(len(cells), np.nan, "float32")
    ids = cells["cell_id"].values
    years = sorted({y for y, _ in ndvi})
    for yr in years:
        parts = []
        for m in range(1, 13):
            k = (yr, m)
            if k not in ndvi:
                continue
            rk = rel.get(k, nan)
            cols = {"cell_id": ids, "year": yr, "month": m, "ndvi": ndvi[k], "ndvi_rel": rk,
                    "ndvi_ok": np.isin(rk, (0, 1)), "ndvi_aqua": aqua.get(k, nan)}
            if wx is not None:
                cols |= {"lst_day": lst_day.get(k, nan), "lst_night": lst_night.get(k, nan), "snow": snow.get(k, nan)}
            parts.append(pd.DataFrame(cols))
        df = pd.concat(parts, ignore_index=True)
        if wx is not None:
            df = df.merge(wx[wx["year"] == yr], on=["cell_id", "year", "month"], how="left")
        df["month"] = df["month"].astype("int8")
        dest = OUT / "monthly" / f"year={yr}"
        dest.mkdir(parents=True, exist_ok=True)
        pq.write_table(pa.Table.from_pandas(df.drop(columns=["year"]), preserve_index=False), dest / "part.parquet")
    return years


def main() -> None:
    global GRID
    ap = argparse.ArgumentParser()
    ap.add_argument("--only", choices=["static", "all"], default="all")
    args = ap.parse_args()
    t0 = time.time()
    OUT.mkdir(parents=True, exist_ok=True)
    manifest: dict = {"created_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "synthetic": False,
                      "partial": True,
                      "units": {"ndvi": "NDVI (scaled, -0.2..1)", "ndvi_rel": "MOD13A3 pixel reliability (0 good, 1 marginal, 2 snow/ice, 3 cloud)",
                                "ndvi_ok": "bool, reliability 0 or 1", "ndvi_aqua": "NDVI from MYD13A3, reliability 0-1 only",
                                "lst_day": "deg C, monthly mean of QA-good 8-day MOD11A2", "lst_night": "deg C",
                                "snow": "fraction 0-1 of 8-day MOD10A2 periods with snow (Mar-Sep only)",
                                "precip": "mm per month (IMERG Final V07, mm/hr x 24 x days)",
                                "soil_moist": "kg m-2 in 0-10 cm (GLDAS-2.1 Noah)", "swe": "kg m-2 (GLDAS-2.1 Noah)",
                                "elev_mean": "m", "slope_mean": "deg", "steep_share": f"share of 30 m pixels steeper than {STEEP_DEG} deg",
                                "lc_type1": "IGBP class (MCD12Q1 LC_Type1, 2020), mode over 500 m pixels"},
                      "datasets": {"MOD13A3": "061, doi:10.5067/MODIS/MOD13A3.061", "MYD13A3": "061, doi:10.5067/MODIS/MYD13A3.061",
                                   "MOD11A2": "061, doi:10.5067/MODIS/MOD11A2.061", "MOD10A2": "61, doi:10.5067/MODIS/MOD10A2.061",
                                   "MCD12Q1": "061, doi:10.5067/MODIS/MCD12Q1.061", "NASADEM_HGT": "001, doi:10.5067/MEaSUREs/NASADEM/NASADEM_HGT.001",
                                   "GPM_3IMERGM": "07, doi:10.5067/GPM/IMERG/3B-MONTH/07", "GLDAS_NOAH025_M": "2.1, doi:10.5067/SXAVCZFAQLNO"},
                      "notes": ["Oblast = 5 official districts (geoBoundaries ADM2). geoBoundaries ADM1 'Naryn Region' "
                                "disagrees (adds Toguz-Toro, misses ~1,900 km2 of Jumgal near Suusamyr); such cells "
                                "have boundary_conflict=true.",
                                f"mask_reason 'too_steep' uses slope_mean > {STEEP_DEG} deg (default, to be agreed)."]}

    GRID = build_master_grid()
    manifest["grid"] = {"crs": SINU, "shape": GRID["shape"], "res_m": PIX1K,
                        "origin_left_top": [GRID["left"], GRID["top"]], "block_cells": BLOCK,
                        "cell_id": "global_row * 43200 + global_col on the MODIS 1 km grid"}
    print(f"master grid {GRID['shape']} ({PIX1K:.3f} m)", flush=True)

    cells, r, c = build_cells()
    for k, v in static_layers((r, c)).items():
        cells[k] = v
    cells["mask_reason"] = mask_reason(cells)
    cells.to_parquet(OUT / "cells.parquet", index=False)
    manifest["cells"] = {"n": len(cells), "in_oblast": int(cells["in_oblast"].sum()),
                         "by_region": cells["region"].value_counts().astype(int).to_dict(),
                         "mask_reason_counts": {str(k): int(v) for k, v in
                                                cells["mask_reason"].value_counts(dropna=False).items()}}
    print(f"cells: {len(cells)} ({cells['in_oblast'].sum()} in oblast) {time.time() - t0:.0f}s", flush=True)
    if args.only == "static":
        manifest["sources_used"] = dict(SOURCES_USED)
        (OUT / "manifest.json").write_text(json.dumps(manifest, indent=2, default=str))
        print("STATIC_DONE", flush=True)
        return

    ndvi, rel, aqua, scaled_flags = {}, {}, {}, set()
    with ProcessPoolExecutor(WORKERS) as pool:
        nd, nr = sources("MOD13A3", "NDVI"), sources("MOD13A3", "pixel_reliability")
        for d, a, b, s in pool.map(ndvi_date, [(d, nd[d], nr.get(d, [])) for d in nd]):
            ndvi[(d.year, d.month)], rel[(d.year, d.month)] = a[r, c], b[r, c]
            scaled_flags.add(s)
        ad, ar = sources("MYD13A3", "NDVI"), sources("MYD13A3", "pixel_reliability")
        for d, a, b, _ in pool.map(ndvi_date, [(d, ad[d], ar.get(d, [])) for d in ad]):
            aqua[(d.year, d.month)] = np.where(np.isin(b[r, c], (0, 1)), a[r, c], np.nan)
        print(f"ndvi: {len(ndvi)} months, aqua: {len(aqua)} {time.time() - t0:.0f}s", flush=True)
        write_monthly(cells, ndvi, rel, aqua, {}, {}, {}, None)
        manifest["sources_used"] = dict(SOURCES_USED)
        manifest["monthly"] = {"columns": ["cell_id", "year", "month", "ndvi", "ndvi_rel", "ndvi_ok", "ndvi_aqua"],
                               "note": "PARTIAL: NDVI only; weather/LST/snow columns follow in the full build"}
        (OUT / "manifest.json").write_text(json.dumps(manifest, indent=2, default=str))
        print("PARTIAL_NDVI_DONE", flush=True)

        L = {k: sources("MOD11A2", k) for k in ("LST_Day_1km", "LST_Night_1km", "QC_Day", "QC_Night")}
        jobs = [(d, L["LST_Day_1km"][d], L["LST_Night_1km"].get(d, []), L["QC_Day"].get(d, []),
                 L["QC_Night"].get(d, [])) for d in L["LST_Day_1km"]]
        res = list(pool.map(lst_date, jobs))
        lst_day = monthly_mean([(d, a) for d, a, _ in res], r, c)
        lst_night = monthly_mean([(d, b) for d, _, b in res], r, c)
        sn = sources("MOD10A2", "Maximum_Snow_Extent")
        snow = monthly_mean(list(pool.map(snow_date, list(sn.items()))), r, c)
        print(f"lst: {len(lst_day)} months, snow: {len(snow)} months {time.time() - t0:.0f}s", flush=True)
    manifest["ndvi_scale_applied_by_us"] = sorted(scaled_flags)
    manifest["sources_used"] = dict(SOURCES_USED)

    wx = weather(cells)
    print(f"weather rows: {len(wx)} {time.time() - t0:.0f}s", flush=True)

    years = write_monthly(cells, ndvi, rel, aqua, lst_day, lst_night, snow, wx)
    manifest["partial"] = False
    manifest["monthly"] = {"years": [years[0], years[-1]], "months_ndvi": len(ndvi), "months_aqua": len(aqua),
                           "months_lst": len(lst_day), "months_snow": len(snow), "weather_rows": int(len(wx))}
    manifest["inputs"] = {f"{base.name}/{p.name}": len(list(p.glob("*")))
                          for base in (RAW / "appeears", RAW / "earthaccess", RAW / "gesdisc") if base.exists()
                          for p in base.iterdir() if p.is_dir()}
    (OUT / "manifest.json").write_text(json.dumps(manifest, indent=2, default=str))
    print(f"CUBE_DONE {time.time() - t0:.0f}s", flush=True)


if __name__ == "__main__":
    main()