Download training_code/scripts/20_build_cube.py from TrendeX90/expected-greenness: direct link, hf CLI and curl.
- Browser
- Download file 24.6 kB
-
https://huggingface.co/TrendeX90/expected-greenness/resolve/refs%2Fpr%2F1/training_code/scripts/20_build_cube.py
- Command line
-
hf download hf://TrendeX90/expected-greenness@refs/pr/1/training_code/scripts/20_build_cube.py
-
curl -L -o 20_build_cube.py https://huggingface.co/TrendeX90/expected-greenness/resolve/refs%2Fpr%2F1/training_code/scripts/20_build_cube.py
24.6 kB
| """Build the shared TrendeX data cube (F1) from the downloaded NASA rasters. | |
| python scripts/20_build_cube.py # everything | |
| python scripts/20_build_cube.py --only static # cells.parquet only (terrain, land cover, districts) | |
| Master grid = the global MODIS sinusoidal 1 km grid, cut to the request area (aoi_request.geojson). | |
| NDVI is never resampled (same grid -> nearest = copy). Other layers: 500 m / 30 m -> average or mode, | |
| 0.1 / 0.25 deg weather -> bilinear at cell centres. | |
| Each MODIS/NASADEM layer is read from AppEEARS GeoTIFFs when that AppEEARS task finished, otherwise | |
| from raw LP DAAC files fetched with earthaccess (HDF4 tiles / NASADEM zips). manifest.json records which. | |
| Outputs in data/cube/: | |
| cells.parquet one row per 1 km cell inside the request area | |
| monthly/year=YYYY/*.parquet one row per cell x month | |
| manifest.json inputs, sources, counts, scale checks, notes | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import calendar | |
| import json | |
| import re | |
| import time | |
| from concurrent.futures import ProcessPoolExecutor | |
| from pathlib import Path | |
| import geopandas as gpd | |
| import numpy as np | |
| import pandas as pd | |
| import pyarrow as pa | |
| import pyarrow.parquet as pq | |
| import rasterio | |
| import xarray as xr | |
| from pyproj import Transformer | |
| from rasterio.warp import Resampling, reproject | |
| from scipy.ndimage import uniform_filter | |
| from shapely import contains_xy | |
| ROOT = Path("/root/trendex-ml/data") | |
| RAW, AOI_DIR, OUT = ROOT / "raw", ROOT / "aoi", ROOT / "cube" | |
| PAT = re.compile(r"^(?P<prod>[A-Z0-9_]+\.\d{3})_(?P<layer>.+)_doy(?P<doy>\d{7})_aid(?P<aid>\d{4})\.tif$") | |
| BLOCK = 11 # 11 x 11 cells ~ 10.2 km blocks (F5, spatial CV) | |
| STEEP_DEG = 35.0 # "too steep" default; to be agreed with the formula owner (F1) | |
| TPI_WINDOW = 5 # cells (~5 km) for the topographic position index | |
| WORKERS = 32 | |
| SINU = "+proj=sinu +lon_0=0 +x_0=0 +y_0=0 +R=6371007.181 +units=m +no_defs" | |
| SINU_CRS = rasterio.crs.CRS.from_proj4(SINU) | |
| TILE = 1111950.5196666666 # MODIS tile size, m | |
| X0, Y0 = -20015109.355798, 10007554.677899 # upper-left corner of the global MODIS grid | |
| PIX1K = TILE / 1200 | |
| # Raw HDF4 tiles (LP DAAC fallback): layer key -> (SDS name, pixels per tile side) | |
| HDF_SDS = { | |
| "MOD13A3": {"NDVI": ("1 km monthly NDVI", 1200), "pixel_reliability": ("1 km monthly pixel reliability", 1200)}, | |
| "MYD13A3": {"NDVI": ("1 km monthly NDVI", 1200), "pixel_reliability": ("1 km monthly pixel reliability", 1200)}, | |
| "MOD11A2": {k: (k, 1200) for k in ("LST_Day_1km", "LST_Night_1km", "QC_Day", "QC_Night")}, | |
| "MOD10A2": {"Maximum_Snow_Extent": ("Maximum_Snow_Extent", 2400)}, | |
| "MCD12Q1": {"LC_Type1": ("LC_Type1", 2400), "LW": ("LW", 2400)}, | |
| } | |
| GRID: dict = {} # filled in main(), read by worker processes after fork | |
| SOURCES_USED: dict = {} | |
| # ----------------------------------------------------------------------------- sources | |
| def appeears_done(product: str) -> bool: | |
| marker = RAW / "appeears_done.json" | |
| return marker.exists() and json.loads(marker.read_text()).get(product) == "done" | |
| def appeears_index(product: str) -> pd.DataFrame: | |
| rows = [] | |
| for p in sorted((RAW / "appeears" / product).glob("*.tif")): | |
| m = PAT.match(p.name) | |
| if not m: | |
| continue | |
| year, doy = int(m["doy"][:4]), int(m["doy"][4:]) | |
| date = pd.Timestamp(year=year, month=1, day=1) + pd.Timedelta(days=doy - 1) | |
| rows.append({"path": str(p), "layer": m["layer"], "date": date}) | |
| if not rows: | |
| raise SystemExit(f"AppEEARS marked {product} done but no GeoTIFFs matched {PAT.pattern}") | |
| return pd.DataFrame(rows) | |
| def sources(product: str, layer: str) -> dict: | |
| """date -> list of raster sources for one layer (AppEEARS if finished, else raw LP DAAC files).""" | |
| if appeears_done(product): | |
| idx = appeears_index(product) | |
| idx = idx[idx["layer"].str.endswith(layer)] | |
| SOURCES_USED[product] = "AppEEARS" | |
| return {d: g["path"].tolist() for d, g in idx.groupby("date")} | |
| folder = RAW / "earthaccess" / product | |
| if product == "NASADEM": | |
| tiles = sorted(folder.glob("*.zip")) | |
| if not tiles: | |
| raise SystemExit("No NASADEM data from AppEEARS (not finished) or LP DAAC") | |
| SOURCES_USED[product] = "LP DAAC (earthaccess)" | |
| return {pd.Timestamp("2000-02-11"): [f"/vsizip/{z}/{z.stem.split('_')[-1]}.hgt" for z in tiles]} | |
| hdfs = sorted(folder.glob("*.hdf")) | |
| if not hdfs: | |
| raise SystemExit(f"No {product} data from AppEEARS (not finished) or LP DAAC ({folder})") | |
| SOURCES_USED[product] = "LP DAAC (earthaccess)" | |
| sds, npix = HDF_SDS[product][layer] | |
| by_date: dict = {} | |
| for p in hdfs: | |
| m = re.search(r"\.A(\d{4})(\d{3})\.", p.name) | |
| d = pd.Timestamp(year=int(m[1]), month=1, day=1) + pd.Timedelta(days=int(m[2]) - 1) | |
| by_date.setdefault(d, []).append(("hdf", str(p), sds, npix)) | |
| return dict(sorted(by_date.items())) | |
| def flat(d: dict) -> list: | |
| return [x for v in d.values() for x in v] | |
| def _open(src): | |
| """-> (raw array, nodata, transform, crs) for a GDAL path or an ('hdf', path, sds, npix) tuple.""" | |
| if isinstance(src, str): | |
| with rasterio.open(src) as s: | |
| return s.read(1), s.nodata, s.transform, s.crs | |
| _, path, sds, npix = src | |
| from pyhdf.SD import SD, SDC | |
| m = re.search(r"\.h(\d{2})v(\d{2})\.", Path(path).name) | |
| h, v = int(m[1]), int(m[2]) | |
| f = SD(path, SDC.READ) | |
| try: | |
| ds = f.select(sds) | |
| raw, attrs = ds.get(), ds.attributes() | |
| finally: | |
| f.end() | |
| pix = TILE / npix | |
| return raw, attrs.get("_FillValue"), rasterio.transform.from_origin(X0 + h * TILE, Y0 - v * TILE, pix, pix), SINU_CRS | |
| # ----------------------------------------------------------------------------- grid helpers | |
| def build_master_grid() -> dict: | |
| minx, miny, maxx, maxy = gpd.read_file(AOI_DIR / "aoi_request.geojson").to_crs(SINU).total_bounds | |
| c0, c1 = int(np.floor((minx - X0) / PIX1K)), int(np.ceil((maxx - X0) / PIX1K)) | |
| r0, r1 = int(np.floor((Y0 - maxy) / PIX1K)), int(np.ceil((Y0 - miny) / PIX1K)) | |
| left, top = X0 + c0 * PIX1K, Y0 - r0 * PIX1K | |
| return {"crs": SINU_CRS, "transform": rasterio.transform.from_origin(left, top, PIX1K, PIX1K), | |
| "shape": (r1 - r0, c1 - c0), "res": (PIX1K, PIX1K), "left": left, "top": top, | |
| "global_row0": r0, "global_col0": c0} | |
| def _nodata_to_nan(raw: np.ndarray, nodata) -> np.ndarray: | |
| a = raw.astype("float32") | |
| if nodata is not None: | |
| a[raw == nodata] = np.nan | |
| return a | |
| def to_master(srcs: list, resampling: Resampling, fn=None) -> np.ndarray: | |
| """Mosaic all pieces (AppEEARS area pieces or MODIS tiles) of one layer/date onto the master grid. | |
| fn(raw_array, nodata) -> float32 array with NaN for invalid pixels. Default: nodata -> NaN. | |
| """ | |
| out = np.full(GRID["shape"], np.nan, dtype="float32") | |
| for src in srcs: | |
| raw, nodata, transform, crs = _open(src) | |
| a = fn(raw, nodata) if fn else _nodata_to_nan(raw, nodata) | |
| tmp = np.full(GRID["shape"], np.nan, dtype="float32") | |
| reproject(a.astype("float32"), tmp, src_transform=transform, src_crs=crs, | |
| dst_transform=GRID["transform"], dst_crs=GRID["crs"], | |
| resampling=resampling, src_nodata=np.nan, dst_nodata=np.nan) | |
| out = np.where(np.isnan(out), tmp, out) | |
| return out | |
| def autoscale(a: np.ndarray, scale: float, raw_if_above: float) -> tuple[np.ndarray, bool]: | |
| """Raw HDF4 is never scaled; AppEEARS output may or may not be. Decide from the data and record it.""" | |
| m = np.nanmax(a) if np.isfinite(a).any() else 0 | |
| return (a * scale, True) if m > raw_if_above else (a, False) | |
| # ----------------------------------------------------------------------------- per-date workers | |
| def ndvi_date(args) -> tuple: | |
| date, ndvi_src, rel_src = args | |
| ndvi, scaled = autoscale(to_master(ndvi_src, Resampling.nearest), 1e-4, 1.5) | |
| rel = to_master(rel_src, Resampling.nearest) if rel_src else np.full(GRID["shape"], np.nan, "float32") | |
| return date, ndvi.astype("float32"), rel.astype("float32"), scaled | |
| def _lst(raw, nodata): | |
| a = raw.astype("float32") | |
| a[(raw == 0) | ((nodata is not None) & (raw == nodata))] = np.nan | |
| return a | |
| def lst_date(args) -> tuple: | |
| date, day_src, night_src, qcd_src, qcn_src = args | |
| out = [] | |
| for vals, qcs in ((day_src, qcd_src), (night_src, qcn_src)): | |
| t, _ = autoscale(to_master(vals, Resampling.nearest, fn=_lst), 0.02, 1000) | |
| qc = to_master(qcs, Resampling.nearest, fn=lambda r, nd: r.astype("float32")) | |
| good = np.isfinite(qc) & ((np.nan_to_num(qc).astype("uint8") & 3) <= 1) # bits 0-1: 00 good, 01 other | |
| out.append(np.where(good, t - 273.15, np.nan).astype("float32")) | |
| return date, out[0], out[1] | |
| def _snow(raw, nodata): # MOD10A2 Maximum_Snow_Extent: 200 snow, 25 no snow; everything else invalid | |
| a = np.full(raw.shape, np.nan, "float32") | |
| a[raw == 200] = 1.0 | |
| a[raw == 25] = 0.0 | |
| return a | |
| def snow_date(args) -> tuple: | |
| date, srcs = args | |
| return date, to_master(srcs, Resampling.average, fn=_snow) | |
| # ----------------------------------------------------------------------------- static layers | |
| def static_layers(rc: tuple[np.ndarray, np.ndarray]) -> dict: | |
| r, c = rc | |
| out = {} | |
| lc_p, lw_p = flat(sources("MCD12Q1", "LC_Type1")), flat(sources("MCD12Q1", "LW")) | |
| out["lc_type1"] = to_master(lc_p, Resampling.mode, | |
| fn=lambda raw, nd: np.where(raw == 255, np.nan, raw).astype("float32"))[r, c] | |
| for name, cls in {"grass_frac": 10, "ice_frac": 15, "urban_frac": 13, "barren_frac": 16}.items(): | |
| out[name] = to_master(lc_p, Resampling.average, | |
| fn=lambda raw, nd, k=cls: np.where(raw == 255, np.nan, raw == k).astype("float32"))[r, c] | |
| out["water_frac"] = to_master(lw_p, Resampling.average, | |
| fn=lambda raw, nd: np.where(raw == 255, np.nan, raw == 1).astype("float32"))[r, c] | |
| acc = {k: np.full(GRID["shape"], np.nan, "float32") for k in ("z", "z2", "slope", "steep", "asin", "acos")} | |
| for src in flat(sources("NASADEM", "NASADEM_HGT")): # slope on the native 30 m grid first, then average | |
| raw, nodata, tr, crs = _open(src) | |
| z = _nodata_to_nan(raw, nodata if nodata is not None else -32768) | |
| lat = tr.f + tr.e * (np.arange(z.shape[0]) + 0.5) | |
| dy = abs(tr.e) * 111_320.0 | |
| dx = tr.a * 111_320.0 * np.cos(np.deg2rad(lat))[:, None] | |
| gy_rows, gx_cols = np.gradient(z) | |
| gx, gy = gx_cols / dx, -gy_rows / dy # east, north | |
| slope = np.degrees(np.arctan(np.hypot(gx, gy))).astype("float32") | |
| aspect = np.arctan2(-gx, -gy) # downslope direction, clockwise from north | |
| layers = {"z": z, "z2": z * z, "slope": slope, | |
| "steep": np.where(np.isnan(slope), np.nan, slope > STEEP_DEG).astype("float32"), | |
| "asin": np.sin(aspect).astype("float32"), "acos": np.cos(aspect).astype("float32")} | |
| for k, arr in layers.items(): | |
| tmp = np.full(GRID["shape"], np.nan, "float32") | |
| reproject(arr.astype("float32"), tmp, src_transform=tr, src_crs=crs, | |
| dst_transform=GRID["transform"], dst_crs=GRID["crs"], | |
| resampling=Resampling.average, src_nodata=np.nan, dst_nodata=np.nan) | |
| acc[k] = np.where(np.isnan(acc[k]), tmp, acc[k]) | |
| elev = acc["z"] | |
| out["elev_mean"] = elev[r, c] | |
| out["elev_std"] = np.sqrt(np.clip(acc["z2"] - elev ** 2, 0, None))[r, c] | |
| out["slope_mean"], out["steep_share"] = acc["slope"][r, c], acc["steep"][r, c] | |
| out["aspect_sin"], out["aspect_cos"] = acc["asin"][r, c], acc["acos"][r, c] | |
| valid = np.isfinite(elev) | |
| num = uniform_filter(np.where(valid, elev, 0), TPI_WINDOW) | |
| den = uniform_filter(valid.astype("float32"), TPI_WINDOW) | |
| out["tpi"] = (elev - num / np.maximum(den, 1e-6))[r, c] | |
| return out | |
| def mask_reason(df: pd.DataFrame) -> pd.Series: | |
| reason = pd.Series(pd.NA, index=df.index, dtype="string") | |
| rules = [("water", (df["water_frac"] > 0.5) | (df["lc_type1"] == 17)), | |
| ("ice", df["lc_type1"] == 15), | |
| ("urban", df["lc_type1"] == 13), | |
| ("too_steep", df["slope_mean"] > STEEP_DEG)] | |
| for name, cond in rules: # first matching rule wins | |
| reason = reason.mask(reason.isna() & cond.fillna(False), name) | |
| return reason | |
| # ----------------------------------------------------------------------------- weather (GES DISC) | |
| def weather(cells: pd.DataFrame) -> pd.DataFrame: | |
| lon = xr.DataArray(cells["lon"].values, dims="cell") | |
| lat = xr.DataArray(cells["lat"].values, dims="cell") | |
| pad = 0.5 | |
| lo0, lo1 = float(lon.min()) - pad, float(lon.max()) + pad | |
| la0, la1 = float(lat.min()) - pad, float(lat.max()) + pad | |
| ids = cells["cell_id"].values | |
| frames = [] | |
| for p in sorted((RAW / "gesdisc" / "GPM_3IMERGM").glob("*.HDF5")): | |
| ym = re.search(r"\.(\d{8})-S", p.name).group(1) | |
| y, m = int(ym[:4]), int(ym[4:6]) | |
| with xr.open_dataset(p, group="Grid", engine="h5netcdf") as ds: | |
| da = ds["precipitation"].squeeze(drop=True).sel(lon=slice(lo0, lo1), lat=slice(la0, la1)).load() | |
| mm_month = da.interp(lon=lon, lat=lat) * 24 * calendar.monthrange(y, m)[1] # mm/hr -> mm/month | |
| frames.append(pd.DataFrame({"cell_id": ids, "year": y, "month": m, | |
| "precip": mm_month.values.astype("float32")})) | |
| precip = pd.concat(frames, ignore_index=True) | |
| frames = [] | |
| for p in sorted((RAW / "gesdisc" / "GLDAS_NOAH025_M").glob("*.nc4")): | |
| ym = re.search(r"\.A(\d{6})\.", p.name).group(1) | |
| y, m = int(ym[:4]), int(ym[4:]) | |
| with xr.open_dataset(p) as ds: | |
| sub = ds[["SoilMoi0_10cm_inst", "SWE_inst"]].squeeze(drop=True).sel( | |
| lon=slice(lo0, lo1), lat=slice(la0, la1)).load() | |
| pts = sub.interp(lon=lon, lat=lat) | |
| frames.append(pd.DataFrame({"cell_id": ids, "year": y, "month": m, | |
| "soil_moist": pts["SoilMoi0_10cm_inst"].values.astype("float32"), | |
| "swe": pts["SWE_inst"].values.astype("float32")})) | |
| gldas = pd.concat(frames, ignore_index=True) | |
| return precip.merge(gldas, on=["cell_id", "year", "month"], how="outer") | |
| # ----------------------------------------------------------------------------- main | |
| def monthly_mean(dated: list[tuple], r, c) -> dict: | |
| """Average 8-day composites into calendar months (by composite start date).""" | |
| groups: dict = {} | |
| for date, arr in dated: | |
| groups.setdefault((date.year, date.month), []).append(arr[r, c]) | |
| with np.errstate(all="ignore"): | |
| return {k: np.nanmean(np.stack(v), axis=0).astype("float32") for k, v in groups.items()} | |
| def build_cells() -> tuple[pd.DataFrame, np.ndarray, np.ndarray]: | |
| nrow, ncol = GRID["shape"] | |
| rr, cc = np.meshgrid(np.arange(nrow), np.arange(ncol), indexing="ij") | |
| x = GRID["left"] + (cc + 0.5) * PIX1K | |
| y = GRID["top"] - (rr + 0.5) * PIX1K | |
| lon, lat = Transformer.from_crs(GRID["crs"], 4326, always_xy=True).transform(x, y) | |
| aoi = gpd.read_file(AOI_DIR / "aoi_request.geojson").to_crs(4326) | |
| region = np.full(x.shape, "", dtype=object) | |
| for _, f in aoi.iterrows(): | |
| region[(region == "") & contains_xy(f.geometry, lon, lat)] = f["name"] | |
| keep = region != "" | |
| r, c = rr[keep], cc[keep] | |
| cells = pd.DataFrame({ | |
| # global MODIS row/col -> stable id that does not depend on our crop | |
| "cell_id": ((r + GRID["global_row0"]) * 43200 + (c + GRID["global_col0"])).astype("int64"), | |
| "row": r, "col": c, "x": x[keep], "y": y[keep], "lon": lon[keep], "lat": lat[keep], | |
| "region": region[keep].astype(str), | |
| "block_id": [f"{a}-{b}" for a, b in zip(r // BLOCK, c // BLOCK)], | |
| "block_row": (r // BLOCK).astype("int16"), "block_col": (c // BLOCK).astype("int16")}) | |
| dist = gpd.read_file(AOI_DIR / "naryn_districts.geojson").to_crs(4326) | |
| cells["district"] = pd.Series(pd.NA, index=cells.index, dtype="string") | |
| for _, d in dist.iterrows(): | |
| cells.loc[contains_xy(d.geometry, cells["lon"].values, cells["lat"].values), "district"] = d["shapeName"] | |
| cells["in_oblast"] = cells["district"].notna() | |
| adm1 = gpd.read_file(AOI_DIR / "geoBoundaries-KGZ-ADM1.geojson").to_crs(4326) | |
| gb = adm1[adm1["shapeName"].str.contains("Naryn")].geometry.iloc[0] | |
| cells["in_adm1_naryn"] = contains_xy(gb, cells["lon"].values, cells["lat"].values) | |
| cells["boundary_conflict"] = cells["in_oblast"] != cells["in_adm1_naryn"] | |
| return cells, r, c | |
| def write_monthly(cells, ndvi, rel, aqua, lst_day, lst_night, snow, wx) -> list[int]: | |
| """One parquet file per year under monthly/year=YYYY/ (hive layout: pd.read_parquet(dir) adds `year`).""" | |
| nan = np.full(len(cells), np.nan, "float32") | |
| ids = cells["cell_id"].values | |
| years = sorted({y for y, _ in ndvi}) | |
| for yr in years: | |
| parts = [] | |
| for m in range(1, 13): | |
| k = (yr, m) | |
| if k not in ndvi: | |
| continue | |
| rk = rel.get(k, nan) | |
| cols = {"cell_id": ids, "year": yr, "month": m, "ndvi": ndvi[k], "ndvi_rel": rk, | |
| "ndvi_ok": np.isin(rk, (0, 1)), "ndvi_aqua": aqua.get(k, nan)} | |
| if wx is not None: | |
| cols |= {"lst_day": lst_day.get(k, nan), "lst_night": lst_night.get(k, nan), "snow": snow.get(k, nan)} | |
| parts.append(pd.DataFrame(cols)) | |
| df = pd.concat(parts, ignore_index=True) | |
| if wx is not None: | |
| df = df.merge(wx[wx["year"] == yr], on=["cell_id", "year", "month"], how="left") | |
| df["month"] = df["month"].astype("int8") | |
| dest = OUT / "monthly" / f"year={yr}" | |
| dest.mkdir(parents=True, exist_ok=True) | |
| pq.write_table(pa.Table.from_pandas(df.drop(columns=["year"]), preserve_index=False), dest / "part.parquet") | |
| return years | |
| def main() -> None: | |
| global GRID | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--only", choices=["static", "all"], default="all") | |
| args = ap.parse_args() | |
| t0 = time.time() | |
| OUT.mkdir(parents=True, exist_ok=True) | |
| manifest: dict = {"created_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "synthetic": False, | |
| "partial": True, | |
| "units": {"ndvi": "NDVI (scaled, -0.2..1)", "ndvi_rel": "MOD13A3 pixel reliability (0 good, 1 marginal, 2 snow/ice, 3 cloud)", | |
| "ndvi_ok": "bool, reliability 0 or 1", "ndvi_aqua": "NDVI from MYD13A3, reliability 0-1 only", | |
| "lst_day": "deg C, monthly mean of QA-good 8-day MOD11A2", "lst_night": "deg C", | |
| "snow": "fraction 0-1 of 8-day MOD10A2 periods with snow (Mar-Sep only)", | |
| "precip": "mm per month (IMERG Final V07, mm/hr x 24 x days)", | |
| "soil_moist": "kg m-2 in 0-10 cm (GLDAS-2.1 Noah)", "swe": "kg m-2 (GLDAS-2.1 Noah)", | |
| "elev_mean": "m", "slope_mean": "deg", "steep_share": f"share of 30 m pixels steeper than {STEEP_DEG} deg", | |
| "lc_type1": "IGBP class (MCD12Q1 LC_Type1, 2020), mode over 500 m pixels"}, | |
| "datasets": {"MOD13A3": "061, doi:10.5067/MODIS/MOD13A3.061", "MYD13A3": "061, doi:10.5067/MODIS/MYD13A3.061", | |
| "MOD11A2": "061, doi:10.5067/MODIS/MOD11A2.061", "MOD10A2": "61, doi:10.5067/MODIS/MOD10A2.061", | |
| "MCD12Q1": "061, doi:10.5067/MODIS/MCD12Q1.061", "NASADEM_HGT": "001, doi:10.5067/MEaSUREs/NASADEM/NASADEM_HGT.001", | |
| "GPM_3IMERGM": "07, doi:10.5067/GPM/IMERG/3B-MONTH/07", "GLDAS_NOAH025_M": "2.1, doi:10.5067/SXAVCZFAQLNO"}, | |
| "notes": ["Oblast = 5 official districts (geoBoundaries ADM2). geoBoundaries ADM1 'Naryn Region' " | |
| "disagrees (adds Toguz-Toro, misses ~1,900 km2 of Jumgal near Suusamyr); such cells " | |
| "have boundary_conflict=true.", | |
| f"mask_reason 'too_steep' uses slope_mean > {STEEP_DEG} deg (default, to be agreed)."]} | |
| GRID = build_master_grid() | |
| manifest["grid"] = {"crs": SINU, "shape": GRID["shape"], "res_m": PIX1K, | |
| "origin_left_top": [GRID["left"], GRID["top"]], "block_cells": BLOCK, | |
| "cell_id": "global_row * 43200 + global_col on the MODIS 1 km grid"} | |
| print(f"master grid {GRID['shape']} ({PIX1K:.3f} m)", flush=True) | |
| cells, r, c = build_cells() | |
| for k, v in static_layers((r, c)).items(): | |
| cells[k] = v | |
| cells["mask_reason"] = mask_reason(cells) | |
| cells.to_parquet(OUT / "cells.parquet", index=False) | |
| manifest["cells"] = {"n": len(cells), "in_oblast": int(cells["in_oblast"].sum()), | |
| "by_region": cells["region"].value_counts().astype(int).to_dict(), | |
| "mask_reason_counts": {str(k): int(v) for k, v in | |
| cells["mask_reason"].value_counts(dropna=False).items()}} | |
| print(f"cells: {len(cells)} ({cells['in_oblast'].sum()} in oblast) {time.time() - t0:.0f}s", flush=True) | |
| if args.only == "static": | |
| manifest["sources_used"] = dict(SOURCES_USED) | |
| (OUT / "manifest.json").write_text(json.dumps(manifest, indent=2, default=str)) | |
| print("STATIC_DONE", flush=True) | |
| return | |
| ndvi, rel, aqua, scaled_flags = {}, {}, {}, set() | |
| with ProcessPoolExecutor(WORKERS) as pool: | |
| nd, nr = sources("MOD13A3", "NDVI"), sources("MOD13A3", "pixel_reliability") | |
| for d, a, b, s in pool.map(ndvi_date, [(d, nd[d], nr.get(d, [])) for d in nd]): | |
| ndvi[(d.year, d.month)], rel[(d.year, d.month)] = a[r, c], b[r, c] | |
| scaled_flags.add(s) | |
| ad, ar = sources("MYD13A3", "NDVI"), sources("MYD13A3", "pixel_reliability") | |
| for d, a, b, _ in pool.map(ndvi_date, [(d, ad[d], ar.get(d, [])) for d in ad]): | |
| aqua[(d.year, d.month)] = np.where(np.isin(b[r, c], (0, 1)), a[r, c], np.nan) | |
| print(f"ndvi: {len(ndvi)} months, aqua: {len(aqua)} {time.time() - t0:.0f}s", flush=True) | |
| write_monthly(cells, ndvi, rel, aqua, {}, {}, {}, None) | |
| manifest["sources_used"] = dict(SOURCES_USED) | |
| manifest["monthly"] = {"columns": ["cell_id", "year", "month", "ndvi", "ndvi_rel", "ndvi_ok", "ndvi_aqua"], | |
| "note": "PARTIAL: NDVI only; weather/LST/snow columns follow in the full build"} | |
| (OUT / "manifest.json").write_text(json.dumps(manifest, indent=2, default=str)) | |
| print("PARTIAL_NDVI_DONE", flush=True) | |
| L = {k: sources("MOD11A2", k) for k in ("LST_Day_1km", "LST_Night_1km", "QC_Day", "QC_Night")} | |
| jobs = [(d, L["LST_Day_1km"][d], L["LST_Night_1km"].get(d, []), L["QC_Day"].get(d, []), | |
| L["QC_Night"].get(d, [])) for d in L["LST_Day_1km"]] | |
| res = list(pool.map(lst_date, jobs)) | |
| lst_day = monthly_mean([(d, a) for d, a, _ in res], r, c) | |
| lst_night = monthly_mean([(d, b) for d, _, b in res], r, c) | |
| sn = sources("MOD10A2", "Maximum_Snow_Extent") | |
| snow = monthly_mean(list(pool.map(snow_date, list(sn.items()))), r, c) | |
| print(f"lst: {len(lst_day)} months, snow: {len(snow)} months {time.time() - t0:.0f}s", flush=True) | |
| manifest["ndvi_scale_applied_by_us"] = sorted(scaled_flags) | |
| manifest["sources_used"] = dict(SOURCES_USED) | |
| wx = weather(cells) | |
| print(f"weather rows: {len(wx)} {time.time() - t0:.0f}s", flush=True) | |
| years = write_monthly(cells, ndvi, rel, aqua, lst_day, lst_night, snow, wx) | |
| manifest["partial"] = False | |
| manifest["monthly"] = {"years": [years[0], years[-1]], "months_ndvi": len(ndvi), "months_aqua": len(aqua), | |
| "months_lst": len(lst_day), "months_snow": len(snow), "weather_rows": int(len(wx))} | |
| manifest["inputs"] = {f"{base.name}/{p.name}": len(list(p.glob("*"))) | |
| for base in (RAW / "appeears", RAW / "earthaccess", RAW / "gesdisc") if base.exists() | |
| for p in base.iterdir() if p.is_dir()} | |
| (OUT / "manifest.json").write_text(json.dumps(manifest, indent=2, default=str)) | |
| print(f"CUBE_DONE {time.time() - t0:.0f}s", flush=True) | |
| if __name__ == "__main__": | |
| main() | |