Add full training code for model v1 (same files as PR #1)
Browse files- training_code/README.md +19 -0
- training_code/configs/model_v1.yaml +87 -0
- training_code/requirements.lock +181 -0
- training_code/scripts/00_smoke.py +190 -0
- training_code/scripts/05_aoi.py +104 -0
- training_code/scripts/10_download.py +233 -0
- training_code/scripts/20_build_cube.py +466 -0
- training_code/scripts/25_qa_cube.py +66 -0
- training_code/scripts/30_train.py +347 -0
- training_code/scripts/40_products.py +154 -0
- training_code/scripts/45_package.py +157 -0
- training_code/scripts/50_push_hf.py +110 -0
- training_code/scripts/55_scenarios.py +90 -0
- training_code/scripts/60_verify_hf.py +57 -0
- training_code/scripts/setup_box.sh +22 -0
- training_code/src/trendex_ml/__init__.py +0 -0
- training_code/src/trendex_ml/predict.py +105 -0
- training_code/src/trendex_ml/stats/__init__.py +0 -0
- training_code/src/trendex_ml/stats/trend.py +159 -0
training_code/README.md
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# TrendeX ML: training code (model v1)
|
| 2 |
+
|
| 3 |
+
Everything that produced `TrendeX90/expected-greenness@v1` and `TrendeX90/naryn-cube`, in run order:
|
| 4 |
+
|
| 5 |
+
| Step | Script | What it does |
|
| 6 |
+
|---|---|---|
|
| 7 |
+
| 0 | `scripts/setup_box.sh`, `scripts/00_smoke.py` | environment; prove each NASA source opens |
|
| 8 |
+
| 1 | `scripts/05_aoi.py` | Naryn oblast = 5 ADM2 districts + 20 km buffer, known-place boxes |
|
| 9 |
+
| 2 | `scripts/10_download.py` | MODIS / NASADEM from LP DAAC, IMERG / GLDAS from GES DISC (earthaccess) |
|
| 10 |
+
| 3 | `scripts/20_build_cube.py`, `scripts/25_qa_cube.py` | 1 km MODIS-grid cube + QA report |
|
| 11 |
+
| 4 | `configs/model_v1.yaml` | protocol, committed in git before the first fit |
|
| 12 |
+
| 5 | `scripts/30_train.py` | tuning, temporal + spatial tests, baselines, pre-registered gate, out-of-fold predictions |
|
| 13 |
+
| 6 | `scripts/40_products.py` | residual trends (shared `src/trendex_ml/stats/trend.py`), placebo, season vs weather, Aqua-Terra |
|
| 14 |
+
| 7 | `scripts/45_package.py`, `scripts/50_push_hf.py` | model repo with golden test; push to HF |
|
| 15 |
+
| 8 | `scripts/55_scenarios.py`, `scripts/60_verify_hf.py` | precomputed what-if grid for the static site; verify HF from a clean machine |
|
| 16 |
+
|
| 17 |
+
Run on one GPU box (RTX 4070, 40 cores) in about 3 hours including downloads; training itself took ~7 minutes.
|
| 18 |
+
`requirements.lock` is the exact environment. Credentials (Earthdata, HF) are read from `~/.netrc` and the HF token cache; none are in this code.
|
| 19 |
+
AI use: implementation was assisted by Claude (Anthropic).
|
training_code/configs/model_v1.yaml
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# TrendeX "expected greenness" model, protocol v1.
|
| 2 |
+
# Locked BEFORE the first model fit. Every output records the git commit of this file.
|
| 3 |
+
# Changing anything below after results are seen = a new version (model_v2.yaml), never an edit.
|
| 4 |
+
|
| 5 |
+
name: expected-greenness
|
| 6 |
+
version: v1
|
| 7 |
+
question: >
|
| 8 |
+
How much of each cell's summer greenness anomaly is explained by weather (precipitation, temperature,
|
| 9 |
+
snow, soil moisture) and terrain alone? Residuals = change not explained by weather (never called a cause).
|
| 10 |
+
|
| 11 |
+
data:
|
| 12 |
+
cube: TrendeX90/naryn-cube # cells.parquet + monthly/
|
| 13 |
+
cells: in_oblast and mask_reason is null # pasture candidates in the 5 districts
|
| 14 |
+
months: [5, 6, 7, 8, 9]
|
| 15 |
+
years: [2001, 2025] # 2000 dropped: IMERG starts 2000-06, no Oct-Apr precipitation
|
| 16 |
+
target_rows: ndvi_ok == true # pixel reliability 0-1 only; others are not imputed
|
| 17 |
+
|
| 18 |
+
target:
|
| 19 |
+
name: ndvi_anomaly
|
| 20 |
+
definition: ndvi - clim(cell, month)
|
| 21 |
+
climatology: >
|
| 22 |
+
mean NDVI of the same cell and month over the TRAINING years of the current split, excluding the row's own
|
| 23 |
+
year (leave-one-year-out inside training years). Never uses test years.
|
| 24 |
+
|
| 25 |
+
features:
|
| 26 |
+
static: [elev_mean, elev_std, slope_mean, steep_share, aspect_sin, aspect_cos, tpi, grass_frac, barren_frac, lc_type1]
|
| 27 |
+
calendar: [month]
|
| 28 |
+
climatology: [clim_ndvi]
|
| 29 |
+
weather_anomalies: # anomaly = value - full-period (2001-2025) mean for the same cell and month
|
| 30 |
+
- precip_m # this month, mm
|
| 31 |
+
- precip_3m # this + 2 previous months
|
| 32 |
+
- precip_winter # Oct (previous year) .. Apr sum
|
| 33 |
+
- lst_day_m
|
| 34 |
+
- lst_night_m
|
| 35 |
+
- lst_day_prev # previous month
|
| 36 |
+
- snow_spring # Mar-May mean snow fraction
|
| 37 |
+
- snow_m
|
| 38 |
+
- soil_m # GLDAS 0-10 cm soil moisture, this month
|
| 39 |
+
- swe_apr # GLDAS snow water equivalent, April
|
| 40 |
+
weather_climatology_note: >
|
| 41 |
+
Weather anomalies use a full-period climatology. These are model inputs, not the target, so this leaks no
|
| 42 |
+
target information; the target climatology is fold-aware (see above).
|
| 43 |
+
excluded_on_purpose: [year, lat, lon, x, y, any NDVI lag]
|
| 44 |
+
|
| 45 |
+
splits:
|
| 46 |
+
temporal_test: {train: [2001, 2019], test: [2020, 2025]}
|
| 47 |
+
spatial_test: {kind: GroupKFold, folds: 5, group: superblock_33x33_cells} # all years, unseen areas
|
| 48 |
+
cross_fit_for_residuals: {kind: contiguous_year_blocks, blocks: [[2001, 2005], [2006, 2010], [2011, 2015], [2016, 2020], [2021, 2025]]}
|
| 49 |
+
inner_validation_for_tuning: {train: [2001, 2015], valid: [2016, 2019]} # never touches 2020-2025
|
| 50 |
+
|
| 51 |
+
baselines:
|
| 52 |
+
B0_climatology: anomaly = 0
|
| 53 |
+
B1_ridge: sklearn Ridge on the same standardized features, alpha chosen on the inner validation split
|
| 54 |
+
|
| 55 |
+
model:
|
| 56 |
+
kind: xgboost
|
| 57 |
+
params_fixed: {tree_method: hist, device: cuda, objective: "reg:squarederror"}
|
| 58 |
+
tuning: {sampler: optuna_tpe, max_minutes: 25, max_trials: 40, metric: rmse_inner_valid}
|
| 59 |
+
seeds: [0, 1, 2, 3, 4]
|
| 60 |
+
|
| 61 |
+
metrics:
|
| 62 |
+
primary: skill_vs_ridge = 1 - MSE_model / MSE_ridge
|
| 63 |
+
others: [rmse, mae, skill_vs_climatology, anomaly_correlation, tercile_balanced_accuracy]
|
| 64 |
+
uncertainty:
|
| 65 |
+
temporal: bootstrap over test years (2000 resamples)
|
| 66 |
+
spatial: bootstrap over superblocks (2000 resamples)
|
| 67 |
+
report_per: [cell_month, block_summer_mean, district_summer_mean]
|
| 68 |
+
|
| 69 |
+
gate:
|
| 70 |
+
rule: ship xgboost only if the 95% CI lower bound of skill_vs_ridge > 0 on BOTH temporal and spatial tests
|
| 71 |
+
otherwise: ship ridge as the expected-greenness model and say so on the Methods page
|
| 72 |
+
|
| 73 |
+
intervals:
|
| 74 |
+
method: split conformal on cross-fitted residuals of training years, per month
|
| 75 |
+
levels: [0.1, 0.9]
|
| 76 |
+
check: empirical coverage on the temporal test years
|
| 77 |
+
|
| 78 |
+
residual_trends:
|
| 79 |
+
series: summer (Jun-Aug) mean of out-of-fold residuals per cell and year
|
| 80 |
+
test: trend.py trend_1d (Sen slope + Mann-Kendall, Hamed-Rao) - copied unchanged from the backend
|
| 81 |
+
multiple_testing: Benjamini-Hochberg across cells, q < 0.05
|
| 82 |
+
min_years: 18
|
| 83 |
+
placebo: shuffle years within each cell, 200 permutations, mk_vectorized
|
| 84 |
+
wording: "change not explained by precipitation, temperature, snow and soil moisture"
|
| 85 |
+
|
| 86 |
+
cross_check:
|
| 87 |
+
aqua_vs_terra: per-block summer NDVI trend 2003-2025 from MYD13A3 vs MOD13A3; report sign agreement
|
training_code/requirements.lock
ADDED
|
@@ -0,0 +1,181 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
affine==3.0.1
|
| 2 |
+
aiobotocore==3.10.0
|
| 3 |
+
aiohappyeyeballs==2.7.1
|
| 4 |
+
aiohttp==3.14.5
|
| 5 |
+
aioitertools==0.13.0
|
| 6 |
+
aiosignal==1.4.0
|
| 7 |
+
alembic==1.20.1
|
| 8 |
+
anyio==4.16.0
|
| 9 |
+
archspec @ file:///home/conda/feedstock_root/build_artifacts/archspec_1708969572489/work
|
| 10 |
+
asttokens @ file:///home/conda/feedstock_root/build_artifacts/asttokens_1698341106958/work
|
| 11 |
+
astunparse==1.6.3
|
| 12 |
+
attrs @ file:///home/conda/feedstock_root/build_artifacts/attrs_1722977137225/work
|
| 13 |
+
beautifulsoup4 @ file:///home/conda/feedstock_root/build_artifacts/beautifulsoup4_1705564648255/work
|
| 14 |
+
boltons @ file:///home/conda/feedstock_root/build_artifacts/boltons_1711936407380/work
|
| 15 |
+
botocore==1.43.117
|
| 16 |
+
bounded-pool-executor==0.0.3
|
| 17 |
+
Brotli @ file:///home/conda/feedstock_root/build_artifacts/brotli-split_1725267488082/work
|
| 18 |
+
certifi @ file:///home/conda/feedstock_root/build_artifacts/certifi_1725278078093/work/certifi
|
| 19 |
+
cffi @ file:///home/conda/feedstock_root/build_artifacts/cffi_1725560564262/work
|
| 20 |
+
cftime==1.6.6.1
|
| 21 |
+
chardet @ file:///home/conda/feedstock_root/build_artifacts/chardet_1724954797915/work
|
| 22 |
+
charset-normalizer @ file:///home/conda/feedstock_root/build_artifacts/charset-normalizer_1728479282467/work
|
| 23 |
+
click==8.5.1
|
| 24 |
+
click-plugins==1.1.1.2
|
| 25 |
+
cligj==0.7.2
|
| 26 |
+
cloudpickle==3.1.2
|
| 27 |
+
colorama @ file:///home/conda/feedstock_root/build_artifacts/colorama_1666700638685/work
|
| 28 |
+
colorlog==6.12.0
|
| 29 |
+
conda @ file:///home/conda/feedstock_root/build_artifacts/conda_1729155142459/work
|
| 30 |
+
conda-build @ file:///home/conda/feedstock_root/build_artifacts/conda-build_1727204250023/work
|
| 31 |
+
conda-libmamba-solver @ file:///home/conda/feedstock_root/build_artifacts/conda-libmamba-solver_1727359833193/work/src
|
| 32 |
+
conda-package-handling @ file:///home/conda/feedstock_root/build_artifacts/conda-package-handling_1729006966809/work
|
| 33 |
+
conda_index @ file:///home/conda/feedstock_root/build_artifacts/conda-index_1718383105992/work
|
| 34 |
+
conda_package_streaming @ file:///home/conda/feedstock_root/build_artifacts/conda-package-streaming_1729004031731/work
|
| 35 |
+
decorator @ file:///home/conda/feedstock_root/build_artifacts/decorator_1641555617451/work
|
| 36 |
+
distro @ file:///home/conda/feedstock_root/build_artifacts/distro_1704321475663/work
|
| 37 |
+
dnspython==2.7.0
|
| 38 |
+
earthaccess==0.17.0
|
| 39 |
+
exceptiongroup @ file:///home/conda/feedstock_root/build_artifacts/exceptiongroup_1720869315914/work
|
| 40 |
+
executing @ file:///home/conda/feedstock_root/build_artifacts/executing_1725214404607/work
|
| 41 |
+
expecttest==0.2.1
|
| 42 |
+
filelock @ file:///home/conda/feedstock_root/build_artifacts/filelock_1726613473834/work
|
| 43 |
+
formulaic==1.2.2
|
| 44 |
+
frozendict @ file:///home/conda/feedstock_root/build_artifacts/frozendict_1728841334936/work
|
| 45 |
+
frozenlist==1.8.0
|
| 46 |
+
fsspec==2026.9.0
|
| 47 |
+
geopandas==1.2.0
|
| 48 |
+
h11==0.16.0
|
| 49 |
+
h2 @ file:///home/conda/feedstock_root/build_artifacts/h2_1634280454336/work
|
| 50 |
+
h5netcdf==1.8.3
|
| 51 |
+
h5py==3.16.0
|
| 52 |
+
hf-xet==1.8.0
|
| 53 |
+
hpack==4.0.0
|
| 54 |
+
httpcore2==2.16.0
|
| 55 |
+
httpx2==2.16.0
|
| 56 |
+
huggingface_hub==2.4.0
|
| 57 |
+
hyperframe @ file:///home/conda/feedstock_root/build_artifacts/hyperframe_1619110129307/work
|
| 58 |
+
hypothesis==6.115.5
|
| 59 |
+
idna==3.20
|
| 60 |
+
importlib_resources @ file:///home/conda/feedstock_root/build_artifacts/importlib_resources_1725921340658/work
|
| 61 |
+
iniconfig==2.3.1
|
| 62 |
+
interface_meta==2.0.1
|
| 63 |
+
ipython @ file:///home/conda/feedstock_root/build_artifacts/ipython_1729866374957/work
|
| 64 |
+
jedi @ file:///home/conda/feedstock_root/build_artifacts/jedi_1696326070614/work
|
| 65 |
+
Jinja2 @ file:///home/conda/feedstock_root/build_artifacts/jinja2_1715127149914/work
|
| 66 |
+
jmespath==1.1.0
|
| 67 |
+
joblib==1.6.0
|
| 68 |
+
jsonpatch @ file:///home/conda/feedstock_root/build_artifacts/jsonpatch_1695536281965/work
|
| 69 |
+
jsonpointer @ file:///home/conda/feedstock_root/build_artifacts/jsonpointer_1725302941992/work
|
| 70 |
+
jsonschema @ file:///home/conda/feedstock_root/build_artifacts/jsonschema_1720529478715/work
|
| 71 |
+
jsonschema-specifications @ file:///tmp/tmpvslgxhz5/src
|
| 72 |
+
libarchive-c @ file:///home/conda/feedstock_root/build_artifacts/python-libarchive-c_1725302626023/work
|
| 73 |
+
libmambapy @ file:///home/conda/feedstock_root/build_artifacts/mamba-split_1727883551957/work/libmambapy
|
| 74 |
+
lief @ file:///home/conda/feedstock_root/build_artifacts/lief_1726040283347/work/api/python
|
| 75 |
+
lightgbm==4.7.0
|
| 76 |
+
lintrunner==0.12.5
|
| 77 |
+
llvmlite==0.50.0
|
| 78 |
+
Mako==1.4.3
|
| 79 |
+
mamba @ file:///home/conda/feedstock_root/build_artifacts/mamba-split_1727883551957/work/mamba
|
| 80 |
+
MarkupSafe @ file:///home/conda/feedstock_root/build_artifacts/markupsafe_1729351299924/work
|
| 81 |
+
matplotlib-inline @ file:///home/conda/feedstock_root/build_artifacts/matplotlib-inline_1713250518406/work
|
| 82 |
+
menuinst @ file:///home/conda/feedstock_root/build_artifacts/menuinst_1725359023560/work
|
| 83 |
+
more-itertools @ file:///home/conda/feedstock_root/build_artifacts/more-itertools_1725630081114/work
|
| 84 |
+
mpmath==1.3.0
|
| 85 |
+
multidict==7.2.1
|
| 86 |
+
multimethod==2.1
|
| 87 |
+
narwhals==2.28.0
|
| 88 |
+
netCDF4==1.7.5.1.post1
|
| 89 |
+
networkx==3.4.2
|
| 90 |
+
ninja==1.11.1.1
|
| 91 |
+
numba==0.68.1
|
| 92 |
+
numpy @ file:///home/conda/feedstock_root/build_artifacts/numpy_1728239949208/work/dist/numpy-2.1.2-cp311-cp311-linux_x86_64.whl#sha256=768addcb66d11bf95f7d2036c7b2595c638ef6539dba5f6a98faa0cdd9170ce8
|
| 93 |
+
nvidia-cublas-cu12==12.4.5.8
|
| 94 |
+
nvidia-cuda-cupti-cu12==12.4.127
|
| 95 |
+
nvidia-cuda-nvrtc-cu12==12.4.127
|
| 96 |
+
nvidia-cuda-runtime-cu12==12.4.127
|
| 97 |
+
nvidia-cudnn-cu12==9.1.0.70
|
| 98 |
+
nvidia-cufft-cu12==11.2.1.3
|
| 99 |
+
nvidia-curand-cu12==10.3.5.147
|
| 100 |
+
nvidia-cusolver-cu12==11.6.1.9
|
| 101 |
+
nvidia-cusparse-cu12==12.3.1.170
|
| 102 |
+
nvidia-nccl-cu12==2.21.5
|
| 103 |
+
nvidia-nvjitlink-cu12==12.4.127
|
| 104 |
+
nvidia-nvtx-cu12==12.4.127
|
| 105 |
+
optree==0.13.0
|
| 106 |
+
optuna==5.0.0
|
| 107 |
+
packaging==26.3
|
| 108 |
+
pandas==3.1.1
|
| 109 |
+
parso @ file:///home/conda/feedstock_root/build_artifacts/parso_1712320355065/work
|
| 110 |
+
patsy==1.0.3
|
| 111 |
+
pexpect @ file:///home/conda/feedstock_root/build_artifacts/pexpect_1706113125309/work
|
| 112 |
+
pickleshare @ file:///home/conda/feedstock_root/build_artifacts/pickleshare_1602536217715/work
|
| 113 |
+
pillow==10.2.0
|
| 114 |
+
pkginfo @ file:///home/conda/feedstock_root/build_artifacts/pkginfo_1728616516923/work
|
| 115 |
+
pkgutil_resolve_name @ file:///home/conda/feedstock_root/build_artifacts/pkgutil-resolve-name_1694617248815/work
|
| 116 |
+
platformdirs @ file:///home/conda/feedstock_root/build_artifacts/platformdirs_1726613481435/work
|
| 117 |
+
pluggy==1.7.0
|
| 118 |
+
polars==2.1.0
|
| 119 |
+
polars-runtime-32==2.1.0
|
| 120 |
+
pqdm==0.2.0
|
| 121 |
+
prompt_toolkit @ file:///home/conda/feedstock_root/build_artifacts/prompt-toolkit_1727341649933/work
|
| 122 |
+
propcache==0.5.4
|
| 123 |
+
psutil @ file:///home/conda/feedstock_root/build_artifacts/psutil_1729847065645/work
|
| 124 |
+
ptyprocess @ file:///home/conda/feedstock_root/build_artifacts/ptyprocess_1609419310487/work/dist/ptyprocess-0.7.0-py2.py3-none-any.whl
|
| 125 |
+
pure_eval @ file:///home/conda/feedstock_root/build_artifacts/pure_eval_1721585709575/work
|
| 126 |
+
pyarrow==26.0.1
|
| 127 |
+
pycosat @ file:///home/conda/feedstock_root/build_artifacts/pycosat_1696355758146/work
|
| 128 |
+
pycparser @ file:///home/conda/feedstock_root/build_artifacts/pycparser_1711811537435/work
|
| 129 |
+
Pygments @ file:///home/conda/feedstock_root/build_artifacts/pygments_1714846767233/work
|
| 130 |
+
pymannkendall==1.4.3
|
| 131 |
+
pyogrio==0.13.0
|
| 132 |
+
pyparsing==3.3.3
|
| 133 |
+
pyproj==3.7.2
|
| 134 |
+
PySocks @ file:///home/conda/feedstock_root/build_artifacts/pysocks_1661604839144/work
|
| 135 |
+
pytest==9.2.0
|
| 136 |
+
python-cmr==0.13.0
|
| 137 |
+
python-dateutil==2.9.0.post0
|
| 138 |
+
python-etcd==0.4.5
|
| 139 |
+
pytz @ file:///home/conda/feedstock_root/build_artifacts/pytz_1726055524169/work
|
| 140 |
+
PyYAML @ file:///home/conda/feedstock_root/build_artifacts/pyyaml_1725456139051/work
|
| 141 |
+
rasterio==1.4.4
|
| 142 |
+
referencing @ file:///home/conda/feedstock_root/build_artifacts/referencing_1714619483868/work
|
| 143 |
+
requests @ file:///home/conda/feedstock_root/build_artifacts/requests_1717057054362/work
|
| 144 |
+
rioxarray==0.19.0
|
| 145 |
+
rpds-py @ file:///home/conda/feedstock_root/build_artifacts/rpds-py_1725327039958/work
|
| 146 |
+
ruamel.yaml @ file:///home/conda/feedstock_root/build_artifacts/ruamel.yaml_1728764976564/work
|
| 147 |
+
ruamel.yaml.clib @ file:///home/conda/feedstock_root/build_artifacts/ruamel.yaml.clib_1728724459810/work
|
| 148 |
+
s3fs==2026.9.0
|
| 149 |
+
scikit-learn==1.9.1
|
| 150 |
+
scipy==1.17.1
|
| 151 |
+
shap==0.51.0
|
| 152 |
+
shapely==2.2.0
|
| 153 |
+
six @ file:///home/conda/feedstock_root/build_artifacts/six_1620240208055/work
|
| 154 |
+
slicer==0.0.8
|
| 155 |
+
sortedcontainers==2.4.0
|
| 156 |
+
soupsieve @ file:///home/conda/feedstock_root/build_artifacts/soupsieve_1693929250441/work
|
| 157 |
+
SQLAlchemy==2.1.8
|
| 158 |
+
stack-data @ file:///home/conda/feedstock_root/build_artifacts/stack_data_1669632077133/work
|
| 159 |
+
statsmodels==0.15.0
|
| 160 |
+
sympy==1.13.1
|
| 161 |
+
tenacity==9.2.1
|
| 162 |
+
threadpoolctl==3.8.0
|
| 163 |
+
tinynetrc==1.3.1
|
| 164 |
+
torch==2.5.1+cu124
|
| 165 |
+
torchaudio==2.5.1+cu124
|
| 166 |
+
torchelastic==0.2.2
|
| 167 |
+
torchvision==0.20.1+cu124
|
| 168 |
+
tqdm @ file:///home/conda/feedstock_root/build_artifacts/tqdm_1722737464726/work
|
| 169 |
+
traitlets @ file:///home/conda/feedstock_root/build_artifacts/traitlets_1713535121073/work
|
| 170 |
+
triton==3.1.0
|
| 171 |
+
truststore==0.10.5
|
| 172 |
+
types-dataclasses==0.6.6
|
| 173 |
+
typing_extensions==4.16.0
|
| 174 |
+
urllib3 @ file:///home/conda/feedstock_root/build_artifacts/urllib3_1726496430923/work
|
| 175 |
+
wcwidth @ file:///home/conda/feedstock_root/build_artifacts/wcwidth_1704731205417/work
|
| 176 |
+
wrapt==2.5.2
|
| 177 |
+
xarray==2026.9.0
|
| 178 |
+
xgboost==3.2.0
|
| 179 |
+
yarl==1.25.1
|
| 180 |
+
zipp @ file:///home/conda/feedstock_root/build_artifacts/zipp_1726248574750/work
|
| 181 |
+
zstandard==0.23.0
|
training_code/scripts/00_smoke.py
ADDED
|
@@ -0,0 +1,190 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Smoke test: prove every NASA data source opens before any bulk request.
|
| 2 |
+
|
| 3 |
+
Needs ~/.netrc with an entry for urs.earthdata.nasa.gov (the user creates it on the box).
|
| 4 |
+
|
| 5 |
+
python scripts/00_smoke.py --out /root/smoke # submit + wait
|
| 6 |
+
python scripts/00_smoke.py --out /root/smoke --resume # re-poll tasks saved earlier
|
| 7 |
+
"""
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import argparse
|
| 11 |
+
import json
|
| 12 |
+
import netrc
|
| 13 |
+
import sys
|
| 14 |
+
import time
|
| 15 |
+
from pathlib import Path
|
| 16 |
+
|
| 17 |
+
import requests
|
| 18 |
+
|
| 19 |
+
APPEEARS = "https://appeears.earthdatacloud.nasa.gov/api"
|
| 20 |
+
# ~20 x 20 km box at Son-Kul: the lake plus the summer pastures around it.
|
| 21 |
+
BBOX = (75.00, 41.75, 75.25, 41.93) # lon_min, lat_min, lon_max, lat_max
|
| 22 |
+
JULY_2020 = ("07-01-2020", "07-31-2020")
|
| 23 |
+
|
| 24 |
+
# One AppEEARS task per product (native projection differs between MODIS and NASADEM).
|
| 25 |
+
TASKS = {
|
| 26 |
+
"MOD13A3": (JULY_2020, [("MOD13A3.061", "_1_km_monthly_NDVI"),
|
| 27 |
+
("MOD13A3.061", "_1_km_monthly_pixel_reliability")]),
|
| 28 |
+
"MYD13A3": (JULY_2020, [("MYD13A3.061", "_1_km_monthly_NDVI")]),
|
| 29 |
+
"MOD11A2": (JULY_2020, [("MOD11A2.061", "LST_Day_1km"),
|
| 30 |
+
("MOD11A2.061", "LST_Night_1km"),
|
| 31 |
+
("MOD11A2.061", "QC_Day")]),
|
| 32 |
+
"MOD10A2": (JULY_2020, [("MOD10A2.061", "Maximum_Snow_Extent")]),
|
| 33 |
+
"MCD12Q1": (("01-01-2020", "01-02-2020"), [("MCD12Q1.061", "LC_Type1")]),
|
| 34 |
+
"NASADEM": (("02-01-2000", "02-29-2000"), [("NASADEM_NC.001", "NASADEM_HGT")]),
|
| 35 |
+
}
|
| 36 |
+
|
| 37 |
+
GESDISC = [ # (short_name, version, variables to print)
|
| 38 |
+
("GPM_3IMERGM", "07", ["precipitation"]),
|
| 39 |
+
("GLDAS_NOAH025_M", "2.1", ["Rainf_f_tavg", "Tair_f_inst", "SoilMoi0_10cm_inst", "SWE_inst"]),
|
| 40 |
+
]
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def earthdata_credentials() -> tuple[str, str]:
|
| 44 |
+
auth = netrc.netrc().authenticators("urs.earthdata.nasa.gov")
|
| 45 |
+
if not auth:
|
| 46 |
+
sys.exit("No urs.earthdata.nasa.gov entry in ~/.netrc")
|
| 47 |
+
return auth[0], auth[2]
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def appeears_token() -> str:
|
| 51 |
+
user, password = earthdata_credentials()
|
| 52 |
+
r = requests.post(f"{APPEEARS}/login", auth=(user, password), timeout=60)
|
| 53 |
+
r.raise_for_status()
|
| 54 |
+
return r.json()["token"]
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def polygon() -> dict:
|
| 58 |
+
x0, y0, x1, y1 = BBOX
|
| 59 |
+
ring = [[x0, y0], [x1, y0], [x1, y1], [x0, y1], [x0, y0]]
|
| 60 |
+
return {"type": "FeatureCollection",
|
| 61 |
+
"features": [{"type": "Feature", "properties": {},
|
| 62 |
+
"geometry": {"type": "Polygon", "coordinates": [ring]}}]}
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def submit(headers: dict) -> dict[str, str]:
|
| 66 |
+
ids = {}
|
| 67 |
+
for name, ((start, end), layers) in TASKS.items():
|
| 68 |
+
body = {"task_type": "area", "task_name": f"trendex-smoke-{name}",
|
| 69 |
+
"params": {"dates": [{"startDate": start, "endDate": end}],
|
| 70 |
+
"layers": [{"product": p, "layer": l} for p, l in layers],
|
| 71 |
+
"geo": polygon(),
|
| 72 |
+
"output": {"format": {"type": "geotiff"}, "projection": "native"}}}
|
| 73 |
+
r = requests.post(f"{APPEEARS}/task", json=body, headers=headers, timeout=60)
|
| 74 |
+
if r.status_code >= 400:
|
| 75 |
+
print(f"[appeears] {name}: submit FAILED {r.status_code} {r.text[:300]}")
|
| 76 |
+
continue
|
| 77 |
+
ids[name] = r.json()["task_id"]
|
| 78 |
+
print(f"[appeears] {name}: submitted {ids[name]}")
|
| 79 |
+
return ids
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def wait_and_download(ids: dict[str, str], headers: dict, out: Path, timeout_min: int) -> dict:
|
| 83 |
+
results, pending = {}, dict(ids)
|
| 84 |
+
deadline = time.time() + timeout_min * 60
|
| 85 |
+
while pending and time.time() < deadline:
|
| 86 |
+
for name, tid in list(pending.items()):
|
| 87 |
+
status = requests.get(f"{APPEEARS}/task/{tid}", headers=headers, timeout=60).json().get("status")
|
| 88 |
+
if status == "done":
|
| 89 |
+
results[name] = download_bundle(name, tid, headers, out)
|
| 90 |
+
pending.pop(name)
|
| 91 |
+
elif status in ("error", "expired", "deleted"):
|
| 92 |
+
print(f"[appeears] {name}: {status}")
|
| 93 |
+
results[name] = {"status": status}
|
| 94 |
+
pending.pop(name)
|
| 95 |
+
if pending:
|
| 96 |
+
print(f"[appeears] waiting on {sorted(pending)} ...", flush=True)
|
| 97 |
+
time.sleep(30)
|
| 98 |
+
for name in pending:
|
| 99 |
+
results[name] = {"status": "still_queued"}
|
| 100 |
+
return results
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def download_bundle(name: str, tid: str, headers: dict, out: Path) -> dict:
|
| 104 |
+
files = requests.get(f"{APPEEARS}/bundle/{tid}", headers=headers, timeout=60).json()["files"]
|
| 105 |
+
dest = out / "appeears" / name
|
| 106 |
+
dest.mkdir(parents=True, exist_ok=True)
|
| 107 |
+
tifs = []
|
| 108 |
+
for f in files:
|
| 109 |
+
if not f["file_name"].endswith(".tif"):
|
| 110 |
+
continue
|
| 111 |
+
path = dest / Path(f["file_name"]).name
|
| 112 |
+
with requests.get(f"{APPEEARS}/bundle/{tid}/{f['file_id']}", headers=headers,
|
| 113 |
+
stream=True, allow_redirects=True, timeout=300) as r:
|
| 114 |
+
r.raise_for_status()
|
| 115 |
+
with open(path, "wb") as fh:
|
| 116 |
+
for chunk in r.iter_content(1 << 20):
|
| 117 |
+
fh.write(chunk)
|
| 118 |
+
tifs.append(path)
|
| 119 |
+
return {"status": "done", "files": [describe_tif(p) for p in tifs]}
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def describe_tif(path: Path) -> dict:
|
| 123 |
+
import numpy as np
|
| 124 |
+
import rasterio
|
| 125 |
+
with rasterio.open(path) as src:
|
| 126 |
+
a = src.read(1, masked=True)
|
| 127 |
+
valid = a.compressed()
|
| 128 |
+
info = {"file": path.name, "crs": str(src.crs)[:60], "shape": a.shape,
|
| 129 |
+
"res": [round(v, 6) for v in src.res], "nodata": src.nodata,
|
| 130 |
+
"valid_share": round(valid.size / a.size, 3) if a.size else 0}
|
| 131 |
+
if valid.size:
|
| 132 |
+
info.update(min=float(np.min(valid)), max=float(np.max(valid)))
|
| 133 |
+
print(f" {info}")
|
| 134 |
+
return info
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
def check_gesdisc(out: Path) -> dict:
|
| 138 |
+
import earthaccess
|
| 139 |
+
import xarray as xr
|
| 140 |
+
earthaccess.login(strategy="netrc")
|
| 141 |
+
lon = (BBOX[0] + BBOX[2]) / 2
|
| 142 |
+
lat = (BBOX[1] + BBOX[3]) / 2
|
| 143 |
+
results = {}
|
| 144 |
+
for short, ver, variables in GESDISC:
|
| 145 |
+
try:
|
| 146 |
+
found = earthaccess.search_data(short_name=short, version=ver,
|
| 147 |
+
temporal=("2020-07-01", "2020-07-31"), bounding_box=BBOX)
|
| 148 |
+
print(f"[gesdisc] {short} v{ver}: {len(found)} granule(s) for July 2020")
|
| 149 |
+
path = Path(earthaccess.download(found[:1], local_path=str(out / "gesdisc" / short))[0])
|
| 150 |
+
kw = {"group": "Grid", "engine": "h5netcdf"} if short.startswith("GPM") else {}
|
| 151 |
+
ds = xr.open_dataset(path, **kw)
|
| 152 |
+
vals = {}
|
| 153 |
+
for v in variables:
|
| 154 |
+
if v in ds:
|
| 155 |
+
pt = ds[v].sel(lon=lon, lat=lat, method="nearest").squeeze()
|
| 156 |
+
vals[v] = f"{float(pt.values):.4g} {ds[v].attrs.get('units', '')}".strip()
|
| 157 |
+
print(f"[gesdisc] {short}: file={path.name} at Son-Kul: {vals}")
|
| 158 |
+
results[short] = {"status": "done", "file": path.name, "values": vals}
|
| 159 |
+
except Exception as exc: # report and keep going: one failure must not hide the others
|
| 160 |
+
print(f"[gesdisc] {short}: FAILED {type(exc).__name__}: {exc}")
|
| 161 |
+
results[short] = {"status": "failed", "error": str(exc)[:300]}
|
| 162 |
+
return results
|
| 163 |
+
|
| 164 |
+
|
| 165 |
+
def main() -> None:
|
| 166 |
+
ap = argparse.ArgumentParser()
|
| 167 |
+
ap.add_argument("--out", type=Path, default=Path("/root/smoke"))
|
| 168 |
+
ap.add_argument("--resume", action="store_true", help="re-poll task ids saved by an earlier run")
|
| 169 |
+
ap.add_argument("--timeout-min", type=int, default=40)
|
| 170 |
+
args = ap.parse_args()
|
| 171 |
+
args.out.mkdir(parents=True, exist_ok=True)
|
| 172 |
+
ids_file = args.out / "appeears_task_ids.json"
|
| 173 |
+
|
| 174 |
+
report = {"gesdisc": check_gesdisc(args.out)}
|
| 175 |
+
|
| 176 |
+
headers = {"Authorization": f"Bearer {appeears_token()}"}
|
| 177 |
+
if args.resume and ids_file.exists():
|
| 178 |
+
ids = json.loads(ids_file.read_text())
|
| 179 |
+
else:
|
| 180 |
+
ids = submit(headers)
|
| 181 |
+
ids_file.write_text(json.dumps(ids, indent=2))
|
| 182 |
+
report["appeears"] = wait_and_download(ids, headers, args.out, args.timeout_min)
|
| 183 |
+
|
| 184 |
+
(args.out / "smoke_report.json").write_text(json.dumps(report, indent=2, default=str))
|
| 185 |
+
bad = [k for sec in report.values() for k, v in sec.items() if v.get("status") != "done"]
|
| 186 |
+
print("SMOKE_OK" if not bad else f"SMOKE_INCOMPLETE: {bad}")
|
| 187 |
+
|
| 188 |
+
|
| 189 |
+
if __name__ == "__main__":
|
| 190 |
+
main()
|
training_code/scripts/05_aoi.py
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Build the area of interest: Naryn oblast, its districts, and a 20 km buffer for AppEEARS.
|
| 2 |
+
|
| 3 |
+
Source: geoBoundaries (gbOpen, KGZ ADM1/ADM2). Writes GeoJSON to --out and prints a summary,
|
| 4 |
+
including whether the PRD's known places fall inside the oblast and inside the buffer.
|
| 5 |
+
"""
|
| 6 |
+
from __future__ import annotations
|
| 7 |
+
|
| 8 |
+
import argparse
|
| 9 |
+
import json
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
|
| 12 |
+
import geopandas as gpd
|
| 13 |
+
import requests
|
| 14 |
+
from shapely.geometry import Point, box
|
| 15 |
+
|
| 16 |
+
API = "https://www.geoboundaries.org/api/current/gbOpen/KGZ/{level}/"
|
| 17 |
+
SINU = "+proj=sinu +lon_0=0 +x_0=0 +y_0=0 +R=6371007.181 +units=m +no_defs" # MODIS grid, equal-area
|
| 18 |
+
BUFFER_M = 20_000
|
| 19 |
+
SIMPLIFY_DEG = 0.005 # ~0.5 km, keeps the AppEEARS polygon small
|
| 20 |
+
|
| 21 |
+
# Official districts of Naryn oblast. geoBoundaries' ADM1 "Naryn Region" also contains Toguz-Toro,
|
| 22 |
+
# which (to our knowledge) belongs to Jalal-Abad oblast, so the oblast is built from these five instead.
|
| 23 |
+
OFFICIAL_DISTRICTS = ["Ak-Talaa", "At-Bashy", "Jumgal", "Kochkor", "Naryn"]
|
| 24 |
+
|
| 25 |
+
# PRD known places outside the oblast get their own box in the AppEEARS request (approximate extents).
|
| 26 |
+
EXTRA_BOXES = {
|
| 27 |
+
"Suusamyr valley (Chuy oblast)": (73.60, 41.95, 74.50, 42.30), # lon_min, lat_min, lon_max, lat_max
|
| 28 |
+
"Ak-Shyirak massif (Issyk-Kul oblast)": (77.90, 41.70, 78.50, 42.00),
|
| 29 |
+
}
|
| 30 |
+
|
| 31 |
+
# Approximate coordinates (lat, lon) of the PRD's "known places"; used only for the inside/outside check.
|
| 32 |
+
KNOWN_PLACES = {
|
| 33 |
+
"Son-Kul lake": (41.83, 75.13),
|
| 34 |
+
"Naryn city": (41.43, 75.99),
|
| 35 |
+
"Ak-Shyirak (Kumtor area, approx.)": (41.86, 78.19),
|
| 36 |
+
"Suusamyr village": (42.17, 73.97),
|
| 37 |
+
}
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def fetch(level: str, cache: Path) -> gpd.GeoDataFrame:
|
| 41 |
+
path = cache / f"geoBoundaries-KGZ-{level}.geojson"
|
| 42 |
+
if not path.exists():
|
| 43 |
+
meta = requests.get(API.format(level=level), timeout=60).json()
|
| 44 |
+
r = requests.get(meta["gjDownloadURL"], timeout=300)
|
| 45 |
+
r.raise_for_status()
|
| 46 |
+
path.write_bytes(r.content)
|
| 47 |
+
return gpd.read_file(path).to_crs(4326)
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def main() -> None:
|
| 51 |
+
ap = argparse.ArgumentParser()
|
| 52 |
+
ap.add_argument("--out", type=Path, default=Path("/root/trendex-ml/data/aoi"))
|
| 53 |
+
args = ap.parse_args()
|
| 54 |
+
args.out.mkdir(parents=True, exist_ok=True)
|
| 55 |
+
|
| 56 |
+
adm1 = fetch("ADM1", args.out)
|
| 57 |
+
adm2 = fetch("ADM2", args.out)
|
| 58 |
+
print("ADM1 names:", sorted(adm1["shapeName"]))
|
| 59 |
+
gb_naryn = adm1[adm1["shapeName"].str.contains("Naryn", case=False)]
|
| 60 |
+
reps = adm2.geometry.representative_point()
|
| 61 |
+
in_gb_naryn = sorted(adm2[reps.within(gb_naryn.geometry.iloc[0])]["shapeName"])
|
| 62 |
+
|
| 63 |
+
districts = adm2[adm2["shapeName"].isin(OFFICIAL_DISTRICTS)][["shapeName", "shapeID", "geometry"]]
|
| 64 |
+
districts = districts.reset_index(drop=True)
|
| 65 |
+
missing = sorted(set(OFFICIAL_DISTRICTS) - set(districts["shapeName"]))
|
| 66 |
+
if missing:
|
| 67 |
+
raise SystemExit(f"Districts not found in geoBoundaries ADM2: {missing}")
|
| 68 |
+
naryn = gpd.GeoDataFrame({"shapeName": ["Naryn oblast (5 districts)"]},
|
| 69 |
+
geometry=[districts.union_all()], crs=4326)
|
| 70 |
+
naryn_geom = naryn.geometry.iloc[0]
|
| 71 |
+
|
| 72 |
+
naryn_sinu = naryn.to_crs(SINU)
|
| 73 |
+
area_km2 = naryn_sinu.area.iloc[0] / 1e6
|
| 74 |
+
buffered = naryn_sinu.buffer(BUFFER_M).to_crs(4326).simplify(SIMPLIFY_DEG)
|
| 75 |
+
names = ["Naryn oblast + 20 km"] + list(EXTRA_BOXES)
|
| 76 |
+
geoms = [buffered.iloc[0]] + [box(*b) for b in EXTRA_BOXES.values()]
|
| 77 |
+
aoi = gpd.GeoDataFrame({"name": names}, geometry=geoms, crs=4326)
|
| 78 |
+
aoi_union = aoi.union_all()
|
| 79 |
+
|
| 80 |
+
naryn.to_file(args.out / "naryn_oblast.geojson", driver="GeoJSON")
|
| 81 |
+
districts.to_file(args.out / "naryn_districts.geojson", driver="GeoJSON")
|
| 82 |
+
aoi.to_file(args.out / "aoi_request.geojson", driver="GeoJSON") # what AppEEARS gets
|
| 83 |
+
|
| 84 |
+
summary = {
|
| 85 |
+
"oblast_name": naryn["shapeName"].iloc[0],
|
| 86 |
+
"oblast_area_km2": round(area_km2),
|
| 87 |
+
"expected_1km_cells_approx": round(area_km2 / 0.8586),
|
| 88 |
+
"districts": sorted(districts["shapeName"]),
|
| 89 |
+
"geoboundaries_adm1_naryn_contains": in_gb_naryn,
|
| 90 |
+
"oblast_area_km2_note": "official figure is about 45,200 km2; compare",
|
| 91 |
+
"aoi_bbox_lon_lat": [round(v, 3) for v in aoi.total_bounds],
|
| 92 |
+
"aoi_features": names,
|
| 93 |
+
"known_places": {
|
| 94 |
+
name: {"in_oblast": naryn_geom.contains(Point(lon, lat)),
|
| 95 |
+
"in_request": aoi_union.contains(Point(lon, lat))}
|
| 96 |
+
for name, (lat, lon) in KNOWN_PLACES.items()
|
| 97 |
+
},
|
| 98 |
+
}
|
| 99 |
+
(args.out / "aoi_summary.json").write_text(json.dumps(summary, indent=2, ensure_ascii=False))
|
| 100 |
+
print(json.dumps(summary, indent=2, ensure_ascii=False))
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
if __name__ == "__main__":
|
| 104 |
+
main()
|
training_code/scripts/10_download.py
ADDED
|
@@ -0,0 +1,233 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Bulk download of every NASA layer for the cube.
|
| 2 |
+
|
| 3 |
+
python scripts/10_download.py submit # AppEEARS: one area task per product, ids saved to JSON
|
| 4 |
+
python scripts/10_download.py fetch # AppEEARS: poll tasks, download finished bundles (re-runnable)
|
| 5 |
+
python scripts/10_download.py gesdisc # GES DISC: IMERG + GLDAS monthly files (re-runnable)
|
| 6 |
+
|
| 7 |
+
Credentials come from ~/.netrc (urs.earthdata.nasa.gov). Every step skips work already done,
|
| 8 |
+
so it is safe to re-run after a crash or a lost SSH session.
|
| 9 |
+
"""
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
import argparse
|
| 13 |
+
import json
|
| 14 |
+
import netrc
|
| 15 |
+
import sys
|
| 16 |
+
import time
|
| 17 |
+
from concurrent.futures import ThreadPoolExecutor
|
| 18 |
+
from pathlib import Path
|
| 19 |
+
|
| 20 |
+
import requests
|
| 21 |
+
|
| 22 |
+
APPEEARS = "https://appeears.earthdatacloud.nasa.gov/api"
|
| 23 |
+
ROOT = Path("/root/trendex-ml/data")
|
| 24 |
+
AOI = ROOT / "aoi" / "aoi_request.geojson"
|
| 25 |
+
RAW = ROOT / "raw"
|
| 26 |
+
TASK_IDS = RAW / "appeears_task_ids.json"
|
| 27 |
+
LAST_YEAR = 2025
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def full(start: str, end: str = f"12-31-{LAST_YEAR}") -> dict:
|
| 31 |
+
return {"startDate": start, "endDate": end}
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def months_each_year(start_mmdd: str, end_mmdd: str, first_year: int) -> dict:
|
| 35 |
+
"""Recurring window, e.g. Mar-Sep of every year."""
|
| 36 |
+
return {"startDate": start_mmdd, "endDate": end_mmdd, "recurring": True,
|
| 37 |
+
"yearRange": [first_year, LAST_YEAR]}
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
# product -> (date window, layers). LST and snow are only needed Mar-Sep (model features).
|
| 41 |
+
PRODUCTS = {
|
| 42 |
+
"MOD13A3": (full("02-01-2000"), [("MOD13A3.061", "_1_km_monthly_NDVI"),
|
| 43 |
+
("MOD13A3.061", "_1_km_monthly_pixel_reliability")]),
|
| 44 |
+
"MYD13A3": (full("07-01-2002"), [("MYD13A3.061", "_1_km_monthly_NDVI"),
|
| 45 |
+
("MYD13A3.061", "_1_km_monthly_pixel_reliability")]),
|
| 46 |
+
"MOD11A2": (months_each_year("03-01", "09-30", 2000), [("MOD11A2.061", "LST_Day_1km"),
|
| 47 |
+
("MOD11A2.061", "LST_Night_1km"),
|
| 48 |
+
("MOD11A2.061", "QC_Day"),
|
| 49 |
+
("MOD11A2.061", "QC_Night")]),
|
| 50 |
+
"MOD10A2": (months_each_year("03-01", "09-30", 2000), [("MOD10A2.061", "Maximum_Snow_Extent")]),
|
| 51 |
+
"MCD12Q1": (full("01-01-2020", "01-02-2020"), [("MCD12Q1.061", "LC_Type1"),
|
| 52 |
+
("MCD12Q1.061", "LW")]),
|
| 53 |
+
"NASADEM": (full("02-01-2000", "02-29-2000"), [("NASADEM_NC.001", "NASADEM_HGT")]),
|
| 54 |
+
}
|
| 55 |
+
ORDER = ["MOD13A3", "NASADEM", "MCD12Q1", "MOD11A2", "MOD10A2", "MYD13A3"] # what the formula needs first
|
| 56 |
+
|
| 57 |
+
LPDAAC = { # fallback: raw HDF4 tiles straight from LP DAAC / NSIDC if the AppEEARS queue is too slow
|
| 58 |
+
"MOD13A3": ("MOD13A3", "061", "2000-02-01"),
|
| 59 |
+
"MYD13A3": ("MYD13A3", "061", "2002-07-01"),
|
| 60 |
+
"MOD11A2": ("MOD11A2", "061", "2000-02-18"),
|
| 61 |
+
"MOD10A2": ("MOD10A2", "61", "2000-02-24"),
|
| 62 |
+
"MCD12Q1": ("MCD12Q1", "061", "2020-01-01"),
|
| 63 |
+
"NASADEM": ("NASADEM_HGT", "001", "2000-02-11"),
|
| 64 |
+
}
|
| 65 |
+
|
| 66 |
+
GESDISC = { # short_name: (version, first month)
|
| 67 |
+
"GPM_3IMERGM": ("07", "2000-06-01"),
|
| 68 |
+
"GLDAS_NOAH025_M": ("2.1", "2000-01-01"),
|
| 69 |
+
}
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def headers() -> dict:
|
| 73 |
+
auth = netrc.netrc().authenticators("urs.earthdata.nasa.gov")
|
| 74 |
+
if not auth:
|
| 75 |
+
sys.exit("No urs.earthdata.nasa.gov entry in ~/.netrc")
|
| 76 |
+
r = requests.post(f"{APPEEARS}/login", auth=(auth[0], auth[2]), timeout=60)
|
| 77 |
+
r.raise_for_status()
|
| 78 |
+
return {"Authorization": f"Bearer {r.json()['token']}"}
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
def submit() -> None:
|
| 82 |
+
RAW.mkdir(parents=True, exist_ok=True)
|
| 83 |
+
ids = json.loads(TASK_IDS.read_text()) if TASK_IDS.exists() else {}
|
| 84 |
+
geo = json.loads(AOI.read_text())
|
| 85 |
+
h = headers()
|
| 86 |
+
for name in ORDER:
|
| 87 |
+
if name in ids:
|
| 88 |
+
print(f"[submit] {name}: already submitted {ids[name]}")
|
| 89 |
+
continue
|
| 90 |
+
window, layers = PRODUCTS[name]
|
| 91 |
+
body = {"task_type": "area", "task_name": f"trendex-{name}",
|
| 92 |
+
"params": {"dates": [window],
|
| 93 |
+
"layers": [{"product": p, "layer": l} for p, l in layers],
|
| 94 |
+
"geo": geo,
|
| 95 |
+
"output": {"format": {"type": "geotiff"}, "projection": "native"}}}
|
| 96 |
+
r = requests.post(f"{APPEEARS}/task", json=body, headers=h, timeout=120)
|
| 97 |
+
if r.status_code >= 400:
|
| 98 |
+
print(f"[submit] {name}: FAILED {r.status_code} {r.text[:400]}")
|
| 99 |
+
continue
|
| 100 |
+
ids[name] = r.json()["task_id"]
|
| 101 |
+
TASK_IDS.write_text(json.dumps(ids, indent=2))
|
| 102 |
+
print(f"[submit] {name}: {ids[name]}")
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
class Session:
|
| 106 |
+
"""AppEEARS client: one login, re-login on 401, polite back-off on 429/5xx (it rate-limits hard)."""
|
| 107 |
+
|
| 108 |
+
def __init__(self) -> None:
|
| 109 |
+
self.h = headers()
|
| 110 |
+
|
| 111 |
+
def get(self, url: str, stream: bool = False, tries: int = 8) -> requests.Response:
|
| 112 |
+
for i in range(tries):
|
| 113 |
+
try:
|
| 114 |
+
r = requests.get(url, headers=self.h, stream=stream, allow_redirects=True, timeout=300)
|
| 115 |
+
except requests.RequestException as exc:
|
| 116 |
+
print(f" network error ({exc}); retry in {20 * (i + 1)}s", flush=True)
|
| 117 |
+
time.sleep(20 * (i + 1))
|
| 118 |
+
continue
|
| 119 |
+
if r.status_code == 401:
|
| 120 |
+
self.h = headers()
|
| 121 |
+
continue
|
| 122 |
+
if r.status_code == 429 or r.status_code >= 500:
|
| 123 |
+
wait = int(r.headers.get("Retry-After", 0) or 60 * (i + 1))
|
| 124 |
+
print(f" HTTP {r.status_code}; waiting {wait}s", flush=True)
|
| 125 |
+
r.close()
|
| 126 |
+
time.sleep(wait)
|
| 127 |
+
continue
|
| 128 |
+
r.raise_for_status()
|
| 129 |
+
return r
|
| 130 |
+
raise RuntimeError(f"giving up on {url}")
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
def _download(s: Session, url: str, path: Path, size: int | None) -> None:
|
| 134 |
+
if path.exists() and (size is None or path.stat().st_size == size):
|
| 135 |
+
return
|
| 136 |
+
tmp = path.with_suffix(path.suffix + ".part")
|
| 137 |
+
with s.get(url, stream=True) as r:
|
| 138 |
+
with open(tmp, "wb") as fh:
|
| 139 |
+
for chunk in r.iter_content(1 << 20):
|
| 140 |
+
fh.write(chunk)
|
| 141 |
+
tmp.rename(path)
|
| 142 |
+
|
| 143 |
+
|
| 144 |
+
def fetch(poll_s: int, max_hours: float) -> None:
|
| 145 |
+
ids = json.loads(TASK_IDS.read_text())
|
| 146 |
+
done_marker = RAW / "appeears_done.json"
|
| 147 |
+
done = json.loads(done_marker.read_text()) if done_marker.exists() else {}
|
| 148 |
+
deadline = time.time() + max_hours * 3600
|
| 149 |
+
s = Session()
|
| 150 |
+
while time.time() < deadline:
|
| 151 |
+
pending = [n for n in ids if n not in done]
|
| 152 |
+
if not pending:
|
| 153 |
+
break
|
| 154 |
+
# one request for every task's status instead of one per task
|
| 155 |
+
statuses = {t["task_id"]: t.get("status") for t in s.get(f"{APPEEARS}/task?limit=100").json()}
|
| 156 |
+
for name in pending:
|
| 157 |
+
tid = ids[name]
|
| 158 |
+
status = statuses.get(tid)
|
| 159 |
+
if status in ("error", "expired", "deleted"):
|
| 160 |
+
print(f"[fetch] {name}: task {status}; resubmit with `submit` after removing it from {TASK_IDS.name}")
|
| 161 |
+
done[name] = status
|
| 162 |
+
continue
|
| 163 |
+
if status != "done":
|
| 164 |
+
continue
|
| 165 |
+
files = s.get(f"{APPEEARS}/bundle/{tid}").json()["files"]
|
| 166 |
+
dest = RAW / "appeears" / name
|
| 167 |
+
dest.mkdir(parents=True, exist_ok=True)
|
| 168 |
+
jobs = [(f"{APPEEARS}/bundle/{tid}/{f['file_id']}", dest / Path(f["file_name"]).name,
|
| 169 |
+
f.get("file_size")) for f in files]
|
| 170 |
+
with ThreadPoolExecutor(6) as pool:
|
| 171 |
+
list(pool.map(lambda j: _download(s, j[0], j[1], j[2]), jobs))
|
| 172 |
+
done[name] = "done"
|
| 173 |
+
done_marker.write_text(json.dumps(done, indent=2))
|
| 174 |
+
print(f"[fetch] {name}: downloaded {len(jobs)} files", flush=True)
|
| 175 |
+
left = [n for n in ids if n not in done]
|
| 176 |
+
if left:
|
| 177 |
+
shown = {n: statuses.get(ids[n]) for n in left}
|
| 178 |
+
print(f"[fetch] waiting on {shown} ({time.strftime('%H:%M:%S')})", flush=True)
|
| 179 |
+
time.sleep(poll_s)
|
| 180 |
+
print("FETCH_COMPLETE" if all(done.get(n) == "done" for n in ids) else f"FETCH_STATUS {done}")
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
def aoi_bbox() -> tuple[float, float, float, float]:
|
| 184 |
+
geo = json.loads(AOI.read_text())
|
| 185 |
+
xs = [c[0] for f in geo["features"] for c in f["geometry"]["coordinates"][0]]
|
| 186 |
+
ys = [c[1] for f in geo["features"] for c in f["geometry"]["coordinates"][0]]
|
| 187 |
+
return (min(xs), min(ys), max(xs), max(ys))
|
| 188 |
+
|
| 189 |
+
|
| 190 |
+
def lpdaac(products: list[str]) -> None:
|
| 191 |
+
import earthaccess
|
| 192 |
+
earthaccess.login(strategy="netrc")
|
| 193 |
+
bbox = aoi_bbox()
|
| 194 |
+
for name in products:
|
| 195 |
+
short, ver, first = LPDAAC[name]
|
| 196 |
+
last = "2020-12-31" if name == "MCD12Q1" else f"{LAST_YEAR}-12-31"
|
| 197 |
+
found = earthaccess.search_data(short_name=short, version=ver, temporal=(first, last), bounding_box=bbox)
|
| 198 |
+
dest = RAW / "earthaccess" / name
|
| 199 |
+
dest.mkdir(parents=True, exist_ok=True)
|
| 200 |
+
print(f"[lpdaac] {name}: {len(found)} granules", flush=True)
|
| 201 |
+
earthaccess.download(found, local_path=str(dest), threads=16)
|
| 202 |
+
print(f"[lpdaac] {name}: {len(list(dest.iterdir()))} files on disk", flush=True)
|
| 203 |
+
print("LPDAAC_COMPLETE", products)
|
| 204 |
+
|
| 205 |
+
|
| 206 |
+
def gesdisc() -> None:
|
| 207 |
+
import earthaccess
|
| 208 |
+
earthaccess.login(strategy="netrc")
|
| 209 |
+
bbox = aoi_bbox()
|
| 210 |
+
for short, (ver, first) in GESDISC.items():
|
| 211 |
+
found = earthaccess.search_data(short_name=short, version=ver,
|
| 212 |
+
temporal=(first, f"{LAST_YEAR}-12-31"), bounding_box=bbox)
|
| 213 |
+
dest = RAW / "gesdisc" / short
|
| 214 |
+
dest.mkdir(parents=True, exist_ok=True)
|
| 215 |
+
print(f"[gesdisc] {short} v{ver}: {len(found)} monthly granules", flush=True)
|
| 216 |
+
earthaccess.download(found, local_path=str(dest), threads=16)
|
| 217 |
+
print(f"[gesdisc] {short}: {len(list(dest.iterdir()))} files on disk", flush=True)
|
| 218 |
+
print("GESDISC_COMPLETE")
|
| 219 |
+
|
| 220 |
+
|
| 221 |
+
def main() -> None:
|
| 222 |
+
ap = argparse.ArgumentParser()
|
| 223 |
+
ap.add_argument("step", choices=["submit", "fetch", "gesdisc", "lpdaac"])
|
| 224 |
+
ap.add_argument("--products", nargs="*", default=["MOD13A3"])
|
| 225 |
+
ap.add_argument("--poll-s", type=int, default=60)
|
| 226 |
+
ap.add_argument("--max-hours", type=float, default=6)
|
| 227 |
+
a = ap.parse_args()
|
| 228 |
+
{"submit": submit, "gesdisc": gesdisc, "lpdaac": lambda: lpdaac(a.products),
|
| 229 |
+
"fetch": lambda: fetch(a.poll_s, a.max_hours)}[a.step]()
|
| 230 |
+
|
| 231 |
+
|
| 232 |
+
if __name__ == "__main__":
|
| 233 |
+
main()
|
training_code/scripts/20_build_cube.py
ADDED
|
@@ -0,0 +1,466 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Build the shared TrendeX data cube (F1) from the downloaded NASA rasters.
|
| 2 |
+
|
| 3 |
+
python scripts/20_build_cube.py # everything
|
| 4 |
+
python scripts/20_build_cube.py --only static # cells.parquet only (terrain, land cover, districts)
|
| 5 |
+
|
| 6 |
+
Master grid = the global MODIS sinusoidal 1 km grid, cut to the request area (aoi_request.geojson).
|
| 7 |
+
NDVI is never resampled (same grid -> nearest = copy). Other layers: 500 m / 30 m -> average or mode,
|
| 8 |
+
0.1 / 0.25 deg weather -> bilinear at cell centres.
|
| 9 |
+
|
| 10 |
+
Each MODIS/NASADEM layer is read from AppEEARS GeoTIFFs when that AppEEARS task finished, otherwise
|
| 11 |
+
from raw LP DAAC files fetched with earthaccess (HDF4 tiles / NASADEM zips). manifest.json records which.
|
| 12 |
+
|
| 13 |
+
Outputs in data/cube/:
|
| 14 |
+
cells.parquet one row per 1 km cell inside the request area
|
| 15 |
+
monthly/year=YYYY/*.parquet one row per cell x month
|
| 16 |
+
manifest.json inputs, sources, counts, scale checks, notes
|
| 17 |
+
"""
|
| 18 |
+
from __future__ import annotations
|
| 19 |
+
|
| 20 |
+
import argparse
|
| 21 |
+
import calendar
|
| 22 |
+
import json
|
| 23 |
+
import re
|
| 24 |
+
import time
|
| 25 |
+
from concurrent.futures import ProcessPoolExecutor
|
| 26 |
+
from pathlib import Path
|
| 27 |
+
|
| 28 |
+
import geopandas as gpd
|
| 29 |
+
import numpy as np
|
| 30 |
+
import pandas as pd
|
| 31 |
+
import pyarrow as pa
|
| 32 |
+
import pyarrow.parquet as pq
|
| 33 |
+
import rasterio
|
| 34 |
+
import xarray as xr
|
| 35 |
+
from pyproj import Transformer
|
| 36 |
+
from rasterio.warp import Resampling, reproject
|
| 37 |
+
from scipy.ndimage import uniform_filter
|
| 38 |
+
from shapely import contains_xy
|
| 39 |
+
|
| 40 |
+
ROOT = Path("/root/trendex-ml/data")
|
| 41 |
+
RAW, AOI_DIR, OUT = ROOT / "raw", ROOT / "aoi", ROOT / "cube"
|
| 42 |
+
PAT = re.compile(r"^(?P<prod>[A-Z0-9_]+\.\d{3})_(?P<layer>.+)_doy(?P<doy>\d{7})_aid(?P<aid>\d{4})\.tif$")
|
| 43 |
+
BLOCK = 11 # 11 x 11 cells ~ 10.2 km blocks (F5, spatial CV)
|
| 44 |
+
STEEP_DEG = 35.0 # "too steep" default; to be agreed with the formula owner (F1)
|
| 45 |
+
TPI_WINDOW = 5 # cells (~5 km) for the topographic position index
|
| 46 |
+
WORKERS = 32
|
| 47 |
+
|
| 48 |
+
SINU = "+proj=sinu +lon_0=0 +x_0=0 +y_0=0 +R=6371007.181 +units=m +no_defs"
|
| 49 |
+
SINU_CRS = rasterio.crs.CRS.from_proj4(SINU)
|
| 50 |
+
TILE = 1111950.5196666666 # MODIS tile size, m
|
| 51 |
+
X0, Y0 = -20015109.355798, 10007554.677899 # upper-left corner of the global MODIS grid
|
| 52 |
+
PIX1K = TILE / 1200
|
| 53 |
+
|
| 54 |
+
# Raw HDF4 tiles (LP DAAC fallback): layer key -> (SDS name, pixels per tile side)
|
| 55 |
+
HDF_SDS = {
|
| 56 |
+
"MOD13A3": {"NDVI": ("1 km monthly NDVI", 1200), "pixel_reliability": ("1 km monthly pixel reliability", 1200)},
|
| 57 |
+
"MYD13A3": {"NDVI": ("1 km monthly NDVI", 1200), "pixel_reliability": ("1 km monthly pixel reliability", 1200)},
|
| 58 |
+
"MOD11A2": {k: (k, 1200) for k in ("LST_Day_1km", "LST_Night_1km", "QC_Day", "QC_Night")},
|
| 59 |
+
"MOD10A2": {"Maximum_Snow_Extent": ("Maximum_Snow_Extent", 2400)},
|
| 60 |
+
"MCD12Q1": {"LC_Type1": ("LC_Type1", 2400), "LW": ("LW", 2400)},
|
| 61 |
+
}
|
| 62 |
+
|
| 63 |
+
GRID: dict = {} # filled in main(), read by worker processes after fork
|
| 64 |
+
SOURCES_USED: dict = {}
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
# ----------------------------------------------------------------------------- sources
|
| 68 |
+
def appeears_done(product: str) -> bool:
|
| 69 |
+
marker = RAW / "appeears_done.json"
|
| 70 |
+
return marker.exists() and json.loads(marker.read_text()).get(product) == "done"
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def appeears_index(product: str) -> pd.DataFrame:
|
| 74 |
+
rows = []
|
| 75 |
+
for p in sorted((RAW / "appeears" / product).glob("*.tif")):
|
| 76 |
+
m = PAT.match(p.name)
|
| 77 |
+
if not m:
|
| 78 |
+
continue
|
| 79 |
+
year, doy = int(m["doy"][:4]), int(m["doy"][4:])
|
| 80 |
+
date = pd.Timestamp(year=year, month=1, day=1) + pd.Timedelta(days=doy - 1)
|
| 81 |
+
rows.append({"path": str(p), "layer": m["layer"], "date": date})
|
| 82 |
+
if not rows:
|
| 83 |
+
raise SystemExit(f"AppEEARS marked {product} done but no GeoTIFFs matched {PAT.pattern}")
|
| 84 |
+
return pd.DataFrame(rows)
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def sources(product: str, layer: str) -> dict:
|
| 88 |
+
"""date -> list of raster sources for one layer (AppEEARS if finished, else raw LP DAAC files)."""
|
| 89 |
+
if appeears_done(product):
|
| 90 |
+
idx = appeears_index(product)
|
| 91 |
+
idx = idx[idx["layer"].str.endswith(layer)]
|
| 92 |
+
SOURCES_USED[product] = "AppEEARS"
|
| 93 |
+
return {d: g["path"].tolist() for d, g in idx.groupby("date")}
|
| 94 |
+
folder = RAW / "earthaccess" / product
|
| 95 |
+
if product == "NASADEM":
|
| 96 |
+
tiles = sorted(folder.glob("*.zip"))
|
| 97 |
+
if not tiles:
|
| 98 |
+
raise SystemExit("No NASADEM data from AppEEARS (not finished) or LP DAAC")
|
| 99 |
+
SOURCES_USED[product] = "LP DAAC (earthaccess)"
|
| 100 |
+
return {pd.Timestamp("2000-02-11"): [f"/vsizip/{z}/{z.stem.split('_')[-1]}.hgt" for z in tiles]}
|
| 101 |
+
hdfs = sorted(folder.glob("*.hdf"))
|
| 102 |
+
if not hdfs:
|
| 103 |
+
raise SystemExit(f"No {product} data from AppEEARS (not finished) or LP DAAC ({folder})")
|
| 104 |
+
SOURCES_USED[product] = "LP DAAC (earthaccess)"
|
| 105 |
+
sds, npix = HDF_SDS[product][layer]
|
| 106 |
+
by_date: dict = {}
|
| 107 |
+
for p in hdfs:
|
| 108 |
+
m = re.search(r"\.A(\d{4})(\d{3})\.", p.name)
|
| 109 |
+
d = pd.Timestamp(year=int(m[1]), month=1, day=1) + pd.Timedelta(days=int(m[2]) - 1)
|
| 110 |
+
by_date.setdefault(d, []).append(("hdf", str(p), sds, npix))
|
| 111 |
+
return dict(sorted(by_date.items()))
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def flat(d: dict) -> list:
|
| 115 |
+
return [x for v in d.values() for x in v]
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
def _open(src):
|
| 119 |
+
"""-> (raw array, nodata, transform, crs) for a GDAL path or an ('hdf', path, sds, npix) tuple."""
|
| 120 |
+
if isinstance(src, str):
|
| 121 |
+
with rasterio.open(src) as s:
|
| 122 |
+
return s.read(1), s.nodata, s.transform, s.crs
|
| 123 |
+
_, path, sds, npix = src
|
| 124 |
+
from pyhdf.SD import SD, SDC
|
| 125 |
+
m = re.search(r"\.h(\d{2})v(\d{2})\.", Path(path).name)
|
| 126 |
+
h, v = int(m[1]), int(m[2])
|
| 127 |
+
f = SD(path, SDC.READ)
|
| 128 |
+
try:
|
| 129 |
+
ds = f.select(sds)
|
| 130 |
+
raw, attrs = ds.get(), ds.attributes()
|
| 131 |
+
finally:
|
| 132 |
+
f.end()
|
| 133 |
+
pix = TILE / npix
|
| 134 |
+
return raw, attrs.get("_FillValue"), rasterio.transform.from_origin(X0 + h * TILE, Y0 - v * TILE, pix, pix), SINU_CRS
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
# ----------------------------------------------------------------------------- grid helpers
|
| 138 |
+
def build_master_grid() -> dict:
|
| 139 |
+
minx, miny, maxx, maxy = gpd.read_file(AOI_DIR / "aoi_request.geojson").to_crs(SINU).total_bounds
|
| 140 |
+
c0, c1 = int(np.floor((minx - X0) / PIX1K)), int(np.ceil((maxx - X0) / PIX1K))
|
| 141 |
+
r0, r1 = int(np.floor((Y0 - maxy) / PIX1K)), int(np.ceil((Y0 - miny) / PIX1K))
|
| 142 |
+
left, top = X0 + c0 * PIX1K, Y0 - r0 * PIX1K
|
| 143 |
+
return {"crs": SINU_CRS, "transform": rasterio.transform.from_origin(left, top, PIX1K, PIX1K),
|
| 144 |
+
"shape": (r1 - r0, c1 - c0), "res": (PIX1K, PIX1K), "left": left, "top": top,
|
| 145 |
+
"global_row0": r0, "global_col0": c0}
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
def _nodata_to_nan(raw: np.ndarray, nodata) -> np.ndarray:
|
| 149 |
+
a = raw.astype("float32")
|
| 150 |
+
if nodata is not None:
|
| 151 |
+
a[raw == nodata] = np.nan
|
| 152 |
+
return a
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
def to_master(srcs: list, resampling: Resampling, fn=None) -> np.ndarray:
|
| 156 |
+
"""Mosaic all pieces (AppEEARS area pieces or MODIS tiles) of one layer/date onto the master grid.
|
| 157 |
+
|
| 158 |
+
fn(raw_array, nodata) -> float32 array with NaN for invalid pixels. Default: nodata -> NaN.
|
| 159 |
+
"""
|
| 160 |
+
out = np.full(GRID["shape"], np.nan, dtype="float32")
|
| 161 |
+
for src in srcs:
|
| 162 |
+
raw, nodata, transform, crs = _open(src)
|
| 163 |
+
a = fn(raw, nodata) if fn else _nodata_to_nan(raw, nodata)
|
| 164 |
+
tmp = np.full(GRID["shape"], np.nan, dtype="float32")
|
| 165 |
+
reproject(a.astype("float32"), tmp, src_transform=transform, src_crs=crs,
|
| 166 |
+
dst_transform=GRID["transform"], dst_crs=GRID["crs"],
|
| 167 |
+
resampling=resampling, src_nodata=np.nan, dst_nodata=np.nan)
|
| 168 |
+
out = np.where(np.isnan(out), tmp, out)
|
| 169 |
+
return out
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
def autoscale(a: np.ndarray, scale: float, raw_if_above: float) -> tuple[np.ndarray, bool]:
|
| 173 |
+
"""Raw HDF4 is never scaled; AppEEARS output may or may not be. Decide from the data and record it."""
|
| 174 |
+
m = np.nanmax(a) if np.isfinite(a).any() else 0
|
| 175 |
+
return (a * scale, True) if m > raw_if_above else (a, False)
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
# ----------------------------------------------------------------------------- per-date workers
|
| 179 |
+
def ndvi_date(args) -> tuple:
|
| 180 |
+
date, ndvi_src, rel_src = args
|
| 181 |
+
ndvi, scaled = autoscale(to_master(ndvi_src, Resampling.nearest), 1e-4, 1.5)
|
| 182 |
+
rel = to_master(rel_src, Resampling.nearest) if rel_src else np.full(GRID["shape"], np.nan, "float32")
|
| 183 |
+
return date, ndvi.astype("float32"), rel.astype("float32"), scaled
|
| 184 |
+
|
| 185 |
+
|
| 186 |
+
def _lst(raw, nodata):
|
| 187 |
+
a = raw.astype("float32")
|
| 188 |
+
a[(raw == 0) | ((nodata is not None) & (raw == nodata))] = np.nan
|
| 189 |
+
return a
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
def lst_date(args) -> tuple:
|
| 193 |
+
date, day_src, night_src, qcd_src, qcn_src = args
|
| 194 |
+
out = []
|
| 195 |
+
for vals, qcs in ((day_src, qcd_src), (night_src, qcn_src)):
|
| 196 |
+
t, _ = autoscale(to_master(vals, Resampling.nearest, fn=_lst), 0.02, 1000)
|
| 197 |
+
qc = to_master(qcs, Resampling.nearest, fn=lambda r, nd: r.astype("float32"))
|
| 198 |
+
good = np.isfinite(qc) & ((np.nan_to_num(qc).astype("uint8") & 3) <= 1) # bits 0-1: 00 good, 01 other
|
| 199 |
+
out.append(np.where(good, t - 273.15, np.nan).astype("float32"))
|
| 200 |
+
return date, out[0], out[1]
|
| 201 |
+
|
| 202 |
+
|
| 203 |
+
def _snow(raw, nodata): # MOD10A2 Maximum_Snow_Extent: 200 snow, 25 no snow; everything else invalid
|
| 204 |
+
a = np.full(raw.shape, np.nan, "float32")
|
| 205 |
+
a[raw == 200] = 1.0
|
| 206 |
+
a[raw == 25] = 0.0
|
| 207 |
+
return a
|
| 208 |
+
|
| 209 |
+
|
| 210 |
+
def snow_date(args) -> tuple:
|
| 211 |
+
date, srcs = args
|
| 212 |
+
return date, to_master(srcs, Resampling.average, fn=_snow)
|
| 213 |
+
|
| 214 |
+
|
| 215 |
+
# ----------------------------------------------------------------------------- static layers
|
| 216 |
+
def static_layers(rc: tuple[np.ndarray, np.ndarray]) -> dict:
|
| 217 |
+
r, c = rc
|
| 218 |
+
out = {}
|
| 219 |
+
lc_p, lw_p = flat(sources("MCD12Q1", "LC_Type1")), flat(sources("MCD12Q1", "LW"))
|
| 220 |
+
out["lc_type1"] = to_master(lc_p, Resampling.mode,
|
| 221 |
+
fn=lambda raw, nd: np.where(raw == 255, np.nan, raw).astype("float32"))[r, c]
|
| 222 |
+
for name, cls in {"grass_frac": 10, "ice_frac": 15, "urban_frac": 13, "barren_frac": 16}.items():
|
| 223 |
+
out[name] = to_master(lc_p, Resampling.average,
|
| 224 |
+
fn=lambda raw, nd, k=cls: np.where(raw == 255, np.nan, raw == k).astype("float32"))[r, c]
|
| 225 |
+
out["water_frac"] = to_master(lw_p, Resampling.average,
|
| 226 |
+
fn=lambda raw, nd: np.where(raw == 255, np.nan, raw == 1).astype("float32"))[r, c]
|
| 227 |
+
|
| 228 |
+
acc = {k: np.full(GRID["shape"], np.nan, "float32") for k in ("z", "z2", "slope", "steep", "asin", "acos")}
|
| 229 |
+
for src in flat(sources("NASADEM", "NASADEM_HGT")): # slope on the native 30 m grid first, then average
|
| 230 |
+
raw, nodata, tr, crs = _open(src)
|
| 231 |
+
z = _nodata_to_nan(raw, nodata if nodata is not None else -32768)
|
| 232 |
+
lat = tr.f + tr.e * (np.arange(z.shape[0]) + 0.5)
|
| 233 |
+
dy = abs(tr.e) * 111_320.0
|
| 234 |
+
dx = tr.a * 111_320.0 * np.cos(np.deg2rad(lat))[:, None]
|
| 235 |
+
gy_rows, gx_cols = np.gradient(z)
|
| 236 |
+
gx, gy = gx_cols / dx, -gy_rows / dy # east, north
|
| 237 |
+
slope = np.degrees(np.arctan(np.hypot(gx, gy))).astype("float32")
|
| 238 |
+
aspect = np.arctan2(-gx, -gy) # downslope direction, clockwise from north
|
| 239 |
+
layers = {"z": z, "z2": z * z, "slope": slope,
|
| 240 |
+
"steep": np.where(np.isnan(slope), np.nan, slope > STEEP_DEG).astype("float32"),
|
| 241 |
+
"asin": np.sin(aspect).astype("float32"), "acos": np.cos(aspect).astype("float32")}
|
| 242 |
+
for k, arr in layers.items():
|
| 243 |
+
tmp = np.full(GRID["shape"], np.nan, "float32")
|
| 244 |
+
reproject(arr.astype("float32"), tmp, src_transform=tr, src_crs=crs,
|
| 245 |
+
dst_transform=GRID["transform"], dst_crs=GRID["crs"],
|
| 246 |
+
resampling=Resampling.average, src_nodata=np.nan, dst_nodata=np.nan)
|
| 247 |
+
acc[k] = np.where(np.isnan(acc[k]), tmp, acc[k])
|
| 248 |
+
elev = acc["z"]
|
| 249 |
+
out["elev_mean"] = elev[r, c]
|
| 250 |
+
out["elev_std"] = np.sqrt(np.clip(acc["z2"] - elev ** 2, 0, None))[r, c]
|
| 251 |
+
out["slope_mean"], out["steep_share"] = acc["slope"][r, c], acc["steep"][r, c]
|
| 252 |
+
out["aspect_sin"], out["aspect_cos"] = acc["asin"][r, c], acc["acos"][r, c]
|
| 253 |
+
valid = np.isfinite(elev)
|
| 254 |
+
num = uniform_filter(np.where(valid, elev, 0), TPI_WINDOW)
|
| 255 |
+
den = uniform_filter(valid.astype("float32"), TPI_WINDOW)
|
| 256 |
+
out["tpi"] = (elev - num / np.maximum(den, 1e-6))[r, c]
|
| 257 |
+
return out
|
| 258 |
+
|
| 259 |
+
|
| 260 |
+
def mask_reason(df: pd.DataFrame) -> pd.Series:
|
| 261 |
+
reason = pd.Series(pd.NA, index=df.index, dtype="string")
|
| 262 |
+
rules = [("water", (df["water_frac"] > 0.5) | (df["lc_type1"] == 17)),
|
| 263 |
+
("ice", df["lc_type1"] == 15),
|
| 264 |
+
("urban", df["lc_type1"] == 13),
|
| 265 |
+
("too_steep", df["slope_mean"] > STEEP_DEG)]
|
| 266 |
+
for name, cond in rules: # first matching rule wins
|
| 267 |
+
reason = reason.mask(reason.isna() & cond.fillna(False), name)
|
| 268 |
+
return reason
|
| 269 |
+
|
| 270 |
+
|
| 271 |
+
# ----------------------------------------------------------------------------- weather (GES DISC)
|
| 272 |
+
def weather(cells: pd.DataFrame) -> pd.DataFrame:
|
| 273 |
+
lon = xr.DataArray(cells["lon"].values, dims="cell")
|
| 274 |
+
lat = xr.DataArray(cells["lat"].values, dims="cell")
|
| 275 |
+
pad = 0.5
|
| 276 |
+
lo0, lo1 = float(lon.min()) - pad, float(lon.max()) + pad
|
| 277 |
+
la0, la1 = float(lat.min()) - pad, float(lat.max()) + pad
|
| 278 |
+
ids = cells["cell_id"].values
|
| 279 |
+
|
| 280 |
+
frames = []
|
| 281 |
+
for p in sorted((RAW / "gesdisc" / "GPM_3IMERGM").glob("*.HDF5")):
|
| 282 |
+
ym = re.search(r"\.(\d{8})-S", p.name).group(1)
|
| 283 |
+
y, m = int(ym[:4]), int(ym[4:6])
|
| 284 |
+
with xr.open_dataset(p, group="Grid", engine="h5netcdf") as ds:
|
| 285 |
+
da = ds["precipitation"].squeeze(drop=True).sel(lon=slice(lo0, lo1), lat=slice(la0, la1)).load()
|
| 286 |
+
mm_month = da.interp(lon=lon, lat=lat) * 24 * calendar.monthrange(y, m)[1] # mm/hr -> mm/month
|
| 287 |
+
frames.append(pd.DataFrame({"cell_id": ids, "year": y, "month": m,
|
| 288 |
+
"precip": mm_month.values.astype("float32")}))
|
| 289 |
+
precip = pd.concat(frames, ignore_index=True)
|
| 290 |
+
|
| 291 |
+
frames = []
|
| 292 |
+
for p in sorted((RAW / "gesdisc" / "GLDAS_NOAH025_M").glob("*.nc4")):
|
| 293 |
+
ym = re.search(r"\.A(\d{6})\.", p.name).group(1)
|
| 294 |
+
y, m = int(ym[:4]), int(ym[4:])
|
| 295 |
+
with xr.open_dataset(p) as ds:
|
| 296 |
+
sub = ds[["SoilMoi0_10cm_inst", "SWE_inst"]].squeeze(drop=True).sel(
|
| 297 |
+
lon=slice(lo0, lo1), lat=slice(la0, la1)).load()
|
| 298 |
+
pts = sub.interp(lon=lon, lat=lat)
|
| 299 |
+
frames.append(pd.DataFrame({"cell_id": ids, "year": y, "month": m,
|
| 300 |
+
"soil_moist": pts["SoilMoi0_10cm_inst"].values.astype("float32"),
|
| 301 |
+
"swe": pts["SWE_inst"].values.astype("float32")}))
|
| 302 |
+
gldas = pd.concat(frames, ignore_index=True)
|
| 303 |
+
return precip.merge(gldas, on=["cell_id", "year", "month"], how="outer")
|
| 304 |
+
|
| 305 |
+
|
| 306 |
+
# ----------------------------------------------------------------------------- main
|
| 307 |
+
def monthly_mean(dated: list[tuple], r, c) -> dict:
|
| 308 |
+
"""Average 8-day composites into calendar months (by composite start date)."""
|
| 309 |
+
groups: dict = {}
|
| 310 |
+
for date, arr in dated:
|
| 311 |
+
groups.setdefault((date.year, date.month), []).append(arr[r, c])
|
| 312 |
+
with np.errstate(all="ignore"):
|
| 313 |
+
return {k: np.nanmean(np.stack(v), axis=0).astype("float32") for k, v in groups.items()}
|
| 314 |
+
|
| 315 |
+
|
| 316 |
+
def build_cells() -> tuple[pd.DataFrame, np.ndarray, np.ndarray]:
|
| 317 |
+
nrow, ncol = GRID["shape"]
|
| 318 |
+
rr, cc = np.meshgrid(np.arange(nrow), np.arange(ncol), indexing="ij")
|
| 319 |
+
x = GRID["left"] + (cc + 0.5) * PIX1K
|
| 320 |
+
y = GRID["top"] - (rr + 0.5) * PIX1K
|
| 321 |
+
lon, lat = Transformer.from_crs(GRID["crs"], 4326, always_xy=True).transform(x, y)
|
| 322 |
+
aoi = gpd.read_file(AOI_DIR / "aoi_request.geojson").to_crs(4326)
|
| 323 |
+
region = np.full(x.shape, "", dtype=object)
|
| 324 |
+
for _, f in aoi.iterrows():
|
| 325 |
+
region[(region == "") & contains_xy(f.geometry, lon, lat)] = f["name"]
|
| 326 |
+
keep = region != ""
|
| 327 |
+
r, c = rr[keep], cc[keep]
|
| 328 |
+
cells = pd.DataFrame({
|
| 329 |
+
# global MODIS row/col -> stable id that does not depend on our crop
|
| 330 |
+
"cell_id": ((r + GRID["global_row0"]) * 43200 + (c + GRID["global_col0"])).astype("int64"),
|
| 331 |
+
"row": r, "col": c, "x": x[keep], "y": y[keep], "lon": lon[keep], "lat": lat[keep],
|
| 332 |
+
"region": region[keep].astype(str),
|
| 333 |
+
"block_id": [f"{a}-{b}" for a, b in zip(r // BLOCK, c // BLOCK)],
|
| 334 |
+
"block_row": (r // BLOCK).astype("int16"), "block_col": (c // BLOCK).astype("int16")})
|
| 335 |
+
dist = gpd.read_file(AOI_DIR / "naryn_districts.geojson").to_crs(4326)
|
| 336 |
+
cells["district"] = pd.Series(pd.NA, index=cells.index, dtype="string")
|
| 337 |
+
for _, d in dist.iterrows():
|
| 338 |
+
cells.loc[contains_xy(d.geometry, cells["lon"].values, cells["lat"].values), "district"] = d["shapeName"]
|
| 339 |
+
cells["in_oblast"] = cells["district"].notna()
|
| 340 |
+
adm1 = gpd.read_file(AOI_DIR / "geoBoundaries-KGZ-ADM1.geojson").to_crs(4326)
|
| 341 |
+
gb = adm1[adm1["shapeName"].str.contains("Naryn")].geometry.iloc[0]
|
| 342 |
+
cells["in_adm1_naryn"] = contains_xy(gb, cells["lon"].values, cells["lat"].values)
|
| 343 |
+
cells["boundary_conflict"] = cells["in_oblast"] != cells["in_adm1_naryn"]
|
| 344 |
+
return cells, r, c
|
| 345 |
+
|
| 346 |
+
|
| 347 |
+
def write_monthly(cells, ndvi, rel, aqua, lst_day, lst_night, snow, wx) -> list[int]:
|
| 348 |
+
"""One parquet file per year under monthly/year=YYYY/ (hive layout: pd.read_parquet(dir) adds `year`)."""
|
| 349 |
+
nan = np.full(len(cells), np.nan, "float32")
|
| 350 |
+
ids = cells["cell_id"].values
|
| 351 |
+
years = sorted({y for y, _ in ndvi})
|
| 352 |
+
for yr in years:
|
| 353 |
+
parts = []
|
| 354 |
+
for m in range(1, 13):
|
| 355 |
+
k = (yr, m)
|
| 356 |
+
if k not in ndvi:
|
| 357 |
+
continue
|
| 358 |
+
rk = rel.get(k, nan)
|
| 359 |
+
cols = {"cell_id": ids, "year": yr, "month": m, "ndvi": ndvi[k], "ndvi_rel": rk,
|
| 360 |
+
"ndvi_ok": np.isin(rk, (0, 1)), "ndvi_aqua": aqua.get(k, nan)}
|
| 361 |
+
if wx is not None:
|
| 362 |
+
cols |= {"lst_day": lst_day.get(k, nan), "lst_night": lst_night.get(k, nan), "snow": snow.get(k, nan)}
|
| 363 |
+
parts.append(pd.DataFrame(cols))
|
| 364 |
+
df = pd.concat(parts, ignore_index=True)
|
| 365 |
+
if wx is not None:
|
| 366 |
+
df = df.merge(wx[wx["year"] == yr], on=["cell_id", "year", "month"], how="left")
|
| 367 |
+
df["month"] = df["month"].astype("int8")
|
| 368 |
+
dest = OUT / "monthly" / f"year={yr}"
|
| 369 |
+
dest.mkdir(parents=True, exist_ok=True)
|
| 370 |
+
pq.write_table(pa.Table.from_pandas(df.drop(columns=["year"]), preserve_index=False), dest / "part.parquet")
|
| 371 |
+
return years
|
| 372 |
+
|
| 373 |
+
|
| 374 |
+
def main() -> None:
|
| 375 |
+
global GRID
|
| 376 |
+
ap = argparse.ArgumentParser()
|
| 377 |
+
ap.add_argument("--only", choices=["static", "all"], default="all")
|
| 378 |
+
args = ap.parse_args()
|
| 379 |
+
t0 = time.time()
|
| 380 |
+
OUT.mkdir(parents=True, exist_ok=True)
|
| 381 |
+
manifest: dict = {"created_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "synthetic": False,
|
| 382 |
+
"partial": True,
|
| 383 |
+
"units": {"ndvi": "NDVI (scaled, -0.2..1)", "ndvi_rel": "MOD13A3 pixel reliability (0 good, 1 marginal, 2 snow/ice, 3 cloud)",
|
| 384 |
+
"ndvi_ok": "bool, reliability 0 or 1", "ndvi_aqua": "NDVI from MYD13A3, reliability 0-1 only",
|
| 385 |
+
"lst_day": "deg C, monthly mean of QA-good 8-day MOD11A2", "lst_night": "deg C",
|
| 386 |
+
"snow": "fraction 0-1 of 8-day MOD10A2 periods with snow (Mar-Sep only)",
|
| 387 |
+
"precip": "mm per month (IMERG Final V07, mm/hr x 24 x days)",
|
| 388 |
+
"soil_moist": "kg m-2 in 0-10 cm (GLDAS-2.1 Noah)", "swe": "kg m-2 (GLDAS-2.1 Noah)",
|
| 389 |
+
"elev_mean": "m", "slope_mean": "deg", "steep_share": f"share of 30 m pixels steeper than {STEEP_DEG} deg",
|
| 390 |
+
"lc_type1": "IGBP class (MCD12Q1 LC_Type1, 2020), mode over 500 m pixels"},
|
| 391 |
+
"datasets": {"MOD13A3": "061, doi:10.5067/MODIS/MOD13A3.061", "MYD13A3": "061, doi:10.5067/MODIS/MYD13A3.061",
|
| 392 |
+
"MOD11A2": "061, doi:10.5067/MODIS/MOD11A2.061", "MOD10A2": "61, doi:10.5067/MODIS/MOD10A2.061",
|
| 393 |
+
"MCD12Q1": "061, doi:10.5067/MODIS/MCD12Q1.061", "NASADEM_HGT": "001, doi:10.5067/MEaSUREs/NASADEM/NASADEM_HGT.001",
|
| 394 |
+
"GPM_3IMERGM": "07, doi:10.5067/GPM/IMERG/3B-MONTH/07", "GLDAS_NOAH025_M": "2.1, doi:10.5067/SXAVCZFAQLNO"},
|
| 395 |
+
"notes": ["Oblast = 5 official districts (geoBoundaries ADM2). geoBoundaries ADM1 'Naryn Region' "
|
| 396 |
+
"disagrees (adds Toguz-Toro, misses ~1,900 km2 of Jumgal near Suusamyr); such cells "
|
| 397 |
+
"have boundary_conflict=true.",
|
| 398 |
+
f"mask_reason 'too_steep' uses slope_mean > {STEEP_DEG} deg (default, to be agreed)."]}
|
| 399 |
+
|
| 400 |
+
GRID = build_master_grid()
|
| 401 |
+
manifest["grid"] = {"crs": SINU, "shape": GRID["shape"], "res_m": PIX1K,
|
| 402 |
+
"origin_left_top": [GRID["left"], GRID["top"]], "block_cells": BLOCK,
|
| 403 |
+
"cell_id": "global_row * 43200 + global_col on the MODIS 1 km grid"}
|
| 404 |
+
print(f"master grid {GRID['shape']} ({PIX1K:.3f} m)", flush=True)
|
| 405 |
+
|
| 406 |
+
cells, r, c = build_cells()
|
| 407 |
+
for k, v in static_layers((r, c)).items():
|
| 408 |
+
cells[k] = v
|
| 409 |
+
cells["mask_reason"] = mask_reason(cells)
|
| 410 |
+
cells.to_parquet(OUT / "cells.parquet", index=False)
|
| 411 |
+
manifest["cells"] = {"n": len(cells), "in_oblast": int(cells["in_oblast"].sum()),
|
| 412 |
+
"by_region": cells["region"].value_counts().astype(int).to_dict(),
|
| 413 |
+
"mask_reason_counts": {str(k): int(v) for k, v in
|
| 414 |
+
cells["mask_reason"].value_counts(dropna=False).items()}}
|
| 415 |
+
print(f"cells: {len(cells)} ({cells['in_oblast'].sum()} in oblast) {time.time() - t0:.0f}s", flush=True)
|
| 416 |
+
if args.only == "static":
|
| 417 |
+
manifest["sources_used"] = dict(SOURCES_USED)
|
| 418 |
+
(OUT / "manifest.json").write_text(json.dumps(manifest, indent=2, default=str))
|
| 419 |
+
print("STATIC_DONE", flush=True)
|
| 420 |
+
return
|
| 421 |
+
|
| 422 |
+
ndvi, rel, aqua, scaled_flags = {}, {}, {}, set()
|
| 423 |
+
with ProcessPoolExecutor(WORKERS) as pool:
|
| 424 |
+
nd, nr = sources("MOD13A3", "NDVI"), sources("MOD13A3", "pixel_reliability")
|
| 425 |
+
for d, a, b, s in pool.map(ndvi_date, [(d, nd[d], nr.get(d, [])) for d in nd]):
|
| 426 |
+
ndvi[(d.year, d.month)], rel[(d.year, d.month)] = a[r, c], b[r, c]
|
| 427 |
+
scaled_flags.add(s)
|
| 428 |
+
ad, ar = sources("MYD13A3", "NDVI"), sources("MYD13A3", "pixel_reliability")
|
| 429 |
+
for d, a, b, _ in pool.map(ndvi_date, [(d, ad[d], ar.get(d, [])) for d in ad]):
|
| 430 |
+
aqua[(d.year, d.month)] = np.where(np.isin(b[r, c], (0, 1)), a[r, c], np.nan)
|
| 431 |
+
print(f"ndvi: {len(ndvi)} months, aqua: {len(aqua)} {time.time() - t0:.0f}s", flush=True)
|
| 432 |
+
write_monthly(cells, ndvi, rel, aqua, {}, {}, {}, None)
|
| 433 |
+
manifest["sources_used"] = dict(SOURCES_USED)
|
| 434 |
+
manifest["monthly"] = {"columns": ["cell_id", "year", "month", "ndvi", "ndvi_rel", "ndvi_ok", "ndvi_aqua"],
|
| 435 |
+
"note": "PARTIAL: NDVI only; weather/LST/snow columns follow in the full build"}
|
| 436 |
+
(OUT / "manifest.json").write_text(json.dumps(manifest, indent=2, default=str))
|
| 437 |
+
print("PARTIAL_NDVI_DONE", flush=True)
|
| 438 |
+
|
| 439 |
+
L = {k: sources("MOD11A2", k) for k in ("LST_Day_1km", "LST_Night_1km", "QC_Day", "QC_Night")}
|
| 440 |
+
jobs = [(d, L["LST_Day_1km"][d], L["LST_Night_1km"].get(d, []), L["QC_Day"].get(d, []),
|
| 441 |
+
L["QC_Night"].get(d, [])) for d in L["LST_Day_1km"]]
|
| 442 |
+
res = list(pool.map(lst_date, jobs))
|
| 443 |
+
lst_day = monthly_mean([(d, a) for d, a, _ in res], r, c)
|
| 444 |
+
lst_night = monthly_mean([(d, b) for d, _, b in res], r, c)
|
| 445 |
+
sn = sources("MOD10A2", "Maximum_Snow_Extent")
|
| 446 |
+
snow = monthly_mean(list(pool.map(snow_date, list(sn.items()))), r, c)
|
| 447 |
+
print(f"lst: {len(lst_day)} months, snow: {len(snow)} months {time.time() - t0:.0f}s", flush=True)
|
| 448 |
+
manifest["ndvi_scale_applied_by_us"] = sorted(scaled_flags)
|
| 449 |
+
manifest["sources_used"] = dict(SOURCES_USED)
|
| 450 |
+
|
| 451 |
+
wx = weather(cells)
|
| 452 |
+
print(f"weather rows: {len(wx)} {time.time() - t0:.0f}s", flush=True)
|
| 453 |
+
|
| 454 |
+
years = write_monthly(cells, ndvi, rel, aqua, lst_day, lst_night, snow, wx)
|
| 455 |
+
manifest["partial"] = False
|
| 456 |
+
manifest["monthly"] = {"years": [years[0], years[-1]], "months_ndvi": len(ndvi), "months_aqua": len(aqua),
|
| 457 |
+
"months_lst": len(lst_day), "months_snow": len(snow), "weather_rows": int(len(wx))}
|
| 458 |
+
manifest["inputs"] = {f"{base.name}/{p.name}": len(list(p.glob("*")))
|
| 459 |
+
for base in (RAW / "appeears", RAW / "earthaccess", RAW / "gesdisc") if base.exists()
|
| 460 |
+
for p in base.iterdir() if p.is_dir()}
|
| 461 |
+
(OUT / "manifest.json").write_text(json.dumps(manifest, indent=2, default=str))
|
| 462 |
+
print(f"CUBE_DONE {time.time() - t0:.0f}s", flush=True)
|
| 463 |
+
|
| 464 |
+
|
| 465 |
+
if __name__ == "__main__":
|
| 466 |
+
main()
|
training_code/scripts/25_qa_cube.py
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Sanity checks on the built cube. Prints a report and writes data/cube/qa_report.json.
|
| 2 |
+
|
| 3 |
+
Checks are descriptive (no thresholds tuned to make things pass); the report is meant to be read.
|
| 4 |
+
"""
|
| 5 |
+
from __future__ import annotations
|
| 6 |
+
|
| 7 |
+
import json
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
|
| 10 |
+
import numpy as np
|
| 11 |
+
import pandas as pd
|
| 12 |
+
|
| 13 |
+
CUBE = Path("/root/trendex-ml/data/cube")
|
| 14 |
+
PLACES = {"Son-Kul pastures (N shore)": (41.90, 75.15), "Naryn city": (41.43, 75.99),
|
| 15 |
+
"At-Bashy valley": (41.17, 75.80), "Suusamyr valley": (42.15, 74.00)}
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def nearest_cell(cells: pd.DataFrame, lat: float, lon: float) -> int:
|
| 19 |
+
d = (cells["lat"] - lat) ** 2 + ((cells["lon"] - lon) * np.cos(np.deg2rad(lat))) ** 2
|
| 20 |
+
return int(cells.loc[d.idxmin(), "cell_id"])
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def main() -> None:
|
| 24 |
+
cells = pd.read_parquet(CUBE / "cells.parquet")
|
| 25 |
+
rep: dict = {"cells": len(cells), "in_oblast": int(cells["in_oblast"].sum()),
|
| 26 |
+
"boundary_conflict": int(cells["boundary_conflict"].sum()),
|
| 27 |
+
"by_district": {str(k): int(v) for k, v in cells["district"].value_counts(dropna=False).items()},
|
| 28 |
+
"mask_reason": {str(k): int(v) for k, v in cells["mask_reason"].value_counts(dropna=False).items()}}
|
| 29 |
+
num = ["elev_mean", "elev_std", "slope_mean", "steep_share", "tpi", "grass_frac", "water_frac", "ice_frac"]
|
| 30 |
+
rep["static_ranges"] = cells[num].describe(percentiles=[0.01, 0.5, 0.99]).round(3).to_dict()
|
| 31 |
+
rep["static_missing_share"] = cells[num + ["lc_type1"]].isna().mean().round(4).to_dict()
|
| 32 |
+
|
| 33 |
+
monthly_dir = CUBE / "monthly"
|
| 34 |
+
if monthly_dir.exists():
|
| 35 |
+
m = pd.read_parquet(monthly_dir)
|
| 36 |
+
m["year"] = m["year"].astype(int)
|
| 37 |
+
m = m.merge(cells[["cell_id", "in_oblast", "mask_reason"]], on="cell_id")
|
| 38 |
+
past = m[m["in_oblast"] & m["mask_reason"].isna()]
|
| 39 |
+
rep["monthly_rows"] = len(m)
|
| 40 |
+
rep["years"] = [int(m["year"].min()), int(m["year"].max())]
|
| 41 |
+
rep["ndvi_ok_share_by_month"] = past.groupby("month")["ndvi_ok"].mean().round(3).to_dict()
|
| 42 |
+
cols = ["ndvi", "ndvi_aqua", "lst_day", "lst_night", "snow", "precip", "soil_moist", "swe"]
|
| 43 |
+
rep["monthly_ranges"] = past[cols].describe(percentiles=[0.01, 0.5, 0.99]).round(3).to_dict()
|
| 44 |
+
rep["monthly_missing_share_MaySep"] = past[past["month"].between(5, 9)][cols].isna().mean().round(3).to_dict()
|
| 45 |
+
both = past[past["ndvi_ok"] & past["ndvi_aqua"].notna()]
|
| 46 |
+
rep["terra_aqua_ndvi_corr"] = round(float(both["ndvi"].corr(both["ndvi_aqua"])), 3) if len(both) else None
|
| 47 |
+
summer = past[past["month"].between(6, 8) & past["ndvi_ok"]]
|
| 48 |
+
rep["oblast_summer_ndvi_by_year"] = summer.groupby("year")["ndvi"].mean().round(4).to_dict()
|
| 49 |
+
rep["places"] = {}
|
| 50 |
+
for name, (lat, lon) in PLACES.items():
|
| 51 |
+
cid = nearest_cell(cells, lat, lon)
|
| 52 |
+
s = m[(m["cell_id"] == cid) & m["month"].between(6, 8)]
|
| 53 |
+
row = cells[cells["cell_id"] == cid].iloc[0]
|
| 54 |
+
rep["places"][name] = {
|
| 55 |
+
"cell_id": cid, "district": None if pd.isna(row["district"]) else row["district"],
|
| 56 |
+
"mask_reason": None if pd.isna(row["mask_reason"]) else row["mask_reason"],
|
| 57 |
+
"elev_m": round(float(row["elev_mean"]), 0),
|
| 58 |
+
"summer_ndvi_mean": round(float(s.loc[s["ndvi_ok"], "ndvi"].mean()), 3),
|
| 59 |
+
"summer_precip_mm_mean": round(float(s["precip"].mean()), 1),
|
| 60 |
+
"summer_lst_day_c_mean": round(float(s["lst_day"].mean()), 1)}
|
| 61 |
+
(CUBE / "qa_report.json").write_text(json.dumps(rep, indent=2, default=str, ensure_ascii=False))
|
| 62 |
+
print(json.dumps(rep, indent=1, default=str, ensure_ascii=False))
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
if __name__ == "__main__":
|
| 66 |
+
main()
|
training_code/scripts/30_train.py
ADDED
|
@@ -0,0 +1,347 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Train and evaluate the expected-greenness model exactly as locked in configs/model_v1.yaml.
|
| 2 |
+
|
| 3 |
+
python scripts/30_train.py --git-commit <hash> [--tune-minutes 25]
|
| 4 |
+
|
| 5 |
+
Writes runs/v1/: metrics.json, expected_oof.parquet, model files for the Space, feature importance.
|
| 6 |
+
Nothing here looks at the test years before the final evaluation step.
|
| 7 |
+
"""
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import argparse
|
| 11 |
+
import json
|
| 12 |
+
import time
|
| 13 |
+
from pathlib import Path
|
| 14 |
+
|
| 15 |
+
import numpy as np
|
| 16 |
+
import pandas as pd
|
| 17 |
+
import xgboost as xgb
|
| 18 |
+
from sklearn.linear_model import Ridge
|
| 19 |
+
from sklearn.model_selection import GroupKFold
|
| 20 |
+
|
| 21 |
+
CUBE = Path("/root/trendex-ml/data/cube")
|
| 22 |
+
RUN = Path("/root/trendex-ml/runs/v1")
|
| 23 |
+
Y0, NY = 2000, 26
|
| 24 |
+
YEARS = np.arange(2001, 2026)
|
| 25 |
+
MONTHS = np.arange(5, 10)
|
| 26 |
+
STATIC = ["elev_mean", "elev_std", "slope_mean", "steep_share", "aspect_sin", "aspect_cos", "tpi",
|
| 27 |
+
"grass_frac", "barren_frac", "lc_type1"]
|
| 28 |
+
WEATHER = ["precip_m", "precip_3m", "precip_winter", "lst_day_m", "lst_night_m", "lst_day_prev",
|
| 29 |
+
"snow_spring", "snow_m", "soil_m", "swe_apr"]
|
| 30 |
+
FEATURES = STATIC + ["month", "clim_ndvi"] + WEATHER
|
| 31 |
+
ABLATION = {"precip": ["precip_m", "precip_3m", "precip_winter"],
|
| 32 |
+
"+lst": ["precip_m", "precip_3m", "precip_winter", "lst_day_m", "lst_night_m", "lst_day_prev"],
|
| 33 |
+
"+snow": ["precip_m", "precip_3m", "precip_winter", "lst_day_m", "lst_night_m", "lst_day_prev",
|
| 34 |
+
"snow_spring", "snow_m"],
|
| 35 |
+
"+gldas (full)": WEATHER}
|
| 36 |
+
SEEDS = [0, 1, 2, 3, 4]
|
| 37 |
+
FIXED = {"tree_method": "hist", "device": "cuda", "objective": "reg:squarederror"}
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def log(msg: str) -> None:
|
| 41 |
+
print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True)
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
# ----------------------------------------------------------------------------- data
|
| 45 |
+
def load():
|
| 46 |
+
cells = pd.read_parquet(CUBE / "cells.parquet")
|
| 47 |
+
sel = cells[cells["in_oblast"] & cells["mask_reason"].isna()].reset_index(drop=True)
|
| 48 |
+
cols = ["cell_id", "month", "ndvi", "ndvi_ok", "precip", "lst_day", "lst_night", "snow", "soil_moist", "swe", "year"]
|
| 49 |
+
m = pd.read_parquet(CUBE / "monthly", columns=cols)
|
| 50 |
+
m["year"] = m["year"].astype(int)
|
| 51 |
+
m = m[m["cell_id"].isin(sel["cell_id"])]
|
| 52 |
+
pos = pd.Series(np.arange(len(sel)), index=sel["cell_id"].values)
|
| 53 |
+
ci, yi, mi = pos[m["cell_id"].values].values, m["year"].values - Y0, m["month"].values - 1
|
| 54 |
+
|
| 55 |
+
def cube(col):
|
| 56 |
+
a = np.full((len(sel), NY, 12), np.nan, np.float32)
|
| 57 |
+
a[ci, yi, mi] = m[col].values.astype("float32")
|
| 58 |
+
return a
|
| 59 |
+
|
| 60 |
+
arr = {k: cube(k) for k in ["ndvi", "precip", "lst_day", "lst_night", "snow", "soil_moist", "swe"]}
|
| 61 |
+
ok = np.zeros((len(sel), NY, 12), bool)
|
| 62 |
+
ok[ci, yi, mi] = m["ndvi_ok"].values
|
| 63 |
+
arr["ndvi"][~ok] = np.nan
|
| 64 |
+
return sel, arr
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def build_features(sel: pd.DataFrame, arr: dict) -> dict:
|
| 68 |
+
"""All features as arrays shaped (cells, 25 years, 5 months). Weather anomalies vs full-period mean."""
|
| 69 |
+
yy, mm = YEARS - Y0, MONTHS - 1
|
| 70 |
+
take = lambda a, dm=0: a[:, yy][:, :, mm - dm] # noqa: E731
|
| 71 |
+
anom_m = lambda x: x - np.nanmean(x, axis=1, keepdims=True) # noqa: E731 per cell-month
|
| 72 |
+
anom_y = lambda x: x - np.nanmean(x, axis=1, keepdims=True) # noqa: E731 per cell (annual)
|
| 73 |
+
rep = lambda x: np.repeat(x[:, :, None], len(MONTHS), axis=2) # noqa: E731
|
| 74 |
+
p, ld, ln, sn, so, sw = (arr[k] for k in ["precip", "lst_day", "lst_night", "snow", "soil_moist", "swe"])
|
| 75 |
+
with np.errstate(all="ignore"):
|
| 76 |
+
raw_m = {"precip_m": take(p), "precip_3m": take(p) + take(p, 1) + take(p, 2),
|
| 77 |
+
"lst_day_m": take(ld), "lst_night_m": take(ln), "lst_day_prev": take(ld, 1),
|
| 78 |
+
"snow_m": take(sn), "soil_m": take(so)}
|
| 79 |
+
raw_y = {"precip_winter": p[:, yy - 1, 9:12].sum(-1) + p[:, yy, 0:4].sum(-1),
|
| 80 |
+
"snow_spring": np.nanmean(sn[:, yy, 2:5], axis=-1), "swe_apr": sw[:, yy, 3]}
|
| 81 |
+
f = {k: anom_m(v) for k, v in raw_m.items()} | {k: rep(anom_y(v)) for k, v in raw_y.items()}
|
| 82 |
+
# weather climatology for inference: raw value - this mean = the anomaly the model sees
|
| 83 |
+
wc = [pd.DataFrame({"cell_id": sel["cell_id"].values, "month": int(mo)} |
|
| 84 |
+
{f"{k}_mean": np.nanmean(v[:, :, j], axis=1) for k, v in raw_m.items()} |
|
| 85 |
+
{f"{k}_mean": np.nanmean(v, axis=1) for k, v in raw_y.items()})
|
| 86 |
+
for j, mo in enumerate(MONTHS)]
|
| 87 |
+
pd.concat(wc, ignore_index=True).to_parquet(RUN / "weather_climatology.parquet", index=False)
|
| 88 |
+
shape = (len(sel), len(YEARS), len(MONTHS))
|
| 89 |
+
for k in STATIC:
|
| 90 |
+
f[k] = np.broadcast_to(sel[k].values.astype("float32")[:, None, None], shape)
|
| 91 |
+
f["month"] = np.broadcast_to(MONTHS.astype("float32")[None, None, :], shape)
|
| 92 |
+
f["ndvi"] = take(arr["ndvi"])
|
| 93 |
+
f["year"] = np.broadcast_to(YEARS[None, :, None], shape)
|
| 94 |
+
f["cell"] = np.broadcast_to(np.arange(len(sel))[:, None, None], shape)
|
| 95 |
+
sb = (sel["block_row"].values // 3) * 1000 + sel["block_col"].values // 3
|
| 96 |
+
f["superblock"] = np.broadcast_to(sb[:, None, None], shape)
|
| 97 |
+
return f
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def clim_fold(ndvi: np.ndarray, train_years: np.ndarray) -> np.ndarray:
|
| 101 |
+
"""Cell-month NDVI climatology from training years only; leave-own-year-out for training rows."""
|
| 102 |
+
tm = np.isin(YEARS, train_years)
|
| 103 |
+
s = np.nansum(np.where(tm[None, :, None], ndvi, np.nan), axis=1, keepdims=True)
|
| 104 |
+
n = np.sum(np.isfinite(ndvi) & tm[None, :, None], axis=1, keepdims=True).astype("float32")
|
| 105 |
+
own = np.where(np.isfinite(ndvi) & tm[None, :, None], ndvi, 0.0)
|
| 106 |
+
own_n = (np.isfinite(ndvi) & tm[None, :, None]).astype("float32")
|
| 107 |
+
with np.errstate(all="ignore"):
|
| 108 |
+
return ((s - own) / (n - own_n)).astype("float32")
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
def table(f: dict, train_years: np.ndarray, features: list[str]) -> pd.DataFrame:
|
| 112 |
+
clim = clim_fold(f["ndvi"], train_years)
|
| 113 |
+
d = {k: f[k].ravel() for k in ["cell", "year", "superblock"]}
|
| 114 |
+
for k in features:
|
| 115 |
+
d[k] = (clim if k == "clim_ndvi" else f[k]).ravel()
|
| 116 |
+
d["clim_ndvi"] = clim.ravel()
|
| 117 |
+
d["target"] = (f["ndvi"] - clim).ravel()
|
| 118 |
+
d["ndvi"] = f["ndvi"].ravel()
|
| 119 |
+
df = pd.DataFrame(d)
|
| 120 |
+
return df[np.isfinite(df["target"])].reset_index(drop=True)
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
# ----------------------------------------------------------------------------- models
|
| 124 |
+
def fit_xgb(Xtr, ytr, params, n_estimators, seed, Xva=None, yva=None):
|
| 125 |
+
m = xgb.XGBRegressor(**FIXED, **params, n_estimators=n_estimators, random_state=seed,
|
| 126 |
+
early_stopping_rounds=50 if Xva is not None else None)
|
| 127 |
+
m.fit(Xtr, ytr, eval_set=[(Xva, yva)] if Xva is not None else None, verbose=False)
|
| 128 |
+
return m
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
class RidgeModel:
|
| 132 |
+
def __init__(self, alpha: float):
|
| 133 |
+
self.alpha = alpha
|
| 134 |
+
|
| 135 |
+
def fit(self, X: pd.DataFrame, y):
|
| 136 |
+
self.cols = list(X.columns)
|
| 137 |
+
self.med = X.median()
|
| 138 |
+
Z = X.fillna(self.med)
|
| 139 |
+
self.mu, self.sd = Z.mean(), Z.std().replace(0, 1)
|
| 140 |
+
self.r = Ridge(alpha=self.alpha).fit(((Z - self.mu) / self.sd).values, y)
|
| 141 |
+
return self
|
| 142 |
+
|
| 143 |
+
def predict(self, X: pd.DataFrame):
|
| 144 |
+
Z = X[self.cols].fillna(self.med)
|
| 145 |
+
return self.r.predict(((Z - self.mu) / self.sd).values)
|
| 146 |
+
|
| 147 |
+
def to_json(self) -> dict:
|
| 148 |
+
return {"kind": "ridge", "alpha": self.alpha, "features": self.cols,
|
| 149 |
+
"fill": self.med.to_dict(), "mean": self.mu.to_dict(), "std": self.sd.to_dict(),
|
| 150 |
+
"coef": self.r.coef_.tolist(), "intercept": float(self.r.intercept_)}
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
def split(f, train_years, test_years, features):
|
| 154 |
+
t = table(f, train_years, features)
|
| 155 |
+
tr = t[np.isin(t["year"], train_years)].reset_index(drop=True)
|
| 156 |
+
te = t[np.isin(t["year"], test_years)].reset_index(drop=True)
|
| 157 |
+
return tr, te
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
# ----------------------------------------------------------------------------- metrics
|
| 161 |
+
def boot_skill(sse_a: np.ndarray, sse_b: np.ndarray, rng, n=2000) -> dict:
|
| 162 |
+
"""Skill 1 - SSE_a/SSE_b with a bootstrap over independent units (years or superblocks)."""
|
| 163 |
+
point = 1 - sse_a.sum() / sse_b.sum()
|
| 164 |
+
idx = rng.integers(0, len(sse_a), size=(n, len(sse_a)))
|
| 165 |
+
dist = 1 - sse_a[idx].sum(1) / sse_b[idx].sum(1)
|
| 166 |
+
return {"point": float(point), "ci_lo": float(np.quantile(dist, 0.025)), "ci_hi": float(np.quantile(dist, 0.975)),
|
| 167 |
+
"units": int(len(sse_a))}
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
def tercile_bacc(y_true, y_pred, cuts) -> float:
|
| 171 |
+
t = np.digitize(y_true, cuts)
|
| 172 |
+
p = np.digitize(y_pred, cuts)
|
| 173 |
+
return float(np.mean([np.mean(p[t == k] == k) for k in range(3) if np.any(t == k)]))
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
def score(te: pd.DataFrame, preds: dict, unit: str, cuts, rng) -> dict:
|
| 177 |
+
y = te["target"].values
|
| 178 |
+
out = {"n_rows": int(len(y))}
|
| 179 |
+
sse = {}
|
| 180 |
+
for name, p in preds.items():
|
| 181 |
+
err = y - p
|
| 182 |
+
out[name] = {"rmse": float(np.sqrt(np.mean(err ** 2))), "mae": float(np.mean(np.abs(err))),
|
| 183 |
+
"anomaly_corr": float(np.corrcoef(y, p)[0, 1]) if np.std(p) > 0 else None,
|
| 184 |
+
"tercile_bacc": tercile_bacc(y, p, cuts)}
|
| 185 |
+
sse[name] = pd.Series(err ** 2).groupby(te[unit].values).sum().values
|
| 186 |
+
out["skill_xgb_vs_ridge"] = boot_skill(sse["xgb"], sse["ridge"], rng)
|
| 187 |
+
out["skill_xgb_vs_clim"] = boot_skill(sse["xgb"], sse["clim"], rng)
|
| 188 |
+
out["skill_ridge_vs_clim"] = boot_skill(sse["ridge"], sse["clim"], rng)
|
| 189 |
+
return out
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
# ----------------------------------------------------------------------------- main
|
| 193 |
+
def main() -> None:
|
| 194 |
+
ap = argparse.ArgumentParser()
|
| 195 |
+
ap.add_argument("--git-commit", required=True)
|
| 196 |
+
ap.add_argument("--tune-minutes", type=float, default=25)
|
| 197 |
+
ap.add_argument("--tune-trials", type=int, default=40)
|
| 198 |
+
a = ap.parse_args()
|
| 199 |
+
RUN.mkdir(parents=True, exist_ok=True)
|
| 200 |
+
rng = np.random.default_rng(0)
|
| 201 |
+
metrics: dict = {"protocol": "configs/model_v1.yaml", "git_commit": a.git_commit, "features": FEATURES}
|
| 202 |
+
|
| 203 |
+
sel, arr = load()
|
| 204 |
+
f = build_features(sel, arr)
|
| 205 |
+
log(f"cells {len(sel)}, feature arrays {f['ndvi'].shape}")
|
| 206 |
+
metrics["data"] = {"cells": len(sel), "years": [int(YEARS[0]), int(YEARS[-1])], "months": MONTHS.tolist()}
|
| 207 |
+
|
| 208 |
+
# --- 1. tuning on the inner split only (2001-2015 -> 2016-2019)
|
| 209 |
+
inner_tr, inner_va = split(f, np.arange(2001, 2016), np.arange(2016, 2020), FEATURES)
|
| 210 |
+
Xi, yi_, Xv, yv = inner_tr[FEATURES], inner_tr["target"], inner_va[FEATURES], inner_va["target"]
|
| 211 |
+
alphas = [0.1, 1, 10, 100, 1000]
|
| 212 |
+
ridge_rmse = {al: float(np.sqrt(np.mean((yv - RidgeModel(al).fit(Xi, yi_).predict(Xv)) ** 2))) for al in alphas}
|
| 213 |
+
alpha = min(ridge_rmse, key=ridge_rmse.get)
|
| 214 |
+
log(f"ridge alpha {alpha} inner rmse {ridge_rmse[alpha]:.4f}")
|
| 215 |
+
|
| 216 |
+
import optuna
|
| 217 |
+
optuna.logging.set_verbosity(optuna.logging.WARNING)
|
| 218 |
+
|
| 219 |
+
def objective(trial):
|
| 220 |
+
params = {"max_depth": trial.suggest_int("max_depth", 4, 10),
|
| 221 |
+
"learning_rate": trial.suggest_float("learning_rate", 0.02, 0.2, log=True),
|
| 222 |
+
"min_child_weight": trial.suggest_float("min_child_weight", 1, 200, log=True),
|
| 223 |
+
"subsample": trial.suggest_float("subsample", 0.5, 1.0),
|
| 224 |
+
"colsample_bytree": trial.suggest_float("colsample_bytree", 0.5, 1.0),
|
| 225 |
+
"reg_lambda": trial.suggest_float("reg_lambda", 1e-3, 30, log=True),
|
| 226 |
+
"reg_alpha": trial.suggest_float("reg_alpha", 1e-3, 10, log=True)}
|
| 227 |
+
m = fit_xgb(Xi, yi_, params, 3000, 0, Xv, yv)
|
| 228 |
+
trial.set_user_attr("n_estimators", int(m.best_iteration) + 1)
|
| 229 |
+
return float(np.sqrt(np.mean((yv - m.predict(Xv)) ** 2)))
|
| 230 |
+
|
| 231 |
+
study = optuna.create_study(direction="minimize", sampler=optuna.samplers.TPESampler(seed=0))
|
| 232 |
+
study.optimize(objective, n_trials=a.tune_trials, timeout=a.tune_minutes * 60)
|
| 233 |
+
best, n_est = study.best_params, study.best_trial.user_attrs["n_estimators"]
|
| 234 |
+
log(f"tuning: {len(study.trials)} trials, best inner rmse {study.best_value:.4f}, n_estimators {n_est}")
|
| 235 |
+
metrics["tuning"] = {"trials": len(study.trials), "best_params": best, "n_estimators": n_est,
|
| 236 |
+
"inner_rmse_xgb": study.best_value, "ridge_alpha": alpha, "inner_rmse_ridge": ridge_rmse}
|
| 237 |
+
cuts = np.quantile(yi_, [1 / 3, 2 / 3])
|
| 238 |
+
|
| 239 |
+
# conformal residual quantiles per month from the inner split (model never saw 2016-2019)
|
| 240 |
+
m_inner = fit_xgb(Xi, yi_, best, n_est, 0)
|
| 241 |
+
res_inner = yv.values - m_inner.predict(Xv)
|
| 242 |
+
conf_inner = {int(mo): [float(np.quantile(res_inner[inner_va["month"] == mo], q)) for q in (0.1, 0.9)]
|
| 243 |
+
for mo in MONTHS}
|
| 244 |
+
|
| 245 |
+
# --- 2. temporal test (train 2001-2019, test 2020-2025), 5 seeds
|
| 246 |
+
tr, te = split(f, np.arange(2001, 2020), np.arange(2020, 2026), FEATURES)
|
| 247 |
+
ridge = RidgeModel(alpha).fit(tr[FEATURES], tr["target"])
|
| 248 |
+
p_ridge = ridge.predict(te[FEATURES])
|
| 249 |
+
seed_scores, p_xgb_seeds = [], []
|
| 250 |
+
for s in SEEDS:
|
| 251 |
+
m = fit_xgb(tr[FEATURES], tr["target"], best, n_est, s)
|
| 252 |
+
p = m.predict(te[FEATURES])
|
| 253 |
+
p_xgb_seeds.append(p)
|
| 254 |
+
seed_scores.append(float(1 - np.mean((te["target"] - p) ** 2) / np.mean((te["target"] - p_ridge) ** 2)))
|
| 255 |
+
p_xgb = np.mean(p_xgb_seeds, axis=0)
|
| 256 |
+
temporal = score(te, {"xgb": p_xgb, "ridge": p_ridge, "clim": np.zeros(len(te))}, "year", cuts, rng)
|
| 257 |
+
temporal["seed_skill_vs_ridge"] = {"values": seed_scores, "mean": float(np.mean(seed_scores)),
|
| 258 |
+
"sd": float(np.std(seed_scores))}
|
| 259 |
+
lo, hi = np.array([conf_inner[int(mo)] for mo in te["month"]]).T
|
| 260 |
+
res = te["target"].values - p_xgb
|
| 261 |
+
temporal["conformal_coverage_p10_p90"] = float(np.mean((res >= lo) & (res <= hi)))
|
| 262 |
+
metrics["temporal_test"] = temporal
|
| 263 |
+
log(f"temporal: skill vs ridge {temporal['skill_xgb_vs_ridge']}")
|
| 264 |
+
|
| 265 |
+
# ablation on the temporal split (one seed)
|
| 266 |
+
abl = {}
|
| 267 |
+
for name, wcols in ABLATION.items():
|
| 268 |
+
cols = STATIC + ["month", "clim_ndvi"] + wcols
|
| 269 |
+
m = fit_xgb(tr[cols], tr["target"], best, n_est, 0)
|
| 270 |
+
abl[name] = float(1 - np.mean((te["target"] - m.predict(te[cols])) ** 2) / np.mean(te["target"] ** 2))
|
| 271 |
+
metrics["ablation_skill_vs_clim_temporal"] = abl
|
| 272 |
+
log(f"ablation {abl}")
|
| 273 |
+
|
| 274 |
+
# --- 3. spatial test: GroupKFold over 33x33-cell superblocks, all years
|
| 275 |
+
full = table(f, YEARS, FEATURES)
|
| 276 |
+
preds = {"xgb": np.zeros(len(full)), "ridge": np.zeros(len(full))}
|
| 277 |
+
for k, (itr, ite) in enumerate(GroupKFold(5).split(full, groups=full["superblock"])):
|
| 278 |
+
Xtr, ytr, Xte = full.loc[itr, FEATURES], full.loc[itr, "target"], full.loc[ite, FEATURES]
|
| 279 |
+
preds["xgb"][ite] = fit_xgb(Xtr, ytr, best, n_est, 0).predict(Xte)
|
| 280 |
+
preds["ridge"][ite] = RidgeModel(alpha).fit(Xtr, ytr).predict(Xte)
|
| 281 |
+
preds["clim"] = np.zeros(len(full))
|
| 282 |
+
spatial = score(full, preds, "superblock", cuts, rng)
|
| 283 |
+
metrics["spatial_test"] = spatial
|
| 284 |
+
log(f"spatial: skill vs ridge {spatial['skill_xgb_vs_ridge']}")
|
| 285 |
+
|
| 286 |
+
# --- 4. gate (pre-registered)
|
| 287 |
+
passed = temporal["skill_xgb_vs_ridge"]["ci_lo"] > 0 and spatial["skill_xgb_vs_ridge"]["ci_lo"] > 0
|
| 288 |
+
chosen = "xgb" if passed else "ridge"
|
| 289 |
+
metrics["gate"] = {"rule": "CI lower bound of skill_xgb_vs_ridge > 0 on temporal AND spatial tests",
|
| 290 |
+
"passed": bool(passed), "shipped_model": chosen}
|
| 291 |
+
log(f"GATE passed={passed} -> shipping {chosen}")
|
| 292 |
+
|
| 293 |
+
# --- 5. cross-fitted out-of-fold predictions (5 contiguous year blocks) with the chosen model
|
| 294 |
+
blocks = [np.arange(s, s + 5) for s in (2001, 2006, 2011, 2016, 2021)]
|
| 295 |
+
oof = []
|
| 296 |
+
for blk in blocks:
|
| 297 |
+
trn = np.setdiff1d(YEARS, blk)
|
| 298 |
+
trb, teb = split(f, trn, blk, FEATURES)
|
| 299 |
+
model = fit_xgb(trb[FEATURES], trb["target"], best, n_est, 0) if chosen == "xgb" else \
|
| 300 |
+
RidgeModel(alpha).fit(trb[FEATURES], trb["target"])
|
| 301 |
+
teb = teb.assign(pred_anom=model.predict(teb[FEATURES]))
|
| 302 |
+
oof.append(teb[["cell", "year", "month", "ndvi", "clim_ndvi", "target", "pred_anom"]])
|
| 303 |
+
oof = pd.concat(oof, ignore_index=True)
|
| 304 |
+
res_all = oof["target"] - oof["pred_anom"]
|
| 305 |
+
conf_prod = {int(mo): [float(np.quantile(res_all[oof["month"] == mo], q)) for q in (0.1, 0.9)] for mo in MONTHS}
|
| 306 |
+
lo = oof["month"].map(lambda mo: conf_prod[int(mo)][0])
|
| 307 |
+
hi = oof["month"].map(lambda mo: conf_prod[int(mo)][1])
|
| 308 |
+
oof["cell_id"] = sel["cell_id"].values[oof["cell"].values]
|
| 309 |
+
oof["expected"] = oof["clim_ndvi"] + oof["pred_anom"]
|
| 310 |
+
oof["p10"], oof["p90"] = oof["expected"] + lo, oof["expected"] + hi
|
| 311 |
+
oof["residual"] = res_all
|
| 312 |
+
oof[["cell_id", "year", "month", "ndvi", "expected", "p10", "p90", "residual"]].rename(
|
| 313 |
+
columns={"ndvi": "observed"}).to_parquet(RUN / "expected_oof.parquet", index=False)
|
| 314 |
+
metrics["conformal_p10_p90_by_month"] = conf_prod
|
| 315 |
+
log(f"oof rows {len(oof)}")
|
| 316 |
+
|
| 317 |
+
# --- 6. production model: refit on all years; climatology tables for inference
|
| 318 |
+
full_all = full
|
| 319 |
+
if chosen == "xgb":
|
| 320 |
+
prod = fit_xgb(full_all[FEATURES], full_all["target"], best, n_est, 0)
|
| 321 |
+
prod.save_model(RUN / "model_a.ubj")
|
| 322 |
+
contrib = prod.get_booster().predict(xgb.DMatrix(full_all[FEATURES].sample(200_000, random_state=0)),
|
| 323 |
+
pred_contribs=True)
|
| 324 |
+
imp = pd.Series(np.abs(contrib[:, :-1]).mean(0), index=FEATURES).sort_values(ascending=False)
|
| 325 |
+
else:
|
| 326 |
+
prod = RidgeModel(alpha).fit(full_all[FEATURES], full_all["target"])
|
| 327 |
+
(RUN / "ridge_a.json").write_text(json.dumps(prod.to_json()))
|
| 328 |
+
imp = pd.Series(np.abs(prod.r.coef_), index=FEATURES).sort_values(ascending=False)
|
| 329 |
+
metrics["feature_importance_mean_abs"] = imp.round(6).to_dict()
|
| 330 |
+
yy = YEARS - Y0
|
| 331 |
+
clim_tab = []
|
| 332 |
+
for k, mo in enumerate(MONTHS):
|
| 333 |
+
rec = {"cell_id": sel["cell_id"].values, "month": int(mo),
|
| 334 |
+
"clim_ndvi": np.nanmean(arr["ndvi"][:, yy, mo - 1], axis=1)}
|
| 335 |
+
clim_tab.append(pd.DataFrame(rec))
|
| 336 |
+
pd.concat(clim_tab).to_parquet(RUN / "climatology.parquet", index=False)
|
| 337 |
+
(RUN / "feature_schema.json").write_text(json.dumps({
|
| 338 |
+
"features": FEATURES, "target": "ndvi_anomaly = ndvi - clim_ndvi", "months": MONTHS.tolist(),
|
| 339 |
+
"weather_anomaly_baseline": "cell-month mean 2001-2025 (see climatology.parquet / weather_climatology.parquet)",
|
| 340 |
+
"shipped_model": chosen}, indent=2))
|
| 341 |
+
(RUN / "conformal.json").write_text(json.dumps(conf_prod, indent=2))
|
| 342 |
+
(RUN / "metrics.json").write_text(json.dumps(metrics, indent=2, default=float))
|
| 343 |
+
log("TRAIN_DONE")
|
| 344 |
+
|
| 345 |
+
|
| 346 |
+
if __name__ == "__main__":
|
| 347 |
+
main()
|
training_code/scripts/40_products.py
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Products built from the out-of-fold predictions (runs/v1 -> runs/v1/products).
|
| 2 |
+
|
| 3 |
+
Residuals for trends use ONE reference per cell-month (the all-years climatology), not the per-block
|
| 4 |
+
training climatology used for evaluation. With per-block climatologies a real trend would be inflated
|
| 5 |
+
(each held-out block's reference comes from the other years). pred_anom (the weather effect) still comes
|
| 6 |
+
from models that never saw that block, so no target information leaks into it.
|
| 7 |
+
|
| 8 |
+
Outputs (cell level, plus block and district aggregates):
|
| 9 |
+
expected_oof.parquet cell_id, year, month, observed, expected, p10, p90, residual
|
| 10 |
+
residual_trend*.parquet Sen slope / CI / Mann-Kendall (Hamed-Rao) / BH q / class of summer residuals
|
| 11 |
+
season_vs_expected*.parquet summer mean residual per year -> below / within / above usual for its weather
|
| 12 |
+
aqua_terra_blocks.parquet per-block summer NDVI trend from Terra and Aqua (2003-2025)
|
| 13 |
+
products_summary.json headline shares, placebo, Aqua-Terra agreement
|
| 14 |
+
"""
|
| 15 |
+
from __future__ import annotations
|
| 16 |
+
|
| 17 |
+
import json
|
| 18 |
+
import sys
|
| 19 |
+
import time
|
| 20 |
+
from pathlib import Path
|
| 21 |
+
|
| 22 |
+
import numpy as np
|
| 23 |
+
import pandas as pd
|
| 24 |
+
|
| 25 |
+
sys.path.insert(0, "/root/trendex-ml/src")
|
| 26 |
+
from trendex_ml.stats.trend import bh_fdr, classify, mk_vectorized, trend_many # noqa: E402
|
| 27 |
+
|
| 28 |
+
CUBE = Path("/root/trendex-ml/data/cube")
|
| 29 |
+
RUN = Path("/root/trendex-ml/runs/v1")
|
| 30 |
+
OUT = RUN / "products"
|
| 31 |
+
YEARS = np.arange(2001, 2026)
|
| 32 |
+
BLOCKS = [np.arange(s, s + 5) for s in (2001, 2006, 2011, 2016, 2021)]
|
| 33 |
+
SUMMER = [6, 7, 8]
|
| 34 |
+
ALPHA, MIN_YEARS, N_PLACEBO = 0.05, 18, 200
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def log(msg):
|
| 38 |
+
print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True)
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def trend_table(wide: pd.DataFrame, key: str) -> pd.DataFrame:
|
| 42 |
+
r = trend_many(wide.values, wide.columns.values.astype(float), method="hamed_rao", min_n=MIN_YEARS)
|
| 43 |
+
df = pd.DataFrame(r)
|
| 44 |
+
df.insert(0, key, wide.index.values)
|
| 45 |
+
df["q"] = bh_fdr(df["p"].values)
|
| 46 |
+
df["class"] = classify(df["slope"].values, df["q"].values, df["n"].values, ALPHA, MIN_YEARS)
|
| 47 |
+
df["slope_per_decade"] = df["slope"] * 10
|
| 48 |
+
return df
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def main() -> None:
|
| 52 |
+
OUT.mkdir(parents=True, exist_ok=True)
|
| 53 |
+
cells = pd.read_parquet(CUBE / "cells.parquet")
|
| 54 |
+
oof = pd.read_parquet(RUN / "expected_oof.parquet")
|
| 55 |
+
conf = {int(k): v for k, v in json.loads((RUN / "conformal.json").read_text()).items()}
|
| 56 |
+
|
| 57 |
+
# recover pred_anom = expected - clim_fold (clim of the other 20 years), then re-reference to clim_all
|
| 58 |
+
ndvi = pd.read_parquet(CUBE / "monthly", columns=["cell_id", "month", "ndvi", "ndvi_ok", "ndvi_aqua", "year"])
|
| 59 |
+
ndvi["year"] = ndvi["year"].astype(int)
|
| 60 |
+
ndvi = ndvi[ndvi["cell_id"].isin(oof["cell_id"].unique()) & ndvi["year"].between(2001, 2025)]
|
| 61 |
+
ok = ndvi[ndvi["ndvi_ok"] & ndvi["month"].between(5, 9)]
|
| 62 |
+
clim_all = ok.groupby(["cell_id", "month"])["ndvi"].mean().rename("clim_all")
|
| 63 |
+
parts = []
|
| 64 |
+
for blk in BLOCKS:
|
| 65 |
+
cf = ok[~ok["year"].isin(blk)].groupby(["cell_id", "month"])["ndvi"].mean().rename("clim_fold").reset_index()
|
| 66 |
+
sub = oof[oof["year"].isin(blk)].merge(cf, on=["cell_id", "month"], how="left")
|
| 67 |
+
parts.append(sub)
|
| 68 |
+
oof = pd.concat(parts, ignore_index=True).merge(clim_all.reset_index(), on=["cell_id", "month"], how="left")
|
| 69 |
+
pred_anom = oof["expected"] - oof["clim_fold"]
|
| 70 |
+
oof["expected"] = oof["clim_all"] + pred_anom
|
| 71 |
+
lo = oof["month"].map(lambda m: conf[int(m)][0])
|
| 72 |
+
hi = oof["month"].map(lambda m: conf[int(m)][1])
|
| 73 |
+
oof["p10"], oof["p90"] = oof["expected"] + lo, oof["expected"] + hi
|
| 74 |
+
oof["residual"] = oof["observed"] - oof["expected"]
|
| 75 |
+
oof = oof[["cell_id", "year", "month", "observed", "expected", "p10", "p90", "residual"]]
|
| 76 |
+
oof.to_parquet(OUT / "expected_oof.parquet", index=False)
|
| 77 |
+
log(f"expected_oof: {len(oof)} rows")
|
| 78 |
+
|
| 79 |
+
# summer residual series
|
| 80 |
+
summer = oof[oof["month"].isin(SUMMER)].groupby(["cell_id", "year"])["residual"].mean().rename("res").reset_index()
|
| 81 |
+
summer = summer.merge(cells[["cell_id", "block_id", "district"]], on="cell_id")
|
| 82 |
+
wide = summer.pivot(index="cell_id", columns="year", values="res").reindex(columns=YEARS)
|
| 83 |
+
|
| 84 |
+
# residual trends: cell, block, district
|
| 85 |
+
rt = trend_table(wide, "cell_id")
|
| 86 |
+
rt.to_parquet(OUT / "residual_trend.parquet", index=False)
|
| 87 |
+
blk_w = summer.groupby(["block_id", "year"])["res"].mean().unstack().reindex(columns=YEARS)
|
| 88 |
+
trend_table(blk_w, "block_id").to_parquet(OUT / "residual_trend_block.parquet", index=False)
|
| 89 |
+
dis_w = summer.groupby(["district", "year"])["res"].mean().unstack().reindex(columns=YEARS)
|
| 90 |
+
rtd = trend_table(dis_w, "district")
|
| 91 |
+
rtd.to_parquet(OUT / "residual_trend_district.parquet", index=False)
|
| 92 |
+
shares = rt["class"].value_counts(normalize=True).round(4).to_dict()
|
| 93 |
+
log(f"residual trend classes (cells): {shares}")
|
| 94 |
+
|
| 95 |
+
# placebo: shuffle years within each cell; how many cells would be flagged by chance?
|
| 96 |
+
rng = np.random.default_rng(0)
|
| 97 |
+
W = wide.values.astype("float32")
|
| 98 |
+
flagged_raw, flagged_q = [], []
|
| 99 |
+
for _ in range(N_PLACEBO):
|
| 100 |
+
perm = np.argsort(rng.random(W.shape), axis=1)
|
| 101 |
+
Wp = np.take_along_axis(W, perm, axis=1)
|
| 102 |
+
_, _, _, p = mk_vectorized(Wp)
|
| 103 |
+
flagged_raw.append(float(np.nanmean(p < ALPHA)))
|
| 104 |
+
flagged_q.append(float(np.nanmean(bh_fdr(p) < ALPHA)))
|
| 105 |
+
_, _, _, p_obs = mk_vectorized(W)
|
| 106 |
+
placebo = {"permutations": N_PLACEBO, "test": "original MK (mk_vectorized), same for observed and placebo",
|
| 107 |
+
"observed_share_p<0.05": float(np.nanmean(p_obs < ALPHA)),
|
| 108 |
+
"observed_share_q<0.05": float(np.nanmean(bh_fdr(p_obs) < ALPHA)),
|
| 109 |
+
"placebo_share_p<0.05_mean": float(np.mean(flagged_raw)),
|
| 110 |
+
"placebo_share_p<0.05_p95": float(np.quantile(flagged_raw, 0.95)),
|
| 111 |
+
"placebo_share_q<0.05_mean": float(np.mean(flagged_q)),
|
| 112 |
+
"placebo_share_q<0.05_p95": float(np.quantile(flagged_q, 0.95))}
|
| 113 |
+
log(f"placebo {placebo}")
|
| 114 |
+
|
| 115 |
+
# season vs expected: summer mean residual vs the empirical P10/P90 of all cell-years
|
| 116 |
+
q10, q90 = summer["res"].quantile([0.1, 0.9])
|
| 117 |
+
summer["season_class"] = np.select([summer["res"] < q10, summer["res"] > q90], ["below", "above"], "within")
|
| 118 |
+
summer.rename(columns={"res": "summer_residual"}).to_parquet(OUT / "season_vs_expected.parquet", index=False)
|
| 119 |
+
sb = summer.groupby(["block_id", "year"])["res"].mean().rename("summer_residual").reset_index()
|
| 120 |
+
bq10, bq90 = sb["summer_residual"].quantile([0.1, 0.9])
|
| 121 |
+
sb["season_class"] = np.select([sb["summer_residual"] < bq10, sb["summer_residual"] > bq90], ["below", "above"], "within")
|
| 122 |
+
sb.to_parquet(OUT / "season_vs_expected_block.parquet", index=False)
|
| 123 |
+
|
| 124 |
+
# Aqua vs Terra: per-block summer NDVI trend 2003-2025, both satellites (raw NDVI, not residuals)
|
| 125 |
+
s = ndvi[ndvi["month"].isin(SUMMER) & ndvi["year"].between(2003, 2025)].merge(cells[["cell_id", "block_id"]], on="cell_id")
|
| 126 |
+
s.loc[~s["ndvi_ok"], "ndvi"] = np.nan
|
| 127 |
+
yrs = np.arange(2003, 2026)
|
| 128 |
+
terra = s.groupby(["block_id", "year"])["ndvi"].mean().unstack().reindex(columns=yrs)
|
| 129 |
+
aqua = s.groupby(["block_id", "year"])["ndvi_aqua"].mean().unstack().reindex(columns=yrs).reindex(terra.index)
|
| 130 |
+
tt, ta = trend_table(terra, "block_id"), trend_table(aqua, "block_id")
|
| 131 |
+
at = tt[["block_id", "slope", "q", "class"]].merge(ta[["block_id", "slope", "q", "class"]], on="block_id",
|
| 132 |
+
suffixes=("_terra", "_aqua"))
|
| 133 |
+
at.to_parquet(OUT / "aqua_terra_blocks.parquet", index=False)
|
| 134 |
+
sig_t = at["class_terra"].isin(["growth", "decline"])
|
| 135 |
+
agreement = {"blocks": int(len(at)),
|
| 136 |
+
"slope_sign_agreement": float(np.mean(np.sign(at["slope_terra"]) == np.sign(at["slope_aqua"]))),
|
| 137 |
+
"terra_significant_blocks": int(sig_t.sum()),
|
| 138 |
+
"same_class_where_terra_significant": float(np.mean(at.loc[sig_t, "class_terra"] == at.loc[sig_t, "class_aqua"])) if sig_t.any() else None,
|
| 139 |
+
"terra_class_shares": at["class_terra"].value_counts(normalize=True).round(4).to_dict(),
|
| 140 |
+
"aqua_class_shares": at["class_aqua"].value_counts(normalize=True).round(4).to_dict()}
|
| 141 |
+
log(f"aqua vs terra {agreement}")
|
| 142 |
+
|
| 143 |
+
summary = {"residual_trend_cell_class_shares": shares,
|
| 144 |
+
"residual_trend_district": rtd[["district", "slope_per_decade", "ci_lo", "ci_hi", "p", "q", "class"]]
|
| 145 |
+
.round(5).to_dict(orient="records"),
|
| 146 |
+
"season_band_summer_residual_q10_q90": [float(q10), float(q90)],
|
| 147 |
+
"placebo": placebo, "aqua_vs_terra_blocks": agreement,
|
| 148 |
+
"wording": "change not explained by precipitation, temperature, snow and soil moisture (not a cause)"}
|
| 149 |
+
(OUT / "products_summary.json").write_text(json.dumps(summary, indent=2, default=float))
|
| 150 |
+
log("PRODUCTS_DONE")
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
if __name__ == "__main__":
|
| 154 |
+
main()
|
training_code/scripts/45_package.py
ADDED
|
@@ -0,0 +1,157 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Assemble the HF model repo folder runs/v1/hf_model: model files, predict.py, golden test, model card.
|
| 2 |
+
|
| 3 |
+
Golden test: 100 fixed feature rows + the outputs of predict.py; the serving side must reproduce them.
|
| 4 |
+
Also checks that predict.py (plain Booster, CPU) gives the same numbers as the training wrapper.
|
| 5 |
+
"""
|
| 6 |
+
from __future__ import annotations
|
| 7 |
+
|
| 8 |
+
import importlib.util
|
| 9 |
+
import json
|
| 10 |
+
import shutil
|
| 11 |
+
import sys
|
| 12 |
+
from pathlib import Path
|
| 13 |
+
|
| 14 |
+
import numpy as np
|
| 15 |
+
import pandas as pd
|
| 16 |
+
import xgboost as xgb
|
| 17 |
+
|
| 18 |
+
ROOT = Path("/root/trendex-ml")
|
| 19 |
+
RUN = ROOT / "runs" / "v1"
|
| 20 |
+
HF = RUN / "hf_model"
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def load_module(path: Path, name: str):
|
| 24 |
+
spec = importlib.util.spec_from_file_location(name, path)
|
| 25 |
+
mod = importlib.util.module_from_spec(spec)
|
| 26 |
+
sys.modules[name] = mod
|
| 27 |
+
spec.loader.exec_module(mod)
|
| 28 |
+
return mod
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def main() -> None:
|
| 32 |
+
HF.mkdir(parents=True, exist_ok=True)
|
| 33 |
+
for f in ["model_a.ubj", "feature_schema.json", "climatology.parquet", "weather_climatology.parquet",
|
| 34 |
+
"conformal.json", "metrics.json"]:
|
| 35 |
+
shutil.copy(RUN / f, HF / f)
|
| 36 |
+
shutil.copy(ROOT / "src" / "trendex_ml" / "predict.py", HF / "predict.py")
|
| 37 |
+
shutil.copy(ROOT / "configs" / "model_v1.yaml", HF / "model_v1.yaml")
|
| 38 |
+
|
| 39 |
+
# golden rows rebuilt with the training code itself
|
| 40 |
+
tr = load_module(ROOT / "scripts" / "30_train.py", "train_mod")
|
| 41 |
+
sel, arr = tr.load()
|
| 42 |
+
f = tr.build_features(sel, arr)
|
| 43 |
+
t = tr.table(f, tr.YEARS, tr.FEATURES).sample(100, random_state=0).reset_index(drop=True)
|
| 44 |
+
t.insert(0, "cell_id", sel["cell_id"].values[t["cell"].values])
|
| 45 |
+
golden_in = t[["cell_id", "year"] + tr.FEATURES]
|
| 46 |
+
(HF / "golden").mkdir(exist_ok=True)
|
| 47 |
+
golden_in.to_parquet(HF / "golden" / "inputs.parquet", index=False)
|
| 48 |
+
|
| 49 |
+
pm = load_module(HF / "predict.py", "predict_mod")
|
| 50 |
+
model = pm.load(HF)
|
| 51 |
+
out = model.predict(golden_in)
|
| 52 |
+
out.to_parquet(HF / "golden" / "expected.parquet", index=False)
|
| 53 |
+
|
| 54 |
+
ref = xgb.XGBRegressor()
|
| 55 |
+
ref.load_model(RUN / "model_a.ubj")
|
| 56 |
+
max_diff = float(np.max(np.abs(ref.predict(golden_in[tr.FEATURES]) - out["pred_anom"].values)))
|
| 57 |
+
print(f"golden: 100 rows, max |predict.py - training wrapper| = {max_diff:.2e}")
|
| 58 |
+
assert max_diff < 1e-5, "predict.py disagrees with the training model"
|
| 59 |
+
|
| 60 |
+
(HF / "test_golden.py").write_text('''"""pytest test_golden.py - the serving code must reproduce these outputs exactly."""
|
| 61 |
+
from pathlib import Path
|
| 62 |
+
|
| 63 |
+
import numpy as np
|
| 64 |
+
import pandas as pd
|
| 65 |
+
|
| 66 |
+
from predict import load
|
| 67 |
+
|
| 68 |
+
HERE = Path(__file__).parent
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def test_golden():
|
| 72 |
+
m = load(HERE)
|
| 73 |
+
got = m.predict(pd.read_parquet(HERE / "golden" / "inputs.parquet"))
|
| 74 |
+
want = pd.read_parquet(HERE / "golden" / "expected.parquet")
|
| 75 |
+
assert np.allclose(got.values, want.values, atol=1e-6)
|
| 76 |
+
''')
|
| 77 |
+
write_card()
|
| 78 |
+
print("PACKAGE_DONE", sorted(p.name for p in HF.iterdir()))
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
def write_card() -> None:
|
| 82 |
+
m = json.loads((RUN / "metrics.json").read_text())
|
| 83 |
+
t, s = m["temporal_test"], m["spatial_test"]
|
| 84 |
+
fmt = lambda d: f"{d['point']:.3f} (95% CI {d['ci_lo']:.3f} to {d['ci_hi']:.3f}, {d['units']} units)" # noqa: E731
|
| 85 |
+
row = lambda name, d: f"| {name} | {d['xgb']['rmse']:.4f} | {d['ridge']['rmse']:.4f} | {d['clim']['rmse']:.4f} | " \
|
| 86 |
+
f"{d['xgb']['anomaly_corr']:.3f} | {d['ridge']['anomaly_corr']:.3f} | {d['xgb']['tercile_bacc']:.3f} |" # noqa: E731
|
| 87 |
+
abl = " -> ".join(f"{k}: {v:.3f}" for k, v in m["ablation_skill_vs_clim_temporal"].items())
|
| 88 |
+
imp = ", ".join(list(m["feature_importance_mean_abs"].keys())[:6])
|
| 89 |
+
card = f"""---
|
| 90 |
+
license: other
|
| 91 |
+
library_name: xgboost
|
| 92 |
+
tags: [nasa, modis, ndvi, imerg, gldas, pastures, kyrgyzstan, tabular-regression]
|
| 93 |
+
datasets: [TrendeX90/naryn-cube]
|
| 94 |
+
---
|
| 95 |
+
# TrendeX expected-greenness model (v1)
|
| 96 |
+
|
| 97 |
+
Predicts the **summer greenness (NDVI) anomaly** of every 1 km pasture cell in Naryn oblast, Kyrgyzstan
|
| 98 |
+
(May-September, 2001-2025) **from weather and terrain alone**. The gap between observed and expected NDVI is
|
| 99 |
+
change **not explained by precipitation, temperature, snow and soil moisture**. It is not a cause and says
|
| 100 |
+
nothing about forage quality. Built for NASA Space Apps 2026 ("Be An Earth System Trend Detective!").
|
| 101 |
+
|
| 102 |
+
## Inputs
|
| 103 |
+
- Weather anomalies vs each cell's 2001-2025 mean for the same month: GPM IMERG precipitation (month, 3-month,
|
| 104 |
+
Oct-Apr), MODIS LST day/night (month, previous month), MODIS snow (Mar-May, month), GLDAS soil moisture
|
| 105 |
+
(month) and April snow water.
|
| 106 |
+
- Terrain from NASADEM (elevation, spread, slope, steep share, aspect, TPI) and land cover (MCD12Q1 2020).
|
| 107 |
+
- The cell's own NDVI climatology for that month. **Not used on purpose:** year, coordinates, past NDVI.
|
| 108 |
+
|
| 109 |
+
## How it was tested (protocol locked in git before the first fit: `model_v1.yaml`, commit {m['git_commit']})
|
| 110 |
+
- Tuning only on 2001-2015 -> 2016-2019. The 2020-2025 years were used once, for the final test.
|
| 111 |
+
- Spatial test: 5-fold GroupKFold over ~30 x 30 km super-blocks (all years; easier than the temporal test
|
| 112 |
+
because the same years' weather appears in training).
|
| 113 |
+
- Baselines: climatology (anomaly = 0) and ridge regression on the same features.
|
| 114 |
+
- Pre-registered gate: ship XGBoost only if the 95% CI lower bound of its skill over ridge is > 0 on both tests.
|
| 115 |
+
**Gate passed: {m['gate']['passed']}. Shipped: {m['gate']['shipped_model']}.**
|
| 116 |
+
|
| 117 |
+
| Test | RMSE xgb | RMSE ridge | RMSE climatology | anomaly corr xgb | anomaly corr ridge | tercile bal. acc. xgb |
|
| 118 |
+
|---|---|---|---|---|---|---|
|
| 119 |
+
{row("Temporal (2020-2025)", t)}
|
| 120 |
+
{row("Spatial (unseen areas)", s)}
|
| 121 |
+
|
| 122 |
+
- Skill vs ridge, temporal: {fmt(t['skill_xgb_vs_ridge'])}. Five seeds: {t['seed_skill_vs_ridge']['mean']:.3f} +/- {t['seed_skill_vs_ridge']['sd']:.3f}.
|
| 123 |
+
- Skill vs ridge, spatial: {fmt(s['skill_xgb_vs_ridge'])}.
|
| 124 |
+
- Skill vs climatology, temporal: {fmt(t['skill_xgb_vs_clim'])}.
|
| 125 |
+
- What each NASA dataset adds (skill vs climatology, temporal test): {abl}.
|
| 126 |
+
- Most used inputs: {imp}.
|
| 127 |
+
|
| 128 |
+
## Limits (read before using)
|
| 129 |
+
- The model explains about a fifth of the year-to-year NDVI anomaly on unseen years; most variation is not weather-explained.
|
| 130 |
+
- P10-P90 band: on 2020-2025 it covered **{t['conformal_coverage_p10_p90']:.0%} instead of 80%**, so treat it as too narrow.
|
| 131 |
+
- Weather comes from coarse grids (IMERG 0.1 deg, GLDAS 0.25 deg) interpolated to 1 km.
|
| 132 |
+
- NDVI is greenness, not forage: shrub encroachment can look like improvement.
|
| 133 |
+
- Land cover is a fixed 2020 map; MODIS Terra drifted in recent years (see Aqua-Terra check in the dataset products).
|
| 134 |
+
- What-if runs: drier and hotter-than-normal conditions lower expected July NDVI, but wetter-than-normal
|
| 135 |
+
precipitation does not raise it (slightly lowers it on average). Treat what-if outputs as model
|
| 136 |
+
sensitivities, not causal effects.
|
| 137 |
+
- No field validation. Relative to Naryn pastures only.
|
| 138 |
+
|
| 139 |
+
## Use
|
| 140 |
+
```python
|
| 141 |
+
from predict import load
|
| 142 |
+
m = load(".") # needs only numpy, pandas, xgboost (CPU)
|
| 143 |
+
out = m.predict(features_df) # columns as in feature_schema.json
|
| 144 |
+
scen = m.what_if(cells, cell_ids, month=7, precip_pct=-30) # cells = naryn-cube cells.parquet
|
| 145 |
+
```
|
| 146 |
+
`pytest test_golden.py` reproduces the 100 golden rows.
|
| 147 |
+
|
| 148 |
+
## Data and tools
|
| 149 |
+
MOD13A3.061, MYD13A3.061, MOD11A2.061, MOD10A2.061, MCD12Q1.061, NASADEM_HGT.001 (LP DAAC / NSIDC via earthaccess),
|
| 150 |
+
GPM_3IMERGM.07 and GLDAS_NOAH025_M.2.1 (GES DISC). XGBoost, Optuna, scikit-learn, pymannkendall.
|
| 151 |
+
AI use: implementation was assisted by Claude (Anthropic); design decisions and the product idea are the team's.
|
| 152 |
+
"""
|
| 153 |
+
(HF / "README.md").write_text(card)
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
if __name__ == "__main__":
|
| 157 |
+
main()
|
training_code/scripts/50_push_hf.py
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Push the cube (and later products/model) to Hugging Face.
|
| 2 |
+
|
| 3 |
+
python scripts/50_push_hf.py cube # data/cube + data/aoi -> TrendeX90/naryn-cube (private dataset)
|
| 4 |
+
|
| 5 |
+
Uses the token in ~/.cache/huggingface/token (a fine-grained token with write access to TrendeX90 only).
|
| 6 |
+
"""
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
import argparse
|
| 10 |
+
import json
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
|
| 13 |
+
from huggingface_hub import HfApi
|
| 14 |
+
|
| 15 |
+
ROOT = Path("/root/trendex-ml/data")
|
| 16 |
+
CUBE_REPO = "TrendeX90/naryn-cube"
|
| 17 |
+
MODEL_REPO = "TrendeX90/expected-greenness"
|
| 18 |
+
RUN = Path("/root/trendex-ml/runs/v1")
|
| 19 |
+
|
| 20 |
+
CARD = """---
|
| 21 |
+
license: other
|
| 22 |
+
pretty_name: TrendeX Naryn pasture cube
|
| 23 |
+
tags: [nasa, modis, ndvi, imerg, gldas, kyrgyzstan, pastures]
|
| 24 |
+
---
|
| 25 |
+
# TrendeX Naryn cube ({status})
|
| 26 |
+
|
| 27 |
+
Shared data cube for TrendeX (NASA Space Apps 2026, "Be An Earth System Trend Detective!"):
|
| 28 |
+
Naryn oblast, Kyrgyzstan, plus a 20 km buffer and two known-place boxes (Suusamyr valley, Ak-Shyirak).
|
| 29 |
+
**synthetic = false** - every value comes from NASA data listed below.
|
| 30 |
+
|
| 31 |
+
## Grid
|
| 32 |
+
Global MODIS sinusoidal 1 km grid (`+proj=sinu +R=6371007.181`), pixel 926.625 m, equal-area.
|
| 33 |
+
`cell_id = global_row * 43200 + global_col`. Blocks: 11 x 11 cells (`block_id` = "row-col", ints in `block_row`, `block_col`).
|
| 34 |
+
|
| 35 |
+
## Files
|
| 36 |
+
- `cells.parquet` - one row per cell: position, district (5 official districts; null outside), terrain (NASADEM),
|
| 37 |
+
land cover (MCD12Q1 2020), `mask_reason` (water / ice / urban / too_steep; null = pasture candidate).
|
| 38 |
+
- `monthly/year=YYYY/part.parquet` - one row per cell x month, 2000-02..2025-12 (`pd.read_parquet("monthly/")`).
|
| 39 |
+
- `aoi/` - oblast, district and request polygons (GeoJSON, EPSG:4326).
|
| 40 |
+
- `manifest.json` - units, dataset versions and DOIs, sources, notes. `qa_report.json` - sanity checks.
|
| 41 |
+
|
| 42 |
+
## Units
|
| 43 |
+
{units}
|
| 44 |
+
|
| 45 |
+
## Data sources
|
| 46 |
+
{datasets}
|
| 47 |
+
|
| 48 |
+
Downloaded with earthaccess from LP DAAC (MODIS, NASADEM) and GES DISC (IMERG, GLDAS).
|
| 49 |
+
|
| 50 |
+
## Notes
|
| 51 |
+
{notes}
|
| 52 |
+
"""
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def push_cube(api: HfApi, message: str) -> None:
|
| 56 |
+
manifest = json.loads((ROOT / "cube" / "manifest.json").read_text())
|
| 57 |
+
status = "PARTIAL - NDVI only" if manifest.get("partial") else "full"
|
| 58 |
+
card = CARD.format(
|
| 59 |
+
status=status,
|
| 60 |
+
units="\n".join(f"- `{k}`: {v}" for k, v in manifest.get("units", {}).items()),
|
| 61 |
+
datasets="\n".join(f"- {k} v{v}" for k, v in manifest.get("datasets", {}).items()),
|
| 62 |
+
notes="\n".join(f"- {n}" for n in manifest.get("notes", [])))
|
| 63 |
+
(ROOT / "cube" / "README.md").write_text(card)
|
| 64 |
+
if not api.repo_exists(CUBE_REPO, repo_type="dataset"): # tokens without org create rights still work
|
| 65 |
+
api.create_repo(CUBE_REPO, repo_type="dataset", private=True)
|
| 66 |
+
api.upload_folder(repo_id=CUBE_REPO, repo_type="dataset", folder_path=str(ROOT / "cube"),
|
| 67 |
+
commit_message=message or f"cube ({status})")
|
| 68 |
+
api.upload_folder(repo_id=CUBE_REPO, repo_type="dataset", folder_path=str(ROOT / "aoi"), path_in_repo="aoi",
|
| 69 |
+
allow_patterns=["naryn_oblast.geojson", "naryn_districts.geojson", "aoi_request.geojson",
|
| 70 |
+
"aoi_summary.json"], commit_message="area of interest polygons")
|
| 71 |
+
print(f"PUSHED {CUBE_REPO} ({status})", flush=True)
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def push_products(api: HfApi, message: str) -> None:
|
| 75 |
+
"""runs/v1/products + metrics.json -> naryn-cube/products/ (what the backend reads)."""
|
| 76 |
+
api.upload_folder(repo_id=CUBE_REPO, repo_type="dataset", folder_path=str(RUN / "products"),
|
| 77 |
+
path_in_repo="products", commit_message=message or "model products v1")
|
| 78 |
+
api.upload_file(path_or_fileobj=str(RUN / "metrics.json"), path_in_repo="products/metrics.json",
|
| 79 |
+
repo_id=CUBE_REPO, repo_type="dataset", commit_message="model metrics v1")
|
| 80 |
+
print(f"PUSHED {CUBE_REPO}/products", flush=True)
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
def push_model(api: HfApi, message: str) -> None:
|
| 84 |
+
if not api.repo_exists(MODEL_REPO):
|
| 85 |
+
api.create_repo(MODEL_REPO, private=True)
|
| 86 |
+
api.upload_folder(repo_id=MODEL_REPO, folder_path=str(RUN / "hf_model"), ignore_patterns=["__pycache__/*", "*.pyc", ".pytest_cache/*"],
|
| 87 |
+
commit_message=message or "expected-greenness v1")
|
| 88 |
+
refs = [t.name for t in api.list_repo_refs(MODEL_REPO).tags]
|
| 89 |
+
if "v1" not in refs:
|
| 90 |
+
api.create_tag(MODEL_REPO, tag="v1", tag_message="protocol model_v1, gate passed")
|
| 91 |
+
print(f"PUSHED {MODEL_REPO} (tag v1)", flush=True)
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def main() -> None:
|
| 95 |
+
ap = argparse.ArgumentParser()
|
| 96 |
+
ap.add_argument("what", choices=["cube", "products", "model", "all"])
|
| 97 |
+
ap.add_argument("-m", "--message", default="")
|
| 98 |
+
a = ap.parse_args()
|
| 99 |
+
api = HfApi()
|
| 100 |
+
print("as:", api.whoami().get("name"), flush=True)
|
| 101 |
+
if a.what in ("cube", "all"):
|
| 102 |
+
push_cube(api, a.message)
|
| 103 |
+
if a.what in ("products", "all"):
|
| 104 |
+
push_products(api, a.message)
|
| 105 |
+
if a.what in ("model", "all"):
|
| 106 |
+
push_model(api, a.message)
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
if __name__ == "__main__":
|
| 110 |
+
main()
|
training_code/scripts/55_scenarios.py
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Precompute what-if scenarios for the static site (no model server needed).
|
| 2 |
+
|
| 3 |
+
uv run --python 3.12 --with xgboost --with pandas --with pyarrow --with huggingface_hub \
|
| 4 |
+
python scripts/55_scenarios.py --out data/scenarios
|
| 5 |
+
|
| 6 |
+
Uses the published model (TrendeX90/expected-greenness@v1, its own predict.py) and cells.parquet from
|
| 7 |
+
TrendeX90/naryn-cube. For every pasture cell, month (May-Sep) and scenario, it computes expected NDVI and the
|
| 8 |
+
change vs typical weather, then averages over blocks, districts and the oblast.
|
| 9 |
+
|
| 10 |
+
Scenario grid: precipitation -30..+30 % (all precipitation inputs), temperature -2..+2 deg C (day and night
|
| 11 |
+
land-surface temperature together). These are model sensitivities, not causal effects.
|
| 12 |
+
"""
|
| 13 |
+
from __future__ import annotations
|
| 14 |
+
|
| 15 |
+
import argparse
|
| 16 |
+
import importlib.util
|
| 17 |
+
import json
|
| 18 |
+
import sys
|
| 19 |
+
import time
|
| 20 |
+
from pathlib import Path
|
| 21 |
+
|
| 22 |
+
import numpy as np
|
| 23 |
+
import pandas as pd
|
| 24 |
+
from huggingface_hub import hf_hub_download, snapshot_download
|
| 25 |
+
|
| 26 |
+
MODEL, CUBE = "TrendeX90/expected-greenness", "TrendeX90/naryn-cube"
|
| 27 |
+
MONTHS = [5, 6, 7, 8, 9]
|
| 28 |
+
PRECIP = [-30, -20, -10, 0, 10, 20, 30]
|
| 29 |
+
TEMP = [-2, -1, 0, 1, 2]
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def main() -> None:
|
| 33 |
+
ap = argparse.ArgumentParser()
|
| 34 |
+
ap.add_argument("--out", type=Path, default=Path("data/scenarios"))
|
| 35 |
+
a = ap.parse_args()
|
| 36 |
+
a.out.mkdir(parents=True, exist_ok=True)
|
| 37 |
+
t0 = time.time()
|
| 38 |
+
|
| 39 |
+
root = Path(snapshot_download(MODEL, revision="v1", local_dir=a.out.parent / "model_v1"))
|
| 40 |
+
spec = importlib.util.spec_from_file_location("predict", root / "predict.py")
|
| 41 |
+
mod = importlib.util.module_from_spec(spec)
|
| 42 |
+
sys.modules["predict"] = mod
|
| 43 |
+
spec.loader.exec_module(mod)
|
| 44 |
+
model = mod.load(root)
|
| 45 |
+
cells = pd.read_parquet(hf_hub_download(CUBE, "cells.parquet", repo_type="dataset"))
|
| 46 |
+
past = cells[cells["in_oblast"] & cells["mask_reason"].isna()]
|
| 47 |
+
ids = past["cell_id"].values
|
| 48 |
+
print(f"{len(ids)} pasture cells, model v1 loaded ({time.time() - t0:.0f}s)", flush=True)
|
| 49 |
+
|
| 50 |
+
rows = []
|
| 51 |
+
for mo in MONTHS:
|
| 52 |
+
for p in PRECIP:
|
| 53 |
+
for t in TEMP:
|
| 54 |
+
r = model.what_if(cells, ids, month=mo, precip_pct=p, lst_day_c=t, lst_night_c=t)
|
| 55 |
+
rows.append(pd.DataFrame({"cell_id": ids, "month": mo, "precip_pct": p, "temp_c": t,
|
| 56 |
+
"typical": r["expected_typical_weather"].values,
|
| 57 |
+
"change": r["change_vs_typical"].values}))
|
| 58 |
+
cellwise = pd.concat(rows, ignore_index=True).merge(past[["cell_id", "block_id", "district"]], on="cell_id")
|
| 59 |
+
cellwise.to_parquet(a.out / "what_if_cells.parquet", index=False)
|
| 60 |
+
print(f"{len(cellwise)} cell-scenario rows ({time.time() - t0:.0f}s)", flush=True)
|
| 61 |
+
|
| 62 |
+
def nested(df: pd.DataFrame) -> dict:
|
| 63 |
+
g = df.groupby(["month", "precip_pct", "temp_c"])["change"].mean()
|
| 64 |
+
change = [[[round(float(g[(mo, p, t)]), 4) for t in TEMP] for p in PRECIP] for mo in MONTHS]
|
| 65 |
+
typ = df[(df["precip_pct"] == 0) & (df["temp_c"] == 0)].groupby("month")["typical"].mean()
|
| 66 |
+
return {"n_cells": int(df["cell_id"].nunique()), "typical": [round(float(typ[mo]), 4) for mo in MONTHS],
|
| 67 |
+
"change": change}
|
| 68 |
+
|
| 69 |
+
out = {"model": f"{MODEL}@v1", "source_cells": f"{CUBE}/cells.parquet (in_oblast and mask_reason is null)",
|
| 70 |
+
"units": "NDVI", "months": MONTHS, "precip_pct": PRECIP, "temp_c": TEMP,
|
| 71 |
+
"layout": "change[month_index][precip_index][temp_index]; expected = typical[month_index] + change",
|
| 72 |
+
"meaning": "Expected greenness change vs typical weather for that month if precipitation and land-surface "
|
| 73 |
+
"temperature differ by the given amounts. Model sensitivity, not a causal effect; "
|
| 74 |
+
"not forage quality. Wetter-than-normal does not raise NDVI in this model (see model card).",
|
| 75 |
+
"oblast": nested(cellwise),
|
| 76 |
+
"districts": {d: nested(g) for d, g in cellwise.groupby("district")},
|
| 77 |
+
"blocks": {b: nested(g) for b, g in cellwise.groupby("block_id")}}
|
| 78 |
+
path = a.out / "what_if.json"
|
| 79 |
+
path.write_text(json.dumps(out, separators=(",", ":")))
|
| 80 |
+
ob = out["oblast"]
|
| 81 |
+
jul = MONTHS.index(7)
|
| 82 |
+
print(f"oblast July typical {ob['typical'][jul]}, -30% precip {ob['change'][jul][0][TEMP.index(0)]}, "
|
| 83 |
+
f"+2C {ob['change'][jul][PRECIP.index(0)][-1]}")
|
| 84 |
+
print(f"what_if.json {path.stat().st_size / 1e6:.2f} MB, {len(out['blocks'])} blocks, "
|
| 85 |
+
f"{len(out['districts'])} districts ({time.time() - t0:.0f}s)")
|
| 86 |
+
print("SCENARIOS_DONE")
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
if __name__ == "__main__":
|
| 90 |
+
main()
|
training_code/scripts/60_verify_hf.py
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Verify what is on Hugging Face from a clean machine (no access to the Vast box).
|
| 2 |
+
|
| 3 |
+
uv run --with xgboost --with pandas --with pyarrow --with huggingface_hub python scripts/60_verify_hf.py
|
| 4 |
+
|
| 5 |
+
1. Downloads TrendeX90/expected-greenness at tag v1, runs the golden test with its own predict.py.
|
| 6 |
+
2. Reads naryn-cube: manifest, cells.parquet, one year of monthly data, products summary.
|
| 7 |
+
"""
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import importlib.util
|
| 11 |
+
import json
|
| 12 |
+
import sys
|
| 13 |
+
import tempfile
|
| 14 |
+
from pathlib import Path
|
| 15 |
+
|
| 16 |
+
import numpy as np
|
| 17 |
+
import pandas as pd
|
| 18 |
+
from huggingface_hub import HfApi, hf_hub_download, snapshot_download
|
| 19 |
+
|
| 20 |
+
MODEL, CUBE = "TrendeX90/expected-greenness", "TrendeX90/naryn-cube"
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def main() -> None:
|
| 24 |
+
api = HfApi()
|
| 25 |
+
tags = [t.name for t in api.list_repo_refs(MODEL).tags]
|
| 26 |
+
print("model tags:", tags)
|
| 27 |
+
with tempfile.TemporaryDirectory() as tmp:
|
| 28 |
+
root = Path(snapshot_download(MODEL, revision="v1", local_dir=Path(tmp) / "model"))
|
| 29 |
+
print("model files:", sorted(p.name for p in root.iterdir() if not p.name.startswith(".")))
|
| 30 |
+
spec = importlib.util.spec_from_file_location("predict", root / "predict.py")
|
| 31 |
+
mod = importlib.util.module_from_spec(spec)
|
| 32 |
+
sys.modules["predict"] = mod
|
| 33 |
+
spec.loader.exec_module(mod)
|
| 34 |
+
m = mod.load(root)
|
| 35 |
+
got = m.predict(pd.read_parquet(root / "golden" / "inputs.parquet"))
|
| 36 |
+
want = pd.read_parquet(root / "golden" / "expected.parquet")
|
| 37 |
+
diff = float(np.max(np.abs(got.values - want.values)))
|
| 38 |
+
print(f"golden test on this machine: max diff {diff:.2e} -> {'PASS' if diff < 1e-6 else 'FAIL'}")
|
| 39 |
+
metrics = json.loads((root / "metrics.json").read_text())
|
| 40 |
+
print("gate:", metrics["gate"])
|
| 41 |
+
|
| 42 |
+
files = api.list_repo_files(CUBE, repo_type="dataset")
|
| 43 |
+
print(f"cube files: {len(files)} | monthly years: {sum(f.startswith('monthly/') for f in files)} | "
|
| 44 |
+
f"products: {sorted(f for f in files if f.startswith('products/'))}")
|
| 45 |
+
man = json.loads(Path(hf_hub_download(CUBE, "manifest.json", repo_type="dataset")).read_text())
|
| 46 |
+
print("manifest: synthetic =", man["synthetic"], "| partial =", man["partial"], "| sources =", man["sources_used"])
|
| 47 |
+
cells = pd.read_parquet(hf_hub_download(CUBE, "cells.parquet", repo_type="dataset"))
|
| 48 |
+
print("cells:", len(cells), "| in oblast:", int(cells["in_oblast"].sum()))
|
| 49 |
+
y = pd.read_parquet(hf_hub_download(CUBE, "monthly/year=2024/part.parquet", repo_type="dataset"))
|
| 50 |
+
print("monthly 2024 rows:", len(y), "| columns:", list(y.columns))
|
| 51 |
+
summ = json.loads(Path(hf_hub_download(CUBE, "products/products_summary.json", repo_type="dataset")).read_text())
|
| 52 |
+
print("residual trend shares:", summ["residual_trend_cell_class_shares"])
|
| 53 |
+
print("VERIFY_DONE")
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
if __name__ == "__main__":
|
| 57 |
+
main()
|
training_code/scripts/setup_box.sh
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
# One-time environment setup on the Vast box (CUDA 12 PyTorch image, /opt/conda Python 3.11).
|
| 3 |
+
set -euo pipefail
|
| 4 |
+
echo "SETUP_START $(date -u +%FT%TZ)"
|
| 5 |
+
python -m pip install -q --upgrade pip
|
| 6 |
+
python -m pip install -q \
|
| 7 |
+
rasterio rioxarray xarray h5netcdf netCDF4 h5py \
|
| 8 |
+
geopandas shapely pyproj pyarrow polars pandas requests tqdm \
|
| 9 |
+
earthaccess xgboost lightgbm scikit-learn optuna shap \
|
| 10 |
+
pymannkendall statsmodels huggingface_hub pytest
|
| 11 |
+
python -m pip freeze > /root/requirements.lock
|
| 12 |
+
python - <<'PY'
|
| 13 |
+
import numpy as np, rasterio, xarray, geopandas, earthaccess, xgboost, lightgbm, sklearn
|
| 14 |
+
import optuna, shap, pymannkendall, statsmodels, huggingface_hub, h5py, netCDF4, polars
|
| 15 |
+
print("rasterio", rasterio.__version__, "| GDAL", rasterio.__gdal_version__)
|
| 16 |
+
print("xgboost", xgboost.__version__, "| lightgbm", lightgbm.__version__, "| earthaccess", earthaccess.__version__)
|
| 17 |
+
from xgboost import XGBRegressor
|
| 18 |
+
X = np.random.rand(2000, 5); y = X.sum(1)
|
| 19 |
+
XGBRegressor(n_estimators=20, tree_method="hist", device="cuda").fit(X, y)
|
| 20 |
+
print("XGBOOST_CUDA_OK")
|
| 21 |
+
PY
|
| 22 |
+
echo "SETUP_DONE $(date -u +%FT%TZ)"
|
training_code/src/trendex_ml/__init__.py
ADDED
|
File without changes
|
training_code/src/trendex_ml/predict.py
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""TrendeX expected-greenness model: CPU inference. Imports only numpy, pandas and xgboost.
|
| 2 |
+
|
| 3 |
+
Files expected next to this module (the HF model repo TrendeX90/expected-greenness):
|
| 4 |
+
model_a.ubj XGBoost model (predicts the NDVI anomaly)
|
| 5 |
+
feature_schema.json feature order
|
| 6 |
+
climatology.parquet cell_id, month, clim_ndvi (2001-2025 mean NDVI of that cell and month)
|
| 7 |
+
weather_climatology.parquet cell_id, month, <feature>_mean (raw weather value minus this = model anomaly)
|
| 8 |
+
conformal.json month -> [q10, q90] of residuals (P10/P90 band)
|
| 9 |
+
|
| 10 |
+
from predict import load
|
| 11 |
+
m = load("path/to/repo")
|
| 12 |
+
out = m.predict(features_df) # features in feature_schema order
|
| 13 |
+
out = m.what_if(cells_df, cell_ids, month=7, precip_pct=-30, lst_day_c=+1.5)
|
| 14 |
+
|
| 15 |
+
Output: pred_anom, expected (= clim_ndvi + pred_anom), p10, p90. NDVI units.
|
| 16 |
+
Meaning: greenness expected from weather and terrain alone. The gap to observed NDVI is change NOT
|
| 17 |
+
explained by precipitation, temperature, snow and soil moisture. It is not a cause and not forage quality.
|
| 18 |
+
"""
|
| 19 |
+
from __future__ import annotations
|
| 20 |
+
|
| 21 |
+
import json
|
| 22 |
+
from pathlib import Path
|
| 23 |
+
|
| 24 |
+
import numpy as np
|
| 25 |
+
import pandas as pd
|
| 26 |
+
import xgboost as xgb
|
| 27 |
+
|
| 28 |
+
STATIC = ["elev_mean", "elev_std", "slope_mean", "steep_share", "aspect_sin", "aspect_cos", "tpi",
|
| 29 |
+
"grass_frac", "barren_frac", "lc_type1"]
|
| 30 |
+
WEATHER = ["precip_m", "precip_3m", "precip_winter", "lst_day_m", "lst_night_m", "lst_day_prev",
|
| 31 |
+
"snow_spring", "snow_m", "soil_m", "swe_apr"]
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
class ExpectedGreenness:
|
| 35 |
+
def __init__(self, root: str | Path):
|
| 36 |
+
root = Path(root)
|
| 37 |
+
self.schema = json.loads((root / "feature_schema.json").read_text())
|
| 38 |
+
self.features = self.schema["features"]
|
| 39 |
+
self.booster = xgb.Booster()
|
| 40 |
+
self.booster.load_model(str(root / "model_a.ubj"))
|
| 41 |
+
self.booster.set_param({"device": "cpu"})
|
| 42 |
+
self.clim = pd.read_parquet(root / "climatology.parquet").set_index(["cell_id", "month"])
|
| 43 |
+
self.wclim = pd.read_parquet(root / "weather_climatology.parquet").set_index(["cell_id", "month"])
|
| 44 |
+
self.conf = {int(k): v for k, v in json.loads((root / "conformal.json").read_text()).items()}
|
| 45 |
+
|
| 46 |
+
# ------------------------------------------------------------------ core
|
| 47 |
+
def predict(self, df: pd.DataFrame) -> pd.DataFrame:
|
| 48 |
+
"""df must contain every feature in feature_schema.json (NaN allowed for weather)."""
|
| 49 |
+
missing = [c for c in self.features if c not in df.columns]
|
| 50 |
+
if missing:
|
| 51 |
+
raise ValueError(f"missing features: {missing}")
|
| 52 |
+
X = df[self.features].astype("float32")
|
| 53 |
+
pred = self.booster.predict(xgb.DMatrix(X, feature_names=self.features))
|
| 54 |
+
months = df["month"].astype(int).values
|
| 55 |
+
lo = np.array([self.conf[m][0] for m in months])
|
| 56 |
+
hi = np.array([self.conf[m][1] for m in months])
|
| 57 |
+
exp = df["clim_ndvi"].values + pred
|
| 58 |
+
return pd.DataFrame({"pred_anom": pred, "expected": exp, "p10": exp + lo, "p90": exp + hi}, index=df.index)
|
| 59 |
+
|
| 60 |
+
# ------------------------------------------------------------------ helpers
|
| 61 |
+
def features_from_raw(self, cells: pd.DataFrame, raw: pd.DataFrame) -> pd.DataFrame:
|
| 62 |
+
"""raw: cell_id, month and raw weather values named like WEATHER (mm, deg C, fractions, kg m-2).
|
| 63 |
+
|
| 64 |
+
precip_3m = this + 2 previous months; precip_winter = Oct..Apr sum; snow_spring = Mar-May mean;
|
| 65 |
+
swe_apr = April SWE. cells: rows of the cube's cells.parquet (for static features).
|
| 66 |
+
"""
|
| 67 |
+
key = pd.MultiIndex.from_arrays([raw["cell_id"].values, raw["month"].astype(int).values])
|
| 68 |
+
out = raw[["cell_id", "month"]].copy()
|
| 69 |
+
for w in WEATHER:
|
| 70 |
+
out[w] = raw[w].values - self.wclim[f"{w}_mean"].reindex(key).values if w in raw else np.nan
|
| 71 |
+
out["clim_ndvi"] = self.clim["clim_ndvi"].reindex(key).values
|
| 72 |
+
out = out.merge(cells[["cell_id"] + STATIC], on="cell_id", how="left")
|
| 73 |
+
return out
|
| 74 |
+
|
| 75 |
+
def what_if(self, cells: pd.DataFrame, cell_ids, month: int, precip_pct: float = 0.0,
|
| 76 |
+
lst_day_c: float = 0.0, lst_night_c: float = 0.0, snow_spring_delta: float = 0.0,
|
| 77 |
+
soil_pct: float = 0.0) -> pd.DataFrame:
|
| 78 |
+
"""Scenario vs typical weather: e.g. precip_pct=-30 means every precipitation input 30% below normal."""
|
| 79 |
+
ids = np.asarray(cell_ids)
|
| 80 |
+
key = pd.MultiIndex.from_arrays([ids, np.full(len(ids), int(month))])
|
| 81 |
+
w = self.wclim.reindex(key)
|
| 82 |
+
|
| 83 |
+
def frame(pp, ld, ln, ss, sp):
|
| 84 |
+
f = pd.DataFrame({"cell_id": ids, "month": float(month)})
|
| 85 |
+
for k in ("precip_m", "precip_3m", "precip_winter"):
|
| 86 |
+
f[k] = w[f"{k}_mean"].values * pp / 100.0
|
| 87 |
+
f["lst_day_m"] = f["lst_day_prev"] = ld
|
| 88 |
+
f["lst_night_m"] = ln
|
| 89 |
+
f["snow_spring"] = ss
|
| 90 |
+
f["snow_m"] = 0.0
|
| 91 |
+
f["soil_m"] = w["soil_m_mean"].values * sp / 100.0
|
| 92 |
+
f["swe_apr"] = 0.0
|
| 93 |
+
f["clim_ndvi"] = self.clim["clim_ndvi"].reindex(key).values
|
| 94 |
+
return f.merge(cells[["cell_id"] + STATIC], on="cell_id", how="left")
|
| 95 |
+
|
| 96 |
+
res = self.predict(frame(precip_pct, lst_day_c, lst_night_c, snow_spring_delta, soil_pct))
|
| 97 |
+
base = self.predict(frame(0.0, 0.0, 0.0, 0.0, 0.0))
|
| 98 |
+
res.insert(0, "cell_id", ids)
|
| 99 |
+
res["expected_typical_weather"] = base["expected"].values
|
| 100 |
+
res["change_vs_typical"] = res["expected"].values - base["expected"].values # the scenario effect
|
| 101 |
+
return res
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def load(root: str | Path = Path(__file__).parent) -> ExpectedGreenness:
|
| 105 |
+
return ExpectedGreenness(root)
|
training_code/src/trendex_ml/stats/__init__.py
ADDED
|
File without changes
|
training_code/src/trendex_ml/stats/trend.py
ADDED
|
@@ -0,0 +1,159 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Trend statistics, shared by the formula (F3) and the ML session's residual trends.
|
| 2 |
+
|
| 3 |
+
One implementation, copied (not forked) into the ML repo.
|
| 4 |
+
|
| 5 |
+
- trend_1d: Sen slope + Mann-Kendall (Hamed-Rao autocorrelation correction by default) + 95% CI (Gilbert 1987).
|
| 6 |
+
- trend_many: trend_1d over rows of a 2-D array (joblib).
|
| 7 |
+
- mk_vectorized / sen_vectorized: fast numpy versions (original MK, no autocorrelation correction),
|
| 8 |
+
used for placebo permutations and sensitivity loops.
|
| 9 |
+
- bh_fdr: Benjamini-Hochberg q-values.
|
| 10 |
+
- classify: growth / decline / unreliable / insufficient.
|
| 11 |
+
"""
|
| 12 |
+
from __future__ import annotations
|
| 13 |
+
|
| 14 |
+
import math
|
| 15 |
+
import warnings
|
| 16 |
+
|
| 17 |
+
import numpy as np
|
| 18 |
+
import pymannkendall as pmk
|
| 19 |
+
from scipy.stats import norm
|
| 20 |
+
|
| 21 |
+
CLASSES = ("growth", "decline", "unreliable", "insufficient")
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def _sen_ci(y: np.ndarray, t: np.ndarray, alpha: float = 0.05) -> tuple[float, float, float]:
|
| 25 |
+
"""Sen slope and its (1-alpha) CI by the rank method (Gilbert 1987). y, t without NaN."""
|
| 26 |
+
n = len(y)
|
| 27 |
+
i, j = np.triu_indices(n, k=1)
|
| 28 |
+
dt = t[j] - t[i]
|
| 29 |
+
slopes = np.sort((y[j] - y[i]) / dt)
|
| 30 |
+
m = len(slopes)
|
| 31 |
+
med = float(np.median(slopes))
|
| 32 |
+
# variance of S without tie correction (ties are rare for float NDVI means)
|
| 33 |
+
var_s = n * (n - 1) * (2 * n + 5) / 18.0
|
| 34 |
+
c = norm.ppf(1 - alpha / 2) * math.sqrt(var_s)
|
| 35 |
+
lo_idx = int(math.floor((m - c) / 2.0))
|
| 36 |
+
hi_idx = int(math.ceil((m + c) / 2.0))
|
| 37 |
+
lo_idx = min(max(lo_idx, 0), m - 1)
|
| 38 |
+
hi_idx = min(max(hi_idx, 0), m - 1)
|
| 39 |
+
return med, float(slopes[lo_idx]), float(slopes[hi_idx])
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def trend_1d(y, years, method: str = "hamed_rao", alpha: float = 0.05, min_n: int = 8) -> dict:
|
| 43 |
+
"""Trend of one series. NaNs are dropped. Returns slope per time unit (per year)."""
|
| 44 |
+
y = np.asarray(y, dtype=float)
|
| 45 |
+
t = np.asarray(years, dtype=float)
|
| 46 |
+
ok = np.isfinite(y)
|
| 47 |
+
y, t = y[ok], t[ok]
|
| 48 |
+
n = int(len(y))
|
| 49 |
+
out = {"slope": np.nan, "intercept": np.nan, "ci_lo": np.nan, "ci_hi": np.nan,
|
| 50 |
+
"p": np.nan, "z": np.nan, "tau": np.nan, "n": n, "method": method}
|
| 51 |
+
if n < min_n or np.nanstd(y) == 0:
|
| 52 |
+
if n >= min_n:
|
| 53 |
+
out.update(slope=0.0, intercept=float(np.median(y)), ci_lo=0.0, ci_hi=0.0, p=1.0, z=0.0, tau=0.0)
|
| 54 |
+
return out
|
| 55 |
+
slope, lo, hi = _sen_ci(y, t, alpha)
|
| 56 |
+
with warnings.catch_warnings():
|
| 57 |
+
warnings.simplefilter("ignore")
|
| 58 |
+
if method == "hamed_rao":
|
| 59 |
+
r = pmk.hamed_rao_modification_test(y)
|
| 60 |
+
else:
|
| 61 |
+
r = pmk.original_test(y)
|
| 62 |
+
p = float(r.p)
|
| 63 |
+
if not np.isfinite(p): # Hamed-Rao can fail on degenerate series; fall back to original MK
|
| 64 |
+
r = pmk.original_test(y)
|
| 65 |
+
p = float(r.p)
|
| 66 |
+
intercept = float(np.median(y - slope * t))
|
| 67 |
+
out.update(slope=slope, intercept=intercept, ci_lo=lo, ci_hi=hi, p=p, z=float(r.z), tau=float(r.Tau))
|
| 68 |
+
return out
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def _trend_rows(Y, years, method, alpha, min_n):
|
| 72 |
+
return [trend_1d(row, years, method, alpha, min_n) for row in Y]
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def trend_many(Y, years, method: str = "hamed_rao", alpha: float = 0.05, min_n: int = 8,
|
| 76 |
+
n_jobs: int = -1, chunk: int = 2000):
|
| 77 |
+
"""trend_1d for every row of Y (n_series, n_years). Returns a dict of 1-D arrays."""
|
| 78 |
+
import pandas as pd
|
| 79 |
+
from joblib import Parallel, delayed
|
| 80 |
+
|
| 81 |
+
Y = np.asarray(Y, dtype=float)
|
| 82 |
+
chunks = [Y[i:i + chunk] for i in range(0, len(Y), chunk)]
|
| 83 |
+
if len(chunks) <= 1 or n_jobs == 1:
|
| 84 |
+
rows = _trend_rows(Y, years, method, alpha, min_n)
|
| 85 |
+
else:
|
| 86 |
+
parts = Parallel(n_jobs=n_jobs)(delayed(_trend_rows)(c, years, method, alpha, min_n) for c in chunks)
|
| 87 |
+
rows = [r for p in parts for r in p]
|
| 88 |
+
df = pd.DataFrame(rows)
|
| 89 |
+
return {k: df[k].to_numpy() for k in df.columns}
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def sen_vectorized(Y, years) -> np.ndarray:
|
| 93 |
+
"""Sen slope for each row (NaN-aware)."""
|
| 94 |
+
Y = np.asarray(Y, dtype=np.float32)
|
| 95 |
+
t = np.asarray(years, dtype=np.float32)
|
| 96 |
+
i, j = np.triu_indices(len(t), k=1)
|
| 97 |
+
s = (Y[:, j] - Y[:, i]) / (t[j] - t[i])
|
| 98 |
+
with warnings.catch_warnings():
|
| 99 |
+
warnings.simplefilter("ignore")
|
| 100 |
+
return np.nanmedian(s, axis=1)
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def mk_vectorized(Y, years=None):
|
| 104 |
+
"""Original Mann-Kendall for each row (NaN-aware, no tie/autocorrelation correction).
|
| 105 |
+
|
| 106 |
+
Returns (S, var_s, z, p)."""
|
| 107 |
+
Y = np.asarray(Y, dtype=np.float32)
|
| 108 |
+
n_t = Y.shape[1]
|
| 109 |
+
i, j = np.triu_indices(n_t, k=1)
|
| 110 |
+
d = Y[:, j] - Y[:, i]
|
| 111 |
+
valid = np.isfinite(d)
|
| 112 |
+
S = np.where(valid, np.sign(d), 0).sum(axis=1).astype(float)
|
| 113 |
+
n = np.isfinite(Y).sum(axis=1).astype(float)
|
| 114 |
+
var_s = n * (n - 1) * (2 * n + 5) / 18.0
|
| 115 |
+
with np.errstate(invalid="ignore", divide="ignore"):
|
| 116 |
+
z = np.where(S > 0, (S - 1) / np.sqrt(var_s), np.where(S < 0, (S + 1) / np.sqrt(var_s), 0.0))
|
| 117 |
+
p = 2 * (1 - norm.cdf(np.abs(z)))
|
| 118 |
+
p = np.where(n >= 3, p, np.nan)
|
| 119 |
+
return S, var_s, z, p
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def bh_fdr(p) -> np.ndarray:
|
| 123 |
+
"""Benjamini-Hochberg q-values; NaN p-values stay NaN and are not counted."""
|
| 124 |
+
p = np.asarray(p, dtype=float)
|
| 125 |
+
q = np.full_like(p, np.nan)
|
| 126 |
+
ok = np.isfinite(p)
|
| 127 |
+
pv = p[ok]
|
| 128 |
+
m = len(pv)
|
| 129 |
+
if m == 0:
|
| 130 |
+
return q
|
| 131 |
+
order = np.argsort(pv)
|
| 132 |
+
ranked = pv[order] * m / np.arange(1, m + 1)
|
| 133 |
+
ranked = np.minimum.accumulate(ranked[::-1])[::-1]
|
| 134 |
+
qv = np.empty(m)
|
| 135 |
+
qv[order] = np.minimum(ranked, 1.0)
|
| 136 |
+
q[ok] = qv
|
| 137 |
+
return q
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
def classify(slope, p_or_q, n, alpha: float = 0.05, min_years: int = 18) -> np.ndarray:
|
| 141 |
+
"""growth / decline / unreliable / insufficient, element-wise."""
|
| 142 |
+
slope = np.atleast_1d(np.asarray(slope, dtype=float))
|
| 143 |
+
pq = np.atleast_1d(np.asarray(p_or_q, dtype=float))
|
| 144 |
+
n = np.atleast_1d(np.asarray(n, dtype=float))
|
| 145 |
+
out = np.full(slope.shape, "unreliable", dtype=object)
|
| 146 |
+
sig = np.isfinite(pq) & (pq < alpha)
|
| 147 |
+
out[sig & (slope > 0)] = "growth"
|
| 148 |
+
out[sig & (slope < 0)] = "decline"
|
| 149 |
+
out[(n < min_years) | ~np.isfinite(slope)] = "insufficient"
|
| 150 |
+
return out
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
def per_decade(slope) -> np.ndarray:
|
| 154 |
+
return np.asarray(slope, dtype=float) * 10.0
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
def pct_per_decade(slope, mean) -> np.ndarray:
|
| 158 |
+
with np.errstate(invalid="ignore", divide="ignore"):
|
| 159 |
+
return np.asarray(slope, dtype=float) * 10.0 / np.asarray(mean, dtype=float) * 100.0
|