File size: 10,566 Bytes
80e070e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
"""
RockGlacierNet β€” Region collection (whole-Sikkim grid).   [lives in patch_dataset/]

Grids the AOI (bounding box of all inventory polygons + AOI_MARGIN_M ~= the Sikkim
glacier belt) into RAW_TILE_PX tiles and downloads, per tile:
  * a 12-band Sentinel-2 + terrain GeoTIFF, and
  * a 1-band mask rasterized from ALL polygons onto that tile's exact grid.

REGION_MODE = "sampled" : every tile touching a glacier + REGION_BG_KEEP background tiles.
REGION_MODE = "dense"   : every tile in the AOI (full mosaic, multi-GB).

Robustness: the median composite is kept light with a scene-level cloud filter
(S2_CLOUD_PCT); if a tile still trips GEE "User memory limit exceeded", it is
auto-split into a finer sub-grid, each part downloaded and mosaicked in memory.

Outputs (nothing outside patch_dataset/):
  patch_dataset/raw_region/features/tile_RRRR_CCCC.tif
  patch_dataset/raw_region/masks/tile_RRRR_CCCC.tif

Run (env with ee+geemap+geopandas+rasterio, e.g. `brats`):
    python patch_dataset/collect_region.py                 # sampled (default)
    python patch_dataset/collect_region.py --mode dense
    python patch_dataset/collect_region.py --limit 3       # smoke test
All tunables live in patch_dataset/config.py.
"""

import os
import sys
import time
import argparse
import requests
import numpy as np

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))   # import local config
import config as C

import ee
import geopandas as gpd
import rasterio
from rasterio.io import MemoryFile
from rasterio.merge import merge as rio_merge
from rasterio.features import rasterize
from shapely.geometry import box


# ─────────────────────────────────────────────────────────────────────────────
# Earth Engine: memory-light 12-band composite (band order == C.BAND_NAMES)
# ─────────────────────────────────────────────────────────────────────────────

def feature_image(roi):
    coll = (ee.ImageCollection(C.S2_COLLECTION)
            .filterBounds(roi)
            .filterDate(C.S2_DATE_START, C.S2_DATE_END)
            .filter(ee.Filter.lt("CLOUDY_PIXEL_PERCENTAGE", C.S2_CLOUD_PCT)))

    def mask_clouds(im):
        qa = im.select("QA60")
        m = qa.bitwiseAnd(1 << 10).eq(0).And(qa.bitwiseAnd(1 << 11).eq(0))
        return im.updateMask(m).divide(C.S2_SR_DIVISOR)

    s2 = coll.map(mask_clouds).median()
    ndvi = s2.normalizedDifference(["B8", "B4"]).rename("NDVI")
    ndwi = s2.normalizedDifference(["B3", "B8"]).rename("NDWI")
    ndsi = s2.normalizedDifference(["B3", "B11"]).rename("NDSI")
    s2 = s2.select(["B2", "B3", "B4", "B8", "B11", "B12"]).addBands([ndvi, ndwi, ndsi])

    dem = ee.ImageCollection(C.DEM_COLLECTION).select("DEM").mosaic()
    # CRITICAL: reproject to a METRIC grid before ee.Terrain. After .mosaic() the
    # image defaults to EPSG:4326 (degrees); computing slope there yields garbage
    # (~1 deg everywhere). Reproject to UTM @30 m so slope/aspect are correct.
    dem_m = dem.reproject(crs=C.TARGET_CRS, scale=30)
    terr = ee.Terrain.products(dem_m)
    img = (s2.addBands(dem_m.rename("Elev"))
             .addBands(terr.select("slope").rename("Slope"))
             .addBands(terr.select("aspect").rename("Aspect")))
    return img.clip(roi)


# ─────────────────────────────────────────────────────────────────────────────
# Download helpers (UTM bbox -> GeoTIFF bytes, with memory-aware sub-tiling)
# ─────────────────────────────────────────────────────────────────────────────

def _utm_to_ee_roi(bbox_utm):
    ll = gpd.GeoSeries([box(*bbox_utm)], crs=C.TARGET_CRS).to_crs("EPSG:4326").total_bounds
    return ee.Geometry.Rectangle([ll[0], ll[1], ll[2], ll[3]])


def _download_bytes(bbox_utm, tries=3):
    """Download one bbox as GeoTIFF bytes. Raises MemoryError on GEE memory limit."""
    roi = _utm_to_ee_roi(bbox_utm)
    url = feature_image(roi).getDownloadURL(
        {"region": roi, "scale": C.SCALE_M, "crs": C.TARGET_CRS, "format": "GEO_TIFF"})
    last = None
    for k in range(tries):
        r = requests.get(url, timeout=300)
        if r.status_code == 200 and r.content[:2] in (b"II", b"MM"):   # valid TIFF magic
            return r.content
        last = r.content[:200]
        if r.status_code == 400 and b"memory limit" in r.content.lower():
            raise MemoryError(last)            # signal: split further
        time.sleep(2 * (k + 1))                # transient: retry
    raise RuntimeError(f"download failed: {last}")


def fetch_tile(bbox_utm, max_split=4):
    """Download bbox; on GEE memory error, recursively split into a finer grid and
    mosaic the parts in memory. Returns (array[C,H,W], transform, crs).

    NB: MemoryFile must stay open while its dataset is read, so we copy the array
    out before closing (otherwise GDAL reads a freed buffer)."""
    split = 1
    while True:
        minx, miny, maxx, maxy = bbox_utm
        xs = np.linspace(minx, maxx, split + 1)
        ys = np.linspace(miny, maxy, split + 1)
        try:
            chunks = [_download_bytes((xs[i], ys[j], xs[i+1], ys[j+1]))
                      for i in range(split) for j in range(split)]
        except MemoryError:
            if split >= max_split:
                raise RuntimeError("memory limit even at max split")
            split *= 2   # 1 -> 2 -> 4 (up to 16 sub-tiles)
            continue

        mfs, dsets = [], []
        try:
            for b in chunks:
                mf = MemoryFile(b); mfs.append(mf); dsets.append(mf.open())
            if len(dsets) == 1:
                ds = dsets[0]
                return ds.read().copy(), ds.transform, ds.crs
            arr, transform = rio_merge(dsets)
            return arr.copy(), transform, dsets[0].crs
        finally:
            for d in dsets: d.close()
            for mf in mfs: mf.close()


# ─────────────────────────────────────────────────────────────────────────────
# Main
# ─────────────────────────────────────────────────────────────────────────────

def main():
    ap = argparse.ArgumentParser(description="Whole-Sikkim region grid collection.")
    ap.add_argument("--shapefile", default=str(C.SHAPEFILE_PATH))
    ap.add_argument("--mode", choices=["sampled", "dense"], default=C.REGION_MODE)
    ap.add_argument("--limit", type=int, default=None, help="download at most N tiles")
    args = ap.parse_args()

    try:
        ee.Initialize(project=C.GEE_PROJECT)
    except Exception:
        ee.Authenticate(); ee.Initialize(project=C.GEE_PROJECT)

    C.RAW_REGION_FEATURES_DIR.mkdir(parents=True, exist_ok=True)
    C.RAW_REGION_MASKS_DIR.mkdir(parents=True, exist_ok=True)

    gdf = gpd.read_file(args.shapefile).to_crs(C.TARGET_CRS)
    gdf = gdf[gdf.geometry.notna() & ~gdf.geometry.is_empty].copy()
    sidx = gdf.sindex

    minx, miny, maxx, maxy = gdf.total_bounds
    m, ts = C.AOI_MARGIN_M, C.RAW_TILE_PX * C.SCALE_M
    minx, miny, maxx, maxy = minx - m, miny - m, maxx + m, maxy + m
    xs = np.arange(minx, maxx, ts)
    ys = np.arange(miny, maxy, ts)
    rng = np.random.RandomState(C.RANDOM_SEED)

    # Plan tiles
    plan = []   # (row, col, bbox, has_glacier)
    for r, y in enumerate(ys):
        for c, x in enumerate(xs):
            bb = (x, y, x + ts, y + ts)
            cand = list(sidx.intersection(bb))
            has = bool(cand) and gdf.geometry.iloc[cand].intersects(box(*bb)).any()
            if args.mode == "dense" or has or rng.rand() < C.REGION_BG_KEEP:
                plan.append((r, c, bb, has))
    if args.limit:
        plan = plan[:args.limit]

    n_g = sum(1 for *_, h in plan if h)
    print(f"AOI {(maxx-minx)/1000:.0f}x{(maxy-miny)/1000:.0f} km | grid {len(xs)}x{len(ys)} "
          f"= {len(xs)*len(ys)} cells | mode={args.mode} | downloading {len(plan)} tiles "
          f"({n_g} glacier, {len(plan)-n_g} background)")

    done = skip = fail = 0
    for n, (r, c, bb, has) in enumerate(plan):
        fpath = C.RAW_REGION_FEATURES_DIR / f"tile_{r:04d}_{c:04d}.tif"
        mpath = C.RAW_REGION_MASKS_DIR / f"tile_{r:04d}_{c:04d}.tif"
        if fpath.exists() and mpath.exists():
            skip += 1; continue
        try:
            arr, transform, crs = fetch_tile(bb)
            H, W = arr.shape[1], arr.shape[2]
            with rasterio.open(fpath, "w", driver="GTiff", height=H, width=W,
                               count=arr.shape[0], dtype="float32",
                               crs=crs, transform=transform) as dst:
                dst.write(arr.astype(np.float32))
            # mask: rasterize all polygons intersecting this tile onto its grid
            cand = list(sidx.intersection(bb))
            geoms = [g for g in gdf.geometry.iloc[cand] if g.intersects(box(*bb))]
            mask = (rasterize([(g, 1) for g in geoms], out_shape=(H, W),
                              transform=transform, fill=0, dtype="uint8")
                    if geoms else np.zeros((H, W), "uint8"))
            with rasterio.open(mpath, "w", driver="GTiff", height=H, width=W, count=1,
                               dtype="uint8", crs=crs, transform=transform, nodata=0) as dst:
                dst.write(mask, 1)
            done += 1
            if done % 20 == 0 or n == len(plan) - 1:
                print(f"  {n+1}/{len(plan)}  done={done} skip={skip} fail={fail} "
                      f"(last tile {H}x{W}, fg={int(mask.sum())})")
        except Exception as e:
            fail += 1
            print(f"  tile_{r:04d}_{c:04d} FAILED: {e}")

    print(f"\nRegion collection complete. done={done} skipped={skip} failed={fail}")
    print(f"Features -> {C.RAW_REGION_FEATURES_DIR}")
    print(f"Masks    -> {C.RAW_REGION_MASKS_DIR}")


if __name__ == "__main__":
    main()