""" Patch extraction pipeline for RockGlacierNet. Reads split GeoTIFFs from output/dataset/{split}/, extracts 128×128 patches with stride 64, filters pure-background patches, saves .npy arrays, and writes rich metadata + diagnostics. Usage: python extract_patches.py Configuration: config.yaml (patching section) """ import os import sys import glob import json import math import yaml import numpy as np import rasterio import pandas as pd BAND_NAMES = ['B2', 'B3', 'B4', 'B8', 'B11', 'B12', 'NDVI', 'NDWI', 'NDSI', 'Elev', 'Slope', 'Aspect'] def load_config(): config_path = os.path.join(os.path.dirname(__file__), 'config.yaml') with open(config_path, 'r') as f: return yaml.safe_load(f) # ───────────────────────────────────────────────────────────────────────────── # Geometry helpers # ───────────────────────────────────────────────────────────────────────────── def padded_size(dim: int, patch: int, stride: int) -> int: """Minimum padded dimension that gives full patch coverage with stride.""" if dim <= patch: return patch n = math.ceil((dim - patch) / stride) return n * stride + patch def pad_image(arr: np.ndarray, pad_h: int, pad_w: int, mode: str) -> np.ndarray: """Pad (H, W, C) array on right and bottom.""" if arr.ndim == 3: return np.pad(arr, ((0, pad_h), (0, pad_w), (0, 0)), mode=mode if arr.shape[2] > 0 else 'constant') return np.pad(arr, ((0, pad_h), (0, pad_w)), mode='constant', constant_values=0) def extract_patches_from_image(feat: np.ndarray, mask: np.ndarray, patch: int, stride: int, padding_mode: str): """ Yield (feat_patch, mask_patch, row, col, padding_applied, pad_h, pad_w) for every patch position covering the entire image. feat: (H, W, 12) float32 mask: (H, W, 1) float32 """ H, W = feat.shape[:2] pH = padded_size(H, patch, stride) pW = padded_size(W, patch, stride) dh, dw = pH - H, pW - W applied = dh > 0 or dw > 0 if applied: feat = pad_image(feat, dh, dw, padding_mode) # Pad the mask with the SAME mode as the feature. Previously the # feature was reflect-padded while the mask was zero-padded, so any # glacier pixels mirrored into the padded region were relabeled as # background. On crops smaller than `patch` (the majority here) the # padded region is most of the patch, which poisoned training by # teaching the model that glacier spectra == background. if padding_mode == 'constant': mask = np.pad(mask, ((0, dh), (0, dw), (0, 0)), mode='constant', constant_values=0) else: mask = np.pad(mask, ((0, dh), (0, dw), (0, 0)), mode=padding_mode) for r in range(0, pH - patch + 1, stride): for c in range(0, pW - patch + 1, stride): yield (feat[r:r+patch, c:c+patch, :], mask[r:r+patch, c:c+patch, :], r, c, applied, dh, dw) # ───────────────────────────────────────────────────────────────────────────── # Main # ───────────────────────────────────────────────────────────────────────────── def run(): config = load_config() cfg = config['patching'] base_dir = os.path.join(os.path.dirname(__file__), config['paths']['output_dir']) srcdir = os.path.join(base_dir, 'dataset') outdir = os.path.join(base_dir, 'patches') rng = np.random.RandomState(cfg['random_seed']) P = cfg['patch_size'] S = cfg['stride'] bg_r = cfg['background_keep_ratio'] min_g = cfg['minimum_glacier_percentage'] mode = cfg['padding_mode'] splits = ['train', 'val', 'test'] os.makedirs(f"{outdir}/metadata", exist_ok=True) with open(f"{outdir}/metadata/config.json", 'w') as f: json.dump(cfg, f, indent=2) patch_records = [] image_records = [] for split in splits: feat_dir = os.path.join(srcdir, split, 'features') mask_dir = os.path.join(srcdir, split, 'masks') os.makedirs(os.path.join(outdir, split, 'features'), exist_ok=True) os.makedirs(os.path.join(outdir, split, 'masks'), exist_ok=True) feat_files = sorted(glob.glob(os.path.join(feat_dir, 'feature_*.tif'))) total = len(feat_files) print(f"\n[{split.upper()}] {total} images") n_gen = n_kept = 0 for idx, fp in enumerate(feat_files): img_id = os.path.basename(fp).replace('feature_', '').replace('.tif', '') mp = os.path.join(mask_dir, f"mask_{img_id}.tif") if not os.path.exists(mp): print(f" Warning: No mask for {img_id}, skipping.") continue with rasterio.open(fp) as fs: feat = np.moveaxis(fs.read().astype(np.float32), 0, -1) with rasterio.open(mp) as ms: mask = ms.read(1).astype(np.float32)[:, :, np.newaxis] # Harmonize: crop both to minimum common (H, W) fH, fW = feat.shape[:2] mH, mW = mask.shape[:2] minH, minW = min(fH, mH), min(fW, mW) feat = feat[:minH, :minW, :] mask = mask[:minH, :minW, :] # Clean no-data values feat = np.where((feat == -9999) | ~np.isfinite(feat), 0.0, feat) oH, oW = feat.shape[:2] img_fg = int((mask > 0.5).sum()) img_bg = mask.size - img_fg img_pct = 100.0 * img_fg / mask.size n_img_gen = n_img_kept = 0 for fp_p, mp_p, r, c, padded, pad_h, pad_w in \ extract_patches_from_image(feat, mask, P, S, mode): fg_px = int((mp_p > 0.5).sum()) bg_px = mp_p.size - fg_px g_pct = 100.0 * fg_px / mp_p.size has_glacier = g_pct >= min_g keep = has_glacier or (rng.rand() < bg_r) n_gen += 1 n_img_gen += 1 b_means = [round(float(fp_p[:, :, b].mean()), 6) for b in range(fp_p.shape[2])] b_stds = [round(float(fp_p[:, :, b].std()), 6) for b in range(fp_p.shape[2])] patch_id = f"{img_id}_r{r:04d}_c{c:04d}" pad_px = int(pad_h * (oW + pad_w) + pad_w * oH) if padded else 0 rec = { 'patch_id': patch_id, 'source_image_id': img_id, 'split': split, 'original_image_height': oH, 'original_image_width': oW, 'patch_row': r, 'patch_col': c, 'patch_height': fp_p.shape[0], 'patch_width': fp_p.shape[1], 'glacier_pixel_count': fg_px, 'background_pixel_count': bg_px, 'glacier_percentage': round(g_pct, 4), 'kept_or_filtered': 'kept' if keep else 'filtered', 'padding_applied': padded, 'padding_bottom': pad_h, 'padding_right': pad_w, 'padding_pixels': pad_px, } for b, name in enumerate(BAND_NAMES): rec[f'mean_{name}'] = b_means[b] rec[f'std_{name}'] = b_stds[b] patch_records.append(rec) if keep: np.save(os.path.join(outdir, split, 'features', f"{patch_id}.npy"), fp_p) np.save(os.path.join(outdir, split, 'masks', f"{patch_id}.npy"), mp_p) n_kept += 1 n_img_kept += 1 image_records.append({ 'image_id': img_id, 'split': split, 'original_height': oH, 'original_width': oW, 'glacier_pixels': img_fg, 'background_pixels': img_bg, 'glacier_percentage': round(img_pct, 4), 'number_of_generated_patches': n_img_gen, 'number_of_kept_patches': n_img_kept, }) if (idx + 1) % 50 == 0 or (idx + 1) == total: print(f" {idx+1}/{total} gen={n_gen} kept={n_kept}") # ── Save metadata ───────────────────────────────────────────── patch_df = pd.DataFrame(patch_records) image_df = pd.DataFrame(image_records) patch_df.to_csv(os.path.join(outdir, 'metadata', 'patch_metadata.csv'), index=False) image_df.to_csv(os.path.join(outdir, 'metadata', 'dataset_summary.csv'), index=False) print("\nMetadata CSV saved.") _write_diagnostics(patch_df, image_df, outdir, cfg) # ───────────────────────────────────────────────────────────────────────────── # Diagnostics # ───────────────────────────────────────────────────────────────────────────── def _write_diagnostics(patch_df: pd.DataFrame, image_df: pd.DataFrame, outdir: str, cfg: dict): kept = patch_df[patch_df['kept_or_filtered'] == 'kept'] lines = [] def h(title): lines.append("") lines.append("─" * 58) lines.append(f" {title}") lines.append("─" * 58) lines.append("=" * 58) lines.append(" ROCKGLACIERNET — PATCH EXTRACTION DIAGNOSTICS") lines.append("=" * 58) lines.append(f" patch_size : {cfg['patch_size']}") lines.append(f" stride : {cfg['stride']}") lines.append(f" background_keep_ratio: {cfg['background_keep_ratio']}") lines.append(f" minimum_glacier_pct : {cfg['minimum_glacier_percentage']}") lines.append(f" padding_mode : {cfg['padding_mode']}") # 1: image counts h("1 IMAGE COUNTS") lines.append(f" Total original images: {len(image_df)}") for sp in ['train', 'val', 'test']: lines.append(f" {sp:5s}: {len(image_df[image_df['split']==sp])}") # 2: patch counts h("2 PATCH COUNTS") for sp in ['train', 'val', 'test']: g = len(patch_df[patch_df['split'] == sp]) k = len(kept[kept['split'] == sp]) lines.append(f" {sp:5s}: generated={g:5d} kept={k:5d} filtered={g-k:5d}") lines.append(f" TOTAL: generated={len(patch_df):5d} kept={len(kept):5d}") # 3: pixel class balance h("3 PIXEL CLASS BALANCE (kept patches)") tfg = kept['glacier_pixel_count'].sum() tbg = kept['background_pixel_count'].sum() tpx = tfg + tbg if tpx > 0: lines.append(f" Glacier : {int(tfg):>10,} ({100*tfg/tpx:.2f}%)") lines.append(f" Background: {int(tbg):>10,} ({100*tbg/tpx:.2f}%)") lines.append(f" Imbalance : {tbg/max(tfg,1):.2f}:1") # 4: glacier coverage distribution h("4 GLACIER COVERAGE DISTRIBUTION (kept patches, %)") bins = [(0, 0.1), (0.1, 5), (5, 20), (20, 50), (50, 80), (80, 100.01)] for lo, hi in bins: n = int(((kept['glacier_percentage'] >= lo) & (kept['glacier_percentage'] < hi)).sum()) bar = '█' * min(n // 5, 40) lines.append(f" {lo:5.1f}–{hi:5.1f}%: {n:5d} {bar}") # 5: patches per image h("5 KEPT PATCHES PER IMAGE") for sp in ['train', 'val', 'test']: sub = image_df[image_df['split'] == sp]['number_of_kept_patches'] if len(sub) > 0: lines.append(f" {sp:5s}: min={sub.min()}" f" max={sub.max()}" f" mean={sub.mean():.1f}" f" median={sub.median():.0f}") # 6: padding stats h("6 PADDING STATS") need_pad = ((image_df['original_height'] < cfg['patch_size']) | (image_df['original_width'] < cfg['patch_size'])).sum() pat_pad = patch_df['padding_applied'].sum() lines.append(f" Images needing padding : {need_pad}/{len(image_df)}" f" ({100*need_pad/len(image_df):.1f}%)") if len(patch_df) > 0: lines.append(f" Patches with padding : {pat_pad}/{len(patch_df)}" f" ({100*pat_pad/len(patch_df):.1f}%)") # 7: expansion factor h("7 DATASET EXPANSION FACTOR") for sp in ['train', 'val', 'test']: n_img = len(image_df[image_df['split'] == sp]) n_kpt = len(kept[kept['split'] == sp]) factor = n_kpt / n_img if n_img else 0 lines.append(f" {sp:5s}: {n_img} images → {n_kpt} patches (×{factor:.1f})") report = "\n".join(lines) print("\n" + report) out_path = os.path.join(outdir, 'metadata', 'diagnostics_report.txt') with open(out_path, 'w') as f: f.write(report) print(f"\nDiagnostics → {out_path}") if __name__ == '__main__': run()