Download datacollection/extract_patches.py from kabiriitb/rockglacierNET: direct link, hf CLI and curl.
- Browser
- Download file 13.9 kB
-
https://huggingface.co/kabiriitb/rockglacierNET/resolve/main/datacollection/extract_patches.py
- Command line
-
hf download hf://kabiriitb/rockglacierNET/datacollection/extract_patches.py
-
curl -L -o extract_patches.py https://huggingface.co/kabiriitb/rockglacierNET/resolve/main/datacollection/extract_patches.py
13.9 kB
| """ | |
| Patch extraction pipeline for RockGlacierNet. | |
| Reads split GeoTIFFs from output/dataset/{split}/, | |
| extracts 128Γ128 patches with stride 64, filters pure-background | |
| patches, saves .npy arrays, and writes rich metadata + diagnostics. | |
| Usage: | |
| python extract_patches.py | |
| Configuration: config.yaml (patching section) | |
| """ | |
| import os | |
| import sys | |
| import glob | |
| import json | |
| import math | |
| import yaml | |
| import numpy as np | |
| import rasterio | |
| import pandas as pd | |
| BAND_NAMES = ['B2', 'B3', 'B4', 'B8', 'B11', 'B12', | |
| 'NDVI', 'NDWI', 'NDSI', 'Elev', 'Slope', 'Aspect'] | |
| def load_config(): | |
| config_path = os.path.join(os.path.dirname(__file__), 'config.yaml') | |
| with open(config_path, 'r') as f: | |
| return yaml.safe_load(f) | |
| # βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| # Geometry helpers | |
| # βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def padded_size(dim: int, patch: int, stride: int) -> int: | |
| """Minimum padded dimension that gives full patch coverage with stride.""" | |
| if dim <= patch: | |
| return patch | |
| n = math.ceil((dim - patch) / stride) | |
| return n * stride + patch | |
| def pad_image(arr: np.ndarray, pad_h: int, pad_w: int, mode: str) -> np.ndarray: | |
| """Pad (H, W, C) array on right and bottom.""" | |
| if arr.ndim == 3: | |
| return np.pad(arr, ((0, pad_h), (0, pad_w), (0, 0)), | |
| mode=mode if arr.shape[2] > 0 else 'constant') | |
| return np.pad(arr, ((0, pad_h), (0, pad_w)), mode='constant', constant_values=0) | |
| def extract_patches_from_image(feat: np.ndarray, mask: np.ndarray, | |
| patch: int, stride: int, padding_mode: str): | |
| """ | |
| Yield (feat_patch, mask_patch, row, col, padding_applied, pad_h, pad_w) | |
| for every patch position covering the entire image. | |
| feat: (H, W, 12) float32 | |
| mask: (H, W, 1) float32 | |
| """ | |
| H, W = feat.shape[:2] | |
| pH = padded_size(H, patch, stride) | |
| pW = padded_size(W, patch, stride) | |
| dh, dw = pH - H, pW - W | |
| applied = dh > 0 or dw > 0 | |
| if applied: | |
| feat = pad_image(feat, dh, dw, padding_mode) | |
| # Pad the mask with the SAME mode as the feature. Previously the | |
| # feature was reflect-padded while the mask was zero-padded, so any | |
| # glacier pixels mirrored into the padded region were relabeled as | |
| # background. On crops smaller than `patch` (the majority here) the | |
| # padded region is most of the patch, which poisoned training by | |
| # teaching the model that glacier spectra == background. | |
| if padding_mode == 'constant': | |
| mask = np.pad(mask, ((0, dh), (0, dw), (0, 0)), | |
| mode='constant', constant_values=0) | |
| else: | |
| mask = np.pad(mask, ((0, dh), (0, dw), (0, 0)), mode=padding_mode) | |
| for r in range(0, pH - patch + 1, stride): | |
| for c in range(0, pW - patch + 1, stride): | |
| yield (feat[r:r+patch, c:c+patch, :], | |
| mask[r:r+patch, c:c+patch, :], | |
| r, c, applied, dh, dw) | |
| # βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| # Main | |
| # βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def run(): | |
| config = load_config() | |
| cfg = config['patching'] | |
| base_dir = os.path.join(os.path.dirname(__file__), config['paths']['output_dir']) | |
| srcdir = os.path.join(base_dir, 'dataset') | |
| outdir = os.path.join(base_dir, 'patches') | |
| rng = np.random.RandomState(cfg['random_seed']) | |
| P = cfg['patch_size'] | |
| S = cfg['stride'] | |
| bg_r = cfg['background_keep_ratio'] | |
| min_g = cfg['minimum_glacier_percentage'] | |
| mode = cfg['padding_mode'] | |
| splits = ['train', 'val', 'test'] | |
| os.makedirs(f"{outdir}/metadata", exist_ok=True) | |
| with open(f"{outdir}/metadata/config.json", 'w') as f: | |
| json.dump(cfg, f, indent=2) | |
| patch_records = [] | |
| image_records = [] | |
| for split in splits: | |
| feat_dir = os.path.join(srcdir, split, 'features') | |
| mask_dir = os.path.join(srcdir, split, 'masks') | |
| os.makedirs(os.path.join(outdir, split, 'features'), exist_ok=True) | |
| os.makedirs(os.path.join(outdir, split, 'masks'), exist_ok=True) | |
| feat_files = sorted(glob.glob(os.path.join(feat_dir, 'feature_*.tif'))) | |
| total = len(feat_files) | |
| print(f"\n[{split.upper()}] {total} images") | |
| n_gen = n_kept = 0 | |
| for idx, fp in enumerate(feat_files): | |
| img_id = os.path.basename(fp).replace('feature_', '').replace('.tif', '') | |
| mp = os.path.join(mask_dir, f"mask_{img_id}.tif") | |
| if not os.path.exists(mp): | |
| print(f" Warning: No mask for {img_id}, skipping.") | |
| continue | |
| with rasterio.open(fp) as fs: | |
| feat = np.moveaxis(fs.read().astype(np.float32), 0, -1) | |
| with rasterio.open(mp) as ms: | |
| mask = ms.read(1).astype(np.float32)[:, :, np.newaxis] | |
| # Harmonize: crop both to minimum common (H, W) | |
| fH, fW = feat.shape[:2] | |
| mH, mW = mask.shape[:2] | |
| minH, minW = min(fH, mH), min(fW, mW) | |
| feat = feat[:minH, :minW, :] | |
| mask = mask[:minH, :minW, :] | |
| # Clean no-data values | |
| feat = np.where((feat == -9999) | ~np.isfinite(feat), 0.0, feat) | |
| oH, oW = feat.shape[:2] | |
| img_fg = int((mask > 0.5).sum()) | |
| img_bg = mask.size - img_fg | |
| img_pct = 100.0 * img_fg / mask.size | |
| n_img_gen = n_img_kept = 0 | |
| for fp_p, mp_p, r, c, padded, pad_h, pad_w in \ | |
| extract_patches_from_image(feat, mask, P, S, mode): | |
| fg_px = int((mp_p > 0.5).sum()) | |
| bg_px = mp_p.size - fg_px | |
| g_pct = 100.0 * fg_px / mp_p.size | |
| has_glacier = g_pct >= min_g | |
| keep = has_glacier or (rng.rand() < bg_r) | |
| n_gen += 1 | |
| n_img_gen += 1 | |
| b_means = [round(float(fp_p[:, :, b].mean()), 6) for b in range(fp_p.shape[2])] | |
| b_stds = [round(float(fp_p[:, :, b].std()), 6) for b in range(fp_p.shape[2])] | |
| patch_id = f"{img_id}_r{r:04d}_c{c:04d}" | |
| pad_px = int(pad_h * (oW + pad_w) + pad_w * oH) if padded else 0 | |
| rec = { | |
| 'patch_id': patch_id, | |
| 'source_image_id': img_id, | |
| 'split': split, | |
| 'original_image_height': oH, | |
| 'original_image_width': oW, | |
| 'patch_row': r, | |
| 'patch_col': c, | |
| 'patch_height': fp_p.shape[0], | |
| 'patch_width': fp_p.shape[1], | |
| 'glacier_pixel_count': fg_px, | |
| 'background_pixel_count': bg_px, | |
| 'glacier_percentage': round(g_pct, 4), | |
| 'kept_or_filtered': 'kept' if keep else 'filtered', | |
| 'padding_applied': padded, | |
| 'padding_bottom': pad_h, | |
| 'padding_right': pad_w, | |
| 'padding_pixels': pad_px, | |
| } | |
| for b, name in enumerate(BAND_NAMES): | |
| rec[f'mean_{name}'] = b_means[b] | |
| rec[f'std_{name}'] = b_stds[b] | |
| patch_records.append(rec) | |
| if keep: | |
| np.save(os.path.join(outdir, split, 'features', f"{patch_id}.npy"), fp_p) | |
| np.save(os.path.join(outdir, split, 'masks', f"{patch_id}.npy"), mp_p) | |
| n_kept += 1 | |
| n_img_kept += 1 | |
| image_records.append({ | |
| 'image_id': img_id, | |
| 'split': split, | |
| 'original_height': oH, | |
| 'original_width': oW, | |
| 'glacier_pixels': img_fg, | |
| 'background_pixels': img_bg, | |
| 'glacier_percentage': round(img_pct, 4), | |
| 'number_of_generated_patches': n_img_gen, | |
| 'number_of_kept_patches': n_img_kept, | |
| }) | |
| if (idx + 1) % 50 == 0 or (idx + 1) == total: | |
| print(f" {idx+1}/{total} gen={n_gen} kept={n_kept}") | |
| # ββ Save metadata βββββββββββββββββββββββββββββββββββββββββββββ | |
| patch_df = pd.DataFrame(patch_records) | |
| image_df = pd.DataFrame(image_records) | |
| patch_df.to_csv(os.path.join(outdir, 'metadata', 'patch_metadata.csv'), index=False) | |
| image_df.to_csv(os.path.join(outdir, 'metadata', 'dataset_summary.csv'), index=False) | |
| print("\nMetadata CSV saved.") | |
| _write_diagnostics(patch_df, image_df, outdir, cfg) | |
| # βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| # Diagnostics | |
| # βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def _write_diagnostics(patch_df: pd.DataFrame, image_df: pd.DataFrame, | |
| outdir: str, cfg: dict): | |
| kept = patch_df[patch_df['kept_or_filtered'] == 'kept'] | |
| lines = [] | |
| def h(title): | |
| lines.append("") | |
| lines.append("β" * 58) | |
| lines.append(f" {title}") | |
| lines.append("β" * 58) | |
| lines.append("=" * 58) | |
| lines.append(" ROCKGLACIERNET β PATCH EXTRACTION DIAGNOSTICS") | |
| lines.append("=" * 58) | |
| lines.append(f" patch_size : {cfg['patch_size']}") | |
| lines.append(f" stride : {cfg['stride']}") | |
| lines.append(f" background_keep_ratio: {cfg['background_keep_ratio']}") | |
| lines.append(f" minimum_glacier_pct : {cfg['minimum_glacier_percentage']}") | |
| lines.append(f" padding_mode : {cfg['padding_mode']}") | |
| # 1: image counts | |
| h("1 IMAGE COUNTS") | |
| lines.append(f" Total original images: {len(image_df)}") | |
| for sp in ['train', 'val', 'test']: | |
| lines.append(f" {sp:5s}: {len(image_df[image_df['split']==sp])}") | |
| # 2: patch counts | |
| h("2 PATCH COUNTS") | |
| for sp in ['train', 'val', 'test']: | |
| g = len(patch_df[patch_df['split'] == sp]) | |
| k = len(kept[kept['split'] == sp]) | |
| lines.append(f" {sp:5s}: generated={g:5d} kept={k:5d} filtered={g-k:5d}") | |
| lines.append(f" TOTAL: generated={len(patch_df):5d} kept={len(kept):5d}") | |
| # 3: pixel class balance | |
| h("3 PIXEL CLASS BALANCE (kept patches)") | |
| tfg = kept['glacier_pixel_count'].sum() | |
| tbg = kept['background_pixel_count'].sum() | |
| tpx = tfg + tbg | |
| if tpx > 0: | |
| lines.append(f" Glacier : {int(tfg):>10,} ({100*tfg/tpx:.2f}%)") | |
| lines.append(f" Background: {int(tbg):>10,} ({100*tbg/tpx:.2f}%)") | |
| lines.append(f" Imbalance : {tbg/max(tfg,1):.2f}:1") | |
| # 4: glacier coverage distribution | |
| h("4 GLACIER COVERAGE DISTRIBUTION (kept patches, %)") | |
| bins = [(0, 0.1), (0.1, 5), (5, 20), (20, 50), (50, 80), (80, 100.01)] | |
| for lo, hi in bins: | |
| n = int(((kept['glacier_percentage'] >= lo) & (kept['glacier_percentage'] < hi)).sum()) | |
| bar = 'β' * min(n // 5, 40) | |
| lines.append(f" {lo:5.1f}β{hi:5.1f}%: {n:5d} {bar}") | |
| # 5: patches per image | |
| h("5 KEPT PATCHES PER IMAGE") | |
| for sp in ['train', 'val', 'test']: | |
| sub = image_df[image_df['split'] == sp]['number_of_kept_patches'] | |
| if len(sub) > 0: | |
| lines.append(f" {sp:5s}: min={sub.min()}" | |
| f" max={sub.max()}" | |
| f" mean={sub.mean():.1f}" | |
| f" median={sub.median():.0f}") | |
| # 6: padding stats | |
| h("6 PADDING STATS") | |
| need_pad = ((image_df['original_height'] < cfg['patch_size']) | | |
| (image_df['original_width'] < cfg['patch_size'])).sum() | |
| pat_pad = patch_df['padding_applied'].sum() | |
| lines.append(f" Images needing padding : {need_pad}/{len(image_df)}" | |
| f" ({100*need_pad/len(image_df):.1f}%)") | |
| if len(patch_df) > 0: | |
| lines.append(f" Patches with padding : {pat_pad}/{len(patch_df)}" | |
| f" ({100*pat_pad/len(patch_df):.1f}%)") | |
| # 7: expansion factor | |
| h("7 DATASET EXPANSION FACTOR") | |
| for sp in ['train', 'val', 'test']: | |
| n_img = len(image_df[image_df['split'] == sp]) | |
| n_kpt = len(kept[kept['split'] == sp]) | |
| factor = n_kpt / n_img if n_img else 0 | |
| lines.append(f" {sp:5s}: {n_img} images β {n_kpt} patches (Γ{factor:.1f})") | |
| report = "\n".join(lines) | |
| print("\n" + report) | |
| out_path = os.path.join(outdir, 'metadata', 'diagnostics_report.txt') | |
| with open(out_path, 'w') as f: | |
| f.write(report) | |
| print(f"\nDiagnostics β {out_path}") | |
| if __name__ == '__main__': | |
| run() | |