File size: 10,749 Bytes
17285c6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
"""Figures that need the rasters (server, CPU only, a few minutes). Queue ON THE SERVER:
  bash ~/pakflood/code/pakflood/run_job.sh cmd figs .venv/bin/python pakflood/make_figures.py --out /home/shahwar/pakflood/figs

Writes into --out:
  fig_tiles_emsr838.png     the 4 official PK test tiles with most flood: post VV | pre VV | CEMS label | hold-out | few-shot | full FT
  fig_emsr838_overview.png  EMSR838 (Pakistan 2022) mosaic at ~200 m, EPSG:4326: flood fraction per model over WorldPop density
  mosaic_emsr838.npz        mosaic arrays (label / per-model flood fraction, mapped mask, people per cell, bounds) for re-plotting
"""
import argparse
import glob
from pathlib import Path

import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import rasterio
from matplotlib.colors import BoundaryNorm, ListedColormap
from matplotlib.patches import Patch
from rasterio.enums import Resampling
from rasterio.transform import from_origin
from rasterio.warp import reproject, transform_bounds
from rasterio.windows import from_bounds

CLS_CMAP = ListedColormap(["#e6e6e6", "#1f77b4", "#d62728"])  # background, permanent water, flood
CLS_NORM = BoundaryNorm([-0.5, 0.5, 1.5, 2.5], 3)
MODELS = [("sa_holdout", "LoRA hold-out (never saw South Asia)"),
          ("sa_fewshot", "LoRA few-shot (+26 SA tiles)"),
          ("full_finetune", "Full fine-tune (trained on PK)")]


def read1(p):
    with rasterio.open(p) as r:
        return r.read(1), r.transform, r.crs, r.bounds


def show_cls(ax, a, title=""):
    ax.imshow(np.ma.masked_equal(a, 255), cmap=CLS_CMAP, norm=CLS_NORM, interpolation="nearest")
    ax.set_title(title, fontsize=8)
    ax.set_axis_off()


def flood_iou(pred, label):
    v = label != 255
    tp = ((pred == 2) & (label == 2) & v).sum()
    fp = ((pred == 2) & (label != 2) & v).sum()
    fn = ((pred != 2) & (label == 2) & v).sum()
    return tp / max(tp + fp + fn, 1)


def fig_tiles(a, out):
    root, preds = Path(a.root), Path(a.preds)
    df = pd.read_csv(root / "data_tiles_s256_st128.csv", usecols=["tile_id", "event_id", "modality", "image_time", "label_id"])
    df = df[(df.modality == "s1grd") & (df.event_id == "EMSR838")].drop_duplicates(["label_id", "image_time", "tile_id"])
    pt = pd.read_csv(preds / "sa_holdout_emsr838" / "per_tile.csv")
    pt = pt[pt.split == "test"].copy()
    pt["lab_flood"] = pt.cm_20 + pt.cm_21 + pt.cm_22
    tiles = a.tiles or pt.sort_values("lab_flood", ascending=False).label_id.head(4).tolist()
    ncol = 3 + len(MODELS)
    fig, axes = plt.subplots(len(tiles), ncol, figsize=(2.5 * ncol, 2.6 * len(tiles)))
    axes = np.atleast_2d(axes)
    for i, lid in enumerate(tiles):
        g = df[df.label_id == lid]
        pre, post = g[g.image_time == "pre"].tile_id.iloc[0], g[g.image_time == "post"].tile_id.iloc[0]
        aoi = lid.replace("_label", "").rsplit("-", 1)[0]  # tiles live in data/geoid-flood/<AOI>/{s1grd,label}
        label = read1(root / aoi / "label" / f"{lid}.tif")[0]
        for j, (tid, name) in enumerate([(post, "post-event S1 VV (dB)"), (pre, "pre-event S1 VV (dB)")]):
            with rasterio.open(root / aoi / "s1grd" / f"{tid}.tif") as r:
                vv = r.read(1).astype(np.float32)
            vv = 10 * np.log10(np.clip(vv, 1e-4, None))
            axes[i, j].imshow(vv, cmap="gray", vmin=-25, vmax=5)
            axes[i, j].set_title(name if i == 0 else "", fontsize=8)
            axes[i, j].set_axis_off()
        show_cls(axes[i, 2], label, "CEMS official label" if i == 0 else "")
        for k, (m, name) in enumerate(MODELS):
            pred = read1(preds / f"{m}_emsr838" / f"{lid}_pred.tif")[0]
            show_cls(axes[i, 3 + k], pred, (name.replace(" (", "\n(") + "\n" if i == 0 else "") + f"flood IoU {flood_iou(pred, label):.2f}")
        axes[i, 0].text(-0.03, 0.5, lid.replace("_label", ""), transform=axes[i, 0].transAxes, rotation=90,
                        va="center", ha="right", fontsize=8)
    fig.legend(handles=[Patch(color=CLS_CMAP(0), label="background"), Patch(color=CLS_CMAP(1), label="permanent water"),
                        Patch(color=CLS_CMAP(2), label="flood")], loc="lower center", ncol=3, frameon=False, fontsize=9)
    fig.tight_layout(rect=(0.02, 0.03, 1, 1))
    fig.savefig(out / "fig_tiles_emsr838.png", dpi=150)
    plt.close(fig)
    print("wrote", out / "fig_tiles_emsr838.png", "tiles:", tiles, flush=True)


def fig_overview(a, out):
    root, preds, res = Path(a.root), Path(a.preds), a.res
    lids = sorted(Path(p).name.replace("_pred.tif", "") for p in glob.glob(f"{preds}/sa_fewshot_emsr838/*_pred.tif"))
    bb = {}
    def aoi_of(lid):
        return lid.replace("_label", "").rsplit("-", 1)[0]
    for lid in lids:
        with rasterio.open(root / aoi_of(lid) / "label" / f"{lid}.tif") as r:
            bb[lid] = transform_bounds(r.crs, "EPSG:4326", *r.bounds)
    arr = np.array(list(bb.values()))
    W, S, E, N = arr[:, 0].min() - res, arr[:, 1].min() - res, arr[:, 2].max() + res, arr[:, 3].max() + res
    width, height = int(np.ceil((E - W) / res)), int(np.ceil((N - S) / res))
    print(f"mosaic grid {width}x{height} at {res} deg, {len(lids)} tiles", flush=True)
    layers = {"label": np.zeros((height, width), np.float32), **{m: np.zeros((height, width), np.float32) for m, _ in MODELS}}
    mapped = np.zeros((height, width), np.float32)
    aoi_pts = {}
    for n, lid in enumerate(lids, 1):
        aoi = aoi_of(lid)
        label, tr, crs, _ = read1(root / aoi / "label" / f"{lid}.tif")
        w, s, e, nn = bb[lid]
        aoi_pts.setdefault(aoi, []).append(((w + e) / 2, (s + nn) / 2))
        c0, r0 = int((w - W) / res), int((N - nn) / res)
        c1, r1 = int(np.ceil((e - W) / res)), int(np.ceil((N - s) / res))
        dst_tr = from_origin(W + c0 * res, N - r0 * res, res, res)
        srcs = {"label": label}
        for m, _ in MODELS:
            srcs[m] = read1(preds / f"{m}_emsr838" / f"{lid}_pred.tif")[0]
        valid = (label != 255).astype(np.float32)
        for key, src in list(srcs.items()) + [("mapped", None)]:
            mask = valid if key == "mapped" else ((src == 2) & (label != 255)).astype(np.float32)
            tmp = np.zeros((r1 - r0, c1 - c0), np.float32)
            reproject(mask, tmp, src_transform=tr, src_crs=crs, dst_transform=dst_tr, dst_crs="EPSG:4326",
                      resampling=Resampling.average)
            target = mapped if key == "mapped" else layers[key]
            target[r0:r1, c0:c1] = np.maximum(target[r0:r1, c0:c1], tmp)
        if n % 50 == 0 or n == len(lids):
            print(f"  {n}/{len(lids)}", flush=True)
    # WorldPop people per grid cell (sum resampling), read only the window we need
    people = np.zeros((height, width), np.float32)
    with rasterio.open(a.worldpop) as pop:
        win = from_bounds(W, S, E, N, transform=pop.transform).round_offsets().round_lengths()
        win = win.intersection(rasterio.windows.Window(0, 0, pop.width, pop.height))
        src = pop.read(1, window=win).astype(np.float32)
        src[(src == pop.nodata) | ~np.isfinite(src) | (src < 0)] = 0
        reproject(src, people, src_transform=pop.window_transform(win), src_crs=pop.crs,
                  dst_transform=from_origin(W, N, res, res), dst_crs="EPSG:4326", resampling=Resampling.sum)
    np.savez_compressed(out / "mosaic_emsr838.npz", mapped=mapped, people=people, bounds=np.array([W, S, E, N]), res=res,
                        aois=np.array(list(aoi_pts.keys())), aoi_xy=np.array([np.mean(v, axis=0) for v in aoi_pts.values()]),
                        **layers)
    pops = {}
    for m, _ in MODELS:
        d = pd.read_csv(preds / f"pop_{m}.csv")
        pops[m] = (d.pop_pred.sum(), d.pop_label.sum())
    panels = [("label", "CEMS official outline")] + MODELS
    fig, axes = plt.subplots(1, len(panels), figsize=(4.6 * len(panels), 5.2), sharex=True, sharey=True)
    ext = (W, E, S, N)
    dens = np.log10(1 + people)
    for ax, (key, name) in zip(axes, panels):
        ax.imshow(dens, cmap="Greys", vmin=0, vmax=np.percentile(dens[dens > 0], 99.5) if (dens > 0).any() else 1,
                  extent=ext, interpolation="nearest")
        rgba = np.zeros((height, width, 4), np.float32)
        rgba[..., 0], rgba[..., 1], rgba[..., 2] = 0.12, 0.47, 0.71  # mapped area tint (blue)
        rgba[..., 3] = 0.18 * (mapped > 0.05)
        ax.imshow(rgba, extent=ext, interpolation="nearest")
        rgba = np.zeros((height, width, 4), np.float32)
        rgba[..., 0], rgba[..., 1], rgba[..., 2] = 0.84, 0.15, 0.16
        rgba[..., 3] = np.clip(layers[key] * 1.3, 0, 1)
        ax.imshow(rgba, extent=ext, interpolation="nearest")
        for aoi, pts in aoi_pts.items():
            x, y = np.mean(pts, axis=0)
            ax.annotate(aoi.replace("EMSR838-", "AOI "), (x, y), fontsize=6, ha="center", color="black", alpha=0.8)
        if key == "label":
            ax.set_title(f"{name}\naffected population {pops[MODELS[0][0]][1]:,.0f}", fontsize=10)
        else:
            p, lab = pops[key]
            ax.set_title(f"{name}\naffected population {p:,.0f} ({100 * (p / lab - 1):+.1f}%)", fontsize=10)
        ax.set_xlabel("longitude")
    axes[0].set_ylabel("latitude")
    fig.legend(handles=[Patch(color="#d62728", label="flooded (fraction of ~200 m cell)"),
                        Patch(color="#1f77b4", alpha=0.3, label="mapped area (AOI)"),
                        Patch(color="grey", label="WorldPop 2020 population density (log)")],
               loc="lower center", ncol=3, frameon=False, fontsize=9)
    fig.suptitle("EMSR838 - Pakistan monsoon floods 2022 (Sentinel-1, 10 m) - flood extent and affected population", fontsize=12)
    fig.tight_layout(rect=(0, 0.05, 1, 0.95))
    fig.savefig(out / "fig_emsr838_overview.png", dpi=150)
    plt.close(fig)
    print("wrote", out / "fig_emsr838_overview.png", flush=True)


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--preds", default=str(Path.home() / "pakflood/preds"))
    ap.add_argument("--root", default="data/geoid-flood")
    ap.add_argument("--worldpop", default=str(Path.home() / "pakflood/data/worldpop/pak_ppp_2020_constrained.tif"))
    ap.add_argument("--out", default=str(Path.home() / "pakflood/figs"))
    ap.add_argument("--res", type=float, default=0.002, help="mosaic cell size in degrees (~200 m)")
    ap.add_argument("--tiles", nargs="*", default=None, help="label ids for the tile panels (default: 4 test tiles with most flood)")
    ap.add_argument("--skip-overview", action="store_true")
    a = ap.parse_args()
    out = Path(a.out)
    out.mkdir(parents=True, exist_ok=True)
    fig_tiles(a, out)
    if not a.skip_overview:
        fig_overview(a, out)


if __name__ == "__main__":
    main()