File size: 3,250 Bytes
17285c6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
"""Bootstrap 95% confidence intervals for flood IoU from per-tile confusion matrices.

Usage:
  .venv/bin/python pakflood/bootstrap.py <name>=<per_tile.csv> [<name>=<per_tile.csv> ...] [--unit tile|event] [--split test] [--n 2000]

Each per_tile.csv comes from pakflood/predict_tiles.py (same tiles for every model). Resamples units (tiles or
events/AOIs) with replacement, recomputes flood IoU from the summed confusion matrices, and reports the 2.5-97.5
percentile interval per model and for every pairwise difference (paired bootstrap: same resample for all models).
"""
import argparse

import numpy as np
import pandas as pd


def iou_from(df: pd.DataFrame, cls: int = 2) -> float:
    cm = np.array([[df[f"cm_{i}{j}"].sum() for j in range(3)] for i in range(3)], dtype=np.float64)
    tp = cm[cls, cls]
    fp = cm[:, cls].sum() - tp
    fn = cm[cls, :].sum() - tp
    return tp / max(tp + fp + fn, 1)


def main() -> None:
    ap = argparse.ArgumentParser()
    ap.add_argument("models", nargs="+", help="name=path/to/per_tile.csv")
    ap.add_argument("--unit", default="event", choices=["tile", "event"])
    ap.add_argument("--split", default=None)
    ap.add_argument("--n", type=int, default=2000)
    ap.add_argument("--seed", type=int, default=0)
    a = ap.parse_args()

    tables = {}
    for m in a.models:
        name, path = m.split("=", 1)
        d = pd.read_csv(path)
        if a.split:
            d = d[d.split == a.split]
        d = d.sort_values("label_id").reset_index(drop=True)
        d["unit"] = d.label_id if a.unit == "tile" else d.event.str.extract(r"(EMSR\d+)")[0]
        tables[name] = d
    names = list(tables)
    ref = tables[names[0]]
    for n in names[1:]:
        assert list(tables[n].label_id) == list(ref.label_id), f"{n}: tile set differs from {names[0]}"
    units = sorted(ref.unit.unique())
    print(f"{len(ref)} tiles, {len(units)} {a.unit}s, {a.n} resamples")

    rng = np.random.default_rng(a.seed)
    groups = {n: {u: g for u, g in t.groupby("unit")} for n, t in tables.items()}
    cols = [f"cm_{i}{j}" for i in range(3) for j in range(3)]
    sums = {n: {u: g[cols].sum().values.astype(np.float64) for u, g in groups[n].items()} for n in names}

    def iou_vec(cmv: np.ndarray) -> float:
        cm = cmv.reshape(3, 3)
        tp = cm[2, 2]
        return tp / max(cm[:, 2].sum() + cm[2, :].sum() - tp, 1)

    boots = {n: np.empty(a.n) for n in names}
    for b in range(a.n):
        pick = rng.choice(units, size=len(units), replace=True)
        for n in names:
            boots[n][b] = iou_vec(sum(sums[n][u] for u in pick))
    print("\nflood IoU with 95% bootstrap CI")
    for n in names:
        lo, hi = np.percentile(boots[n], [2.5, 97.5])
        print(f"  {n:24s} {iou_from(tables[n]):.4f}  [{lo:.4f}, {hi:.4f}]")
    if len(names) > 1:
        print("\npairwise differences (row minus column), 95% CI; interval excluding 0 = significant")
        for i, n1 in enumerate(names):
            for n2 in names[i + 1 :]:
                d = boots[n1] - boots[n2]
                lo, hi = np.percentile(d, [2.5, 97.5])
                print(f"  {n1} - {n2}: {iou_from(tables[n1]) - iou_from(tables[n2]):+.4f}  [{lo:+.4f}, {hi:+.4f}]")


if __name__ == "__main__":
    main()