Spaces:
Sleeping
Sleeping
Commit ·
99e1f27
1
Parent(s): 995884c
Improve detection accuracy with full-res tiling, config flags, and evaluation.
Browse filesCentralize detection tuning in detection_config, add windowed GeoTIFF inference
for large rasters, optional hysteresis fusion and multi-scale DL, synthetic
benchmark metrics, and local stability fixes for rescan and job queuing.
- .env.example +27 -0
- DEPLOYMENT.md +20 -0
- app/cd_models/change_model.py +12 -43
- app/cd_models/model_utils.py +89 -0
- app/dda/bootstrap.py +61 -3
- app/dda/config.py +2 -6
- app/dda/detect_service.py +25 -0
- app/dda/geotiff_io.py +73 -0
- app/dda/job_runner.py +4 -1
- app/dda/local_routes.py +10 -3
- app/dda/tree/sync_service.py +36 -4
- app/detection_config.py +225 -0
- app/detection_engine.py +186 -25
- app/evaluation/__init__.py +4 -0
- app/evaluation/metrics.py +79 -0
- app/model_inference.py +155 -52
- scripts/validate_detection.py +116 -2
- static/js/dda/compare.js +2 -1
- templates/index_dda.html +1 -1
.env.example
CHANGED
|
@@ -17,6 +17,33 @@
|
|
| 17 |
# Max pixel dimension for detection (lower = less RAM, default 4096 local)
|
| 18 |
# DETECTION_MAX_SIDE=4096
|
| 19 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 20 |
# Public URL for report links in emails (default http://localhost:8000 locally)
|
| 21 |
# PUBLIC_BASE_URL=http://localhost:8000
|
| 22 |
|
|
|
|
| 17 |
# Max pixel dimension for detection (lower = less RAM, default 4096 local)
|
| 18 |
# DETECTION_MAX_SIDE=4096
|
| 19 |
|
| 20 |
+
# --- Accuracy controls (all optional; defaults preserve current behavior) ---
|
| 21 |
+
# Inference mode: downscaled (default) or fullres_tiled (native detail for large GeoTIFFs)
|
| 22 |
+
# DETECTION_INFERENCE_MODE=downscaled
|
| 23 |
+
# Full-res cap (px) in fullres_tiled mode; 0 = native resolution
|
| 24 |
+
# DETECTION_FULLRES_MAX_SIDE=8192
|
| 25 |
+
# Native side (px) above which GeoTIFFs stream from disk windows (avoids OOM)
|
| 26 |
+
# DETECTION_WINDOWED_THRESHOLD=8192
|
| 27 |
+
# Soft RAM budget (MB) per in-memory array before switching to windowed streaming
|
| 28 |
+
# DETECTION_TILE_MEMORY_MB=1536
|
| 29 |
+
# Tile size / overlap for full-res scoring
|
| 30 |
+
# DETECTION_TILE_SIZE=512
|
| 31 |
+
# DETECTION_TILE_OVERLAP=0.25
|
| 32 |
+
# Multi-scale DL fusion: off or a comma list, e.g. 0.5,1.0,1.5
|
| 33 |
+
# DETECTION_MULTISCALE=off
|
| 34 |
+
# Fusion strategy: smart_union (default) or hysteresis
|
| 35 |
+
# DETECTION_FUSION=smart_union
|
| 36 |
+
# Preprocessing toggles
|
| 37 |
+
# DETECTION_CLAHE=true
|
| 38 |
+
# DETECTION_HIST_MATCH=false
|
| 39 |
+
# DETECTION_SKIP_PREBLUR= # auto: on in fullres_tiled mode
|
| 40 |
+
# Border pixels zeroed in mask cleanup (auto: 4 fullres / 12 downscaled)
|
| 41 |
+
# DETECTION_BORDER_MARGIN=
|
| 42 |
+
# Tiles per GPU forward pass (1 = no batching)
|
| 43 |
+
# DETECTION_TILE_BATCH=1
|
| 44 |
+
# Save a downsampled probability-map PNG per run for debugging
|
| 45 |
+
# DETECTION_SAVE_PROB_MAP=false
|
| 46 |
+
|
| 47 |
# Public URL for report links in emails (default http://localhost:8000 locally)
|
| 48 |
# PUBLIC_BASE_URL=http://localhost:8000
|
| 49 |
|
DEPLOYMENT.md
CHANGED
|
@@ -145,6 +145,26 @@ Set these in each Space’s **Settings → Repository secrets / Variables** if n
|
|
| 145 |
| `DDA_ADMIN_EMAIL` / `DDA_ADMIN_PASSWORD` | Seed admin user on dev (role: admin) |
|
| 146 |
| `DDA_TRAINING_EXPORT_KEY` | Header `X-DDA-Training-Key` for false-positive export |
|
| 147 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 148 |
Dev Space can omit `SECRET_KEY` (login is disabled on both Spaces).
|
| 149 |
|
| 150 |
---
|
|
|
|
| 145 |
| `DDA_ADMIN_EMAIL` / `DDA_ADMIN_PASSWORD` | Seed admin user on dev (role: admin) |
|
| 146 |
| `DDA_TRAINING_EXPORT_KEY` | Header `X-DDA-Training-Key` for false-positive export |
|
| 147 |
|
| 148 |
+
### Detection accuracy controls (optional, dev-first)
|
| 149 |
+
|
| 150 |
+
All default to the previous behavior, so leaving them unset changes nothing.
|
| 151 |
+
|
| 152 |
+
| Variable | Purpose |
|
| 153 |
+
|----------|---------|
|
| 154 |
+
| `DETECTION_MAX_SIDE` | Downscaled-path pixel cap (default 4096 local / 2048 hosted) |
|
| 155 |
+
| `DETECTION_INFERENCE_MODE` | `downscaled` (default) or `fullres_tiled` (native detail) |
|
| 156 |
+
| `DETECTION_FULLRES_MAX_SIDE` | Full-res cap in fullres mode; 0 = native (default 8192) |
|
| 157 |
+
| `DETECTION_WINDOWED_THRESHOLD` | Native side above which GeoTIFFs stream from disk windows (default 8192) |
|
| 158 |
+
| `DETECTION_TILE_MEMORY_MB` | RAM budget per array before switching to windowed streaming (default 1536) |
|
| 159 |
+
| `DETECTION_TILE_SIZE` / `DETECTION_TILE_OVERLAP` | Full-res tile geometry (default 512 / 0.25) |
|
| 160 |
+
| `DETECTION_MULTISCALE` | `off` or scale list e.g. `0.5,1.0,1.5` (max-fused for recall) |
|
| 161 |
+
| `DETECTION_FUSION` | `smart_union` (default) or `hysteresis` |
|
| 162 |
+
| `DETECTION_CLAHE` / `DETECTION_HIST_MATCH` | Preprocessing toggles (CLAHE on, hist-match off) |
|
| 163 |
+
| `DETECTION_SKIP_PREBLUR` | Skip denoise (auto-on in fullres mode) |
|
| 164 |
+
| `DETECTION_BORDER_MARGIN` | Mask border zeroing (auto: 4 fullres / 12 downscaled) |
|
| 165 |
+
| `DETECTION_TILE_BATCH` | Tiles per GPU forward pass (default 1) |
|
| 166 |
+
| `DETECTION_SAVE_PROB_MAP` | Save a per-run probability PNG for debugging (default off) |
|
| 167 |
+
|
| 168 |
Dev Space can omit `SECRET_KEY` (login is disabled on both Spaces).
|
| 169 |
|
| 170 |
---
|
app/cd_models/change_model.py
CHANGED
|
@@ -189,54 +189,23 @@ def predict_siamese(img1, img2, threshold=0.5):
|
|
| 189 |
if img1.shape != img2.shape:
|
| 190 |
img2 = cv2.resize(img2, (img1.shape[1], img1.shape[0]))
|
| 191 |
|
| 192 |
-
|
| 193 |
-
tile = _TILE
|
| 194 |
-
overlap = tile // 4
|
| 195 |
-
stride = tile - overlap
|
| 196 |
-
|
| 197 |
-
pad_h = (tile - h % tile) % tile
|
| 198 |
-
pad_w = (tile - w % tile) % tile
|
| 199 |
-
if pad_h or pad_w:
|
| 200 |
-
img1 = np.pad(img1, ((0, pad_h), (0, pad_w), (0, 0)), mode="reflect")
|
| 201 |
-
img2 = np.pad(img2, ((0, pad_h), (0, pad_w), (0, 0)), mode="reflect")
|
| 202 |
-
|
| 203 |
-
ph, pw = img1.shape[:2]
|
| 204 |
-
score_sum = np.zeros((ph, pw), dtype=np.float32)
|
| 205 |
-
count = np.zeros((ph, pw), dtype=np.float32)
|
| 206 |
-
|
| 207 |
-
ramp = np.linspace(0, 1, overlap)
|
| 208 |
-
flat = np.ones(tile - 2 * overlap)
|
| 209 |
-
profile = np.concatenate([ramp, flat, ramp[::-1]])
|
| 210 |
-
weight_2d = np.outer(profile, profile).astype(np.float32)
|
| 211 |
|
| 212 |
mean = np.array([0.485, 0.456, 0.406], dtype=np.float32)
|
| 213 |
std = np.array([0.229, 0.224, 0.225], dtype=np.float32)
|
| 214 |
|
| 215 |
-
|
| 216 |
-
|
| 217 |
-
|
| 218 |
-
|
| 219 |
-
|
| 220 |
-
|
| 221 |
-
|
| 222 |
-
|
| 223 |
-
|
| 224 |
-
ta = torch.from_numpy(t1.transpose(2, 0, 1)).unsqueeze(0).to(_DEVICE)
|
| 225 |
-
tb = torch.from_numpy(t2.transpose(2, 0, 1)).unsqueeze(0).to(_DEVICE)
|
| 226 |
-
|
| 227 |
-
logits = model(ta, tb)
|
| 228 |
-
probs = torch.softmax(logits, dim=1)
|
| 229 |
-
prob_map = probs[0, 1].cpu().numpy()
|
| 230 |
|
| 231 |
-
|
| 232 |
-
|
| 233 |
-
|
| 234 |
-
score_sum[y0:y0+tile, x0:x0+tile] += prob_map * weight_2d
|
| 235 |
-
count[y0:y0+tile, x0:x0+tile] += weight_2d
|
| 236 |
-
|
| 237 |
-
count = np.maximum(count, 1e-6)
|
| 238 |
-
avg = score_sum / count
|
| 239 |
-
avg = avg[:h, :w]
|
| 240 |
|
| 241 |
mask = (avg >= threshold).astype(np.uint8) * 255
|
| 242 |
return mask, avg
|
|
|
|
| 189 |
if img1.shape != img2.shape:
|
| 190 |
img2 = cv2.resize(img2, (img1.shape[1], img1.shape[0]))
|
| 191 |
|
| 192 |
+
from .model_utils import tiled_score_map
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 193 |
|
| 194 |
mean = np.array([0.485, 0.456, 0.406], dtype=np.float32)
|
| 195 |
std = np.array([0.229, 0.224, 0.225], dtype=np.float32)
|
| 196 |
|
| 197 |
+
def _score_tile(t1, t2):
|
| 198 |
+
n1 = (t1.astype(np.float32) / 255.0 - mean) / std
|
| 199 |
+
n2 = (t2.astype(np.float32) / 255.0 - mean) / std
|
| 200 |
+
ta = torch.from_numpy(n1.transpose(2, 0, 1)).unsqueeze(0).to(_DEVICE)
|
| 201 |
+
tb = torch.from_numpy(n2.transpose(2, 0, 1)).unsqueeze(0).to(_DEVICE)
|
| 202 |
+
logits = model(ta, tb)
|
| 203 |
+
probs = torch.softmax(logits, dim=1)
|
| 204 |
+
return probs[0, 1].cpu().numpy()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 205 |
|
| 206 |
+
with torch.no_grad():
|
| 207 |
+
avg = tiled_score_map(_score_tile, img1, img2,
|
| 208 |
+
tile_size=_TILE, overlap=_TILE // 4)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 209 |
|
| 210 |
mask = (avg >= threshold).astype(np.uint8) * 255
|
| 211 |
return mask, avg
|
app/cd_models/model_utils.py
CHANGED
|
@@ -5,6 +5,95 @@ import cv2
|
|
| 5 |
import numpy as np
|
| 6 |
|
| 7 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8 |
def split_into_tiles(img, tile_size=512, overlap=64):
|
| 9 |
"""
|
| 10 |
Split an image into overlapping tiles.
|
|
|
|
| 5 |
import numpy as np
|
| 6 |
|
| 7 |
|
| 8 |
+
def make_blend_weights(tile_size: int, overlap: int) -> np.ndarray:
|
| 9 |
+
"""Raised-cosine (trapezoid) 2D weight for seamless tile stitching.
|
| 10 |
+
|
| 11 |
+
Ramps from 0 to 1 across the overlap band on each edge and stays flat in
|
| 12 |
+
the interior, so overlapping tile predictions blend without visible seams.
|
| 13 |
+
"""
|
| 14 |
+
overlap = max(0, min(overlap, tile_size // 2))
|
| 15 |
+
if overlap == 0:
|
| 16 |
+
return np.ones((tile_size, tile_size), dtype=np.float32)
|
| 17 |
+
ramp = np.linspace(0.0, 1.0, overlap, dtype=np.float32)
|
| 18 |
+
flat = np.ones(tile_size - 2 * overlap, dtype=np.float32)
|
| 19 |
+
profile = np.concatenate([ramp, flat, ramp[::-1]])
|
| 20 |
+
return np.outer(profile, profile).astype(np.float32)
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def tiled_score_map(score_tile_fn, img1, img2, tile_size: int = 256,
|
| 24 |
+
overlap: int | None = None,
|
| 25 |
+
score_batch_fn=None, batch: int = 1):
|
| 26 |
+
"""Sliding-window tiled scoring with reflect padding + cosine blending.
|
| 27 |
+
|
| 28 |
+
``score_tile_fn(tile1, tile2)`` must return a float32 array in [0, 1] the
|
| 29 |
+
same height/width as the input tile. Returns a float32 change-probability
|
| 30 |
+
map cropped back to the original (h, w). This is the single shared
|
| 31 |
+
implementation used by every deep model so blending stays consistent.
|
| 32 |
+
|
| 33 |
+
When ``batch > 1`` and ``score_batch_fn`` is provided, tiles are scored in
|
| 34 |
+
groups (``score_batch_fn(list[(t1, t2)]) -> list[prob]``) so GPUs can run
|
| 35 |
+
several tiles per forward pass.
|
| 36 |
+
"""
|
| 37 |
+
h, w = img1.shape[:2]
|
| 38 |
+
if overlap is None:
|
| 39 |
+
overlap = tile_size // 4
|
| 40 |
+
overlap = max(0, min(overlap, tile_size // 2))
|
| 41 |
+
stride = max(1, tile_size - overlap)
|
| 42 |
+
|
| 43 |
+
pad_h = (tile_size - h % tile_size) % tile_size
|
| 44 |
+
pad_w = (tile_size - w % tile_size) % tile_size
|
| 45 |
+
# Guarantee at least one full tile even for small inputs
|
| 46 |
+
pad_h = max(pad_h, max(0, tile_size - h))
|
| 47 |
+
pad_w = max(pad_w, max(0, tile_size - w))
|
| 48 |
+
if pad_h or pad_w:
|
| 49 |
+
img1 = np.pad(img1, ((0, pad_h), (0, pad_w), (0, 0)), mode="reflect")
|
| 50 |
+
img2 = np.pad(img2, ((0, pad_h), (0, pad_w), (0, 0)), mode="reflect")
|
| 51 |
+
|
| 52 |
+
ph, pw = img1.shape[:2]
|
| 53 |
+
score_sum = np.zeros((ph, pw), dtype=np.float32)
|
| 54 |
+
count = np.zeros((ph, pw), dtype=np.float32)
|
| 55 |
+
weight_2d = make_blend_weights(tile_size, overlap)
|
| 56 |
+
|
| 57 |
+
ys = list(range(0, ph - tile_size + 1, stride))
|
| 58 |
+
xs = list(range(0, pw - tile_size + 1, stride))
|
| 59 |
+
# Ensure the far edges are always covered
|
| 60 |
+
if ys and ys[-1] != ph - tile_size:
|
| 61 |
+
ys.append(ph - tile_size)
|
| 62 |
+
if xs and xs[-1] != pw - tile_size:
|
| 63 |
+
xs.append(pw - tile_size)
|
| 64 |
+
|
| 65 |
+
coords = [(y0, x0) for y0 in ys for x0 in xs]
|
| 66 |
+
|
| 67 |
+
def _accumulate(y0, x0, prob):
|
| 68 |
+
if prob.shape != (tile_size, tile_size):
|
| 69 |
+
prob = cv2.resize(prob.astype(np.float32), (tile_size, tile_size),
|
| 70 |
+
interpolation=cv2.INTER_LINEAR)
|
| 71 |
+
score_sum[y0:y0 + tile_size, x0:x0 + tile_size] += prob * weight_2d
|
| 72 |
+
count[y0:y0 + tile_size, x0:x0 + tile_size] += weight_2d
|
| 73 |
+
|
| 74 |
+
use_batch = score_batch_fn is not None and batch > 1
|
| 75 |
+
if use_batch:
|
| 76 |
+
for i in range(0, len(coords), batch):
|
| 77 |
+
chunk = coords[i:i + batch]
|
| 78 |
+
pairs = [
|
| 79 |
+
(np.ascontiguousarray(img1[y:y + tile_size, x:x + tile_size]),
|
| 80 |
+
np.ascontiguousarray(img2[y:y + tile_size, x:x + tile_size]))
|
| 81 |
+
for (y, x) in chunk
|
| 82 |
+
]
|
| 83 |
+
probs = score_batch_fn(pairs)
|
| 84 |
+
for (y0, x0), prob in zip(chunk, probs):
|
| 85 |
+
_accumulate(y0, x0, prob)
|
| 86 |
+
else:
|
| 87 |
+
for (y0, x0) in coords:
|
| 88 |
+
t1 = np.ascontiguousarray(img1[y0:y0 + tile_size, x0:x0 + tile_size])
|
| 89 |
+
t2 = np.ascontiguousarray(img2[y0:y0 + tile_size, x0:x0 + tile_size])
|
| 90 |
+
_accumulate(y0, x0, score_tile_fn(t1, t2))
|
| 91 |
+
|
| 92 |
+
count = np.maximum(count, 1e-6)
|
| 93 |
+
avg = score_sum / count
|
| 94 |
+
return avg[:h, :w]
|
| 95 |
+
|
| 96 |
+
|
| 97 |
def split_into_tiles(img, tile_size=512, overlap=64):
|
| 98 |
"""
|
| 99 |
Split an image into overlapping tiles.
|
app/dda/bootstrap.py
CHANGED
|
@@ -18,6 +18,60 @@ from .dda_auth import seed_dda_admin
|
|
| 18 |
logger = logging.getLogger(__name__)
|
| 19 |
|
| 20 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 21 |
def init_dda_database():
|
| 22 |
"""Run DDA-specific startup tasks (dirs, seed, migrations)."""
|
| 23 |
if not IS_DDA_MODE:
|
|
@@ -43,6 +97,7 @@ def init_dda_database():
|
|
| 43 |
conn.commit()
|
| 44 |
except Exception:
|
| 45 |
conn.rollback()
|
|
|
|
| 46 |
except Exception as exc:
|
| 47 |
logger.warning("DDA schema migration skipped: %s", exc)
|
| 48 |
|
|
@@ -63,9 +118,12 @@ def init_dda_database():
|
|
| 63 |
from .tree.migration import run_tree_migration
|
| 64 |
mig = run_tree_migration(db)
|
| 65 |
logger.info("Tree migration: %s", mig)
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
|
|
|
|
|
|
|
|
|
| 69 |
from .job_runner import reconcile_stale_jobs
|
| 70 |
reconcile_stale_jobs(db)
|
| 71 |
finally:
|
|
|
|
| 18 |
logger = logging.getLogger(__name__)
|
| 19 |
|
| 20 |
|
| 21 |
+
def _migrate_detection_jobs_nullable() -> None:
|
| 22 |
+
"""SQLite legacy schema required image asset FKs; tree library jobs use paths only."""
|
| 23 |
+
try:
|
| 24 |
+
with engine.connect() as conn:
|
| 25 |
+
rows = conn.execute(sa_text("PRAGMA table_info(dda_detection_jobs)")).fetchall()
|
| 26 |
+
if not rows:
|
| 27 |
+
return
|
| 28 |
+
by_name = {r[1]: r for r in rows}
|
| 29 |
+
base_col = by_name.get("base_image_id")
|
| 30 |
+
if not base_col or base_col[3] == 0:
|
| 31 |
+
return
|
| 32 |
+
logger.info("Migrating dda_detection_jobs to allow NULL image IDs for path-based jobs")
|
| 33 |
+
conn.execute(sa_text("PRAGMA foreign_keys=OFF"))
|
| 34 |
+
conn.execute(sa_text("""
|
| 35 |
+
CREATE TABLE dda_detection_jobs_new (
|
| 36 |
+
id INTEGER NOT NULL PRIMARY KEY,
|
| 37 |
+
status VARCHAR(32),
|
| 38 |
+
base_image_id INTEGER,
|
| 39 |
+
comparison_image_id INTEGER,
|
| 40 |
+
method VARCHAR(64),
|
| 41 |
+
params_json TEXT,
|
| 42 |
+
run_id INTEGER,
|
| 43 |
+
error_message TEXT,
|
| 44 |
+
notify_email VARCHAR(255),
|
| 45 |
+
created_by INTEGER,
|
| 46 |
+
started_at DATETIME,
|
| 47 |
+
completed_at DATETIME,
|
| 48 |
+
created_at DATETIME,
|
| 49 |
+
FOREIGN KEY(base_image_id) REFERENCES dda_image_assets (id),
|
| 50 |
+
FOREIGN KEY(comparison_image_id) REFERENCES dda_image_assets (id),
|
| 51 |
+
FOREIGN KEY(run_id) REFERENCES detection_runs (id),
|
| 52 |
+
FOREIGN KEY(created_by) REFERENCES users (id)
|
| 53 |
+
)
|
| 54 |
+
"""))
|
| 55 |
+
conn.execute(sa_text("""
|
| 56 |
+
INSERT INTO dda_detection_jobs_new (
|
| 57 |
+
id, status, base_image_id, comparison_image_id, method, params_json,
|
| 58 |
+
run_id, error_message, notify_email, created_by,
|
| 59 |
+
started_at, completed_at, created_at
|
| 60 |
+
)
|
| 61 |
+
SELECT
|
| 62 |
+
id, status, base_image_id, comparison_image_id, method, params_json,
|
| 63 |
+
run_id, error_message, notify_email, created_by,
|
| 64 |
+
started_at, completed_at, created_at
|
| 65 |
+
FROM dda_detection_jobs
|
| 66 |
+
"""))
|
| 67 |
+
conn.execute(sa_text("DROP TABLE dda_detection_jobs"))
|
| 68 |
+
conn.execute(sa_text("ALTER TABLE dda_detection_jobs_new RENAME TO dda_detection_jobs"))
|
| 69 |
+
conn.execute(sa_text("PRAGMA foreign_keys=ON"))
|
| 70 |
+
conn.commit()
|
| 71 |
+
except Exception as exc:
|
| 72 |
+
logger.warning("Detection jobs nullable migration skipped: %s", exc)
|
| 73 |
+
|
| 74 |
+
|
| 75 |
def init_dda_database():
|
| 76 |
"""Run DDA-specific startup tasks (dirs, seed, migrations)."""
|
| 77 |
if not IS_DDA_MODE:
|
|
|
|
| 97 |
conn.commit()
|
| 98 |
except Exception:
|
| 99 |
conn.rollback()
|
| 100 |
+
_migrate_detection_jobs_nullable()
|
| 101 |
except Exception as exc:
|
| 102 |
logger.warning("DDA schema migration skipped: %s", exc)
|
| 103 |
|
|
|
|
| 118 |
from .tree.migration import run_tree_migration
|
| 119 |
mig = run_tree_migration(db)
|
| 120 |
logger.info("Tree migration: %s", mig)
|
| 121 |
+
try:
|
| 122 |
+
from .tree.sync_service import sync_from_filesystem
|
| 123 |
+
sync_stats = sync_from_filesystem(db)
|
| 124 |
+
logger.info("Filesystem sync at startup: %s", sync_stats)
|
| 125 |
+
except Exception as exc:
|
| 126 |
+
logger.warning("Filesystem sync at startup failed: %s", exc)
|
| 127 |
from .job_runner import reconcile_stale_jobs
|
| 128 |
reconcile_stale_jobs(db)
|
| 129 |
finally:
|
app/dda/config.py
CHANGED
|
@@ -134,9 +134,5 @@ def geotiff_io_available() -> bool:
|
|
| 134 |
|
| 135 |
def get_detection_max_side() -> int:
|
| 136 |
"""Max pixel dimension for GeoTIFF load + detection pipeline (higher = sharper, more RAM)."""
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
value = int(os.environ.get("DETECTION_MAX_SIDE", default))
|
| 140 |
-
except ValueError:
|
| 141 |
-
value = int(default)
|
| 142 |
-
return max(1024, min(8192, value))
|
|
|
|
| 134 |
|
| 135 |
def get_detection_max_side() -> int:
|
| 136 |
"""Max pixel dimension for GeoTIFF load + detection pipeline (higher = sharper, more RAM)."""
|
| 137 |
+
from ..detection_config import get_detection_max_side as _central
|
| 138 |
+
return _central()
|
|
|
|
|
|
|
|
|
|
|
|
app/dda/detect_service.py
CHANGED
|
@@ -79,6 +79,7 @@ def run_detection_and_save(
|
|
| 79 |
notify_email: Optional[str] = None,
|
| 80 |
max_size: Optional[int] = None,
|
| 81 |
geo_bounds_path: Optional[Path] = None,
|
|
|
|
| 82 |
base_path: str = "",
|
| 83 |
user_id: Optional[int] = None,
|
| 84 |
job_id: Optional[int] = None,
|
|
@@ -107,6 +108,12 @@ def run_detection_and_save(
|
|
| 107 |
_report(job_pct, stage)
|
| 108 |
|
| 109 |
_report(15, "Running detection")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 110 |
change_mask, result_image, stats, change_regions = run_detection(
|
| 111 |
before_pil,
|
| 112 |
after_pil,
|
|
@@ -117,6 +124,8 @@ def run_detection_and_save(
|
|
| 117 |
min_region_area=min_region_area,
|
| 118 |
max_size=max_size,
|
| 119 |
on_progress=_on_engine_progress,
|
|
|
|
|
|
|
| 120 |
)
|
| 121 |
|
| 122 |
_report(80, "Saving results")
|
|
@@ -133,6 +142,21 @@ def run_detection_and_save(
|
|
| 133 |
Image.fromarray(result_image).save(overlay_path)
|
| 134 |
relative_overlay = f"overlays/{overlay_filename}"
|
| 135 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 136 |
relative_before_full = ""
|
| 137 |
relative_before_thumb = ""
|
| 138 |
relative_after_thumb = ""
|
|
@@ -261,6 +285,7 @@ def run_detection_and_save(
|
|
| 261 |
"alignmentWarning": stats.get("alignment_warning"),
|
| 262 |
"registrationOk": stats.get("params", {}).get("registration_ok"),
|
| 263 |
"geo": geo_debug,
|
|
|
|
| 264 |
},
|
| 265 |
"regions": regions_serializable,
|
| 266 |
"overlayBase64Png": overlay_b64,
|
|
|
|
| 79 |
notify_email: Optional[str] = None,
|
| 80 |
max_size: Optional[int] = None,
|
| 81 |
geo_bounds_path: Optional[Path] = None,
|
| 82 |
+
comparison_file: Optional[Path] = None,
|
| 83 |
base_path: str = "",
|
| 84 |
user_id: Optional[int] = None,
|
| 85 |
job_id: Optional[int] = None,
|
|
|
|
| 108 |
_report(job_pct, stage)
|
| 109 |
|
| 110 |
_report(15, "Running detection")
|
| 111 |
+
|
| 112 |
+
def _geotiff_path(p: Optional[Path]) -> Optional[str]:
|
| 113 |
+
if p is not None and str(p).lower().endswith((".tif", ".tiff")):
|
| 114 |
+
return str(p)
|
| 115 |
+
return None
|
| 116 |
+
|
| 117 |
change_mask, result_image, stats, change_regions = run_detection(
|
| 118 |
before_pil,
|
| 119 |
after_pil,
|
|
|
|
| 124 |
min_region_area=min_region_area,
|
| 125 |
max_size=max_size,
|
| 126 |
on_progress=_on_engine_progress,
|
| 127 |
+
before_path=_geotiff_path(geo_bounds_path),
|
| 128 |
+
after_path=_geotiff_path(comparison_file),
|
| 129 |
)
|
| 130 |
|
| 131 |
_report(80, "Saving results")
|
|
|
|
| 142 |
Image.fromarray(result_image).save(overlay_path)
|
| 143 |
relative_overlay = f"overlays/{overlay_filename}"
|
| 144 |
|
| 145 |
+
relative_prob = ""
|
| 146 |
+
try:
|
| 147 |
+
from ..detection_config import get_save_prob_map
|
| 148 |
+
score_map = stats.pop("_score_map", None)
|
| 149 |
+
if get_save_prob_map() and score_map is not None:
|
| 150 |
+
import numpy as _np
|
| 151 |
+
prob_u8 = _np.clip(score_map * 255.0, 0, 255).astype("uint8")
|
| 152 |
+
prob_img = Image.fromarray(prob_u8)
|
| 153 |
+
prob_img.thumbnail((1024, 1024), Image.Resampling.LANCZOS)
|
| 154 |
+
prob_file = OVERLAYS_DIR / f"{base_name}_prob.png"
|
| 155 |
+
prob_img.save(prob_file)
|
| 156 |
+
relative_prob = f"overlays/{base_name}_prob.png"
|
| 157 |
+
except Exception as exc:
|
| 158 |
+
logger.warning("Probability map export failed: %s", exc)
|
| 159 |
+
|
| 160 |
relative_before_full = ""
|
| 161 |
relative_before_thumb = ""
|
| 162 |
relative_after_thumb = ""
|
|
|
|
| 285 |
"alignmentWarning": stats.get("alignment_warning"),
|
| 286 |
"registrationOk": stats.get("params", {}).get("registration_ok"),
|
| 287 |
"geo": geo_debug,
|
| 288 |
+
"probabilityMapUrl": f"/api/overlay/{relative_prob}" if relative_prob else None,
|
| 289 |
},
|
| 290 |
"regions": regions_serializable,
|
| 291 |
"overlayBase64Png": overlay_b64,
|
app/dda/geotiff_io.py
CHANGED
|
@@ -210,6 +210,79 @@ def _rasterio_read_rgb(path: Path, max_side: int):
|
|
| 210 |
return Image.fromarray(rgb.astype("uint8"), mode="RGB")
|
| 211 |
|
| 212 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 213 |
def load_rgb_pil(path: Path, max_side: Optional[int] = None) -> Image.Image:
|
| 214 |
"""Load image as RGB PIL, downscaling large GeoTIFFs via rasterio."""
|
| 215 |
if max_side is None:
|
|
|
|
| 210 |
return Image.fromarray(rgb.astype("uint8"), mode="RGB")
|
| 211 |
|
| 212 |
|
| 213 |
+
def read_native_size(path: Path) -> Optional[Tuple[int, int]]:
|
| 214 |
+
"""Return (width, height) of a GeoTIFF without decoding pixels, or None."""
|
| 215 |
+
ext = path.suffix.lower()
|
| 216 |
+
if ext not in (".tif", ".tiff"):
|
| 217 |
+
return None
|
| 218 |
+
try:
|
| 219 |
+
import rasterio
|
| 220 |
+
with rasterio.open(path) as src:
|
| 221 |
+
return int(src.width), int(src.height)
|
| 222 |
+
except Exception:
|
| 223 |
+
return None
|
| 224 |
+
|
| 225 |
+
|
| 226 |
+
def _normalize_window_rgb(data):
|
| 227 |
+
"""Convert a rasterio (bands, h, w) window read to RGB uint8 (h, w, 3)."""
|
| 228 |
+
import numpy as np
|
| 229 |
+
count = data.shape[0]
|
| 230 |
+
if count == 1:
|
| 231 |
+
rgb = np.stack([data[0], data[0], data[0]])
|
| 232 |
+
else:
|
| 233 |
+
rgb = data[:3]
|
| 234 |
+
rgb = np.transpose(rgb, (1, 2, 0)).astype("float32")
|
| 235 |
+
if rgb.max() > 255 or rgb.min() < 0:
|
| 236 |
+
lo, hi = np.percentile(rgb, (2, 98))
|
| 237 |
+
rgb = np.clip((rgb - lo) / max(hi - lo, 1e-6), 0, 1) * 255
|
| 238 |
+
return rgb.astype("uint8")
|
| 239 |
+
|
| 240 |
+
|
| 241 |
+
def iter_geotiff_window_pairs(path_a: Path, path_b: Path,
|
| 242 |
+
tile_size: int = 512, overlap: float = 0.25):
|
| 243 |
+
"""Stream paired native-resolution RGB windows from two same-size GeoTIFFs.
|
| 244 |
+
|
| 245 |
+
Reads corresponding pixel windows from disk via rasterio so very large
|
| 246 |
+
rasters never load fully into RAM. Yields
|
| 247 |
+
``(tile_a, tile_b, y0, x0, full_h, full_w)`` where tiles are RGB uint8.
|
| 248 |
+
Requires both rasters to share native pixel dimensions; raises ValueError
|
| 249 |
+
otherwise so the caller can fall back to the in-memory path.
|
| 250 |
+
"""
|
| 251 |
+
import rasterio
|
| 252 |
+
from rasterio.windows import Window
|
| 253 |
+
|
| 254 |
+
tile_size = max(64, int(tile_size))
|
| 255 |
+
ov = min(0.5, max(0.0, float(overlap)))
|
| 256 |
+
step = max(1, int(round(tile_size * (1.0 - ov))))
|
| 257 |
+
|
| 258 |
+
with rasterio.open(path_a) as src_a, rasterio.open(path_b) as src_b:
|
| 259 |
+
if (src_a.width, src_a.height) != (src_b.width, src_b.height):
|
| 260 |
+
raise ValueError(
|
| 261 |
+
f"GeoTIFF dimensions differ: {src_a.width}x{src_a.height} "
|
| 262 |
+
f"vs {src_b.width}x{src_b.height}"
|
| 263 |
+
)
|
| 264 |
+
full_w, full_h = int(src_a.width), int(src_a.height)
|
| 265 |
+
bands_a = list(range(1, min(3, src_a.count) + 1))
|
| 266 |
+
bands_b = list(range(1, min(3, src_b.count) + 1))
|
| 267 |
+
|
| 268 |
+
ys = list(range(0, max(1, full_h - tile_size + 1), step))
|
| 269 |
+
xs = list(range(0, max(1, full_w - tile_size + 1), step))
|
| 270 |
+
if ys[-1] != full_h - tile_size and full_h > tile_size:
|
| 271 |
+
ys.append(full_h - tile_size)
|
| 272 |
+
if xs[-1] != full_w - tile_size and full_w > tile_size:
|
| 273 |
+
xs.append(full_w - tile_size)
|
| 274 |
+
|
| 275 |
+
for y0 in ys:
|
| 276 |
+
for x0 in xs:
|
| 277 |
+
wh = min(tile_size, full_h - y0)
|
| 278 |
+
ww = min(tile_size, full_w - x0)
|
| 279 |
+
win = Window(x0, y0, ww, wh)
|
| 280 |
+
da = src_a.read(indexes=bands_a, window=win)
|
| 281 |
+
db = src_b.read(indexes=bands_b, window=win)
|
| 282 |
+
yield (_normalize_window_rgb(da), _normalize_window_rgb(db),
|
| 283 |
+
y0, x0, full_h, full_w)
|
| 284 |
+
|
| 285 |
+
|
| 286 |
def load_rgb_pil(path: Path, max_side: Optional[int] = None) -> Image.Image:
|
| 287 |
"""Load image as RGB PIL, downscaling large GeoTIFFs via rasterio."""
|
| 288 |
if max_side is None:
|
app/dda/job_runner.py
CHANGED
|
@@ -32,9 +32,10 @@ def _utcnow():
|
|
| 32 |
|
| 33 |
|
| 34 |
def _load_pair(base_path: str, comparison_path: str) -> tuple[Image.Image, Image.Image, Path]:
|
|
|
|
| 35 |
base_file = safe_resolve(base_path)
|
| 36 |
comp_file = safe_resolve(comparison_path)
|
| 37 |
-
max_side =
|
| 38 |
before_pil = load_rgb_pil(base_file, max_side=max_side)
|
| 39 |
after_pil = load_rgb_pil(comp_file, max_side=max_side)
|
| 40 |
if before_pil.size != after_pil.size:
|
|
@@ -71,6 +72,7 @@ def _run_job_sync(job_id: int) -> None:
|
|
| 71 |
|
| 72 |
update_job_progress(job_id, 8, "Loading images")
|
| 73 |
before_pil, after_pil, base_file = _load_pair(base_path, comparison_path)
|
|
|
|
| 74 |
update_job_progress(job_id, 12, "Images loaded")
|
| 75 |
title = params.get("title") or f"{Path(base_path).name} vs {Path(comparison_path).name}"
|
| 76 |
result = run_detection_and_save(
|
|
@@ -88,6 +90,7 @@ def _run_job_sync(job_id: int) -> None:
|
|
| 88 |
notify_email=job.notify_email or params.get("notify_email"),
|
| 89 |
max_size=get_detection_max_side(),
|
| 90 |
geo_bounds_path=base_file,
|
|
|
|
| 91 |
base_path=base_path,
|
| 92 |
user_id=job.created_by,
|
| 93 |
job_id=job_id,
|
|
|
|
| 32 |
|
| 33 |
|
| 34 |
def _load_pair(base_path: str, comparison_path: str) -> tuple[Image.Image, Image.Image, Path]:
|
| 35 |
+
from ..detection_config import get_load_max_side
|
| 36 |
base_file = safe_resolve(base_path)
|
| 37 |
comp_file = safe_resolve(comparison_path)
|
| 38 |
+
max_side = get_load_max_side()
|
| 39 |
before_pil = load_rgb_pil(base_file, max_side=max_side)
|
| 40 |
after_pil = load_rgb_pil(comp_file, max_side=max_side)
|
| 41 |
if before_pil.size != after_pil.size:
|
|
|
|
| 72 |
|
| 73 |
update_job_progress(job_id, 8, "Loading images")
|
| 74 |
before_pil, after_pil, base_file = _load_pair(base_path, comparison_path)
|
| 75 |
+
comp_file = safe_resolve(comparison_path)
|
| 76 |
update_job_progress(job_id, 12, "Images loaded")
|
| 77 |
title = params.get("title") or f"{Path(base_path).name} vs {Path(comparison_path).name}"
|
| 78 |
result = run_detection_and_save(
|
|
|
|
| 90 |
notify_email=job.notify_email or params.get("notify_email"),
|
| 91 |
max_size=get_detection_max_side(),
|
| 92 |
geo_bounds_path=base_file,
|
| 93 |
+
comparison_file=comp_file,
|
| 94 |
base_path=base_path,
|
| 95 |
user_id=job.created_by,
|
| 96 |
job_id=job_id,
|
app/dda/local_routes.py
CHANGED
|
@@ -111,7 +111,11 @@ def local_rescan(db: Session = Depends(get_db)):
|
|
| 111 |
_require_dda()
|
| 112 |
from .tree.sync_service import sync_from_filesystem
|
| 113 |
|
| 114 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 115 |
tree = build_tree(db)
|
| 116 |
images = list_all_images(db)
|
| 117 |
return {
|
|
@@ -158,8 +162,10 @@ async def detect_from_library(
|
|
| 158 |
raise HTTPException(status_code=400, detail=f"Invalid library path: {exc}") from exc
|
| 159 |
|
| 160 |
try:
|
| 161 |
-
|
| 162 |
-
|
|
|
|
|
|
|
| 163 |
except RuntimeError as exc:
|
| 164 |
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
| 165 |
except Exception as exc:
|
|
@@ -187,6 +193,7 @@ async def detect_from_library(
|
|
| 187 |
notify_email=notify_email,
|
| 188 |
max_size=get_detection_max_side(),
|
| 189 |
geo_bounds_path=base_file,
|
|
|
|
| 190 |
base_path=base_norm,
|
| 191 |
user_id=user.id,
|
| 192 |
)
|
|
|
|
| 111 |
_require_dda()
|
| 112 |
from .tree.sync_service import sync_from_filesystem
|
| 113 |
|
| 114 |
+
try:
|
| 115 |
+
sync_stats = sync_from_filesystem(db)
|
| 116 |
+
except Exception as exc:
|
| 117 |
+
logger.exception("Filesystem rescan failed")
|
| 118 |
+
raise HTTPException(status_code=500, detail=f"Rescan failed: {exc}") from exc
|
| 119 |
tree = build_tree(db)
|
| 120 |
images = list_all_images(db)
|
| 121 |
return {
|
|
|
|
| 162 |
raise HTTPException(status_code=400, detail=f"Invalid library path: {exc}") from exc
|
| 163 |
|
| 164 |
try:
|
| 165 |
+
from ..detection_config import get_load_max_side
|
| 166 |
+
load_side = get_load_max_side()
|
| 167 |
+
before_pil = load_rgb_pil(base_file, max_side=load_side)
|
| 168 |
+
after_pil = load_rgb_pil(comp_file, max_side=load_side)
|
| 169 |
except RuntimeError as exc:
|
| 170 |
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
| 171 |
except Exception as exc:
|
|
|
|
| 193 |
notify_email=notify_email,
|
| 194 |
max_size=get_detection_max_side(),
|
| 195 |
geo_bounds_path=base_file,
|
| 196 |
+
comparison_file=comp_file,
|
| 197 |
base_path=base_norm,
|
| 198 |
user_id=user.id,
|
| 199 |
)
|
app/dda/tree/sync_service.py
CHANGED
|
@@ -15,10 +15,13 @@ from .models import ImageLibrary, TreeNode
|
|
| 15 |
from .path_service import ensure_node_directory, storage_root
|
| 16 |
from .path_slugs import RESERVED
|
| 17 |
|
|
|
|
|
|
|
| 18 |
logger = logging.getLogger(__name__)
|
| 19 |
|
| 20 |
_NODE_TYPES = ("Zone", "Area", "Year", "Folder")
|
| 21 |
_SKIP_DIRS = frozenset({".git", ".thumbs", "__pycache__", "cache", "thumbs"})
|
|
|
|
| 22 |
|
| 23 |
|
| 24 |
def _infer_node_type(depth: int) -> str:
|
|
@@ -157,6 +160,16 @@ def _index_image_file(db: Session, node: TreeNode, file_path: Path, rel_file: st
|
|
| 157 |
return True
|
| 158 |
|
| 159 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 160 |
def _sync_directory(db: Session, abs_dir: Path, rel_path: str, stats: dict) -> None:
|
| 161 |
"""Recursively sync nodes and images under rel_path."""
|
| 162 |
if not abs_dir.is_dir():
|
|
@@ -169,10 +182,16 @@ def _sync_directory(db: Session, abs_dir: Path, rel_path: str, stats: dict) -> N
|
|
| 169 |
child_rel = f"{rel_path}/{child.name}".strip("/") if rel_path else child.name
|
| 170 |
|
| 171 |
if child.name.lower() == "images":
|
| 172 |
-
|
| 173 |
-
|
|
|
|
| 174 |
if not node:
|
| 175 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 176 |
for f in sorted(child.iterdir()):
|
| 177 |
if not f.is_file() or f.suffix.lower() not in ALLOWED_EXTENSIONS:
|
| 178 |
continue
|
|
@@ -183,8 +202,20 @@ def _sync_directory(db: Session, abs_dir: Path, rel_path: str, stats: dict) -> N
|
|
| 183 |
stats["imagesUpdated"] += 1
|
| 184 |
continue
|
| 185 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 186 |
before = _find_node_by_physical_path(db, child_rel)
|
| 187 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 188 |
if not before:
|
| 189 |
stats["nodesCreated"] += 1
|
| 190 |
_sync_directory(db, child, child_rel, stats)
|
|
@@ -198,6 +229,7 @@ def sync_from_filesystem(db: Session) -> dict:
|
|
| 198 |
"imagesIndexed": 0,
|
| 199 |
"imagesUpdated": 0,
|
| 200 |
"orphansFlagged": 0,
|
|
|
|
| 201 |
}
|
| 202 |
if not root.exists():
|
| 203 |
return stats
|
|
|
|
| 15 |
from .path_service import ensure_node_directory, storage_root
|
| 16 |
from .path_slugs import RESERVED
|
| 17 |
|
| 18 |
+
from .path_slugs import RESERVED
|
| 19 |
+
|
| 20 |
logger = logging.getLogger(__name__)
|
| 21 |
|
| 22 |
_NODE_TYPES = ("Zone", "Area", "Year", "Folder")
|
| 23 |
_SKIP_DIRS = frozenset({".git", ".thumbs", "__pycache__", "cache", "thumbs"})
|
| 24 |
+
_RESERVED_DIR_NAMES = RESERVED
|
| 25 |
|
| 26 |
|
| 27 |
def _infer_node_type(depth: int) -> str:
|
|
|
|
| 160 |
return True
|
| 161 |
|
| 162 |
|
| 163 |
+
def _node_path_for_images_folder(rel_path: str) -> Optional[str]:
|
| 164 |
+
"""Resolve which tree node should own an on-disk Images/ folder."""
|
| 165 |
+
rel = (rel_path or "").strip("/")
|
| 166 |
+
while rel:
|
| 167 |
+
if rel.split("/")[-1].lower() not in _RESERVED_DIR_NAMES:
|
| 168 |
+
return rel
|
| 169 |
+
rel = "/".join(rel.split("/")[:-1])
|
| 170 |
+
return None
|
| 171 |
+
|
| 172 |
+
|
| 173 |
def _sync_directory(db: Session, abs_dir: Path, rel_path: str, stats: dict) -> None:
|
| 174 |
"""Recursively sync nodes and images under rel_path."""
|
| 175 |
if not abs_dir.is_dir():
|
|
|
|
| 182 |
child_rel = f"{rel_path}/{child.name}".strip("/") if rel_path else child.name
|
| 183 |
|
| 184 |
if child.name.lower() == "images":
|
| 185 |
+
node_rel = _node_path_for_images_folder(rel_path)
|
| 186 |
+
if node_rel:
|
| 187 |
+
node = _find_node_by_physical_path(db, node_rel)
|
| 188 |
if not node:
|
| 189 |
+
try:
|
| 190 |
+
node = ensure_node_from_disk(db, node_rel)
|
| 191 |
+
except ValueError as exc:
|
| 192 |
+
logger.warning("Skipping images under %s: %s", rel_path, exc)
|
| 193 |
+
stats["foldersSkipped"] = stats.get("foldersSkipped", 0) + 1
|
| 194 |
+
continue
|
| 195 |
for f in sorted(child.iterdir()):
|
| 196 |
if not f.is_file() or f.suffix.lower() not in ALLOWED_EXTENSIONS:
|
| 197 |
continue
|
|
|
|
| 202 |
stats["imagesUpdated"] += 1
|
| 203 |
continue
|
| 204 |
|
| 205 |
+
if child.name.lower() in _RESERVED_DIR_NAMES:
|
| 206 |
+
logger.warning("Skipping reserved folder on disk: %s", child_rel)
|
| 207 |
+
stats["foldersSkipped"] = stats.get("foldersSkipped", 0) + 1
|
| 208 |
+
_sync_directory(db, child, child_rel, stats)
|
| 209 |
+
continue
|
| 210 |
+
|
| 211 |
before = _find_node_by_physical_path(db, child_rel)
|
| 212 |
+
try:
|
| 213 |
+
ensure_node_from_disk(db, child_rel)
|
| 214 |
+
except ValueError as exc:
|
| 215 |
+
logger.warning("Skipping folder %s: %s", child_rel, exc)
|
| 216 |
+
stats["foldersSkipped"] = stats.get("foldersSkipped", 0) + 1
|
| 217 |
+
_sync_directory(db, child, child_rel, stats)
|
| 218 |
+
continue
|
| 219 |
if not before:
|
| 220 |
stats["nodesCreated"] += 1
|
| 221 |
_sync_directory(db, child, child_rel, stats)
|
|
|
|
| 229 |
"imagesIndexed": 0,
|
| 230 |
"imagesUpdated": 0,
|
| 231 |
"orphansFlagged": 0,
|
| 232 |
+
"foldersSkipped": 0,
|
| 233 |
}
|
| 234 |
if not root.exists():
|
| 235 |
return stats
|
app/detection_config.py
ADDED
|
@@ -0,0 +1,225 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Centralized, env-driven detection configuration.
|
| 2 |
+
|
| 3 |
+
Single source of truth for detection tuning knobs so the engine, model
|
| 4 |
+
inference, and tiling utilities stay consistent. Every value is read from the
|
| 5 |
+
environment with conservative defaults that preserve the previous behavior
|
| 6 |
+
(``downscaled`` inference, ``smart_union`` fusion, no multi-scale).
|
| 7 |
+
|
| 8 |
+
Env vars
|
| 9 |
+
--------
|
| 10 |
+
DETECTION_MAX_SIDE Max pixel dimension for the legacy downscaled path.
|
| 11 |
+
DETECTION_INFERENCE_MODE ``downscaled`` (default) | ``fullres_tiled``.
|
| 12 |
+
DETECTION_FULLRES_MAX_SIDE Cap for full-resolution tiled mode (0 = native).
|
| 13 |
+
DETECTION_TILE_SIZE Tile size for full-res scoring (256..2048).
|
| 14 |
+
DETECTION_TILE_OVERLAP Fractional tile overlap (0.0..0.5).
|
| 15 |
+
DETECTION_MULTISCALE ``off`` | comma list e.g. ``0.5,1.0,1.5``.
|
| 16 |
+
DETECTION_FUSION ``smart_union`` (default) | ``hysteresis``.
|
| 17 |
+
DETECTION_SKIP_PREBLUR ``true`` | ``false`` (auto: on for fullres_tiled).
|
| 18 |
+
DETECTION_TTA ``off`` | ``hflip`` | ``full`` | ``auto`` (model_inference).
|
| 19 |
+
"""
|
| 20 |
+
from __future__ import annotations
|
| 21 |
+
|
| 22 |
+
import logging
|
| 23 |
+
import os
|
| 24 |
+
from typing import List
|
| 25 |
+
|
| 26 |
+
_log = logging.getLogger(__name__)
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def _env(name: str, default: str = "") -> str:
|
| 30 |
+
return os.environ.get(name, default).strip()
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def get_detection_max_side() -> int:
|
| 34 |
+
"""Max pixel dimension for image load + the downscaled detection path."""
|
| 35 |
+
hosted = bool(_env("SPACE_ID"))
|
| 36 |
+
default = "2048" if hosted else "4096"
|
| 37 |
+
try:
|
| 38 |
+
value = int(os.environ.get("DETECTION_MAX_SIDE", default))
|
| 39 |
+
except ValueError:
|
| 40 |
+
value = int(default)
|
| 41 |
+
return max(1024, min(8192, value))
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def get_inference_mode() -> str:
|
| 45 |
+
"""Detection inference mode: ``downscaled`` or ``fullres_tiled``."""
|
| 46 |
+
mode = _env("DETECTION_INFERENCE_MODE", "downscaled").lower()
|
| 47 |
+
return mode if mode in ("downscaled", "fullres_tiled") else "downscaled"
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def get_fullres_max_side() -> int:
|
| 51 |
+
"""Upper bound for full-res tiled mode. 0 means use native resolution.
|
| 52 |
+
|
| 53 |
+
Defaults to 8192 so very large GeoTIFFs stay bounded in RAM while still
|
| 54 |
+
running detection at far higher detail than the 2048/4096 downscaled cap.
|
| 55 |
+
"""
|
| 56 |
+
raw = _env("DETECTION_FULLRES_MAX_SIDE", "8192")
|
| 57 |
+
try:
|
| 58 |
+
value = int(raw)
|
| 59 |
+
except ValueError:
|
| 60 |
+
value = 8192
|
| 61 |
+
if value <= 0:
|
| 62 |
+
return 0 # native resolution, no cap
|
| 63 |
+
return max(2048, min(20000, value))
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def get_load_max_side() -> int:
|
| 67 |
+
"""Effective image-load / inference pixel cap for the active mode.
|
| 68 |
+
|
| 69 |
+
In ``fullres_tiled`` mode this returns the (much larger) full-res cap so
|
| 70 |
+
images keep their detail; otherwise it returns the standard downscale cap.
|
| 71 |
+
Always a concrete positive int so callers can pass it straight to loaders.
|
| 72 |
+
"""
|
| 73 |
+
if get_inference_mode() == "fullres_tiled":
|
| 74 |
+
cap = get_fullres_max_side()
|
| 75 |
+
return cap if cap > 0 else 20000
|
| 76 |
+
return get_detection_max_side()
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
def get_windowed_threshold() -> int:
|
| 80 |
+
"""Native max-side above which GeoTIFF inference streams via rasterio windows.
|
| 81 |
+
|
| 82 |
+
Keeps peak RAM bounded for very large rasters: when a GeoTIFF's native
|
| 83 |
+
longest side exceeds this, the deep score map is built tile-by-tile from
|
| 84 |
+
disk instead of loading the whole array. 0 disables windowed streaming.
|
| 85 |
+
"""
|
| 86 |
+
raw = _env("DETECTION_WINDOWED_THRESHOLD", "8192")
|
| 87 |
+
try:
|
| 88 |
+
value = int(raw)
|
| 89 |
+
except ValueError:
|
| 90 |
+
value = 8192
|
| 91 |
+
return max(0, value)
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def get_tile_size() -> int:
|
| 95 |
+
"""Tile size (px) for full-resolution scoring."""
|
| 96 |
+
try:
|
| 97 |
+
value = int(os.environ.get("DETECTION_TILE_SIZE", "512"))
|
| 98 |
+
except ValueError:
|
| 99 |
+
value = 512
|
| 100 |
+
return max(256, min(2048, value))
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def get_tile_overlap() -> float:
|
| 104 |
+
"""Fractional overlap between adjacent tiles (0.0..0.5)."""
|
| 105 |
+
try:
|
| 106 |
+
value = float(os.environ.get("DETECTION_TILE_OVERLAP", "0.25"))
|
| 107 |
+
except ValueError:
|
| 108 |
+
value = 0.25
|
| 109 |
+
return min(0.5, max(0.0, value))
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
def get_multiscale_scales() -> List[float]:
|
| 113 |
+
"""Scales for multi-scale fusion. Empty list disables it."""
|
| 114 |
+
raw = _env("DETECTION_MULTISCALE", "off").lower()
|
| 115 |
+
if raw in ("off", "", "none", "0", "false"):
|
| 116 |
+
return []
|
| 117 |
+
scales: List[float] = []
|
| 118 |
+
for part in raw.split(","):
|
| 119 |
+
part = part.strip()
|
| 120 |
+
if not part:
|
| 121 |
+
continue
|
| 122 |
+
try:
|
| 123 |
+
scale = float(part)
|
| 124 |
+
except ValueError:
|
| 125 |
+
continue
|
| 126 |
+
if 0.1 <= scale <= 4.0:
|
| 127 |
+
scales.append(scale)
|
| 128 |
+
# Always include native scale so recall never drops below single-scale
|
| 129 |
+
if scales and 1.0 not in scales:
|
| 130 |
+
scales.append(1.0)
|
| 131 |
+
return sorted(set(scales))
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
def get_fusion_mode() -> str:
|
| 135 |
+
"""DL + classical fusion strategy: ``smart_union`` or ``hysteresis``."""
|
| 136 |
+
mode = _env("DETECTION_FUSION", "smart_union").lower()
|
| 137 |
+
return mode if mode in ("smart_union", "hysteresis") else "smart_union"
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
def get_skip_preblur() -> bool:
|
| 141 |
+
"""Whether to skip the preprocessing Gaussian/bilateral blur.
|
| 142 |
+
|
| 143 |
+
Explicit env wins; otherwise auto-skip in full-res mode to preserve the
|
| 144 |
+
fine detail that motivates running detection at native resolution.
|
| 145 |
+
"""
|
| 146 |
+
raw = _env("DETECTION_SKIP_PREBLUR", "").lower()
|
| 147 |
+
if raw in ("1", "true", "yes", "on"):
|
| 148 |
+
return True
|
| 149 |
+
if raw in ("0", "false", "no", "off"):
|
| 150 |
+
return False
|
| 151 |
+
return get_inference_mode() == "fullres_tiled"
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
def _flag(name: str, default: bool) -> bool:
|
| 155 |
+
raw = _env(name, "").lower()
|
| 156 |
+
if raw in ("1", "true", "yes", "on"):
|
| 157 |
+
return True
|
| 158 |
+
if raw in ("0", "false", "no", "off"):
|
| 159 |
+
return False
|
| 160 |
+
return default
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
def get_enable_clahe() -> bool:
|
| 164 |
+
"""CLAHE contrast equalization during radiometric normalization (default on)."""
|
| 165 |
+
return _flag("DETECTION_CLAHE", True)
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
def get_hist_match() -> bool:
|
| 169 |
+
"""Optional histogram matching of the after image to the before (default off)."""
|
| 170 |
+
return _flag("DETECTION_HIST_MATCH", False)
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
def get_tile_batch() -> int:
|
| 174 |
+
"""Tiles per model forward pass (GPU throughput). 1 = no batching (default)."""
|
| 175 |
+
try:
|
| 176 |
+
value = int(os.environ.get("DETECTION_TILE_BATCH", "1"))
|
| 177 |
+
except ValueError:
|
| 178 |
+
value = 1
|
| 179 |
+
return max(1, min(64, value))
|
| 180 |
+
|
| 181 |
+
|
| 182 |
+
def get_tile_memory_budget_mb() -> int:
|
| 183 |
+
"""Soft RAM budget (MB) for a single in-memory detection array.
|
| 184 |
+
|
| 185 |
+
When loading both timestamps at the full-res cap would exceed this, large
|
| 186 |
+
GeoTIFFs switch to disk-windowed streaming so peak memory stays bounded
|
| 187 |
+
(lets 15 GB rasters run without OOM). 0 disables the budget check.
|
| 188 |
+
"""
|
| 189 |
+
raw = _env("DETECTION_TILE_MEMORY_MB", "1536")
|
| 190 |
+
try:
|
| 191 |
+
value = int(raw)
|
| 192 |
+
except ValueError:
|
| 193 |
+
value = 1536
|
| 194 |
+
return max(0, value)
|
| 195 |
+
|
| 196 |
+
|
| 197 |
+
def get_save_prob_map() -> bool:
|
| 198 |
+
"""Save a downsampled probability-map PNG per run for debugging (default off)."""
|
| 199 |
+
return _flag("DETECTION_SAVE_PROB_MAP", False)
|
| 200 |
+
|
| 201 |
+
|
| 202 |
+
def get_border_margin() -> int:
|
| 203 |
+
"""Border pixels zeroed in mask cleanup. Smaller in full-res mode."""
|
| 204 |
+
raw = _env("DETECTION_BORDER_MARGIN", "")
|
| 205 |
+
if raw:
|
| 206 |
+
try:
|
| 207 |
+
return max(0, min(64, int(raw)))
|
| 208 |
+
except ValueError:
|
| 209 |
+
pass
|
| 210 |
+
return 4 if get_inference_mode() == "fullres_tiled" else 12
|
| 211 |
+
|
| 212 |
+
|
| 213 |
+
def summary() -> dict:
|
| 214 |
+
"""Snapshot of effective detection config (for logs and debug output)."""
|
| 215 |
+
return {
|
| 216 |
+
"maxSide": get_detection_max_side(),
|
| 217 |
+
"inferenceMode": get_inference_mode(),
|
| 218 |
+
"fullresMaxSide": get_fullres_max_side(),
|
| 219 |
+
"tileSize": get_tile_size(),
|
| 220 |
+
"tileOverlap": get_tile_overlap(),
|
| 221 |
+
"multiscale": get_multiscale_scales(),
|
| 222 |
+
"fusion": get_fusion_mode(),
|
| 223 |
+
"skipPreblur": get_skip_preblur(),
|
| 224 |
+
"borderMargin": get_border_margin(),
|
| 225 |
+
}
|
app/detection_engine.py
CHANGED
|
@@ -6,6 +6,8 @@ SIFT+FLANN registration, tile-based + multi-scale processing, Excess Green
|
|
| 6 |
vegetation index, confidence maps, and improved object classification.
|
| 7 |
"""
|
| 8 |
import logging
|
|
|
|
|
|
|
| 9 |
import numpy as np
|
| 10 |
import cv2
|
| 11 |
from PIL import Image
|
|
@@ -22,14 +24,8 @@ _log = logging.getLogger(__name__)
|
|
| 22 |
|
| 23 |
def get_detection_max_size() -> int:
|
| 24 |
"""Max pixel dimension for detection (override with DETECTION_MAX_SIDE env)."""
|
| 25 |
-
import
|
| 26 |
-
|
| 27 |
-
default = "2048" if hosted else "4096"
|
| 28 |
-
try:
|
| 29 |
-
value = int(os.environ.get("DETECTION_MAX_SIDE", default))
|
| 30 |
-
except ValueError:
|
| 31 |
-
value = int(default)
|
| 32 |
-
return max(1024, min(8192, value))
|
| 33 |
|
| 34 |
|
| 35 |
def _ensure_rgb_uint8(img_array):
|
|
@@ -50,8 +46,13 @@ def _to_float32(img):
|
|
| 50 |
return img.astype(np.float32) / 255.0
|
| 51 |
|
| 52 |
|
| 53 |
-
def preprocess_image(image, max_size=None):
|
| 54 |
-
"""Preprocess image: convert to RGB, limit size, light denoise.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 55 |
if max_size is None:
|
| 56 |
max_size = get_detection_max_size()
|
| 57 |
img_array = np.array(image)
|
|
@@ -63,6 +64,9 @@ def preprocess_image(image, max_size=None):
|
|
| 63 |
new_w, new_h = max(1, int(width * scale)), max(1, int(height * scale))
|
| 64 |
img_array = cv2.resize(img_array, (new_w, new_h), interpolation=cv2.INTER_AREA)
|
| 65 |
|
|
|
|
|
|
|
|
|
|
| 66 |
# Light denoise — smaller kernel preserves fine change detail at high resolution
|
| 67 |
blur_ksize = 3 if max_size >= 3000 else 5
|
| 68 |
img_array = cv2.GaussianBlur(img_array, (blur_ksize, blur_ksize), 0)
|
|
@@ -311,8 +315,27 @@ def _register_images_ecc_multiscale(img1, img2):
|
|
| 311 |
# 3. Improved radiometric normalization
|
| 312 |
# ---------------------------------------------------------------------------
|
| 313 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 314 |
def normalize_radiometry(img1, img2):
|
| 315 |
-
"""Match after image radiometry to before; symmetric CLAHE on L channel.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 316 |
lab1 = cv2.cvtColor(img1, cv2.COLOR_RGB2LAB).astype(np.float32)
|
| 317 |
lab2 = cv2.cvtColor(img2, cv2.COLOR_RGB2LAB).astype(np.float32)
|
| 318 |
|
|
@@ -323,11 +346,17 @@ def normalize_radiometry(img1, img2):
|
|
| 323 |
if std2 > 1e-6:
|
| 324 |
result[:, :, ch] = (lab2[:, :, ch] - mean2) * (std1 / std2) + mean1
|
| 325 |
|
| 326 |
-
|
|
|
|
|
|
|
|
|
|
| 327 |
lab1_u = cv2.cvtColor(img1, cv2.COLOR_RGB2LAB)
|
| 328 |
lab2_u = np.clip(result, 0, 255).astype(np.uint8)
|
| 329 |
-
|
| 330 |
-
|
|
|
|
|
|
|
|
|
|
| 331 |
|
| 332 |
return cv2.cvtColor(lab1_u, cv2.COLOR_LAB2RGB), cv2.cvtColor(lab2_u, cv2.COLOR_LAB2RGB)
|
| 333 |
|
|
@@ -524,6 +553,22 @@ def compute_lbp(gray, radius=1, n_points=8):
|
|
| 524 |
return lbp / n_points
|
| 525 |
|
| 526 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 527 |
def _hysteresis_threshold(score, high_thr, low_thr):
|
| 528 |
"""Two-level (hysteresis) thresholding on a [0,1] score map.
|
| 529 |
|
|
@@ -937,9 +982,14 @@ def _resolve_classification(scores, diff, feat_a):
|
|
| 937 |
return best, conf
|
| 938 |
|
| 939 |
|
| 940 |
-
def ai_deep_learning_method(img1, img2, sensitivity=0.5, registration_ok=True
|
|
|
|
| 941 |
"""
|
| 942 |
Dual-engine: AdaptFormer + classical fusion with confidence-pruned union.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 943 |
"""
|
| 944 |
from .model_inference import is_model_available, predict_change_mask
|
| 945 |
|
|
@@ -948,7 +998,14 @@ def ai_deep_learning_method(img1, img2, sensitivity=0.5, registration_ok=True):
|
|
| 948 |
model_ok = False
|
| 949 |
threshold = 0.30 + (1.0 - float(np.clip(sensitivity, 0, 1))) * 0.22
|
| 950 |
|
| 951 |
-
if
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 952 |
try:
|
| 953 |
model_mask, dl_score = predict_change_mask(img1, img2, threshold=threshold)
|
| 954 |
model_ok = dl_score is not None
|
|
@@ -961,18 +1018,33 @@ def ai_deep_learning_method(img1, img2, sensitivity=0.5, registration_ok=True):
|
|
| 961 |
if model_ok and dl_score is not None:
|
| 962 |
if model_mask is None:
|
| 963 |
model_mask = (dl_score >= threshold).astype(np.uint8) * 255
|
| 964 |
-
|
| 965 |
-
|
| 966 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 967 |
debug = {
|
| 968 |
"method": "AI-Based Deep Learning (AdaptFormer + confidence union)",
|
| 969 |
"model": "adaptformer-levir-cd",
|
| 970 |
-
"fusion":
|
| 971 |
"threshold_used": int(threshold * 255),
|
| 972 |
"sensitivity": float(sensitivity),
|
| 973 |
"model_changed_px": int(np.sum(model_mask > 127)),
|
| 974 |
"rule_changed_px": int(np.sum(rule_mask > 127)),
|
| 975 |
"combined_changed_px": int(np.sum(combined > 127)),
|
|
|
|
|
|
|
|
|
|
|
|
|
| 976 |
}
|
| 977 |
return combined, debug
|
| 978 |
|
|
@@ -1129,7 +1201,7 @@ ALIGNMENT_WARNING_MSG = (
|
|
| 1129 |
# 11. Robust post-processing
|
| 1130 |
# ---------------------------------------------------------------------------
|
| 1131 |
|
| 1132 |
-
def _clean_mask(mask, sensitivity=0.5, border_margin=
|
| 1133 |
"""
|
| 1134 |
Robust morphological cleaning:
|
| 1135 |
1. Zero-out border pixels (registration artifacts)
|
|
@@ -1139,7 +1211,13 @@ def _clean_mask(mask, sensitivity=0.5, border_margin=12):
|
|
| 1139 |
5. Fill holes inside regions
|
| 1140 |
6. Erode-then-dilate to break thin noise bridges
|
| 1141 |
7. Connected-component area + circularity filtering
|
|
|
|
|
|
|
|
|
|
| 1142 |
"""
|
|
|
|
|
|
|
|
|
|
| 1143 |
mask = mask.copy()
|
| 1144 |
h, w = mask.shape[:2]
|
| 1145 |
|
|
@@ -2639,16 +2717,57 @@ def analyze_change_regions(change_mask, image, min_area=400, use_ensemble=True,
|
|
| 2639 |
def run_detection(before_pil, after_pil, method="AI-Based Deep Learning",
|
| 2640 |
enable_registration=True, enable_normalization=True,
|
| 2641 |
detection_sensitivity=0.5, min_region_area=None,
|
| 2642 |
-
max_size=None, on_progress=None
|
|
|
|
| 2643 |
"""Run full detection pipeline; returns change_mask, result_image, stats, regions."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2644 |
def _prog(pct, stage):
|
| 2645 |
if on_progress:
|
| 2646 |
on_progress(int(pct), stage)
|
| 2647 |
|
| 2648 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2649 |
_prog(5, "Preprocessing images")
|
| 2650 |
-
before_array = preprocess_image(before_pil, max_size=ms)
|
| 2651 |
-
after_array = preprocess_image(after_pil, max_size=ms)
|
| 2652 |
|
| 2653 |
registration_ok = False
|
| 2654 |
reg_meta = {}
|
|
@@ -2664,12 +2783,33 @@ def run_detection(before_pil, after_pil, method="AI-Based Deep Learning",
|
|
| 2664 |
if enable_registration and not registration_ok:
|
| 2665 |
alignment_warning = ALIGNMENT_WARNING_MSG
|
| 2666 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2667 |
_prog(50, f"Running {method}")
|
| 2668 |
if method == "AI-Based Deep Learning":
|
| 2669 |
change_mask, threshold_debug = ai_deep_learning_method(
|
| 2670 |
before_array, after_array,
|
| 2671 |
sensitivity=detection_sensitivity,
|
| 2672 |
registration_ok=registration_ok,
|
|
|
|
| 2673 |
)
|
| 2674 |
elif method == "Image Difference":
|
| 2675 |
change_mask, threshold_debug = image_difference_method(
|
|
@@ -2696,6 +2836,22 @@ def run_detection(before_pil, after_pil, method="AI-Based Deep Learning",
|
|
| 2696 |
registration_ok=registration_ok,
|
| 2697 |
)
|
| 2698 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2699 |
total_pixels = int(change_mask.shape[0] * change_mask.shape[1])
|
| 2700 |
changed_pixels_ratio = (
|
| 2701 |
float(np.sum(change_mask > 127)) / float(total_pixels) if total_pixels else 0.0
|
|
@@ -2758,7 +2914,12 @@ def run_detection(before_pil, after_pil, method="AI-Based Deep Learning",
|
|
| 2758 |
"enable_normalization": bool(enable_normalization),
|
| 2759 |
"registration_ok": bool(registration_ok),
|
| 2760 |
"registration": reg_meta,
|
|
|
|
|
|
|
|
|
|
| 2761 |
},
|
| 2762 |
}
|
|
|
|
|
|
|
| 2763 |
|
| 2764 |
return change_mask, result_image, stats, change_regions
|
|
|
|
| 6 |
vegetation index, confidence maps, and improved object classification.
|
| 7 |
"""
|
| 8 |
import logging
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
|
| 11 |
import numpy as np
|
| 12 |
import cv2
|
| 13 |
from PIL import Image
|
|
|
|
| 24 |
|
| 25 |
def get_detection_max_size() -> int:
|
| 26 |
"""Max pixel dimension for detection (override with DETECTION_MAX_SIDE env)."""
|
| 27 |
+
from .detection_config import get_detection_max_side
|
| 28 |
+
return get_detection_max_side()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 29 |
|
| 30 |
|
| 31 |
def _ensure_rgb_uint8(img_array):
|
|
|
|
| 46 |
return img.astype(np.float32) / 255.0
|
| 47 |
|
| 48 |
|
| 49 |
+
def preprocess_image(image, max_size=None, skip_blur=False):
|
| 50 |
+
"""Preprocess image: convert to RGB, limit size, light denoise.
|
| 51 |
+
|
| 52 |
+
``skip_blur`` disables the Gaussian/bilateral denoise so fine change detail
|
| 53 |
+
is preserved — used by full-resolution tiled inference where the whole point
|
| 54 |
+
is to keep native sharpness.
|
| 55 |
+
"""
|
| 56 |
if max_size is None:
|
| 57 |
max_size = get_detection_max_size()
|
| 58 |
img_array = np.array(image)
|
|
|
|
| 64 |
new_w, new_h = max(1, int(width * scale)), max(1, int(height * scale))
|
| 65 |
img_array = cv2.resize(img_array, (new_w, new_h), interpolation=cv2.INTER_AREA)
|
| 66 |
|
| 67 |
+
if skip_blur:
|
| 68 |
+
return img_array
|
| 69 |
+
|
| 70 |
# Light denoise — smaller kernel preserves fine change detail at high resolution
|
| 71 |
blur_ksize = 3 if max_size >= 3000 else 5
|
| 72 |
img_array = cv2.GaussianBlur(img_array, (blur_ksize, blur_ksize), 0)
|
|
|
|
| 315 |
# 3. Improved radiometric normalization
|
| 316 |
# ---------------------------------------------------------------------------
|
| 317 |
|
| 318 |
+
def _match_histogram(src, ref):
|
| 319 |
+
"""Map ``src`` intensities so its histogram matches ``ref`` (single channel)."""
|
| 320 |
+
src_u = np.clip(src, 0, 255).astype(np.uint8)
|
| 321 |
+
ref_u = np.clip(ref, 0, 255).astype(np.uint8)
|
| 322 |
+
src_hist = np.bincount(src_u.ravel(), minlength=256).astype(np.float64)
|
| 323 |
+
ref_hist = np.bincount(ref_u.ravel(), minlength=256).astype(np.float64)
|
| 324 |
+
src_cdf = np.cumsum(src_hist) / max(src_u.size, 1)
|
| 325 |
+
ref_cdf = np.cumsum(ref_hist) / max(ref_u.size, 1)
|
| 326 |
+
lut = np.interp(src_cdf, ref_cdf, np.arange(256)).astype(np.float32)
|
| 327 |
+
return lut[src_u]
|
| 328 |
+
|
| 329 |
+
|
| 330 |
def normalize_radiometry(img1, img2):
|
| 331 |
+
"""Match after image radiometry to before; symmetric CLAHE on L channel.
|
| 332 |
+
|
| 333 |
+
CLAHE and histogram matching are env-toggleable (``DETECTION_CLAHE``,
|
| 334 |
+
``DETECTION_HIST_MATCH``) so each technique can be A/B benchmarked before
|
| 335 |
+
being promoted to a default.
|
| 336 |
+
"""
|
| 337 |
+
from .detection_config import get_enable_clahe, get_hist_match
|
| 338 |
+
|
| 339 |
lab1 = cv2.cvtColor(img1, cv2.COLOR_RGB2LAB).astype(np.float32)
|
| 340 |
lab2 = cv2.cvtColor(img2, cv2.COLOR_RGB2LAB).astype(np.float32)
|
| 341 |
|
|
|
|
| 346 |
if std2 > 1e-6:
|
| 347 |
result[:, :, ch] = (lab2[:, :, ch] - mean2) * (std1 / std2) + mean1
|
| 348 |
|
| 349 |
+
if get_hist_match():
|
| 350 |
+
# Stronger than mean/std matching: align the full L-channel distribution
|
| 351 |
+
result[:, :, 0] = _match_histogram(result[:, :, 0], lab1[:, :, 0])
|
| 352 |
+
|
| 353 |
lab1_u = cv2.cvtColor(img1, cv2.COLOR_RGB2LAB)
|
| 354 |
lab2_u = np.clip(result, 0, 255).astype(np.uint8)
|
| 355 |
+
|
| 356 |
+
if get_enable_clahe():
|
| 357 |
+
clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8))
|
| 358 |
+
lab1_u[:, :, 0] = clahe.apply(lab1_u[:, :, 0])
|
| 359 |
+
lab2_u[:, :, 0] = clahe.apply(lab2_u[:, :, 0])
|
| 360 |
|
| 361 |
return cv2.cvtColor(lab1_u, cv2.COLOR_LAB2RGB), cv2.cvtColor(lab2_u, cv2.COLOR_LAB2RGB)
|
| 362 |
|
|
|
|
| 553 |
return lbp / n_points
|
| 554 |
|
| 555 |
|
| 556 |
+
def _prob_map_stats(score) -> dict:
|
| 557 |
+
"""Summary stats of a [0,1] probability/score map for threshold tuning."""
|
| 558 |
+
if score is None or score.size == 0:
|
| 559 |
+
return {}
|
| 560 |
+
s = score.astype(np.float32)
|
| 561 |
+
return {
|
| 562 |
+
"min": round(float(s.min()), 4),
|
| 563 |
+
"max": round(float(s.max()), 4),
|
| 564 |
+
"mean": round(float(s.mean()), 4),
|
| 565 |
+
"std": round(float(s.std()), 4),
|
| 566 |
+
"p50": round(float(np.percentile(s, 50)), 4),
|
| 567 |
+
"p90": round(float(np.percentile(s, 90)), 4),
|
| 568 |
+
"p99": round(float(np.percentile(s, 99)), 4),
|
| 569 |
+
}
|
| 570 |
+
|
| 571 |
+
|
| 572 |
def _hysteresis_threshold(score, high_thr, low_thr):
|
| 573 |
"""Two-level (hysteresis) thresholding on a [0,1] score map.
|
| 574 |
|
|
|
|
| 982 |
return best, conf
|
| 983 |
|
| 984 |
|
| 985 |
+
def ai_deep_learning_method(img1, img2, sensitivity=0.5, registration_ok=True,
|
| 986 |
+
dl_score_override=None):
|
| 987 |
"""
|
| 988 |
Dual-engine: AdaptFormer + classical fusion with confidence-pruned union.
|
| 989 |
+
|
| 990 |
+
``dl_score_override`` lets callers supply a precomputed deep score map (e.g.
|
| 991 |
+
full-resolution windowed inference) instead of running AdaptFormer on the
|
| 992 |
+
working-resolution arrays.
|
| 993 |
"""
|
| 994 |
from .model_inference import is_model_available, predict_change_mask
|
| 995 |
|
|
|
|
| 998 |
model_ok = False
|
| 999 |
threshold = 0.30 + (1.0 - float(np.clip(sensitivity, 0, 1))) * 0.22
|
| 1000 |
|
| 1001 |
+
if dl_score_override is not None:
|
| 1002 |
+
dl_score = dl_score_override.astype(np.float32)
|
| 1003 |
+
if dl_score.shape != img1.shape[:2]:
|
| 1004 |
+
dl_score = cv2.resize(dl_score, (img1.shape[1], img1.shape[0]),
|
| 1005 |
+
interpolation=cv2.INTER_LINEAR)
|
| 1006 |
+
model_mask = (dl_score >= threshold).astype(np.uint8) * 255
|
| 1007 |
+
model_ok = True
|
| 1008 |
+
elif is_model_available():
|
| 1009 |
try:
|
| 1010 |
model_mask, dl_score = predict_change_mask(img1, img2, threshold=threshold)
|
| 1011 |
model_ok = dl_score is not None
|
|
|
|
| 1018 |
if model_ok and dl_score is not None:
|
| 1019 |
if model_mask is None:
|
| 1020 |
model_mask = (dl_score >= threshold).astype(np.uint8) * 255
|
| 1021 |
+
|
| 1022 |
+
from .detection_config import get_fusion_mode
|
| 1023 |
+
fusion_mode = get_fusion_mode()
|
| 1024 |
+
if fusion_mode == "hysteresis":
|
| 1025 |
+
# Confidence-gated hysteresis fusion keeps complete change blobs
|
| 1026 |
+
# while pruning isolated weak responses (better recall on big blobs).
|
| 1027 |
+
combined, final_score, fuse_dbg = fuse_dl_and_classical(
|
| 1028 |
+
dl_score, classical_score, img1, img2, sensitivity=sensitivity)
|
| 1029 |
+
else:
|
| 1030 |
+
combined = _smart_union_fusion(
|
| 1031 |
+
model_mask, rule_mask, dl_score, classical_score, sensitivity=sensitivity)
|
| 1032 |
+
combined = _clean_mask(combined, sensitivity=sensitivity)
|
| 1033 |
+
final_score = np.maximum(dl_score, classical_score)
|
| 1034 |
+
fuse_dbg = {}
|
| 1035 |
debug = {
|
| 1036 |
"method": "AI-Based Deep Learning (AdaptFormer + confidence union)",
|
| 1037 |
"model": "adaptformer-levir-cd",
|
| 1038 |
+
"fusion": fusion_mode,
|
| 1039 |
"threshold_used": int(threshold * 255),
|
| 1040 |
"sensitivity": float(sensitivity),
|
| 1041 |
"model_changed_px": int(np.sum(model_mask > 127)),
|
| 1042 |
"rule_changed_px": int(np.sum(rule_mask > 127)),
|
| 1043 |
"combined_changed_px": int(np.sum(combined > 127)),
|
| 1044 |
+
"fusion_debug": fuse_dbg,
|
| 1045 |
+
"dl_score_source": "windowed_fullres" if dl_score_override is not None else "tiled",
|
| 1046 |
+
"probabilityMapStats": _prob_map_stats(final_score),
|
| 1047 |
+
"_score_map": final_score.astype(np.float32),
|
| 1048 |
}
|
| 1049 |
return combined, debug
|
| 1050 |
|
|
|
|
| 1201 |
# 11. Robust post-processing
|
| 1202 |
# ---------------------------------------------------------------------------
|
| 1203 |
|
| 1204 |
+
def _clean_mask(mask, sensitivity=0.5, border_margin=None):
|
| 1205 |
"""
|
| 1206 |
Robust morphological cleaning:
|
| 1207 |
1. Zero-out border pixels (registration artifacts)
|
|
|
|
| 1211 |
5. Fill holes inside regions
|
| 1212 |
6. Erode-then-dilate to break thin noise bridges
|
| 1213 |
7. Connected-component area + circularity filtering
|
| 1214 |
+
|
| 1215 |
+
``border_margin`` defaults to the env-configured value (smaller in full-res
|
| 1216 |
+
mode so genuine edge changes are not discarded).
|
| 1217 |
"""
|
| 1218 |
+
if border_margin is None:
|
| 1219 |
+
from .detection_config import get_border_margin
|
| 1220 |
+
border_margin = get_border_margin()
|
| 1221 |
mask = mask.copy()
|
| 1222 |
h, w = mask.shape[:2]
|
| 1223 |
|
|
|
|
| 2717 |
def run_detection(before_pil, after_pil, method="AI-Based Deep Learning",
|
| 2718 |
enable_registration=True, enable_normalization=True,
|
| 2719 |
detection_sensitivity=0.5, min_region_area=None,
|
| 2720 |
+
max_size=None, on_progress=None,
|
| 2721 |
+
before_path=None, after_path=None):
|
| 2722 |
"""Run full detection pipeline; returns change_mask, result_image, stats, regions."""
|
| 2723 |
+
from .detection_config import (
|
| 2724 |
+
get_inference_mode, get_load_max_side, get_skip_preblur,
|
| 2725 |
+
get_windowed_threshold, get_tile_size, get_tile_overlap,
|
| 2726 |
+
)
|
| 2727 |
+
|
| 2728 |
def _prog(pct, stage):
|
| 2729 |
if on_progress:
|
| 2730 |
on_progress(int(pct), stage)
|
| 2731 |
|
| 2732 |
+
inference_mode = get_inference_mode()
|
| 2733 |
+
skip_blur = get_skip_preblur()
|
| 2734 |
+
|
| 2735 |
+
# Decide whether to stream the deep score from native GeoTIFF windows.
|
| 2736 |
+
# Triggered for very large rasters (native side > threshold) or when loading
|
| 2737 |
+
# the full-res array would blow the memory budget, so peak RAM stays bounded
|
| 2738 |
+
# while the model still sees full-resolution detail.
|
| 2739 |
+
windowed = False
|
| 2740 |
+
if inference_mode == "fullres_tiled" and before_path and after_path:
|
| 2741 |
+
try:
|
| 2742 |
+
from .dda.geotiff_io import read_native_size
|
| 2743 |
+
from .detection_config import get_tile_memory_budget_mb
|
| 2744 |
+
thr = get_windowed_threshold()
|
| 2745 |
+
na = read_native_size(Path(before_path))
|
| 2746 |
+
nb = read_native_size(Path(after_path))
|
| 2747 |
+
if na and nb and na == nb:
|
| 2748 |
+
cap = get_load_max_side()
|
| 2749 |
+
long_side = min(max(na), cap)
|
| 2750 |
+
# ~3 bytes/px/array, two timestamps held in memory at once
|
| 2751 |
+
est_mb = (long_side * long_side * 3 * 2) / (1024 * 1024)
|
| 2752 |
+
budget = get_tile_memory_budget_mb()
|
| 2753 |
+
over_threshold = bool(thr and max(na) > thr)
|
| 2754 |
+
over_budget = bool(budget and est_mb > budget)
|
| 2755 |
+
if over_threshold or over_budget:
|
| 2756 |
+
windowed = True
|
| 2757 |
+
except Exception as exc:
|
| 2758 |
+
_log.warning("Windowed-size probe failed: %s", exc)
|
| 2759 |
+
|
| 2760 |
+
if inference_mode == "fullres_tiled" and not windowed:
|
| 2761 |
+
# Keep full imagery at the full-res cap and skip denoise.
|
| 2762 |
+
ms = max_size if (max_size and max_size > get_detection_max_size()) else get_load_max_side()
|
| 2763 |
+
else:
|
| 2764 |
+
# Windowed mode keeps the in-memory arrays bounded (registration +
|
| 2765 |
+
# classical run on the preview); detail comes from the streamed score.
|
| 2766 |
+
ms = max_size or get_detection_max_size()
|
| 2767 |
+
|
| 2768 |
_prog(5, "Preprocessing images")
|
| 2769 |
+
before_array = preprocess_image(before_pil, max_size=ms, skip_blur=skip_blur)
|
| 2770 |
+
after_array = preprocess_image(after_pil, max_size=ms, skip_blur=skip_blur)
|
| 2771 |
|
| 2772 |
registration_ok = False
|
| 2773 |
reg_meta = {}
|
|
|
|
| 2783 |
if enable_registration and not registration_ok:
|
| 2784 |
alignment_warning = ALIGNMENT_WARNING_MSG
|
| 2785 |
|
| 2786 |
+
dl_score_override = None
|
| 2787 |
+
if windowed and method in ("AI-Based Deep Learning", "Hybrid AI"):
|
| 2788 |
+
try:
|
| 2789 |
+
from .model_inference import is_model_available, predict_change_score_windowed
|
| 2790 |
+
if is_model_available():
|
| 2791 |
+
_prog(45, "Full-res tiled inference")
|
| 2792 |
+
out_h, out_w = before_array.shape[:2]
|
| 2793 |
+
|
| 2794 |
+
def _win_prog(frac):
|
| 2795 |
+
_prog(45 + int(frac * 5), "Full-res tiled inference")
|
| 2796 |
+
|
| 2797 |
+
dl_score_override = predict_change_score_windowed(
|
| 2798 |
+
before_path, after_path, out_h, out_w,
|
| 2799 |
+
tile_size=get_tile_size(), overlap=get_tile_overlap(),
|
| 2800 |
+
on_progress=_win_prog,
|
| 2801 |
+
)
|
| 2802 |
+
except Exception as exc:
|
| 2803 |
+
_log.warning("Windowed inference failed, falling back to in-memory: %s", exc)
|
| 2804 |
+
dl_score_override = None
|
| 2805 |
+
|
| 2806 |
_prog(50, f"Running {method}")
|
| 2807 |
if method == "AI-Based Deep Learning":
|
| 2808 |
change_mask, threshold_debug = ai_deep_learning_method(
|
| 2809 |
before_array, after_array,
|
| 2810 |
sensitivity=detection_sensitivity,
|
| 2811 |
registration_ok=registration_ok,
|
| 2812 |
+
dl_score_override=dl_score_override,
|
| 2813 |
)
|
| 2814 |
elif method == "Image Difference":
|
| 2815 |
change_mask, threshold_debug = image_difference_method(
|
|
|
|
| 2836 |
registration_ok=registration_ok,
|
| 2837 |
)
|
| 2838 |
|
| 2839 |
+
# Pull the (numpy) probability map out of the debug dict so the returned
|
| 2840 |
+
# threshold_debug stays JSON-serializable; stash it privately in stats for
|
| 2841 |
+
# optional probability-PNG export downstream.
|
| 2842 |
+
score_map = None
|
| 2843 |
+
if isinstance(threshold_debug, dict):
|
| 2844 |
+
score_map = threshold_debug.pop("_score_map", None)
|
| 2845 |
+
from .detection_config import (
|
| 2846 |
+
get_tile_size as _gts, get_tile_overlap as _gto,
|
| 2847 |
+
get_multiscale_scales as _gms, get_fusion_mode as _gfm,
|
| 2848 |
+
)
|
| 2849 |
+
threshold_debug["inferenceMode"] = inference_mode
|
| 2850 |
+
threshold_debug["windowed"] = bool(windowed)
|
| 2851 |
+
threshold_debug["fusionMode"] = _gfm()
|
| 2852 |
+
threshold_debug["tileConfig"] = {"tileSize": _gts(), "overlap": _gto(),
|
| 2853 |
+
"multiscale": _gms()}
|
| 2854 |
+
|
| 2855 |
total_pixels = int(change_mask.shape[0] * change_mask.shape[1])
|
| 2856 |
changed_pixels_ratio = (
|
| 2857 |
float(np.sum(change_mask > 127)) / float(total_pixels) if total_pixels else 0.0
|
|
|
|
| 2914 |
"enable_normalization": bool(enable_normalization),
|
| 2915 |
"registration_ok": bool(registration_ok),
|
| 2916 |
"registration": reg_meta,
|
| 2917 |
+
"inference_mode": inference_mode,
|
| 2918 |
+
"windowed": bool(windowed),
|
| 2919 |
+
"working_resolution": [int(before_array.shape[1]), int(before_array.shape[0])],
|
| 2920 |
},
|
| 2921 |
}
|
| 2922 |
+
if score_map is not None:
|
| 2923 |
+
stats["_score_map"] = score_map # numpy; not serialized, consumed downstream
|
| 2924 |
|
| 2925 |
return change_mask, result_image, stats, change_regions
|
app/evaluation/__init__.py
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Evaluation utilities for change detection (binary-mask metrics)."""
|
| 2 |
+
from .metrics import binary_metrics, confusion_counts
|
| 3 |
+
|
| 4 |
+
__all__ = ["binary_metrics", "confusion_counts"]
|
app/evaluation/metrics.py
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Binary change-mask evaluation metrics.
|
| 2 |
+
|
| 3 |
+
All functions take boolean / 0-or-255 / 0-or-1 masks of identical shape and
|
| 4 |
+
return plain Python floats so results are easy to log and serialize. These are
|
| 5 |
+
the standard pixel-wise change-detection metrics: IoU, Dice/F1, precision,
|
| 6 |
+
recall, pixel accuracy and the false-positive / false-negative rates.
|
| 7 |
+
"""
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
from dataclasses import asdict, dataclass
|
| 11 |
+
|
| 12 |
+
import numpy as np
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def _to_bool(mask: np.ndarray) -> np.ndarray:
|
| 16 |
+
arr = np.asarray(mask)
|
| 17 |
+
if arr.dtype == bool:
|
| 18 |
+
return arr
|
| 19 |
+
return arr > (127 if arr.max() > 1 else 0)
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
@dataclass
|
| 23 |
+
class ConfusionCounts:
|
| 24 |
+
tp: int
|
| 25 |
+
fp: int
|
| 26 |
+
fn: int
|
| 27 |
+
tn: int
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def confusion_counts(pred: np.ndarray, gt: np.ndarray) -> ConfusionCounts:
|
| 31 |
+
"""Pixel-wise true/false positive/negative counts for two binary masks."""
|
| 32 |
+
p = _to_bool(pred)
|
| 33 |
+
g = _to_bool(gt)
|
| 34 |
+
if p.shape != g.shape:
|
| 35 |
+
raise ValueError(f"shape mismatch: pred {p.shape} vs gt {g.shape}")
|
| 36 |
+
tp = int(np.sum(p & g))
|
| 37 |
+
fp = int(np.sum(p & ~g))
|
| 38 |
+
fn = int(np.sum(~p & g))
|
| 39 |
+
tn = int(np.sum(~p & ~g))
|
| 40 |
+
return ConfusionCounts(tp=tp, fp=fp, fn=fn, tn=tn)
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def _safe_div(num: float, den: float) -> float:
|
| 44 |
+
return float(num) / float(den) if den else 0.0
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def binary_metrics(pred: np.ndarray, gt: np.ndarray) -> dict:
|
| 48 |
+
"""Return IoU, Dice/F1, precision, recall, accuracy, FPR and FNR.
|
| 49 |
+
|
| 50 |
+
Edge case: when ground truth has no positive pixels and the prediction is
|
| 51 |
+
also empty, IoU/Dice/precision/recall are defined as 1.0 (perfect).
|
| 52 |
+
"""
|
| 53 |
+
c = confusion_counts(pred, gt)
|
| 54 |
+
tp, fp, fn, tn = c.tp, c.fp, c.fn, c.tn
|
| 55 |
+
|
| 56 |
+
if (tp + fp + fn) == 0:
|
| 57 |
+
precision = recall = iou = dice = 1.0
|
| 58 |
+
else:
|
| 59 |
+
precision = _safe_div(tp, tp + fp)
|
| 60 |
+
recall = _safe_div(tp, tp + fn)
|
| 61 |
+
iou = _safe_div(tp, tp + fp + fn)
|
| 62 |
+
dice = _safe_div(2 * tp, 2 * tp + fp + fn)
|
| 63 |
+
|
| 64 |
+
f1 = _safe_div(2 * precision * recall, precision + recall) if (precision + recall) else dice
|
| 65 |
+
accuracy = _safe_div(tp + tn, tp + fp + fn + tn)
|
| 66 |
+
fpr = _safe_div(fp, fp + tn)
|
| 67 |
+
fnr = _safe_div(fn, fn + tp)
|
| 68 |
+
|
| 69 |
+
return {
|
| 70 |
+
"iou": round(iou, 4),
|
| 71 |
+
"dice": round(dice, 4),
|
| 72 |
+
"f1": round(f1, 4),
|
| 73 |
+
"precision": round(precision, 4),
|
| 74 |
+
"recall": round(recall, 4),
|
| 75 |
+
"pixelAccuracy": round(accuracy, 4),
|
| 76 |
+
"falsePositiveRate": round(fpr, 4),
|
| 77 |
+
"falseNegativeRate": round(fnr, 4),
|
| 78 |
+
"counts": asdict(c),
|
| 79 |
+
}
|
app/model_inference.py
CHANGED
|
@@ -169,76 +169,140 @@ def _unflip_map(score, op):
|
|
| 169 |
|
| 170 |
|
| 171 |
def _infer_score_map(img1, img2):
|
| 172 |
-
"""Single-pass tiled inference returning a float32 change-probability map at (h, w).
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 173 |
torch, _, _ = _try_import()
|
| 174 |
model, processor = _load_model()
|
| 175 |
from PIL import Image as PILImage
|
| 176 |
|
| 177 |
-
|
| 178 |
-
|
| 179 |
-
|
| 180 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 181 |
|
| 182 |
-
|
| 183 |
-
|
| 184 |
-
|
| 185 |
-
|
| 186 |
-
|
|
|
|
| 187 |
|
| 188 |
-
ph, pw = img1.shape[:2]
|
| 189 |
-
score_sum = np.zeros((ph, pw), dtype=np.float32)
|
| 190 |
-
count = np.zeros((ph, pw), dtype=np.float32)
|
| 191 |
|
| 192 |
-
|
| 193 |
-
|
| 194 |
-
|
| 195 |
-
|
|
|
|
| 196 |
|
| 197 |
-
with torch.no_grad():
|
| 198 |
-
for y0 in range(0, ph - tile + 1, stride):
|
| 199 |
-
for x0 in range(0, pw - tile + 1, stride):
|
| 200 |
-
t1 = np.ascontiguousarray(img1[y0:y0+tile, x0:x0+tile])
|
| 201 |
-
t2 = np.ascontiguousarray(img2[y0:y0+tile, x0:x0+tile])
|
| 202 |
|
| 203 |
-
|
| 204 |
-
|
|
|
|
| 205 |
|
| 206 |
-
|
| 207 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 208 |
|
| 209 |
-
|
| 210 |
-
logits = outputs.logits
|
| 211 |
-
prob_map = _logits_to_change_prob(logits, torch).cpu().numpy()
|
| 212 |
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
|
| 216 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 217 |
|
| 218 |
-
|
| 219 |
-
|
|
|
|
|
|
|
| 220 |
|
| 221 |
-
|
| 222 |
-
|
| 223 |
-
return avg_score[:h, :w]
|
| 224 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 225 |
|
| 226 |
-
|
| 227 |
-
|
| 228 |
-
Run AdaptFormer inference on two RGB numpy arrays (H, W, 3).
|
| 229 |
-
Averages predictions over test-time augmentation flips (DETECTION_TTA) for
|
| 230 |
-
higher-accuracy, less boundary-sensitive change maps.
|
| 231 |
-
Returns (uint8 mask [0 or 255], float32 score map [0-1]).
|
| 232 |
-
Use threshold > 1.0 to obtain score map only (empty mask).
|
| 233 |
-
"""
|
| 234 |
-
_load_model()
|
| 235 |
|
| 236 |
-
if img1.shape != img2.shape:
|
| 237 |
-
img2 = cv2.resize(img2, (img1.shape[1], img1.shape[0]))
|
| 238 |
|
|
|
|
|
|
|
| 239 |
h, w = img1.shape[:2]
|
| 240 |
ops = _resolve_tta_ops()
|
| 241 |
-
|
| 242 |
acc = np.zeros((h, w), dtype=np.float32)
|
| 243 |
n = 0
|
| 244 |
for op in ops:
|
|
@@ -251,10 +315,49 @@ def predict_change_mask(img1, img2, threshold=0.5):
|
|
| 251 |
continue
|
| 252 |
acc += _unflip_map(s, op)
|
| 253 |
n += 1
|
| 254 |
-
|
| 255 |
if n == 0:
|
| 256 |
raise RuntimeError("AdaptFormer inference produced no predictions")
|
| 257 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 258 |
|
| 259 |
mask = (avg_score >= threshold).astype(np.uint8) * 255
|
| 260 |
return mask, avg_score
|
|
|
|
| 169 |
|
| 170 |
|
| 171 |
def _infer_score_map(img1, img2):
|
| 172 |
+
"""Single-pass tiled inference returning a float32 change-probability map at (h, w).
|
| 173 |
+
|
| 174 |
+
The model always sees its native 256px patches; the shared tiler handles
|
| 175 |
+
padding, sliding windows and cosine blending so stitching stays consistent
|
| 176 |
+
across every deep model in the project.
|
| 177 |
+
"""
|
| 178 |
torch, _, _ = _try_import()
|
| 179 |
model, processor = _load_model()
|
| 180 |
from PIL import Image as PILImage
|
| 181 |
|
| 182 |
+
from .cd_models.model_utils import tiled_score_map
|
| 183 |
+
from .detection_config import get_tile_batch
|
| 184 |
+
|
| 185 |
+
def _score_tile(t1, t2):
|
| 186 |
+
pil1 = PILImage.fromarray(t1)
|
| 187 |
+
pil2 = PILImage.fromarray(t2)
|
| 188 |
+
inputs = processor(images=(pil1, pil2), return_tensors="pt")
|
| 189 |
+
inputs = {k: v.to(_DEVICE) for k, v in inputs.items()}
|
| 190 |
+
outputs = model(**inputs)
|
| 191 |
+
logits = outputs.logits
|
| 192 |
+
return _logits_to_change_prob(logits, torch).cpu().numpy()
|
| 193 |
+
|
| 194 |
+
def _score_batch(pairs):
|
| 195 |
+
# Stack each pair's processed tensors along the batch dim for one forward
|
| 196 |
+
# pass. Only used when the model's output batch dim matches the number of
|
| 197 |
+
# input pairs; otherwise fall back to correct per-tile scoring.
|
| 198 |
+
n = len(pairs)
|
| 199 |
+
try:
|
| 200 |
+
per_pair = [
|
| 201 |
+
processor(images=(PILImage.fromarray(t1), PILImage.fromarray(t2)),
|
| 202 |
+
return_tensors="pt")
|
| 203 |
+
for t1, t2 in pairs
|
| 204 |
+
]
|
| 205 |
+
keys = per_pair[0].keys()
|
| 206 |
+
per_pair_bs = per_pair[0][next(iter(keys))].shape[0]
|
| 207 |
+
stacked = {k: torch.cat([p[k] for p in per_pair], dim=0).to(_DEVICE)
|
| 208 |
+
for k in keys}
|
| 209 |
+
outputs = model(**stacked)
|
| 210 |
+
logits = outputs.logits
|
| 211 |
+
if logits.shape[0] != n * per_pair_bs:
|
| 212 |
+
raise RuntimeError("unexpected batched logits layout")
|
| 213 |
+
group = per_pair_bs
|
| 214 |
+
return [
|
| 215 |
+
_logits_to_change_prob(logits[i * group:(i + 1) * group], torch).cpu().numpy()
|
| 216 |
+
for i in range(n)
|
| 217 |
+
]
|
| 218 |
+
except Exception as exc:
|
| 219 |
+
logger.warning("Batched tile inference failed (%s); using per-tile", exc)
|
| 220 |
+
return [_score_tile(t1, t2) for t1, t2 in pairs]
|
| 221 |
|
| 222 |
+
batch = get_tile_batch()
|
| 223 |
+
with torch.no_grad():
|
| 224 |
+
return tiled_score_map(_score_tile, img1, img2,
|
| 225 |
+
tile_size=_TILE_SIZE, overlap=_TILE_SIZE // 4,
|
| 226 |
+
score_batch_fn=_score_batch if batch > 1 else None,
|
| 227 |
+
batch=batch)
|
| 228 |
|
|
|
|
|
|
|
|
|
|
| 229 |
|
| 230 |
+
def _count_windows(full_h, full_w, tile_size, overlap):
|
| 231 |
+
step = max(1, int(round(tile_size * (1.0 - overlap))))
|
| 232 |
+
ny = len(range(0, max(1, full_h - tile_size + 1), step)) + (1 if full_h > tile_size else 0)
|
| 233 |
+
nx = len(range(0, max(1, full_w - tile_size + 1), step)) + (1 if full_w > tile_size else 0)
|
| 234 |
+
return max(1, ny) * max(1, nx)
|
| 235 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 236 |
|
| 237 |
+
def predict_change_score_windowed(path_a, path_b, out_h, out_w,
|
| 238 |
+
tile_size=512, overlap=0.25, on_progress=None):
|
| 239 |
+
"""Build a change-probability map from two large GeoTIFFs via disk windows.
|
| 240 |
|
| 241 |
+
Reads paired native-resolution windows (so the model sees full detail),
|
| 242 |
+
scores each with AdaptFormer, then accumulates into a bounded
|
| 243 |
+
``(out_h, out_w)`` canvas with cosine blending. Peak memory stays at one
|
| 244 |
+
native window plus the small output canvas, so 10k+ rasters never OOM.
|
| 245 |
+
``on_progress(frac)`` (0..1) is called as windows complete.
|
| 246 |
+
Returns a float32 score map in [0, 1] at (out_h, out_w).
|
| 247 |
+
"""
|
| 248 |
+
from .dda.geotiff_io import iter_geotiff_window_pairs, read_native_size
|
| 249 |
|
| 250 |
+
_load_model()
|
|
|
|
|
|
|
| 251 |
|
| 252 |
+
score_sum = np.zeros((out_h, out_w), dtype=np.float32)
|
| 253 |
+
count = np.zeros((out_h, out_w), dtype=np.float32)
|
| 254 |
+
scale_y = scale_x = None
|
| 255 |
+
|
| 256 |
+
total = None
|
| 257 |
+
native = read_native_size(__import__("pathlib").Path(path_a))
|
| 258 |
+
if native:
|
| 259 |
+
total = _count_windows(native[1], native[0], tile_size, overlap)
|
| 260 |
+
done = 0
|
| 261 |
+
|
| 262 |
+
for tile_a, tile_b, y0, x0, full_h, full_w in iter_geotiff_window_pairs(
|
| 263 |
+
path_a, path_b, tile_size=tile_size, overlap=overlap):
|
| 264 |
+
if scale_y is None:
|
| 265 |
+
scale_y = out_h / float(full_h)
|
| 266 |
+
scale_x = out_w / float(full_w)
|
| 267 |
+
if total is None:
|
| 268 |
+
total = _count_windows(full_h, full_w, tile_size, overlap)
|
| 269 |
+
|
| 270 |
+
wh, ww = tile_a.shape[:2]
|
| 271 |
+
score = _infer_score_map(tile_a, tile_b)
|
| 272 |
+
|
| 273 |
+
dy0 = int(round(y0 * scale_y))
|
| 274 |
+
dx0 = int(round(x0 * scale_x))
|
| 275 |
+
dh = max(1, int(round(wh * scale_y)))
|
| 276 |
+
dw = max(1, int(round(ww * scale_x)))
|
| 277 |
+
dy1 = min(out_h, dy0 + dh)
|
| 278 |
+
dx1 = min(out_w, dx0 + dw)
|
| 279 |
+
dh, dw = dy1 - dy0, dx1 - dx0
|
| 280 |
+
if dh <= 0 or dw <= 0:
|
| 281 |
+
continue
|
| 282 |
|
| 283 |
+
score_ds = cv2.resize(score, (dw, dh), interpolation=cv2.INTER_AREA)
|
| 284 |
+
wy = np.hanning(dh + 2)[1:-1] if dh > 2 else np.ones(dh)
|
| 285 |
+
wx = np.hanning(dw + 2)[1:-1] if dw > 2 else np.ones(dw)
|
| 286 |
+
weight = np.maximum(np.outer(wy, wx).astype(np.float32), 1e-3)
|
| 287 |
|
| 288 |
+
score_sum[dy0:dy1, dx0:dx1] += score_ds * weight
|
| 289 |
+
count[dy0:dy1, dx0:dx1] += weight
|
|
|
|
| 290 |
|
| 291 |
+
done += 1
|
| 292 |
+
if on_progress and total:
|
| 293 |
+
try:
|
| 294 |
+
on_progress(min(1.0, done / float(total)))
|
| 295 |
+
except Exception:
|
| 296 |
+
pass
|
| 297 |
|
| 298 |
+
count = np.maximum(count, 1e-6)
|
| 299 |
+
return score_sum / count
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 300 |
|
|
|
|
|
|
|
| 301 |
|
| 302 |
+
def _predict_score_tta(img1, img2):
|
| 303 |
+
"""TTA-averaged change score for one (already same-size) image pair."""
|
| 304 |
h, w = img1.shape[:2]
|
| 305 |
ops = _resolve_tta_ops()
|
|
|
|
| 306 |
acc = np.zeros((h, w), dtype=np.float32)
|
| 307 |
n = 0
|
| 308 |
for op in ops:
|
|
|
|
| 315 |
continue
|
| 316 |
acc += _unflip_map(s, op)
|
| 317 |
n += 1
|
|
|
|
| 318 |
if n == 0:
|
| 319 |
raise RuntimeError("AdaptFormer inference produced no predictions")
|
| 320 |
+
return acc / float(n)
|
| 321 |
+
|
| 322 |
+
|
| 323 |
+
def predict_change_mask(img1, img2, threshold=0.5):
|
| 324 |
+
"""
|
| 325 |
+
Run AdaptFormer inference on two RGB numpy arrays (H, W, 3).
|
| 326 |
+
Averages predictions over test-time augmentation flips (DETECTION_TTA) and,
|
| 327 |
+
when DETECTION_MULTISCALE is set, fuses scores across scales (max) so both
|
| 328 |
+
small and large changes are captured.
|
| 329 |
+
Returns (uint8 mask [0 or 255], float32 score map [0-1]).
|
| 330 |
+
Use threshold > 1.0 to obtain score map only (empty mask).
|
| 331 |
+
"""
|
| 332 |
+
_load_model()
|
| 333 |
+
|
| 334 |
+
if img1.shape != img2.shape:
|
| 335 |
+
img2 = cv2.resize(img2, (img1.shape[1], img1.shape[0]))
|
| 336 |
+
|
| 337 |
+
h, w = img1.shape[:2]
|
| 338 |
+
|
| 339 |
+
from .detection_config import get_multiscale_scales
|
| 340 |
+
scales = get_multiscale_scales()
|
| 341 |
+
|
| 342 |
+
if not scales:
|
| 343 |
+
avg_score = _predict_score_tta(img1, img2)
|
| 344 |
+
else:
|
| 345 |
+
fused = None
|
| 346 |
+
for scale in scales:
|
| 347 |
+
if scale == 1.0:
|
| 348 |
+
s1, s2 = img1, img2
|
| 349 |
+
else:
|
| 350 |
+
nh = max(64, int(round(h * scale)))
|
| 351 |
+
nw = max(64, int(round(w * scale)))
|
| 352 |
+
interp = cv2.INTER_AREA if scale < 1.0 else cv2.INTER_CUBIC
|
| 353 |
+
s1 = cv2.resize(img1, (nw, nh), interpolation=interp)
|
| 354 |
+
s2 = cv2.resize(img2, (nw, nh), interpolation=interp)
|
| 355 |
+
score = _predict_score_tta(s1, s2)
|
| 356 |
+
if score.shape != (h, w):
|
| 357 |
+
score = cv2.resize(score, (w, h), interpolation=cv2.INTER_LINEAR)
|
| 358 |
+
# Max fusion favors recall on small structures that only one scale sees
|
| 359 |
+
fused = score if fused is None else np.maximum(fused, score)
|
| 360 |
+
avg_score = fused
|
| 361 |
|
| 362 |
mask = (avg_score >= threshold).astype(np.uint8) * 255
|
| 363 |
return mask, avg_score
|
scripts/validate_detection.py
CHANGED
|
@@ -1,7 +1,18 @@
|
|
| 1 |
"""
|
| 2 |
-
Lightweight validation for the change detection pipeline.
|
| 3 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4 |
"""
|
|
|
|
|
|
|
| 5 |
import sys
|
| 6 |
from pathlib import Path
|
| 7 |
|
|
@@ -16,6 +27,7 @@ from app.detection_engine import ( # noqa: E402
|
|
| 16 |
run_detection,
|
| 17 |
fuse_dl_and_classical,
|
| 18 |
)
|
|
|
|
| 19 |
|
| 20 |
|
| 21 |
def test_registration_identical_pair():
|
|
@@ -69,7 +81,105 @@ def test_run_detection_synthetic():
|
|
| 69 |
print(" run_detection: change%=", f"{ratio:.2f}", "regions=", len(regions))
|
| 70 |
|
| 71 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 72 |
def main():
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 73 |
print("validate_detection.py")
|
| 74 |
test_registration_identical_pair()
|
| 75 |
test_registration_with_shift()
|
|
@@ -77,6 +187,10 @@ def main():
|
|
| 77 |
test_run_detection_synthetic()
|
| 78 |
print("All checks passed.")
|
| 79 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 80 |
|
| 81 |
if __name__ == "__main__":
|
| 82 |
main()
|
|
|
|
| 1 |
"""
|
| 2 |
+
Lightweight validation + accuracy benchmark for the change detection pipeline.
|
| 3 |
+
|
| 4 |
+
Run from change_detection_webapp:
|
| 5 |
+
python scripts/validate_detection.py # unit checks
|
| 6 |
+
python scripts/validate_detection.py --benchmark # synthetic IoU/Dice/F1 report
|
| 7 |
+
python scripts/validate_detection.py --benchmark --out runs/eval # + comparison PNGs
|
| 8 |
+
|
| 9 |
+
Benchmarks run on synthetic pairs with known ground-truth change masks, so they
|
| 10 |
+
work even without labeled imagery (per project decision: synthetic + review
|
| 11 |
+
labels for now). The same harness is used to A/B preprocessing and fusion
|
| 12 |
+
toggles by setting the relevant DETECTION_* env vars before running.
|
| 13 |
"""
|
| 14 |
+
import argparse
|
| 15 |
+
import json
|
| 16 |
import sys
|
| 17 |
from pathlib import Path
|
| 18 |
|
|
|
|
| 27 |
run_detection,
|
| 28 |
fuse_dl_and_classical,
|
| 29 |
)
|
| 30 |
+
from app.evaluation.metrics import binary_metrics # noqa: E402
|
| 31 |
|
| 32 |
|
| 33 |
def test_registration_identical_pair():
|
|
|
|
| 81 |
print(" run_detection: change%=", f"{ratio:.2f}", "regions=", len(regions))
|
| 82 |
|
| 83 |
|
| 84 |
+
# ---------------------------------------------------------------------------
|
| 85 |
+
# Synthetic benchmark suite (known ground-truth masks)
|
| 86 |
+
# ---------------------------------------------------------------------------
|
| 87 |
+
|
| 88 |
+
def _base_scene(size=384, seed=0):
|
| 89 |
+
"""A textured pseudo-aerial scene so registration/feature matching has signal."""
|
| 90 |
+
rng = np.random.default_rng(seed)
|
| 91 |
+
img = rng.integers(40, 200, (size, size, 3), dtype=np.uint8)
|
| 92 |
+
img = np.array(Image.fromarray(img).resize((size, size)))
|
| 93 |
+
# A few stable structures (roads / fields) common to both timestamps
|
| 94 |
+
img[:, size // 3: size // 3 + 6] = [90, 90, 90]
|
| 95 |
+
img[size // 2: size // 2 + 6, :] = [110, 100, 80]
|
| 96 |
+
return img
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
def _case_inserted_buildings(size=384):
|
| 100 |
+
before = _base_scene(size, seed=1)
|
| 101 |
+
after = before.copy()
|
| 102 |
+
gt = np.zeros((size, size), dtype=np.uint8)
|
| 103 |
+
boxes = [(60, 70, 50, 40), (220, 90, 60, 55), (150, 250, 70, 45)]
|
| 104 |
+
for (x, y, w, h) in boxes:
|
| 105 |
+
after[y:y + h, x:x + w] = [205, 200, 190]
|
| 106 |
+
gt[y:y + h, x:x + w] = 255
|
| 107 |
+
return before, after, gt, "inserted_buildings"
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
def _case_brightness_only(size=384):
|
| 111 |
+
# Global illumination change with NO structural change -> GT empty, expect low FP.
|
| 112 |
+
before = _base_scene(size, seed=2)
|
| 113 |
+
after = np.clip(before.astype(np.float32) * 1.18 + 12, 0, 255).astype(np.uint8)
|
| 114 |
+
gt = np.zeros((size, size), dtype=np.uint8)
|
| 115 |
+
return before, after, gt, "brightness_only"
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
def _case_misaligned_change(size=384):
|
| 119 |
+
before = _base_scene(size, seed=3)
|
| 120 |
+
shifted = np.roll(np.roll(before, 6, axis=0), 4, axis=1)
|
| 121 |
+
after = shifted.copy()
|
| 122 |
+
gt = np.zeros((size, size), dtype=np.uint8)
|
| 123 |
+
x, y, w, h = 180, 160, 80, 60
|
| 124 |
+
after[y:y + h, x:x + w] = [210, 60, 60]
|
| 125 |
+
gt[y:y + h, x:x + w] = 255
|
| 126 |
+
return before, after, gt, "misaligned_change"
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
def _save_comparison(out_dir: Path, name: str, before, after, pred_mask, gt):
|
| 130 |
+
out_dir.mkdir(parents=True, exist_ok=True)
|
| 131 |
+
h, w = gt.shape
|
| 132 |
+
pred = (pred_mask > 127).astype(np.uint8) * 255
|
| 133 |
+
overlay = after.copy()
|
| 134 |
+
overlay[pred > 0] = (0.45 * overlay[pred > 0] + 0.55 * np.array([255, 40, 40])).astype(np.uint8)
|
| 135 |
+
diff = np.zeros((h, w, 3), dtype=np.uint8)
|
| 136 |
+
diff[(pred > 0) & (gt > 0)] = [0, 200, 0] # true positive
|
| 137 |
+
diff[(pred > 0) & (gt == 0)] = [255, 0, 0] # false positive
|
| 138 |
+
diff[(pred == 0) & (gt > 0)] = [0, 0, 255] # false negative
|
| 139 |
+
panels = [
|
| 140 |
+
before, after,
|
| 141 |
+
np.dstack([gt] * 3), overlay, diff,
|
| 142 |
+
]
|
| 143 |
+
strip = np.concatenate([np.asarray(p, dtype=np.uint8) for p in panels], axis=1)
|
| 144 |
+
Image.fromarray(strip).save(out_dir / f"{name}.png")
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
def benchmark_synthetic(out_dir: Path | None = None, sensitivity=0.5):
|
| 148 |
+
cases = [_case_inserted_buildings(), _case_brightness_only(), _case_misaligned_change()]
|
| 149 |
+
report = {}
|
| 150 |
+
print("\nSynthetic benchmark (IoU / Dice / F1 / Precision / Recall):")
|
| 151 |
+
for before, after, gt, name in cases:
|
| 152 |
+
mask, _img, stats, regions = run_detection(
|
| 153 |
+
Image.fromarray(before), Image.fromarray(after),
|
| 154 |
+
method="AI-Based Deep Learning",
|
| 155 |
+
enable_registration=True, enable_normalization=True,
|
| 156 |
+
detection_sensitivity=sensitivity,
|
| 157 |
+
)
|
| 158 |
+
if mask.shape != gt.shape:
|
| 159 |
+
from cv2 import resize, INTER_NEAREST
|
| 160 |
+
mask = resize(mask, (gt.shape[1], gt.shape[0]), interpolation=INTER_NEAREST)
|
| 161 |
+
m = binary_metrics(mask, gt)
|
| 162 |
+
report[name] = {"metrics": m, "regions": len(regions),
|
| 163 |
+
"changePct": round(stats["change_percentage"], 3)}
|
| 164 |
+
print(f" {name:20s} IoU={m['iou']:.3f} Dice={m['dice']:.3f} "
|
| 165 |
+
f"F1={m['f1']:.3f} P={m['precision']:.3f} R={m['recall']:.3f} "
|
| 166 |
+
f"FPR={m['falsePositiveRate']:.3f}")
|
| 167 |
+
if out_dir is not None:
|
| 168 |
+
_save_comparison(out_dir, name, before, after, mask, gt)
|
| 169 |
+
|
| 170 |
+
if out_dir is not None:
|
| 171 |
+
(out_dir / "metrics.json").write_text(json.dumps(report, indent=2), encoding="utf-8")
|
| 172 |
+
print(f"\n Wrote comparison images + metrics.json to {out_dir}")
|
| 173 |
+
return report
|
| 174 |
+
|
| 175 |
+
|
| 176 |
def main():
|
| 177 |
+
parser = argparse.ArgumentParser(description="Validate + benchmark change detection")
|
| 178 |
+
parser.add_argument("--benchmark", action="store_true", help="run synthetic accuracy benchmark")
|
| 179 |
+
parser.add_argument("--out", type=str, default="", help="dir for comparison images + metrics.json")
|
| 180 |
+
parser.add_argument("--sensitivity", type=float, default=0.5)
|
| 181 |
+
args = parser.parse_args()
|
| 182 |
+
|
| 183 |
print("validate_detection.py")
|
| 184 |
test_registration_identical_pair()
|
| 185 |
test_registration_with_shift()
|
|
|
|
| 187 |
test_run_detection_synthetic()
|
| 188 |
print("All checks passed.")
|
| 189 |
|
| 190 |
+
if args.benchmark:
|
| 191 |
+
out_dir = Path(args.out).resolve() if args.out else None
|
| 192 |
+
benchmark_synthetic(out_dir=out_dir, sensitivity=args.sensitivity)
|
| 193 |
+
|
| 194 |
|
| 195 |
if __name__ == "__main__":
|
| 196 |
main()
|
static/js/dda/compare.js
CHANGED
|
@@ -383,7 +383,8 @@ async function runDetectionWithFallback(form) {
|
|
| 383 |
} catch (err) {
|
| 384 |
const msg = String(err.message || '');
|
| 385 |
const useSync = msg.includes('Not Found') || msg.includes('404')
|
| 386 |
-
|| msg.includes('503') || msg.includes('409') || msg.includes('busy')
|
|
|
|
| 387 |
if (!useSync) throw err;
|
| 388 |
return runSyncDetectionWithProgress(form);
|
| 389 |
}
|
|
|
|
| 383 |
} catch (err) {
|
| 384 |
const msg = String(err.message || '');
|
| 385 |
const useSync = msg.includes('Not Found') || msg.includes('404')
|
| 386 |
+
|| msg.includes('503') || msg.includes('409') || msg.includes('busy')
|
| 387 |
+
|| msg.includes('Internal Server Error') || msg.includes('NOT NULL');
|
| 388 |
if (!useSync) throw err;
|
| 389 |
return runSyncDetectionWithProgress(form);
|
| 390 |
}
|
templates/index_dda.html
CHANGED
|
@@ -326,7 +326,7 @@
|
|
| 326 |
<script src="/static/js/dda/tree.js?v=2"></script>
|
| 327 |
<script src="/static/js/dda/library.js?v=10"></script>
|
| 328 |
<script src="/static/js/dda/result.js?v=7"></script>
|
| 329 |
-
<script src="/static/js/dda/compare.js?v=
|
| 330 |
<script src="/static/js/dda/reports.js?v=4"></script>
|
| 331 |
<script src="/static/js/dda/notifications.js?v=1"></script>
|
| 332 |
</body>
|
|
|
|
| 326 |
<script src="/static/js/dda/tree.js?v=2"></script>
|
| 327 |
<script src="/static/js/dda/library.js?v=10"></script>
|
| 328 |
<script src="/static/js/dda/result.js?v=7"></script>
|
| 329 |
+
<script src="/static/js/dda/compare.js?v=13"></script>
|
| 330 |
<script src="/static/js/dda/reports.js?v=4"></script>
|
| 331 |
<script src="/static/js/dda/notifications.js?v=1"></script>
|
| 332 |
</body>
|