Spaces:
Sleeping
Sleeping
File size: 6,197 Bytes
ada7b0f d298346 ada7b0f d298346 ada7b0f 9163dd2 ada7b0f 9163dd2 ada7b0f 9163dd2 ada7b0f 0c81eb2 ada7b0f 0c81eb2 ada7b0f 0c81eb2 ada7b0f d298346 ada7b0f d298346 ada7b0f d298346 ada7b0f d298346 ada7b0f d298346 ada7b0f d298346 ada7b0f 0c81eb2 ada7b0f d298346 ada7b0f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 | """Explainability module for LULC models."""
import torch
import torch.nn.functional as F
import numpy as np
import cv2
import base64
import matplotlib.pyplot as plt
from io import BytesIO
import PIL.Image
from pytorch_grad_cam import GradCAM, GradCAMPlusPlus
from pytorch_grad_cam.utils.image import show_cam_on_image
from lime import lime_image
from skimage.segmentation import mark_boundaries
import logging
# Set up logging
logger = logging.getLogger("lulc-explainers")
def array_to_base64(img_array):
"""Convert numpy RGB array to base64 jpeg."""
# Convert RGB to BGR for cv2
img_bgr = cv2.cvtColor(img_array, cv2.COLOR_RGB2BGR)
_, buffer = cv2.imencode('.jpg', img_bgr)
return base64.b64encode(buffer).decode('utf-8')
def get_saliency_map(model, image_tensor):
"""Generate vanilla Saliency map."""
# Make a copy attached to gradients
x = image_tensor.clone()
x.requires_grad = True
model.eval()
# Forward pass
logits = model(x)
# Get top class
score, indices = torch.max(logits, 1)
# Backward pass
model.zero_grad()
score.backward()
# Saliency map = max absolute gradient across channels
saliency, _ = torch.max(x.grad.data.abs(), dim=1)
saliency = saliency.squeeze().cpu().numpy()
# Normalize to 0-1
saliency = (saliency - saliency.min()) / (saliency.max() - saliency.min() + 1e-8)
# Apply colormap (jet)
saliency_heatmap = cv2.applyColorMap(np.uint8(255 * saliency), cv2.COLORMAP_JET)
saliency_heatmap = cv2.cvtColor(saliency_heatmap, cv2.COLOR_BGR2RGB)
# Resize to original tensor size for consistency
_, _, H, W = image_tensor.shape
saliency_heatmap = cv2.resize(saliency_heatmap, (W, H))
return array_to_base64(saliency_heatmap)
def get_gradcam_maps(model, image_tensor, orig_image_np):
"""Generate GradCAM and GradCAM++."""
# Target layer for AMFRNet is the last block
target_layers = [model.block3]
# Normalize original image for overlay
orig_img_norm = orig_image_np.astype(np.float32) / 255.0
h, w = orig_img_norm.shape[:2]
# 1. GradCAM
cam_algo = GradCAM(model=model, target_layers=target_layers)
grayscale_cam = cam_algo(input_tensor=image_tensor, targets=None)[0]
# Force resize grayscale_cam to match orig_img_norm exactly to avoid broadcast errors
if grayscale_cam.shape[0] != h or grayscale_cam.shape[1] != w:
logger.warning(f"[explainers] Shape mismatch! Resizing GradCAM heatmap from {grayscale_cam.shape} to ({h}, {w})")
grayscale_cam = cv2.resize(grayscale_cam, (w, h))
cam_image = show_cam_on_image(orig_img_norm, grayscale_cam, use_rgb=True)
# 2. GradCAM++
cam_plus_algo = GradCAMPlusPlus(model=model, target_layers=target_layers)
grayscale_cam_plus = cam_plus_algo(input_tensor=image_tensor, targets=None)[0]
if grayscale_cam_plus.shape[0] != h or grayscale_cam_plus.shape[1] != w:
grayscale_cam_plus = cv2.resize(grayscale_cam_plus, (w, h))
cam_plus_image = show_cam_on_image(orig_img_norm, grayscale_cam_plus, use_rgb=True)
return array_to_base64(cam_image), array_to_base64(cam_plus_image)
def get_lime_map(model, preprocessor, image_bytes, orig_image_np, device, model_type):
"""Generate LIME explanation."""
explainer = lime_image.LimeImageExplainer()
def predict_fn(images):
tensors = []
for img_arr in images:
img_pil = PIL.Image.fromarray(img_arr)
transform = preprocessor.get_transforms(model_type, img_pil.width, img_pil.height)
tensor = transform(img_pil).to(device)
tensors.append(tensor)
batch_tensor = torch.stack(tensors)
with torch.no_grad():
logits = model(batch_tensor)
probs = torch.softmax(logits, dim=1)
return probs.cpu().numpy()
# Run explanation (num_samples=100 for decent speed, default 1000 is too slow for API)
explanation = explainer.explain_instance(
orig_image_np,
predict_fn,
top_labels=1,
hide_color=0,
num_samples=150
)
temp, mask = explanation.get_image_and_mask(
explanation.top_labels[0],
positive_only=True,
num_features=5,
hide_rest=False
)
img_boundry = mark_boundaries(temp / 255.0, mask)
img_boundry_uint8 = np.uint8(img_boundry * 255)
return array_to_base64(img_boundry_uint8)
def generate_all_explanations(model, preprocessor, image_tensor, image_bytes, model_type):
"""Wrapper function to generate all maps and return a dict of base64 strings."""
device = image_tensor.device
# Dynamically determine the target size from the image tensor (1, 3, H, W)
_, _, H, W = image_tensor.shape
target_size = (W, H) # PIL uses (width, height)
logger.info(f"[explainers] Target size for explanations: {target_size}")
# Original image decoding for overlay
orig_pil = PIL.Image.open(BytesIO(image_bytes)).convert("RGB")
orig_pil = orig_pil.resize(target_size)
orig_np = np.array(orig_pil)
result = {}
try:
logger.info("[explainers] generating Saliency...")
saliency_b64 = get_saliency_map(model, image_tensor)
result["Saliency"] = f"data:image/jpeg;base64,{saliency_b64}"
except Exception as e:
logger.error(f"[explainers] Saliency failed: {e}")
try:
logger.info("[explainers] generating GradCAM & GradCAM++...")
gc_b64, gcp_b64 = get_gradcam_maps(model, image_tensor, orig_np)
result["GradCAM"] = f"data:image/jpeg;base64,{gc_b64}"
result["GradCAM++"] = f"data:image/jpeg;base64,{gcp_b64}"
except Exception as e:
logger.error(f"[explainers] GradCAM failed: {e}")
try:
print("[explainers] generating LIME...")
lime_b64 = get_lime_map(model, preprocessor, image_bytes, orig_np, device, model_type)
result["LIME"] = f"data:image/jpeg;base64,{lime_b64}"
except Exception as e:
logger.error(f"[explainers] LIME failed: {e}")
return result
|