Spaces:
Sleeping
Sleeping
File size: 5,256 Bytes
02b6b5d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 | """Explainability module for AMSI-Net LULC model."""
import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
import cv2
import base64
from io import BytesIO
import PIL.Image
from pytorch_grad_cam import GradCAM, GradCAMPlusPlus
from pytorch_grad_cam.utils.image import show_cam_on_image
from lime import lime_image
from skimage.segmentation import mark_boundaries
class ModelWrapper(nn.Module):
"""Wrapper to make AMSI-Net return only logits for explainability tools."""
def __init__(self, model):
super().__init__()
self.model = model
def forward(self, x):
outputs = self.model(x)
return outputs['logits']
def array_to_base64(img_array):
"""Convert numpy RGB array to base64 jpeg string."""
# Convert RGB to BGR for cv2
img_bgr = cv2.cvtColor(img_array, cv2.COLOR_RGB2BGR)
_, buffer = cv2.imencode('.jpg', img_bgr)
return base64.b64encode(buffer).decode('utf-8')
def get_saliency_map(model, image_tensor):
"""Generate vanilla Saliency map."""
x = image_tensor.clone()
x.requires_grad = True
model.eval()
# Forward pass
outputs = model(x)
logits = outputs['logits']
# Get top class
score, indices = torch.max(logits, 1)
# Backward pass
model.zero_grad()
score.backward()
# Saliency map = max absolute gradient across channels
saliency, _ = torch.max(x.grad.data.abs(), dim=1)
saliency = saliency.squeeze().cpu().numpy()
# Normalize
saliency = (saliency - saliency.min()) / (saliency.max() - saliency.min() + 1e-8)
# Apply colormap
saliency_heatmap = cv2.applyColorMap(np.uint8(255 * saliency), cv2.COLORMAP_JET)
saliency_heatmap = cv2.cvtColor(saliency_heatmap, cv2.COLOR_BGR2RGB)
saliency_heatmap = cv2.resize(saliency_heatmap, (224, 224))
return array_to_base64(saliency_heatmap)
def get_gradcam_maps(model, image_tensor, orig_image_np):
"""Generate GradCAM and GradCAM++."""
# Target the last convolutional layer of the ResNet backbone in AMSI-Net
# In AMSI-Net, backbone.layer4 is the final block of the ResNet parts
target_layers = [model.layer4[-1]]
wrapper = ModelWrapper(model)
# 1. GradCAM
cam_algo = GradCAM(model=wrapper, target_layers=target_layers)
grayscale_cam = cam_algo(input_tensor=image_tensor, targets=None)[0]
orig_img_norm = orig_image_np.astype(np.float32) / 255.0
cam_image = show_cam_on_image(orig_img_norm, grayscale_cam, use_rgb=True)
# 2. GradCAM++
cam_plus_algo = GradCAMPlusPlus(model=wrapper, target_layers=target_layers)
grayscale_cam_plus = cam_plus_algo(input_tensor=image_tensor, targets=None)[0]
cam_plus_image = show_cam_on_image(orig_img_norm, grayscale_cam_plus, use_rgb=True)
return array_to_base64(cam_image), array_to_base64(cam_plus_image)
def get_lime_map(model, preprocessor, image_bytes, orig_image_np, device):
"""Generate LIME explanation."""
explainer = lime_image.LimeImageExplainer()
def predict_fn(images):
tensors = []
for img_arr in images:
# Re-apply preprocessing to each LIME perturbation
img_pil = PIL.Image.fromarray(img_arr)
tensor = preprocessor.transform(img_pil).to(device)
tensors.append(tensor)
batch_tensor = torch.stack(tensors)
with torch.no_grad():
outputs = model(batch_tensor)
probs = torch.sigmoid(outputs['logits']) # Using sigmoid for multi-label
return probs.cpu().numpy()
explanation = explainer.explain_instance(
orig_image_np,
predict_fn,
top_labels=1,
hide_color=0,
num_samples=100
)
temp, mask = explanation.get_image_and_mask(
explanation.top_labels[0],
positive_only=True,
num_features=5,
hide_rest=False
)
img_boundary = mark_boundaries(temp / 255.0, mask)
img_boundary_uint8 = np.uint8(img_boundary * 255)
return array_to_base64(img_boundary_uint8)
def generate_all_explanations(model, preprocessor, image_tensor, image_bytes):
"""Generate all explainability maps."""
device = image_tensor.device
orig_pil = PIL.Image.open(BytesIO(image_bytes)).convert("RGB")
orig_pil = orig_pil.resize((224, 224))
orig_np = np.array(orig_pil)
result = {}
# Saliency
try:
saliency_b64 = get_saliency_map(model, image_tensor)
result["Saliency"] = f"data:image/jpeg;base64,{saliency_b64}"
except Exception as e:
print(f"Error generating Saliency: {e}")
# GradCAM
try:
gc_b64, gcp_b64 = get_gradcam_maps(model, image_tensor, orig_np)
result["GradCAM"] = f"data:image/jpeg;base64,{gc_b64}"
result["GradCAM++"] = f"data:image/jpeg;base64,{gcp_b64}"
except Exception as e:
print(f"Error generating GradCAM: {e}")
# LIME
try:
lime_b64 = get_lime_map(model, preprocessor, image_bytes, orig_np, device)
result["LIME"] = f"data:image/jpeg;base64,{lime_b64}"
except Exception as e:
print(f"Error generating LIME: {e}")
return result
|