AMSI-Net / backend /explainers.py
numanabubakar's picture
Initialize AMSI-Net Multi-label Backend with Explainability
02b6b5d
Raw History Blame Contribute Delete
5.26 kB
"""Explainability module for AMSI-Net LULC model."""
import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
import cv2
import base64
from io import BytesIO
import PIL.Image
from pytorch_grad_cam import GradCAM, GradCAMPlusPlus
from pytorch_grad_cam.utils.image import show_cam_on_image
from lime import lime_image
from skimage.segmentation import mark_boundaries
class ModelWrapper(nn.Module):
"""Wrapper to make AMSI-Net return only logits for explainability tools."""
def __init__(self, model):
super().__init__()
self.model = model
def forward(self, x):
outputs = self.model(x)
return outputs['logits']
def array_to_base64(img_array):
"""Convert numpy RGB array to base64 jpeg string."""
# Convert RGB to BGR for cv2
img_bgr = cv2.cvtColor(img_array, cv2.COLOR_RGB2BGR)
_, buffer = cv2.imencode('.jpg', img_bgr)
return base64.b64encode(buffer).decode('utf-8')
def get_saliency_map(model, image_tensor):
"""Generate vanilla Saliency map."""
x = image_tensor.clone()
x.requires_grad = True
model.eval()
# Forward pass
outputs = model(x)
logits = outputs['logits']
# Get top class
score, indices = torch.max(logits, 1)
# Backward pass
model.zero_grad()
score.backward()
# Saliency map = max absolute gradient across channels
saliency, _ = torch.max(x.grad.data.abs(), dim=1)
saliency = saliency.squeeze().cpu().numpy()
# Normalize
saliency = (saliency - saliency.min()) / (saliency.max() - saliency.min() + 1e-8)
# Apply colormap
saliency_heatmap = cv2.applyColorMap(np.uint8(255 * saliency), cv2.COLORMAP_JET)
saliency_heatmap = cv2.cvtColor(saliency_heatmap, cv2.COLOR_BGR2RGB)
saliency_heatmap = cv2.resize(saliency_heatmap, (224, 224))
return array_to_base64(saliency_heatmap)
def get_gradcam_maps(model, image_tensor, orig_image_np):
"""Generate GradCAM and GradCAM++."""
# Target the last convolutional layer of the ResNet backbone in AMSI-Net
# In AMSI-Net, backbone.layer4 is the final block of the ResNet parts
target_layers = [model.layer4[-1]]
wrapper = ModelWrapper(model)
# 1. GradCAM
cam_algo = GradCAM(model=wrapper, target_layers=target_layers)
grayscale_cam = cam_algo(input_tensor=image_tensor, targets=None)[0]
orig_img_norm = orig_image_np.astype(np.float32) / 255.0
cam_image = show_cam_on_image(orig_img_norm, grayscale_cam, use_rgb=True)
# 2. GradCAM++
cam_plus_algo = GradCAMPlusPlus(model=wrapper, target_layers=target_layers)
grayscale_cam_plus = cam_plus_algo(input_tensor=image_tensor, targets=None)[0]
cam_plus_image = show_cam_on_image(orig_img_norm, grayscale_cam_plus, use_rgb=True)
return array_to_base64(cam_image), array_to_base64(cam_plus_image)
def get_lime_map(model, preprocessor, image_bytes, orig_image_np, device):
"""Generate LIME explanation."""
explainer = lime_image.LimeImageExplainer()
def predict_fn(images):
tensors = []
for img_arr in images:
# Re-apply preprocessing to each LIME perturbation
img_pil = PIL.Image.fromarray(img_arr)
tensor = preprocessor.transform(img_pil).to(device)
tensors.append(tensor)
batch_tensor = torch.stack(tensors)
with torch.no_grad():
outputs = model(batch_tensor)
probs = torch.sigmoid(outputs['logits']) # Using sigmoid for multi-label
return probs.cpu().numpy()
explanation = explainer.explain_instance(
orig_image_np,
predict_fn,
top_labels=1,
hide_color=0,
num_samples=100
)
temp, mask = explanation.get_image_and_mask(
explanation.top_labels[0],
positive_only=True,
num_features=5,
hide_rest=False
)
img_boundary = mark_boundaries(temp / 255.0, mask)
img_boundary_uint8 = np.uint8(img_boundary * 255)
return array_to_base64(img_boundary_uint8)
def generate_all_explanations(model, preprocessor, image_tensor, image_bytes):
"""Generate all explainability maps."""
device = image_tensor.device
orig_pil = PIL.Image.open(BytesIO(image_bytes)).convert("RGB")
orig_pil = orig_pil.resize((224, 224))
orig_np = np.array(orig_pil)
result = {}
# Saliency
try:
saliency_b64 = get_saliency_map(model, image_tensor)
result["Saliency"] = f"data:image/jpeg;base64,{saliency_b64}"
except Exception as e:
print(f"Error generating Saliency: {e}")
# GradCAM
try:
gc_b64, gcp_b64 = get_gradcam_maps(model, image_tensor, orig_np)
result["GradCAM"] = f"data:image/jpeg;base64,{gc_b64}"
result["GradCAM++"] = f"data:image/jpeg;base64,{gcp_b64}"
except Exception as e:
print(f"Error generating GradCAM: {e}")
# LIME
try:
lime_b64 = get_lime_map(model, preprocessor, image_bytes, orig_np, device)
result["LIME"] = f"data:image/jpeg;base64,{lime_b64}"
except Exception as e:
print(f"Error generating LIME: {e}")
return result