"""Explainability module for LULC models.""" import torch import torch.nn.functional as F import numpy as np import cv2 import base64 import matplotlib.pyplot as plt from io import BytesIO import PIL.Image from pytorch_grad_cam import GradCAM, GradCAMPlusPlus from pytorch_grad_cam.utils.image import show_cam_on_image from lime import lime_image from skimage.segmentation import mark_boundaries import logging # Set up logging logger = logging.getLogger("lulc-explainers") def array_to_base64(img_array): """Convert numpy RGB array to base64 jpeg.""" # Convert RGB to BGR for cv2 img_bgr = cv2.cvtColor(img_array, cv2.COLOR_RGB2BGR) _, buffer = cv2.imencode('.jpg', img_bgr) return base64.b64encode(buffer).decode('utf-8') def get_saliency_map(model, image_tensor): """Generate vanilla Saliency map.""" # Make a copy attached to gradients x = image_tensor.clone() x.requires_grad = True model.eval() # Forward pass logits = model(x) # Get top class score, indices = torch.max(logits, 1) # Backward pass model.zero_grad() score.backward() # Saliency map = max absolute gradient across channels saliency, _ = torch.max(x.grad.data.abs(), dim=1) saliency = saliency.squeeze().cpu().numpy() # Normalize to 0-1 saliency = (saliency - saliency.min()) / (saliency.max() - saliency.min() + 1e-8) # Apply colormap (jet) saliency_heatmap = cv2.applyColorMap(np.uint8(255 * saliency), cv2.COLORMAP_JET) saliency_heatmap = cv2.cvtColor(saliency_heatmap, cv2.COLOR_BGR2RGB) # Resize to original tensor size for consistency _, _, H, W = image_tensor.shape saliency_heatmap = cv2.resize(saliency_heatmap, (W, H)) return array_to_base64(saliency_heatmap) def get_gradcam_maps(model, image_tensor, orig_image_np): """Generate GradCAM and GradCAM++.""" # Target layer for AMFRNet is the last block target_layers = [model.block3] # Normalize original image for overlay orig_img_norm = orig_image_np.astype(np.float32) / 255.0 h, w = orig_img_norm.shape[:2] # 1. GradCAM cam_algo = GradCAM(model=model, target_layers=target_layers) grayscale_cam = cam_algo(input_tensor=image_tensor, targets=None)[0] # Force resize grayscale_cam to match orig_img_norm exactly to avoid broadcast errors if grayscale_cam.shape[0] != h or grayscale_cam.shape[1] != w: logger.warning(f"[explainers] Shape mismatch! Resizing GradCAM heatmap from {grayscale_cam.shape} to ({h}, {w})") grayscale_cam = cv2.resize(grayscale_cam, (w, h)) cam_image = show_cam_on_image(orig_img_norm, grayscale_cam, use_rgb=True) # 2. GradCAM++ cam_plus_algo = GradCAMPlusPlus(model=model, target_layers=target_layers) grayscale_cam_plus = cam_plus_algo(input_tensor=image_tensor, targets=None)[0] if grayscale_cam_plus.shape[0] != h or grayscale_cam_plus.shape[1] != w: grayscale_cam_plus = cv2.resize(grayscale_cam_plus, (w, h)) cam_plus_image = show_cam_on_image(orig_img_norm, grayscale_cam_plus, use_rgb=True) return array_to_base64(cam_image), array_to_base64(cam_plus_image) def get_lime_map(model, preprocessor, image_bytes, orig_image_np, device, model_type): """Generate LIME explanation.""" explainer = lime_image.LimeImageExplainer() def predict_fn(images): tensors = [] for img_arr in images: img_pil = PIL.Image.fromarray(img_arr) transform = preprocessor.get_transforms(model_type, img_pil.width, img_pil.height) tensor = transform(img_pil).to(device) tensors.append(tensor) batch_tensor = torch.stack(tensors) with torch.no_grad(): logits = model(batch_tensor) probs = torch.softmax(logits, dim=1) return probs.cpu().numpy() # Run explanation (num_samples=100 for decent speed, default 1000 is too slow for API) explanation = explainer.explain_instance( orig_image_np, predict_fn, top_labels=1, hide_color=0, num_samples=150 ) temp, mask = explanation.get_image_and_mask( explanation.top_labels[0], positive_only=True, num_features=5, hide_rest=False ) img_boundry = mark_boundaries(temp / 255.0, mask) img_boundry_uint8 = np.uint8(img_boundry * 255) return array_to_base64(img_boundry_uint8) def generate_all_explanations(model, preprocessor, image_tensor, image_bytes, model_type): """Wrapper function to generate all maps and return a dict of base64 strings.""" device = image_tensor.device # Dynamically determine the target size from the image tensor (1, 3, H, W) _, _, H, W = image_tensor.shape target_size = (W, H) # PIL uses (width, height) logger.info(f"[explainers] Target size for explanations: {target_size}") # Original image decoding for overlay orig_pil = PIL.Image.open(BytesIO(image_bytes)).convert("RGB") orig_pil = orig_pil.resize(target_size) orig_np = np.array(orig_pil) result = {} try: logger.info("[explainers] generating Saliency...") saliency_b64 = get_saliency_map(model, image_tensor) result["Saliency"] = f"data:image/jpeg;base64,{saliency_b64}" except Exception as e: logger.error(f"[explainers] Saliency failed: {e}") try: logger.info("[explainers] generating GradCAM & GradCAM++...") gc_b64, gcp_b64 = get_gradcam_maps(model, image_tensor, orig_np) result["GradCAM"] = f"data:image/jpeg;base64,{gc_b64}" result["GradCAM++"] = f"data:image/jpeg;base64,{gcp_b64}" except Exception as e: logger.error(f"[explainers] GradCAM failed: {e}") try: print("[explainers] generating LIME...") lime_b64 = get_lime_map(model, preprocessor, image_bytes, orig_np, device, model_type) result["LIME"] = f"data:image/jpeg;base64,{lime_b64}" except Exception as e: logger.error(f"[explainers] LIME failed: {e}") return result