code/src/gradcam.py

"""
Grad-CAM visualization for SpiceFusionNet.
Hooks into EfficientNet-B4's last convolutional layer.

Bug fixes:
  - Display image now matches the *exact* image the model sees
    (resize -> center crop), so heatmap pixels align correctly.
  - Optional fusion-mode Grad-CAM: computes gradients w.r.t. the
    fusion_head logits (matches the actual prediction pathway).
  - Uses last MBConv block features (slightly richer than conv_head).
"""
import numpy as np
import cv2
import torch
import torch.nn.functional as F
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
from pathlib import Path
from PIL import Image
import albumentations as A
from albumentations.pytorch import ToTensorV2

import config
from src.features import extract_all as _extract_all


# Resize-then-centercrop dimensions (must match get_val_transform exactly)
_RESIZE = int(config.IMG_SIZE * 256 / 224)
_CROP   = config.IMG_SIZE


class GradCAM:
    def __init__(self, model, target_layer=None, mode: str = "fusion"):
        """mode = "fusion" or "image"  — which forward pass to backprop through."""
        self.model = model
        self.mode = mode
        self.gradients = None
        self.activations = None

        # Last conv block of EfficientNet-B4. `conv_head` is a 1x1 conv;
        # `blocks[-1]` carries the spatial features that flow into it.
        if target_layer is None:
            target_layer = model.backbone.conv_head

        self._fwd = target_layer.register_forward_hook(self._save_activation)
        self._bwd = target_layer.register_full_backward_hook(self._save_gradient)

    def _save_activation(self, module, inp, out):
        self.activations = out.detach()

    def _save_gradient(self, module, grad_in, grad_out):
        self.gradients = grad_out[0].detach()

    def remove(self):
        self._fwd.remove()
        self._bwd.remove()

    def __call__(self, x: torch.Tensor, tex=None, col=None, class_idx: int = None):
        """Returns (cam, pred_idx). cam is H×W float32 in [0,1]."""
        self.model.eval()
        self.model.zero_grad()

        if self.mode == "fusion" and tex is not None and col is not None:
            logits, _ = self.model.forward_fusion(x, tex, col)
        else:
            logits = self.model.forward_image(x)

        if class_idx is None:
            class_idx = logits.argmax(dim=1).item()

        logits[0, class_idx].backward()

        weights = self.gradients.mean(dim=(2, 3), keepdim=True)
        cam = (weights * self.activations).sum(dim=1, keepdim=True)
        cam = F.relu(cam).squeeze().cpu().numpy()

        cam = cam - cam.min()
        if cam.max() > 0:
            cam = cam / cam.max()
        return cam, class_idx


def _model_view(img_path: str):
    """Return BOTH the tensor the model sees AND the matching uint8 display
    image, guaranteed to be the exact same pixels (resize -> center crop)."""
    img = np.array(Image.open(img_path).convert("RGB"))

    # Step 1: resize + center crop to produce display image (uint8)
    crop_tf = A.Compose([
        A.Resize(_RESIZE, _RESIZE),
        A.CenterCrop(_CROP, _CROP),
    ])
    display = crop_tf(image=img)["image"]   # (224, 224, 3) uint8

    # Step 2: normalize + to-tensor for the model
    norm_tf = A.Compose([
        A.Normalize(mean=config.IMG_MEAN, std=config.IMG_STD),
        ToTensorV2(),
    ])
    tensor = norm_tf(image=display)["image"].unsqueeze(0)   # (1, 3, 224, 224)

    # Step 3: hand-crafted features on the ORIGINAL image (matches dataset.py)
    tex_np, col_np = _extract_all(img)
    tex = torch.from_numpy(tex_np).unsqueeze(0).float()
    col = torch.from_numpy(col_np).unsqueeze(0).float()

    return tensor, display, tex, col


def visualize_gradcam(
    model,
    img_paths: list,
    true_labels: list,
    classes: list,
    device: torch.device,
    output_path: Path,
    n_samples: int = 16,
    mode: str = "fusion",
):
    """Plot Grad-CAM for up to n_samples images.

    mode='fusion' computes gradients through the fusion_head (matches the
    actual prediction). mode='image' uses the Phase-1 img_head.
    """
    model.eval()
    gcam = GradCAM(model, mode=mode)

    cols = 4
    rows = (n_samples + cols - 1) // cols
    fig, axes = plt.subplots(rows, cols * 2, figsize=(cols * 6, rows * 3))
    axes = axes.flatten()

    plotted = 0
    for img_path, true_label in zip(img_paths, true_labels):
        if plotted >= n_samples:
            break

        tensor, display, tex, col = _model_view(img_path)
        tensor = tensor.to(device); tex = tex.to(device); col = col.to(device)

        cam, pred_idx = gcam(tensor, tex=tex, col=col)

        # Overlay — display & cam now share identical 224x224 spatial grid
        heatmap = cv2.resize(cam, (display.shape[1], display.shape[0]))
        heatmap = cv2.applyColorMap(np.uint8(255 * heatmap), cv2.COLORMAP_JET)
        heatmap = cv2.cvtColor(heatmap, cv2.COLOR_BGR2RGB)
        overlay = cv2.addWeighted(display, 0.5, heatmap, 0.5, 0)

        ax_img = axes[plotted * 2]
        ax_cam = axes[plotted * 2 + 1]

        ax_img.imshow(display)
        ax_img.set_title(f"True: {classes[true_label]}", fontsize=8)
        ax_img.axis("off")

        ax_cam.imshow(overlay)
        color = "green" if pred_idx == true_label else "red"
        ax_cam.set_title(f"Pred: {classes[pred_idx]}", fontsize=8, color=color)
        ax_cam.axis("off")

        plotted += 1

    for i in range(plotted * 2, len(axes)):
        axes[i].axis("off")

    plt.suptitle(f"Grad-CAM ({mode} mode)", fontsize=13)
    plt.tight_layout()
    plt.savefig(output_path, dpi=150)
    plt.close()
    gcam.remove()
    print(f"Grad-CAM saved to {output_path}")