Blame for model/src/model.py

import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
from typing import List, Tuple, Optional


BAND_CONFIGS = {
    3: [(0.0, 0.15), (0.15, 0.45), (0.45, 1.0)],
    4: [(0.0, 0.1), (0.1, 0.3), (0.3, 0.6), (0.6, 1.0)],
}

class FrequencyDecomposition(nn.Module):
    """Radial band-pass decomposition via FFT.



    soft_masks=False reproduces the pilot behavior (binary masks).

    soft_masks=True replaces the hard cutoff with a sigmoid transition of

    width `soft_tau * r_max`, removing the ringing artifacts that hard

    masks imprint on the band images.



    Masks are cached per (H, W, device) instead of being rebuilt for every

    band of every forward pass.

    """
    def __init__(self, num_bands: int = 3, soft_masks: bool = False,

                 soft_tau: float = 0.02):
        super().__init__()
        self.bands = BAND_CONFIGS.get(num_bands, BAND_CONFIGS[3])
        self.num_bands = len(self.bands)
        self.soft_masks = soft_masks
        self.soft_tau = soft_tau
        self._mask_cache = {}

    def _get_masks(self, H: int, W: int, device) -> torch.Tensor:
        key = (H, W, str(device), self.soft_masks)
        cached = self._mask_cache.get(key)
        if cached is not None:
            return cached

        ny, nx = H // 2, W // 2
        r_max = min(ny, nx)
        y_grid, x_grid = torch.meshgrid(
            torch.arange(H, device=device),
            torch.arange(W, device=device),
            indexing='ij'
        )
        dist = torch.sqrt((y_grid - ny).float() ** 2 + (x_grid - nx).float() ** 2)

        masks = []
        for lo, hi in self.bands:
            r_low, r_high = r_max * lo, r_max * hi
            if self.soft_masks:
                tau = max(self.soft_tau * r_max, 1e-3)
                lower = torch.sigmoid((dist - r_low) / tau) if lo > 0 \
                    else torch.ones_like(dist)
                upper = torch.sigmoid((r_high - dist) / tau) if hi < 1.0 \
                    else torch.ones_like(dist)
                masks.append(lower * upper)
            else:
                masks.append(((dist >= int(r_low)) & (dist < int(r_high))).float())
        stacked = torch.stack(masks).unsqueeze(1).unsqueeze(1)  # (B_bands,1,1,H,W)
        self._mask_cache[key] = stacked
        return stacked

    def forward(self, x: torch.Tensor) -> List[torch.Tensor]:
        B, C, H, W = x.shape
        x_fft = torch.fft.fft2(x, norm='ortho')
        x_shifted = torch.fft.fftshift(x_fft)
        masks = self._get_masks(H, W, x.device)

        outputs = []
        for i in range(self.num_bands):
            filtered = x_shifted * masks[i]
            band = torch.fft.ifft2(torch.fft.ifftshift(filtered), norm='ortho').real
            outputs.append(band)
        return outputs


def npr_residual(x: torch.Tensor, factor: int = 2) -> torch.Tensor:
    """Neighboring-pixel-relation residual (after Tan et al., CVPR 2024):

    the difference between the image and its down-up resampled version,

    isolating the local interpolation traces that generator upsampling

    layers imprint. Returns a tensor shaped like x."""
    down = F.interpolate(x, scale_factor=1.0 / factor,
                         mode='bilinear', align_corners=False)
    up = F.interpolate(down, size=x.shape[2:],
                       mode='bilinear', align_corners=False)
    return x - up


class FrequencyFeatureExtractor(nn.Module):
    """

    CNN backbone for each frequency band.

    Uses depthwise separable convolutions for efficiency.

    """
    def __init__(self, in_channels: int = 3, feat_dim: int = 256):
        super().__init__()
        self.stem = nn.Sequential(
            nn.Conv2d(in_channels, 32, 3, stride=2, padding=1, bias=False),
            nn.BatchNorm2d(32),
            nn.GELU(),
        )
        self.blocks = nn.ModuleList([
            self._make_block(32, 64, 3, 2),
            self._make_block(64, 128, 3, 2),
            self._make_block(128, 256, 3, 2),
        ])
        self.head = nn.Sequential(
            nn.AdaptiveAvgPool2d(1),
            nn.Flatten(),
            nn.Linear(256, feat_dim),
            nn.LayerNorm(feat_dim),
        )

    def _make_block(self, cin: int, cout: int, k: int, s: int) -> nn.Module:
        return nn.Sequential(
            nn.Conv2d(cin, cin, k, stride=s, padding=k//2, groups=cin, bias=False),
            nn.Conv2d(cin, cout, 1, bias=False),
            nn.BatchNorm2d(cout),
            nn.GELU(),
        )

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        x = self.stem(x)
        for block in self.blocks:
            x = block(x)
        return self.head(x)


class PretrainedFrequencyFeatureExtractor(nn.Module):
    """

    Drop-in replacement for FrequencyFeatureExtractor backed by an

    ImageNet-pretrained MobileNetV3-Small feature trunk. Used for the

    matched-pretraining MFFT variant that decouples architecture from

    initialization in comparisons against pretrained baselines.



    Band images are 3-channel (per-RGB-channel FFT filtering), so the

    pretrained stem is used unchanged.

    """
    def __init__(self, in_channels: int = 3, feat_dim: int = 256,

                 weights: str = "DEFAULT"):
        super().__init__()
        from torchvision.models import mobilenet_v3_small
        try:
            backbone = mobilenet_v3_small(weights=weights)
        except Exception as e:  # offline or weights unavailable
            import warnings
            warnings.warn(f"Pretrained weights unavailable ({e}); "
                          "falling back to random init.")
            backbone = mobilenet_v3_small(weights=None)
        if in_channels != 3:
            # inflate the stem conv: keep RGB filters, tile them (scaled)
            # over the extra channels so pretrained features are preserved
            old = backbone.features[0][0]
            new = nn.Conv2d(in_channels, old.out_channels,
                            old.kernel_size, old.stride, old.padding,
                            bias=old.bias is not None)
            with torch.no_grad():
                reps = -(-in_channels // 3)  # ceil division
                w = old.weight.repeat(1, reps, 1, 1)[:, :in_channels]
                new.weight.copy_(w * (3.0 / in_channels))
            backbone.features[0][0] = new
        self.features = backbone.features  # output: (B, 576, H/32, W/32)
        self.head = nn.Sequential(
            nn.AdaptiveAvgPool2d(1),
            nn.Flatten(),
            nn.Linear(576, feat_dim),
            nn.LayerNorm(feat_dim),
        )

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        return self.head(self.features(x))


class CrossAttentionFusion(nn.Module):
    """

    Cross-attention fusion across frequency bands.

    Each band attends to all others to produce a fused representation.

    """
    def __init__(self, dim: int = 256, num_heads: int = 8):
        super().__init__()
        self.dim = dim
        self.num_heads = num_heads
        self.head_dim = dim // num_heads
        self.scale = self.head_dim ** -0.5

        self.to_qkv = nn.Linear(dim, dim * 3, bias=False)
        self.proj = nn.Linear(dim, dim)
        self.proj_drop = nn.Dropout(0.1)

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        B, N, D = x.shape
        qkv = self.to_qkv(x).reshape(B, N, 3, self.num_heads, self.head_dim)
        qkv = qkv.permute(2, 0, 3, 1, 4)
        q, k, v = qkv[0], qkv[1], qkv[2]

        attn = (q @ k.transpose(-2, -1)) * self.scale
        attn = attn.softmax(dim=-1)

        out = (attn @ v).transpose(1, 2).reshape(B, N, D)
        out = self.proj(out)
        out = self.proj_drop(out)
        return out


class FrequencyGuidedAttention(nn.Module):
    """

    Novel module: uses frequency information to guide spatial attention.

    High-freq regions (edges, textures) get higher attention weights.

    """
    def __init__(self, dim: int = 256):
        super().__init__()
        self.freq_gate = nn.Sequential(
            nn.Linear(dim, dim // 4),
            nn.ReLU(),
            nn.Linear(dim // 4, dim),
            nn.Sigmoid(),
        )

    def forward(self, x: torch.Tensor, freq_weights: torch.Tensor) -> torch.Tensor:
        gate = self.freq_gate(x)
        return x * gate * freq_weights


class MFFT(nn.Module):
    """

    Multi-Frequency Fusion Transformer (MFFT)

    ===========================================

    A novel architecture for AI-generated image detection.



    Key innovations:

    1. Frequency decomposition into Low/Mid/High bands via DCT

    2. Per-band CNN feature extraction

    3. Cross-attention fusion across frequency bands

    4. Frequency-guided spatial attention

    5. Multi-scale feature aggregation



    Input:  (B, 3, H, W)  RGB image

    Output: (B, 2)         logits [real, ai_generated]



    Ablation config (dict):

        spatial_only : bool  — skip frequency decomposition, use raw image

        skip_bands   : list  — band names to exclude: ["low", "mid", "high"]

        fusion_mode  : str   — "attention" | "concat" | "avg" | "max"

        use_fga      : bool  — enable/disable FrequencyGuidedAttention

    """
    def __init__(

        self,

        in_channels: int = 3,

        feat_dim: int = 256,

        num_bands: int = 3,

        num_heads: int = 8,

        num_classes: int = 2,

        ablation: Optional[dict] = None,

        pretrained_extractors: bool = False,

        use_npr: bool = False,

        soft_masks: bool = False,

    ):
        super().__init__()
        self.num_bands = num_bands
        self.feat_dim = feat_dim
        self.ablation = ablation or {}
        self.pretrained_extractors = pretrained_extractors
        self.use_npr = use_npr

        self.decomposer = FrequencyDecomposition(
            num_bands=num_bands, soft_masks=soft_masks)
        extractor_cls = (PretrainedFrequencyFeatureExtractor
                         if pretrained_extractors else FrequencyFeatureExtractor)
        # with NPR enabled, the highest band additionally receives the
        # 3-channel neighboring-pixel-relation residual
        self.extractors = nn.ModuleList([
            extractor_cls(
                in_channels + (3 if (use_npr and i == num_bands - 1) else 0),
                feat_dim)
            for i in range(num_bands)
        ])

        if self.ablation.get("fusion_mode", "attention") == "attention":
            self.fusion = CrossAttentionFusion(feat_dim, num_heads)
        else:
            self.fusion = None

        self.use_fga = self.ablation.get("use_fga", True)
        if self.use_fga:
            self.freq_guided_attn = FrequencyGuidedAttention(feat_dim)
        else:
            self.freq_guided_attn = None

        self.classifier = nn.Sequential(
            nn.LayerNorm(feat_dim * num_bands),
            nn.Linear(feat_dim * num_bands, feat_dim),
            nn.GELU(),
            nn.Dropout(0.2),
            nn.Linear(feat_dim, feat_dim // 2),
            nn.GELU(),
            nn.Dropout(0.1),
            nn.Linear(feat_dim // 2, num_classes),
        )

        # Ablation configs whose fused representation is narrower than the
        # classifier input (avg/max fusion, skipped bands, spatial-only)
        # get a learned projection instead of the old zero-padding, which
        # wasted classifier capacity on constant inputs.
        combined_dim = self._fused_dim(feat_dim, num_bands)
        if combined_dim != feat_dim * num_bands:
            self.input_proj = nn.Linear(combined_dim, feat_dim * num_bands)
        else:
            self.input_proj = nn.Identity()

        self._init_weights()

    def _fused_dim(self, feat_dim: int, num_bands: int) -> int:
        """Width of the flattened fused representation under the current

        ablation config (mirrors the branching in forward())."""
        if self.ablation.get("spatial_only", False):
            return feat_dim
        num_active = num_bands - len(self.ablation.get("skip_bands", []))
        mode = self.ablation.get("fusion_mode", "attention")
        if mode in ("avg", "max") and num_active > 1:
            return feat_dim
        return num_active * feat_dim  # attention / concat / single band

    def _init_weights(self):
        # pretrained extractor trunks must keep their ImageNet weights
        skip = set()
        if self.pretrained_extractors:
            for ext in self.extractors:
                for m in ext.features.modules():
                    skip.add(id(m))
        for m in self.modules():
            if id(m) in skip:
                continue
            if isinstance(m, (nn.Conv2d, nn.Linear)):
                nn.init.trunc_normal_(m.weight, std=0.02)
                if m.bias is not None:
                    nn.init.constant_(m.bias, 0)
            elif isinstance(m, nn.LayerNorm):
                nn.init.constant_(m.bias, 0)
                nn.init.constant_(m.weight, 1.0)

    def forward(self, x: torch.Tensor, return_heatmap: bool = False):
        spatial_only = self.ablation.get("spatial_only", False)
        skip_bands = self.ablation.get("skip_bands", [])
        fusion_mode = self.ablation.get("fusion_mode", "attention")
        if self.num_bands == 4:
            band_names = ["low", "low_mid", "mid_high", "high"]
        else:
            band_names = ["low", "mid", "high"]

        if spatial_only:
            bands = [x]
            extractors = self.extractors[:1]
            extractor_inputs = bands
        else:
            all_bands = self.decomposer(x)
            keep_indices = [i for i, name in enumerate(band_names) if name not in skip_bands]
            bands = [all_bands[i] for i in keep_indices]
            extractors = [self.extractors[i] for i in keep_indices]
            extractor_inputs = []
            for i, band in zip(keep_indices, bands):
                if self.use_npr and i == self.num_bands - 1:
                    band = torch.cat([band, npr_residual(x)], dim=1)
                extractor_inputs.append(band)

        features = []
        for band, extractor in zip(extractor_inputs, extractors):
            feat = extractor(band)
            features.append(feat)

        B = x.shape[0]
        num_active = len(features)

        if num_active == 1 or fusion_mode == "concat":
            fused = torch.cat(features, dim=-1)
            N = num_active
            D = fused.shape[-1] // N
            fused = fused.unsqueeze(1)
        elif fusion_mode == "avg":
            fused = torch.stack(features, dim=0).mean(dim=0)
            N = 1
            D = fused.shape[-1]
            fused = fused.unsqueeze(1)
        elif fusion_mode == "max":
            fused = torch.stack(features, dim=0).max(dim=0).values
            N = 1
            D = fused.shape[-1]
            fused = fused.unsqueeze(1)
        elif fusion_mode == "attention" and self.fusion is not None and num_active > 1:
            feat_stack = torch.stack(features, dim=1)
            fused = self.fusion(feat_stack)
            N = fused.shape[1]
            D = fused.shape[2]
        else:
            fused = torch.cat(features, dim=-1)
            N = num_active
            D = fused.shape[-1] // N
            fused = fused.unsqueeze(1)

        # FGA operates on per-band tokens, so it only applies when the fused
        # representation kept one token per band (attention path). Pooled
        # representations (avg/max/concat) have no band axis to weight.
        fga_applicable = (
            self.use_fga and self.freq_guided_attn is not None
            and not spatial_only
            and N == num_active and D == fused.shape[-1]
            and fused.shape[-1] == self.feat_dim
        )
        if fga_applicable:
            all_bands_for_fga = bands
            freq_magnitudes = torch.stack([
                torch.abs(band).mean(dim=(1, 2, 3))
                for band in all_bands_for_fga
            ], dim=1)
            freq_weights = F.softmax(freq_magnitudes, dim=1).unsqueeze(-1)
            guided = self.freq_guided_attn(fused, freq_weights)
        else:
            guided = fused

        combined = guided.reshape(B, N * D)
        combined = self.input_proj(combined)
        logits = self.classifier(combined)

        if return_heatmap:
            heatmaps = []
            for band in bands:
                h = torch.abs(band).mean(dim=1, keepdim=True)
                h = F.interpolate(h, size=x.shape[2:], mode='bilinear', align_corners=False)
                heatmaps.append(h)
            return logits, torch.cat(heatmaps, dim=1)
        return logits


class MFFTWithExplainability(MFFT):
    """

    Extension of MFFT that also produces:

    - Confidence score (0-1)

    - Per-region anomaly heatmap

    - Per-frequency-band contribution scores

    """
    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        self.register_buffer(
            'mean', torch.tensor([0.485, 0.456, 0.406]).view(1, 3, 1, 1)
        )
        self.register_buffer(
            'std', torch.tensor([0.229, 0.224, 0.225]).view(1, 3, 1, 1)
        )

    def preprocess(self, x: torch.Tensor) -> torch.Tensor:
        if x.shape[1] == 1:
            x = x.repeat(1, 3, 1, 1)
        x = (x / 255.0 - self.mean) / self.std
        return x

    @torch.no_grad()
    def predict(self, x: torch.Tensor) -> dict:
        x = self.preprocess(x)
        self.eval()
        logits, heatmaps = self.forward(x, return_heatmap=True)
        probs = F.softmax(logits, dim=-1)
        preds = torch.argmax(probs, dim=-1)

        return {
            "prediction": preds,
            "real_prob": probs[:, 0],
            "ai_prob": probs[:, 1],
            "confidence": probs.max(dim=-1).values,
            "heatmaps": heatmaps,
            "logits": logits,
        }


def count_parameters(model: nn.Module) -> int:
    return sum(p.numel() for p in model.parameters() if p.requires_grad)


def build_mfft(variant: str = "base", ablation: Optional[dict] = None,

               pretrained_extractors: bool = False,

               use_npr: bool = False, soft_masks: bool = False) -> MFFT:
    configs = {
        "tiny":  {"feat_dim": 128, "num_heads": 4, "num_bands": 3},
        "base":  {"feat_dim": 384, "num_heads": 6, "num_bands": 3},
        "large": {"feat_dim": 768, "num_heads": 12, "num_bands": 4},
    }
    cfg = configs.get(variant, configs["base"])
    model = MFFT(**cfg, ablation=ablation,
                 pretrained_extractors=pretrained_extractors,
                 use_npr=use_npr, soft_masks=soft_masks)
    return model