File size: 13,469 Bytes
151f388
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
"""
ConvNeXt Deepfake Detector Adapter (Chen et al. 2024 / RAID Constituent)
Provides a clean, standardized wrapper around the chen2024_convnext checkpoint from aimagelab/RAID_ckpt.
Adheres strictly to the DetectorResult schema, supporting lazy loading and robust error handling.
"""

import os
import sys
import time
from typing import Optional, Union, Any, Dict
from PIL import Image
import cv2
import numpy as np
import torch
import torch.nn as nn
from torchvision import transforms

from src.detector_schema import (
    DetectorResult,
    PREDICTION_REAL,
    PREDICTION_FAKE,
    PREDICTION_ERROR,
)

BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
DEFAULT_CHECKPOINT_PATH = os.path.join(
    BASE_DIR, "models", "chen2024_convnext", "16_acc0.9993.pth"
)


class Chen2024ConvNeXtArchitecture(nn.Module):
    """
    ConvNeXt-Base model with linear embedding and binary classification head,
    matching the exact Chen et al. 2024 / RAID architecture:
      - Backbone: timm convnext_base (stem 128, stage blocks [3, 3, 27, 3], head 1024)
      - Classifier: nn.Linear(1024, 2)
    """

    def __init__(self, num_classes: int = 2, embedding_size: int = 1024):
        super().__init__()
        import timm
        # convnext_base produces 1024-dim embedding through its linear head
        self.model = timm.create_model(
            "convnext_base",
            pretrained=False,
            num_classes=embedding_size,
        )
        self.fc = nn.Linear(embedding_size, num_classes)

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        feat = self.model(x)
        return self.fc(feat)


class DRCTNativeCropTransform:
    """
    Official DRCT evaluation transform (--is_crop, Chen et al. ICML 2024):
      - Native-resolution center cropping without prior resizing.
      - If image height or width < 224, pads symmetrically with constant 0 (black).
      - Center crops exactly (224, 224).
      - Normalizes using ImageNet mean [0.485, 0.456, 0.406] and std [0.229, 0.224, 0.225].
      - Converts to torch float32 tensor of shape [3, 224, 224].
    """

    def __init__(
        self,
        size: int = 224,
        mean=(0.485, 0.456, 0.406),
        std=(0.229, 0.224, 0.225),
    ):
        self.size = size
        self.mean = np.array(mean, dtype=np.float32)
        self.std = np.array(std, dtype=np.float32)

    def __call__(self, img: Union[Image.Image, np.ndarray, torch.Tensor]) -> torch.Tensor:
        if isinstance(img, Image.Image):
            img_np = np.array(img.convert("RGB"))
        elif isinstance(img, np.ndarray):
            if img.ndim == 2:
                img_np = np.stack([img] * 3, axis=-1)
            elif img.ndim == 3 and img.shape[2] == 4:
                img_np = img[:, :, :3]
            elif img.ndim == 3 and img.shape[2] == 3:
                img_np = img
            else:
                raise ValueError(f"Unsupported numpy image shape: {img.shape}")
        elif isinstance(img, torch.Tensor):
            if img.ndim == 3 and img.shape[0] == 3:
                img_np = (img.permute(1, 2, 0).cpu().numpy() * 255.0).clip(0, 255).astype(np.uint8)
            else:
                raise ValueError(f"Unsupported tensor shape: {img.shape}")
        else:
            raise TypeError(f"Unsupported image type: {type(img)}")

        h, w = img_np.shape[:2]
        pad_h = max(0, self.size - h)
        pad_w = max(0, self.size - w)
        if pad_h > 0 or pad_w > 0:
            top = pad_h // 2
            bottom = pad_h - top
            left = pad_w // 2
            right = pad_w - left
            img_np = cv2.copyMakeBorder(
                img_np, top, bottom, left, right, cv2.BORDER_CONSTANT, value=[0, 0, 0]
            )
            h, w = img_np.shape[:2]

        start_y = (h - self.size) // 2
        start_x = (w - self.size) // 2
        cropped = img_np[start_y:start_y + self.size, start_x:start_x + self.size]

        norm = (cropped.astype(np.float32) / 255.0 - self.mean) / self.std
        tensor = torch.from_numpy(norm.transpose(2, 0, 1)).float()
        return tensor


class ConvNeXtDetectorAdapter:
    """
    Adapter for Chen et al. 2024 ConvNeXt deepfake detector from RAID / DRCT.

    Model Details:
      - Architecture: ConvNeXt-Base + Linear Head (1024 -> 2)
      - Checkpoint: models/chen2024_convnext/16_acc0.9993.pth (aimagelab/RAID_ckpt)
      - Preprocessing: Native-resolution CenterCrop(224) matching official DRCT --is_crop
      - Decision Threshold: 0.50 Fake Probability
      - Label Mapping: Index 0 = Real, Index 1 = Fake
    """

    DETECTOR_NAME = "Chen2024ConvNeXtDetector"
    MODEL_NAME = "chen2024_convnext (ConvNeXt-Base)"
    PREPROCESSING_NAME = (
        "PadIfNeeded(224, border_mode=BORDER_CONSTANT, value=0) -> "
        "CenterCrop(224) [native resolution, no resize] -> ToTensor() -> "
        "Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])"
    )
    DECISION_THRESHOLD = 0.50
    LIMITATION_WARNING = (
        "ConvNeXt detector evaluated on synthetic/generator artifacts from RAID benchmark. "
        "Does not guarantee authenticity."
    )

    def __init__(
        self,
        checkpoint_path: Optional[str] = None,
        device: Optional[Union[str, torch.device]] = None,
        lazy_load: bool = True,
    ):
        self.checkpoint_path = checkpoint_path or DEFAULT_CHECKPOINT_PATH
        self.device_requested = device
        self.model = None
        self.device = None
        self.transform = None
        self.real_idx = 0
        self.fake_idx = 1
        self.load_error = None
        self.load_time_sec = None

        if not lazy_load:
            self.load_model()

    def _init_transform(self):
        """Construct the official DRCT evaluation preprocessing transform (--is_crop)."""
        self.transform = DRCTNativeCropTransform(size=224)

    def load_model(self) -> bool:
        """
        Loads the ConvNeXt model architecture and checkpoint weights if not already loaded.
        Ensures lazy loading so module import has zero weight overhead.
        """
        if self.model is not None:
            return True

        start_t = time.perf_counter()
        try:
            # 1. Determine compute device
            if self.device_requested is not None:
                self.device = torch.device(self.device_requested)
            else:
                self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

            # 2. Validate checkpoint existence
            if not os.path.exists(self.checkpoint_path):
                raise FileNotFoundError(
                    f"Checkpoint not found at: {self.checkpoint_path}"
                )

            # 3. Instantiate architecture
            net = Chen2024ConvNeXtArchitecture(num_classes=2, embedding_size=1024)

            # 4. Load weights strictly
            state_dict = torch.load(self.checkpoint_path, map_location="cpu", weights_only=True)
            missing, unexpected = net.load_state_dict(state_dict, strict=True)
            if missing or unexpected:
                raise RuntimeError(
                    f"State dict mismatch! Missing: {missing}, Unexpected: {unexpected}"
                )

            net.to(self.device)
            net.eval()
            self.model = net

            # 5. Initialize preprocessing pipeline
            self._init_transform()

            self.load_time_sec = time.perf_counter() - start_t
            return True

        except Exception as e:
            self.load_error = str(e)
            self.model = None
            return False

    def unload_model(self) -> None:
        """
        Explicitly releases the neural network model and preprocessing pipeline
        to free system and GPU memory.
        """
        self.model = None
        self.transform = None
        if torch.cuda.is_available():
            torch.cuda.empty_cache()

    def predict(self, image_input: Any) -> DetectorResult:
        """
        Execute single-image inference and return a standardized DetectorResult.

        Accepts:
          - PIL Image
          - File path string
          - Numpy image array (or any object convertible to PIL Image)
        """
        start_time = time.perf_counter()

        # 1. Validation for empty input
        if image_input is None:
            return DetectorResult.create_error(
                detector_name=self.DETECTOR_NAME,
                model_name=self.MODEL_NAME,
                error_message="No image provided (input was None).",
                warning=self.LIMITATION_WARNING,
                device=str(self.device) if self.device else None,
                preprocessing=self.PREPROCESSING_NAME,
            )

        # 2. Safe image loading and RGB conversion
        try:
            if isinstance(image_input, str):
                if not os.path.exists(image_input):
                    return DetectorResult.create_error(
                        detector_name=self.DETECTOR_NAME,
                        model_name=self.MODEL_NAME,
                        error_message=f"Image file does not exist: {image_input}",
                        warning=self.LIMITATION_WARNING,
                        device=str(self.device) if self.device else None,
                        preprocessing=self.PREPROCESSING_NAME,
                    )
                with Image.open(image_input) as opened_img:
                    img_rgb = opened_img.convert("RGB")
            elif isinstance(image_input, Image.Image):
                img_rgb = image_input.convert("RGB")
            elif hasattr(image_input, "__array__"):
                img_rgb = Image.fromarray(image_input).convert("RGB")
            else:
                return DetectorResult.create_error(
                    detector_name=self.DETECTOR_NAME,
                    model_name=self.MODEL_NAME,
                    error_message=f"Unsupported image input type: {type(image_input).__name__}",
                    warning=self.LIMITATION_WARNING,
                    device=str(self.device) if self.device else None,
                    preprocessing=self.PREPROCESSING_NAME,
                )
        except Exception as img_err:
            return DetectorResult.create_error(
                detector_name=self.DETECTOR_NAME,
                model_name=self.MODEL_NAME,
                error_message=f"Invalid or corrupted image: {str(img_err)}",
                warning=self.LIMITATION_WARNING,
                device=str(self.device) if self.device else None,
                preprocessing=self.PREPROCESSING_NAME,
            )

        # 3. Ensure model is loaded
        if not self.load_model():
            return DetectorResult.create_error(
                detector_name=self.DETECTOR_NAME,
                model_name=self.MODEL_NAME,
                error_message=f"Model loading failed: {self.load_error}",
                warning=self.LIMITATION_WARNING,
                device=str(self.device) if self.device else None,
                preprocessing=self.PREPROCESSING_NAME,
            )

        # 4. Inference execution under torch.inference_mode()
        try:
            # Dynamic device adjustment for ZeroGPU runtime
            current_device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
            if next(self.model.parameters()).device != current_device:
                self.model.to(current_device)
                self.device = current_device

            tensor_input = self.transform(img_rgb).unsqueeze(0).to(self.device)

            with torch.inference_mode():
                logits = self.model(tensor_input)  # shape: [1, 2]
                probs = torch.softmax(logits, dim=1).squeeze(0)

            real_prob = float(probs[self.real_idx].item())
            fake_prob = float(probs[self.fake_idx].item())

            real_logit = float(logits[0, self.real_idx].item())
            fake_logit = float(logits[0, self.fake_idx].item())
            relative_logit = fake_logit - real_logit

            latency_ms = (time.perf_counter() - start_time) * 1000

            # Decision rule: index 1 is FAKE, threshold = 0.50
            is_fake = fake_prob >= self.DECISION_THRESHOLD
            prediction = PREDICTION_FAKE if is_fake else PREDICTION_REAL
            confidence = fake_prob if is_fake else real_prob

            return DetectorResult(
                detector_name=self.DETECTOR_NAME,
                model_name=self.MODEL_NAME,
                prediction=prediction,
                fake_probability=round(fake_prob, 6),
                real_probability=round(real_prob, 6),
                raw_logit=round(relative_logit, 6),
                confidence=round(confidence, 6),
                threshold=self.DECISION_THRESHOLD,
                device=str(self.device),
                preprocessing=self.PREPROCESSING_NAME,
                inference_time_ms=round(latency_ms, 2),
                error=None,
                warning=self.LIMITATION_WARNING,
            )

        except Exception as inf_err:
            return DetectorResult.create_error(
                detector_name=self.DETECTOR_NAME,
                model_name=self.MODEL_NAME,
                error_message=f"Inference execution failed: {str(inf_err)}",
                warning=self.LIMITATION_WARNING,
                device=str(self.device) if self.device else None,
                preprocessing=self.PREPROCESSING_NAME,
            )