StillWorking commited on
Commit
53ecd06
Β·
verified Β·
1 Parent(s): e8fd2e2

Upload backend/core/models.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. backend/core/models.py +458 -0
backend/core/models.py ADDED
@@ -0,0 +1,458 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ core/models.py β€” Sentinel-AI
3
+ ===============================
4
+ Singleton loaders for the local PyTorch models:
5
+
6
+ * MTCNN β€” face detection + 5-point alignment (vendored with AdaFace)
7
+ * AdaFace IR-50 β€” face embedding (512-D output, FP16 on CUDA)
8
+
9
+ RF-DETR is kept as an optional, explicitly opt-in detector for backwards
10
+ compatibility. It is *not* used by default: ``RFDETRSmall`` is a COCO object
11
+ detector (90 everyday classes, no "face" class) and was being asked to do face
12
+ detection, which produced labels such as "bicycle" and "boat" for human faces.
13
+ The AdaFace repository in ``models_cache/adaface`` already ships the MTCNN
14
+ weights, so no extra download is required.
15
+
16
+ Both heavy models are module-level singletons: loaded once at startup and
17
+ reused across every request.
18
+ """
19
+
20
+ from __future__ import annotations
21
+
22
+ import os
23
+ import sys
24
+ from pathlib import Path
25
+ from typing import Optional
26
+
27
+ import numpy as np
28
+ import torch
29
+ import torch.nn as nn
30
+
31
+ from core.paths import env_float, env_int, resolve_path
32
+
33
+ # ── Lazy import guard for RF-DETR (optional) ─────────────────────────────────
34
+ try:
35
+ from rfdetr import RFDETRSmall # type: ignore
36
+ _RFDETR_AVAILABLE = True
37
+ except ImportError:
38
+ _RFDETR_AVAILABLE = False
39
+
40
+ # ── AdaFace + MTCNN live in the vendored repository ──────────────────────────
41
+ # models.py is at backend/core/models.py, so parent.parent == backend/
42
+ _ADAFACE_REPO = Path(__file__).resolve().parent.parent / "models_cache" / "adaface"
43
+ if _ADAFACE_REPO.exists() and str(_ADAFACE_REPO) not in sys.path:
44
+ sys.path.insert(0, str(_ADAFACE_REPO))
45
+
46
+ try:
47
+ import net as adaface_net # type: ignore
48
+ _ADAFACE_AVAILABLE = True
49
+ except ImportError:
50
+ _ADAFACE_AVAILABLE = False
51
+
52
+ try:
53
+ from face_alignment import mtcnn as _mtcnn # type: ignore
54
+ _MTCNN_AVAILABLE = True
55
+ except Exception: # pragma: no cover
56
+ _MTCNN_AVAILABLE = False
57
+
58
+ DEVICE = os.getenv("TORCH_DEVICE", "cuda") if torch.cuda.is_available() else "cpu"
59
+ if os.getenv("TORCH_DEVICE", "cuda").lower() == "cpu":
60
+ DEVICE = "cpu"
61
+
62
+ ADAFACE_WEIGHTS = resolve_path(
63
+ os.getenv("ADAFACE_WEIGHTS"),
64
+ "./models_cache/adaface_ir50_ms1mv2.ckpt",
65
+ )
66
+
67
+ # A face crop must include some hair / shoulder context so that it resembles
68
+ # the CelebA framing the FAISS index was built from. 1.5 was measured on the
69
+ # bundled dataset (see SENTINEL_FIX_LOG.md).
70
+ FACE_CROP_MARGIN = env_float("FACE_CROP_MARGIN", 1.5)
71
+
72
+ # MTCNN missed ~1% of CelebA samples at the default settings. If the first
73
+ # pass finds nothing, retry once with a smaller minimum size and lower
74
+ # detection thresholds. Downstream identity matching (cosine similarity)
75
+ # still rejects the extra low-confidence detections that are not real matches.
76
+ MTCNN_RETRY_MIN_FACE_SIZE = env_int("MTCNN_RETRY_MIN_FACE_SIZE", 12)
77
+ MTCNN_RETRY_CONFIDENCE = env_float("MTCNN_RETRY_CONFIDENCE", 0.45)
78
+
79
+ # ── Module-level singletons ───────────────────────────────────────────────────
80
+ _detector: Optional[nn.Module] = None
81
+ _embedder: Optional[nn.Module] = None
82
+ _face_detector = None
83
+
84
+
85
+ def detector_ready() -> bool:
86
+ """True once the MTCNN detector has been loaded."""
87
+ return _face_detector is not None
88
+
89
+
90
+ def embedder_ready() -> bool:
91
+ """True once the AdaFace embedder has been loaded."""
92
+ return _embedder is not None
93
+
94
+
95
+ # ─────────────────────────────────────────────────────────────────────────────
96
+ # MTCNN face detection (default)
97
+ # ─────────────────────────────────────────────────────────────────────────────
98
+
99
+ def load_face_detector(device: str = DEVICE):
100
+ """Return the MTCNN face detector singleton."""
101
+ global _face_detector
102
+ if _face_detector is not None:
103
+ return _face_detector
104
+
105
+ if not _MTCNN_AVAILABLE:
106
+ raise RuntimeError(
107
+ "MTCNN face detector is not available. "
108
+ "Expected the vendored AdaFace repository at "
109
+ f"{_ADAFACE_REPO}."
110
+ )
111
+
112
+ mtcnn_device = "cuda:0" if str(device).startswith("cuda") and torch.cuda.is_available() else "cpu"
113
+ _face_detector = _mtcnn.MTCNN(device=mtcnn_device, crop_size=(112, 112))
114
+ # Conservative defaults; exposed through env for ops tuning.
115
+ _face_detector.min_face_size = env_int("MTCNN_MIN_FACE_SIZE", 20)
116
+ print(f"[Sentinel-AI] MTCNN face detector loaded on {mtcnn_device}")
117
+ return _face_detector
118
+
119
+
120
+ def _detect_pass(
121
+ detector,
122
+ image,
123
+ min_face_size: int,
124
+ confidence_threshold: float,
125
+ thresholds,
126
+ ) -> list[dict]:
127
+ """Run one MTCNN pass and return score-sorted face dictionaries."""
128
+ boxes, landmarks = detector.detect_faces(
129
+ image,
130
+ min_face_size,
131
+ thresholds,
132
+ detector.nms_thresholds,
133
+ detector.factor,
134
+ )
135
+
136
+ faces: list[dict] = []
137
+ for box, points in zip(boxes, landmarks):
138
+ score = float(box[4]) if len(box) > 4 else 1.0
139
+ if score < confidence_threshold:
140
+ continue
141
+ faces.append({
142
+ "box": [int(round(v)) for v in box[:4]],
143
+ "score": round(score, 4),
144
+ "landmarks": [float(v) for v in points[:10]],
145
+ })
146
+
147
+ faces.sort(key=lambda item: item["score"], reverse=True)
148
+ return faces
149
+
150
+
151
+ def detect_faces(image, confidence_threshold: float = 0.6) -> list[dict]:
152
+ """
153
+ Detect faces with MTCNN, retrying once with relaxed thresholds.
154
+
155
+ The first pass uses the detector's normal settings. If it finds no face,
156
+ a second pass lowers the minimum face size and the P/R/O thresholds
157
+ (``MTCNN_RETRY_MIN_FACE_SIZE`` / ``MTCNN_RETRY_CONFIDENCE``). The caller
158
+ still compares the resulting embedding against ``MATCH_SIMILARITY_THRESHOLD``,
159
+ so a relaxed detection cannot by itself produce a false identity match.
160
+
161
+ Parameters
162
+ ----------
163
+ image : PIL.Image.Image RGB image
164
+ confidence_threshold : minimum MTCNN probability to keep a face
165
+
166
+ Returns
167
+ -------
168
+ List of ``{"box": [x1, y1, x2, y2], "score": float,
169
+ "landmarks": [x0, …, x4, y0, …, y4]}`` sorted by score.
170
+ """
171
+ detector = load_face_detector()
172
+ rgb_image = image.convert("RGB")
173
+
174
+ faces = _detect_pass(
175
+ detector,
176
+ rgb_image,
177
+ detector.min_face_size,
178
+ confidence_threshold,
179
+ detector.thresholds,
180
+ )
181
+ if faces:
182
+ return faces
183
+
184
+ retry_min_size = min(int(detector.min_face_size), int(MTCNN_RETRY_MIN_FACE_SIZE))
185
+ retry_confidence = min(float(confidence_threshold), float(MTCNN_RETRY_CONFIDENCE))
186
+ # Lower each P/R/O stage threshold to the configured retry confidence.
187
+ # Using ``retry_confidence`` here would wrongly lower the later stages when
188
+ # the retry confidence is *higher* than the caller's threshold.
189
+ retry_stage_threshold = float(MTCNN_RETRY_CONFIDENCE)
190
+ retry_thresholds = [
191
+ min(float(threshold), retry_stage_threshold)
192
+ for threshold in detector.thresholds
193
+ ]
194
+
195
+ # Skip a pointless second pass when the retry settings are not actually
196
+ # more permissive than the first pass.
197
+ if (
198
+ retry_min_size >= int(detector.min_face_size)
199
+ and retry_confidence >= float(confidence_threshold)
200
+ and retry_thresholds == list(detector.thresholds)
201
+ ):
202
+ return faces
203
+
204
+ return _detect_pass(
205
+ detector,
206
+ rgb_image,
207
+ retry_min_size,
208
+ retry_confidence,
209
+ retry_thresholds,
210
+ )
211
+
212
+
213
+ def align_face(image, landmarks: list[float], detector=None):
214
+ """
215
+ Warp a face to the canonical 112x112 template using MTCNN's 5 landmarks.
216
+
217
+ This is the standard AdaFace preprocessing. It removes pose, scale, and
218
+ framing variance, so embeddings are comparable regardless of how the
219
+ source image was cropped. ``landmarks`` is the flat 10-value list
220
+ returned by :func:`detect_faces`.
221
+
222
+ Falls back to a plain context crop if the warp fails.
223
+ """
224
+ from PIL import Image as _Image
225
+ from mtcnn_pytorch.src.align_trans import warp_and_crop_face # type: ignore
226
+
227
+ if not landmarks or len(landmarks) < 10:
228
+ raise ValueError("MTCNN landmarks are required for alignment.")
229
+
230
+ detector = detector or load_face_detector()
231
+ points = [[landmarks[j], landmarks[j + 5]] for j in range(5)]
232
+ warped = warp_and_crop_face(
233
+ np.array(image.convert("RGB")),
234
+ points,
235
+ detector.refrence,
236
+ crop_size=detector.crop_size,
237
+ )
238
+ return _Image.fromarray(warped).convert("RGB")
239
+
240
+
241
+
242
+ def square_context_crop(image, box: list[int], margin: float = FACE_CROP_MARGIN):
243
+ """
244
+ Crop a square region around a face box, including hair/shoulder context.
245
+
246
+ CelebA images (used to build the index) are tightly framed portraits. A
247
+ raw MTCNN box crops only the inner face and loses that context, which
248
+ measurably degrades identity matching. Expanding the box by ``margin``
249
+ and forcing a square aspect reproduces the index framing.
250
+
251
+ PIL pads out-of-bounds crop areas with black, so faces near an image edge
252
+ are handled without special-casing.
253
+ """
254
+ x1, y1, x2, y2 = (float(v) for v in box[:4])
255
+ cx, cy = (x1 + x2) / 2.0, (y1 + y2) / 2.0
256
+ side = max(x2 - x1, y2 - y1) * max(margin, 1.0)
257
+ half = side / 2.0
258
+
259
+ left = int(round(cx - half))
260
+ top = int(round(cy - half))
261
+ right = int(round(cx + half))
262
+ bottom = int(round(cy + half))
263
+
264
+ if right <= left or bottom <= top: # degenerate box
265
+ return image.convert("RGB")
266
+
267
+ return image.crop((left, top, right, bottom)).convert("RGB")
268
+
269
+
270
+ # ─────────────────────────────────────────────────────────────────────────────
271
+ # Optional RF-DETR detector (kept for backwards compatibility)
272
+ # ─────────────────────────────────────────────────────────────────────────────
273
+
274
+ def load_rfdetr(device: str = DEVICE):
275
+ """
276
+ Return the RF-DETR detector singleton, if the optional dependency is
277
+ installed. RF-DETR is a general COCO object detector and is *not* used by
278
+ the default inference path.
279
+ """
280
+ global _detector
281
+ if _detector is not None:
282
+ return _detector
283
+
284
+ if not _RFDETR_AVAILABLE:
285
+ raise RuntimeError(
286
+ "RF-DETR is not installed. "
287
+ "Install it with: pip install 'rfdetr>=1.11'"
288
+ )
289
+
290
+ print("[Sentinel-AI] Loading optional RF-DETR Small weights...")
291
+ wrapper = RFDETRSmall()
292
+
293
+ # rfdetr >= 1.11 exposes .inference(); older releases used
294
+ # .optimize_for_inference(). Support both rather than crashing at startup.
295
+ try:
296
+ if hasattr(wrapper, "inference"):
297
+ wrapper.inference(compile=False, inplace=True, dtype=torch.float16)
298
+ elif hasattr(wrapper, "optimize_for_inference"):
299
+ wrapper.optimize_for_inference(compile=False, inplace=True, dtype="float16")
300
+ except Exception as exc: # pragma: no cover
301
+ print(f"[Sentinel-AI] RF-DETR FP16 optimisation skipped: {exc}")
302
+
303
+ _detector = wrapper
304
+ print("[Sentinel-AI] RF-DETR Small loaded")
305
+ return _detector
306
+
307
+
308
+ def _safe_parse_detr_output(
309
+ outputs: dict,
310
+ image_h: int,
311
+ image_w: int,
312
+ confidence_threshold: float,
313
+ ) -> list[dict]:
314
+ """
315
+ Parse raw DETR forward-pass output into detection dicts.
316
+
317
+ Raises KeyError if RF-DETR changes its output format (fail-fast).
318
+ """
319
+ logits = outputs["pred_logits"][0].softmax(-1) # (queries, classes+1)
320
+ boxes = outputs["pred_boxes"][0] # (queries, 4)
321
+
322
+ scores, labels = logits[..., :-1].max(-1)
323
+
324
+ detections: list[dict] = []
325
+ for score, label, box in zip(scores, labels, boxes):
326
+ s = float(score)
327
+ if s < confidence_threshold:
328
+ continue
329
+ cx, cy, w, h = box.float().tolist()
330
+ x1 = int(max(0, (cx - w / 2) * image_w))
331
+ y1 = int(max(0, (cy - h / 2) * image_h))
332
+ x2 = int(min(image_w, (cx + w / 2) * image_w))
333
+ y2 = int(min(image_h, (cy + h / 2) * image_h))
334
+ detections.append({
335
+ "box": [x1, y1, x2, y2],
336
+ "score": round(s, 4),
337
+ "label": int(label.item()),
338
+ })
339
+
340
+ return detections
341
+
342
+
343
+ def rfdetr_detect(
344
+ image_tensor: torch.Tensor,
345
+ model,
346
+ confidence_threshold: float = 0.6,
347
+ ) -> list[dict]:
348
+ """Run the optional RF-DETR detector on a CHW tensor in [0, 1]."""
349
+ from PIL import Image
350
+
351
+ image_np = (image_tensor.detach().permute(1, 2, 0).cpu().numpy() * 255).clip(0, 255).astype(np.uint8)
352
+ pil_image = Image.fromarray(image_np)
353
+
354
+ detections_sv = model.predict(pil_image, confidence=confidence_threshold)
355
+
356
+ detections = []
357
+ for i in range(len(detections_sv.xyxy)):
358
+ x1, y1, x2, y2 = detections_sv.xyxy[i].astype(int).tolist()
359
+ score = float(detections_sv.confidence[i])
360
+ label = int(detections_sv.class_id[i]) if detections_sv.class_id is not None else 0
361
+
362
+ detections.append({
363
+ "box": [x1, y1, x2, y2],
364
+ "score": round(score, 4),
365
+ "label": label,
366
+ })
367
+
368
+ detections.sort(key=lambda d: d["score"], reverse=True)
369
+ return detections
370
+
371
+
372
+ # ─────────────────────────────────────────────────────────────────────────────
373
+ # AdaFace embedding
374
+ # ─────────────────────────────────────────────────────────────────────────────
375
+
376
+ def load_adaface(
377
+ architecture: str = "ir_50",
378
+ weights_path: Path = ADAFACE_WEIGHTS,
379
+ device: str = DEVICE,
380
+ ) -> nn.Module:
381
+ """
382
+ Return the AdaFace face-embedding model singleton.
383
+
384
+ Raises RuntimeError if the checkpoint is missing or if the checkpoint does
385
+ not cover the model (the previous ``strict=False`` silently produced
386
+ random embeddings in that case).
387
+ """
388
+ global _embedder
389
+ if _embedder is not None:
390
+ return _embedder
391
+
392
+ if not _ADAFACE_AVAILABLE:
393
+ raise RuntimeError(
394
+ "AdaFace is not available. "
395
+ "Clone https://github.com/mk-minchul/AdaFace into models_cache/adaface "
396
+ "and download adaface_ir50_ms1mv2.ckpt."
397
+ )
398
+
399
+ weights_path = Path(weights_path)
400
+ if not weights_path.exists():
401
+ raise RuntimeError(
402
+ f"AdaFace weights not found at {weights_path}. "
403
+ "Download adaface_ir50_ms1mv2.ckpt before starting the server."
404
+ )
405
+
406
+ model = adaface_net.build_model(architecture)
407
+
408
+ checkpoint = torch.load(str(weights_path), map_location="cpu", weights_only=False)
409
+ state_dict = checkpoint.get("state_dict", checkpoint)
410
+ # Strip the Lightning "model." prefix used by the official checkpoint.
411
+ state_dict = {k.replace("model.", "", 1): v for k, v in state_dict.items()}
412
+
413
+ result = model.load_state_dict(state_dict, strict=False)
414
+ missing = [k for k in result.missing_keys if "head." not in k]
415
+ if missing:
416
+ raise RuntimeError(
417
+ "AdaFace checkpoint does not match the model architecture; "
418
+ f"{len(missing)} required tensors are missing (e.g. {missing[:3]}). "
419
+ "Refusing to run with partially random weights."
420
+ )
421
+
422
+ model = model.to(device)
423
+ if str(device).startswith("cuda"):
424
+ model = model.half() # FP16
425
+ model.eval()
426
+ _embedder = model
427
+ return _embedder
428
+
429
+
430
+ def adaface_embed(
431
+ face_tensor: torch.Tensor,
432
+ model: nn.Module,
433
+ ) -> torch.Tensor:
434
+ """
435
+ Embed a single 112x112 face crop.
436
+
437
+ Parameters
438
+ ----------
439
+ face_tensor : torch.Tensor shape (3, 112, 112), normalised to [-1, 1]
440
+ model : AdaFace model from load_adaface()
441
+
442
+ Returns
443
+ -------
444
+ torch.Tensor shape (512,), L2-normalised float32 on CPU
445
+ """
446
+ device = next(model.parameters()).device
447
+ x = face_tensor.unsqueeze(0).to(device)
448
+ if device.type == "cuda":
449
+ x = x.half()
450
+
451
+ with torch.no_grad():
452
+ with torch.amp.autocast(device_type=device.type, enabled=(device.type == "cuda")):
453
+ # AdaFace always returns (embeddings, norms)
454
+ embeddings, _ = model(x)
455
+
456
+ embedding = embeddings[0].float().cpu()
457
+ embedding = embedding / embedding.norm().clamp(min=1e-12)
458
+ return embedding