StillWorking commited on
Commit
23cafee
Β·
verified Β·
1 Parent(s): 91d148e

Upload backend/api/routes/analyze.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. backend/api/routes/analyze.py +256 -0
backend/api/routes/analyze.py ADDED
@@ -0,0 +1,256 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ api/routes/analyze.py β€” Sentinel-AI
3
+ ========================================
4
+ POST /api/v1/analyze-frame
5
+
6
+ Accepts a raw image upload (JPEG/PNG/WebP), runs the full inference pipeline:
7
+ 1. MTCNN detects faces β†’ aligns the highest-confidence face to 112Γ—112
8
+ 2. AdaFace embeds the aligned face β†’ 512-D vector
9
+ 3. FAISS nearest-neighbour search β†’ cosine similarity + attributes
10
+ 4. cascadeflow + Gemini generates the security alert
11
+
12
+ Returns the JSON payload defined in AnalyzeFrameResponse.
13
+
14
+ Blocking notes
15
+ --------------
16
+ PyTorch inference, PIL decoding, and FAISS search are all CPU/GPU blocking
17
+ operations. Running them inside an async function directly would stall the
18
+ uvicorn event loop. We therefore use two strategies:
19
+
20
+ β€’ The entire CV+FAISS pipeline is wrapped in asyncio.to_thread() so it
21
+ executes in the default thread-pool and releases the event loop.
22
+ β€’ The Gemini call (generate_alert) is natively async and runs normally.
23
+ """
24
+
25
+ from __future__ import annotations
26
+
27
+ import asyncio
28
+ import io
29
+ import os
30
+ from typing import Annotated, Any
31
+
32
+ import torch
33
+ import torchvision.transforms as T
34
+ from fastapi import APIRouter, File, HTTPException, UploadFile
35
+ from PIL import Image, UnidentifiedImageError
36
+
37
+ from api.schemas import AnalyzeFrameResponse, CascadeflowMetrics
38
+ from core.faiss_store import search_by_vector
39
+ from core.gemini_agent import generate_alert
40
+ from core.models import (
41
+ DEVICE,
42
+ FACE_CROP_MARGIN,
43
+ adaface_embed,
44
+ align_face,
45
+ detect_faces,
46
+ load_adaface,
47
+ square_context_crop,
48
+ )
49
+ from core.paths import env_float, env_int
50
+
51
+ router = APIRouter()
52
+
53
+ # Reject oversized uploads before decoding them into memory.
54
+ MAX_UPLOAD_BYTES = env_int("MAX_UPLOAD_BYTES", 10 * 1024 * 1024)
55
+
56
+ # MTCNN probability below which a detection is discarded.
57
+ DETECTION_CONFIDENCE = env_float("DETECTION_CONFIDENCE", 0.6)
58
+
59
+ # Cosine similarity below which the nearest neighbour is *not* considered a
60
+ # match. With the index and the query both using MTCNN landmark alignment,
61
+ # correct identities scored ~0.98 (min 0.90 over a 48-image sample) while
62
+ # unrelated portraits stayed below ~0.35.
63
+ MATCH_SIMILARITY_THRESHOLD = env_float("MATCH_SIMILARITY_THRESHOLD", 0.50)
64
+
65
+ # AdaFace input normalisation (must match build_index.py)
66
+ _FACE_TRANSFORM = T.Compose([
67
+ T.Resize((112, 112)),
68
+ T.ToTensor(),
69
+ T.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]),
70
+ ])
71
+
72
+
73
+ class InvalidImageError(ValueError):
74
+ """Raised when the uploaded bytes are not a decodable image."""
75
+
76
+
77
+ def _crop_face(image: Image.Image, face: dict) -> torch.Tensor:
78
+ """
79
+ Produce an AdaFace-ready 3x112x112 tensor for one detected face.
80
+
81
+ Landmark alignment is the default: it matches the preprocessing used to
82
+ build the index. If the warp fails for a malformed landmark set, fall
83
+ back to a context-expanded square crop.
84
+ """
85
+ try:
86
+ aligned = align_face(image, face.get("landmarks", []))
87
+ except Exception:
88
+ aligned = square_context_crop(image, face["box"], margin=FACE_CROP_MARGIN)
89
+ return _FACE_TRANSFORM(aligned)
90
+
91
+
92
+ # ─────────────────────────────────────────────────────────────────────────────
93
+ # Blocking CV + FAISS pipeline β€” runs in a thread-pool via asyncio.to_thread
94
+ # ─────────────────────────────────────────────────────────────────────────────
95
+
96
+ def _run_cv_pipeline(raw_bytes: bytes) -> dict[str, Any]:
97
+ """
98
+ Decode image, detect faces, embed the best face, search FAISS.
99
+
100
+ Returns a plain dict to keep the result serialisable across threads:
101
+ status : "no_face" | "no_match" | "ok"
102
+ faces_detected : int
103
+ detection_score : float
104
+ attributes : list[str]
105
+ similarity : float
106
+ """
107
+ # ── 1. Decode (invalid uploads must not surface as a 500) ─────────────────
108
+ try:
109
+ with Image.open(io.BytesIO(raw_bytes)) as opened:
110
+ image = opened.convert("RGB")
111
+ except (UnidentifiedImageError, OSError, ValueError) as exc:
112
+ raise InvalidImageError("Uploaded file is not a valid image.") from exc
113
+
114
+ # ── 2. Detect faces with MTCNN ────────────────────────────────────────────
115
+ try:
116
+ faces = detect_faces(image, confidence_threshold=DETECTION_CONFIDENCE)
117
+ except Exception as exc: # model/runtime error
118
+ raise RuntimeError(f"Face detection failed: {exc}") from exc
119
+
120
+ if not faces:
121
+ return {
122
+ "status": "no_face",
123
+ "faces_detected": 0,
124
+ "detection_score": 0.0,
125
+ "attributes": [],
126
+ "similarity": 0.0,
127
+ }
128
+
129
+ best = faces[0] # sorted by score
130
+ faces_detected = len(faces)
131
+
132
+ # ── 3. Embed best face ────────────────────────────────────────────────────
133
+ embedder = load_adaface(device=DEVICE)
134
+ face_tensor = _crop_face(image, best) # (3, 112, 112)
135
+ embedding = adaface_embed(face_tensor, embedder) # (512,) float32 CPU
136
+
137
+ # ── 4. FAISS search ───────────────────────────────────────────────────────
138
+ results = search_by_vector(embedding.numpy(), top_k=1)
139
+ if not results:
140
+ return {
141
+ "status": "no_match",
142
+ "faces_detected": faces_detected,
143
+ "detection_score": round(float(best["score"]), 4),
144
+ "attributes": [],
145
+ "similarity": 0.0,
146
+ }
147
+
148
+ top_match = results[0]
149
+ similarity = float(top_match["similarity"])
150
+
151
+ # ── 5. Threshold: a nearest neighbour is not automatically a match ────────
152
+ if similarity < MATCH_SIMILARITY_THRESHOLD:
153
+ return {
154
+ "status": "no_match",
155
+ "faces_detected": faces_detected,
156
+ "detection_score": round(float(best["score"]), 4),
157
+ "attributes": [],
158
+ "similarity": similarity,
159
+ }
160
+
161
+ return {
162
+ "status": "ok",
163
+ "faces_detected": faces_detected,
164
+ "detection_score": round(float(best["score"]), 4),
165
+ "attributes": top_match["attributes"],
166
+ "similarity": similarity,
167
+ }
168
+
169
+
170
+ # ─────────────────────────────────────────────────────────────────────────────
171
+ # Endpoint
172
+ # ─────────────────────────────────────────────────────────────────────────────
173
+
174
+ @router.post(
175
+ "/analyze-frame",
176
+ response_model=AnalyzeFrameResponse,
177
+ summary="Analyse a camera frame: detect faces, embed, match, and generate alert",
178
+ )
179
+ async def analyze_frame(
180
+ file: Annotated[UploadFile, File(description="Raw image file (JPEG, PNG, WebP, …)")],
181
+ ) -> AnalyzeFrameResponse:
182
+ """
183
+ Processes a live camera frame through the full Sentinel-AI pipeline.
184
+
185
+ - **file**: raw image file (JPEG, PNG, WebP, …), max 10 MB by default
186
+ """
187
+ # ── Read upload bytes, enforcing a hard size cap ──────────────────────────
188
+ try:
189
+ raw_bytes = await file.read(MAX_UPLOAD_BYTES + 1)
190
+ except Exception:
191
+ raise HTTPException(status_code=422, detail="Could not read uploaded file.")
192
+
193
+ if not raw_bytes:
194
+ raise HTTPException(status_code=422, detail="Uploaded file is empty.")
195
+
196
+ if len(raw_bytes) > MAX_UPLOAD_BYTES:
197
+ raise HTTPException(
198
+ status_code=413,
199
+ detail=f"Uploaded file exceeds the {MAX_UPLOAD_BYTES} byte limit.",
200
+ )
201
+
202
+ # ── CV + FAISS in thread-pool (blocking) ──────────────────────────────────
203
+ try:
204
+ cv_result: dict[str, Any] = await asyncio.to_thread(_run_cv_pipeline, raw_bytes)
205
+ except InvalidImageError as exc:
206
+ raise HTTPException(status_code=422, detail=str(exc))
207
+ except Exception:
208
+ # Never leak stack traces / absolute paths / internal model details.
209
+ raise HTTPException(status_code=500, detail="Inference pipeline error.")
210
+
211
+ # ── Early exits ───────────────────────────────────────────────────────────
212
+ if cv_result["status"] == "no_face":
213
+ return AnalyzeFrameResponse(
214
+ status="no_face_detected",
215
+ faces_detected=0,
216
+ match_confidence=None,
217
+ matched_attributes=[],
218
+ agent_report=None,
219
+ cascadeflow_metrics=None,
220
+ )
221
+
222
+ if cv_result["status"] == "no_match":
223
+ return AnalyzeFrameResponse(
224
+ status="no_match_found",
225
+ faces_detected=cv_result["faces_detected"],
226
+ match_confidence=None,
227
+ matched_attributes=[],
228
+ agent_report=(
229
+ "No identity in the database exceeds the "
230
+ f"{MATCH_SIMILARITY_THRESHOLD:.2f} cosine-similarity threshold."
231
+ ),
232
+ cascadeflow_metrics=None,
233
+ )
234
+
235
+ # ── Gemini alert (async, no blocking) ────────────────────────────────────
236
+ agent_result: dict[str, Any] = await generate_alert(
237
+ attributes=cv_result["attributes"],
238
+ similarity=cv_result["similarity"],
239
+ )
240
+
241
+ cf_raw = agent_result.get("cascadeflow_metrics", {})
242
+ cf_metrics = CascadeflowMetrics(
243
+ latency_ms=cf_raw.get("latency_ms", 0.0),
244
+ cost_usd=cf_raw.get("cost_usd", 0.0),
245
+ )
246
+
247
+ clamped_confidence = min(max(float(cv_result["similarity"]), 0.0), 1.0)
248
+
249
+ return AnalyzeFrameResponse(
250
+ status="success",
251
+ faces_detected=cv_result["faces_detected"],
252
+ match_confidence=round(clamped_confidence, 4),
253
+ matched_attributes=cv_result["attributes"],
254
+ agent_report=agent_result.get("agent_report", ""),
255
+ cascadeflow_metrics=cf_metrics,
256
+ )