webolavo commited on
Commit
e4988d4
·
verified ·
1 Parent(s): 5a2aa04

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +217 -17
app.py CHANGED
@@ -1,5 +1,6 @@
1
  import io
2
  import os
 
3
  import threading
4
  from contextlib import asynccontextmanager
5
  from inspect import signature
@@ -26,11 +27,37 @@ DEVICE = os.getenv("DEVICE", "cpu").strip().lower()
26
 
27
  TEXT_PROMPT = (
28
  "woman . man . young child . baby . female hand . male hand . "
29
- "female belly . male belly . female stomach . female leg . male leg"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
30
  )
31
  BOX_THRESHOLD = 0.25
32
- TEXT_THRESHOLD = 0.35
33
  NMS_IOU_THRESHOLD = 0.50
 
 
 
34
  MAX_IMAGE_BYTES = 20 * 1024 * 1024
35
 
36
  ARABIC_LABELS: Dict[str, str] = {
@@ -40,9 +67,12 @@ ARABIC_LABELS: Dict[str, str] = {
40
  "baby": "طفل أو رضيع",
41
  "female hand": "يد إمرأة",
42
  "male hand": "يد رجل",
 
 
 
 
43
  "female belly": "بطن إمرأة",
44
  "male belly": "بطن رجل",
45
- "female stomach": "hhdhhd",
46
  "female leg": "ساق إمرأة",
47
  "male leg": "ساق رجل",
48
  }
@@ -54,12 +84,77 @@ LABEL_FAMILIES = {
54
  "baby": "person",
55
  "female hand": "hand",
56
  "male hand": "hand",
 
 
 
 
57
  "female belly": "belly",
58
  "male belly": "belly",
59
  "female leg": "leg",
60
  "male leg": "leg",
61
  }
62
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
63
  processor: Optional[Any] = None
64
  model: Optional[Any] = None
65
  inference_lock = threading.Lock()
@@ -104,10 +199,19 @@ app = FastAPI(
104
 
105
 
106
  def canonical_label(raw_phrase: str) -> Optional[str]:
 
 
 
 
 
 
 
 
 
107
  normalized = " ".join(raw_phrase.lower().replace(".", " ").split())
108
- for label in sorted(ARABIC_LABELS, key=len, reverse=True):
109
- if normalized == label or label in normalized:
110
- return label
111
  return None
112
 
113
 
@@ -148,15 +252,13 @@ def contrastive_nms(detections: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
148
  return sorted(kept, key=lambda detection: detection["confidence"], reverse=True)
149
 
150
 
151
- def run_detection(image: Image.Image) -> List[Dict[str, Any]]:
152
- if processor is None or model is None:
153
- raise RuntimeError("The model is not ready.")
154
-
155
- inputs = processor(
156
- images=image,
157
- text=TEXT_PROMPT,
158
- return_tensors="pt",
159
- ).to(DEVICE)
160
  with torch.inference_mode():
161
  outputs = model(**inputs)
162
 
@@ -167,8 +269,8 @@ def run_detection(image: Image.Image) -> List[Dict[str, Any]]:
167
  outputs,
168
  inputs.input_ids,
169
  **{
170
- threshold_name: BOX_THRESHOLD,
171
- "text_threshold": TEXT_THRESHOLD,
172
  "target_sizes": [image.size[::-1]],
173
  },
174
  )[0]
@@ -177,6 +279,7 @@ def run_detection(image: Image.Image) -> List[Dict[str, Any]]:
177
  scores = result.get("scores", [])
178
  phrases = result.get("text_labels", result.get("labels", []))
179
  detections: List[Dict[str, Any]] = []
 
180
 
181
  for index, box in enumerate(boxes):
182
  if index >= len(scores) or index >= len(phrases):
@@ -186,6 +289,15 @@ def run_detection(image: Image.Image) -> List[Dict[str, Any]]:
186
  phrase = phrase.item()
187
  label = canonical_label(str(phrase))
188
  if label is None:
 
 
 
 
 
 
 
 
 
189
  continue
190
 
191
  detections.append(
@@ -197,6 +309,94 @@ def run_detection(image: Image.Image) -> List[Dict[str, Any]]:
197
  }
198
  )
199
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
200
  return contrastive_nms(detections)
201
 
202
 
 
1
  import io
2
  import os
3
+ import re
4
  import threading
5
  from contextlib import asynccontextmanager
6
  from inspect import signature
 
27
 
28
  TEXT_PROMPT = (
29
  "woman . man . young child . baby . female hand . male hand . "
30
+ "female belly . male belly . female leg . male leg"
31
+ )
32
+
33
+ # Small or printed body regions are much easier for Grounding DINO to find
34
+ # when semantically related labels are queried separately. Generic terms help
35
+ # create region proposals, while only gendered labels are returned to clients.
36
+ PROMPT_GROUPS = (
37
+ "woman . man . young child . baby",
38
+ (
39
+ "female eye . male eye . woman eye . man eye . "
40
+ "female eyebrow . male eyebrow . woman eyebrow . man eyebrow . "
41
+ "human eye . eye . eyebrow . face . head"
42
+ ),
43
+ (
44
+ "female belly . male belly . woman belly . man belly . "
45
+ "female stomach . male stomach . woman stomach . man stomach . "
46
+ "female abdomen . male abdomen . woman abdomen . man abdomen . "
47
+ "belly . stomach . abdomen . torso"
48
+ ),
49
+ (
50
+ "female hand . male hand . woman hand . man hand . "
51
+ "female leg . male leg . woman leg . man leg . "
52
+ "human hand . hand . human leg . leg"
53
+ ),
54
  )
55
  BOX_THRESHOLD = 0.25
56
+ TEXT_THRESHOLD = 0.22
57
  NMS_IOU_THRESHOLD = 0.50
58
+ REFINEMENT_BOX_THRESHOLD = 0.15
59
+ REFINEMENT_TEXT_THRESHOLD = 0.18
60
+ MAX_REFINEMENTS_PER_FAMILY = 2
61
  MAX_IMAGE_BYTES = 20 * 1024 * 1024
62
 
63
  ARABIC_LABELS: Dict[str, str] = {
 
67
  "baby": "طفل أو رضيع",
68
  "female hand": "يد إمرأة",
69
  "male hand": "يد رجل",
70
+ "female eye": "عين إمرأة",
71
+ "male eye": "عين رجل",
72
+ "female eyebrow": "حاجب إمرأة",
73
+ "male eyebrow": "حاجب رجل",
74
  "female belly": "بطن إمرأة",
75
  "male belly": "بطن رجل",
 
76
  "female leg": "ساق إمرأة",
77
  "male leg": "ساق رجل",
78
  }
 
84
  "baby": "person",
85
  "female hand": "hand",
86
  "male hand": "hand",
87
+ "female eye": "eye",
88
+ "male eye": "eye",
89
+ "female eyebrow": "eyebrow",
90
+ "male eyebrow": "eyebrow",
91
  "female belly": "belly",
92
  "male belly": "belly",
93
  "female leg": "leg",
94
  "male leg": "leg",
95
  }
96
 
97
+ LABEL_ALIASES = {
98
+ "woman eye": "female eye",
99
+ "female eye": "female eye",
100
+ "man eye": "male eye",
101
+ "male eye": "male eye",
102
+ "woman eyebrow": "female eyebrow",
103
+ "female eyebrow": "female eyebrow",
104
+ "man eyebrow": "male eyebrow",
105
+ "male eyebrow": "male eyebrow",
106
+ "woman stomach": "female belly",
107
+ "female stomach": "female belly",
108
+ "woman abdomen": "female belly",
109
+ "female abdomen": "female belly",
110
+ "woman belly": "female belly",
111
+ "female belly": "female belly",
112
+ "man stomach": "male belly",
113
+ "male stomach": "male belly",
114
+ "man abdomen": "male belly",
115
+ "male abdomen": "male belly",
116
+ "man belly": "male belly",
117
+ "male belly": "male belly",
118
+ "woman hand": "female hand",
119
+ "female hand": "female hand",
120
+ "man hand": "male hand",
121
+ "male hand": "male hand",
122
+ "woman leg": "female leg",
123
+ "female leg": "female leg",
124
+ "man leg": "male leg",
125
+ "male leg": "male leg",
126
+ "young child": "young child",
127
+ "woman": "woman",
128
+ "baby": "baby",
129
+ "man": "man",
130
+ }
131
+
132
+ GENERIC_FAMILY_ALIASES = {
133
+ "human eye": "eye",
134
+ "eye": "eye",
135
+ "eyebrow": "eyebrow",
136
+ "belly": "belly",
137
+ "stomach": "belly",
138
+ "abdomen": "belly",
139
+ "torso": "belly",
140
+ "human hand": "hand",
141
+ "hand": "hand",
142
+ "human leg": "leg",
143
+ "leg": "leg",
144
+ }
145
+
146
+ GENDER_REFINEMENT_PROMPTS = {
147
+ "eye": "female eye . woman eye . male eye . man eye",
148
+ "eyebrow": "female eyebrow . woman eyebrow . male eyebrow . man eyebrow",
149
+ "belly": (
150
+ "female belly . woman belly . female stomach . woman stomach . "
151
+ "female abdomen . woman abdomen . male belly . man belly . "
152
+ "male stomach . man stomach . male abdomen . man abdomen"
153
+ ),
154
+ "hand": "female hand . woman hand . male hand . man hand",
155
+ "leg": "female leg . woman leg . male leg . man leg",
156
+ }
157
+
158
  processor: Optional[Any] = None
159
  model: Optional[Any] = None
160
  inference_lock = threading.Lock()
 
199
 
200
 
201
  def canonical_label(raw_phrase: str) -> Optional[str]:
202
+ normalized = raw_phrase.lower().replace("'s", "").replace(".", " ")
203
+ normalized = " ".join(normalized.split())
204
+ for alias in sorted(LABEL_ALIASES, key=len, reverse=True):
205
+ if normalized == alias or re.search(rf"\b{re.escape(alias)}\b", normalized):
206
+ return LABEL_ALIASES[alias]
207
+ return None
208
+
209
+
210
+ def generic_family(raw_phrase: str) -> Optional[str]:
211
  normalized = " ".join(raw_phrase.lower().replace(".", " ").split())
212
+ for alias in sorted(GENERIC_FAMILY_ALIASES, key=len, reverse=True):
213
+ if normalized == alias or re.search(rf"\b{re.escape(alias)}\b", normalized):
214
+ return GENERIC_FAMILY_ALIASES[alias]
215
  return None
216
 
217
 
 
252
  return sorted(kept, key=lambda detection: detection["confidence"], reverse=True)
253
 
254
 
255
+ def run_prompt(
256
+ image: Image.Image,
257
+ prompt: str,
258
+ box_threshold: float = BOX_THRESHOLD,
259
+ text_threshold: float = TEXT_THRESHOLD,
260
+ ) -> tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
261
+ inputs = processor(images=image, text=prompt, return_tensors="pt").to(DEVICE)
 
 
262
  with torch.inference_mode():
263
  outputs = model(**inputs)
264
 
 
269
  outputs,
270
  inputs.input_ids,
271
  **{
272
+ threshold_name: box_threshold,
273
+ "text_threshold": text_threshold,
274
  "target_sizes": [image.size[::-1]],
275
  },
276
  )[0]
 
279
  scores = result.get("scores", [])
280
  phrases = result.get("text_labels", result.get("labels", []))
281
  detections: List[Dict[str, Any]] = []
282
+ generic_regions: List[Dict[str, Any]] = []
283
 
284
  for index, box in enumerate(boxes):
285
  if index >= len(scores) or index >= len(phrases):
 
289
  phrase = phrase.item()
290
  label = canonical_label(str(phrase))
291
  if label is None:
292
+ family = generic_family(str(phrase))
293
+ if family is not None:
294
+ generic_regions.append(
295
+ {
296
+ "family": family,
297
+ "confidence": float(scores[index].item()),
298
+ "box": [float(value) for value in box.tolist()],
299
+ }
300
+ )
301
  continue
302
 
303
  detections.append(
 
309
  }
310
  )
311
 
312
+ return detections, generic_regions
313
+
314
+
315
+ def suppress_generic_regions(regions: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
316
+ kept: List[Dict[str, Any]] = []
317
+ family_counts: Dict[str, int] = {}
318
+
319
+ for candidate in sorted(
320
+ regions,
321
+ key=lambda region: region["confidence"],
322
+ reverse=True,
323
+ ):
324
+ family = candidate["family"]
325
+ if family_counts.get(family, 0) >= MAX_REFINEMENTS_PER_FAMILY:
326
+ continue
327
+ if any(
328
+ selected["family"] == family
329
+ and box_iou(candidate["box"], selected["box"]) > NMS_IOU_THRESHOLD
330
+ for selected in kept
331
+ ):
332
+ continue
333
+ kept.append(candidate)
334
+ family_counts[family] = family_counts.get(family, 0) + 1
335
+ return kept
336
+
337
+
338
+ def crop_region(
339
+ image: Image.Image,
340
+ box: Sequence[float],
341
+ ) -> tuple[Image.Image, float, float]:
342
+ x1, y1, x2, y2 = box
343
+ box_width = max(1.0, x2 - x1)
344
+ box_height = max(1.0, y2 - y1)
345
+ padding_x = box_width * 0.20
346
+ padding_y = box_height * 0.20
347
+ left = max(0.0, x1 - padding_x)
348
+ top = max(0.0, y1 - padding_y)
349
+ right = min(float(image.width), x2 + padding_x)
350
+ bottom = min(float(image.height), y2 + padding_y)
351
+ return image.crop((left, top, right, bottom)), left, top
352
+
353
+
354
+ def run_detection(image: Image.Image) -> List[Dict[str, Any]]:
355
+ if processor is None or model is None:
356
+ raise RuntimeError("The model is not ready.")
357
+
358
+ detections: List[Dict[str, Any]] = []
359
+ generic_regions: List[Dict[str, Any]] = []
360
+ for prompt in PROMPT_GROUPS:
361
+ prompt_detections, prompt_generic_regions = run_prompt(image, prompt)
362
+ detections.extend(prompt_detections)
363
+ generic_regions.extend(prompt_generic_regions)
364
+
365
+ # Generic detections are localized first, then the cropped region is
366
+ # contrasted with female/male prompts. This recovers small printed body
367
+ # regions without guessing a gender from a generic label.
368
+ for region in suppress_generic_regions(generic_regions):
369
+ family = region["family"]
370
+ if any(
371
+ LABEL_FAMILIES[detection["label"]] == family
372
+ and box_iou(detection["box"], region["box"]) > 0.20
373
+ for detection in detections
374
+ ):
375
+ continue
376
+
377
+ cropped_image, offset_x, offset_y = crop_region(image, region["box"])
378
+ refined, _ = run_prompt(
379
+ cropped_image,
380
+ GENDER_REFINEMENT_PROMPTS[family],
381
+ box_threshold=REFINEMENT_BOX_THRESHOLD,
382
+ text_threshold=REFINEMENT_TEXT_THRESHOLD,
383
+ )
384
+ matching = [
385
+ detection
386
+ for detection in refined
387
+ if LABEL_FAMILIES[detection["label"]] == family
388
+ ]
389
+ if not matching:
390
+ continue
391
+
392
+ best = max(matching, key=lambda detection: detection["confidence"])
393
+ best["box"] = [
394
+ best["box"][0] + offset_x,
395
+ best["box"][1] + offset_y,
396
+ best["box"][2] + offset_x,
397
+ best["box"][3] + offset_y,
398
+ ]
399
+ detections.append(best)
400
  return contrastive_nms(detections)
401
 
402