Update app.py
Browse files
app.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
| 1 |
import io
|
| 2 |
import os
|
|
|
|
| 3 |
import threading
|
| 4 |
from contextlib import asynccontextmanager
|
| 5 |
from inspect import signature
|
|
@@ -26,11 +27,37 @@ DEVICE = os.getenv("DEVICE", "cpu").strip().lower()
|
|
| 26 |
|
| 27 |
TEXT_PROMPT = (
|
| 28 |
"woman . man . young child . baby . female hand . male hand . "
|
| 29 |
-
"female belly . male belly . female
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 30 |
)
|
| 31 |
BOX_THRESHOLD = 0.25
|
| 32 |
-
TEXT_THRESHOLD = 0.
|
| 33 |
NMS_IOU_THRESHOLD = 0.50
|
|
|
|
|
|
|
|
|
|
| 34 |
MAX_IMAGE_BYTES = 20 * 1024 * 1024
|
| 35 |
|
| 36 |
ARABIC_LABELS: Dict[str, str] = {
|
|
@@ -40,9 +67,12 @@ ARABIC_LABELS: Dict[str, str] = {
|
|
| 40 |
"baby": "طفل أو رضيع",
|
| 41 |
"female hand": "يد إمرأة",
|
| 42 |
"male hand": "يد رجل",
|
|
|
|
|
|
|
|
|
|
|
|
|
| 43 |
"female belly": "بطن إمرأة",
|
| 44 |
"male belly": "بطن رجل",
|
| 45 |
-
"female stomach": "hhdhhd",
|
| 46 |
"female leg": "ساق إمرأة",
|
| 47 |
"male leg": "ساق رجل",
|
| 48 |
}
|
|
@@ -54,12 +84,77 @@ LABEL_FAMILIES = {
|
|
| 54 |
"baby": "person",
|
| 55 |
"female hand": "hand",
|
| 56 |
"male hand": "hand",
|
|
|
|
|
|
|
|
|
|
|
|
|
| 57 |
"female belly": "belly",
|
| 58 |
"male belly": "belly",
|
| 59 |
"female leg": "leg",
|
| 60 |
"male leg": "leg",
|
| 61 |
}
|
| 62 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 63 |
processor: Optional[Any] = None
|
| 64 |
model: Optional[Any] = None
|
| 65 |
inference_lock = threading.Lock()
|
|
@@ -104,10 +199,19 @@ app = FastAPI(
|
|
| 104 |
|
| 105 |
|
| 106 |
def canonical_label(raw_phrase: str) -> Optional[str]:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 107 |
normalized = " ".join(raw_phrase.lower().replace(".", " ").split())
|
| 108 |
-
for
|
| 109 |
-
if normalized ==
|
| 110 |
-
return
|
| 111 |
return None
|
| 112 |
|
| 113 |
|
|
@@ -148,15 +252,13 @@ def contrastive_nms(detections: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
|
| 148 |
return sorted(kept, key=lambda detection: detection["confidence"], reverse=True)
|
| 149 |
|
| 150 |
|
| 151 |
-
def
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
return_tensors="pt",
|
| 159 |
-
).to(DEVICE)
|
| 160 |
with torch.inference_mode():
|
| 161 |
outputs = model(**inputs)
|
| 162 |
|
|
@@ -167,8 +269,8 @@ def run_detection(image: Image.Image) -> List[Dict[str, Any]]:
|
|
| 167 |
outputs,
|
| 168 |
inputs.input_ids,
|
| 169 |
**{
|
| 170 |
-
threshold_name:
|
| 171 |
-
"text_threshold":
|
| 172 |
"target_sizes": [image.size[::-1]],
|
| 173 |
},
|
| 174 |
)[0]
|
|
@@ -177,6 +279,7 @@ def run_detection(image: Image.Image) -> List[Dict[str, Any]]:
|
|
| 177 |
scores = result.get("scores", [])
|
| 178 |
phrases = result.get("text_labels", result.get("labels", []))
|
| 179 |
detections: List[Dict[str, Any]] = []
|
|
|
|
| 180 |
|
| 181 |
for index, box in enumerate(boxes):
|
| 182 |
if index >= len(scores) or index >= len(phrases):
|
|
@@ -186,6 +289,15 @@ def run_detection(image: Image.Image) -> List[Dict[str, Any]]:
|
|
| 186 |
phrase = phrase.item()
|
| 187 |
label = canonical_label(str(phrase))
|
| 188 |
if label is None:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 189 |
continue
|
| 190 |
|
| 191 |
detections.append(
|
|
@@ -197,6 +309,94 @@ def run_detection(image: Image.Image) -> List[Dict[str, Any]]:
|
|
| 197 |
}
|
| 198 |
)
|
| 199 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 200 |
return contrastive_nms(detections)
|
| 201 |
|
| 202 |
|
|
|
|
| 1 |
import io
|
| 2 |
import os
|
| 3 |
+
import re
|
| 4 |
import threading
|
| 5 |
from contextlib import asynccontextmanager
|
| 6 |
from inspect import signature
|
|
|
|
| 27 |
|
| 28 |
TEXT_PROMPT = (
|
| 29 |
"woman . man . young child . baby . female hand . male hand . "
|
| 30 |
+
"female belly . male belly . female leg . male leg"
|
| 31 |
+
)
|
| 32 |
+
|
| 33 |
+
# Small or printed body regions are much easier for Grounding DINO to find
|
| 34 |
+
# when semantically related labels are queried separately. Generic terms help
|
| 35 |
+
# create region proposals, while only gendered labels are returned to clients.
|
| 36 |
+
PROMPT_GROUPS = (
|
| 37 |
+
"woman . man . young child . baby",
|
| 38 |
+
(
|
| 39 |
+
"female eye . male eye . woman eye . man eye . "
|
| 40 |
+
"female eyebrow . male eyebrow . woman eyebrow . man eyebrow . "
|
| 41 |
+
"human eye . eye . eyebrow . face . head"
|
| 42 |
+
),
|
| 43 |
+
(
|
| 44 |
+
"female belly . male belly . woman belly . man belly . "
|
| 45 |
+
"female stomach . male stomach . woman stomach . man stomach . "
|
| 46 |
+
"female abdomen . male abdomen . woman abdomen . man abdomen . "
|
| 47 |
+
"belly . stomach . abdomen . torso"
|
| 48 |
+
),
|
| 49 |
+
(
|
| 50 |
+
"female hand . male hand . woman hand . man hand . "
|
| 51 |
+
"female leg . male leg . woman leg . man leg . "
|
| 52 |
+
"human hand . hand . human leg . leg"
|
| 53 |
+
),
|
| 54 |
)
|
| 55 |
BOX_THRESHOLD = 0.25
|
| 56 |
+
TEXT_THRESHOLD = 0.22
|
| 57 |
NMS_IOU_THRESHOLD = 0.50
|
| 58 |
+
REFINEMENT_BOX_THRESHOLD = 0.15
|
| 59 |
+
REFINEMENT_TEXT_THRESHOLD = 0.18
|
| 60 |
+
MAX_REFINEMENTS_PER_FAMILY = 2
|
| 61 |
MAX_IMAGE_BYTES = 20 * 1024 * 1024
|
| 62 |
|
| 63 |
ARABIC_LABELS: Dict[str, str] = {
|
|
|
|
| 67 |
"baby": "طفل أو رضيع",
|
| 68 |
"female hand": "يد إمرأة",
|
| 69 |
"male hand": "يد رجل",
|
| 70 |
+
"female eye": "عين إمرأة",
|
| 71 |
+
"male eye": "عين رجل",
|
| 72 |
+
"female eyebrow": "حاجب إمرأة",
|
| 73 |
+
"male eyebrow": "حاجب رجل",
|
| 74 |
"female belly": "بطن إمرأة",
|
| 75 |
"male belly": "بطن رجل",
|
|
|
|
| 76 |
"female leg": "ساق إمرأة",
|
| 77 |
"male leg": "ساق رجل",
|
| 78 |
}
|
|
|
|
| 84 |
"baby": "person",
|
| 85 |
"female hand": "hand",
|
| 86 |
"male hand": "hand",
|
| 87 |
+
"female eye": "eye",
|
| 88 |
+
"male eye": "eye",
|
| 89 |
+
"female eyebrow": "eyebrow",
|
| 90 |
+
"male eyebrow": "eyebrow",
|
| 91 |
"female belly": "belly",
|
| 92 |
"male belly": "belly",
|
| 93 |
"female leg": "leg",
|
| 94 |
"male leg": "leg",
|
| 95 |
}
|
| 96 |
|
| 97 |
+
LABEL_ALIASES = {
|
| 98 |
+
"woman eye": "female eye",
|
| 99 |
+
"female eye": "female eye",
|
| 100 |
+
"man eye": "male eye",
|
| 101 |
+
"male eye": "male eye",
|
| 102 |
+
"woman eyebrow": "female eyebrow",
|
| 103 |
+
"female eyebrow": "female eyebrow",
|
| 104 |
+
"man eyebrow": "male eyebrow",
|
| 105 |
+
"male eyebrow": "male eyebrow",
|
| 106 |
+
"woman stomach": "female belly",
|
| 107 |
+
"female stomach": "female belly",
|
| 108 |
+
"woman abdomen": "female belly",
|
| 109 |
+
"female abdomen": "female belly",
|
| 110 |
+
"woman belly": "female belly",
|
| 111 |
+
"female belly": "female belly",
|
| 112 |
+
"man stomach": "male belly",
|
| 113 |
+
"male stomach": "male belly",
|
| 114 |
+
"man abdomen": "male belly",
|
| 115 |
+
"male abdomen": "male belly",
|
| 116 |
+
"man belly": "male belly",
|
| 117 |
+
"male belly": "male belly",
|
| 118 |
+
"woman hand": "female hand",
|
| 119 |
+
"female hand": "female hand",
|
| 120 |
+
"man hand": "male hand",
|
| 121 |
+
"male hand": "male hand",
|
| 122 |
+
"woman leg": "female leg",
|
| 123 |
+
"female leg": "female leg",
|
| 124 |
+
"man leg": "male leg",
|
| 125 |
+
"male leg": "male leg",
|
| 126 |
+
"young child": "young child",
|
| 127 |
+
"woman": "woman",
|
| 128 |
+
"baby": "baby",
|
| 129 |
+
"man": "man",
|
| 130 |
+
}
|
| 131 |
+
|
| 132 |
+
GENERIC_FAMILY_ALIASES = {
|
| 133 |
+
"human eye": "eye",
|
| 134 |
+
"eye": "eye",
|
| 135 |
+
"eyebrow": "eyebrow",
|
| 136 |
+
"belly": "belly",
|
| 137 |
+
"stomach": "belly",
|
| 138 |
+
"abdomen": "belly",
|
| 139 |
+
"torso": "belly",
|
| 140 |
+
"human hand": "hand",
|
| 141 |
+
"hand": "hand",
|
| 142 |
+
"human leg": "leg",
|
| 143 |
+
"leg": "leg",
|
| 144 |
+
}
|
| 145 |
+
|
| 146 |
+
GENDER_REFINEMENT_PROMPTS = {
|
| 147 |
+
"eye": "female eye . woman eye . male eye . man eye",
|
| 148 |
+
"eyebrow": "female eyebrow . woman eyebrow . male eyebrow . man eyebrow",
|
| 149 |
+
"belly": (
|
| 150 |
+
"female belly . woman belly . female stomach . woman stomach . "
|
| 151 |
+
"female abdomen . woman abdomen . male belly . man belly . "
|
| 152 |
+
"male stomach . man stomach . male abdomen . man abdomen"
|
| 153 |
+
),
|
| 154 |
+
"hand": "female hand . woman hand . male hand . man hand",
|
| 155 |
+
"leg": "female leg . woman leg . male leg . man leg",
|
| 156 |
+
}
|
| 157 |
+
|
| 158 |
processor: Optional[Any] = None
|
| 159 |
model: Optional[Any] = None
|
| 160 |
inference_lock = threading.Lock()
|
|
|
|
| 199 |
|
| 200 |
|
| 201 |
def canonical_label(raw_phrase: str) -> Optional[str]:
|
| 202 |
+
normalized = raw_phrase.lower().replace("'s", "").replace(".", " ")
|
| 203 |
+
normalized = " ".join(normalized.split())
|
| 204 |
+
for alias in sorted(LABEL_ALIASES, key=len, reverse=True):
|
| 205 |
+
if normalized == alias or re.search(rf"\b{re.escape(alias)}\b", normalized):
|
| 206 |
+
return LABEL_ALIASES[alias]
|
| 207 |
+
return None
|
| 208 |
+
|
| 209 |
+
|
| 210 |
+
def generic_family(raw_phrase: str) -> Optional[str]:
|
| 211 |
normalized = " ".join(raw_phrase.lower().replace(".", " ").split())
|
| 212 |
+
for alias in sorted(GENERIC_FAMILY_ALIASES, key=len, reverse=True):
|
| 213 |
+
if normalized == alias or re.search(rf"\b{re.escape(alias)}\b", normalized):
|
| 214 |
+
return GENERIC_FAMILY_ALIASES[alias]
|
| 215 |
return None
|
| 216 |
|
| 217 |
|
|
|
|
| 252 |
return sorted(kept, key=lambda detection: detection["confidence"], reverse=True)
|
| 253 |
|
| 254 |
|
| 255 |
+
def run_prompt(
|
| 256 |
+
image: Image.Image,
|
| 257 |
+
prompt: str,
|
| 258 |
+
box_threshold: float = BOX_THRESHOLD,
|
| 259 |
+
text_threshold: float = TEXT_THRESHOLD,
|
| 260 |
+
) -> tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
|
| 261 |
+
inputs = processor(images=image, text=prompt, return_tensors="pt").to(DEVICE)
|
|
|
|
|
|
|
| 262 |
with torch.inference_mode():
|
| 263 |
outputs = model(**inputs)
|
| 264 |
|
|
|
|
| 269 |
outputs,
|
| 270 |
inputs.input_ids,
|
| 271 |
**{
|
| 272 |
+
threshold_name: box_threshold,
|
| 273 |
+
"text_threshold": text_threshold,
|
| 274 |
"target_sizes": [image.size[::-1]],
|
| 275 |
},
|
| 276 |
)[0]
|
|
|
|
| 279 |
scores = result.get("scores", [])
|
| 280 |
phrases = result.get("text_labels", result.get("labels", []))
|
| 281 |
detections: List[Dict[str, Any]] = []
|
| 282 |
+
generic_regions: List[Dict[str, Any]] = []
|
| 283 |
|
| 284 |
for index, box in enumerate(boxes):
|
| 285 |
if index >= len(scores) or index >= len(phrases):
|
|
|
|
| 289 |
phrase = phrase.item()
|
| 290 |
label = canonical_label(str(phrase))
|
| 291 |
if label is None:
|
| 292 |
+
family = generic_family(str(phrase))
|
| 293 |
+
if family is not None:
|
| 294 |
+
generic_regions.append(
|
| 295 |
+
{
|
| 296 |
+
"family": family,
|
| 297 |
+
"confidence": float(scores[index].item()),
|
| 298 |
+
"box": [float(value) for value in box.tolist()],
|
| 299 |
+
}
|
| 300 |
+
)
|
| 301 |
continue
|
| 302 |
|
| 303 |
detections.append(
|
|
|
|
| 309 |
}
|
| 310 |
)
|
| 311 |
|
| 312 |
+
return detections, generic_regions
|
| 313 |
+
|
| 314 |
+
|
| 315 |
+
def suppress_generic_regions(regions: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
| 316 |
+
kept: List[Dict[str, Any]] = []
|
| 317 |
+
family_counts: Dict[str, int] = {}
|
| 318 |
+
|
| 319 |
+
for candidate in sorted(
|
| 320 |
+
regions,
|
| 321 |
+
key=lambda region: region["confidence"],
|
| 322 |
+
reverse=True,
|
| 323 |
+
):
|
| 324 |
+
family = candidate["family"]
|
| 325 |
+
if family_counts.get(family, 0) >= MAX_REFINEMENTS_PER_FAMILY:
|
| 326 |
+
continue
|
| 327 |
+
if any(
|
| 328 |
+
selected["family"] == family
|
| 329 |
+
and box_iou(candidate["box"], selected["box"]) > NMS_IOU_THRESHOLD
|
| 330 |
+
for selected in kept
|
| 331 |
+
):
|
| 332 |
+
continue
|
| 333 |
+
kept.append(candidate)
|
| 334 |
+
family_counts[family] = family_counts.get(family, 0) + 1
|
| 335 |
+
return kept
|
| 336 |
+
|
| 337 |
+
|
| 338 |
+
def crop_region(
|
| 339 |
+
image: Image.Image,
|
| 340 |
+
box: Sequence[float],
|
| 341 |
+
) -> tuple[Image.Image, float, float]:
|
| 342 |
+
x1, y1, x2, y2 = box
|
| 343 |
+
box_width = max(1.0, x2 - x1)
|
| 344 |
+
box_height = max(1.0, y2 - y1)
|
| 345 |
+
padding_x = box_width * 0.20
|
| 346 |
+
padding_y = box_height * 0.20
|
| 347 |
+
left = max(0.0, x1 - padding_x)
|
| 348 |
+
top = max(0.0, y1 - padding_y)
|
| 349 |
+
right = min(float(image.width), x2 + padding_x)
|
| 350 |
+
bottom = min(float(image.height), y2 + padding_y)
|
| 351 |
+
return image.crop((left, top, right, bottom)), left, top
|
| 352 |
+
|
| 353 |
+
|
| 354 |
+
def run_detection(image: Image.Image) -> List[Dict[str, Any]]:
|
| 355 |
+
if processor is None or model is None:
|
| 356 |
+
raise RuntimeError("The model is not ready.")
|
| 357 |
+
|
| 358 |
+
detections: List[Dict[str, Any]] = []
|
| 359 |
+
generic_regions: List[Dict[str, Any]] = []
|
| 360 |
+
for prompt in PROMPT_GROUPS:
|
| 361 |
+
prompt_detections, prompt_generic_regions = run_prompt(image, prompt)
|
| 362 |
+
detections.extend(prompt_detections)
|
| 363 |
+
generic_regions.extend(prompt_generic_regions)
|
| 364 |
+
|
| 365 |
+
# Generic detections are localized first, then the cropped region is
|
| 366 |
+
# contrasted with female/male prompts. This recovers small printed body
|
| 367 |
+
# regions without guessing a gender from a generic label.
|
| 368 |
+
for region in suppress_generic_regions(generic_regions):
|
| 369 |
+
family = region["family"]
|
| 370 |
+
if any(
|
| 371 |
+
LABEL_FAMILIES[detection["label"]] == family
|
| 372 |
+
and box_iou(detection["box"], region["box"]) > 0.20
|
| 373 |
+
for detection in detections
|
| 374 |
+
):
|
| 375 |
+
continue
|
| 376 |
+
|
| 377 |
+
cropped_image, offset_x, offset_y = crop_region(image, region["box"])
|
| 378 |
+
refined, _ = run_prompt(
|
| 379 |
+
cropped_image,
|
| 380 |
+
GENDER_REFINEMENT_PROMPTS[family],
|
| 381 |
+
box_threshold=REFINEMENT_BOX_THRESHOLD,
|
| 382 |
+
text_threshold=REFINEMENT_TEXT_THRESHOLD,
|
| 383 |
+
)
|
| 384 |
+
matching = [
|
| 385 |
+
detection
|
| 386 |
+
for detection in refined
|
| 387 |
+
if LABEL_FAMILIES[detection["label"]] == family
|
| 388 |
+
]
|
| 389 |
+
if not matching:
|
| 390 |
+
continue
|
| 391 |
+
|
| 392 |
+
best = max(matching, key=lambda detection: detection["confidence"])
|
| 393 |
+
best["box"] = [
|
| 394 |
+
best["box"][0] + offset_x,
|
| 395 |
+
best["box"][1] + offset_y,
|
| 396 |
+
best["box"][2] + offset_x,
|
| 397 |
+
best["box"][3] + offset_y,
|
| 398 |
+
]
|
| 399 |
+
detections.append(best)
|
| 400 |
return contrastive_nms(detections)
|
| 401 |
|
| 402 |
|