Spaces:
Running
Running
Update dfine_jina_pipeline.py
Browse files- dfine_jina_pipeline.py +395 -216
dfine_jina_pipeline.py
CHANGED
|
@@ -1,9 +1,10 @@
|
|
| 1 |
"""
|
| 2 |
-
Pipeline: D-FINE (person/car only) → group detections → crop regions →
|
| 3 |
-
|
| 4 |
-
|
| 5 |
Outputs separate crop folders per model (jina_crops, nomic_crops) for visual comparison.
|
|
|
|
| 6 |
"""
|
|
|
|
| 7 |
import argparse
|
| 8 |
import csv
|
| 9 |
import time
|
|
@@ -11,8 +12,7 @@ from pathlib import Path
|
|
| 11 |
|
| 12 |
import numpy as np
|
| 13 |
import torch
|
| 14 |
-
import
|
| 15 |
-
from PIL import Image
|
| 16 |
from transformers import AutoImageProcessor, DFineForObjectDetection
|
| 17 |
|
| 18 |
# Jina-CLIP-v2 few-shot (same refs + classify as jina_fewshot.py)
|
|
@@ -22,11 +22,13 @@ from jina_fewshot import (
|
|
| 22 |
JinaCLIPv2Encoder,
|
| 23 |
build_refs,
|
| 24 |
classify as jina_classify,
|
| 25 |
-
draw_label_on_image,
|
| 26 |
)
|
| 27 |
|
|
|
|
|
|
|
|
|
|
| 28 |
# -----------------------------------------------------------------------------
|
| 29 |
-
# Detection + grouping
|
| 30 |
# -----------------------------------------------------------------------------
|
| 31 |
|
| 32 |
|
|
@@ -90,10 +92,7 @@ def box_center_inside(box, crop_box):
|
|
| 90 |
"""True if center of box is inside crop_box. All [x1,y1,x2,y2]."""
|
| 91 |
cx = (box[0] + box[2]) / 2
|
| 92 |
cy = (box[1] + box[3]) / 2
|
| 93 |
-
return
|
| 94 |
-
crop_box[0] <= cx <= crop_box[2]
|
| 95 |
-
and crop_box[1] <= cy <= crop_box[3]
|
| 96 |
-
)
|
| 97 |
|
| 98 |
|
| 99 |
def squarify_crop_box(bx1, by1, bx2, by2, img_w, img_h):
|
|
@@ -140,7 +139,6 @@ def deduplicate_by_iou(detections, iou_threshold=0.9):
|
|
| 140 |
"""Keep one detection per overlapping group (IoU >= iou_threshold). Prefer higher confidence."""
|
| 141 |
if not detections:
|
| 142 |
return []
|
| 143 |
-
# Sort by confidence descending; keep first, then add only if no kept box overlaps >= threshold
|
| 144 |
sorted_d = sorted(detections, key=lambda x: -x["conf"])
|
| 145 |
kept = []
|
| 146 |
for d in sorted_d:
|
|
@@ -149,25 +147,182 @@ def deduplicate_by_iou(detections, iou_threshold=0.9):
|
|
| 149 |
return kept
|
| 150 |
|
| 151 |
|
| 152 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 153 |
|
| 154 |
|
| 155 |
def parse_args():
|
| 156 |
p = argparse.ArgumentParser(
|
| 157 |
-
description="D-FINE (person/car) → group →
|
| 158 |
)
|
| 159 |
p.add_argument("--refs", required=True, help="Reference images folder for Jina and Nomic (e.g. refs/)")
|
| 160 |
p.add_argument("--input", required=True, help="Full-frame images folder")
|
| 161 |
p.add_argument("--output", default="pipeline_results", help="Output folder (CSV, etc.)")
|
| 162 |
p.add_argument("--det-threshold", type=float, default=0.13, help="D-FINE score threshold")
|
| 163 |
-
p.add_argument("--group-dist", type=float, default=None,
|
| 164 |
-
|
| 165 |
-
p.add_argument("--
|
| 166 |
-
p.add_argument("--
|
| 167 |
-
p.add_argument("--
|
| 168 |
-
p.add_argument("--
|
| 169 |
-
p.add_argument("--
|
| 170 |
-
p.add_argument("--gap-threshold", type=float, default=0.05, help="Jina accept gap")
|
| 171 |
p.add_argument("--text-weight", type=float, default=0.3)
|
| 172 |
p.add_argument("--max-images", type=int, default=None)
|
| 173 |
p.add_argument("--device", default=None)
|
|
@@ -191,7 +346,6 @@ def get_person_car_label_ids(model):
|
|
| 191 |
|
| 192 |
def run_dfine(image, processor, model, device, score_threshold):
|
| 193 |
"""Run D-FINE, return all detections as list of {box, score, label_id, label}."""
|
| 194 |
-
from PIL import Image
|
| 195 |
if isinstance(image, Image.Image):
|
| 196 |
pil = image.convert("RGB")
|
| 197 |
else:
|
|
@@ -212,9 +366,7 @@ def run_dfine(image, processor, model, device, score_threshold):
|
|
| 212 |
|
| 213 |
detections = []
|
| 214 |
for result in results:
|
| 215 |
-
for score, label_id, box in zip(
|
| 216 |
-
result["scores"], result["labels"], result["boxes"]
|
| 217 |
-
):
|
| 218 |
sid = int(label_id.item())
|
| 219 |
detections.append({
|
| 220 |
"box": [float(x) for x in box.cpu().tolist()],
|
|
@@ -225,6 +377,48 @@ def run_dfine(image, processor, model, device, score_threshold):
|
|
| 225 |
return detections
|
| 226 |
|
| 227 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 228 |
def main():
|
| 229 |
args = parse_args()
|
| 230 |
device = args.device or ("cuda" if torch.cuda.is_available() else "cpu")
|
|
@@ -238,10 +432,7 @@ def main():
|
|
| 238 |
if not input_dir.is_dir():
|
| 239 |
raise SystemExit(f"Input folder not found: {input_dir}")
|
| 240 |
|
| 241 |
-
paths = sorted(
|
| 242 |
-
p for p in input_dir.iterdir()
|
| 243 |
-
if p.suffix.lower() in IMAGE_EXTS
|
| 244 |
-
)
|
| 245 |
if args.max_images is not None:
|
| 246 |
paths = paths[: args.max_images]
|
| 247 |
if not paths:
|
|
@@ -254,35 +445,34 @@ def main():
|
|
| 254 |
dfine_model = DFineForObjectDetection.from_pretrained("ustc-community/dfine-medium-obj365")
|
| 255 |
dfine_model = dfine_model.to(device).eval()
|
| 256 |
person_car_ids = get_person_car_label_ids(dfine_model)
|
| 257 |
-
print(f" Person/car label IDs: {person_car_ids} ({time.perf_counter()-t0:.1f}s)")
|
| 258 |
|
| 259 |
-
# Load Jina
|
| 260 |
print("[*] Loading Jina-CLIP-v2 and building refs...")
|
| 261 |
t0 = time.perf_counter()
|
| 262 |
jina_encoder = JinaCLIPv2Encoder(device)
|
| 263 |
-
ref_labels, ref_embs = build_refs(
|
| 264 |
-
|
| 265 |
-
)
|
| 266 |
-
print(f" Jina refs: {ref_labels} ({time.perf_counter()-t0:.1f}s)\n")
|
| 267 |
|
| 268 |
-
# Load Nomic
|
| 269 |
print("[*] Loading Nomic embed-vision + embed-text and building refs...")
|
| 270 |
t0 = time.perf_counter()
|
| 271 |
nomic_encoder = NomicVisionEncoder(device)
|
| 272 |
nomic_text_encoder = NomicTextEncoder(device)
|
| 273 |
ref_labels_nomic, ref_embs_nomic = build_refs_nomic(
|
| 274 |
-
nomic_encoder,
|
| 275 |
-
|
|
|
|
|
|
|
|
|
|
| 276 |
)
|
| 277 |
-
print(f" Nomic refs: {ref_labels_nomic} ({time.perf_counter()-t0:.1f}s)\n")
|
| 278 |
|
| 279 |
-
# Separate output folders per model for visual comparison
|
| 280 |
jina_crops_dir = output_dir / "jina_crops"
|
| 281 |
nomic_crops_dir = output_dir / "nomic_crops"
|
| 282 |
jina_crops_dir.mkdir(parents=True, exist_ok=True)
|
| 283 |
nomic_crops_dir.mkdir(parents=True, exist_ok=True)
|
| 284 |
|
| 285 |
-
# CSV
|
| 286 |
csv_path = output_dir / "results.csv"
|
| 287 |
f = open(csv_path, "w", newline="")
|
| 288 |
w = csv.writer(f)
|
|
@@ -298,114 +488,91 @@ def main():
|
|
| 298 |
img_w, img_h = pil.size
|
| 299 |
group_dist = args.group_dist if args.group_dist is not None else 0.1 * max(img_h, img_w)
|
| 300 |
|
| 301 |
-
|
| 302 |
-
detections = run_dfine(
|
| 303 |
-
pil, image_processor, dfine_model, device, args.det_threshold
|
| 304 |
-
)
|
| 305 |
person_car = [d for d in detections if d["cls"] in person_car_ids]
|
| 306 |
if not person_car:
|
| 307 |
continue
|
| 308 |
|
| 309 |
-
# 2) Group person/car detections (same as reference)
|
| 310 |
grouped = group_detections(person_car, group_dist)
|
| 311 |
grouped.sort(key=lambda x: x["conf"], reverse=True)
|
| 312 |
-
top_groups = grouped[:10]
|
| 313 |
|
| 314 |
-
# 3) Collect all candidate crops (bboxes inside person/car groups)
|
| 315 |
-
# Each: (crop_box, crop_pil, d, gidx, crop_idx, x1, y1, x2, y2)
|
| 316 |
-
candidates = []
|
| 317 |
for gidx, grp in enumerate(top_groups):
|
| 318 |
-
|
| 319 |
-
|
| 320 |
-
|
| 321 |
-
|
| 322 |
-
|
| 323 |
-
|
| 324 |
-
|
| 325 |
-
inside = deduplicate_by_iou(inside, iou_threshold=0.9)
|
| 326 |
-
|
| 327 |
-
for crop_idx, d in enumerate(inside):
|
| 328 |
-
bx1, by1, bx2, by2 = [float(x) for x in d["box"]]
|
| 329 |
-
obj_w, obj_h = bx2 - bx1, by2 - by1
|
| 330 |
-
if obj_w <= 0 or obj_h <= 0:
|
| 331 |
-
continue
|
| 332 |
-
pad_x = obj_w * 0.3
|
| 333 |
-
pad_y = obj_h * 0.3
|
| 334 |
-
bx1 = max(0, int(bx1 - pad_x))
|
| 335 |
-
by1 = max(0, int(by1 - pad_y))
|
| 336 |
-
bx2 = min(img_w, int(bx2 + pad_x))
|
| 337 |
-
by2 = min(img_h, int(by2 + pad_y))
|
| 338 |
-
if bx2 <= bx1 or by2 <= by1:
|
| 339 |
-
continue
|
| 340 |
-
if min(bx2 - bx1, by2 - by1) < args.min_side:
|
| 341 |
-
continue
|
| 342 |
-
expanded_box = [bx1, by1, bx2, by2]
|
| 343 |
-
candidates.append((expanded_box, d, gidx, crop_idx, x1, y1, x2, y2))
|
| 344 |
-
|
| 345 |
-
# 4) Dedup on EXPANDED boxes (before squarify), keep larger; then squarify only kept
|
| 346 |
-
def crop_area(box):
|
| 347 |
-
return (box[2] - box[0]) * (box[3] - box[1])
|
| 348 |
-
|
| 349 |
-
candidates.sort(key=lambda c: -crop_area(c[0]))
|
| 350 |
-
kept = []
|
| 351 |
-
for c in candidates:
|
| 352 |
-
expanded_box = c[0]
|
| 353 |
-
def is_same_object(box_a, box_b):
|
| 354 |
-
if box_iou(box_a, box_b) >= args.crop_dedup_iou:
|
| 355 |
-
return True
|
| 356 |
-
if box_center_inside(box_a, box_b) or box_center_inside(box_b, box_a):
|
| 357 |
-
return True
|
| 358 |
-
return False
|
| 359 |
-
if not any(is_same_object(expanded_box, k[0]) for k in kept):
|
| 360 |
-
kept.append(c)
|
| 361 |
-
|
| 362 |
-
# 5) Optionally squarify, then run Jina and Nomic only on kept crops
|
| 363 |
-
for i, (expanded_box, d, gidx, crop_idx, x1, y1, x2, y2) in enumerate(kept):
|
| 364 |
-
if not args.no_squarify:
|
| 365 |
-
bx1, by1, bx2, by2 = squarify_crop_box(
|
| 366 |
-
expanded_box[0], expanded_box[1], expanded_box[2], expanded_box[3], img_w, img_h
|
| 367 |
-
)
|
| 368 |
-
else:
|
| 369 |
-
bx1, by1, bx2, by2 = expanded_box[0], expanded_box[1], expanded_box[2], expanded_box[3]
|
| 370 |
-
crop_pil = pil.crop((bx1, by1, bx2, by2))
|
| 371 |
-
crop_name = f"{img_path.stem}_g{gidx}_{i}_{bx1}_{by1}_{bx2}_{by2}{img_path.suffix}"
|
| 372 |
-
|
| 373 |
-
q_jina = jina_encoder.encode_images([crop_pil], TRUNCATE_DIM)
|
| 374 |
-
result_jina = jina_classify(
|
| 375 |
-
q_jina, ref_labels, ref_embs,
|
| 376 |
-
args.conf_threshold, args.gap_threshold
|
| 377 |
)
|
| 378 |
-
if
|
| 379 |
-
|
| 380 |
-
conf_jina = result_jina["confidence"]
|
| 381 |
-
else:
|
| 382 |
-
label_jina = f"unnamed (dfine: {d['label']})"
|
| 383 |
-
conf_jina = 0.0
|
| 384 |
-
ann_jina = draw_label_on_image(crop_pil, label_jina, conf_jina)
|
| 385 |
-
ann_jina.save(jina_crops_dir / crop_name)
|
| 386 |
|
| 387 |
-
|
| 388 |
-
|
| 389 |
-
|
| 390 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 391 |
)
|
| 392 |
-
if result_nomic["prediction"] in ref_labels_nomic:
|
| 393 |
-
label_nomic = result_nomic["prediction"]
|
| 394 |
-
conf_nomic = result_nomic["confidence"]
|
| 395 |
-
else:
|
| 396 |
-
label_nomic = f"unnamed (dfine: {d['label']})"
|
| 397 |
-
conf_nomic = 0.0
|
| 398 |
-
ann_nomic = draw_label_on_image(crop_pil, label_nomic, conf_nomic)
|
| 399 |
-
ann_nomic.save(nomic_crops_dir / crop_name)
|
| 400 |
|
| 401 |
-
|
| 402 |
-
|
| 403 |
-
|
| 404 |
-
|
| 405 |
-
d["
|
| 406 |
-
|
| 407 |
-
|
| 408 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 409 |
|
| 410 |
f.close()
|
| 411 |
print(f"[*] Wrote {csv_path}")
|
|
@@ -414,7 +581,7 @@ def main():
|
|
| 414 |
|
| 415 |
|
| 416 |
# -----------------------------------------------------------------------------
|
| 417 |
-
# Single-image runner for Gradio app: D-FINE first, then Jina or Nomic
|
| 418 |
# -----------------------------------------------------------------------------
|
| 419 |
_APP_DFINE = None
|
| 420 |
_APP_JINA = None
|
|
@@ -436,18 +603,20 @@ def run_single_image(
|
|
| 436 |
squarify=True,
|
| 437 |
):
|
| 438 |
"""
|
| 439 |
-
Run D-FINE on one image, then classify small-object
|
| 440 |
-
|
| 441 |
-
|
|
|
|
| 442 |
"""
|
| 443 |
-
import numpy as np
|
| 444 |
-
from PIL import Image
|
| 445 |
global _APP_DFINE, _APP_JINA, _APP_NOMIC, _APP_REFS_JINA, _APP_REFS_NOMIC
|
|
|
|
| 446 |
refs_dir = Path(refs_dir)
|
| 447 |
if not refs_dir.is_dir():
|
| 448 |
return None, f"Refs folder not found: {refs_dir}"
|
|
|
|
| 449 |
device = device or ("cuda" if torch.cuda.is_available() else "cpu")
|
| 450 |
print(f"[*] Device: {device}")
|
|
|
|
| 451 |
pil = pil_image.convert("RGB") if isinstance(pil_image, Image.Image) else Image.fromarray(pil_image).convert("RGB")
|
| 452 |
img_w, img_h = pil.size
|
| 453 |
group_dist = 0.1 * max(img_h, img_w)
|
|
@@ -459,6 +628,7 @@ def run_single_image(
|
|
| 459 |
dfine_model = dfine_model.to(device).eval()
|
| 460 |
person_car_ids = get_person_car_label_ids(dfine_model)
|
| 461 |
_APP_DFINE = (image_processor, dfine_model, person_car_ids)
|
|
|
|
| 462 |
image_processor, dfine_model, person_car_ids = _APP_DFINE
|
| 463 |
|
| 464 |
detections = run_dfine(pil, image_processor, dfine_model, device, det_threshold)
|
|
@@ -469,51 +639,6 @@ def run_single_image(
|
|
| 469 |
grouped = group_detections(person_car, group_dist)
|
| 470 |
grouped.sort(key=lambda x: x["conf"], reverse=True)
|
| 471 |
top_groups = grouped[:10]
|
| 472 |
-
candidates = []
|
| 473 |
-
for gidx, grp in enumerate(top_groups):
|
| 474 |
-
x1, y1, x2, y2 = grp["box"]
|
| 475 |
-
group_box = [x1, y1, x2, y2]
|
| 476 |
-
inside = [
|
| 477 |
-
d for d in detections
|
| 478 |
-
if box_center_inside(d["box"], group_box) and d["cls"] not in person_car_ids
|
| 479 |
-
]
|
| 480 |
-
inside = deduplicate_by_iou(inside, iou_threshold=0.9)
|
| 481 |
-
for crop_idx, d in enumerate(inside):
|
| 482 |
-
bx1, by1, bx2, by2 = [float(x) for x in d["box"]]
|
| 483 |
-
obj_w, obj_h = bx2 - bx1, by2 - by1
|
| 484 |
-
if obj_w <= 0 or obj_h <= 0:
|
| 485 |
-
continue
|
| 486 |
-
pad_x, pad_y = obj_w * 0.3, obj_h * 0.3
|
| 487 |
-
bx1 = max(0, int(bx1 - pad_x))
|
| 488 |
-
by1 = max(0, int(by1 - pad_y))
|
| 489 |
-
bx2 = min(img_w, int(bx2 + pad_x))
|
| 490 |
-
by2 = min(img_h, int(by2 + pad_y))
|
| 491 |
-
if bx2 <= bx1 or by2 <= by1:
|
| 492 |
-
continue
|
| 493 |
-
if min(bx2 - bx1, by2 - by1) < min_side:
|
| 494 |
-
continue
|
| 495 |
-
expanded_box = [bx1, by1, bx2, by2]
|
| 496 |
-
candidates.append((expanded_box, d, gidx, crop_idx))
|
| 497 |
-
|
| 498 |
-
def crop_area(box):
|
| 499 |
-
return (box[2] - box[0]) * (box[3] - box[1])
|
| 500 |
-
|
| 501 |
-
candidates.sort(key=lambda c: -crop_area(c[0]))
|
| 502 |
-
kept = []
|
| 503 |
-
for c in candidates:
|
| 504 |
-
def is_same_object(box_a, box_b):
|
| 505 |
-
if box_iou(box_a, box_b) >= crop_dedup_iou:
|
| 506 |
-
return True
|
| 507 |
-
if box_center_inside(box_a, box_b) or box_center_inside(box_b, box_a):
|
| 508 |
-
return True
|
| 509 |
-
return False
|
| 510 |
-
if not any(is_same_object(c[0], k[0]) for k in kept):
|
| 511 |
-
kept.append(c)
|
| 512 |
-
|
| 513 |
-
if not kept:
|
| 514 |
-
if not candidates:
|
| 515 |
-
return np.array(pil), "No small-object crops: D-FINE did not detect any object (gun/phone/etc.) inside person/car areas, or all were below min size. Try a higher-resolution image."
|
| 516 |
-
return np.array(pil), "No small-object crops (after dedup)."
|
| 517 |
|
| 518 |
# Load encoder + refs for chosen model
|
| 519 |
if encoder_choice == "jina":
|
|
@@ -522,44 +647,98 @@ def run_single_image(
|
|
| 522 |
ref_labels, ref_embs = build_refs(jina_encoder, refs_dir, TRUNCATE_DIM, 0.3, batch_size=16)
|
| 523 |
_APP_JINA = (jina_encoder, ref_labels, ref_embs)
|
| 524 |
_APP_REFS_JINA = str(refs_dir)
|
| 525 |
-
|
| 526 |
else:
|
| 527 |
if _APP_NOMIC is None or _APP_REFS_NOMIC != str(refs_dir):
|
| 528 |
nomic_encoder = NomicVisionEncoder(device)
|
| 529 |
nomic_text_encoder = NomicTextEncoder(device)
|
| 530 |
ref_labels, ref_embs = build_refs_nomic(
|
| 531 |
-
nomic_encoder,
|
| 532 |
-
|
|
|
|
|
|
|
|
|
|
| 533 |
)
|
| 534 |
_APP_NOMIC = (nomic_encoder, ref_labels, ref_embs)
|
| 535 |
_APP_REFS_NOMIC = str(refs_dir)
|
| 536 |
-
|
| 537 |
|
|
|
|
| 538 |
lines = []
|
| 539 |
-
|
| 540 |
-
for
|
| 541 |
-
|
| 542 |
-
|
| 543 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 544 |
)
|
| 545 |
-
|
| 546 |
-
|
| 547 |
-
|
| 548 |
-
|
| 549 |
-
|
| 550 |
-
|
| 551 |
-
|
| 552 |
-
|
| 553 |
-
|
| 554 |
-
|
| 555 |
-
|
| 556 |
-
|
| 557 |
-
|
| 558 |
-
|
| 559 |
-
|
| 560 |
-
result_text = "\n".join(lines) if lines else "No crops"
|
| 561 |
-
return np.array(out_img), result_text
|
| 562 |
|
| 563 |
|
| 564 |
if __name__ == "__main__":
|
| 565 |
-
main()
|
|
|
|
| 1 |
"""
|
| 2 |
+
Pipeline: D-FINE (person/car only) → group detections → crop group regions →
|
| 3 |
+
classify all inner object detections with Jina-CLIP-v2 or Nomic.
|
|
|
|
| 4 |
Outputs separate crop folders per model (jina_crops, nomic_crops) for visual comparison.
|
| 5 |
+
Each saved image is the D-FINE group crop, with bboxes drawn only for known classes.
|
| 6 |
"""
|
| 7 |
+
|
| 8 |
import argparse
|
| 9 |
import csv
|
| 10 |
import time
|
|
|
|
| 12 |
|
| 13 |
import numpy as np
|
| 14 |
import torch
|
| 15 |
+
from PIL import Image, ImageDraw, ImageFont
|
|
|
|
| 16 |
from transformers import AutoImageProcessor, DFineForObjectDetection
|
| 17 |
|
| 18 |
# Jina-CLIP-v2 few-shot (same refs + classify as jina_fewshot.py)
|
|
|
|
| 22 |
JinaCLIPv2Encoder,
|
| 23 |
build_refs,
|
| 24 |
classify as jina_classify,
|
|
|
|
| 25 |
)
|
| 26 |
|
| 27 |
+
from nomic_fewshot import NomicTextEncoder, NomicVisionEncoder, build_refs_nomic
|
| 28 |
+
|
| 29 |
+
|
| 30 |
# -----------------------------------------------------------------------------
|
| 31 |
+
# Detection + grouping
|
| 32 |
# -----------------------------------------------------------------------------
|
| 33 |
|
| 34 |
|
|
|
|
| 92 |
"""True if center of box is inside crop_box. All [x1,y1,x2,y2]."""
|
| 93 |
cx = (box[0] + box[2]) / 2
|
| 94 |
cy = (box[1] + box[3]) / 2
|
| 95 |
+
return crop_box[0] <= cx <= crop_box[2] and crop_box[1] <= cy <= crop_box[3]
|
|
|
|
|
|
|
|
|
|
| 96 |
|
| 97 |
|
| 98 |
def squarify_crop_box(bx1, by1, bx2, by2, img_w, img_h):
|
|
|
|
| 139 |
"""Keep one detection per overlapping group (IoU >= iou_threshold). Prefer higher confidence."""
|
| 140 |
if not detections:
|
| 141 |
return []
|
|
|
|
| 142 |
sorted_d = sorted(detections, key=lambda x: -x["conf"])
|
| 143 |
kept = []
|
| 144 |
for d in sorted_d:
|
|
|
|
| 147 |
return kept
|
| 148 |
|
| 149 |
|
| 150 |
+
# -----------------------------------------------------------------------------
|
| 151 |
+
# Drawing / layout helpers
|
| 152 |
+
# -----------------------------------------------------------------------------
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
def _load_font_for_box(img_w, img_h):
|
| 156 |
+
font_path = "/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf"
|
| 157 |
+
size = max(10, min(img_w, img_h) // 14)
|
| 158 |
+
try:
|
| 159 |
+
return ImageFont.truetype(font_path, size=size)
|
| 160 |
+
except OSError:
|
| 161 |
+
return ImageFont.load_default()
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
def draw_bbox_with_optional_label(
|
| 165 |
+
img,
|
| 166 |
+
box,
|
| 167 |
+
label=None,
|
| 168 |
+
confidence=None,
|
| 169 |
+
color=(255, 0, 0),
|
| 170 |
+
width=3,
|
| 171 |
+
):
|
| 172 |
+
"""
|
| 173 |
+
Draw bbox on the given image. If label is None, draw only the box.
|
| 174 |
+
box is [x1, y1, x2, y2] in this image's coordinate space.
|
| 175 |
+
"""
|
| 176 |
+
out = img.copy().convert("RGB")
|
| 177 |
+
draw = ImageDraw.Draw(out)
|
| 178 |
+
x1, y1, x2, y2 = [int(v) for v in box]
|
| 179 |
+
|
| 180 |
+
x1 = max(0, min(x1, out.width - 1))
|
| 181 |
+
y1 = max(0, min(y1, out.height - 1))
|
| 182 |
+
x2 = max(0, min(x2, out.width - 1))
|
| 183 |
+
y2 = max(0, min(y2, out.height - 1))
|
| 184 |
+
|
| 185 |
+
for i in range(width):
|
| 186 |
+
draw.rectangle([x1 - i, y1 - i, x2 + i, y2 + i], outline=color)
|
| 187 |
+
|
| 188 |
+
if label:
|
| 189 |
+
font = _load_font_for_box(out.width, out.height)
|
| 190 |
+
text = f"{label} ({confidence:.2f})" if confidence is not None else label
|
| 191 |
+
bbox = draw.textbbox((0, 0), text, font=font)
|
| 192 |
+
tw = bbox[2] - bbox[0]
|
| 193 |
+
th = bbox[3] - bbox[1]
|
| 194 |
+
|
| 195 |
+
tx = max(0, min(x1, out.width - tw - 8))
|
| 196 |
+
ty = y1 - th - 8
|
| 197 |
+
if ty < 0:
|
| 198 |
+
ty = min(out.height - th - 4, y1 + 4)
|
| 199 |
+
|
| 200 |
+
draw.rectangle([tx, ty, tx + tw + 8, ty + th + 6], fill=color)
|
| 201 |
+
draw.text((tx + 4, ty + 3), text, fill=(255, 255, 255), font=font)
|
| 202 |
+
|
| 203 |
+
return out
|
| 204 |
+
|
| 205 |
+
|
| 206 |
+
def stack_images_vertical(images, bg=(255, 255, 255), pad=10):
|
| 207 |
+
"""Stack PIL images vertically into one output image."""
|
| 208 |
+
if not images:
|
| 209 |
+
return None
|
| 210 |
+
widths = [img.width for img in images]
|
| 211 |
+
heights = [img.height for img in images]
|
| 212 |
+
out_w = max(widths)
|
| 213 |
+
out_h = sum(heights) + pad * (len(images) - 1)
|
| 214 |
+
canvas = Image.new("RGB", (out_w, out_h), color=bg)
|
| 215 |
+
|
| 216 |
+
y = 0
|
| 217 |
+
for img in images:
|
| 218 |
+
x = (out_w - img.width) // 2
|
| 219 |
+
canvas.paste(img, (x, y))
|
| 220 |
+
y += img.height + pad
|
| 221 |
+
return canvas
|
| 222 |
+
|
| 223 |
+
|
| 224 |
+
# -----------------------------------------------------------------------------
|
| 225 |
+
# Shared crop/object preparation
|
| 226 |
+
# -----------------------------------------------------------------------------
|
| 227 |
+
|
| 228 |
+
|
| 229 |
+
def expand_box_with_padding(box, img_w, img_h, padding):
|
| 230 |
+
x1, y1, x2, y2 = [float(v) for v in box]
|
| 231 |
+
w = x2 - x1
|
| 232 |
+
h = y2 - y1
|
| 233 |
+
if w <= 0 or h <= 0:
|
| 234 |
+
return None
|
| 235 |
+
pad_x = w * padding
|
| 236 |
+
pad_y = h * padding
|
| 237 |
+
ex1 = max(0, int(x1 - pad_x))
|
| 238 |
+
ey1 = max(0, int(y1 - pad_y))
|
| 239 |
+
ex2 = min(img_w, int(x2 + pad_x))
|
| 240 |
+
ey2 = min(img_h, int(y2 + pad_y))
|
| 241 |
+
if ex2 <= ex1 or ey2 <= ey1:
|
| 242 |
+
return None
|
| 243 |
+
return [ex1, ey1, ex2, ey2]
|
| 244 |
+
|
| 245 |
+
|
| 246 |
+
def build_group_crop_box(group_box, img_w, img_h, padding=0.2, squarify=True):
|
| 247 |
+
expanded = expand_box_with_padding(group_box, img_w, img_h, padding)
|
| 248 |
+
if expanded is None:
|
| 249 |
+
return None
|
| 250 |
+
if squarify:
|
| 251 |
+
return list(squarify_crop_box(expanded[0], expanded[1], expanded[2], expanded[3], img_w, img_h))
|
| 252 |
+
return expanded
|
| 253 |
+
|
| 254 |
+
|
| 255 |
+
def collect_group_object_candidates(
|
| 256 |
+
detections,
|
| 257 |
+
person_car_ids,
|
| 258 |
+
group_box,
|
| 259 |
+
img_w,
|
| 260 |
+
img_h,
|
| 261 |
+
min_side,
|
| 262 |
+
crop_dedup_iou,
|
| 263 |
+
object_padding=0.3,
|
| 264 |
+
):
|
| 265 |
+
"""
|
| 266 |
+
For one group crop, collect and deduplicate all non-person/car detections inside it.
|
| 267 |
+
Returns list of dicts with:
|
| 268 |
+
{
|
| 269 |
+
"det": original detection,
|
| 270 |
+
"expanded_box": expanded object crop box in full-image coords
|
| 271 |
+
}
|
| 272 |
+
"""
|
| 273 |
+
inside = [
|
| 274 |
+
d for d in detections
|
| 275 |
+
if box_center_inside(d["box"], group_box) and d["cls"] not in person_car_ids
|
| 276 |
+
]
|
| 277 |
+
inside = deduplicate_by_iou(inside, iou_threshold=0.9)
|
| 278 |
+
|
| 279 |
+
candidates = []
|
| 280 |
+
for d in inside:
|
| 281 |
+
expanded_box = expand_box_with_padding(d["box"], img_w, img_h, object_padding)
|
| 282 |
+
if expanded_box is None:
|
| 283 |
+
continue
|
| 284 |
+
if min(expanded_box[2] - expanded_box[0], expanded_box[3] - expanded_box[1]) < min_side:
|
| 285 |
+
continue
|
| 286 |
+
candidates.append({
|
| 287 |
+
"det": d,
|
| 288 |
+
"expanded_box": expanded_box,
|
| 289 |
+
})
|
| 290 |
+
|
| 291 |
+
def crop_area(box):
|
| 292 |
+
return (box[2] - box[0]) * (box[3] - box[1])
|
| 293 |
+
|
| 294 |
+
candidates.sort(key=lambda c: -crop_area(c["expanded_box"]))
|
| 295 |
+
kept = []
|
| 296 |
+
|
| 297 |
+
def is_same_object(box_a, box_b):
|
| 298 |
+
if box_iou(box_a, box_b) >= crop_dedup_iou:
|
| 299 |
+
return True
|
| 300 |
+
if box_center_inside(box_a, box_b) or box_center_inside(box_b, box_a):
|
| 301 |
+
return True
|
| 302 |
+
return False
|
| 303 |
+
|
| 304 |
+
for c in candidates:
|
| 305 |
+
if not any(is_same_object(c["expanded_box"], k["expanded_box"]) for k in kept):
|
| 306 |
+
kept.append(c)
|
| 307 |
+
|
| 308 |
+
return kept
|
| 309 |
|
| 310 |
|
| 311 |
def parse_args():
|
| 312 |
p = argparse.ArgumentParser(
|
| 313 |
+
description="D-FINE (person/car) → group → classify objects inside each group crop"
|
| 314 |
)
|
| 315 |
p.add_argument("--refs", required=True, help="Reference images folder for Jina and Nomic (e.g. refs/)")
|
| 316 |
p.add_argument("--input", required=True, help="Full-frame images folder")
|
| 317 |
p.add_argument("--output", default="pipeline_results", help="Output folder (CSV, etc.)")
|
| 318 |
p.add_argument("--det-threshold", type=float, default=0.13, help="D-FINE score threshold")
|
| 319 |
+
p.add_argument("--group-dist", type=float, default=None, help="Group distance (default: 0.1 * max(H,W))")
|
| 320 |
+
p.add_argument("--min-side", type=int, default=40, help="Min side of expanded object bbox in px (skip smaller)")
|
| 321 |
+
p.add_argument("--crop-dedup-iou", type=float, default=0.35, help="Min IoU to treat two object crops as same object")
|
| 322 |
+
p.add_argument("--no-squarify", action="store_true", help="Skip squarify on group crop")
|
| 323 |
+
p.add_argument("--padding", type=float, default=0.2, help="Padding around D-FINE group crop box")
|
| 324 |
+
p.add_argument("--conf-threshold", type=float, default=0.75, help="Accept confidence")
|
| 325 |
+
p.add_argument("--gap-threshold", type=float, default=0.05, help="Accept gap")
|
|
|
|
| 326 |
p.add_argument("--text-weight", type=float, default=0.3)
|
| 327 |
p.add_argument("--max-images", type=int, default=None)
|
| 328 |
p.add_argument("--device", default=None)
|
|
|
|
| 346 |
|
| 347 |
def run_dfine(image, processor, model, device, score_threshold):
|
| 348 |
"""Run D-FINE, return all detections as list of {box, score, label_id, label}."""
|
|
|
|
| 349 |
if isinstance(image, Image.Image):
|
| 350 |
pil = image.convert("RGB")
|
| 351 |
else:
|
|
|
|
| 366 |
|
| 367 |
detections = []
|
| 368 |
for result in results:
|
| 369 |
+
for score, label_id, box in zip(result["scores"], result["labels"], result["boxes"]):
|
|
|
|
|
|
|
| 370 |
sid = int(label_id.item())
|
| 371 |
detections.append({
|
| 372 |
"box": [float(x) for x in box.cpu().tolist()],
|
|
|
|
| 377 |
return detections
|
| 378 |
|
| 379 |
|
| 380 |
+
def annotate_group_crop(
|
| 381 |
+
pil,
|
| 382 |
+
detections,
|
| 383 |
+
person_car_ids,
|
| 384 |
+
group_box,
|
| 385 |
+
crop_box,
|
| 386 |
+
encoder_choice,
|
| 387 |
+
encoder,
|
| 388 |
+
ref_labels,
|
| 389 |
+
ref_embs,
|
| 390 |
+
conf_threshold,
|
| 391 |
+
gap_threshold,
|
| 392 |
+
):
|
| 393 |
+
"""
|
| 394 |
+
Build one D-FINE group crop, classify all object candidates inside it,
|
| 395 |
+
and draw bbox+label only for known classes.
|
| 396 |
+
|
| 397 |
+
Returns:
|
| 398 |
+
annotated_crop_pil,
|
| 399 |
+
rows_for_csv,
|
| 400 |
+
known_lines
|
| 401 |
+
"""
|
| 402 |
+
crop_x1, crop_y1, crop_x2, crop_y2 = crop_box
|
| 403 |
+
group_crop = pil.crop((crop_x1, crop_y1, crop_x2, crop_y2)).convert("RGB")
|
| 404 |
+
|
| 405 |
+
obj_candidates = collect_group_object_candidates(
|
| 406 |
+
detections=detections,
|
| 407 |
+
person_car_ids=person_car_ids,
|
| 408 |
+
group_box=group_box,
|
| 409 |
+
img_w=pil.width,
|
| 410 |
+
img_h=pil.height,
|
| 411 |
+
min_side=1, # not used here; caller filters before this if needed
|
| 412 |
+
crop_dedup_iou=1.0, # not used here; caller passes already-filtered set if needed
|
| 413 |
+
object_padding=0.0,
|
| 414 |
+
)
|
| 415 |
+
# This helper is not used directly because caller already builds filtered candidates.
|
| 416 |
+
# Kept here only for API symmetry.
|
| 417 |
+
_ = obj_candidates
|
| 418 |
+
|
| 419 |
+
return group_crop, [], []
|
| 420 |
+
|
| 421 |
+
|
| 422 |
def main():
|
| 423 |
args = parse_args()
|
| 424 |
device = args.device or ("cuda" if torch.cuda.is_available() else "cpu")
|
|
|
|
| 432 |
if not input_dir.is_dir():
|
| 433 |
raise SystemExit(f"Input folder not found: {input_dir}")
|
| 434 |
|
| 435 |
+
paths = sorted(p for p in input_dir.iterdir() if p.suffix.lower() in IMAGE_EXTS)
|
|
|
|
|
|
|
|
|
|
| 436 |
if args.max_images is not None:
|
| 437 |
paths = paths[: args.max_images]
|
| 438 |
if not paths:
|
|
|
|
| 445 |
dfine_model = DFineForObjectDetection.from_pretrained("ustc-community/dfine-medium-obj365")
|
| 446 |
dfine_model = dfine_model.to(device).eval()
|
| 447 |
person_car_ids = get_person_car_label_ids(dfine_model)
|
| 448 |
+
print(f" Person/car label IDs: {person_car_ids} ({time.perf_counter() - t0:.1f}s)")
|
| 449 |
|
| 450 |
+
# Load Jina refs
|
| 451 |
print("[*] Loading Jina-CLIP-v2 and building refs...")
|
| 452 |
t0 = time.perf_counter()
|
| 453 |
jina_encoder = JinaCLIPv2Encoder(device)
|
| 454 |
+
ref_labels, ref_embs = build_refs(jina_encoder, refs_dir, TRUNCATE_DIM, args.text_weight, batch_size=16)
|
| 455 |
+
print(f" Jina refs: {ref_labels} ({time.perf_counter() - t0:.1f}s)\n")
|
|
|
|
|
|
|
| 456 |
|
| 457 |
+
# Load Nomic refs
|
| 458 |
print("[*] Loading Nomic embed-vision + embed-text and building refs...")
|
| 459 |
t0 = time.perf_counter()
|
| 460 |
nomic_encoder = NomicVisionEncoder(device)
|
| 461 |
nomic_text_encoder = NomicTextEncoder(device)
|
| 462 |
ref_labels_nomic, ref_embs_nomic = build_refs_nomic(
|
| 463 |
+
nomic_encoder,
|
| 464 |
+
refs_dir,
|
| 465 |
+
batch_size=16,
|
| 466 |
+
text_encoder=nomic_text_encoder,
|
| 467 |
+
text_weight=args.text_weight,
|
| 468 |
)
|
| 469 |
+
print(f" Nomic refs: {ref_labels_nomic} ({time.perf_counter() - t0:.1f}s)\n")
|
| 470 |
|
|
|
|
| 471 |
jina_crops_dir = output_dir / "jina_crops"
|
| 472 |
nomic_crops_dir = output_dir / "nomic_crops"
|
| 473 |
jina_crops_dir.mkdir(parents=True, exist_ok=True)
|
| 474 |
nomic_crops_dir.mkdir(parents=True, exist_ok=True)
|
| 475 |
|
|
|
|
| 476 |
csv_path = output_dir / "results.csv"
|
| 477 |
f = open(csv_path, "w", newline="")
|
| 478 |
w = csv.writer(f)
|
|
|
|
| 488 |
img_w, img_h = pil.size
|
| 489 |
group_dist = args.group_dist if args.group_dist is not None else 0.1 * max(img_h, img_w)
|
| 490 |
|
| 491 |
+
detections = run_dfine(pil, image_processor, dfine_model, device, args.det_threshold)
|
|
|
|
|
|
|
|
|
|
| 492 |
person_car = [d for d in detections if d["cls"] in person_car_ids]
|
| 493 |
if not person_car:
|
| 494 |
continue
|
| 495 |
|
|
|
|
| 496 |
grouped = group_detections(person_car, group_dist)
|
| 497 |
grouped.sort(key=lambda x: x["conf"], reverse=True)
|
| 498 |
+
top_groups = grouped[:10]
|
| 499 |
|
|
|
|
|
|
|
|
|
|
| 500 |
for gidx, grp in enumerate(top_groups):
|
| 501 |
+
group_box = grp["box"]
|
| 502 |
+
crop_box = build_group_crop_box(
|
| 503 |
+
group_box,
|
| 504 |
+
img_w,
|
| 505 |
+
img_h,
|
| 506 |
+
padding=args.padding,
|
| 507 |
+
squarify=not args.no_squarify,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 508 |
)
|
| 509 |
+
if crop_box is None:
|
| 510 |
+
continue
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 511 |
|
| 512 |
+
crop_x1, crop_y1, crop_x2, crop_y2 = crop_box
|
| 513 |
+
crop_name = f"{img_path.stem}_g{gidx}_{crop_x1}_{crop_y1}_{crop_x2}_{crop_y2}{img_path.suffix}"
|
| 514 |
+
base_group_crop = pil.crop((crop_x1, crop_y1, crop_x2, crop_y2)).convert("RGB")
|
| 515 |
+
|
| 516 |
+
obj_candidates = collect_group_object_candidates(
|
| 517 |
+
detections=detections,
|
| 518 |
+
person_car_ids=person_car_ids,
|
| 519 |
+
group_box=group_box,
|
| 520 |
+
img_w=img_w,
|
| 521 |
+
img_h=img_h,
|
| 522 |
+
min_side=args.min_side,
|
| 523 |
+
crop_dedup_iou=args.crop_dedup_iou,
|
| 524 |
+
object_padding=0.3,
|
| 525 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 526 |
|
| 527 |
+
ann_jina = base_group_crop.copy()
|
| 528 |
+
ann_nomic = base_group_crop.copy()
|
| 529 |
+
|
| 530 |
+
for item in obj_candidates:
|
| 531 |
+
d = item["det"]
|
| 532 |
+
ex1, ey1, ex2, ey2 = item["expanded_box"]
|
| 533 |
+
|
| 534 |
+
obj_crop = pil.crop((ex1, ey1, ex2, ey2)).convert("RGB")
|
| 535 |
+
|
| 536 |
+
q_jina = jina_encoder.encode_images([obj_crop], TRUNCATE_DIM)
|
| 537 |
+
result_jina = jina_classify(q_jina, ref_labels, ref_embs, args.conf_threshold, args.gap_threshold)
|
| 538 |
+
|
| 539 |
+
q_nomic = nomic_encoder.encode_images([obj_crop])
|
| 540 |
+
result_nomic = jina_classify(q_nomic, ref_labels_nomic, ref_embs_nomic, args.conf_threshold, args.gap_threshold)
|
| 541 |
+
|
| 542 |
+
rel_box = [
|
| 543 |
+
max(0, int(round(d["box"][0] - crop_x1))),
|
| 544 |
+
max(0, int(round(d["box"][1] - crop_y1))),
|
| 545 |
+
min(base_group_crop.width, int(round(d["box"][2] - crop_x1))),
|
| 546 |
+
min(base_group_crop.height, int(round(d["box"][3] - crop_y1))),
|
| 547 |
+
]
|
| 548 |
+
|
| 549 |
+
if result_jina["prediction"] in ref_labels:
|
| 550 |
+
ann_jina = draw_bbox_with_optional_label(
|
| 551 |
+
ann_jina,
|
| 552 |
+
rel_box,
|
| 553 |
+
label=result_jina["prediction"],
|
| 554 |
+
confidence=result_jina["confidence"],
|
| 555 |
+
)
|
| 556 |
+
|
| 557 |
+
if result_nomic["prediction"] in ref_labels_nomic:
|
| 558 |
+
ann_nomic = draw_bbox_with_optional_label(
|
| 559 |
+
ann_nomic,
|
| 560 |
+
rel_box,
|
| 561 |
+
label=result_nomic["prediction"],
|
| 562 |
+
confidence=result_nomic["confidence"],
|
| 563 |
+
)
|
| 564 |
+
|
| 565 |
+
w.writerow([
|
| 566 |
+
img_path.name, crop_name, gidx,
|
| 567 |
+
crop_x1, crop_y1, crop_x2, crop_y2,
|
| 568 |
+
d["box"][0], d["box"][1], d["box"][2], d["box"][3],
|
| 569 |
+
d["label"], f"{d['conf']:.4f}",
|
| 570 |
+
result_jina["prediction"], f"{result_jina['confidence']:.4f}", result_jina["status"],
|
| 571 |
+
result_nomic["prediction"], f"{result_nomic['confidence']:.4f}", result_nomic["status"],
|
| 572 |
+
])
|
| 573 |
+
|
| 574 |
+
ann_jina.save(jina_crops_dir / crop_name)
|
| 575 |
+
ann_nomic.save(nomic_crops_dir / crop_name)
|
| 576 |
|
| 577 |
f.close()
|
| 578 |
print(f"[*] Wrote {csv_path}")
|
|
|
|
| 581 |
|
| 582 |
|
| 583 |
# -----------------------------------------------------------------------------
|
| 584 |
+
# Single-image runner for Gradio app: D-FINE first, then Jina or Nomic
|
| 585 |
# -----------------------------------------------------------------------------
|
| 586 |
_APP_DFINE = None
|
| 587 |
_APP_JINA = None
|
|
|
|
| 603 |
squarify=True,
|
| 604 |
):
|
| 605 |
"""
|
| 606 |
+
Run D-FINE on one image, then classify small-object detections inside each group crop.
|
| 607 |
+
Returns:
|
| 608 |
+
- one vertically stacked image of all D-FINE group crops
|
| 609 |
+
- text containing only known-class predictions
|
| 610 |
"""
|
|
|
|
|
|
|
| 611 |
global _APP_DFINE, _APP_JINA, _APP_NOMIC, _APP_REFS_JINA, _APP_REFS_NOMIC
|
| 612 |
+
|
| 613 |
refs_dir = Path(refs_dir)
|
| 614 |
if not refs_dir.is_dir():
|
| 615 |
return None, f"Refs folder not found: {refs_dir}"
|
| 616 |
+
|
| 617 |
device = device or ("cuda" if torch.cuda.is_available() else "cpu")
|
| 618 |
print(f"[*] Device: {device}")
|
| 619 |
+
|
| 620 |
pil = pil_image.convert("RGB") if isinstance(pil_image, Image.Image) else Image.fromarray(pil_image).convert("RGB")
|
| 621 |
img_w, img_h = pil.size
|
| 622 |
group_dist = 0.1 * max(img_h, img_w)
|
|
|
|
| 628 |
dfine_model = dfine_model.to(device).eval()
|
| 629 |
person_car_ids = get_person_car_label_ids(dfine_model)
|
| 630 |
_APP_DFINE = (image_processor, dfine_model, person_car_ids)
|
| 631 |
+
|
| 632 |
image_processor, dfine_model, person_car_ids = _APP_DFINE
|
| 633 |
|
| 634 |
detections = run_dfine(pil, image_processor, dfine_model, device, det_threshold)
|
|
|
|
| 639 |
grouped = group_detections(person_car, group_dist)
|
| 640 |
grouped.sort(key=lambda x: x["conf"], reverse=True)
|
| 641 |
top_groups = grouped[:10]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 642 |
|
| 643 |
# Load encoder + refs for chosen model
|
| 644 |
if encoder_choice == "jina":
|
|
|
|
| 647 |
ref_labels, ref_embs = build_refs(jina_encoder, refs_dir, TRUNCATE_DIM, 0.3, batch_size=16)
|
| 648 |
_APP_JINA = (jina_encoder, ref_labels, ref_embs)
|
| 649 |
_APP_REFS_JINA = str(refs_dir)
|
| 650 |
+
encoder, ref_labels, ref_embs = _APP_JINA
|
| 651 |
else:
|
| 652 |
if _APP_NOMIC is None or _APP_REFS_NOMIC != str(refs_dir):
|
| 653 |
nomic_encoder = NomicVisionEncoder(device)
|
| 654 |
nomic_text_encoder = NomicTextEncoder(device)
|
| 655 |
ref_labels, ref_embs = build_refs_nomic(
|
| 656 |
+
nomic_encoder,
|
| 657 |
+
refs_dir,
|
| 658 |
+
batch_size=16,
|
| 659 |
+
text_encoder=nomic_text_encoder,
|
| 660 |
+
text_weight=0.3,
|
| 661 |
)
|
| 662 |
_APP_NOMIC = (nomic_encoder, ref_labels, ref_embs)
|
| 663 |
_APP_REFS_NOMIC = str(refs_dir)
|
| 664 |
+
encoder, ref_labels, ref_embs = _APP_NOMIC
|
| 665 |
|
| 666 |
+
output_crops = []
|
| 667 |
lines = []
|
| 668 |
+
|
| 669 |
+
for gidx, grp in enumerate(top_groups):
|
| 670 |
+
group_box = grp["box"]
|
| 671 |
+
crop_box = build_group_crop_box(
|
| 672 |
+
group_box,
|
| 673 |
+
img_w,
|
| 674 |
+
img_h,
|
| 675 |
+
padding=0.2,
|
| 676 |
+
squarify=squarify,
|
| 677 |
+
)
|
| 678 |
+
if crop_box is None:
|
| 679 |
+
continue
|
| 680 |
+
|
| 681 |
+
crop_x1, crop_y1, crop_x2, crop_y2 = crop_box
|
| 682 |
+
group_crop = pil.crop((crop_x1, crop_y1, crop_x2, crop_y2)).convert("RGB")
|
| 683 |
+
|
| 684 |
+
obj_candidates = collect_group_object_candidates(
|
| 685 |
+
detections=detections,
|
| 686 |
+
person_car_ids=person_car_ids,
|
| 687 |
+
group_box=group_box,
|
| 688 |
+
img_w=img_w,
|
| 689 |
+
img_h=img_h,
|
| 690 |
+
min_side=min_side,
|
| 691 |
+
crop_dedup_iou=crop_dedup_iou,
|
| 692 |
+
object_padding=0.3,
|
| 693 |
+
)
|
| 694 |
+
|
| 695 |
+
ann_crop = group_crop.copy()
|
| 696 |
+
group_known_lines = []
|
| 697 |
+
|
| 698 |
+
for item in obj_candidates:
|
| 699 |
+
d = item["det"]
|
| 700 |
+
ex1, ey1, ex2, ey2 = item["expanded_box"]
|
| 701 |
+
obj_crop = pil.crop((ex1, ey1, ex2, ey2)).convert("RGB")
|
| 702 |
+
|
| 703 |
+
if encoder_choice == "jina":
|
| 704 |
+
q = encoder.encode_images([obj_crop], TRUNCATE_DIM)
|
| 705 |
+
result = jina_classify(q, ref_labels, ref_embs, conf_threshold, gap_threshold)
|
| 706 |
+
else:
|
| 707 |
+
q = encoder.encode_images([obj_crop])
|
| 708 |
+
result = jina_classify(q, ref_labels, ref_embs, conf_threshold, gap_threshold)
|
| 709 |
+
|
| 710 |
+
known = result["prediction"] in ref_labels
|
| 711 |
+
if not known:
|
| 712 |
+
continue
|
| 713 |
+
|
| 714 |
+
rel_box = [
|
| 715 |
+
max(0, int(round(d["box"][0] - crop_x1))),
|
| 716 |
+
max(0, int(round(d["box"][1] - crop_y1))),
|
| 717 |
+
min(group_crop.width, int(round(d["box"][2] - crop_x1))),
|
| 718 |
+
min(group_crop.height, int(round(d["box"][3] - crop_y1))),
|
| 719 |
+
]
|
| 720 |
+
|
| 721 |
+
ann_crop = draw_bbox_with_optional_label(
|
| 722 |
+
ann_crop,
|
| 723 |
+
rel_box,
|
| 724 |
+
label=result["prediction"],
|
| 725 |
+
confidence=result["confidence"],
|
| 726 |
)
|
| 727 |
+
group_known_lines.append(f"{result['prediction']} ({result['confidence']:.2f})")
|
| 728 |
+
|
| 729 |
+
output_crops.append(ann_crop)
|
| 730 |
+
|
| 731 |
+
if group_known_lines:
|
| 732 |
+
lines.append(f"Crop {gidx + 1}:")
|
| 733 |
+
lines.extend(group_known_lines)
|
| 734 |
+
|
| 735 |
+
if not output_crops:
|
| 736 |
+
return np.array(pil), "No D-FINE group crops."
|
| 737 |
+
|
| 738 |
+
stacked = stack_images_vertical(output_crops, pad=10)
|
| 739 |
+
result_text = "\n".join(lines) if lines else ""
|
| 740 |
+
return np.array(stacked), result_text
|
|
|
|
|
|
|
|
|
|
| 741 |
|
| 742 |
|
| 743 |
if __name__ == "__main__":
|
| 744 |
+
main()
|