Napron commited on
Commit
0930a73
·
verified ·
1 Parent(s): e500a2a

Update dfine_jina_pipeline.py

Browse files
Files changed (1) hide show
  1. dfine_jina_pipeline.py +395 -216
dfine_jina_pipeline.py CHANGED
@@ -1,9 +1,10 @@
1
  """
2
- Pipeline: D-FINE (person/car only) → group detections → crop regions →
3
- find all bboxes inside each crop → Jina-CLIP-v2 and Nomic embeddings on those crops.
4
-
5
  Outputs separate crop folders per model (jina_crops, nomic_crops) for visual comparison.
 
6
  """
 
7
  import argparse
8
  import csv
9
  import time
@@ -11,8 +12,7 @@ from pathlib import Path
11
 
12
  import numpy as np
13
  import torch
14
- import torch.nn.functional as F
15
- from PIL import Image
16
  from transformers import AutoImageProcessor, DFineForObjectDetection
17
 
18
  # Jina-CLIP-v2 few-shot (same refs + classify as jina_fewshot.py)
@@ -22,11 +22,13 @@ from jina_fewshot import (
22
  JinaCLIPv2Encoder,
23
  build_refs,
24
  classify as jina_classify,
25
- draw_label_on_image,
26
  )
27
 
 
 
 
28
  # -----------------------------------------------------------------------------
29
- # Detection + grouping (from reference_detection.py)
30
  # -----------------------------------------------------------------------------
31
 
32
 
@@ -90,10 +92,7 @@ def box_center_inside(box, crop_box):
90
  """True if center of box is inside crop_box. All [x1,y1,x2,y2]."""
91
  cx = (box[0] + box[2]) / 2
92
  cy = (box[1] + box[3]) / 2
93
- return (
94
- crop_box[0] <= cx <= crop_box[2]
95
- and crop_box[1] <= cy <= crop_box[3]
96
- )
97
 
98
 
99
  def squarify_crop_box(bx1, by1, bx2, by2, img_w, img_h):
@@ -140,7 +139,6 @@ def deduplicate_by_iou(detections, iou_threshold=0.9):
140
  """Keep one detection per overlapping group (IoU >= iou_threshold). Prefer higher confidence."""
141
  if not detections:
142
  return []
143
- # Sort by confidence descending; keep first, then add only if no kept box overlaps >= threshold
144
  sorted_d = sorted(detections, key=lambda x: -x["conf"])
145
  kept = []
146
  for d in sorted_d:
@@ -149,25 +147,182 @@ def deduplicate_by_iou(detections, iou_threshold=0.9):
149
  return kept
150
 
151
 
152
- from nomic_fewshot import NomicTextEncoder, NomicVisionEncoder, build_refs_nomic
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
153
 
154
 
155
  def parse_args():
156
  p = argparse.ArgumentParser(
157
- description="D-FINE (person/car) → group → Jina-CLIP-v2 on crops inside groups"
158
  )
159
  p.add_argument("--refs", required=True, help="Reference images folder for Jina and Nomic (e.g. refs/)")
160
  p.add_argument("--input", required=True, help="Full-frame images folder")
161
  p.add_argument("--output", default="pipeline_results", help="Output folder (CSV, etc.)")
162
  p.add_argument("--det-threshold", type=float, default=0.13, help="D-FINE score threshold")
163
- p.add_argument("--group-dist", type=float, default=None,
164
- help="Group distance (default: 0.1 * max(H,W))")
165
- p.add_argument("--min-side", type=int, default=40, help="Min side of expanded bbox in px (skip smaller)")
166
- p.add_argument("--crop-dedup-iou", type=float, default=0.35, help="Min IoU to treat two crops as same object (keep larger)")
167
- p.add_argument("--no-squarify", action="store_true", help="Skip squarify; use expanded bbox only (tighter crops, often better recognition)")
168
- p.add_argument("--padding", type=float, default=0.2, help="Crop padding around group box (0.2 = 20%%)")
169
- p.add_argument("--conf-threshold", type=float, default=0.75, help="Jina accept confidence")
170
- p.add_argument("--gap-threshold", type=float, default=0.05, help="Jina accept gap")
171
  p.add_argument("--text-weight", type=float, default=0.3)
172
  p.add_argument("--max-images", type=int, default=None)
173
  p.add_argument("--device", default=None)
@@ -191,7 +346,6 @@ def get_person_car_label_ids(model):
191
 
192
  def run_dfine(image, processor, model, device, score_threshold):
193
  """Run D-FINE, return all detections as list of {box, score, label_id, label}."""
194
- from PIL import Image
195
  if isinstance(image, Image.Image):
196
  pil = image.convert("RGB")
197
  else:
@@ -212,9 +366,7 @@ def run_dfine(image, processor, model, device, score_threshold):
212
 
213
  detections = []
214
  for result in results:
215
- for score, label_id, box in zip(
216
- result["scores"], result["labels"], result["boxes"]
217
- ):
218
  sid = int(label_id.item())
219
  detections.append({
220
  "box": [float(x) for x in box.cpu().tolist()],
@@ -225,6 +377,48 @@ def run_dfine(image, processor, model, device, score_threshold):
225
  return detections
226
 
227
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
228
  def main():
229
  args = parse_args()
230
  device = args.device or ("cuda" if torch.cuda.is_available() else "cpu")
@@ -238,10 +432,7 @@ def main():
238
  if not input_dir.is_dir():
239
  raise SystemExit(f"Input folder not found: {input_dir}")
240
 
241
- paths = sorted(
242
- p for p in input_dir.iterdir()
243
- if p.suffix.lower() in IMAGE_EXTS
244
- )
245
  if args.max_images is not None:
246
  paths = paths[: args.max_images]
247
  if not paths:
@@ -254,35 +445,34 @@ def main():
254
  dfine_model = DFineForObjectDetection.from_pretrained("ustc-community/dfine-medium-obj365")
255
  dfine_model = dfine_model.to(device).eval()
256
  person_car_ids = get_person_car_label_ids(dfine_model)
257
- print(f" Person/car label IDs: {person_car_ids} ({time.perf_counter()-t0:.1f}s)")
258
 
259
- # Load Jina-CLIP-v2 + build refs
260
  print("[*] Loading Jina-CLIP-v2 and building refs...")
261
  t0 = time.perf_counter()
262
  jina_encoder = JinaCLIPv2Encoder(device)
263
- ref_labels, ref_embs = build_refs(
264
- jina_encoder, refs_dir, TRUNCATE_DIM, args.text_weight, batch_size=16
265
- )
266
- print(f" Jina refs: {ref_labels} ({time.perf_counter()-t0:.1f}s)\n")
267
 
268
- # Load Nomic vision + text, build refs (same as Jina: image + text prompts, text_weight 0.3)
269
  print("[*] Loading Nomic embed-vision + embed-text and building refs...")
270
  t0 = time.perf_counter()
271
  nomic_encoder = NomicVisionEncoder(device)
272
  nomic_text_encoder = NomicTextEncoder(device)
273
  ref_labels_nomic, ref_embs_nomic = build_refs_nomic(
274
- nomic_encoder, refs_dir, batch_size=16,
275
- text_encoder=nomic_text_encoder, text_weight=args.text_weight,
 
 
 
276
  )
277
- print(f" Nomic refs: {ref_labels_nomic} ({time.perf_counter()-t0:.1f}s)\n")
278
 
279
- # Separate output folders per model for visual comparison
280
  jina_crops_dir = output_dir / "jina_crops"
281
  nomic_crops_dir = output_dir / "nomic_crops"
282
  jina_crops_dir.mkdir(parents=True, exist_ok=True)
283
  nomic_crops_dir.mkdir(parents=True, exist_ok=True)
284
 
285
- # CSV
286
  csv_path = output_dir / "results.csv"
287
  f = open(csv_path, "w", newline="")
288
  w = csv.writer(f)
@@ -298,114 +488,91 @@ def main():
298
  img_w, img_h = pil.size
299
  group_dist = args.group_dist if args.group_dist is not None else 0.1 * max(img_h, img_w)
300
 
301
- # 1) D-FINE: detect everything, keep all bboxes for the image
302
- detections = run_dfine(
303
- pil, image_processor, dfine_model, device, args.det_threshold
304
- )
305
  person_car = [d for d in detections if d["cls"] in person_car_ids]
306
  if not person_car:
307
  continue
308
 
309
- # 2) Group person/car detections (same as reference)
310
  grouped = group_detections(person_car, group_dist)
311
  grouped.sort(key=lambda x: x["conf"], reverse=True)
312
- top_groups = grouped[:10] # limit groups per image
313
 
314
- # 3) Collect all candidate crops (bboxes inside person/car groups)
315
- # Each: (crop_box, crop_pil, d, gidx, crop_idx, x1, y1, x2, y2)
316
- candidates = []
317
  for gidx, grp in enumerate(top_groups):
318
- x1, y1, x2, y2 = grp["box"]
319
- group_box = [x1, y1, x2, y2]
320
- inside = [
321
- d for d in detections
322
- if box_center_inside(d["box"], group_box)
323
- and d["cls"] not in person_car_ids
324
- ]
325
- inside = deduplicate_by_iou(inside, iou_threshold=0.9)
326
-
327
- for crop_idx, d in enumerate(inside):
328
- bx1, by1, bx2, by2 = [float(x) for x in d["box"]]
329
- obj_w, obj_h = bx2 - bx1, by2 - by1
330
- if obj_w <= 0 or obj_h <= 0:
331
- continue
332
- pad_x = obj_w * 0.3
333
- pad_y = obj_h * 0.3
334
- bx1 = max(0, int(bx1 - pad_x))
335
- by1 = max(0, int(by1 - pad_y))
336
- bx2 = min(img_w, int(bx2 + pad_x))
337
- by2 = min(img_h, int(by2 + pad_y))
338
- if bx2 <= bx1 or by2 <= by1:
339
- continue
340
- if min(bx2 - bx1, by2 - by1) < args.min_side:
341
- continue
342
- expanded_box = [bx1, by1, bx2, by2]
343
- candidates.append((expanded_box, d, gidx, crop_idx, x1, y1, x2, y2))
344
-
345
- # 4) Dedup on EXPANDED boxes (before squarify), keep larger; then squarify only kept
346
- def crop_area(box):
347
- return (box[2] - box[0]) * (box[3] - box[1])
348
-
349
- candidates.sort(key=lambda c: -crop_area(c[0]))
350
- kept = []
351
- for c in candidates:
352
- expanded_box = c[0]
353
- def is_same_object(box_a, box_b):
354
- if box_iou(box_a, box_b) >= args.crop_dedup_iou:
355
- return True
356
- if box_center_inside(box_a, box_b) or box_center_inside(box_b, box_a):
357
- return True
358
- return False
359
- if not any(is_same_object(expanded_box, k[0]) for k in kept):
360
- kept.append(c)
361
-
362
- # 5) Optionally squarify, then run Jina and Nomic only on kept crops
363
- for i, (expanded_box, d, gidx, crop_idx, x1, y1, x2, y2) in enumerate(kept):
364
- if not args.no_squarify:
365
- bx1, by1, bx2, by2 = squarify_crop_box(
366
- expanded_box[0], expanded_box[1], expanded_box[2], expanded_box[3], img_w, img_h
367
- )
368
- else:
369
- bx1, by1, bx2, by2 = expanded_box[0], expanded_box[1], expanded_box[2], expanded_box[3]
370
- crop_pil = pil.crop((bx1, by1, bx2, by2))
371
- crop_name = f"{img_path.stem}_g{gidx}_{i}_{bx1}_{by1}_{bx2}_{by2}{img_path.suffix}"
372
-
373
- q_jina = jina_encoder.encode_images([crop_pil], TRUNCATE_DIM)
374
- result_jina = jina_classify(
375
- q_jina, ref_labels, ref_embs,
376
- args.conf_threshold, args.gap_threshold
377
  )
378
- if result_jina["prediction"] in ref_labels:
379
- label_jina = result_jina["prediction"]
380
- conf_jina = result_jina["confidence"]
381
- else:
382
- label_jina = f"unnamed (dfine: {d['label']})"
383
- conf_jina = 0.0
384
- ann_jina = draw_label_on_image(crop_pil, label_jina, conf_jina)
385
- ann_jina.save(jina_crops_dir / crop_name)
386
 
387
- q_nomic = nomic_encoder.encode_images([crop_pil])
388
- result_nomic = jina_classify(
389
- q_nomic, ref_labels_nomic, ref_embs_nomic,
390
- args.conf_threshold, args.gap_threshold
 
 
 
 
 
 
 
 
 
391
  )
392
- if result_nomic["prediction"] in ref_labels_nomic:
393
- label_nomic = result_nomic["prediction"]
394
- conf_nomic = result_nomic["confidence"]
395
- else:
396
- label_nomic = f"unnamed (dfine: {d['label']})"
397
- conf_nomic = 0.0
398
- ann_nomic = draw_label_on_image(crop_pil, label_nomic, conf_nomic)
399
- ann_nomic.save(nomic_crops_dir / crop_name)
400
 
401
- w.writerow([
402
- img_path.name, crop_name, gidx,
403
- x1, y1, x2, y2,
404
- bx1, by1, bx2, by2,
405
- d["label"], f"{d['conf']:.4f}",
406
- result_jina["prediction"], f"{result_jina['confidence']:.4f}", result_jina["status"],
407
- result_nomic["prediction"], f"{result_nomic['confidence']:.4f}", result_nomic["status"],
408
- ])
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
409
 
410
  f.close()
411
  print(f"[*] Wrote {csv_path}")
@@ -414,7 +581,7 @@ def main():
414
 
415
 
416
  # -----------------------------------------------------------------------------
417
- # Single-image runner for Gradio app: D-FINE first, then Jina or Nomic (user choice)
418
  # -----------------------------------------------------------------------------
419
  _APP_DFINE = None
420
  _APP_JINA = None
@@ -436,18 +603,20 @@ def run_single_image(
436
  squarify=True,
437
  ):
438
  """
439
- Run D-FINE on one image, then classify small-object crops with Jina or Nomic.
440
- refs_dir: path to refs folder (str or Path). encoder_choice: "jina" or "nomic".
441
- Returns (annotated_pil, result_text) for display in app.
 
442
  """
443
- import numpy as np
444
- from PIL import Image
445
  global _APP_DFINE, _APP_JINA, _APP_NOMIC, _APP_REFS_JINA, _APP_REFS_NOMIC
 
446
  refs_dir = Path(refs_dir)
447
  if not refs_dir.is_dir():
448
  return None, f"Refs folder not found: {refs_dir}"
 
449
  device = device or ("cuda" if torch.cuda.is_available() else "cpu")
450
  print(f"[*] Device: {device}")
 
451
  pil = pil_image.convert("RGB") if isinstance(pil_image, Image.Image) else Image.fromarray(pil_image).convert("RGB")
452
  img_w, img_h = pil.size
453
  group_dist = 0.1 * max(img_h, img_w)
@@ -459,6 +628,7 @@ def run_single_image(
459
  dfine_model = dfine_model.to(device).eval()
460
  person_car_ids = get_person_car_label_ids(dfine_model)
461
  _APP_DFINE = (image_processor, dfine_model, person_car_ids)
 
462
  image_processor, dfine_model, person_car_ids = _APP_DFINE
463
 
464
  detections = run_dfine(pil, image_processor, dfine_model, device, det_threshold)
@@ -469,51 +639,6 @@ def run_single_image(
469
  grouped = group_detections(person_car, group_dist)
470
  grouped.sort(key=lambda x: x["conf"], reverse=True)
471
  top_groups = grouped[:10]
472
- candidates = []
473
- for gidx, grp in enumerate(top_groups):
474
- x1, y1, x2, y2 = grp["box"]
475
- group_box = [x1, y1, x2, y2]
476
- inside = [
477
- d for d in detections
478
- if box_center_inside(d["box"], group_box) and d["cls"] not in person_car_ids
479
- ]
480
- inside = deduplicate_by_iou(inside, iou_threshold=0.9)
481
- for crop_idx, d in enumerate(inside):
482
- bx1, by1, bx2, by2 = [float(x) for x in d["box"]]
483
- obj_w, obj_h = bx2 - bx1, by2 - by1
484
- if obj_w <= 0 or obj_h <= 0:
485
- continue
486
- pad_x, pad_y = obj_w * 0.3, obj_h * 0.3
487
- bx1 = max(0, int(bx1 - pad_x))
488
- by1 = max(0, int(by1 - pad_y))
489
- bx2 = min(img_w, int(bx2 + pad_x))
490
- by2 = min(img_h, int(by2 + pad_y))
491
- if bx2 <= bx1 or by2 <= by1:
492
- continue
493
- if min(bx2 - bx1, by2 - by1) < min_side:
494
- continue
495
- expanded_box = [bx1, by1, bx2, by2]
496
- candidates.append((expanded_box, d, gidx, crop_idx))
497
-
498
- def crop_area(box):
499
- return (box[2] - box[0]) * (box[3] - box[1])
500
-
501
- candidates.sort(key=lambda c: -crop_area(c[0]))
502
- kept = []
503
- for c in candidates:
504
- def is_same_object(box_a, box_b):
505
- if box_iou(box_a, box_b) >= crop_dedup_iou:
506
- return True
507
- if box_center_inside(box_a, box_b) or box_center_inside(box_b, box_a):
508
- return True
509
- return False
510
- if not any(is_same_object(c[0], k[0]) for k in kept):
511
- kept.append(c)
512
-
513
- if not kept:
514
- if not candidates:
515
- return np.array(pil), "No small-object crops: D-FINE did not detect any object (gun/phone/etc.) inside person/car areas, or all were below min size. Try a higher-resolution image."
516
- return np.array(pil), "No small-object crops (after dedup)."
517
 
518
  # Load encoder + refs for chosen model
519
  if encoder_choice == "jina":
@@ -522,44 +647,98 @@ def run_single_image(
522
  ref_labels, ref_embs = build_refs(jina_encoder, refs_dir, TRUNCATE_DIM, 0.3, batch_size=16)
523
  _APP_JINA = (jina_encoder, ref_labels, ref_embs)
524
  _APP_REFS_JINA = str(refs_dir)
525
- jina_encoder, ref_labels, ref_embs = _APP_JINA
526
  else:
527
  if _APP_NOMIC is None or _APP_REFS_NOMIC != str(refs_dir):
528
  nomic_encoder = NomicVisionEncoder(device)
529
  nomic_text_encoder = NomicTextEncoder(device)
530
  ref_labels, ref_embs = build_refs_nomic(
531
- nomic_encoder, refs_dir, batch_size=16,
532
- text_encoder=nomic_text_encoder, text_weight=0.3,
 
 
 
533
  )
534
  _APP_NOMIC = (nomic_encoder, ref_labels, ref_embs)
535
  _APP_REFS_NOMIC = str(refs_dir)
536
- nomic_encoder, ref_labels, ref_embs = _APP_NOMIC
537
 
 
538
  lines = []
539
- out_img = pil.copy()
540
- for i, (expanded_box, d, gidx, crop_idx) in enumerate(kept):
541
- if squarify:
542
- bx1, by1, bx2, by2 = squarify_crop_box(
543
- expanded_box[0], expanded_box[1], expanded_box[2], expanded_box[3], img_w, img_h
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
544
  )
545
- else:
546
- bx1, by1, bx2, by2 = expanded_box[0], expanded_box[1], expanded_box[2], expanded_box[3]
547
- crop_pil = pil.crop((bx1, by1, bx2, by2))
548
- if encoder_choice == "jina":
549
- q = jina_encoder.encode_images([crop_pil], TRUNCATE_DIM)
550
- result = jina_classify(q, ref_labels, ref_embs, conf_threshold, gap_threshold)
551
- else:
552
- q = nomic_encoder.encode_images([crop_pil])
553
- result = jina_classify(q, ref_labels, ref_embs, conf_threshold, gap_threshold)
554
- pred = result["prediction"] if result["prediction"] in ref_labels else f"unknown ({d['label']})"
555
- conf = result["confidence"]
556
- lines.append(f"Crop {i+1}: {pred} ({conf:.2f})")
557
- labeled = draw_label_on_image(crop_pil, pred, conf)
558
- out_img.paste(labeled, (bx1, by1))
559
-
560
- result_text = "\n".join(lines) if lines else "No crops"
561
- return np.array(out_img), result_text
562
 
563
 
564
  if __name__ == "__main__":
565
- main()
 
1
  """
2
+ Pipeline: D-FINE (person/car only) → group detections → crop group regions →
3
+ classify all inner object detections with Jina-CLIP-v2 or Nomic.
 
4
  Outputs separate crop folders per model (jina_crops, nomic_crops) for visual comparison.
5
+ Each saved image is the D-FINE group crop, with bboxes drawn only for known classes.
6
  """
7
+
8
  import argparse
9
  import csv
10
  import time
 
12
 
13
  import numpy as np
14
  import torch
15
+ from PIL import Image, ImageDraw, ImageFont
 
16
  from transformers import AutoImageProcessor, DFineForObjectDetection
17
 
18
  # Jina-CLIP-v2 few-shot (same refs + classify as jina_fewshot.py)
 
22
  JinaCLIPv2Encoder,
23
  build_refs,
24
  classify as jina_classify,
 
25
  )
26
 
27
+ from nomic_fewshot import NomicTextEncoder, NomicVisionEncoder, build_refs_nomic
28
+
29
+
30
  # -----------------------------------------------------------------------------
31
+ # Detection + grouping
32
  # -----------------------------------------------------------------------------
33
 
34
 
 
92
  """True if center of box is inside crop_box. All [x1,y1,x2,y2]."""
93
  cx = (box[0] + box[2]) / 2
94
  cy = (box[1] + box[3]) / 2
95
+ return crop_box[0] <= cx <= crop_box[2] and crop_box[1] <= cy <= crop_box[3]
 
 
 
96
 
97
 
98
  def squarify_crop_box(bx1, by1, bx2, by2, img_w, img_h):
 
139
  """Keep one detection per overlapping group (IoU >= iou_threshold). Prefer higher confidence."""
140
  if not detections:
141
  return []
 
142
  sorted_d = sorted(detections, key=lambda x: -x["conf"])
143
  kept = []
144
  for d in sorted_d:
 
147
  return kept
148
 
149
 
150
+ # -----------------------------------------------------------------------------
151
+ # Drawing / layout helpers
152
+ # -----------------------------------------------------------------------------
153
+
154
+
155
+ def _load_font_for_box(img_w, img_h):
156
+ font_path = "/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf"
157
+ size = max(10, min(img_w, img_h) // 14)
158
+ try:
159
+ return ImageFont.truetype(font_path, size=size)
160
+ except OSError:
161
+ return ImageFont.load_default()
162
+
163
+
164
+ def draw_bbox_with_optional_label(
165
+ img,
166
+ box,
167
+ label=None,
168
+ confidence=None,
169
+ color=(255, 0, 0),
170
+ width=3,
171
+ ):
172
+ """
173
+ Draw bbox on the given image. If label is None, draw only the box.
174
+ box is [x1, y1, x2, y2] in this image's coordinate space.
175
+ """
176
+ out = img.copy().convert("RGB")
177
+ draw = ImageDraw.Draw(out)
178
+ x1, y1, x2, y2 = [int(v) for v in box]
179
+
180
+ x1 = max(0, min(x1, out.width - 1))
181
+ y1 = max(0, min(y1, out.height - 1))
182
+ x2 = max(0, min(x2, out.width - 1))
183
+ y2 = max(0, min(y2, out.height - 1))
184
+
185
+ for i in range(width):
186
+ draw.rectangle([x1 - i, y1 - i, x2 + i, y2 + i], outline=color)
187
+
188
+ if label:
189
+ font = _load_font_for_box(out.width, out.height)
190
+ text = f"{label} ({confidence:.2f})" if confidence is not None else label
191
+ bbox = draw.textbbox((0, 0), text, font=font)
192
+ tw = bbox[2] - bbox[0]
193
+ th = bbox[3] - bbox[1]
194
+
195
+ tx = max(0, min(x1, out.width - tw - 8))
196
+ ty = y1 - th - 8
197
+ if ty < 0:
198
+ ty = min(out.height - th - 4, y1 + 4)
199
+
200
+ draw.rectangle([tx, ty, tx + tw + 8, ty + th + 6], fill=color)
201
+ draw.text((tx + 4, ty + 3), text, fill=(255, 255, 255), font=font)
202
+
203
+ return out
204
+
205
+
206
+ def stack_images_vertical(images, bg=(255, 255, 255), pad=10):
207
+ """Stack PIL images vertically into one output image."""
208
+ if not images:
209
+ return None
210
+ widths = [img.width for img in images]
211
+ heights = [img.height for img in images]
212
+ out_w = max(widths)
213
+ out_h = sum(heights) + pad * (len(images) - 1)
214
+ canvas = Image.new("RGB", (out_w, out_h), color=bg)
215
+
216
+ y = 0
217
+ for img in images:
218
+ x = (out_w - img.width) // 2
219
+ canvas.paste(img, (x, y))
220
+ y += img.height + pad
221
+ return canvas
222
+
223
+
224
+ # -----------------------------------------------------------------------------
225
+ # Shared crop/object preparation
226
+ # -----------------------------------------------------------------------------
227
+
228
+
229
+ def expand_box_with_padding(box, img_w, img_h, padding):
230
+ x1, y1, x2, y2 = [float(v) for v in box]
231
+ w = x2 - x1
232
+ h = y2 - y1
233
+ if w <= 0 or h <= 0:
234
+ return None
235
+ pad_x = w * padding
236
+ pad_y = h * padding
237
+ ex1 = max(0, int(x1 - pad_x))
238
+ ey1 = max(0, int(y1 - pad_y))
239
+ ex2 = min(img_w, int(x2 + pad_x))
240
+ ey2 = min(img_h, int(y2 + pad_y))
241
+ if ex2 <= ex1 or ey2 <= ey1:
242
+ return None
243
+ return [ex1, ey1, ex2, ey2]
244
+
245
+
246
+ def build_group_crop_box(group_box, img_w, img_h, padding=0.2, squarify=True):
247
+ expanded = expand_box_with_padding(group_box, img_w, img_h, padding)
248
+ if expanded is None:
249
+ return None
250
+ if squarify:
251
+ return list(squarify_crop_box(expanded[0], expanded[1], expanded[2], expanded[3], img_w, img_h))
252
+ return expanded
253
+
254
+
255
+ def collect_group_object_candidates(
256
+ detections,
257
+ person_car_ids,
258
+ group_box,
259
+ img_w,
260
+ img_h,
261
+ min_side,
262
+ crop_dedup_iou,
263
+ object_padding=0.3,
264
+ ):
265
+ """
266
+ For one group crop, collect and deduplicate all non-person/car detections inside it.
267
+ Returns list of dicts with:
268
+ {
269
+ "det": original detection,
270
+ "expanded_box": expanded object crop box in full-image coords
271
+ }
272
+ """
273
+ inside = [
274
+ d for d in detections
275
+ if box_center_inside(d["box"], group_box) and d["cls"] not in person_car_ids
276
+ ]
277
+ inside = deduplicate_by_iou(inside, iou_threshold=0.9)
278
+
279
+ candidates = []
280
+ for d in inside:
281
+ expanded_box = expand_box_with_padding(d["box"], img_w, img_h, object_padding)
282
+ if expanded_box is None:
283
+ continue
284
+ if min(expanded_box[2] - expanded_box[0], expanded_box[3] - expanded_box[1]) < min_side:
285
+ continue
286
+ candidates.append({
287
+ "det": d,
288
+ "expanded_box": expanded_box,
289
+ })
290
+
291
+ def crop_area(box):
292
+ return (box[2] - box[0]) * (box[3] - box[1])
293
+
294
+ candidates.sort(key=lambda c: -crop_area(c["expanded_box"]))
295
+ kept = []
296
+
297
+ def is_same_object(box_a, box_b):
298
+ if box_iou(box_a, box_b) >= crop_dedup_iou:
299
+ return True
300
+ if box_center_inside(box_a, box_b) or box_center_inside(box_b, box_a):
301
+ return True
302
+ return False
303
+
304
+ for c in candidates:
305
+ if not any(is_same_object(c["expanded_box"], k["expanded_box"]) for k in kept):
306
+ kept.append(c)
307
+
308
+ return kept
309
 
310
 
311
  def parse_args():
312
  p = argparse.ArgumentParser(
313
+ description="D-FINE (person/car) → group → classify objects inside each group crop"
314
  )
315
  p.add_argument("--refs", required=True, help="Reference images folder for Jina and Nomic (e.g. refs/)")
316
  p.add_argument("--input", required=True, help="Full-frame images folder")
317
  p.add_argument("--output", default="pipeline_results", help="Output folder (CSV, etc.)")
318
  p.add_argument("--det-threshold", type=float, default=0.13, help="D-FINE score threshold")
319
+ p.add_argument("--group-dist", type=float, default=None, help="Group distance (default: 0.1 * max(H,W))")
320
+ p.add_argument("--min-side", type=int, default=40, help="Min side of expanded object bbox in px (skip smaller)")
321
+ p.add_argument("--crop-dedup-iou", type=float, default=0.35, help="Min IoU to treat two object crops as same object")
322
+ p.add_argument("--no-squarify", action="store_true", help="Skip squarify on group crop")
323
+ p.add_argument("--padding", type=float, default=0.2, help="Padding around D-FINE group crop box")
324
+ p.add_argument("--conf-threshold", type=float, default=0.75, help="Accept confidence")
325
+ p.add_argument("--gap-threshold", type=float, default=0.05, help="Accept gap")
 
326
  p.add_argument("--text-weight", type=float, default=0.3)
327
  p.add_argument("--max-images", type=int, default=None)
328
  p.add_argument("--device", default=None)
 
346
 
347
  def run_dfine(image, processor, model, device, score_threshold):
348
  """Run D-FINE, return all detections as list of {box, score, label_id, label}."""
 
349
  if isinstance(image, Image.Image):
350
  pil = image.convert("RGB")
351
  else:
 
366
 
367
  detections = []
368
  for result in results:
369
+ for score, label_id, box in zip(result["scores"], result["labels"], result["boxes"]):
 
 
370
  sid = int(label_id.item())
371
  detections.append({
372
  "box": [float(x) for x in box.cpu().tolist()],
 
377
  return detections
378
 
379
 
380
+ def annotate_group_crop(
381
+ pil,
382
+ detections,
383
+ person_car_ids,
384
+ group_box,
385
+ crop_box,
386
+ encoder_choice,
387
+ encoder,
388
+ ref_labels,
389
+ ref_embs,
390
+ conf_threshold,
391
+ gap_threshold,
392
+ ):
393
+ """
394
+ Build one D-FINE group crop, classify all object candidates inside it,
395
+ and draw bbox+label only for known classes.
396
+
397
+ Returns:
398
+ annotated_crop_pil,
399
+ rows_for_csv,
400
+ known_lines
401
+ """
402
+ crop_x1, crop_y1, crop_x2, crop_y2 = crop_box
403
+ group_crop = pil.crop((crop_x1, crop_y1, crop_x2, crop_y2)).convert("RGB")
404
+
405
+ obj_candidates = collect_group_object_candidates(
406
+ detections=detections,
407
+ person_car_ids=person_car_ids,
408
+ group_box=group_box,
409
+ img_w=pil.width,
410
+ img_h=pil.height,
411
+ min_side=1, # not used here; caller filters before this if needed
412
+ crop_dedup_iou=1.0, # not used here; caller passes already-filtered set if needed
413
+ object_padding=0.0,
414
+ )
415
+ # This helper is not used directly because caller already builds filtered candidates.
416
+ # Kept here only for API symmetry.
417
+ _ = obj_candidates
418
+
419
+ return group_crop, [], []
420
+
421
+
422
  def main():
423
  args = parse_args()
424
  device = args.device or ("cuda" if torch.cuda.is_available() else "cpu")
 
432
  if not input_dir.is_dir():
433
  raise SystemExit(f"Input folder not found: {input_dir}")
434
 
435
+ paths = sorted(p for p in input_dir.iterdir() if p.suffix.lower() in IMAGE_EXTS)
 
 
 
436
  if args.max_images is not None:
437
  paths = paths[: args.max_images]
438
  if not paths:
 
445
  dfine_model = DFineForObjectDetection.from_pretrained("ustc-community/dfine-medium-obj365")
446
  dfine_model = dfine_model.to(device).eval()
447
  person_car_ids = get_person_car_label_ids(dfine_model)
448
+ print(f" Person/car label IDs: {person_car_ids} ({time.perf_counter() - t0:.1f}s)")
449
 
450
+ # Load Jina refs
451
  print("[*] Loading Jina-CLIP-v2 and building refs...")
452
  t0 = time.perf_counter()
453
  jina_encoder = JinaCLIPv2Encoder(device)
454
+ ref_labels, ref_embs = build_refs(jina_encoder, refs_dir, TRUNCATE_DIM, args.text_weight, batch_size=16)
455
+ print(f" Jina refs: {ref_labels} ({time.perf_counter() - t0:.1f}s)\n")
 
 
456
 
457
+ # Load Nomic refs
458
  print("[*] Loading Nomic embed-vision + embed-text and building refs...")
459
  t0 = time.perf_counter()
460
  nomic_encoder = NomicVisionEncoder(device)
461
  nomic_text_encoder = NomicTextEncoder(device)
462
  ref_labels_nomic, ref_embs_nomic = build_refs_nomic(
463
+ nomic_encoder,
464
+ refs_dir,
465
+ batch_size=16,
466
+ text_encoder=nomic_text_encoder,
467
+ text_weight=args.text_weight,
468
  )
469
+ print(f" Nomic refs: {ref_labels_nomic} ({time.perf_counter() - t0:.1f}s)\n")
470
 
 
471
  jina_crops_dir = output_dir / "jina_crops"
472
  nomic_crops_dir = output_dir / "nomic_crops"
473
  jina_crops_dir.mkdir(parents=True, exist_ok=True)
474
  nomic_crops_dir.mkdir(parents=True, exist_ok=True)
475
 
 
476
  csv_path = output_dir / "results.csv"
477
  f = open(csv_path, "w", newline="")
478
  w = csv.writer(f)
 
488
  img_w, img_h = pil.size
489
  group_dist = args.group_dist if args.group_dist is not None else 0.1 * max(img_h, img_w)
490
 
491
+ detections = run_dfine(pil, image_processor, dfine_model, device, args.det_threshold)
 
 
 
492
  person_car = [d for d in detections if d["cls"] in person_car_ids]
493
  if not person_car:
494
  continue
495
 
 
496
  grouped = group_detections(person_car, group_dist)
497
  grouped.sort(key=lambda x: x["conf"], reverse=True)
498
+ top_groups = grouped[:10]
499
 
 
 
 
500
  for gidx, grp in enumerate(top_groups):
501
+ group_box = grp["box"]
502
+ crop_box = build_group_crop_box(
503
+ group_box,
504
+ img_w,
505
+ img_h,
506
+ padding=args.padding,
507
+ squarify=not args.no_squarify,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
508
  )
509
+ if crop_box is None:
510
+ continue
 
 
 
 
 
 
511
 
512
+ crop_x1, crop_y1, crop_x2, crop_y2 = crop_box
513
+ crop_name = f"{img_path.stem}_g{gidx}_{crop_x1}_{crop_y1}_{crop_x2}_{crop_y2}{img_path.suffix}"
514
+ base_group_crop = pil.crop((crop_x1, crop_y1, crop_x2, crop_y2)).convert("RGB")
515
+
516
+ obj_candidates = collect_group_object_candidates(
517
+ detections=detections,
518
+ person_car_ids=person_car_ids,
519
+ group_box=group_box,
520
+ img_w=img_w,
521
+ img_h=img_h,
522
+ min_side=args.min_side,
523
+ crop_dedup_iou=args.crop_dedup_iou,
524
+ object_padding=0.3,
525
  )
 
 
 
 
 
 
 
 
526
 
527
+ ann_jina = base_group_crop.copy()
528
+ ann_nomic = base_group_crop.copy()
529
+
530
+ for item in obj_candidates:
531
+ d = item["det"]
532
+ ex1, ey1, ex2, ey2 = item["expanded_box"]
533
+
534
+ obj_crop = pil.crop((ex1, ey1, ex2, ey2)).convert("RGB")
535
+
536
+ q_jina = jina_encoder.encode_images([obj_crop], TRUNCATE_DIM)
537
+ result_jina = jina_classify(q_jina, ref_labels, ref_embs, args.conf_threshold, args.gap_threshold)
538
+
539
+ q_nomic = nomic_encoder.encode_images([obj_crop])
540
+ result_nomic = jina_classify(q_nomic, ref_labels_nomic, ref_embs_nomic, args.conf_threshold, args.gap_threshold)
541
+
542
+ rel_box = [
543
+ max(0, int(round(d["box"][0] - crop_x1))),
544
+ max(0, int(round(d["box"][1] - crop_y1))),
545
+ min(base_group_crop.width, int(round(d["box"][2] - crop_x1))),
546
+ min(base_group_crop.height, int(round(d["box"][3] - crop_y1))),
547
+ ]
548
+
549
+ if result_jina["prediction"] in ref_labels:
550
+ ann_jina = draw_bbox_with_optional_label(
551
+ ann_jina,
552
+ rel_box,
553
+ label=result_jina["prediction"],
554
+ confidence=result_jina["confidence"],
555
+ )
556
+
557
+ if result_nomic["prediction"] in ref_labels_nomic:
558
+ ann_nomic = draw_bbox_with_optional_label(
559
+ ann_nomic,
560
+ rel_box,
561
+ label=result_nomic["prediction"],
562
+ confidence=result_nomic["confidence"],
563
+ )
564
+
565
+ w.writerow([
566
+ img_path.name, crop_name, gidx,
567
+ crop_x1, crop_y1, crop_x2, crop_y2,
568
+ d["box"][0], d["box"][1], d["box"][2], d["box"][3],
569
+ d["label"], f"{d['conf']:.4f}",
570
+ result_jina["prediction"], f"{result_jina['confidence']:.4f}", result_jina["status"],
571
+ result_nomic["prediction"], f"{result_nomic['confidence']:.4f}", result_nomic["status"],
572
+ ])
573
+
574
+ ann_jina.save(jina_crops_dir / crop_name)
575
+ ann_nomic.save(nomic_crops_dir / crop_name)
576
 
577
  f.close()
578
  print(f"[*] Wrote {csv_path}")
 
581
 
582
 
583
  # -----------------------------------------------------------------------------
584
+ # Single-image runner for Gradio app: D-FINE first, then Jina or Nomic
585
  # -----------------------------------------------------------------------------
586
  _APP_DFINE = None
587
  _APP_JINA = None
 
603
  squarify=True,
604
  ):
605
  """
606
+ Run D-FINE on one image, then classify small-object detections inside each group crop.
607
+ Returns:
608
+ - one vertically stacked image of all D-FINE group crops
609
+ - text containing only known-class predictions
610
  """
 
 
611
  global _APP_DFINE, _APP_JINA, _APP_NOMIC, _APP_REFS_JINA, _APP_REFS_NOMIC
612
+
613
  refs_dir = Path(refs_dir)
614
  if not refs_dir.is_dir():
615
  return None, f"Refs folder not found: {refs_dir}"
616
+
617
  device = device or ("cuda" if torch.cuda.is_available() else "cpu")
618
  print(f"[*] Device: {device}")
619
+
620
  pil = pil_image.convert("RGB") if isinstance(pil_image, Image.Image) else Image.fromarray(pil_image).convert("RGB")
621
  img_w, img_h = pil.size
622
  group_dist = 0.1 * max(img_h, img_w)
 
628
  dfine_model = dfine_model.to(device).eval()
629
  person_car_ids = get_person_car_label_ids(dfine_model)
630
  _APP_DFINE = (image_processor, dfine_model, person_car_ids)
631
+
632
  image_processor, dfine_model, person_car_ids = _APP_DFINE
633
 
634
  detections = run_dfine(pil, image_processor, dfine_model, device, det_threshold)
 
639
  grouped = group_detections(person_car, group_dist)
640
  grouped.sort(key=lambda x: x["conf"], reverse=True)
641
  top_groups = grouped[:10]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
642
 
643
  # Load encoder + refs for chosen model
644
  if encoder_choice == "jina":
 
647
  ref_labels, ref_embs = build_refs(jina_encoder, refs_dir, TRUNCATE_DIM, 0.3, batch_size=16)
648
  _APP_JINA = (jina_encoder, ref_labels, ref_embs)
649
  _APP_REFS_JINA = str(refs_dir)
650
+ encoder, ref_labels, ref_embs = _APP_JINA
651
  else:
652
  if _APP_NOMIC is None or _APP_REFS_NOMIC != str(refs_dir):
653
  nomic_encoder = NomicVisionEncoder(device)
654
  nomic_text_encoder = NomicTextEncoder(device)
655
  ref_labels, ref_embs = build_refs_nomic(
656
+ nomic_encoder,
657
+ refs_dir,
658
+ batch_size=16,
659
+ text_encoder=nomic_text_encoder,
660
+ text_weight=0.3,
661
  )
662
  _APP_NOMIC = (nomic_encoder, ref_labels, ref_embs)
663
  _APP_REFS_NOMIC = str(refs_dir)
664
+ encoder, ref_labels, ref_embs = _APP_NOMIC
665
 
666
+ output_crops = []
667
  lines = []
668
+
669
+ for gidx, grp in enumerate(top_groups):
670
+ group_box = grp["box"]
671
+ crop_box = build_group_crop_box(
672
+ group_box,
673
+ img_w,
674
+ img_h,
675
+ padding=0.2,
676
+ squarify=squarify,
677
+ )
678
+ if crop_box is None:
679
+ continue
680
+
681
+ crop_x1, crop_y1, crop_x2, crop_y2 = crop_box
682
+ group_crop = pil.crop((crop_x1, crop_y1, crop_x2, crop_y2)).convert("RGB")
683
+
684
+ obj_candidates = collect_group_object_candidates(
685
+ detections=detections,
686
+ person_car_ids=person_car_ids,
687
+ group_box=group_box,
688
+ img_w=img_w,
689
+ img_h=img_h,
690
+ min_side=min_side,
691
+ crop_dedup_iou=crop_dedup_iou,
692
+ object_padding=0.3,
693
+ )
694
+
695
+ ann_crop = group_crop.copy()
696
+ group_known_lines = []
697
+
698
+ for item in obj_candidates:
699
+ d = item["det"]
700
+ ex1, ey1, ex2, ey2 = item["expanded_box"]
701
+ obj_crop = pil.crop((ex1, ey1, ex2, ey2)).convert("RGB")
702
+
703
+ if encoder_choice == "jina":
704
+ q = encoder.encode_images([obj_crop], TRUNCATE_DIM)
705
+ result = jina_classify(q, ref_labels, ref_embs, conf_threshold, gap_threshold)
706
+ else:
707
+ q = encoder.encode_images([obj_crop])
708
+ result = jina_classify(q, ref_labels, ref_embs, conf_threshold, gap_threshold)
709
+
710
+ known = result["prediction"] in ref_labels
711
+ if not known:
712
+ continue
713
+
714
+ rel_box = [
715
+ max(0, int(round(d["box"][0] - crop_x1))),
716
+ max(0, int(round(d["box"][1] - crop_y1))),
717
+ min(group_crop.width, int(round(d["box"][2] - crop_x1))),
718
+ min(group_crop.height, int(round(d["box"][3] - crop_y1))),
719
+ ]
720
+
721
+ ann_crop = draw_bbox_with_optional_label(
722
+ ann_crop,
723
+ rel_box,
724
+ label=result["prediction"],
725
+ confidence=result["confidence"],
726
  )
727
+ group_known_lines.append(f"{result['prediction']} ({result['confidence']:.2f})")
728
+
729
+ output_crops.append(ann_crop)
730
+
731
+ if group_known_lines:
732
+ lines.append(f"Crop {gidx + 1}:")
733
+ lines.extend(group_known_lines)
734
+
735
+ if not output_crops:
736
+ return np.array(pil), "No D-FINE group crops."
737
+
738
+ stacked = stack_images_vertical(output_crops, pad=10)
739
+ result_text = "\n".join(lines) if lines else ""
740
+ return np.array(stacked), result_text
 
 
 
741
 
742
 
743
  if __name__ == "__main__":
744
+ main()