happydev93 commited on
Commit
d4c1535
Β·
verified Β·
1 Parent(s): bf7a6e5

deploy_chutes: upload project folder

Browse files
Files changed (3) hide show
  1. chute_config.yml +21 -0
  2. miner.py +707 -0
  3. weights.onnx +3 -0
chute_config.yml ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Image:
2
+ from_base: parachutes/python:3.12
3
+ run_command:
4
+ - pip install --upgrade setuptools wheel
5
+ - pip install 'numpy>=1.23' 'onnxruntime-gpu>=1.16' 'opencv-python>=4.7' 'pillow>=9.5' 'huggingface_hub>=0.19.4' 'pydantic>=2.0' 'pyyaml>=6.0' 'aiohttp>=3.9'
6
+ - pip install torch==2.8.0 torchvision==0.23.0 torchaudio==2.8.0 --index-url https://download.pytorch.org/whl/cu128
7
+
8
+ NodeSelector:
9
+ gpu_count: 1
10
+ min_vram_gb_per_gpu: 16
11
+ max_hourly_price_per_gpu: 2
12
+ include:
13
+ - pro_6000
14
+
15
+ Chute:
16
+ timeout_seconds: 900
17
+ concurrency: 4
18
+ max_instances: 5
19
+ scaling_threshold: 0.5
20
+ shutdown_after_seconds: 288000
21
+ tee: true
miner.py ADDED
@@ -0,0 +1,707 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import math
3
+ from pathlib import Path
4
+
5
+ import cv2
6
+ import numpy as np
7
+ import onnxruntime as ort
8
+ from numpy import ndarray
9
+ from pydantic import BaseModel
10
+
11
+
12
+ # =============================================================================
13
+ # Person miner (manak0/Detect-Person) β€” port of the proven carwash001 miner,
14
+ # same structure as VehicleDetection/miner/my_miner.
15
+ #
16
+ # Model: yolo26n end2end ONNX @704 (person_0904_704 / powerful_selected soup),
17
+ # single element-canonical class: 0=person.
18
+ #
19
+ # Carried over (all validated on the carwash/vehicle elements):
20
+ # * robust model loading, FP16 auto-detect, metadata class order, ORT thread
21
+ # pinning, raw-YOLO decode fallback, NO CLAHE (train/test match)
22
+ # * vectorized hard NMS + cluster-max boost, per-class NMS IoU / min-area,
23
+ # pre_nms_topk, cross-class dedup (a no-op guard with one class),
24
+ # square-domain hook
25
+ # * soft-NMS with post-decay re-filter (use_soft_nms), rescue bonus β€” both
26
+ # SWEEPABLE via tune_miner.py; which setting wins is element-specific
27
+ # (carwash: soft on / cluster on / rescue off; vehicle: the opposite).
28
+ #
29
+ # The vehicle miner's dual-label bus/truck lever is intentionally absent:
30
+ # person has no sibling class to hedge against.
31
+ #
32
+ # Run tune_miner.py after ANY retrain/re-export and paste its printout below.
33
+ # A failing conformity tuple (latency/format) removes the miner from scoring
34
+ # entirely β€” bench latency with bench_latency.py (2 pinned CPUs) too.
35
+ # =============================================================================
36
+
37
+
38
+ class BoundingBox(BaseModel):
39
+ x1: int
40
+ y1: int
41
+ x2: int
42
+ y2: int
43
+ cls_id: int
44
+ conf: float
45
+
46
+
47
+ class TVFrameResult(BaseModel):
48
+ frame_id: int
49
+ boxes: list[BoundingBox]
50
+ keypoints: list[tuple[int, int]]
51
+
52
+
53
+ class Miner:
54
+ def __init__(self, path_hf_repo: Path) -> None:
55
+ model_path = self._resolve_model_path(path_hf_repo)
56
+ # Canonical class order every downstream consumer (validator,
57
+ # BoundingBox.cls_id) sees β€” MUST equal element.objects order:
58
+ self.class_names = ["person"]
59
+ # Fallback model-emit order (used only when ONNX metadata is missing).
60
+ self._model_class_order = ["person"]
61
+ print("ORT version:", ort.__version__)
62
+
63
+ try:
64
+ ort.preload_dlls()
65
+ print("onnxruntime.preload_dlls() success")
66
+ except Exception as e:
67
+ print(f"preload_dlls failed: {e}")
68
+
69
+ print("ORT available providers BEFORE session:", ort.get_available_providers())
70
+
71
+ sess_options = ort.SessionOptions()
72
+ sess_options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
73
+ # Pin threads for the CPU latency gate.
74
+ sess_options.intra_op_num_threads = 2
75
+ sess_options.inter_op_num_threads = 1
76
+ sess_options.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL
77
+
78
+ # MINER_FORCE_CPU=1 skips the CUDA attempt β€” used by bench_latency.py to
79
+ # measure the conformity-gate path (this box has onnxruntime-gpu too, so
80
+ # an unforced "CPU" benchmark silently runs on the GPU).
81
+ force_cpu = os.environ.get("MINER_FORCE_CPU", "") == "1"
82
+ try:
83
+ if force_cpu:
84
+ raise RuntimeError("MINER_FORCE_CPU=1")
85
+ self.session = ort.InferenceSession(
86
+ str(model_path),
87
+ sess_options=sess_options,
88
+ providers=["CUDAExecutionProvider", "CPUExecutionProvider"],
89
+ )
90
+ print("Created ORT session with preferred CUDA provider list")
91
+ except Exception as e:
92
+ print(f"CUDA session creation skipped/failed, using CPU: {e}")
93
+ self.session = ort.InferenceSession(
94
+ str(model_path),
95
+ sess_options=sess_options,
96
+ providers=["CPUExecutionProvider"],
97
+ )
98
+ print("ORT session providers:", self.session.get_providers())
99
+
100
+ # cls_remap[i] = self.class_names.index(model_class_order[i]); order comes
101
+ # from ONNX metadata when present, else the static fallback.
102
+ model_class_order = self._read_model_class_order()
103
+ if model_class_order is None:
104
+ model_class_order = list(self._model_class_order)
105
+ print(f"cls order: no usable ONNX metadata, FALLBACK {model_class_order}")
106
+ else:
107
+ print(f"cls order: from ONNX metadata {model_class_order}")
108
+ self.cls_remap = np.array(
109
+ [self.class_names.index(n) for n in model_class_order], dtype=np.int32
110
+ )
111
+
112
+ for inp in self.session.get_inputs():
113
+ print("INPUT:", inp.name, inp.shape, inp.type)
114
+ for out in self.session.get_outputs():
115
+ print("OUTPUT:", out.name, out.shape, out.type)
116
+
117
+ self.input_name = self.session.get_inputs()[0].name
118
+ self.output_names = [o.name for o in self.session.get_outputs()]
119
+ self.input_shape = self.session.get_inputs()[0].shape
120
+
121
+ # Match the ONNX input dtype (FP16 export needs float16 input).
122
+ input_type = self.session.get_inputs()[0].type
123
+ self.np_dtype = np.float16 if "float16" in input_type else np.float32
124
+ print(f"ONNX input dtype: {input_type} -> numpy {self.np_dtype}")
125
+
126
+ # The miner MUST run at the ONNX's baked input size (704). Dropping to
127
+ # 640 is a latency lever but an EXPORT decision (re-export + re-eval),
128
+ # not a miner-side change.
129
+ self.input_height = self._safe_dim(self.input_shape[2], default=704)
130
+ self.input_width = self._safe_dim(self.input_shape[3], default=704)
131
+
132
+ # ── Post-processing config (tune on person1-powerful/val) ─────────────
133
+ # Per-class NMS IoU ([person]).
134
+ self._iou_thres_array = np.array([0.45], dtype=np.float32)
135
+ self.cross_iou_thresh = 0.9 # near-duplicate guard (no-op across classes
136
+ # with one class; catches post-NMS dupes)
137
+ self.max_det = 300 # person frames can be busy
138
+ self.pre_nms_topk = 1000 # cap candidates before NMS (crowded-frame speed guard)
139
+ self.use_tta = False # 2nd flipped pass; doubles latency -> off for CPU gate
140
+
141
+ # Single-view cluster boost: raise each survivor's conf to its
142
+ # IoU-cluster max. OFF per the 2026-09-04 person sweep (carwash: on,
143
+ # vehicle: off -- element-specific, re-sweep on new weights).
144
+ self.use_cluster_boost = False
145
+
146
+ # Soft-NMS: Gaussian decay of overlapping scores instead of hard
147
+ # deletion, then RE-FILTER decayed scores against the confidence
148
+ # threshold (keeps decayed junk away from the FP pillar).
149
+ # OFF per the 2026-09-04 person sweep (off 0.5670 vs on 0.5662) --
150
+ # element- and weights-specific, re-sweep after every retrain.
151
+ self.use_soft_nms = True
152
+ self.soft_nms_sigma = 0.4
153
+
154
+ # Confidence threshold ([person]). `_conf_thres_array` is the SQUARE /
155
+ # validator-eval set (every scored frame is 1024x1024 squished, so this
156
+ # is the one that matters); `_extra` covers non-square inputs (warmup /
157
+ # any 16:9 frame). Tuned 2026-09-04 (tune_miner.py, 1853-img
158
+ # person1-powerful/val, adaptive-IoU matcher): comp 0.5385 -> 0.5670.
159
+ # NOTE: the val is dense (29 GT/frame avg) and recall is the binding
160
+ # constraint. A low-edge extension sweep found a plateau below the
161
+ # floor (0.05-0.09 -> comp 0.5713, +0.004) but on REAL challenge
162
+ # frames the 0.05-0.12 band adds ~25 boxes/frame -- too FP-risky to
163
+ # ship on dense-val evidence alone. Decide with test_miner.py + SAM3
164
+ # GT on real frames; retune after every retrain (especially once the
165
+ # sparse regenerated data lands).
166
+ self._conf_thres_array = np.array([0.3], dtype=np.float32)
167
+ self._extra_conf_thres_array = np.array([0.30], dtype=np.float32)
168
+
169
+ # Rescue bonus: if NO box passes the threshold, admit the top-1 when
170
+ # score >= (threshold - bonus). ON per the 2026-09-04 sweep.
171
+ self._bonus_array = np.full(1, 0.03, dtype=np.float32)
172
+
173
+ # Min box area (px^2), loose: distant people are tiny in 1024x1024
174
+ # frames and tiny-GT matches only need IoU 0.30 under the adaptive
175
+ # matcher β€” do not silently drop them.
176
+ self._min_box_area_array = np.array([12.0], dtype=np.float32)
177
+ self.min_side = 3
178
+ self.max_aspect_ratio = 12.0
179
+ self.max_area_frac = 0.95
180
+
181
+ # Density conf floors applied on the FINAL box list (after NMS).
182
+ # Count is taken once; boxes below the matching floor are dropped.
183
+ # n > 70 -> drop conf < 0.40
184
+ # 50 < n <= 70 -> drop conf < 0.36
185
+ # 30 < n <= 50 -> drop conf < 0.34 (n==50 uses this band)
186
+ # n <= 30 -> keep existing conf threshold
187
+ self._density_conf_floors = (
188
+ (70, 0.40),
189
+ (50, 0.36),
190
+ (30, 0.34),
191
+ )
192
+
193
+ # Square-domain de-squish: widen a square (validator) frame by this
194
+ # fraction before letterboxing, then drop pad-center boxes and un-pad.
195
+ # 0.0 = off β€” the model was TRAINED on squished 1024 frames, so the
196
+ # domains already match; enabling changes aspect -> validate first.
197
+ self.square_pad_frac = 0.0
198
+
199
+ self._avg_iou = float(np.mean(self._iou_thres_array))
200
+ print(f"ONNX model loaded from: {model_path}")
201
+ print(f"ONNX input: name={self.input_name}, shape={self.input_shape}")
202
+ self._warmup()
203
+
204
+ # ── Setup helpers ────────────────────────────────────────────────────────
205
+
206
+ def _warmup(self, iters: int = 3) -> None:
207
+ try:
208
+ dummy = np.zeros((1024, 1024, 3), dtype=np.uint8) # validator domain
209
+ for _ in range(max(1, iters)):
210
+ self.predict_batch(batch_images=[dummy], offset=0, n_keypoints=0)
211
+ print(f"warmup: {iters} dummy predict_batch call(s) done")
212
+ except Exception as e:
213
+ print(f"warmup skipped: {e}")
214
+
215
+ def __repr__(self) -> str:
216
+ return f"PersonMiner(classes={len(self.class_names)}, providers={self.session.get_providers()})"
217
+
218
+ @staticmethod
219
+ def _safe_dim(value, default: int) -> int:
220
+ return value if isinstance(value, int) and value > 0 else default
221
+
222
+ @staticmethod
223
+ def _resolve_model_path(repo: Path) -> Path:
224
+ """Prefer weights.onnx, then weights_int8.onnx, then any .onnx."""
225
+ for name in ("weights.onnx", "weights_int8.onnx"):
226
+ p = repo / name
227
+ if p.exists():
228
+ if name != "weights.onnx":
229
+ print(f"model: weights.onnx not found, using {name}")
230
+ return p
231
+ candidates = sorted(repo.glob("*.onnx"))
232
+ if candidates:
233
+ print(f"model: using {candidates[0].name}")
234
+ return candidates[0]
235
+ return repo / "weights.onnx" # let session creation raise the error
236
+
237
+ def _read_model_class_order(self) -> list[str] | None:
238
+ """Read class order from Ultralytics ONNX `names` metadata, or None."""
239
+ try:
240
+ import ast
241
+ meta = self.session.get_modelmeta().custom_metadata_map
242
+ names = ast.literal_eval(meta["names"])
243
+ order = ([str(names[i]) for i in sorted(names)] if isinstance(names, dict)
244
+ else [str(n) for n in names])
245
+ except Exception as e:
246
+ print(f"cls order: could not read ONNX names metadata ({e})")
247
+ return None
248
+ if sorted(order) != sorted(self.class_names):
249
+ print(f"cls order: ONNX names {order} != expected {self.class_names}; ignoring")
250
+ return None
251
+ return order
252
+
253
+ # ── Preprocessing ────────────────────────────────────────────────────────
254
+
255
+ def _letterbox(self, image: ndarray, new_shape: tuple[int, int],
256
+ color=(114, 114, 114)) -> tuple[ndarray, float, tuple[float, float]]:
257
+ h, w = image.shape[:2]
258
+ new_w, new_h = new_shape
259
+ ratio = min(new_w / w, new_h / h)
260
+ rw, rh = int(round(w * ratio)), int(round(h * ratio))
261
+ if (rw, rh) != (w, h):
262
+ interp = cv2.INTER_CUBIC if ratio > 1.0 else cv2.INTER_LINEAR
263
+ image = cv2.resize(image, (rw, rh), interpolation=interp)
264
+ dw = (new_w - rw) / 2.0
265
+ dh = (new_h - rh) / 2.0
266
+ left, right = int(round(dw - 0.1)), int(round(dw + 0.1))
267
+ top, bottom = int(round(dh - 0.1)), int(round(dh + 0.1))
268
+ padded = cv2.copyMakeBorder(image, top, bottom, left, right,
269
+ cv2.BORDER_CONSTANT, value=color)
270
+ return padded, ratio, (dw, dh)
271
+
272
+ def _preprocess(self, image: ndarray) -> tuple[np.ndarray, dict]:
273
+ """Letterbox to the ONNX input size; NO CLAHE/denoise/sharpen (any
274
+ enhancement the model wasn't trained on is a train/test mismatch that
275
+ HURTS accuracy and also risks the CPU latency gate).
276
+
277
+ Square-domain de-squish: when the frame is square (the validator's
278
+ 1024x1024 squished eval image) and square_pad_frac > 0, widen it with
279
+ gray bars first -- objects come out horizontally compressed in those
280
+ frames, and mild widening partly restores their aspect.
281
+ """
282
+ orig_h, orig_w = image.shape[:2]
283
+ extra_left = extra_right = 0
284
+ if self.square_pad_frac > 0.0 and orig_w == orig_h:
285
+ target_w = int(orig_w * (1.0 + self.square_pad_frac))
286
+ if target_w > orig_w:
287
+ total = target_w - orig_w
288
+ extra_left = total // 2
289
+ extra_right = total - extra_left
290
+ image = cv2.copyMakeBorder(image, 0, 0, extra_left, extra_right,
291
+ cv2.BORDER_CONSTANT, value=(114, 114, 114))
292
+
293
+ img, ratio, pad = self._letterbox(image, (self.input_width, self.input_height))
294
+ img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
295
+ img = img.astype(np.float32) / 255.0
296
+ img = np.transpose(img, (2, 0, 1))[None, ...]
297
+ img = np.ascontiguousarray(img, dtype=self.np_dtype)
298
+ return img, {
299
+ "ratio": ratio, "pad": pad, "orig_size": (orig_w, orig_h),
300
+ "extra_left": extra_left, "extra_right": extra_right,
301
+ }
302
+
303
+ # ── Vectorized box ops ───────────────────────────────────────────────────
304
+
305
+ @staticmethod
306
+ def _clip_boxes(boxes: np.ndarray, image_size: tuple[int, int]) -> np.ndarray:
307
+ w, h = image_size
308
+ boxes[:, 0] = np.clip(boxes[:, 0], 0, w - 1)
309
+ boxes[:, 1] = np.clip(boxes[:, 1], 0, h - 1)
310
+ boxes[:, 2] = np.clip(boxes[:, 2], 0, w - 1)
311
+ boxes[:, 3] = np.clip(boxes[:, 3], 0, h - 1)
312
+ return boxes
313
+
314
+ @staticmethod
315
+ def _xywh_to_xyxy(boxes: np.ndarray) -> np.ndarray:
316
+ out = np.empty_like(boxes)
317
+ out[:, 0] = boxes[:, 0] - boxes[:, 2] / 2.0
318
+ out[:, 1] = boxes[:, 1] - boxes[:, 3] / 2.0
319
+ out[:, 2] = boxes[:, 0] + boxes[:, 2] / 2.0
320
+ out[:, 3] = boxes[:, 1] + boxes[:, 3] / 2.0
321
+ return out
322
+
323
+ @staticmethod
324
+ def _hard_nms(boxes: np.ndarray, scores: np.ndarray, iou_thresh: float) -> np.ndarray:
325
+ """Vectorized greedy NMS: areas precomputed once, each step is a single
326
+ numpy IoU vector -- no inner Python loop."""
327
+ n = len(boxes)
328
+ if n == 0:
329
+ return np.array([], dtype=np.intp)
330
+ x1, y1, x2, y2 = boxes[:, 0], boxes[:, 1], boxes[:, 2], boxes[:, 3]
331
+ areas = np.maximum(0.0, x2 - x1) * np.maximum(0.0, y2 - y1)
332
+ order = np.argsort(-scores)
333
+ keep = []
334
+ while order.size > 0:
335
+ i = int(order[0])
336
+ keep.append(i)
337
+ if order.size == 1:
338
+ break
339
+ rest = order[1:]
340
+ xx1 = np.maximum(x1[i], x1[rest])
341
+ yy1 = np.maximum(y1[i], y1[rest])
342
+ xx2 = np.minimum(x2[i], x2[rest])
343
+ yy2 = np.minimum(y2[i], y2[rest])
344
+ inter = np.maximum(0.0, xx2 - xx1) * np.maximum(0.0, yy2 - yy1)
345
+ iou = inter / (areas[i] + areas[rest] - inter + 1e-7)
346
+ order = rest[iou <= iou_thresh]
347
+ return np.array(keep, dtype=np.intp)
348
+
349
+ @staticmethod
350
+ def _soft_nms(boxes: np.ndarray, scores: np.ndarray, sigma: float
351
+ ) -> tuple[np.ndarray, np.ndarray]:
352
+ """Gaussian soft-NMS: decay overlapping scores instead of deleting.
353
+ Returns (original indices in decay order, decayed scores)."""
354
+ n = len(boxes)
355
+ if n == 0:
356
+ return np.array([], dtype=np.intp), np.array([], dtype=np.float32)
357
+ b = boxes.astype(np.float32, copy=True)
358
+ sc = scores.astype(np.float32, copy=True)
359
+ order = np.arange(n)
360
+ for i in range(n):
361
+ mp = i + int(np.argmax(sc[i:]))
362
+ b[[i, mp]] = b[[mp, i]]; sc[[i, mp]] = sc[[mp, i]]
363
+ order[[i, mp]] = order[[mp, i]]
364
+ if i + 1 >= n:
365
+ break
366
+ xx1 = np.maximum(b[i, 0], b[i+1:, 0]); yy1 = np.maximum(b[i, 1], b[i+1:, 1])
367
+ xx2 = np.minimum(b[i, 2], b[i+1:, 2]); yy2 = np.minimum(b[i, 3], b[i+1:, 3])
368
+ inter = np.maximum(0.0, xx2 - xx1) * np.maximum(0.0, yy2 - yy1)
369
+ ai = max(0.0, float((b[i, 2]-b[i, 0]) * (b[i, 3]-b[i, 1])))
370
+ aj = np.maximum(0.0, b[i+1:, 2]-b[i+1:, 0]) * np.maximum(0.0, b[i+1:, 3]-b[i+1:, 1])
371
+ iou = inter / (ai + aj - inter + 1e-7)
372
+ sc[i+1:] *= np.exp(-(iou ** 2) / sigma)
373
+ return order, sc
374
+
375
+ def _per_class_soft_nms(self, boxes: np.ndarray, scores: np.ndarray,
376
+ cls_ids: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
377
+ """Soft-NMS per class, then re-filter decayed scores against the
378
+ per-class confidence thresholds so no decayed box reaches the FP pillar
379
+ below its class bar."""
380
+ keep_idx: list[int] = []
381
+ keep_sc: list[float] = []
382
+ for c in np.unique(cls_ids):
383
+ idx = np.where(cls_ids == c)[0]
384
+ order, dec = self._soft_nms(boxes[idx], scores[idx], self.soft_nms_sigma)
385
+ thr = float(self._conf_thres_array[c])
386
+ for k, s_dec in zip(order, dec):
387
+ if s_dec >= thr:
388
+ keep_idx.append(int(idx[k])); keep_sc.append(float(s_dec))
389
+ return np.array(keep_idx, dtype=np.intp), np.array(keep_sc, dtype=np.float32)
390
+
391
+ def _per_class_hard_nms(self, boxes: np.ndarray, scores: np.ndarray,
392
+ cls_ids: np.ndarray) -> np.ndarray:
393
+ """Per-class NMS using the per-class IoU thresholds."""
394
+ if len(boxes) == 0:
395
+ return np.array([], dtype=np.intp)
396
+ all_keep: list[int] = []
397
+ for c in np.unique(cls_ids):
398
+ idx = np.where(cls_ids == c)[0]
399
+ cls_iou = float(self._iou_thres_array[c])
400
+ keep = self._hard_nms(boxes[idx], scores[idx], cls_iou)
401
+ all_keep.extend(idx[keep].tolist())
402
+ all_keep.sort()
403
+ return np.array(all_keep, dtype=np.intp)
404
+
405
+ @staticmethod
406
+ def _max_score_per_cluster(post_boxes: np.ndarray, post_cls: np.ndarray,
407
+ full_boxes: np.ndarray, full_scores: np.ndarray,
408
+ full_cls: np.ndarray, iou_thresh: float) -> np.ndarray:
409
+ """Each survivor's confidence -> max score in its SAME-CLASS IoU cluster.
410
+ Vectorized single (n_post x n_full) IoU matrix."""
411
+ n = len(post_boxes)
412
+ if n == 0:
413
+ return np.empty(0, dtype=np.float32)
414
+ m = len(full_boxes)
415
+ if m == 0:
416
+ return np.zeros(n, dtype=np.float32)
417
+ pa = (np.maximum(0.0, post_boxes[:, 2] - post_boxes[:, 0]) *
418
+ np.maximum(0.0, post_boxes[:, 3] - post_boxes[:, 1]))
419
+ fa = (np.maximum(0.0, full_boxes[:, 2] - full_boxes[:, 0]) *
420
+ np.maximum(0.0, full_boxes[:, 3] - full_boxes[:, 1]))
421
+ xx1 = np.maximum(post_boxes[:, 0][:, None], full_boxes[:, 0][None, :])
422
+ yy1 = np.maximum(post_boxes[:, 1][:, None], full_boxes[:, 1][None, :])
423
+ xx2 = np.minimum(post_boxes[:, 2][:, None], full_boxes[:, 2][None, :])
424
+ yy2 = np.minimum(post_boxes[:, 3][:, None], full_boxes[:, 3][None, :])
425
+ inter = np.maximum(0.0, xx2 - xx1) * np.maximum(0.0, yy2 - yy1)
426
+ iou = inter / (pa[:, None] + fa[None, :] - inter + 1e-7)
427
+ mask = (iou >= iou_thresh) & (post_cls[:, None] == full_cls[None, :])
428
+ tiled = np.where(mask, full_scores[None, :], -np.inf)
429
+ out = tiled.max(axis=1)
430
+ out[~np.isfinite(out)] = 0.0
431
+ return out.astype(np.float32)
432
+
433
+ def _conf_filter_mask(self, scores: np.ndarray, cls_ids: np.ndarray,
434
+ extra_left: int) -> np.ndarray:
435
+ """Per-class threshold (square vs non-square set) + per-class rescue."""
436
+ if len(scores) == 0:
437
+ return np.zeros(0, dtype=bool)
438
+ thr_arr = self._extra_conf_thres_array if extra_left > 0 else self._conf_thres_array
439
+ keep = scores >= thr_arr[cls_ids]
440
+ for c in np.unique(cls_ids):
441
+ b = float(self._bonus_array[c])
442
+ if b <= 0.0:
443
+ continue
444
+ cm = cls_ids == c
445
+ if keep[cm].any():
446
+ continue
447
+ idx = np.where(cm)[0]
448
+ top = int(idx[int(np.argmax(scores[idx]))])
449
+ if scores[top] >= float(self._conf_thres_array[c]) - b:
450
+ keep[top] = True
451
+ return keep
452
+
453
+ def _filter_sane_boxes(self, boxes: np.ndarray, scores: np.ndarray,
454
+ cls_ids: np.ndarray, orig_size: tuple[int, int]
455
+ ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
456
+ """Vectorized per-class min-area / max-area / min-side / aspect filter."""
457
+ if len(boxes) == 0:
458
+ return boxes, scores, cls_ids
459
+ orig_w, orig_h = orig_size
460
+ image_area = float(orig_w * orig_h)
461
+ bw = np.maximum(0.0, boxes[:, 2] - boxes[:, 0])
462
+ bh = np.maximum(0.0, boxes[:, 3] - boxes[:, 1])
463
+ area = bw * bh
464
+ ar = np.maximum(bw / np.maximum(bh, 1e-6), bh / np.maximum(bw, 1e-6))
465
+ keep = (
466
+ (bw >= self.min_side) & (bh >= self.min_side) &
467
+ (area >= self._min_box_area_array[cls_ids]) &
468
+ (area <= self.max_area_frac * image_area) &
469
+ (ar <= self.max_aspect_ratio)
470
+ )
471
+ return boxes[keep], scores[keep], cls_ids[keep]
472
+
473
+ def _cross_class_dedup_op(self, boxes: np.ndarray, scores: np.ndarray,
474
+ cls_ids: np.ndarray, iou_thresh: float
475
+ ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
476
+ """Suppress near-duplicate boxes at IoU>thresh regardless of class.
477
+ With a single class this only catches rare post-NMS duplicates."""
478
+ n = len(boxes)
479
+ if n <= 1:
480
+ return boxes, scores, cls_ids
481
+ boxes = np.asarray(boxes, dtype=np.float32)
482
+ scores = np.asarray(scores, dtype=np.float32)
483
+ cls_ids = np.asarray(cls_ids, dtype=np.int32)
484
+ areas = (np.maximum(0.0, boxes[:, 2] - boxes[:, 0]) *
485
+ np.maximum(0.0, boxes[:, 3] - boxes[:, 1]))
486
+ margins = scores - self._conf_thres_array[cls_ids]
487
+ order = np.lexsort((-areas, -margins))
488
+ suppressed = np.zeros(n, dtype=bool)
489
+ keep: list[int] = []
490
+ for i in order:
491
+ if suppressed[i]:
492
+ continue
493
+ keep.append(int(i))
494
+ bi = boxes[i]
495
+ xx1 = np.maximum(bi[0], boxes[:, 0])
496
+ yy1 = np.maximum(bi[1], boxes[:, 1])
497
+ xx2 = np.minimum(bi[2], boxes[:, 2])
498
+ yy2 = np.minimum(bi[3], boxes[:, 3])
499
+ inter = np.maximum(0.0, xx2 - xx1) * np.maximum(0.0, yy2 - yy1)
500
+ a_i = max(1e-7, float((bi[2] - bi[0]) * (bi[3] - bi[1])))
501
+ iou = inter / (a_i + areas - inter + 1e-7)
502
+ dup = iou > iou_thresh
503
+ dup[i] = False
504
+ suppressed |= dup
505
+ keep_idx = np.array(keep, dtype=np.intp)
506
+ return boxes[keep_idx], scores[keep_idx], cls_ids[keep_idx]
507
+
508
+ def _per_view_pipeline(self, boxes: np.ndarray, scores: np.ndarray,
509
+ cls_ids: np.ndarray, orig_size: tuple[int, int]
510
+ ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
511
+ """sane filter -> top-k cap -> per-class NMS -> max_det cap -> dedup."""
512
+ boxes, scores, cls_ids = self._filter_sane_boxes(boxes, scores, cls_ids, orig_size)
513
+ if len(boxes) == 0:
514
+ return boxes, scores, cls_ids
515
+ if len(scores) > self.pre_nms_topk:
516
+ top = np.argpartition(-scores, self.pre_nms_topk)[: self.pre_nms_topk]
517
+ boxes, scores, cls_ids = boxes[top], scores[top], cls_ids[top]
518
+ if len(boxes) > 1:
519
+ if self.use_soft_nms:
520
+ keep, scores = self._per_class_soft_nms(boxes, scores, cls_ids)
521
+ boxes, cls_ids = boxes[keep], cls_ids[keep]
522
+ else:
523
+ keep = self._per_class_hard_nms(boxes, scores, cls_ids)
524
+ boxes, scores, cls_ids = boxes[keep], scores[keep], cls_ids[keep]
525
+ if len(scores) > self.max_det:
526
+ top = np.argsort(-scores)[: self.max_det]
527
+ boxes, scores, cls_ids = boxes[top], scores[top], cls_ids[top]
528
+ if len(boxes) > 1:
529
+ boxes, scores, cls_ids = self._cross_class_dedup_op(
530
+ boxes, scores, cls_ids, self.cross_iou_thresh)
531
+ return boxes, scores, cls_ids
532
+
533
+ # ── Coordinate un-mapping (shared by both decode paths) ──────────────────
534
+
535
+ def _unmap_and_finish(self, boxes: np.ndarray, scores: np.ndarray,
536
+ cls_ids: np.ndarray, meta: dict
537
+ ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
538
+ """Undo letterbox + square-pad, drop pad-center boxes, clip, post-process."""
539
+ pad_w, pad_h = meta["pad"]
540
+ ratio = meta["ratio"]
541
+ orig_w, orig_h = meta["orig_size"]
542
+ extra_left, extra_right = meta["extra_left"], meta["extra_right"]
543
+
544
+ boxes[:, [0, 2]] -= pad_w
545
+ boxes[:, [1, 3]] -= pad_h
546
+ boxes /= ratio
547
+ if extra_left:
548
+ boxes[:, [0, 2]] -= extra_left
549
+ if extra_left or extra_right:
550
+ cx = (boxes[:, 0] + boxes[:, 2]) * 0.5
551
+ inside = (cx >= 0) & (cx <= orig_w)
552
+ boxes, scores, cls_ids = boxes[inside], scores[inside], cls_ids[inside]
553
+ if len(boxes) == 0:
554
+ return boxes, scores, cls_ids
555
+ boxes = self._clip_boxes(boxes, (orig_w, orig_h))
556
+ return self._per_view_pipeline(boxes, scores, cls_ids, (orig_w, orig_h))
557
+
558
+ # ── Decoding ─────────────────────────────────────────────────────────────
559
+
560
+ def _decode_final_dets(self, preds: np.ndarray, meta: dict
561
+ ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
562
+ """End2end output rows [x1, y1, x2, y2, conf, cls_id] (NMS in graph)."""
563
+ empty = (np.empty((0, 4), np.float32), np.empty(0, np.float32), np.empty(0, np.int32))
564
+ if preds.ndim == 3 and preds.shape[0] == 1:
565
+ preds = preds[0]
566
+ if preds.ndim != 2 or preds.shape[1] < 6:
567
+ raise ValueError(f"Unexpected final-det output shape: {preds.shape}")
568
+ boxes = preds[:, :4].astype(np.float32)
569
+ scores = preds[:, 4].astype(np.float32)
570
+ cls_ids = self.cls_remap[preds[:, 5].astype(np.int32)]
571
+ keep = self._conf_filter_mask(scores, cls_ids, meta["extra_left"])
572
+ boxes, scores, cls_ids = boxes[keep], scores[keep], cls_ids[keep]
573
+ if len(boxes) == 0:
574
+ return empty
575
+ return self._unmap_and_finish(boxes, scores, cls_ids, meta)
576
+
577
+ def _decode_raw_yolo(self, preds: np.ndarray, meta: dict
578
+ ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
579
+ """Raw YOLO output [1,C,N] or [1,N,C]; xywh + per-class scores."""
580
+ empty = (np.empty((0, 4), np.float32), np.empty(0, np.float32), np.empty(0, np.int32))
581
+ if preds.ndim != 3 or preds.shape[0] != 1:
582
+ raise ValueError(f"Unexpected raw output shape: {preds.shape}")
583
+ preds = preds[0]
584
+ if preds.shape[0] <= 16 and preds.shape[1] > preds.shape[0]:
585
+ preds = preds.T
586
+ if preds.ndim != 2 or preds.shape[1] < 5:
587
+ raise ValueError(f"Unexpected normalized raw shape: {preds.shape}")
588
+ boxes_xywh = preds[:, :4].astype(np.float32)
589
+ cls_part = preds[:, 4:].astype(np.float32)
590
+ if cls_part.shape[1] == 1:
591
+ scores = cls_part[:, 0]
592
+ cls_ids = np.zeros(len(scores), dtype=np.int32)
593
+ else:
594
+ cls_ids = np.argmax(cls_part, axis=1).astype(np.int32)
595
+ scores = cls_part[np.arange(len(cls_part)), cls_ids]
596
+ cls_ids = self.cls_remap[cls_ids]
597
+ keep = self._conf_filter_mask(scores, cls_ids, meta["extra_left"])
598
+ boxes_xywh, scores, cls_ids = boxes_xywh[keep], scores[keep], cls_ids[keep]
599
+ if len(boxes_xywh) == 0:
600
+ return empty
601
+ boxes = self._xywh_to_xyxy(boxes_xywh)
602
+ return self._unmap_and_finish(boxes, scores, cls_ids, meta)
603
+
604
+ def _decode(self, output: np.ndarray, meta: dict
605
+ ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
606
+ if output.ndim == 2 and output.shape[1] >= 6:
607
+ return self._decode_final_dets(output, meta)
608
+ if output.ndim == 3 and output.shape[0] == 1 and output.shape[2] == 6:
609
+ return self._decode_final_dets(output, meta)
610
+ return self._decode_raw_yolo(output, meta)
611
+
612
+ # ── Inference ────────────────────────────────────────────────────────────
613
+
614
+ def _predict_single_arrays(self, image: np.ndarray
615
+ ) -> tuple[np.ndarray, np.ndarray, np.ndarray, dict]:
616
+ if not isinstance(image, np.ndarray) or image.ndim != 3 or image.shape[2] != 3:
617
+ raise ValueError(f"Expected HWC BGR image, got {type(image)} {getattr(image,'shape',None)}")
618
+ if image.dtype != np.uint8:
619
+ image = image.astype(np.uint8)
620
+ inp, meta = self._preprocess(image)
621
+ outputs = self.session.run(self.output_names, {self.input_name: inp})
622
+ boxes, scores, cls_ids = self._decode(outputs[0], meta)
623
+ return boxes, scores, cls_ids, meta
624
+
625
+ def _density_conf_filter(self, boxes: np.ndarray, scores: np.ndarray,
626
+ cls_ids: np.ndarray
627
+ ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
628
+ """Raise the conf floor when a frame is crowded; one-shot on current count."""
629
+ n = len(scores)
630
+ floor = 0.0
631
+ for count_gt, conf_floor in self._density_conf_floors:
632
+ if n > count_gt:
633
+ floor = conf_floor
634
+ break
635
+ if floor <= 0.0:
636
+ return boxes, scores, cls_ids
637
+ keep = scores >= floor
638
+ return boxes[keep], scores[keep], cls_ids[keep]
639
+
640
+ @staticmethod
641
+ def _to_boxes(boxes: np.ndarray, scores: np.ndarray, cls_ids: np.ndarray,
642
+ orig_size: tuple[int, int]) -> list[BoundingBox]:
643
+ results: list[BoundingBox] = []
644
+ orig_w, orig_h = orig_size
645
+ for box, conf, cls_id in zip(boxes, scores, cls_ids):
646
+ x1, y1, x2, y2 = box.tolist()
647
+ if x2 <= x1 or y2 <= y1:
648
+ continue
649
+ results.append(BoundingBox(
650
+ x1=max(0, min(orig_w, int(math.floor(x1)))),
651
+ y1=max(0, min(orig_h, int(math.floor(y1)))),
652
+ x2=max(0, min(orig_w, int(math.ceil(x2)))),
653
+ y2=max(0, min(orig_h, int(math.ceil(y2)))),
654
+ cls_id=int(cls_id),
655
+ conf=float(max(0.0, min(1.0, conf))),
656
+ ))
657
+ return results
658
+
659
+ def _infer_single(self, image: np.ndarray) -> list[BoundingBox]:
660
+ boxes, scores, cls_ids, meta = self._predict_single_arrays(image)
661
+ if len(boxes) == 0:
662
+ return []
663
+ if self.use_cluster_boost and len(boxes) > 1:
664
+ scores = self._max_score_per_cluster(
665
+ boxes, cls_ids, boxes, scores, cls_ids, self._avg_iou)
666
+ boxes, scores, cls_ids = self._density_conf_filter(boxes, scores, cls_ids)
667
+ return self._to_boxes(boxes, scores, cls_ids, meta["orig_size"])
668
+
669
+ def _infer_tta(self, image: np.ndarray) -> list[BoundingBox]:
670
+ """Horizontal-flip TTA: union of original + flipped, per-class NMS,
671
+ same-class cluster-max boost, dedup."""
672
+ b0, s0, c0, meta = self._predict_single_arrays(image)
673
+ w = image.shape[1]
674
+ bf, sf, cf, _ = self._predict_single_arrays(cv2.flip(image, 1))
675
+ if len(bf):
676
+ bf = bf.copy()
677
+ bf[:, [0, 2]] = w - bf[:, [2, 0]] # mirror x back to original coords
678
+ coords = np.concatenate([b0, bf], axis=0) if len(bf) else b0
679
+ scores = np.concatenate([s0, sf], axis=0) if len(sf) else s0
680
+ cls_ids = np.concatenate([c0, cf], axis=0) if len(cf) else c0
681
+ if len(coords) == 0:
682
+ return []
683
+ keep = self._per_class_hard_nms(coords, scores, cls_ids)
684
+ if len(keep) == 0:
685
+ return []
686
+ if len(keep) > self.max_det:
687
+ keep = keep[np.argsort(-scores[keep])[: self.max_det]]
688
+ boosted = self._max_score_per_cluster(
689
+ coords[keep], cls_ids[keep], coords, scores, cls_ids, self._avg_iou)
690
+ kb, kc = coords[keep], cls_ids[keep]
691
+ if len(kb) > 1:
692
+ kb, boosted, kc = self._cross_class_dedup_op(kb, boosted, kc, self.cross_iou_thresh)
693
+ kb, boosted, kc = self._density_conf_filter(kb, boosted, kc)
694
+ return self._to_boxes(kb, boosted, kc, meta["orig_size"])
695
+
696
+ def predict_batch(self, batch_images: list[ndarray], offset: int,
697
+ n_keypoints: int) -> list[TVFrameResult]:
698
+ results: list[TVFrameResult] = []
699
+ for idx, image in enumerate(batch_images):
700
+ try:
701
+ boxes = self._infer_tta(image) if self.use_tta else self._infer_single(image)
702
+ except Exception as e:
703
+ print(f"Inference failed for frame {offset + idx}: {e}")
704
+ boxes = []
705
+ keypoints = [(0, 0) for _ in range(max(0, int(n_keypoints)))]
706
+ results.append(TVFrameResult(frame_id=offset + idx, boxes=boxes, keypoints=keypoints))
707
+ return results
weights.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:abfe40d544c6c923f85b7294c4f0f16ea4b42af71890368b9132f0f8489d69ee
3
+ size 9840060