happydev93 commited on
Commit
8acd5fc
Β·
verified Β·
1 Parent(s): cc776a9

deploy_chutes: upload project folder

Browse files
Files changed (3) hide show
  1. chute_config.yml +21 -0
  2. miner.py +647 -0
  3. weights.onnx +3 -0
chute_config.yml ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Image:
2
+ from_base: parachutes/python:3.12
3
+ run_command:
4
+ - pip install --upgrade setuptools wheel
5
+ - pip install 'numpy>=1.23' 'onnxruntime-gpu>=1.16' 'opencv-python>=4.7' 'pillow>=9.5' 'huggingface_hub>=0.19.4' 'pydantic>=2.0' 'pyyaml>=6.0' 'aiohttp>=3.9'
6
+ - pip install torch==2.8.0 torchvision==0.23.0 torchaudio==2.8.0 --index-url https://download.pytorch.org/whl/cu128
7
+
8
+ NodeSelector:
9
+ gpu_count: 1
10
+ min_vram_gb_per_gpu: 16
11
+ max_hourly_price_per_gpu: 2
12
+ include:
13
+ - pro_6000
14
+
15
+ Chute:
16
+ timeout_seconds: 900
17
+ concurrency: 4
18
+ max_instances: 5
19
+ scaling_threshold: 0.5
20
+ shutdown_after_seconds: 288000
21
+ tee: true
miner.py ADDED
@@ -0,0 +1,647 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import math
3
+ from pathlib import Path
4
+
5
+ import cv2
6
+ import numpy as np
7
+ import onnxruntime as ort
8
+ from numpy import ndarray
9
+ from pydantic import BaseModel
10
+
11
+
12
+ # =============================================================================
13
+ # Detection miner: end-to-end ONNX, letterboxed input, per-class confidence
14
+ # thresholds and NMS.
15
+ # Class order is read from the model when present, else from the static order
16
+ # below; the two must agree.
17
+ # =============================================================================
18
+
19
+
20
+ class BoundingBox(BaseModel):
21
+ x1: int
22
+ y1: int
23
+ x2: int
24
+ y2: int
25
+ cls_id: int
26
+ conf: float
27
+
28
+
29
+ class TVFrameResult(BaseModel):
30
+ frame_id: int
31
+ boxes: list[BoundingBox]
32
+ keypoints: list[tuple[int, int]]
33
+
34
+
35
+ class Miner:
36
+ def __init__(self, path_hf_repo: Path) -> None:
37
+ model_path = self._resolve_model_path(path_hf_repo)
38
+ # cls_id is a positional index into the element's object list.
39
+ self.class_names = ["cup", "can", "bottle"]
40
+ # What this ONNX emits (used when it carries no names metadata).
41
+ self._model_class_order = ["cup", "can", "bottle"]
42
+ print("ORT version:", ort.__version__)
43
+
44
+ try:
45
+ ort.preload_dlls()
46
+ print("onnxruntime.preload_dlls() success")
47
+ except Exception as e:
48
+ print(f"preload_dlls failed: {e}")
49
+
50
+ print("ORT available providers BEFORE session:", ort.get_available_providers())
51
+
52
+ sess_options = ort.SessionOptions()
53
+ sess_options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
54
+ # Pin threads for the CPU latency gate.
55
+ sess_options.intra_op_num_threads = 2
56
+ sess_options.inter_op_num_threads = 1
57
+ sess_options.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL
58
+ # Flush denormal attention scores so a hot checkpoint stays under the CPU gate.
59
+ sess_options.add_session_config_entry("session.set_denormal_as_zero", "1")
60
+
61
+ # MINER_FORCE_CPU=1 skips the CUDA attempt (so the CPU path can be timed;
62
+ # an unforced "CPU" benchmark silently runs on the GPU).
63
+ force_cpu = os.environ.get("MINER_FORCE_CPU", "") == "1"
64
+ try:
65
+ if force_cpu:
66
+ raise RuntimeError("MINER_FORCE_CPU=1")
67
+ self.session = ort.InferenceSession(
68
+ str(model_path),
69
+ sess_options=sess_options,
70
+ providers=["CUDAExecutionProvider", "CPUExecutionProvider"],
71
+ )
72
+ print("Created ORT session with preferred CUDA provider list")
73
+ except Exception as e:
74
+ print(f"CUDA session creation skipped/failed, using CPU: {e}")
75
+ self.session = ort.InferenceSession(
76
+ str(model_path),
77
+ sess_options=sess_options,
78
+ providers=["CPUExecutionProvider"],
79
+ )
80
+ print("ORT session providers:", self.session.get_providers())
81
+
82
+ # cls_remap[i] = self.class_names.index(model_class_order[i]); order comes
83
+ # from ONNX metadata when present, else the static fallback.
84
+ model_class_order = self._read_model_class_order()
85
+ if model_class_order is None:
86
+ model_class_order = list(self._model_class_order)
87
+ print(f"cls order: no usable ONNX metadata, FALLBACK {model_class_order}")
88
+ else:
89
+ print(f"cls order: from ONNX metadata {model_class_order}")
90
+ self.cls_remap = np.array(
91
+ [self.class_names.index(n) for n in model_class_order], dtype=np.int32
92
+ )
93
+
94
+ for inp in self.session.get_inputs():
95
+ print("INPUT:", inp.name, inp.shape, inp.type)
96
+ for out in self.session.get_outputs():
97
+ print("OUTPUT:", out.name, out.shape, out.type)
98
+
99
+ self.input_name = self.session.get_inputs()[0].name
100
+ self.output_names = [o.name for o in self.session.get_outputs()]
101
+ self.input_shape = self.session.get_inputs()[0].shape
102
+
103
+ # Match the ONNX input dtype (FP16 export needs float16 input).
104
+ input_type = self.session.get_inputs()[0].type
105
+ self.np_dtype = np.float16 if "float16" in input_type else np.float32
106
+ print(f"ONNX input dtype: {input_type} -> numpy {self.np_dtype}")
107
+
108
+ # The miner MUST run at the ONNX's baked input size (704). Dropping to
109
+ # 640 is a latency lever but an EXPORT decision (re-export + re-eval),
110
+ # not a miner-side change.
111
+ self.input_height = self._safe_dim(self.input_shape[2], default=704)
112
+ self.input_width = self._safe_dim(self.input_shape[3], default=704)
113
+
114
+ #
115
+ # ALL per-class arrays below are in CANONICAL order (cup, can, bottle).
116
+ # Per-class NMS IoU (cup, can, bottle).
117
+ self._iou_thres_array = np.array([0.40, 0.50, 0.30], dtype=np.float32)
118
+ self.cross_iou_thresh = 0.9
119
+ # classes, e.g. can -> can+cup)
120
+ self.max_det = 300
121
+ self.pre_nms_topk = 1000
122
+ self.use_tta = False
123
+
124
+ # Single-view cluster boost.
125
+ self.use_cluster_boost = True
126
+
127
+ # Soft-NMS: Gaussian decay of overlapping same-class scores instead of
128
+ # hard deletion, then RE-FILTER decayed scores against the per-class
129
+ # confidence thresholds.
130
+ self.use_soft_nms = False
131
+ self.soft_nms_sigma = 0.5
132
+
133
+ # Per-class confidence thresholds (cup, can, bottle); the second array
134
+ # applies to non-square inputs.
135
+ self._conf_thres_array = np.array([0.42, 0.46, 0.26], dtype=np.float32)
136
+ self._extra_conf_thres_array = np.array([0.37, 0.37, 0.17], dtype=np.float32)
137
+
138
+ # Per-class rescue bonus: if a class has ZERO boxes passing, admit its
139
+ # top-1 when score >= (threshold - bonus).
140
+ self._bonus_array = np.array([0.0, 0.0, 0.0], dtype=np.float32)
141
+
142
+ # Per-class min box area (px^2), loose: beverages are TINY on CCTV
143
+ # (median GT ~0.001 of the frame area) and tiny-GT matches only need
144
+ # IoU 0.30 under the adaptive matcher β€” do not silently drop them.
145
+ self._min_box_area_array = np.array([8, 8, 8], dtype=np.float32)
146
+ self.min_side = 3
147
+ self.max_aspect_ratio = 12
148
+ self.max_area_frac = 0.95
149
+
150
+ #
151
+ # fraction before letterboxing, then drop pad-center boxes and un-pad.
152
+ # 0.0 = off β€” the model was TRAINED on squished 1024 frames, so the
153
+ # domains already match; enabling changes aspect -> validate first.
154
+ self.square_pad_frac = 0
155
+
156
+ self._avg_iou = float(np.mean(self._iou_thres_array))
157
+ print(f"ONNX model loaded from: {model_path}")
158
+ print(f"ONNX input: name={self.input_name}, shape={self.input_shape}")
159
+ self._warmup()
160
+
161
+ # ── Setup helpers ────────────────────────────────────────────────────────
162
+
163
+ def _warmup(self, iters: int = 3) -> None:
164
+ try:
165
+ dummy = np.zeros((1024, 1024, 3), dtype=np.uint8)
166
+ for _ in range(max(1, iters)):
167
+ self.predict_batch(batch_images=[dummy], offset=0, n_keypoints=0)
168
+ print(f"warmup: {iters} dummy predict_batch call(s) done")
169
+ except Exception as e:
170
+ print(f"warmup skipped: {e}")
171
+
172
+ def __repr__(self) -> str:
173
+ return f"BeverageMiner(classes={len(self.class_names)}, providers={self.session.get_providers()})"
174
+
175
+ @staticmethod
176
+ def _safe_dim(value, default: int) -> int:
177
+ return value if isinstance(value, int) and value > 0 else default
178
+
179
+ @staticmethod
180
+ def _resolve_model_path(repo: Path) -> Path:
181
+ """Prefer weights.onnx, then weights_int8.onnx, then any .onnx."""
182
+ for name in ("weights.onnx", "weights_int8.onnx"):
183
+ p = repo / name
184
+ if p.exists():
185
+ if name != "weights.onnx":
186
+ print(f"model: weights.onnx not found, using {name}")
187
+ return p
188
+ candidates = sorted(repo.glob("*.onnx"))
189
+ if candidates:
190
+ print(f"model: using {candidates[0].name}")
191
+ return candidates[0]
192
+ return repo / "weights.onnx" # let session creation raise the error
193
+
194
+ def _read_model_class_order(self) -> list[str] | None:
195
+ """Read class order from Ultralytics ONNX `names` metadata, or None."""
196
+ try:
197
+ import ast
198
+ meta = self.session.get_modelmeta().custom_metadata_map
199
+ names = ast.literal_eval(meta["names"])
200
+ order = ([str(names[i]) for i in sorted(names)] if isinstance(names, dict)
201
+ else [str(n) for n in names])
202
+ except Exception as e:
203
+ print(f"cls order: could not read ONNX names metadata ({e})")
204
+ return None
205
+ if sorted(order) != sorted(self.class_names):
206
+ print(f"cls order: ONNX names {order} != expected {self.class_names}; ignoring")
207
+ return None
208
+ return order
209
+
210
+ # ── Preprocessing ────────────────────────────────────────────────────────
211
+
212
+ def _letterbox(self, image: ndarray, new_shape: tuple[int, int],
213
+ color=(114, 114, 114)) -> tuple[ndarray, float, tuple[float, float]]:
214
+ h, w = image.shape[:2]
215
+ new_w, new_h = new_shape
216
+ ratio = min(new_w / w, new_h / h)
217
+ rw, rh = int(round(w * ratio)), int(round(h * ratio))
218
+ if (rw, rh) != (w, h):
219
+ interp = cv2.INTER_CUBIC if ratio > 1.0 else cv2.INTER_LINEAR
220
+ image = cv2.resize(image, (rw, rh), interpolation=interp)
221
+ dw = (new_w - rw) / 2.0
222
+ dh = (new_h - rh) / 2.0
223
+ left, right = int(round(dw - 0.1)), int(round(dw + 0.1))
224
+ top, bottom = int(round(dh - 0.1)), int(round(dh + 0.1))
225
+ padded = cv2.copyMakeBorder(image, top, bottom, left, right,
226
+ cv2.BORDER_CONSTANT, value=color)
227
+ return padded, ratio, (dw, dh)
228
+
229
+ def _preprocess(self, image: ndarray) -> tuple[np.ndarray, dict]:
230
+ """Letterbox to the ONNX input size; NO CLAHE/denoise/sharpen (any
231
+ enhancement the model wasn't trained on is a train/test mismatch that
232
+ HURTS accuracy and also risks the CPU latency gate).
233
+ Square-domain de-squish: when the frame is square (the 's
234
+ 1024x1024 squished eval image) and square_pad_frac > 0, widen it with
235
+ gray bars first -- objects come out horizontally compressed in those
236
+ frames, and mild widening partly restores their aspect.
237
+ """
238
+ orig_h, orig_w = image.shape[:2]
239
+ extra_left = extra_right = 0
240
+ if self.square_pad_frac > 0.0 and orig_w == orig_h:
241
+ target_w = int(orig_w * (1.0 + self.square_pad_frac))
242
+ if target_w > orig_w:
243
+ total = target_w - orig_w
244
+ extra_left = total // 2
245
+ extra_right = total - extra_left
246
+ image = cv2.copyMakeBorder(image, 0, 0, extra_left, extra_right,
247
+ cv2.BORDER_CONSTANT, value=(114, 114, 114))
248
+
249
+ img, ratio, pad = self._letterbox(image, (self.input_width, self.input_height))
250
+ img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
251
+ img = img.astype(np.float32) / 255.0
252
+ img = np.transpose(img, (2, 0, 1))[None, ...]
253
+ img = np.ascontiguousarray(img, dtype=self.np_dtype)
254
+ return img, {
255
+ "ratio": ratio, "pad": pad, "orig_size": (orig_w, orig_h),
256
+ "extra_left": extra_left, "extra_right": extra_right,
257
+ }
258
+
259
+ # ── Vectorized box ops ───────────────────────────────────────────────────
260
+
261
+ @staticmethod
262
+ def _clip_boxes(boxes: np.ndarray, image_size: tuple[int, int]) -> np.ndarray:
263
+ w, h = image_size
264
+ boxes[:, 0] = np.clip(boxes[:, 0], 0, w - 1)
265
+ boxes[:, 1] = np.clip(boxes[:, 1], 0, h - 1)
266
+ boxes[:, 2] = np.clip(boxes[:, 2], 0, w - 1)
267
+ boxes[:, 3] = np.clip(boxes[:, 3], 0, h - 1)
268
+ return boxes
269
+
270
+ @staticmethod
271
+ def _xywh_to_xyxy(boxes: np.ndarray) -> np.ndarray:
272
+ out = np.empty_like(boxes)
273
+ out[:, 0] = boxes[:, 0] - boxes[:, 2] / 2.0
274
+ out[:, 1] = boxes[:, 1] - boxes[:, 3] / 2.0
275
+ out[:, 2] = boxes[:, 0] + boxes[:, 2] / 2.0
276
+ out[:, 3] = boxes[:, 1] + boxes[:, 3] / 2.0
277
+ return out
278
+
279
+ @staticmethod
280
+ def _hard_nms(boxes: np.ndarray, scores: np.ndarray, iou_thresh: float) -> np.ndarray:
281
+ """Vectorized greedy NMS: areas precomputed once, each step is a single
282
+ numpy IoU vector -- no inner Python loop."""
283
+ n = len(boxes)
284
+ if n == 0:
285
+ return np.array([], dtype=np.intp)
286
+ x1, y1, x2, y2 = boxes[:, 0], boxes[:, 1], boxes[:, 2], boxes[:, 3]
287
+ areas = np.maximum(0.0, x2 - x1) * np.maximum(0.0, y2 - y1)
288
+ order = np.argsort(-scores)
289
+ keep = []
290
+ while order.size > 0:
291
+ i = int(order[0])
292
+ keep.append(i)
293
+ if order.size == 1:
294
+ break
295
+ rest = order[1:]
296
+ xx1 = np.maximum(x1[i], x1[rest])
297
+ yy1 = np.maximum(y1[i], y1[rest])
298
+ xx2 = np.minimum(x2[i], x2[rest])
299
+ yy2 = np.minimum(y2[i], y2[rest])
300
+ inter = np.maximum(0.0, xx2 - xx1) * np.maximum(0.0, yy2 - yy1)
301
+ iou = inter / (areas[i] + areas[rest] - inter + 1e-7)
302
+ order = rest[iou <= iou_thresh]
303
+ return np.array(keep, dtype=np.intp)
304
+
305
+ @staticmethod
306
+ def _soft_nms(boxes: np.ndarray, scores: np.ndarray, sigma: float
307
+ ) -> tuple[np.ndarray, np.ndarray]:
308
+ """Gaussian soft-NMS: decay overlapping scores instead of deleting.
309
+ Returns (original indices in decay order, decayed scores)."""
310
+ n = len(boxes)
311
+ if n == 0:
312
+ return np.array([], dtype=np.intp), np.array([], dtype=np.float32)
313
+ b = boxes.astype(np.float32, copy=True)
314
+ sc = scores.astype(np.float32, copy=True)
315
+ order = np.arange(n)
316
+ for i in range(n):
317
+ mp = i + int(np.argmax(sc[i:]))
318
+ b[[i, mp]] = b[[mp, i]]; sc[[i, mp]] = sc[[mp, i]]
319
+ order[[i, mp]] = order[[mp, i]]
320
+ if i + 1 >= n:
321
+ break
322
+ xx1 = np.maximum(b[i, 0], b[i+1:, 0]); yy1 = np.maximum(b[i, 1], b[i+1:, 1])
323
+ xx2 = np.minimum(b[i, 2], b[i+1:, 2]); yy2 = np.minimum(b[i, 3], b[i+1:, 3])
324
+ inter = np.maximum(0.0, xx2 - xx1) * np.maximum(0.0, yy2 - yy1)
325
+ ai = max(0.0, float((b[i, 2]-b[i, 0]) * (b[i, 3]-b[i, 1])))
326
+ aj = np.maximum(0.0, b[i+1:, 2]-b[i+1:, 0]) * np.maximum(0.0, b[i+1:, 3]-b[i+1:, 1])
327
+ iou = inter / (ai + aj - inter + 1e-7)
328
+ sc[i+1:] *= np.exp(-(iou ** 2) / sigma)
329
+ return order, sc
330
+
331
+ def _per_class_soft_nms(self, boxes: np.ndarray, scores: np.ndarray,
332
+ cls_ids: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
333
+ """Soft-NMS per class, then re-filter decayed scores against the
334
+ per-class confidence thresholds so no decayed box reaches the
335
+ below its class bar."""
336
+ keep_idx: list[int] = []
337
+ keep_sc: list[float] = []
338
+ for c in np.unique(cls_ids):
339
+ idx = np.where(cls_ids == c)[0]
340
+ order, dec = self._soft_nms(boxes[idx], scores[idx], self.soft_nms_sigma)
341
+ thr = float(self._conf_thres_array[c])
342
+ for k, s_dec in zip(order, dec):
343
+ if s_dec >= thr:
344
+ keep_idx.append(int(idx[k])); keep_sc.append(float(s_dec))
345
+ return np.array(keep_idx, dtype=np.intp), np.array(keep_sc, dtype=np.float32)
346
+
347
+ def _per_class_hard_nms(self, boxes: np.ndarray, scores: np.ndarray,
348
+ cls_ids: np.ndarray) -> np.ndarray:
349
+ """Per-class NMS using the per-class IoU thresholds."""
350
+ if len(boxes) == 0:
351
+ return np.array([], dtype=np.intp)
352
+ all_keep: list[int] = []
353
+ for c in np.unique(cls_ids):
354
+ idx = np.where(cls_ids == c)[0]
355
+ cls_iou = float(self._iou_thres_array[c])
356
+ keep = self._hard_nms(boxes[idx], scores[idx], cls_iou)
357
+ all_keep.extend(idx[keep].tolist())
358
+ all_keep.sort()
359
+ return np.array(all_keep, dtype=np.intp)
360
+
361
+ @staticmethod
362
+ def _max_score_per_cluster(post_boxes: np.ndarray, post_cls: np.ndarray,
363
+ full_boxes: np.ndarray, full_scores: np.ndarray,
364
+ full_cls: np.ndarray, iou_thresh: float) -> np.ndarray:
365
+ """Each survivor's confidence -> max score in its SAME-CLASS IoU cluster.
366
+ Vectorized single (n_post x n_full) IoU matrix."""
367
+ n = len(post_boxes)
368
+ if n == 0:
369
+ return np.empty(0, dtype=np.float32)
370
+ m = len(full_boxes)
371
+ if m == 0:
372
+ return np.zeros(n, dtype=np.float32)
373
+ pa = (np.maximum(0.0, post_boxes[:, 2] - post_boxes[:, 0]) *
374
+ np.maximum(0.0, post_boxes[:, 3] - post_boxes[:, 1]))
375
+ fa = (np.maximum(0.0, full_boxes[:, 2] - full_boxes[:, 0]) *
376
+ np.maximum(0.0, full_boxes[:, 3] - full_boxes[:, 1]))
377
+ xx1 = np.maximum(post_boxes[:, 0][:, None], full_boxes[:, 0][None, :])
378
+ yy1 = np.maximum(post_boxes[:, 1][:, None], full_boxes[:, 1][None, :])
379
+ xx2 = np.minimum(post_boxes[:, 2][:, None], full_boxes[:, 2][None, :])
380
+ yy2 = np.minimum(post_boxes[:, 3][:, None], full_boxes[:, 3][None, :])
381
+ inter = np.maximum(0.0, xx2 - xx1) * np.maximum(0.0, yy2 - yy1)
382
+ iou = inter / (pa[:, None] + fa[None, :] - inter + 1e-7)
383
+ mask = (iou >= iou_thresh) & (post_cls[:, None] == full_cls[None, :])
384
+ tiled = np.where(mask, full_scores[None, :], -np.inf)
385
+ out = tiled.max(axis=1)
386
+ out[~np.isfinite(out)] = 0.0
387
+ return out.astype(np.float32)
388
+
389
+ def _conf_filter_mask(self, scores: np.ndarray, cls_ids: np.ndarray,
390
+ extra_left: int) -> np.ndarray:
391
+ """Per-class threshold (square vs non-square set) + per-class rescue."""
392
+ if len(scores) == 0:
393
+ return np.zeros(0, dtype=bool)
394
+ thr_arr = self._extra_conf_thres_array if extra_left > 0 else self._conf_thres_array
395
+ keep = scores >= thr_arr[cls_ids]
396
+ for c in np.unique(cls_ids):
397
+ b = float(self._bonus_array[c])
398
+ if b <= 0.0:
399
+ continue
400
+ cm = cls_ids == c
401
+ if keep[cm].any():
402
+ continue
403
+ idx = np.where(cm)[0]
404
+ top = int(idx[int(np.argmax(scores[idx]))])
405
+ if scores[top] >= float(self._conf_thres_array[c]) - b:
406
+ keep[top] = True
407
+ return keep
408
+
409
+ def _filter_sane_boxes(self, boxes: np.ndarray, scores: np.ndarray,
410
+ cls_ids: np.ndarray, orig_size: tuple[int, int]
411
+ ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
412
+ """Vectorized per-class min-area / max-area / min-side / aspect filter."""
413
+ if len(boxes) == 0:
414
+ return boxes, scores, cls_ids
415
+ orig_w, orig_h = orig_size
416
+ image_area = float(orig_w * orig_h)
417
+ bw = np.maximum(0.0, boxes[:, 2] - boxes[:, 0])
418
+ bh = np.maximum(0.0, boxes[:, 3] - boxes[:, 1])
419
+ area = bw * bh
420
+ ar = np.maximum(bw / np.maximum(bh, 1e-6), bh / np.maximum(bw, 1e-6))
421
+ keep = (
422
+ (bw >= self.min_side) & (bh >= self.min_side) &
423
+ (area >= self._min_box_area_array[cls_ids]) &
424
+ (area <= self.max_area_frac * image_area) &
425
+ (ar <= self.max_aspect_ratio)
426
+ )
427
+ return boxes[keep], scores[keep], cls_ids[keep]
428
+
429
+ def _cross_class_dedup_op(self, boxes: np.ndarray, scores: np.ndarray,
430
+ cls_ids: np.ndarray, iou_thresh: float
431
+ ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
432
+ """Suppress near-duplicate boxes ACROSS classes (same physical object
433
+ firing 2 classes, e.g. a can also scoring as cup). Order by
434
+ conf-margin then area; keep highest, drop IoU>thresh others."""
435
+ n = len(boxes)
436
+ if n <= 1:
437
+ return boxes, scores, cls_ids
438
+ boxes = np.asarray(boxes, dtype=np.float32)
439
+ scores = np.asarray(scores, dtype=np.float32)
440
+ cls_ids = np.asarray(cls_ids, dtype=np.int32)
441
+ areas = (np.maximum(0.0, boxes[:, 2] - boxes[:, 0]) *
442
+ np.maximum(0.0, boxes[:, 3] - boxes[:, 1]))
443
+ margins = scores - self._conf_thres_array[cls_ids]
444
+ order = np.lexsort((-areas, -margins))
445
+ suppressed = np.zeros(n, dtype=bool)
446
+ keep: list[int] = []
447
+ for i in order:
448
+ if suppressed[i]:
449
+ continue
450
+ keep.append(int(i))
451
+ bi = boxes[i]
452
+ xx1 = np.maximum(bi[0], boxes[:, 0])
453
+ yy1 = np.maximum(bi[1], boxes[:, 1])
454
+ xx2 = np.minimum(bi[2], boxes[:, 2])
455
+ yy2 = np.minimum(bi[3], boxes[:, 3])
456
+ inter = np.maximum(0.0, xx2 - xx1) * np.maximum(0.0, yy2 - yy1)
457
+ a_i = max(1e-7, float((bi[2] - bi[0]) * (bi[3] - bi[1])))
458
+ iou = inter / (a_i + areas - inter + 1e-7)
459
+ dup = iou > iou_thresh
460
+ dup[i] = False
461
+ suppressed |= dup
462
+ keep_idx = np.array(keep, dtype=np.intp)
463
+ return boxes[keep_idx], scores[keep_idx], cls_ids[keep_idx]
464
+
465
+ def _per_view_pipeline(self, boxes: np.ndarray, scores: np.ndarray,
466
+ cls_ids: np.ndarray, orig_size: tuple[int, int]
467
+ ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
468
+ """sane filter -> top-k cap -> per-class NMS -> max_det cap -> dedup."""
469
+ boxes, scores, cls_ids = self._filter_sane_boxes(boxes, scores, cls_ids, orig_size)
470
+ if len(boxes) == 0:
471
+ return boxes, scores, cls_ids
472
+ if len(scores) > self.pre_nms_topk:
473
+ top = np.argpartition(-scores, self.pre_nms_topk)[: self.pre_nms_topk]
474
+ boxes, scores, cls_ids = boxes[top], scores[top], cls_ids[top]
475
+ if len(boxes) > 1:
476
+ if self.use_soft_nms:
477
+ keep, scores = self._per_class_soft_nms(boxes, scores, cls_ids)
478
+ boxes, cls_ids = boxes[keep], cls_ids[keep]
479
+ else:
480
+ keep = self._per_class_hard_nms(boxes, scores, cls_ids)
481
+ boxes, scores, cls_ids = boxes[keep], scores[keep], cls_ids[keep]
482
+ if len(scores) > self.max_det:
483
+ top = np.argsort(-scores)[: self.max_det]
484
+ boxes, scores, cls_ids = boxes[top], scores[top], cls_ids[top]
485
+ if len(boxes) > 1:
486
+ boxes, scores, cls_ids = self._cross_class_dedup_op(
487
+ boxes, scores, cls_ids, self.cross_iou_thresh)
488
+ return boxes, scores, cls_ids
489
+
490
+ # ── Coordinate un-mapping (shared by both decode paths) ──────────────────
491
+
492
+ def _unmap_and_finish(self, boxes: np.ndarray, scores: np.ndarray,
493
+ cls_ids: np.ndarray, meta: dict
494
+ ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
495
+ """Undo letterbox + square-pad, drop pad-center boxes, clip, post-process."""
496
+ pad_w, pad_h = meta["pad"]
497
+ ratio = meta["ratio"]
498
+ orig_w, orig_h = meta["orig_size"]
499
+ extra_left, extra_right = meta["extra_left"], meta["extra_right"]
500
+
501
+ boxes[:, [0, 2]] -= pad_w
502
+ boxes[:, [1, 3]] -= pad_h
503
+ boxes /= ratio
504
+ if extra_left:
505
+ boxes[:, [0, 2]] -= extra_left
506
+ if extra_left or extra_right:
507
+ cx = (boxes[:, 0] + boxes[:, 2]) * 0.5
508
+ inside = (cx >= 0) & (cx <= orig_w)
509
+ boxes, scores, cls_ids = boxes[inside], scores[inside], cls_ids[inside]
510
+ if len(boxes) == 0:
511
+ return boxes, scores, cls_ids
512
+ boxes = self._clip_boxes(boxes, (orig_w, orig_h))
513
+ return self._per_view_pipeline(boxes, scores, cls_ids, (orig_w, orig_h))
514
+
515
+ # ── Decoding ─────────────────────────────────────────────────────────────
516
+
517
+ def _decode_final_dets(self, preds: np.ndarray, meta: dict
518
+ ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
519
+ """End2end output rows [x1, y1, x2, y2, conf, cls_id] (NMS in graph)."""
520
+ empty = (np.empty((0, 4), np.float32), np.empty(0, np.float32), np.empty(0, np.int32))
521
+ if preds.ndim == 3 and preds.shape[0] == 1:
522
+ preds = preds[0]
523
+ if preds.ndim != 2 or preds.shape[1] < 6:
524
+ raise ValueError(f"Unexpected final-det output shape: {preds.shape}")
525
+ boxes = preds[:, :4].astype(np.float32)
526
+ scores = preds[:, 4].astype(np.float32)
527
+ cls_ids = self.cls_remap[preds[:, 5].astype(np.int32)]
528
+ keep = self._conf_filter_mask(scores, cls_ids, meta["extra_left"])
529
+ boxes, scores, cls_ids = boxes[keep], scores[keep], cls_ids[keep]
530
+ if len(boxes) == 0:
531
+ return empty
532
+ return self._unmap_and_finish(boxes, scores, cls_ids, meta)
533
+
534
+ def _decode_raw_yolo(self, preds: np.ndarray, meta: dict
535
+ ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
536
+ """Raw YOLO output [1,C,N] or [1,N,C]; xywh + per-class scores."""
537
+ empty = (np.empty((0, 4), np.float32), np.empty(0, np.float32), np.empty(0, np.int32))
538
+ if preds.ndim != 3 or preds.shape[0] != 1:
539
+ raise ValueError(f"Unexpected raw output shape: {preds.shape}")
540
+ preds = preds[0]
541
+ if preds.shape[0] <= 16 and preds.shape[1] > preds.shape[0]:
542
+ preds = preds.T
543
+ if preds.ndim != 2 or preds.shape[1] < 5:
544
+ raise ValueError(f"Unexpected normalized raw shape: {preds.shape}")
545
+ boxes_xywh = preds[:, :4].astype(np.float32)
546
+ cls_part = preds[:, 4:].astype(np.float32)
547
+ if cls_part.shape[1] == 1:
548
+ scores = cls_part[:, 0]
549
+ cls_ids = np.zeros(len(scores), dtype=np.int32)
550
+ else:
551
+ cls_ids = np.argmax(cls_part, axis=1).astype(np.int32)
552
+ scores = cls_part[np.arange(len(cls_part)), cls_ids]
553
+ cls_ids = self.cls_remap[cls_ids]
554
+ keep = self._conf_filter_mask(scores, cls_ids, meta["extra_left"])
555
+ boxes_xywh, scores, cls_ids = boxes_xywh[keep], scores[keep], cls_ids[keep]
556
+ if len(boxes_xywh) == 0:
557
+ return empty
558
+ boxes = self._xywh_to_xyxy(boxes_xywh)
559
+ return self._unmap_and_finish(boxes, scores, cls_ids, meta)
560
+
561
+ def _decode(self, output: np.ndarray, meta: dict
562
+ ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
563
+ if output.ndim == 2 and output.shape[1] >= 6:
564
+ return self._decode_final_dets(output, meta)
565
+ if output.ndim == 3 and output.shape[0] == 1 and output.shape[2] == 6:
566
+ return self._decode_final_dets(output, meta)
567
+ return self._decode_raw_yolo(output, meta)
568
+
569
+ # ── Inference ────────────────────────────────────────────────────────────
570
+
571
+ def _predict_single_arrays(self, image: np.ndarray
572
+ ) -> tuple[np.ndarray, np.ndarray, np.ndarray, dict]:
573
+ if not isinstance(image, np.ndarray) or image.ndim != 3 or image.shape[2] != 3:
574
+ raise ValueError(f"Expected HWC BGR image, got {type(image)} {getattr(image,'shape',None)}")
575
+ if image.dtype != np.uint8:
576
+ image = image.astype(np.uint8)
577
+ inp, meta = self._preprocess(image)
578
+ outputs = self.session.run(self.output_names, {self.input_name: inp})
579
+ boxes, scores, cls_ids = self._decode(outputs[0], meta)
580
+ return boxes, scores, cls_ids, meta
581
+
582
+ @staticmethod
583
+ def _to_boxes(boxes: np.ndarray, scores: np.ndarray, cls_ids: np.ndarray,
584
+ orig_size: tuple[int, int]) -> list[BoundingBox]:
585
+ results: list[BoundingBox] = []
586
+ orig_w, orig_h = orig_size
587
+ for box, conf, cls_id in zip(boxes, scores, cls_ids):
588
+ x1, y1, x2, y2 = box.tolist()
589
+ if x2 <= x1 or y2 <= y1:
590
+ continue
591
+ results.append(BoundingBox(
592
+ x1=max(0, min(orig_w, int(math.floor(x1)))),
593
+ y1=max(0, min(orig_h, int(math.floor(y1)))),
594
+ x2=max(0, min(orig_w, int(math.ceil(x2)))),
595
+ y2=max(0, min(orig_h, int(math.ceil(y2)))),
596
+ cls_id=int(cls_id),
597
+ conf=float(max(0.0, min(1.0, conf))),
598
+ ))
599
+ return results
600
+
601
+ def _infer_single(self, image: np.ndarray) -> list[BoundingBox]:
602
+ boxes, scores, cls_ids, meta = self._predict_single_arrays(image)
603
+ if len(boxes) == 0:
604
+ return []
605
+ if self.use_cluster_boost and len(boxes) > 1:
606
+ scores = self._max_score_per_cluster(
607
+ boxes, cls_ids, boxes, scores, cls_ids, self._avg_iou)
608
+ return self._to_boxes(boxes, scores, cls_ids, meta["orig_size"])
609
+
610
+ def _infer_tta(self, image: np.ndarray) -> list[BoundingBox]:
611
+ """Horizontal-flip TTA: union of original + flipped, per-class NMS,
612
+ same-class cluster-max boost, dedup."""
613
+ b0, s0, c0, meta = self._predict_single_arrays(image)
614
+ w = image.shape[1]
615
+ bf, sf, cf, _ = self._predict_single_arrays(cv2.flip(image, 1))
616
+ if len(bf):
617
+ bf = bf.copy()
618
+ bf[:, [0, 2]] = w - bf[:, [2, 0]] # mirror x back to original coords
619
+ coords = np.concatenate([b0, bf], axis=0) if len(bf) else b0
620
+ scores = np.concatenate([s0, sf], axis=0) if len(sf) else s0
621
+ cls_ids = np.concatenate([c0, cf], axis=0) if len(cf) else c0
622
+ if len(coords) == 0:
623
+ return []
624
+ keep = self._per_class_hard_nms(coords, scores, cls_ids)
625
+ if len(keep) == 0:
626
+ return []
627
+ if len(keep) > self.max_det:
628
+ keep = keep[np.argsort(-scores[keep])[: self.max_det]]
629
+ boosted = self._max_score_per_cluster(
630
+ coords[keep], cls_ids[keep], coords, scores, cls_ids, self._avg_iou)
631
+ kb, kc = coords[keep], cls_ids[keep]
632
+ if len(kb) > 1:
633
+ kb, boosted, kc = self._cross_class_dedup_op(kb, boosted, kc, self.cross_iou_thresh)
634
+ return self._to_boxes(kb, boosted, kc, meta["orig_size"])
635
+
636
+ def predict_batch(self, batch_images: list[ndarray], offset: int,
637
+ n_keypoints: int) -> list[TVFrameResult]:
638
+ results: list[TVFrameResult] = []
639
+ for idx, image in enumerate(batch_images):
640
+ try:
641
+ boxes = self._infer_tta(image) if self.use_tta else self._infer_single(image)
642
+ except Exception as e:
643
+ print(f"Inference failed for frame {offset + idx}: {e}")
644
+ boxes = []
645
+ keypoints = [(0, 0) for _ in range(max(0, int(n_keypoints)))]
646
+ results.append(TVFrameResult(frame_id=offset + idx, boxes=boxes, keypoints=keypoints))
647
+ return results
weights.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:36162aa8abae59303765949094da75185f9bdc0ee125ad042367d94aff50c412
3
+ size 6396897