MTerryJack commited on
Commit
af7fc07
·
verified ·
1 Parent(s): ffd29b2

subnet_bridge: copy winning miner repo into library

Browse files
README.md CHANGED
@@ -1,31 +1,24 @@
1
  ---
2
  tags:
3
  - element_type:detect
4
- - model:ultralytics-yolo
5
  - subnet:winner
6
  - object:person
 
7
  manako:
8
  source: winner_fetch
9
  manifest_element_name: manak0/Detect-Person
10
- winner_repo_id: smile0123/nmodel262
11
- winner_revision: df364ffd90211f8688b9b1fa219ace083dcd4c05
12
- score:
13
- size_mb: 10.710215
14
- performance_map50: 0.7599817092537609
15
- E: 0.07095858572902233
16
- last_benchmark:
17
- type: synthetic_fixed
18
- ran_at: '2026-09-22T09:44:06.248024Z'
19
- result_path: benchmark/synthetic/f2b3a0a0-3c7a-40c2-82e7-86005ced7e92.json
20
  ---
21
 
22
  # Detect-Person-winner
23
 
24
  Published winning miner converted into a library element.
25
 
26
- - Source winner repo: `smile0123/nmodel262`
27
- - Source revision: `df364ffd90211f8688b9b1fa219ace083dcd4c05`
28
  - Manifest element: `manak0/Detect-Person`
29
  - Element type: `Detect`
30
  - Objects: person
31
- - Runtime model type: `ultralytics-yolo`
 
1
  ---
2
  tags:
3
  - element_type:detect
4
+ - model:onnxruntime
5
  - subnet:winner
6
  - object:person
7
+
8
  manako:
9
  source: winner_fetch
10
  manifest_element_name: manak0/Detect-Person
11
+ winner_repo_id: Lucky221115/Person-ML3
12
+ winner_revision: e4bfc487e3460e23b6ccea155de9f35b8a7f3fd0
 
 
 
 
 
 
 
 
13
  ---
14
 
15
  # Detect-Person-winner
16
 
17
  Published winning miner converted into a library element.
18
 
19
+ - Source winner repo: `Lucky221115/Person-ML3`
20
+ - Source revision: `e4bfc487e3460e23b6ccea155de9f35b8a7f3fd0`
21
  - Manifest element: `manak0/Detect-Person`
22
  - Element type: `Detect`
23
  - Objects: person
24
+ - Runtime model type: `onnxruntime`
benchmark/synthetic/f2b3a0a0-3c7a-40c2-82e7-86005ced7e92.json DELETED
@@ -1,80 +0,0 @@
1
- {
2
- "meta": {
3
- "element_id": "manak0/Detect-Person-winner",
4
- "run_id": "f2b3a0a0-3c7a-40c2-82e7-86005ced7e92",
5
- "benchmark_type": "synthetic_fixed",
6
- "saved_at": "2026-09-22T09:44:06.248024Z",
7
- "result_path": "benchmark/synthetic/f2b3a0a0-3c7a-40c2-82e7-86005ced7e92.json",
8
- "split_ref": "private_leaderboard"
9
- },
10
- "results": {
11
- "overall_iou": 0.7417574323431301,
12
- "per_class_iou": {
13
- "person": 0.7417574323431301
14
- },
15
- "map_50": 0.7599817092537609,
16
- "map_50_90": 0.5303797022812917,
17
- "precision": 0.8327219369038885,
18
- "recall": 0.7948179271708683,
19
- "per_class_map_50": {
20
- "person": 0.7599817092537609
21
- },
22
- "per_class_map_50_90": {
23
- "person": 0.5303797022812917
24
- },
25
- "per_class_precision": {
26
- "person": 0.8327219369038885
27
- },
28
- "per_class_recall": {
29
- "person": 0.7948179271708683
30
- },
31
- "matched_images": 30,
32
- "gt_count": 1428,
33
- "pred_count": 1363,
34
- "classes": [
35
- "person"
36
- ],
37
- "evaluated_element_revision": null,
38
- "dataset_id": "bdf4d5b4-7dec-4b72-8747-ab005ca92b3b",
39
- "dataset_version_id": "f2b3a0a0-3c7a-40c2-82e7-86005ced7e92",
40
- "run_id": "f2b3a0a0-3c7a-40c2-82e7-86005ced7e92",
41
- "split_ref": "private_leaderboard",
42
- "selected_frame_count": 30,
43
- "selected_frame_indices": [
44
- 1,
45
- 2,
46
- 3,
47
- 4,
48
- 5,
49
- 6,
50
- 7,
51
- 8,
52
- 9,
53
- 10,
54
- 11,
55
- 12,
56
- 13,
57
- 14,
58
- 15,
59
- 16,
60
- 17,
61
- 18,
62
- 19,
63
- 20,
64
- 21,
65
- 22,
66
- 23,
67
- 24,
68
- 25,
69
- 26,
70
- 27,
71
- 28,
72
- 29,
73
- 30
74
- ],
75
- "evaluated_version_ids": [
76
- "f2b3a0a0-3c7a-40c2-82e7-86005ced7e92"
77
- ],
78
- "skipped_missing_pairs": 0
79
- }
80
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
benchmark/synthetic/index.json DELETED
@@ -1,16 +0,0 @@
1
- [
2
- {
3
- "run_id": "f2b3a0a0-3c7a-40c2-82e7-86005ced7e92",
4
- "saved_at": "2026-09-22T09:44:06.248024Z",
5
- "result_path": "benchmark/synthetic/f2b3a0a0-3c7a-40c2-82e7-86005ced7e92.json",
6
- "overall_iou": 0.7417574323431301,
7
- "map_50": 0.7599817092537609,
8
- "map_50_90": 0.5303797022812917,
9
- "precision": 0.8327219369038885,
10
- "recall": 0.7948179271708683,
11
- "matched_images": 30,
12
- "gt_count": 1428,
13
- "pred_count": 1363,
14
- "split_ref": "private_leaderboard"
15
- }
16
- ]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
benchmark/synthetic/latest.json DELETED
@@ -1,80 +0,0 @@
1
- {
2
- "meta": {
3
- "element_id": "manak0/Detect-Person-winner",
4
- "run_id": "f2b3a0a0-3c7a-40c2-82e7-86005ced7e92",
5
- "benchmark_type": "synthetic_fixed",
6
- "saved_at": "2026-09-22T09:44:06.248024Z",
7
- "result_path": "benchmark/synthetic/f2b3a0a0-3c7a-40c2-82e7-86005ced7e92.json",
8
- "split_ref": "private_leaderboard"
9
- },
10
- "results": {
11
- "overall_iou": 0.7417574323431301,
12
- "per_class_iou": {
13
- "person": 0.7417574323431301
14
- },
15
- "map_50": 0.7599817092537609,
16
- "map_50_90": 0.5303797022812917,
17
- "precision": 0.8327219369038885,
18
- "recall": 0.7948179271708683,
19
- "per_class_map_50": {
20
- "person": 0.7599817092537609
21
- },
22
- "per_class_map_50_90": {
23
- "person": 0.5303797022812917
24
- },
25
- "per_class_precision": {
26
- "person": 0.8327219369038885
27
- },
28
- "per_class_recall": {
29
- "person": 0.7948179271708683
30
- },
31
- "matched_images": 30,
32
- "gt_count": 1428,
33
- "pred_count": 1363,
34
- "classes": [
35
- "person"
36
- ],
37
- "evaluated_element_revision": null,
38
- "dataset_id": "bdf4d5b4-7dec-4b72-8747-ab005ca92b3b",
39
- "dataset_version_id": "f2b3a0a0-3c7a-40c2-82e7-86005ced7e92",
40
- "run_id": "f2b3a0a0-3c7a-40c2-82e7-86005ced7e92",
41
- "split_ref": "private_leaderboard",
42
- "selected_frame_count": 30,
43
- "selected_frame_indices": [
44
- 1,
45
- 2,
46
- 3,
47
- 4,
48
- 5,
49
- 6,
50
- 7,
51
- 8,
52
- 9,
53
- 10,
54
- 11,
55
- 12,
56
- 13,
57
- 14,
58
- 15,
59
- 16,
60
- 17,
61
- 18,
62
- 19,
63
- 20,
64
- 21,
65
- 22,
66
- 23,
67
- 24,
68
- 25,
69
- 26,
70
- 27,
71
- 28,
72
- 29,
73
- 30
74
- ],
75
- "evaluated_version_ids": [
76
- "f2b3a0a0-3c7a-40c2-82e7-86005ced7e92"
77
- ],
78
- "skipped_missing_pairs": 0
79
- }
80
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
main.py CHANGED
@@ -28,7 +28,7 @@ Miner = _load_local_miner_class()
28
 
29
 
30
  CLASS_NAMES = ['person']
31
- MODEL_TYPE = 'ultralytics-yolo'
32
 
33
 
34
  def _to_dict(value: Any) -> dict[str, Any]:
 
28
 
29
 
30
  CLASS_NAMES = ['person']
31
+ MODEL_TYPE = 'onnxruntime'
32
 
33
 
34
  def _to_dict(value: Any) -> dict[str, Any]:
miner.py CHANGED
@@ -1,5 +1,9 @@
 
 
 
1
  import os
2
  from pathlib import Path
 
3
 
4
  import cv2
5
  import numpy as np
@@ -7,31 +11,28 @@ import onnxruntime as ort
7
  from numpy import ndarray
8
  from pydantic import BaseModel
9
 
 
 
 
 
10
 
11
- # =============================================================================
12
- # Person miner (manak0/Detect-Person) — port of the proven carwash001 miner,
13
- # same structure as VehicleDetection/miner/my_miner.
14
- #
15
- # Model: yolo26n end2end ONNX @704 (person_0904_704 / powerful_selected soup),
16
- # single element-canonical class: 0=person.
17
- #
18
- # Carried over (all validated on the carwash/vehicle elements):
19
- # * robust model loading, FP16 auto-detect, metadata class order, ORT thread
20
- # pinning, raw-YOLO decode fallback, NO CLAHE (train/test match)
21
- # * vectorized hard NMS + cluster-max boost, per-class NMS IoU / min-area,
22
- # pre_nms_topk, cross-class dedup (a no-op guard with one class),
23
- # square-domain hook
24
- # * soft-NMS with post-decay re-filter (use_soft_nms), rescue bonus — both
25
- # SWEEPABLE via tune_miner.py; which setting wins is element-specific
26
- # (carwash: soft on / cluster on / rescue off; vehicle: the opposite).
27
- #
28
- # The vehicle miner's dual-label bus/truck lever is intentionally absent:
29
- # person has no sibling class to hedge against.
30
- #
31
- # Run tune_miner.py after ANY retrain/re-export and paste its printout below.
32
- # A failing conformity tuple (latency/format) removes the miner from scoring
33
- # entirely — bench latency with bench_latency.py (2 pinned CPUs) too.
34
- # =============================================================================
35
 
36
 
37
  class BoundingBox(BaseModel):
@@ -49,788 +50,220 @@ class TVFrameResult(BaseModel):
49
  keypoints: list[tuple[int, int]]
50
 
51
 
52
- class Miner:
53
- def __init__(self, path_hf_repo: Path) -> None:
54
- model_path = self._resolve_model_path(path_hf_repo)
55
- # Canonical class order every downstream consumer (validator,
56
- # BoundingBox.cls_id) sees — MUST equal element.objects order:
57
- self.class_names = ["person"]
58
- # Fallback model-emit order (used only when ONNX metadata is missing).
59
- self._model_class_order = ["person"]
60
- print("ORT version:", ort.__version__)
61
-
62
- try:
63
- ort.preload_dlls()
64
- print("onnxruntime.preload_dlls() success")
65
- except Exception as e:
66
- print(f"preload_dlls failed: {e}")
67
 
68
- print("ORT available providers BEFORE session:", ort.get_available_providers())
69
 
70
- sess_options = ort.SessionOptions()
71
- sess_options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
72
- # Pin threads for the CPU latency gate.
73
- sess_options.intra_op_num_threads = 2
74
- sess_options.inter_op_num_threads = 1
75
- sess_options.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL
76
 
77
- # MINER_FORCE_CPU=1 skips the CUDA attempt — used by bench_latency.py to
78
- # measure the conformity-gate path (this box has onnxruntime-gpu too, so
79
- # an unforced "CPU" benchmark silently runs on the GPU).
80
- force_cpu = os.environ.get("MINER_FORCE_CPU", "") == "1"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
81
  try:
82
- if force_cpu:
83
- raise RuntimeError("MINER_FORCE_CPU=1")
84
  self.session = ort.InferenceSession(
85
- str(model_path),
86
- sess_options=sess_options,
87
  providers=["CUDAExecutionProvider", "CPUExecutionProvider"],
88
  )
89
- print("Created ORT session with preferred CUDA provider list")
90
- except Exception as e:
91
- print(f"CUDA session creation skipped/failed, using CPU: {e}")
92
  self.session = ort.InferenceSession(
93
- str(model_path),
94
- sess_options=sess_options,
95
- providers=["CPUExecutionProvider"],
96
  )
97
- print("ORT session providers:", self.session.get_providers())
98
-
99
- # cls_remap[i] = self.class_names.index(model_class_order[i]); order comes
100
- # from ONNX metadata when present, else the static fallback.
101
- model_class_order = self._read_model_class_order()
102
- if model_class_order is None:
103
- model_class_order = list(self._model_class_order)
104
- print(f"cls order: no usable ONNX metadata, FALLBACK {model_class_order}")
105
- else:
106
- print(f"cls order: from ONNX metadata {model_class_order}")
107
- self.cls_remap = np.array(
108
- [self.class_names.index(n) for n in model_class_order], dtype=np.int32
109
- )
110
-
111
- for inp in self.session.get_inputs():
112
- print("INPUT:", inp.name, inp.shape, inp.type)
113
- for out in self.session.get_outputs():
114
- print("OUTPUT:", out.name, out.shape, out.type)
115
 
116
  self.input_name = self.session.get_inputs()[0].name
117
  self.output_names = [o.name for o in self.session.get_outputs()]
118
- self.input_shape = self.session.get_inputs()[0].shape
119
-
120
- # Match the ONNX input dtype (FP16 export needs float16 input).
121
- input_type = self.session.get_inputs()[0].type
122
- self.np_dtype = np.float16 if "float16" in input_type else np.float32
123
- print(f"ONNX input dtype: {input_type} -> numpy {self.np_dtype}")
124
-
125
- # The miner MUST run at the ONNX's baked input size (704). Dropping to
126
- # 640 is a latency lever but an EXPORT decision (re-export + re-eval),
127
- # not a miner-side change.
128
- self.input_height = self._safe_dim(self.input_shape[2], default=704)
129
- self.input_width = self._safe_dim(self.input_shape[3], default=704)
130
-
131
- # ── Post-processing config (tune on person1-powerful/val) ─────────────
132
- # Per-class NMS IoU ([person]).
133
- self._iou_thres_array = np.array([0.45], dtype=np.float32)
134
- self.cross_iou_thresh = 0.9 # near-duplicate guard (no-op across classes
135
- # with one class; catches post-NMS dupes)
136
- self.max_det = 300 # person frames can be busy
137
- self.pre_nms_topk = 1000 # cap candidates before NMS (crowded-frame speed guard)
138
- self.use_tta = False # 2nd flipped pass; doubles latency -> off for CPU gate
139
-
140
- # Single-view cluster boost: raise each survivor's conf to its
141
- # IoU-cluster max. OFF per the 2026-09-04 person sweep (carwash: on,
142
- # vehicle: off -- element-specific, re-sweep on new weights).
143
- self.use_cluster_boost = False
144
-
145
- # Soft-NMS: Gaussian decay of overlapping scores instead of hard
146
- # deletion, then RE-FILTER decayed scores against the confidence
147
- # threshold (keeps decayed junk away from the FP pillar).
148
- # OFF per the 2026-09-04 person sweep (off 0.5670 vs on 0.5662) --
149
- # element- and weights-specific, re-sweep after every retrain.
150
- self.use_soft_nms = True
151
- self.soft_nms_sigma = 0.4
152
-
153
- # Confidence threshold ([person]). `_conf_thres_array` is the SQUARE /
154
- # validator-eval set (every scored frame is 1024x1024 squished, so this
155
- # is the one that matters); `_extra` covers non-square inputs (warmup /
156
- # any 16:9 frame). Tuned 2026-09-04 (tune_miner.py, 1853-img
157
- # person1-powerful/val, adaptive-IoU matcher): comp 0.5385 -> 0.5670.
158
- # NOTE: the val is dense (29 GT/frame avg) and recall is the binding
159
- # constraint. A low-edge extension sweep found a plateau below the
160
- # floor (0.05-0.09 -> comp 0.5713, +0.004) but on REAL challenge
161
- # frames the 0.05-0.12 band adds ~25 boxes/frame -- too FP-risky to
162
- # ship on dense-val evidence alone. Decide with test_miner.py + SAM3
163
- # GT on real frames; retune after every retrain (especially once the
164
- # sparse regenerated data lands).
165
- self._conf_thres_array = np.array([0.25], dtype=np.float32)
166
- self._extra_conf_thres_array = np.array([0.25], dtype=np.float32)
167
-
168
- # Rescue bonus: if NO box passes the threshold, admit the top-1 when
169
- # score >= (threshold - bonus). ON per the 2026-09-04 sweep.
170
- self._bonus_array = np.full(1, 0.03, dtype=np.float32)
171
-
172
- # Min box area (px^2), loose: distant people are tiny in 1024x1024
173
- # frames and tiny-GT matches only need IoU 0.30 under the adaptive
174
- # matcher — do not silently drop them.
175
- self._min_box_area_array = np.array([12.0], dtype=np.float32)
176
- self.min_side = 3
177
- self.max_aspect_ratio = 12.0
178
- self.max_area_frac = 0.95
179
-
180
- # Density-adaptive confidence ("CONF_STEPS"): after NMS, count the
181
- # frame's survivors and re-gate them at a threshold that rises with the
182
- # crowd. Rationale: the validator charges false positives PER FRAME
183
- # (false_positive = max(0, 1 - FP_per_image / 10)), so a busy frame --
184
- # which generates the most FP -- can in principle afford a stricter bar.
185
- #
186
- # Entries are (upper_bound, threshold), first bound the count falls
187
- # under wins, None = catch-all; if none match, the last threshold is
188
- # used. () disables the lever. The config-driven miner shipped
189
- # ((30, 0.5), (50, 0.55), (None, 0.6)).
190
- #
191
- # SET to the "gentle" schedule (chosen 2026-09-10). Measured on THESE
192
- # weights (md5 4ca5622f = person4_cctv_yolo26n_704_0909) against the
193
- # real turbovision pillars, delta composite vs the lever OFF:
194
- #
195
- # schedule p4/test p-11/197 merged/test
196
- # ((30,.45),(60,.50),(None,.55)) -0.0001 -0.0011 -0.0006 <- set
197
- # ((30,.5),(50,.55),(None,.6)) -0.0119 -0.0081 -0.0161
198
- # ((40,.47),(None,.50)) -0.0044 -0.0026 -0.0063
199
- # inverted (loosen when crowded) -0.0261 -0.0074 -0.0311
200
- #
201
- # This schedule is the cheapest of the four: it costs ~0.001 composite,
202
- # which is within run-to-run noise, where the ported one costs 10-16x
203
- # that. It is cheap because its first step (0.45) EQUALS
204
- # `_conf_thres_array`, so frames under 30 boxes are untouched -- only
205
- # the 0.50 and 0.55 tiers ever bite:
206
- #
207
- # set frames @0.45 @0.50 @0.55 frames changed boxes cut
208
- # p4/test 1702 1573 120 9 117 ( 6.9%) 1.68%
209
- # person-11 197 153 37 7 34 (17.3%) 2.55%
210
- # merged/test 1605 1591 14 0 14 ( 0.9%) 0.39%
211
- #
212
- # COUPLING: if `_conf_thres_array` is ever lowered below 0.45, the first
213
- # step stops being a no-op and starts gating every sparse frame -- re-run
214
- # sweep_conf_steps.py before changing either value.
215
- #
216
- # Verified over all 3504 frames above: 3310 land in the 0.45 tier and
217
- # NONE loses a box. Exactly one holds a sub-0.45 box (admitted by the
218
- # rescue bonus in `_conf_filter_mask`, floor - 0.03 = 0.42); it survives
219
- # only because `_apply_density_gate` keeps the top-1 rather than
220
- # emptying the frame. Drop that guarantee and this schedule silently
221
- # zeroes that frame's recall.
222
- #
223
- # No schedule tested beat OFF; this one is a near-tie. Inverting the
224
- # direction is worse than the ported schedule, which says the premise --
225
- # that crowded frames carry the excess FP -- does not hold for this
226
- # model: its FP are spread across sparse frames too.
227
- #
228
- # SWEEPABLE: sweep_conf_steps.py (same dir) re-measures this table
229
- # against the real turbovision pillars -- re-run it after any retrain.
230
- # NOT tune_miner.py: that is a scoring library
231
- # (evaluate_validator/adaptive_thr), it does not sweep levers.
232
- self.conf_steps: tuple[tuple[int | None, float], ...] = (
233
- (50, 0.25), (90, 0.33), (None, 0.38),
234
  )
235
 
236
-
237
- # Square-domain de-squish: widen a square (validator) frame by this
238
- # fraction before letterboxing, then drop pad-center boxes and un-pad.
239
- # 0.0 = off — the model was TRAINED on squished 1024 frames, so the
240
- # domains already match; enabling changes aspect -> validate first.
241
- self.square_pad_frac = 0.0
242
-
243
- # uint8 -> float32/255 as a 256-entry table lookup: same numbers as a
244
- # multiply, measurably cheaper and steadier per frame. Read-only after
245
- # construction, so it stays safe to share across concurrent calls.
246
- self._scale_lut = (np.arange(256, dtype=np.float32) / 255.0).reshape(1, 256)
247
-
248
- self._avg_iou = float(np.mean(self._iou_thres_array))
249
- print(f"ONNX model loaded from: {model_path}")
250
- print(f"ONNX input: name={self.input_name}, shape={self.input_shape}")
251
- self._warmup()
252
-
253
- # ── Setup helpers ────────────────────────────────────────────────────────
254
-
255
- def _warmup(self, iters: int = 3) -> None:
256
  try:
257
- dummy = np.zeros((1024, 1024, 3), dtype=np.uint8) # validator domain
258
- for _ in range(max(1, iters)):
259
- self.predict_batch(batch_images=[dummy], offset=0, n_keypoints=0)
260
- print(f"warmup: {iters} dummy predict_batch call(s) done")
261
- except Exception as e:
262
- print(f"warmup skipped: {e}")
263
-
264
- def __repr__(self) -> str:
265
- return f"PersonMiner(classes={len(self.class_names)}, providers={self.session.get_providers()})"
266
-
267
- @staticmethod
268
- def _safe_dim(value, default: int) -> int:
269
- return value if isinstance(value, int) and value > 0 else default
270
 
271
  @staticmethod
272
- def _resolve_model_path(repo: Path) -> Path:
273
- """Prefer weights.onnx, then weights_int8.onnx, then any .onnx."""
274
- for name in ("weights.onnx", "weights_int8.onnx"):
275
- p = repo / name
276
- if p.exists():
277
- if name != "weights.onnx":
278
- print(f"model: weights.onnx not found, using {name}")
279
- return p
280
- candidates = sorted(repo.glob("*.onnx"))
281
- if candidates:
282
- print(f"model: using {candidates[0].name}")
283
- return candidates[0]
284
- return repo / "weights.onnx" # let session creation raise the error
285
-
286
- def _read_model_class_order(self) -> list[str] | None:
287
- """Read class order from Ultralytics ONNX `names` metadata, or None."""
288
- try:
289
- import ast
290
- meta = self.session.get_modelmeta().custom_metadata_map
291
- names = ast.literal_eval(meta["names"])
292
- order = ([str(names[i]) for i in sorted(names)] if isinstance(names, dict)
293
- else [str(n) for n in names])
294
- except Exception as e:
295
- print(f"cls order: could not read ONNX names metadata ({e})")
296
- return None
297
- if sorted(order) != sorted(self.class_names):
298
- print(f"cls order: ONNX names {order} != expected {self.class_names}; ignoring")
299
- return None
300
- return order
301
-
302
- # ── Preprocessing ────────────────────────────────────────────────────────
303
-
304
- def _letterbox(self, image: ndarray, new_shape: tuple[int, int],
305
- color=(114, 114, 114)) -> tuple[ndarray, float, tuple[float, float]]:
306
- h, w = image.shape[:2]
307
- new_w, new_h = new_shape
308
- ratio = min(new_w / w, new_h / h)
309
- rw, rh = int(round(w * ratio)), int(round(h * ratio))
310
- if (rw, rh) != (w, h):
311
- interp = cv2.INTER_CUBIC if ratio > 1.0 else cv2.INTER_LINEAR
312
- image = cv2.resize(image, (rw, rh), interpolation=interp)
313
- dw = (new_w - rw) / 2.0
314
- dh = (new_h - rh) / 2.0
315
- left, right = int(round(dw - 0.1)), int(round(dw + 0.1))
316
- top, bottom = int(round(dh - 0.1)), int(round(dh + 0.1))
317
- if left or right or top or bottom:
318
- image = cv2.copyMakeBorder(image, top, bottom, left, right,
319
- cv2.BORDER_CONSTANT, value=color)
320
- # A square validator frame letterboxes to zero padding -- copying the
321
- # image just to add no border is pure latency, so skip it.
322
- return image, ratio, (dw, dh)
323
-
324
- def _to_blob(self, img: ndarray) -> ndarray:
325
- """HWC-BGR uint8 -> NCHW-RGB float32 /255.
326
-
327
- cv2.split + cv2.LUT write the de-interleaved, scaled planes straight
328
- into the output blob: two C passes and one allocation, where the old
329
- `cvtColor -> astype -> /255 -> transpose -> ascontiguousarray` chain
330
- reached the same numbers through four full-size temporaries and a
331
- strided 6 MB copy.
332
-
333
- The blob is allocated per call ON PURPOSE. Hoisting it (and the split
334
- planes) into reused instance buffers measured only ~0.07 ms/frame
335
- better and makes the miner unsafe to call from two threads at once --
336
- chute_config.yml sets `concurrency: 4`, and a shared buffer there
337
- silently returns one request's boxes for another request's frame.
338
- """
339
- h, w = img.shape[:2]
340
- blob = np.empty((1, 3, h, w), dtype=np.float32)
341
- b, g, r = cv2.split(img)
342
- lut = self._scale_lut
343
- cv2.LUT(r, lut, dst=blob[0, 0])
344
- cv2.LUT(g, lut, dst=blob[0, 1])
345
- cv2.LUT(b, lut, dst=blob[0, 2])
346
- if self.np_dtype != np.float32: # FP16 export
347
- return blob.astype(self.np_dtype)
348
- return blob
349
-
350
- def _preprocess(self, image: ndarray) -> tuple[np.ndarray, dict]:
351
- """Letterbox to the ONNX input size; NO CLAHE/denoise/sharpen (any
352
- enhancement the model wasn't trained on is a train/test mismatch that
353
- HURTS accuracy and also risks the CPU latency gate).
354
-
355
- Square-domain de-squish: when the frame is square (the validator's
356
- 1024x1024 squished eval image) and square_pad_frac > 0, widen it with
357
- gray bars first -- objects come out horizontally compressed in those
358
- frames, and mild widening partly restores their aspect.
359
- """
360
- orig_h, orig_w = image.shape[:2]
361
- extra_left = extra_right = 0
362
- if self.square_pad_frac > 0.0 and orig_w == orig_h:
363
- target_w = int(orig_w * (1.0 + self.square_pad_frac))
364
- if target_w > orig_w:
365
- total = target_w - orig_w
366
- extra_left = total // 2
367
- extra_right = total - extra_left
368
- image = cv2.copyMakeBorder(image, 0, 0, extra_left, extra_right,
369
- cv2.BORDER_CONSTANT, value=(114, 114, 114))
370
-
371
- img, ratio, pad = self._letterbox(image, (self.input_width, self.input_height))
372
- img = self._to_blob(img)
373
- return img, {
374
- "ratio": ratio, "pad": pad, "orig_size": (orig_w, orig_h),
375
- "extra_left": extra_left, "extra_right": extra_right,
376
- }
377
-
378
- # ── Vectorized box ops ───────────────────────────────────────────────────
379
-
380
- @staticmethod
381
- def _clip_boxes(boxes: np.ndarray, image_size: tuple[int, int]) -> np.ndarray:
382
- w, h = image_size
383
- boxes[:, 0] = np.clip(boxes[:, 0], 0, w - 1)
384
- boxes[:, 1] = np.clip(boxes[:, 1], 0, h - 1)
385
- boxes[:, 2] = np.clip(boxes[:, 2], 0, w - 1)
386
- boxes[:, 3] = np.clip(boxes[:, 3], 0, h - 1)
387
- return boxes
388
-
389
- @staticmethod
390
- def _xywh_to_xyxy(boxes: np.ndarray) -> np.ndarray:
391
- out = np.empty_like(boxes)
392
- out[:, 0] = boxes[:, 0] - boxes[:, 2] / 2.0
393
- out[:, 1] = boxes[:, 1] - boxes[:, 3] / 2.0
394
- out[:, 2] = boxes[:, 0] + boxes[:, 2] / 2.0
395
- out[:, 3] = boxes[:, 1] + boxes[:, 3] / 2.0
396
- return out
397
-
398
- @staticmethod
399
- def _hard_nms(boxes: np.ndarray, scores: np.ndarray, iou_thresh: float) -> np.ndarray:
400
- """Vectorized greedy NMS: areas precomputed once, each step is a single
401
- numpy IoU vector -- no inner Python loop."""
402
- n = len(boxes)
403
- if n == 0:
404
- return np.array([], dtype=np.intp)
405
- x1, y1, x2, y2 = boxes[:, 0], boxes[:, 1], boxes[:, 2], boxes[:, 3]
406
- areas = np.maximum(0.0, x2 - x1) * np.maximum(0.0, y2 - y1)
407
- order = np.argsort(-scores)
408
- keep = []
409
- while order.size > 0:
410
- i = int(order[0])
411
- keep.append(i)
412
- if order.size == 1:
413
- break
414
- rest = order[1:]
415
- xx1 = np.maximum(x1[i], x1[rest])
416
- yy1 = np.maximum(y1[i], y1[rest])
417
- xx2 = np.minimum(x2[i], x2[rest])
418
- yy2 = np.minimum(y2[i], y2[rest])
419
- inter = np.maximum(0.0, xx2 - xx1) * np.maximum(0.0, yy2 - yy1)
420
- iou = inter / (areas[i] + areas[rest] - inter + 1e-7)
421
- order = rest[iou <= iou_thresh]
422
- return np.array(keep, dtype=np.intp)
423
-
424
- @staticmethod
425
- def _soft_nms(boxes: np.ndarray, scores: np.ndarray, sigma: float
426
- ) -> tuple[np.ndarray, np.ndarray]:
427
- """Gaussian soft-NMS: decay overlapping scores instead of deleting.
428
- Returns (original indices in decay order, decayed scores).
429
-
430
- Pairwise IoU does not change as scores decay, so the IoU -> decay
431
- matrix is built ONCE and each step is an argmax plus one multiply.
432
- Same arithmetic as recomputing a geometry row per step, at a fraction
433
- of the numpy call overhead -- which, at a few dozen boxes per frame, is
434
- what this loop actually costs. Verified box-for-box identical to the
435
- per-step version on real frames; the only behaviour that can differ is
436
- which of two byte-identical duplicate boxes with equal scores is listed
437
- first."""
438
- n = len(boxes)
439
- if n == 0:
440
- return np.array([], dtype=np.intp), np.array([], dtype=np.float32)
441
- b = boxes.astype(np.float32, copy=False)
442
- x1, y1, x2, y2 = b[:, 0], b[:, 1], b[:, 2], b[:, 3]
443
- areas = np.maximum(0.0, x2 - x1) * np.maximum(0.0, y2 - y1)
444
- inter = (np.maximum(0.0, np.minimum(x2[:, None], x2[None, :])
445
- - np.maximum(x1[:, None], x1[None, :])) *
446
- np.maximum(0.0, np.minimum(y2[:, None], y2[None, :])
447
- - np.maximum(y1[:, None], y1[None, :])))
448
- iou = inter / (areas[:, None] + areas[None, :] - inter + 1e-7)
449
- decay = np.exp(-(iou ** 2) / sigma)
450
-
451
- # Picked boxes are parked at -inf instead of being deleted from the
452
- # working arrays, so each step is just an argmax and one in-place
453
- # multiply over a fixed-size vector -- no per-step reallocation.
454
- # `where=alive` keeps the multiply off the parked entries: a small
455
- # enough soft_nms_sigma (a swept knob) underflows exp() to 0.0 in
456
- # float32, and -inf * 0.0 would be NaN -- which argmax then picks,
457
- # re-selecting a dead box.
458
- sc = scores.astype(np.float32, copy=True)
459
- alive = np.ones(n, dtype=bool)
460
- order = np.empty(n, dtype=np.intp)
461
- out_sc = np.empty(n, dtype=np.float32)
462
- for i in range(n):
463
- k = int(np.argmax(sc))
464
- order[i] = k
465
- out_sc[i] = sc[k]
466
- if i + 1 >= n:
467
- break
468
- sc[k] = -np.inf
469
- alive[k] = False
470
- np.multiply(sc, decay[k], out=sc, where=alive)
471
- return order, out_sc
472
-
473
- def _per_class_soft_nms(self, boxes: np.ndarray, scores: np.ndarray,
474
- cls_ids: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
475
- """Soft-NMS per class, then re-filter decayed scores against the
476
- per-class confidence thresholds so no decayed box reaches the FP pillar
477
- below its class bar."""
478
- keep_idx: list[np.ndarray] = []
479
- keep_sc: list[np.ndarray] = []
480
- for c in np.unique(cls_ids):
481
- idx = np.where(cls_ids == c)[0]
482
- order, dec = self._soft_nms(boxes[idx], scores[idx], self.soft_nms_sigma)
483
- thr = float(self._conf_thres_array[c])
484
- surv = dec >= thr
485
- keep_idx.append(idx[order[surv]])
486
- keep_sc.append(dec[surv])
487
- if not keep_idx:
488
- return np.array([], dtype=np.intp), np.array([], dtype=np.float32)
489
- return (np.concatenate(keep_idx).astype(np.intp, copy=False),
490
- np.concatenate(keep_sc).astype(np.float32, copy=False))
491
-
492
- def _per_class_hard_nms(self, boxes: np.ndarray, scores: np.ndarray,
493
- cls_ids: np.ndarray) -> np.ndarray:
494
- """Per-class NMS using the per-class IoU thresholds."""
495
- if len(boxes) == 0:
496
- return np.array([], dtype=np.intp)
497
- all_keep: list[int] = []
498
- for c in np.unique(cls_ids):
499
- idx = np.where(cls_ids == c)[0]
500
- cls_iou = float(self._iou_thres_array[c])
501
- keep = self._hard_nms(boxes[idx], scores[idx], cls_iou)
502
- all_keep.extend(idx[keep].tolist())
503
- all_keep.sort()
504
- return np.array(all_keep, dtype=np.intp)
505
-
506
- @staticmethod
507
- def _max_score_per_cluster(post_boxes: np.ndarray, post_cls: np.ndarray,
508
- full_boxes: np.ndarray, full_scores: np.ndarray,
509
- full_cls: np.ndarray, iou_thresh: float) -> np.ndarray:
510
- """Each survivor's confidence -> max score in its SAME-CLASS IoU cluster.
511
- Vectorized single (n_post x n_full) IoU matrix."""
512
- n = len(post_boxes)
513
- if n == 0:
514
- return np.empty(0, dtype=np.float32)
515
- m = len(full_boxes)
516
- if m == 0:
517
- return np.zeros(n, dtype=np.float32)
518
- pa = (np.maximum(0.0, post_boxes[:, 2] - post_boxes[:, 0]) *
519
- np.maximum(0.0, post_boxes[:, 3] - post_boxes[:, 1]))
520
- fa = (np.maximum(0.0, full_boxes[:, 2] - full_boxes[:, 0]) *
521
- np.maximum(0.0, full_boxes[:, 3] - full_boxes[:, 1]))
522
- xx1 = np.maximum(post_boxes[:, 0][:, None], full_boxes[:, 0][None, :])
523
- yy1 = np.maximum(post_boxes[:, 1][:, None], full_boxes[:, 1][None, :])
524
- xx2 = np.minimum(post_boxes[:, 2][:, None], full_boxes[:, 2][None, :])
525
- yy2 = np.minimum(post_boxes[:, 3][:, None], full_boxes[:, 3][None, :])
526
- inter = np.maximum(0.0, xx2 - xx1) * np.maximum(0.0, yy2 - yy1)
527
- iou = inter / (pa[:, None] + fa[None, :] - inter + 1e-7)
528
- mask = (iou >= iou_thresh) & (post_cls[:, None] == full_cls[None, :])
529
- tiled = np.where(mask, full_scores[None, :], -np.inf)
530
- out = tiled.max(axis=1)
531
- out[~np.isfinite(out)] = 0.0
532
- return out.astype(np.float32)
533
-
534
- def _conf_filter_mask(self, scores: np.ndarray, cls_ids: np.ndarray,
535
- extra_left: int) -> np.ndarray:
536
- """Per-class threshold (square vs non-square set) + per-class rescue."""
537
- if len(scores) == 0:
538
- return np.zeros(0, dtype=bool)
539
- thr_arr = self._extra_conf_thres_array if extra_left > 0 else self._conf_thres_array
540
- keep = scores >= thr_arr[cls_ids]
541
- for c in np.unique(cls_ids):
542
- b = float(self._bonus_array[c])
543
- if b <= 0.0:
544
- continue
545
- cm = cls_ids == c
546
- if keep[cm].any():
547
- continue
548
- idx = np.where(cm)[0]
549
- top = int(idx[int(np.argmax(scores[idx]))])
550
- if scores[top] >= float(self._conf_thres_array[c]) - b:
551
- keep[top] = True
552
- return keep
553
-
554
- def _filter_sane_boxes(self, boxes: np.ndarray, scores: np.ndarray,
555
- cls_ids: np.ndarray, orig_size: tuple[int, int]
556
- ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
557
- """Vectorized per-class min-area / max-area / min-side / aspect filter."""
558
- if len(boxes) == 0:
559
- return boxes, scores, cls_ids
560
- orig_w, orig_h = orig_size
561
- image_area = float(orig_w * orig_h)
562
- bw = np.maximum(0.0, boxes[:, 2] - boxes[:, 0])
563
- bh = np.maximum(0.0, boxes[:, 3] - boxes[:, 1])
564
- area = bw * bh
565
- ar = np.maximum(bw / np.maximum(bh, 1e-6), bh / np.maximum(bw, 1e-6))
566
- keep = (
567
- (bw >= self.min_side) & (bh >= self.min_side) &
568
- (area >= self._min_box_area_array[cls_ids]) &
569
- (area <= self.max_area_frac * image_area) &
570
- (ar <= self.max_aspect_ratio)
571
  )
572
- return boxes[keep], scores[keep], cls_ids[keep]
573
-
574
- def _cross_class_dedup_op(self, boxes: np.ndarray, scores: np.ndarray,
575
- cls_ids: np.ndarray, iou_thresh: float
576
- ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
577
- """Suppress near-duplicate boxes at IoU>thresh regardless of class.
578
- With a single class this only catches rare post-NMS duplicates."""
579
- n = len(boxes)
580
- if n <= 1:
581
- return boxes, scores, cls_ids
582
- boxes = np.asarray(boxes, dtype=np.float32)
583
- scores = np.asarray(scores, dtype=np.float32)
584
- cls_ids = np.asarray(cls_ids, dtype=np.int32)
585
- x1, y1, x2, y2 = boxes[:, 0], boxes[:, 1], boxes[:, 2], boxes[:, 3]
586
- areas = np.maximum(0.0, x2 - x1) * np.maximum(0.0, y2 - y1)
587
- margins = scores - self._conf_thres_array[cls_ids]
588
- order = np.lexsort((-areas, -margins))
589
- # One (n x n) IoU matrix instead of an n-long numpy pass per survivor;
590
- # `a_i` keeps the old per-row 1e-7 area floor so the numbers match.
591
- inter = (np.maximum(0.0, np.minimum(x2[:, None], x2[None, :])
592
- - np.maximum(x1[:, None], x1[None, :])) *
593
- np.maximum(0.0, np.minimum(y2[:, None], y2[None, :])
594
- - np.maximum(y1[:, None], y1[None, :])))
595
- a_i = np.maximum(1e-7, (x2 - x1) * (y2 - y1))
596
- dup = (inter / (a_i[:, None] + areas[None, :] - inter + 1e-7)) > iou_thresh
597
- np.fill_diagonal(dup, False)
598
- if not dup.any():
599
- # Nothing overlaps at this threshold (the common case for a lever
600
- # meant to catch rare post-NMS dupes) -- skip the greedy scan, but
601
- # keep the margin/area ordering it would have produced.
602
- return boxes[order], scores[order], cls_ids[order]
603
- suppressed = np.zeros(n, dtype=bool)
604
- keep: list[int] = []
605
- for i in order:
606
- if suppressed[i]:
607
- continue
608
- keep.append(int(i))
609
- suppressed |= dup[i]
610
- keep_idx = np.array(keep, dtype=np.intp)
611
- return boxes[keep_idx], scores[keep_idx], cls_ids[keep_idx]
612
-
613
- def _density_conf(self, n_objects: int) -> float | None:
614
- """Confidence floor for a frame holding `n_objects` boxes.
615
-
616
- Returns None when the lever is off, so callers can skip the gate
617
- entirely rather than compare against a sentinel.
618
- """
619
- if not self.conf_steps:
620
- return None
621
- for limit, thr in self.conf_steps:
622
- if limit is None or n_objects < limit:
623
- return float(thr)
624
- return float(self.conf_steps[-1][1])
625
-
626
- def _apply_density_gate(self, boxes: np.ndarray, scores: np.ndarray,
627
- cls_ids: np.ndarray
628
- ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
629
- """Re-gate one frame's survivors at a crowd-size-dependent threshold.
630
-
631
- Runs once per FRAME on final scores (after NMS, max_det, dedup and any
632
- cluster boost) -- not per view, so TTA counts the merged set once.
633
-
634
- Never empties a frame that had boxes: the top-1 is always kept, which
635
- mirrors the rescue-bonus guarantee in `_conf_filter_mask`. Wiping a
636
- frame to zero would forfeit the recall pillar outright for at most one
637
- frame's worth of FP.
638
- """
639
- if len(scores) == 0:
640
- return boxes, scores, cls_ids
641
- thr = self._density_conf(len(scores))
642
- if thr is None:
643
- return boxes, scores, cls_ids
644
- keep = scores >= np.float32(thr)
645
  if not keep.any():
646
- keep[int(np.argmax(scores))] = True
647
- return boxes[keep], scores[keep], cls_ids[keep]
648
-
649
- def _per_view_pipeline(self, boxes: np.ndarray, scores: np.ndarray,
650
- cls_ids: np.ndarray, orig_size: tuple[int, int]
651
- ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
652
- """sane filter -> top-k cap -> per-class NMS -> max_det cap -> dedup."""
653
- boxes, scores, cls_ids = self._filter_sane_boxes(boxes, scores, cls_ids, orig_size)
654
- if len(boxes) == 0:
655
- return boxes, scores, cls_ids
656
- if len(scores) > self.pre_nms_topk:
657
- top = np.argpartition(-scores, self.pre_nms_topk)[: self.pre_nms_topk]
658
- boxes, scores, cls_ids = boxes[top], scores[top], cls_ids[top]
659
- if len(boxes) > 1:
660
- if self.use_soft_nms:
661
- keep, scores = self._per_class_soft_nms(boxes, scores, cls_ids)
662
- boxes, cls_ids = boxes[keep], cls_ids[keep]
663
- else:
664
- keep = self._per_class_hard_nms(boxes, scores, cls_ids)
665
- boxes, scores, cls_ids = boxes[keep], scores[keep], cls_ids[keep]
666
- if len(scores) > self.max_det:
667
- top = np.argsort(-scores)[: self.max_det]
668
- boxes, scores, cls_ids = boxes[top], scores[top], cls_ids[top]
669
- if len(boxes) > 1:
670
- boxes, scores, cls_ids = self._cross_class_dedup_op(
671
- boxes, scores, cls_ids, self.cross_iou_thresh)
672
- return boxes, scores, cls_ids
673
-
674
- # ── Coordinate un-mapping (shared by both decode paths) ──────────────────
675
 
676
- def _unmap_and_finish(self, boxes: np.ndarray, scores: np.ndarray,
677
- cls_ids: np.ndarray, meta: dict
678
- ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
679
- """Undo letterbox + square-pad, drop pad-center boxes, clip, post-process."""
680
- pad_w, pad_h = meta["pad"]
681
- ratio = meta["ratio"]
682
- orig_w, orig_h = meta["orig_size"]
683
- extra_left, extra_right = meta["extra_left"], meta["extra_right"]
684
-
685
- boxes[:, [0, 2]] -= pad_w
686
- boxes[:, [1, 3]] -= pad_h
687
- boxes /= ratio
688
- if extra_left:
689
- boxes[:, [0, 2]] -= extra_left
690
- if extra_left or extra_right:
691
  cx = (boxes[:, 0] + boxes[:, 2]) * 0.5
692
- inside = (cx >= 0) & (cx <= orig_w)
693
- boxes, scores, cls_ids = boxes[inside], scores[inside], cls_ids[inside]
694
- if len(boxes) == 0:
695
- return boxes, scores, cls_ids
696
- boxes = self._clip_boxes(boxes, (orig_w, orig_h))
697
- return self._per_view_pipeline(boxes, scores, cls_ids, (orig_w, orig_h))
698
-
699
- # ── Decoding ─────────────────────────────────────────────────────────────
700
-
701
- def _decode_final_dets(self, preds: np.ndarray, meta: dict
702
- ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
703
- """End2end output rows [x1, y1, x2, y2, conf, cls_id] (NMS in graph)."""
704
- empty = (np.empty((0, 4), np.float32), np.empty(0, np.float32), np.empty(0, np.int32))
705
- if preds.ndim == 3 and preds.shape[0] == 1:
706
- preds = preds[0]
707
- if preds.ndim != 2 or preds.shape[1] < 6:
708
- raise ValueError(f"Unexpected final-det output shape: {preds.shape}")
709
- boxes = np.asarray(preds[:, :4], dtype=np.float32)
710
- scores = np.asarray(preds[:, 4], dtype=np.float32)
711
- cls_ids = self.cls_remap[preds[:, 5].astype(np.int32)]
712
- keep = self._conf_filter_mask(scores, cls_ids, meta["extra_left"])
713
- boxes, scores, cls_ids = boxes[keep], scores[keep], cls_ids[keep]
714
- if len(boxes) == 0:
715
- return empty
716
- return self._unmap_and_finish(boxes, scores, cls_ids, meta)
717
-
718
- def _decode_raw_yolo(self, preds: np.ndarray, meta: dict
719
- ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
720
- """Raw YOLO output [1,C,N] or [1,N,C]; xywh + per-class scores."""
721
- empty = (np.empty((0, 4), np.float32), np.empty(0, np.float32), np.empty(0, np.int32))
722
- if preds.ndim != 3 or preds.shape[0] != 1:
723
- raise ValueError(f"Unexpected raw output shape: {preds.shape}")
724
- preds = preds[0]
725
- if preds.shape[0] <= 16 and preds.shape[1] > preds.shape[0]:
726
- preds = preds.T
727
- if preds.ndim != 2 or preds.shape[1] < 5:
728
- raise ValueError(f"Unexpected normalized raw shape: {preds.shape}")
729
- boxes_xywh = preds[:, :4].astype(np.float32)
730
- cls_part = preds[:, 4:].astype(np.float32)
731
- if cls_part.shape[1] == 1:
732
- scores = cls_part[:, 0]
733
- cls_ids = np.zeros(len(scores), dtype=np.int32)
734
- else:
735
- cls_ids = np.argmax(cls_part, axis=1).astype(np.int32)
736
- scores = cls_part[np.arange(len(cls_part)), cls_ids]
737
- cls_ids = self.cls_remap[cls_ids]
738
- keep = self._conf_filter_mask(scores, cls_ids, meta["extra_left"])
739
- boxes_xywh, scores, cls_ids = boxes_xywh[keep], scores[keep], cls_ids[keep]
740
- if len(boxes_xywh) == 0:
741
- return empty
742
- boxes = self._xywh_to_xyxy(boxes_xywh)
743
- return self._unmap_and_finish(boxes, scores, cls_ids, meta)
744
-
745
- def _decode(self, output: np.ndarray, meta: dict
746
- ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
747
- if output.ndim == 2 and output.shape[1] >= 6:
748
- return self._decode_final_dets(output, meta)
749
- if output.ndim == 3 and output.shape[0] == 1 and output.shape[2] == 6:
750
- return self._decode_final_dets(output, meta)
751
- return self._decode_raw_yolo(output, meta)
752
-
753
- # ── Inference ────────────────────────────────────────────────────────────
754
-
755
- def _predict_single_arrays(self, image: np.ndarray
756
- ) -> tuple[np.ndarray, np.ndarray, np.ndarray, dict]:
757
  if not isinstance(image, np.ndarray) or image.ndim != 3 or image.shape[2] != 3:
758
- raise ValueError(f"Expected HWC BGR image, got {type(image)} {getattr(image,'shape',None)}")
759
  if image.dtype != np.uint8:
760
  image = image.astype(np.uint8)
761
- inp, meta = self._preprocess(image)
762
- outputs = self.session.run(self.output_names, {self.input_name: inp})
763
- boxes, scores, cls_ids = self._decode(outputs[0], meta)
764
- return boxes, scores, cls_ids, meta
765
-
766
- @staticmethod
767
- def _to_boxes(boxes: np.ndarray, scores: np.ndarray, cls_ids: np.ndarray,
768
- orig_size: tuple[int, int]) -> list[BoundingBox]:
769
- if len(boxes) == 0:
770
- return []
771
- orig_w, orig_h = orig_size
772
- valid = (boxes[:, 2] > boxes[:, 0]) & (boxes[:, 3] > boxes[:, 1])
773
- if not valid.all():
774
- boxes, scores, cls_ids = boxes[valid], scores[valid], cls_ids[valid]
775
- if len(boxes) == 0:
776
- return []
777
- # floor/ceil + clamp for every box in six numpy calls; the per-box
778
- # math.floor/max/min chain was the bulk of this function.
779
- x1 = np.clip(np.floor(boxes[:, 0]), 0, orig_w).astype(np.int64).tolist()
780
- y1 = np.clip(np.floor(boxes[:, 1]), 0, orig_h).astype(np.int64).tolist()
781
- x2 = np.clip(np.ceil(boxes[:, 2]), 0, orig_w).astype(np.int64).tolist()
782
- y2 = np.clip(np.ceil(boxes[:, 3]), 0, orig_h).astype(np.int64).tolist()
783
- confs = np.clip(scores, 0.0, 1.0).tolist()
784
- clss = cls_ids.astype(np.int64).tolist()
785
- return [BoundingBox(x1=a, y1=b, x2=c, y2=d, cls_id=k, conf=s)
786
- for a, b, c, d, k, s in zip(x1, y1, x2, y2, clss, confs)]
787
-
788
- def _infer_single(self, image: np.ndarray) -> list[BoundingBox]:
789
- boxes, scores, cls_ids, meta = self._predict_single_arrays(image)
790
- if len(boxes) == 0:
791
- return []
792
- if self.use_cluster_boost and len(boxes) > 1:
793
- scores = self._max_score_per_cluster(
794
- boxes, cls_ids, boxes, scores, cls_ids, self._avg_iou)
795
- boxes, scores, cls_ids = self._apply_density_gate(boxes, scores, cls_ids)
796
- return self._to_boxes(boxes, scores, cls_ids, meta["orig_size"])
797
-
798
- def _infer_tta(self, image: np.ndarray) -> list[BoundingBox]:
799
- """Horizontal-flip TTA: union of original + flipped, per-class NMS,
800
- same-class cluster-max boost, dedup."""
801
- b0, s0, c0, meta = self._predict_single_arrays(image)
802
- w = image.shape[1]
803
- bf, sf, cf, _ = self._predict_single_arrays(cv2.flip(image, 1))
804
- if len(bf):
805
- bf = bf.copy()
806
- bf[:, [0, 2]] = w - bf[:, [2, 0]] # mirror x back to original coords
807
- coords = np.concatenate([b0, bf], axis=0) if len(bf) else b0
808
- scores = np.concatenate([s0, sf], axis=0) if len(sf) else s0
809
- cls_ids = np.concatenate([c0, cf], axis=0) if len(cf) else c0
810
- if len(coords) == 0:
811
- return []
812
- keep = self._per_class_hard_nms(coords, scores, cls_ids)
813
- if len(keep) == 0:
814
- return []
815
- if len(keep) > self.max_det:
816
- keep = keep[np.argsort(-scores[keep])[: self.max_det]]
817
- boosted = self._max_score_per_cluster(
818
- coords[keep], cls_ids[keep], coords, scores, cls_ids, self._avg_iou)
819
- kb, kc = coords[keep], cls_ids[keep]
820
- if len(kb) > 1:
821
- kb, boosted, kc = self._cross_class_dedup_op(kb, boosted, kc, self.cross_iou_thresh)
822
- kb, boosted, kc = self._apply_density_gate(kb, boosted, kc)
823
- return self._to_boxes(kb, boosted, kc, meta["orig_size"])
824
 
825
- def predict_batch(self, batch_images: list[ndarray], offset: int,
826
- n_keypoints: int) -> list[TVFrameResult]:
827
  results: list[TVFrameResult] = []
828
- for idx, image in enumerate(batch_images):
 
829
  try:
830
- boxes = self._infer_tta(image) if self.use_tta else self._infer_single(image)
831
- except Exception as e:
832
- print(f"Inference failed for frame {offset + idx}: {e}")
 
 
 
 
 
 
 
 
 
833
  boxes = []
834
- keypoints = [(0, 0) for _ in range(max(0, int(n_keypoints)))]
835
- results.append(TVFrameResult(frame_id=offset + idx, boxes=boxes, keypoints=keypoints))
836
  return results
 
 
 
 
1
+ import ast
2
+ import json
3
+ import math
4
  import os
5
  from pathlib import Path
6
+ from typing import Any
7
 
8
  import cv2
9
  import numpy as np
 
11
  from numpy import ndarray
12
  from pydantic import BaseModel
13
 
14
+ API_VERSION = "c2-strict-v1"
15
+ STRICT_INFERENCE_ENV = "SV_STRICT_INFERENCE"
16
+ OUTPUT_ROWS = 300
17
+ OUTPUT_COLUMNS = 6
18
 
19
+ ELEMENT_CONFIG: dict[str, Any] = {
20
+ "element_id": "person",
21
+ "classes": ["person"],
22
+ "imgsz": 704,
23
+ "conf": 0.3,
24
+ "max_det": 300,
25
+ "min_side": 3,
26
+ "max_aspect": 10,
27
+ "max_area_frac": 0.95,
28
+ "scale_w": 1,
29
+ "scale_h": 1,
30
+ "conf_steps": ((30, 0.39), (50, 0.46), (None, 0.58)),
31
+ }
32
+
33
+
34
+ class InferenceInvariantError(RuntimeError):
35
+ pass
 
 
 
 
 
 
 
36
 
37
 
38
  class BoundingBox(BaseModel):
 
50
  keypoints: list[tuple[int, int]]
51
 
52
 
53
+ def _conf_for(n_raw: int) -> float:
54
+ for limit, conf in ELEMENT_CONFIG.get("conf_steps", ()):
55
+ if limit is None or n_raw < limit:
56
+ return float(conf)
57
+ return float(ELEMENT_CONFIG["conf"])
 
 
 
 
 
 
 
 
 
 
58
 
 
59
 
60
+ class Miner:
61
+ api_version = API_VERSION
62
+ strict_inference = False
 
 
 
63
 
64
+ def __init__(self, path_hf_repo: Path) -> None:
65
+ cfg = ELEMENT_CONFIG
66
+ self.config = cfg
67
+ self.class_names = list(cfg["classes"])
68
+ self.conf_threshold = float(cfg["conf"])
69
+ self.max_det = int(cfg["max_det"])
70
+ self.min_side = float(cfg.get("min_side", 3.0))
71
+ self.max_aspect_ratio = float(cfg.get("max_aspect", 10.0))
72
+ self.max_area_frac = float(cfg.get("max_area_frac", 0.95))
73
+ self.conf_steps = tuple(tuple(s) for s in cfg.get("conf_steps", ()))
74
+ self.scale_w = float(cfg.get("scale_w", 1.0))
75
+ self.scale_h = float(cfg.get("scale_h", 1.0))
76
+ raw = os.environ.get(STRICT_INFERENCE_ENV, "")
77
+ if raw not in ("", "0", "1"):
78
+ raise ValueError(f"{STRICT_INFERENCE_ENV} must be '1' or unset/'0', got {raw!r}")
79
+ self.strict_inference = raw == "1"
80
+ self.inference_failures = 0
81
+
82
+ cv2.setNumThreads(1)
83
+ options = ort.SessionOptions()
84
+ options.intra_op_num_threads = 2
85
+ options.inter_op_num_threads = 1
86
+ options.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL
87
+ options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
88
+ model_path = Path(path_hf_repo) / "weights.onnx"
89
  try:
 
 
90
  self.session = ort.InferenceSession(
91
+ str(model_path), sess_options=options,
 
92
  providers=["CUDAExecutionProvider", "CPUExecutionProvider"],
93
  )
94
+ except Exception as exc:
95
+ print(f"CUDA session failed, falling back to CPU: {exc}")
 
96
  self.session = ort.InferenceSession(
97
+ str(model_path), sess_options=options, providers=["CPUExecutionProvider"]
 
 
98
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
99
 
100
  self.input_name = self.session.get_inputs()[0].name
101
  self.output_names = [o.name for o in self.session.get_outputs()]
102
+ shape = self.session.get_inputs()[0].shape
103
+ if len(shape) != 4 or shape[1] != 3:
104
+ raise ValueError(f"model input must be rank-4 NCHW with 3 channels, got {shape!r}")
105
+ if not all(isinstance(v, int) and v > 0 for v in shape[2:]):
106
+ raise ValueError(f"e2e export must have static spatial dims, got {shape[2:]!r}")
107
+ if (shape[2], shape[3]) != (cfg["imgsz"], cfg["imgsz"]):
108
+ raise ValueError(f"model input {shape[2]}x{shape[3]} != config imgsz {cfg['imgsz']}")
109
+ self.input_height = self.input_width = int(cfg["imgsz"])
110
+ out_shape = self.session.get_outputs()[0].shape
111
+ if list(out_shape) != [1, OUTPUT_ROWS, OUTPUT_COLUMNS]:
112
+ raise ValueError(f"e2e output must be [1, {OUTPUT_ROWS}, {OUTPUT_COLUMNS}], got {out_shape!r}")
113
+ self.model_names = self._read_model_names()
114
+ self._warmup(3)
115
+ print(
116
+ f"api_version={self.api_version} element={cfg['element_id']} imgsz={cfg['imgsz']} "
117
+ f"conf={self.conf_threshold} steps={self.conf_steps} "
118
+ f"providers={self.session.get_providers()}"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
119
  )
120
 
121
+ def _read_model_names(self) -> tuple[str, ...]:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
122
  try:
123
+ raw = self.session.get_modelmeta().custom_metadata_map["names"]
124
+ parsed = ast.literal_eval(raw)
125
+ if not isinstance(parsed, dict):
126
+ raise TypeError("names metadata is not a dict")
127
+ keys = sorted(parsed)
128
+ if keys != list(range(len(keys))):
129
+ raise ValueError(f"names metadata keys not consecutive from zero: {keys[:8]}")
130
+ return tuple(str(parsed[i]) for i in keys)
131
+ except Exception as exc:
132
+ raise ValueError(f"e2e ONNX requires valid names metadata: {exc}") from exc
 
 
 
133
 
134
  @staticmethod
135
+ def _letterbox(image: ndarray, new_shape: tuple[int, int], color=(114, 114, 114)):
136
+ height, width = image.shape[:2]
137
+ new_width, new_height = new_shape
138
+ ratio = min(new_width / width, new_height / height)
139
+ rw, rh = int(round(width * ratio)), int(round(height * ratio))
140
+ if (rw, rh) != (width, height):
141
+ image = cv2.resize(
142
+ image, (rw, rh),
143
+ interpolation=cv2.INTER_CUBIC if ratio > 1.0 else cv2.INTER_LINEAR,
144
+ )
145
+ dw, dh = (new_width - rw) / 2.0, (new_height - rh) / 2.0
146
+ padded = cv2.copyMakeBorder(
147
+ image,
148
+ int(round(dh - 0.1)), int(round(dh + 0.1)),
149
+ int(round(dw - 0.1)), int(round(dw + 0.1)),
150
+ cv2.BORDER_CONSTANT, value=color,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
151
  )
152
+ return padded, ratio, (dw, dh)
153
+
154
+ def _decode(self, output, ratio, pad, original_size):
155
+ rows = output[0]
156
+ boxes = rows[:, :4].astype(np.float32, copy=True)
157
+ scores = rows[:, 4].astype(np.float32, copy=False)
158
+ raw_classes = rows[:, 5]
159
+ if (scores < 0).any() or (scores > 1).any():
160
+ raise InferenceInvariantError("e2e confidence outside [0, 1]")
161
+ rounded = np.rint(raw_classes)
162
+ if not np.array_equal(raw_classes, rounded):
163
+ raise InferenceInvariantError("e2e model class ids must be exact integers")
164
+ model_classes = rounded.astype(np.int64)
165
+ if (model_classes < 0).any() or (model_classes >= len(self.model_names)).any():
166
+ raise InferenceInvariantError("e2e model class id outside range")
167
+
168
+ active = scores > 0.0
169
+ reversed_rows = active & ((boxes[:, 2] < boxes[:, 0]) | (boxes[:, 3] < boxes[:, 1]))
170
+ if reversed_rows.any():
171
+ raise InferenceInvariantError("e2e output contains reversed boxes")
172
+
173
+ floor = min([self.conf_threshold] + [float(c) for _, c in self.conf_steps])
174
+ keep = active & (model_classes == 0) & (scores >= np.float32(floor))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
175
  if not keep.any():
176
+ return np.empty((0, 4), np.float32), np.empty((0,), np.float32)
177
+ boxes, scores = boxes[keep], scores[keep]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
178
 
179
+ boxes[:, [0, 2]] = (boxes[:, [0, 2]] - pad[0]) / ratio
180
+ boxes[:, [1, 3]] = (boxes[:, [1, 3]] - pad[1]) / ratio
181
+ width, height = original_size
182
+ if self.scale_w != 1.0 or self.scale_h != 1.0:
 
 
 
 
 
 
 
 
 
 
 
183
  cx = (boxes[:, 0] + boxes[:, 2]) * 0.5
184
+ cy = (boxes[:, 1] + boxes[:, 3]) * 0.5
185
+ hw = (boxes[:, 2] - boxes[:, 0]) * 0.5 * np.float32(self.scale_w)
186
+ hh = (boxes[:, 3] - boxes[:, 1]) * 0.5 * np.float32(self.scale_h)
187
+ boxes = np.stack([cx - hw, cy - hh, cx + hw, cy + hh], 1).astype(np.float32)
188
+ boxes[:, [0, 2]] = np.clip(boxes[:, [0, 2]], 0, width - 1)
189
+ boxes[:, [1, 3]] = np.clip(boxes[:, [1, 3]], 0, height - 1)
190
+
191
+ bw = boxes[:, 2] - boxes[:, 0]
192
+ bh = boxes[:, 3] - boxes[:, 1]
193
+ aspect = np.maximum(bw / np.maximum(bh, 1e-6), bh / np.maximum(bw, 1e-6))
194
+ sane = (
195
+ (bw >= self.min_side)
196
+ & (bh >= self.min_side)
197
+ & (bw * bh <= self.max_area_frac * float(width * height))
198
+ & (aspect <= self.max_aspect_ratio)
199
+ )
200
+ boxes, scores = boxes[sane], scores[sane]
201
+ if not len(scores):
202
+ return boxes, scores
203
+ order = np.argsort(-scores, kind="stable")[: self.max_det]
204
+ boxes, scores = boxes[order], scores[order]
205
+ gate = _conf_for(len(scores)) if self.conf_steps else self.conf_threshold
206
+ emit = scores >= np.float32(gate)
207
+ return boxes[emit], scores[emit]
208
+
209
+ def _predict_single(self, image: np.ndarray) -> list[BoundingBox]:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
210
  if not isinstance(image, np.ndarray) or image.ndim != 3 or image.shape[2] != 3:
211
+ raise ValueError(f"expected HWC 3-channel image, got {getattr(image, 'shape', type(image))}")
212
  if image.dtype != np.uint8:
213
  image = image.astype(np.uint8)
214
+ original_height, original_width = image.shape[:2]
215
+ padded, ratio, pad = self._letterbox(image, (self.input_width, self.input_height))
216
+ blob = cv2.dnn.blobFromImage(padded, scalefactor=1 / 255.0, swapRB=True)
217
+ outputs = self.session.run(self.output_names, {self.input_name: blob})
218
+ output = outputs[0]
219
+ if not np.isfinite(output).all():
220
+ raise InferenceInvariantError("model output must be finite everywhere")
221
+ if list(output.shape) != [1, OUTPUT_ROWS, OUTPUT_COLUMNS]:
222
+ raise ValueError(f"unexpected output shape {list(output.shape)}")
223
+ boxes, scores = self._decode(output, ratio, pad, (original_width, original_height))
224
+ return [
225
+ BoundingBox(
226
+ x1=int(math.floor(b[0])), y1=int(math.floor(b[1])),
227
+ x2=int(math.ceil(b[2])), y2=int(math.ceil(b[3])),
228
+ cls_id=0, conf=float(s),
229
+ )
230
+ for b, s in zip(boxes, scores)
231
+ if b[2] > b[0] and b[3] > b[1]
232
+ ]
233
+
234
+ def _warmup(self, iterations: int) -> None:
235
+ dummy = np.zeros((720, 1280, 3), dtype=np.uint8)
236
+ original = self.strict_inference
237
+ self.strict_inference = True
238
+ try:
239
+ for _ in range(iterations):
240
+ self.predict_batch(batch_images=[dummy], offset=0, n_keypoints=0)
241
+ except Exception as exc:
242
+ raise RuntimeError(f"warmup failed, refusing to serve: {type(exc).__name__}: {exc}") from exc
243
+ finally:
244
+ self.strict_inference = original
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
245
 
246
+ def predict_batch(self, batch_images, offset: int, n_keypoints: int) -> list[TVFrameResult]:
247
+ _ = n_keypoints
248
  results: list[TVFrameResult] = []
249
+ for index, image in enumerate(batch_images):
250
+ frame_id = offset + index
251
  try:
252
+ boxes = self._predict_single(image)
253
+ except Exception as exc:
254
+ if self.strict_inference:
255
+ raise
256
+ self.inference_failures += 1
257
+ print(json.dumps({
258
+ "event": "inference_failure",
259
+ "frame_id": frame_id,
260
+ "error_type": type(exc).__name__,
261
+ "error": str(exc),
262
+ "inference_failures": self.inference_failures,
263
+ }, sort_keys=True))
264
  boxes = []
265
+ results.append(TVFrameResult(frame_id=frame_id, boxes=boxes, keypoints=[]))
 
266
  return results
267
+
268
+ def __repr__(self) -> str:
269
+ return f"E2EONNXRuntime(providers={self.session.get_providers()})"
pyproject.toml CHANGED
@@ -1,7 +1,7 @@
1
  [project]
2
  name = "miner-element-adapter"
3
  version = "0.1.0"
4
- requires-python = ">=3.9"
5
 
6
  dependencies = [
7
  "numpy>=1.23",
 
1
  [project]
2
  name = "miner-element-adapter"
3
  version = "0.1.0"
4
+ requires-python = ">=3.11"
5
 
6
  dependencies = [
7
  "numpy>=1.23",
weights.onnx CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:ec627a7e406e2ba42c91f738bd73863f317a6b63cf3f6b5c713eb13a8184a4d5
3
- size 10654740
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f2adc7fc20e19817198873a52dabfbac62d2e312b75253a54622b7aeda7f7bb8
3
+ size 9860783