ninja990621 commited on
Commit
c599dd5
·
verified ·
1 Parent(s): d7df189

scorevision: push artifact

Browse files
Files changed (1) hide show
  1. miner.py +113 -91
miner.py CHANGED
@@ -1,12 +1,20 @@
1
- """Room-dirtiness detector for Score (SN44) element manako/Detect-room-dirtiness-detection.
2
 
3
  Classes (cls_id order fixed by the element manifest):
4
  0 liquid spillage, 1 bin, 2 litter, 3 mould, 4 stain
5
 
6
- Runs a YOLO-style ONNX detector on CPU with onnxruntime, so the chute and Score's
7
- CPU compliance re-run produce the same boxes. Pure numpy/cv2 pre/post-processing.
 
 
 
 
 
 
 
8
  """
9
 
 
10
  from pathlib import Path
11
 
12
  import cv2
@@ -17,17 +25,13 @@ from pydantic import BaseModel
17
 
18
  NUM_CLASSES = 5
19
  DEFAULT_INPUT_SIZE = 640
20
- # Per-class score thresholds, tuned offline with Score's own per-image scorer.
21
- CONF_THRESHOLDS = (0.25, 0.25, 0.25, 0.25, 0.35)
22
  NMS_IOU = 0.5
23
- # Score's answer key gives each object one class: drop a weaker box of another class sitting on the same spot.
24
- CROSS_CLASS_IOU = 2
25
- # Score's labeller gives one box per object and drops a mask lying inside a bigger mask of the same prompt, so a
26
- # same-class box mostly inside a bigger, about as confident box is a piece of that object (e.g. one item of a pile).
27
- INSIDE_SHARE = 0.7
28
- INSIDE_CONF_RATIO = 0.9
29
- MAX_DETECTIONS = 30
30
  PAD_VALUE = 114
 
 
31
 
32
 
33
  class BoundingBox(BaseModel):
@@ -45,51 +49,61 @@ class TVFrameResult(BaseModel):
45
  keypoints: list[tuple[int, int]] | None = None
46
 
47
 
48
- def letterbox(image: ndarray, size: int) -> tuple[ndarray, float, float, float]:
49
- """Resize keeping aspect ratio and pad to size x size (Ultralytics LetterBox, centred)."""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
50
  h, w = image.shape[:2]
51
- r = min(size / h, size / w)
52
- new_w, new_h = int(round(w * r)), int(round(h * r))
53
- if (new_w, new_h) != (w, h):
 
54
  image = cv2.resize(image, (new_w, new_h), interpolation=cv2.INTER_LINEAR)
55
- dw, dh = (size - new_w) / 2, (size - new_h) / 2
56
- top, bottom = int(round(dh - 0.1)), int(round(dh + 0.1))
57
- left, right = int(round(dw - 0.1)), int(round(dw + 0.1))
58
- image = cv2.copyMakeBorder(image, top, bottom, left, right, cv2.BORDER_CONSTANT,
59
- value=(PAD_VALUE, PAD_VALUE, PAD_VALUE))
60
- return image, r, left, top
61
-
62
-
63
- def box_iou(a: ndarray, b: ndarray) -> float:
64
- iw = max(0.0, min(a[2], b[2]) - max(a[0], b[0]))
65
- ih = max(0.0, min(a[3], b[3]) - max(a[1], b[1]))
66
- inter = iw * ih
67
- union = (a[2] - a[0]) * (a[3] - a[1]) + (b[2] - b[0]) * (b[3] - b[1]) - inter
68
- return float(inter / union) if union > 0 else 0.0
69
-
70
-
71
- def inside_share(a: ndarray, b: ndarray) -> float:
72
- """Share of the smaller box's area covered by the other box."""
73
- iw = max(0.0, min(a[2], b[2]) - max(a[0], b[0]))
74
- ih = max(0.0, min(a[3], b[3]) - max(a[1], b[1]))
75
- smaller = min((a[2] - a[0]) * (a[3] - a[1]), (b[2] - b[0]) * (b[3] - b[1]))
76
- return float(iw * ih / smaller) if smaller > 0 else 0.0
77
-
78
-
79
- def nms(boxes: ndarray, scores: ndarray, iou_threshold: float) -> list[int]:
80
- x1, y1, x2, y2 = boxes.T
81
- areas = (x2 - x1).clip(0) * (y2 - y1).clip(0)
82
- order = scores.argsort()[::-1]
83
  keep = []
84
- while order.size:
85
- i = order[0]
86
- keep.append(int(i))
87
- rest = order[1:]
88
- iw = (np.minimum(x2[i], x2[rest]) - np.maximum(x1[i], x1[rest])).clip(0)
89
- ih = (np.minimum(y2[i], y2[rest]) - np.maximum(y1[i], y1[rest])).clip(0)
 
 
 
 
90
  inter = iw * ih
91
- iou = inter / (areas[i] + areas[rest] - inter + 1e-9)
92
- order = rest[iou <= iou_threshold]
 
93
  return keep
94
 
95
 
@@ -100,61 +114,69 @@ class Miner:
100
  options.inter_op_num_threads = 1
101
  options.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL
102
  options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
 
 
103
  self.session = ort.InferenceSession(str(Path(path_hf_repo) / "model.onnx"), sess_options=options,
104
  providers=["CPUExecutionProvider"])
105
  model_input = self.session.get_inputs()[0]
106
- self.input_name = model_input.name
107
- size = model_input.shape[2]
108
- self.input_size = size if isinstance(size, int) and size > 0 else DEFAULT_INPUT_SIZE
 
 
 
 
 
 
 
109
  self.conf_thresholds = np.array(CONF_THRESHOLDS, dtype=np.float32)
110
- self.min_conf = float(self.conf_thresholds.min())
 
111
 
112
  def __repr__(self) -> str:
113
- return f"Room-dirtiness ONNX detector ({self.input_size}px, providers={self.session.get_providers()})"
 
114
 
115
  def detect(self, image: ndarray) -> list[tuple[int, int, int, int, int, float]]:
116
  """Boxes for one BGR image: (x1, y1, x2, y2, cls_id, conf) in original pixel coordinates."""
 
117
  h, w = image.shape[:2]
118
- padded, r, left, top = letterbox(image, self.input_size)
119
- x = cv2.cvtColor(padded, cv2.COLOR_BGR2RGB).transpose(2, 0, 1)[None].astype(np.float32) / 255.0
120
- out = self.session.run(None, {self.input_name: x})[0][0] # (4 + classes [+ extras], anchors)
121
- pred = out[: 4 + NUM_CLASSES].T
122
  scores = pred[:, 4:]
123
- anchor_idx, cls_ids = np.nonzero(scores >= np.maximum(self.conf_thresholds, self.min_conf))
124
- if anchor_idx.size == 0:
 
 
125
  return []
126
- conf = scores[anchor_idx, cls_ids]
127
- cx, cy, bw, bh = pred[anchor_idx, :4].T
128
- boxes = np.stack([cx - bw / 2, cy - bh / 2, cx + bw / 2, cy + bh / 2], axis=1)
129
- boxes[:, [0, 2]] = ((boxes[:, [0, 2]] - left) / r).clip(0, w)
130
- boxes[:, [1, 3]] = ((boxes[:, [1, 3]] - top) / r).clip(0, h)
131
-
 
 
 
 
 
 
 
132
  results = []
133
- for c in np.unique(cls_ids):
134
- sel = np.nonzero(cls_ids == c)[0]
135
- for k in nms(boxes[sel], conf[sel], NMS_IOU):
136
- i = sel[k]
137
- results.append((boxes[i], int(c), float(conf[i])))
138
- results.sort(key=lambda t: -t[2])
139
- if CROSS_CLASS_IOU <= 1.0:
140
- kept = []
141
- for b, c, sc in results:
142
- if all(kc == c or box_iou(b, kb) < CROSS_CLASS_IOU for kb, kc, _ in kept):
143
- kept.append((b, c, sc))
144
- results = kept
145
- kept = []
146
- for b, c, sc in sorted(results, key=lambda t: -(t[0][2] - t[0][0]) * (t[0][3] - t[0][1])):
147
- if not any(kc == c and ks >= INSIDE_CONF_RATIO * sc and inside_share(b, kb) >= INSIDE_SHARE
148
- for kb, kc, ks in kept):
149
- kept.append((b, c, sc))
150
- results = sorted(kept, key=lambda t: -t[2])
151
- return [(int(round(b[0])), int(round(b[1])), int(round(b[2])), int(round(b[3])), c, s)
152
- for b, c, s in results[:MAX_DETECTIONS] if b[2] - b[0] >= 1 and b[3] - b[1] >= 1]
153
 
154
  def predict_batch(self, batch_images: list[ndarray], offset: int, n_keypoints: int) -> list[TVFrameResult]:
155
  results = []
156
  for i, image in enumerate(batch_images):
157
- boxes = [BoundingBox(x1=x1, y1=y1, x2=x2, y2=y2, cls_id=c, conf=round(s, 4))
158
  for x1, y1, x2, y2, c, s in self.detect(image)]
159
  results.append(TVFrameResult(frame_id=offset + i, boxes=boxes, keypoints=[(0, 0)] * n_keypoints))
160
  return results
 
1
+ """Room-dirtiness detector for Score (SN44) element manako/Detect-room-dirtiness-detection, YOLO26n graph.
2
 
3
  Classes (cls_id order fixed by the element manifest):
4
  0 liquid spillage, 1 bin, 2 litter, 3 mould, 4 stain
5
 
6
+ Runs a YOLO26n ONNX detector on CPU with onnxruntime, so the chute and Score's CPU compliance re-run produce the same
7
+ boxes. The graph's one-to-many head has no NMS: output0 is [1, 4 + 5, anchors], per anchor the box centre x, y, width
8
+ and height in input pixels and the 5 class scores. Pure numpy/cv2 pre/post-processing:
9
+ pre centred letterbox to the graph's input size (704 x 704), INTER_LINEAR, grey 114 border; BGR -> RGB planes
10
+ multiplied by float32(1 / 255)
11
+ post one class per anchor (its highest score), keep scores >= 0.25, boxes back to picture pixels and clipped to
12
+ the picture, ordered by score (ties: left edge, then top edge), class-wise greedy NMS (a box drops every
13
+ later box of its class with IoU > 0.5), the first 100 boxes; integer corners floor(x1), floor(y1),
14
+ ceil(x2), ceil(y2) kept inside the picture; the score is kept at full precision (not rounded)
15
  """
16
 
17
+ import math
18
  from pathlib import Path
19
 
20
  import cv2
 
25
 
26
  NUM_CLASSES = 5
27
  DEFAULT_INPUT_SIZE = 640
28
+ # One flat score threshold for every class.
29
+ CONF_THRESHOLDS = (0.25, 0.25, 0.25, 0.25, 0.25)
30
  NMS_IOU = 0.5
31
+ MAX_DETECTIONS = 100
 
 
 
 
 
 
32
  PAD_VALUE = 114
33
+ # A few runs at start-up, so the first scored frame does not pay for onnxruntime's first-run allocations.
34
+ WARMUP_RUNS = 3
35
 
36
 
37
  class BoundingBox(BaseModel):
 
49
  keypoints: list[tuple[int, int]] | None = None
50
 
51
 
52
+ def as_bgr(image: ndarray) -> ndarray:
53
+ """The frame as a contiguous 3-channel BGR uint8 picture (grey, BGRA and non-uint8 frames are converted)."""
54
+ arr = np.asarray(image)
55
+ if arr.ndim == 2:
56
+ arr = cv2.cvtColor(arr if arr.dtype == np.uint8 else np.clip(arr, 0, 255).astype(np.uint8), cv2.COLOR_GRAY2BGR)
57
+ if arr.dtype != np.uint8:
58
+ arr = np.clip(arr, 0, 255).astype(np.uint8)
59
+ if arr.ndim == 3 and arr.shape[2] == 4:
60
+ arr = cv2.cvtColor(arr, cv2.COLOR_BGRA2BGR)
61
+ elif arr.ndim == 3 and arr.shape[2] == 1:
62
+ arr = cv2.cvtColor(arr, cv2.COLOR_GRAY2BGR)
63
+ return np.ascontiguousarray(arr)
64
+
65
+
66
+ def letterbox(image: ndarray, in_h: int, in_w: int) -> tuple[ndarray, float, int, int]:
67
+ """Resize keeping the aspect ratio and pad to in_h x in_w, centred (Ultralytics LetterBox).
68
+ Returns the (1, 3, in_h, in_w) float32 RGB input, the scale and the left / top border."""
69
  h, w = image.shape[:2]
70
+ r = min(in_h / h, in_w / w)
71
+ new_h = min(max(1, int(round(h * r))), in_h)
72
+ new_w = min(max(1, int(round(w * r))), in_w)
73
+ if (new_h, new_w) != (h, w):
74
  image = cv2.resize(image, (new_w, new_h), interpolation=cv2.INTER_LINEAR)
75
+ top, left = (in_h - new_h) // 2, (in_w - new_w) // 2
76
+ if (new_h, new_w) != (in_h, in_w):
77
+ canvas = np.full((in_h, in_w, 3), PAD_VALUE, dtype=np.uint8)
78
+ canvas[top:top + new_h, left:left + new_w] = image
79
+ image = canvas
80
+ x = np.empty((1, 3, in_h, in_w), dtype=np.float32)
81
+ scale = np.float32(1.0 / 255.0)
82
+ for c in range(3): # plane c of the input = RGB channel c = BGR channel 2 - c
83
+ np.multiply(image[:, :, 2 - c], scale, out=x[0, c], dtype=np.float32)
84
+ return x, r, left, top
85
+
86
+
87
+ def nms_classwise(boxes: ndarray, cls_ids: ndarray, iou_threshold: float) -> list[int]:
88
+ """Greedy NMS in the given order: each kept box drops every later box of its class with IoU > iou_threshold."""
89
+ n = boxes.shape[0]
90
+ areas = (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1])
91
+ dropped = np.zeros(n, dtype=bool)
 
 
 
 
 
 
 
 
 
 
 
92
  keep = []
93
+ for i in range(n):
94
+ if dropped[i]:
95
+ continue
96
+ keep.append(i)
97
+ rest = np.arange(i + 1, n)
98
+ rest = rest[(~dropped[rest]) & (cls_ids[rest] == cls_ids[i])]
99
+ if rest.size == 0:
100
+ continue
101
+ iw = np.clip(np.minimum(boxes[i, 2], boxes[rest, 2]) - np.maximum(boxes[i, 0], boxes[rest, 0]), 0, None)
102
+ ih = np.clip(np.minimum(boxes[i, 3], boxes[rest, 3]) - np.maximum(boxes[i, 1], boxes[rest, 1]), 0, None)
103
  inter = iw * ih
104
+ union = areas[i] + areas[rest] - inter
105
+ iou = np.where(union > 0, inter / np.maximum(union, 1e-9), 0.0)
106
+ dropped[rest[iou > iou_threshold]] = True
107
  return keep
108
 
109
 
 
114
  options.inter_op_num_threads = 1
115
  options.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL
116
  options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
117
+ options.add_session_config_entry("session.intra_op.allow_spinning", "1")
118
+ options.add_session_config_entry("session.inter_op.allow_spinning", "1")
119
  self.session = ort.InferenceSession(str(Path(path_hf_repo) / "model.onnx"), sess_options=options,
120
  providers=["CPUExecutionProvider"])
121
  model_input = self.session.get_inputs()[0]
122
+ model_output = self.session.get_outputs()[0]
123
+ self.input_name, self.output_name = model_input.name, model_output.name
124
+ shape = model_input.shape
125
+ if len(shape) == 4 and all(isinstance(d, int) and d > 0 for d in shape[2:]):
126
+ self.input_h, self.input_w = int(shape[2]), int(shape[3])
127
+ else:
128
+ self.input_h = self.input_w = DEFAULT_INPUT_SIZE
129
+ out_shape = model_output.shape
130
+ if len(out_shape) != 3 or (isinstance(out_shape[1], int) and out_shape[1] != 4 + NUM_CLASSES):
131
+ raise ValueError(f"model.onnx output {model_output.name} {out_shape} is not [1, {4 + NUM_CLASSES}, anchors]")
132
  self.conf_thresholds = np.array(CONF_THRESHOLDS, dtype=np.float32)
133
+ for _ in range(WARMUP_RUNS):
134
+ self.detect(np.full((1024, 1024, 3), PAD_VALUE, dtype=np.uint8))
135
 
136
  def __repr__(self) -> str:
137
+ return (f"Room-dirtiness YOLO26 ONNX detector ({self.input_h}x{self.input_w}px, "
138
+ f"providers={self.session.get_providers()})")
139
 
140
  def detect(self, image: ndarray) -> list[tuple[int, int, int, int, int, float]]:
141
  """Boxes for one BGR image: (x1, y1, x2, y2, cls_id, conf) in original pixel coordinates."""
142
+ image = as_bgr(image)
143
  h, w = image.shape[:2]
144
+ x, r, left, top = letterbox(image, self.input_h, self.input_w)
145
+ out = np.asarray(self.session.run([self.output_name], {self.input_name: x})[0], dtype=np.float32)
146
+ pred = out[0][: 4 + NUM_CLASSES].T # (anchors, 4 + classes)
 
147
  scores = pred[:, 4:]
148
+ cls_ids = scores.argmax(1)
149
+ conf = scores[np.arange(scores.shape[0]), cls_ids]
150
+ keep = np.isfinite(conf) & (conf >= self.conf_thresholds[cls_ids])
151
+ if not np.any(keep):
152
  return []
153
+ box, conf, cls_ids = pred[keep, :4], conf[keep], cls_ids[keep]
154
+ cx, cy, bw, bh = box[:, 0], box[:, 1], box[:, 2], box[:, 3]
155
+ boxes = np.stack([(cx - bw / 2 - left) / r, (cy - bh / 2 - top) / r,
156
+ (cx + bw / 2 - left) / r, (cy + bh / 2 - top) / r], 1)
157
+ finite = np.isfinite(boxes).all(1) & np.isfinite(conf)
158
+ if not np.all(finite):
159
+ boxes, conf, cls_ids = boxes[finite], conf[finite], cls_ids[finite]
160
+ if boxes.shape[0] == 0:
161
+ return []
162
+ boxes[:, [0, 2]] = np.clip(boxes[:, [0, 2]], 0, w)
163
+ boxes[:, [1, 3]] = np.clip(boxes[:, [1, 3]], 0, h)
164
+ order = np.lexsort((boxes[:, 1], boxes[:, 0], -conf)) # score high to low, then left edge, then top edge
165
+ boxes, conf, cls_ids = boxes[order], conf[order], cls_ids[order]
166
  results = []
167
+ for i in nms_classwise(boxes, cls_ids, NMS_IOU)[:MAX_DETECTIONS]:
168
+ xa, ya, xb, yb = (float(v) for v in boxes[i])
169
+ x1 = max(0, min(int(math.floor(xa)), w - 1))
170
+ y1 = max(0, min(int(math.floor(ya)), h - 1))
171
+ x2 = max(x1 + 1, min(int(math.ceil(xb)), w))
172
+ y2 = max(y1 + 1, min(int(math.ceil(yb)), h))
173
+ results.append((x1, y1, x2, y2, int(cls_ids[i]), min(1.0, max(0.0, float(conf[i])))))
174
+ return results
 
 
 
 
 
 
 
 
 
 
 
 
175
 
176
  def predict_batch(self, batch_images: list[ndarray], offset: int, n_keypoints: int) -> list[TVFrameResult]:
177
  results = []
178
  for i, image in enumerate(batch_images):
179
+ boxes = [BoundingBox(x1=x1, y1=y1, x2=x2, y2=y2, cls_id=c, conf=s)
180
  for x1, y1, x2, y2, c, s in self.detect(image)]
181
  results.append(TVFrameResult(frame_id=offset + i, boxes=boxes, keypoints=[(0, 0)] * n_keypoints))
182
  return results