scorevision: push artifact
Browse files
miner.py
CHANGED
|
@@ -1,12 +1,20 @@
|
|
| 1 |
-
"""Room-dirtiness detector for Score (SN44) element manako/Detect-room-dirtiness-detection.
|
| 2 |
|
| 3 |
Classes (cls_id order fixed by the element manifest):
|
| 4 |
0 liquid spillage, 1 bin, 2 litter, 3 mould, 4 stain
|
| 5 |
|
| 6 |
-
Runs a
|
| 7 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8 |
"""
|
| 9 |
|
|
|
|
| 10 |
from pathlib import Path
|
| 11 |
|
| 12 |
import cv2
|
|
@@ -17,17 +25,13 @@ from pydantic import BaseModel
|
|
| 17 |
|
| 18 |
NUM_CLASSES = 5
|
| 19 |
DEFAULT_INPUT_SIZE = 640
|
| 20 |
-
#
|
| 21 |
-
CONF_THRESHOLDS = (0.25, 0.25, 0.25, 0.25, 0.
|
| 22 |
NMS_IOU = 0.5
|
| 23 |
-
|
| 24 |
-
CROSS_CLASS_IOU = 2
|
| 25 |
-
# Score's labeller gives one box per object and drops a mask lying inside a bigger mask of the same prompt, so a
|
| 26 |
-
# same-class box mostly inside a bigger, about as confident box is a piece of that object (e.g. one item of a pile).
|
| 27 |
-
INSIDE_SHARE = 0.7
|
| 28 |
-
INSIDE_CONF_RATIO = 0.9
|
| 29 |
-
MAX_DETECTIONS = 30
|
| 30 |
PAD_VALUE = 114
|
|
|
|
|
|
|
| 31 |
|
| 32 |
|
| 33 |
class BoundingBox(BaseModel):
|
|
@@ -45,51 +49,61 @@ class TVFrameResult(BaseModel):
|
|
| 45 |
keypoints: list[tuple[int, int]] | None = None
|
| 46 |
|
| 47 |
|
| 48 |
-
def
|
| 49 |
-
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 50 |
h, w = image.shape[:2]
|
| 51 |
-
r = min(
|
| 52 |
-
|
| 53 |
-
|
|
|
|
| 54 |
image = cv2.resize(image, (new_w, new_h), interpolation=cv2.INTER_LINEAR)
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
"""Share of the smaller box's area covered by the other box."""
|
| 73 |
-
iw = max(0.0, min(a[2], b[2]) - max(a[0], b[0]))
|
| 74 |
-
ih = max(0.0, min(a[3], b[3]) - max(a[1], b[1]))
|
| 75 |
-
smaller = min((a[2] - a[0]) * (a[3] - a[1]), (b[2] - b[0]) * (b[3] - b[1]))
|
| 76 |
-
return float(iw * ih / smaller) if smaller > 0 else 0.0
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
def nms(boxes: ndarray, scores: ndarray, iou_threshold: float) -> list[int]:
|
| 80 |
-
x1, y1, x2, y2 = boxes.T
|
| 81 |
-
areas = (x2 - x1).clip(0) * (y2 - y1).clip(0)
|
| 82 |
-
order = scores.argsort()[::-1]
|
| 83 |
keep = []
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 90 |
inter = iw * ih
|
| 91 |
-
|
| 92 |
-
|
|
|
|
| 93 |
return keep
|
| 94 |
|
| 95 |
|
|
@@ -100,61 +114,69 @@ class Miner:
|
|
| 100 |
options.inter_op_num_threads = 1
|
| 101 |
options.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL
|
| 102 |
options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
|
|
|
|
|
|
|
| 103 |
self.session = ort.InferenceSession(str(Path(path_hf_repo) / "model.onnx"), sess_options=options,
|
| 104 |
providers=["CPUExecutionProvider"])
|
| 105 |
model_input = self.session.get_inputs()[0]
|
| 106 |
-
|
| 107 |
-
|
| 108 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 109 |
self.conf_thresholds = np.array(CONF_THRESHOLDS, dtype=np.float32)
|
| 110 |
-
|
|
|
|
| 111 |
|
| 112 |
def __repr__(self) -> str:
|
| 113 |
-
return f"Room-dirtiness ONNX detector ({self.
|
|
|
|
| 114 |
|
| 115 |
def detect(self, image: ndarray) -> list[tuple[int, int, int, int, int, float]]:
|
| 116 |
"""Boxes for one BGR image: (x1, y1, x2, y2, cls_id, conf) in original pixel coordinates."""
|
|
|
|
| 117 |
h, w = image.shape[:2]
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
pred = out[: 4 + NUM_CLASSES].T
|
| 122 |
scores = pred[:, 4:]
|
| 123 |
-
|
| 124 |
-
|
|
|
|
|
|
|
| 125 |
return []
|
| 126 |
-
conf =
|
| 127 |
-
cx, cy, bw, bh =
|
| 128 |
-
boxes = np.stack([cx - bw / 2
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 132 |
results = []
|
| 133 |
-
for
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
for b, c, sc in results:
|
| 142 |
-
if all(kc == c or box_iou(b, kb) < CROSS_CLASS_IOU for kb, kc, _ in kept):
|
| 143 |
-
kept.append((b, c, sc))
|
| 144 |
-
results = kept
|
| 145 |
-
kept = []
|
| 146 |
-
for b, c, sc in sorted(results, key=lambda t: -(t[0][2] - t[0][0]) * (t[0][3] - t[0][1])):
|
| 147 |
-
if not any(kc == c and ks >= INSIDE_CONF_RATIO * sc and inside_share(b, kb) >= INSIDE_SHARE
|
| 148 |
-
for kb, kc, ks in kept):
|
| 149 |
-
kept.append((b, c, sc))
|
| 150 |
-
results = sorted(kept, key=lambda t: -t[2])
|
| 151 |
-
return [(int(round(b[0])), int(round(b[1])), int(round(b[2])), int(round(b[3])), c, s)
|
| 152 |
-
for b, c, s in results[:MAX_DETECTIONS] if b[2] - b[0] >= 1 and b[3] - b[1] >= 1]
|
| 153 |
|
| 154 |
def predict_batch(self, batch_images: list[ndarray], offset: int, n_keypoints: int) -> list[TVFrameResult]:
|
| 155 |
results = []
|
| 156 |
for i, image in enumerate(batch_images):
|
| 157 |
-
boxes = [BoundingBox(x1=x1, y1=y1, x2=x2, y2=y2, cls_id=c, conf=
|
| 158 |
for x1, y1, x2, y2, c, s in self.detect(image)]
|
| 159 |
results.append(TVFrameResult(frame_id=offset + i, boxes=boxes, keypoints=[(0, 0)] * n_keypoints))
|
| 160 |
return results
|
|
|
|
| 1 |
+
"""Room-dirtiness detector for Score (SN44) element manako/Detect-room-dirtiness-detection, YOLO26n graph.
|
| 2 |
|
| 3 |
Classes (cls_id order fixed by the element manifest):
|
| 4 |
0 liquid spillage, 1 bin, 2 litter, 3 mould, 4 stain
|
| 5 |
|
| 6 |
+
Runs a YOLO26n ONNX detector on CPU with onnxruntime, so the chute and Score's CPU compliance re-run produce the same
|
| 7 |
+
boxes. The graph's one-to-many head has no NMS: output0 is [1, 4 + 5, anchors], per anchor the box centre x, y, width
|
| 8 |
+
and height in input pixels and the 5 class scores. Pure numpy/cv2 pre/post-processing:
|
| 9 |
+
pre centred letterbox to the graph's input size (704 x 704), INTER_LINEAR, grey 114 border; BGR -> RGB planes
|
| 10 |
+
multiplied by float32(1 / 255)
|
| 11 |
+
post one class per anchor (its highest score), keep scores >= 0.25, boxes back to picture pixels and clipped to
|
| 12 |
+
the picture, ordered by score (ties: left edge, then top edge), class-wise greedy NMS (a box drops every
|
| 13 |
+
later box of its class with IoU > 0.5), the first 100 boxes; integer corners floor(x1), floor(y1),
|
| 14 |
+
ceil(x2), ceil(y2) kept inside the picture; the score is kept at full precision (not rounded)
|
| 15 |
"""
|
| 16 |
|
| 17 |
+
import math
|
| 18 |
from pathlib import Path
|
| 19 |
|
| 20 |
import cv2
|
|
|
|
| 25 |
|
| 26 |
NUM_CLASSES = 5
|
| 27 |
DEFAULT_INPUT_SIZE = 640
|
| 28 |
+
# One flat score threshold for every class.
|
| 29 |
+
CONF_THRESHOLDS = (0.25, 0.25, 0.25, 0.25, 0.25)
|
| 30 |
NMS_IOU = 0.5
|
| 31 |
+
MAX_DETECTIONS = 100
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 32 |
PAD_VALUE = 114
|
| 33 |
+
# A few runs at start-up, so the first scored frame does not pay for onnxruntime's first-run allocations.
|
| 34 |
+
WARMUP_RUNS = 3
|
| 35 |
|
| 36 |
|
| 37 |
class BoundingBox(BaseModel):
|
|
|
|
| 49 |
keypoints: list[tuple[int, int]] | None = None
|
| 50 |
|
| 51 |
|
| 52 |
+
def as_bgr(image: ndarray) -> ndarray:
|
| 53 |
+
"""The frame as a contiguous 3-channel BGR uint8 picture (grey, BGRA and non-uint8 frames are converted)."""
|
| 54 |
+
arr = np.asarray(image)
|
| 55 |
+
if arr.ndim == 2:
|
| 56 |
+
arr = cv2.cvtColor(arr if arr.dtype == np.uint8 else np.clip(arr, 0, 255).astype(np.uint8), cv2.COLOR_GRAY2BGR)
|
| 57 |
+
if arr.dtype != np.uint8:
|
| 58 |
+
arr = np.clip(arr, 0, 255).astype(np.uint8)
|
| 59 |
+
if arr.ndim == 3 and arr.shape[2] == 4:
|
| 60 |
+
arr = cv2.cvtColor(arr, cv2.COLOR_BGRA2BGR)
|
| 61 |
+
elif arr.ndim == 3 and arr.shape[2] == 1:
|
| 62 |
+
arr = cv2.cvtColor(arr, cv2.COLOR_GRAY2BGR)
|
| 63 |
+
return np.ascontiguousarray(arr)
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def letterbox(image: ndarray, in_h: int, in_w: int) -> tuple[ndarray, float, int, int]:
|
| 67 |
+
"""Resize keeping the aspect ratio and pad to in_h x in_w, centred (Ultralytics LetterBox).
|
| 68 |
+
Returns the (1, 3, in_h, in_w) float32 RGB input, the scale and the left / top border."""
|
| 69 |
h, w = image.shape[:2]
|
| 70 |
+
r = min(in_h / h, in_w / w)
|
| 71 |
+
new_h = min(max(1, int(round(h * r))), in_h)
|
| 72 |
+
new_w = min(max(1, int(round(w * r))), in_w)
|
| 73 |
+
if (new_h, new_w) != (h, w):
|
| 74 |
image = cv2.resize(image, (new_w, new_h), interpolation=cv2.INTER_LINEAR)
|
| 75 |
+
top, left = (in_h - new_h) // 2, (in_w - new_w) // 2
|
| 76 |
+
if (new_h, new_w) != (in_h, in_w):
|
| 77 |
+
canvas = np.full((in_h, in_w, 3), PAD_VALUE, dtype=np.uint8)
|
| 78 |
+
canvas[top:top + new_h, left:left + new_w] = image
|
| 79 |
+
image = canvas
|
| 80 |
+
x = np.empty((1, 3, in_h, in_w), dtype=np.float32)
|
| 81 |
+
scale = np.float32(1.0 / 255.0)
|
| 82 |
+
for c in range(3): # plane c of the input = RGB channel c = BGR channel 2 - c
|
| 83 |
+
np.multiply(image[:, :, 2 - c], scale, out=x[0, c], dtype=np.float32)
|
| 84 |
+
return x, r, left, top
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def nms_classwise(boxes: ndarray, cls_ids: ndarray, iou_threshold: float) -> list[int]:
|
| 88 |
+
"""Greedy NMS in the given order: each kept box drops every later box of its class with IoU > iou_threshold."""
|
| 89 |
+
n = boxes.shape[0]
|
| 90 |
+
areas = (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1])
|
| 91 |
+
dropped = np.zeros(n, dtype=bool)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 92 |
keep = []
|
| 93 |
+
for i in range(n):
|
| 94 |
+
if dropped[i]:
|
| 95 |
+
continue
|
| 96 |
+
keep.append(i)
|
| 97 |
+
rest = np.arange(i + 1, n)
|
| 98 |
+
rest = rest[(~dropped[rest]) & (cls_ids[rest] == cls_ids[i])]
|
| 99 |
+
if rest.size == 0:
|
| 100 |
+
continue
|
| 101 |
+
iw = np.clip(np.minimum(boxes[i, 2], boxes[rest, 2]) - np.maximum(boxes[i, 0], boxes[rest, 0]), 0, None)
|
| 102 |
+
ih = np.clip(np.minimum(boxes[i, 3], boxes[rest, 3]) - np.maximum(boxes[i, 1], boxes[rest, 1]), 0, None)
|
| 103 |
inter = iw * ih
|
| 104 |
+
union = areas[i] + areas[rest] - inter
|
| 105 |
+
iou = np.where(union > 0, inter / np.maximum(union, 1e-9), 0.0)
|
| 106 |
+
dropped[rest[iou > iou_threshold]] = True
|
| 107 |
return keep
|
| 108 |
|
| 109 |
|
|
|
|
| 114 |
options.inter_op_num_threads = 1
|
| 115 |
options.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL
|
| 116 |
options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
|
| 117 |
+
options.add_session_config_entry("session.intra_op.allow_spinning", "1")
|
| 118 |
+
options.add_session_config_entry("session.inter_op.allow_spinning", "1")
|
| 119 |
self.session = ort.InferenceSession(str(Path(path_hf_repo) / "model.onnx"), sess_options=options,
|
| 120 |
providers=["CPUExecutionProvider"])
|
| 121 |
model_input = self.session.get_inputs()[0]
|
| 122 |
+
model_output = self.session.get_outputs()[0]
|
| 123 |
+
self.input_name, self.output_name = model_input.name, model_output.name
|
| 124 |
+
shape = model_input.shape
|
| 125 |
+
if len(shape) == 4 and all(isinstance(d, int) and d > 0 for d in shape[2:]):
|
| 126 |
+
self.input_h, self.input_w = int(shape[2]), int(shape[3])
|
| 127 |
+
else:
|
| 128 |
+
self.input_h = self.input_w = DEFAULT_INPUT_SIZE
|
| 129 |
+
out_shape = model_output.shape
|
| 130 |
+
if len(out_shape) != 3 or (isinstance(out_shape[1], int) and out_shape[1] != 4 + NUM_CLASSES):
|
| 131 |
+
raise ValueError(f"model.onnx output {model_output.name} {out_shape} is not [1, {4 + NUM_CLASSES}, anchors]")
|
| 132 |
self.conf_thresholds = np.array(CONF_THRESHOLDS, dtype=np.float32)
|
| 133 |
+
for _ in range(WARMUP_RUNS):
|
| 134 |
+
self.detect(np.full((1024, 1024, 3), PAD_VALUE, dtype=np.uint8))
|
| 135 |
|
| 136 |
def __repr__(self) -> str:
|
| 137 |
+
return (f"Room-dirtiness YOLO26 ONNX detector ({self.input_h}x{self.input_w}px, "
|
| 138 |
+
f"providers={self.session.get_providers()})")
|
| 139 |
|
| 140 |
def detect(self, image: ndarray) -> list[tuple[int, int, int, int, int, float]]:
|
| 141 |
"""Boxes for one BGR image: (x1, y1, x2, y2, cls_id, conf) in original pixel coordinates."""
|
| 142 |
+
image = as_bgr(image)
|
| 143 |
h, w = image.shape[:2]
|
| 144 |
+
x, r, left, top = letterbox(image, self.input_h, self.input_w)
|
| 145 |
+
out = np.asarray(self.session.run([self.output_name], {self.input_name: x})[0], dtype=np.float32)
|
| 146 |
+
pred = out[0][: 4 + NUM_CLASSES].T # (anchors, 4 + classes)
|
|
|
|
| 147 |
scores = pred[:, 4:]
|
| 148 |
+
cls_ids = scores.argmax(1)
|
| 149 |
+
conf = scores[np.arange(scores.shape[0]), cls_ids]
|
| 150 |
+
keep = np.isfinite(conf) & (conf >= self.conf_thresholds[cls_ids])
|
| 151 |
+
if not np.any(keep):
|
| 152 |
return []
|
| 153 |
+
box, conf, cls_ids = pred[keep, :4], conf[keep], cls_ids[keep]
|
| 154 |
+
cx, cy, bw, bh = box[:, 0], box[:, 1], box[:, 2], box[:, 3]
|
| 155 |
+
boxes = np.stack([(cx - bw / 2 - left) / r, (cy - bh / 2 - top) / r,
|
| 156 |
+
(cx + bw / 2 - left) / r, (cy + bh / 2 - top) / r], 1)
|
| 157 |
+
finite = np.isfinite(boxes).all(1) & np.isfinite(conf)
|
| 158 |
+
if not np.all(finite):
|
| 159 |
+
boxes, conf, cls_ids = boxes[finite], conf[finite], cls_ids[finite]
|
| 160 |
+
if boxes.shape[0] == 0:
|
| 161 |
+
return []
|
| 162 |
+
boxes[:, [0, 2]] = np.clip(boxes[:, [0, 2]], 0, w)
|
| 163 |
+
boxes[:, [1, 3]] = np.clip(boxes[:, [1, 3]], 0, h)
|
| 164 |
+
order = np.lexsort((boxes[:, 1], boxes[:, 0], -conf)) # score high to low, then left edge, then top edge
|
| 165 |
+
boxes, conf, cls_ids = boxes[order], conf[order], cls_ids[order]
|
| 166 |
results = []
|
| 167 |
+
for i in nms_classwise(boxes, cls_ids, NMS_IOU)[:MAX_DETECTIONS]:
|
| 168 |
+
xa, ya, xb, yb = (float(v) for v in boxes[i])
|
| 169 |
+
x1 = max(0, min(int(math.floor(xa)), w - 1))
|
| 170 |
+
y1 = max(0, min(int(math.floor(ya)), h - 1))
|
| 171 |
+
x2 = max(x1 + 1, min(int(math.ceil(xb)), w))
|
| 172 |
+
y2 = max(y1 + 1, min(int(math.ceil(yb)), h))
|
| 173 |
+
results.append((x1, y1, x2, y2, int(cls_ids[i]), min(1.0, max(0.0, float(conf[i])))))
|
| 174 |
+
return results
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 175 |
|
| 176 |
def predict_batch(self, batch_images: list[ndarray], offset: int, n_keypoints: int) -> list[TVFrameResult]:
|
| 177 |
results = []
|
| 178 |
for i, image in enumerate(batch_images):
|
| 179 |
+
boxes = [BoundingBox(x1=x1, y1=y1, x2=x2, y2=y2, cls_id=c, conf=s)
|
| 180 |
for x1, y1, x2, y2, c, s in self.detect(image)]
|
| 181 |
results.append(TVFrameResult(frame_id=offset + i, boxes=boxes, keypoints=[(0, 0)] * n_keypoints))
|
| 182 |
return results
|