ScoreVision / miner.py
ninja990621's picture
scorevision: push artifact
c599dd5 verified
Raw History Blame Contribute Delete
8.92 kB
"""Room-dirtiness detector for Score (SN44) element manako/Detect-room-dirtiness-detection, YOLO26n graph.
Classes (cls_id order fixed by the element manifest):
0 liquid spillage, 1 bin, 2 litter, 3 mould, 4 stain
Runs a YOLO26n ONNX detector on CPU with onnxruntime, so the chute and Score's CPU compliance re-run produce the same
boxes. The graph's one-to-many head has no NMS: output0 is [1, 4 + 5, anchors], per anchor the box centre x, y, width
and height in input pixels and the 5 class scores. Pure numpy/cv2 pre/post-processing:
pre centred letterbox to the graph's input size (704 x 704), INTER_LINEAR, grey 114 border; BGR -> RGB planes
multiplied by float32(1 / 255)
post one class per anchor (its highest score), keep scores >= 0.25, boxes back to picture pixels and clipped to
the picture, ordered by score (ties: left edge, then top edge), class-wise greedy NMS (a box drops every
later box of its class with IoU > 0.5), the first 100 boxes; integer corners floor(x1), floor(y1),
ceil(x2), ceil(y2) kept inside the picture; the score is kept at full precision (not rounded)
"""
import math
from pathlib import Path
import cv2
import numpy as np
import onnxruntime as ort
from numpy import ndarray
from pydantic import BaseModel
NUM_CLASSES = 5
DEFAULT_INPUT_SIZE = 640
# One flat score threshold for every class.
CONF_THRESHOLDS = (0.25, 0.25, 0.25, 0.25, 0.25)
NMS_IOU = 0.5
MAX_DETECTIONS = 100
PAD_VALUE = 114
# A few runs at start-up, so the first scored frame does not pay for onnxruntime's first-run allocations.
WARMUP_RUNS = 3
class BoundingBox(BaseModel):
x1: int
y1: int
x2: int
y2: int
cls_id: int
conf: float
class TVFrameResult(BaseModel):
frame_id: int
boxes: list[BoundingBox] | None = None
keypoints: list[tuple[int, int]] | None = None
def as_bgr(image: ndarray) -> ndarray:
"""The frame as a contiguous 3-channel BGR uint8 picture (grey, BGRA and non-uint8 frames are converted)."""
arr = np.asarray(image)
if arr.ndim == 2:
arr = cv2.cvtColor(arr if arr.dtype == np.uint8 else np.clip(arr, 0, 255).astype(np.uint8), cv2.COLOR_GRAY2BGR)
if arr.dtype != np.uint8:
arr = np.clip(arr, 0, 255).astype(np.uint8)
if arr.ndim == 3 and arr.shape[2] == 4:
arr = cv2.cvtColor(arr, cv2.COLOR_BGRA2BGR)
elif arr.ndim == 3 and arr.shape[2] == 1:
arr = cv2.cvtColor(arr, cv2.COLOR_GRAY2BGR)
return np.ascontiguousarray(arr)
def letterbox(image: ndarray, in_h: int, in_w: int) -> tuple[ndarray, float, int, int]:
"""Resize keeping the aspect ratio and pad to in_h x in_w, centred (Ultralytics LetterBox).
Returns the (1, 3, in_h, in_w) float32 RGB input, the scale and the left / top border."""
h, w = image.shape[:2]
r = min(in_h / h, in_w / w)
new_h = min(max(1, int(round(h * r))), in_h)
new_w = min(max(1, int(round(w * r))), in_w)
if (new_h, new_w) != (h, w):
image = cv2.resize(image, (new_w, new_h), interpolation=cv2.INTER_LINEAR)
top, left = (in_h - new_h) // 2, (in_w - new_w) // 2
if (new_h, new_w) != (in_h, in_w):
canvas = np.full((in_h, in_w, 3), PAD_VALUE, dtype=np.uint8)
canvas[top:top + new_h, left:left + new_w] = image
image = canvas
x = np.empty((1, 3, in_h, in_w), dtype=np.float32)
scale = np.float32(1.0 / 255.0)
for c in range(3): # plane c of the input = RGB channel c = BGR channel 2 - c
np.multiply(image[:, :, 2 - c], scale, out=x[0, c], dtype=np.float32)
return x, r, left, top
def nms_classwise(boxes: ndarray, cls_ids: ndarray, iou_threshold: float) -> list[int]:
"""Greedy NMS in the given order: each kept box drops every later box of its class with IoU > iou_threshold."""
n = boxes.shape[0]
areas = (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1])
dropped = np.zeros(n, dtype=bool)
keep = []
for i in range(n):
if dropped[i]:
continue
keep.append(i)
rest = np.arange(i + 1, n)
rest = rest[(~dropped[rest]) & (cls_ids[rest] == cls_ids[i])]
if rest.size == 0:
continue
iw = np.clip(np.minimum(boxes[i, 2], boxes[rest, 2]) - np.maximum(boxes[i, 0], boxes[rest, 0]), 0, None)
ih = np.clip(np.minimum(boxes[i, 3], boxes[rest, 3]) - np.maximum(boxes[i, 1], boxes[rest, 1]), 0, None)
inter = iw * ih
union = areas[i] + areas[rest] - inter
iou = np.where(union > 0, inter / np.maximum(union, 1e-9), 0.0)
dropped[rest[iou > iou_threshold]] = True
return keep
class Miner:
def __init__(self, path_hf_repo: Path) -> None:
options = ort.SessionOptions()
options.intra_op_num_threads = 2
options.inter_op_num_threads = 1
options.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL
options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
options.add_session_config_entry("session.intra_op.allow_spinning", "1")
options.add_session_config_entry("session.inter_op.allow_spinning", "1")
self.session = ort.InferenceSession(str(Path(path_hf_repo) / "model.onnx"), sess_options=options,
providers=["CPUExecutionProvider"])
model_input = self.session.get_inputs()[0]
model_output = self.session.get_outputs()[0]
self.input_name, self.output_name = model_input.name, model_output.name
shape = model_input.shape
if len(shape) == 4 and all(isinstance(d, int) and d > 0 for d in shape[2:]):
self.input_h, self.input_w = int(shape[2]), int(shape[3])
else:
self.input_h = self.input_w = DEFAULT_INPUT_SIZE
out_shape = model_output.shape
if len(out_shape) != 3 or (isinstance(out_shape[1], int) and out_shape[1] != 4 + NUM_CLASSES):
raise ValueError(f"model.onnx output {model_output.name} {out_shape} is not [1, {4 + NUM_CLASSES}, anchors]")
self.conf_thresholds = np.array(CONF_THRESHOLDS, dtype=np.float32)
for _ in range(WARMUP_RUNS):
self.detect(np.full((1024, 1024, 3), PAD_VALUE, dtype=np.uint8))
def __repr__(self) -> str:
return (f"Room-dirtiness YOLO26 ONNX detector ({self.input_h}x{self.input_w}px, "
f"providers={self.session.get_providers()})")
def detect(self, image: ndarray) -> list[tuple[int, int, int, int, int, float]]:
"""Boxes for one BGR image: (x1, y1, x2, y2, cls_id, conf) in original pixel coordinates."""
image = as_bgr(image)
h, w = image.shape[:2]
x, r, left, top = letterbox(image, self.input_h, self.input_w)
out = np.asarray(self.session.run([self.output_name], {self.input_name: x})[0], dtype=np.float32)
pred = out[0][: 4 + NUM_CLASSES].T # (anchors, 4 + classes)
scores = pred[:, 4:]
cls_ids = scores.argmax(1)
conf = scores[np.arange(scores.shape[0]), cls_ids]
keep = np.isfinite(conf) & (conf >= self.conf_thresholds[cls_ids])
if not np.any(keep):
return []
box, conf, cls_ids = pred[keep, :4], conf[keep], cls_ids[keep]
cx, cy, bw, bh = box[:, 0], box[:, 1], box[:, 2], box[:, 3]
boxes = np.stack([(cx - bw / 2 - left) / r, (cy - bh / 2 - top) / r,
(cx + bw / 2 - left) / r, (cy + bh / 2 - top) / r], 1)
finite = np.isfinite(boxes).all(1) & np.isfinite(conf)
if not np.all(finite):
boxes, conf, cls_ids = boxes[finite], conf[finite], cls_ids[finite]
if boxes.shape[0] == 0:
return []
boxes[:, [0, 2]] = np.clip(boxes[:, [0, 2]], 0, w)
boxes[:, [1, 3]] = np.clip(boxes[:, [1, 3]], 0, h)
order = np.lexsort((boxes[:, 1], boxes[:, 0], -conf)) # score high to low, then left edge, then top edge
boxes, conf, cls_ids = boxes[order], conf[order], cls_ids[order]
results = []
for i in nms_classwise(boxes, cls_ids, NMS_IOU)[:MAX_DETECTIONS]:
xa, ya, xb, yb = (float(v) for v in boxes[i])
x1 = max(0, min(int(math.floor(xa)), w - 1))
y1 = max(0, min(int(math.floor(ya)), h - 1))
x2 = max(x1 + 1, min(int(math.ceil(xb)), w))
y2 = max(y1 + 1, min(int(math.ceil(yb)), h))
results.append((x1, y1, x2, y2, int(cls_ids[i]), min(1.0, max(0.0, float(conf[i])))))
return results
def predict_batch(self, batch_images: list[ndarray], offset: int, n_keypoints: int) -> list[TVFrameResult]:
results = []
for i, image in enumerate(batch_images):
boxes = [BoundingBox(x1=x1, y1=y1, x2=x2, y2=y2, cls_id=c, conf=s)
for x1, y1, x2, y2, c, s in self.detect(image)]
results.append(TVFrameResult(frame_id=offset + i, boxes=boxes, keypoints=[(0, 0)] * n_keypoints))
return results