"""Room-dirtiness detector for Score (SN44) element manako/Detect-room-dirtiness-detection, YOLO26n graph. Classes (cls_id order fixed by the element manifest): 0 liquid spillage, 1 bin, 2 litter, 3 mould, 4 stain Runs a YOLO26n ONNX detector on CPU with onnxruntime, so the chute and Score's CPU compliance re-run produce the same boxes. The graph's one-to-many head has no NMS: output0 is [1, 4 + 5, anchors], per anchor the box centre x, y, width and height in input pixels and the 5 class scores. Pure numpy/cv2 pre/post-processing: pre centred letterbox to the graph's input size (704 x 704), INTER_LINEAR, grey 114 border; BGR -> RGB planes multiplied by float32(1 / 255) post one class per anchor (its highest score), keep scores >= 0.25, boxes back to picture pixels and clipped to the picture, ordered by score (ties: left edge, then top edge), class-wise greedy NMS (a box drops every later box of its class with IoU > 0.5), the first 100 boxes; integer corners floor(x1), floor(y1), ceil(x2), ceil(y2) kept inside the picture; the score is kept at full precision (not rounded) """ import math from pathlib import Path import cv2 import numpy as np import onnxruntime as ort from numpy import ndarray from pydantic import BaseModel NUM_CLASSES = 5 DEFAULT_INPUT_SIZE = 640 # One flat score threshold for every class. CONF_THRESHOLDS = (0.25, 0.25, 0.25, 0.25, 0.25) NMS_IOU = 0.5 MAX_DETECTIONS = 100 PAD_VALUE = 114 # A few runs at start-up, so the first scored frame does not pay for onnxruntime's first-run allocations. WARMUP_RUNS = 3 class BoundingBox(BaseModel): x1: int y1: int x2: int y2: int cls_id: int conf: float class TVFrameResult(BaseModel): frame_id: int boxes: list[BoundingBox] | None = None keypoints: list[tuple[int, int]] | None = None def as_bgr(image: ndarray) -> ndarray: """The frame as a contiguous 3-channel BGR uint8 picture (grey, BGRA and non-uint8 frames are converted).""" arr = np.asarray(image) if arr.ndim == 2: arr = cv2.cvtColor(arr if arr.dtype == np.uint8 else np.clip(arr, 0, 255).astype(np.uint8), cv2.COLOR_GRAY2BGR) if arr.dtype != np.uint8: arr = np.clip(arr, 0, 255).astype(np.uint8) if arr.ndim == 3 and arr.shape[2] == 4: arr = cv2.cvtColor(arr, cv2.COLOR_BGRA2BGR) elif arr.ndim == 3 and arr.shape[2] == 1: arr = cv2.cvtColor(arr, cv2.COLOR_GRAY2BGR) return np.ascontiguousarray(arr) def letterbox(image: ndarray, in_h: int, in_w: int) -> tuple[ndarray, float, int, int]: """Resize keeping the aspect ratio and pad to in_h x in_w, centred (Ultralytics LetterBox). Returns the (1, 3, in_h, in_w) float32 RGB input, the scale and the left / top border.""" h, w = image.shape[:2] r = min(in_h / h, in_w / w) new_h = min(max(1, int(round(h * r))), in_h) new_w = min(max(1, int(round(w * r))), in_w) if (new_h, new_w) != (h, w): image = cv2.resize(image, (new_w, new_h), interpolation=cv2.INTER_LINEAR) top, left = (in_h - new_h) // 2, (in_w - new_w) // 2 if (new_h, new_w) != (in_h, in_w): canvas = np.full((in_h, in_w, 3), PAD_VALUE, dtype=np.uint8) canvas[top:top + new_h, left:left + new_w] = image image = canvas x = np.empty((1, 3, in_h, in_w), dtype=np.float32) scale = np.float32(1.0 / 255.0) for c in range(3): # plane c of the input = RGB channel c = BGR channel 2 - c np.multiply(image[:, :, 2 - c], scale, out=x[0, c], dtype=np.float32) return x, r, left, top def nms_classwise(boxes: ndarray, cls_ids: ndarray, iou_threshold: float) -> list[int]: """Greedy NMS in the given order: each kept box drops every later box of its class with IoU > iou_threshold.""" n = boxes.shape[0] areas = (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1]) dropped = np.zeros(n, dtype=bool) keep = [] for i in range(n): if dropped[i]: continue keep.append(i) rest = np.arange(i + 1, n) rest = rest[(~dropped[rest]) & (cls_ids[rest] == cls_ids[i])] if rest.size == 0: continue iw = np.clip(np.minimum(boxes[i, 2], boxes[rest, 2]) - np.maximum(boxes[i, 0], boxes[rest, 0]), 0, None) ih = np.clip(np.minimum(boxes[i, 3], boxes[rest, 3]) - np.maximum(boxes[i, 1], boxes[rest, 1]), 0, None) inter = iw * ih union = areas[i] + areas[rest] - inter iou = np.where(union > 0, inter / np.maximum(union, 1e-9), 0.0) dropped[rest[iou > iou_threshold]] = True return keep class Miner: def __init__(self, path_hf_repo: Path) -> None: options = ort.SessionOptions() options.intra_op_num_threads = 2 options.inter_op_num_threads = 1 options.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL options.add_session_config_entry("session.intra_op.allow_spinning", "1") options.add_session_config_entry("session.inter_op.allow_spinning", "1") self.session = ort.InferenceSession(str(Path(path_hf_repo) / "model.onnx"), sess_options=options, providers=["CPUExecutionProvider"]) model_input = self.session.get_inputs()[0] model_output = self.session.get_outputs()[0] self.input_name, self.output_name = model_input.name, model_output.name shape = model_input.shape if len(shape) == 4 and all(isinstance(d, int) and d > 0 for d in shape[2:]): self.input_h, self.input_w = int(shape[2]), int(shape[3]) else: self.input_h = self.input_w = DEFAULT_INPUT_SIZE out_shape = model_output.shape if len(out_shape) != 3 or (isinstance(out_shape[1], int) and out_shape[1] != 4 + NUM_CLASSES): raise ValueError(f"model.onnx output {model_output.name} {out_shape} is not [1, {4 + NUM_CLASSES}, anchors]") self.conf_thresholds = np.array(CONF_THRESHOLDS, dtype=np.float32) for _ in range(WARMUP_RUNS): self.detect(np.full((1024, 1024, 3), PAD_VALUE, dtype=np.uint8)) def __repr__(self) -> str: return (f"Room-dirtiness YOLO26 ONNX detector ({self.input_h}x{self.input_w}px, " f"providers={self.session.get_providers()})") def detect(self, image: ndarray) -> list[tuple[int, int, int, int, int, float]]: """Boxes for one BGR image: (x1, y1, x2, y2, cls_id, conf) in original pixel coordinates.""" image = as_bgr(image) h, w = image.shape[:2] x, r, left, top = letterbox(image, self.input_h, self.input_w) out = np.asarray(self.session.run([self.output_name], {self.input_name: x})[0], dtype=np.float32) pred = out[0][: 4 + NUM_CLASSES].T # (anchors, 4 + classes) scores = pred[:, 4:] cls_ids = scores.argmax(1) conf = scores[np.arange(scores.shape[0]), cls_ids] keep = np.isfinite(conf) & (conf >= self.conf_thresholds[cls_ids]) if not np.any(keep): return [] box, conf, cls_ids = pred[keep, :4], conf[keep], cls_ids[keep] cx, cy, bw, bh = box[:, 0], box[:, 1], box[:, 2], box[:, 3] boxes = np.stack([(cx - bw / 2 - left) / r, (cy - bh / 2 - top) / r, (cx + bw / 2 - left) / r, (cy + bh / 2 - top) / r], 1) finite = np.isfinite(boxes).all(1) & np.isfinite(conf) if not np.all(finite): boxes, conf, cls_ids = boxes[finite], conf[finite], cls_ids[finite] if boxes.shape[0] == 0: return [] boxes[:, [0, 2]] = np.clip(boxes[:, [0, 2]], 0, w) boxes[:, [1, 3]] = np.clip(boxes[:, [1, 3]], 0, h) order = np.lexsort((boxes[:, 1], boxes[:, 0], -conf)) # score high to low, then left edge, then top edge boxes, conf, cls_ids = boxes[order], conf[order], cls_ids[order] results = [] for i in nms_classwise(boxes, cls_ids, NMS_IOU)[:MAX_DETECTIONS]: xa, ya, xb, yb = (float(v) for v in boxes[i]) x1 = max(0, min(int(math.floor(xa)), w - 1)) y1 = max(0, min(int(math.floor(ya)), h - 1)) x2 = max(x1 + 1, min(int(math.ceil(xb)), w)) y2 = max(y1 + 1, min(int(math.ceil(yb)), h)) results.append((x1, y1, x2, y2, int(cls_ids[i]), min(1.0, max(0.0, float(conf[i]))))) return results def predict_batch(self, batch_images: list[ndarray], offset: int, n_keypoints: int) -> list[TVFrameResult]: results = [] for i, image in enumerate(batch_images): boxes = [BoundingBox(x1=x1, y1=y1, x2=x2, y2=y2, cls_id=c, conf=s) for x1, y1, x2, y2, c, s in self.detect(image)] results.append(TVFrameResult(frame_id=offset + i, boxes=boxes, keypoints=[(0, 0)] * n_keypoints)) return results