import json from pathlib import Path import cv2 import numpy as np import onnxruntime as ort from numpy import ndarray from pydantic import BaseModel class BoundingBox(BaseModel): x1: int y1: int x2: int y2: int cls_id: int conf: float class TVFrameResult(BaseModel): frame_id: int boxes: list[BoundingBox] | None = None polygons: list = [] keypoints: list[tuple[int, int]] | None = None def _nms(boxes: ndarray, scores: ndarray, iou_thr: float) -> list[int]: order = scores.argsort()[::-1] areas = (boxes[:, 2] - boxes[:, 0]).clip(0) * (boxes[:, 3] - boxes[:, 1]).clip(0) keep: list[int] = [] while order.size: i = int(order[0]) keep.append(i) if order.size == 1: break rest = order[1:] xx1 = np.maximum(boxes[i, 0], boxes[rest, 0]) yy1 = np.maximum(boxes[i, 1], boxes[rest, 1]) xx2 = np.minimum(boxes[i, 2], boxes[rest, 2]) yy2 = np.minimum(boxes[i, 3], boxes[rest, 3]) inter = (xx2 - xx1).clip(0) * (yy2 - yy1).clip(0) iou = inter / (areas[i] + areas[rest] - inter + 1e-9) order = rest[iou <= iou_thr] return keep class Miner: """Room-dirtiness detector: a single ONNX YOLO model run on CPU. CPU execution keeps chute outputs identical to the validator's CPU compliance run. """ def __init__(self, path_hf_repo: Path) -> None: path_hf_repo = Path(path_hf_repo) cfg = json.loads((path_hf_repo / "config.json").read_text()) self.classes: list[str] = cfg["classes"] self.imgsz = int(cfg.get("imgsz", 640)) self.conf_thr = np.asarray(cfg.get("conf", [0.25] * len(self.classes)), dtype=np.float32) nms = cfg.get("nms_iou", 0.6) self.nms_iou = [float(v) for v in nms] if isinstance(nms, list) else [float(nms)] * len(self.classes) self.max_det = int(cfg.get("max_det", 100)) mpc = cfg.get("max_per_class", self.max_det) self.max_per_class = [int(v) for v in mpc] if isinstance(mpc, list) else [int(mpc)] * len(self.classes) # per class: drop a box that lies mostly inside a bigger, higher-scoring box of the same class # (one piece of a larger object boxed on its own); null disables self.contain_drop = cfg.get("contain_drop", [None] * len(self.classes)) # size-aware rules: [area %, conf]; a far bin needs more confidence, a floor-wide spill must not be a faint guess, # and an image whose best box is weak gets no boxes (without GT every false positive costs 0.1) self.small_bin = cfg.get("small_bin") self.big_spill = cfg.get("big_spill") self.weak_image = float(cfg.get("weak_image", 0.0)) # OpenCV's own thread pool is not needed (one resize per frame) and every extra thread counts # against the checker sandbox's process limit. cv2.setNumThreads(1) model_path = str(path_hf_repo / cfg.get("model", "model.onnx")) threads = int(cfg.get("threads", 2)) try: self.session = self._make_session(model_path, threads) except Exception: # The checker sandbox caps processes per user; if no worker thread can be spawned, # run the graph on the calling thread instead of failing to load. self.session = self._make_session(model_path, 1) self.input_name = self.session.get_inputs()[0].name warm = np.zeros((self.imgsz, self.imgsz, 3), dtype=np.uint8) self._detect(warm) @staticmethod def _make_session(model_path: str, threads: int) -> ort.InferenceSession: opts = ort.SessionOptions() opts.intra_op_num_threads = threads opts.inter_op_num_threads = 1 opts.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL opts.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL return ort.InferenceSession(model_path, sess_options=opts, providers=["CPUExecutionProvider"]) def __repr__(self) -> str: return f"ONNX YOLO room-dirtiness imgsz={self.imgsz} classes={self.classes}" def _letterbox(self, img: ndarray) -> tuple[ndarray, float, int, int]: h, w = img.shape[:2] r = min(self.imgsz / h, self.imgsz / w) nh, nw = int(round(h * r)), int(round(w * r)) resized = cv2.resize(img, (nw, nh), interpolation=cv2.INTER_LINEAR) top = (self.imgsz - nh) // 2 left = (self.imgsz - nw) // 2 canvas = np.full((self.imgsz, self.imgsz, 3), 114, dtype=np.uint8) canvas[top : top + nh, left : left + nw] = resized return canvas, r, left, top def _postprocess(self, boxes: ndarray, scores: ndarray, cls: ndarray) -> tuple[ndarray, ndarray, ndarray]: """Score-ordered dedup: containment rule and a per-class cap.""" order = np.argsort(-scores, kind="stable") keep: list[int] = [] per_class: dict[int, int] = {} for i in order: c = int(cls[i]) frac = self.contain_drop[c] if c < len(self.contain_drop) else None if frac is not None: bx = boxes[i] area_i = max(1e-6, (bx[2] - bx[0]) * (bx[3] - bx[1])) dup = False for k in keep: if int(cls[k]) != c: continue kb = boxes[k] iw = max(0.0, min(bx[2], kb[2]) - max(bx[0], kb[0])) ih = max(0.0, min(bx[3], kb[3]) - max(bx[1], kb[1])) area_k = max(1e-6, (kb[2] - kb[0]) * (kb[3] - kb[1])) if area_k > area_i and iw * ih / area_i >= frac: dup = True break if dup: continue if per_class.get(c, 0) >= self.max_per_class[c]: continue per_class[c] = per_class.get(c, 0) + 1 keep.append(int(i)) return boxes[keep], scores[keep], cls[keep] def _detect(self, img_bgr: ndarray) -> list[BoundingBox]: h, w = img_bgr.shape[:2] canvas, r, left, top = self._letterbox(img_bgr) x = cv2.cvtColor(canvas, cv2.COLOR_BGR2RGB).astype(np.float32) / 255.0 x = np.ascontiguousarray(x.transpose(2, 0, 1)[None]) out = self.session.run(None, {self.input_name: x})[0][0] if out.shape[-1] == 6: # End-to-end export (NMS-free): rows of x1, y1, x2, y2, score, cls. boxes, scores, cls = out[:, :4], out[:, 4], out[:, 5].astype(np.int64) keep = scores >= self.conf_thr[np.clip(cls, 0, len(self.classes) - 1)] boxes, scores, cls = boxes[keep], scores[keep], cls[keep] else: # Raw head: (4 + nc, N) with cx, cy, w, h then per-class scores. pred = out.T cls_scores = pred[:, 4 : 4 + len(self.classes)] # seg heads append mask coefficients # every class of an anchor above its threshold: an ambiguous patch is boxed as both spill and stain, # one extra false positive costs far less than missing the class the label used anchor, cls = np.nonzero(cls_scores >= self.conf_thr) pred, scores = pred[anchor], cls_scores[anchor, cls] cx, cy, bw, bh = pred[:, 0], pred[:, 1], pred[:, 2], pred[:, 3] boxes = np.stack([cx - bw / 2, cy - bh / 2, cx + bw / 2, cy + bh / 2], axis=1) selected: list[int] = [] for c in np.unique(cls): idx = np.where(cls == c)[0] selected.extend(idx[_nms(boxes[idx], scores[idx], self.nms_iou[int(c)])].tolist()) selected.sort(key=lambda i: -float(scores[i])) boxes, scores, cls = boxes[selected], scores[selected], cls[selected] boxes, scores, cls = self._postprocess(boxes, scores, cls) results: list[BoundingBox] = [] for (x1, y1, x2, y2), s, c in zip(boxes[: self.max_det], scores[: self.max_det], cls[: self.max_det]): x1 = int(np.clip(np.floor((x1 - left) / r), 0, w)) y1 = int(np.clip(np.floor((y1 - top) / r), 0, h)) x2 = int(np.clip(np.ceil((x2 - left) / r), 0, w)) y2 = int(np.clip(np.ceil((y2 - top) / r), 0, h)) if x2 <= x1 or y2 <= y1: continue area = (x2 - x1) * (y2 - y1) / (w * h) * 100 if self.small_bin and c == self.classes.index("bin") and area < self.small_bin[0] and s < self.small_bin[1]: continue if self.big_spill and c == self.classes.index("liquid spillage") and area > self.big_spill[0] and s < self.big_spill[1]: continue results.append(BoundingBox(x1=x1, y1=y1, x2=x2, y2=y2, cls_id=int(c), conf=round(float(s), 4))) if results and max(b.conf for b in results) < self.weak_image: return [] return results def predict_batch( self, batch_images: list[ndarray], offset: int, n_keypoints: int, ) -> list[TVFrameResult]: results: list[TVFrameResult] = [] for i, img in enumerate(batch_images): results.append( TVFrameResult( frame_id=offset + i, boxes=self._detect(img), polygons=[], keypoints=[(0, 0)] * n_keypoints, ) ) return results