File size: 9,417 Bytes
8a316d2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 | import json
from pathlib import Path
import cv2
import numpy as np
import onnxruntime as ort
from numpy import ndarray
from pydantic import BaseModel
class BoundingBox(BaseModel):
x1: int
y1: int
x2: int
y2: int
cls_id: int
conf: float
class TVFrameResult(BaseModel):
frame_id: int
boxes: list[BoundingBox] | None = None
polygons: list = []
keypoints: list[tuple[int, int]] | None = None
def _nms(boxes: ndarray, scores: ndarray, iou_thr: float) -> list[int]:
order = scores.argsort()[::-1]
areas = (boxes[:, 2] - boxes[:, 0]).clip(0) * (boxes[:, 3] - boxes[:, 1]).clip(0)
keep: list[int] = []
while order.size:
i = int(order[0])
keep.append(i)
if order.size == 1:
break
rest = order[1:]
xx1 = np.maximum(boxes[i, 0], boxes[rest, 0])
yy1 = np.maximum(boxes[i, 1], boxes[rest, 1])
xx2 = np.minimum(boxes[i, 2], boxes[rest, 2])
yy2 = np.minimum(boxes[i, 3], boxes[rest, 3])
inter = (xx2 - xx1).clip(0) * (yy2 - yy1).clip(0)
iou = inter / (areas[i] + areas[rest] - inter + 1e-9)
order = rest[iou <= iou_thr]
return keep
class Miner:
"""Room-dirtiness detector: a single ONNX YOLO model run on CPU.
CPU execution keeps chute outputs identical to the validator's CPU compliance run.
"""
def __init__(self, path_hf_repo: Path) -> None:
path_hf_repo = Path(path_hf_repo)
cfg = json.loads((path_hf_repo / "config.json").read_text())
self.classes: list[str] = cfg["classes"]
self.imgsz = int(cfg.get("imgsz", 640))
self.conf_thr = np.asarray(cfg.get("conf", [0.25] * len(self.classes)), dtype=np.float32)
nms = cfg.get("nms_iou", 0.6)
self.nms_iou = [float(v) for v in nms] if isinstance(nms, list) else [float(nms)] * len(self.classes)
self.max_det = int(cfg.get("max_det", 100))
mpc = cfg.get("max_per_class", self.max_det)
self.max_per_class = [int(v) for v in mpc] if isinstance(mpc, list) else [int(mpc)] * len(self.classes)
# per class: drop a box that lies mostly inside a bigger, higher-scoring box of the same class
# (one piece of a larger object boxed on its own); null disables
self.contain_drop = cfg.get("contain_drop", [None] * len(self.classes))
# size-aware rules: [area %, conf]; a far bin needs more confidence, a floor-wide spill must not be a faint guess,
# and an image whose best box is weak gets no boxes (without GT every false positive costs 0.1)
self.small_bin = cfg.get("small_bin")
self.big_spill = cfg.get("big_spill")
self.weak_image = float(cfg.get("weak_image", 0.0))
# OpenCV's own thread pool is not needed (one resize per frame) and every extra thread counts
# against the checker sandbox's process limit.
cv2.setNumThreads(1)
model_path = str(path_hf_repo / cfg.get("model", "model.onnx"))
threads = int(cfg.get("threads", 2))
try:
self.session = self._make_session(model_path, threads)
except Exception:
# The checker sandbox caps processes per user; if no worker thread can be spawned,
# run the graph on the calling thread instead of failing to load.
self.session = self._make_session(model_path, 1)
self.input_name = self.session.get_inputs()[0].name
warm = np.zeros((self.imgsz, self.imgsz, 3), dtype=np.uint8)
self._detect(warm)
@staticmethod
def _make_session(model_path: str, threads: int) -> ort.InferenceSession:
opts = ort.SessionOptions()
opts.intra_op_num_threads = threads
opts.inter_op_num_threads = 1
opts.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL
opts.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
return ort.InferenceSession(model_path, sess_options=opts, providers=["CPUExecutionProvider"])
def __repr__(self) -> str:
return f"ONNX YOLO room-dirtiness imgsz={self.imgsz} classes={self.classes}"
def _letterbox(self, img: ndarray) -> tuple[ndarray, float, int, int]:
h, w = img.shape[:2]
r = min(self.imgsz / h, self.imgsz / w)
nh, nw = int(round(h * r)), int(round(w * r))
resized = cv2.resize(img, (nw, nh), interpolation=cv2.INTER_LINEAR)
top = (self.imgsz - nh) // 2
left = (self.imgsz - nw) // 2
canvas = np.full((self.imgsz, self.imgsz, 3), 114, dtype=np.uint8)
canvas[top : top + nh, left : left + nw] = resized
return canvas, r, left, top
def _postprocess(self, boxes: ndarray, scores: ndarray, cls: ndarray) -> tuple[ndarray, ndarray, ndarray]:
"""Score-ordered dedup: containment rule and a per-class cap."""
order = np.argsort(-scores, kind="stable")
keep: list[int] = []
per_class: dict[int, int] = {}
for i in order:
c = int(cls[i])
frac = self.contain_drop[c] if c < len(self.contain_drop) else None
if frac is not None:
bx = boxes[i]
area_i = max(1e-6, (bx[2] - bx[0]) * (bx[3] - bx[1]))
dup = False
for k in keep:
if int(cls[k]) != c:
continue
kb = boxes[k]
iw = max(0.0, min(bx[2], kb[2]) - max(bx[0], kb[0]))
ih = max(0.0, min(bx[3], kb[3]) - max(bx[1], kb[1]))
area_k = max(1e-6, (kb[2] - kb[0]) * (kb[3] - kb[1]))
if area_k > area_i and iw * ih / area_i >= frac:
dup = True
break
if dup:
continue
if per_class.get(c, 0) >= self.max_per_class[c]:
continue
per_class[c] = per_class.get(c, 0) + 1
keep.append(int(i))
return boxes[keep], scores[keep], cls[keep]
def _detect(self, img_bgr: ndarray) -> list[BoundingBox]:
h, w = img_bgr.shape[:2]
canvas, r, left, top = self._letterbox(img_bgr)
x = cv2.cvtColor(canvas, cv2.COLOR_BGR2RGB).astype(np.float32) / 255.0
x = np.ascontiguousarray(x.transpose(2, 0, 1)[None])
out = self.session.run(None, {self.input_name: x})[0][0]
if out.shape[-1] == 6:
# End-to-end export (NMS-free): rows of x1, y1, x2, y2, score, cls.
boxes, scores, cls = out[:, :4], out[:, 4], out[:, 5].astype(np.int64)
keep = scores >= self.conf_thr[np.clip(cls, 0, len(self.classes) - 1)]
boxes, scores, cls = boxes[keep], scores[keep], cls[keep]
else:
# Raw head: (4 + nc, N) with cx, cy, w, h then per-class scores.
pred = out.T
cls_scores = pred[:, 4 : 4 + len(self.classes)] # seg heads append mask coefficients
# every class of an anchor above its threshold: an ambiguous patch is boxed as both spill and stain,
# one extra false positive costs far less than missing the class the label used
anchor, cls = np.nonzero(cls_scores >= self.conf_thr)
pred, scores = pred[anchor], cls_scores[anchor, cls]
cx, cy, bw, bh = pred[:, 0], pred[:, 1], pred[:, 2], pred[:, 3]
boxes = np.stack([cx - bw / 2, cy - bh / 2, cx + bw / 2, cy + bh / 2], axis=1)
selected: list[int] = []
for c in np.unique(cls):
idx = np.where(cls == c)[0]
selected.extend(idx[_nms(boxes[idx], scores[idx], self.nms_iou[int(c)])].tolist())
selected.sort(key=lambda i: -float(scores[i]))
boxes, scores, cls = boxes[selected], scores[selected], cls[selected]
boxes, scores, cls = self._postprocess(boxes, scores, cls)
results: list[BoundingBox] = []
for (x1, y1, x2, y2), s, c in zip(boxes[: self.max_det], scores[: self.max_det], cls[: self.max_det]):
x1 = int(np.clip(np.floor((x1 - left) / r), 0, w))
y1 = int(np.clip(np.floor((y1 - top) / r), 0, h))
x2 = int(np.clip(np.ceil((x2 - left) / r), 0, w))
y2 = int(np.clip(np.ceil((y2 - top) / r), 0, h))
if x2 <= x1 or y2 <= y1:
continue
area = (x2 - x1) * (y2 - y1) / (w * h) * 100
if self.small_bin and c == self.classes.index("bin") and area < self.small_bin[0] and s < self.small_bin[1]:
continue
if self.big_spill and c == self.classes.index("liquid spillage") and area > self.big_spill[0] and s < self.big_spill[1]:
continue
results.append(BoundingBox(x1=x1, y1=y1, x2=x2, y2=y2, cls_id=int(c), conf=round(float(s), 4)))
if results and max(b.conf for b in results) < self.weak_image:
return []
return results
def predict_batch(
self,
batch_images: list[ndarray],
offset: int,
n_keypoints: int,
) -> list[TVFrameResult]:
results: list[TVFrameResult] = []
for i, img in enumerate(batch_images):
results.append(
TVFrameResult(
frame_id=offset + i,
boxes=self._detect(img),
polygons=[],
keypoints=[(0, 0)] * n_keypoints,
)
)
return results
|