File size: 9,417 Bytes
8a316d2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
import json
from pathlib import Path

import cv2
import numpy as np
import onnxruntime as ort
from numpy import ndarray
from pydantic import BaseModel


class BoundingBox(BaseModel):
    x1: int
    y1: int
    x2: int
    y2: int
    cls_id: int
    conf: float


class TVFrameResult(BaseModel):
    frame_id: int
    boxes: list[BoundingBox] | None = None
    polygons: list = []
    keypoints: list[tuple[int, int]] | None = None


def _nms(boxes: ndarray, scores: ndarray, iou_thr: float) -> list[int]:
    order = scores.argsort()[::-1]
    areas = (boxes[:, 2] - boxes[:, 0]).clip(0) * (boxes[:, 3] - boxes[:, 1]).clip(0)
    keep: list[int] = []
    while order.size:
        i = int(order[0])
        keep.append(i)
        if order.size == 1:
            break
        rest = order[1:]
        xx1 = np.maximum(boxes[i, 0], boxes[rest, 0])
        yy1 = np.maximum(boxes[i, 1], boxes[rest, 1])
        xx2 = np.minimum(boxes[i, 2], boxes[rest, 2])
        yy2 = np.minimum(boxes[i, 3], boxes[rest, 3])
        inter = (xx2 - xx1).clip(0) * (yy2 - yy1).clip(0)
        iou = inter / (areas[i] + areas[rest] - inter + 1e-9)
        order = rest[iou <= iou_thr]
    return keep


class Miner:
    """Room-dirtiness detector: a single ONNX YOLO model run on CPU.

    CPU execution keeps chute outputs identical to the validator's CPU compliance run.
    """

    def __init__(self, path_hf_repo: Path) -> None:
        path_hf_repo = Path(path_hf_repo)
        cfg = json.loads((path_hf_repo / "config.json").read_text())
        self.classes: list[str] = cfg["classes"]
        self.imgsz = int(cfg.get("imgsz", 640))
        self.conf_thr = np.asarray(cfg.get("conf", [0.25] * len(self.classes)), dtype=np.float32)
        nms = cfg.get("nms_iou", 0.6)
        self.nms_iou = [float(v) for v in nms] if isinstance(nms, list) else [float(nms)] * len(self.classes)
        self.max_det = int(cfg.get("max_det", 100))
        mpc = cfg.get("max_per_class", self.max_det)
        self.max_per_class = [int(v) for v in mpc] if isinstance(mpc, list) else [int(mpc)] * len(self.classes)
        # per class: drop a box that lies mostly inside a bigger, higher-scoring box of the same class
        # (one piece of a larger object boxed on its own); null disables
        self.contain_drop = cfg.get("contain_drop", [None] * len(self.classes))
        # size-aware rules: [area %, conf]; a far bin needs more confidence, a floor-wide spill must not be a faint guess,
        # and an image whose best box is weak gets no boxes (without GT every false positive costs 0.1)
        self.small_bin = cfg.get("small_bin")
        self.big_spill = cfg.get("big_spill")
        self.weak_image = float(cfg.get("weak_image", 0.0))

        # OpenCV's own thread pool is not needed (one resize per frame) and every extra thread counts
        # against the checker sandbox's process limit.
        cv2.setNumThreads(1)
        model_path = str(path_hf_repo / cfg.get("model", "model.onnx"))
        threads = int(cfg.get("threads", 2))
        try:
            self.session = self._make_session(model_path, threads)
        except Exception:
            # The checker sandbox caps processes per user; if no worker thread can be spawned,
            # run the graph on the calling thread instead of failing to load.
            self.session = self._make_session(model_path, 1)
        self.input_name = self.session.get_inputs()[0].name
        warm = np.zeros((self.imgsz, self.imgsz, 3), dtype=np.uint8)
        self._detect(warm)

    @staticmethod
    def _make_session(model_path: str, threads: int) -> ort.InferenceSession:
        opts = ort.SessionOptions()
        opts.intra_op_num_threads = threads
        opts.inter_op_num_threads = 1
        opts.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL
        opts.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
        return ort.InferenceSession(model_path, sess_options=opts, providers=["CPUExecutionProvider"])

    def __repr__(self) -> str:
        return f"ONNX YOLO room-dirtiness imgsz={self.imgsz} classes={self.classes}"

    def _letterbox(self, img: ndarray) -> tuple[ndarray, float, int, int]:
        h, w = img.shape[:2]
        r = min(self.imgsz / h, self.imgsz / w)
        nh, nw = int(round(h * r)), int(round(w * r))
        resized = cv2.resize(img, (nw, nh), interpolation=cv2.INTER_LINEAR)
        top = (self.imgsz - nh) // 2
        left = (self.imgsz - nw) // 2
        canvas = np.full((self.imgsz, self.imgsz, 3), 114, dtype=np.uint8)
        canvas[top : top + nh, left : left + nw] = resized
        return canvas, r, left, top

    def _postprocess(self, boxes: ndarray, scores: ndarray, cls: ndarray) -> tuple[ndarray, ndarray, ndarray]:
        """Score-ordered dedup: containment rule and a per-class cap."""
        order = np.argsort(-scores, kind="stable")
        keep: list[int] = []
        per_class: dict[int, int] = {}
        for i in order:
            c = int(cls[i])
            frac = self.contain_drop[c] if c < len(self.contain_drop) else None
            if frac is not None:
                bx = boxes[i]
                area_i = max(1e-6, (bx[2] - bx[0]) * (bx[3] - bx[1]))
                dup = False
                for k in keep:
                    if int(cls[k]) != c:
                        continue
                    kb = boxes[k]
                    iw = max(0.0, min(bx[2], kb[2]) - max(bx[0], kb[0]))
                    ih = max(0.0, min(bx[3], kb[3]) - max(bx[1], kb[1]))
                    area_k = max(1e-6, (kb[2] - kb[0]) * (kb[3] - kb[1]))
                    if area_k > area_i and iw * ih / area_i >= frac:
                        dup = True
                        break
                if dup:
                    continue
            if per_class.get(c, 0) >= self.max_per_class[c]:
                continue
            per_class[c] = per_class.get(c, 0) + 1
            keep.append(int(i))
        return boxes[keep], scores[keep], cls[keep]

    def _detect(self, img_bgr: ndarray) -> list[BoundingBox]:
        h, w = img_bgr.shape[:2]
        canvas, r, left, top = self._letterbox(img_bgr)
        x = cv2.cvtColor(canvas, cv2.COLOR_BGR2RGB).astype(np.float32) / 255.0
        x = np.ascontiguousarray(x.transpose(2, 0, 1)[None])
        out = self.session.run(None, {self.input_name: x})[0][0]

        if out.shape[-1] == 6:
            # End-to-end export (NMS-free): rows of x1, y1, x2, y2, score, cls.
            boxes, scores, cls = out[:, :4], out[:, 4], out[:, 5].astype(np.int64)
            keep = scores >= self.conf_thr[np.clip(cls, 0, len(self.classes) - 1)]
            boxes, scores, cls = boxes[keep], scores[keep], cls[keep]
        else:
            # Raw head: (4 + nc, N) with cx, cy, w, h then per-class scores.
            pred = out.T
            cls_scores = pred[:, 4 : 4 + len(self.classes)]  # seg heads append mask coefficients
            # every class of an anchor above its threshold: an ambiguous patch is boxed as both spill and stain,
            # one extra false positive costs far less than missing the class the label used
            anchor, cls = np.nonzero(cls_scores >= self.conf_thr)
            pred, scores = pred[anchor], cls_scores[anchor, cls]
            cx, cy, bw, bh = pred[:, 0], pred[:, 1], pred[:, 2], pred[:, 3]
            boxes = np.stack([cx - bw / 2, cy - bh / 2, cx + bw / 2, cy + bh / 2], axis=1)
            selected: list[int] = []
            for c in np.unique(cls):
                idx = np.where(cls == c)[0]
                selected.extend(idx[_nms(boxes[idx], scores[idx], self.nms_iou[int(c)])].tolist())
            selected.sort(key=lambda i: -float(scores[i]))
            boxes, scores, cls = boxes[selected], scores[selected], cls[selected]
        boxes, scores, cls = self._postprocess(boxes, scores, cls)

        results: list[BoundingBox] = []
        for (x1, y1, x2, y2), s, c in zip(boxes[: self.max_det], scores[: self.max_det], cls[: self.max_det]):
            x1 = int(np.clip(np.floor((x1 - left) / r), 0, w))
            y1 = int(np.clip(np.floor((y1 - top) / r), 0, h))
            x2 = int(np.clip(np.ceil((x2 - left) / r), 0, w))
            y2 = int(np.clip(np.ceil((y2 - top) / r), 0, h))
            if x2 <= x1 or y2 <= y1:
                continue
            area = (x2 - x1) * (y2 - y1) / (w * h) * 100
            if self.small_bin and c == self.classes.index("bin") and area < self.small_bin[0] and s < self.small_bin[1]:
                continue
            if self.big_spill and c == self.classes.index("liquid spillage") and area > self.big_spill[0] and s < self.big_spill[1]:
                continue
            results.append(BoundingBox(x1=x1, y1=y1, x2=x2, y2=y2, cls_id=int(c), conf=round(float(s), 4)))
        if results and max(b.conf for b in results) < self.weak_image:
            return []
        return results

    def predict_batch(
        self,
        batch_images: list[ndarray],
        offset: int,
        n_keypoints: int,
    ) -> list[TVFrameResult]:
        results: list[TVFrameResult] = []
        for i, img in enumerate(batch_images):
            results.append(
                TVFrameResult(
                    frame_id=offset + i,
                    boxes=self._detect(img),
                    polygons=[],
                    keypoints=[(0, 0)] * n_keypoints,
                )
            )
        return results