File size: 8,922 Bytes
c599dd5
4b183f8
 
 
 
c599dd5
 
 
 
 
 
 
 
 
4b183f8
 
c599dd5
4b183f8
 
 
 
 
 
 
 
 
 
c599dd5
 
4b183f8
c599dd5
4b183f8
c599dd5
 
4b183f8
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
c599dd5
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4b183f8
c599dd5
 
 
 
4b183f8
c599dd5
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4b183f8
c599dd5
 
 
 
 
 
 
 
 
 
4b183f8
c599dd5
 
 
4b183f8
 
 
 
 
 
 
 
 
 
c599dd5
 
4b183f8
 
 
c599dd5
 
 
 
 
 
 
 
 
 
4b183f8
c599dd5
 
4b183f8
 
c599dd5
 
4b183f8
 
 
c599dd5
4b183f8
c599dd5
 
 
4b183f8
c599dd5
 
 
 
4b183f8
c599dd5
 
 
 
 
 
 
 
 
 
 
 
 
4b183f8
c599dd5
 
 
 
 
 
 
 
4b183f8
 
 
 
c599dd5
4b183f8
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
"""Room-dirtiness detector for Score (SN44) element manako/Detect-room-dirtiness-detection, YOLO26n graph.

Classes (cls_id order fixed by the element manifest):
    0 liquid spillage, 1 bin, 2 litter, 3 mould, 4 stain

Runs a YOLO26n ONNX detector on CPU with onnxruntime, so the chute and Score's CPU compliance re-run produce the same
boxes. The graph's one-to-many head has no NMS: output0 is [1, 4 + 5, anchors], per anchor the box centre x, y, width
and height in input pixels and the 5 class scores. Pure numpy/cv2 pre/post-processing:
    pre   centred letterbox to the graph's input size (704 x 704), INTER_LINEAR, grey 114 border; BGR -> RGB planes
          multiplied by float32(1 / 255)
    post  one class per anchor (its highest score), keep scores >= 0.25, boxes back to picture pixels and clipped to
          the picture, ordered by score (ties: left edge, then top edge), class-wise greedy NMS (a box drops every
          later box of its class with IoU > 0.5), the first 100 boxes; integer corners floor(x1), floor(y1),
          ceil(x2), ceil(y2) kept inside the picture; the score is kept at full precision (not rounded)
"""

import math
from pathlib import Path

import cv2
import numpy as np
import onnxruntime as ort
from numpy import ndarray
from pydantic import BaseModel

NUM_CLASSES = 5
DEFAULT_INPUT_SIZE = 640
# One flat score threshold for every class.
CONF_THRESHOLDS = (0.25, 0.25, 0.25, 0.25, 0.25)
NMS_IOU = 0.5
MAX_DETECTIONS = 100
PAD_VALUE = 114
# A few runs at start-up, so the first scored frame does not pay for onnxruntime's first-run allocations.
WARMUP_RUNS = 3


class BoundingBox(BaseModel):
    x1: int
    y1: int
    x2: int
    y2: int
    cls_id: int
    conf: float


class TVFrameResult(BaseModel):
    frame_id: int
    boxes: list[BoundingBox] | None = None
    keypoints: list[tuple[int, int]] | None = None


def as_bgr(image: ndarray) -> ndarray:
    """The frame as a contiguous 3-channel BGR uint8 picture (grey, BGRA and non-uint8 frames are converted)."""
    arr = np.asarray(image)
    if arr.ndim == 2:
        arr = cv2.cvtColor(arr if arr.dtype == np.uint8 else np.clip(arr, 0, 255).astype(np.uint8), cv2.COLOR_GRAY2BGR)
    if arr.dtype != np.uint8:
        arr = np.clip(arr, 0, 255).astype(np.uint8)
    if arr.ndim == 3 and arr.shape[2] == 4:
        arr = cv2.cvtColor(arr, cv2.COLOR_BGRA2BGR)
    elif arr.ndim == 3 and arr.shape[2] == 1:
        arr = cv2.cvtColor(arr, cv2.COLOR_GRAY2BGR)
    return np.ascontiguousarray(arr)


def letterbox(image: ndarray, in_h: int, in_w: int) -> tuple[ndarray, float, int, int]:
    """Resize keeping the aspect ratio and pad to in_h x in_w, centred (Ultralytics LetterBox).
    Returns the (1, 3, in_h, in_w) float32 RGB input, the scale and the left / top border."""
    h, w = image.shape[:2]
    r = min(in_h / h, in_w / w)
    new_h = min(max(1, int(round(h * r))), in_h)
    new_w = min(max(1, int(round(w * r))), in_w)
    if (new_h, new_w) != (h, w):
        image = cv2.resize(image, (new_w, new_h), interpolation=cv2.INTER_LINEAR)
    top, left = (in_h - new_h) // 2, (in_w - new_w) // 2
    if (new_h, new_w) != (in_h, in_w):
        canvas = np.full((in_h, in_w, 3), PAD_VALUE, dtype=np.uint8)
        canvas[top:top + new_h, left:left + new_w] = image
        image = canvas
    x = np.empty((1, 3, in_h, in_w), dtype=np.float32)
    scale = np.float32(1.0 / 255.0)
    for c in range(3):  # plane c of the input = RGB channel c = BGR channel 2 - c
        np.multiply(image[:, :, 2 - c], scale, out=x[0, c], dtype=np.float32)
    return x, r, left, top


def nms_classwise(boxes: ndarray, cls_ids: ndarray, iou_threshold: float) -> list[int]:
    """Greedy NMS in the given order: each kept box drops every later box of its class with IoU > iou_threshold."""
    n = boxes.shape[0]
    areas = (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1])
    dropped = np.zeros(n, dtype=bool)
    keep = []
    for i in range(n):
        if dropped[i]:
            continue
        keep.append(i)
        rest = np.arange(i + 1, n)
        rest = rest[(~dropped[rest]) & (cls_ids[rest] == cls_ids[i])]
        if rest.size == 0:
            continue
        iw = np.clip(np.minimum(boxes[i, 2], boxes[rest, 2]) - np.maximum(boxes[i, 0], boxes[rest, 0]), 0, None)
        ih = np.clip(np.minimum(boxes[i, 3], boxes[rest, 3]) - np.maximum(boxes[i, 1], boxes[rest, 1]), 0, None)
        inter = iw * ih
        union = areas[i] + areas[rest] - inter
        iou = np.where(union > 0, inter / np.maximum(union, 1e-9), 0.0)
        dropped[rest[iou > iou_threshold]] = True
    return keep


class Miner:
    def __init__(self, path_hf_repo: Path) -> None:
        options = ort.SessionOptions()
        options.intra_op_num_threads = 2
        options.inter_op_num_threads = 1
        options.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL
        options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
        options.add_session_config_entry("session.intra_op.allow_spinning", "1")
        options.add_session_config_entry("session.inter_op.allow_spinning", "1")
        self.session = ort.InferenceSession(str(Path(path_hf_repo) / "model.onnx"), sess_options=options,
                                            providers=["CPUExecutionProvider"])
        model_input = self.session.get_inputs()[0]
        model_output = self.session.get_outputs()[0]
        self.input_name, self.output_name = model_input.name, model_output.name
        shape = model_input.shape
        if len(shape) == 4 and all(isinstance(d, int) and d > 0 for d in shape[2:]):
            self.input_h, self.input_w = int(shape[2]), int(shape[3])
        else:
            self.input_h = self.input_w = DEFAULT_INPUT_SIZE
        out_shape = model_output.shape
        if len(out_shape) != 3 or (isinstance(out_shape[1], int) and out_shape[1] != 4 + NUM_CLASSES):
            raise ValueError(f"model.onnx output {model_output.name} {out_shape} is not [1, {4 + NUM_CLASSES}, anchors]")
        self.conf_thresholds = np.array(CONF_THRESHOLDS, dtype=np.float32)
        for _ in range(WARMUP_RUNS):
            self.detect(np.full((1024, 1024, 3), PAD_VALUE, dtype=np.uint8))

    def __repr__(self) -> str:
        return (f"Room-dirtiness YOLO26 ONNX detector ({self.input_h}x{self.input_w}px, "
                f"providers={self.session.get_providers()})")

    def detect(self, image: ndarray) -> list[tuple[int, int, int, int, int, float]]:
        """Boxes for one BGR image: (x1, y1, x2, y2, cls_id, conf) in original pixel coordinates."""
        image = as_bgr(image)
        h, w = image.shape[:2]
        x, r, left, top = letterbox(image, self.input_h, self.input_w)
        out = np.asarray(self.session.run([self.output_name], {self.input_name: x})[0], dtype=np.float32)
        pred = out[0][: 4 + NUM_CLASSES].T  # (anchors, 4 + classes)
        scores = pred[:, 4:]
        cls_ids = scores.argmax(1)
        conf = scores[np.arange(scores.shape[0]), cls_ids]
        keep = np.isfinite(conf) & (conf >= self.conf_thresholds[cls_ids])
        if not np.any(keep):
            return []
        box, conf, cls_ids = pred[keep, :4], conf[keep], cls_ids[keep]
        cx, cy, bw, bh = box[:, 0], box[:, 1], box[:, 2], box[:, 3]
        boxes = np.stack([(cx - bw / 2 - left) / r, (cy - bh / 2 - top) / r,
                          (cx + bw / 2 - left) / r, (cy + bh / 2 - top) / r], 1)
        finite = np.isfinite(boxes).all(1) & np.isfinite(conf)
        if not np.all(finite):
            boxes, conf, cls_ids = boxes[finite], conf[finite], cls_ids[finite]
        if boxes.shape[0] == 0:
            return []
        boxes[:, [0, 2]] = np.clip(boxes[:, [0, 2]], 0, w)
        boxes[:, [1, 3]] = np.clip(boxes[:, [1, 3]], 0, h)
        order = np.lexsort((boxes[:, 1], boxes[:, 0], -conf))  # score high to low, then left edge, then top edge
        boxes, conf, cls_ids = boxes[order], conf[order], cls_ids[order]
        results = []
        for i in nms_classwise(boxes, cls_ids, NMS_IOU)[:MAX_DETECTIONS]:
            xa, ya, xb, yb = (float(v) for v in boxes[i])
            x1 = max(0, min(int(math.floor(xa)), w - 1))
            y1 = max(0, min(int(math.floor(ya)), h - 1))
            x2 = max(x1 + 1, min(int(math.ceil(xb)), w))
            y2 = max(y1 + 1, min(int(math.ceil(yb)), h))
            results.append((x1, y1, x2, y2, int(cls_ids[i]), min(1.0, max(0.0, float(conf[i])))))
        return results

    def predict_batch(self, batch_images: list[ndarray], offset: int, n_keypoints: int) -> list[TVFrameResult]:
        results = []
        for i, image in enumerate(batch_images):
            boxes = [BoundingBox(x1=x1, y1=y1, x2=x2, y2=y2, cls_id=c, conf=s)
                     for x1, y1, x2, y2, c, s in self.detect(image)]
            results.append(TVFrameResult(frame_id=offset + i, boxes=boxes, keypoints=[(0, 0)] * n_keypoints))
        return results