ScoreVision / miner.py
artur7236's picture
scorevision: push artifact
8a316d2 verified
Raw History Blame Contribute Delete
9.42 kB
import json
from pathlib import Path
import cv2
import numpy as np
import onnxruntime as ort
from numpy import ndarray
from pydantic import BaseModel
class BoundingBox(BaseModel):
x1: int
y1: int
x2: int
y2: int
cls_id: int
conf: float
class TVFrameResult(BaseModel):
frame_id: int
boxes: list[BoundingBox] | None = None
polygons: list = []
keypoints: list[tuple[int, int]] | None = None
def _nms(boxes: ndarray, scores: ndarray, iou_thr: float) -> list[int]:
order = scores.argsort()[::-1]
areas = (boxes[:, 2] - boxes[:, 0]).clip(0) * (boxes[:, 3] - boxes[:, 1]).clip(0)
keep: list[int] = []
while order.size:
i = int(order[0])
keep.append(i)
if order.size == 1:
break
rest = order[1:]
xx1 = np.maximum(boxes[i, 0], boxes[rest, 0])
yy1 = np.maximum(boxes[i, 1], boxes[rest, 1])
xx2 = np.minimum(boxes[i, 2], boxes[rest, 2])
yy2 = np.minimum(boxes[i, 3], boxes[rest, 3])
inter = (xx2 - xx1).clip(0) * (yy2 - yy1).clip(0)
iou = inter / (areas[i] + areas[rest] - inter + 1e-9)
order = rest[iou <= iou_thr]
return keep
class Miner:
"""Room-dirtiness detector: a single ONNX YOLO model run on CPU.
CPU execution keeps chute outputs identical to the validator's CPU compliance run.
"""
def __init__(self, path_hf_repo: Path) -> None:
path_hf_repo = Path(path_hf_repo)
cfg = json.loads((path_hf_repo / "config.json").read_text())
self.classes: list[str] = cfg["classes"]
self.imgsz = int(cfg.get("imgsz", 640))
self.conf_thr = np.asarray(cfg.get("conf", [0.25] * len(self.classes)), dtype=np.float32)
nms = cfg.get("nms_iou", 0.6)
self.nms_iou = [float(v) for v in nms] if isinstance(nms, list) else [float(nms)] * len(self.classes)
self.max_det = int(cfg.get("max_det", 100))
mpc = cfg.get("max_per_class", self.max_det)
self.max_per_class = [int(v) for v in mpc] if isinstance(mpc, list) else [int(mpc)] * len(self.classes)
# per class: drop a box that lies mostly inside a bigger, higher-scoring box of the same class
# (one piece of a larger object boxed on its own); null disables
self.contain_drop = cfg.get("contain_drop", [None] * len(self.classes))
# size-aware rules: [area %, conf]; a far bin needs more confidence, a floor-wide spill must not be a faint guess,
# and an image whose best box is weak gets no boxes (without GT every false positive costs 0.1)
self.small_bin = cfg.get("small_bin")
self.big_spill = cfg.get("big_spill")
self.weak_image = float(cfg.get("weak_image", 0.0))
# OpenCV's own thread pool is not needed (one resize per frame) and every extra thread counts
# against the checker sandbox's process limit.
cv2.setNumThreads(1)
model_path = str(path_hf_repo / cfg.get("model", "model.onnx"))
threads = int(cfg.get("threads", 2))
try:
self.session = self._make_session(model_path, threads)
except Exception:
# The checker sandbox caps processes per user; if no worker thread can be spawned,
# run the graph on the calling thread instead of failing to load.
self.session = self._make_session(model_path, 1)
self.input_name = self.session.get_inputs()[0].name
warm = np.zeros((self.imgsz, self.imgsz, 3), dtype=np.uint8)
self._detect(warm)
@staticmethod
def _make_session(model_path: str, threads: int) -> ort.InferenceSession:
opts = ort.SessionOptions()
opts.intra_op_num_threads = threads
opts.inter_op_num_threads = 1
opts.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL
opts.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
return ort.InferenceSession(model_path, sess_options=opts, providers=["CPUExecutionProvider"])
def __repr__(self) -> str:
return f"ONNX YOLO room-dirtiness imgsz={self.imgsz} classes={self.classes}"
def _letterbox(self, img: ndarray) -> tuple[ndarray, float, int, int]:
h, w = img.shape[:2]
r = min(self.imgsz / h, self.imgsz / w)
nh, nw = int(round(h * r)), int(round(w * r))
resized = cv2.resize(img, (nw, nh), interpolation=cv2.INTER_LINEAR)
top = (self.imgsz - nh) // 2
left = (self.imgsz - nw) // 2
canvas = np.full((self.imgsz, self.imgsz, 3), 114, dtype=np.uint8)
canvas[top : top + nh, left : left + nw] = resized
return canvas, r, left, top
def _postprocess(self, boxes: ndarray, scores: ndarray, cls: ndarray) -> tuple[ndarray, ndarray, ndarray]:
"""Score-ordered dedup: containment rule and a per-class cap."""
order = np.argsort(-scores, kind="stable")
keep: list[int] = []
per_class: dict[int, int] = {}
for i in order:
c = int(cls[i])
frac = self.contain_drop[c] if c < len(self.contain_drop) else None
if frac is not None:
bx = boxes[i]
area_i = max(1e-6, (bx[2] - bx[0]) * (bx[3] - bx[1]))
dup = False
for k in keep:
if int(cls[k]) != c:
continue
kb = boxes[k]
iw = max(0.0, min(bx[2], kb[2]) - max(bx[0], kb[0]))
ih = max(0.0, min(bx[3], kb[3]) - max(bx[1], kb[1]))
area_k = max(1e-6, (kb[2] - kb[0]) * (kb[3] - kb[1]))
if area_k > area_i and iw * ih / area_i >= frac:
dup = True
break
if dup:
continue
if per_class.get(c, 0) >= self.max_per_class[c]:
continue
per_class[c] = per_class.get(c, 0) + 1
keep.append(int(i))
return boxes[keep], scores[keep], cls[keep]
def _detect(self, img_bgr: ndarray) -> list[BoundingBox]:
h, w = img_bgr.shape[:2]
canvas, r, left, top = self._letterbox(img_bgr)
x = cv2.cvtColor(canvas, cv2.COLOR_BGR2RGB).astype(np.float32) / 255.0
x = np.ascontiguousarray(x.transpose(2, 0, 1)[None])
out = self.session.run(None, {self.input_name: x})[0][0]
if out.shape[-1] == 6:
# End-to-end export (NMS-free): rows of x1, y1, x2, y2, score, cls.
boxes, scores, cls = out[:, :4], out[:, 4], out[:, 5].astype(np.int64)
keep = scores >= self.conf_thr[np.clip(cls, 0, len(self.classes) - 1)]
boxes, scores, cls = boxes[keep], scores[keep], cls[keep]
else:
# Raw head: (4 + nc, N) with cx, cy, w, h then per-class scores.
pred = out.T
cls_scores = pred[:, 4 : 4 + len(self.classes)] # seg heads append mask coefficients
# every class of an anchor above its threshold: an ambiguous patch is boxed as both spill and stain,
# one extra false positive costs far less than missing the class the label used
anchor, cls = np.nonzero(cls_scores >= self.conf_thr)
pred, scores = pred[anchor], cls_scores[anchor, cls]
cx, cy, bw, bh = pred[:, 0], pred[:, 1], pred[:, 2], pred[:, 3]
boxes = np.stack([cx - bw / 2, cy - bh / 2, cx + bw / 2, cy + bh / 2], axis=1)
selected: list[int] = []
for c in np.unique(cls):
idx = np.where(cls == c)[0]
selected.extend(idx[_nms(boxes[idx], scores[idx], self.nms_iou[int(c)])].tolist())
selected.sort(key=lambda i: -float(scores[i]))
boxes, scores, cls = boxes[selected], scores[selected], cls[selected]
boxes, scores, cls = self._postprocess(boxes, scores, cls)
results: list[BoundingBox] = []
for (x1, y1, x2, y2), s, c in zip(boxes[: self.max_det], scores[: self.max_det], cls[: self.max_det]):
x1 = int(np.clip(np.floor((x1 - left) / r), 0, w))
y1 = int(np.clip(np.floor((y1 - top) / r), 0, h))
x2 = int(np.clip(np.ceil((x2 - left) / r), 0, w))
y2 = int(np.clip(np.ceil((y2 - top) / r), 0, h))
if x2 <= x1 or y2 <= y1:
continue
area = (x2 - x1) * (y2 - y1) / (w * h) * 100
if self.small_bin and c == self.classes.index("bin") and area < self.small_bin[0] and s < self.small_bin[1]:
continue
if self.big_spill and c == self.classes.index("liquid spillage") and area > self.big_spill[0] and s < self.big_spill[1]:
continue
results.append(BoundingBox(x1=x1, y1=y1, x2=x2, y2=y2, cls_id=int(c), conf=round(float(s), 4)))
if results and max(b.conf for b in results) < self.weak_image:
return []
return results
def predict_batch(
self,
batch_images: list[ndarray],
offset: int,
n_keypoints: int,
) -> list[TVFrameResult]:
results: list[TVFrameResult] = []
for i, img in enumerate(batch_images):
results.append(
TVFrameResult(
frame_id=offset + i,
boxes=self._detect(img),
polygons=[],
keypoints=[(0, 0)] * n_keypoints,
)
)
return results