Detect-worksite-ppe-compliance-detection-probe-set miner

chute_config.yml

@@ -0,0 +1,23 @@
+Image:
+  from_base: parachutes/python:3.12
+  run_command:
+    - pip install --upgrade setuptools wheel
+    # CPU onnxruntime pinned to the conformity checker's version (Dockerfile.latency_conformity):
+    # identical kernels => identical boxes between the live chute and the offline FAIL_OUTPUT check.
+    - pip install 'numpy>=1.23,<3' 'onnxruntime==1.26.0' 'opencv-python-headless>=4.8' 'pillow>=9.5' 'huggingface_hub>=0.24' 'pydantic>=2.0' 'pyyaml>=6.0' 'aiohttp>=3.9'
+    - pip install torch==2.8.0 torchvision==0.23.0 --index-url https://download.pytorch.org/whl/cu128
+
+NodeSelector:
+  gpu_count: 1
+  min_vram_gb_per_gpu: 16
+  max_hourly_price_per_gpu: 2
+  include:
+    - pro_6000
+
+Chute:
+  timeout_seconds: 900
+  concurrency: 4
+  max_instances: 5
+  scaling_threshold: 0.5
+  shutdown_after_seconds: 288000
+  tee: true

config.json

@@ -0,0 +1 @@
+{"classes": ["person", "PPE compliant person", "PPE noncompliant person", "safety helmet", "high-visibility safety vest", "protective gloves", "protective eyewear"], "thr": [0.02, 0.02, 0.02, 0.02, 0.02, 0.02, 0.02], "thr2": [0.02, 0.02, 0.02, 0.02, 0.02, 0.02, 0.02], "nms_iou": [0.6, 0.6, 0.6, 0.6, 0.6, 0.6, 0.6], "out_map": [-1, -1, -1, 0, 1, 2, 3], "threads": 2, "max_det": 300, "policy": {"w": [-0.25850449337380316, 1.4355176067249764, 1.8575488532194198, 0.021346864206550166, 0.14557662012226255, -0.47041897170304936, 0.4042205001520079, 1.2644728763974595, -0.6910133086193161, -0.10128821638931128, -0.11051447883517954, -0.8747434916161727, -0.024137525391398303, -0.4371693129346642, -1.3307217263107136, -1.387105394867884, 0.6117582713656641, 0.43068893594018554, -0.1898949861661965, 0.38328867448057324, 0.5480339434453338, -1.138265415362644], "cal": [[[0.02171427522215151, 0.03200429160185535, 0.04118453932530952, 0.0580075006894391, 0.09302379868247292, 0.2000282296628663, 0.45800774446641557, 0.6478584583359536, 0.717738576609679, 0.7553269161118401, 0.7904511045927953, 0.8021226981673578, 0.8196076578564114, 0.8388257302419104, 0.849534008358464, 0.8722090582654933, 0.883531017134888, 0.916295185113194, 0.9221355429803482, 0.9297653213895932, 0.9431003808013855], [0.007063572149344097, 0.012121212121212121, 0.012121212121212121, 0.024242424242424242, 0.052525252525252523, 0.12323232323232323, 0.4686868686868687, 0.8363636363636363, 0.9292929292929293, 0.9595959595959596, 0.9838383838383838, 0.9929292929292929, 0.9952861952861953, 0.996969696969697, 0.9987878787878788, 0.9993265993265993, 0.9994949494949495, 1.0, 1.0, 1.0, 1.0]], [[0.022106543251995667, 0.027303524418360627, 0.03513205175026221, 0.06556730450713882, 0.09778129910269091, 0.16266566700375026, 0.2887591053538608, 0.5120982754175437, 0.7192948185903136, 0.8027083014013581, 0.8345992962061535, 0.8636636628014386, 0.8718806285462621, 0.8913694541849848, 0.9132355483171577, 0.916160672008303, 0.9190547121559969, 0.9221312612432488, 0.9251604117281426, 0.9281763169347965, 0.9316660545714733, 0.9353992341575534, 0.9393211851746256, 0.9436757192358992, 0.9481326012633414, 0.9525797920567649, 0.956845249173828, 0.9615678875127696, 0.968130467536812], [0.004608294930875576, 0.016129032258064516, 0.018433179723502304, 0.020785219399538105, 0.029953917050691243, 0.059907834101382486, 0.15207373271889402, 0.42725173210161665, 0.8433179723502304, 0.9792626728110599, 0.9965437788018433, 0.9976905311778291, 0.9982708933717579, 0.9996707276918011, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]], [[0.020868378363806627, 0.02263093634964167, 0.024687080546840705, 0.027135139348899835, 0.02993820174384261, 0.04162003234915943, 0.0471438938065579, 0.05377447479135317, 0.061573292439718305, 0.07173815770741677, 0.08420612127540233, 0.12109002422113888, 0.1473319429431731, 0.18115477789775167, 0.2225713448034884, 0.2719214830910779, 0.32507406778125747, 0.3822114515448563, 0.4377281193181911, 0.48872739228325945, 0.5326959685686851, 0.5716481154802239, 0.6055050094750958, 0.6348809822558122, 0.6599474227870714, 0.6812789685464913, 0.6991994514662814, 0.7584199795854616, 0.7715795261783781, 0.7846514004587202, 0.8308257490422214, 0.8617337103547721], [0.022413793103448276, 0.023295944779982744, 0.028472821397756688, 0.030198446937014668, 0.03192407247627265, 0.04659188955996549, 0.057808455565142365, 0.058671268334771355, 0.06471095772217429, 0.091458153580673, 0.10440034512510785, 0.11993097497842968, 0.182053494391717, 0.23382226056945643, 0.28731665228645387, 0.34339948231233824, 0.41069887834339946, 0.4969801553062985, 0.5918895599654875, 0.6824849007765315, 0.7411561691113029, 0.7989646246764452, 0.8886971527178602, 0.9283865401207938, 0.9413287316652287, 0.9542709232096636, 0.9859792924935289, 0.9879206212251941, 0.994823123382226, 0.996548748921484, 0.998274374460742, 0.9991379310344828]], [[0.020474641714706893, 0.029248317021020452, 0.03249462698557362, 0.04358960407367651, 0.04920868504615057, 0.05247903750701384, 0.0558932319931362, 0.05970598597171878, 0.06411615198801372, 0.06860553743182748, 0.07918603338810228, 0.08541398944933552, 0.0921977498639938, 0.10041019265070435, 0.1304611877715292, 0.14196583091842463, 0.15509569053807534, 0.17126487457727546, 0.18811128731847795, 0.21059386204343197, 0.23962976695587918, 0.27597109837965533, 0.32356487859356725, 0.3927764003434457, 0.5281871974714532], [0.04134672179562906, 0.045501551189245086, 0.06120760959470637, 0.0703205791106515, 0.07660455486542443, 0.08057851239669421, 0.08902691511387163, 0.09297520661157024, 0.09917355371900827, 0.10134436401240951, 0.12629399585921325, 0.13429752066115702, 0.14492753623188406, 0.1543762922122674, 0.17768595041322313, 0.2070393374741201, 0.23553719008264462, 0.2505175983436853, 0.2603305785123967, 0.3512396694214876, 0.36024844720496896, 0.4462809917355372, 0.47619047619047616, 0.5950413223140496, 0.8285123966942148]]]}}

miner.py

@@ -0,0 +1,153 @@
+"""ScoreVision multi-label detection miner (first-mover kit).
+
+The validator ground truth for object elements is produced per prompt (one SAM3 query per manifest object), so one physical
+object can carry several labels (for example "person" and "fallen person"). This miner therefore decodes the raw one2many
+head [1, 4+nc, N] per class instead of taking the arg-max: an anchor emits class c when its score passes thr[c] (c is the
+anchor's top class) or thr2[c] (c is a secondary label). NMS runs per class only. CPU execution provider everywhere so the
+live chute and the offline conformity checker produce identical boxes. Thresholds live in config.json and can be re-fitted
+without retraining; out_map maps kit classes onto the manifest object list (-1 drops a class).
+"""
+
+import json
+from pathlib import Path
+
+import cv2
+import numpy as np
+import onnxruntime as ort
+from numpy import ndarray
+from pydantic import BaseModel
+
+
+# ---- learned emission policy (linear over 22 box/image features, trained by ES on the exact SN44 metric) ----
+def _pol_iou(a):
+    lo = np.maximum(a[:, None, :2], a[None, :, :2]); hi = np.minimum(a[:, None, 2:], a[None, :, 2:])
+    inter = np.maximum(hi - lo, 0).prod(2); ar = np.maximum(a[:, 2:] - a[:, :2], 0).prod(1)
+    u = ar[:, None] + ar[None, :] - inter
+    return np.divide(inter, u, out=np.zeros_like(inter), where=u > 0)
+
+def _pol_features(det, W, H, cal):
+    n = len(det); c = det; cl = c[:, 5].astype(int)
+    pi = np.zeros(n)
+    for k in range(4):
+        m = cl == k
+        if m.any(): pi[m] = np.interp(c[m, 4], np.asarray(cal[k][0]), np.asarray(cal[k][1]))
+    nh = np.array([pi[cl == k].sum() for k in range(4)]); nt = pi.sum(); npos = (pi > 0.5).sum()
+    area = np.maximum(c[:, 2] - c[:, 0], 1) * np.maximum(c[:, 3] - c[:, 1], 1) / (W * H)
+    asp = np.log(np.maximum(c[:, 3] - c[:, 1], 1) / np.maximum(c[:, 2] - c[:, 0], 1))
+    io = _pol_iou(c[:, :4]); same = cl[:, None] == cl[None, :]; hi = np.tril(np.ones((n, n), bool), -1)
+    dup = np.where(same & hi, io, 0).max(1) if n > 1 else np.zeros(n)
+    rank = np.zeros(n); mx = np.zeros(4)
+    for k in range(4):
+        m = cl == k; rank[m] = np.arange(m.sum())
+        if m.any(): mx[k] = c[m, 4].max()
+    f = np.zeros((n, 22))
+    f[:, 0] = 1; f[:, 1] = np.log(c[:, 4] / (1 - c[:, 4] + 1e-6) + 1e-6) / 4; f[:, 2] = pi
+    for k in range(4): f[:, 3 + k] = (cl == k)
+    f[:, 7] = c[:, 6]; f[:, 8] = np.log1p(rank) / 3; f[:, 9] = (np.log10(area) + 3) / 2; f[:, 10] = asp
+    f[:, 11] = np.log1p(nt) / 3; f[:, 12] = np.log1p(nh[cl]) / 3; f[:, 13] = np.log1p(npos) / 3
+    f[:, 14] = dup; f[:, 15] = (dup > 0.5); f[:, 16] = mx[cl]; f[:, 17] = pi / (nh[cl] + 1)
+    f[:, 18] = (pi > 0.5) * 1.0; f[:, 19] = np.log1p(nh[cl] + 0.01) * pi; f[:, 20] = pi * (rank + 1) / (nh[cl] + 1); f[:, 21] = (nh > 0.5).sum() / 4
+    return f
+
+def _pol_decide(det, W, H, P):
+    if len(det) == 0: return np.zeros(0, bool)
+    return _pol_features(np.asarray(det, np.float64), W, H, P["cal"]) @ np.asarray(P["w"]) > 0
+
+
+class BoundingBox(BaseModel):
+    x1: int
+    y1: int
+    x2: int
+    y2: int
+    cls_id: int
+    conf: float
+
+
+class TVFrameResult(BaseModel):
+    frame_id: int
+    boxes: list[BoundingBox]
+    keypoints: list[tuple[int, int]]
+
+
+class Miner:
+    def __init__(self, path_hf_repo: Path) -> None:
+        repo = Path(path_hf_repo)
+        cfg = json.loads((repo / "config.json").read_text())
+        self.nc = len(cfg["classes"])
+        self.thr = np.asarray(cfg["thr"], dtype=np.float32)
+        self.thr2 = np.asarray(cfg.get("thr2", cfg["thr"]), dtype=np.float32)
+        self.nms_iou = np.asarray(cfg.get("nms_iou", [0.6] * self.nc), dtype=np.float32)
+        self.max_det = int(cfg.get("max_det", 300))
+        # out_map[c] = manifest cls_id emitted for kit class c (-1 = never emitted); default identity
+        self.out_map = [int(v) for v in cfg.get("out_map", list(range(self.nc)))]
+        self.policy = cfg.get("policy")   # {"w": [22], "cal": [[xs, ps] x 4]}: emit/skip decided per candidate box
+        self.min_side = float(cfg.get("min_side", 2.0))
+        so = ort.SessionOptions()
+        so.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
+        so.intra_op_num_threads = int(cfg.get("threads", 2))
+        so.inter_op_num_threads = 1
+        so.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL
+        so.add_session_config_entry("session.intra_op.allow_spinning", "0")
+        self.session = ort.InferenceSession(str(repo / "weights.onnx"), sess_options=so, providers=["CPUExecutionProvider"])
+        inp = self.session.get_inputs()[0]
+        self.input_name = inp.name
+        self.u8 = "uint8" in inp.type  # uint8 NHWC BGR input with normalisation fused in-graph (fast path)
+        self.h_in, self.w_in = (int(inp.shape[1]), int(inp.shape[2])) if self.u8 else (int(inp.shape[2]), int(inp.shape[3]))
+        dummy = np.zeros((1, self.h_in, self.w_in, 3), np.uint8) if self.u8 else np.zeros((1, 3, self.h_in, self.w_in), np.float32)
+        for _ in range(3):
+            self.session.run(None, {self.input_name: dummy})
+
+    def __repr__(self) -> str:
+        return f"MultiLabelMiner(nc={self.nc}, input={self.h_in}x{self.w_in})"
+
+    def _prep(self, img: ndarray):
+        h, w = img.shape[:2]
+        r = min(self.h_in / h, self.w_in / w)
+        nh, nw = int(round(h * r)), int(round(w * r))
+        x = np.full((self.h_in, self.w_in, 3), 114, np.uint8)
+        top, left = (self.h_in - nh) // 2, (self.w_in - nw) // 2
+        x[top:top + nh, left:left + nw] = cv2.resize(img, (nw, nh), interpolation=cv2.INTER_LINEAR)
+        if self.u8:
+            return np.ascontiguousarray(x[None]), r, left, top
+        x = x[:, :, ::-1].transpose(2, 0, 1)[None].astype(np.float32) * (1.0 / 255.0)
+        return np.ascontiguousarray(x), r, left, top
+
+    def _decode(self, y: ndarray) -> ndarray:
+        y = y.T  # [N, 4+nc]: cx, cy, w, h, class scores
+        s = y[:, 4:]
+        top = s.argmax(1)
+        need = np.where(np.arange(self.nc)[None, :] == top[:, None], self.thr[None, :], self.thr2[None, :])
+        out = []
+        for c in range(self.nc):
+            if self.out_map[c] < 0:
+                continue
+            m = s[:, c] >= need[:, c]
+            if not m.any():
+                continue
+            b = y[m, :4]
+            sc = s[m, c]
+            xyxy = np.stack([b[:, 0] - b[:, 2] / 2, b[:, 1] - b[:, 3] / 2, b[:, 0] + b[:, 2] / 2, b[:, 1] + b[:, 3] / 2], 1)
+            xywh = np.concatenate([xyxy[:, :2], xyxy[:, 2:] - xyxy[:, :2]], 1).astype(np.float64)
+            k = np.asarray(cv2.dnn.NMSBoxes(xywh.tolist(), sc.astype(np.float32).tolist(), 0.0, float(self.nms_iou[c])), dtype=int).reshape(-1)
+            if k.size:
+                out.append(np.concatenate([xyxy[k], sc[k, None], np.full((k.size, 1), self.out_map[c], np.float32), (top[m][k] == c).astype(np.float32)[:, None]], 1))
+        if not out:
+            return np.zeros((0, 7), np.float32)
+        det = np.concatenate(out, 0)
+        return det[np.argsort(-det[:, 4], kind="stable")][: self.max_det]
+
+    def predict_batch(self, batch_images: list[ndarray], offset: int, n_keypoints: int) -> list[TVFrameResult]:
+        results = []
+        for i, img in enumerate(batch_images):
+            h, w = img.shape[:2]
+            x, r, left, top = self._prep(img)
+            det = self._decode(self.session.run(None, {self.input_name: x})[0][0]).astype(np.float64)
+            det[:, [0, 2]] = np.clip((det[:, [0, 2]] - left) / r, 0, w); det[:, [1, 3]] = np.clip((det[:, [1, 3]] - top) / r, 0, h)
+            det[:, :4] = np.round(det[:, :4])
+            det = det[((det[:, 2] - det[:, 0]) >= self.min_side) & ((det[:, 3] - det[:, 1]) >= self.min_side)].reshape(-1, 7)
+            if self.policy is not None and len(det):
+                det = det[_pol_decide(det, w, h, self.policy)]
+            boxes = [BoundingBox(x1=int(x1), y1=int(y1), x2=int(x2), y2=int(y2), cls_id=int(c), conf=float(round(float(conf), 5)))
+                     for x1, y1, x2, y2, conf, c, _ in det]
+            results.append(TVFrameResult(frame_id=offset + i, boxes=boxes, keypoints=[]))
+        return results

weights.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66401a1dd6988a4ca980e13e06db8b32387f3f447fa468f3c1ad754a4be69cca
+size 9833859