firedetect1007 / miner.py
happydev93's picture
scorevision: push artifact
02fcd4d verified
Raw History Blame Contribute Delete
129 kB
from pathlib import Path import math import cv2 import numpy as np import onnxruntime as ort from numpy import ndarray from pydantic import BaseModel class BoundingBox(BaseModel): x1: int y1: int x2: int y2: int cls_id: int conf: float class TVFrameResult(BaseModel): frame_id: int boxes: list[BoundingBox] keypoints: list[tuple[int, int]] class Miner: class_names = ["fire", "smoke", "fire extinguisher"] _model_class_order = ["fire", "fire extinguisher", "smoke"] _parent_split_class_names = ("fire", "fire extinguisher") conf_thresh = 0.000005 isolated_conf_thresh = 0.008 same_class_iou_thresh = 0.6 edge_conf_thresh = 0.005 edge_tol = 1.0 overlap_low_conf_thresh = 0.003 overlap_same_class_iou_thresh = 0.5 group_iou_thresh = 0.7 flip_low_conf_thresh = 0.2 fire_ext_same_object_overlap_thresh = 0.7 fire_ext_group_involve_thresh = 0.9 smoke_involve_thresh = 0.7 smoke_prefer_larger_conf_thresh = 0.3 smoke_cluster_merge_involve_thresh = 0.3 smoke_cluster_merge_min_size = 2 smoke_cluster_merge_early_min_count = 3 small_group_min_count = 2 small_group_low_conf_thresh = 0.02 fire_parent_min_small_count = 2 fire_parent_involve_thresh = 0.7 fire_split_inner_involve_thresh = 0.5 fire_split_max_inner_area_ratio = 0.5 fire_split_cluster_iou_thresh = 0.5 fire_split_min_clusters = 2 fire_split_min_inner_conf = 0.08 fire_split_peak_member_min_conf = 0.05 fire_split_peak_dist_ratio = 0.10 fire_split_peak_dist_min_px = 12.0 fire_split_peak_dist_max_px = 48.0 fire_split_centroid_preserve_px = 12.0 fire_split_conf_boost = 0.2 fire_split_parent_blob_conf_thresh = 0.5 fire_min_conf = 0.165 smoke_min_conf = 0.07 fire_ext_min_conf = 0.037 fire_ext_color_filter_max_conf = 0.20 fire_ext_min_red_dom_frac = 0.03 fire_ext_color_min_mean_r = 50.0 fire_ext_min_r_minus_g = 0.0 smoke_anchor_min_smoke_conf = 0.63 smoke_anchor_base_height_frac = 0.35 smoke_anchor_crop_width_pad_ratio = 0.15 smoke_anchor_crop_down_pad_px = 24.0 smoke_anchor_corroborate_involve_thresh = 0.5 smoke_anchor_fire_floor_conf = 0.165 smoke_anchor_min_fire_width_px = 10.0 smoke_anchor_min_fire_height_px = 10.0 smoke_anchor_strong_smoke_conf = 0.7 smoke_anchor_corroborated_min_fire_width_px = 4.0 smoke_anchor_corroborated_min_fire_height_px = 4.0 smoke_anchor_max_fire_area_ratio = 0.25 smoke_anchor_corroborate_area_ratio_max = 4.0 smoke_anchor_max_fires_per_smoke = 2 smoke_anchor_max_crop_fires = 2 smoke_anchor_max_probe_smokes = 2 smoke_anchor_separate_fire_involve_thresh = 0.5 smoke_anchor_max_probe_fire_width_px = 52.0 smoke_anchor_max_probe_fire_height_px = 48.0 smoke_anchor_max_probe_fire_area_ratio = 0.04 smoke_anchor_max_probe_crop_width_ratio = 0.25 smoke_anchor_max_probe_crop_height_ratio = 0.60 smoke_anchor_max_probe_fire_bottom_dist_frac = 0.10 smoke_anchor_probe_rescue_min_conf = 0.000005 smoke_anchor_corroborated_probe_min_conf = 0.00005 smoke_anchor_corroborated_refine_pad_px = 3.0 smoke_anchor_expanded_width_ratio = 1.25 ext_probe_min_cluster_count = 2 ext_probe_cluster_centroid_dist_px = 28.0 ext_probe_min_cluster_best_conf = 0.003 ext_probe_crop_pad_ratio = 0.85 ext_probe_min_crop_size_px = 200.0 ext_probe_crop_rescue_min_conf = 0.5 ext_probe_crop_rescue_min_width_px = 6.0 ext_probe_rescue_min_conf = 0.000005 ext_probe_min_width_px = 10.0 ext_probe_min_height_px = 10.0 ext_probe_max_width_px = 120.0 ext_probe_max_height_px = 160.0 ext_probe_fire_exclude_involve_thresh = 0.5 ext_probe_near_fire_gap_px = 24.0 ext_probe_duplicate_involve_thresh = 0.5 ext_probe_max_per_frame = 2 ext_probe_edge_tol = 1.0 ext_probe_singleton_min_conf = 0.00005 ext_probe_singleton_max_area = 500.0 ext_probe_singleton_max_width_px = 20.0 ext_probe_singleton_max_height_px = 35.0 ext_probe_singleton_min_edge_margin_px = 12.0 smoke_anchor_probe_min_mean_r = 65.0 smoke_anchor_probe_min_max_rgb = 80.0 smoke_anchor_probe_min_warm_frac = 0.05 smoke_anchor_probe_warm_min_max_rgb = 120.0 smoke_anchor_probe_warm_min_mean_r = 120.0 smoke_anchor_probe_strong_warm_frac = 0.15 smoke_anchor_probe_min_bright_frac = 0.12 smoke_anchor_probe_bright_thresh = 150 smoke_anchor_probe_min_hot_max_rgb = 180.0 smoke_anchor_probe_min_r_minus_g = 2.0 smoke_anchor_probe_sparse_hot_min_max_rgb = 200.0 smoke_anchor_probe_sparse_hot_min_bright_frac = 0.01 smoke_expand_min_seed_conf = 0.5 smoke_expand_crop_width_pad_ratio = 0.45 smoke_expand_crop_up_pad_ratio = 0.15 smoke_expand_crop_down_pad_px = 32.0 smoke_expand_corroborate_involve_thresh = 0.3 smoke_expand_vertical_band_pad_ratio = 0.10 smoke_expand_band_max_horizontal_gap_ratio = 0.25 smoke_expand_upward_diagonal_max_gap_ratio = 0.45 smoke_expand_min_probe_smoke_conf = 0.10 smoke_expand_min_tta_smoke_conf = 0.12 smoke_expand_min_tta_extend_conf = 0.15 smoke_expand_tta_view_iou_thresh = 0.5 smoke_expand_wide_seed_max_width_ratio = 0.25 smoke_expand_upper_plume_max_y_ratio = 0.72 smoke_expand_wide_crop_x_start_ratio = 0.05 smoke_expand_wide_crop_x_end_ratio = 0.95 smoke_expand_min_crop_width_ratio = 0.60 smoke_expand_border_strip_ratio = 0.15 smoke_expand_max_color_dist = 45.0 smoke_expand_max_frame_ratio = 0.20 smoke_expand_min_extend_probe_conf = 0.15 smoke_expand_min_bidi_crop_conf = 0.40 smoke_expand_min_upward_tta_conf = 0.012 smoke_expand_min_upward_tta_involve_conf = 0.0002 smoke_expand_min_upward_crop_conf = 0.012 smoke_expand_max_seeds = 1 smoke_expand_skip_min_conf = 0.85 smoke_expand_skip_min_width_ratio = 0.15 smoke_expand_skip_max_y1_ratio = 0.04 def __init__(self, path_hf_repo: Path) -> None: model_path = path_hf_repo / "weights.onnx" self.group_object_counts: list[int] = [] self.group_max_confidences: list[float] = [] self.group_avg_confidences: list[float] = [] self.cls_remap = np.array( [self.class_names.index(n) for n in self._model_class_order], dtype=np.int32, ) print("ORT version:", ort.__version__) try: ort.preload_dlls() print("✅ onnxruntime.preload_dlls() success") except Exception as e: print(f"⚠️ preload_dlls failed: {e}") print("ORT available providers BEFORE session:", ort.get_available_providers()) sess_options = ort.SessionOptions() sess_options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL try: self.session = ort.InferenceSession( str(model_path), sess_options=sess_options, providers=["CUDAExecutionProvider", "CPUExecutionProvider"], ) print("✅ Created ORT session with preferred CUDA provider list") except Exception as e: print(f"⚠️ CUDA session creation failed, falling back to CPU: {e}") self.session = ort.InferenceSession( str(model_path), sess_options=sess_options, providers=["CPUExecutionProvider"], ) print("ORT session providers:", self.session.get_providers()) for inp in self.session.get_inputs(): print("INPUT:", inp.name, inp.shape, inp.type) for out in self.session.get_outputs(): print("OUTPUT:", out.name, out.shape, out.type) self.input_name = self.session.get_inputs()[0].name self.output_names = [output.name for output in self.session.get_outputs()] self.input_shape = self.session.get_inputs()[0].shape self.input_height = self._safe_dim(self.input_shape[2], default=1280) self.input_width = self._safe_dim(self.input_shape[3], default=1280) self.use_tta = True print(f"✅ ONNX model loaded from: {model_path}") print(f"✅ ONNX providers: {self.session.get_providers()}") print(f"✅ ONNX input: name={self.input_name}, shape={self.input_shape}") def __repr__(self) -> str: return ( f"ONNXRuntime(session={type(self.session).__name__}, " f"providers={self.session.get_providers()})" ) @staticmethod def _safe_dim(value, default: int) -> int: return value if isinstance(value, int) and value > 0 else default def _letterbox( self, image: ndarray, new_shape: tuple[int, int], color=(114, 114, 114), ) -> tuple[ndarray, float, tuple[float, float]]: h, w = image.shape[:2] new_w, new_h = new_shape ratio = min(new_w / w, new_h / h) resized_w = int(round(w * ratio)) resized_h = int(round(h * ratio)) if (resized_w, resized_h) != (w, h): interp = cv2.INTER_CUBIC if ratio > 1.0 else cv2.INTER_LINEAR image = cv2.resize(image, (resized_w, resized_h), interpolation=interp) dw = (new_w - resized_w) / 2.0 dh = (new_h - resized_h) / 2.0 left = int(round(dw - 0.1)) right = int(round(dw + 0.1)) top = int(round(dh - 0.1)) bottom = int(round(dh + 0.1)) padded = cv2.copyMakeBorder( image, top, bottom, left, right, borderType=cv2.BORDER_CONSTANT, value=color, ) return padded, ratio, (dw, dh) def _preprocess( self, image: ndarray ) -> tuple[np.ndarray, float, tuple[float, float], tuple[int, int]]: orig_h, orig_w = image.shape[:2] img, ratio, pad = self._letterbox( image, (self.input_width, self.input_height) ) img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) img = img.astype(np.float32) / 255.0 img = np.transpose(img, (2, 0, 1))[None, ...] img = np.ascontiguousarray(img, dtype=np.float32) return img, ratio, pad, (orig_w, orig_h) @staticmethod def _clip_boxes(boxes: np.ndarray, image_size: tuple[int, int]) -> np.ndarray: w, h = image_size boxes[:, 0] = np.clip(boxes[:, 0], 0, w - 1) boxes[:, 1] = np.clip(boxes[:, 1], 0, h - 1) boxes[:, 2] = np.clip(boxes[:, 2], 0, w - 1) boxes[:, 3] = np.clip(boxes[:, 3], 0, h - 1) return boxes @staticmethod def _build_results( boxes: np.ndarray, scores: np.ndarray, cls_ids: np.ndarray ) -> list[BoundingBox]: results: list[BoundingBox] = [] for box, conf, cls_id in zip(boxes, scores, cls_ids): x1, y1, x2, y2 = box.tolist() if x2 <= x1 or y2 <= y1: continue results.append( BoundingBox( x1=int(math.floor(x1)), y1=int(math.floor(y1)), x2=int(math.ceil(x2)), y2=int(math.ceil(y2)), cls_id=int(cls_id), conf=float(conf), ) ) return results def _map_boxes_to_orig( self, boxes: np.ndarray, ratio: float, pad: tuple[float, float], orig_size: tuple[int, int], ) -> np.ndarray: pad_w, pad_h = pad boxes = boxes.copy() boxes[:, [0, 2]] -= pad_w boxes[:, [1, 3]] -= pad_h boxes /= ratio return self._clip_boxes(boxes, orig_size) @staticmethod def _compute_iou_matrix(boxes: np.ndarray) -> np.ndarray: n = len(boxes) if n == 0: return np.empty((0, 0), dtype=np.float32) areas = (np.maximum(0.0, boxes[:, 2] - boxes[:, 0]) * np.maximum(0.0, boxes[:, 3] - boxes[:, 1])) iou = np.zeros((n, n), dtype=np.float32) for i in range(n): bi = boxes[i] xx1 = np.maximum(bi[0], boxes[i + 1:, 0]) yy1 = np.maximum(bi[1], boxes[i + 1:, 1]) xx2 = np.minimum(bi[2], boxes[i + 1:, 2]) yy2 = np.minimum(bi[3], boxes[i + 1:, 3]) inter = np.maximum(0.0, xx2 - xx1) * np.maximum(0.0, yy2 - yy1) a_i = max(1e-7, float((bi[2] - bi[0]) * (bi[3] - bi[1]))) iou[i, i + 1:] = inter / (a_i + areas[i + 1:] - inter + 1e-7) iou = iou + iou.T np.fill_diagonal(iou, 1.0) return iou def _group_overlapping_detections( self, boxes: np.ndarray, scores: np.ndarray, cls_ids: np.ndarray, iou_thresh: float | None = None, view_ids: np.ndarray | None = None, record_stats: bool = True, ) -> tuple[np.ndarray, np.ndarray, np.ndarray]: if iou_thresh is None: iou_thresh = self.group_iou_thresh n = len(boxes) if record_stats: self.group_object_counts = [] self.group_max_confidences = [] self.group_avg_confidences = [] if n == 0: return boxes, scores, cls_ids boxes = np.asarray(boxes, dtype=np.float32) scores = np.asarray(scores, dtype=np.float32) cls_ids = np.asarray(cls_ids, dtype=np.int32) if view_ids is not None: view_ids = np.asarray(view_ids, dtype=np.int32) parent = list(range(n)) def find(x: int) -> int: while parent[x] != x: parent[x] = parent[parent[x]] x = parent[x] return x def union(a: int, b: int) -> None: ra, rb = find(a), find(b) if ra != rb: parent[rb] = ra iou = self._compute_iou_matrix(boxes) for i in range(n): for j in range(i + 1, n): if cls_ids[i] != cls_ids[j]: continue if view_ids is not None and view_ids[i] != view_ids[j]: continue if iou[i, j] > iou_thresh: union(i, j) groups: dict[int, list[int]] = {} for i in range(n): groups.setdefault(find(i), []).append(i) kept_groups: list[tuple[int, int, float, float]] = [] for members in groups.values(): member_scores = scores[members] count = len(members) max_conf = float(np.max(member_scores)) avg_conf = float(np.mean(member_scores)) best = members[int(np.argmax(member_scores))] kept_groups.append((best, count, max_conf, avg_conf)) kept_groups.sort(key=lambda item: item[0]) keep = np.array([item[0] for item in kept_groups], dtype=np.intp) if record_stats: self.group_object_counts = [item[1] for item in kept_groups] self.group_max_confidences = [item[2] for item in kept_groups] self.group_avg_confidences = [item[3] for item in kept_groups] return boxes[keep], scores[keep], cls_ids[keep] def _remove_isolated_low_conf_same_class( self, boxes: np.ndarray, scores: np.ndarray, cls_ids: np.ndarray, view_ids: np.ndarray | None = None, ) -> tuple[np.ndarray, np.ndarray, np.ndarray] | tuple[ np.ndarray, np.ndarray, np.ndarray, np.ndarray ]: n = len(boxes) if n == 0: if view_ids is None: return boxes, scores, cls_ids return boxes, scores, cls_ids, view_ids if n == 1: if scores[0] < self.isolated_conf_thresh: empty_boxes = np.empty((0, 4), dtype=np.float32) empty_scores = np.empty((0,), dtype=np.float32) empty_cls = np.empty((0,), dtype=np.int32) if view_ids is None: return empty_boxes, empty_scores, empty_cls return empty_boxes, empty_scores, empty_cls, np.empty((0,), dtype=np.int32) if view_ids is None: return boxes, scores, cls_ids return boxes, scores, cls_ids, view_ids boxes = np.asarray(boxes, dtype=np.float32) scores = np.asarray(scores, dtype=np.float32) cls_ids = np.asarray(cls_ids, dtype=np.int32) areas = (np.maximum(0.0, boxes[:, 2] - boxes[:, 0]) * np.maximum(0.0, boxes[:, 3] - boxes[:, 1])) keep = np.ones(n, dtype=bool) low_conf = scores < self.isolated_conf_thresh if not np.any(low_conf): if view_ids is None: return boxes, scores, cls_ids return boxes, scores, cls_ids, view_ids for i in np.where(low_conf)[0]: bi = boxes[i] xx1 = np.maximum(bi[0], boxes[:, 0]) yy1 = np.maximum(bi[1], boxes[:, 1]) xx2 = np.minimum(bi[2], boxes[:, 2]) yy2 = np.minimum(bi[3], boxes[:, 3]) inter = np.maximum(0.0, xx2 - xx1) * np.maximum(0.0, yy2 - yy1) a_i = max(1e-7, float((bi[2] - bi[0]) * (bi[3] - bi[1]))) iou = inter / (a_i + areas - inter + 1e-7) same_class = cls_ids == cls_ids[i] other = np.arange(n) != i if not np.any((iou > self.same_class_iou_thresh) & same_class & other): keep[i] = False if view_ids is None: return boxes[keep], scores[keep], cls_ids[keep] return boxes[keep], scores[keep], cls_ids[keep], view_ids[keep] def _remove_edge_low_conf( self, boxes: np.ndarray, scores: np.ndarray, cls_ids: np.ndarray, orig_size: tuple[int, int], view_ids: np.ndarray | None = None, ) -> tuple[np.ndarray, np.ndarray, np.ndarray] | tuple[ np.ndarray, np.ndarray, np.ndarray, np.ndarray ]: if len(boxes) == 0: if view_ids is None: return boxes, scores, cls_ids return boxes, scores, cls_ids, view_ids orig_w, orig_h = orig_size tol = self.edge_tol on_edge = ( (boxes[:, 0] <= tol) | (boxes[:, 1] <= tol) | (boxes[:, 2] >= orig_w - 1 - tol) | (boxes[:, 3] >= orig_h - 1 - tol) ) keep = ~(on_edge & (scores < self.edge_conf_thresh)) if view_ids is None: return boxes[keep], scores[keep], cls_ids[keep] return boxes[keep], scores[keep], cls_ids[keep], view_ids[keep] def _remove_overlapping_low_conf_same_class( self, boxes: np.ndarray, scores: np.ndarray, cls_ids: np.ndarray, view_ids: np.ndarray | None = None, ) -> tuple[np.ndarray, np.ndarray, np.ndarray] | tuple[ np.ndarray, np.ndarray, np.ndarray, np.ndarray ]: n = len(boxes) if n <= 1: if view_ids is None: return boxes, scores, cls_ids return boxes, scores, cls_ids, view_ids boxes = np.asarray(boxes, dtype=np.float32) scores = np.asarray(scores, dtype=np.float32) cls_ids = np.asarray(cls_ids, dtype=np.int32) areas = (np.maximum(0.0, boxes[:, 2] - boxes[:, 0]) * np.maximum(0.0, boxes[:, 3] - boxes[:, 1])) keep = np.ones(n, dtype=bool) low_conf = scores < self.overlap_low_conf_thresh if not np.any(low_conf): if view_ids is None: return boxes, scores, cls_ids return boxes, scores, cls_ids, view_ids for i in np.where(low_conf)[0]: bi = boxes[i] xx1 = np.maximum(bi[0], boxes[:, 0]) yy1 = np.maximum(bi[1], boxes[:, 1]) xx2 = np.minimum(bi[2], boxes[:, 2]) yy2 = np.minimum(bi[3], boxes[:, 3]) inter = np.maximum(0.0, xx2 - xx1) * np.maximum(0.0, yy2 - yy1) a_i = max(1e-7, float((bi[2] - bi[0]) * (bi[3] - bi[1]))) iou = inter / (a_i + areas - inter + 1e-7) same_class = cls_ids == cls_ids[i] other = np.arange(n) != i if np.any((iou >= self.overlap_same_class_iou_thresh) & same_class & other): keep[i] = False if view_ids is None: return boxes[keep], scores[keep], cls_ids[keep] return boxes[keep], scores[keep], cls_ids[keep], view_ids[keep] def _decode_preds_to_arrays( self, preds: np.ndarray, ratio: float, pad: tuple[float, float], orig_size: tuple[int, int], ) -> tuple[np.ndarray, np.ndarray, np.ndarray]: if preds.ndim == 3 and preds.shape[0] == 1: preds = preds[0] if preds.ndim != 2 or preds.shape[1] < 6: raise ValueError(f"Unexpected ONNX final-det output shape: {preds.shape}") boxes = preds[:, :4].astype(np.float32) scores = preds[:, 4].astype(np.float32) cls_ids = self.cls_remap[preds[:, 5].astype(np.int32)] keep = scores >= self.conf_thresh boxes = boxes[keep] scores = scores[keep] cls_ids = cls_ids[keep] if len(boxes) == 0: return ( np.empty((0, 4), dtype=np.float32), np.empty((0,), dtype=np.float32), np.empty((0,), dtype=np.int32), ) boxes = self._map_boxes_to_orig(boxes, ratio, pad, orig_size) valid = (boxes[:, 2] > boxes[:, 0]) & (boxes[:, 3] > boxes[:, 1]) return boxes[valid], scores[valid], cls_ids[valid] @staticmethod def _map_flipped_boxes_arrays( boxes: np.ndarray, orig_size: tuple[int, int], flip_code: int, ) -> np.ndarray: orig_w, _orig_h = orig_size out = boxes.copy() if flip_code == 1: x1 = orig_w - out[:, 2] x2 = orig_w - out[:, 0] out[:, 0] = x1 out[:, 2] = x2 else: raise ValueError(f"Unsupported flip_code: {flip_code}") return out def _remove_unconfirmed_single_view_low_conf( self, boxes: np.ndarray, scores: np.ndarray, cls_ids: np.ndarray, view_ids: np.ndarray, conf_thresh: float | None = None, match_iou_thresh: float | None = None, protect_mask: np.ndarray | None = None, ) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: if conf_thresh is None: conf_thresh = self.flip_low_conf_thresh if match_iou_thresh is None: match_iou_thresh = self.overlap_same_class_iou_thresh n = len(boxes) if n == 0: return boxes, scores, cls_ids, view_ids low_conf = scores < conf_thresh if not np.any(low_conf): return boxes, scores, cls_ids, view_ids iou = self._compute_iou_matrix(boxes) keep = np.ones(n, dtype=bool) for i in np.where(low_conf)[0]: if protect_mask is not None and protect_mask[i]: continue other = np.arange(n) != i other_view = view_ids != view_ids[i] same_class = cls_ids == cls_ids[i] corroborated = np.any( other & other_view & same_class & (iou[i] >= match_iou_thresh) ) if not corroborated: keep[i] = False return boxes[keep], scores[keep], cls_ids[keep], view_ids[keep] @staticmethod def _boxes_involve_each_other( boxes: np.ndarray, i: int, j: int, overlap_thresh: float ) -> bool: bi, bj = boxes[i], boxes[j] xx1 = max(float(bi[0]), float(bj[0])) yy1 = max(float(bi[1]), float(bj[1])) xx2 = min(float(bi[2]), float(bj[2])) yy2 = min(float(bi[3]), float(bj[3])) inter = max(0.0, xx2 - xx1) * max(0.0, yy2 - yy1) if inter <= 0.0: return False area_i = max(1e-7, float((bi[2] - bi[0]) * (bi[3] - bi[1]))) area_j = max(1e-7, float((bj[2] - bj[0]) * (bj[3] - bj[1]))) return (inter / min(area_i, area_j)) >= overlap_thresh @staticmethod def _box_area(box: np.ndarray) -> float: return max(1e-7, float((box[2] - box[0]) * (box[3] - box[1]))) @staticmethod def _box_intersection_area(box_a: np.ndarray, box_b: np.ndarray) -> float: xx1 = max(float(box_a[0]), float(box_b[0])) yy1 = max(float(box_a[1]), float(box_b[1])) xx2 = min(float(box_a[2]), float(box_b[2])) yy2 = min(float(box_a[3]), float(box_b[3])) return max(0.0, xx2 - xx1) * max(0.0, yy2 - yy1) @staticmethod def _larger_involves_smaller( boxes: np.ndarray, big_idx: int, small_idx: int, involve_thresh: float ) -> bool: inter = Miner._box_intersection_area(boxes[big_idx], boxes[small_idx]) if inter <= 0.0: return False area_small = Miner._box_area(boxes[small_idx]) return (inter / area_small) >= involve_thresh @staticmethod def _either_box_involves_other( boxes: np.ndarray, i: int, j: int, involve_thresh: float ) -> bool: inter = Miner._box_intersection_area(boxes[i], boxes[j]) if inter <= 0.0: return False area_i = Miner._box_area(boxes[i]) area_j = Miner._box_area(boxes[j]) return (inter / area_i >= involve_thresh) or (inter / area_j >= involve_thresh) @staticmethod def _boxes_involve_each_other_arrays( box_a: np.ndarray, box_b: np.ndarray, involve_thresh: float ) -> bool: inter = Miner._box_intersection_area(box_a, box_b) if inter <= 0.0: return False area_a = Miner._box_area(box_a) area_b = Miner._box_area(box_b) return (inter / area_a >= involve_thresh) or (inter / area_b >= involve_thresh) def _resolve_smoke_overlap( self, boxes: np.ndarray, scores: np.ndarray, cls_ids: np.ndarray, involve_thresh: float | None = None, prefer_larger_conf_thresh: float | None = None, ) -> tuple[np.ndarray, np.ndarray, np.ndarray]: if involve_thresh is None: involve_thresh = self.smoke_involve_thresh if prefer_larger_conf_thresh is None: prefer_larger_conf_thresh = self.smoke_prefer_larger_conf_thresh n = len(boxes) if n <= 1: return boxes, scores, cls_ids cls_smoke = self.class_names.index("smoke") keep = np.ones(n, dtype=bool) for i in range(n): if not keep[i] or int(cls_ids[i]) != cls_smoke: continue for j in range(i + 1, n): if not keep[j] or int(cls_ids[j]) != cls_smoke: continue area_i = self._box_area(boxes[i]) area_j = self._box_area(boxes[j]) if area_i == area_j: big_idx, small_idx = (i, j) if scores[i] >= scores[j] else (j, i) elif area_i > area_j: big_idx, small_idx = i, j else: big_idx, small_idx = j, i if not self._larger_involves_smaller( boxes, big_idx, small_idx, involve_thresh ): continue big_conf = float(scores[big_idx]) small_conf = float(scores[small_idx]) if ( big_conf > prefer_larger_conf_thresh and small_conf > prefer_larger_conf_thresh ): keep[small_idx] = False elif big_conf >= small_conf: keep[small_idx] = False else: keep[big_idx] = False if big_idx == i: break return boxes[keep], scores[keep], cls_ids[keep] def _merge_overlapping_smoke_clusters( self, boxes: np.ndarray, scores: np.ndarray, cls_ids: np.ndarray, involve_thresh: float | None = None, min_cluster_size: int | None = None, ) -> tuple[np.ndarray, np.ndarray, np.ndarray]: if involve_thresh is None: involve_thresh = self.smoke_cluster_merge_involve_thresh if min_cluster_size is None: min_cluster_size = self.smoke_cluster_merge_min_size n = len(boxes) if n == 0: return boxes, scores, cls_ids cls_smoke = self.class_names.index("smoke") smoke_indices = [ i for i in range(n) if int(cls_ids[i]) == cls_smoke ] if len(smoke_indices) < min_cluster_size: return boxes, scores, cls_ids n_smoke = len(smoke_indices) parent = list(range(n_smoke)) def find(x: int) -> int: while parent[x] != x: parent[x] = parent[parent[x]] x = parent[x] return x def union(a: int, b: int) -> None: ra, rb = find(a), find(b) if ra != rb: parent[rb] = ra for ai in range(n_smoke): for bi in range(ai + 1, n_smoke): idx_i = smoke_indices[ai] idx_j = smoke_indices[bi] if self._either_box_involves_other( boxes, idx_i, idx_j, involve_thresh ): union(ai, bi) clusters: dict[int, list[int]] = {} for ai in range(n_smoke): root = find(ai) clusters.setdefault(root, []).append(smoke_indices[ai]) remove_indices: set[int] = set() merged_boxes: list[np.ndarray] = [] merged_scores: list[float] = [] for cluster in clusters.values(): if len(cluster) < min_cluster_size: continue remove_indices.update(cluster) union_box = boxes[cluster[0]].copy() max_conf = float(scores[cluster[0]]) for idx in cluster[1:]: union_box[0] = min(union_box[0], boxes[idx][0]) union_box[1] = min(union_box[1], boxes[idx][1]) union_box[2] = max(union_box[2], boxes[idx][2]) union_box[3] = max(union_box[3], boxes[idx][3]) max_conf = max(max_conf, float(scores[idx])) merged_boxes.append(union_box) merged_scores.append(max_conf) if not remove_indices: return boxes, scores, cls_ids keep = np.ones(n, dtype=bool) for idx in remove_indices: keep[idx] = False out_boxes = list(boxes[keep]) out_scores = [float(s) for s in scores[keep]] out_cls = [int(c) for c in cls_ids[keep]] out_boxes.extend(merged_boxes) out_scores.extend(merged_scores) out_cls.extend([cls_smoke] * len(merged_boxes)) return ( np.asarray(out_boxes, dtype=np.float32), np.asarray(out_scores, dtype=np.float32), np.asarray(out_cls, dtype=np.int32), ) def _remove_class_min_conf( self, boxes: np.ndarray, scores: np.ndarray, cls_ids: np.ndarray, fire_min_conf: float | None = None, smoke_min_conf: float | None = None, fire_ext_min_conf: float | None = None, ) -> tuple[np.ndarray, np.ndarray, np.ndarray]: if fire_min_conf is None: fire_min_conf = self.fire_min_conf if smoke_min_conf is None: smoke_min_conf = self.smoke_min_conf if fire_ext_min_conf is None: fire_ext_min_conf = self.fire_ext_min_conf n = len(boxes) if n == 0: return boxes, scores, cls_ids cls_fire = self.class_names.index("fire") cls_smoke = self.class_names.index("smoke") cls_fire_ext = self.class_names.index("fire extinguisher") keep = np.ones(n, dtype=bool) for i in range(n): cls_id = int(cls_ids[i]) if cls_id == cls_fire and float(scores[i]) < fire_min_conf: keep[i] = False elif cls_id == cls_smoke and float(scores[i]) < smoke_min_conf: keep[i] = False elif cls_id == cls_fire_ext and float(scores[i]) < fire_ext_min_conf: keep[i] = False return boxes[keep], scores[keep], cls_ids[keep] def _filter_results_min_conf( self, results: list[BoundingBox] ) -> list[BoundingBox]: if not results: return results cls_fire = self.class_names.index("fire") cls_smoke = self.class_names.index("smoke") cls_fire_ext = self.class_names.index("fire extinguisher") filtered: list[BoundingBox] = [] for box in results: if box.cls_id == cls_fire and box.conf < self.fire_min_conf: continue if box.cls_id == cls_smoke and box.conf < self.smoke_min_conf: continue if box.cls_id == cls_fire_ext and box.conf < self.fire_ext_min_conf: continue filtered.append(box) return filtered def _resolve_fire_and_ext_same_class_overlap( self, boxes: np.ndarray, scores: np.ndarray, cls_ids: np.ndarray, overlap_thresh: float | None = None, group_involve_thresh: float | None = None, ) -> tuple[np.ndarray, np.ndarray, np.ndarray]: if overlap_thresh is None: overlap_thresh = self.fire_ext_same_object_overlap_thresh if group_involve_thresh is None: group_involve_thresh = self.fire_ext_group_involve_thresh n = len(boxes) if n <= 1: return boxes, scores, cls_ids cls_fire = self.class_names.index("fire") dedupe_classes = { cls_fire, self.class_names.index("fire extinguisher"), } keep = np.ones(n, dtype=bool) for i in range(n): if not keep[i] or int(cls_ids[i]) not in dedupe_classes: continue for j in range(i + 1, n): if not keep[j] or cls_ids[i] != cls_ids[j]: continue should_dedupe = self._either_box_involves_other( boxes, i, j, group_involve_thresh ) if not should_dedupe: if not self._boxes_involve_each_other( boxes, i, j, overlap_thresh ): continue if int(cls_ids[i]) == cls_fire: cx_i, cy_i = self._box_centroid(boxes[i]) cx_j, cy_j = self._box_centroid(boxes[j]) dx = cx_i - cx_j dy = cy_i - cy_j if ( dx * dx + dy * dy > self.fire_split_centroid_preserve_px * self.fire_split_centroid_preserve_px ): continue should_dedupe = True if not should_dedupe: continue if scores[i] >= scores[j]: keep[j] = False else: keep[i] = False break return boxes[keep], scores[keep], cls_ids[keep] def _parent_split_cls_ids(self) -> list[int]: return [self.class_names.index(name) for name in self._parent_split_class_names] def _remove_spanning_parent_fire( self, boxes: np.ndarray, scores: np.ndarray, cls_ids: np.ndarray, view_ids: np.ndarray | None = None, involve_thresh: float | None = None, min_small_count: int | None = None, ) -> tuple[np.ndarray, np.ndarray, np.ndarray] | tuple[ np.ndarray, np.ndarray, np.ndarray, np.ndarray ]: if involve_thresh is None: involve_thresh = self.fire_parent_involve_thresh if min_small_count is None: min_small_count = self.fire_parent_min_small_count n = len(boxes) if n <= 1: if view_ids is None: return boxes, scores, cls_ids return boxes, scores, cls_ids, view_ids keep = np.ones(n, dtype=bool) for target_cls in self._parent_split_cls_ids(): class_indices = [i for i in range(n) if int(cls_ids[i]) == target_cls] if len(class_indices) <= 1: continue for a_idx in class_indices: area_a = self._box_area(boxes[a_idx]) small_indices = [ j for j in class_indices if j != a_idx and self._box_area(boxes[j]) < area_a and self._larger_involves_smaller( boxes, a_idx, j, involve_thresh ) ] if len(small_indices) < min_small_count: continue smalls_are_separate = True for i_pos in range(len(small_indices)): for j_pos in range(i_pos + 1, len(small_indices)): if self._boxes_involve_each_other( boxes, small_indices[i_pos], small_indices[j_pos], involve_thresh, ): smalls_are_separate = False break if not smalls_are_separate: break if smalls_are_separate: keep[a_idx] = False if view_ids is None: return boxes[keep], scores[keep], cls_ids[keep] return boxes[keep], scores[keep], cls_ids[keep], view_ids[keep] @staticmethod def _box_centroid(box: np.ndarray) -> tuple[float, float]: return ((box[0] + box[2]) / 2.0, (box[1] + box[3]) / 2.0) @staticmethod def _centroid_inside_box( boxes: np.ndarray, parent_idx: int, child_idx: int ) -> bool: parent = boxes[parent_idx] cx, cy = Miner._box_centroid(boxes[child_idx]) return parent[0] <= cx <= parent[2] and parent[1] <= cy <= parent[3] @staticmethod def _mostly_inside_parent( boxes: np.ndarray, parent_idx: int, child_idx: int, involve_thresh: float, ) -> bool: inter = Miner._box_intersection_area(boxes[parent_idx], boxes[child_idx]) if inter <= 0.0: return False area_child = Miner._box_area(boxes[child_idx]) if (inter / area_child) >= involve_thresh: return True child = boxes[child_idx] parent = boxes[parent_idx] cx = (child[0] + child[2]) / 2.0 cy = (child[1] + child[3]) / 2.0 return ( parent[0] <= cx <= parent[2] and parent[1] <= cy <= parent[3] ) def _cluster_indices_by_box_iou( self, boxes: np.ndarray, indices: list[int], iou_thresh: float, ) -> list[list[int]]: if not indices: return [] if len(indices) == 1: return [indices] parent_map = {idx: idx for idx in indices} def find(x: int) -> int: while parent_map[x] != x: parent_map[x] = parent_map[parent_map[x]] x = parent_map[x] return x def union(a: int, b: int) -> None: ra, rb = find(a), find(b) if ra != rb: parent_map[rb] = ra iou = self._compute_iou_matrix(boxes) for i_pos in range(len(indices)): for j_pos in range(i_pos + 1, len(indices)): a_idx = indices[i_pos] b_idx = indices[j_pos] if iou[a_idx, b_idx] > iou_thresh: union(a_idx, b_idx) groups: dict[int, list[int]] = {} for idx in indices: groups.setdefault(find(idx), []).append(idx) return list(groups.values()) def _cluster_indices_by_centroid_distance( self, boxes: np.ndarray, indices: list[int], max_dist: float, ) -> list[list[int]]: if not indices: return [] if len(indices) == 1: return [indices] parent_map = {idx: idx for idx in indices} def find(x: int) -> int: while parent_map[x] != x: parent_map[x] = parent_map[parent_map[x]] x = parent_map[x] return x def union(a: int, b: int) -> None: ra, rb = find(a), find(b) if ra != rb: parent_map[rb] = ra max_dist_sq = max_dist * max_dist centroids = [self._box_centroid(boxes[idx]) for idx in indices] for i_pos in range(len(indices)): cx_i, cy_i = centroids[i_pos] for j_pos in range(i_pos + 1, len(indices)): cx_j, cy_j = centroids[j_pos] dx = cx_i - cx_j dy = cy_i - cy_j if dx * dx + dy * dy <= max_dist_sq: union(indices[i_pos], indices[j_pos]) groups: dict[int, list[int]] = {} for idx in indices: groups.setdefault(find(idx), []).append(idx) return list(groups.values()) def _split_fire_parent_with_inner_clusters( self, boxes: np.ndarray, scores: np.ndarray, cls_ids: np.ndarray, view_ids: np.ndarray | None = None, inner_involve_thresh: float | None = None, cluster_iou_thresh: float | None = None, min_clusters: int | None = None, min_inner_conf: float | None = None, max_inner_area_ratio: float | None = None, peak_dist_ratio: float | None = None, peak_dist_min_px: float | None = None, peak_dist_max_px: float | None = None, peak_member_min_conf: float | None = None, ) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray | None, np.ndarray]: if inner_involve_thresh is None: inner_involve_thresh = self.fire_split_inner_involve_thresh if cluster_iou_thresh is None: cluster_iou_thresh = self.fire_split_cluster_iou_thresh if min_clusters is None: min_clusters = self.fire_split_min_clusters if min_inner_conf is None: min_inner_conf = self.fire_split_min_inner_conf if max_inner_area_ratio is None: max_inner_area_ratio = self.fire_split_max_inner_area_ratio if peak_dist_ratio is None: peak_dist_ratio = self.fire_split_peak_dist_ratio if peak_dist_min_px is None: peak_dist_min_px = self.fire_split_peak_dist_min_px if peak_dist_max_px is None: peak_dist_max_px = self.fire_split_peak_dist_max_px if peak_member_min_conf is None: peak_member_min_conf = self.fire_split_peak_member_min_conf n = len(boxes) if n <= 1: protect = np.zeros(n, dtype=bool) return boxes, scores, cls_ids, view_ids, protect remove_indices: set[int] = set() split_boxes: list[np.ndarray] = [] split_scores: list[float] = [] split_cls: list[int] = [] split_views: list[int] = [] for target_cls in self._parent_split_cls_ids(): class_indices = [i for i in range(n) if int(cls_ids[i]) == target_cls] if len(class_indices) <= 1: continue class_indices_sorted = sorted( class_indices, key=lambda idx: self._box_area(boxes[idx]), reverse=True, ) for parent_idx in class_indices_sorted: if parent_idx in remove_indices: continue parent_area = self._box_area(boxes[parent_idx]) parent_scale = math.sqrt(parent_area) peak_dist = max( peak_dist_min_px, min(parent_scale * peak_dist_ratio, peak_dist_max_px), ) peak_indices = [ j for j in class_indices if j != parent_idx and j not in remove_indices and self._centroid_inside_box(boxes, parent_idx, j) ] max_inner_area = parent_area * max_inner_area_ratio small_inner_indices = [ j for j in peak_indices if self._box_area(boxes[j]) <= max_inner_area and self._mostly_inside_parent( boxes, parent_idx, j, inner_involve_thresh ) ] strong_clusters: list[list[int]] = [] member_indices: list[int] = [] use_small_split_outputs = False peak_clusters: list[list[int]] = [] peak_strong: list[list[int]] = [] peak_members = [parent_idx] + [ j for j in peak_indices if float(scores[j]) >= peak_member_min_conf ] if len(peak_members) >= min_clusters: peak_clusters = self._cluster_indices_by_centroid_distance( boxes, peak_members, peak_dist ) peak_strong = [ cluster for cluster in peak_clusters if max(float(scores[m]) for m in cluster) >= min_inner_conf ] if len(peak_strong) >= min_clusters: strong_clusters = peak_strong member_indices = peak_indices if len(strong_clusters) < min_clusters and small_inner_indices: iou_clusters = self._cluster_indices_by_box_iou( boxes, small_inner_indices, cluster_iou_thresh ) iou_strong = [ cluster for cluster in iou_clusters if max(float(scores[m]) for m in cluster) >= min_inner_conf ] if len(iou_strong) >= min_clusters: strong_clusters = iou_strong member_indices = small_inner_indices parent_blob_conf = 0.0 for cluster in peak_clusters: if parent_idx not in cluster: continue non_parent = [m for m in cluster if m != parent_idx] if non_parent: parent_blob_conf = max( parent_blob_conf, max(float(scores[m]) for m in non_parent), ) if ( len(strong_clusters) < min_clusters and parent_blob_conf >= self.fire_split_parent_blob_conf_thresh and len(small_inner_indices) >= min_clusters ): small_peak_clusters = self._cluster_indices_by_centroid_distance( boxes, small_inner_indices, peak_dist ) small_strong = [ cluster for cluster in small_peak_clusters if min( 1.0, max(float(scores[m]) for m in cluster) + self.fire_split_conf_boost, ) >= min_inner_conf ] if len(small_strong) >= min_clusters: cluster_best = [ cluster[int(np.argmax(scores[cluster]))] for cluster in small_strong ] smalls_are_separate = True for i_pos in range(len(cluster_best)): for j_pos in range(i_pos + 1, len(cluster_best)): if self._boxes_involve_each_other( boxes, cluster_best[i_pos], cluster_best[j_pos], self.fire_parent_involve_thresh, ): smalls_are_separate = False break if not smalls_are_separate: break if smalls_are_separate: strong_clusters = small_strong member_indices = peak_indices use_small_split_outputs = True if len(strong_clusters) < min_clusters: continue remove_indices.add(parent_idx) remove_indices.update(member_indices) for j in class_indices: if j in remove_indices: continue if self._boxes_involve_each_other( boxes, parent_idx, j, self.fire_parent_involve_thresh ): remove_indices.add(j) for cluster in strong_clusters: if use_small_split_outputs: best_member = cluster[int(np.argmax(scores[cluster]))] else: cluster_candidates = [ m for m in cluster if m != parent_idx ] or cluster best_member = cluster_candidates[ int(np.argmax(scores[cluster_candidates])) ] boosted_conf = float(scores[best_member]) member_area = self._box_area(boxes[best_member]) if ( use_small_split_outputs or member_area <= parent_area * max_inner_area_ratio ): boosted_conf = min( 1.0, boosted_conf + self.fire_split_conf_boost ) split_boxes.append(boxes[best_member].copy()) split_scores.append(boosted_conf) split_cls.append(target_cls) if view_ids is not None: split_views.append(int(view_ids[best_member])) if not remove_indices: protect = np.zeros(n, dtype=bool) return boxes, scores, cls_ids, view_ids, protect keep = np.array([i not in remove_indices for i in range(n)], dtype=bool) boxes = boxes[keep] scores = scores[keep] cls_ids = cls_ids[keep] if view_ids is not None: view_ids = view_ids[keep] protect = np.zeros(len(boxes), dtype=bool) if split_boxes: split_count = len(split_boxes) boxes = np.concatenate([boxes, np.stack(split_boxes, axis=0)], axis=0) scores = np.concatenate( [scores, np.asarray(split_scores, dtype=np.float32)], axis=0 ) cls_ids = np.concatenate( [cls_ids, np.asarray(split_cls, dtype=np.int32)], axis=0 ) if view_ids is not None: view_ids = np.concatenate( [view_ids, np.asarray(split_views, dtype=np.int32)], axis=0 ) else: view_ids = np.full(split_count, -1, dtype=np.int32) protect = np.concatenate( [protect, np.ones(split_count, dtype=bool)], axis=0 ) return boxes, scores, cls_ids, view_ids, protect def _remove_small_group_low_conf( self, boxes: np.ndarray, scores: np.ndarray, cls_ids: np.ndarray, min_count: int | None = None, conf_thresh: float | None = None, ) -> tuple[np.ndarray, np.ndarray, np.ndarray]: if min_count is None: min_count = self.small_group_min_count if conf_thresh is None: conf_thresh = self.small_group_low_conf_thresh n = len(boxes) if n == 0 or len(self.group_object_counts) != n: return boxes, scores, cls_ids keep = np.ones(n, dtype=bool) for i in range(n): if self.group_object_counts[i] < min_count and scores[i] < conf_thresh: keep[i] = False self.group_object_counts = [ count for count, k in zip(self.group_object_counts, keep) if k ] self.group_max_confidences = [ value for value, k in zip(self.group_max_confidences, keep) if k ] self.group_avg_confidences = [ value for value, k in zip(self.group_avg_confidences, keep) if k ] return boxes[keep], scores[keep], cls_ids[keep] def _apply_post_filters( self, boxes: np.ndarray, scores: np.ndarray, cls_ids: np.ndarray, orig_size: tuple[int, int], view_ids: np.ndarray | None = None, ) -> list[BoundingBox]: if len(boxes) == 0: return [] if view_ids is not None: boxes, scores, cls_ids, view_ids = self._remove_edge_low_conf( boxes, scores, cls_ids, orig_size, view_ids ) else: boxes, scores, cls_ids = self._remove_edge_low_conf( boxes, scores, cls_ids, orig_size ) if len(boxes) == 0: return [] split_protect: np.ndarray | None = None if view_ids is not None: boxes, scores, cls_ids, view_ids = ( self._remove_isolated_low_conf_same_class( boxes, scores, cls_ids, view_ids ) ) boxes, scores, cls_ids, view_ids = ( self._remove_overlapping_low_conf_same_class( boxes, scores, cls_ids, view_ids ) ) boxes, scores, cls_ids, view_ids, split_protect = ( self._split_fire_parent_with_inner_clusters( boxes, scores, cls_ids, view_ids ) ) if len(boxes) == 0: return [] boxes, scores, cls_ids, view_ids = ( self._remove_unconfirmed_single_view_low_conf( boxes, scores, cls_ids, view_ids, protect_mask=split_protect ) ) if len(boxes) == 0: return [] boxes, scores, cls_ids, view_ids = self._remove_spanning_parent_fire( boxes, scores, cls_ids, view_ids ) if len(boxes) == 0: return [] boxes, scores, cls_ids = self._group_overlapping_detections( boxes, scores, cls_ids, view_ids=view_ids, record_stats=False ) else: boxes, scores, cls_ids = self._remove_isolated_low_conf_same_class( boxes, scores, cls_ids ) boxes, scores, cls_ids = self._remove_overlapping_low_conf_same_class( boxes, scores, cls_ids ) boxes, scores, cls_ids, _, split_protect = ( self._split_fire_parent_with_inner_clusters( boxes, scores, cls_ids ) ) if len(boxes) == 0: return [] boxes, scores, cls_ids = self._remove_spanning_parent_fire( boxes, scores, cls_ids ) if len(boxes) == 0: return [] boxes, scores, cls_ids = self._group_overlapping_detections( boxes, scores, cls_ids, record_stats=True ) boxes, scores, cls_ids = self._remove_small_group_low_conf( boxes, scores, cls_ids ) boxes, scores, cls_ids = self._resolve_fire_and_ext_same_class_overlap( boxes, scores, cls_ids ) cls_smoke = self.class_names.index("smoke") if int(np.sum(cls_ids == cls_smoke)) >= self.smoke_cluster_merge_early_min_count: boxes, scores, cls_ids = self._merge_overlapping_smoke_clusters( boxes, scores, cls_ids ) boxes, scores, cls_ids = self._resolve_smoke_overlap( boxes, scores, cls_ids ) boxes, scores, cls_ids = self._merge_overlapping_smoke_clusters( boxes, scores, cls_ids ) return self._build_results(boxes, scores, cls_ids) @staticmethod def _bbox_to_array(box: BoundingBox) -> np.ndarray: return np.array([box.x1, box.y1, box.x2, box.y2], dtype=np.float32) def _smoke_base_band_box(self, smoke: BoundingBox) -> np.ndarray: smoke_h = smoke.y2 - smoke.y1 base_y1 = smoke.y1 + smoke_h * (1.0 - self.smoke_anchor_base_height_frac) return np.array( [float(smoke.x1), base_y1, float(smoke.x2), float(smoke.y2)], dtype=np.float32, ) def _fire_overlaps_smoke_base( self, fire_box: np.ndarray, smoke: BoundingBox ) -> bool: return self._boxes_involve_each_other_arrays( fire_box, self._smoke_base_band_box(smoke), self.smoke_anchor_corroborate_involve_thresh, ) def _fire_sits_under_smoke( self, fire_box: np.ndarray, smoke: BoundingBox ) -> bool: cx = (fire_box[0] + fire_box[2]) / 2.0 cy = (fire_box[1] + fire_box[3]) / 2.0 smoke_w = float(smoke.x2 - smoke.x1) smoke_cx = (smoke.x1 + smoke.x2) / 2.0 if abs(cx - smoke_cx) > smoke_w * 0.55: return False if cy < smoke.y1 + (smoke.y2 - smoke.y1) * 0.45: return False return True def _fire_centroid_in_smoke_base_band( self, fire_box: np.ndarray, smoke: BoundingBox ) -> bool: base = self._smoke_base_band_box(smoke) cx = (fire_box[0] + fire_box[2]) / 2.0 cy = (fire_box[1] + fire_box[3]) / 2.0 return ( base[0] <= cx <= base[2] and base[1] <= cy <= base[3] ) def _corroborating_merged_tta_fire_boxes( self, fire_box: np.ndarray, merged_boxes: np.ndarray, merged_cls: np.ndarray, smoke: BoundingBox, ) -> list[np.ndarray]: cls_fire = self.class_names.index("fire") involve_thresh = self.smoke_anchor_corroborate_involve_thresh probe_area = self._box_area(fire_box) corroborators: list[np.ndarray] = [] for i in range(len(merged_boxes)): if int(merged_cls[i]) != cls_fire: continue merged_box = merged_boxes[i] if not self._boxes_involve_each_other_arrays( fire_box, merged_box, involve_thresh ): continue if not self._fire_centroid_in_smoke_base_band(merged_box, smoke): continue merged_area = self._box_area(merged_box) if merged_area <= 0.0 or probe_area <= 0.0: continue area_ratio = probe_area / merged_area if area_ratio > self.smoke_anchor_corroborate_area_ratio_max: continue if area_ratio < 1.0 / self.smoke_anchor_corroborate_area_ratio_max: continue corroborators.append(merged_box.copy()) return corroborators def _snap_smoke_anchor_probe_to_corroborator( self, fire_box: np.ndarray, merged_boxes: np.ndarray, merged_cls: np.ndarray, smoke: BoundingBox, crop_size: tuple[int, int], ) -> np.ndarray | None: corroborators = self._corroborating_merged_tta_fire_boxes( fire_box, merged_boxes, merged_cls, smoke ) if not corroborators: return None corroborators.sort(key=self._box_area) for corr in corroborators: if self._passes_smoke_anchor_probe_fire_max_size( corr, smoke, crop_size ): return corr.copy() return None def _corroborated_by_merged_tta_fire( self, fire_box: np.ndarray, merged_boxes: np.ndarray, merged_cls: np.ndarray, smoke: BoundingBox, ) -> bool: return bool( self._corroborating_merged_tta_fire_boxes( fire_box, merged_boxes, merged_cls, smoke ) ) def _refine_smoke_anchor_probe_fire_box( self, probe_box: np.ndarray, image: np.ndarray, smoke: BoundingBox, merged_boxes: np.ndarray, merged_cls: np.ndarray, crop_size: tuple[int, int], ) -> np.ndarray: corroborators = self._corroborating_merged_tta_fire_boxes( probe_box, merged_boxes, merged_cls, smoke ) if not corroborators: return probe_box def _passes_refined(box: np.ndarray) -> bool: return ( self._passes_smoke_anchor_probe_fire_max_size(box, smoke, crop_size) and self._passes_smoke_anchor_probe_fire_bottom_dist(box, smoke) and self._passes_smoke_anchor_probe_fire_color(image, box) ) probe_area = self._box_area(probe_box) if probe_area <= 0.0: return probe_box expanders: list[np.ndarray] = [] for corr in corroborators: if not self._passes_smoke_anchor_probe_fire_max_size( corr, smoke, crop_size ): continue for candidate in ( corr, np.array( [ min(probe_box[0], corr[0]), min(probe_box[1], corr[1]), max(probe_box[2], corr[2]), max(probe_box[3], corr[3]), ], dtype=np.float32, ), ): if not _passes_refined(candidate): continue if ( candidate[0] > probe_box[0] + 1.0 and candidate[1] > probe_box[1] + 1.0 ): continue inter = self._box_intersection_area(probe_box, candidate) if inter / probe_area < 0.95: continue expanders.append(candidate) if expanders: refined = min( expanders, key=lambda box: (box[0], box[2] - box[0], box[1]), ) else: refined = probe_box pad = self.smoke_anchor_corroborated_refine_pad_px if pad > 0.0 and corroborators: padded = refined.copy() padded[0] = max(0.0, padded[0] - pad) padded[1] = max(0.0, padded[1] - pad) if _passes_refined(padded): refined = padded return refined def _smoke_anchor_min_fire_size( self, corroborated: bool, smoke_conf: float, under_base: bool = False, ) -> tuple[float, float]: if smoke_conf >= self.smoke_anchor_strong_smoke_conf and ( corroborated or under_base ): return ( self.smoke_anchor_corroborated_min_fire_width_px, self.smoke_anchor_corroborated_min_fire_height_px, ) return ( self.smoke_anchor_min_fire_width_px, self.smoke_anchor_min_fire_height_px, ) def _passes_smoke_anchor_fire_size( self, fire_box: np.ndarray, corroborated: bool, smoke_conf: float, under_base: bool = False, ) -> bool: box_w = float(fire_box[2] - fire_box[0]) box_h = float(fire_box[3] - fire_box[1]) min_w, min_h = self._smoke_anchor_min_fire_size( corroborated, smoke_conf, under_base ) return box_w >= min_w and box_h >= min_h def _passes_smoke_anchor_probe_fire_max_size( self, fire_box: np.ndarray, smoke: BoundingBox, crop_size: tuple[int, int], ) -> bool: box_w = float(fire_box[2] - fire_box[0]) box_h = float(fire_box[3] - fire_box[1]) if box_w > self.smoke_anchor_max_probe_fire_width_px: return False if box_h > self.smoke_anchor_max_probe_fire_height_px: return False smoke_area = max( 1.0, float((smoke.x2 - smoke.x1) * (smoke.y2 - smoke.y1)) ) if self._box_area(fire_box) / smoke_area > self.smoke_anchor_max_probe_fire_area_ratio: return False crop_w, crop_h = crop_size if crop_w > 0 and box_w / float(crop_w) > self.smoke_anchor_max_probe_crop_width_ratio: return False if crop_h > 0 and box_h / float(crop_h) > self.smoke_anchor_max_probe_crop_height_ratio: return False return True def _passes_smoke_anchor_probe_fire_bottom_dist( self, fire_box: np.ndarray, smoke: BoundingBox ) -> bool: smoke_h = max(1.0, float(smoke.y2 - smoke.y1)) bottom_gap = float(smoke.y2) - float(fire_box[3]) return bottom_gap / smoke_h <= self.smoke_anchor_max_probe_fire_bottom_dist_frac def _passes_smoke_anchor_probe_fire_color( self, image: np.ndarray, fire_box: np.ndarray ) -> bool: h, w = image.shape[:2] x1 = max(0, int(math.floor(fire_box[0]))) y1 = max(0, int(math.floor(fire_box[1]))) x2 = min(w, int(math.ceil(fire_box[2]))) y2 = min(h, int(math.ceil(fire_box[3]))) if x2 <= x1 or y2 <= y1: return False roi = image[y1:y2, x1:x2] if roi.size == 0: return False blue = roi[:, :, 0].astype(np.float32) green = roi[:, :, 1].astype(np.float32) red = roi[:, :, 2].astype(np.float32) mean_r = float(np.mean(red)) max_rgb = float( max(np.max(red), np.max(green), np.max(blue)) ) bright_frac = float( np.mean(np.max(roi, axis=2) >= self.smoke_anchor_probe_bright_thresh) ) if ( max_rgb >= self.smoke_anchor_probe_sparse_hot_min_max_rgb and bright_frac >= self.smoke_anchor_probe_sparse_hot_min_bright_frac ): return True if mean_r < self.smoke_anchor_probe_min_mean_r: return False if max_rgb < self.smoke_anchor_probe_min_max_rgb: return False warm_mask = (red > green + 10.0) & (red > blue + 10.0) warm_frac = float(np.mean(warm_mask)) r_minus_g = mean_r - float(np.mean(green)) if warm_frac >= self.smoke_anchor_probe_min_warm_frac: if ( max_rgb >= self.smoke_anchor_probe_warm_min_max_rgb or mean_r >= self.smoke_anchor_probe_warm_min_mean_r or warm_frac >= self.smoke_anchor_probe_strong_warm_frac ): return True if ( bright_frac >= self.smoke_anchor_probe_min_bright_frac and r_minus_g >= self.smoke_anchor_probe_min_r_minus_g ): return True if ( max_rgb >= self.smoke_anchor_probe_min_hot_max_rgb and r_minus_g >= self.smoke_anchor_probe_min_r_minus_g ): return True return False def _fire_ext_red_distribution( self, image: np.ndarray, ext_box: np.ndarray ) -> tuple[float, float, float]: h, w = image.shape[:2] x1 = max(0, int(math.floor(float(ext_box[0])))) y1 = max(0, int(math.floor(float(ext_box[1])))) x2 = min(w, int(math.ceil(float(ext_box[2])))) y2 = min(h, int(math.ceil(float(ext_box[3])))) if x2 <= x1 or y2 <= y1: return 0.0, 0.0, 0.0 roi = image[y1:y2, x1:x2] if roi.size == 0: return 0.0, 0.0, 0.0 blue = roi[:, :, 0].astype(np.float32) green = roi[:, :, 1].astype(np.float32) red = roi[:, :, 2].astype(np.float32) red_dom = (red > green + 10.0) & (red > blue + 10.0) return ( float(np.mean(red)), float(np.mean(red_dom)), float(np.mean(red - green)), ) def _passes_fire_ext_red_color( self, image: np.ndarray, ext_box: np.ndarray ) -> bool: mean_r, red_dom, r_minus_g = self._fire_ext_red_distribution( image, ext_box ) if red_dom >= self.fire_ext_min_red_dom_frac: return True if ( r_minus_g >= self.fire_ext_min_r_minus_g and mean_r >= self.fire_ext_color_min_mean_r ): return True return False def _filter_fire_ext_by_red_color( self, image: np.ndarray, results: list[BoundingBox] ) -> list[BoundingBox]: cls_fire_ext = self.class_names.index("fire extinguisher") max_conf = self.fire_ext_color_filter_max_conf filtered: list[BoundingBox] = [] for box in results: if ( box.cls_id == cls_fire_ext and box.conf <= max_conf + 1e-4 and not self._passes_fire_ext_red_color( image, self._bbox_to_array(box) ) ): continue filtered.append(box) return filtered def _filter_probe_fires_by_color( self, image: np.ndarray, results: list[BoundingBox] ) -> list[BoundingBox]: cls_fire = self.class_names.index("fire") floor_conf = self.smoke_anchor_fire_floor_conf filtered: list[BoundingBox] = [] for box in results: if ( box.cls_id == cls_fire and box.conf <= floor_conf + 1e-4 and not self._passes_smoke_anchor_probe_fire_color( image, self._bbox_to_array(box) ) ): continue filtered.append(box) return filtered def _smoke_anchor_fire_is_duplicate( self, fire_box: np.ndarray, others: list[BoundingBox], cls_fire: int, involve_thresh: float | None = None, ) -> bool: if involve_thresh is None: involve_thresh = self.smoke_anchor_corroborate_involve_thresh for fire in others: if fire.cls_id != cls_fire: continue if self._boxes_involve_each_other_arrays( self._bbox_to_array(fire), fire_box, involve_thresh ): return True return False def _collect_smoke_anchor_probe_fires( self, image: np.ndarray, boxes: np.ndarray, scores: np.ndarray, cls_ids: np.ndarray, smoke: BoundingBox, merged_boxes: np.ndarray, merged_cls: np.ndarray, existing_fires: list[BoundingBox], cls_fire: int, crop_size: tuple[int, int], ) -> list[tuple[np.ndarray, float]]: smoke_area = max( 1.0, float((smoke.x2 - smoke.x1) * (smoke.y2 - smoke.y1)) ) candidates: list[tuple[float, np.ndarray]] = [] for i in range(len(boxes)): if int(cls_ids[i]) != cls_fire: continue fire_box = boxes[i] corroborated = self._corroborated_by_merged_tta_fire( fire_box, merged_boxes, merged_cls, smoke ) under_smoke = self._fire_sits_under_smoke(fire_box, smoke) in_base = self._fire_centroid_in_smoke_base_band(fire_box, smoke) under_base = under_smoke and in_base if not corroborated and not under_base: continue if not self._passes_smoke_anchor_fire_size( fire_box, corroborated, smoke.conf, under_base ): continue if not self._passes_smoke_anchor_probe_fire_max_size( fire_box, smoke, crop_size ): if corroborated: fire_box = self._snap_smoke_anchor_probe_to_corroborator( fire_box, merged_boxes, merged_cls, smoke, crop_size ) if fire_box is None: continue else: continue if not self._passes_smoke_anchor_probe_fire_bottom_dist(fire_box, smoke): continue if not self._passes_smoke_anchor_probe_fire_color(image, fire_box): continue if self._box_area(fire_box) > smoke_area * self.smoke_anchor_max_fire_area_ratio: continue candidates.append((float(scores[i]), fire_box.copy())) candidates.sort(key=lambda item: item[0], reverse=True) selected: list[tuple[np.ndarray, float]] = [] separate_thresh = self.smoke_anchor_separate_fire_involve_thresh max_fires = self.smoke_anchor_max_fires_per_smoke floor_conf = self.smoke_anchor_fire_floor_conf for score, fire_box in candidates: corroborated = self._corroborated_by_merged_tta_fire( fire_box, merged_boxes, merged_cls, smoke ) under_smoke = self._fire_sits_under_smoke(fire_box, smoke) in_base = self._fire_centroid_in_smoke_base_band(fire_box, smoke) min_conf = floor_conf if corroborated and under_smoke and in_base: min_conf = self.smoke_anchor_corroborated_probe_min_conf elif not corroborated and under_smoke and in_base: min_conf = self.smoke_anchor_probe_rescue_min_conf if score < min_conf: continue if len(selected) >= max_fires: break if self._smoke_anchor_fire_is_duplicate( fire_box, existing_fires, cls_fire ): continue overlaps_selected = False for chosen_box, _ in selected: if self._boxes_involve_each_other_arrays( fire_box, chosen_box, separate_thresh ): overlaps_selected = True break if overlaps_selected: continue selected.append((fire_box, score)) return selected def _smoke_anchor_crop_region( self, smoke: BoundingBox, orig_size: tuple[int, int] ) -> tuple[int, int, int, int]: orig_w, orig_h = orig_size smoke_w = float(smoke.x2 - smoke.x1) smoke_h = float(smoke.y2 - smoke.y1) pad_x = smoke_w * self.smoke_anchor_crop_width_pad_ratio y1 = smoke.y1 + smoke_h * (1.0 - self.smoke_anchor_base_height_frac) y2 = min(orig_h, float(smoke.y2) + self.smoke_anchor_crop_down_pad_px) x1 = max(0, int(math.floor(smoke.x1 - pad_x))) x2 = min(orig_w, int(math.ceil(smoke.x2 + pad_x))) y1i = max(0, int(math.floor(y1))) y2i = min(orig_h, max(y1i + 1, int(math.ceil(y2)))) x2 = max(x1 + 1, x2) return x1, x2, y1i, y2i def _smoke_anchor_probe_target_smoke( self, smoke: BoundingBox, smoke_seeds: list[BoundingBox] | None, ) -> BoundingBox: if not smoke_seeds: return smoke cls_smoke = self.class_names.index("smoke") smoke_arr = self._bbox_to_array(smoke) smoke_w = float(smoke.x2 - smoke.x1) best_seed: BoundingBox | None = None for seed in smoke_seeds: if seed.cls_id != cls_smoke: continue if seed.conf < self.smoke_anchor_min_smoke_conf: continue seed_arr = self._bbox_to_array(seed) if not self._boxes_involve_each_other_arrays( smoke_arr, seed_arr, self.smoke_expand_corroborate_involve_thresh ): continue seed_w = float(seed.x2 - seed.x1) if seed_w >= smoke_w * 0.95: continue if best_seed is None or seed.conf > best_seed.conf: best_seed = seed if best_seed is None: return smoke seed_w = float(best_seed.x2 - best_seed.x1) if smoke_w > seed_w * self.smoke_anchor_expanded_width_ratio: return best_seed return smoke def _probe_smoke_anchored_fire( self, image: np.ndarray, results: list[BoundingBox], orig_size: tuple[int, int], merged_boxes: np.ndarray, merged_cls: np.ndarray, smoke_seeds: list[BoundingBox] | None = None, ) -> list[BoundingBox]: if not self.use_tta: return results cls_fire = self.class_names.index("fire") cls_smoke = self.class_names.index("smoke") floor_conf = self.smoke_anchor_fire_floor_conf added: list[BoundingBox] = [] smoke_candidates = sorted( ( box for box in results if box.cls_id == cls_smoke and box.conf >= self.smoke_anchor_min_smoke_conf ), key=lambda box: box.conf, reverse=True, )[: self.smoke_anchor_max_probe_smokes] if not smoke_candidates: return results for smoke in smoke_candidates: smoke_arr = self._bbox_to_array(smoke) fire_on_smoke_base = False for fire in results: if fire.cls_id != cls_fire or fire.conf < self.fire_min_conf: continue if self._fire_overlaps_smoke_base( self._bbox_to_array(fire), smoke ): fire_on_smoke_base = True break if fire_on_smoke_base: continue anchor_smoke = smoke x1, x2, y1, y2 = self._smoke_anchor_crop_region(anchor_smoke, orig_size) crop = image[y1:y2, x1:x2] if crop.size == 0: continue crop_size = (crop.shape[1], crop.shape[0]) boxes, scores, cls_ids = self._infer_view_arrays( crop, crop_size, flip_code=None ) if len(boxes) == 0: continue boxes = boxes.copy() boxes[:, [0, 2]] += x1 boxes[:, [1, 3]] += y1 boxes = self._clip_boxes(boxes, orig_size) probe_fires = self._collect_smoke_anchor_probe_fires( image, boxes, scores, cls_ids, anchor_smoke, merged_boxes, merged_cls, results + added, cls_fire, crop_size, ) if not probe_fires: seed_smoke = self._smoke_anchor_probe_target_smoke(smoke, smoke_seeds) if seed_smoke is not smoke: x1, x2, y1, y2 = self._smoke_anchor_crop_region( seed_smoke, orig_size ) crop = image[y1:y2, x1:x2] if crop.size == 0: continue crop_size = (crop.shape[1], crop.shape[0]) boxes, scores, cls_ids = self._infer_view_arrays( crop, crop_size, flip_code=None ) if len(boxes) == 0: continue boxes = boxes.copy() boxes[:, [0, 2]] += x1 boxes[:, [1, 3]] += y1 boxes = self._clip_boxes(boxes, orig_size) probe_fires = self._collect_smoke_anchor_probe_fires( image, boxes, scores, cls_ids, seed_smoke, merged_boxes, merged_cls, results + added, cls_fire, crop_size, ) anchor_smoke = seed_smoke if not probe_fires: continue for probe_box, probe_raw_score in probe_fires: probe_box = self._refine_smoke_anchor_probe_fire_box( probe_box, image, anchor_smoke, merged_boxes, merged_cls, crop_size, ) probe_conf = min(1.0, max(probe_raw_score, floor_conf)) added.append( BoundingBox( x1=int(math.floor(probe_box[0])), y1=int(math.floor(probe_box[1])), x2=int(math.ceil(probe_box[2])), y2=int(math.ceil(probe_box[3])), cls_id=cls_fire, conf=probe_conf, ) ) if len(added) > self.smoke_anchor_max_crop_fires: added.sort(key=lambda box: box.conf, reverse=True) added = added[: self.smoke_anchor_max_crop_fires] return results + added def _ext_probe_union_box( self, boxes: np.ndarray, indices: list[int] ) -> np.ndarray: cluster_boxes = boxes[indices] return np.array( [ float(np.min(cluster_boxes[:, 0])), float(np.min(cluster_boxes[:, 1])), float(np.max(cluster_boxes[:, 2])), float(np.max(cluster_boxes[:, 3])), ], dtype=np.float32, ) def _ext_probe_seed_on_border( self, seed_box: np.ndarray, orig_size: tuple[int, int] ) -> bool: orig_w, orig_h = orig_size tol = self.ext_probe_edge_tol return bool( seed_box[0] <= tol or seed_box[1] <= tol or seed_box[2] >= orig_w - 1 - tol or seed_box[3] >= orig_h - 1 - tol ) def _ext_probe_singleton_too_close_to_border( self, seed_box: np.ndarray, orig_size: tuple[int, int] ) -> bool: orig_w, orig_h = orig_size margin = self.ext_probe_singleton_min_edge_margin_px return bool( seed_box[0] < margin or seed_box[1] < margin or seed_box[2] > orig_w - 1 - margin or seed_box[3] > orig_h - 1 - margin ) def _ext_probe_overlaps_strong_fire( self, seed_box: np.ndarray, results: list[BoundingBox] ) -> bool: cls_fire = self.class_names.index("fire") for box in results: if box.cls_id != cls_fire or box.conf < self.fire_min_conf: continue if self._boxes_involve_each_other_arrays( seed_box, self._bbox_to_array(box), self.ext_probe_fire_exclude_involve_thresh, ): return True return False def _ext_probe_near_any_fire( self, seed_box: np.ndarray, results: list[BoundingBox] ) -> bool: cls_fire = self.class_names.index("fire") max_gap = self.ext_probe_near_fire_gap_px for box in results: if box.cls_id != cls_fire: continue fire_arr = self._bbox_to_array(box) if self._box_axis_gap(seed_box, fire_arr, 0) > max_gap: continue if self._box_axis_gap(seed_box, fire_arr, 1) <= max_gap: return True return False def _ext_probe_is_duplicate( self, ext_box: np.ndarray, results: list[BoundingBox], cls_fire_ext: int, min_conf: float = 0.0, ) -> bool: for box in results: if box.cls_id != cls_fire_ext or box.conf < min_conf: continue if self._boxes_involve_each_other_arrays( ext_box, self._bbox_to_array(box), self.ext_probe_duplicate_involve_thresh, ): return True return False def _passes_ext_probe_size( self, ext_box: np.ndarray, crop_score: float = 0.0 ) -> bool: box_w = float(ext_box[2] - ext_box[0]) box_h = float(ext_box[3] - ext_box[1]) min_w = self.ext_probe_min_width_px if crop_score >= self.ext_probe_crop_rescue_min_conf: min_w = min(min_w, self.ext_probe_crop_rescue_min_width_px) return ( box_w >= min_w and box_h >= self.ext_probe_min_height_px and box_w <= self.ext_probe_max_width_px and box_h <= self.ext_probe_max_height_px ) def _passes_ext_probe_singleton_size(self, ext_box: np.ndarray) -> bool: box_w = float(ext_box[2] - ext_box[0]) box_h = float(ext_box[3] - ext_box[1]) return ( box_w >= self.ext_probe_min_width_px and box_h >= self.ext_probe_min_height_px and box_w <= self.ext_probe_singleton_max_width_px and box_h <= self.ext_probe_singleton_max_height_px and self._box_area(ext_box) <= self.ext_probe_singleton_max_area ) def _ext_probe_validate_seed( self, seed_box: np.ndarray, best_score: float, min_conf: float, results: list[BoundingBox], orig_size: tuple[int, int], cls_fire_ext: int, ) -> bool: if best_score < min_conf: return False if best_score >= self.fire_ext_min_conf: return False if not self._passes_ext_probe_size(seed_box): return False if self._ext_probe_seed_on_border(seed_box, orig_size): return False if self._ext_probe_overlaps_strong_fire(seed_box, results): return False if self._ext_probe_near_any_fire(seed_box, results): return False if self._ext_probe_is_duplicate( seed_box, results, cls_fire_ext, min_conf=self.fire_ext_min_conf ): return False return True def _ext_probe_crop_region( self, seed_box: np.ndarray, orig_size: tuple[int, int] ) -> tuple[int, int, int, int]: orig_w, orig_h = orig_size seed_w = float(seed_box[2] - seed_box[0]) seed_h = float(seed_box[3] - seed_box[1]) pad_x = seed_w * self.ext_probe_crop_pad_ratio pad_y = seed_h * self.ext_probe_crop_pad_ratio x1 = max(0, int(math.floor(seed_box[0] - pad_x))) x2 = min(orig_w, int(math.ceil(seed_box[2] + pad_x))) y1 = max(0, int(math.floor(seed_box[1] - pad_y))) y2 = min(orig_h, int(math.ceil(seed_box[3] + pad_y))) min_crop = self.ext_probe_min_crop_size_px crop_w = float(x2 - x1) crop_h = float(y2 - y1) if crop_w < min_crop: cx = 0.5 * (float(seed_box[0]) + float(seed_box[2])) half = 0.5 * min_crop x1 = max(0, int(math.floor(cx - half))) x2 = min(orig_w, int(math.ceil(cx + half))) if crop_h < min_crop: cy = 0.5 * (float(seed_box[1]) + float(seed_box[3])) half = 0.5 * min_crop y1 = max(0, int(math.floor(cy - half))) y2 = min(orig_h, int(math.ceil(cy + half))) x2 = max(x1 + 1, x2) y2 = max(y1 + 1, y2) return x1, x2, y1, y2 def _collect_merged_ext_probe_seeds( self, merged_boxes: np.ndarray, merged_scores: np.ndarray, merged_cls: np.ndarray, results: list[BoundingBox], orig_size: tuple[int, int], ) -> list[tuple[np.ndarray, float, list[int]]]: cls_fire_ext = self.class_names.index("fire extinguisher") ext_indices = [ i for i in range(len(merged_boxes)) if int(merged_cls[i]) == cls_fire_ext ] if len(ext_indices) == 0: return [] clusters = self._cluster_indices_by_centroid_distance( merged_boxes, ext_indices, self.ext_probe_cluster_centroid_dist_px, ) seeds: list[tuple[np.ndarray, float, list[int]]] = [] for cluster in clusters: best_idx = cluster[int(np.argmax(merged_scores[cluster]))] best_score = float(merged_scores[best_idx]) if len(cluster) >= self.ext_probe_min_cluster_count: if best_score < self.ext_probe_min_cluster_best_conf: continue seed_box = self._ext_probe_union_box(merged_boxes, cluster) min_conf = self.ext_probe_min_cluster_best_conf elif len(cluster) == 1: seed_box = merged_boxes[best_idx].copy() if not self._passes_ext_probe_singleton_size(seed_box): continue if self._ext_probe_singleton_too_close_to_border(seed_box, orig_size): continue min_conf = self.ext_probe_singleton_min_conf else: continue if not self._ext_probe_validate_seed( seed_box, best_score, min_conf, results, orig_size, cls_fire_ext ): continue seeds.append((seed_box, best_score, cluster)) seeds.sort(key=lambda item: item[1], reverse=True) return seeds def _collect_ext_probe_crop_candidates( self, boxes: np.ndarray, scores: np.ndarray, cls_ids: np.ndarray, seed_box: np.ndarray, cls_fire_ext: int, ) -> list[tuple[np.ndarray, float]]: candidates: list[tuple[float, np.ndarray]] = [] for i in range(len(boxes)): if int(cls_ids[i]) != cls_fire_ext: continue ext_box = boxes[i] if not self._boxes_involve_each_other_arrays( seed_box, ext_box, self.ext_probe_duplicate_involve_thresh ): continue if not self._passes_ext_probe_size(ext_box, float(scores[i])): continue candidates.append((float(scores[i]), ext_box.copy())) candidates.sort(key=lambda item: item[0], reverse=True) return [(box, score) for score, box in candidates] def _refine_ext_probe_box( self, seed_box: np.ndarray, crop_candidates: list[tuple[np.ndarray, float]], ) -> tuple[np.ndarray, float]: if not crop_candidates: return seed_box.copy(), 0.0 crop_box, crop_score = crop_candidates[0] if crop_score < self.ext_probe_rescue_min_conf: return seed_box.copy(), 0.0 refined = np.array( [ min(float(seed_box[0]), float(crop_box[0])), min(float(seed_box[1]), float(crop_box[1])), max(float(seed_box[2]), float(crop_box[2])), max(float(seed_box[3]), float(crop_box[3])), ], dtype=np.float32, ) if not self._passes_ext_probe_size(refined, crop_score): return crop_box.copy(), crop_score return refined, crop_score def _probe_merged_anchored_fire_ext( self, image: np.ndarray, results: list[BoundingBox], orig_size: tuple[int, int], merged_boxes: np.ndarray, merged_scores: np.ndarray, merged_cls: np.ndarray, ) -> list[BoundingBox]: if not self.use_tta: return results cls_fire_ext = self.class_names.index("fire extinguisher") floor_conf = self.fire_ext_min_conf seeds = self._collect_merged_ext_probe_seeds( merged_boxes, merged_scores, merged_cls, results, orig_size ) if not seeds: return results added: list[BoundingBox] = [] for seed_box, seed_score, _cluster in seeds: x1, x2, y1, y2 = self._ext_probe_crop_region(seed_box, orig_size) crop = image[y1:y2, x1:x2] crop_score = 0.0 probe_box = seed_box.copy() if crop.size > 0: crop_size = (crop.shape[1], crop.shape[0]) boxes, scores, cls_ids = self._infer_view_arrays( crop, crop_size, flip_code=None ) if len(boxes) > 0: boxes = boxes.copy() boxes[:, [0, 2]] += x1 boxes[:, [1, 3]] += y1 boxes = self._clip_boxes(boxes, orig_size) crop_candidates = self._collect_ext_probe_crop_candidates( boxes, scores, cls_ids, seed_box, cls_fire_ext ) probe_box, crop_score = self._refine_ext_probe_box( seed_box, crop_candidates ) if not self._passes_ext_probe_size(probe_box, crop_score): continue if self._ext_probe_is_duplicate( probe_box, results + added, cls_fire_ext, min_conf=self.fire_ext_min_conf, ): continue probe_conf = min( 1.0, max(seed_score, crop_score, floor_conf) ) added.append( BoundingBox( x1=int(math.floor(probe_box[0])), y1=int(math.floor(probe_box[1])), x2=int(math.ceil(probe_box[2])), y2=int(math.ceil(probe_box[3])), cls_id=cls_fire_ext, conf=probe_conf, ) ) if len(added) >= self.ext_probe_max_per_frame: break if not added: return results kept: list[BoundingBox] = [] for box in results: if box.cls_id == cls_fire_ext and box.conf < floor_conf: if any( self._boxes_involve_each_other_arrays( self._bbox_to_array(box), self._bbox_to_array(added_box), self.ext_probe_duplicate_involve_thresh, ) for added_box in added ): continue kept.append(box) return kept + added def _smoke_expansion_max_deltas( self, orig_size: tuple[int, int] ) -> tuple[float, float]: orig_w, orig_h = orig_size return ( float(orig_w) * self.smoke_expand_max_frame_ratio, float(orig_h) * self.smoke_expand_max_frame_ratio, ) def _box_axis_gap( self, box_a: np.ndarray, box_b: np.ndarray, axis: int ) -> float: start_a = float(box_a[axis]) end_a = float(box_a[axis + 2]) start_b = float(box_b[axis]) end_b = float(box_b[axis + 2]) if end_a >= start_b and end_b >= start_a: return 0.0 return max(start_b - end_a, start_a - end_b) def _smoke_probe_within_expansion_reach( self, seed_arr: np.ndarray, probe_box: np.ndarray, orig_size: tuple[int, int], ) -> bool: max_dx, max_dy = self._smoke_expansion_max_deltas(orig_size) if self._box_axis_gap(seed_arr, probe_box, 0) > max_dx: return False if self._box_axis_gap(seed_arr, probe_box, 1) > max_dy: return False return True def _clamp_smoke_expansion_union( self, union_box: np.ndarray, seed_arr: np.ndarray, orig_size: tuple[int, int], ) -> np.ndarray: max_dx, max_dy = self._smoke_expansion_max_deltas(orig_size) union_box = union_box.copy() union_box[0] = max(float(union_box[0]), float(seed_arr[0]) - max_dx) union_box[2] = min(float(union_box[2]), float(seed_arr[2]) + max_dx) union_box[1] = max(float(union_box[1]), float(seed_arr[1]) - max_dy) union_box[3] = min(float(union_box[3]), float(seed_arr[3]) + max_dy) return union_box def _smoke_crop_horizontal_reach_limit(self, seed_arr: np.ndarray) -> float: seed_w = max(1.0, float(seed_arr[2]) - float(seed_arr[0])) return max( 32.0, seed_w * self.smoke_expand_band_max_horizontal_gap_ratio, ) def _clip_smoke_crop_candidate_for_union( self, candidate_box: np.ndarray, seed_arr: np.ndarray, ) -> np.ndarray: if self._probe_qualifies_for_upward_low_conf(candidate_box, seed_arr): return candidate_box.copy() max_h_gap = self._smoke_crop_horizontal_reach_limit(seed_arr) clipped = candidate_box.copy() clipped[0] = max(float(clipped[0]), float(seed_arr[0]) - max_h_gap) clipped[2] = min(float(clipped[2]), float(seed_arr[2]) + max_h_gap) if float(clipped[2]) <= float(clipped[0]): clipped[0] = float(seed_arr[0]) clipped[2] = float(seed_arr[2]) return clipped def _smoke_expansion_crop_region( self, smoke: BoundingBox, orig_size: tuple[int, int] ) -> tuple[int, int, int, int]: orig_w, orig_h = orig_size max_dx, max_dy = self._smoke_expansion_max_deltas(orig_size) x1 = max(0, int(math.floor(float(smoke.x1) - max_dx))) x2 = min(orig_w, int(math.ceil(float(smoke.x2) + max_dx))) y1 = max(0, int(math.floor(float(smoke.y1) - max_dy))) y2 = min( orig_h, max(y1 + 1, int(math.ceil(float(smoke.y2) + max_dy))), ) x2 = max(x1 + 1, min(orig_w, x2)) y2 = max(y1 + 1, min(orig_h, y2)) return x1, x2, y1, y2 def _smoke_expansion_uses_wide_crop( self, smoke: BoundingBox, orig_size: tuple[int, int] ) -> bool: orig_w, orig_h = orig_size smoke_w = float(smoke.x2 - smoke.x1) return ( float(smoke.y2) < orig_h * self.smoke_expand_upper_plume_max_y_ratio and smoke_w < orig_w * self.smoke_expand_wide_seed_max_width_ratio ) def _smoke_expansion_crop_regions( self, smoke: BoundingBox, orig_size: tuple[int, int] ) -> list[tuple[int, int, int, int]]: primary = self._smoke_expansion_crop_region(smoke, orig_size) regions = [primary] if not self._smoke_expansion_uses_wide_crop(smoke, orig_size): return regions orig_w, orig_h = orig_size px1, px2, py1, py2 = primary max_dy = self._smoke_expansion_max_deltas(orig_size)[1] smoke_h = max(1.0, float(smoke.y2 - smoke.y1)) slice_y1 = max( py1, int(math.floor(float(smoke.y1) - min(smoke_h * 0.15, max_dy))), ) slice_y2 = min( py2, max(slice_y1 + 1, int(math.ceil(float(smoke.y2) + max_dy))), ) left_x2 = max(px1 + 1, min(px2, int(math.ceil(float(smoke.x2))))) right_x1 = min(px2 - 1, max(px1, int(math.floor(float(smoke.x1))))) min_slice_w = max(40, int(orig_w * 0.08)) if left_x2 - px1 >= min_slice_w: regions.append((px1, left_x2, slice_y1, slice_y2)) if px2 - right_x1 >= min_slice_w: regions.append((right_x1, px2, slice_y1, slice_y2)) return regions def _smoke_corroborates_upward_diagonal_plume( self, probe_box: np.ndarray, seed_arr: np.ndarray, crop_x1: int, crop_x2: int, ) -> bool: if not self._probe_extends_smoke_upward(probe_box, seed_arr): return False smoke_h = max(1.0, float(seed_arr[3]) - seed_arr[1]) smoke_w = max(1.0, float(seed_arr[2]) - seed_arr[0]) if float(probe_box[3]) < float(seed_arr[1]) - smoke_h * 0.15: return False cy = (float(probe_box[1]) + float(probe_box[3])) / 2.0 if cy > float(seed_arr[1]) + smoke_h * 0.35: return False gap_x = self._box_axis_gap(probe_box, seed_arr, 0) max_gap = max( 32.0, smoke_w * self.smoke_expand_upward_diagonal_max_gap_ratio, ) if gap_x > max_gap: return False cx = (float(probe_box[0]) + float(probe_box[2])) / 2.0 return float(crop_x1) <= cx <= float(crop_x2) def _smoke_corroborates_expansion_seed( self, probe_box: np.ndarray, seed: BoundingBox, seed_arr: np.ndarray, crop_x1: int, crop_x2: int, ) -> bool: if self._boxes_involve_each_other_arrays( probe_box, seed_arr, self.smoke_expand_corroborate_involve_thresh, ): return True if self._smoke_corroborates_upward_diagonal_plume( probe_box, seed_arr, crop_x1, crop_x2 ): return True smoke_h = max(1.0, float(seed.y2 - seed.y1)) band_pad = smoke_h * self.smoke_expand_vertical_band_pad_ratio cy = (float(probe_box[1]) + float(probe_box[3])) / 2.0 cx = (float(probe_box[0]) + float(probe_box[2])) / 2.0 if cy < float(seed.y1) - band_pad or cy > float(seed.y2) + band_pad: return False gap_x = self._box_axis_gap(probe_box, seed_arr, 0) seed_w = max(1.0, float(seed.x2 - seed.x1)) max_band_gap = max( 32.0, seed_w * self.smoke_expand_band_max_horizontal_gap_ratio, ) if gap_x > max_band_gap: return False return float(crop_x1) <= cx <= float(crop_x2) def _smoke_box_border_strip(self, bw: int, bh: int) -> int: return max( 2, min( int(min(bw, bh) * self.smoke_expand_border_strip_ratio), bw // 3, bh // 3, ), ) def _smoke_expansion_side_border_rgb( self, image: np.ndarray, x1: int, y1: int, x2: int, y2: int, side: str, ) -> np.ndarray | None: bw = x2 - x1 bh = y2 - y1 if bw < 2 or bh < 2: return None strip = self._smoke_box_border_strip(bw, bh) if side == "left": block = image[y1:y2, x1 : x1 + strip] elif side == "right": block = image[y1:y2, x2 - strip : x2] elif side == "top": block = image[y1 : y1 + strip, x1:x2] else: block = image[y2 - strip : y2, x1:x2] if block.size == 0: return None return np.array( [ float(np.mean(block[:, :, 2])), float(np.mean(block[:, :, 1])), float(np.mean(block[:, :, 0])), ], dtype=np.float32, ) def _smoke_expansion_border_reference_rgb( self, image: np.ndarray, seed: BoundingBox ) -> np.ndarray | None: h, w = image.shape[:2] x1 = max(0, int(math.floor(seed.x1))) y1 = max(0, int(math.floor(seed.y1))) x2 = min(w, int(math.ceil(seed.x2))) y2 = min(h, int(math.ceil(seed.y2))) block_means: list[np.ndarray] = [] for side in ("top", "bottom", "left", "right"): rgb = self._smoke_expansion_side_border_rgb(image, x1, y1, x2, y2, side) if rgb is not None: block_means.append(rgb) if not block_means: return None return np.mean(block_means, axis=0) def _smoke_expansion_tta_view_corroborated( self, index: int, boxes: np.ndarray, cls_ids: np.ndarray, view_ids: np.ndarray, cls_smoke: int, ) -> bool: n = len(boxes) if n <= 1 or int(cls_ids[index]) != cls_smoke: return False iou = self._compute_iou_matrix(boxes) other = np.arange(n) != index other_view = view_ids != view_ids[index] same_class = cls_ids == cls_smoke return bool( np.any( other & other_view & same_class & (iou[index] >= self.smoke_expand_tta_view_iou_thresh) ) ) def _probe_extends_smoke_upward( self, probe_box: np.ndarray, seed_arr: np.ndarray, margin: float = 2.0 ) -> bool: return float(probe_box[1]) < float(seed_arr[1]) - margin def _probe_extends_smoke_downward( self, probe_box: np.ndarray, seed_arr: np.ndarray, margin: float = 2.0 ) -> bool: return float(probe_box[3]) > float(seed_arr[3]) + margin def _probe_qualifies_for_upward_low_conf( self, probe_box: np.ndarray, seed_arr: np.ndarray, margin: float = 2.0 ) -> bool: if not self._probe_extends_smoke_upward(probe_box, seed_arr, margin): return False return not self._probe_extends_smoke_downward(probe_box, seed_arr, margin) def _probe_extends_smoke_horizontally( self, probe_box: np.ndarray, seed_arr: np.ndarray, margin: float = 2.0 ) -> bool: return ( float(probe_box[0]) < float(seed_arr[0]) - margin or float(probe_box[2]) > float(seed_arr[2]) + margin ) def _probe_extends_smoke_seed( self, probe_box: np.ndarray, seed_arr: np.ndarray, margin: float = 2.0 ) -> bool: return ( self._probe_extends_smoke_horizontally(probe_box, seed_arr, margin) or self._probe_extends_smoke_upward(probe_box, seed_arr, margin) or float(probe_box[3]) > float(seed_arr[3]) + margin ) def _smoke_expansion_tta_candidate_conf_ok( self, score: float, candidate_box: np.ndarray, seed_arr: np.ndarray ) -> bool: if self._probe_qualifies_for_upward_low_conf(candidate_box, seed_arr): if score >= self.smoke_expand_min_upward_tta_conf: return True if score >= self.smoke_expand_min_upward_tta_involve_conf: return self._boxes_involve_each_other_arrays( candidate_box, seed_arr, self.smoke_expand_corroborate_involve_thresh, ) return False if score < self.smoke_expand_min_tta_smoke_conf: return False if self._probe_extends_smoke_horizontally( candidate_box, seed_arr ) or self._probe_extends_smoke_downward(candidate_box, seed_arr): return score >= self.smoke_expand_min_tta_extend_conf return True def _smoke_expansion_crop_candidate_conf_ok( self, score: float, candidate_box: np.ndarray, seed_arr: np.ndarray ) -> bool: if ( self._probe_extends_smoke_horizontally(candidate_box, seed_arr) and self._probe_extends_smoke_downward(candidate_box, seed_arr) and score < self.smoke_expand_min_bidi_crop_conf ): return False if self._probe_qualifies_for_upward_low_conf(candidate_box, seed_arr): return score >= self.smoke_expand_min_upward_crop_conf if score < self.smoke_expand_min_probe_smoke_conf: return False if self._probe_extends_smoke_horizontally( candidate_box, seed_arr ) or self._probe_extends_smoke_downward(candidate_box, seed_arr): return score >= self.smoke_expand_min_extend_probe_conf return True def _smoke_expand_mean_rgb_distance( self, image: np.ndarray, box: np.ndarray, ref_rgb: np.ndarray ) -> float: h, w = image.shape[:2] x1 = max(0, int(math.floor(float(box[0])))) y1 = max(0, int(math.floor(float(box[1])))) x2 = min(w, int(math.ceil(float(box[2])))) y2 = min(h, int(math.ceil(float(box[3])))) if x2 <= x1 or y2 <= y1: return float("inf") roi = image[y1:y2, x1:x2] if roi.size == 0: return float("inf") mean_rgb = np.array( [ float(np.mean(roi[:, :, 2])), float(np.mean(roi[:, :, 1])), float(np.mean(roi[:, :, 0])), ], dtype=np.float32, ) return float(np.linalg.norm(mean_rgb - ref_rgb)) def _passes_smoke_expand_probe_color( self, image: np.ndarray, probe_box: np.ndarray, ref_rgb: np.ndarray, ) -> bool: return ( self._smoke_expand_mean_rgb_distance(image, probe_box, ref_rgb) <= self.smoke_expand_max_color_dist ) def _smoke_expansion_union_candidate( self, union_box: np.ndarray, seed_arr: np.ndarray, candidate_box: np.ndarray, candidate_score: float, max_conf: float, ) -> tuple[np.ndarray, float, bool]: union_box = union_box.copy() union_box[0] = min(union_box[0], candidate_box[0]) union_box[1] = min(union_box[1], candidate_box[1]) union_box[2] = max(union_box[2], candidate_box[2]) union_box[3] = max(union_box[3], candidate_box[3]) margin = 2.0 found_extra = ( float(candidate_box[0]) < float(seed_arr[0]) - margin or float(candidate_box[2]) > float(seed_arr[2]) + margin or float(candidate_box[1]) < float(seed_arr[1]) - margin or float(candidate_box[3]) > float(seed_arr[3]) + margin ) return union_box, max(max_conf, candidate_score), found_extra def _collect_smoke_expansion_cluster( self, image: np.ndarray, seed: BoundingBox, orig_size: tuple[int, int], crop_x1: int, crop_x2: int, ref_rgb: np.ndarray | None, tta_boxes: np.ndarray, tta_scores: np.ndarray, tta_cls: np.ndarray, tta_view_ids: np.ndarray | None, crop_boxes: np.ndarray | None = None, crop_scores: np.ndarray | None = None, crop_cls: np.ndarray | None = None, ) -> tuple[np.ndarray, float] | None: cls_smoke = self.class_names.index("smoke") seed_arr = self._bbox_to_array(seed) union_box = seed_arr.copy() max_conf = float(seed.conf) found_extra = False for i in range(len(tta_boxes)): if int(tta_cls[i]) != cls_smoke: continue candidate_box = tta_boxes[i] if not self._smoke_expansion_tta_candidate_conf_ok( float(tta_scores[i]), candidate_box, seed_arr ): continue if tta_view_ids is not None and not self._smoke_expansion_tta_view_corroborated( i, tta_boxes, tta_cls, tta_view_ids, cls_smoke ): continue if not self._smoke_corroborates_expansion_seed( candidate_box, seed, seed_arr, crop_x1, crop_x2 ): continue if not self._smoke_probe_within_expansion_reach( seed_arr, candidate_box, orig_size ): continue union_box, max_conf, extra = self._smoke_expansion_union_candidate( union_box, seed_arr, candidate_box, float(tta_scores[i]), max_conf ) found_extra = found_extra or extra if ( crop_boxes is not None and crop_scores is not None and crop_cls is not None and ref_rgb is not None and len(crop_boxes) > 0 ): for i in range(len(crop_boxes)): if int(crop_cls[i]) != cls_smoke: continue candidate_box = crop_boxes[i] if not self._smoke_expansion_crop_candidate_conf_ok( float(crop_scores[i]), candidate_box, seed_arr ): continue if not self._smoke_corroborates_expansion_seed( candidate_box, seed, seed_arr, crop_x1, crop_x2 ): continue if not self._smoke_probe_within_expansion_reach( seed_arr, candidate_box, orig_size ): continue if not self._passes_smoke_expand_probe_color( image, candidate_box, ref_rgb ): continue candidate_box = self._clip_smoke_crop_candidate_for_union( candidate_box, seed_arr ) union_box, max_conf, extra = self._smoke_expansion_union_candidate( union_box, seed_arr, candidate_box, float(crop_scores[i]), max_conf, ) found_extra = found_extra or extra union_box = self._clamp_smoke_expansion_union(union_box, seed_arr, orig_size) margin = 2.0 refined_seed = ( float(union_box[0]) < float(seed_arr[0]) - margin or float(union_box[2]) > float(seed_arr[2]) + margin or float(union_box[1]) < float(seed_arr[1]) - margin or float(union_box[3]) > float(seed_arr[3]) + margin or float(union_box[0]) > float(seed_arr[0]) + margin or float(union_box[2]) < float(seed_arr[2]) - margin or float(union_box[1]) > float(seed_arr[1]) + margin or float(union_box[3]) < float(seed_arr[3]) - margin ) if not found_extra and not refined_seed: return None return union_box, max_conf def _smoke_expansion_seed_is_complete( self, seed: BoundingBox, orig_size: tuple[int, int], tta_boxes: np.ndarray, tta_cls: np.ndarray, tta_view_ids: np.ndarray | None, ) -> bool: orig_w, orig_h = orig_size seed_w = float(seed.x2 - seed.x1) width_ratio = seed_w / max(1.0, float(orig_w)) y1_ratio = float(seed.y1) / max(1.0, float(orig_h)) if y1_ratio > self.smoke_expand_skip_max_y1_ratio: return False if width_ratio < self.smoke_expand_skip_min_width_ratio: return False if tta_view_ids is None or len(tta_boxes) == 0: return False cls_smoke = self.class_names.index("smoke") seed_arr = self._bbox_to_array(seed) matched_views: set[int] = set() for i in range(len(tta_boxes)): if int(tta_cls[i]) != cls_smoke: continue if self._boxes_involve_each_other_arrays( tta_boxes[i], seed_arr, self.smoke_expand_corroborate_involve_thresh, ): matched_views.add(int(tta_view_ids[i])) if len(matched_views) < 2: return False return seed.conf >= self.smoke_expand_skip_min_conf @staticmethod def _smoke_expansion_union_extended_seed( seed_arr: np.ndarray, union_box: np.ndarray, margin: float = 2.0 ) -> bool: return ( float(union_box[0]) < float(seed_arr[0]) - margin or float(union_box[2]) > float(seed_arr[2]) + margin or float(union_box[1]) < float(seed_arr[1]) - margin or float(union_box[3]) > float(seed_arr[3]) + margin ) def _probe_smoke_expansion( self, image: np.ndarray, results: list[BoundingBox], orig_size: tuple[int, int], tta_boxes: np.ndarray, tta_scores: np.ndarray, tta_cls: np.ndarray, tta_view_ids: np.ndarray | None, ) -> list[BoundingBox]: if not self.use_tta: return results cls_smoke = self.class_names.index("smoke") seeds = sorted( ( box for box in results if box.cls_id == cls_smoke and box.conf >= self.smoke_expand_min_seed_conf ), key=lambda box: box.conf, reverse=True, ) if not seeds: return results seeds = seeds[: self.smoke_expand_max_seeds] absorbed: set[int] = set() replacements: dict[int, BoundingBox] = {} for seed in seeds: seed_key = id(seed) if seed_key in absorbed: continue skip_crops_only = self._smoke_expansion_seed_is_complete( seed, orig_size, tta_boxes, tta_cls, tta_view_ids ) crop_regions = self._smoke_expansion_crop_regions(seed, orig_size) primary_x1 = crop_regions[0][0] primary_x2 = crop_regions[0][1] ref_rgb = self._smoke_expansion_border_reference_rgb(image, seed) expanded = self._collect_smoke_expansion_cluster( image, seed, orig_size, primary_x1, primary_x2, ref_rgb, tta_boxes, tta_scores, tta_cls, tta_view_ids, None, None, None, ) seed_arr = self._bbox_to_array(seed) orig_h = orig_size[1] upward_open = float(seed.y1) / max(1.0, float(orig_h)) > ( self.smoke_expand_skip_max_y1_ratio ) if skip_crops_only: pass elif ( expanded is not None and self._smoke_expansion_union_extended_seed(seed_arr, expanded[0]) and not upward_open ): pass else: crop_boxes_list: list[np.ndarray] = [] crop_scores_list: list[np.ndarray] = [] crop_cls_list: list[np.ndarray] = [] for x1, x2, y1, y2 in crop_regions: crop = image[y1:y2, x1:x2] if crop.size == 0 or crop.shape[0] < 2 or crop.shape[1] < 2: continue crop_size = (crop.shape[1], crop.shape[0]) boxes, scores, cls_ids = self._infer_view_arrays( crop, crop_size, flip_code=None ) if len(boxes) == 0: continue boxes = boxes.copy() boxes[:, [0, 2]] += x1 boxes[:, [1, 3]] += y1 boxes = self._clip_boxes(boxes, orig_size) crop_boxes_list.append(boxes) crop_scores_list.append(scores) crop_cls_list.append(cls_ids) merged_crop_boxes: np.ndarray | None = None merged_crop_scores: np.ndarray | None = None merged_crop_cls: np.ndarray | None = None if crop_boxes_list and ref_rgb is not None: merged_crop_boxes = np.concatenate(crop_boxes_list, axis=0) merged_crop_scores = np.concatenate(crop_scores_list, axis=0) merged_crop_cls = np.concatenate(crop_cls_list, axis=0) crop_expanded = self._collect_smoke_expansion_cluster( image, seed, orig_size, primary_x1, primary_x2, ref_rgb, tta_boxes, tta_scores, tta_cls, tta_view_ids, merged_crop_boxes, merged_crop_scores, merged_crop_cls, ) if crop_expanded is not None: expanded = crop_expanded if expanded is None: continue union_box, max_conf = expanded seed_arr = self._bbox_to_array(seed) for other in results: if other.cls_id != cls_smoke or other is seed: continue other_key = id(other) if other_key in absorbed: continue if other_key in replacements: other_arr = self._bbox_to_array(replacements[other_key]) else: other_arr = self._bbox_to_array(other) if self._boxes_involve_each_other_arrays( other_arr, seed_arr, self.smoke_expand_corroborate_involve_thresh, ) or self._boxes_involve_each_other_arrays( other_arr, union_box, self.smoke_expand_corroborate_involve_thresh, ): union_box[0] = min(union_box[0], other_arr[0]) union_box[1] = min(union_box[1], other_arr[1]) union_box[2] = max(union_box[2], other_arr[2]) union_box[3] = max(union_box[3], other_arr[3]) other_conf = ( float(replacements[other_key].conf) if other_key in replacements else float(other.conf) ) max_conf = max(max_conf, other_conf) absorbed.add(other_key) expanded_smoke = BoundingBox( x1=int(math.floor(union_box[0])), y1=int(math.floor(union_box[1])), x2=int(math.ceil(union_box[2])), y2=int(math.ceil(union_box[3])), cls_id=cls_smoke, conf=min(1.0, max_conf), ) replacements[seed_key] = expanded_smoke if not replacements and not absorbed: return results updated: list[BoundingBox] = [] for box in results: box_key = id(box) if box_key in absorbed: continue if box_key in replacements: updated.append(replacements[box_key]) continue updated.append(box) return updated def _infer_view_arrays( self, image: np.ndarray, orig_size: tuple[int, int], flip_code: int | None = None, ) -> tuple[np.ndarray, np.ndarray, np.ndarray]: if flip_code is not None: image = cv2.flip(image, flip_code) decode_size = (image.shape[1], image.shape[0]) input_tensor, ratio, pad, _ = self._preprocess(image) expected = (1, 3, self.input_height, self.input_width) if input_tensor.shape != expected: raise ValueError( f"Bad input tensor shape={input_tensor.shape}, expected={expected}" ) outputs = self.session.run(self.output_names, {self.input_name: input_tensor}) boxes, scores, cls_ids = self._decode_preds_to_arrays( outputs[0], ratio, pad, decode_size ) if flip_code is not None and len(boxes) > 0: boxes = self._map_flipped_boxes_arrays(boxes, decode_size, flip_code) return boxes, scores, cls_ids def _collect_merged_arrays( self, image: np.ndarray ) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, tuple[int, int]]: orig_size = (image.shape[1], image.shape[0]) if self.use_tta: view_flip_codes: tuple[int | None, ...] = (None, 1) else: view_flip_codes = (None,) all_boxes: list[np.ndarray] = [] all_scores: list[np.ndarray] = [] all_cls: list[np.ndarray] = [] all_view_ids: list[np.ndarray] = [] view_id = 0 for flip_code in view_flip_codes: boxes, scores, cls_ids = self._infer_view_arrays( image, orig_size, flip_code ) if len(boxes) > 0: all_boxes.append(boxes) all_scores.append(scores) all_cls.append(cls_ids) all_view_ids.append( np.full(len(boxes), view_id, dtype=np.int32) ) view_id += 1 if not all_boxes: return ( np.empty((0, 4), dtype=np.float32), np.empty((0,), dtype=np.float32), np.empty((0,), dtype=np.int32), np.empty((0,), dtype=np.int32), orig_size, ) return ( np.concatenate(all_boxes, axis=0), np.concatenate(all_scores, axis=0), np.concatenate(all_cls, axis=0), np.concatenate(all_view_ids, axis=0), orig_size, ) def _decode_final_dets( self, preds: np.ndarray, ratio: float, pad: tuple[float, float], orig_size: tuple[int, int], ) -> list[BoundingBox]: boxes, scores, cls_ids = self._decode_preds_to_arrays( preds, ratio, pad, orig_size ) return self._apply_post_filters(boxes, scores, cls_ids, orig_size) def _postprocess( self, output: np.ndarray, ratio: float, pad: tuple[float, float], orig_size: tuple[int, int], ) -> list[BoundingBox]: return self._decode_final_dets(output, ratio, pad, orig_size) @staticmethod def _validate_image(image: np.ndarray) -> np.ndarray: if image is None: raise ValueError("Input image is None") if not isinstance(image, np.ndarray): raise TypeError(f"Input is not numpy array: {type(image)}") if image.ndim != 3: raise ValueError(f"Expected HWC image, got shape={image.shape}") if image.shape[0] <= 0 or image.shape[1] <= 0: raise ValueError(f"Invalid image shape={image.shape}") if image.shape[2] != 3: raise ValueError(f"Expected 3 channels, got shape={image.shape}") if image.dtype != np.uint8: image = image.astype(np.uint8) return image def _predict_single(self, image: np.ndarray) -> list[BoundingBox]: image = self._validate_image(image) boxes, scores, cls_ids, view_ids, orig_size = self._collect_merged_arrays( image ) results = self._apply_post_filters( boxes, scores, cls_ids, orig_size, view_ids, ) cls_smoke = self.class_names.index("smoke") has_expandable_smoke = any( box.cls_id == cls_smoke and box.conf >= self.smoke_expand_min_seed_conf for box in results ) smoke_seeds = [ BoundingBox( x1=box.x1, y1=box.y1, x2=box.x2, y2=box.y2, cls_id=box.cls_id, conf=box.conf, ) for box in results if box.cls_id == cls_smoke ] if has_expandable_smoke: results = self._probe_smoke_expansion( image, results, orig_size, boxes, scores, cls_ids, view_ids ) has_anchor_smoke = any( box.cls_id == cls_smoke and box.conf >= self.smoke_anchor_min_smoke_conf for box in results ) if has_anchor_smoke: results = self._probe_smoke_anchored_fire( image, results, orig_size, boxes, cls_ids, smoke_seeds=smoke_seeds ) results = self._probe_merged_anchored_fire_ext( image, results, orig_size, boxes, scores, cls_ids ) results = self._filter_probe_fires_by_color(image, results) results = self._filter_fire_ext_by_red_color(image, results) return self._filter_results_min_conf(results) def predict_batch( self, batch_images: list[ndarray], offset: int, n_keypoints: int, ) -> list[TVFrameResult]: results: list[TVFrameResult] = [] for frame_number_in_batch, image in enumerate(batch_images): try: boxes = self._predict_single(image) except Exception as e: print( f"⚠️ Inference failed for frame " f"{offset + frame_number_in_batch}: {e}" ) boxes = [] results.append( TVFrameResult( frame_id=offset + frame_number_in_batch, boxes=boxes, keypoints=[(0, 0) for _ in range(max(0, int(n_keypoints)))], ) ) return results
from pathlib import Path
import math
import cv2
import numpy as np
import onnxruntime as ort
from numpy import ndarray
from pydantic import BaseModel
class BoundingBox(BaseModel):
x1: int
y1: int
x2: int
y2: int
cls_id: int
conf: float
class TVFrameResult(BaseModel):
frame_id: int
boxes: list[BoundingBox]
keypoints: list[tuple[int, int]]
class Miner:
class_names = ["fire", "smoke", "fire extinguisher"]
_model_class_order = ["fire", "fire extinguisher", "smoke"]
_parent_split_class_names = ("fire", "fire extinguisher")
conf_thresh = 0.000005
isolated_conf_thresh = 0.008
same_class_iou_thresh = 0.6
edge_conf_thresh = 0.005
edge_tol = 1.0
overlap_low_conf_thresh = 0.003
overlap_same_class_iou_thresh = 0.5
group_iou_thresh = 0.7
flip_low_conf_thresh = 0.2
fire_ext_same_object_overlap_thresh = 0.7
fire_ext_group_involve_thresh = 0.9
smoke_involve_thresh = 0.7
smoke_prefer_larger_conf_thresh = 0.3
smoke_cluster_merge_involve_thresh = 0.3
smoke_cluster_merge_min_size = 2
smoke_cluster_merge_early_min_count = 3
small_group_min_count = 2
small_group_low_conf_thresh = 0.02
fire_parent_min_small_count = 2
fire_parent_involve_thresh = 0.7
fire_split_inner_involve_thresh = 0.5
fire_split_max_inner_area_ratio = 0.5
fire_split_cluster_iou_thresh = 0.5
fire_split_min_clusters = 2
fire_split_min_inner_conf = 0.08
fire_split_peak_member_min_conf = 0.05
fire_split_peak_dist_ratio = 0.10
fire_split_peak_dist_min_px = 12.0
fire_split_peak_dist_max_px = 48.0
fire_split_centroid_preserve_px = 12.0
fire_split_conf_boost = 0.2
fire_split_parent_blob_conf_thresh = 0.5
fire_min_conf = 0.165
smoke_min_conf = 0.07
fire_ext_min_conf = 0.037
fire_ext_color_filter_max_conf = 0.20
fire_ext_min_red_dom_frac = 0.03
fire_ext_color_min_mean_r = 50.0
fire_ext_min_r_minus_g = 0.0
smoke_anchor_min_smoke_conf = 0.63
smoke_anchor_base_height_frac = 0.35
smoke_anchor_crop_width_pad_ratio = 0.15
smoke_anchor_crop_down_pad_px = 24.0
smoke_anchor_corroborate_involve_thresh = 0.5
smoke_anchor_fire_floor_conf = 0.165
smoke_anchor_min_fire_width_px = 10.0
smoke_anchor_min_fire_height_px = 10.0
smoke_anchor_strong_smoke_conf = 0.7
smoke_anchor_corroborated_min_fire_width_px = 4.0
smoke_anchor_corroborated_min_fire_height_px = 4.0
smoke_anchor_max_fire_area_ratio = 0.25
smoke_anchor_corroborate_area_ratio_max = 4.0
smoke_anchor_max_fires_per_smoke = 2
smoke_anchor_max_crop_fires = 2
smoke_anchor_max_probe_smokes = 2
smoke_anchor_separate_fire_involve_thresh = 0.5
smoke_anchor_max_probe_fire_width_px = 52.0
smoke_anchor_max_probe_fire_height_px = 48.0
smoke_anchor_max_probe_fire_area_ratio = 0.04
smoke_anchor_max_probe_crop_width_ratio = 0.25
smoke_anchor_max_probe_crop_height_ratio = 0.60
smoke_anchor_max_probe_fire_bottom_dist_frac = 0.10
smoke_anchor_probe_rescue_min_conf = 0.000005
smoke_anchor_corroborated_probe_min_conf = 0.00005
smoke_anchor_corroborated_refine_pad_px = 3.0
smoke_anchor_expanded_width_ratio = 1.25
ext_probe_min_cluster_count = 2
ext_probe_cluster_centroid_dist_px = 28.0
ext_probe_min_cluster_best_conf = 0.003
ext_probe_crop_pad_ratio = 0.85