Spaces:
Sleeping
Sleeping
Download code/pipeline/segmentation.py from acra-thesis/acra-backend: direct link, hf CLI and curl.
- Browser
- Download file 31.2 kB
-
https://huggingface.co/spaces/acra-thesis/acra-backend/resolve/main/code/pipeline/segmentation.py
- Command line
-
hf download hf://spaces/acra-thesis/acra-backend/code/pipeline/segmentation.py
-
curl -L -o segmentation.py https://huggingface.co/spaces/acra-thesis/acra-backend/resolve/main/code/pipeline/segmentation.py
31.2 kB
| """ | |
| segmentation.py | |
| --------------- | |
| Gallo, Dave Andre A. - YOLOv8 Semantic Segmentation + Per-ROI FCM | |
| Pipeline: | |
| 1. YOLOv8 detects semantic regions (roi-symbol, roi-color, etc.) | |
| 2. FCM runs *within* each detected region to find color sub-clusters | |
| 3. Sub-clusters feed into the existing conflict-detection and LCH | |
| re-encoding stages - only targeted poster regions are corrected | |
| Color ROI fallback (HSV red/green thresholding) activates automatically | |
| when YOLO finds nothing, ensuring check/X marks are always captured. | |
| Uses onnxruntime directly - no ultralytics / PyTorch dependency. | |
| """ | |
| import os | |
| import threading | |
| # See main.py: the second image used to kill the process on Windows. | |
| os.environ.setdefault("KMP_DUPLICATE_LIB_OK", "TRUE") | |
| os.environ.setdefault("OMP_NUM_THREADS", "1") | |
| os.environ.setdefault("OPENBLAS_NUM_THREADS", "1") | |
| os.environ.setdefault("MKL_NUM_THREADS", "1") | |
| import numpy as np | |
| from concurrent.futures import ThreadPoolExecutor | |
| from pathlib import Path | |
| from typing import Dict, List, Tuple, Optional | |
| from PIL import Image, ImageFilter | |
| try: | |
| _RESAMPLE_NEAREST = Image.Resampling.NEAREST | |
| except AttributeError: | |
| _RESAMPLE_NEAREST = Image.NEAREST | |
| _MODEL_CACHE: dict = {} | |
| # One Run() at a time. The CPU provider is not safe to call from the request | |
| # thread pool in parallel, and a crash there takes the whole server down. | |
| _INFER_LOCK = threading.Lock() | |
| _CODE_DIR = Path(__file__).resolve().parent.parent | |
| _MODEL_CANDIDATES = ("acra-v124-best.onnx", "acra_v124_1280.onnx", "acra-v79-stage1-best.onnx", "acra_medium_v7_best.onnx") | |
| def resolve_seg_model(explicit: Optional[str] = None) -> Optional[str]: | |
| """Return the first usable YOLO weights path (absolute).""" | |
| import os | |
| def _try(path: Path) -> Optional[str]: | |
| return str(path.resolve()) if path.is_file() else None | |
| if explicit: | |
| p = Path(explicit) | |
| found = _try(p) or _try(_CODE_DIR / explicit) | |
| if found: | |
| return found | |
| env = os.getenv("SEG_MODEL_PATH", "").strip() | |
| if env: | |
| found = _try(Path(env)) or _try(_CODE_DIR / env) | |
| if found: | |
| return found | |
| for name in _MODEL_CANDIDATES: | |
| found = _try(_CODE_DIR / name) | |
| if found: | |
| return found | |
| return None | |
| # Chroma below this marks a pixel/center as near-achromatic (text/ink/paper). | |
| # Kept in sync with reencoding.NEUTRAL_CHROMA so protection is consistent. | |
| _NEUTRAL_CHROMA = 12.0 | |
| # ββ Model loading βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def _load_model(model_path: str): | |
| """Load and cache an ONNX inference session.""" | |
| if model_path not in _MODEL_CACHE: | |
| import onnxruntime as ort | |
| opts = ort.SessionOptions() | |
| opts.log_severity_level = 3 # suppress onnxruntime warnings | |
| # One intra-op thread, no memory arena. Reusing the arena across | |
| # requests is what access-violated on the second image on Windows. | |
| opts.intra_op_num_threads = 1 | |
| opts.inter_op_num_threads = 1 | |
| opts.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL | |
| opts.enable_cpu_mem_arena = False | |
| opts.enable_mem_pattern = False | |
| _MODEL_CACHE[model_path] = ort.InferenceSession( | |
| model_path, | |
| sess_options=opts, | |
| providers=["CPUExecutionProvider"], | |
| ) | |
| return _MODEL_CACHE[model_path] | |
| def _get_class_names(session) -> dict: | |
| """Extract class idβname map from ONNX model metadata (ultralytics export format).""" | |
| try: | |
| import ast | |
| meta = session.get_modelmeta().custom_metadata_map | |
| # ultralytics exports names as a Python dict repr ({0: 'roi-color', ...}), | |
| # which is not valid JSON - parse with literal_eval | |
| parsed = ast.literal_eval(meta["names"]) | |
| return {int(k): v for k, v in parsed.items()} | |
| except Exception: | |
| # Fallback for a model with no names in its metadata. | |
| # | |
| # This list used to read roi-color, roi-object, roi-text, roi-symbol, | |
| # exclude-person -- which is NOT what the exports contain. Roboflow | |
| # writes classes ALPHABETICALLY, verified against the metadata of both | |
| # acra_medium_v7_best.onnx and the v17 export: | |
| # {0: exclude-person, 1: roi-color, 2: roi-object, | |
| # 3: roi-symbol, 4: roi-text} | |
| # With the old order every class was mislabelled, and class 0 -- | |
| # exclude-person -- was read as roi-color, so real people would have | |
| # been recoloured instead of protected. | |
| return {0: "exclude-person", 1: "roi-color", 2: "roi-object", | |
| 3: "roi-symbol", 4: "roi-text"} | |
| def _preprocess(img_uint8: np.ndarray, input_size: int = 640): | |
| """Letterbox-resize + normalize to [1,3,H,W] float32 for YOLOv8.""" | |
| h, w = img_uint8.shape[:2] | |
| scale = input_size / max(h, w) | |
| new_h, new_w = int(round(h * scale)), int(round(w * scale)) | |
| # BILINEAR on purpose. PIL defaults to BICUBIC, but ultralytics letterboxes | |
| # with cv2.INTER_LINEAR during training AND validation, so leaving the | |
| # default fed the model slightly different pixels than the ones its mAP was | |
| # measured on. Measured on four real pages: bilinear found one extra | |
| # detection on two of them. Small, but there is no reason to serve under | |
| # different rules than the model was scored under. | |
| resized = np.array(Image.fromarray(img_uint8).resize((new_w, new_h), | |
| resample=Image.BILINEAR)) | |
| canvas = np.full((input_size, input_size, 3), 114, dtype=np.uint8) | |
| pad_y = (input_size - new_h) // 2 | |
| pad_x = (input_size - new_w) // 2 | |
| canvas[pad_y:pad_y + new_h, pad_x:pad_x + new_w] = resized | |
| blob = canvas.astype(np.float32) / 255.0 | |
| blob = blob.transpose(2, 0, 1)[np.newaxis] # BCHW | |
| return blob, scale, pad_x, pad_y | |
| def _nms(boxes_xyxy: np.ndarray, scores: np.ndarray, iou_threshold: float = 0.45): | |
| """Greedy NMS - returns indices of kept boxes.""" | |
| if len(boxes_xyxy) == 0: | |
| return [] | |
| x1, y1, x2, y2 = boxes_xyxy[:, 0], boxes_xyxy[:, 1], boxes_xyxy[:, 2], boxes_xyxy[:, 3] | |
| areas = np.maximum(0, x2 - x1) * np.maximum(0, y2 - y1) | |
| order = scores.argsort()[::-1] | |
| keep = [] | |
| while order.size: | |
| i = order[0] | |
| keep.append(i) | |
| xx1 = np.maximum(x1[i], x1[order[1:]]) | |
| yy1 = np.maximum(y1[i], y1[order[1:]]) | |
| xx2 = np.minimum(x2[i], x2[order[1:]]) | |
| yy2 = np.minimum(y2[i], y2[order[1:]]) | |
| inter = np.maximum(0, xx2 - xx1) * np.maximum(0, yy2 - yy1) | |
| iou = inter / np.maximum(areas[i] + areas[order[1:]] - inter, 1e-6) | |
| order = order[1:][iou <= iou_threshold] | |
| return keep | |
| def _postprocess(raw_output: np.ndarray, conf_threshold: float, iou_threshold: float, | |
| scale: float, pad_x: int, pad_y: int, orig_h: int, orig_w: int, | |
| id_to_name: dict): | |
| """ | |
| Parse YOLOv8 detection ONNX output [1, 4+nc, anchors]. | |
| The anchor count follows the input size, it is not fixed: 8400 at 640, | |
| 48384 at 1536. Nothing here depends on the number. | |
| Returns (boxes_xyxy_int, confidences, class_name_list) in original image coords. | |
| """ | |
| pred = raw_output[0].T # [8400, 4+nc] | |
| boxes_cxcywh = pred[:, :4] | |
| scores_all = pred[:, 4:] | |
| class_ids = scores_all.argmax(axis=1) | |
| confidences = scores_all[np.arange(len(scores_all)), class_ids] | |
| mask = confidences >= conf_threshold | |
| if not mask.any(): | |
| return np.empty((0, 4), dtype=np.float32), np.array([]), [] | |
| boxes_cxcywh = boxes_cxcywh[mask] | |
| confidences = confidences[mask] | |
| class_ids = class_ids[mask] | |
| # cx,cy,w,h β x1,y1,x2,y2 | |
| x1 = boxes_cxcywh[:, 0] - boxes_cxcywh[:, 2] / 2 | |
| y1 = boxes_cxcywh[:, 1] - boxes_cxcywh[:, 3] / 2 | |
| x2 = boxes_cxcywh[:, 0] + boxes_cxcywh[:, 2] / 2 | |
| y2 = boxes_cxcywh[:, 1] + boxes_cxcywh[:, 3] / 2 | |
| boxes_xyxy = np.stack([x1, y1, x2, y2], axis=1) | |
| # CLASS-AWARE NMS. Suppress duplicates of the SAME class only. | |
| # | |
| # ACRA's labelling rules put several boxes on one region on purpose: a stop | |
| # sign is roi-object + roi-color + roi-text, and roi-color is drawn over | |
| # text and symbols by design. Boxes on the same pixels have IoU near 1.0, | |
| # so comparing across classes deleted all but the most confident one. The | |
| # dangerous case is exclude-person inside a photo panel: the panel wins, | |
| # the person box disappears, and that face gets recoloured. | |
| # | |
| # Shifting each class into its own coordinate band makes boxes of different | |
| # classes non-overlapping by construction, so NMS can only ever compare | |
| # like with like. The offsets are dropped straight afterwards. This is also | |
| # what ultralytics does when it measures mAP, so the server now scores | |
| # under the same rules the model was evaluated with. | |
| _CLASS_STRIDE = 100_000.0 # far larger than any image dimension | |
| _offsets = class_ids.astype(np.float32)[:, None] * _CLASS_STRIDE | |
| keep = _nms(boxes_xyxy + _offsets, confidences, iou_threshold) | |
| boxes_xyxy = boxes_xyxy[keep] | |
| confidences = confidences[keep] | |
| class_ids = class_ids[keep] | |
| # Remove letterbox padding and undo scale β original image coords | |
| boxes_xyxy[:, [0, 2]] -= pad_x | |
| boxes_xyxy[:, [1, 3]] -= pad_y | |
| boxes_xyxy /= scale | |
| boxes_xyxy[:, [0, 2]] = np.clip(boxes_xyxy[:, [0, 2]], 0, orig_w) | |
| boxes_xyxy[:, [1, 3]] = np.clip(boxes_xyxy[:, [1, 3]], 0, orig_h) | |
| names = [id_to_name.get(int(cid), f"class_{cid}") for cid in class_ids] | |
| return boxes_xyxy.astype(int), confidences, names | |
| # ββ Helpers βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def _bbox_to_mask(xyxy: np.ndarray, h: int, w: int) -> np.ndarray: | |
| """Convert a [x1,y1,x2,y2] box to a binary mask.""" | |
| x1, y1, x2, y2 = xyxy.astype(int) | |
| x1, y1 = max(0, x1), max(0, y1) | |
| x2, y2 = min(w, x2), min(h, y2) | |
| mask = np.zeros((h, w), dtype=np.float32) | |
| mask[y1:y2, x1:x2] = 1.0 | |
| return mask | |
| def _bbox_to_color_mask( | |
| xyxy: np.ndarray, | |
| img_uint8: np.ndarray, | |
| color_mask: Optional[np.ndarray] = None, | |
| ) -> np.ndarray: | |
| """ | |
| Convert a [x1,y1,x2,y2] box into a color-constrained ROI mask. | |
| Plain rectangular boxes are too broad for re-encoding: background pixels | |
| inside the box get cluster memberships and can be visibly recolored. This | |
| keeps only red/green foreground pixels inside the detected box. | |
| Pass a precomputed `color_mask` when masking many boxes - the full-image | |
| HSV analysis is by far the dominant cost and is identical for every box. | |
| """ | |
| h, w = img_uint8.shape[:2] | |
| bbox_mask = _bbox_to_mask(xyxy, h, w) | |
| if color_mask is None: | |
| red_f, green_f = _hsv_rg_masks(img_uint8) | |
| color_mask = np.clip(red_f + green_f, 0.0, 1.0) | |
| return bbox_mask * color_mask | |
| PROTECTED_CLASSES = ("exclude-person",) | |
| def build_protection_mask( | |
| boxes: List[Dict], | |
| h: int, | |
| w: int, | |
| soft_edge_radius: float = 3.0, | |
| ) -> Optional[np.ndarray]: | |
| """ | |
| Union of every protected detection as a float mask, 1.0 = never re-encode. | |
| Returns None when nothing protected was detected, so the caller can skip | |
| the work entirely. | |
| The feather ramp is grown OUTSIDE each box. A blur applied straight to the | |
| box would fade the person's own outline down to ~0.5 and let half the color | |
| shift back in exactly where it matters most. So the box is padded before | |
| blurring and then re-stamped at 1.0 at its true coordinates: every pixel the | |
| detector called a person is fully protected, and only the surrounding | |
| padding ramps down to blend with re-encoded neighbours. | |
| """ | |
| if not boxes: | |
| return None | |
| protected = [ | |
| b for b in boxes | |
| if str(b.get("class", "")).strip().lower() in PROTECTED_CLASSES | |
| ] | |
| if not protected: | |
| return None | |
| def _clip(b: Dict, pad: int) -> Tuple[int, int, int, int]: | |
| return (max(0, int(b["x1"]) - pad), max(0, int(b["y1"]) - pad), | |
| min(w, int(b["x2"]) + pad), min(h, int(b["y2"]) + pad)) | |
| pad = int(round(max(soft_edge_radius, 0.0) * 2.0)) + 1 | |
| mask = np.zeros((h, w), dtype=np.float32) | |
| for b in protected: | |
| x1, y1, x2, y2 = _clip(b, pad) | |
| if x2 > x1 and y2 > y1: | |
| mask[y1:y2, x1:x2] = 1.0 | |
| if soft_edge_radius > 0: | |
| mask = _soften_mask(mask, soft_edge_radius) | |
| for b in protected: | |
| x1, y1, x2, y2 = _clip(b, 0) | |
| if x2 > x1 and y2 > y1: | |
| mask[y1:y2, x1:x2] = 1.0 | |
| return np.clip(mask, 0.0, 1.0) | |
| def _soften_mask(mask_f: np.ndarray, radius: float) -> np.ndarray: | |
| """Feather mask edges using dilation, multi-pass blur, and smoothstep.""" | |
| if radius <= 0: | |
| return mask_f | |
| mask_uint8 = (np.clip(mask_f, 0.0, 1.0) * 255).astype(np.uint8) | |
| mask_pil = Image.fromarray(mask_uint8) | |
| dilate_px = max(0, int(round(radius * 0.15))) | |
| if dilate_px > 0: | |
| kernel = dilate_px * 2 + 1 | |
| mask_pil = mask_pil.filter(ImageFilter.MaxFilter(size=kernel)) | |
| mask_pil = mask_pil.filter(ImageFilter.GaussianBlur(radius=radius)) | |
| mask_pil = mask_pil.filter(ImageFilter.GaussianBlur(radius=radius * 0.6)) | |
| mask_f = np.array(mask_pil, dtype=np.float32) / 255.0 | |
| edge0, edge1 = 0.12, 0.88 | |
| t = np.clip((mask_f - edge0) / (edge1 - edge0), 0.0, 1.0) | |
| mask_f = t * t * (3.0 - 2.0 * t) | |
| mask_f = np.power(mask_f, 1.15) | |
| mask_pil = Image.fromarray((mask_f * 255).astype(np.uint8)) | |
| mask_pil = mask_pil.filter(ImageFilter.GaussianBlur(radius=radius * 0.2)) | |
| return np.array(mask_pil, dtype=np.float32) / 255.0 | |
| def _merge_close_centers(centers: np.ndarray, threshold: float = 20.0) -> np.ndarray: | |
| """ | |
| Merge cluster centers a normal viewer perceives as the same colour. | |
| A flat colour region (a solid green/red poster cell) gets split by FCM into | |
| several near-identical sub-clusters. If re-encoding shifts those by even | |
| slightly different amounts, the per-pixel blend turns the cell into a rough | |
| patchwork. Collapsing them to one center β one shift β one uniform recolor. | |
| Centers within `threshold` CIELAB ΞE (CIE76) are unioned and replaced by | |
| their mean; genuinely different colours (ΞE β₯ threshold) are kept apart. | |
| """ | |
| n = len(centers) | |
| if n <= 1: | |
| return np.asarray(centers, dtype=np.float64) | |
| parent = list(range(n)) | |
| def _find(x: int) -> int: | |
| while parent[x] != x: | |
| parent[x] = parent[parent[x]] | |
| x = parent[x] | |
| return x | |
| for i in range(n): | |
| for j in range(i + 1, n): | |
| if np.sqrt(((centers[i] - centers[j]) ** 2).sum()) < threshold: | |
| parent[_find(i)] = _find(j) | |
| groups: dict = {} | |
| for i in range(n): | |
| groups.setdefault(_find(i), []).append(i) | |
| merged = [np.asarray(centers)[idx].mean(axis=0) for idx in groups.values()] | |
| return np.array(merged, dtype=np.float64) | |
| def _hsv_rg_masks( | |
| img_uint8: np.ndarray, | |
| red_sat_min: float = 0.10, | |
| green_sat_min: float = 0.10, | |
| val_min: float = 0.10, | |
| ) -> Tuple[np.ndarray, np.ndarray]: | |
| """HSV red/green masks with no size filter. The expensive pass - call once per image.""" | |
| h, w = img_uint8.shape[:2] | |
| img_f = img_uint8.astype(np.float32) / 255.0 | |
| r, g, b = img_f[..., 0], img_f[..., 1], img_f[..., 2] | |
| cmax = np.maximum(np.maximum(r, g), b) | |
| cmin = np.minimum(np.minimum(r, g), b) | |
| delta = cmax - cmin | |
| hue = np.zeros((h, w), dtype=np.float32) | |
| safe_d = np.where(delta > 0, delta, 1.0) | |
| is_r = (cmax == r) & (delta > 0) | |
| is_g = (cmax == g) & (delta > 0) & ~is_r | |
| is_b = (cmax == b) & (delta > 0) & ~is_r & ~is_g | |
| hue[is_r] = (60.0 * ((g[is_r] - b[is_r]) / safe_d[is_r])) % 360.0 | |
| hue[is_g] = 60.0 * ((b[is_g] - r[is_g]) / safe_d[is_g] + 2.0) | |
| hue[is_b] = 60.0 * ((r[is_b] - g[is_b]) / safe_d[is_b] + 4.0) | |
| sat = np.divide(delta, cmax, out=np.zeros_like(delta), where=cmax > 1e-6) | |
| val = cmax | |
| vivid = val > val_min | |
| # CVD red-green confusion encompasses coral/orange/red through rose/pink/magenta/purple (hue <= 45 or >= 295) | |
| # and chartreuse/lime/yellow-green through emerald/sea-green/teal/cyan (hue 45..195). | |
| red_mask = vivid & (sat > red_sat_min) & ((hue <= 45.0) | (hue >= 295.0)) | |
| green_mask = vivid & (sat > green_sat_min) & (hue >= 45.0) & (hue <= 195.0) | |
| return red_mask.astype(np.float32), green_mask.astype(np.float32) | |
| def _filter_small_regions( | |
| red_mask: np.ndarray, | |
| green_mask: np.ndarray, | |
| min_region_frac: float, | |
| ) -> Tuple[np.ndarray, np.ndarray]: | |
| """Zero out red/green masks that cover fewer than min_region_frac of pixels.""" | |
| if min_region_frac <= 0: | |
| return red_mask, green_mask | |
| h, w = red_mask.shape | |
| min_px = max(1, int(h * w * min_region_frac)) | |
| if float(red_mask.sum()) < min_px: | |
| red_mask = np.zeros_like(red_mask) | |
| if float(green_mask.sum()) < min_px: | |
| green_mask = np.zeros_like(green_mask) | |
| return red_mask, green_mask | |
| # ββ YOLO mask extraction βββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def _get_yolo_masks( | |
| img_uint8: np.ndarray, | |
| model_path: str, | |
| conf_threshold: float, | |
| soft_edge_radius: float, | |
| imgsz: Optional[int] = None, | |
| ) -> Tuple[List[np.ndarray], List[str], List[Dict], Tuple[np.ndarray, np.ndarray]]: | |
| """ | |
| Run YOLO inference and return per-region float32 masks plus detection boxes. | |
| Index 0 is always background. Boxes are in `img_uint8` coordinates. | |
| HSV red/green masks are computed once and returned for reuse. | |
| """ | |
| h, w = img_uint8.shape[:2] | |
| background = np.ones((h, w), dtype=np.float32) | |
| masks_list: List[np.ndarray] = [background] | |
| class_names: List[str] = ["background"] | |
| resolved = resolve_seg_model(model_path) | |
| if not resolved: | |
| raise FileNotFoundError( | |
| f"No YOLO model found. Place acra_medium_v7_best.onnx in {_CODE_DIR} " | |
| "or set SEG_MODEL_PATH." | |
| ) | |
| session = _load_model(resolved) | |
| # Ask the MODEL what size it wants instead of assuming 640. The exported | |
| # ONNX has a fixed input -- 640 for acra_medium_v7_best, 1536 for the v17 | |
| # export -- and feeding the wrong size is an instant shape-mismatch error | |
| # on every request. An explicit imgsz still wins; 640 is only the last | |
| # resort for a model with a dynamic input dimension. | |
| _in_shape = session.get_inputs()[0].shape # [1, 3, H, W] | |
| _model_size = _in_shape[2] if isinstance(_in_shape[2], int) else None | |
| input_size = imgsz or _model_size or 640 | |
| blob, scale, pad_x, pad_y = _preprocess(img_uint8, input_size) | |
| input_name = session.get_inputs()[0].name | |
| with _INFER_LOCK: | |
| raw_output = session.run(None, {input_name: blob})[0] # [1, 4+nc, anchors] | |
| id_to_name = _get_class_names(session) | |
| det_boxes, det_confs, det_names = _postprocess( | |
| raw_output, conf_threshold, 0.45, scale, pad_x, pad_y, h, w, id_to_name | |
| ) | |
| has_box = len(det_boxes) > 0 | |
| yolo_boxes: List[Dict] = [] | |
| for xyxy, conf, class_name in zip(det_boxes, det_confs, det_names): | |
| yolo_boxes.append({ | |
| "x1": int(xyxy[0]), | |
| "y1": int(xyxy[1]), | |
| "x2": int(xyxy[2]), | |
| "y2": int(xyxy[3]), | |
| "class": class_name, | |
| "conf": round(float(conf), 4), | |
| }) | |
| red_hsv, green_hsv = _hsv_rg_masks(img_uint8) | |
| def _apply_mask(mask_f: np.ndarray, name: str) -> None: | |
| nonlocal background | |
| background -= mask_f | |
| masks_list.append(mask_f) | |
| class_names.append(name) | |
| if has_box: | |
| # Reuse the single HSV pass for every box (was recomputed per box, then again for fallback) | |
| color_mask_all = np.clip(red_hsv + green_hsv, 0.0, 1.0) | |
| # Merge boxes of the same class into one mask. Detection-dense images | |
| # (infographics) can yield 50+ boxes; per-box ROIs each pay their own | |
| # FCM run and weight column. Same-class regions share one FCM - the | |
| # dominant colors are the same and identical icons recolor uniformly. | |
| class_masks: Dict[str, np.ndarray] = {} | |
| for box_xyxy, class_name in zip(det_boxes, det_names): | |
| mask_f = _bbox_to_color_mask(box_xyxy, img_uint8, color_mask_all) | |
| if mask_f.sum() <= 0: | |
| continue | |
| # Feather only the box's local crop - the mask is zero elsewhere, | |
| # so blurring the full image per box is wasted work. | |
| pad = int(round(soft_edge_radius * 4 + 2)) | |
| cx1 = max(0, int(box_xyxy[0]) - pad) | |
| cy1 = max(0, int(box_xyxy[1]) - pad) | |
| cx2 = min(w, int(box_xyxy[2]) + pad) | |
| cy2 = min(h, int(box_xyxy[3]) + pad) | |
| soft = np.zeros_like(mask_f) | |
| soft[cy1:cy2, cx1:cx2] = _soften_mask(mask_f[cy1:cy2, cx1:cx2], soft_edge_radius) | |
| if class_name in class_masks: | |
| np.maximum(class_masks[class_name], soft, out=class_masks[class_name]) | |
| else: | |
| class_masks[class_name] = soft | |
| for class_name, merged in class_masks.items(): | |
| _apply_mask(merged, class_name) | |
| masks_list[0] = np.clip(background, 0.0, 1.0) | |
| return masks_list, class_names, yolo_boxes, (red_hsv, green_hsv) | |
| # ββ Main entry point ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def run_yolo_segmentation( | |
| img_uint8: np.ndarray, | |
| img_lab: np.ndarray, | |
| model_path: Optional[str] = None, | |
| # 0.45, not 0.25. Measured on the 50-image bench with v34: dropping the | |
| # floor's junk costs little and buys a visibly cleaner page. | |
| # conf random boxes off-position wrong class recall | |
| # 0.25 11.5% 10.4% 3.8% 79.9% | |
| # 0.45 8.5% 9.0% 3.1% 74.4% | |
| # A quarter of the spurious boxes disappear for 5.5 points of recall. That | |
| # trade favours re-encoding: a false box RECOLOURS a region that should not | |
| # be touched and the user sees it, while a missed box only leaves a region | |
| # alone. See Kaggle/bench/WHY_THE_BOXES_ARE_WRONG.md. | |
| conf_threshold: float = 0.45, | |
| soft_edge_radius: float = 3.0, | |
| imgsz: Optional[int] = None, | |
| require_red_green: bool = True, | |
| clusters_per_roi: Optional[int] = None, | |
| ) -> Tuple[np.ndarray, np.ndarray, List[str], List[Dict]]: | |
| """ | |
| Detect ROIs with YOLOv8, run FCM within each region, then return | |
| the combined (centers, memberships) for conflict detection and | |
| LCH re-encoding. | |
| Flow per detected region | |
| ------------------------ | |
| mask β extract ROI pixels β FCM (k sub-clusters) | |
| β scale memberships by mask weight | |
| β merge into global (N, K_total) membership matrix | |
| Background pixels (outside all ROIs) get a single center = their | |
| mean CIELAB color and are not re-encoded unless they conflict. | |
| Parameters | |
| ---------- | |
| img_uint8 : (H, W, 3) uint8 RGB | |
| img_lab : (H, W, 3) float CIELAB | |
| model_path : path to trained .pt weights | |
| conf_threshold : YOLO detection confidence cutoff | |
| soft_edge_radius : Gaussian blur on mask edges for smooth blending | |
| imgsz : YOLO inference input size (None = model default, 640) | |
| clusters_per_roi : FCM sub-clusters per detected region (None = adaptive) | |
| require_red_green : bool | |
| If True, only keep detected ROIs that overlap red or green pixels. | |
| Returns | |
| ------- | |
| centers : (K_total, 3) mean CIELAB per sub-cluster | |
| memberships : (N, K_total) per-pixel weights, rows sum to 1 | |
| class_names : list[str] label per sub-cluster | |
| yolo_boxes : list[dict] YOLO detections in img_uint8 coordinates | |
| """ | |
| from .fcm import run_fcm, _compute_memberships | |
| from .auto_clusters import estimate_n_clusters | |
| h, w = img_uint8.shape[:2] | |
| n_pixels = h * w | |
| lab_flat = img_lab.reshape(n_pixels, 3).astype(np.float64) | |
| import time as _time | |
| _t_masks = _time.perf_counter() | |
| masks_list, raw_names, yolo_boxes, (red_hsv, green_hsv) = _get_yolo_masks( | |
| img_uint8, model_path, conf_threshold, soft_edge_radius, imgsz | |
| ) | |
| print(f"[ACRA] seg: yolo+masks ({len(masks_list) - 1} rois) {_time.perf_counter() - _t_masks:.2f}s") | |
| # Optional filter: only keep ROIs that overlap red/green pixels. | |
| if require_red_green and len(masks_list) > 1: | |
| red_f, green_f = _filter_small_regions(red_hsv, green_hsv, 0.003) | |
| keep_masks: List[np.ndarray] = [np.ones((h, w), dtype=np.float32)] | |
| keep_names: List[str] = ["background"] | |
| kept_any = False | |
| for mask_f, name in zip(masks_list[1:], raw_names[1:]): | |
| overlap = (mask_f * (red_f + green_f)).sum() | |
| overlap_frac = overlap / max(mask_f.sum(), 1e-6) | |
| if overlap_frac >= 0.002: | |
| keep_masks.append(mask_f) | |
| keep_names.append(name) | |
| kept_any = True | |
| if not kept_any: | |
| keep_masks = [m.copy() for m in masks_list] | |
| keep_names = list(raw_names) | |
| # Rebuild background so skipped ROIs are treated as background | |
| if len(keep_masks) > 1: | |
| background = np.ones((h, w), dtype=np.float32) | |
| for mask_f in keep_masks[1:]: | |
| background -= mask_f | |
| keep_masks[0] = np.clip(background, 0.0, 1.0) | |
| masks_list, raw_names = keep_masks, keep_names | |
| def _process_roi(mask_f, name): | |
| mask_flat = mask_f.reshape(n_pixels).astype(np.float64) | |
| total_w = mask_flat.sum() | |
| region_center = ( | |
| (lab_flat * mask_flat[:, None]).sum(axis=0) / max(total_w, 1e-6) | |
| ) | |
| pixel_idx = np.where(mask_flat > 0.15)[0] | |
| if len(pixel_idx) == 0: | |
| return region_center[None], mask_flat[:, None], [name] | |
| # Split ROI pixels into chromatic (the colored cell) and neutral | |
| # (black/white text, ink, paper). The split only decides which pixels | |
| # define each cluster *center* - text pixels must not contaminate the | |
| # colored center, and neutral centers stay near-zero-chroma so | |
| # reencoding._is_neutral() pins them. Per-pixel memberships are still | |
| # computed softly against ALL centers further below, so the text/ | |
| # background transition blends smoothly (no hard chroma-threshold step). | |
| roi_lab = lab_flat[pixel_idx] | |
| roi_chroma = np.hypot(roi_lab[:, 1], roi_lab[:, 2]) | |
| chrom_sel = roi_chroma >= _NEUTRAL_CHROMA | |
| chrom_idx = pixel_idx[chrom_sel] | |
| neut_idx = pixel_idx[~chrom_sel] | |
| centers_out: List[np.ndarray] = [] | |
| names_out: List[str] = [] | |
| # ββ Chromatic sub-clusters - centers from chromatic pixels only ββββββ | |
| if clusters_per_roi is None: | |
| chrom_pixels = lab_flat[chrom_idx] if len(chrom_idx) else lab_flat[:0] | |
| target_k = estimate_n_clusters(chrom_pixels, k_min=2, k_max=4) | |
| else: | |
| target_k = clusters_per_roi | |
| min_for_fcm = max(target_k * 10, 30) | |
| if len(chrom_idx) >= min_for_fcm: | |
| k = max(2, min(target_k, len(chrom_idx) // 10)) | |
| try: | |
| roi_centers, _ = run_fcm(lab_flat[chrom_idx], n_clusters=k) | |
| # Collapse near-identical sub-clusters so a flat colour cell | |
| # re-encodes as one uniform shift, not a rough patchwork. | |
| roi_centers = _merge_close_centers(roi_centers) | |
| centers_out.append(roi_centers) | |
| names_out.extend( | |
| f"{name} #{j + 1}" for j in range(len(roi_centers)) | |
| ) | |
| except Exception: | |
| # FCM failed - fall back to one mean center over all pixels. | |
| chrom_idx = pixel_idx | |
| neut_idx = pixel_idx[:0] | |
| if not centers_out and len(chrom_idx): | |
| centers_out.append(lab_flat[chrom_idx].mean(axis=0)[None]) | |
| names_out.append(name) | |
| # ββ Neutral clusters - text/ink (dark) and paper (light), kept apart β | |
| if len(neut_idx): | |
| neut_L = lab_flat[neut_idx, 0] | |
| for sel, suffix in ((neut_L < 55.0, "text"), (neut_L >= 55.0, "paper")): | |
| grp = neut_idx[sel] | |
| if not len(grp): | |
| continue | |
| centers_out.append(lab_flat[grp].mean(axis=0)[None]) | |
| names_out.append(f"{name} ({suffix})") | |
| if not centers_out: | |
| return region_center[None], mask_flat[:, None], [name] | |
| roi_centers_all = np.vstack(centers_out) | |
| # Soft memberships for EVERY ROI pixel against ALL centers. This is what | |
| # keeps re-encoding smooth: edge/anti-aliased pixels blend gradually | |
| # instead of snapping at the chromatic/neutral boundary. | |
| k_total = roi_centers_all.shape[0] | |
| full_W = np.zeros((n_pixels, k_total), dtype=np.float64) | |
| CHUNK = 50000 | |
| for s in range(0, len(pixel_idx), CHUNK): | |
| idx = pixel_idx[s:s + CHUNK] | |
| full_W[idx] = ( | |
| _compute_memberships(lab_flat[idx], roi_centers_all, m=2.0) | |
| * mask_flat[idx, np.newaxis] | |
| ) | |
| return roi_centers_all, full_W, names_out | |
| all_centers: List[np.ndarray] = [] | |
| all_weights: List[np.ndarray] = [] | |
| final_names: List[str] = [] | |
| # Background is always index 0 - single cluster, no FCM | |
| bg_mask_flat = masks_list[0].reshape(n_pixels).astype(np.float64) | |
| bg_center = (lab_flat * bg_mask_flat[:, None]).sum(axis=0) / max(bg_mask_flat.sum(), 1e-6) | |
| all_centers.append(bg_center[None]) | |
| all_weights.append(bg_mask_flat[:, None]) | |
| final_names.append(raw_names[0]) | |
| # ROIs: each FCM run is independent - process in parallel | |
| _t_fcm = _time.perf_counter() | |
| roi_pairs = list(zip(masks_list[1:], raw_names[1:])) | |
| if roi_pairs: | |
| with ThreadPoolExecutor(max_workers=2) as _ex: | |
| futures = [_ex.submit(_process_roi, mf, nm) for mf, nm in roi_pairs] | |
| for fut in futures: | |
| roi_centers, roi_W, roi_names = fut.result() | |
| all_centers.append(roi_centers) | |
| all_weights.append(roi_W) | |
| final_names.extend(roi_names) | |
| print(f"[ACRA] seg: fcm ({len(roi_pairs)} rois) {_time.perf_counter() - _t_fcm:.2f}s") | |
| # Combine all regions into flat arrays | |
| centers = np.vstack(all_centers) # (K_total, 3) | |
| W = np.hstack(all_weights).astype(np.float32) # (N, K_total) | |
| # Normalize rows so every pixel's weights sum to 1 | |
| row_sum = W.sum(axis=1, keepdims=True).clip(min=1e-6) | |
| W /= row_sum | |
| return centers, W, final_names, yolo_boxes | |