acra-backend / code /pipeline /segmentation.py
Davionnic
fix(pipeline): prevent excessive green lightness shift and unneeded re-encoding
b13e551
Raw History Blame Contribute Delete
31.2 kB
"""
segmentation.py
---------------
Gallo, Dave Andre A. - YOLOv8 Semantic Segmentation + Per-ROI FCM
Pipeline:
1. YOLOv8 detects semantic regions (roi-symbol, roi-color, etc.)
2. FCM runs *within* each detected region to find color sub-clusters
3. Sub-clusters feed into the existing conflict-detection and LCH
re-encoding stages - only targeted poster regions are corrected
Color ROI fallback (HSV red/green thresholding) activates automatically
when YOLO finds nothing, ensuring check/X marks are always captured.
Uses onnxruntime directly - no ultralytics / PyTorch dependency.
"""
import os
import threading
# See main.py: the second image used to kill the process on Windows.
os.environ.setdefault("KMP_DUPLICATE_LIB_OK", "TRUE")
os.environ.setdefault("OMP_NUM_THREADS", "1")
os.environ.setdefault("OPENBLAS_NUM_THREADS", "1")
os.environ.setdefault("MKL_NUM_THREADS", "1")
import numpy as np
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from typing import Dict, List, Tuple, Optional
from PIL import Image, ImageFilter
try:
_RESAMPLE_NEAREST = Image.Resampling.NEAREST
except AttributeError:
_RESAMPLE_NEAREST = Image.NEAREST
_MODEL_CACHE: dict = {}
# One Run() at a time. The CPU provider is not safe to call from the request
# thread pool in parallel, and a crash there takes the whole server down.
_INFER_LOCK = threading.Lock()
_CODE_DIR = Path(__file__).resolve().parent.parent
_MODEL_CANDIDATES = ("acra-v124-best.onnx", "acra_v124_1280.onnx", "acra-v79-stage1-best.onnx", "acra_medium_v7_best.onnx")
def resolve_seg_model(explicit: Optional[str] = None) -> Optional[str]:
"""Return the first usable YOLO weights path (absolute)."""
import os
def _try(path: Path) -> Optional[str]:
return str(path.resolve()) if path.is_file() else None
if explicit:
p = Path(explicit)
found = _try(p) or _try(_CODE_DIR / explicit)
if found:
return found
env = os.getenv("SEG_MODEL_PATH", "").strip()
if env:
found = _try(Path(env)) or _try(_CODE_DIR / env)
if found:
return found
for name in _MODEL_CANDIDATES:
found = _try(_CODE_DIR / name)
if found:
return found
return None
# Chroma below this marks a pixel/center as near-achromatic (text/ink/paper).
# Kept in sync with reencoding.NEUTRAL_CHROMA so protection is consistent.
_NEUTRAL_CHROMA = 12.0
# ── Model loading ─────────────────────────────────────────────────────────────
def _load_model(model_path: str):
"""Load and cache an ONNX inference session."""
if model_path not in _MODEL_CACHE:
import onnxruntime as ort
opts = ort.SessionOptions()
opts.log_severity_level = 3 # suppress onnxruntime warnings
# One intra-op thread, no memory arena. Reusing the arena across
# requests is what access-violated on the second image on Windows.
opts.intra_op_num_threads = 1
opts.inter_op_num_threads = 1
opts.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL
opts.enable_cpu_mem_arena = False
opts.enable_mem_pattern = False
_MODEL_CACHE[model_path] = ort.InferenceSession(
model_path,
sess_options=opts,
providers=["CPUExecutionProvider"],
)
return _MODEL_CACHE[model_path]
def _get_class_names(session) -> dict:
"""Extract class id→name map from ONNX model metadata (ultralytics export format)."""
try:
import ast
meta = session.get_modelmeta().custom_metadata_map
# ultralytics exports names as a Python dict repr ({0: 'roi-color', ...}),
# which is not valid JSON - parse with literal_eval
parsed = ast.literal_eval(meta["names"])
return {int(k): v for k, v in parsed.items()}
except Exception:
# Fallback for a model with no names in its metadata.
#
# This list used to read roi-color, roi-object, roi-text, roi-symbol,
# exclude-person -- which is NOT what the exports contain. Roboflow
# writes classes ALPHABETICALLY, verified against the metadata of both
# acra_medium_v7_best.onnx and the v17 export:
# {0: exclude-person, 1: roi-color, 2: roi-object,
# 3: roi-symbol, 4: roi-text}
# With the old order every class was mislabelled, and class 0 --
# exclude-person -- was read as roi-color, so real people would have
# been recoloured instead of protected.
return {0: "exclude-person", 1: "roi-color", 2: "roi-object",
3: "roi-symbol", 4: "roi-text"}
def _preprocess(img_uint8: np.ndarray, input_size: int = 640):
"""Letterbox-resize + normalize to [1,3,H,W] float32 for YOLOv8."""
h, w = img_uint8.shape[:2]
scale = input_size / max(h, w)
new_h, new_w = int(round(h * scale)), int(round(w * scale))
# BILINEAR on purpose. PIL defaults to BICUBIC, but ultralytics letterboxes
# with cv2.INTER_LINEAR during training AND validation, so leaving the
# default fed the model slightly different pixels than the ones its mAP was
# measured on. Measured on four real pages: bilinear found one extra
# detection on two of them. Small, but there is no reason to serve under
# different rules than the model was scored under.
resized = np.array(Image.fromarray(img_uint8).resize((new_w, new_h),
resample=Image.BILINEAR))
canvas = np.full((input_size, input_size, 3), 114, dtype=np.uint8)
pad_y = (input_size - new_h) // 2
pad_x = (input_size - new_w) // 2
canvas[pad_y:pad_y + new_h, pad_x:pad_x + new_w] = resized
blob = canvas.astype(np.float32) / 255.0
blob = blob.transpose(2, 0, 1)[np.newaxis] # BCHW
return blob, scale, pad_x, pad_y
def _nms(boxes_xyxy: np.ndarray, scores: np.ndarray, iou_threshold: float = 0.45):
"""Greedy NMS - returns indices of kept boxes."""
if len(boxes_xyxy) == 0:
return []
x1, y1, x2, y2 = boxes_xyxy[:, 0], boxes_xyxy[:, 1], boxes_xyxy[:, 2], boxes_xyxy[:, 3]
areas = np.maximum(0, x2 - x1) * np.maximum(0, y2 - y1)
order = scores.argsort()[::-1]
keep = []
while order.size:
i = order[0]
keep.append(i)
xx1 = np.maximum(x1[i], x1[order[1:]])
yy1 = np.maximum(y1[i], y1[order[1:]])
xx2 = np.minimum(x2[i], x2[order[1:]])
yy2 = np.minimum(y2[i], y2[order[1:]])
inter = np.maximum(0, xx2 - xx1) * np.maximum(0, yy2 - yy1)
iou = inter / np.maximum(areas[i] + areas[order[1:]] - inter, 1e-6)
order = order[1:][iou <= iou_threshold]
return keep
def _postprocess(raw_output: np.ndarray, conf_threshold: float, iou_threshold: float,
scale: float, pad_x: int, pad_y: int, orig_h: int, orig_w: int,
id_to_name: dict):
"""
Parse YOLOv8 detection ONNX output [1, 4+nc, anchors].
The anchor count follows the input size, it is not fixed: 8400 at 640,
48384 at 1536. Nothing here depends on the number.
Returns (boxes_xyxy_int, confidences, class_name_list) in original image coords.
"""
pred = raw_output[0].T # [8400, 4+nc]
boxes_cxcywh = pred[:, :4]
scores_all = pred[:, 4:]
class_ids = scores_all.argmax(axis=1)
confidences = scores_all[np.arange(len(scores_all)), class_ids]
mask = confidences >= conf_threshold
if not mask.any():
return np.empty((0, 4), dtype=np.float32), np.array([]), []
boxes_cxcywh = boxes_cxcywh[mask]
confidences = confidences[mask]
class_ids = class_ids[mask]
# cx,cy,w,h β†’ x1,y1,x2,y2
x1 = boxes_cxcywh[:, 0] - boxes_cxcywh[:, 2] / 2
y1 = boxes_cxcywh[:, 1] - boxes_cxcywh[:, 3] / 2
x2 = boxes_cxcywh[:, 0] + boxes_cxcywh[:, 2] / 2
y2 = boxes_cxcywh[:, 1] + boxes_cxcywh[:, 3] / 2
boxes_xyxy = np.stack([x1, y1, x2, y2], axis=1)
# CLASS-AWARE NMS. Suppress duplicates of the SAME class only.
#
# ACRA's labelling rules put several boxes on one region on purpose: a stop
# sign is roi-object + roi-color + roi-text, and roi-color is drawn over
# text and symbols by design. Boxes on the same pixels have IoU near 1.0,
# so comparing across classes deleted all but the most confident one. The
# dangerous case is exclude-person inside a photo panel: the panel wins,
# the person box disappears, and that face gets recoloured.
#
# Shifting each class into its own coordinate band makes boxes of different
# classes non-overlapping by construction, so NMS can only ever compare
# like with like. The offsets are dropped straight afterwards. This is also
# what ultralytics does when it measures mAP, so the server now scores
# under the same rules the model was evaluated with.
_CLASS_STRIDE = 100_000.0 # far larger than any image dimension
_offsets = class_ids.astype(np.float32)[:, None] * _CLASS_STRIDE
keep = _nms(boxes_xyxy + _offsets, confidences, iou_threshold)
boxes_xyxy = boxes_xyxy[keep]
confidences = confidences[keep]
class_ids = class_ids[keep]
# Remove letterbox padding and undo scale β†’ original image coords
boxes_xyxy[:, [0, 2]] -= pad_x
boxes_xyxy[:, [1, 3]] -= pad_y
boxes_xyxy /= scale
boxes_xyxy[:, [0, 2]] = np.clip(boxes_xyxy[:, [0, 2]], 0, orig_w)
boxes_xyxy[:, [1, 3]] = np.clip(boxes_xyxy[:, [1, 3]], 0, orig_h)
names = [id_to_name.get(int(cid), f"class_{cid}") for cid in class_ids]
return boxes_xyxy.astype(int), confidences, names
# ── Helpers ───────────────────────────────────────────────────────────────────
def _bbox_to_mask(xyxy: np.ndarray, h: int, w: int) -> np.ndarray:
"""Convert a [x1,y1,x2,y2] box to a binary mask."""
x1, y1, x2, y2 = xyxy.astype(int)
x1, y1 = max(0, x1), max(0, y1)
x2, y2 = min(w, x2), min(h, y2)
mask = np.zeros((h, w), dtype=np.float32)
mask[y1:y2, x1:x2] = 1.0
return mask
def _bbox_to_color_mask(
xyxy: np.ndarray,
img_uint8: np.ndarray,
color_mask: Optional[np.ndarray] = None,
) -> np.ndarray:
"""
Convert a [x1,y1,x2,y2] box into a color-constrained ROI mask.
Plain rectangular boxes are too broad for re-encoding: background pixels
inside the box get cluster memberships and can be visibly recolored. This
keeps only red/green foreground pixels inside the detected box.
Pass a precomputed `color_mask` when masking many boxes - the full-image
HSV analysis is by far the dominant cost and is identical for every box.
"""
h, w = img_uint8.shape[:2]
bbox_mask = _bbox_to_mask(xyxy, h, w)
if color_mask is None:
red_f, green_f = _hsv_rg_masks(img_uint8)
color_mask = np.clip(red_f + green_f, 0.0, 1.0)
return bbox_mask * color_mask
PROTECTED_CLASSES = ("exclude-person",)
def build_protection_mask(
boxes: List[Dict],
h: int,
w: int,
soft_edge_radius: float = 3.0,
) -> Optional[np.ndarray]:
"""
Union of every protected detection as a float mask, 1.0 = never re-encode.
Returns None when nothing protected was detected, so the caller can skip
the work entirely.
The feather ramp is grown OUTSIDE each box. A blur applied straight to the
box would fade the person's own outline down to ~0.5 and let half the color
shift back in exactly where it matters most. So the box is padded before
blurring and then re-stamped at 1.0 at its true coordinates: every pixel the
detector called a person is fully protected, and only the surrounding
padding ramps down to blend with re-encoded neighbours.
"""
if not boxes:
return None
protected = [
b for b in boxes
if str(b.get("class", "")).strip().lower() in PROTECTED_CLASSES
]
if not protected:
return None
def _clip(b: Dict, pad: int) -> Tuple[int, int, int, int]:
return (max(0, int(b["x1"]) - pad), max(0, int(b["y1"]) - pad),
min(w, int(b["x2"]) + pad), min(h, int(b["y2"]) + pad))
pad = int(round(max(soft_edge_radius, 0.0) * 2.0)) + 1
mask = np.zeros((h, w), dtype=np.float32)
for b in protected:
x1, y1, x2, y2 = _clip(b, pad)
if x2 > x1 and y2 > y1:
mask[y1:y2, x1:x2] = 1.0
if soft_edge_radius > 0:
mask = _soften_mask(mask, soft_edge_radius)
for b in protected:
x1, y1, x2, y2 = _clip(b, 0)
if x2 > x1 and y2 > y1:
mask[y1:y2, x1:x2] = 1.0
return np.clip(mask, 0.0, 1.0)
def _soften_mask(mask_f: np.ndarray, radius: float) -> np.ndarray:
"""Feather mask edges using dilation, multi-pass blur, and smoothstep."""
if radius <= 0:
return mask_f
mask_uint8 = (np.clip(mask_f, 0.0, 1.0) * 255).astype(np.uint8)
mask_pil = Image.fromarray(mask_uint8)
dilate_px = max(0, int(round(radius * 0.15)))
if dilate_px > 0:
kernel = dilate_px * 2 + 1
mask_pil = mask_pil.filter(ImageFilter.MaxFilter(size=kernel))
mask_pil = mask_pil.filter(ImageFilter.GaussianBlur(radius=radius))
mask_pil = mask_pil.filter(ImageFilter.GaussianBlur(radius=radius * 0.6))
mask_f = np.array(mask_pil, dtype=np.float32) / 255.0
edge0, edge1 = 0.12, 0.88
t = np.clip((mask_f - edge0) / (edge1 - edge0), 0.0, 1.0)
mask_f = t * t * (3.0 - 2.0 * t)
mask_f = np.power(mask_f, 1.15)
mask_pil = Image.fromarray((mask_f * 255).astype(np.uint8))
mask_pil = mask_pil.filter(ImageFilter.GaussianBlur(radius=radius * 0.2))
return np.array(mask_pil, dtype=np.float32) / 255.0
def _merge_close_centers(centers: np.ndarray, threshold: float = 20.0) -> np.ndarray:
"""
Merge cluster centers a normal viewer perceives as the same colour.
A flat colour region (a solid green/red poster cell) gets split by FCM into
several near-identical sub-clusters. If re-encoding shifts those by even
slightly different amounts, the per-pixel blend turns the cell into a rough
patchwork. Collapsing them to one center β†’ one shift β†’ one uniform recolor.
Centers within `threshold` CIELAB Ξ”E (CIE76) are unioned and replaced by
their mean; genuinely different colours (Ξ”E β‰₯ threshold) are kept apart.
"""
n = len(centers)
if n <= 1:
return np.asarray(centers, dtype=np.float64)
parent = list(range(n))
def _find(x: int) -> int:
while parent[x] != x:
parent[x] = parent[parent[x]]
x = parent[x]
return x
for i in range(n):
for j in range(i + 1, n):
if np.sqrt(((centers[i] - centers[j]) ** 2).sum()) < threshold:
parent[_find(i)] = _find(j)
groups: dict = {}
for i in range(n):
groups.setdefault(_find(i), []).append(i)
merged = [np.asarray(centers)[idx].mean(axis=0) for idx in groups.values()]
return np.array(merged, dtype=np.float64)
def _hsv_rg_masks(
img_uint8: np.ndarray,
red_sat_min: float = 0.10,
green_sat_min: float = 0.10,
val_min: float = 0.10,
) -> Tuple[np.ndarray, np.ndarray]:
"""HSV red/green masks with no size filter. The expensive pass - call once per image."""
h, w = img_uint8.shape[:2]
img_f = img_uint8.astype(np.float32) / 255.0
r, g, b = img_f[..., 0], img_f[..., 1], img_f[..., 2]
cmax = np.maximum(np.maximum(r, g), b)
cmin = np.minimum(np.minimum(r, g), b)
delta = cmax - cmin
hue = np.zeros((h, w), dtype=np.float32)
safe_d = np.where(delta > 0, delta, 1.0)
is_r = (cmax == r) & (delta > 0)
is_g = (cmax == g) & (delta > 0) & ~is_r
is_b = (cmax == b) & (delta > 0) & ~is_r & ~is_g
hue[is_r] = (60.0 * ((g[is_r] - b[is_r]) / safe_d[is_r])) % 360.0
hue[is_g] = 60.0 * ((b[is_g] - r[is_g]) / safe_d[is_g] + 2.0)
hue[is_b] = 60.0 * ((r[is_b] - g[is_b]) / safe_d[is_b] + 4.0)
sat = np.divide(delta, cmax, out=np.zeros_like(delta), where=cmax > 1e-6)
val = cmax
vivid = val > val_min
# CVD red-green confusion encompasses coral/orange/red through rose/pink/magenta/purple (hue <= 45 or >= 295)
# and chartreuse/lime/yellow-green through emerald/sea-green/teal/cyan (hue 45..195).
red_mask = vivid & (sat > red_sat_min) & ((hue <= 45.0) | (hue >= 295.0))
green_mask = vivid & (sat > green_sat_min) & (hue >= 45.0) & (hue <= 195.0)
return red_mask.astype(np.float32), green_mask.astype(np.float32)
def _filter_small_regions(
red_mask: np.ndarray,
green_mask: np.ndarray,
min_region_frac: float,
) -> Tuple[np.ndarray, np.ndarray]:
"""Zero out red/green masks that cover fewer than min_region_frac of pixels."""
if min_region_frac <= 0:
return red_mask, green_mask
h, w = red_mask.shape
min_px = max(1, int(h * w * min_region_frac))
if float(red_mask.sum()) < min_px:
red_mask = np.zeros_like(red_mask)
if float(green_mask.sum()) < min_px:
green_mask = np.zeros_like(green_mask)
return red_mask, green_mask
# ── YOLO mask extraction ───────────────────────────────────────────────────────
def _get_yolo_masks(
img_uint8: np.ndarray,
model_path: str,
conf_threshold: float,
soft_edge_radius: float,
imgsz: Optional[int] = None,
) -> Tuple[List[np.ndarray], List[str], List[Dict], Tuple[np.ndarray, np.ndarray]]:
"""
Run YOLO inference and return per-region float32 masks plus detection boxes.
Index 0 is always background. Boxes are in `img_uint8` coordinates.
HSV red/green masks are computed once and returned for reuse.
"""
h, w = img_uint8.shape[:2]
background = np.ones((h, w), dtype=np.float32)
masks_list: List[np.ndarray] = [background]
class_names: List[str] = ["background"]
resolved = resolve_seg_model(model_path)
if not resolved:
raise FileNotFoundError(
f"No YOLO model found. Place acra_medium_v7_best.onnx in {_CODE_DIR} "
"or set SEG_MODEL_PATH."
)
session = _load_model(resolved)
# Ask the MODEL what size it wants instead of assuming 640. The exported
# ONNX has a fixed input -- 640 for acra_medium_v7_best, 1536 for the v17
# export -- and feeding the wrong size is an instant shape-mismatch error
# on every request. An explicit imgsz still wins; 640 is only the last
# resort for a model with a dynamic input dimension.
_in_shape = session.get_inputs()[0].shape # [1, 3, H, W]
_model_size = _in_shape[2] if isinstance(_in_shape[2], int) else None
input_size = imgsz or _model_size or 640
blob, scale, pad_x, pad_y = _preprocess(img_uint8, input_size)
input_name = session.get_inputs()[0].name
with _INFER_LOCK:
raw_output = session.run(None, {input_name: blob})[0] # [1, 4+nc, anchors]
id_to_name = _get_class_names(session)
det_boxes, det_confs, det_names = _postprocess(
raw_output, conf_threshold, 0.45, scale, pad_x, pad_y, h, w, id_to_name
)
has_box = len(det_boxes) > 0
yolo_boxes: List[Dict] = []
for xyxy, conf, class_name in zip(det_boxes, det_confs, det_names):
yolo_boxes.append({
"x1": int(xyxy[0]),
"y1": int(xyxy[1]),
"x2": int(xyxy[2]),
"y2": int(xyxy[3]),
"class": class_name,
"conf": round(float(conf), 4),
})
red_hsv, green_hsv = _hsv_rg_masks(img_uint8)
def _apply_mask(mask_f: np.ndarray, name: str) -> None:
nonlocal background
background -= mask_f
masks_list.append(mask_f)
class_names.append(name)
if has_box:
# Reuse the single HSV pass for every box (was recomputed per box, then again for fallback)
color_mask_all = np.clip(red_hsv + green_hsv, 0.0, 1.0)
# Merge boxes of the same class into one mask. Detection-dense images
# (infographics) can yield 50+ boxes; per-box ROIs each pay their own
# FCM run and weight column. Same-class regions share one FCM - the
# dominant colors are the same and identical icons recolor uniformly.
class_masks: Dict[str, np.ndarray] = {}
for box_xyxy, class_name in zip(det_boxes, det_names):
mask_f = _bbox_to_color_mask(box_xyxy, img_uint8, color_mask_all)
if mask_f.sum() <= 0:
continue
# Feather only the box's local crop - the mask is zero elsewhere,
# so blurring the full image per box is wasted work.
pad = int(round(soft_edge_radius * 4 + 2))
cx1 = max(0, int(box_xyxy[0]) - pad)
cy1 = max(0, int(box_xyxy[1]) - pad)
cx2 = min(w, int(box_xyxy[2]) + pad)
cy2 = min(h, int(box_xyxy[3]) + pad)
soft = np.zeros_like(mask_f)
soft[cy1:cy2, cx1:cx2] = _soften_mask(mask_f[cy1:cy2, cx1:cx2], soft_edge_radius)
if class_name in class_masks:
np.maximum(class_masks[class_name], soft, out=class_masks[class_name])
else:
class_masks[class_name] = soft
for class_name, merged in class_masks.items():
_apply_mask(merged, class_name)
masks_list[0] = np.clip(background, 0.0, 1.0)
return masks_list, class_names, yolo_boxes, (red_hsv, green_hsv)
# ── Main entry point ──────────────────────────────────────────────────────────
def run_yolo_segmentation(
img_uint8: np.ndarray,
img_lab: np.ndarray,
model_path: Optional[str] = None,
# 0.45, not 0.25. Measured on the 50-image bench with v34: dropping the
# floor's junk costs little and buys a visibly cleaner page.
# conf random boxes off-position wrong class recall
# 0.25 11.5% 10.4% 3.8% 79.9%
# 0.45 8.5% 9.0% 3.1% 74.4%
# A quarter of the spurious boxes disappear for 5.5 points of recall. That
# trade favours re-encoding: a false box RECOLOURS a region that should not
# be touched and the user sees it, while a missed box only leaves a region
# alone. See Kaggle/bench/WHY_THE_BOXES_ARE_WRONG.md.
conf_threshold: float = 0.45,
soft_edge_radius: float = 3.0,
imgsz: Optional[int] = None,
require_red_green: bool = True,
clusters_per_roi: Optional[int] = None,
) -> Tuple[np.ndarray, np.ndarray, List[str], List[Dict]]:
"""
Detect ROIs with YOLOv8, run FCM within each region, then return
the combined (centers, memberships) for conflict detection and
LCH re-encoding.
Flow per detected region
------------------------
mask β†’ extract ROI pixels β†’ FCM (k sub-clusters)
β†’ scale memberships by mask weight
β†’ merge into global (N, K_total) membership matrix
Background pixels (outside all ROIs) get a single center = their
mean CIELAB color and are not re-encoded unless they conflict.
Parameters
----------
img_uint8 : (H, W, 3) uint8 RGB
img_lab : (H, W, 3) float CIELAB
model_path : path to trained .pt weights
conf_threshold : YOLO detection confidence cutoff
soft_edge_radius : Gaussian blur on mask edges for smooth blending
imgsz : YOLO inference input size (None = model default, 640)
clusters_per_roi : FCM sub-clusters per detected region (None = adaptive)
require_red_green : bool
If True, only keep detected ROIs that overlap red or green pixels.
Returns
-------
centers : (K_total, 3) mean CIELAB per sub-cluster
memberships : (N, K_total) per-pixel weights, rows sum to 1
class_names : list[str] label per sub-cluster
yolo_boxes : list[dict] YOLO detections in img_uint8 coordinates
"""
from .fcm import run_fcm, _compute_memberships
from .auto_clusters import estimate_n_clusters
h, w = img_uint8.shape[:2]
n_pixels = h * w
lab_flat = img_lab.reshape(n_pixels, 3).astype(np.float64)
import time as _time
_t_masks = _time.perf_counter()
masks_list, raw_names, yolo_boxes, (red_hsv, green_hsv) = _get_yolo_masks(
img_uint8, model_path, conf_threshold, soft_edge_radius, imgsz
)
print(f"[ACRA] seg: yolo+masks ({len(masks_list) - 1} rois) {_time.perf_counter() - _t_masks:.2f}s")
# Optional filter: only keep ROIs that overlap red/green pixels.
if require_red_green and len(masks_list) > 1:
red_f, green_f = _filter_small_regions(red_hsv, green_hsv, 0.003)
keep_masks: List[np.ndarray] = [np.ones((h, w), dtype=np.float32)]
keep_names: List[str] = ["background"]
kept_any = False
for mask_f, name in zip(masks_list[1:], raw_names[1:]):
overlap = (mask_f * (red_f + green_f)).sum()
overlap_frac = overlap / max(mask_f.sum(), 1e-6)
if overlap_frac >= 0.002:
keep_masks.append(mask_f)
keep_names.append(name)
kept_any = True
if not kept_any:
keep_masks = [m.copy() for m in masks_list]
keep_names = list(raw_names)
# Rebuild background so skipped ROIs are treated as background
if len(keep_masks) > 1:
background = np.ones((h, w), dtype=np.float32)
for mask_f in keep_masks[1:]:
background -= mask_f
keep_masks[0] = np.clip(background, 0.0, 1.0)
masks_list, raw_names = keep_masks, keep_names
def _process_roi(mask_f, name):
mask_flat = mask_f.reshape(n_pixels).astype(np.float64)
total_w = mask_flat.sum()
region_center = (
(lab_flat * mask_flat[:, None]).sum(axis=0) / max(total_w, 1e-6)
)
pixel_idx = np.where(mask_flat > 0.15)[0]
if len(pixel_idx) == 0:
return region_center[None], mask_flat[:, None], [name]
# Split ROI pixels into chromatic (the colored cell) and neutral
# (black/white text, ink, paper). The split only decides which pixels
# define each cluster *center* - text pixels must not contaminate the
# colored center, and neutral centers stay near-zero-chroma so
# reencoding._is_neutral() pins them. Per-pixel memberships are still
# computed softly against ALL centers further below, so the text/
# background transition blends smoothly (no hard chroma-threshold step).
roi_lab = lab_flat[pixel_idx]
roi_chroma = np.hypot(roi_lab[:, 1], roi_lab[:, 2])
chrom_sel = roi_chroma >= _NEUTRAL_CHROMA
chrom_idx = pixel_idx[chrom_sel]
neut_idx = pixel_idx[~chrom_sel]
centers_out: List[np.ndarray] = []
names_out: List[str] = []
# ── Chromatic sub-clusters - centers from chromatic pixels only ──────
if clusters_per_roi is None:
chrom_pixels = lab_flat[chrom_idx] if len(chrom_idx) else lab_flat[:0]
target_k = estimate_n_clusters(chrom_pixels, k_min=2, k_max=4)
else:
target_k = clusters_per_roi
min_for_fcm = max(target_k * 10, 30)
if len(chrom_idx) >= min_for_fcm:
k = max(2, min(target_k, len(chrom_idx) // 10))
try:
roi_centers, _ = run_fcm(lab_flat[chrom_idx], n_clusters=k)
# Collapse near-identical sub-clusters so a flat colour cell
# re-encodes as one uniform shift, not a rough patchwork.
roi_centers = _merge_close_centers(roi_centers)
centers_out.append(roi_centers)
names_out.extend(
f"{name} #{j + 1}" for j in range(len(roi_centers))
)
except Exception:
# FCM failed - fall back to one mean center over all pixels.
chrom_idx = pixel_idx
neut_idx = pixel_idx[:0]
if not centers_out and len(chrom_idx):
centers_out.append(lab_flat[chrom_idx].mean(axis=0)[None])
names_out.append(name)
# ── Neutral clusters - text/ink (dark) and paper (light), kept apart ─
if len(neut_idx):
neut_L = lab_flat[neut_idx, 0]
for sel, suffix in ((neut_L < 55.0, "text"), (neut_L >= 55.0, "paper")):
grp = neut_idx[sel]
if not len(grp):
continue
centers_out.append(lab_flat[grp].mean(axis=0)[None])
names_out.append(f"{name} ({suffix})")
if not centers_out:
return region_center[None], mask_flat[:, None], [name]
roi_centers_all = np.vstack(centers_out)
# Soft memberships for EVERY ROI pixel against ALL centers. This is what
# keeps re-encoding smooth: edge/anti-aliased pixels blend gradually
# instead of snapping at the chromatic/neutral boundary.
k_total = roi_centers_all.shape[0]
full_W = np.zeros((n_pixels, k_total), dtype=np.float64)
CHUNK = 50000
for s in range(0, len(pixel_idx), CHUNK):
idx = pixel_idx[s:s + CHUNK]
full_W[idx] = (
_compute_memberships(lab_flat[idx], roi_centers_all, m=2.0)
* mask_flat[idx, np.newaxis]
)
return roi_centers_all, full_W, names_out
all_centers: List[np.ndarray] = []
all_weights: List[np.ndarray] = []
final_names: List[str] = []
# Background is always index 0 - single cluster, no FCM
bg_mask_flat = masks_list[0].reshape(n_pixels).astype(np.float64)
bg_center = (lab_flat * bg_mask_flat[:, None]).sum(axis=0) / max(bg_mask_flat.sum(), 1e-6)
all_centers.append(bg_center[None])
all_weights.append(bg_mask_flat[:, None])
final_names.append(raw_names[0])
# ROIs: each FCM run is independent - process in parallel
_t_fcm = _time.perf_counter()
roi_pairs = list(zip(masks_list[1:], raw_names[1:]))
if roi_pairs:
with ThreadPoolExecutor(max_workers=2) as _ex:
futures = [_ex.submit(_process_roi, mf, nm) for mf, nm in roi_pairs]
for fut in futures:
roi_centers, roi_W, roi_names = fut.result()
all_centers.append(roi_centers)
all_weights.append(roi_W)
final_names.extend(roi_names)
print(f"[ACRA] seg: fcm ({len(roi_pairs)} rois) {_time.perf_counter() - _t_fcm:.2f}s")
# Combine all regions into flat arrays
centers = np.vstack(all_centers) # (K_total, 3)
W = np.hstack(all_weights).astype(np.float32) # (N, K_total)
# Normalize rows so every pixel's weights sum to 1
row_sum = W.sum(axis=1, keepdims=True).clip(min=1e-6)
W /= row_sum
return centers, W, final_names, yolo_boxes