Image Segmentation
PyTorch
English
painting-vision-robotics-kit
semantic-segmentation
robotics
edge-ai
construction-ai
autonomous-painting
wall-painting-robot
paint-coverage-estimation
building-facade
drywall
skirting-detection
window-detection
lidar
depth-validation
segformer
deeplabv3
Eval Results (legacy)
Update code: real-machine G-code adapter, commissioning gate, machine_job runner, pump watchdog firmware, twin completion gate, realism and masking modules (card and weights unchanged)
4eecd03 verified Download drawing_match.py from constructelligence/painting-vision-robotics-kit: direct link, hf CLI and curl.
- Browser
- Download file 17.3 kB
-
https://huggingface.co/constructelligence/painting-vision-robotics-kit/resolve/main/drawing_match.py
- Command line
-
hf download hf://constructelligence/painting-vision-robotics-kit/drawing_match.py
-
curl -L -o drawing_match.py https://huggingface.co/constructelligence/painting-vision-robotics-kit/resolve/main/drawing_match.py
17.3 kB
| """Match a photo of a wall to its drawing, and measure the wall from the drawing's own scale. | |
| A photo (texture, light, clutter) and an elevation drawing (line art) share almost no pixels, so feature matching fails. They do share | |
| STRUCTURE: the layout of the openings. The model's mask gives each window and door in the photo as a quad; the drawing gives the same | |
| openings in millimetres (a JSON export from CAD/BIM, or a raster elevation with a stated mm-per-pixel). One opening of the same kind | |
| in both fixes a plane homography (four corners); every candidate is scored by how many OTHER openings it also lands on, then refined | |
| on all matched corners. The result is a homography photo-pixel -> drawing-mm. | |
| What that buys: | |
| * **Metric scale without assumptions.** ``rectify`` warps the photo's labels into the drawing plane at a fixed mm/px, so painted, | |
| unpainted, window and unseen areas are counted in true square millimetres. predict.py's default ("floor-to-ceiling assumed 8 ft") is | |
| not needed. | |
| * **Where the photo sits** on the wall (its footprint on the drawing), so views of one wall can be filed by position. | |
| * **As-built vs drawing:** openings seen in the photo that the drawing lacks, and drawn openings the photo should show but does not. | |
| Honest limits, enforced in the result rather than the prose: | |
| * A layout that repeats (a row of identical windows) matches several places equally. When the best match is not clearly better than a | |
| different one, ``ambiguous`` is True, ``usable_for_scale`` is False and the candidates are listed, never one silently picked. | |
| * One opening cannot be cross-checked: confidence is ``low`` and the scale is not offered. | |
| * Only planar, roughly frontal walls (the plane homography). Matching a photo to a floor plan is a different problem and is not done here. | |
| Pure numpy + cv2: no model is needed to run it (the labels come from predict.py / the model). | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import math | |
| import cv2 | |
| import numpy as np | |
| from scipy import ndimage | |
| DOOR, WINDOW = 7, 8 # label_schema classes | |
| KIND_OF = {DOOR: "door", WINDOW: "window"} | |
| MIN_IOU = 0.5 | |
| MM2_PER_FT2 = 304.8 ** 2 | |
| def order_quad(pts): | |
| """Corners as TL, TR, BR, BL for a y-down image.""" | |
| pts = np.asarray(pts, np.float32).reshape(-1, 2) | |
| s, d = pts.sum(1), pts[:, 0] - pts[:, 1] | |
| return np.float32([pts[s.argmin()], pts[d.argmax()], pts[s.argmax()], pts[d.argmin()]]) | |
| def _quad_of(component_mask): | |
| cnts, _ = cv2.findContours(component_mask.astype(np.uint8), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) | |
| if not cnts: | |
| return None | |
| c = max(cnts, key=cv2.contourArea) | |
| hull = cv2.convexHull(c) | |
| approx = cv2.approxPolyDP(hull, 0.04 * cv2.arcLength(hull, True), True) | |
| pts = approx.reshape(-1, 2) if len(approx) == 4 else cv2.boxPoints(cv2.minAreaRect(hull)) | |
| return order_quad(pts) | |
| def openings_from_mask(labels, min_area_frac: float = 0.0008): | |
| """Windows and doors in a predicted semantic mask, as [{kind, quad(px), area_px}]; specks below ``min_area_frac`` are ignored.""" | |
| labels = np.asarray(labels) | |
| out = [] | |
| for cls, kind in KIND_OF.items(): | |
| comp, n = ndimage.label(labels == cls) | |
| for i in range(1, n + 1): | |
| m = comp == i | |
| if m.sum() < min_area_frac * labels.size: | |
| continue | |
| q = _quad_of(m) | |
| if q is not None: | |
| out.append({"kind": kind, "quad": q, "area_px": int(m.sum())}) | |
| return out | |
| def drawing_from_json(spec): | |
| """{"wall": {"width_mm", "height_mm"}, "openings": [{"kind", "x_mm", "y_mm", "w_mm", "h_mm"}]} (x,y from the wall's bottom-left) | |
| -> (wall_w_mm, wall_h_mm, openings with quads in a y-DOWN mm frame whose origin is the wall's top-left).""" | |
| spec = json.loads(spec) if isinstance(spec, str) else spec | |
| W, H = float(spec["wall"]["width_mm"]), float(spec["wall"]["height_mm"]) | |
| ops = [] | |
| for o in spec["openings"]: | |
| x, y, w, h = float(o["x_mm"]), float(o["y_mm"]), float(o["w_mm"]), float(o["h_mm"]) | |
| top = H - (y + h) | |
| ops.append({"kind": o["kind"], "quad": np.float32([[x, top], [x + w, top], [x + w, top + h], [x, top + h]]), | |
| "area_mm2": w * h, "id": o.get("id")}) | |
| return W, H, ops | |
| def drawing_from_image(gray, mm_per_px: float, min_area_frac=0.0005, max_area_frac=0.4): | |
| """Openings of a raster elevation (dark lines on light paper). The outer wall rectangle is the largest closed rectangle; an | |
| opening whose bottom edge sits on the wall's base line is a door, any other is a window.""" | |
| g = np.asarray(gray) | |
| _, ink = cv2.threshold(g, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU) | |
| ink = cv2.dilate(ink, np.ones((3, 3), np.uint8)) | |
| # The INTERIOR of every closed line shape (a hole of the ink): one contour per drawn rectangle, not one per line edge. | |
| cnts, hier = cv2.findContours(ink, cv2.RETR_CCOMP, cv2.CHAIN_APPROX_SIMPLE) | |
| rects = [] | |
| for c, hh in zip(cnts, hier[0]): | |
| if hh[3] == -1: # an outer contour of ink, not a hole | |
| continue | |
| approx = cv2.approxPolyDP(c, 0.03 * cv2.arcLength(c, True), True) | |
| if len(approx) == 4 and cv2.isContourConvex(approx): | |
| rects.append((cv2.contourArea(approx), order_quad(approx.reshape(-1, 2)))) | |
| if not rects: | |
| raise ValueError("no closed rectangles found in the drawing") | |
| # The wall = the outer edge of the largest drawn shape, inset by about half a line width. Not the largest interior: a door | |
| # standing on the base line shares that line, so the wall's own interior is no longer a clean rectangle. | |
| outers = [c for c, hh in zip(cnts, hier[0]) if hh[3] == -1] | |
| if not outers: | |
| raise ValueError("no drawn wall found") | |
| bx, by, bw, bh = cv2.boundingRect(max(outers, key=cv2.contourArea)) | |
| half_line = 0.8 * float(cv2.distanceTransform(ink, cv2.DIST_L2, 3).max()) | |
| x0, y0, x1, y1 = bx + half_line, by + half_line, bx + bw - half_line, by + bh - half_line | |
| W, H = (x1 - x0) * mm_per_px, (y1 - y0) * mm_per_px | |
| ops = [] | |
| for area, q in rects: | |
| if area >= 0.95 * (x1 - x0) * (y1 - y0) or not (min_area_frac * g.size <= area <= max_area_frac * g.size): | |
| continue | |
| if not (q[:, 0].min() >= x0 - 3 and q[:, 0].max() <= x1 + 3 and q[:, 1].min() >= y0 - 3 and q[:, 1].max() <= y1 + 3): | |
| continue | |
| door = abs(q[:, 1].max() - y1) < 0.03 * (y1 - y0) + 4 | |
| qq = (q - [x0, y0]) * mm_per_px | |
| ops.append({"kind": "door" if door else "window", "quad": np.float32(qq), "area_mm2": float(area * mm_per_px ** 2)}) | |
| return float(W), float(H), ops | |
| def _iou(a, b): | |
| inter, _ = cv2.intersectConvexConvex(np.float32(a), np.float32(b)) | |
| if inter <= 0: | |
| return 0.0 | |
| union = cv2.contourArea(np.float32(a)) + cv2.contourArea(np.float32(b)) - inter | |
| return float(inter / union) if union > 0 else 0.0 | |
| def _project(h, pts): | |
| p = cv2.perspectiveTransform(np.float32(pts).reshape(-1, 1, 2), h).reshape(-1, 2) | |
| return p | |
| def _score(h, photo, drawing): | |
| """Greedy one-to-one matches of projected photo openings onto same-kind drawing openings with IoU >= MIN_IOU.""" | |
| pairs, used = [], set() | |
| cands = [] | |
| for i, p in enumerate(photo): | |
| pq = _project(h, p["quad"]) | |
| if not np.all(np.isfinite(pq)): | |
| return [], 0.0 | |
| for j, d in enumerate(drawing): | |
| if d["kind"] == p["kind"]: | |
| v = _iou(pq, d["quad"]) | |
| if v >= MIN_IOU: | |
| cands.append((v, i, j)) | |
| for v, i, j in sorted(cands, reverse=True): | |
| if i not in {a for a, _, _ in pairs} and j not in used: | |
| pairs.append((i, j, v)) | |
| used.add(j) | |
| return pairs, (float(np.mean([v for *_, v in pairs])) if pairs else 0.0) | |
| def _inside_fraction(h, shape, wall_w, wall_h): | |
| fh, fw = shape | |
| foot = _project(h, [[0, 0], [fw, 0], [fw, fh], [0, fh]]) | |
| poly = np.float32(foot) | |
| wall = np.float32([[0, 0], [wall_w, 0], [wall_w, wall_h], [0, wall_h]]) | |
| inter, _ = cv2.intersectConvexConvex(poly, wall) | |
| area = abs(cv2.contourArea(poly)) | |
| return float(inter / area) if area > 1e-9 else 0.0, foot | |
| def _sane(h, shape, wall_w, wall_h): | |
| """A placement a photo of this wall could really have: not mirrored or folded, some of the drawn wall inside the frame, and a frame | |
| footprint of believable size. The frame does NOT have to lie inside the wall: real photos include floor, ceiling and the next wall.""" | |
| if h is None or not np.all(np.isfinite(h)) or np.linalg.det(h[:2, :2]) <= 0: | |
| return False | |
| fh, fw = shape | |
| foot = _project(h, [[0, 0], [fw, 0], [fw, fh], [0, fh]]) | |
| if not np.all(np.isfinite(foot)): | |
| return False | |
| x, y = foot[:, 0], foot[:, 1] | |
| area = 0.5 * (np.dot(x, np.roll(y, -1)) - np.dot(y, np.roll(x, -1))) | |
| if area <= 0 or area > 60 * wall_w * wall_h: | |
| return False | |
| wall = np.float32([[0, 0], [wall_w, 0], [wall_w, wall_h], [0, wall_h]]) | |
| inter, _ = cv2.intersectConvexConvex(np.float32(foot), wall) | |
| return inter >= 0.03 * wall_w * wall_h | |
| def match(photo, drawing, photo_shape, ambiguity_margin: float = 0.05): | |
| """Best homography photo-px -> drawing-mm, or an explicit reason there is none. | |
| ``photo``: openings_from_mask(...). ``drawing``: (wall_w_mm, wall_h_mm, openings) from drawing_from_json / drawing_from_image. | |
| """ | |
| wall_w, wall_h, dops = drawing | |
| if not photo: | |
| return {"matched": False, "reason": "no windows or doors found in the photo's mask, so there is nothing to anchor to the drawing"} | |
| if not dops: | |
| return {"matched": False, "reason": "the drawing has no openings"} | |
| results = {} | |
| for p in photo: | |
| for d in dops: | |
| if p["kind"] != d["kind"]: | |
| continue | |
| try: | |
| h0 = cv2.getPerspectiveTransform(np.float32(p["quad"]), np.float32(d["quad"])) | |
| except cv2.error: | |
| continue | |
| if not _sane(h0, photo_shape, wall_w, wall_h): | |
| continue | |
| pairs, _ = _score(h0, photo, dops) | |
| if len(pairs) >= 2: # refine on every matched corner pair | |
| src = np.vstack([photo[i]["quad"] for i, _, _ in pairs]) | |
| dst = np.vstack([dops[j]["quad"] for _, j, _ in pairs]) | |
| h1, _ = cv2.findHomography(src, dst, 0) | |
| if h1 is not None and _sane(h1, photo_shape, wall_w, wall_h): | |
| p2, _ = _score(h1, photo, dops) | |
| if len(p2) >= len(pairs): | |
| h0, pairs = h1, p2 | |
| key = frozenset((i, j) for i, j, _ in pairs) | |
| mean = float(np.mean([v for *_, v in pairs])) if pairs else 0.0 | |
| if key not in results or (len(pairs), mean) > (len(results[key]["pairs"]), results[key]["iou"]): | |
| results[key] = {"h": h0, "pairs": pairs, "iou": mean} | |
| if not results: | |
| return {"matched": False, "reason": "no opening of the same kind could be placed inside the drawn wall"} | |
| ranked = sorted(results.values(), key=lambda r: (len(r["pairs"]), r["iou"]), reverse=True) | |
| best = ranked[0] | |
| rivals = [r for r in ranked[1:] if len(r["pairs"]) >= len(best["pairs"]) and r["iou"] >= best["iou"] - ambiguity_margin] | |
| n = len(best["pairs"]) | |
| ambiguous = bool(rivals) | |
| confidence = "low" if (n < 2 or ambiguous) else ("high" if n >= 3 and best["iou"] >= 0.7 else "medium") | |
| h = best["h"] | |
| fh, fw = photo_shape | |
| cx = _project(h, [[fw / 2, fh / 2], [fw / 2 + 1, fh / 2], [fw / 2, fh / 2 + 1]]) | |
| mm_per_px = float(math.sqrt(abs(np.linalg.det(np.column_stack([cx[1] - cx[0], cx[2] - cx[0]]))))) | |
| _, foot = _inside_fraction(h, photo_shape, wall_w, wall_h) | |
| matched_photo = {i for i, _, _ in best["pairs"]} | |
| matched_draw = {j for _, j, _ in best["pairs"]} | |
| in_view = [j for j, d in enumerate(dops) if _inside_fraction_quad(h, photo_shape, d["quad"])] | |
| return { | |
| "matched": True, "confidence": confidence, "ambiguous": ambiguous, "usable_for_scale": confidence in ("medium", "high"), | |
| "homography_px_to_mm": h.tolist(), "mm_per_px_at_centre": round(mm_per_px, 3), | |
| "matched_openings": [{"photo": i, "drawing": j, "kind": photo[i]["kind"], "iou": round(v, 3), "drawing_id": dops[j].get("id")} | |
| for i, j, v in best["pairs"]], | |
| "photo_footprint_mm": np.round(foot, 1).tolist(), | |
| "in_photo_not_in_drawing": [i for i in range(len(photo)) if i not in matched_photo], | |
| "in_drawing_not_in_photo": [j for j in in_view if j not in matched_draw], | |
| "candidates": [{"matches": len(r["pairs"]), "mean_iou": round(r["iou"], 3)} for r in ranked[:3]] if ambiguous else [], | |
| "wall_mm": [wall_w, wall_h], | |
| "note": ("several placements fit equally well (a repeated layout): the scale is not offered; add a distinguishing feature " | |
| "(a corner, a door) to the photo" if ambiguous else | |
| "one opening cannot be cross-checked: confidence is low" if n < 2 else None), | |
| } | |
| def _inside_fraction_quad(h, shape, quad): | |
| """True when the drawn opening lies (mostly) inside the photo's footprint, i.e. the photo should show it.""" | |
| fh, fw = shape | |
| inv = np.linalg.inv(h) | |
| px = _project(inv, quad) | |
| frame = np.float32([[0, 0], [fw, 0], [fw, fh], [0, fh]]) | |
| inter, _ = cv2.intersectConvexConvex(np.float32(px), frame) | |
| area = abs(cv2.contourArea(np.float32(px))) | |
| return area > 1e-9 and inter / area > 0.8 | |
| def rectify(labels, m, mm_per_px: float = 10.0): | |
| """Warp the photo's labels into the drawing plane at ``mm_per_px``: a (wall_h/mm_per_px, wall_w/mm_per_px) map over the whole | |
| drawn wall; pixels the photo did not see are 255. Needs a match with usable_for_scale.""" | |
| if not (m.get("matched") and m["usable_for_scale"]): | |
| raise ValueError("cannot rectify: " + (m.get("reason") or m.get("note") or "match is not usable for scale")) | |
| wall_w, wall_h = m["wall_mm"] | |
| w, h = int(round(wall_w / mm_per_px)), int(round(wall_h / mm_per_px)) | |
| s = np.diag([1.0 / mm_per_px, 1.0 / mm_per_px, 1.0]) | |
| out = cv2.warpPerspective(np.asarray(labels).astype(np.uint8), s @ np.array(m["homography_px_to_mm"]), (w, h), | |
| flags=cv2.INTER_NEAREST, borderValue=255) | |
| return out | |
| def areas_ft2(rectified, mm_per_px: float = 10.0): | |
| """True areas per class (sq ft) from a rectified map, plus the share of the drawn wall the photo covered.""" | |
| px_ft2 = mm_per_px ** 2 / MM2_PER_FT2 | |
| seen = rectified != 255 | |
| return {"seen_fraction": round(float(seen.mean()), 3), | |
| "by_class_ft2": {int(c): round(float((rectified == c).sum() * px_ft2), 2) for c in np.unique(rectified) if c != 255}, | |
| "wall_total_ft2": round(float(rectified.size * px_ft2), 1)} | |
| def assign(walls, drawings, ambiguity_margin: float = 0.05): | |
| """Which drawing is each wall? ``walls``: [(openings, label_shape)] (one per wall group, from a photo or a wall mosaic); | |
| ``drawings``: [(name, (wall_w_mm, wall_h_mm, openings))]. Every wall is matched to every drawing and the walls are assigned | |
| one-to-one, strongest evidence first. | |
| Per wall the result says which drawing, the match itself, and the alternatives it beat. A wall is ``ambiguous_between_drawings`` | |
| when a different drawing fits it equally well (two elevations with the same openings), and then no drawing is assigned: the | |
| operator picks. A drawing already taken by a stronger wall is not reused.""" | |
| scored = [] # (n_matches, mean_iou, wall, drawing, match) | |
| table = {w: [] for w in range(len(walls))} | |
| for w, (ops, shape) in enumerate(walls): | |
| for d, (name, drawing) in enumerate(drawings): | |
| m = match(ops, drawing, shape, ambiguity_margin) | |
| n = len(m.get("matched_openings", [])) if m.get("matched") else 0 | |
| iou = float(np.mean([o["iou"] for o in m["matched_openings"]])) if n else 0.0 | |
| table[w].append((n, iou, d, m)) | |
| if n: | |
| scored.append((n, iou, w, d, m)) | |
| out = [{"drawing": None, "match": None, "alternatives": [], "ambiguous_between_drawings": False, | |
| "reason": "no drawing fits this wall's openings"} for _ in walls] | |
| for w in table: | |
| ranked = sorted(table[w], key=lambda t: (t[0], t[1]), reverse=True) | |
| out[w]["alternatives"] = [{"drawing": drawings[d][0], "matches": n, "mean_iou": round(i, 3)} for n, i, d, _ in ranked if n] | |
| if len(ranked) >= 2 and ranked[0][0] and ranked[1][0] >= ranked[0][0] and ranked[1][1] >= ranked[0][1] - ambiguity_margin: | |
| out[w].update(ambiguous_between_drawings=True, | |
| reason=f"{drawings[ranked[0][2]][0]} and {drawings[ranked[1][2]][0]} fit equally well: choose which sheet this is") | |
| taken = set() | |
| for n, iou, w, d, m in sorted(scored, key=lambda t: (t[0], t[1]), reverse=True): | |
| if out[w]["drawing"] is not None or out[w]["ambiguous_between_drawings"] or d in taken: | |
| continue | |
| out[w].update(drawing=drawings[d][0], match=m, reason=None) | |
| taken.add(d) | |
| return out | |