"""Match a photo of a wall to its drawing, and measure the wall from the drawing's own scale. A photo (texture, light, clutter) and an elevation drawing (line art) share almost no pixels, so feature matching fails. They do share STRUCTURE: the layout of the openings. The model's mask gives each window and door in the photo as a quad; the drawing gives the same openings in millimetres (a JSON export from CAD/BIM, or a raster elevation with a stated mm-per-pixel). One opening of the same kind in both fixes a plane homography (four corners); every candidate is scored by how many OTHER openings it also lands on, then refined on all matched corners. The result is a homography photo-pixel -> drawing-mm. What that buys: * **Metric scale without assumptions.** ``rectify`` warps the photo's labels into the drawing plane at a fixed mm/px, so painted, unpainted, window and unseen areas are counted in true square millimetres. predict.py's default ("floor-to-ceiling assumed 8 ft") is not needed. * **Where the photo sits** on the wall (its footprint on the drawing), so views of one wall can be filed by position. * **As-built vs drawing:** openings seen in the photo that the drawing lacks, and drawn openings the photo should show but does not. Honest limits, enforced in the result rather than the prose: * A layout that repeats (a row of identical windows) matches several places equally. When the best match is not clearly better than a different one, ``ambiguous`` is True, ``usable_for_scale`` is False and the candidates are listed, never one silently picked. * One opening cannot be cross-checked: confidence is ``low`` and the scale is not offered. * Only planar, roughly frontal walls (the plane homography). Matching a photo to a floor plan is a different problem and is not done here. Pure numpy + cv2: no model is needed to run it (the labels come from predict.py / the model). """ from __future__ import annotations import json import math import cv2 import numpy as np from scipy import ndimage DOOR, WINDOW = 7, 8 # label_schema classes KIND_OF = {DOOR: "door", WINDOW: "window"} MIN_IOU = 0.5 MM2_PER_FT2 = 304.8 ** 2 def order_quad(pts): """Corners as TL, TR, BR, BL for a y-down image.""" pts = np.asarray(pts, np.float32).reshape(-1, 2) s, d = pts.sum(1), pts[:, 0] - pts[:, 1] return np.float32([pts[s.argmin()], pts[d.argmax()], pts[s.argmax()], pts[d.argmin()]]) def _quad_of(component_mask): cnts, _ = cv2.findContours(component_mask.astype(np.uint8), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) if not cnts: return None c = max(cnts, key=cv2.contourArea) hull = cv2.convexHull(c) approx = cv2.approxPolyDP(hull, 0.04 * cv2.arcLength(hull, True), True) pts = approx.reshape(-1, 2) if len(approx) == 4 else cv2.boxPoints(cv2.minAreaRect(hull)) return order_quad(pts) def openings_from_mask(labels, min_area_frac: float = 0.0008): """Windows and doors in a predicted semantic mask, as [{kind, quad(px), area_px}]; specks below ``min_area_frac`` are ignored.""" labels = np.asarray(labels) out = [] for cls, kind in KIND_OF.items(): comp, n = ndimage.label(labels == cls) for i in range(1, n + 1): m = comp == i if m.sum() < min_area_frac * labels.size: continue q = _quad_of(m) if q is not None: out.append({"kind": kind, "quad": q, "area_px": int(m.sum())}) return out def drawing_from_json(spec): """{"wall": {"width_mm", "height_mm"}, "openings": [{"kind", "x_mm", "y_mm", "w_mm", "h_mm"}]} (x,y from the wall's bottom-left) -> (wall_w_mm, wall_h_mm, openings with quads in a y-DOWN mm frame whose origin is the wall's top-left).""" spec = json.loads(spec) if isinstance(spec, str) else spec W, H = float(spec["wall"]["width_mm"]), float(spec["wall"]["height_mm"]) ops = [] for o in spec["openings"]: x, y, w, h = float(o["x_mm"]), float(o["y_mm"]), float(o["w_mm"]), float(o["h_mm"]) top = H - (y + h) ops.append({"kind": o["kind"], "quad": np.float32([[x, top], [x + w, top], [x + w, top + h], [x, top + h]]), "area_mm2": w * h, "id": o.get("id")}) return W, H, ops def drawing_from_image(gray, mm_per_px: float, min_area_frac=0.0005, max_area_frac=0.4): """Openings of a raster elevation (dark lines on light paper). The outer wall rectangle is the largest closed rectangle; an opening whose bottom edge sits on the wall's base line is a door, any other is a window.""" g = np.asarray(gray) _, ink = cv2.threshold(g, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU) ink = cv2.dilate(ink, np.ones((3, 3), np.uint8)) # The INTERIOR of every closed line shape (a hole of the ink): one contour per drawn rectangle, not one per line edge. cnts, hier = cv2.findContours(ink, cv2.RETR_CCOMP, cv2.CHAIN_APPROX_SIMPLE) rects = [] for c, hh in zip(cnts, hier[0]): if hh[3] == -1: # an outer contour of ink, not a hole continue approx = cv2.approxPolyDP(c, 0.03 * cv2.arcLength(c, True), True) if len(approx) == 4 and cv2.isContourConvex(approx): rects.append((cv2.contourArea(approx), order_quad(approx.reshape(-1, 2)))) if not rects: raise ValueError("no closed rectangles found in the drawing") # The wall = the outer edge of the largest drawn shape, inset by about half a line width. Not the largest interior: a door # standing on the base line shares that line, so the wall's own interior is no longer a clean rectangle. outers = [c for c, hh in zip(cnts, hier[0]) if hh[3] == -1] if not outers: raise ValueError("no drawn wall found") bx, by, bw, bh = cv2.boundingRect(max(outers, key=cv2.contourArea)) half_line = 0.8 * float(cv2.distanceTransform(ink, cv2.DIST_L2, 3).max()) x0, y0, x1, y1 = bx + half_line, by + half_line, bx + bw - half_line, by + bh - half_line W, H = (x1 - x0) * mm_per_px, (y1 - y0) * mm_per_px ops = [] for area, q in rects: if area >= 0.95 * (x1 - x0) * (y1 - y0) or not (min_area_frac * g.size <= area <= max_area_frac * g.size): continue if not (q[:, 0].min() >= x0 - 3 and q[:, 0].max() <= x1 + 3 and q[:, 1].min() >= y0 - 3 and q[:, 1].max() <= y1 + 3): continue door = abs(q[:, 1].max() - y1) < 0.03 * (y1 - y0) + 4 qq = (q - [x0, y0]) * mm_per_px ops.append({"kind": "door" if door else "window", "quad": np.float32(qq), "area_mm2": float(area * mm_per_px ** 2)}) return float(W), float(H), ops def _iou(a, b): inter, _ = cv2.intersectConvexConvex(np.float32(a), np.float32(b)) if inter <= 0: return 0.0 union = cv2.contourArea(np.float32(a)) + cv2.contourArea(np.float32(b)) - inter return float(inter / union) if union > 0 else 0.0 def _project(h, pts): p = cv2.perspectiveTransform(np.float32(pts).reshape(-1, 1, 2), h).reshape(-1, 2) return p def _score(h, photo, drawing): """Greedy one-to-one matches of projected photo openings onto same-kind drawing openings with IoU >= MIN_IOU.""" pairs, used = [], set() cands = [] for i, p in enumerate(photo): pq = _project(h, p["quad"]) if not np.all(np.isfinite(pq)): return [], 0.0 for j, d in enumerate(drawing): if d["kind"] == p["kind"]: v = _iou(pq, d["quad"]) if v >= MIN_IOU: cands.append((v, i, j)) for v, i, j in sorted(cands, reverse=True): if i not in {a for a, _, _ in pairs} and j not in used: pairs.append((i, j, v)) used.add(j) return pairs, (float(np.mean([v for *_, v in pairs])) if pairs else 0.0) def _inside_fraction(h, shape, wall_w, wall_h): fh, fw = shape foot = _project(h, [[0, 0], [fw, 0], [fw, fh], [0, fh]]) poly = np.float32(foot) wall = np.float32([[0, 0], [wall_w, 0], [wall_w, wall_h], [0, wall_h]]) inter, _ = cv2.intersectConvexConvex(poly, wall) area = abs(cv2.contourArea(poly)) return float(inter / area) if area > 1e-9 else 0.0, foot def _sane(h, shape, wall_w, wall_h): """A placement a photo of this wall could really have: not mirrored or folded, some of the drawn wall inside the frame, and a frame footprint of believable size. The frame does NOT have to lie inside the wall: real photos include floor, ceiling and the next wall.""" if h is None or not np.all(np.isfinite(h)) or np.linalg.det(h[:2, :2]) <= 0: return False fh, fw = shape foot = _project(h, [[0, 0], [fw, 0], [fw, fh], [0, fh]]) if not np.all(np.isfinite(foot)): return False x, y = foot[:, 0], foot[:, 1] area = 0.5 * (np.dot(x, np.roll(y, -1)) - np.dot(y, np.roll(x, -1))) if area <= 0 or area > 60 * wall_w * wall_h: return False wall = np.float32([[0, 0], [wall_w, 0], [wall_w, wall_h], [0, wall_h]]) inter, _ = cv2.intersectConvexConvex(np.float32(foot), wall) return inter >= 0.03 * wall_w * wall_h def match(photo, drawing, photo_shape, ambiguity_margin: float = 0.05): """Best homography photo-px -> drawing-mm, or an explicit reason there is none. ``photo``: openings_from_mask(...). ``drawing``: (wall_w_mm, wall_h_mm, openings) from drawing_from_json / drawing_from_image. """ wall_w, wall_h, dops = drawing if not photo: return {"matched": False, "reason": "no windows or doors found in the photo's mask, so there is nothing to anchor to the drawing"} if not dops: return {"matched": False, "reason": "the drawing has no openings"} results = {} for p in photo: for d in dops: if p["kind"] != d["kind"]: continue try: h0 = cv2.getPerspectiveTransform(np.float32(p["quad"]), np.float32(d["quad"])) except cv2.error: continue if not _sane(h0, photo_shape, wall_w, wall_h): continue pairs, _ = _score(h0, photo, dops) if len(pairs) >= 2: # refine on every matched corner pair src = np.vstack([photo[i]["quad"] for i, _, _ in pairs]) dst = np.vstack([dops[j]["quad"] for _, j, _ in pairs]) h1, _ = cv2.findHomography(src, dst, 0) if h1 is not None and _sane(h1, photo_shape, wall_w, wall_h): p2, _ = _score(h1, photo, dops) if len(p2) >= len(pairs): h0, pairs = h1, p2 key = frozenset((i, j) for i, j, _ in pairs) mean = float(np.mean([v for *_, v in pairs])) if pairs else 0.0 if key not in results or (len(pairs), mean) > (len(results[key]["pairs"]), results[key]["iou"]): results[key] = {"h": h0, "pairs": pairs, "iou": mean} if not results: return {"matched": False, "reason": "no opening of the same kind could be placed inside the drawn wall"} ranked = sorted(results.values(), key=lambda r: (len(r["pairs"]), r["iou"]), reverse=True) best = ranked[0] rivals = [r for r in ranked[1:] if len(r["pairs"]) >= len(best["pairs"]) and r["iou"] >= best["iou"] - ambiguity_margin] n = len(best["pairs"]) ambiguous = bool(rivals) confidence = "low" if (n < 2 or ambiguous) else ("high" if n >= 3 and best["iou"] >= 0.7 else "medium") h = best["h"] fh, fw = photo_shape cx = _project(h, [[fw / 2, fh / 2], [fw / 2 + 1, fh / 2], [fw / 2, fh / 2 + 1]]) mm_per_px = float(math.sqrt(abs(np.linalg.det(np.column_stack([cx[1] - cx[0], cx[2] - cx[0]]))))) _, foot = _inside_fraction(h, photo_shape, wall_w, wall_h) matched_photo = {i for i, _, _ in best["pairs"]} matched_draw = {j for _, j, _ in best["pairs"]} in_view = [j for j, d in enumerate(dops) if _inside_fraction_quad(h, photo_shape, d["quad"])] return { "matched": True, "confidence": confidence, "ambiguous": ambiguous, "usable_for_scale": confidence in ("medium", "high"), "homography_px_to_mm": h.tolist(), "mm_per_px_at_centre": round(mm_per_px, 3), "matched_openings": [{"photo": i, "drawing": j, "kind": photo[i]["kind"], "iou": round(v, 3), "drawing_id": dops[j].get("id")} for i, j, v in best["pairs"]], "photo_footprint_mm": np.round(foot, 1).tolist(), "in_photo_not_in_drawing": [i for i in range(len(photo)) if i not in matched_photo], "in_drawing_not_in_photo": [j for j in in_view if j not in matched_draw], "candidates": [{"matches": len(r["pairs"]), "mean_iou": round(r["iou"], 3)} for r in ranked[:3]] if ambiguous else [], "wall_mm": [wall_w, wall_h], "note": ("several placements fit equally well (a repeated layout): the scale is not offered; add a distinguishing feature " "(a corner, a door) to the photo" if ambiguous else "one opening cannot be cross-checked: confidence is low" if n < 2 else None), } def _inside_fraction_quad(h, shape, quad): """True when the drawn opening lies (mostly) inside the photo's footprint, i.e. the photo should show it.""" fh, fw = shape inv = np.linalg.inv(h) px = _project(inv, quad) frame = np.float32([[0, 0], [fw, 0], [fw, fh], [0, fh]]) inter, _ = cv2.intersectConvexConvex(np.float32(px), frame) area = abs(cv2.contourArea(np.float32(px))) return area > 1e-9 and inter / area > 0.8 def rectify(labels, m, mm_per_px: float = 10.0): """Warp the photo's labels into the drawing plane at ``mm_per_px``: a (wall_h/mm_per_px, wall_w/mm_per_px) map over the whole drawn wall; pixels the photo did not see are 255. Needs a match with usable_for_scale.""" if not (m.get("matched") and m["usable_for_scale"]): raise ValueError("cannot rectify: " + (m.get("reason") or m.get("note") or "match is not usable for scale")) wall_w, wall_h = m["wall_mm"] w, h = int(round(wall_w / mm_per_px)), int(round(wall_h / mm_per_px)) s = np.diag([1.0 / mm_per_px, 1.0 / mm_per_px, 1.0]) out = cv2.warpPerspective(np.asarray(labels).astype(np.uint8), s @ np.array(m["homography_px_to_mm"]), (w, h), flags=cv2.INTER_NEAREST, borderValue=255) return out def areas_ft2(rectified, mm_per_px: float = 10.0): """True areas per class (sq ft) from a rectified map, plus the share of the drawn wall the photo covered.""" px_ft2 = mm_per_px ** 2 / MM2_PER_FT2 seen = rectified != 255 return {"seen_fraction": round(float(seen.mean()), 3), "by_class_ft2": {int(c): round(float((rectified == c).sum() * px_ft2), 2) for c in np.unique(rectified) if c != 255}, "wall_total_ft2": round(float(rectified.size * px_ft2), 1)} def assign(walls, drawings, ambiguity_margin: float = 0.05): """Which drawing is each wall? ``walls``: [(openings, label_shape)] (one per wall group, from a photo or a wall mosaic); ``drawings``: [(name, (wall_w_mm, wall_h_mm, openings))]. Every wall is matched to every drawing and the walls are assigned one-to-one, strongest evidence first. Per wall the result says which drawing, the match itself, and the alternatives it beat. A wall is ``ambiguous_between_drawings`` when a different drawing fits it equally well (two elevations with the same openings), and then no drawing is assigned: the operator picks. A drawing already taken by a stronger wall is not reused.""" scored = [] # (n_matches, mean_iou, wall, drawing, match) table = {w: [] for w in range(len(walls))} for w, (ops, shape) in enumerate(walls): for d, (name, drawing) in enumerate(drawings): m = match(ops, drawing, shape, ambiguity_margin) n = len(m.get("matched_openings", [])) if m.get("matched") else 0 iou = float(np.mean([o["iou"] for o in m["matched_openings"]])) if n else 0.0 table[w].append((n, iou, d, m)) if n: scored.append((n, iou, w, d, m)) out = [{"drawing": None, "match": None, "alternatives": [], "ambiguous_between_drawings": False, "reason": "no drawing fits this wall's openings"} for _ in walls] for w in table: ranked = sorted(table[w], key=lambda t: (t[0], t[1]), reverse=True) out[w]["alternatives"] = [{"drawing": drawings[d][0], "matches": n, "mean_iou": round(i, 3)} for n, i, d, _ in ranked if n] if len(ranked) >= 2 and ranked[0][0] and ranked[1][0] >= ranked[0][0] and ranked[1][1] >= ranked[0][1] - ambiguity_margin: out[w].update(ambiguous_between_drawings=True, reason=f"{drawings[ranked[0][2]][0]} and {drawings[ranked[1][2]][0]} fit equally well: choose which sheet this is") taken = set() for n, iou, w, d, m in sorted(scored, key=lambda t: (t[0], t[1]), reverse=True): if out[w]["drawing"] is not None or out[w]["ambiguous_between_drawings"] or d in taken: continue out[w].update(drawing=drawings[d][0], match=m, reason=None) taken.add(d) return out