"""Material rendering on cached detection results. Segmentation is never re-run from here.""" import json import math import numpy as np from PIL import Image, ImageDraw import torch import torch.nn.functional as F import geometry import imgops from textures import parse_color, sample_mipmapped PREVIEW_SIDE = 1500 LUMA = imgops.LUMA PALETTE = [(40, 44, 52), (66, 135, 245), (245, 166, 35), (120, 200, 120), (200, 90, 160), (90, 200, 200)] # ------------------------------------------------------------------ session / cache class Layer: def __init__(self, arr, base, conf, label, raw_label, band): self.arr, self.base, self.conf = arr, base, conf self.label, self.raw_label, self.band = label, raw_label, band self.H, self.W = arr.shape[:2] def _down_layer(L: Layer, side: int) -> Layer: s = side / max(L.H, L.W) if s >= 1: return L size = (max(8, round(L.H * s)), max(8, round(L.W * s))) nn = lambda a: F.interpolate(torch.from_numpy(a.astype(np.float32))[None, None], size=size, mode="nearest")[0, 0].numpy().astype(np.uint8) return Layer(imgops.area_down(L.arr, size), {k: imgops.area_down(v, size) for k, v in L.base.items()}, imgops.area_down(L.conf, size), nn(L.label), nn(L.raw_label), nn(L.band)) class Session: def __init__(self, det): self.det = det self.full = Layer(det.rgb, det.alpha, det.conf, det.labelmap, det.raw_label, det.band) self.prev = _down_layer(self.full, PREVIEW_SIDE) self._edit = {"full": self._blank(), "prev": self._blank()} self._light, self._geo = {}, {} @staticmethod def _blank(): return {"keys": None, "alpha": None, "version": 0} def layer(self, q): return self.full if q == "full" else self.prev # --- user edits (brush strokes), applied incrementally and cached def alphas(self, q, strokes): L, st = self.layer(q), self._edit[q] keys = [json.dumps(s, sort_keys=True) for s in strokes] if st["keys"] is not None and keys[:len(st["keys"])] == st["keys"] and st["alpha"] is not None: new = keys[len(st["keys"]):] todo = strokes[len(st["keys"]):] else: st["alpha"] = {k: v.copy() for k, v in L.base.items()} new, todo = keys, strokes for s in todo: apply_stroke(st["alpha"], s, L.H, L.W) if st["keys"] != keys: st["version"] += 1 st["keys"] = keys return st["alpha"], st["version"] def geo(self, q, strokes): al, ver = self.alphas(q, strokes) # geometry only needs a small mask, so one cache entry per edit-state is enough key = (self._edit[q]["keys"] and "|".join(self._edit[q]["keys"])) or "" if key not in self._geo: self._geo = {key: geometry.scene_geometry(al["wall"], al["floor"])} return self._geo[key] def lighting(self, q, surface, alpha, ver): key = (q, ver, surface) if key not in self._light: if len(self._light) > 6: self._light.clear() L = self.layer(q) lum = (L.arr @ LUMA).astype(np.float32) r = max(3, min(L.H, L.W) // 85) smooth = imgops.masked_blur(lum, alpha, r) ref = float((smooth * alpha).sum() / max(alpha.sum(), 1.0)) ref = max(ref, 1e-3) self._light[key] = (smooth / ref, (lum - smooth) / ref) return self._light[key] def apply_stroke(al, st, H, W): """Rasterise one brush stroke in its bounding box (fast) and edit the mattes.""" pts = [(x * W, y * H) for x, y in st["pts"]] if not pts: return r = max(1.0, float(st["r"]) * W) hard = float(st.get("hard", 0.7)) op = float(st.get("op", 1.0)) rf = int((1 - hard) * r * 0.6) mg = int(r + 2 * rf + 3) x0 = max(0, int(min(p[0] for p in pts)) - mg) y0 = max(0, int(min(p[1] for p in pts)) - mg) x1 = min(W, int(max(p[0] for p in pts)) + mg + 1) y1 = min(H, int(max(p[1] for p in pts)) + mg + 1) if x1 <= x0 or y1 <= y0: return im = Image.new("L", (x1 - x0, y1 - y0), 0) d = ImageDraw.Draw(im) loc = [(x - x0, y - y0) for x, y in pts] if len(loc) > 1: d.line(loc, fill=255, width=max(1, round(2 * r)), joint="curve") for x, y in loc[:: max(1, len(loc) // 300)] + [loc[-1]]: d.ellipse([x - r, y - r, x + r, y + r], fill=255) m = np.asarray(im, np.float32) / 255.0 m = imgops.box_filter(m, max(1, rf)) if rf else imgops.box_filter(m, 1) if rf > 1: m = imgops.box_filter(m, max(1, rf // 2)) m = np.clip(m * op, 0, 1) sl = (slice(y0, y1), slice(x0, x1)) aw, af = al["wall"][sl], al["floor"][sl] mode = st["m"] if mode == "floor": al["floor"][sl] = af + (1 - af) * m al["wall"][sl] = aw * (1 - m) elif mode == "wall": al["wall"][sl] = aw + (1 - aw) * m al["floor"][sl] = af * (1 - m) else: al["wall"][sl] = aw * (1 - m) al["floor"][sl] = af * (1 - m) # ------------------------------------------------------------------ mapping def map_floor(tex, xs, ys, H, W, p, geo, bottom): scale = max(float(p.get("scale", 0.3)), 0.02) aspect = tex.h / tex.w if p.get("persp", True): if p.get("persp_mode", "auto") == "manual": yh, vpx = float(p.get("horizon", -0.2)) * H, W / 2 else: yh, vpx = geo["floor"]["horizon"] * H, geo["floor"]["vp_x"] * W d = np.maximum(ys - yh, 1.0) db = max(bottom - yh, 1.0) f = float(p.get("depth", 0.9)) * W k = db / (W * scale) u = (xs - vpx) * k / d v = (k * f / d) / aspect fp = np.sqrt((k / d * tex.w) * (k * f / d ** 2 * tex.h / aspect)) else: u, v = xs / (W * scale), ys / (W * scale * aspect) fp = np.full(u.shape, tex.w / (W * scale), np.float32) return _finish(tex, u, v, fp, p) def slider_vp(p): """Manual 'wall angle' slider -> vanishing-point x (image-width units from centre); None = front-facing.""" p = float(p) return None if abs(p) < 0.03 else (0.5 / p) def map_wall(tex, xs, ys, H, W, p, geo): """Planar homography per wall plane: u ~ 1/(x - x_vp), v ~ (y - horizon)/(x - x_vp), normalised so a tile spans `scale` of the image width at the plane's visible centre. vp=None -> front-facing (affine).""" scale = max(float(p.get("scale", 0.25)), 0.02) aspect = tex.h / tex.w manual = p.get("persp_mode", "auto") == "manual" yh = (float(p["horizon"]) if manual and "horizon" in p else geo["floor"]["horizon"]) * H segs = ([{"x0": 0.0, "x1": 1.0, "vp": slider_vp(p.get("wpersp", 0.0))}] if manual else geo["wall"]["segments"]) xn, yn = (xs - W / 2) / W, (ys - yh) / W fx = xs / W u, v, fp = np.empty_like(xn), np.empty_like(xn), np.empty_like(xn) done = np.zeros(xn.shape, bool) fn = 0.9 # focal length in image-width units (same assumption as the floor) for i, s in enumerate(segs): sel = (fx >= s["x0"]) & ((fx < s["x1"]) if i < len(segs) - 1 else (fx <= 1.0 + 1e-6)) & ~done if not sel.any(): continue done |= sel vp = s["vp"] if vp is None: u[sel], v[sel] = xn[sel] / scale, yn[sel] / (scale * aspect) fp[sel] = tex.w / (W * scale) else: xr = float(np.median(xn[sel])) sg = 1.0 if xr - vp >= 0 else -1.0 dx = xn[sel] - vp dx = np.where(sg * dx < 0.02, sg * 0.02, dx) # exact plane homography: metric along-wall coord = |D| * t, height = y * t, t = K / (x_vp - x) # with |D| = sqrt(vp^2 + f^2) (the wall's rotation relative to the camera) D = float(np.hypot(vp, fn)) Q = (xr - vp) ** 2 / (D * scale) u[sel] = Q * D / dx v[sel] = Q * yn[sel] / dx / aspect fp[sel] = np.sqrt((tex.w * Q * D / (dx ** 2 * W)) * (tex.w * Q / (np.abs(dx) * W))) return _finish(tex, u, v, fp, p) def _finish(tex, u, v, fp, p): ang = math.radians(float(p.get("angle", 0))) if ang: c, s = math.cos(ang), math.sin(ang) u, v = u * c - v * s, u * s + v * c return sample_mipmapped(tex, u.astype(np.float32), v.astype(np.float32), fp.astype(np.float32)) # ------------------------------------------------------------------ views def _img(a): return Image.fromarray((np.clip(a, 0, 1) * 255 + 0.5).astype(np.uint8)) def _contour(mask): inner = ~imgops.dilate_bool(~mask, 1) return mask & ~inner def _dim(arr, f=0.28): return arr * f + 0.0 def view_detection(L, al, ids): out = L.arr.copy() for name, col in (("wall", (0.26, 0.53, 0.96)), ("floor", (0.96, 0.65, 0.14))): a = al[name][..., None] * 0.30 out = out * (1 - a) + np.array(col, np.float32) * a c = _contour(al[name] > 0.5)[..., None] out = np.where(c, np.array(col, np.float32), out) return _img(out) def view_surface(L, al, name): col = (0.96, 0.65, 0.14) if name == "floor" else (0.26, 0.53, 0.96) a = al[name][..., None] out = L.arr * a + _dim(L.arr) * (1 - a) c = _contour(al[name] > 0.5)[..., None] return _img(np.where(c, np.array(col, np.float32), out)) def view_confidence(L, al): c = np.clip((L.conf - 0.5) / 0.5, 0, 1) r, g = np.clip(2 * (1 - c), 0, 1), np.clip(2 * c, 0, 1) heat = np.stack([r, g, np.full_like(c, 0.15)], -1) return _img(_dim(L.arr, 0.45) * 0.45 + heat * 0.55) def _palette(lbl, n=None): pal = np.array([PALETTE[i % len(PALETTE)] for i in range(256)], np.float32) / 255.0 return pal[lbl] def debug_view(sess, name, L, al, q): det = sess.det H, W = L.H, L.W if name == "preprocessed": v = det.preproc s = min(W / v.width, H / v.height) v = v.resize((max(1, round(v.width * s)), max(1, round(v.height * s))), Image.NEAREST) bg = Image.new("RGB", (W, H), (24, 26, 30)) bg.paste(v, ((W - v.width) // 2, (H - v.height) // 2)) return bg if name == "rawclass": return _img(0.35 * L.arr + 0.65 * _palette(L.raw_label)) if name == "band": b = L.band[..., None].astype(np.float32) return _img(L.arr * (1 - 0.55 * b) + np.array([1.0, 0.2, 0.5], np.float32) * 0.55 * b) if name == "refined": return _img(0.35 * L.arr + 0.65 * _palette(L.label)) return view_detection(L, al, sess.det.ids) # ------------------------------------------------------------------ main entry def render(sess: Session, settings: dict, store, view="result", quality="prev", strokes=()): q = "full" if quality == "full" else "prev" L = sess.layer(q) al, ver = sess.alphas(q, list(strokes)) arr = L.arr H, W = L.H, L.W if view == "original": return _img(arr) if view == "detection": return view_detection(L, al, sess.det.ids) if view in ("floor", "wall"): return view_surface(L, al, view) if view == "confidence": return view_confidence(L, al) if view.startswith("debug:"): return debug_view(sess, view[6:], L, al, q) out = arr.copy() geo = None for name in ("wall", "floor"): p = settings.get(name) if not p or not p.get("tex") or not store.valid(p["tex"]): continue a = al[name] m = a > 0.004 if not m.any(): continue if geo is None: geo = sess.geo(q, list(strokes)) ys, xs = np.nonzero(m) fx, fy = xs.astype(np.float32), ys.astype(np.float32) color = parse_color(p["tex"]) if color is not None: layer = np.empty((len(xs), 3), np.float32) layer[:] = color tmean = None else: tex = store.get(p["tex"], p.get("tiling", "auto")) tmean = tex.mean if name == "floor": layer = map_floor(tex, fx, fy, H, W, p, geo, float(ys.max())) else: layer = map_wall(tex, fx, fy, H, W, p, geo) con = float(p.get("contrast", 1.0)) if tmean is not None and con != 1.0: layer = tmean + (layer - tmean) * con ratio0, hp = sess.lighting(q, name, a, ver) shade = float(p.get("shade", 0.2)) ratio = np.clip(ratio0[ys, xs], 0.3, 2.0) ** shade refl = float(p.get("reflect", 0.2)) spec = 1.0 + refl * np.clip(hp[ys, xs], -0.5, 0.8) layer = np.clip(layer * (ratio * spec * float(p.get("bright", 0.7)))[:, None], 0, 1) w = (a[ys, xs] * float(p.get("blend", 1.0)))[:, None] out[ys, xs] = out[ys, xs] * (1 - w) + layer * w return _img(out)