Spaces:
Sleeping
Sleeping
Download app/render.py from umang2191/room-surface-studio: direct link, hf CLI and curl.
- Browser
- Download file 12.7 kB
-
https://huggingface.co/spaces/umang2191/room-surface-studio/resolve/main/app/render.py
- Command line
-
hf download hf://spaces/umang2191/room-surface-studio/app/render.py
-
curl -L -o render.py https://huggingface.co/spaces/umang2191/room-surface-studio/resolve/main/app/render.py
12.7 kB
| """Material rendering on cached detection results. Segmentation is never re-run from here.""" | |
| import json | |
| import math | |
| import numpy as np | |
| from PIL import Image, ImageDraw | |
| import torch | |
| import torch.nn.functional as F | |
| import geometry | |
| import imgops | |
| from textures import parse_color, sample_mipmapped | |
| PREVIEW_SIDE = 1500 | |
| LUMA = imgops.LUMA | |
| PALETTE = [(40, 44, 52), (66, 135, 245), (245, 166, 35), (120, 200, 120), (200, 90, 160), (90, 200, 200)] | |
| # ------------------------------------------------------------------ session / cache | |
| class Layer: | |
| def __init__(self, arr, base, conf, label, raw_label, band): | |
| self.arr, self.base, self.conf = arr, base, conf | |
| self.label, self.raw_label, self.band = label, raw_label, band | |
| self.H, self.W = arr.shape[:2] | |
| def _down_layer(L: Layer, side: int) -> Layer: | |
| s = side / max(L.H, L.W) | |
| if s >= 1: | |
| return L | |
| size = (max(8, round(L.H * s)), max(8, round(L.W * s))) | |
| nn = lambda a: F.interpolate(torch.from_numpy(a.astype(np.float32))[None, None], size=size, mode="nearest")[0, 0].numpy().astype(np.uint8) | |
| return Layer(imgops.area_down(L.arr, size), {k: imgops.area_down(v, size) for k, v in L.base.items()}, | |
| imgops.area_down(L.conf, size), nn(L.label), nn(L.raw_label), nn(L.band)) | |
| class Session: | |
| def __init__(self, det): | |
| self.det = det | |
| self.full = Layer(det.rgb, det.alpha, det.conf, det.labelmap, det.raw_label, det.band) | |
| self.prev = _down_layer(self.full, PREVIEW_SIDE) | |
| self._edit = {"full": self._blank(), "prev": self._blank()} | |
| self._light, self._geo = {}, {} | |
| def _blank(): | |
| return {"keys": None, "alpha": None, "version": 0} | |
| def layer(self, q): | |
| return self.full if q == "full" else self.prev | |
| # --- user edits (brush strokes), applied incrementally and cached | |
| def alphas(self, q, strokes): | |
| L, st = self.layer(q), self._edit[q] | |
| keys = [json.dumps(s, sort_keys=True) for s in strokes] | |
| if st["keys"] is not None and keys[:len(st["keys"])] == st["keys"] and st["alpha"] is not None: | |
| new = keys[len(st["keys"]):] | |
| todo = strokes[len(st["keys"]):] | |
| else: | |
| st["alpha"] = {k: v.copy() for k, v in L.base.items()} | |
| new, todo = keys, strokes | |
| for s in todo: | |
| apply_stroke(st["alpha"], s, L.H, L.W) | |
| if st["keys"] != keys: | |
| st["version"] += 1 | |
| st["keys"] = keys | |
| return st["alpha"], st["version"] | |
| def geo(self, q, strokes): | |
| al, ver = self.alphas(q, strokes) | |
| # geometry only needs a small mask, so one cache entry per edit-state is enough | |
| key = (self._edit[q]["keys"] and "|".join(self._edit[q]["keys"])) or "" | |
| if key not in self._geo: | |
| self._geo = {key: geometry.scene_geometry(al["wall"], al["floor"])} | |
| return self._geo[key] | |
| def lighting(self, q, surface, alpha, ver): | |
| key = (q, ver, surface) | |
| if key not in self._light: | |
| if len(self._light) > 6: | |
| self._light.clear() | |
| L = self.layer(q) | |
| lum = (L.arr @ LUMA).astype(np.float32) | |
| r = max(3, min(L.H, L.W) // 85) | |
| smooth = imgops.masked_blur(lum, alpha, r) | |
| ref = float((smooth * alpha).sum() / max(alpha.sum(), 1.0)) | |
| ref = max(ref, 1e-3) | |
| self._light[key] = (smooth / ref, (lum - smooth) / ref) | |
| return self._light[key] | |
| def apply_stroke(al, st, H, W): | |
| """Rasterise one brush stroke in its bounding box (fast) and edit the mattes.""" | |
| pts = [(x * W, y * H) for x, y in st["pts"]] | |
| if not pts: | |
| return | |
| r = max(1.0, float(st["r"]) * W) | |
| hard = float(st.get("hard", 0.7)) | |
| op = float(st.get("op", 1.0)) | |
| rf = int((1 - hard) * r * 0.6) | |
| mg = int(r + 2 * rf + 3) | |
| x0 = max(0, int(min(p[0] for p in pts)) - mg) | |
| y0 = max(0, int(min(p[1] for p in pts)) - mg) | |
| x1 = min(W, int(max(p[0] for p in pts)) + mg + 1) | |
| y1 = min(H, int(max(p[1] for p in pts)) + mg + 1) | |
| if x1 <= x0 or y1 <= y0: | |
| return | |
| im = Image.new("L", (x1 - x0, y1 - y0), 0) | |
| d = ImageDraw.Draw(im) | |
| loc = [(x - x0, y - y0) for x, y in pts] | |
| if len(loc) > 1: | |
| d.line(loc, fill=255, width=max(1, round(2 * r)), joint="curve") | |
| for x, y in loc[:: max(1, len(loc) // 300)] + [loc[-1]]: | |
| d.ellipse([x - r, y - r, x + r, y + r], fill=255) | |
| m = np.asarray(im, np.float32) / 255.0 | |
| m = imgops.box_filter(m, max(1, rf)) if rf else imgops.box_filter(m, 1) | |
| if rf > 1: | |
| m = imgops.box_filter(m, max(1, rf // 2)) | |
| m = np.clip(m * op, 0, 1) | |
| sl = (slice(y0, y1), slice(x0, x1)) | |
| aw, af = al["wall"][sl], al["floor"][sl] | |
| mode = st["m"] | |
| if mode == "floor": | |
| al["floor"][sl] = af + (1 - af) * m | |
| al["wall"][sl] = aw * (1 - m) | |
| elif mode == "wall": | |
| al["wall"][sl] = aw + (1 - aw) * m | |
| al["floor"][sl] = af * (1 - m) | |
| else: | |
| al["wall"][sl] = aw * (1 - m) | |
| al["floor"][sl] = af * (1 - m) | |
| # ------------------------------------------------------------------ mapping | |
| def map_floor(tex, xs, ys, H, W, p, geo, bottom): | |
| scale = max(float(p.get("scale", 0.3)), 0.02) | |
| aspect = tex.h / tex.w | |
| if p.get("persp", True): | |
| if p.get("persp_mode", "auto") == "manual": | |
| yh, vpx = float(p.get("horizon", -0.2)) * H, W / 2 | |
| else: | |
| yh, vpx = geo["floor"]["horizon"] * H, geo["floor"]["vp_x"] * W | |
| d = np.maximum(ys - yh, 1.0) | |
| db = max(bottom - yh, 1.0) | |
| f = float(p.get("depth", 0.9)) * W | |
| k = db / (W * scale) | |
| u = (xs - vpx) * k / d | |
| v = (k * f / d) / aspect | |
| fp = np.sqrt((k / d * tex.w) * (k * f / d ** 2 * tex.h / aspect)) | |
| else: | |
| u, v = xs / (W * scale), ys / (W * scale * aspect) | |
| fp = np.full(u.shape, tex.w / (W * scale), np.float32) | |
| return _finish(tex, u, v, fp, p) | |
| def slider_vp(p): | |
| """Manual 'wall angle' slider -> vanishing-point x (image-width units from centre); None = front-facing.""" | |
| p = float(p) | |
| return None if abs(p) < 0.03 else (0.5 / p) | |
| def map_wall(tex, xs, ys, H, W, p, geo): | |
| """Planar homography per wall plane: u ~ 1/(x - x_vp), v ~ (y - horizon)/(x - x_vp), normalised so a | |
| tile spans `scale` of the image width at the plane's visible centre. vp=None -> front-facing (affine).""" | |
| scale = max(float(p.get("scale", 0.25)), 0.02) | |
| aspect = tex.h / tex.w | |
| manual = p.get("persp_mode", "auto") == "manual" | |
| yh = (float(p["horizon"]) if manual and "horizon" in p else geo["floor"]["horizon"]) * H | |
| segs = ([{"x0": 0.0, "x1": 1.0, "vp": slider_vp(p.get("wpersp", 0.0))}] if manual else geo["wall"]["segments"]) | |
| xn, yn = (xs - W / 2) / W, (ys - yh) / W | |
| fx = xs / W | |
| u, v, fp = np.empty_like(xn), np.empty_like(xn), np.empty_like(xn) | |
| done = np.zeros(xn.shape, bool) | |
| fn = 0.9 # focal length in image-width units (same assumption as the floor) | |
| for i, s in enumerate(segs): | |
| sel = (fx >= s["x0"]) & ((fx < s["x1"]) if i < len(segs) - 1 else (fx <= 1.0 + 1e-6)) & ~done | |
| if not sel.any(): | |
| continue | |
| done |= sel | |
| vp = s["vp"] | |
| if vp is None: | |
| u[sel], v[sel] = xn[sel] / scale, yn[sel] / (scale * aspect) | |
| fp[sel] = tex.w / (W * scale) | |
| else: | |
| xr = float(np.median(xn[sel])) | |
| sg = 1.0 if xr - vp >= 0 else -1.0 | |
| dx = xn[sel] - vp | |
| dx = np.where(sg * dx < 0.02, sg * 0.02, dx) | |
| # exact plane homography: metric along-wall coord = |D| * t, height = y * t, t = K / (x_vp - x) | |
| # with |D| = sqrt(vp^2 + f^2) (the wall's rotation relative to the camera) | |
| D = float(np.hypot(vp, fn)) | |
| Q = (xr - vp) ** 2 / (D * scale) | |
| u[sel] = Q * D / dx | |
| v[sel] = Q * yn[sel] / dx / aspect | |
| fp[sel] = np.sqrt((tex.w * Q * D / (dx ** 2 * W)) * (tex.w * Q / (np.abs(dx) * W))) | |
| return _finish(tex, u, v, fp, p) | |
| def _finish(tex, u, v, fp, p): | |
| ang = math.radians(float(p.get("angle", 0))) | |
| if ang: | |
| c, s = math.cos(ang), math.sin(ang) | |
| u, v = u * c - v * s, u * s + v * c | |
| return sample_mipmapped(tex, u.astype(np.float32), v.astype(np.float32), fp.astype(np.float32)) | |
| # ------------------------------------------------------------------ views | |
| def _img(a): | |
| return Image.fromarray((np.clip(a, 0, 1) * 255 + 0.5).astype(np.uint8)) | |
| def _contour(mask): | |
| inner = ~imgops.dilate_bool(~mask, 1) | |
| return mask & ~inner | |
| def _dim(arr, f=0.28): | |
| return arr * f + 0.0 | |
| def view_detection(L, al, ids): | |
| out = L.arr.copy() | |
| for name, col in (("wall", (0.26, 0.53, 0.96)), ("floor", (0.96, 0.65, 0.14))): | |
| a = al[name][..., None] * 0.30 | |
| out = out * (1 - a) + np.array(col, np.float32) * a | |
| c = _contour(al[name] > 0.5)[..., None] | |
| out = np.where(c, np.array(col, np.float32), out) | |
| return _img(out) | |
| def view_surface(L, al, name): | |
| col = (0.96, 0.65, 0.14) if name == "floor" else (0.26, 0.53, 0.96) | |
| a = al[name][..., None] | |
| out = L.arr * a + _dim(L.arr) * (1 - a) | |
| c = _contour(al[name] > 0.5)[..., None] | |
| return _img(np.where(c, np.array(col, np.float32), out)) | |
| def view_confidence(L, al): | |
| c = np.clip((L.conf - 0.5) / 0.5, 0, 1) | |
| r, g = np.clip(2 * (1 - c), 0, 1), np.clip(2 * c, 0, 1) | |
| heat = np.stack([r, g, np.full_like(c, 0.15)], -1) | |
| return _img(_dim(L.arr, 0.45) * 0.45 + heat * 0.55) | |
| def _palette(lbl, n=None): | |
| pal = np.array([PALETTE[i % len(PALETTE)] for i in range(256)], np.float32) / 255.0 | |
| return pal[lbl] | |
| def debug_view(sess, name, L, al, q): | |
| det = sess.det | |
| H, W = L.H, L.W | |
| if name == "preprocessed": | |
| v = det.preproc | |
| s = min(W / v.width, H / v.height) | |
| v = v.resize((max(1, round(v.width * s)), max(1, round(v.height * s))), Image.NEAREST) | |
| bg = Image.new("RGB", (W, H), (24, 26, 30)) | |
| bg.paste(v, ((W - v.width) // 2, (H - v.height) // 2)) | |
| return bg | |
| if name == "rawclass": | |
| return _img(0.35 * L.arr + 0.65 * _palette(L.raw_label)) | |
| if name == "band": | |
| b = L.band[..., None].astype(np.float32) | |
| return _img(L.arr * (1 - 0.55 * b) + np.array([1.0, 0.2, 0.5], np.float32) * 0.55 * b) | |
| if name == "refined": | |
| return _img(0.35 * L.arr + 0.65 * _palette(L.label)) | |
| return view_detection(L, al, sess.det.ids) | |
| # ------------------------------------------------------------------ main entry | |
| def render(sess: Session, settings: dict, store, view="result", quality="prev", strokes=()): | |
| q = "full" if quality == "full" else "prev" | |
| L = sess.layer(q) | |
| al, ver = sess.alphas(q, list(strokes)) | |
| arr = L.arr | |
| H, W = L.H, L.W | |
| if view == "original": | |
| return _img(arr) | |
| if view == "detection": | |
| return view_detection(L, al, sess.det.ids) | |
| if view in ("floor", "wall"): | |
| return view_surface(L, al, view) | |
| if view == "confidence": | |
| return view_confidence(L, al) | |
| if view.startswith("debug:"): | |
| return debug_view(sess, view[6:], L, al, q) | |
| out = arr.copy() | |
| geo = None | |
| for name in ("wall", "floor"): | |
| p = settings.get(name) | |
| if not p or not p.get("tex") or not store.valid(p["tex"]): | |
| continue | |
| a = al[name] | |
| m = a > 0.004 | |
| if not m.any(): | |
| continue | |
| if geo is None: | |
| geo = sess.geo(q, list(strokes)) | |
| ys, xs = np.nonzero(m) | |
| fx, fy = xs.astype(np.float32), ys.astype(np.float32) | |
| color = parse_color(p["tex"]) | |
| if color is not None: | |
| layer = np.empty((len(xs), 3), np.float32) | |
| layer[:] = color | |
| tmean = None | |
| else: | |
| tex = store.get(p["tex"], p.get("tiling", "auto")) | |
| tmean = tex.mean | |
| if name == "floor": | |
| layer = map_floor(tex, fx, fy, H, W, p, geo, float(ys.max())) | |
| else: | |
| layer = map_wall(tex, fx, fy, H, W, p, geo) | |
| con = float(p.get("contrast", 1.0)) | |
| if tmean is not None and con != 1.0: | |
| layer = tmean + (layer - tmean) * con | |
| ratio0, hp = sess.lighting(q, name, a, ver) | |
| shade = float(p.get("shade", 0.2)) | |
| ratio = np.clip(ratio0[ys, xs], 0.3, 2.0) ** shade | |
| refl = float(p.get("reflect", 0.2)) | |
| spec = 1.0 + refl * np.clip(hp[ys, xs], -0.5, 0.8) | |
| layer = np.clip(layer * (ratio * spec * float(p.get("bright", 0.7)))[:, None], 0, 1) | |
| w = (a[ys, xs] * float(p.get("blend", 1.0)))[:, None] | |
| out[ys, xs] = out[ys, xs] * (1 - w) + layer * w | |
| return _img(out) | |