Spaces:
Running on Zero
Running on Zero
| """Interpenetration in a rendered clip, from the fine depth/semantic, camera poses and bones. | |
| buried : the protagonist's feet are below the visible ground -- at the projected toe/foot | |
| bones the depth buffer shows a surface clearly in front of the bone (Town04). | |
| inside : the protagonist's hips are over the footprint of a hedge, planter, wall, fence or | |
| static prop that spans its shins. | |
| pierced : a pole or other thin solid runs through the body: in the middle 60% of the | |
| protagonist's box, a column that is mostly non-protagonist surface at its own depth. | |
| occluded : something (typically a wall the camera is inside) covers > 30% of the protagonist's | |
| projected bone box with surfaces well in front of it and raised above its feet | |
| (the ground in front of the feet and walls beside the path do not count). | |
| A window is bad if buried / inside / occluded holds for > 0.5 s in a row, or pierced for >= 3 frames | |
| (walking through a post takes ~0.3 s). | |
| """ | |
| from __future__ import annotations | |
| import math | |
| from pathlib import Path | |
| import numpy as np | |
| from ..clip import FPS, n_windows, read_cameras, read_proxy, read_scene_frames, read_scene_static, window_frames | |
| from ..geometry import (CLS_HERO, CLS_SKY, FEET_BONES, PROXY_CX, PROXY_CY, PROXY_F, PROXY_H, PROXY_W, camera_basis, | |
| longest_run) | |
| RUN = int(0.5 * FPS) | |
| PIERCED_RUN = 3 | |
| SOLID_LABELS = ("Walls", "Fences", "GuardRail", "Static") # + vegetation under 2.4 m (hedges, planters) | |
| def _solid_boxes(clip: Path): | |
| """Low solid things a walker must not stand in. Tree boxes are excluded (their canopy footprint | |
| covers sidewalks), and so are buildings (their boxes include awnings).""" | |
| try: | |
| boxes = read_scene_static(clip)["boxes"] | |
| except OSError: | |
| return [] | |
| out = [] | |
| for b in boxes: | |
| e = [abs(x) for x in b["e"]] | |
| if (b["label"] in SOLID_LABELS or (b["label"] == "Vegetation" and e[2] < 1.2)) and max(e[0], e[1]) > 0.2: | |
| out.append((np.array(b["c"]), np.array(e), math.radians(b["r"][1]))) | |
| return out | |
| def _buried(d, s, bones, C, fwd, right, up) -> bool: | |
| sunk = 0 | |
| for b in FEET_BONES: | |
| q = np.array(bones[b]) - C | |
| z = q @ fwd | |
| if z < 0.3: | |
| continue | |
| iu, iv = int(PROXY_CX + PROXY_F * (q @ right) / z), int(PROXY_CY - PROXY_F * (q @ up) / z) | |
| if not (1 <= iu < PROXY_W - 1 and 1 <= iv < PROXY_H - 1): | |
| continue | |
| win_d, win_s = d[iv - 1:iv + 2, iu - 1:iu + 2], s[iv - 1:iv + 2, iu - 1:iu + 2] | |
| if not (win_s == CLS_HERO).any() and np.median(win_d) < z - 0.15: | |
| sunk += 1 | |
| return sunk >= 3 | |
| def _inside(bones, foot_z, solid) -> bool: | |
| hp = np.array(bones["crl_hips__C"]) | |
| for bc, be, yaw in solid: | |
| if not (bc[2] - be[2] < foot_z + 0.5 and bc[2] + be[2] > foot_z + 0.3): | |
| continue | |
| dx, dy = hp[0] - bc[0], hp[1] - bc[1] | |
| u, v = dx * math.cos(yaw) + dy * math.sin(yaw), -dx * math.sin(yaw) + dy * math.cos(yaw) | |
| if abs(u) < be[0] - 0.1 and abs(v) < be[1] - 0.1: | |
| return True | |
| return False | |
| def _box_checks(d, s, bones, C, fwd, right, up, foot_z, xn, yn): | |
| """(occluded fraction of the protagonist's box, pole through the body).""" | |
| hd = (np.array(bones["crl_hips__C"]) - C) @ fwd | |
| pts = np.array(list(bones.values())) - C | |
| zs = pts @ fwd | |
| if not (zs > 0.3).all(): | |
| return 0.0, False | |
| us, vs = PROXY_CX + PROXY_F * (pts @ right) / zs, PROXY_CY - PROXY_F * (pts @ up) / zs | |
| u0, u1 = int(max(0, us.min() - 3)), int(min(PROXY_W, us.max() + 4)) | |
| v0, v1 = int(max(0, vs.min() - 3)), int(min(PROXY_H, vs.max() + 4)) | |
| if u1 - u0 < 3 or v1 - v0 < 3: | |
| return 0.0, False | |
| bd, bs = d[v0:v1, u0:u1], s[v0:v1, u0:u1] | |
| height = C[2] + bd * (fwd[2] + xn[v0:v1, u0:u1] * right[2] + yn[v0:v1, u0:u1] * up[2]) - foot_z | |
| occ = float(((bd > 0) & (bd < 0.75 * hd) & (height > 0.3) & (bs != CLS_HERO) & (bs != CLS_SKY)).mean()) | |
| hz = np.median(bd[bs == CLS_HERO]) if (bs == CLS_HERO).sum() > 20 else hd | |
| same = (np.abs(bd - hz) < 0.3) & (bs != CLS_HERO) & (bs != CLS_SKY) & (height > 0.25) | |
| w0 = int(0.2 * (u1 - u0)) | |
| cols = same[:, w0:(u1 - u0) - w0].mean(0) if (u1 - u0) - 2 * w0 > 0 else np.zeros(1) | |
| return occ, bool(cols.max() > 0.5) | |
| def scan(clip: Path) -> dict: | |
| clip = Path(clip) | |
| cams, frames = read_cameras(clip), read_scene_frames(clip) | |
| n = min(len(cams), len(frames)) | |
| vv, uu = np.mgrid[0:PROXY_H, 0:PROXY_W] | |
| xn, yn = (uu + 0.5 - PROXY_CX) / PROXY_F, -(vv + 0.5 - PROXY_CY) / PROXY_F | |
| solid = _solid_boxes(clip) | |
| buried, inside, occluded, occ_frac, pierced = [], [], [], [], [] | |
| for i in range(n): | |
| d, s = read_proxy(clip, i) | |
| C = np.array(cams[i]["loc"]) | |
| fwd, right, up = camera_basis(*cams[i]["rot"][:2]) | |
| bones = frames[i]["bones"] | |
| foot_z = min(bones[b][2] for b in FEET_BONES) | |
| buried.append(_buried(d, s, bones, C, fwd, right, up)) | |
| inside.append(_inside(bones, foot_z, solid)) | |
| frac, pole = _box_checks(d, s, bones, C, fwd, right, up, foot_z, xn, yn) | |
| occ_frac.append(round(frac, 3)) | |
| occluded.append(frac > 0.3) | |
| pierced.append(pole) | |
| windows = [] | |
| for w in range(n_windows(n)): | |
| sl = window_frames(w) | |
| r = dict(w=w, buried_run=longest_run(buried[sl]), occluded_run=longest_run(occluded[sl]), | |
| inside_run=longest_run(inside[sl]), pierced_run=longest_run(pierced[sl])) | |
| r["bad"] = r["pierced_run"] >= PIERCED_RUN or max(r["buried_run"], r["occluded_run"], r["inside_run"]) > RUN | |
| windows.append(r) | |
| return dict(clip=clip.name, frames=n, | |
| max_buried_run=longest_run(buried), max_occluded_run=longest_run(occluded), | |
| max_inside_run=longest_run(inside), max_pierced_run=longest_run(pierced), | |
| bad_windows=[x["w"] for x in windows if x["bad"]], windows=windows, | |
| buried=[int(b) for b in buried], inside=[int(b) for b in inside], | |
| pierced=[int(b) for b in pierced], occ_frac=occ_frac) | |