Spaces:
Running on Zero
Running on Zero
| """Wider render audit (all from saved outputs). | |
| camera : single-frame pops (a step > 2.5x both neighbours), roll | |
| hero : walking backwards (toe direction vs velocity), standing still, off screen, too small / | |
| too large, other walkers or parked cars overlapping the protagonist | |
| video : black / blown-out / duplicated frames, mean luma, exposure changes (largest frame-to-frame | |
| luma jump, largest luma swing within 1 s -- CARLA's auto exposure adapting) | |
| proxy : protagonist present in the fine proxy but missing from the coarse one, coarse-vs-fine | |
| depth disagreement | |
| """ | |
| from __future__ import annotations | |
| import math | |
| from pathlib import Path | |
| import av | |
| import numpy as np | |
| from ..clip import FPS, n_windows, read_cameras, read_meta, read_proxy, read_scene_frames, read_scene_static, window_frames | |
| from ..geometry import CLS_HERO, CLS_SKY, PROXY_H, PROXY_W, longest_run, project | |
| def camera_pops(cams) -> np.ndarray: | |
| """Frames where the camera jumps: position or yaw step much larger than both neighbours. The | |
| smooth ~0.6 s swing round the protagonist at the turn-round is FollowCamera behaviour, not a pop.""" | |
| C = np.array([c["loc"] for c in cams]) | |
| rot = np.array([c["rot"] for c in cams]) | |
| s = np.r_[0, np.linalg.norm(np.diff(C, axis=0), axis=1), 0] | |
| y = np.r_[0, np.abs((np.diff(rot[:, 1]) + 180) % 360 - 180), 0] | |
| pop = np.zeros(len(C), bool) | |
| for i in range(1, len(C)): | |
| pop[i] = (s[i] > 0.15 and s[i] > 2.5 * max(s[i - 1], s[i + 1], 0.02)) or \ | |
| (y[i] > 4 and y[i] > 2.5 * max(y[i - 1], y[i + 1], 0.3)) | |
| return pop | |
| def video_stats(path: Path): | |
| luma, blown, dup, prev = [], [], [], None | |
| with av.open(str(path)) as c: | |
| s = c.streams.video[0] | |
| s.thread_type = "AUTO" | |
| for f in c.decode(s): | |
| g = f.reformat(width=PROXY_W, height=PROXY_H, format="gray").to_ndarray().astype(np.int16) | |
| luma.append(float(g.mean())) | |
| blown.append(float((g >= 250).mean())) | |
| dup.append(prev is not None and float(np.abs(g - prev).mean()) < 0.05) | |
| prev = g | |
| return np.array(luma), np.array(blown), np.array(dup) | |
| def _touching(p, z, walkers, vehicles) -> bool: | |
| for w in walkers: | |
| if math.hypot(w[0] - p[0], w[1] - p[1]) < 0.45 and abs(w[2] - z) < 1.0: | |
| return True | |
| for v in vehicles: | |
| yaw = math.radians(v["r"][1]) | |
| d = p - np.array(v["c"][:2]) | |
| lx, ly = d[0] * math.cos(yaw) + d[1] * math.sin(yaw), -d[0] * math.sin(yaw) + d[1] * math.cos(yaw) | |
| if abs(lx) < v["e"][0] + 0.15 and abs(ly) < v["e"][1] + 0.15: | |
| return True | |
| return False | |
| def audit(clip: Path) -> dict: | |
| clip = Path(clip) | |
| meta, cams, frames = read_meta(clip), read_cameras(clip), read_scene_frames(clip) | |
| vehicles = read_scene_static(clip)["vehicles"] | |
| n = min(len(cams), len(frames)) | |
| traj = np.array(meta["hero_traj"])[:n] | |
| rot = np.array([c["rot"] for c in cams[:n]]) | |
| pops = camera_pops(cams[:n]) | |
| vel = np.zeros((n, 2)) | |
| vel[6:-6] = (traj[12:, :2] - traj[:-12, :2]) / (12 / FPS) | |
| speed = np.linalg.norm(vel, axis=1) | |
| back = np.zeros(n, bool) | |
| for i in range(n): | |
| b = frames[i]["bones"] | |
| face = sum(np.array(b[f"crl_toe__{s}"][:2]) - np.array(b[f"crl_foot__{s}"][:2]) for s in "LR") | |
| if speed[i] > 0.5 and np.linalg.norm(face) > 1e-3: | |
| back[i] = face @ vel[i] / (np.linalg.norm(face) * speed[i]) < -0.3 | |
| touch = np.array([_touching(traj[i, :2], traj[i, 2], frames[i]["walkers"], vehicles) for i in range(n)]) | |
| hero_frac, coarse_miss, depth_err, offscreen = [], [], [], [] | |
| for i in range(0, n, 4): | |
| df, sf = read_proxy(clip, i) | |
| dc, sc = read_proxy(clip, i, "conditions_coarse") | |
| hf = sf == CLS_HERO | |
| hero_frac.append(float(hf.mean())) | |
| coarse_miss.append(hf.sum() > 50 and (hf & (sc == CLS_HERO)).sum() < 0.3 * hf.sum()) | |
| u, v, z = project(frames[i]["bones"]["crl_hips__C"], cams[i]["loc"], cams[i]["rot"]) | |
| offscreen.append(not (z[0] > 0 and 0 <= u[0] < PROXY_W and 0 <= v[0] < PROXY_H)) | |
| bg = (df > 0) & (dc > 0) & (sf != CLS_SKY) & (sc != CLS_SKY) | |
| depth_err.append(float(np.median(np.abs(np.log(dc[bg] / df[bg])))) if bg.sum() > 100 else 0.0) | |
| hero_frac, depth_err = np.array(hero_frac), np.array(depth_err) | |
| luma, blown, dup = video_stats(clip / "rgb.mp4") | |
| m = min(len(luma), n) | |
| windows = [] | |
| for w in range(n_windows(n)): | |
| sl = window_frames(w) | |
| a, b = sl.start, sl.stop | |
| a4, b4 = a // 4, (b + 3) // 4 | |
| windows.append(dict( | |
| w=w, camera_pops=int(pops[a:b].sum()), max_roll=float(np.abs(rot[a:b, 2]).max()), | |
| backwards_s=longest_run(back[a:b]) / FPS, still_s=longest_run(speed[a:b] < 0.2) / FPS, | |
| touch_s=longest_run(touch[a:b]) / FPS, offscreen_s=longest_run(offscreen[a4:b4]) * 4 / FPS, | |
| hero_frac_min=float(hero_frac[a4:b4].min()), hero_frac_max=float(hero_frac[a4:b4].max()), | |
| coarse_hero_miss=int(sum(coarse_miss[a4:b4])), depth_logerr_med=float(np.median(depth_err[a4:b4])), | |
| luma=float(luma[a:min(b, m)].mean()), luma_min=float(luma[a:min(b, m)].min()), | |
| blown_max=float(blown[a:min(b, m)].max()), dup_frames=int(dup[a:min(b, m)].sum()), | |
| luma_jump_max=float(np.abs(np.diff(luma[a:min(b, m)])).max()), | |
| luma_swing_1s=float(max(np.ptp(luma[i:i + FPS]) for i in range(a, max(a + 1, min(b, m) - FPS)))))) | |
| return dict(clip=clip.name, map=meta["map"], weather=meta.get("weather"), frames=n, video_frames=int(len(luma)), | |
| windows=windows) | |