DepthPro2 / app.py
akhaliq's picture
akhaliq HF Staff
Video: return H.264 re-encoded source alongside depth (input window playable for HEVC uploads)
59ee84a verified
Raw History Blame Contribute Delete
10.1 kB
from __future__ import annotations
import base64
import io
from functools import partial
from pathlib import Path
import av
import gradio as gr
import numpy as np
import torch
from fastapi.responses import FileResponse, HTMLResponse
from gradio import Server
from gradio.data_classes import FileData
from PIL import Image
try:
import spaces # ZeroGPU: available when the Space runs on ZeroGPU hardware
except ImportError:
spaces = None
from depth_pro_2 import DepthPro2, load_rgb, upright
from depth_pro_2.model import to_normalized_tensor
from depth_pro_2.network.fov import FOVEstimator
from huggingface_hub import hf_hub_download
from safetensors.torch import load_file
MAX_VIDEO_FRAMES = 60
MAX_IMAGE_SIDE = 1536
MAX_VIDEO_SIDE = 960
# Weights load from the Hub and go to CUDA at module level: ZeroGPU's CUDA emulation allows
# placements at startup, and lazy .to("cuda") inside @spaces.GPU is discouraged (docs:
# huggingface.co/docs/hub/spaces-zerogpu, Model loading) — transfers are optimized for startup.
# The backbone comes from apple/DepthPro2-H (whose config has "fov": false, so the focal
# estimator is attached separately from apple/DepthPro2-focal, its own repo). Hub downloads
# resume instead of corrupting, unlike the preset loader's CDN checkpoints. bfloat16 is how the
# models run on a GPU.
model = DepthPro2.from_pretrained("apple/DepthPro2-H", precision=torch.bfloat16)
model.fov = FOVEstimator(precision=torch.bfloat16)
model.fov.load_state_dict(
load_file(hf_hub_download("apple/DepthPro2-focal", "model.safetensors"))
)
model.eval().to("cuda")
# Self-host the @gradio/client bundle: browsers on some networks cannot reach public CDNs, and
# the page's dynamic import is what failed for them. Download it once at startup (server-side
# package installs work here) and serve it from this Space's own origin; the page falls back to
# the CDNs only if this file is missing.
_CLIENT_SOURCES = [
"https://cdn.jsdelivr.net/npm/@gradio/client@2.7.1/dist/index.min.js",
"https://unpkg.com/@gradio/client@2.7.1/dist/index.min.js",
"https://cdn.statically.io/npm/@gradio/client@2.7.1/dist/index.min.js",
]
_client_path = Path(__file__).resolve().parent / "gradio_client.min.js"
if not _client_path.exists():
import urllib.request
for _src in _CLIENT_SOURCES:
try:
urllib.request.urlretrieve(_src, _client_path)
break
except OSError:
continue
def _resize(img: np.ndarray, f_px: float | None, max_side: int):
"""Downscale so the longer side is at most ``max_side``, scaling ``f_px`` to match."""
height, width = img.shape[:2]
scale = max_side / max(height, width)
if scale >= 1.0:
return img, f_px
out = Image.fromarray(img).resize((round(width * scale), round(height * scale)), Image.LANCZOS)
return np.asarray(out), None if f_px is None else f_px * scale
# ColorBrewer "Spectral" stops, dark red (near) to violet (far), interpolated to a 256-entry LUT.
_SPECTRAL = np.array(
[
[0x9E, 0x01, 0x42], [0xD5, 0x3E, 0x4F], [0xF4, 0x6D, 0x43], [0xFD, 0xAE, 0x61],
[0xFE, 0xE0, 0x8B], [0xFF, 0xFF, 0xBF], [0xE6, 0xF5, 0x98], [0xAB, 0xDD, 0xA4],
[0x66, 0xC2, 0xA5], [0x32, 0x88, 0xBD], [0x5E, 0x4F, 0xA2],
],
dtype=np.float64,
) / 255.0
_LUT = np.stack(
[
np.interp(np.linspace(0.0, 1.0, 256), np.linspace(0.0, 1.0, len(_SPECTRAL)), _SPECTRAL[:, c])
for c in range(3)
],
axis=1,
)
def _gpu(duration: int):
"""ZeroGPU wrapper; a no-op where the spaces package is absent (plain CPU)."""
if spaces is not None and hasattr(spaces, "GPU"):
return spaces.GPU(duration=duration)
return lambda fn: fn
def _device_and_model():
return model, partial(to_normalized_tensor, device=torch.device("cuda"))
def _colorize(depth: np.ndarray) -> np.ndarray:
"""Metric depth in meters to RGB, red near and blue far; white where infinitely far."""
finite = np.isfinite(depth)
lo, hi = np.percentile(depth[finite], [2, 98])
norm = np.clip((depth - lo) / max(hi - lo, 1e-6), 0.0, 1.0)
rgb = _LUT[(norm * 255).astype(np.uint8)]
rgb[~finite] = 1.0
return (rgb * 255).astype(np.uint8)
@_gpu(60)
def _infer_image(img: np.ndarray, f_px: float | None) -> np.ndarray:
"""Depth colorization for one photo, downscaled to at most 1536 px on the longer side."""
img, f_px = _resize(img, f_px, MAX_IMAGE_SIDE)
net, transform = _device_and_model()
prediction = net.infer(transform(img), f_px=f_px)
if prediction["depth"] is None:
raise ValueError("No focal length recorded in the photo and none could be estimated.")
return _colorize(prediction["depth"].float().cpu().numpy())
@_gpu(240)
def _infer_video(path: str) -> dict:
"""Depth colorization for a video (first 60 frames, up to 960 px); returns source + depth data URIs.
The source is re-encoded to H.264 from the already-decoded frames: phone videos are often
HEVC, which Chrome cannot decode, and an unplayable input window otherwise reads as if the
depth result failed. The frames are already in memory, so the second encode is cheap.
"""
container = av.open(path)
rate = container.streams.video[0].average_rate
fps = int(round(float(rate))) if rate else 24
imgs = []
for frame in container.decode(video=0):
imgs.append(_resize(upright(frame), None, MAX_VIDEO_SIDE)[0])
if len(imgs) >= MAX_VIDEO_FRAMES:
break
if not imgs:
raise ValueError("Could not decode any frames from that video.")
net, transform = _device_and_model()
predictions = net.infer_video(transform(f) for f in imgs)
out_path = Path(path).with_suffix(".depth.mp4")
src_path = Path(path).with_suffix(".source.mp4")
written = 0
with av.open(out_path, mode="w") as out, av.open(src_path, mode="w") as sout:
writer = None
src_writer = None
for img, prediction in zip(imgs, predictions):
rgb = _colorize(prediction["depth"].float().cpu().numpy())
if writer is None:
height, width = rgb.shape[:2]
writer = out.add_stream("libx264", rate=fps)
writer.width, writer.height = width - width % 2, height - height % 2
writer.pix_fmt = "yuv420p"
# The source at the depth map's size, so both windows encode the same frames.
src_writer = sout.add_stream("libx264", rate=fps)
src_writer.width, src_writer.height = writer.width, writer.height
src_writer.pix_fmt = "yuv420p"
for stream, muxer, pixels in ((writer, out, rgb), (src_writer, sout, img)):
frame = av.VideoFrame.from_ndarray(
np.ascontiguousarray(pixels[: writer.height, : writer.width]), format="rgb24"
)
for packet in stream.encode(frame):
muxer.mux(packet)
written += 1
if writer is not None:
for packet in writer.encode():
out.mux(packet)
for packet in src_writer.encode():
sout.mux(packet)
if writer is None:
raise ValueError("Could not decode any frames from that video.")
print(f"infer_video: {written} frames, depth {out_path.stat().st_size} B, source {src_path.stat().st_size} B", flush=True)
return {
"image_data": "data:video/mp4;base64," + base64.b64encode(out_path.read_bytes()).decode(),
"source": "data:video/mp4;base64," + base64.b64encode(src_path.read_bytes()).decode(),
}
app = Server(
title="Depth Pro 2",
description="Metric monocular depth from photos and videos (apple/DepthPro2).",
)
@app.api(name="predict_depth")
def predict_depth(image: FileData, request: gr.Request) -> str:
"""Estimate metric depth for an uploaded photo; returns it inline as a data URI."""
# request is taken explicitly: Server-mode ZeroGPU token forwarding is more reliable with it
# in the signature (see MiniMaxAI/MiniMax-H3-Turbo-Lora commit 7db57ff).
try:
img, f_px = load_rgb(image["path"]) # f_px: the focal length recorded in the photo, or None
except Exception as err:
raise gr.Error(
f"That upload could not be read as an image ({type(err).__name__}) \u2014 try a JPG or PNG."
) from err
print(f"predict_depth: {img.shape[1]}x{img.shape[0]} px, f_px={f_px}", flush=True)
rgb = _infer_image(img, f_px)
# Encode at display resolution (long side 1024): the depth map is for viewing, and a compact
# data URI sidesteps any size limits between the API and the browser.
out = Image.fromarray(rgb)
if max(out.size) > 1024:
scale = 1024 / max(out.size)
out = out.resize((round(out.width * scale), round(out.height * scale)), Image.LANCZOS)
buffer = io.BytesIO()
out.save(buffer, "PNG")
print(f"predict_depth: encoded {len(buffer.getvalue())} bytes", flush=True)
# A dict (not a bare string) is the proven-safe return: gradio's postprocess leaves dicts
# without file metadata untouched, while a bare data-URI string can get mangled as a path
# (the P-Image-OSS pattern).
return {"image_data": "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode()}
@app.api(name="predict_depth_video")
def predict_depth_video(video: FileData, request: gr.Request) -> dict:
"""Estimate metric depth for an uploaded video (first 60 frames); returns source + depth data URIs."""
return _infer_video(video["path"])
@app.get("/gradio-client.js")
def gradio_client_js():
"""The @gradio/client bundle, served same-origin so no CDN is needed in the browser."""
if _client_path.exists():
return FileResponse(_client_path, media_type="text/javascript")
return HTMLResponse("// client bundle not available", status_code=404)
@app.get("/", response_class=HTMLResponse)
async def homepage():
return (Path(__file__).resolve().parent / "index.html").read_text(encoding="utf-8")
app.launch(show_error=True)