File size: 10,094 Bytes
c66ee68
 
6df2866
e6ce011
c66ee68
 
 
 
ffd1a97
c66ee68
 
d1846d3
b9f01b8
 
c66ee68
 
 
 
 
 
 
54be9ba
c66ee68
54be9ba
 
 
c66ee68
afa74dd
 
 
 
 
 
 
 
 
 
 
54be9ba
 
 
 
 
afa74dd
 
d1846d3
 
 
 
 
babd6b4
 
 
d1846d3
 
 
 
 
 
 
 
 
 
 
 
afa74dd
 
 
 
 
 
 
 
 
c66ee68
b9f01b8
1feea84
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
b9f01b8
 
 
 
 
 
 
 
afa74dd
b9f01b8
 
c66ee68
 
 
 
 
1feea84
c66ee68
 
 
 
afa74dd
b9f01b8
afa74dd
 
c66ee68
84501a5
 
b9f01b8
 
c66ee68
 
afa74dd
59ee84a
 
 
 
 
 
 
b9f01b8
c66ee68
 
 
59ee84a
 
 
 
 
 
 
c66ee68
 
59ee84a
c66ee68
e1950cc
59ee84a
c66ee68
59ee84a
 
 
 
b9f01b8
c66ee68
 
 
 
 
59ee84a
 
 
 
 
 
 
 
 
 
c66ee68
 
 
 
59ee84a
 
c66ee68
b9f01b8
59ee84a
 
 
 
 
c66ee68
 
b9f01b8
 
 
c66ee68
 
b9f01b8
 
6df2866
 
ffd1a97
 
ca006c6
 
 
 
 
 
6df2866
b9f01b8
e6ce011
 
 
 
 
 
 
 
 
85b3388
 
 
e6ce011
b9f01b8
 
 
85b3388
59ee84a
 
b9f01b8
 
d1846d3
 
 
 
 
 
 
 
b9f01b8
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
from __future__ import annotations

import base64
import io
from functools import partial
from pathlib import Path

import av
import gradio as gr
import numpy as np
import torch
from fastapi.responses import FileResponse, HTMLResponse
from gradio import Server
from gradio.data_classes import FileData
from PIL import Image

try:
    import spaces  # ZeroGPU: available when the Space runs on ZeroGPU hardware
except ImportError:
    spaces = None

from depth_pro_2 import DepthPro2, load_rgb, upright
from depth_pro_2.model import to_normalized_tensor
from depth_pro_2.network.fov import FOVEstimator
from huggingface_hub import hf_hub_download
from safetensors.torch import load_file

MAX_VIDEO_FRAMES = 60
MAX_IMAGE_SIDE = 1536
MAX_VIDEO_SIDE = 960

# Weights load from the Hub and go to CUDA at module level: ZeroGPU's CUDA emulation allows
# placements at startup, and lazy .to("cuda") inside @spaces.GPU is discouraged (docs:
# huggingface.co/docs/hub/spaces-zerogpu, Model loading) — transfers are optimized for startup.
# The backbone comes from apple/DepthPro2-H (whose config has "fov": false, so the focal
# estimator is attached separately from apple/DepthPro2-focal, its own repo). Hub downloads
# resume instead of corrupting, unlike the preset loader's CDN checkpoints. bfloat16 is how the
# models run on a GPU.
model = DepthPro2.from_pretrained("apple/DepthPro2-H", precision=torch.bfloat16)
model.fov = FOVEstimator(precision=torch.bfloat16)
model.fov.load_state_dict(
    load_file(hf_hub_download("apple/DepthPro2-focal", "model.safetensors"))
)
model.eval().to("cuda")

# Self-host the @gradio/client bundle: browsers on some networks cannot reach public CDNs, and
# the page's dynamic import is what failed for them. Download it once at startup (server-side
# package installs work here) and serve it from this Space's own origin; the page falls back to
# the CDNs only if this file is missing.
_CLIENT_SOURCES = [
    "https://cdn.jsdelivr.net/npm/@gradio/client@2.7.1/dist/index.min.js",
    "https://unpkg.com/@gradio/client@2.7.1/dist/index.min.js",
    "https://cdn.statically.io/npm/@gradio/client@2.7.1/dist/index.min.js",
]
_client_path = Path(__file__).resolve().parent / "gradio_client.min.js"
if not _client_path.exists():
    import urllib.request

    for _src in _CLIENT_SOURCES:
        try:
            urllib.request.urlretrieve(_src, _client_path)
            break
        except OSError:
            continue


def _resize(img: np.ndarray, f_px: float | None, max_side: int):
    """Downscale so the longer side is at most ``max_side``, scaling ``f_px`` to match."""
    height, width = img.shape[:2]
    scale = max_side / max(height, width)
    if scale >= 1.0:
        return img, f_px
    out = Image.fromarray(img).resize((round(width * scale), round(height * scale)), Image.LANCZOS)
    return np.asarray(out), None if f_px is None else f_px * scale

# ColorBrewer "Spectral" stops, dark red (near) to violet (far), interpolated to a 256-entry LUT.
_SPECTRAL = np.array(
    [
        [0x9E, 0x01, 0x42], [0xD5, 0x3E, 0x4F], [0xF4, 0x6D, 0x43], [0xFD, 0xAE, 0x61],
        [0xFE, 0xE0, 0x8B], [0xFF, 0xFF, 0xBF], [0xE6, 0xF5, 0x98], [0xAB, 0xDD, 0xA4],
        [0x66, 0xC2, 0xA5], [0x32, 0x88, 0xBD], [0x5E, 0x4F, 0xA2],
    ],
    dtype=np.float64,
) / 255.0
_LUT = np.stack(
    [
        np.interp(np.linspace(0.0, 1.0, 256), np.linspace(0.0, 1.0, len(_SPECTRAL)), _SPECTRAL[:, c])
        for c in range(3)
    ],
    axis=1,
)


def _gpu(duration: int):
    """ZeroGPU wrapper; a no-op where the spaces package is absent (plain CPU)."""
    if spaces is not None and hasattr(spaces, "GPU"):
        return spaces.GPU(duration=duration)
    return lambda fn: fn


def _device_and_model():
    return model, partial(to_normalized_tensor, device=torch.device("cuda"))


def _colorize(depth: np.ndarray) -> np.ndarray:
    """Metric depth in meters to RGB, red near and blue far; white where infinitely far."""
    finite = np.isfinite(depth)
    lo, hi = np.percentile(depth[finite], [2, 98])
    norm = np.clip((depth - lo) / max(hi - lo, 1e-6), 0.0, 1.0)
    rgb = _LUT[(norm * 255).astype(np.uint8)]
    rgb[~finite] = 1.0
    return (rgb * 255).astype(np.uint8)


@_gpu(60)
def _infer_image(img: np.ndarray, f_px: float | None) -> np.ndarray:
    """Depth colorization for one photo, downscaled to at most 1536 px on the longer side."""
    img, f_px = _resize(img, f_px, MAX_IMAGE_SIDE)
    net, transform = _device_and_model()
    prediction = net.infer(transform(img), f_px=f_px)
    if prediction["depth"] is None:
        raise ValueError("No focal length recorded in the photo and none could be estimated.")
    return _colorize(prediction["depth"].float().cpu().numpy())


@_gpu(240)
def _infer_video(path: str) -> dict:
    """Depth colorization for a video (first 60 frames, up to 960 px); returns source + depth data URIs.

    The source is re-encoded to H.264 from the already-decoded frames: phone videos are often
    HEVC, which Chrome cannot decode, and an unplayable input window otherwise reads as if the
    depth result failed. The frames are already in memory, so the second encode is cheap.
    """
    container = av.open(path)
    rate = container.streams.video[0].average_rate
    fps = int(round(float(rate))) if rate else 24

    imgs = []
    for frame in container.decode(video=0):
        imgs.append(_resize(upright(frame), None, MAX_VIDEO_SIDE)[0])
        if len(imgs) >= MAX_VIDEO_FRAMES:
            break
    if not imgs:
        raise ValueError("Could not decode any frames from that video.")

    net, transform = _device_and_model()
    predictions = net.infer_video(transform(f) for f in imgs)

    out_path = Path(path).with_suffix(".depth.mp4")
    src_path = Path(path).with_suffix(".source.mp4")
    written = 0
    with av.open(out_path, mode="w") as out, av.open(src_path, mode="w") as sout:
        writer = None
        src_writer = None
        for img, prediction in zip(imgs, predictions):
            rgb = _colorize(prediction["depth"].float().cpu().numpy())
            if writer is None:
                height, width = rgb.shape[:2]
                writer = out.add_stream("libx264", rate=fps)
                writer.width, writer.height = width - width % 2, height - height % 2
                writer.pix_fmt = "yuv420p"
                # The source at the depth map's size, so both windows encode the same frames.
                src_writer = sout.add_stream("libx264", rate=fps)
                src_writer.width, src_writer.height = writer.width, writer.height
                src_writer.pix_fmt = "yuv420p"
            for stream, muxer, pixels in ((writer, out, rgb), (src_writer, sout, img)):
                frame = av.VideoFrame.from_ndarray(
                    np.ascontiguousarray(pixels[: writer.height, : writer.width]), format="rgb24"
                )
                for packet in stream.encode(frame):
                    muxer.mux(packet)
            written += 1
        if writer is not None:
            for packet in writer.encode():
                out.mux(packet)
            for packet in src_writer.encode():
                sout.mux(packet)
    if writer is None:
        raise ValueError("Could not decode any frames from that video.")
    print(f"infer_video: {written} frames, depth {out_path.stat().st_size} B, source {src_path.stat().st_size} B", flush=True)
    return {
        "image_data": "data:video/mp4;base64," + base64.b64encode(out_path.read_bytes()).decode(),
        "source": "data:video/mp4;base64," + base64.b64encode(src_path.read_bytes()).decode(),
    }


app = Server(
    title="Depth Pro 2",
    description="Metric monocular depth from photos and videos (apple/DepthPro2).",
)


@app.api(name="predict_depth")
def predict_depth(image: FileData, request: gr.Request) -> str:
    """Estimate metric depth for an uploaded photo; returns it inline as a data URI."""
    # request is taken explicitly: Server-mode ZeroGPU token forwarding is more reliable with it
    # in the signature (see MiniMaxAI/MiniMax-H3-Turbo-Lora commit 7db57ff).
    try:
        img, f_px = load_rgb(image["path"])  # f_px: the focal length recorded in the photo, or None
    except Exception as err:
        raise gr.Error(
            f"That upload could not be read as an image ({type(err).__name__}) \u2014 try a JPG or PNG."
        ) from err
    print(f"predict_depth: {img.shape[1]}x{img.shape[0]} px, f_px={f_px}", flush=True)
    rgb = _infer_image(img, f_px)
    # Encode at display resolution (long side 1024): the depth map is for viewing, and a compact
    # data URI sidesteps any size limits between the API and the browser.
    out = Image.fromarray(rgb)
    if max(out.size) > 1024:
        scale = 1024 / max(out.size)
        out = out.resize((round(out.width * scale), round(out.height * scale)), Image.LANCZOS)
    buffer = io.BytesIO()
    out.save(buffer, "PNG")
    print(f"predict_depth: encoded {len(buffer.getvalue())} bytes", flush=True)
    # A dict (not a bare string) is the proven-safe return: gradio's postprocess leaves dicts
    # without file metadata untouched, while a bare data-URI string can get mangled as a path
    # (the P-Image-OSS pattern).
    return {"image_data": "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode()}


@app.api(name="predict_depth_video")
def predict_depth_video(video: FileData, request: gr.Request) -> dict:
    """Estimate metric depth for an uploaded video (first 60 frames); returns source + depth data URIs."""
    return _infer_video(video["path"])


@app.get("/gradio-client.js")
def gradio_client_js():
    """The @gradio/client bundle, served same-origin so no CDN is needed in the browser."""
    if _client_path.exists():
        return FileResponse(_client_path, media_type="text/javascript")
    return HTMLResponse("// client bundle not available", status_code=404)


@app.get("/", response_class=HTMLResponse)
async def homepage():
    return (Path(__file__).resolve().parent / "index.html").read_text(encoding="utf-8")


app.launch(show_error=True)