Switch to gradio.Server mode: @app .api endpoints + custom HTML front page with comparison slider
Browse files
app.py
CHANGED
|
@@ -5,9 +5,11 @@ from functools import partial
|
|
| 5 |
from pathlib import Path
|
| 6 |
|
| 7 |
import av
|
| 8 |
-
import gradio as gr
|
| 9 |
import numpy as np
|
| 10 |
import torch
|
|
|
|
|
|
|
|
|
|
| 11 |
from PIL import Image
|
| 12 |
|
| 13 |
try:
|
|
@@ -20,30 +22,14 @@ from depth_pro_2.model import to_normalized_tensor
|
|
| 20 |
|
| 21 |
MAX_VIDEO_FRAMES = 100
|
| 22 |
|
| 23 |
-
#
|
| 24 |
-
#
|
| 25 |
-
# infer return depth=None for photos without an EXIF focal length).
|
| 26 |
-
# and moved to CUDA inside the GPU-decorated functions, since
|
| 27 |
-
# during those calls. bfloat16 is how the models run on a GPU.
|
| 28 |
-
model, _ = create_model_and_transforms(
|
| 29 |
-
"depthpro2-h", device="cpu", precision=torch.bfloat16
|
| 30 |
-
)
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
def _gpu(duration: int):
|
| 34 |
-
"""ZeroGPU wrapper; a no-op where the spaces package is absent (plain CPU)."""
|
| 35 |
-
if spaces is not None and hasattr(spaces, "GPU"):
|
| 36 |
-
return spaces.GPU(duration=duration)
|
| 37 |
-
return lambda fn: fn
|
| 38 |
-
|
| 39 |
|
| 40 |
-
|
| 41 |
-
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
| 42 |
-
return model.to(device), partial(to_normalized_tensor, device=device)
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
# ColorBrewer "Spectral" stops, dark red (near) to violet (far), interpolated to a 256-entry LUT
|
| 46 |
-
# so no matplotlib version-specific colormap registry is needed.
|
| 47 |
_SPECTRAL = np.array(
|
| 48 |
[
|
| 49 |
[0x9E, 0x01, 0x42], [0xD5, 0x3E, 0x4F], [0xF4, 0x6D, 0x43], [0xFD, 0xAE, 0x61],
|
|
@@ -61,6 +47,18 @@ _LUT = np.stack(
|
|
| 61 |
)
|
| 62 |
|
| 63 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 64 |
def _colorize(depth: np.ndarray) -> np.ndarray:
|
| 65 |
"""Metric depth in meters to RGB, red near and blue far; white where infinitely far."""
|
| 66 |
finite = np.isfinite(depth)
|
|
@@ -72,22 +70,19 @@ def _colorize(depth: np.ndarray) -> np.ndarray:
|
|
| 72 |
|
| 73 |
|
| 74 |
@_gpu(120)
|
| 75 |
-
def
|
| 76 |
-
|
| 77 |
-
path = Path(tmp) / "input.png"
|
| 78 |
-
image.save(path)
|
| 79 |
-
img, f_px = load_rgb(path) # f_px: the focal length recorded in the photo, or None
|
| 80 |
net, transform = _device_and_model()
|
| 81 |
prediction = net.infer(transform(img), f_px=f_px)
|
| 82 |
if prediction["depth"] is None:
|
| 83 |
-
raise
|
| 84 |
-
|
| 85 |
-
return Image.fromarray(_colorize(depth.float().cpu().numpy()))
|
| 86 |
|
| 87 |
|
| 88 |
@_gpu(300)
|
| 89 |
-
def
|
| 90 |
-
|
|
|
|
| 91 |
rate = container.streams.video[0].average_rate
|
| 92 |
fps = int(round(float(rate))) if rate else 24
|
| 93 |
|
|
@@ -96,15 +91,14 @@ def run_video(video: str) -> str:
|
|
| 96 |
yield upright(frame)
|
| 97 |
|
| 98 |
net, transform = _device_and_model()
|
| 99 |
-
predictions = net.infer_video(transform(f) for f in frames())
|
| 100 |
|
| 101 |
out_path = Path(tempfile.gettempdir()) / "depthpro2-output.mp4"
|
| 102 |
writer = None
|
| 103 |
written = 0
|
| 104 |
with av.open(out_path, mode="w") as out:
|
| 105 |
for prediction in predictions:
|
| 106 |
-
|
| 107 |
-
rgb = _colorize(depth)
|
| 108 |
if writer is None:
|
| 109 |
height, width = rgb.shape[:2]
|
| 110 |
writer = out.add_stream("libx264", rate=fps)
|
|
@@ -120,42 +114,35 @@ def run_video(video: str) -> str:
|
|
| 120 |
for packet in writer.encode():
|
| 121 |
out.mux(packet)
|
| 122 |
if writer is None:
|
| 123 |
-
raise
|
| 124 |
return str(out_path)
|
| 125 |
|
| 126 |
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
"
|
| 130 |
-
"pixel from a single photo or a whole video. Depth is sharp at object edges and steady from "
|
| 131 |
-
"frame to frame; red is near, blue is far, and white marks the infinitely far (sky). "
|
| 132 |
-
"For research use only."
|
| 133 |
)
|
| 134 |
|
| 135 |
-
|
| 136 |
-
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
|
| 145 |
-
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
|
| 149 |
-
|
| 150 |
-
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
run_video, inputs=video_in, outputs=video_out
|
| 159 |
-
)
|
| 160 |
-
|
| 161 |
-
demo.launch()
|
|
|
|
| 5 |
from pathlib import Path
|
| 6 |
|
| 7 |
import av
|
|
|
|
| 8 |
import numpy as np
|
| 9 |
import torch
|
| 10 |
+
from fastapi.responses import HTMLResponse
|
| 11 |
+
from gradio import Server
|
| 12 |
+
from gradio.data_classes import FileData
|
| 13 |
from PIL import Image
|
| 14 |
|
| 15 |
try:
|
|
|
|
| 22 |
|
| 23 |
MAX_VIDEO_FRAMES = 100
|
| 24 |
|
| 25 |
+
# Weights and focal-length estimator load at startup via the library's canonical loader
|
| 26 |
+
# (create_model_and_transforms; from_pretrained on the per-size repo leaves the FOV estimator
|
| 27 |
+
# unloaded, which makes infer return depth=None for photos without an EXIF focal length).
|
| 28 |
+
# The model is created on CPU and moved to CUDA inside the GPU-decorated functions, since
|
| 29 |
+
# ZeroGPU exposes the GPU only during those calls. bfloat16 is how the models run on a GPU.
|
| 30 |
+
model, _ = create_model_and_transforms("depthpro2-h", device="cpu", precision=torch.bfloat16)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 31 |
|
| 32 |
+
# ColorBrewer "Spectral" stops, dark red (near) to violet (far), interpolated to a 256-entry LUT.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
_SPECTRAL = np.array(
|
| 34 |
[
|
| 35 |
[0x9E, 0x01, 0x42], [0xD5, 0x3E, 0x4F], [0xF4, 0x6D, 0x43], [0xFD, 0xAE, 0x61],
|
|
|
|
| 47 |
)
|
| 48 |
|
| 49 |
|
| 50 |
+
def _gpu(duration: int):
|
| 51 |
+
"""ZeroGPU wrapper; a no-op where the spaces package is absent (plain CPU)."""
|
| 52 |
+
if spaces is not None and hasattr(spaces, "GPU"):
|
| 53 |
+
return spaces.GPU(duration=duration)
|
| 54 |
+
return lambda fn: fn
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def _device_and_model():
|
| 58 |
+
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
| 59 |
+
return model.to(device), partial(to_normalized_tensor, device=device)
|
| 60 |
+
|
| 61 |
+
|
| 62 |
def _colorize(depth: np.ndarray) -> np.ndarray:
|
| 63 |
"""Metric depth in meters to RGB, red near and blue far; white where infinitely far."""
|
| 64 |
finite = np.isfinite(depth)
|
|
|
|
| 70 |
|
| 71 |
|
| 72 |
@_gpu(120)
|
| 73 |
+
def _infer_image(img: np.ndarray, f_px: float | None) -> np.ndarray:
|
| 74 |
+
"""Depth colorization for one photo; f_px from EXIF, or None to estimate it."""
|
|
|
|
|
|
|
|
|
|
| 75 |
net, transform = _device_and_model()
|
| 76 |
prediction = net.infer(transform(img), f_px=f_px)
|
| 77 |
if prediction["depth"] is None:
|
| 78 |
+
raise ValueError("No focal length recorded in the photo and none could be estimated.")
|
| 79 |
+
return _colorize(prediction["depth"].float().cpu().numpy())
|
|
|
|
| 80 |
|
| 81 |
|
| 82 |
@_gpu(300)
|
| 83 |
+
def _infer_video(path: str) -> str:
|
| 84 |
+
"""Depth colorization for a video, one prediction per frame, in order."""
|
| 85 |
+
container = av.open(path)
|
| 86 |
rate = container.streams.video[0].average_rate
|
| 87 |
fps = int(round(float(rate))) if rate else 24
|
| 88 |
|
|
|
|
| 91 |
yield upright(frame)
|
| 92 |
|
| 93 |
net, transform = _device_and_model()
|
| 94 |
+
predictions = net.infer_video(transform(f) for f in frames())
|
| 95 |
|
| 96 |
out_path = Path(tempfile.gettempdir()) / "depthpro2-output.mp4"
|
| 97 |
writer = None
|
| 98 |
written = 0
|
| 99 |
with av.open(out_path, mode="w") as out:
|
| 100 |
for prediction in predictions:
|
| 101 |
+
rgb = _colorize(prediction["depth"].float().cpu().numpy())
|
|
|
|
| 102 |
if writer is None:
|
| 103 |
height, width = rgb.shape[:2]
|
| 104 |
writer = out.add_stream("libx264", rate=fps)
|
|
|
|
| 114 |
for packet in writer.encode():
|
| 115 |
out.mux(packet)
|
| 116 |
if writer is None:
|
| 117 |
+
raise ValueError("Could not decode any frames from that video.")
|
| 118 |
return str(out_path)
|
| 119 |
|
| 120 |
|
| 121 |
+
app = Server(
|
| 122 |
+
title="Depth Pro 2",
|
| 123 |
+
description="Metric monocular depth from photos and videos (apple/DepthPro2).",
|
|
|
|
|
|
|
|
|
|
| 124 |
)
|
| 125 |
|
| 126 |
+
|
| 127 |
+
@app.api(name="predict_depth")
|
| 128 |
+
def predict_depth(image: FileData) -> FileData:
|
| 129 |
+
"""Estimate metric depth for an uploaded photo; returns the colorized depth map."""
|
| 130 |
+
img, f_px = load_rgb(image["path"]) # f_px: the focal length recorded in the photo, or None
|
| 131 |
+
rgb = _infer_image(img, f_px)
|
| 132 |
+
out_path = Path(tempfile.gettempdir()) / "depthpro2-depth.png"
|
| 133 |
+
Image.fromarray(rgb).save(out_path)
|
| 134 |
+
return FileData(path=str(out_path))
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
@app.api(name="predict_depth_video")
|
| 138 |
+
def predict_depth_video(video: FileData) -> FileData:
|
| 139 |
+
"""Estimate metric depth for an uploaded video (first 100 frames); returns the depth video."""
|
| 140 |
+
return FileData(path=_infer_video(video["path"]))
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
@app.get("/", response_class=HTMLResponse)
|
| 144 |
+
async def homepage():
|
| 145 |
+
return (Path(__file__).resolve().parent / "index.html").read_text(encoding="utf-8")
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
app.launch(show_error=True)
|
|
|
|
|
|
|
|
|
|
|
|