ZeroGPU-friendly compute caps: CUDA at startup, 1536px images, 60 frames at 960px, tighter durations
Browse files
app.py
CHANGED
|
@@ -23,20 +23,33 @@ from depth_pro_2.network.fov import FOVEstimator
|
|
| 23 |
from huggingface_hub import hf_hub_download
|
| 24 |
from safetensors.torch import load_file
|
| 25 |
|
| 26 |
-
MAX_VIDEO_FRAMES =
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
#
|
| 31 |
-
#
|
| 32 |
-
#
|
| 33 |
-
#
|
|
|
|
|
|
|
|
|
|
| 34 |
model = DepthPro2.from_pretrained("apple/DepthPro2-H", precision=torch.bfloat16)
|
| 35 |
model.fov = FOVEstimator(precision=torch.bfloat16)
|
| 36 |
model.fov.load_state_dict(
|
| 37 |
load_file(hf_hub_download("apple/DepthPro2-focal", "model.safetensors"))
|
| 38 |
)
|
| 39 |
-
model.eval()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40 |
|
| 41 |
# ColorBrewer "Spectral" stops, dark red (near) to violet (far), interpolated to a 256-entry LUT.
|
| 42 |
_SPECTRAL = np.array(
|
|
@@ -64,8 +77,7 @@ def _gpu(duration: int):
|
|
| 64 |
|
| 65 |
|
| 66 |
def _device_and_model():
|
| 67 |
-
|
| 68 |
-
return model.to(device), partial(to_normalized_tensor, device=device)
|
| 69 |
|
| 70 |
|
| 71 |
def _colorize(depth: np.ndarray) -> np.ndarray:
|
|
@@ -78,9 +90,10 @@ def _colorize(depth: np.ndarray) -> np.ndarray:
|
|
| 78 |
return (rgb * 255).astype(np.uint8)
|
| 79 |
|
| 80 |
|
| 81 |
-
@_gpu(
|
| 82 |
def _infer_image(img: np.ndarray, f_px: float | None) -> np.ndarray:
|
| 83 |
-
"""Depth colorization for one photo
|
|
|
|
| 84 |
net, transform = _device_and_model()
|
| 85 |
prediction = net.infer(transform(img), f_px=f_px)
|
| 86 |
if prediction["depth"] is None:
|
|
@@ -88,16 +101,17 @@ def _infer_image(img: np.ndarray, f_px: float | None) -> np.ndarray:
|
|
| 88 |
return _colorize(prediction["depth"].float().cpu().numpy())
|
| 89 |
|
| 90 |
|
| 91 |
-
@_gpu(
|
| 92 |
def _infer_video(path: str) -> str:
|
| 93 |
-
"""Depth colorization for a video,
|
| 94 |
container = av.open(path)
|
| 95 |
rate = container.streams.video[0].average_rate
|
| 96 |
fps = int(round(float(rate))) if rate else 24
|
| 97 |
|
| 98 |
def frames():
|
| 99 |
for frame in container.decode(video=0):
|
| 100 |
-
|
|
|
|
| 101 |
|
| 102 |
net, transform = _device_and_model()
|
| 103 |
predictions = net.infer_video(transform(f) for f in frames())
|
|
|
|
| 23 |
from huggingface_hub import hf_hub_download
|
| 24 |
from safetensors.torch import load_file
|
| 25 |
|
| 26 |
+
MAX_VIDEO_FRAMES = 60
|
| 27 |
+
MAX_IMAGE_SIDE = 1536
|
| 28 |
+
MAX_VIDEO_SIDE = 960
|
| 29 |
+
|
| 30 |
+
# Weights load from the Hub and go to CUDA at module level: ZeroGPU's CUDA emulation allows
|
| 31 |
+
# placements at startup, and lazy .to("cuda") inside @spaces.GPU is discouraged (docs:
|
| 32 |
+
# huggingface.co/docs/hub/spaces-zerogpu, Model loading) — transfers are optimized for startup.
|
| 33 |
+
# The backbone comes from apple/DepthPro2-H (whose config has "fov": false, so the focal
|
| 34 |
+
# estimator is attached separately from apple/DepthPro2-focal, its own repo). Hub downloads
|
| 35 |
+
# resume instead of corrupting, unlike the preset loader's CDN checkpoints. bfloat16 is how the
|
| 36 |
+
# models run on a GPU.
|
| 37 |
model = DepthPro2.from_pretrained("apple/DepthPro2-H", precision=torch.bfloat16)
|
| 38 |
model.fov = FOVEstimator(precision=torch.bfloat16)
|
| 39 |
model.fov.load_state_dict(
|
| 40 |
load_file(hf_hub_download("apple/DepthPro2-focal", "model.safetensors"))
|
| 41 |
)
|
| 42 |
+
model.eval().to("cuda")
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def _resize(img: np.ndarray, f_px: float | None, max_side: int):
|
| 46 |
+
"""Downscale so the longer side is at most ``max_side``, scaling ``f_px`` to match."""
|
| 47 |
+
height, width = img.shape[:2]
|
| 48 |
+
scale = max_side / max(height, width)
|
| 49 |
+
if scale >= 1.0:
|
| 50 |
+
return img, f_px
|
| 51 |
+
out = Image.fromarray(img).resize((round(width * scale), round(height * scale)), Image.LANCZOS)
|
| 52 |
+
return np.asarray(out), None if f_px is None else f_px * scale
|
| 53 |
|
| 54 |
# ColorBrewer "Spectral" stops, dark red (near) to violet (far), interpolated to a 256-entry LUT.
|
| 55 |
_SPECTRAL = np.array(
|
|
|
|
| 77 |
|
| 78 |
|
| 79 |
def _device_and_model():
|
| 80 |
+
return model, partial(to_normalized_tensor, device=torch.device("cuda"))
|
|
|
|
| 81 |
|
| 82 |
|
| 83 |
def _colorize(depth: np.ndarray) -> np.ndarray:
|
|
|
|
| 90 |
return (rgb * 255).astype(np.uint8)
|
| 91 |
|
| 92 |
|
| 93 |
+
@_gpu(60)
|
| 94 |
def _infer_image(img: np.ndarray, f_px: float | None) -> np.ndarray:
|
| 95 |
+
"""Depth colorization for one photo, downscaled to at most 1536 px on the longer side."""
|
| 96 |
+
img, f_px = _resize(img, f_px, MAX_IMAGE_SIDE)
|
| 97 |
net, transform = _device_and_model()
|
| 98 |
prediction = net.infer(transform(img), f_px=f_px)
|
| 99 |
if prediction["depth"] is None:
|
|
|
|
| 101 |
return _colorize(prediction["depth"].float().cpu().numpy())
|
| 102 |
|
| 103 |
|
| 104 |
+
@_gpu(240)
|
| 105 |
def _infer_video(path: str) -> str:
|
| 106 |
+
"""Depth colorization for a video: first 60 frames, each at most 960 px on the longer side."""
|
| 107 |
container = av.open(path)
|
| 108 |
rate = container.streams.video[0].average_rate
|
| 109 |
fps = int(round(float(rate))) if rate else 24
|
| 110 |
|
| 111 |
def frames():
|
| 112 |
for frame in container.decode(video=0):
|
| 113 |
+
img, _ = _resize(upright(frame), None, MAX_VIDEO_SIDE)
|
| 114 |
+
yield img
|
| 115 |
|
| 116 |
net, transform = _device_and_model()
|
| 117 |
predictions = net.infer_video(transform(f) for f in frames())
|