akhaliq HF Staff commited on
Commit
b9f01b8
·
verified ·
1 Parent(s): 84501a5

Switch to gradio.Server mode: @app .api endpoints + custom HTML front page with comparison slider

Browse files
Files changed (1) hide show
  1. app.py +58 -71
app.py CHANGED
@@ -5,9 +5,11 @@ from functools import partial
5
  from pathlib import Path
6
 
7
  import av
8
- import gradio as gr
9
  import numpy as np
10
  import torch
 
 
 
11
  from PIL import Image
12
 
13
  try:
@@ -20,30 +22,14 @@ from depth_pro_2.model import to_normalized_tensor
20
 
21
  MAX_VIDEO_FRAMES = 100
22
 
23
- # The weights and the focal-length estimator load at startup via the library's canonical
24
- # loader (from_pretrained on the per-size repo leaves the FOV estimator unloaded, which makes
25
- # infer return depth=None for photos without an EXIF focal length). The model is created on CPU
26
- # and moved to CUDA inside the GPU-decorated functions, since ZeroGPU exposes the GPU only
27
- # during those calls. bfloat16 is how the models run on a GPU.
28
- model, _ = create_model_and_transforms(
29
- "depthpro2-h", device="cpu", precision=torch.bfloat16
30
- )
31
-
32
-
33
- def _gpu(duration: int):
34
- """ZeroGPU wrapper; a no-op where the spaces package is absent (plain CPU)."""
35
- if spaces is not None and hasattr(spaces, "GPU"):
36
- return spaces.GPU(duration=duration)
37
- return lambda fn: fn
38
-
39
 
40
- def _device_and_model():
41
- device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
42
- return model.to(device), partial(to_normalized_tensor, device=device)
43
-
44
-
45
- # ColorBrewer "Spectral" stops, dark red (near) to violet (far), interpolated to a 256-entry LUT
46
- # so no matplotlib version-specific colormap registry is needed.
47
  _SPECTRAL = np.array(
48
  [
49
  [0x9E, 0x01, 0x42], [0xD5, 0x3E, 0x4F], [0xF4, 0x6D, 0x43], [0xFD, 0xAE, 0x61],
@@ -61,6 +47,18 @@ _LUT = np.stack(
61
  )
62
 
63
 
 
 
 
 
 
 
 
 
 
 
 
 
64
  def _colorize(depth: np.ndarray) -> np.ndarray:
65
  """Metric depth in meters to RGB, red near and blue far; white where infinitely far."""
66
  finite = np.isfinite(depth)
@@ -72,22 +70,19 @@ def _colorize(depth: np.ndarray) -> np.ndarray:
72
 
73
 
74
  @_gpu(120)
75
- def run_image(image: Image.Image) -> Image.Image:
76
- with tempfile.TemporaryDirectory() as tmp:
77
- path = Path(tmp) / "input.png"
78
- image.save(path)
79
- img, f_px = load_rgb(path) # f_px: the focal length recorded in the photo, or None
80
  net, transform = _device_and_model()
81
  prediction = net.infer(transform(img), f_px=f_px)
82
  if prediction["depth"] is None:
83
- raise gr.Error("No focal length recorded in the photo and none could be estimated.")
84
- depth = prediction["depth"] # meters, (H, W)
85
- return Image.fromarray(_colorize(depth.float().cpu().numpy()))
86
 
87
 
88
  @_gpu(300)
89
- def run_video(video: str) -> str:
90
- container = av.open(video)
 
91
  rate = container.streams.video[0].average_rate
92
  fps = int(round(float(rate))) if rate else 24
93
 
@@ -96,15 +91,14 @@ def run_video(video: str) -> str:
96
  yield upright(frame)
97
 
98
  net, transform = _device_and_model()
99
- predictions = net.infer_video(transform(f) for f in frames()) # one per frame, in order
100
 
101
  out_path = Path(tempfile.gettempdir()) / "depthpro2-output.mp4"
102
  writer = None
103
  written = 0
104
  with av.open(out_path, mode="w") as out:
105
  for prediction in predictions:
106
- depth = prediction["depth"].float().cpu().numpy()
107
- rgb = _colorize(depth)
108
  if writer is None:
109
  height, width = rgb.shape[:2]
110
  writer = out.add_stream("libx264", rate=fps)
@@ -120,42 +114,35 @@ def run_video(video: str) -> str:
120
  for packet in writer.encode():
121
  out.mux(packet)
122
  if writer is None:
123
- raise gr.Error("Could not decode any frames from that video.")
124
  return str(out_path)
125
 
126
 
127
- TITLE = "# Depth Pro 2 — metric monocular depth"
128
- DESCRIPTION = (
129
- "[apple/DepthPro2](https://huggingface.co/apple/DepthPro2) estimates metric depth for every "
130
- "pixel from a single photo or a whole video. Depth is sharp at object edges and steady from "
131
- "frame to frame; red is near, blue is far, and white marks the infinitely far (sky). "
132
- "For research use only."
133
  )
134
 
135
- with gr.Blocks(title="Depth Pro 2") as demo:
136
- gr.Markdown(TITLE + "\n\n" + DESCRIPTION)
137
- with gr.Tabs():
138
- with gr.Tab("Image"):
139
- with gr.Row():
140
- image_in = gr.Image(type="pil", label="Input photo")
141
- image_out = gr.Image(label="Predicted depth", interactive=False)
142
- gr.Button("Estimate depth", variant="primary").click(
143
- run_image, inputs=image_in, outputs=image_out
144
- )
145
- gr.Examples(
146
- [
147
- [
148
- "https://huggingface.co/apple/DepthPro2/resolve/main/assets/teaser.jpg"
149
- ]
150
- ],
151
- inputs=image_in,
152
- )
153
- with gr.Tab("Video"):
154
- with gr.Row():
155
- video_in = gr.Video(label="Input video (first 100 frames)")
156
- video_out = gr.Video(label="Predicted depth", interactive=False)
157
- gr.Button("Estimate depth", variant="primary").click(
158
- run_video, inputs=video_in, outputs=video_out
159
- )
160
-
161
- demo.launch()
 
5
  from pathlib import Path
6
 
7
  import av
 
8
  import numpy as np
9
  import torch
10
+ from fastapi.responses import HTMLResponse
11
+ from gradio import Server
12
+ from gradio.data_classes import FileData
13
  from PIL import Image
14
 
15
  try:
 
22
 
23
  MAX_VIDEO_FRAMES = 100
24
 
25
+ # Weights and focal-length estimator load at startup via the library's canonical loader
26
+ # (create_model_and_transforms; from_pretrained on the per-size repo leaves the FOV estimator
27
+ # unloaded, which makes infer return depth=None for photos without an EXIF focal length).
28
+ # The model is created on CPU and moved to CUDA inside the GPU-decorated functions, since
29
+ # ZeroGPU exposes the GPU only during those calls. bfloat16 is how the models run on a GPU.
30
+ model, _ = create_model_and_transforms("depthpro2-h", device="cpu", precision=torch.bfloat16)
 
 
 
 
 
 
 
 
 
 
31
 
32
+ # ColorBrewer "Spectral" stops, dark red (near) to violet (far), interpolated to a 256-entry LUT.
 
 
 
 
 
 
33
  _SPECTRAL = np.array(
34
  [
35
  [0x9E, 0x01, 0x42], [0xD5, 0x3E, 0x4F], [0xF4, 0x6D, 0x43], [0xFD, 0xAE, 0x61],
 
47
  )
48
 
49
 
50
+ def _gpu(duration: int):
51
+ """ZeroGPU wrapper; a no-op where the spaces package is absent (plain CPU)."""
52
+ if spaces is not None and hasattr(spaces, "GPU"):
53
+ return spaces.GPU(duration=duration)
54
+ return lambda fn: fn
55
+
56
+
57
+ def _device_and_model():
58
+ device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
59
+ return model.to(device), partial(to_normalized_tensor, device=device)
60
+
61
+
62
  def _colorize(depth: np.ndarray) -> np.ndarray:
63
  """Metric depth in meters to RGB, red near and blue far; white where infinitely far."""
64
  finite = np.isfinite(depth)
 
70
 
71
 
72
  @_gpu(120)
73
+ def _infer_image(img: np.ndarray, f_px: float | None) -> np.ndarray:
74
+ """Depth colorization for one photo; f_px from EXIF, or None to estimate it."""
 
 
 
75
  net, transform = _device_and_model()
76
  prediction = net.infer(transform(img), f_px=f_px)
77
  if prediction["depth"] is None:
78
+ raise ValueError("No focal length recorded in the photo and none could be estimated.")
79
+ return _colorize(prediction["depth"].float().cpu().numpy())
 
80
 
81
 
82
  @_gpu(300)
83
+ def _infer_video(path: str) -> str:
84
+ """Depth colorization for a video, one prediction per frame, in order."""
85
+ container = av.open(path)
86
  rate = container.streams.video[0].average_rate
87
  fps = int(round(float(rate))) if rate else 24
88
 
 
91
  yield upright(frame)
92
 
93
  net, transform = _device_and_model()
94
+ predictions = net.infer_video(transform(f) for f in frames())
95
 
96
  out_path = Path(tempfile.gettempdir()) / "depthpro2-output.mp4"
97
  writer = None
98
  written = 0
99
  with av.open(out_path, mode="w") as out:
100
  for prediction in predictions:
101
+ rgb = _colorize(prediction["depth"].float().cpu().numpy())
 
102
  if writer is None:
103
  height, width = rgb.shape[:2]
104
  writer = out.add_stream("libx264", rate=fps)
 
114
  for packet in writer.encode():
115
  out.mux(packet)
116
  if writer is None:
117
+ raise ValueError("Could not decode any frames from that video.")
118
  return str(out_path)
119
 
120
 
121
+ app = Server(
122
+ title="Depth Pro 2",
123
+ description="Metric monocular depth from photos and videos (apple/DepthPro2).",
 
 
 
124
  )
125
 
126
+
127
+ @app.api(name="predict_depth")
128
+ def predict_depth(image: FileData) -> FileData:
129
+ """Estimate metric depth for an uploaded photo; returns the colorized depth map."""
130
+ img, f_px = load_rgb(image["path"]) # f_px: the focal length recorded in the photo, or None
131
+ rgb = _infer_image(img, f_px)
132
+ out_path = Path(tempfile.gettempdir()) / "depthpro2-depth.png"
133
+ Image.fromarray(rgb).save(out_path)
134
+ return FileData(path=str(out_path))
135
+
136
+
137
+ @app.api(name="predict_depth_video")
138
+ def predict_depth_video(video: FileData) -> FileData:
139
+ """Estimate metric depth for an uploaded video (first 100 frames); returns the depth video."""
140
+ return FileData(path=_infer_video(video["path"]))
141
+
142
+
143
+ @app.get("/", response_class=HTMLResponse)
144
+ async def homepage():
145
+ return (Path(__file__).resolve().parent / "index.html").read_text(encoding="utf-8")
146
+
147
+
148
+ app.launch(show_error=True)