Video: return H.264 re-encoded source alongside depth (input window playable for HEVC uploads)
Browse files
app.py
CHANGED
|
@@ -124,44 +124,64 @@ def _infer_image(img: np.ndarray, f_px: float | None) -> np.ndarray:
|
|
| 124 |
|
| 125 |
|
| 126 |
@_gpu(240)
|
| 127 |
-
def _infer_video(path: str) ->
|
| 128 |
-
"""Depth colorization for a video
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 129 |
container = av.open(path)
|
| 130 |
rate = container.streams.video[0].average_rate
|
| 131 |
fps = int(round(float(rate))) if rate else 24
|
| 132 |
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
|
|
|
|
|
|
|
|
|
|
| 137 |
|
| 138 |
net, transform = _device_and_model()
|
| 139 |
-
predictions = net.infer_video(transform(f) for f in
|
| 140 |
|
| 141 |
-
# Beside the uploaded input, which is in Gradio's servable upload cache.
|
| 142 |
out_path = Path(path).with_suffix(".depth.mp4")
|
| 143 |
-
|
| 144 |
written = 0
|
| 145 |
-
with av.open(out_path, mode="w") as out:
|
| 146 |
-
|
|
|
|
|
|
|
| 147 |
rgb = _colorize(prediction["depth"].float().cpu().numpy())
|
| 148 |
if writer is None:
|
| 149 |
height, width = rgb.shape[:2]
|
| 150 |
writer = out.add_stream("libx264", rate=fps)
|
| 151 |
writer.width, writer.height = width - width % 2, height - height % 2
|
| 152 |
writer.pix_fmt = "yuv420p"
|
| 153 |
-
|
| 154 |
-
|
| 155 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 156 |
written += 1
|
| 157 |
-
if written >= MAX_VIDEO_FRAMES:
|
| 158 |
-
break
|
| 159 |
if writer is not None:
|
| 160 |
for packet in writer.encode():
|
| 161 |
out.mux(packet)
|
|
|
|
|
|
|
| 162 |
if writer is None:
|
| 163 |
raise ValueError("Could not decode any frames from that video.")
|
| 164 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 165 |
|
| 166 |
|
| 167 |
app = Server(
|
|
@@ -200,8 +220,8 @@ def predict_depth(image: FileData, request: gr.Request) -> str:
|
|
| 200 |
|
| 201 |
@app.api(name="predict_depth_video")
|
| 202 |
def predict_depth_video(video: FileData, request: gr.Request) -> dict:
|
| 203 |
-
"""Estimate metric depth for an uploaded video (first 60 frames); returns
|
| 204 |
-
return
|
| 205 |
|
| 206 |
|
| 207 |
@app.get("/gradio-client.js")
|
|
|
|
| 124 |
|
| 125 |
|
| 126 |
@_gpu(240)
|
| 127 |
+
def _infer_video(path: str) -> dict:
|
| 128 |
+
"""Depth colorization for a video (first 60 frames, up to 960 px); returns source + depth data URIs.
|
| 129 |
+
|
| 130 |
+
The source is re-encoded to H.264 from the already-decoded frames: phone videos are often
|
| 131 |
+
HEVC, which Chrome cannot decode, and an unplayable input window otherwise reads as if the
|
| 132 |
+
depth result failed. The frames are already in memory, so the second encode is cheap.
|
| 133 |
+
"""
|
| 134 |
container = av.open(path)
|
| 135 |
rate = container.streams.video[0].average_rate
|
| 136 |
fps = int(round(float(rate))) if rate else 24
|
| 137 |
|
| 138 |
+
imgs = []
|
| 139 |
+
for frame in container.decode(video=0):
|
| 140 |
+
imgs.append(_resize(upright(frame), None, MAX_VIDEO_SIDE)[0])
|
| 141 |
+
if len(imgs) >= MAX_VIDEO_FRAMES:
|
| 142 |
+
break
|
| 143 |
+
if not imgs:
|
| 144 |
+
raise ValueError("Could not decode any frames from that video.")
|
| 145 |
|
| 146 |
net, transform = _device_and_model()
|
| 147 |
+
predictions = net.infer_video(transform(f) for f in imgs)
|
| 148 |
|
|
|
|
| 149 |
out_path = Path(path).with_suffix(".depth.mp4")
|
| 150 |
+
src_path = Path(path).with_suffix(".source.mp4")
|
| 151 |
written = 0
|
| 152 |
+
with av.open(out_path, mode="w") as out, av.open(src_path, mode="w") as sout:
|
| 153 |
+
writer = None
|
| 154 |
+
src_writer = None
|
| 155 |
+
for img, prediction in zip(imgs, predictions):
|
| 156 |
rgb = _colorize(prediction["depth"].float().cpu().numpy())
|
| 157 |
if writer is None:
|
| 158 |
height, width = rgb.shape[:2]
|
| 159 |
writer = out.add_stream("libx264", rate=fps)
|
| 160 |
writer.width, writer.height = width - width % 2, height - height % 2
|
| 161 |
writer.pix_fmt = "yuv420p"
|
| 162 |
+
# The source at the depth map's size, so both windows encode the same frames.
|
| 163 |
+
src_writer = sout.add_stream("libx264", rate=fps)
|
| 164 |
+
src_writer.width, src_writer.height = writer.width, writer.height
|
| 165 |
+
src_writer.pix_fmt = "yuv420p"
|
| 166 |
+
for stream, muxer, pixels in ((writer, out, rgb), (src_writer, sout, img)):
|
| 167 |
+
frame = av.VideoFrame.from_ndarray(
|
| 168 |
+
np.ascontiguousarray(pixels[: writer.height, : writer.width]), format="rgb24"
|
| 169 |
+
)
|
| 170 |
+
for packet in stream.encode(frame):
|
| 171 |
+
muxer.mux(packet)
|
| 172 |
written += 1
|
|
|
|
|
|
|
| 173 |
if writer is not None:
|
| 174 |
for packet in writer.encode():
|
| 175 |
out.mux(packet)
|
| 176 |
+
for packet in src_writer.encode():
|
| 177 |
+
sout.mux(packet)
|
| 178 |
if writer is None:
|
| 179 |
raise ValueError("Could not decode any frames from that video.")
|
| 180 |
+
print(f"infer_video: {written} frames, depth {out_path.stat().st_size} B, source {src_path.stat().st_size} B", flush=True)
|
| 181 |
+
return {
|
| 182 |
+
"image_data": "data:video/mp4;base64," + base64.b64encode(out_path.read_bytes()).decode(),
|
| 183 |
+
"source": "data:video/mp4;base64," + base64.b64encode(src_path.read_bytes()).decode(),
|
| 184 |
+
}
|
| 185 |
|
| 186 |
|
| 187 |
app = Server(
|
|
|
|
| 220 |
|
| 221 |
@app.api(name="predict_depth_video")
|
| 222 |
def predict_depth_video(video: FileData, request: gr.Request) -> dict:
|
| 223 |
+
"""Estimate metric depth for an uploaded video (first 60 frames); returns source + depth data URIs."""
|
| 224 |
+
return _infer_video(video["path"])
|
| 225 |
|
| 226 |
|
| 227 |
@app.get("/gradio-client.js")
|