akhaliq HF Staff commited on
Commit
59ee84a
·
verified ·
1 Parent(s): 8883091

Video: return H.264 re-encoded source alongside depth (input window playable for HEVC uploads)

Browse files
Files changed (1) hide show
  1. app.py +39 -19
app.py CHANGED
@@ -124,44 +124,64 @@ def _infer_image(img: np.ndarray, f_px: float | None) -> np.ndarray:
124
 
125
 
126
  @_gpu(240)
127
- def _infer_video(path: str) -> str:
128
- """Depth colorization for a video: first 60 frames, each at most 960 px on the longer side."""
 
 
 
 
 
129
  container = av.open(path)
130
  rate = container.streams.video[0].average_rate
131
  fps = int(round(float(rate))) if rate else 24
132
 
133
- def frames():
134
- for frame in container.decode(video=0):
135
- img, _ = _resize(upright(frame), None, MAX_VIDEO_SIDE)
136
- yield img
 
 
 
137
 
138
  net, transform = _device_and_model()
139
- predictions = net.infer_video(transform(f) for f in frames())
140
 
141
- # Beside the uploaded input, which is in Gradio's servable upload cache.
142
  out_path = Path(path).with_suffix(".depth.mp4")
143
- writer = None
144
  written = 0
145
- with av.open(out_path, mode="w") as out:
146
- for prediction in predictions:
 
 
147
  rgb = _colorize(prediction["depth"].float().cpu().numpy())
148
  if writer is None:
149
  height, width = rgb.shape[:2]
150
  writer = out.add_stream("libx264", rate=fps)
151
  writer.width, writer.height = width - width % 2, height - height % 2
152
  writer.pix_fmt = "yuv420p"
153
- frame = av.VideoFrame.from_ndarray(rgb[: writer.height, : writer.width], format="rgb24")
154
- for packet in writer.encode(frame):
155
- out.mux(packet)
 
 
 
 
 
 
 
156
  written += 1
157
- if written >= MAX_VIDEO_FRAMES:
158
- break
159
  if writer is not None:
160
  for packet in writer.encode():
161
  out.mux(packet)
 
 
162
  if writer is None:
163
  raise ValueError("Could not decode any frames from that video.")
164
- return str(out_path)
 
 
 
 
165
 
166
 
167
  app = Server(
@@ -200,8 +220,8 @@ def predict_depth(image: FileData, request: gr.Request) -> str:
200
 
201
  @app.api(name="predict_depth_video")
202
  def predict_depth_video(video: FileData, request: gr.Request) -> dict:
203
- """Estimate metric depth for an uploaded video (first 60 frames); returns a data URI."""
204
- return {"image_data": _infer_video(video["path"])}
205
 
206
 
207
  @app.get("/gradio-client.js")
 
124
 
125
 
126
  @_gpu(240)
127
+ def _infer_video(path: str) -> dict:
128
+ """Depth colorization for a video (first 60 frames, up to 960 px); returns source + depth data URIs.
129
+
130
+ The source is re-encoded to H.264 from the already-decoded frames: phone videos are often
131
+ HEVC, which Chrome cannot decode, and an unplayable input window otherwise reads as if the
132
+ depth result failed. The frames are already in memory, so the second encode is cheap.
133
+ """
134
  container = av.open(path)
135
  rate = container.streams.video[0].average_rate
136
  fps = int(round(float(rate))) if rate else 24
137
 
138
+ imgs = []
139
+ for frame in container.decode(video=0):
140
+ imgs.append(_resize(upright(frame), None, MAX_VIDEO_SIDE)[0])
141
+ if len(imgs) >= MAX_VIDEO_FRAMES:
142
+ break
143
+ if not imgs:
144
+ raise ValueError("Could not decode any frames from that video.")
145
 
146
  net, transform = _device_and_model()
147
+ predictions = net.infer_video(transform(f) for f in imgs)
148
 
 
149
  out_path = Path(path).with_suffix(".depth.mp4")
150
+ src_path = Path(path).with_suffix(".source.mp4")
151
  written = 0
152
+ with av.open(out_path, mode="w") as out, av.open(src_path, mode="w") as sout:
153
+ writer = None
154
+ src_writer = None
155
+ for img, prediction in zip(imgs, predictions):
156
  rgb = _colorize(prediction["depth"].float().cpu().numpy())
157
  if writer is None:
158
  height, width = rgb.shape[:2]
159
  writer = out.add_stream("libx264", rate=fps)
160
  writer.width, writer.height = width - width % 2, height - height % 2
161
  writer.pix_fmt = "yuv420p"
162
+ # The source at the depth map's size, so both windows encode the same frames.
163
+ src_writer = sout.add_stream("libx264", rate=fps)
164
+ src_writer.width, src_writer.height = writer.width, writer.height
165
+ src_writer.pix_fmt = "yuv420p"
166
+ for stream, muxer, pixels in ((writer, out, rgb), (src_writer, sout, img)):
167
+ frame = av.VideoFrame.from_ndarray(
168
+ np.ascontiguousarray(pixels[: writer.height, : writer.width]), format="rgb24"
169
+ )
170
+ for packet in stream.encode(frame):
171
+ muxer.mux(packet)
172
  written += 1
 
 
173
  if writer is not None:
174
  for packet in writer.encode():
175
  out.mux(packet)
176
+ for packet in src_writer.encode():
177
+ sout.mux(packet)
178
  if writer is None:
179
  raise ValueError("Could not decode any frames from that video.")
180
+ print(f"infer_video: {written} frames, depth {out_path.stat().st_size} B, source {src_path.stat().st_size} B", flush=True)
181
+ return {
182
+ "image_data": "data:video/mp4;base64," + base64.b64encode(out_path.read_bytes()).decode(),
183
+ "source": "data:video/mp4;base64," + base64.b64encode(src_path.read_bytes()).decode(),
184
+ }
185
 
186
 
187
  app = Server(
 
220
 
221
  @app.api(name="predict_depth_video")
222
  def predict_depth_video(video: FileData, request: gr.Request) -> dict:
223
+ """Estimate metric depth for an uploaded video (first 60 frames); returns source + depth data URIs."""
224
+ return _infer_video(video["path"])
225
 
226
 
227
  @app.get("/gradio-client.js")