phrographica commited on
Commit
89b4b4d
·
verified ·
1 Parent(s): 7c179c3

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +291 -505
app.py CHANGED
@@ -12,16 +12,17 @@
12
  # See the License for the specific language governing permissions and
13
  # limitations under the License.
14
 
 
 
15
  import gc
16
  import os
17
  import subprocess
 
18
  import uuid
19
- import zipfile
20
 
21
  import gradio as gr
22
- import Imath
23
  import numpy as np
24
- import OpenEXR
25
  import imageio_ffmpeg
26
  import spaces
27
  import torch
@@ -59,22 +60,10 @@ def _sdpa_shim(
59
 
60
  xFormers uses tensors shaped [B, M, H, K].
61
  PyTorch SDPA expects [B, H, M, K].
62
-
63
- Video Depth Anything inference normally uses attn_bias=None. If a
64
- non-Tensor xFormers-specific bias object is encountered, fall back to
65
- xFormers' original implementation.
66
  """
67
 
68
  if attn_bias is not None and not isinstance(attn_bias, torch.Tensor):
69
- return _xf_original(
70
- query,
71
- key,
72
- value,
73
- attn_bias,
74
- p,
75
- scale,
76
- op,
77
- )
78
 
79
  squeeze = query.dim() == 3
80
 
@@ -84,18 +73,12 @@ def _sdpa_shim(
84
  q, k, v = (t.transpose(1, 2) for t in (query, key, value))
85
 
86
  out = F.scaled_dot_product_attention(
87
- q,
88
- k,
89
- v,
90
- attn_mask=attn_bias,
91
- dropout_p=p,
92
- scale=scale,
93
  )
94
 
95
  return out.squeeze(1) if squeeze else out.transpose(1, 2)
96
 
97
 
98
- # Patch xFormers BEFORE importing VideoDepthAnything / bundled DINOv2.
99
  xformers.ops.memory_efficient_attention = _sdpa_shim
100
 
101
  print(
@@ -109,7 +92,7 @@ print(
109
  # =============================================================================
110
 
111
  from video_depth_anything.video_depth import VideoDepthAnything
112
- from utils.dc_utils import read_video_frames, save_video
113
 
114
 
115
  # =============================================================================
@@ -137,30 +120,17 @@ examples = [
137
  DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
138
 
139
  model_configs = {
140
- "vits": {
141
- "encoder": "vits",
142
- "features": 64,
143
- "out_channels": [48, 96, 192, 384],
144
- },
145
- "vitl": {
146
- "encoder": "vitl",
147
- "features": 256,
148
- "out_channels": [256, 512, 1024, 1024],
149
- },
150
  }
151
 
152
- encoder2name = {
153
- "vits": "Small",
154
- "vitl": "Large",
155
- }
156
 
157
- # The Large model processes 32 temporal frames together. Values above the
158
- # standard 518 setting can exceed a ZeroGPU allocation during the DPT decoder,
159
- # especially for 16:9 footage, even when the GPU reports substantial total
160
- # memory. Output resolution is controlled separately by max_res.
161
  SAFE_VITL_INPUT_SIZE = 518
162
 
163
- # Large model
164
  encoder = "vitl"
165
  model_name = encoder2name[encoder]
166
 
@@ -172,257 +142,242 @@ filepath = hf_hub_download(
172
  repo_type="model",
173
  )
174
 
175
- video_depth_anything.load_state_dict(
176
- torch.load(filepath, map_location="cpu")
177
- )
178
-
179
  video_depth_anything = video_depth_anything.to(DEVICE).eval()
180
 
181
 
182
  # =============================================================================
183
- # UI TEXT
184
  # =============================================================================
185
 
186
- title = "# Video Depth Anything"
187
-
188
- description = """
189
- Official demo for **Video Depth Anything**, modified to export the model's
190
- depth prediction in three useful formats.
191
-
192
- - **MP4 depth preview:** normalized/compressed visualization for viewing.
193
- - **16-bit grayscale depth video:** lossless FFV1 master with 65,536 gray
194
- levels, full processed resolution, and no chroma subsampling.
195
- - **EXR ZIP:** raw floating-point depth values for Blender displacement.
196
 
197
- For Blender displacement, the EXR sequence remains the best choice. The
198
- 16-bit MKV is an additional high-quality grayscale video for editing,
199
- archiving, and workflows that do not need the raw floating-point values.
200
- """
201
 
202
 
203
  # =============================================================================
204
- # EXR EXPORT
205
  # =============================================================================
206
 
207
- def save_depths_as_rgb_exr(depths: np.ndarray, output_dir: str) -> None:
208
- """Save raw depth values as 32-bit float RGB OpenEXR frames.
209
-
210
- The same raw depth value is written to R, G, and B so Blender can read
211
- the sequence directly through a regular Image Texture node.
212
 
213
- No 0-1 normalization, uint8 conversion, colormap, or video compression
214
- is applied to these files.
 
215
  """
216
 
217
- os.makedirs(output_dir, exist_ok=True)
 
218
 
219
- float_channel = Imath.PixelType(Imath.PixelType.FLOAT)
 
220
 
221
- for i, depth in enumerate(depths):
222
- depth = np.ascontiguousarray(depth, dtype=np.float32)
 
 
 
223
 
224
- if depth.ndim != 2:
225
- raise ValueError(
226
- f"Expected each depth frame to have shape HxW, got {depth.shape}"
227
- )
228
 
229
- height, width = depth.shape
230
- exr_path = os.path.join(output_dir, f"frame_{i:05d}.exr")
231
 
232
- header = OpenEXR.Header(width, height)
233
- header["channels"] = {
234
- "R": Imath.Channel(float_channel),
235
- "G": Imath.Channel(float_channel),
236
- "B": Imath.Channel(float_channel),
237
- }
238
 
239
- depth_bytes = depth.tobytes()
240
 
241
- exr_file = OpenEXR.OutputFile(exr_path, header)
242
-
243
- try:
244
- exr_file.writePixels(
245
- {
246
- "R": depth_bytes,
247
- "G": depth_bytes,
248
- "B": depth_bytes,
249
- }
250
- )
251
- finally:
252
- exr_file.close()
253
 
 
 
 
254
 
255
- def zip_directory(source_dir: str, zip_path: str) -> str:
256
- """Zip an EXR sequence so Gradio can return it as one download."""
257
 
258
- with zipfile.ZipFile(
259
- zip_path,
260
- mode="w",
261
- compression=zipfile.ZIP_DEFLATED,
262
- ) as zip_file:
263
 
264
- for filename in sorted(os.listdir(source_dir)):
265
- file_path = os.path.join(source_dir, filename)
266
 
267
- if os.path.isfile(file_path):
268
- zip_file.write(
269
- file_path,
270
- arcname=filename,
271
- )
272
 
273
- return zip_path
274
 
275
 
276
  # =============================================================================
277
- # LOSSLESS 16-BIT GRAYSCALE VIDEO EXPORT
278
  # =============================================================================
279
 
280
- def save_depths_as_lossless_gray16_video(
281
  depths: np.ndarray,
282
  output_path: str,
283
  fps: float,
284
- ) -> str:
285
- """Save a full-resolution, lossless 16-bit grayscale FFV1 video.
286
-
287
- Raw VDA values are relative depth and may extend outside 0-1. They are
288
- normalized once using the minimum and maximum of the entire clip, then
289
- quantized to 16 bits. Clip-wide normalization avoids the brightness
290
- pumping that per-frame normalization would introduce.
291
-
292
- FFV1 in Matroska is used because normal H.264 MP4 output is generally
293
- 8-bit and lossy. FFV1 preserves all 65,536 grayscale code values without
294
- chroma subsampling or inter-frame loss. The raw 32-bit values are still
295
- preserved separately in the EXR sequence.
 
 
 
 
 
 
296
  """
297
 
298
- depths = np.asarray(depths, dtype=np.float32)
299
 
300
  if depths.ndim != 3 or depths.shape[0] == 0:
301
- raise ValueError(
302
- f"Expected depth video with shape TxHxW, got {depths.shape}"
303
- )
304
 
305
  if not np.isfinite(fps) or fps <= 0:
306
  raise ValueError(f"FPS must be positive, got {fps}")
307
 
308
- depth_min = float(np.nanmin(depths))
309
- depth_max = float(np.nanmax(depths))
310
-
311
- if not np.isfinite(depth_min) or not np.isfinite(depth_max):
312
- raise ValueError("Depth output contains no usable finite range")
313
-
314
- depth_range = depth_max - depth_min
315
-
316
- if depth_range <= np.finfo(np.float32).eps:
317
- raise ValueError(
318
- "Depth output is effectively constant, so a grayscale video "
319
- "cannot be normalized"
320
- )
321
-
322
- os.makedirs(os.path.dirname(output_path) or ".", exist_ok=True)
323
 
324
  height, width = depths.shape[1:]
325
- ffmpeg_exe = imageio_ffmpeg.get_ffmpeg_exe()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
326
 
327
  command = [
328
- ffmpeg_exe,
329
- "-y",
330
- "-hide_banner",
331
- "-loglevel",
332
- "error",
333
- "-f",
334
- "rawvideo",
335
- "-pixel_format",
336
- "gray16le",
337
- "-video_size",
338
- f"{width}x{height}",
339
- "-framerate",
340
- f"{float(fps):.8f}",
341
- "-i",
342
- "-",
343
  "-an",
344
- "-c:v",
345
- "ffv1",
346
- "-level",
347
- "3",
348
- "-coder",
349
- "1",
350
- "-context",
351
- "1",
352
- "-g",
353
- "1",
354
- "-slicecrc",
355
- "1",
356
- "-pix_fmt",
357
- "gray16le",
358
  output_path,
359
  ]
360
 
361
- process = subprocess.Popen(
362
- command,
363
- stdin=subprocess.PIPE,
364
- stdout=subprocess.DEVNULL,
365
- stderr=subprocess.PIPE,
366
- )
367
 
368
- try:
369
- for depth in depths:
370
- # Replace exceptional values only in the downloadable video. The
371
- # raw EXR sequence remains untouched.
372
- clean_depth = np.nan_to_num(
373
- depth,
374
- nan=depth_min,
375
- posinf=depth_max,
376
- neginf=depth_min,
377
- )
378
- normalized = np.clip(
379
- (clean_depth - depth_min) / depth_range,
380
- 0.0,
381
- 1.0,
382
- )
383
- gray16 = np.rint(normalized * 65535.0).astype("<u2")
384
- process.stdin.write(np.ascontiguousarray(gray16).tobytes())
385
-
386
- process.stdin.close()
387
- process.stdin = None
388
- stderr = process.stderr.read().decode("utf-8", errors="replace")
389
- return_code = process.wait()
390
-
391
- if return_code != 0:
392
- raise RuntimeError(
393
- "FFmpeg could not create the lossless 16-bit depth video: "
394
- + (stderr.strip() or f"exit code {return_code}")
395
- )
396
 
397
- except Exception:
398
- if process.poll() is None:
399
- process.kill()
400
- process.wait()
401
 
402
- if os.path.exists(output_path):
403
- os.remove(output_path)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
404
 
405
- raise
 
 
 
406
 
407
- finally:
408
- if process.stderr is not None:
409
- process.stderr.close()
 
410
 
411
- return output_path
 
 
 
 
 
 
 
 
412
 
413
 
414
  # =============================================================================
415
  # INFERENCE
416
  # =============================================================================
417
 
418
- @spaces.GPU(duration=240)
419
  def infer_video_depth(
420
  input_video: str,
421
  max_len: int = -1,
422
  target_fps: int = -1,
423
  max_res: int = 1280,
424
  input_size: int = 518,
425
- grayscale: bool = True,
 
 
 
426
  output_dir: str = "./outputs",
427
  ):
428
 
@@ -433,52 +388,27 @@ def infer_video_depth(
433
  target_fps = int(target_fps)
434
  max_res = int(max_res)
435
 
436
- # VDA/DINOv2 patch size is 14, so keep inference size on a multiple of 14.
437
- input_size = int(round(float(input_size) / 14.0) * 14)
438
- input_size = max(input_size, 14)
439
 
440
  if encoder == "vitl" and input_size > SAFE_VITL_INPUT_SIZE:
441
- gr.Warning(
442
- "The Large model is capped at inference size 518 on ZeroGPU to "
443
- "avoid GPU-memory crashes. Max Side Resolution still controls "
444
- "the resolution of the exported video and EXRs."
445
- )
446
  input_size = SAFE_VITL_INPUT_SIZE
447
 
448
- print(f"Using model inference size: {input_size}")
449
-
450
- # -------------------------------------------------------------------------
451
- # Read / preprocess source video
452
- # -------------------------------------------------------------------------
453
-
454
- frames, actual_fps = read_video_frames(
455
- input_video,
456
- max_len,
457
- target_fps,
458
- max_res,
459
- )
460
-
461
- print(f"Loaded {len(frames)} frames")
462
-
463
- # -------------------------------------------------------------------------
464
- # Raw Video Depth Anything inference
465
- # -------------------------------------------------------------------------
466
 
467
  gc.collect()
468
-
469
  if torch.cuda.is_available():
470
  torch.cuda.empty_cache()
471
 
472
  try:
473
  depth_list, fps = video_depth_anything.infer_video_depth(
474
- frames,
475
- actual_fps,
476
- input_size=input_size,
477
- device=DEVICE,
478
  )
479
  except RuntimeError as error:
480
  error_text = str(error).lower()
481
- gpu_memory_error = any(
482
  marker in error_text
483
  for marker in (
484
  "out of memory",
@@ -486,13 +416,10 @@ def infer_video_depth(
486
  "cudacachingallocator",
487
  "nvml_success",
488
  )
489
- )
490
-
491
- if not gpu_memory_error:
492
  raise
493
 
494
  gc.collect()
495
-
496
  if torch.cuda.is_available():
497
  try:
498
  torch.cuda.empty_cache()
@@ -501,142 +428,69 @@ def infer_video_depth(
501
 
502
  raise gr.Error(
503
  "ZeroGPU ran out of memory during depth inference. Try Model "
504
- "Inference Size 448, or 392 if necessary. You can leave Max Side "
505
- "Resolution at 1920 because it controls export resolution rather "
506
- "than model-inference memory."
507
  ) from error
508
 
509
- # Preserve the model output as float32 BEFORE making any visualization.
510
- depths = np.asarray(depth_list, dtype=np.float32)
511
-
512
- print("Raw depth shape:", depths.shape)
513
- print(
514
- "Raw depth range:",
515
- float(depths.min()),
516
- "to",
517
- float(depths.max()),
518
- )
519
-
520
- # -------------------------------------------------------------------------
521
- # Unique output folder for each run
522
- # -------------------------------------------------------------------------
523
-
524
- video_name = os.path.basename(input_video)
525
- base_name = os.path.splitext(video_name)[0]
526
- run_id = uuid.uuid4().hex[:8]
527
-
528
- run_dir = os.path.join(
529
- output_dir,
530
- f"{base_name}_{run_id}",
531
- )
532
-
533
- os.makedirs(run_dir, exist_ok=True)
534
-
535
- processed_video_path = os.path.join(
536
- run_dir,
537
- f"{base_name}_src.mp4",
538
- )
539
-
540
- depth_vis_path = os.path.join(
541
- run_dir,
542
- f"{base_name}_vis.mp4",
543
- )
544
-
545
- depth_gray16_path = os.path.join(
546
- run_dir,
547
- f"{base_name}_depth_gray16_lossless.mkv",
548
- )
549
-
550
- depth_exr_dir = os.path.join(
551
- run_dir,
552
- f"{base_name}_depth_EXR",
553
- )
554
-
555
- zip_path = os.path.join(
556
- run_dir,
557
- f"{base_name}_depth_EXR.zip",
558
- )
559
-
560
- # -------------------------------------------------------------------------
561
- # Save preprocessed source video
562
- # -------------------------------------------------------------------------
563
-
564
- save_video(
565
- frames,
566
- processed_video_path,
567
- fps=fps,
568
- )
569
-
570
- # -------------------------------------------------------------------------
571
- # Save normalized MP4 preview
572
- # -------------------------------------------------------------------------
573
-
574
- save_video(
575
- depths,
576
- depth_vis_path,
577
- fps=fps,
578
- is_depths=True,
579
- grayscale=bool(grayscale),
580
- )
581
-
582
- # -------------------------------------------------------------------------
583
- # Save lossless 16-bit grayscale depth video
584
- # -------------------------------------------------------------------------
585
-
586
- print("Saving lossless 16-bit grayscale FFV1 video...")
587
-
588
- save_depths_as_lossless_gray16_video(
589
- depths,
590
- depth_gray16_path,
591
- fps=fps,
592
- )
593
-
594
- print("16-bit grayscale video:", depth_gray16_path)
595
-
596
- # -------------------------------------------------------------------------
597
- # Save RAW floating-point EXR sequence
598
- # -------------------------------------------------------------------------
599
-
600
- print("Saving raw 32-bit EXR sequence...")
601
 
602
- save_depths_as_rgb_exr(
603
- depths,
604
- depth_exr_dir,
605
- )
606
 
607
- print("Creating EXR ZIP...")
 
 
 
608
 
609
- zip_directory(
610
- depth_exr_dir,
611
- zip_path,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
612
  )
613
 
614
- print("EXR ZIP:", zip_path)
615
-
616
- # -------------------------------------------------------------------------
617
- # Cleanup
618
- # -------------------------------------------------------------------------
619
 
620
- del depths
621
- del depth_list
622
- del frames
623
 
624
- gc.collect()
 
 
625
 
626
- if torch.cuda.is_available():
627
- torch.cuda.empty_cache()
628
 
629
- return [
630
- processed_video_path,
631
- depth_vis_path,
632
- depth_gray16_path,
633
- zip_path,
634
- ]
635
 
 
 
 
 
 
636
 
637
- # =============================================================================
638
- # GRADIO UI
639
- # =============================================================================
640
 
641
  def construct_demo():
642
 
@@ -645,154 +499,91 @@ def construct_demo():
645
  gr.Markdown(title)
646
  gr.Markdown(description)
647
 
648
- with gr.Row(equal_height=True):
649
 
650
  with gr.Column(scale=1):
651
 
652
- input_video = gr.Video(
653
- label="Input Video"
654
- )
655
-
656
- with gr.Column(scale=2):
657
-
658
- with gr.Row(equal_height=True):
659
-
660
- processed_video = gr.Video(
661
- label="Preprocessed Video",
662
- interactive=False,
663
- autoplay=True,
664
- loop=True,
665
- show_share_button=True,
666
- scale=5,
667
- )
668
-
669
- depth_vis_video = gr.Video(
670
- label="Generated Depth Preview",
671
- interactive=False,
672
- autoplay=True,
673
- loop=True,
674
- show_share_button=True,
675
- scale=5,
676
- )
677
-
678
- depth_exr_file = gr.File(
679
- label="Raw 32-bit EXR Depth Sequence (.zip)",
680
- interactive=False,
681
- )
682
-
683
- depth_gray16_file = gr.File(
684
- label=(
685
- "Lossless 16-bit Grayscale Depth Video "
686
- "(FFV1 .mkv)"
687
- ),
688
- interactive=False,
689
- )
690
-
691
- with gr.Row(equal_height=True):
692
-
693
- with gr.Column(scale=1):
694
 
695
- with gr.Accordion(
696
- "Advanced Settings",
697
- open=True,
698
- ):
699
 
700
  max_len = gr.Slider(
701
  label="Max Process Length (frames; -1 = all)",
702
- minimum=-1,
703
- maximum=1000,
704
- value=500,
705
- step=1,
706
  )
707
 
708
  target_fps = gr.Slider(
709
  label="Target FPS (-1 = original)",
710
- minimum=-1,
711
- maximum=30,
712
- value=15,
713
- step=1,
714
  )
715
 
716
  max_res = gr.Slider(
717
  label="Max Side Resolution",
718
- minimum=480,
719
- maximum=2560,
720
- value=1280,
721
- step=1,
722
- info=(
723
- "Controls the maximum size of the source/depth frames. "
724
- "This is separate from the model inference size."
725
- ),
726
  )
727
 
728
  input_size = gr.Slider(
729
  label="Model Inference Size",
730
- minimum=392,
731
- maximum=SAFE_VITL_INPUT_SIZE,
732
- value=518,
733
- step=14,
734
- info=(
735
- "518 is the standard safe setting for the Large "
736
- "model on ZeroGPU. If memory remains tight, use "
737
- "448 or 392. This does not set export resolution."
738
- ),
739
  )
740
 
741
- grayscale = gr.Checkbox(
742
- label="Grayscale MP4 Preview",
743
- value=True,
 
 
 
 
 
 
 
744
  info=(
745
- "Only affects the preview MP4. The EXRs always contain "
746
- "the raw floating-point depth."
747
  ),
748
  )
749
 
750
- generate_btn = gr.Button(
751
- "Generate Depth + 16-bit Video + EXR",
752
- variant="primary",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
753
  )
754
 
755
- with gr.Column(scale=2):
756
 
757
  gr.Markdown(
758
  """
759
- ### Blender EXR setup
760
-
761
- 1. Unzip the downloaded EXR file.
762
- 2. Load `frame_00000.exr` as an **Image Sequence**.
763
- 3. Set the Image Texture to **Non-Color**.
764
- 4. Use **Cubic** interpolation.
765
- 5. Read the **R** channel.
766
- 6. Feed it through **Map Range** before displacement.
767
 
768
- The EXRs may look very dark in a normal image viewer. That is normal: they
769
- contain raw floating-point model values instead of a normalized 8-bit preview.
 
770
 
771
- ### 16-bit grayscale video
772
-
773
- The MKV download uses lossless FFV1 at the full processed frame resolution.
774
- It stores 65,536 grayscale levels instead of the 256 levels in a normal 8-bit
775
- MP4. VLC and FFmpeg-based tools can read FFV1/MKV. Some web browsers will not
776
- preview it, so it is provided as a download rather than an embedded player.
777
  """
778
  )
779
 
780
- # Examples retain the original four settings. The inference-size and
781
- # grayscale parameters use their function defaults for example runs.
782
  gr.Examples(
783
  examples=examples,
784
- inputs=[
785
- input_video,
786
- max_len,
787
- target_fps,
788
- max_res,
789
- ],
790
- outputs=[
791
- processed_video,
792
- depth_vis_video,
793
- depth_gray16_file,
794
- depth_exr_file,
795
- ],
796
  fn=infer_video_depth,
797
  cache_examples="lazy",
798
  )
@@ -805,23 +596,18 @@ preview it, so it is provided as a download rather than an embedded player.
805
  target_fps,
806
  max_res,
807
  input_size,
808
- grayscale,
809
- ],
810
- outputs=[
811
- processed_video,
812
- depth_vis_video,
813
- depth_gray16_file,
814
- depth_exr_file,
815
  ],
 
816
  )
817
 
818
  return demo
819
 
820
 
821
  if __name__ == "__main__":
822
-
823
  demo = construct_demo()
824
  demo.queue()
825
-
826
- # On Hugging Face Spaces, a public share URL is unnecessary.
827
- demo.launch()
 
12
  # See the License for the specific language governing permissions and
13
  # limitations under the License.
14
 
15
+ # Modified: single banding-free depth MP4 output (10-bit HEVC by default).
16
+
17
  import gc
18
  import os
19
  import subprocess
20
+ import tempfile
21
  import uuid
22
+ from fractions import Fraction
23
 
24
  import gradio as gr
 
25
  import numpy as np
 
26
  import imageio_ffmpeg
27
  import spaces
28
  import torch
 
60
 
61
  xFormers uses tensors shaped [B, M, H, K].
62
  PyTorch SDPA expects [B, H, M, K].
 
 
 
 
63
  """
64
 
65
  if attn_bias is not None and not isinstance(attn_bias, torch.Tensor):
66
+ return _xf_original(query, key, value, attn_bias, p, scale, op)
 
 
 
 
 
 
 
 
67
 
68
  squeeze = query.dim() == 3
69
 
 
73
  q, k, v = (t.transpose(1, 2) for t in (query, key, value))
74
 
75
  out = F.scaled_dot_product_attention(
76
+ q, k, v, attn_mask=attn_bias, dropout_p=p, scale=scale
 
 
 
 
 
77
  )
78
 
79
  return out.squeeze(1) if squeeze else out.transpose(1, 2)
80
 
81
 
 
82
  xformers.ops.memory_efficient_attention = _sdpa_shim
83
 
84
  print(
 
92
  # =============================================================================
93
 
94
  from video_depth_anything.video_depth import VideoDepthAnything
95
+ from utils.dc_utils import read_video_frames
96
 
97
 
98
  # =============================================================================
 
120
  DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
121
 
122
  model_configs = {
123
+ "vits": {"encoder": "vits", "features": 64, "out_channels": [48, 96, 192, 384]},
124
+ "vitl": {"encoder": "vitl", "features": 256, "out_channels": [256, 512, 1024, 1024]},
 
 
 
 
 
 
 
 
125
  }
126
 
127
+ encoder2name = {"vits": "Small", "vitl": "Large"}
 
 
 
128
 
129
+ # The Large model processes 32 temporal frames together. Values above 518 can
130
+ # exceed a ZeroGPU allocation in the DPT decoder. Output resolution is set
131
+ # separately by max_res.
 
132
  SAFE_VITL_INPUT_SIZE = 518
133
 
 
134
  encoder = "vitl"
135
  model_name = encoder2name[encoder]
136
 
 
142
  repo_type="model",
143
  )
144
 
145
+ video_depth_anything.load_state_dict(torch.load(filepath, map_location="cpu"))
 
 
 
146
  video_depth_anything = video_depth_anything.to(DEVICE).eval()
147
 
148
 
149
  # =============================================================================
150
+ # ENCODING OPTIONS
151
  # =============================================================================
152
 
153
+ ENC_HEVC10 = "10-bit HEVC (no banding — recommended)"
154
+ ENC_H264_DITHER = "8-bit H.264 + dither (plays everywhere)"
 
 
 
 
 
 
 
 
155
 
156
+ NORM_FULL = "Full clip range"
157
+ NORM_ROBUST = "Robust (ignore 0.5% outliers)"
 
 
158
 
159
 
160
  # =============================================================================
161
+ # NORMALIZATION
162
  # =============================================================================
163
 
164
+ def compute_depth_range(depths: np.ndarray, mode: str) -> tuple[float, float]:
165
+ """Return one (low, high) pair for the whole clip.
 
 
 
166
 
167
+ A single clip-wide range keeps brightness stable over time (no pumping).
168
+ Robust mode uses percentiles on a strided sample so a few extreme pixels
169
+ (sky, specular hits, edge spikes) don't eat most of the code values.
170
  """
171
 
172
+ finite_sample = depths[:, ::4, ::4]
173
+ finite_sample = finite_sample[np.isfinite(finite_sample)]
174
 
175
+ if finite_sample.size == 0:
176
+ raise ValueError("Depth output contains no finite values")
177
 
178
+ if mode == NORM_ROBUST:
179
+ low, high = np.percentile(finite_sample, [0.5, 99.5])
180
+ else:
181
+ low = float(np.nanmin(depths))
182
+ high = float(np.nanmax(depths))
183
 
184
+ low, high = float(low), float(high)
 
 
 
185
 
186
+ if high - low <= np.finfo(np.float32).eps:
187
+ raise ValueError("Depth output is effectively constant; nothing to encode")
188
 
189
+ return low, high
 
 
 
 
 
190
 
 
191
 
192
+ def normalize_frame(
193
+ depth: np.ndarray,
194
+ low: float,
195
+ high: float,
196
+ invert: bool,
197
+ ) -> np.ndarray:
198
+ """Map one raw depth frame to float64 0..1 (near = 1 unless inverted)."""
 
 
 
 
 
199
 
200
+ clean = np.nan_to_num(depth.astype(np.float64), nan=low, posinf=high, neginf=low)
201
+ norm = np.clip((clean - low) / (high - low), 0.0, 1.0)
202
+ return 1.0 - norm if invert else norm
203
 
 
 
204
 
205
+ def pad_to_even(img: np.ndarray) -> np.ndarray:
206
+ """4:2:0 video needs even dimensions; replicate the last row/column if odd."""
 
 
 
207
 
208
+ h, w = img.shape
209
+ pad_h, pad_w = h % 2, w % 2
210
 
211
+ if pad_h or pad_w:
212
+ img = np.pad(img, ((0, pad_h), (0, pad_w)), mode="edge")
 
 
 
213
 
214
+ return img
215
 
216
 
217
  # =============================================================================
218
+ # MP4 ENCODER
219
  # =============================================================================
220
 
221
+ def encode_depth_mp4(
222
  depths: np.ndarray,
223
  output_path: str,
224
  fps: float,
225
+ encoding: str = ENC_HEVC10,
226
+ normalization: str = NORM_FULL,
227
+ invert: bool = False,
228
+ crf: int = 10,
229
+ ) -> dict:
230
+ """Encode raw float depth straight to a high-quality grayscale MP4.
231
+
232
+ Banding comes from too few gray levels plus an encoder that flattens
233
+ smooth gradients. This avoids both:
234
+
235
+ * Quantization is done here in float64 with exact rounding and the planes
236
+ are piped to FFmpeg as raw YUV, so no swscale range/precision step
237
+ sits in between.
238
+ * 10-bit HEVC: 877 luma steps (limited range 64-940) instead of 220 in
239
+ 8-bit video, encoded with SAO off (SAO smears gradients into steps) and
240
+ aq-mode 3 (spends bits on flat/dark regions where banding shows).
241
+ * 8-bit H.264 fallback: triangular (TPDF) dither before quantizing, so
242
+ the 220 levels read as a smooth gradient instead of contour lines.
243
  """
244
 
245
+ depths = np.asarray(depths)
246
 
247
  if depths.ndim != 3 or depths.shape[0] == 0:
248
+ raise ValueError(f"Expected depth video shaped TxHxW, got {depths.shape}")
 
 
249
 
250
  if not np.isfinite(fps) or fps <= 0:
251
  raise ValueError(f"FPS must be positive, got {fps}")
252
 
253
+ low, high = compute_depth_range(depths, normalization)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
254
 
255
  height, width = depths.shape[1:]
256
+ enc_h, enc_w = height + height % 2, width + width % 2
257
+
258
+ rate = Fraction(float(fps)).limit_denominator(1001)
259
+ rate_str = f"{rate.numerator}/{rate.denominator}"
260
+
261
+ is_hevc10 = encoding == ENC_HEVC10
262
+
263
+ if is_hevc10:
264
+ raw_pix_fmt = "yuv420p10le"
265
+ chroma = np.full((enc_h // 2, enc_w // 2), 512, dtype="<u2").tobytes()
266
+ codec_args = [
267
+ "-c:v", "libx265",
268
+ "-preset", "medium",
269
+ "-crf", str(int(crf)),
270
+ "-pix_fmt", "yuv420p10le",
271
+ "-profile:v", "main10",
272
+ "-x265-params", "aq-mode=3:no-sao=1:log-level=error",
273
+ "-tag:v", "hvc1", # needed for QuickTime / Safari / Apple apps
274
+ ]
275
+ else:
276
+ raw_pix_fmt = "yuv420p"
277
+ chroma = np.full((enc_h // 2, enc_w // 2), 128, dtype=np.uint8).tobytes()
278
+ codec_args = [
279
+ "-c:v", "libx264",
280
+ "-preset", "slow",
281
+ "-crf", str(int(crf)),
282
+ "-pix_fmt", "yuv420p",
283
+ "-profile:v", "high",
284
+ "-tune", "grain", # keep the dither instead of smoothing it away
285
+ ]
286
 
287
  command = [
288
+ imageio_ffmpeg.get_ffmpeg_exe(),
289
+ "-y", "-hide_banner", "-loglevel", "error",
290
+ "-f", "rawvideo",
291
+ "-pix_fmt", raw_pix_fmt,
292
+ "-video_size", f"{enc_w}x{enc_h}",
293
+ "-framerate", rate_str,
294
+ "-i", "-",
 
 
 
 
 
 
 
 
295
  "-an",
296
+ *codec_args,
297
+ "-color_range", "tv",
298
+ "-colorspace", "bt709",
299
+ "-color_primaries", "bt709",
300
+ "-color_trc", "bt709",
301
+ "-movflags", "+faststart",
 
 
 
 
 
 
 
 
302
  output_path,
303
  ]
304
 
305
+ os.makedirs(os.path.dirname(output_path) or ".", exist_ok=True)
 
 
 
 
 
306
 
307
+ # stderr goes to a temp file so a chatty encoder can never fill a pipe
308
+ # and deadlock against our writes to stdin.
309
+ with tempfile.TemporaryFile() as err_file:
310
+ process = subprocess.Popen(
311
+ command,
312
+ stdin=subprocess.PIPE,
313
+ stdout=subprocess.DEVNULL,
314
+ stderr=err_file,
315
+ )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
316
 
317
+ rng = np.random.default_rng(0)
 
 
 
318
 
319
+ try:
320
+ for depth in depths:
321
+ norm = pad_to_even(normalize_frame(depth, low, high, invert))
322
+
323
+ if is_hevc10:
324
+ luma = np.rint(64.0 + norm * 876.0).astype("<u2")
325
+ else:
326
+ # TPDF dither, +/-1 code value, new pattern every frame.
327
+ noise = rng.random(norm.shape) - rng.random(norm.shape)
328
+ luma = np.clip(
329
+ np.rint(16.0 + norm * 219.0 + noise), 16, 235
330
+ ).astype(np.uint8)
331
+
332
+ process.stdin.write(luma.tobytes() + chroma + chroma)
333
+
334
+ process.stdin.close()
335
+ return_code = process.wait()
336
+
337
+ if return_code != 0:
338
+ err_file.seek(0)
339
+ message = err_file.read().decode("utf-8", errors="replace").strip()
340
+ raise RuntimeError(
341
+ "FFmpeg failed to encode the depth MP4: "
342
+ + (message or f"exit code {return_code}")
343
+ )
344
 
345
+ except Exception:
346
+ if process.poll() is None:
347
+ process.kill()
348
+ process.wait()
349
 
350
+ if os.path.exists(output_path):
351
+ os.remove(output_path)
352
+
353
+ raise
354
 
355
+ return {
356
+ "low": low,
357
+ "high": high,
358
+ "width": width,
359
+ "height": height,
360
+ "padded": (enc_w, enc_h) != (width, height),
361
+ "frames": int(depths.shape[0]),
362
+ "fps": float(rate),
363
+ }
364
 
365
 
366
  # =============================================================================
367
  # INFERENCE
368
  # =============================================================================
369
 
370
+ @spaces.GPU(duration=300)
371
  def infer_video_depth(
372
  input_video: str,
373
  max_len: int = -1,
374
  target_fps: int = -1,
375
  max_res: int = 1280,
376
  input_size: int = 518,
377
+ encoding: str = ENC_HEVC10,
378
+ normalization: str = NORM_FULL,
379
+ invert: bool = False,
380
+ crf: int = 10,
381
  output_dir: str = "./outputs",
382
  ):
383
 
 
388
  target_fps = int(target_fps)
389
  max_res = int(max_res)
390
 
391
+ # DINOv2 patch size is 14.
392
+ input_size = max(int(round(float(input_size) / 14.0) * 14), 14)
 
393
 
394
  if encoder == "vitl" and input_size > SAFE_VITL_INPUT_SIZE:
395
+ gr.Warning("Inference size capped at 518 on ZeroGPU to avoid OOM.")
 
 
 
 
396
  input_size = SAFE_VITL_INPUT_SIZE
397
 
398
+ frames, actual_fps = read_video_frames(input_video, max_len, target_fps, max_res)
399
+ print(f"Loaded {len(frames)} frames, inference size {input_size}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
400
 
401
  gc.collect()
 
402
  if torch.cuda.is_available():
403
  torch.cuda.empty_cache()
404
 
405
  try:
406
  depth_list, fps = video_depth_anything.infer_video_depth(
407
+ frames, actual_fps, input_size=input_size, device=DEVICE
 
 
 
408
  )
409
  except RuntimeError as error:
410
  error_text = str(error).lower()
411
+ if not any(
412
  marker in error_text
413
  for marker in (
414
  "out of memory",
 
416
  "cudacachingallocator",
417
  "nvml_success",
418
  )
419
+ ):
 
 
420
  raise
421
 
422
  gc.collect()
 
423
  if torch.cuda.is_available():
424
  try:
425
  torch.cuda.empty_cache()
 
428
 
429
  raise gr.Error(
430
  "ZeroGPU ran out of memory during depth inference. Try Model "
431
+ "Inference Size 448, or 392 if necessary. Max Side Resolution "
432
+ "controls export resolution, not inference memory."
 
433
  ) from error
434
 
435
+ del frames
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
436
 
437
+ depths = np.asarray(depth_list, dtype=np.float32)
438
+ del depth_list
439
+ print("Raw depth:", depths.shape, float(depths.min()), "to", float(depths.max()))
 
440
 
441
+ base_name = os.path.splitext(os.path.basename(input_video))[0]
442
+ run_dir = os.path.join(output_dir, f"{base_name}_{uuid.uuid4().hex[:8]}")
443
+ tag = "hevc10" if encoding == ENC_HEVC10 else "h264_dither"
444
+ mp4_path = os.path.join(run_dir, f"{base_name}_depth_{tag}.mp4")
445
 
446
+ try:
447
+ stats = encode_depth_mp4(
448
+ depths,
449
+ mp4_path,
450
+ fps=fps,
451
+ encoding=encoding,
452
+ normalization=normalization,
453
+ invert=bool(invert),
454
+ crf=int(crf),
455
+ )
456
+ except (RuntimeError, ValueError) as error:
457
+ raise gr.Error(str(error)) from error
458
+ finally:
459
+ del depths
460
+ gc.collect()
461
+ if torch.cuda.is_available():
462
+ torch.cuda.empty_cache()
463
+
464
+ size_mb = os.path.getsize(mp4_path) / (1024 * 1024)
465
+ pad_note = " (padded by 1 px to an even size for 4:2:0)" if stats["padded"] else ""
466
+
467
+ info = (
468
+ f"**{os.path.basename(mp4_path)}** — {size_mb:.1f} MB\n\n"
469
+ f"{stats['width']}×{stats['height']}{pad_note}, "
470
+ f"{stats['frames']} frames @ {stats['fps']:.3f} fps\n\n"
471
+ f"Encoding: {encoding} · CRF {int(crf)} · {normalization}"
472
+ f"{' · inverted (near = black)' if invert else ' · near = white'}\n\n"
473
+ f"Raw depth mapped to black/white: {stats['low']:.5f} → {stats['high']:.5f}"
474
  )
475
 
476
+ return mp4_path, info
 
 
 
 
477
 
 
 
 
478
 
479
+ # =============================================================================
480
+ # GRADIO UI
481
+ # =============================================================================
482
 
483
+ title = "# Video Depth Anything — Depth MP4"
 
484
 
485
+ description = """
486
+ Runs **Video Depth Anything (Large)** and returns one grayscale depth MP4.
 
 
 
 
487
 
488
+ The default is **10-bit HEVC**, which has four times the gray levels of a normal
489
+ 8-bit MP4 and is tuned so smooth depth gradients don't break into contour bands.
490
+ It plays in QuickTime, Safari, Premiere, Resolve and Blender. Use the 8-bit
491
+ dithered option only if something downstream can't read HEVC.
492
+ """
493
 
 
 
 
494
 
495
  def construct_demo():
496
 
 
499
  gr.Markdown(title)
500
  gr.Markdown(description)
501
 
502
+ with gr.Row(equal_height=False):
503
 
504
  with gr.Column(scale=1):
505
 
506
+ input_video = gr.Video(label="Input Video")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
507
 
508
+ with gr.Accordion("Settings", open=True):
 
 
 
509
 
510
  max_len = gr.Slider(
511
  label="Max Process Length (frames; -1 = all)",
512
+ minimum=-1, maximum=1000, value=500, step=1,
 
 
 
513
  )
514
 
515
  target_fps = gr.Slider(
516
  label="Target FPS (-1 = original)",
517
+ minimum=-1, maximum=30, value=15, step=1,
 
 
 
518
  )
519
 
520
  max_res = gr.Slider(
521
  label="Max Side Resolution",
522
+ minimum=480, maximum=2560, value=1280, step=1,
523
+ info="Output resolution. Separate from inference size.",
 
 
 
 
 
 
524
  )
525
 
526
  input_size = gr.Slider(
527
  label="Model Inference Size",
528
+ minimum=392, maximum=SAFE_VITL_INPUT_SIZE, value=518, step=14,
529
+ info="518 is standard. Drop to 448 or 392 on OOM.",
 
 
 
 
 
 
 
530
  )
531
 
532
+ encoding = gr.Radio(
533
+ label="MP4 Encoding",
534
+ choices=[ENC_HEVC10, ENC_H264_DITHER],
535
+ value=ENC_HEVC10,
536
+ )
537
+
538
+ normalization = gr.Radio(
539
+ label="Depth Normalization",
540
+ choices=[NORM_FULL, NORM_ROBUST],
541
+ value=NORM_FULL,
542
  info=(
543
+ "Robust spends more gray levels on the actual scene "
544
+ "by clipping extreme outlier pixels."
545
  ),
546
  )
547
 
548
+ invert = gr.Checkbox(
549
+ label="Invert (near = black)",
550
+ value=False,
551
+ )
552
+
553
+ crf = gr.Slider(
554
+ label="Quality (CRF — lower is better / larger)",
555
+ minimum=0, maximum=28, value=10, step=1,
556
+ )
557
+
558
+ generate_btn = gr.Button("Generate Depth MP4", variant="primary")
559
+
560
+ with gr.Column(scale=1):
561
+
562
+ depth_mp4 = gr.File(
563
+ label="Depth MP4 (download)",
564
+ interactive=False,
565
  )
566
 
567
+ info = gr.Markdown()
568
 
569
  gr.Markdown(
570
  """
571
+ ### Notes
 
 
 
 
 
 
 
572
 
573
+ The MP4 is delivered as a file download rather than an inline player on
574
+ purpose: Gradio's video player re-encodes anything that isn't 8-bit H.264,
575
+ which would bring the banding right back.
576
 
577
+ **Blender:** load as a Movie in an Image Texture, set it to **Non-Color**,
578
+ use **Cubic** interpolation, and run it through **Map Range** before
579
+ displacement. Bright = near unless you ticked Invert.
 
 
 
580
  """
581
  )
582
 
 
 
583
  gr.Examples(
584
  examples=examples,
585
+ inputs=[input_video, max_len, target_fps, max_res],
586
+ outputs=[depth_mp4, info],
 
 
 
 
 
 
 
 
 
 
587
  fn=infer_video_depth,
588
  cache_examples="lazy",
589
  )
 
596
  target_fps,
597
  max_res,
598
  input_size,
599
+ encoding,
600
+ normalization,
601
+ invert,
602
+ crf,
 
 
 
603
  ],
604
+ outputs=[depth_mp4, info],
605
  )
606
 
607
  return demo
608
 
609
 
610
  if __name__ == "__main__":
 
611
  demo = construct_demo()
612
  demo.queue()
613
+ demo.launch()