phrographica commited on
Commit
5697a94
·
verified ·
1 Parent(s): c8713b0

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +207 -753
app.py CHANGED
@@ -12,7 +12,6 @@
12
  # See the License for the specific language governing permissions and
13
  # limitations under the License.
14
 
15
-
16
  import gc
17
  import os
18
  import uuid
@@ -29,13 +28,17 @@ import xformers
29
  import xformers.ops
30
 
31
  from huggingface_hub import hf_hub_download
32
- from video_depth_anything.video_depth import VideoDepthAnything
33
- from utils.dc_utils import read_video_frames, save_video
34
 
35
 
36
- # ============================================================
37
- # XFORMERS -> PYTORCH SDPA COMPATIBILITY SHIM
38
- # ============================================================
 
 
 
 
 
 
39
 
40
  _xf_original = xformers.ops.memory_efficient_attention
41
 
@@ -50,21 +53,17 @@ def _sdpa_shim(
50
  op=None,
51
  **kwargs,
52
  ):
53
- """
54
- Route xformers memory-efficient attention through
55
- PyTorch scaled_dot_product_attention.
56
 
57
- xformers uses:
58
- [B, M, H, K]
59
 
60
- torch SDPA uses:
61
- [B, H, M, K]
 
62
  """
63
 
64
- if attn_bias is not None and not isinstance(
65
- attn_bias,
66
- torch.Tensor
67
- ):
68
  return _xf_original(
69
  query,
70
  key,
@@ -72,21 +71,15 @@ def _sdpa_shim(
72
  attn_bias,
73
  p,
74
  scale,
75
- op
76
  )
77
 
78
  squeeze = query.dim() == 3
79
 
80
  if squeeze:
81
- q, k, v = (
82
- t.unsqueeze(1)
83
- for t in (query, key, value)
84
- )
85
  else:
86
- q, k, v = (
87
- t.transpose(1, 2)
88
- for t in (query, key, value)
89
- )
90
 
91
  out = F.scaled_dot_product_attention(
92
  q,
@@ -94,1049 +87,510 @@ def _sdpa_shim(
94
  v,
95
  attn_mask=attn_bias,
96
  dropout_p=p,
97
- scale=scale
98
  )
99
 
100
- return (
101
- out.squeeze(1)
102
- if squeeze
103
- else out.transpose(1, 2)
104
- )
105
 
106
 
 
107
  xformers.ops.memory_efficient_attention = _sdpa_shim
108
 
 
 
 
 
 
109
 
110
- # ============================================================
 
 
 
 
 
 
 
 
111
  # EXAMPLES
112
- # ============================================================
113
 
114
  examples = [
115
- [
116
- "assets/example_videos/davis_rollercoaster.mp4",
117
- -1,
118
- -1,
119
- 1280
120
- ],
121
- [
122
- "assets/example_videos/Tokyo-Walk_rgb.mp4",
123
- -1,
124
- -1,
125
- 1280
126
- ],
127
- [
128
- "assets/example_videos/4158877-uhd_3840_2160_30fps_rgb.mp4",
129
- -1,
130
- -1,
131
- 1280
132
- ],
133
- [
134
- "assets/example_videos/4511004-uhd_3840_2160_24fps_rgb.mp4",
135
- -1,
136
- -1,
137
- 1280
138
- ],
139
- [
140
- "assets/example_videos/1753029-hd_1920_1080_30fps.mp4",
141
- -1,
142
- -1,
143
- 1280
144
- ],
145
- [
146
- "assets/example_videos/davis_burnout.mp4",
147
- -1,
148
- -1,
149
- 1280
150
- ],
151
- [
152
- "assets/example_videos/example_5473765-l.mp4",
153
- -1,
154
- -1,
155
- 1280
156
- ],
157
- [
158
- "assets/example_videos/Istanbul-26920.mp4",
159
- -1,
160
- -1,
161
- 1280
162
- ],
163
- [
164
- "assets/example_videos/obj_1.mp4",
165
- -1,
166
- -1,
167
- 1280
168
- ],
169
- [
170
- "assets/example_videos/sheep_cut1.mp4",
171
- -1,
172
- -1,
173
- 1280
174
- ],
175
  ]
176
 
177
 
178
- # ============================================================
179
- # DEVICE
180
- # ============================================================
181
-
182
- DEVICE = (
183
- "cuda"
184
- if torch.cuda.is_available()
185
- else "cpu"
186
- )
187
-
188
 
189
- # ============================================================
190
- # MODEL CONFIGURATION
191
- # ============================================================
192
 
193
  model_configs = {
194
-
195
  "vits": {
196
  "encoder": "vits",
197
  "features": 64,
198
- "out_channels": [
199
- 48,
200
- 96,
201
- 192,
202
- 384
203
- ],
204
  },
205
-
206
  "vitl": {
207
  "encoder": "vitl",
208
  "features": 256,
209
- "out_channels": [
210
- 256,
211
- 512,
212
- 1024,
213
- 1024
214
- ],
215
  },
216
-
217
  }
218
 
219
-
220
  encoder2name = {
221
  "vits": "Small",
222
  "vitl": "Large",
223
  }
224
 
225
-
226
- # Use LARGE model
227
  encoder = "vitl"
228
-
229
  model_name = encoder2name[encoder]
230
 
231
-
232
- video_depth_anything = VideoDepthAnything(
233
- **model_configs[encoder]
234
- )
235
-
236
 
237
  filepath = hf_hub_download(
238
-
239
- repo_id=(
240
- f"depth-anything/"
241
- f"Video-Depth-Anything-{model_name}"
242
- ),
243
-
244
- filename=(
245
- f"video_depth_anything_{encoder}.pth"
246
- ),
247
-
248
  repo_type="model",
249
-
250
  )
251
 
252
-
253
  video_depth_anything.load_state_dict(
254
-
255
- torch.load(
256
- filepath,
257
- map_location="cpu"
258
- )
259
-
260
  )
261
 
262
-
263
- video_depth_anything = (
264
- video_depth_anything
265
- .to(DEVICE)
266
- .eval()
267
- )
268
 
269
 
270
- # ============================================================
271
  # UI TEXT
272
- # ============================================================
273
 
274
  title = "# Video Depth Anything"
275
 
276
-
277
  description = """
278
- Official demo for **Video Depth Anything**, modified to also
279
- export the **raw depth prediction as a 32-bit floating-point
280
- OpenEXR image sequence**.
281
 
282
- The MP4 depth video is only a visualization.
 
283
 
284
- For Blender displacement, use the **EXR ZIP**.
285
-
286
- The EXR files contain the raw floating-point depth values before
287
- conversion to an 8-bit video.
288
  """
289
 
290
 
291
- # ============================================================
292
- # SAVE RAW DEPTH AS OPENEXR
293
- # ============================================================
294
 
295
- def save_depths_as_rgb_exr(
296
- depths: np.ndarray,
297
- output_dir: str
298
- ):
299
 
300
- """
301
- Save raw depth frames as 32-bit floating-point EXRs.
302
 
303
- The same depth value is written into:
304
-
305
- R
306
- G
307
- B
308
-
309
- This makes the EXRs easy to use with Blender's
310
- Image Texture node.
311
-
312
- IMPORTANT:
313
-
314
- No 0-1 normalization.
315
- No conversion to uint8.
316
- No MP4 compression.
317
  """
318
 
319
- os.makedirs(
320
- output_dir,
321
- exist_ok=True
322
- )
323
-
324
-
325
- float_channel = Imath.PixelType(
326
- Imath.PixelType.FLOAT
327
- )
328
 
 
329
 
330
  for i, depth in enumerate(depths):
331
-
332
- # Explicitly keep data as 32-bit float
333
- depth = np.ascontiguousarray(
334
- depth,
335
- dtype=np.float32
336
- )
337
-
338
 
339
  if depth.ndim != 2:
340
-
341
  raise ValueError(
342
- "Expected depth frame to be H x W. "
343
- f"Received: {depth.shape}"
344
  )
345
 
346
-
347
  height, width = depth.shape
 
348
 
349
-
350
- output_exr = os.path.join(
351
-
352
- output_dir,
353
-
354
- f"frame_{i:05d}.exr"
355
-
356
- )
357
-
358
-
359
- header = OpenEXR.Header(
360
- width,
361
- height
362
- )
363
-
364
-
365
  header["channels"] = {
366
-
367
- "R": Imath.Channel(
368
- float_channel
369
- ),
370
-
371
- "G": Imath.Channel(
372
- float_channel
373
- ),
374
-
375
- "B": Imath.Channel(
376
- float_channel
377
- ),
378
-
379
  }
380
 
381
-
382
  depth_bytes = depth.tobytes()
383
 
384
-
385
- exr_file = OpenEXR.OutputFile(
386
- output_exr,
387
- header
388
- )
389
-
390
 
391
  try:
392
-
393
- exr_file.writePixels({
394
-
395
- "R": depth_bytes,
396
-
397
- "G": depth_bytes,
398
-
399
- "B": depth_bytes,
400
-
401
- })
402
-
403
  finally:
404
-
405
  exr_file.close()
406
 
407
 
408
- # ============================================================
409
- # ZIP EXR SEQUENCE
410
- # ============================================================
411
-
412
- def zip_directory(
413
- source_dir: str,
414
- zip_path: str
415
- ):
416
 
417
  with zipfile.ZipFile(
418
-
419
  zip_path,
420
-
421
  mode="w",
422
-
423
- compression=zipfile.ZIP_DEFLATED
424
-
425
  ) as zip_file:
426
 
427
-
428
- for filename in sorted(
429
- os.listdir(source_dir)
430
- ):
431
-
432
-
433
- file_path = os.path.join(
434
- source_dir,
435
- filename
436
- )
437
-
438
 
439
  if os.path.isfile(file_path):
440
-
441
  zip_file.write(
442
-
443
  file_path,
444
-
445
- arcname=filename
446
-
447
  )
448
 
449
-
450
  return zip_path
451
 
452
 
453
- # ============================================================
454
- # DEPTH INFERENCE
455
- # ============================================================
456
 
457
  @spaces.GPU(duration=240)
458
-
459
  def infer_video_depth(
460
-
461
  input_video: str,
462
-
463
  max_len: int = -1,
464
-
465
  target_fps: int = -1,
466
-
467
  max_res: int = 1280,
468
-
469
  input_size: int = 518,
470
-
471
  grayscale: bool = True,
472
-
473
  output_dir: str = "./outputs",
474
-
475
  ):
476
 
477
-
478
  if not input_video:
 
479
 
480
- raise gr.Error(
481
- "Please upload a video first."
482
- )
483
-
484
 
485
- # --------------------------------------------------------
486
- # FORCE INPUT SIZE TO MULTIPLE OF 14
487
- # --------------------------------------------------------
488
 
489
- input_size = int(
490
 
491
- round(
492
- float(input_size) / 14.0
493
- ) * 14
494
-
495
- )
496
-
497
-
498
- input_size = max(
499
- input_size,
500
- 14
501
- )
502
-
503
-
504
- print(
505
- f"Using model inference size: "
506
- f"{input_size}"
507
- )
508
-
509
-
510
- # --------------------------------------------------------
511
- # READ VIDEO
512
- # --------------------------------------------------------
513
-
514
- frames, target_fps = read_video_frames(
515
 
 
516
  input_video,
517
-
518
- int(max_len),
519
-
520
- int(target_fps),
521
-
522
- int(max_res),
523
-
524
  )
525
 
 
526
 
527
- print(
528
- f"Loaded {len(frames)} frames"
529
- )
530
-
531
-
532
- # --------------------------------------------------------
533
- # VIDEO DEPTH ANYTHING
534
- # --------------------------------------------------------
535
-
536
- depths, fps = (
537
- video_depth_anything
538
- .infer_video_depth(
539
-
540
- frames,
541
 
542
- target_fps,
543
-
544
- input_size=input_size,
545
-
546
- device=DEVICE,
547
-
548
- )
549
- )
550
-
551
-
552
- # --------------------------------------------------------
553
- # PRESERVE RAW DEPTH
554
- # --------------------------------------------------------
555
-
556
- depths = np.asarray(
557
- depths,
558
- dtype=np.float32
559
- )
560
-
561
-
562
- print(
563
- "Raw depth shape:",
564
- depths.shape
565
  )
566
 
 
 
567
 
 
568
  print(
569
  "Raw depth range:",
570
  float(depths.min()),
571
  "to",
572
- float(depths.max())
573
- )
574
-
575
-
576
- # --------------------------------------------------------
577
- # OUTPUT DIRECTORY
578
- # --------------------------------------------------------
579
-
580
- video_name = os.path.basename(
581
- input_video
582
  )
583
 
 
 
 
584
 
585
- base_name = os.path.splitext(
586
- video_name
587
- )[0]
588
-
589
-
590
- # Unique folder for every generation.
591
- #
592
- # This prevents EXRs from an older generation accidentally
593
- # getting included if you process another video with the
594
- # same filename but fewer frames.
595
-
596
  run_id = uuid.uuid4().hex[:8]
597
 
598
-
599
  run_dir = os.path.join(
600
-
601
  output_dir,
602
-
603
- f"{base_name}_{run_id}"
604
-
605
- )
606
-
607
-
608
- os.makedirs(
609
- run_dir,
610
- exist_ok=True
611
  )
612
 
613
-
614
- # --------------------------------------------------------
615
- # FILE PATHS
616
- # --------------------------------------------------------
617
 
618
  processed_video_path = os.path.join(
619
-
620
  run_dir,
621
-
622
- f"{base_name}_src.mp4"
623
-
624
  )
625
 
626
-
627
  depth_vis_path = os.path.join(
628
-
629
  run_dir,
630
-
631
- f"{base_name}_vis.mp4"
632
-
633
  )
634
 
635
-
636
  depth_exr_dir = os.path.join(
637
-
638
  run_dir,
639
-
640
- f"{base_name}_depth_EXR"
641
-
642
  )
643
 
644
-
645
  zip_path = os.path.join(
646
-
647
  run_dir,
648
-
649
- f"{base_name}_depth_EXR.zip"
650
-
651
  )
652
 
653
-
654
- # --------------------------------------------------------
655
- # SAVE PREPROCESSED INPUT VIDEO
656
- # --------------------------------------------------------
657
 
658
  save_video(
659
-
660
  frames,
661
-
662
  processed_video_path,
663
-
664
- fps=fps
665
-
666
  )
667
 
668
-
669
- # --------------------------------------------------------
670
- # SAVE MP4 DEPTH PREVIEW
671
- #
672
- # This IS quantized for viewing.
673
- #
674
- # DO NOT use this file as the displacement source.
675
- # --------------------------------------------------------
676
 
677
  save_video(
678
-
679
  depths,
680
-
681
  depth_vis_path,
682
-
683
  fps=fps,
684
-
685
  is_depths=True,
686
-
687
- grayscale=bool(grayscale)
688
-
689
  )
690
 
 
 
 
691
 
692
- # --------------------------------------------------------
693
- # SAVE RAW 32-BIT FLOAT DEPTH
694
- # --------------------------------------------------------
695
-
696
- print(
697
- "Saving raw 32-bit EXR sequence..."
698
- )
699
-
700
 
701
  save_depths_as_rgb_exr(
702
-
703
  depths,
704
-
705
- depth_exr_dir
706
-
707
- )
708
-
709
-
710
- # --------------------------------------------------------
711
- # ZIP EXR FRAMES
712
- # --------------------------------------------------------
713
-
714
- print(
715
- "Creating EXR ZIP..."
716
  )
717
 
 
718
 
719
  zip_directory(
720
-
721
  depth_exr_dir,
722
-
723
- zip_path
724
-
725
- )
726
-
727
-
728
- print(
729
- "EXR ZIP:",
730
- zip_path
731
  )
732
 
 
733
 
734
- # --------------------------------------------------------
735
- # CLEAN GPU / RAM
736
- # --------------------------------------------------------
737
 
738
  del depths
 
739
  del frames
740
 
741
-
742
  gc.collect()
743
 
744
-
745
  if torch.cuda.is_available():
746
-
747
  torch.cuda.empty_cache()
748
 
749
-
750
- # --------------------------------------------------------
751
- # RETURN TO GRADIO
752
- # --------------------------------------------------------
753
-
754
  return [
755
-
756
  processed_video_path,
757
-
758
  depth_vis_path,
759
-
760
- zip_path
761
-
762
  ]
763
 
764
 
765
- # ============================================================
766
  # GRADIO UI
767
- # ============================================================
768
 
769
  def construct_demo():
770
 
 
771
 
772
- with gr.Blocks(
773
- analytics_enabled=False
774
- ) as demo:
775
-
776
-
777
- gr.Markdown(
778
- title
779
- )
780
-
781
-
782
- gr.Markdown(
783
- description
784
- )
785
-
786
-
787
- # ----------------------------------------------------
788
- # INPUT / OUTPUT
789
- # ----------------------------------------------------
790
 
791
- with gr.Row(
792
- equal_height=True
793
- ):
794
-
795
-
796
- with gr.Column(
797
- scale=1
798
- ):
799
 
 
800
 
801
  input_video = gr.Video(
802
-
803
  label="Input Video"
804
-
805
  )
806
 
 
807
 
808
- with gr.Column(
809
- scale=2
810
- ):
811
-
812
-
813
- with gr.Row(
814
- equal_height=True
815
- ):
816
-
817
 
818
  processed_video = gr.Video(
819
-
820
- label=(
821
- "Preprocessed Video"
822
- ),
823
-
824
  interactive=False,
825
-
826
  autoplay=True,
827
-
828
  loop=True,
829
-
830
  show_share_button=True,
831
-
832
  scale=5,
833
-
834
  )
835
 
836
-
837
  depth_vis_video = gr.Video(
838
-
839
- label=(
840
- "Generated Depth Preview"
841
- ),
842
-
843
  interactive=False,
844
-
845
  autoplay=True,
846
-
847
  loop=True,
848
-
849
  show_share_button=True,
850
-
851
  scale=5,
852
-
853
  )
854
 
855
-
856
  depth_exr_file = gr.File(
857
-
858
- label=(
859
- "Raw 32-bit EXR "
860
- "Depth Sequence (.zip)"
861
- ),
862
-
863
- interactive=False
864
-
865
  )
866
 
 
867
 
868
- # ----------------------------------------------------
869
- # SETTINGS
870
- # ----------------------------------------------------
871
-
872
- with gr.Row(
873
- equal_height=True
874
- ):
875
-
876
-
877
- with gr.Column(
878
- scale=1
879
- ):
880
-
881
 
882
  with gr.Accordion(
883
-
884
  "Advanced Settings",
885
-
886
- open=True
887
-
888
  ):
889
 
890
-
891
  max_len = gr.Slider(
892
-
893
- label=(
894
- "Max Process Length "
895
- "(frames; -1 = all)"
896
- ),
897
-
898
  minimum=-1,
899
-
900
  maximum=1000,
901
-
902
  value=500,
903
-
904
  step=1,
905
-
906
  )
907
 
908
-
909
  target_fps = gr.Slider(
910
-
911
- label=(
912
- "Target FPS "
913
- "(-1 = original)"
914
- ),
915
-
916
  minimum=-1,
917
-
918
  maximum=30,
919
-
920
  value=15,
921
-
922
  step=1,
923
-
924
  )
925
 
926
-
927
  max_res = gr.Slider(
928
-
929
- label=(
930
- "Max Output Side Resolution"
931
- ),
932
-
933
  minimum=480,
934
-
935
  maximum=2560,
936
-
937
  value=1280,
938
-
939
  step=1,
940
-
941
  info=(
942
- "Controls the maximum "
943
- "resolution of the video "
944
- "frames and final EXRs. "
945
- "This is different from "
946
- "the model inference size."
947
  ),
948
-
949
  )
950
 
951
-
952
- # ----------------------------------------
953
- # NEW:
954
- # ACTUAL MODEL INFERENCE RESOLUTION
955
- # ----------------------------------------
956
-
957
  input_size = gr.Slider(
958
-
959
- label=(
960
- "Model Inference Size"
961
- ),
962
-
963
  minimum=392,
964
-
965
  maximum=896,
966
-
967
  value=518,
968
-
969
  step=14,
970
-
971
  info=(
972
- "518 is the normal setting. "
973
- "Increasing this gives the "
974
- "model more spatial resolution, "
975
- "but VRAM usage rises quickly."
976
  ),
977
-
978
  )
979
 
980
-
981
  grayscale = gr.Checkbox(
982
-
983
- label=(
984
- "Grayscale MP4 Preview"
985
- ),
986
-
987
  value=True,
988
-
989
  info=(
990
- "Only affects the preview MP4. "
991
- "The EXR always contains raw "
992
- "floating-point depth."
993
  ),
994
-
995
  )
996
 
997
-
998
  generate_btn = gr.Button(
999
-
1000
  "Generate Depth + EXR",
1001
-
1002
- variant="primary"
1003
-
1004
  )
1005
 
1006
-
1007
- # ------------------------------------------------
1008
- # BLENDER INFO
1009
- # ------------------------------------------------
1010
-
1011
- with gr.Column(
1012
- scale=2
1013
- ):
1014
-
1015
 
1016
  gr.Markdown(
1017
  """
1018
  ### Blender EXR setup
1019
 
1020
- Unzip the EXR file first.
1021
-
1022
- Then load:
1023
-
1024
- `frame_00000.exr`
1025
-
1026
- as an **Image Sequence** in Blender.
1027
 
1028
- For the depth Image Texture:
1029
-
1030
- **Color Space**
1031
- `Non-Color`
1032
-
1033
- **Interpolation**
1034
- `Cubic`
1035
-
1036
- Then:
1037
-
1038
- `Image Texture`
1039
- → `Separate Color`
1040
- → `Red`
1041
- → `Map Range`
1042
- → displacement
1043
-
1044
- The EXRs may look almost black in a normal image viewer.
1045
- That does **not** mean the depth data is missing.
1046
-
1047
- They contain raw floating-point depth rather than a normalized
1048
- 8-bit image.
1049
  """
1050
  )
1051
 
1052
-
1053
- # ----------------------------------------------------
1054
- # EXAMPLES
1055
- # ----------------------------------------------------
1056
-
1057
  gr.Examples(
1058
-
1059
  examples=examples,
1060
-
1061
  inputs=[
1062
-
1063
  input_video,
1064
-
1065
  max_len,
1066
-
1067
  target_fps,
1068
-
1069
- max_res
1070
-
1071
  ],
1072
-
1073
  outputs=[
1074
-
1075
  processed_video,
1076
-
1077
  depth_vis_video,
1078
-
1079
- depth_exr_file
1080
-
1081
  ],
1082
-
1083
  fn=infer_video_depth,
1084
-
1085
- cache_examples="lazy"
1086
-
1087
  )
1088
 
1089
-
1090
- # ----------------------------------------------------
1091
- # GENERATE BUTTON
1092
- # ----------------------------------------------------
1093
-
1094
  generate_btn.click(
1095
-
1096
  fn=infer_video_depth,
1097
-
1098
  inputs=[
1099
-
1100
  input_video,
1101
-
1102
  max_len,
1103
-
1104
  target_fps,
1105
-
1106
  max_res,
1107
-
1108
  input_size,
1109
-
1110
- grayscale
1111
-
1112
  ],
1113
-
1114
  outputs=[
1115
-
1116
  processed_video,
1117
-
1118
  depth_vis_video,
1119
-
1120
- depth_exr_file
1121
-
1122
  ],
1123
-
1124
  )
1125
 
1126
-
1127
  return demo
1128
 
1129
 
1130
- # ============================================================
1131
- # START
1132
- # ============================================================
1133
-
1134
  if __name__ == "__main__":
1135
 
1136
  demo = construct_demo()
1137
-
1138
  demo.queue()
1139
 
1140
- demo.launch(
1141
- share=True
1142
- )
 
12
  # See the License for the specific language governing permissions and
13
  # limitations under the License.
14
 
 
15
  import gc
16
  import os
17
  import uuid
 
28
  import xformers.ops
29
 
30
  from huggingface_hub import hf_hub_download
 
 
31
 
32
 
33
+ # =============================================================================
34
+ # BLACKWELL / XFORMERS COMPATIBILITY SHIM
35
+ #
36
+ # IMPORTANT:
37
+ # This patch MUST run before VideoDepthAnything is imported.
38
+ # The bundled DINOv2 attention module imports memory_efficient_attention by
39
+ # name, so patching xformers.ops first ensures it receives this PyTorch SDPA
40
+ # replacement instead of the incompatible xFormers CUDA kernel.
41
+ # =============================================================================
42
 
43
  _xf_original = xformers.ops.memory_efficient_attention
44
 
 
53
  op=None,
54
  **kwargs,
55
  ):
56
+ """Route xFormers memory-efficient attention through PyTorch SDPA.
 
 
57
 
58
+ xFormers uses tensors shaped [B, M, H, K].
59
+ PyTorch SDPA expects [B, H, M, K].
60
 
61
+ Video Depth Anything inference normally uses attn_bias=None. If a
62
+ non-Tensor xFormers-specific bias object is encountered, fall back to
63
+ xFormers' original implementation.
64
  """
65
 
66
+ if attn_bias is not None and not isinstance(attn_bias, torch.Tensor):
 
 
 
67
  return _xf_original(
68
  query,
69
  key,
 
71
  attn_bias,
72
  p,
73
  scale,
74
+ op,
75
  )
76
 
77
  squeeze = query.dim() == 3
78
 
79
  if squeeze:
80
+ q, k, v = (t.unsqueeze(1) for t in (query, key, value))
 
 
 
81
  else:
82
+ q, k, v = (t.transpose(1, 2) for t in (query, key, value))
 
 
 
83
 
84
  out = F.scaled_dot_product_attention(
85
  q,
 
87
  v,
88
  attn_mask=attn_bias,
89
  dropout_p=p,
90
+ scale=scale,
91
  )
92
 
93
+ return out.squeeze(1) if squeeze else out.transpose(1, 2)
 
 
 
 
94
 
95
 
96
+ # Patch xFormers BEFORE importing VideoDepthAnything / bundled DINOv2.
97
  xformers.ops.memory_efficient_attention = _sdpa_shim
98
 
99
+ print(
100
+ "Blackwell attention patch installed:",
101
+ xformers.ops.memory_efficient_attention is _sdpa_shim,
102
+ )
103
+
104
 
105
+ # =============================================================================
106
+ # THESE IMPORTS MUST REMAIN AFTER THE XFORMERS PATCH ABOVE
107
+ # =============================================================================
108
+
109
+ from video_depth_anything.video_depth import VideoDepthAnything
110
+ from utils.dc_utils import read_video_frames, save_video
111
+
112
+
113
+ # =============================================================================
114
  # EXAMPLES
115
+ # =============================================================================
116
 
117
  examples = [
118
+ ["assets/example_videos/davis_rollercoaster.mp4", -1, -1, 1280],
119
+ ["assets/example_videos/Tokyo-Walk_rgb.mp4", -1, -1, 1280],
120
+ ["assets/example_videos/4158877-uhd_3840_2160_30fps_rgb.mp4", -1, -1, 1280],
121
+ ["assets/example_videos/4511004-uhd_3840_2160_24fps_rgb.mp4", -1, -1, 1280],
122
+ ["assets/example_videos/1753029-hd_1920_1080_30fps.mp4", -1, -1, 1280],
123
+ ["assets/example_videos/davis_burnout.mp4", -1, -1, 1280],
124
+ ["assets/example_videos/example_5473765-l.mp4", -1, -1, 1280],
125
+ ["assets/example_videos/Istanbul-26920.mp4", -1, -1, 1280],
126
+ ["assets/example_videos/obj_1.mp4", -1, -1, 1280],
127
+ ["assets/example_videos/sheep_cut1.mp4", -1, -1, 1280],
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
128
  ]
129
 
130
 
131
+ # =============================================================================
132
+ # MODEL
133
+ # =============================================================================
 
 
 
 
 
 
 
134
 
135
+ DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
 
 
136
 
137
  model_configs = {
 
138
  "vits": {
139
  "encoder": "vits",
140
  "features": 64,
141
+ "out_channels": [48, 96, 192, 384],
 
 
 
 
 
142
  },
 
143
  "vitl": {
144
  "encoder": "vitl",
145
  "features": 256,
146
+ "out_channels": [256, 512, 1024, 1024],
 
 
 
 
 
147
  },
 
148
  }
149
 
 
150
  encoder2name = {
151
  "vits": "Small",
152
  "vitl": "Large",
153
  }
154
 
155
+ # Large model
 
156
  encoder = "vitl"
 
157
  model_name = encoder2name[encoder]
158
 
159
+ video_depth_anything = VideoDepthAnything(**model_configs[encoder])
 
 
 
 
160
 
161
  filepath = hf_hub_download(
162
+ repo_id=f"depth-anything/Video-Depth-Anything-{model_name}",
163
+ filename=f"video_depth_anything_{encoder}.pth",
 
 
 
 
 
 
 
 
164
  repo_type="model",
 
165
  )
166
 
 
167
  video_depth_anything.load_state_dict(
168
+ torch.load(filepath, map_location="cpu")
 
 
 
 
 
169
  )
170
 
171
+ video_depth_anything = video_depth_anything.to(DEVICE).eval()
 
 
 
 
 
172
 
173
 
174
+ # =============================================================================
175
  # UI TEXT
176
+ # =============================================================================
177
 
178
  title = "# Video Depth Anything"
179
 
 
180
  description = """
181
+ Official demo for **Video Depth Anything**, modified to export the model's
182
+ **raw depth prediction as a 32-bit floating-point OpenEXR image sequence**.
 
183
 
184
+ - **MP4 depth preview:** normalized/compressed visualization for viewing.
185
+ - **EXR ZIP:** raw floating-point depth values for Blender displacement.
186
 
187
+ For Blender, use the EXR sequence rather than the MP4 depth preview.
 
 
 
188
  """
189
 
190
 
191
+ # =============================================================================
192
+ # EXR EXPORT
193
+ # =============================================================================
194
 
195
+ def save_depths_as_rgb_exr(depths: np.ndarray, output_dir: str) -> None:
196
+ """Save raw depth values as 32-bit float RGB OpenEXR frames.
 
 
197
 
198
+ The same raw depth value is written to R, G, and B so Blender can read
199
+ the sequence directly through a regular Image Texture node.
200
 
201
+ No 0-1 normalization, uint8 conversion, colormap, or video compression
202
+ is applied to these files.
 
 
 
 
 
 
 
 
 
 
 
 
203
  """
204
 
205
+ os.makedirs(output_dir, exist_ok=True)
 
 
 
 
 
 
 
 
206
 
207
+ float_channel = Imath.PixelType(Imath.PixelType.FLOAT)
208
 
209
  for i, depth in enumerate(depths):
210
+ depth = np.ascontiguousarray(depth, dtype=np.float32)
 
 
 
 
 
 
211
 
212
  if depth.ndim != 2:
 
213
  raise ValueError(
214
+ f"Expected each depth frame to have shape HxW, got {depth.shape}"
 
215
  )
216
 
 
217
  height, width = depth.shape
218
+ exr_path = os.path.join(output_dir, f"frame_{i:05d}.exr")
219
 
220
+ header = OpenEXR.Header(width, height)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
221
  header["channels"] = {
222
+ "R": Imath.Channel(float_channel),
223
+ "G": Imath.Channel(float_channel),
224
+ "B": Imath.Channel(float_channel),
 
 
 
 
 
 
 
 
 
 
225
  }
226
 
 
227
  depth_bytes = depth.tobytes()
228
 
229
+ exr_file = OpenEXR.OutputFile(exr_path, header)
 
 
 
 
 
230
 
231
  try:
232
+ exr_file.writePixels(
233
+ {
234
+ "R": depth_bytes,
235
+ "G": depth_bytes,
236
+ "B": depth_bytes,
237
+ }
238
+ )
 
 
 
 
239
  finally:
 
240
  exr_file.close()
241
 
242
 
243
+ def zip_directory(source_dir: str, zip_path: str) -> str:
244
+ """Zip an EXR sequence so Gradio can return it as one download."""
 
 
 
 
 
 
245
 
246
  with zipfile.ZipFile(
 
247
  zip_path,
 
248
  mode="w",
249
+ compression=zipfile.ZIP_DEFLATED,
 
 
250
  ) as zip_file:
251
 
252
+ for filename in sorted(os.listdir(source_dir)):
253
+ file_path = os.path.join(source_dir, filename)
 
 
 
 
 
 
 
 
 
254
 
255
  if os.path.isfile(file_path):
 
256
  zip_file.write(
 
257
  file_path,
258
+ arcname=filename,
 
 
259
  )
260
 
 
261
  return zip_path
262
 
263
 
264
+ # =============================================================================
265
+ # INFERENCE
266
+ # =============================================================================
267
 
268
  @spaces.GPU(duration=240)
 
269
  def infer_video_depth(
 
270
  input_video: str,
 
271
  max_len: int = -1,
 
272
  target_fps: int = -1,
 
273
  max_res: int = 1280,
 
274
  input_size: int = 518,
 
275
  grayscale: bool = True,
 
276
  output_dir: str = "./outputs",
 
277
  ):
278
 
 
279
  if not input_video:
280
+ raise gr.Error("Please upload a video first.")
281
 
282
+ max_len = int(max_len)
283
+ target_fps = int(target_fps)
284
+ max_res = int(max_res)
 
285
 
286
+ # VDA/DINOv2 patch size is 14, so keep inference size on a multiple of 14.
287
+ input_size = int(round(float(input_size) / 14.0) * 14)
288
+ input_size = max(input_size, 14)
289
 
290
+ print(f"Using model inference size: {input_size}")
291
 
292
+ # -------------------------------------------------------------------------
293
+ # Read / preprocess source video
294
+ # -------------------------------------------------------------------------
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
295
 
296
+ frames, actual_fps = read_video_frames(
297
  input_video,
298
+ max_len,
299
+ target_fps,
300
+ max_res,
 
 
 
 
301
  )
302
 
303
+ print(f"Loaded {len(frames)} frames")
304
 
305
+ # -------------------------------------------------------------------------
306
+ # Raw Video Depth Anything inference
307
+ # -------------------------------------------------------------------------
 
 
 
 
 
 
 
 
 
 
 
308
 
309
+ depth_list, fps = video_depth_anything.infer_video_depth(
310
+ frames,
311
+ actual_fps,
312
+ input_size=input_size,
313
+ device=DEVICE,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
314
  )
315
 
316
+ # Preserve the model output as float32 BEFORE making any visualization.
317
+ depths = np.asarray(depth_list, dtype=np.float32)
318
 
319
+ print("Raw depth shape:", depths.shape)
320
  print(
321
  "Raw depth range:",
322
  float(depths.min()),
323
  "to",
324
+ float(depths.max()),
 
 
 
 
 
 
 
 
 
325
  )
326
 
327
+ # -------------------------------------------------------------------------
328
+ # Unique output folder for each run
329
+ # -------------------------------------------------------------------------
330
 
331
+ video_name = os.path.basename(input_video)
332
+ base_name = os.path.splitext(video_name)[0]
 
 
 
 
 
 
 
 
 
333
  run_id = uuid.uuid4().hex[:8]
334
 
 
335
  run_dir = os.path.join(
 
336
  output_dir,
337
+ f"{base_name}_{run_id}",
 
 
 
 
 
 
 
 
338
  )
339
 
340
+ os.makedirs(run_dir, exist_ok=True)
 
 
 
341
 
342
  processed_video_path = os.path.join(
 
343
  run_dir,
344
+ f"{base_name}_src.mp4",
 
 
345
  )
346
 
 
347
  depth_vis_path = os.path.join(
 
348
  run_dir,
349
+ f"{base_name}_vis.mp4",
 
 
350
  )
351
 
 
352
  depth_exr_dir = os.path.join(
 
353
  run_dir,
354
+ f"{base_name}_depth_EXR",
 
 
355
  )
356
 
 
357
  zip_path = os.path.join(
 
358
  run_dir,
359
+ f"{base_name}_depth_EXR.zip",
 
 
360
  )
361
 
362
+ # -------------------------------------------------------------------------
363
+ # Save preprocessed source video
364
+ # -------------------------------------------------------------------------
 
365
 
366
  save_video(
 
367
  frames,
 
368
  processed_video_path,
369
+ fps=fps,
 
 
370
  )
371
 
372
+ # -------------------------------------------------------------------------
373
+ # Save normalized MP4 preview
374
+ # -------------------------------------------------------------------------
 
 
 
 
 
375
 
376
  save_video(
 
377
  depths,
 
378
  depth_vis_path,
 
379
  fps=fps,
 
380
  is_depths=True,
381
+ grayscale=bool(grayscale),
 
 
382
  )
383
 
384
+ # -------------------------------------------------------------------------
385
+ # Save RAW floating-point EXR sequence
386
+ # -------------------------------------------------------------------------
387
 
388
+ print("Saving raw 32-bit EXR sequence...")
 
 
 
 
 
 
 
389
 
390
  save_depths_as_rgb_exr(
 
391
  depths,
392
+ depth_exr_dir,
 
 
 
 
 
 
 
 
 
 
 
393
  )
394
 
395
+ print("Creating EXR ZIP...")
396
 
397
  zip_directory(
 
398
  depth_exr_dir,
399
+ zip_path,
 
 
 
 
 
 
 
 
400
  )
401
 
402
+ print("EXR ZIP:", zip_path)
403
 
404
+ # -------------------------------------------------------------------------
405
+ # Cleanup
406
+ # -------------------------------------------------------------------------
407
 
408
  del depths
409
+ del depth_list
410
  del frames
411
 
 
412
  gc.collect()
413
 
 
414
  if torch.cuda.is_available():
 
415
  torch.cuda.empty_cache()
416
 
 
 
 
 
 
417
  return [
 
418
  processed_video_path,
 
419
  depth_vis_path,
420
+ zip_path,
 
 
421
  ]
422
 
423
 
424
+ # =============================================================================
425
  # GRADIO UI
426
+ # =============================================================================
427
 
428
  def construct_demo():
429
 
430
+ with gr.Blocks(analytics_enabled=False) as demo:
431
 
432
+ gr.Markdown(title)
433
+ gr.Markdown(description)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
434
 
435
+ with gr.Row(equal_height=True):
 
 
 
 
 
 
 
436
 
437
+ with gr.Column(scale=1):
438
 
439
  input_video = gr.Video(
 
440
  label="Input Video"
 
441
  )
442
 
443
+ with gr.Column(scale=2):
444
 
445
+ with gr.Row(equal_height=True):
 
 
 
 
 
 
 
 
446
 
447
  processed_video = gr.Video(
448
+ label="Preprocessed Video",
 
 
 
 
449
  interactive=False,
 
450
  autoplay=True,
 
451
  loop=True,
 
452
  show_share_button=True,
 
453
  scale=5,
 
454
  )
455
 
 
456
  depth_vis_video = gr.Video(
457
+ label="Generated Depth Preview",
 
 
 
 
458
  interactive=False,
 
459
  autoplay=True,
 
460
  loop=True,
 
461
  show_share_button=True,
 
462
  scale=5,
 
463
  )
464
 
 
465
  depth_exr_file = gr.File(
466
+ label="Raw 32-bit EXR Depth Sequence (.zip)",
467
+ interactive=False,
 
 
 
 
 
 
468
  )
469
 
470
+ with gr.Row(equal_height=True):
471
 
472
+ with gr.Column(scale=1):
 
 
 
 
 
 
 
 
 
 
 
 
473
 
474
  with gr.Accordion(
 
475
  "Advanced Settings",
476
+ open=True,
 
 
477
  ):
478
 
 
479
  max_len = gr.Slider(
480
+ label="Max Process Length (frames; -1 = all)",
 
 
 
 
 
481
  minimum=-1,
 
482
  maximum=1000,
 
483
  value=500,
 
484
  step=1,
 
485
  )
486
 
 
487
  target_fps = gr.Slider(
488
+ label="Target FPS (-1 = original)",
 
 
 
 
 
489
  minimum=-1,
 
490
  maximum=30,
 
491
  value=15,
 
492
  step=1,
 
493
  )
494
 
 
495
  max_res = gr.Slider(
496
+ label="Max Side Resolution",
 
 
 
 
497
  minimum=480,
 
498
  maximum=2560,
 
499
  value=1280,
 
500
  step=1,
 
501
  info=(
502
+ "Controls the maximum size of the source/depth frames. "
503
+ "This is separate from the model inference size."
 
 
 
504
  ),
 
505
  )
506
 
 
 
 
 
 
 
507
  input_size = gr.Slider(
508
+ label="Model Inference Size",
 
 
 
 
509
  minimum=392,
 
510
  maximum=896,
 
511
  value=518,
 
512
  step=14,
 
513
  info=(
514
+ "518 is the normal VDA setting. Higher values can "
515
+ "preserve more spatial detail but use much more VRAM."
 
 
516
  ),
 
517
  )
518
 
 
519
  grayscale = gr.Checkbox(
520
+ label="Grayscale MP4 Preview",
 
 
 
 
521
  value=True,
 
522
  info=(
523
+ "Only affects the preview MP4. The EXRs always contain "
524
+ "the raw floating-point depth."
 
525
  ),
 
526
  )
527
 
 
528
  generate_btn = gr.Button(
 
529
  "Generate Depth + EXR",
530
+ variant="primary",
 
 
531
  )
532
 
533
+ with gr.Column(scale=2):
 
 
 
 
 
 
 
 
534
 
535
  gr.Markdown(
536
  """
537
  ### Blender EXR setup
538
 
539
+ 1. Unzip the downloaded EXR file.
540
+ 2. Load `frame_00000.exr` as an **Image Sequence**.
541
+ 3. Set the Image Texture to **Non-Color**.
542
+ 4. Use **Cubic** interpolation.
543
+ 5. Read the **R** channel.
544
+ 6. Feed it through **Map Range** before displacement.
 
545
 
546
+ The EXRs may look very dark in a normal image viewer. That is normal: they
547
+ contain raw floating-point model values instead of a normalized 8-bit preview.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
548
  """
549
  )
550
 
551
+ # Examples retain the original four settings. The inference-size and
552
+ # grayscale parameters use their function defaults for example runs.
 
 
 
553
  gr.Examples(
 
554
  examples=examples,
 
555
  inputs=[
 
556
  input_video,
 
557
  max_len,
 
558
  target_fps,
559
+ max_res,
 
 
560
  ],
 
561
  outputs=[
 
562
  processed_video,
 
563
  depth_vis_video,
564
+ depth_exr_file,
 
 
565
  ],
 
566
  fn=infer_video_depth,
567
+ cache_examples="lazy",
 
 
568
  )
569
 
 
 
 
 
 
570
  generate_btn.click(
 
571
  fn=infer_video_depth,
 
572
  inputs=[
 
573
  input_video,
 
574
  max_len,
 
575
  target_fps,
 
576
  max_res,
 
577
  input_size,
578
+ grayscale,
 
 
579
  ],
 
580
  outputs=[
 
581
  processed_video,
 
582
  depth_vis_video,
583
+ depth_exr_file,
 
 
584
  ],
 
585
  )
586
 
 
587
  return demo
588
 
589
 
 
 
 
 
590
  if __name__ == "__main__":
591
 
592
  demo = construct_demo()
 
593
  demo.queue()
594
 
595
+ # On Hugging Face Spaces, a public share URL is unnecessary.
596
+ demo.launch()