linjohnss Claude Fable 5.2 commited on
Commit
5dda48b
·
1 Parent(s): 619e007

Fix torch.compile AssertionError in ZeroGPU workers on torch 2.13

Browse files

Rebuilding the Space picked up torch 2.13.0 from the ZeroGPU base image.
In a GPU worker, spaces 0.51.3 restores torch.cuda._maybe_exchange_device
to the _exchange_device builtin; torch 2.13's dynamo registers one handler
per function and fails on the duplicate ("Handler already registered for
<built-in function _cuda_exchangeDevice>"), so every viewer session died
during renderer warm-up.

Restore the real builtin at the start of both @spaces.GPU functions,
before anything compiles. No-op when the two are already distinct.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>

Files changed (2) hide show
  1. app.py +3 -0
  2. viser_app/zerogpu_compat.py +28 -0
app.py CHANGED
@@ -56,6 +56,7 @@ import gradio as gr
56
  from viser_app.scene_loaders import (
57
  IMAGE_EXTS, VIDEO_EXTS, extract_video_frames,
58
  )
 
59
 
60
  # ---------------------------------------------------------------------------
61
  # Config
@@ -151,6 +152,7 @@ def reconstruct_gpu(img_dir: str, target_h: int, target_w: int, max_views: int)
151
  from viser_app.pi3_estimator import _list_images, pi3_to_scene_data, run_pi3
152
  from viser_app.recon_scene import reconstruct, save_artifact
153
 
 
154
  image_paths = _list_images(Path(img_dir))
155
  if max_views and len(image_paths) > max_views:
156
  idxs = np.linspace(0, len(image_paths) - 1, max_views).round().astype(int)
@@ -202,6 +204,7 @@ def _live_session_worker(artifact_path: str):
202
  from viser_app.viewer import install_viser_handlers
203
  from viser_app.viewer_optim import install_optimizations
204
 
 
205
  yield ("prep", 0.15, "Loading scene…")
206
  model = DVSM_MODEL_CPU.cuda().eval()
207
  artifact = torch.load(artifact_path, weights_only=False, map_location="cpu")
 
56
  from viser_app.scene_loaders import (
57
  IMAGE_EXTS, VIDEO_EXTS, extract_video_frames,
58
  )
59
+ from viser_app.zerogpu_compat import fix_cuda_exchange_device
60
 
61
  # ---------------------------------------------------------------------------
62
  # Config
 
152
  from viser_app.pi3_estimator import _list_images, pi3_to_scene_data, run_pi3
153
  from viser_app.recon_scene import reconstruct, save_artifact
154
 
155
+ fix_cuda_exchange_device() # torch.compile vs ZeroGPU, see zerogpu_compat
156
  image_paths = _list_images(Path(img_dir))
157
  if max_views and len(image_paths) > max_views:
158
  idxs = np.linspace(0, len(image_paths) - 1, max_views).round().astype(int)
 
204
  from viser_app.viewer import install_viser_handlers
205
  from viser_app.viewer_optim import install_optimizations
206
 
207
+ fix_cuda_exchange_device() # torch.compile vs ZeroGPU, see zerogpu_compat
208
  yield ("prep", 0.15, "Loading scene…")
209
  model = DVSM_MODEL_CPU.cuda().eval()
210
  artifact = torch.load(artifact_path, weights_only=False, map_location="cpu")
viser_app/zerogpu_compat.py ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Workarounds for the ZeroGPU (`spaces`) runtime."""
2
+
3
+
4
+ def fix_cuda_exchange_device() -> bool:
5
+ """Make torch.compile usable in a ZeroGPU worker on torch >= 2.13.
6
+
7
+ When a GPU worker starts, spaces.zero.torch.patching.unpatch() (seen in
8
+ spaces 0.51.3) restores torch.cuda._maybe_exchange_device to the
9
+ *_exchange_device* builtin, so both names point at one function. torch 2.13's
10
+ dynamo registers one handler per function and dies on the duplicate with
11
+ "AssertionError: Handler already registered for <built-in function
12
+ _cuda_exchangeDevice>" the first time a compiled function is traced.
13
+
14
+ Call this inside the @spaces.GPU function before anything compiles. It is a
15
+ no-op when the two functions are already distinct. Returns True if it had to
16
+ repair the alias.
17
+ """
18
+ import torch
19
+
20
+ exchange = getattr(torch.cuda, "_exchange_device", None)
21
+ if exchange is None or getattr(torch.cuda, "_maybe_exchange_device", None) is not exchange:
22
+ return False
23
+ real = getattr(torch._C, "_cuda_maybeExchangeDevice", None)
24
+ if real is None or real is exchange:
25
+ def real(device): # distinct object, same behaviour as the alias
26
+ return exchange(device)
27
+ torch.cuda._maybe_exchange_device = real
28
+ return True