Fix torch.compile AssertionError in ZeroGPU workers on torch 2.13
Browse filesRebuilding the Space picked up torch 2.13.0 from the ZeroGPU base image.
In a GPU worker, spaces 0.51.3 restores torch.cuda._maybe_exchange_device
to the _exchange_device builtin; torch 2.13's dynamo registers one handler
per function and fails on the duplicate ("Handler already registered for
<built-in function _cuda_exchangeDevice>"), so every viewer session died
during renderer warm-up.
Restore the real builtin at the start of both @spaces.GPU functions,
before anything compiles. No-op when the two are already distinct.
Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
- app.py +3 -0
- viser_app/zerogpu_compat.py +28 -0
app.py
CHANGED
|
@@ -56,6 +56,7 @@ import gradio as gr
|
|
| 56 |
from viser_app.scene_loaders import (
|
| 57 |
IMAGE_EXTS, VIDEO_EXTS, extract_video_frames,
|
| 58 |
)
|
|
|
|
| 59 |
|
| 60 |
# ---------------------------------------------------------------------------
|
| 61 |
# Config
|
|
@@ -151,6 +152,7 @@ def reconstruct_gpu(img_dir: str, target_h: int, target_w: int, max_views: int)
|
|
| 151 |
from viser_app.pi3_estimator import _list_images, pi3_to_scene_data, run_pi3
|
| 152 |
from viser_app.recon_scene import reconstruct, save_artifact
|
| 153 |
|
|
|
|
| 154 |
image_paths = _list_images(Path(img_dir))
|
| 155 |
if max_views and len(image_paths) > max_views:
|
| 156 |
idxs = np.linspace(0, len(image_paths) - 1, max_views).round().astype(int)
|
|
@@ -202,6 +204,7 @@ def _live_session_worker(artifact_path: str):
|
|
| 202 |
from viser_app.viewer import install_viser_handlers
|
| 203 |
from viser_app.viewer_optim import install_optimizations
|
| 204 |
|
|
|
|
| 205 |
yield ("prep", 0.15, "Loading scene…")
|
| 206 |
model = DVSM_MODEL_CPU.cuda().eval()
|
| 207 |
artifact = torch.load(artifact_path, weights_only=False, map_location="cpu")
|
|
|
|
| 56 |
from viser_app.scene_loaders import (
|
| 57 |
IMAGE_EXTS, VIDEO_EXTS, extract_video_frames,
|
| 58 |
)
|
| 59 |
+
from viser_app.zerogpu_compat import fix_cuda_exchange_device
|
| 60 |
|
| 61 |
# ---------------------------------------------------------------------------
|
| 62 |
# Config
|
|
|
|
| 152 |
from viser_app.pi3_estimator import _list_images, pi3_to_scene_data, run_pi3
|
| 153 |
from viser_app.recon_scene import reconstruct, save_artifact
|
| 154 |
|
| 155 |
+
fix_cuda_exchange_device() # torch.compile vs ZeroGPU, see zerogpu_compat
|
| 156 |
image_paths = _list_images(Path(img_dir))
|
| 157 |
if max_views and len(image_paths) > max_views:
|
| 158 |
idxs = np.linspace(0, len(image_paths) - 1, max_views).round().astype(int)
|
|
|
|
| 204 |
from viser_app.viewer import install_viser_handlers
|
| 205 |
from viser_app.viewer_optim import install_optimizations
|
| 206 |
|
| 207 |
+
fix_cuda_exchange_device() # torch.compile vs ZeroGPU, see zerogpu_compat
|
| 208 |
yield ("prep", 0.15, "Loading scene…")
|
| 209 |
model = DVSM_MODEL_CPU.cuda().eval()
|
| 210 |
artifact = torch.load(artifact_path, weights_only=False, map_location="cpu")
|
viser_app/zerogpu_compat.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Workarounds for the ZeroGPU (`spaces`) runtime."""
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
def fix_cuda_exchange_device() -> bool:
|
| 5 |
+
"""Make torch.compile usable in a ZeroGPU worker on torch >= 2.13.
|
| 6 |
+
|
| 7 |
+
When a GPU worker starts, spaces.zero.torch.patching.unpatch() (seen in
|
| 8 |
+
spaces 0.51.3) restores torch.cuda._maybe_exchange_device to the
|
| 9 |
+
*_exchange_device* builtin, so both names point at one function. torch 2.13's
|
| 10 |
+
dynamo registers one handler per function and dies on the duplicate with
|
| 11 |
+
"AssertionError: Handler already registered for <built-in function
|
| 12 |
+
_cuda_exchangeDevice>" the first time a compiled function is traced.
|
| 13 |
+
|
| 14 |
+
Call this inside the @spaces.GPU function before anything compiles. It is a
|
| 15 |
+
no-op when the two functions are already distinct. Returns True if it had to
|
| 16 |
+
repair the alias.
|
| 17 |
+
"""
|
| 18 |
+
import torch
|
| 19 |
+
|
| 20 |
+
exchange = getattr(torch.cuda, "_exchange_device", None)
|
| 21 |
+
if exchange is None or getattr(torch.cuda, "_maybe_exchange_device", None) is not exchange:
|
| 22 |
+
return False
|
| 23 |
+
real = getattr(torch._C, "_cuda_maybeExchangeDevice", None)
|
| 24 |
+
if real is None or real is exchange:
|
| 25 |
+
def real(device): # distinct object, same behaviour as the alias
|
| 26 |
+
return exchange(device)
|
| 27 |
+
torch.cuda._maybe_exchange_device = real
|
| 28 |
+
return True
|