Spaces:
Running on Zero
Running on Zero
Run in 32 GB VRAM with phased offload; fix NVML assert and queue stalls
Browse files- Build all models on CPU and move them to the GPU one pipeline phase at a
time (Gemma -> embeddings processor -> transformer/VAE encoder/upsampler
-> decoders), offloading the previous phase first. Peak VRAM is ~29 GiB
at 10 s 1536x1024, instead of 53 GB resident, so the Space no longer
depends on ZeroGPU handing it a full GPU.
- Use the cudaMallocAsync allocator: on ZeroGPU MIG slices the native
allocator turns any CUDA OOM into "NVML_SUCCESS == r INTERNAL ASSERT
FAILED" (same fix as the LTX-2.5 Space).
- Keep the Gradio queue alive when a session's message queue is lost,
instead of stopping it permanently ("503 Queue is stopped.").
app.py
CHANGED
|
@@ -6,6 +6,13 @@ import sys
|
|
| 6 |
os.environ["TORCH_COMPILE_DISABLE"] = "1"
|
| 7 |
os.environ["TORCHDYNAMO_DISABLE"] = "1"
|
| 8 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
# Install xformers for memory-efficient attention
|
| 10 |
subprocess.run([sys.executable, "-m", "pip", "install", "xformers==0.0.32.post2", "--no-build-isolation"], check=False)
|
| 11 |
|
|
@@ -34,6 +41,7 @@ subprocess.run(
|
|
| 34 |
sys.path.insert(0, os.path.join(LTX_REPO_DIR, "packages", "ltx-pipelines", "src"))
|
| 35 |
sys.path.insert(0, os.path.join(LTX_REPO_DIR, "packages", "ltx-core", "src"))
|
| 36 |
|
|
|
|
| 37 |
import logging
|
| 38 |
import random
|
| 39 |
import tempfile
|
|
@@ -45,6 +53,27 @@ torch._dynamo.config.disable = True
|
|
| 45 |
|
| 46 |
import spaces
|
| 47 |
import gradio as gr
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 48 |
import numpy as np
|
| 49 |
from huggingface_hub import hf_hub_download, snapshot_download
|
| 50 |
|
|
@@ -176,27 +205,96 @@ pipeline = DistilledPipeline(
|
|
| 176 |
quantization=QuantizationPolicy.fp8_cast(),
|
| 177 |
)
|
| 178 |
|
| 179 |
-
# Preload all models
|
| 180 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 181 |
ledger = pipeline.model_ledger
|
| 182 |
-
|
| 183 |
-
|
| 184 |
-
|
| 185 |
-
|
| 186 |
-
|
| 187 |
-
|
| 188 |
-
|
| 189 |
-
|
| 190 |
-
|
| 191 |
-
|
| 192 |
-
|
| 193 |
-
|
| 194 |
-
ledger.
|
| 195 |
-
|
| 196 |
-
|
| 197 |
-
|
| 198 |
-
|
| 199 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 200 |
|
| 201 |
print("=" * 80)
|
| 202 |
print("Pipeline ready!")
|
|
|
|
| 6 |
os.environ["TORCH_COMPILE_DISABLE"] = "1"
|
| 7 |
os.environ["TORCHDYNAMO_DISABLE"] = "1"
|
| 8 |
|
| 9 |
+
# ZeroGPU's MIG slice trips an NVML assert in torch's native caching allocator
|
| 10 |
+
# ("NVML_SUCCESS == r INTERNAL ASSERT FAILED") whenever an allocation fails, which
|
| 11 |
+
# hides the real CUDA OOM. cudaMallocAsync is NVML-safe (same fix as the LTX-2.5
|
| 12 |
+
# Space). Must be set before torch initializes its allocator.
|
| 13 |
+
os.environ.setdefault("PYTORCH_ALLOC_CONF", "backend:cudaMallocAsync")
|
| 14 |
+
os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "backend:cudaMallocAsync")
|
| 15 |
+
|
| 16 |
# Install xformers for memory-efficient attention
|
| 17 |
subprocess.run([sys.executable, "-m", "pip", "install", "xformers==0.0.32.post2", "--no-build-isolation"], check=False)
|
| 18 |
|
|
|
|
| 41 |
sys.path.insert(0, os.path.join(LTX_REPO_DIR, "packages", "ltx-pipelines", "src"))
|
| 42 |
sys.path.insert(0, os.path.join(LTX_REPO_DIR, "packages", "ltx-core", "src"))
|
| 43 |
|
| 44 |
+
import gc
|
| 45 |
import logging
|
| 46 |
import random
|
| 47 |
import tempfile
|
|
|
|
| 53 |
|
| 54 |
import spaces
|
| 55 |
import gradio as gr
|
| 56 |
+
import gradio.queueing
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
# Gradio 6.x bug workaround: Queue.send_message raises KeyError when an event's
|
| 60 |
+
# session is no longer in pending_messages_per_session (an LRUCache capped at 2000
|
| 61 |
+
# sessions, also cleared on client disconnect). If that happens inside
|
| 62 |
+
# Queue.start_processing (via broadcast_estimations), its `finally` sets
|
| 63 |
+
# queue.stopped = True permanently and every request gets "Queue is stopped."
|
| 64 |
+
# until the Space restarts. Drop the orphaned event instead of killing the queue.
|
| 65 |
+
_original_send_message = gradio.queueing.Queue.send_message
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def _send_message_tolerating_lost_sessions(self, event, event_message):
|
| 69 |
+
try:
|
| 70 |
+
_original_send_message(self, event, event_message)
|
| 71 |
+
except KeyError:
|
| 72 |
+
event.alive = False
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
gradio.queueing.Queue.send_message = _send_message_tolerating_lost_sessions
|
| 76 |
+
|
| 77 |
import numpy as np
|
| 78 |
from huggingface_hub import hf_hub_download, snapshot_download
|
| 79 |
|
|
|
|
| 205 |
quantization=QuantizationPolicy.fp8_cast(),
|
| 206 |
)
|
| 207 |
|
| 208 |
+
# Preload all models into CPU RAM and move them to the GPU in phases.
|
| 209 |
+
#
|
| 210 |
+
# Keeping everything resident on the GPU needs ~53 GB (Gemma alone is ~24 GB).
|
| 211 |
+
# Instead, models are built on CPU at startup (so ZeroGPU packs nothing) and
|
| 212 |
+
# grouped by pipeline phase (as upstream encode_prompts already frees Gemma before
|
| 213 |
+
# the embeddings processor). Requesting a model from a group offloads whatever
|
| 214 |
+
# other group is on the GPU, so Gemma is gone before the transformer arrives and
|
| 215 |
+
# the transformer is gone before VAE decoding. Offloading just re-points the
|
| 216 |
+
# parameters at their original CPU tensors, so it costs no device-to-host copy.
|
| 217 |
+
print("Preloading all models into CPU RAM (including Gemma)...")
|
| 218 |
ledger = pipeline.model_ledger
|
| 219 |
+
_compute_device = ledger.device
|
| 220 |
+
ledger.device = torch.device("cpu")
|
| 221 |
+
_models = {
|
| 222 |
+
"text_encoder": ledger.text_encoder(),
|
| 223 |
+
"gemma_embeddings_processor": ledger.gemma_embeddings_processor(),
|
| 224 |
+
"transformer": ledger.transformer(),
|
| 225 |
+
"video_encoder": ledger.video_encoder(),
|
| 226 |
+
"spatial_upsampler": ledger.spatial_upsampler(),
|
| 227 |
+
"video_decoder": ledger.video_decoder(),
|
| 228 |
+
"audio_decoder": ledger.audio_decoder(),
|
| 229 |
+
"vocoder": ledger.vocoder(),
|
| 230 |
+
}
|
| 231 |
+
ledger.device = _compute_device
|
| 232 |
+
|
| 233 |
+
_PHASES = {
|
| 234 |
+
"text": ("text_encoder",),
|
| 235 |
+
"embed": ("gemma_embeddings_processor",),
|
| 236 |
+
"denoise": ("transformer", "video_encoder", "spatial_upsampler"),
|
| 237 |
+
"decode": ("video_decoder", "audio_decoder", "vocoder"),
|
| 238 |
+
}
|
| 239 |
+
_PHASE_OF = {name: phase for phase, names in _PHASES.items() for name in names}
|
| 240 |
+
|
| 241 |
+
|
| 242 |
+
class _PhasedOffloader:
|
| 243 |
+
"""Keeps at most one phase's models on the GPU."""
|
| 244 |
+
|
| 245 |
+
def __init__(self):
|
| 246 |
+
self.active = None
|
| 247 |
+
self.cpu_tensors = {} # model name -> list of (module, kind, key, cpu tensor)
|
| 248 |
+
|
| 249 |
+
def _upload(self, name):
|
| 250 |
+
saved = self.cpu_tensors[name] = []
|
| 251 |
+
for module in _models[name].modules():
|
| 252 |
+
for key, param in module._parameters.items():
|
| 253 |
+
if param is not None:
|
| 254 |
+
saved.append((module, "param", key, param.data))
|
| 255 |
+
param.data = param.data.to(_compute_device, non_blocking=True)
|
| 256 |
+
for key, buf in module._buffers.items():
|
| 257 |
+
if buf is not None:
|
| 258 |
+
saved.append((module, "buffer", key, buf))
|
| 259 |
+
module._buffers[key] = buf.to(_compute_device, non_blocking=True)
|
| 260 |
+
|
| 261 |
+
def _offload(self, name):
|
| 262 |
+
for module, kind, key, cpu_tensor in self.cpu_tensors.pop(name):
|
| 263 |
+
if kind == "param":
|
| 264 |
+
module._parameters[key].data = cpu_tensor
|
| 265 |
+
else:
|
| 266 |
+
module._buffers[key] = cpu_tensor
|
| 267 |
+
|
| 268 |
+
def activate(self, phase):
|
| 269 |
+
if self.active == phase:
|
| 270 |
+
return
|
| 271 |
+
self.offload_all()
|
| 272 |
+
try:
|
| 273 |
+
for name in _PHASES[phase]:
|
| 274 |
+
self._upload(name)
|
| 275 |
+
torch.cuda.synchronize()
|
| 276 |
+
except BaseException:
|
| 277 |
+
self.offload_all()
|
| 278 |
+
raise
|
| 279 |
+
self.active = phase
|
| 280 |
+
log_memory(f"phase {phase} loaded")
|
| 281 |
+
|
| 282 |
+
def offload_all(self):
|
| 283 |
+
for name in list(self.cpu_tensors):
|
| 284 |
+
self._offload(name)
|
| 285 |
+
self.active = None
|
| 286 |
+
gc.collect()
|
| 287 |
+
torch.cuda.empty_cache()
|
| 288 |
+
|
| 289 |
+
def get(self, name):
|
| 290 |
+
self.activate(_PHASE_OF[name])
|
| 291 |
+
return _models[name]
|
| 292 |
+
|
| 293 |
+
|
| 294 |
+
_offloader = _PhasedOffloader()
|
| 295 |
+
for _name in _models:
|
| 296 |
+
setattr(ledger, _name, lambda _name=_name: _offloader.get(_name))
|
| 297 |
+
print("All models preloaded on CPU (including Gemma text encoder)!")
|
| 298 |
|
| 299 |
print("=" * 80)
|
| 300 |
print("Pipeline ready!")
|