art-alex commited on
Commit
6de521a
·
verified ·
1 Parent(s): da03295

Run in 32 GB VRAM with phased offload; fix NVML assert and queue stalls

Browse files

- Build all models on CPU and move them to the GPU one pipeline phase at a
time (Gemma -> embeddings processor -> transformer/VAE encoder/upsampler
-> decoders), offloading the previous phase first. Peak VRAM is ~29 GiB
at 10 s 1536x1024, instead of 53 GB resident, so the Space no longer
depends on ZeroGPU handing it a full GPU.
- Use the cudaMallocAsync allocator: on ZeroGPU MIG slices the native
allocator turns any CUDA OOM into "NVML_SUCCESS == r INTERNAL ASSERT
FAILED" (same fix as the LTX-2.5 Space).
- Keep the Gradio queue alive when a session's message queue is lost,
instead of stopping it permanently ("503 Queue is stopped.").

Files changed (1) hide show
  1. app.py +118 -20
app.py CHANGED
@@ -6,6 +6,13 @@ import sys
6
  os.environ["TORCH_COMPILE_DISABLE"] = "1"
7
  os.environ["TORCHDYNAMO_DISABLE"] = "1"
8
 
 
 
 
 
 
 
 
9
  # Install xformers for memory-efficient attention
10
  subprocess.run([sys.executable, "-m", "pip", "install", "xformers==0.0.32.post2", "--no-build-isolation"], check=False)
11
 
@@ -34,6 +41,7 @@ subprocess.run(
34
  sys.path.insert(0, os.path.join(LTX_REPO_DIR, "packages", "ltx-pipelines", "src"))
35
  sys.path.insert(0, os.path.join(LTX_REPO_DIR, "packages", "ltx-core", "src"))
36
 
 
37
  import logging
38
  import random
39
  import tempfile
@@ -45,6 +53,27 @@ torch._dynamo.config.disable = True
45
 
46
  import spaces
47
  import gradio as gr
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
48
  import numpy as np
49
  from huggingface_hub import hf_hub_download, snapshot_download
50
 
@@ -176,27 +205,96 @@ pipeline = DistilledPipeline(
176
  quantization=QuantizationPolicy.fp8_cast(),
177
  )
178
 
179
- # Preload all models for ZeroGPU tensor packing.
180
- print("Preloading all models (including Gemma)...")
 
 
 
 
 
 
 
 
181
  ledger = pipeline.model_ledger
182
- _transformer = ledger.transformer()
183
- _video_encoder = ledger.video_encoder()
184
- _video_decoder = ledger.video_decoder()
185
- _audio_decoder = ledger.audio_decoder()
186
- _vocoder = ledger.vocoder()
187
- _spatial_upsampler = ledger.spatial_upsampler()
188
- _text_encoder = ledger.text_encoder()
189
- _embeddings_processor = ledger.gemma_embeddings_processor()
190
-
191
- ledger.transformer = lambda: _transformer
192
- ledger.video_encoder = lambda: _video_encoder
193
- ledger.video_decoder = lambda: _video_decoder
194
- ledger.audio_decoder = lambda: _audio_decoder
195
- ledger.vocoder = lambda: _vocoder
196
- ledger.spatial_upsampler = lambda: _spatial_upsampler
197
- ledger.text_encoder = lambda: _text_encoder
198
- ledger.gemma_embeddings_processor = lambda: _embeddings_processor
199
- print("All models preloaded (including Gemma text encoder)!")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
200
 
201
  print("=" * 80)
202
  print("Pipeline ready!")
 
6
  os.environ["TORCH_COMPILE_DISABLE"] = "1"
7
  os.environ["TORCHDYNAMO_DISABLE"] = "1"
8
 
9
+ # ZeroGPU's MIG slice trips an NVML assert in torch's native caching allocator
10
+ # ("NVML_SUCCESS == r INTERNAL ASSERT FAILED") whenever an allocation fails, which
11
+ # hides the real CUDA OOM. cudaMallocAsync is NVML-safe (same fix as the LTX-2.5
12
+ # Space). Must be set before torch initializes its allocator.
13
+ os.environ.setdefault("PYTORCH_ALLOC_CONF", "backend:cudaMallocAsync")
14
+ os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "backend:cudaMallocAsync")
15
+
16
  # Install xformers for memory-efficient attention
17
  subprocess.run([sys.executable, "-m", "pip", "install", "xformers==0.0.32.post2", "--no-build-isolation"], check=False)
18
 
 
41
  sys.path.insert(0, os.path.join(LTX_REPO_DIR, "packages", "ltx-pipelines", "src"))
42
  sys.path.insert(0, os.path.join(LTX_REPO_DIR, "packages", "ltx-core", "src"))
43
 
44
+ import gc
45
  import logging
46
  import random
47
  import tempfile
 
53
 
54
  import spaces
55
  import gradio as gr
56
+ import gradio.queueing
57
+
58
+
59
+ # Gradio 6.x bug workaround: Queue.send_message raises KeyError when an event's
60
+ # session is no longer in pending_messages_per_session (an LRUCache capped at 2000
61
+ # sessions, also cleared on client disconnect). If that happens inside
62
+ # Queue.start_processing (via broadcast_estimations), its `finally` sets
63
+ # queue.stopped = True permanently and every request gets "Queue is stopped."
64
+ # until the Space restarts. Drop the orphaned event instead of killing the queue.
65
+ _original_send_message = gradio.queueing.Queue.send_message
66
+
67
+
68
+ def _send_message_tolerating_lost_sessions(self, event, event_message):
69
+ try:
70
+ _original_send_message(self, event, event_message)
71
+ except KeyError:
72
+ event.alive = False
73
+
74
+
75
+ gradio.queueing.Queue.send_message = _send_message_tolerating_lost_sessions
76
+
77
  import numpy as np
78
  from huggingface_hub import hf_hub_download, snapshot_download
79
 
 
205
  quantization=QuantizationPolicy.fp8_cast(),
206
  )
207
 
208
+ # Preload all models into CPU RAM and move them to the GPU in phases.
209
+ #
210
+ # Keeping everything resident on the GPU needs ~53 GB (Gemma alone is ~24 GB).
211
+ # Instead, models are built on CPU at startup (so ZeroGPU packs nothing) and
212
+ # grouped by pipeline phase (as upstream encode_prompts already frees Gemma before
213
+ # the embeddings processor). Requesting a model from a group offloads whatever
214
+ # other group is on the GPU, so Gemma is gone before the transformer arrives and
215
+ # the transformer is gone before VAE decoding. Offloading just re-points the
216
+ # parameters at their original CPU tensors, so it costs no device-to-host copy.
217
+ print("Preloading all models into CPU RAM (including Gemma)...")
218
  ledger = pipeline.model_ledger
219
+ _compute_device = ledger.device
220
+ ledger.device = torch.device("cpu")
221
+ _models = {
222
+ "text_encoder": ledger.text_encoder(),
223
+ "gemma_embeddings_processor": ledger.gemma_embeddings_processor(),
224
+ "transformer": ledger.transformer(),
225
+ "video_encoder": ledger.video_encoder(),
226
+ "spatial_upsampler": ledger.spatial_upsampler(),
227
+ "video_decoder": ledger.video_decoder(),
228
+ "audio_decoder": ledger.audio_decoder(),
229
+ "vocoder": ledger.vocoder(),
230
+ }
231
+ ledger.device = _compute_device
232
+
233
+ _PHASES = {
234
+ "text": ("text_encoder",),
235
+ "embed": ("gemma_embeddings_processor",),
236
+ "denoise": ("transformer", "video_encoder", "spatial_upsampler"),
237
+ "decode": ("video_decoder", "audio_decoder", "vocoder"),
238
+ }
239
+ _PHASE_OF = {name: phase for phase, names in _PHASES.items() for name in names}
240
+
241
+
242
+ class _PhasedOffloader:
243
+ """Keeps at most one phase's models on the GPU."""
244
+
245
+ def __init__(self):
246
+ self.active = None
247
+ self.cpu_tensors = {} # model name -> list of (module, kind, key, cpu tensor)
248
+
249
+ def _upload(self, name):
250
+ saved = self.cpu_tensors[name] = []
251
+ for module in _models[name].modules():
252
+ for key, param in module._parameters.items():
253
+ if param is not None:
254
+ saved.append((module, "param", key, param.data))
255
+ param.data = param.data.to(_compute_device, non_blocking=True)
256
+ for key, buf in module._buffers.items():
257
+ if buf is not None:
258
+ saved.append((module, "buffer", key, buf))
259
+ module._buffers[key] = buf.to(_compute_device, non_blocking=True)
260
+
261
+ def _offload(self, name):
262
+ for module, kind, key, cpu_tensor in self.cpu_tensors.pop(name):
263
+ if kind == "param":
264
+ module._parameters[key].data = cpu_tensor
265
+ else:
266
+ module._buffers[key] = cpu_tensor
267
+
268
+ def activate(self, phase):
269
+ if self.active == phase:
270
+ return
271
+ self.offload_all()
272
+ try:
273
+ for name in _PHASES[phase]:
274
+ self._upload(name)
275
+ torch.cuda.synchronize()
276
+ except BaseException:
277
+ self.offload_all()
278
+ raise
279
+ self.active = phase
280
+ log_memory(f"phase {phase} loaded")
281
+
282
+ def offload_all(self):
283
+ for name in list(self.cpu_tensors):
284
+ self._offload(name)
285
+ self.active = None
286
+ gc.collect()
287
+ torch.cuda.empty_cache()
288
+
289
+ def get(self, name):
290
+ self.activate(_PHASE_OF[name])
291
+ return _models[name]
292
+
293
+
294
+ _offloader = _PhasedOffloader()
295
+ for _name in _models:
296
+ setattr(ledger, _name, lambda _name=_name: _offloader.get(_name))
297
+ print("All models preloaded on CPU (including Gemma text encoder)!")
298
 
299
  print("=" * 80)
300
  print("Pipeline ready!")