Spaces:
Running on Zero
Running on Zero
File size: 2,828 Bytes
6febba8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 | """One H3 GPU job at a time. The guard refuses; it never kills anything."""
from __future__ import annotations
import math
import os
import subprocess
GPU_SCRIPTS = ("train_synth.py", "parity_h3.py", "procrustes_h3.py", "make_pairs_h3.py", "render_h3.py", "time_decode_h3.py", "train_h3.py", "build_cache_h3.py", "time_lot_h3.py", "mixed_res_h3.py", "two_track_h3.py", "train_tt.py", "build_pairs_tt.py")
def other_gpu_job() -> str | None:
"""Command line of another LoT GPU script on this machine, or ``None``."""
listing = subprocess.run(["ps", "-eo", "pid=,args="], capture_output=True, text=True)
me = os.getpid()
for line in listing.stdout.splitlines():
pid, _, cmd = line.strip().partition(" ")
argv = cmd.split()
if not pid.isdigit() or int(pid) == me or not argv:
continue
# argv[0] must be the interpreter: a shell whose command string names a
# script is a wrapper, not the job.
if not os.path.basename(argv[0]).startswith("python"):
continue
if any(os.path.basename(arg) in GPU_SCRIPTS for arg in argv[1:]):
return cmd
return None
def refuse_if_busy(name: str) -> None:
job = other_gpu_job()
if job is not None:
raise SystemExit(f"{name}: another LoT GPU job is running ({job}). It will not be killed.")
def card_gb() -> float:
"""Free memory on the current CUDA device, GB (other processes count against it)."""
import torch
free, _total = torch.cuda.mem_get_info()
return free / 2**30
def auto_swap(value: str, inference: bool = False, latent_t: int = 7) -> int:
"""``--swap auto``: DiT blocks to stream from CPU, from the card's free memory.
Measured with ``train_tt.py lora --check`` (two-track LoRA, 1344x768 pairs): 8 blocks peak at
19.6 GB on an RTX 5090; 24 blocks peak at 13.3 GB on an RTX PRO 4000 with 6 GB already taken.
0 (all resident) on 64 GB+ free, e.g. an RTX PRO 6000 (96 GB), is not measured yet."""
if value != "auto":
return int(value)
gb = card_gb()
if inference:
# Rendering (no LoRA grads, no checkpointing), measured 2026-11-13 on an RTX 5090 at 1344x768, 22 frames,
# VAE decode included: peak 26.8 GB with every block resident, about 0.42 GB less per streamed block.
# Streaming costs two-track the most (LR steps are short): 14.0 s at 0-2 streamed, 20.9 s at 8.
# Longer clips need more (activations, VAE decode): 22 latent frames (73 pixel frames) ran out of memory
# with none streamed and fit with 6. Assumed 0.35 GB per latent frame beyond 7.
need = 26.8 + 0.35 * max(0, latent_t - 7) + 1.0
return 0 if gb >= need else min(50, int(math.ceil((need - gb) / 0.42)))
return 0 if gb >= 64 else 8 if gb >= 28 else 16 if gb >= 21 else 24
|