code-Movens-Zero_train / scale /script /preview_gpu_lease.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
13cc860 verified
Raw History Blame Contribute Delete
3.75 kB
"""Lend DSW GPU 2 from an isolated keeper; never touch other GPU owners."""
import contextlib
import os
from pathlib import Path
import signal
import subprocess
import time
ROOT = Path(__file__).resolve().parents[2]
MONITOR = (ROOT.parent / "monitor").resolve()
PREVIEW_GPU = 2
KEEPER_PREFIX = '/tmp/movens_scale_preview_gpu2_keeper'
PID_FILES = (Path(KEEPER_PREFIX + '_supervisor.pid'), Path(KEEPER_PREFIX + '.lock'))
def argv(pid):
try:
return Path(f'/proc/{pid}/cmdline').read_bytes().decode().strip('\0').split('\0')
except (FileNotFoundError, ProcessLookupError):
return []
def keeper_pids():
pids = []
for path, script in zip(PID_FILES, ('monitor.sh', 'monitor.py')):
if not path.exists():
continue
pid = int(path.read_text().strip())
command = argv(pid)
if command:
if str(MONITOR / script) not in command:
raise RuntimeError(f'Refusing unrelated process in keeper PID file: {pid}')
pids.append(pid)
return pids
def gpu_processes(gpu=PREVIEW_GPU):
uuid = subprocess.check_output(['nvidia-smi', f'--id={gpu}', '--query-gpu=uuid', '--format=csv,noheader'], text=True, timeout=10).strip()
raw = subprocess.check_output(['nvidia-smi', '--query-compute-apps=gpu_uuid,pid', '--format=csv,noheader,nounits'], text=True, timeout=10)
return {int(line.split(',')[1]) for line in raw.splitlines() if line.split(',')[0].strip() == uuid}
def stop_keeper():
pids = keeper_pids()
for pid in pids:
try:
os.kill(pid, signal.SIGTERM)
except ProcessLookupError:
pass
for _ in range(40):
if not any(argv(pid) for pid in pids):
return
time.sleep(.5)
raise RuntimeError('Keeper did not exit; not using GPU')
def start_keeper():
env = os.environ.copy()
env.pop('CUDA_VISIBLE_DEVICES', None)
env.pop('MONITOR_GPU_INDICES', None)
# Other projects can deliberately hold the shared keeper lock for their
# full job lifetime. Own only GPU 2 with separate supervisor/worker locks;
# do not stop or reconfigure unrelated keepers or jobs on other GPUs.
env.update(MONITOR_GPU_INDICES=str(PREVIEW_GPU),
MONITOR_LOCK_FILE=str(PID_FILES[1]),
MONITOR_SUPERVISOR_PID_FILE=str(PID_FILES[0]),
MONITOR_SUPERVISOR_LOCK_FILE=KEEPER_PREFIX + '_supervisor.lock')
subprocess.run(['bash', '-c', 'source "$1"', 'keeper', str(MONITOR / 'cmd.sh')], env=env, check=True, timeout=15)
def restore_keeper():
stop_keeper()
start_keeper()
for _ in range(45):
raw = subprocess.check_output(['nvidia-smi', f'--id={PREVIEW_GPU}', '--query-gpu=memory.used', '--format=csv,noheader,nounits'], text=True, timeout=10)
if int(raw.strip()) >= 500:
return
time.sleep(1)
raise RuntimeError(f'Preview GPU {PREVIEW_GPU} occupancy restoration not confirmed')
@contextlib.contextmanager
def lease_gpu2():
foreign = gpu_processes() - set(keeper_pids())
if foreign:
raise RuntimeError(f'GPU {PREVIEW_GPU} has non-keeper process(es) {sorted(foreign)}; waiting without stopping them')
changed = False
try:
changed = True
stop_keeper()
raw = subprocess.check_output(['nvidia-smi', f'--id={PREVIEW_GPU}', '--query-gpu=memory.used', '--format=csv,noheader,nounits'], text=True, timeout=10)
if gpu_processes() or int(raw.strip()) >= 500:
raise RuntimeError(f'GPU {PREVIEW_GPU} is not free after keeper handoff')
yield
finally:
if changed:
restore_keeper()
# Compatibility for older callers; the lease now targets PREVIEW_GPU (2).
lease_gpu3 = lease_gpu2