"""Lend DSW GPU 2 from an isolated keeper; never touch other GPU owners.""" import contextlib import os from pathlib import Path import signal import subprocess import time ROOT = Path(__file__).resolve().parents[2] MONITOR = (ROOT.parent / "monitor").resolve() PREVIEW_GPU = 2 KEEPER_PREFIX = '/tmp/movens_scale_preview_gpu2_keeper' PID_FILES = (Path(KEEPER_PREFIX + '_supervisor.pid'), Path(KEEPER_PREFIX + '.lock')) def argv(pid): try: return Path(f'/proc/{pid}/cmdline').read_bytes().decode().strip('\0').split('\0') except (FileNotFoundError, ProcessLookupError): return [] def keeper_pids(): pids = [] for path, script in zip(PID_FILES, ('monitor.sh', 'monitor.py')): if not path.exists(): continue pid = int(path.read_text().strip()) command = argv(pid) if command: if str(MONITOR / script) not in command: raise RuntimeError(f'Refusing unrelated process in keeper PID file: {pid}') pids.append(pid) return pids def gpu_processes(gpu=PREVIEW_GPU): uuid = subprocess.check_output(['nvidia-smi', f'--id={gpu}', '--query-gpu=uuid', '--format=csv,noheader'], text=True, timeout=10).strip() raw = subprocess.check_output(['nvidia-smi', '--query-compute-apps=gpu_uuid,pid', '--format=csv,noheader,nounits'], text=True, timeout=10) return {int(line.split(',')[1]) for line in raw.splitlines() if line.split(',')[0].strip() == uuid} def stop_keeper(): pids = keeper_pids() for pid in pids: try: os.kill(pid, signal.SIGTERM) except ProcessLookupError: pass for _ in range(40): if not any(argv(pid) for pid in pids): return time.sleep(.5) raise RuntimeError('Keeper did not exit; not using GPU') def start_keeper(): env = os.environ.copy() env.pop('CUDA_VISIBLE_DEVICES', None) env.pop('MONITOR_GPU_INDICES', None) # Other projects can deliberately hold the shared keeper lock for their # full job lifetime. Own only GPU 2 with separate supervisor/worker locks; # do not stop or reconfigure unrelated keepers or jobs on other GPUs. env.update(MONITOR_GPU_INDICES=str(PREVIEW_GPU), MONITOR_LOCK_FILE=str(PID_FILES[1]), MONITOR_SUPERVISOR_PID_FILE=str(PID_FILES[0]), MONITOR_SUPERVISOR_LOCK_FILE=KEEPER_PREFIX + '_supervisor.lock') subprocess.run(['bash', '-c', 'source "$1"', 'keeper', str(MONITOR / 'cmd.sh')], env=env, check=True, timeout=15) def restore_keeper(): stop_keeper() start_keeper() for _ in range(45): raw = subprocess.check_output(['nvidia-smi', f'--id={PREVIEW_GPU}', '--query-gpu=memory.used', '--format=csv,noheader,nounits'], text=True, timeout=10) if int(raw.strip()) >= 500: return time.sleep(1) raise RuntimeError(f'Preview GPU {PREVIEW_GPU} occupancy restoration not confirmed') @contextlib.contextmanager def lease_gpu2(): foreign = gpu_processes() - set(keeper_pids()) if foreign: raise RuntimeError(f'GPU {PREVIEW_GPU} has non-keeper process(es) {sorted(foreign)}; waiting without stopping them') changed = False try: changed = True stop_keeper() raw = subprocess.check_output(['nvidia-smi', f'--id={PREVIEW_GPU}', '--query-gpu=memory.used', '--format=csv,noheader,nounits'], text=True, timeout=10) if gpu_processes() or int(raw.strip()) >= 500: raise RuntimeError(f'GPU {PREVIEW_GPU} is not free after keeper handoff') yield finally: if changed: restore_keeper() # Compatibility for older callers; the lease now targets PREVIEW_GPU (2). lease_gpu3 = lease_gpu2