Download scale/script/preview_gpu_lease.py from Tinghong-Ye22/code-Movens-Zero_train: direct link, hf CLI and curl.
- Browser
- Download file 3.75 kB
-
https://huggingface.co/Tinghong-Ye22/code-Movens-Zero_train/resolve/main/scale/script/preview_gpu_lease.py
- Command line
-
hf download hf://Tinghong-Ye22/code-Movens-Zero_train/scale/script/preview_gpu_lease.py
-
curl -L -o preview_gpu_lease.py https://huggingface.co/Tinghong-Ye22/code-Movens-Zero_train/resolve/main/scale/script/preview_gpu_lease.py
3.75 kB
| """Lend DSW GPU 2 from an isolated keeper; never touch other GPU owners.""" | |
| import contextlib | |
| import os | |
| from pathlib import Path | |
| import signal | |
| import subprocess | |
| import time | |
| ROOT = Path(__file__).resolve().parents[2] | |
| MONITOR = (ROOT.parent / "monitor").resolve() | |
| PREVIEW_GPU = 2 | |
| KEEPER_PREFIX = '/tmp/movens_scale_preview_gpu2_keeper' | |
| PID_FILES = (Path(KEEPER_PREFIX + '_supervisor.pid'), Path(KEEPER_PREFIX + '.lock')) | |
| def argv(pid): | |
| try: | |
| return Path(f'/proc/{pid}/cmdline').read_bytes().decode().strip('\0').split('\0') | |
| except (FileNotFoundError, ProcessLookupError): | |
| return [] | |
| def keeper_pids(): | |
| pids = [] | |
| for path, script in zip(PID_FILES, ('monitor.sh', 'monitor.py')): | |
| if not path.exists(): | |
| continue | |
| pid = int(path.read_text().strip()) | |
| command = argv(pid) | |
| if command: | |
| if str(MONITOR / script) not in command: | |
| raise RuntimeError(f'Refusing unrelated process in keeper PID file: {pid}') | |
| pids.append(pid) | |
| return pids | |
| def gpu_processes(gpu=PREVIEW_GPU): | |
| uuid = subprocess.check_output(['nvidia-smi', f'--id={gpu}', '--query-gpu=uuid', '--format=csv,noheader'], text=True, timeout=10).strip() | |
| raw = subprocess.check_output(['nvidia-smi', '--query-compute-apps=gpu_uuid,pid', '--format=csv,noheader,nounits'], text=True, timeout=10) | |
| return {int(line.split(',')[1]) for line in raw.splitlines() if line.split(',')[0].strip() == uuid} | |
| def stop_keeper(): | |
| pids = keeper_pids() | |
| for pid in pids: | |
| try: | |
| os.kill(pid, signal.SIGTERM) | |
| except ProcessLookupError: | |
| pass | |
| for _ in range(40): | |
| if not any(argv(pid) for pid in pids): | |
| return | |
| time.sleep(.5) | |
| raise RuntimeError('Keeper did not exit; not using GPU') | |
| def start_keeper(): | |
| env = os.environ.copy() | |
| env.pop('CUDA_VISIBLE_DEVICES', None) | |
| env.pop('MONITOR_GPU_INDICES', None) | |
| # Other projects can deliberately hold the shared keeper lock for their | |
| # full job lifetime. Own only GPU 2 with separate supervisor/worker locks; | |
| # do not stop or reconfigure unrelated keepers or jobs on other GPUs. | |
| env.update(MONITOR_GPU_INDICES=str(PREVIEW_GPU), | |
| MONITOR_LOCK_FILE=str(PID_FILES[1]), | |
| MONITOR_SUPERVISOR_PID_FILE=str(PID_FILES[0]), | |
| MONITOR_SUPERVISOR_LOCK_FILE=KEEPER_PREFIX + '_supervisor.lock') | |
| subprocess.run(['bash', '-c', 'source "$1"', 'keeper', str(MONITOR / 'cmd.sh')], env=env, check=True, timeout=15) | |
| def restore_keeper(): | |
| stop_keeper() | |
| start_keeper() | |
| for _ in range(45): | |
| raw = subprocess.check_output(['nvidia-smi', f'--id={PREVIEW_GPU}', '--query-gpu=memory.used', '--format=csv,noheader,nounits'], text=True, timeout=10) | |
| if int(raw.strip()) >= 500: | |
| return | |
| time.sleep(1) | |
| raise RuntimeError(f'Preview GPU {PREVIEW_GPU} occupancy restoration not confirmed') | |
| def lease_gpu2(): | |
| foreign = gpu_processes() - set(keeper_pids()) | |
| if foreign: | |
| raise RuntimeError(f'GPU {PREVIEW_GPU} has non-keeper process(es) {sorted(foreign)}; waiting without stopping them') | |
| changed = False | |
| try: | |
| changed = True | |
| stop_keeper() | |
| raw = subprocess.check_output(['nvidia-smi', f'--id={PREVIEW_GPU}', '--query-gpu=memory.used', '--format=csv,noheader,nounits'], text=True, timeout=10) | |
| if gpu_processes() or int(raw.strip()) >= 500: | |
| raise RuntimeError(f'GPU {PREVIEW_GPU} is not free after keeper handoff') | |
| yield | |
| finally: | |
| if changed: | |
| restore_keeper() | |
| # Compatibility for older callers; the lease now targets PREVIEW_GPU (2). | |
| lease_gpu3 = lease_gpu2 | |