File size: 7,384 Bytes
9fc9873 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 | #!/usr/bin/env python3
"""One DLC job: local training, memory-admitted inference, CPFS mirror, dashboard."""
import json
import fcntl
import os
from pathlib import Path
import signal
import subprocess
import sys
import threading
import time
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT/'src'))
from psi.dlc_storage import atomic_json, synchronize, restore
def main():
env = dict(os.environ)
env.setdefault('HFD_RUN_TIMESTAMP', time.strftime('%y%m%d%H%M%S'))
env.setdefault('HFD_NPROC_PER_NODE', '8')
if "--print-config" in sys.argv[1:] or env.get("HFD_CONFIG_ONLY", "0") in {"1", "json"}:
# The same real shell/recipe, without creating a DLC run, mirror,
# dashboard or inference worker. Useful for pre-flight verification.
if not env.get('HFD_DLC_RESUME_RUN'):
env.setdefault('HFD_TRAIN_BATCH_SIZE', '8')
env.setdefault('HFD_EFFECTIVE_SOURCES_PER_GPU', env['HFD_TRAIN_BATCH_SIZE'])
elif not env.get('HFD_RESUME_FROM_CHECKPOINT') or env.get('HFD_RESUME_LOAD_FULL_STATE') != '1':
raise ValueError("Resume config inspection requires HFD_RESUME_FROM_CHECKPOINT and HFD_RESUME_LOAD_FULL_STATE")
raise SystemExit(subprocess.call(
['bash', str(ROOT/'posttrain-full-duplex-audio-command.sh'), *sys.argv[1:]], env=env))
world = int(env['HFD_NPROC_PER_NODE'])
local = Path(env.get('HFD_DLC_LOCAL_ROOT', '/mnt/workspace/hfd_dlc')).resolve()
remote = Path(env.get('HFD_DLC_BACKUP_ROOT', str(ROOT/'runs'))).resolve()
local.mkdir(parents=True, exist_ok=True)
remote.mkdir(parents=True, exist_ok=True)
if local == remote or local.stat().st_dev == remote.stat().st_dev:
raise ValueError('DLC local root must be on a different filesystem from CPFS backup')
env['HFD_OUTPUT_DIR'] = str(local/'runs')
if not env.get('HFD_DLC_RESUME_RUN'):
env.setdefault('HFD_TRAIN_BATCH_SIZE', '8')
env.setdefault('HFD_EFFECTIVE_SOURCES_PER_GPU', env['HFD_TRAIN_BATCH_SIZE'])
env.setdefault('HFD_NUM_WORKERS', '4')
env.setdefault('HFD_INFER_MIN_FREE_GIB', '48')
env.setdefault('HFD_INFER_MEMORY_BUDGET_GIB', '36')
env.setdefault('HFD_INFER_MAX_CONCURRENT', '1')
env.setdefault('HFD_INFER_CHECKPOINT_INTERVAL', '20000')
env.setdefault('HFD_INFER_GPUS', env.get('CUDA_VISIBLE_DEVICES', ','.join(map(str, range(world)))))
env.setdefault('HFD_RENDER_GPU', env['HFD_INFER_GPUS'].split(',')[0])
env['HFD_MANAGE_GPU_MONITOR'] = '0'
resume_run = Path(env['HFD_DLC_RESUME_RUN']).resolve() if env.get('HFD_DLC_RESUME_RUN') else None
if resume_run:
saved = json.loads((resume_run/'run_config.json').read_text())
env['HFD_RUN_TIMESTAMP'] = saved['timestamp']
env['HFD_WANDB_ID'] = saved['wandb']['id'] or ''
env['HFD_WANDB_RESUME'] = 'must' if env['HFD_WANDB_ID'] else 'never'
run_name = 'hfd.gpus'+str(world)+'.'+env['HFD_RUN_TIMESTAMP']
run = local/'runs/posttrain_full_duplex'/run_name
backup = remote/'posttrain_full_duplex'/run_name
# Reservation prevents accidental overwrite or concurrent launches.
run.mkdir(parents=True, exist_ok=bool(resume_run))
lock = (run/'.dlc.lock').open('a')
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
if resume_run:
if resume_run.name != run_name:
raise ValueError('Resume must preserve rank count and original run name')
checkpoint = restore(resume_run, run, world)
env['HFD_RESUME_FROM_CHECKPOINT'] = str(checkpoint)
env['HFD_RESUME_LOAD_FULL_STATE'] = '1'
atomic_json(run/'dlc_launch.json', dict(world_size=world, local=str(run), backup=str(backup),
batch_per_gpu=int(env.get('HFD_TRAIN_BATCH_SIZE', saved['train']['train_batch_size'] if resume_run else 8)), started=time.time()))
stop = threading.Event()
def mirror():
while not stop.is_set():
try:
step = synchronize(run, backup, world)
atomic_json(run/'backup_status.json', dict(ok=True, step=step, updated=time.time()))
except Exception as exc:
atomic_json(run/'backup_status.json', dict(ok=False, error=str(exc), updated=time.time()))
print('[backup:retry]', exc, flush=True)
stop.wait(float(env.get('HFD_DLC_SYNC_SECONDS', '120')))
worker = threading.Thread(target=mirror, daemon=True)
worker.start()
children, streams = [], []
def start(command, filename):
stream = (run/filename).open('ab')
streams.append(stream)
child = subprocess.Popen(command, cwd=ROOT, env=env, stdout=stream, stderr=subprocess.STDOUT,
start_new_session=True)
children.append(child)
return child
training = start(['bash', str(ROOT/'posttrain-full-duplex-audio-command.sh'), *sys.argv[1:]], 'training.log')
print(f'[DLC] training PID={training.pid} local={run} backup={backup}', flush=True)
interrupted = False
def shutdown(signum, frame):
nonlocal interrupted
interrupted = True
for child in children:
if child.poll() is None:
os.killpg(child.pid, signal.SIGTERM)
signal.signal(signal.SIGTERM, shutdown)
signal.signal(signal.SIGINT, shutdown)
infer = dashboard = None
try:
while training.poll() is None:
if (run/'run_config.json').is_file():
if infer is None or infer.poll() is not None:
infer = start(['bash', str(ROOT/'infer_render_auto.sh'), str(run)], 'auto_infer.log')
if dashboard is None or dashboard.poll() is not None:
dashboard = start([sys.executable, str(ROOT/'scripts/training_dashboard.py'), str(run),
'--host', '0.0.0.0', '--port', env.get('HFD_DASHBOARD_PORT', '8772')], 'dashboard.log')
time.sleep(10)
print('[DLC] training exit=', training.returncode, flush=True)
# The watcher remains alive after training so the final checkpoint renders.
if training.returncode == 0 and not interrupted and infer is not None:
while not interrupted:
time.sleep(10)
if infer.poll() is not None:
infer = start(['bash', str(ROOT/'infer_render_auto.sh'), str(run)], 'auto_infer.log')
if dashboard is not None and dashboard.poll() is not None:
dashboard = start([sys.executable, str(ROOT/'scripts/training_dashboard.py'), str(run),
'--host', '0.0.0.0', '--port', env.get('HFD_DASHBOARD_PORT', '8772')], 'dashboard.log')
finally:
shutdown(signal.SIGTERM, None)
for child in children:
try:
child.wait(timeout=30)
except subprocess.TimeoutExpired:
os.killpg(child.pid, signal.SIGKILL)
child.wait()
stop.set()
worker.join(timeout=300)
if not worker.is_alive():
try:
synchronize(run, backup, world)
except Exception as exc:
print('[backup:pending] Local artifacts retained:', run, exc, flush=True)
for stream in streams:
stream.close()
raise SystemExit(training.returncode or 0)
if __name__ == '__main__':
main()
|