code-HFD / scripts /launch_hfd_dlc.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
9fc9873 verified
Raw History Blame Contribute Delete
7.38 kB
#!/usr/bin/env python3
"""One DLC job: local training, memory-admitted inference, CPFS mirror, dashboard."""
import json
import fcntl
import os
from pathlib import Path
import signal
import subprocess
import sys
import threading
import time
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT/'src'))
from psi.dlc_storage import atomic_json, synchronize, restore
def main():
env = dict(os.environ)
env.setdefault('HFD_RUN_TIMESTAMP', time.strftime('%y%m%d%H%M%S'))
env.setdefault('HFD_NPROC_PER_NODE', '8')
if "--print-config" in sys.argv[1:] or env.get("HFD_CONFIG_ONLY", "0") in {"1", "json"}:
# The same real shell/recipe, without creating a DLC run, mirror,
# dashboard or inference worker. Useful for pre-flight verification.
if not env.get('HFD_DLC_RESUME_RUN'):
env.setdefault('HFD_TRAIN_BATCH_SIZE', '8')
env.setdefault('HFD_EFFECTIVE_SOURCES_PER_GPU', env['HFD_TRAIN_BATCH_SIZE'])
elif not env.get('HFD_RESUME_FROM_CHECKPOINT') or env.get('HFD_RESUME_LOAD_FULL_STATE') != '1':
raise ValueError("Resume config inspection requires HFD_RESUME_FROM_CHECKPOINT and HFD_RESUME_LOAD_FULL_STATE")
raise SystemExit(subprocess.call(
['bash', str(ROOT/'posttrain-full-duplex-audio-command.sh'), *sys.argv[1:]], env=env))
world = int(env['HFD_NPROC_PER_NODE'])
local = Path(env.get('HFD_DLC_LOCAL_ROOT', '/mnt/workspace/hfd_dlc')).resolve()
remote = Path(env.get('HFD_DLC_BACKUP_ROOT', str(ROOT/'runs'))).resolve()
local.mkdir(parents=True, exist_ok=True)
remote.mkdir(parents=True, exist_ok=True)
if local == remote or local.stat().st_dev == remote.stat().st_dev:
raise ValueError('DLC local root must be on a different filesystem from CPFS backup')
env['HFD_OUTPUT_DIR'] = str(local/'runs')
if not env.get('HFD_DLC_RESUME_RUN'):
env.setdefault('HFD_TRAIN_BATCH_SIZE', '8')
env.setdefault('HFD_EFFECTIVE_SOURCES_PER_GPU', env['HFD_TRAIN_BATCH_SIZE'])
env.setdefault('HFD_NUM_WORKERS', '4')
env.setdefault('HFD_INFER_MIN_FREE_GIB', '48')
env.setdefault('HFD_INFER_MEMORY_BUDGET_GIB', '36')
env.setdefault('HFD_INFER_MAX_CONCURRENT', '1')
env.setdefault('HFD_INFER_CHECKPOINT_INTERVAL', '20000')
env.setdefault('HFD_INFER_GPUS', env.get('CUDA_VISIBLE_DEVICES', ','.join(map(str, range(world)))))
env.setdefault('HFD_RENDER_GPU', env['HFD_INFER_GPUS'].split(',')[0])
env['HFD_MANAGE_GPU_MONITOR'] = '0'
resume_run = Path(env['HFD_DLC_RESUME_RUN']).resolve() if env.get('HFD_DLC_RESUME_RUN') else None
if resume_run:
saved = json.loads((resume_run/'run_config.json').read_text())
env['HFD_RUN_TIMESTAMP'] = saved['timestamp']
env['HFD_WANDB_ID'] = saved['wandb']['id'] or ''
env['HFD_WANDB_RESUME'] = 'must' if env['HFD_WANDB_ID'] else 'never'
run_name = 'hfd.gpus'+str(world)+'.'+env['HFD_RUN_TIMESTAMP']
run = local/'runs/posttrain_full_duplex'/run_name
backup = remote/'posttrain_full_duplex'/run_name
# Reservation prevents accidental overwrite or concurrent launches.
run.mkdir(parents=True, exist_ok=bool(resume_run))
lock = (run/'.dlc.lock').open('a')
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
if resume_run:
if resume_run.name != run_name:
raise ValueError('Resume must preserve rank count and original run name')
checkpoint = restore(resume_run, run, world)
env['HFD_RESUME_FROM_CHECKPOINT'] = str(checkpoint)
env['HFD_RESUME_LOAD_FULL_STATE'] = '1'
atomic_json(run/'dlc_launch.json', dict(world_size=world, local=str(run), backup=str(backup),
batch_per_gpu=int(env.get('HFD_TRAIN_BATCH_SIZE', saved['train']['train_batch_size'] if resume_run else 8)), started=time.time()))
stop = threading.Event()
def mirror():
while not stop.is_set():
try:
step = synchronize(run, backup, world)
atomic_json(run/'backup_status.json', dict(ok=True, step=step, updated=time.time()))
except Exception as exc:
atomic_json(run/'backup_status.json', dict(ok=False, error=str(exc), updated=time.time()))
print('[backup:retry]', exc, flush=True)
stop.wait(float(env.get('HFD_DLC_SYNC_SECONDS', '120')))
worker = threading.Thread(target=mirror, daemon=True)
worker.start()
children, streams = [], []
def start(command, filename):
stream = (run/filename).open('ab')
streams.append(stream)
child = subprocess.Popen(command, cwd=ROOT, env=env, stdout=stream, stderr=subprocess.STDOUT,
start_new_session=True)
children.append(child)
return child
training = start(['bash', str(ROOT/'posttrain-full-duplex-audio-command.sh'), *sys.argv[1:]], 'training.log')
print(f'[DLC] training PID={training.pid} local={run} backup={backup}', flush=True)
interrupted = False
def shutdown(signum, frame):
nonlocal interrupted
interrupted = True
for child in children:
if child.poll() is None:
os.killpg(child.pid, signal.SIGTERM)
signal.signal(signal.SIGTERM, shutdown)
signal.signal(signal.SIGINT, shutdown)
infer = dashboard = None
try:
while training.poll() is None:
if (run/'run_config.json').is_file():
if infer is None or infer.poll() is not None:
infer = start(['bash', str(ROOT/'infer_render_auto.sh'), str(run)], 'auto_infer.log')
if dashboard is None or dashboard.poll() is not None:
dashboard = start([sys.executable, str(ROOT/'scripts/training_dashboard.py'), str(run),
'--host', '0.0.0.0', '--port', env.get('HFD_DASHBOARD_PORT', '8772')], 'dashboard.log')
time.sleep(10)
print('[DLC] training exit=', training.returncode, flush=True)
# The watcher remains alive after training so the final checkpoint renders.
if training.returncode == 0 and not interrupted and infer is not None:
while not interrupted:
time.sleep(10)
if infer.poll() is not None:
infer = start(['bash', str(ROOT/'infer_render_auto.sh'), str(run)], 'auto_infer.log')
if dashboard is not None and dashboard.poll() is not None:
dashboard = start([sys.executable, str(ROOT/'scripts/training_dashboard.py'), str(run),
'--host', '0.0.0.0', '--port', env.get('HFD_DASHBOARD_PORT', '8772')], 'dashboard.log')
finally:
shutdown(signal.SIGTERM, None)
for child in children:
try:
child.wait(timeout=30)
except subprocess.TimeoutExpired:
os.killpg(child.pid, signal.SIGKILL)
child.wait()
stop.set()
worker.join(timeout=300)
if not worker.is_alive():
try:
synchronize(run, backup, world)
except Exception as exc:
print('[backup:pending] Local artifacts retained:', run, exc, flush=True)
for stream in streams:
stream.close()
raise SystemExit(training.returncode or 0)
if __name__ == '__main__':
main()