#!/usr/bin/env python3 """One DLC job: local training, memory-admitted inference, CPFS mirror, dashboard.""" import json import fcntl import os from pathlib import Path import signal import subprocess import sys import threading import time ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT/'src')) from psi.dlc_storage import atomic_json, synchronize, restore def main(): env = dict(os.environ) env.setdefault('HFD_RUN_TIMESTAMP', time.strftime('%y%m%d%H%M%S')) env.setdefault('HFD_NPROC_PER_NODE', '8') if "--print-config" in sys.argv[1:] or env.get("HFD_CONFIG_ONLY", "0") in {"1", "json"}: # The same real shell/recipe, without creating a DLC run, mirror, # dashboard or inference worker. Useful for pre-flight verification. if not env.get('HFD_DLC_RESUME_RUN'): env.setdefault('HFD_TRAIN_BATCH_SIZE', '8') env.setdefault('HFD_EFFECTIVE_SOURCES_PER_GPU', env['HFD_TRAIN_BATCH_SIZE']) elif not env.get('HFD_RESUME_FROM_CHECKPOINT') or env.get('HFD_RESUME_LOAD_FULL_STATE') != '1': raise ValueError("Resume config inspection requires HFD_RESUME_FROM_CHECKPOINT and HFD_RESUME_LOAD_FULL_STATE") raise SystemExit(subprocess.call( ['bash', str(ROOT/'posttrain-full-duplex-audio-command.sh'), *sys.argv[1:]], env=env)) world = int(env['HFD_NPROC_PER_NODE']) local = Path(env.get('HFD_DLC_LOCAL_ROOT', '/mnt/workspace/hfd_dlc')).resolve() remote = Path(env.get('HFD_DLC_BACKUP_ROOT', str(ROOT/'runs'))).resolve() local.mkdir(parents=True, exist_ok=True) remote.mkdir(parents=True, exist_ok=True) if local == remote or local.stat().st_dev == remote.stat().st_dev: raise ValueError('DLC local root must be on a different filesystem from CPFS backup') env['HFD_OUTPUT_DIR'] = str(local/'runs') if not env.get('HFD_DLC_RESUME_RUN'): env.setdefault('HFD_TRAIN_BATCH_SIZE', '8') env.setdefault('HFD_EFFECTIVE_SOURCES_PER_GPU', env['HFD_TRAIN_BATCH_SIZE']) env.setdefault('HFD_NUM_WORKERS', '4') env.setdefault('HFD_INFER_MIN_FREE_GIB', '48') env.setdefault('HFD_INFER_MEMORY_BUDGET_GIB', '36') env.setdefault('HFD_INFER_MAX_CONCURRENT', '1') env.setdefault('HFD_INFER_CHECKPOINT_INTERVAL', '20000') env.setdefault('HFD_INFER_GPUS', env.get('CUDA_VISIBLE_DEVICES', ','.join(map(str, range(world))))) env.setdefault('HFD_RENDER_GPU', env['HFD_INFER_GPUS'].split(',')[0]) env['HFD_MANAGE_GPU_MONITOR'] = '0' resume_run = Path(env['HFD_DLC_RESUME_RUN']).resolve() if env.get('HFD_DLC_RESUME_RUN') else None if resume_run: saved = json.loads((resume_run/'run_config.json').read_text()) env['HFD_RUN_TIMESTAMP'] = saved['timestamp'] env['HFD_WANDB_ID'] = saved['wandb']['id'] or '' env['HFD_WANDB_RESUME'] = 'must' if env['HFD_WANDB_ID'] else 'never' run_name = 'hfd.gpus'+str(world)+'.'+env['HFD_RUN_TIMESTAMP'] run = local/'runs/posttrain_full_duplex'/run_name backup = remote/'posttrain_full_duplex'/run_name # Reservation prevents accidental overwrite or concurrent launches. run.mkdir(parents=True, exist_ok=bool(resume_run)) lock = (run/'.dlc.lock').open('a') fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) if resume_run: if resume_run.name != run_name: raise ValueError('Resume must preserve rank count and original run name') checkpoint = restore(resume_run, run, world) env['HFD_RESUME_FROM_CHECKPOINT'] = str(checkpoint) env['HFD_RESUME_LOAD_FULL_STATE'] = '1' atomic_json(run/'dlc_launch.json', dict(world_size=world, local=str(run), backup=str(backup), batch_per_gpu=int(env.get('HFD_TRAIN_BATCH_SIZE', saved['train']['train_batch_size'] if resume_run else 8)), started=time.time())) stop = threading.Event() def mirror(): while not stop.is_set(): try: step = synchronize(run, backup, world) atomic_json(run/'backup_status.json', dict(ok=True, step=step, updated=time.time())) except Exception as exc: atomic_json(run/'backup_status.json', dict(ok=False, error=str(exc), updated=time.time())) print('[backup:retry]', exc, flush=True) stop.wait(float(env.get('HFD_DLC_SYNC_SECONDS', '120'))) worker = threading.Thread(target=mirror, daemon=True) worker.start() children, streams = [], [] def start(command, filename): stream = (run/filename).open('ab') streams.append(stream) child = subprocess.Popen(command, cwd=ROOT, env=env, stdout=stream, stderr=subprocess.STDOUT, start_new_session=True) children.append(child) return child training = start(['bash', str(ROOT/'posttrain-full-duplex-audio-command.sh'), *sys.argv[1:]], 'training.log') print(f'[DLC] training PID={training.pid} local={run} backup={backup}', flush=True) interrupted = False def shutdown(signum, frame): nonlocal interrupted interrupted = True for child in children: if child.poll() is None: os.killpg(child.pid, signal.SIGTERM) signal.signal(signal.SIGTERM, shutdown) signal.signal(signal.SIGINT, shutdown) infer = dashboard = None try: while training.poll() is None: if (run/'run_config.json').is_file(): if infer is None or infer.poll() is not None: infer = start(['bash', str(ROOT/'infer_render_auto.sh'), str(run)], 'auto_infer.log') if dashboard is None or dashboard.poll() is not None: dashboard = start([sys.executable, str(ROOT/'scripts/training_dashboard.py'), str(run), '--host', '0.0.0.0', '--port', env.get('HFD_DASHBOARD_PORT', '8772')], 'dashboard.log') time.sleep(10) print('[DLC] training exit=', training.returncode, flush=True) # The watcher remains alive after training so the final checkpoint renders. if training.returncode == 0 and not interrupted and infer is not None: while not interrupted: time.sleep(10) if infer.poll() is not None: infer = start(['bash', str(ROOT/'infer_render_auto.sh'), str(run)], 'auto_infer.log') if dashboard is not None and dashboard.poll() is not None: dashboard = start([sys.executable, str(ROOT/'scripts/training_dashboard.py'), str(run), '--host', '0.0.0.0', '--port', env.get('HFD_DASHBOARD_PORT', '8772')], 'dashboard.log') finally: shutdown(signal.SIGTERM, None) for child in children: try: child.wait(timeout=30) except subprocess.TimeoutExpired: os.killpg(child.pid, signal.SIGKILL) child.wait() stop.set() worker.join(timeout=300) if not worker.is_alive(): try: synchronize(run, backup, world) except Exception as exc: print('[backup:pending] Local artifacts retained:', run, exc, flush=True) for stream in streams: stream.close() raise SystemExit(training.returncode or 0) if __name__ == '__main__': main()