Download scripts/launch_hfd_dlc.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 7.38 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/launch_hfd_dlc.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/scripts/launch_hfd_dlc.py
-
curl -L -o launch_hfd_dlc.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/launch_hfd_dlc.py
7.38 kB
| #!/usr/bin/env python3 | |
| """One DLC job: local training, memory-admitted inference, CPFS mirror, dashboard.""" | |
| import json | |
| import fcntl | |
| import os | |
| from pathlib import Path | |
| import signal | |
| import subprocess | |
| import sys | |
| import threading | |
| import time | |
| ROOT = Path(__file__).resolve().parents[1] | |
| sys.path.insert(0, str(ROOT/'src')) | |
| from psi.dlc_storage import atomic_json, synchronize, restore | |
| def main(): | |
| env = dict(os.environ) | |
| env.setdefault('HFD_RUN_TIMESTAMP', time.strftime('%y%m%d%H%M%S')) | |
| env.setdefault('HFD_NPROC_PER_NODE', '8') | |
| if "--print-config" in sys.argv[1:] or env.get("HFD_CONFIG_ONLY", "0") in {"1", "json"}: | |
| # The same real shell/recipe, without creating a DLC run, mirror, | |
| # dashboard or inference worker. Useful for pre-flight verification. | |
| if not env.get('HFD_DLC_RESUME_RUN'): | |
| env.setdefault('HFD_TRAIN_BATCH_SIZE', '8') | |
| env.setdefault('HFD_EFFECTIVE_SOURCES_PER_GPU', env['HFD_TRAIN_BATCH_SIZE']) | |
| elif not env.get('HFD_RESUME_FROM_CHECKPOINT') or env.get('HFD_RESUME_LOAD_FULL_STATE') != '1': | |
| raise ValueError("Resume config inspection requires HFD_RESUME_FROM_CHECKPOINT and HFD_RESUME_LOAD_FULL_STATE") | |
| raise SystemExit(subprocess.call( | |
| ['bash', str(ROOT/'posttrain-full-duplex-audio-command.sh'), *sys.argv[1:]], env=env)) | |
| world = int(env['HFD_NPROC_PER_NODE']) | |
| local = Path(env.get('HFD_DLC_LOCAL_ROOT', '/mnt/workspace/hfd_dlc')).resolve() | |
| remote = Path(env.get('HFD_DLC_BACKUP_ROOT', str(ROOT/'runs'))).resolve() | |
| local.mkdir(parents=True, exist_ok=True) | |
| remote.mkdir(parents=True, exist_ok=True) | |
| if local == remote or local.stat().st_dev == remote.stat().st_dev: | |
| raise ValueError('DLC local root must be on a different filesystem from CPFS backup') | |
| env['HFD_OUTPUT_DIR'] = str(local/'runs') | |
| if not env.get('HFD_DLC_RESUME_RUN'): | |
| env.setdefault('HFD_TRAIN_BATCH_SIZE', '8') | |
| env.setdefault('HFD_EFFECTIVE_SOURCES_PER_GPU', env['HFD_TRAIN_BATCH_SIZE']) | |
| env.setdefault('HFD_NUM_WORKERS', '4') | |
| env.setdefault('HFD_INFER_MIN_FREE_GIB', '48') | |
| env.setdefault('HFD_INFER_MEMORY_BUDGET_GIB', '36') | |
| env.setdefault('HFD_INFER_MAX_CONCURRENT', '1') | |
| env.setdefault('HFD_INFER_CHECKPOINT_INTERVAL', '20000') | |
| env.setdefault('HFD_INFER_GPUS', env.get('CUDA_VISIBLE_DEVICES', ','.join(map(str, range(world))))) | |
| env.setdefault('HFD_RENDER_GPU', env['HFD_INFER_GPUS'].split(',')[0]) | |
| env['HFD_MANAGE_GPU_MONITOR'] = '0' | |
| resume_run = Path(env['HFD_DLC_RESUME_RUN']).resolve() if env.get('HFD_DLC_RESUME_RUN') else None | |
| if resume_run: | |
| saved = json.loads((resume_run/'run_config.json').read_text()) | |
| env['HFD_RUN_TIMESTAMP'] = saved['timestamp'] | |
| env['HFD_WANDB_ID'] = saved['wandb']['id'] or '' | |
| env['HFD_WANDB_RESUME'] = 'must' if env['HFD_WANDB_ID'] else 'never' | |
| run_name = 'hfd.gpus'+str(world)+'.'+env['HFD_RUN_TIMESTAMP'] | |
| run = local/'runs/posttrain_full_duplex'/run_name | |
| backup = remote/'posttrain_full_duplex'/run_name | |
| # Reservation prevents accidental overwrite or concurrent launches. | |
| run.mkdir(parents=True, exist_ok=bool(resume_run)) | |
| lock = (run/'.dlc.lock').open('a') | |
| fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) | |
| if resume_run: | |
| if resume_run.name != run_name: | |
| raise ValueError('Resume must preserve rank count and original run name') | |
| checkpoint = restore(resume_run, run, world) | |
| env['HFD_RESUME_FROM_CHECKPOINT'] = str(checkpoint) | |
| env['HFD_RESUME_LOAD_FULL_STATE'] = '1' | |
| atomic_json(run/'dlc_launch.json', dict(world_size=world, local=str(run), backup=str(backup), | |
| batch_per_gpu=int(env.get('HFD_TRAIN_BATCH_SIZE', saved['train']['train_batch_size'] if resume_run else 8)), started=time.time())) | |
| stop = threading.Event() | |
| def mirror(): | |
| while not stop.is_set(): | |
| try: | |
| step = synchronize(run, backup, world) | |
| atomic_json(run/'backup_status.json', dict(ok=True, step=step, updated=time.time())) | |
| except Exception as exc: | |
| atomic_json(run/'backup_status.json', dict(ok=False, error=str(exc), updated=time.time())) | |
| print('[backup:retry]', exc, flush=True) | |
| stop.wait(float(env.get('HFD_DLC_SYNC_SECONDS', '120'))) | |
| worker = threading.Thread(target=mirror, daemon=True) | |
| worker.start() | |
| children, streams = [], [] | |
| def start(command, filename): | |
| stream = (run/filename).open('ab') | |
| streams.append(stream) | |
| child = subprocess.Popen(command, cwd=ROOT, env=env, stdout=stream, stderr=subprocess.STDOUT, | |
| start_new_session=True) | |
| children.append(child) | |
| return child | |
| training = start(['bash', str(ROOT/'posttrain-full-duplex-audio-command.sh'), *sys.argv[1:]], 'training.log') | |
| print(f'[DLC] training PID={training.pid} local={run} backup={backup}', flush=True) | |
| interrupted = False | |
| def shutdown(signum, frame): | |
| nonlocal interrupted | |
| interrupted = True | |
| for child in children: | |
| if child.poll() is None: | |
| os.killpg(child.pid, signal.SIGTERM) | |
| signal.signal(signal.SIGTERM, shutdown) | |
| signal.signal(signal.SIGINT, shutdown) | |
| infer = dashboard = None | |
| try: | |
| while training.poll() is None: | |
| if (run/'run_config.json').is_file(): | |
| if infer is None or infer.poll() is not None: | |
| infer = start(['bash', str(ROOT/'infer_render_auto.sh'), str(run)], 'auto_infer.log') | |
| if dashboard is None or dashboard.poll() is not None: | |
| dashboard = start([sys.executable, str(ROOT/'scripts/training_dashboard.py'), str(run), | |
| '--host', '0.0.0.0', '--port', env.get('HFD_DASHBOARD_PORT', '8772')], 'dashboard.log') | |
| time.sleep(10) | |
| print('[DLC] training exit=', training.returncode, flush=True) | |
| # The watcher remains alive after training so the final checkpoint renders. | |
| if training.returncode == 0 and not interrupted and infer is not None: | |
| while not interrupted: | |
| time.sleep(10) | |
| if infer.poll() is not None: | |
| infer = start(['bash', str(ROOT/'infer_render_auto.sh'), str(run)], 'auto_infer.log') | |
| if dashboard is not None and dashboard.poll() is not None: | |
| dashboard = start([sys.executable, str(ROOT/'scripts/training_dashboard.py'), str(run), | |
| '--host', '0.0.0.0', '--port', env.get('HFD_DASHBOARD_PORT', '8772')], 'dashboard.log') | |
| finally: | |
| shutdown(signal.SIGTERM, None) | |
| for child in children: | |
| try: | |
| child.wait(timeout=30) | |
| except subprocess.TimeoutExpired: | |
| os.killpg(child.pid, signal.SIGKILL) | |
| child.wait() | |
| stop.set() | |
| worker.join(timeout=300) | |
| if not worker.is_alive(): | |
| try: | |
| synchronize(run, backup, world) | |
| except Exception as exc: | |
| print('[backup:pending] Local artifacts retained:', run, exc, flush=True) | |
| for stream in streams: | |
| stream.close() | |
| raise SystemExit(training.returncode or 0) | |
| if __name__ == '__main__': | |
| main() | |