File size: 7,384 Bytes
9fc9873
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
#!/usr/bin/env python3
"""One DLC job: local training, memory-admitted inference, CPFS mirror, dashboard."""
import json
import fcntl
import os
from pathlib import Path
import signal
import subprocess
import sys
import threading
import time

ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT/'src'))
from psi.dlc_storage import atomic_json, synchronize, restore


def main():
    env = dict(os.environ)
    env.setdefault('HFD_RUN_TIMESTAMP', time.strftime('%y%m%d%H%M%S'))
    env.setdefault('HFD_NPROC_PER_NODE', '8')
    if "--print-config" in sys.argv[1:] or env.get("HFD_CONFIG_ONLY", "0") in {"1", "json"}:
        # The same real shell/recipe, without creating a DLC run, mirror,
        # dashboard or inference worker. Useful for pre-flight verification.
        if not env.get('HFD_DLC_RESUME_RUN'):
            env.setdefault('HFD_TRAIN_BATCH_SIZE', '8')
            env.setdefault('HFD_EFFECTIVE_SOURCES_PER_GPU', env['HFD_TRAIN_BATCH_SIZE'])
        elif not env.get('HFD_RESUME_FROM_CHECKPOINT') or env.get('HFD_RESUME_LOAD_FULL_STATE') != '1':
            raise ValueError("Resume config inspection requires HFD_RESUME_FROM_CHECKPOINT and HFD_RESUME_LOAD_FULL_STATE")
        raise SystemExit(subprocess.call(
            ['bash', str(ROOT/'posttrain-full-duplex-audio-command.sh'), *sys.argv[1:]], env=env))
    world = int(env['HFD_NPROC_PER_NODE'])
    local = Path(env.get('HFD_DLC_LOCAL_ROOT', '/mnt/workspace/hfd_dlc')).resolve()
    remote = Path(env.get('HFD_DLC_BACKUP_ROOT', str(ROOT/'runs'))).resolve()
    local.mkdir(parents=True, exist_ok=True)
    remote.mkdir(parents=True, exist_ok=True)
    if local == remote or local.stat().st_dev == remote.stat().st_dev:
        raise ValueError('DLC local root must be on a different filesystem from CPFS backup')
    env['HFD_OUTPUT_DIR'] = str(local/'runs')
    if not env.get('HFD_DLC_RESUME_RUN'):
        env.setdefault('HFD_TRAIN_BATCH_SIZE', '8')
        env.setdefault('HFD_EFFECTIVE_SOURCES_PER_GPU', env['HFD_TRAIN_BATCH_SIZE'])
    env.setdefault('HFD_NUM_WORKERS', '4')
    env.setdefault('HFD_INFER_MIN_FREE_GIB', '48')
    env.setdefault('HFD_INFER_MEMORY_BUDGET_GIB', '36')
    env.setdefault('HFD_INFER_MAX_CONCURRENT', '1')
    env.setdefault('HFD_INFER_CHECKPOINT_INTERVAL', '20000')
    env.setdefault('HFD_INFER_GPUS', env.get('CUDA_VISIBLE_DEVICES', ','.join(map(str, range(world)))))
    env.setdefault('HFD_RENDER_GPU', env['HFD_INFER_GPUS'].split(',')[0])
    env['HFD_MANAGE_GPU_MONITOR'] = '0'
    resume_run = Path(env['HFD_DLC_RESUME_RUN']).resolve() if env.get('HFD_DLC_RESUME_RUN') else None
    if resume_run:
        saved = json.loads((resume_run/'run_config.json').read_text())
        env['HFD_RUN_TIMESTAMP'] = saved['timestamp']
        env['HFD_WANDB_ID'] = saved['wandb']['id'] or ''
        env['HFD_WANDB_RESUME'] = 'must' if env['HFD_WANDB_ID'] else 'never'
    run_name = 'hfd.gpus'+str(world)+'.'+env['HFD_RUN_TIMESTAMP']
    run = local/'runs/posttrain_full_duplex'/run_name
    backup = remote/'posttrain_full_duplex'/run_name
    # Reservation prevents accidental overwrite or concurrent launches.
    run.mkdir(parents=True, exist_ok=bool(resume_run))
    lock = (run/'.dlc.lock').open('a')
    fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
    if resume_run:
        if resume_run.name != run_name:
            raise ValueError('Resume must preserve rank count and original run name')
        checkpoint = restore(resume_run, run, world)
        env['HFD_RESUME_FROM_CHECKPOINT'] = str(checkpoint)
        env['HFD_RESUME_LOAD_FULL_STATE'] = '1'
    atomic_json(run/'dlc_launch.json', dict(world_size=world, local=str(run), backup=str(backup),
                batch_per_gpu=int(env.get('HFD_TRAIN_BATCH_SIZE', saved['train']['train_batch_size'] if resume_run else 8)), started=time.time()))
    stop = threading.Event()
    def mirror():
        while not stop.is_set():
            try:
                step = synchronize(run, backup, world)
                atomic_json(run/'backup_status.json', dict(ok=True, step=step, updated=time.time()))
            except Exception as exc:
                atomic_json(run/'backup_status.json', dict(ok=False, error=str(exc), updated=time.time()))
                print('[backup:retry]', exc, flush=True)
            stop.wait(float(env.get('HFD_DLC_SYNC_SECONDS', '120')))
    worker = threading.Thread(target=mirror, daemon=True)
    worker.start()
    children, streams = [], []
    def start(command, filename):
        stream = (run/filename).open('ab')
        streams.append(stream)
        child = subprocess.Popen(command, cwd=ROOT, env=env, stdout=stream, stderr=subprocess.STDOUT,
                                 start_new_session=True)
        children.append(child)
        return child
    training = start(['bash', str(ROOT/'posttrain-full-duplex-audio-command.sh'), *sys.argv[1:]], 'training.log')
    print(f'[DLC] training PID={training.pid} local={run} backup={backup}', flush=True)
    interrupted = False
    def shutdown(signum, frame):
        nonlocal interrupted
        interrupted = True
        for child in children:
            if child.poll() is None:
                os.killpg(child.pid, signal.SIGTERM)
    signal.signal(signal.SIGTERM, shutdown)
    signal.signal(signal.SIGINT, shutdown)
    infer = dashboard = None
    try:
        while training.poll() is None:
            if (run/'run_config.json').is_file():
                if infer is None or infer.poll() is not None:
                    infer = start(['bash', str(ROOT/'infer_render_auto.sh'), str(run)], 'auto_infer.log')
                if dashboard is None or dashboard.poll() is not None:
                    dashboard = start([sys.executable, str(ROOT/'scripts/training_dashboard.py'), str(run),
                                      '--host', '0.0.0.0', '--port', env.get('HFD_DASHBOARD_PORT', '8772')], 'dashboard.log')
            time.sleep(10)
        print('[DLC] training exit=', training.returncode, flush=True)
        # The watcher remains alive after training so the final checkpoint renders.
        if training.returncode == 0 and not interrupted and infer is not None:
            while not interrupted:
                time.sleep(10)
                if infer.poll() is not None:
                    infer = start(['bash', str(ROOT/'infer_render_auto.sh'), str(run)], 'auto_infer.log')
                if dashboard is not None and dashboard.poll() is not None:
                    dashboard = start([sys.executable, str(ROOT/'scripts/training_dashboard.py'), str(run),
                                      '--host', '0.0.0.0', '--port', env.get('HFD_DASHBOARD_PORT', '8772')], 'dashboard.log')
    finally:
        shutdown(signal.SIGTERM, None)
        for child in children:
            try:
                child.wait(timeout=30)
            except subprocess.TimeoutExpired:
                os.killpg(child.pid, signal.SIGKILL)
                child.wait()
        stop.set()
        worker.join(timeout=300)
        if not worker.is_alive():
            try:
                synchronize(run, backup, world)
            except Exception as exc:
                print('[backup:pending] Local artifacts retained:', run, exc, flush=True)
        for stream in streams:
            stream.close()
    raise SystemExit(training.returncode or 0)


if __name__ == '__main__':
    main()