File size: 11,086 Bytes
d0f2b17
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
"""Launch and supervise RAQEEB training on both RTX 3090s (native Windows DDP over gloo).

  python train_raqeeb.py --name raqeeb_v1 --epochs 40 --batch 20     # full run (global batch over 2 GPUs)
  python train_raqeeb.py --name raqeeb_smoke --smoke                 # short check on a subset
  python train_raqeeb.py --name raqeeb_v1 --resume                   # continue from runs/raqeeb_v1/weights/last.pt
  python train_raqeeb.py --name raqeeb_v1 --resume --single-gpu      # continue on GPU 1 only

Supervision: if a training process dies, the launcher checks which GPUs are still present. If GPU 1 (the headless
card) is healthy it resumes from the last saved epoch on GPU 1 alone (same per-GPU batch); if both GPUs are healthy
(software failure) it retries the two-GPU resume once. Every attempt is recorded in <run>/launcher_status.json.
Graceful stop: create the file <run>/STOP; training stops after the current epoch and keeps best/last.pt.
Recommended before long two-GPU runs on this PC (admin, resets at reboot): nvidia-smi -pl 250
"""
import argparse
import json
import os
import subprocess
import sys
import time
from datetime import datetime, timezone
from pathlib import Path

HERE = Path(__file__).resolve().parent
RAQ = Path(r'D:\CCTV_AI_FLOORPLAN_data\RAQEEB CCTV')
DATA = RAQ / 'dataset' / 'data.yaml'
PROJECT = RAQ / 'runs'
BASE_MODEL = RAQ / 'pretrained' / 'yolo11s-seg.pt'
# Physical GPUs in PCI order. GPU 0 drives the monitor: leave it headroom. GPU 1 is headless.
GPUS = {0: dict(pci='00000000:73:00.0', mem_fraction=0.72),
        1: dict(pci='00000000:D5:00.0', mem_fraction=0.92, uuid='GPU-f4eadf9c-df66-df50-7149-fe323b79330e')}
MAX_ATTEMPTS = 4


def now():
    return datetime.now(timezone.utc).isoformat()


def hyp(a):
    h = dict(
        task='segment', mode='train', data=str(a.data), epochs=a.epochs, imgsz=a.imgsz, batch=a.batch,
        device='0,1', workers=a.workers, project=str(PROJECT), name=a.name, exist_ok=True, seed=20261111,
        deterministic=False, amp=True, cache=False, optimizer='SGD', lr0=0.01, lrf=0.05, momentum=0.937,
        weight_decay=0.0005, warmup_epochs=2.0, cos_lr=True, close_mosaic=8, patience=25, save_period=5,
        mosaic=1.0, mixup=0.0, copy_paste=0.0, cutmix=0.0, degrees=0.0, translate=0.1, scale=0.5,
        shear=0.0, perspective=0.0, fliplr=0.5, flipud=0.5, hsv_h=0.015, hsv_s=0.5, hsv_v=0.4,
        overlap_mask=True, mask_ratio=a.mask_ratio, max_det=800, plots=True, val=True, verbose=True,
        nbs=64)
    if a.smoke:
        h.update(epochs=2, fraction=a.fraction, save_period=-1, close_mosaic=1, patience=100)
    return h


# ----------------------------------------------------------------------------- child (one training process)
def child(a):
    single = os.environ.get('RAQEEB_SINGLE') == '1'
    rank = 0 if single else int(os.environ['LOCAL_RANK'])
    import torch
    torch.cuda.set_per_process_memory_fraction(float(os.environ['RAQEEB_MEM_FRACTION']), rank)
    sys.path.insert(0, str(HERE))
    from raqeeb_trainer import RaqeebTrainer
    device = '0' if single else '0,1'
    last = PROJECT / a.name / 'weights' / 'last.pt'
    if os.environ.get('RAQEEB_RESUME') == '1':
        overrides = dict(resume=str(last), device=device, workers=a.workers)
        if single:
            overrides['batch'] = max(1, a.batch // 2)  # keep the per-GPU batch of the two-GPU run
    else:
        overrides = hyp(a)
        overrides.update(model=str(a.model), device=device)
        if single:
            overrides['batch'] = max(1, a.batch // 2)
    RaqeebTrainer(overrides=overrides).train()


# ----------------------------------------------------------------------------- parent (supervisor)
def gpu_health():
    """Physical GPU index -> present and queryable (a lost GPU drops out of nvidia-smi)."""
    try:
        r = subprocess.run(['nvidia-smi', '--query-gpu=pci.bus_id,temperature.gpu', '--format=csv,noheader'],
                           capture_output=True, text=True, timeout=60)
        text = r.stdout + r.stderr
    except Exception as e:
        return {i: False for i in GPUS}, str(e)
    ok = {}
    for i, g in GPUS.items():
        ok[i] = False
        for line in r.stdout.splitlines():
            parts = [x.strip() for x in line.split(',')]
            if len(parts) == 2 and parts[0].upper() == g['pci'].upper():
                try:
                    float(parts[1])
                    ok[i] = True
                except ValueError:
                    pass
    return ok, text.strip()[-500:]


def base_env():
    return dict(os.environ, PYTHONUNBUFFERED='1', KMP_DUPLICATE_LIB_OK='TRUE', OMP_NUM_THREADS='2',
                CUDA_DEVICE_ORDER='PCI_BUS_ID', YOLO_CONFIG_DIR=str(RAQ / 'ultralytics_settings'),
                YOLO_OFFLINE='true', WANDB_MODE='disabled', COMET_MODE='DISABLED',
                MPLCONFIGDIR=str(RAQ / 'matplotlib'), USE_LIBUV='0',
                PYTHONPATH=str(HERE) + os.pathsep + os.environ.get('PYTHONPATH', ''))


def run_attempt(a, mode, resume, logs, attempt):
    """mode 'ddp' (both GPUs) or 'gpu1' (GPU 1 only). Returns list of exit codes."""
    script = [sys.executable, str(Path(__file__).resolve()), *[x for x in sys.argv[1:] if x != '--single-gpu'],
              '--child']
    procs = []
    if mode == 'ddp':
        port = str(29500 + (int(time.time()) % 400))
        for r in range(2):
            env = dict(base_env(), RANK=str(r), LOCAL_RANK=str(r), WORLD_SIZE='2', LOCAL_WORLD_SIZE='2',
                       MASTER_ADDR='127.0.0.1', MASTER_PORT=port, RAQEEB_MEM_FRACTION=str(GPUS[r]['mem_fraction']),
                       RAQEEB_RESUME='1' if resume else '0')
            out = open(logs / f'{a.name}_rank{r}.log', 'a', encoding='utf-8')
            out.write(f'\n===== attempt {attempt} mode={mode} resume={resume} {now()} =====\n')
            out.flush()
            procs.append((subprocess.Popen(script, env=env, stdout=out, stderr=subprocess.STDOUT), out))
    else:
        env = dict(base_env(), RAQEEB_SINGLE='1', CUDA_VISIBLE_DEVICES=GPUS[1]['uuid'],
                   RAQEEB_MEM_FRACTION=str(GPUS[1]['mem_fraction']), RAQEEB_RESUME='1' if resume else '0')
        for k in ('RANK', 'LOCAL_RANK', 'WORLD_SIZE', 'LOCAL_WORLD_SIZE', 'MASTER_ADDR', 'MASTER_PORT'):
            env.pop(k, None)
        out = open(logs / f'{a.name}_gpu1.log', 'a', encoding='utf-8')
        out.write(f'\n===== attempt {attempt} mode={mode} resume={resume} {now()} =====\n')
        out.flush()
        procs.append((subprocess.Popen(script, env=env, stdout=out, stderr=subprocess.STDOUT), out))
    codes = [None] * len(procs)
    while any(c is None for c in codes):
        for i, (p, _) in enumerate(procs):
            if codes[i] is None:
                codes[i] = p.poll()
        if any(c not in (None, 0) for c in codes):  # one rank died: stop the other so nothing hangs
            time.sleep(60)
            for i, (p, _) in enumerate(procs):
                if p.poll() is None:
                    p.terminate()
                    codes[i] = 'terminated'
        time.sleep(5)
    for _, f in procs:
        f.close()
    return codes, [p.pid for p, _ in procs]


def parent(a):
    run = PROJECT / a.name
    if run.exists() and not a.resume:
        sys.exit(f'{run} already exists; use a new --name or --resume')
    run.mkdir(parents=True, exist_ok=True)
    logs = RAQ / 'logs'
    logs.mkdir(exist_ok=True)
    os.environ.update(YOLO_CONFIG_DIR=str(RAQ / 'ultralytics_settings'), YOLO_OFFLINE='true')
    from ultralytics import settings  # configure once, before the ranks start (avoids concurrent writes)
    settings.update({k: False for k in ('sync', 'hub', 'clearml', 'comet', 'dvc', 'mlflow', 'neptune', 'raytune',
                                        'tensorboard', 'wandb') if k in settings})
    settings.update(runs_dir=str(PROJECT), datasets_dir=str(RAQ / 'dataset'))
    status_path = run / 'launcher_status.json'
    prev = json.loads(status_path.read_text()) if status_path.exists() else {}
    status = dict(name=a.name, started_utc=prev.get('started_utc', now()), argv=sys.argv, python=sys.executable,
                  gpus=GPUS, hyp=hyp(a), status='running', attempts=prev.get('attempts', []))
    mode = 'gpu1' if a.single_gpu else 'ddp'
    resume, ddp_retries = a.resume, 0
    last = run / 'weights' / 'last.pt'
    while True:
        health, smi = gpu_health()
        rec = dict(attempt=len(status['attempts']) + 1, mode=mode, resume=resume, started_utc=now(),
                   gpu_health_before={str(k): v for k, v in health.items()}, power_limit_note='see nvidia-smi')
        status['attempts'].append(rec)
        status_path.write_text(json.dumps(status, indent=2))
        codes, pids = run_attempt(a, mode, resume, logs, rec['attempt'])
        health, smi = gpu_health()
        rec.update(ended_utc=now(), exit_codes=codes, pids=pids,
                   gpu_health_after={str(k): v for k, v in health.items()}, nvidia_smi_tail=smi)
        if all(c == 0 for c in codes):
            status['status'] = 'completed'
            break
        n_attempts = len(status['attempts'])
        if n_attempts >= MAX_ATTEMPTS:
            status['status'] = 'failed'
            status['reason'] = 'maximum attempts reached'
            break
        resume = last.exists()
        if mode == 'ddp' and health.get(0) and health.get(1) and ddp_retries < 1:
            ddp_retries += 1
            rec['next'] = 'retry two-GPU run' + (' from last.pt' if resume else ' from scratch')
        elif health.get(1):
            mode = 'gpu1'
            rec['next'] = 'GPU 1 only' + (' from last.pt' if resume else ' from scratch')
        else:
            status['status'] = 'failed'
            status['reason'] = 'GPU 1 unavailable; restart Windows and run with --resume'
            break
        status_path.write_text(json.dumps(status, indent=2))
        time.sleep(30)
    status['ended_utc'] = now()
    status_path.write_text(json.dumps(status, indent=2))
    print(json.dumps(status, indent=2))
    sys.exit(0 if status['status'] == 'completed' else 1)


if __name__ == '__main__':
    ap = argparse.ArgumentParser()
    ap.add_argument('--name', required=True)
    ap.add_argument('--data', default=str(DATA))
    ap.add_argument('--model', default=str(BASE_MODEL))
    ap.add_argument('--epochs', type=int, default=40)
    ap.add_argument('--imgsz', type=int, default=1024)
    ap.add_argument('--batch', type=int, default=20, help='global batch of the two-GPU run (GPU 1 alone uses half)')
    ap.add_argument('--workers', type=int, default=10, help='dataloader workers per GPU')
    ap.add_argument('--mask-ratio', dest='mask_ratio', type=int, default=4)
    ap.add_argument('--smoke', action='store_true')
    ap.add_argument('--fraction', type=float, default=0.03)
    ap.add_argument('--resume', action='store_true')
    ap.add_argument('--single-gpu', dest='single_gpu', action='store_true', help='train on GPU 1 only')
    ap.add_argument('--child', action='store_true')
    a = ap.parse_args()
    child(a) if a.child else parent(a)