"""Launch and supervise RAQEEB training on both RTX 3090s (native Windows DDP over gloo). python train_raqeeb.py --name raqeeb_v1 --epochs 40 --batch 20 # full run (global batch over 2 GPUs) python train_raqeeb.py --name raqeeb_smoke --smoke # short check on a subset python train_raqeeb.py --name raqeeb_v1 --resume # continue from runs/raqeeb_v1/weights/last.pt python train_raqeeb.py --name raqeeb_v1 --resume --single-gpu # continue on GPU 1 only Supervision: if a training process dies, the launcher checks which GPUs are still present. If GPU 1 (the headless card) is healthy it resumes from the last saved epoch on GPU 1 alone (same per-GPU batch); if both GPUs are healthy (software failure) it retries the two-GPU resume once. Every attempt is recorded in /launcher_status.json. Graceful stop: create the file /STOP; training stops after the current epoch and keeps best/last.pt. Recommended before long two-GPU runs on this PC (admin, resets at reboot): nvidia-smi -pl 250 """ import argparse import json import os import subprocess import sys import time from datetime import datetime, timezone from pathlib import Path HERE = Path(__file__).resolve().parent RAQ = Path(r'D:\CCTV_AI_FLOORPLAN_data\RAQEEB CCTV') DATA = RAQ / 'dataset' / 'data.yaml' PROJECT = RAQ / 'runs' BASE_MODEL = RAQ / 'pretrained' / 'yolo11s-seg.pt' # Physical GPUs in PCI order. GPU 0 drives the monitor: leave it headroom. GPU 1 is headless. GPUS = {0: dict(pci='00000000:73:00.0', mem_fraction=0.72), 1: dict(pci='00000000:D5:00.0', mem_fraction=0.92, uuid='GPU-f4eadf9c-df66-df50-7149-fe323b79330e')} MAX_ATTEMPTS = 4 def now(): return datetime.now(timezone.utc).isoformat() def hyp(a): h = dict( task='segment', mode='train', data=str(a.data), epochs=a.epochs, imgsz=a.imgsz, batch=a.batch, device='0,1', workers=a.workers, project=str(PROJECT), name=a.name, exist_ok=True, seed=20261111, deterministic=False, amp=True, cache=False, optimizer='SGD', lr0=0.01, lrf=0.05, momentum=0.937, weight_decay=0.0005, warmup_epochs=2.0, cos_lr=True, close_mosaic=8, patience=25, save_period=5, mosaic=1.0, mixup=0.0, copy_paste=0.0, cutmix=0.0, degrees=0.0, translate=0.1, scale=0.5, shear=0.0, perspective=0.0, fliplr=0.5, flipud=0.5, hsv_h=0.015, hsv_s=0.5, hsv_v=0.4, overlap_mask=True, mask_ratio=a.mask_ratio, max_det=800, plots=True, val=True, verbose=True, nbs=64) if a.smoke: h.update(epochs=2, fraction=a.fraction, save_period=-1, close_mosaic=1, patience=100) return h # ----------------------------------------------------------------------------- child (one training process) def child(a): single = os.environ.get('RAQEEB_SINGLE') == '1' rank = 0 if single else int(os.environ['LOCAL_RANK']) import torch torch.cuda.set_per_process_memory_fraction(float(os.environ['RAQEEB_MEM_FRACTION']), rank) sys.path.insert(0, str(HERE)) from raqeeb_trainer import RaqeebTrainer device = '0' if single else '0,1' last = PROJECT / a.name / 'weights' / 'last.pt' if os.environ.get('RAQEEB_RESUME') == '1': overrides = dict(resume=str(last), device=device, workers=a.workers) if single: overrides['batch'] = max(1, a.batch // 2) # keep the per-GPU batch of the two-GPU run else: overrides = hyp(a) overrides.update(model=str(a.model), device=device) if single: overrides['batch'] = max(1, a.batch // 2) RaqeebTrainer(overrides=overrides).train() # ----------------------------------------------------------------------------- parent (supervisor) def gpu_health(): """Physical GPU index -> present and queryable (a lost GPU drops out of nvidia-smi).""" try: r = subprocess.run(['nvidia-smi', '--query-gpu=pci.bus_id,temperature.gpu', '--format=csv,noheader'], capture_output=True, text=True, timeout=60) text = r.stdout + r.stderr except Exception as e: return {i: False for i in GPUS}, str(e) ok = {} for i, g in GPUS.items(): ok[i] = False for line in r.stdout.splitlines(): parts = [x.strip() for x in line.split(',')] if len(parts) == 2 and parts[0].upper() == g['pci'].upper(): try: float(parts[1]) ok[i] = True except ValueError: pass return ok, text.strip()[-500:] def base_env(): return dict(os.environ, PYTHONUNBUFFERED='1', KMP_DUPLICATE_LIB_OK='TRUE', OMP_NUM_THREADS='2', CUDA_DEVICE_ORDER='PCI_BUS_ID', YOLO_CONFIG_DIR=str(RAQ / 'ultralytics_settings'), YOLO_OFFLINE='true', WANDB_MODE='disabled', COMET_MODE='DISABLED', MPLCONFIGDIR=str(RAQ / 'matplotlib'), USE_LIBUV='0', PYTHONPATH=str(HERE) + os.pathsep + os.environ.get('PYTHONPATH', '')) def run_attempt(a, mode, resume, logs, attempt): """mode 'ddp' (both GPUs) or 'gpu1' (GPU 1 only). Returns list of exit codes.""" script = [sys.executable, str(Path(__file__).resolve()), *[x for x in sys.argv[1:] if x != '--single-gpu'], '--child'] procs = [] if mode == 'ddp': port = str(29500 + (int(time.time()) % 400)) for r in range(2): env = dict(base_env(), RANK=str(r), LOCAL_RANK=str(r), WORLD_SIZE='2', LOCAL_WORLD_SIZE='2', MASTER_ADDR='127.0.0.1', MASTER_PORT=port, RAQEEB_MEM_FRACTION=str(GPUS[r]['mem_fraction']), RAQEEB_RESUME='1' if resume else '0') out = open(logs / f'{a.name}_rank{r}.log', 'a', encoding='utf-8') out.write(f'\n===== attempt {attempt} mode={mode} resume={resume} {now()} =====\n') out.flush() procs.append((subprocess.Popen(script, env=env, stdout=out, stderr=subprocess.STDOUT), out)) else: env = dict(base_env(), RAQEEB_SINGLE='1', CUDA_VISIBLE_DEVICES=GPUS[1]['uuid'], RAQEEB_MEM_FRACTION=str(GPUS[1]['mem_fraction']), RAQEEB_RESUME='1' if resume else '0') for k in ('RANK', 'LOCAL_RANK', 'WORLD_SIZE', 'LOCAL_WORLD_SIZE', 'MASTER_ADDR', 'MASTER_PORT'): env.pop(k, None) out = open(logs / f'{a.name}_gpu1.log', 'a', encoding='utf-8') out.write(f'\n===== attempt {attempt} mode={mode} resume={resume} {now()} =====\n') out.flush() procs.append((subprocess.Popen(script, env=env, stdout=out, stderr=subprocess.STDOUT), out)) codes = [None] * len(procs) while any(c is None for c in codes): for i, (p, _) in enumerate(procs): if codes[i] is None: codes[i] = p.poll() if any(c not in (None, 0) for c in codes): # one rank died: stop the other so nothing hangs time.sleep(60) for i, (p, _) in enumerate(procs): if p.poll() is None: p.terminate() codes[i] = 'terminated' time.sleep(5) for _, f in procs: f.close() return codes, [p.pid for p, _ in procs] def parent(a): run = PROJECT / a.name if run.exists() and not a.resume: sys.exit(f'{run} already exists; use a new --name or --resume') run.mkdir(parents=True, exist_ok=True) logs = RAQ / 'logs' logs.mkdir(exist_ok=True) os.environ.update(YOLO_CONFIG_DIR=str(RAQ / 'ultralytics_settings'), YOLO_OFFLINE='true') from ultralytics import settings # configure once, before the ranks start (avoids concurrent writes) settings.update({k: False for k in ('sync', 'hub', 'clearml', 'comet', 'dvc', 'mlflow', 'neptune', 'raytune', 'tensorboard', 'wandb') if k in settings}) settings.update(runs_dir=str(PROJECT), datasets_dir=str(RAQ / 'dataset')) status_path = run / 'launcher_status.json' prev = json.loads(status_path.read_text()) if status_path.exists() else {} status = dict(name=a.name, started_utc=prev.get('started_utc', now()), argv=sys.argv, python=sys.executable, gpus=GPUS, hyp=hyp(a), status='running', attempts=prev.get('attempts', [])) mode = 'gpu1' if a.single_gpu else 'ddp' resume, ddp_retries = a.resume, 0 last = run / 'weights' / 'last.pt' while True: health, smi = gpu_health() rec = dict(attempt=len(status['attempts']) + 1, mode=mode, resume=resume, started_utc=now(), gpu_health_before={str(k): v for k, v in health.items()}, power_limit_note='see nvidia-smi') status['attempts'].append(rec) status_path.write_text(json.dumps(status, indent=2)) codes, pids = run_attempt(a, mode, resume, logs, rec['attempt']) health, smi = gpu_health() rec.update(ended_utc=now(), exit_codes=codes, pids=pids, gpu_health_after={str(k): v for k, v in health.items()}, nvidia_smi_tail=smi) if all(c == 0 for c in codes): status['status'] = 'completed' break n_attempts = len(status['attempts']) if n_attempts >= MAX_ATTEMPTS: status['status'] = 'failed' status['reason'] = 'maximum attempts reached' break resume = last.exists() if mode == 'ddp' and health.get(0) and health.get(1) and ddp_retries < 1: ddp_retries += 1 rec['next'] = 'retry two-GPU run' + (' from last.pt' if resume else ' from scratch') elif health.get(1): mode = 'gpu1' rec['next'] = 'GPU 1 only' + (' from last.pt' if resume else ' from scratch') else: status['status'] = 'failed' status['reason'] = 'GPU 1 unavailable; restart Windows and run with --resume' break status_path.write_text(json.dumps(status, indent=2)) time.sleep(30) status['ended_utc'] = now() status_path.write_text(json.dumps(status, indent=2)) print(json.dumps(status, indent=2)) sys.exit(0 if status['status'] == 'completed' else 1) if __name__ == '__main__': ap = argparse.ArgumentParser() ap.add_argument('--name', required=True) ap.add_argument('--data', default=str(DATA)) ap.add_argument('--model', default=str(BASE_MODEL)) ap.add_argument('--epochs', type=int, default=40) ap.add_argument('--imgsz', type=int, default=1024) ap.add_argument('--batch', type=int, default=20, help='global batch of the two-GPU run (GPU 1 alone uses half)') ap.add_argument('--workers', type=int, default=10, help='dataloader workers per GPU') ap.add_argument('--mask-ratio', dest='mask_ratio', type=int, default=4) ap.add_argument('--smoke', action='store_true') ap.add_argument('--fraction', type=float, default=0.03) ap.add_argument('--resume', action='store_true') ap.add_argument('--single-gpu', dest='single_gpu', action='store_true', help='train on GPU 1 only') ap.add_argument('--child', action='store_true') a = ap.parse_args() child(a) if a.child else parent(a)