RAQEEB-CCTV / code /train_raqeeb.py
swedishtech's picture
RAQEEB-CCTV v1.0.0
d0f2b17 verified
Raw History Blame Contribute Delete
11.1 kB
"""Launch and supervise RAQEEB training on both RTX 3090s (native Windows DDP over gloo).
python train_raqeeb.py --name raqeeb_v1 --epochs 40 --batch 20 # full run (global batch over 2 GPUs)
python train_raqeeb.py --name raqeeb_smoke --smoke # short check on a subset
python train_raqeeb.py --name raqeeb_v1 --resume # continue from runs/raqeeb_v1/weights/last.pt
python train_raqeeb.py --name raqeeb_v1 --resume --single-gpu # continue on GPU 1 only
Supervision: if a training process dies, the launcher checks which GPUs are still present. If GPU 1 (the headless
card) is healthy it resumes from the last saved epoch on GPU 1 alone (same per-GPU batch); if both GPUs are healthy
(software failure) it retries the two-GPU resume once. Every attempt is recorded in <run>/launcher_status.json.
Graceful stop: create the file <run>/STOP; training stops after the current epoch and keeps best/last.pt.
Recommended before long two-GPU runs on this PC (admin, resets at reboot): nvidia-smi -pl 250
"""
import argparse
import json
import os
import subprocess
import sys
import time
from datetime import datetime, timezone
from pathlib import Path
HERE = Path(__file__).resolve().parent
RAQ = Path(r'D:\CCTV_AI_FLOORPLAN_data\RAQEEB CCTV')
DATA = RAQ / 'dataset' / 'data.yaml'
PROJECT = RAQ / 'runs'
BASE_MODEL = RAQ / 'pretrained' / 'yolo11s-seg.pt'
# Physical GPUs in PCI order. GPU 0 drives the monitor: leave it headroom. GPU 1 is headless.
GPUS = {0: dict(pci='00000000:73:00.0', mem_fraction=0.72),
1: dict(pci='00000000:D5:00.0', mem_fraction=0.92, uuid='GPU-f4eadf9c-df66-df50-7149-fe323b79330e')}
MAX_ATTEMPTS = 4
def now():
return datetime.now(timezone.utc).isoformat()
def hyp(a):
h = dict(
task='segment', mode='train', data=str(a.data), epochs=a.epochs, imgsz=a.imgsz, batch=a.batch,
device='0,1', workers=a.workers, project=str(PROJECT), name=a.name, exist_ok=True, seed=20261111,
deterministic=False, amp=True, cache=False, optimizer='SGD', lr0=0.01, lrf=0.05, momentum=0.937,
weight_decay=0.0005, warmup_epochs=2.0, cos_lr=True, close_mosaic=8, patience=25, save_period=5,
mosaic=1.0, mixup=0.0, copy_paste=0.0, cutmix=0.0, degrees=0.0, translate=0.1, scale=0.5,
shear=0.0, perspective=0.0, fliplr=0.5, flipud=0.5, hsv_h=0.015, hsv_s=0.5, hsv_v=0.4,
overlap_mask=True, mask_ratio=a.mask_ratio, max_det=800, plots=True, val=True, verbose=True,
nbs=64)
if a.smoke:
h.update(epochs=2, fraction=a.fraction, save_period=-1, close_mosaic=1, patience=100)
return h
# ----------------------------------------------------------------------------- child (one training process)
def child(a):
single = os.environ.get('RAQEEB_SINGLE') == '1'
rank = 0 if single else int(os.environ['LOCAL_RANK'])
import torch
torch.cuda.set_per_process_memory_fraction(float(os.environ['RAQEEB_MEM_FRACTION']), rank)
sys.path.insert(0, str(HERE))
from raqeeb_trainer import RaqeebTrainer
device = '0' if single else '0,1'
last = PROJECT / a.name / 'weights' / 'last.pt'
if os.environ.get('RAQEEB_RESUME') == '1':
overrides = dict(resume=str(last), device=device, workers=a.workers)
if single:
overrides['batch'] = max(1, a.batch // 2) # keep the per-GPU batch of the two-GPU run
else:
overrides = hyp(a)
overrides.update(model=str(a.model), device=device)
if single:
overrides['batch'] = max(1, a.batch // 2)
RaqeebTrainer(overrides=overrides).train()
# ----------------------------------------------------------------------------- parent (supervisor)
def gpu_health():
"""Physical GPU index -> present and queryable (a lost GPU drops out of nvidia-smi)."""
try:
r = subprocess.run(['nvidia-smi', '--query-gpu=pci.bus_id,temperature.gpu', '--format=csv,noheader'],
capture_output=True, text=True, timeout=60)
text = r.stdout + r.stderr
except Exception as e:
return {i: False for i in GPUS}, str(e)
ok = {}
for i, g in GPUS.items():
ok[i] = False
for line in r.stdout.splitlines():
parts = [x.strip() for x in line.split(',')]
if len(parts) == 2 and parts[0].upper() == g['pci'].upper():
try:
float(parts[1])
ok[i] = True
except ValueError:
pass
return ok, text.strip()[-500:]
def base_env():
return dict(os.environ, PYTHONUNBUFFERED='1', KMP_DUPLICATE_LIB_OK='TRUE', OMP_NUM_THREADS='2',
CUDA_DEVICE_ORDER='PCI_BUS_ID', YOLO_CONFIG_DIR=str(RAQ / 'ultralytics_settings'),
YOLO_OFFLINE='true', WANDB_MODE='disabled', COMET_MODE='DISABLED',
MPLCONFIGDIR=str(RAQ / 'matplotlib'), USE_LIBUV='0',
PYTHONPATH=str(HERE) + os.pathsep + os.environ.get('PYTHONPATH', ''))
def run_attempt(a, mode, resume, logs, attempt):
"""mode 'ddp' (both GPUs) or 'gpu1' (GPU 1 only). Returns list of exit codes."""
script = [sys.executable, str(Path(__file__).resolve()), *[x for x in sys.argv[1:] if x != '--single-gpu'],
'--child']
procs = []
if mode == 'ddp':
port = str(29500 + (int(time.time()) % 400))
for r in range(2):
env = dict(base_env(), RANK=str(r), LOCAL_RANK=str(r), WORLD_SIZE='2', LOCAL_WORLD_SIZE='2',
MASTER_ADDR='127.0.0.1', MASTER_PORT=port, RAQEEB_MEM_FRACTION=str(GPUS[r]['mem_fraction']),
RAQEEB_RESUME='1' if resume else '0')
out = open(logs / f'{a.name}_rank{r}.log', 'a', encoding='utf-8')
out.write(f'\n===== attempt {attempt} mode={mode} resume={resume} {now()} =====\n')
out.flush()
procs.append((subprocess.Popen(script, env=env, stdout=out, stderr=subprocess.STDOUT), out))
else:
env = dict(base_env(), RAQEEB_SINGLE='1', CUDA_VISIBLE_DEVICES=GPUS[1]['uuid'],
RAQEEB_MEM_FRACTION=str(GPUS[1]['mem_fraction']), RAQEEB_RESUME='1' if resume else '0')
for k in ('RANK', 'LOCAL_RANK', 'WORLD_SIZE', 'LOCAL_WORLD_SIZE', 'MASTER_ADDR', 'MASTER_PORT'):
env.pop(k, None)
out = open(logs / f'{a.name}_gpu1.log', 'a', encoding='utf-8')
out.write(f'\n===== attempt {attempt} mode={mode} resume={resume} {now()} =====\n')
out.flush()
procs.append((subprocess.Popen(script, env=env, stdout=out, stderr=subprocess.STDOUT), out))
codes = [None] * len(procs)
while any(c is None for c in codes):
for i, (p, _) in enumerate(procs):
if codes[i] is None:
codes[i] = p.poll()
if any(c not in (None, 0) for c in codes): # one rank died: stop the other so nothing hangs
time.sleep(60)
for i, (p, _) in enumerate(procs):
if p.poll() is None:
p.terminate()
codes[i] = 'terminated'
time.sleep(5)
for _, f in procs:
f.close()
return codes, [p.pid for p, _ in procs]
def parent(a):
run = PROJECT / a.name
if run.exists() and not a.resume:
sys.exit(f'{run} already exists; use a new --name or --resume')
run.mkdir(parents=True, exist_ok=True)
logs = RAQ / 'logs'
logs.mkdir(exist_ok=True)
os.environ.update(YOLO_CONFIG_DIR=str(RAQ / 'ultralytics_settings'), YOLO_OFFLINE='true')
from ultralytics import settings # configure once, before the ranks start (avoids concurrent writes)
settings.update({k: False for k in ('sync', 'hub', 'clearml', 'comet', 'dvc', 'mlflow', 'neptune', 'raytune',
'tensorboard', 'wandb') if k in settings})
settings.update(runs_dir=str(PROJECT), datasets_dir=str(RAQ / 'dataset'))
status_path = run / 'launcher_status.json'
prev = json.loads(status_path.read_text()) if status_path.exists() else {}
status = dict(name=a.name, started_utc=prev.get('started_utc', now()), argv=sys.argv, python=sys.executable,
gpus=GPUS, hyp=hyp(a), status='running', attempts=prev.get('attempts', []))
mode = 'gpu1' if a.single_gpu else 'ddp'
resume, ddp_retries = a.resume, 0
last = run / 'weights' / 'last.pt'
while True:
health, smi = gpu_health()
rec = dict(attempt=len(status['attempts']) + 1, mode=mode, resume=resume, started_utc=now(),
gpu_health_before={str(k): v for k, v in health.items()}, power_limit_note='see nvidia-smi')
status['attempts'].append(rec)
status_path.write_text(json.dumps(status, indent=2))
codes, pids = run_attempt(a, mode, resume, logs, rec['attempt'])
health, smi = gpu_health()
rec.update(ended_utc=now(), exit_codes=codes, pids=pids,
gpu_health_after={str(k): v for k, v in health.items()}, nvidia_smi_tail=smi)
if all(c == 0 for c in codes):
status['status'] = 'completed'
break
n_attempts = len(status['attempts'])
if n_attempts >= MAX_ATTEMPTS:
status['status'] = 'failed'
status['reason'] = 'maximum attempts reached'
break
resume = last.exists()
if mode == 'ddp' and health.get(0) and health.get(1) and ddp_retries < 1:
ddp_retries += 1
rec['next'] = 'retry two-GPU run' + (' from last.pt' if resume else ' from scratch')
elif health.get(1):
mode = 'gpu1'
rec['next'] = 'GPU 1 only' + (' from last.pt' if resume else ' from scratch')
else:
status['status'] = 'failed'
status['reason'] = 'GPU 1 unavailable; restart Windows and run with --resume'
break
status_path.write_text(json.dumps(status, indent=2))
time.sleep(30)
status['ended_utc'] = now()
status_path.write_text(json.dumps(status, indent=2))
print(json.dumps(status, indent=2))
sys.exit(0 if status['status'] == 'completed' else 1)
if __name__ == '__main__':
ap = argparse.ArgumentParser()
ap.add_argument('--name', required=True)
ap.add_argument('--data', default=str(DATA))
ap.add_argument('--model', default=str(BASE_MODEL))
ap.add_argument('--epochs', type=int, default=40)
ap.add_argument('--imgsz', type=int, default=1024)
ap.add_argument('--batch', type=int, default=20, help='global batch of the two-GPU run (GPU 1 alone uses half)')
ap.add_argument('--workers', type=int, default=10, help='dataloader workers per GPU')
ap.add_argument('--mask-ratio', dest='mask_ratio', type=int, default=4)
ap.add_argument('--smoke', action='store_true')
ap.add_argument('--fraction', type=float, default=0.03)
ap.add_argument('--resume', action='store_true')
ap.add_argument('--single-gpu', dest='single_gpu', action='store_true', help='train on GPU 1 only')
ap.add_argument('--child', action='store_true')
a = ap.parse_args()
child(a) if a.child else parent(a)