Virtual-LPU / lpu /hardware /multi_gpu.py
iparaco's picture
Upload 118 files
e59bd9d verified
Raw History Blame Contribute Delete
9.36 kB
"""Multi-GPU execution strategies.
Windows-specific engineering decisions (documented, measured, not assumed):
- DataParallel (nn.DataParallel): the DEFAULT for 2x P100. Replicates the
model per forward (EDSR-baseline is ~5 MB fp32 - negligible over PCIe),
scatters the batch, gathers outputs. Single-process, no NCCL needed,
works identically on Windows/Linux.
- DDP (gloo backend): Windows has no NCCL. gloo-CUDA on Windows has
historically been unreliable; therefore DDP is implemented, benchmarked
when the user opts in, and NEVER chosen automatically unless it both
initializes AND measures faster than DataParallel.
- ModelParallel: explicit pipeline split for nn.Sequential models. Only
chosen when the model is large AND batch is too small for DP to help.
On dual-P100 PCIe without P2P, the boundary activation crosses through
pinned host memory each iteration - the scheduler accounts for that.
Strategy selection is MEASURED (see core/scheduler.py): with probe_on_start,
we time the real model on synthetic batches for candidate strategies and
pick the fastest. Estimates are labeled estimates.
"""
from __future__ import annotations
import copy
import os
import tempfile
from pathlib import Path
from typing import List, Optional
from ..compat import require_torch
class StrategyError(RuntimeError):
pass
class SingleGPU:
"""Run the model on one GPU (most free VRAM by default)."""
name = "single"
def __init__(self, model, device_index: Optional[int] = None):
torch = require_torch()
if not torch.cuda.is_available():
raise StrategyError("SingleGPU requires CUDA")
if device_index is None:
best, best_free = 0, None
for i in range(torch.cuda.device_count()):
free_b, _ = torch.cuda.mem_get_info(i)
free = free_b / (1024 * 1024)
if best_free is None or free > best_free:
best, best_free = i, free
device_index = best
self.device = torch.device(f"cuda:{device_index}")
self.model = model.to(self.device)
def __call__(self, *args, **kwargs):
return self.model(*args, **kwargs)
class DataParallel:
"""torch.nn.DataParallel wrapper - the default multi-GPU strategy.
Kept as a thin, named wrapper so the scheduler/benchmarks reference one
authoritative implementation and so we can attach measurement metadata.
"""
name = "data_parallel"
def __init__(self, model, device_ids: Optional[List[int]] = None, output_device=None):
torch = require_torch()
if not torch.cuda.is_available():
raise StrategyError("DataParallel requires CUDA")
n = torch.cuda.device_count()
if device_ids is None:
device_ids = list(range(n))
if len(device_ids) < 2:
# Honest degradation: DP on a single GPU is just the model.
self._single = True
self.model = model.to(f"cuda:{device_ids[0] if device_ids else 0}")
self.device_ids = device_ids or [0]
return
self._single = False
self.device_ids = device_ids
# torch.nn.DataParallel REQUIRES the module resident on device_ids[0];
# a CPU-resident module crashes at the first forward.
model = model.to(f"cuda:{device_ids[0]}")
self.model = torch.nn.DataParallel(
model, device_ids=device_ids,
output_device=device_ids[0] if output_device is None else output_device,
)
@property
def underlying(self):
return self.model.module if (not self._single and hasattr(self.model, "module")) else self.model
def __call__(self, *args, **kwargs):
return self.model(*args, **kwargs)
class ModelParallel:
"""Explicit pipeline split of nn.Sequential-compatible models.
Splits children across GPUs by parameter memory (greedy balance). The
boundary activation transfers device->device (P2P when available, else
through the host). We do NOT split small SR models unless the scheduler
measured a justification (large params or DP infeasible batch).
"""
name = "model_parallel"
def __init__(self, model, device_ids: Optional[List[int]] = None):
torch = require_torch()
nn = torch.nn
if not torch.cuda.is_available():
raise StrategyError("ModelParallel requires CUDA")
n = torch.cuda.device_count()
self.device_ids = device_ids or list(range(n))
if len(self.device_ids) < 2:
raise StrategyError("ModelParallel requires >= 2 GPUs")
if not isinstance(model, nn.Sequential):
raise StrategyError(
"ModelParallel V3 splits nn.Sequential containers only "
f"(got {type(model).__name__}); wrap your blocks sequentially "
"or use data_parallel"
)
# greedy balance by parameter bytes
children = list(model.children())
if len(children) < 2:
raise StrategyError("model has fewer than 2 children to split")
weights = [
sum(p.numel() * p.element_size() for p in c.parameters()) for c in children
]
groups: List[List[int]] = [[] for _ in self.device_ids]
loads = [0] * len(self.device_ids)
order = sorted(range(len(children)), key=lambda i: -weights[i])
for idx in order:
g = loads.index(min(loads))
groups[g].append(idx)
loads[g] += weights[idx]
# preserve original ordering inside each partition
groups = [sorted(g) for g in groups if g]
self._parts = nn.ModuleList()
self._part_devices = []
for gi, group in enumerate(groups):
dev = self.device_ids[gi % len(self.device_ids)]
part = nn.Sequential(*[children[i] for i in group]).to(f"cuda:{dev}")
self._parts.append(part)
self._part_devices.append(dev)
self.split_plan = {
f"cuda:{d}": [type(children[i]).__name__ for i in g]
for g, d in zip(groups, self._part_devices)
}
def __call__(self, x, *args, **kwargs):
torch = require_torch()
for part, dev in zip(self._parts, self._part_devices):
if x.device.index != dev or x.device.type != "cuda":
x = x.to(f"cuda:{dev}", non_blocking=True)
x = part(x)
return x
def parameters(self):
for part in self._parts:
yield from part.parameters()
def state_dict(self):
"""Checkpoint-compatible: original child order, no part prefixes."""
out = {}
for part in self._parts:
out.update(part.state_dict())
return out
class DDPWorker:
"""Opt-in DistributedDataParallel launcher (gloo backend, file init).
Windows notes: NCCL does not exist on Windows; gloo is the only backend.
Because gloo-CUDA on Windows has been unreliable across torch versions,
this strategy is NEVER auto-selected: config gpu.strategy='ddp' or an
explicit benchmark win are required. init uses a temp file:// URL.
"""
name = "ddp"
_LAUNCH_LOCK = "LPU_DDP_ACTIVE"
@staticmethod
def available() -> bool:
torch = require_torch()
try:
import torch.distributed as dist # noqa: F401
return torch.cuda.is_available()
except Exception: # noqa: BLE001
return False
@staticmethod
def _init_process_group(rank: int, world_size: int, init_file: Path):
import torch.distributed as dist
dist.init_process_group(
backend="gloo",
init_method=f"file:///{init_file.as_posix().lstrip('/')}"
if os.name == "nt"
else f"file://{init_file}",
rank=rank,
world_size=world_size,
)
@staticmethod
def wrap_model(model, rank: int, world_size: int, init_file: Path):
"""Wrap a model in DDP; call inside each worker process."""
import torch.distributed as dist
DDPWorker._init_process_group(rank, world_size, init_file)
torch = require_torch()
model = model.to(f"cuda:{rank % torch.cuda.device_count()}")
ddp = torch.nn.parallel.DistributedDataParallel(model, device_ids=[rank % torch.cuda.device_count()])
return ddp
@staticmethod
def shutdown():
try:
import torch.distributed as dist
if dist.is_initialized():
dist.destroy_process_group()
except Exception: # noqa: BLE001
pass
def build_strategy(model, strategy: str, device_ids: Optional[List[int]] = None):
"""Factory used by the scheduler once a strategy name is decided."""
if strategy == "single":
return SingleGPU(model, device_ids[0] if device_ids else None)
if strategy == "data_parallel":
return DataParallel(model, device_ids)
if strategy == "model_parallel":
return ModelParallel(model, device_ids)
if strategy == "ddp":
raise StrategyError(
"DDP requires its multi-process launcher; use "
"lpu.ddp_launch(fn, args) or config gpu.strategy=ddp via the "
"training session, not build_strategy"
)
raise StrategyError(f"unknown strategy {strategy!r}")