"""Multi-GPU execution strategies. Windows-specific engineering decisions (documented, measured, not assumed): - DataParallel (nn.DataParallel): the DEFAULT for 2x P100. Replicates the model per forward (EDSR-baseline is ~5 MB fp32 - negligible over PCIe), scatters the batch, gathers outputs. Single-process, no NCCL needed, works identically on Windows/Linux. - DDP (gloo backend): Windows has no NCCL. gloo-CUDA on Windows has historically been unreliable; therefore DDP is implemented, benchmarked when the user opts in, and NEVER chosen automatically unless it both initializes AND measures faster than DataParallel. - ModelParallel: explicit pipeline split for nn.Sequential models. Only chosen when the model is large AND batch is too small for DP to help. On dual-P100 PCIe without P2P, the boundary activation crosses through pinned host memory each iteration - the scheduler accounts for that. Strategy selection is MEASURED (see core/scheduler.py): with probe_on_start, we time the real model on synthetic batches for candidate strategies and pick the fastest. Estimates are labeled estimates. """ from __future__ import annotations import copy import os import tempfile from pathlib import Path from typing import List, Optional from ..compat import require_torch class StrategyError(RuntimeError): pass class SingleGPU: """Run the model on one GPU (most free VRAM by default).""" name = "single" def __init__(self, model, device_index: Optional[int] = None): torch = require_torch() if not torch.cuda.is_available(): raise StrategyError("SingleGPU requires CUDA") if device_index is None: best, best_free = 0, None for i in range(torch.cuda.device_count()): free_b, _ = torch.cuda.mem_get_info(i) free = free_b / (1024 * 1024) if best_free is None or free > best_free: best, best_free = i, free device_index = best self.device = torch.device(f"cuda:{device_index}") self.model = model.to(self.device) def __call__(self, *args, **kwargs): return self.model(*args, **kwargs) class DataParallel: """torch.nn.DataParallel wrapper - the default multi-GPU strategy. Kept as a thin, named wrapper so the scheduler/benchmarks reference one authoritative implementation and so we can attach measurement metadata. """ name = "data_parallel" def __init__(self, model, device_ids: Optional[List[int]] = None, output_device=None): torch = require_torch() if not torch.cuda.is_available(): raise StrategyError("DataParallel requires CUDA") n = torch.cuda.device_count() if device_ids is None: device_ids = list(range(n)) if len(device_ids) < 2: # Honest degradation: DP on a single GPU is just the model. self._single = True self.model = model.to(f"cuda:{device_ids[0] if device_ids else 0}") self.device_ids = device_ids or [0] return self._single = False self.device_ids = device_ids # torch.nn.DataParallel REQUIRES the module resident on device_ids[0]; # a CPU-resident module crashes at the first forward. model = model.to(f"cuda:{device_ids[0]}") self.model = torch.nn.DataParallel( model, device_ids=device_ids, output_device=device_ids[0] if output_device is None else output_device, ) @property def underlying(self): return self.model.module if (not self._single and hasattr(self.model, "module")) else self.model def __call__(self, *args, **kwargs): return self.model(*args, **kwargs) class ModelParallel: """Explicit pipeline split of nn.Sequential-compatible models. Splits children across GPUs by parameter memory (greedy balance). The boundary activation transfers device->device (P2P when available, else through the host). We do NOT split small SR models unless the scheduler measured a justification (large params or DP infeasible batch). """ name = "model_parallel" def __init__(self, model, device_ids: Optional[List[int]] = None): torch = require_torch() nn = torch.nn if not torch.cuda.is_available(): raise StrategyError("ModelParallel requires CUDA") n = torch.cuda.device_count() self.device_ids = device_ids or list(range(n)) if len(self.device_ids) < 2: raise StrategyError("ModelParallel requires >= 2 GPUs") if not isinstance(model, nn.Sequential): raise StrategyError( "ModelParallel V3 splits nn.Sequential containers only " f"(got {type(model).__name__}); wrap your blocks sequentially " "or use data_parallel" ) # greedy balance by parameter bytes children = list(model.children()) if len(children) < 2: raise StrategyError("model has fewer than 2 children to split") weights = [ sum(p.numel() * p.element_size() for p in c.parameters()) for c in children ] groups: List[List[int]] = [[] for _ in self.device_ids] loads = [0] * len(self.device_ids) order = sorted(range(len(children)), key=lambda i: -weights[i]) for idx in order: g = loads.index(min(loads)) groups[g].append(idx) loads[g] += weights[idx] # preserve original ordering inside each partition groups = [sorted(g) for g in groups if g] self._parts = nn.ModuleList() self._part_devices = [] for gi, group in enumerate(groups): dev = self.device_ids[gi % len(self.device_ids)] part = nn.Sequential(*[children[i] for i in group]).to(f"cuda:{dev}") self._parts.append(part) self._part_devices.append(dev) self.split_plan = { f"cuda:{d}": [type(children[i]).__name__ for i in g] for g, d in zip(groups, self._part_devices) } def __call__(self, x, *args, **kwargs): torch = require_torch() for part, dev in zip(self._parts, self._part_devices): if x.device.index != dev or x.device.type != "cuda": x = x.to(f"cuda:{dev}", non_blocking=True) x = part(x) return x def parameters(self): for part in self._parts: yield from part.parameters() def state_dict(self): """Checkpoint-compatible: original child order, no part prefixes.""" out = {} for part in self._parts: out.update(part.state_dict()) return out class DDPWorker: """Opt-in DistributedDataParallel launcher (gloo backend, file init). Windows notes: NCCL does not exist on Windows; gloo is the only backend. Because gloo-CUDA on Windows has been unreliable across torch versions, this strategy is NEVER auto-selected: config gpu.strategy='ddp' or an explicit benchmark win are required. init uses a temp file:// URL. """ name = "ddp" _LAUNCH_LOCK = "LPU_DDP_ACTIVE" @staticmethod def available() -> bool: torch = require_torch() try: import torch.distributed as dist # noqa: F401 return torch.cuda.is_available() except Exception: # noqa: BLE001 return False @staticmethod def _init_process_group(rank: int, world_size: int, init_file: Path): import torch.distributed as dist dist.init_process_group( backend="gloo", init_method=f"file:///{init_file.as_posix().lstrip('/')}" if os.name == "nt" else f"file://{init_file}", rank=rank, world_size=world_size, ) @staticmethod def wrap_model(model, rank: int, world_size: int, init_file: Path): """Wrap a model in DDP; call inside each worker process.""" import torch.distributed as dist DDPWorker._init_process_group(rank, world_size, init_file) torch = require_torch() model = model.to(f"cuda:{rank % torch.cuda.device_count()}") ddp = torch.nn.parallel.DistributedDataParallel(model, device_ids=[rank % torch.cuda.device_count()]) return ddp @staticmethod def shutdown(): try: import torch.distributed as dist if dist.is_initialized(): dist.destroy_process_group() except Exception: # noqa: BLE001 pass def build_strategy(model, strategy: str, device_ids: Optional[List[int]] = None): """Factory used by the scheduler once a strategy name is decided.""" if strategy == "single": return SingleGPU(model, device_ids[0] if device_ids else None) if strategy == "data_parallel": return DataParallel(model, device_ids) if strategy == "model_parallel": return ModelParallel(model, device_ids) if strategy == "ddp": raise StrategyError( "DDP requires its multi-process launcher; use " "lpu.ddp_launch(fn, args) or config gpu.strategy=ddp via the " "training session, not build_strategy" ) raise StrategyError(f"unknown strategy {strategy!r}")