Spaces:
Running
Running
Download lpu/hardware/multi_gpu.py from iparaco/Virtual-LPU: direct link, hf CLI and curl.
- Browser
- Download file 9.36 kB
-
https://huggingface.co/spaces/iparaco/Virtual-LPU/resolve/main/lpu/hardware/multi_gpu.py
- Command line
-
hf download hf://spaces/iparaco/Virtual-LPU/lpu/hardware/multi_gpu.py
-
curl -L -o multi_gpu.py https://huggingface.co/spaces/iparaco/Virtual-LPU/resolve/main/lpu/hardware/multi_gpu.py
9.36 kB
| """Multi-GPU execution strategies. | |
| Windows-specific engineering decisions (documented, measured, not assumed): | |
| - DataParallel (nn.DataParallel): the DEFAULT for 2x P100. Replicates the | |
| model per forward (EDSR-baseline is ~5 MB fp32 - negligible over PCIe), | |
| scatters the batch, gathers outputs. Single-process, no NCCL needed, | |
| works identically on Windows/Linux. | |
| - DDP (gloo backend): Windows has no NCCL. gloo-CUDA on Windows has | |
| historically been unreliable; therefore DDP is implemented, benchmarked | |
| when the user opts in, and NEVER chosen automatically unless it both | |
| initializes AND measures faster than DataParallel. | |
| - ModelParallel: explicit pipeline split for nn.Sequential models. Only | |
| chosen when the model is large AND batch is too small for DP to help. | |
| On dual-P100 PCIe without P2P, the boundary activation crosses through | |
| pinned host memory each iteration - the scheduler accounts for that. | |
| Strategy selection is MEASURED (see core/scheduler.py): with probe_on_start, | |
| we time the real model on synthetic batches for candidate strategies and | |
| pick the fastest. Estimates are labeled estimates. | |
| """ | |
| from __future__ import annotations | |
| import copy | |
| import os | |
| import tempfile | |
| from pathlib import Path | |
| from typing import List, Optional | |
| from ..compat import require_torch | |
| class StrategyError(RuntimeError): | |
| pass | |
| class SingleGPU: | |
| """Run the model on one GPU (most free VRAM by default).""" | |
| name = "single" | |
| def __init__(self, model, device_index: Optional[int] = None): | |
| torch = require_torch() | |
| if not torch.cuda.is_available(): | |
| raise StrategyError("SingleGPU requires CUDA") | |
| if device_index is None: | |
| best, best_free = 0, None | |
| for i in range(torch.cuda.device_count()): | |
| free_b, _ = torch.cuda.mem_get_info(i) | |
| free = free_b / (1024 * 1024) | |
| if best_free is None or free > best_free: | |
| best, best_free = i, free | |
| device_index = best | |
| self.device = torch.device(f"cuda:{device_index}") | |
| self.model = model.to(self.device) | |
| def __call__(self, *args, **kwargs): | |
| return self.model(*args, **kwargs) | |
| class DataParallel: | |
| """torch.nn.DataParallel wrapper - the default multi-GPU strategy. | |
| Kept as a thin, named wrapper so the scheduler/benchmarks reference one | |
| authoritative implementation and so we can attach measurement metadata. | |
| """ | |
| name = "data_parallel" | |
| def __init__(self, model, device_ids: Optional[List[int]] = None, output_device=None): | |
| torch = require_torch() | |
| if not torch.cuda.is_available(): | |
| raise StrategyError("DataParallel requires CUDA") | |
| n = torch.cuda.device_count() | |
| if device_ids is None: | |
| device_ids = list(range(n)) | |
| if len(device_ids) < 2: | |
| # Honest degradation: DP on a single GPU is just the model. | |
| self._single = True | |
| self.model = model.to(f"cuda:{device_ids[0] if device_ids else 0}") | |
| self.device_ids = device_ids or [0] | |
| return | |
| self._single = False | |
| self.device_ids = device_ids | |
| # torch.nn.DataParallel REQUIRES the module resident on device_ids[0]; | |
| # a CPU-resident module crashes at the first forward. | |
| model = model.to(f"cuda:{device_ids[0]}") | |
| self.model = torch.nn.DataParallel( | |
| model, device_ids=device_ids, | |
| output_device=device_ids[0] if output_device is None else output_device, | |
| ) | |
| def underlying(self): | |
| return self.model.module if (not self._single and hasattr(self.model, "module")) else self.model | |
| def __call__(self, *args, **kwargs): | |
| return self.model(*args, **kwargs) | |
| class ModelParallel: | |
| """Explicit pipeline split of nn.Sequential-compatible models. | |
| Splits children across GPUs by parameter memory (greedy balance). The | |
| boundary activation transfers device->device (P2P when available, else | |
| through the host). We do NOT split small SR models unless the scheduler | |
| measured a justification (large params or DP infeasible batch). | |
| """ | |
| name = "model_parallel" | |
| def __init__(self, model, device_ids: Optional[List[int]] = None): | |
| torch = require_torch() | |
| nn = torch.nn | |
| if not torch.cuda.is_available(): | |
| raise StrategyError("ModelParallel requires CUDA") | |
| n = torch.cuda.device_count() | |
| self.device_ids = device_ids or list(range(n)) | |
| if len(self.device_ids) < 2: | |
| raise StrategyError("ModelParallel requires >= 2 GPUs") | |
| if not isinstance(model, nn.Sequential): | |
| raise StrategyError( | |
| "ModelParallel V3 splits nn.Sequential containers only " | |
| f"(got {type(model).__name__}); wrap your blocks sequentially " | |
| "or use data_parallel" | |
| ) | |
| # greedy balance by parameter bytes | |
| children = list(model.children()) | |
| if len(children) < 2: | |
| raise StrategyError("model has fewer than 2 children to split") | |
| weights = [ | |
| sum(p.numel() * p.element_size() for p in c.parameters()) for c in children | |
| ] | |
| groups: List[List[int]] = [[] for _ in self.device_ids] | |
| loads = [0] * len(self.device_ids) | |
| order = sorted(range(len(children)), key=lambda i: -weights[i]) | |
| for idx in order: | |
| g = loads.index(min(loads)) | |
| groups[g].append(idx) | |
| loads[g] += weights[idx] | |
| # preserve original ordering inside each partition | |
| groups = [sorted(g) for g in groups if g] | |
| self._parts = nn.ModuleList() | |
| self._part_devices = [] | |
| for gi, group in enumerate(groups): | |
| dev = self.device_ids[gi % len(self.device_ids)] | |
| part = nn.Sequential(*[children[i] for i in group]).to(f"cuda:{dev}") | |
| self._parts.append(part) | |
| self._part_devices.append(dev) | |
| self.split_plan = { | |
| f"cuda:{d}": [type(children[i]).__name__ for i in g] | |
| for g, d in zip(groups, self._part_devices) | |
| } | |
| def __call__(self, x, *args, **kwargs): | |
| torch = require_torch() | |
| for part, dev in zip(self._parts, self._part_devices): | |
| if x.device.index != dev or x.device.type != "cuda": | |
| x = x.to(f"cuda:{dev}", non_blocking=True) | |
| x = part(x) | |
| return x | |
| def parameters(self): | |
| for part in self._parts: | |
| yield from part.parameters() | |
| def state_dict(self): | |
| """Checkpoint-compatible: original child order, no part prefixes.""" | |
| out = {} | |
| for part in self._parts: | |
| out.update(part.state_dict()) | |
| return out | |
| class DDPWorker: | |
| """Opt-in DistributedDataParallel launcher (gloo backend, file init). | |
| Windows notes: NCCL does not exist on Windows; gloo is the only backend. | |
| Because gloo-CUDA on Windows has been unreliable across torch versions, | |
| this strategy is NEVER auto-selected: config gpu.strategy='ddp' or an | |
| explicit benchmark win are required. init uses a temp file:// URL. | |
| """ | |
| name = "ddp" | |
| _LAUNCH_LOCK = "LPU_DDP_ACTIVE" | |
| def available() -> bool: | |
| torch = require_torch() | |
| try: | |
| import torch.distributed as dist # noqa: F401 | |
| return torch.cuda.is_available() | |
| except Exception: # noqa: BLE001 | |
| return False | |
| def _init_process_group(rank: int, world_size: int, init_file: Path): | |
| import torch.distributed as dist | |
| dist.init_process_group( | |
| backend="gloo", | |
| init_method=f"file:///{init_file.as_posix().lstrip('/')}" | |
| if os.name == "nt" | |
| else f"file://{init_file}", | |
| rank=rank, | |
| world_size=world_size, | |
| ) | |
| def wrap_model(model, rank: int, world_size: int, init_file: Path): | |
| """Wrap a model in DDP; call inside each worker process.""" | |
| import torch.distributed as dist | |
| DDPWorker._init_process_group(rank, world_size, init_file) | |
| torch = require_torch() | |
| model = model.to(f"cuda:{rank % torch.cuda.device_count()}") | |
| ddp = torch.nn.parallel.DistributedDataParallel(model, device_ids=[rank % torch.cuda.device_count()]) | |
| return ddp | |
| def shutdown(): | |
| try: | |
| import torch.distributed as dist | |
| if dist.is_initialized(): | |
| dist.destroy_process_group() | |
| except Exception: # noqa: BLE001 | |
| pass | |
| def build_strategy(model, strategy: str, device_ids: Optional[List[int]] = None): | |
| """Factory used by the scheduler once a strategy name is decided.""" | |
| if strategy == "single": | |
| return SingleGPU(model, device_ids[0] if device_ids else None) | |
| if strategy == "data_parallel": | |
| return DataParallel(model, device_ids) | |
| if strategy == "model_parallel": | |
| return ModelParallel(model, device_ids) | |
| if strategy == "ddp": | |
| raise StrategyError( | |
| "DDP requires its multi-process launcher; use " | |
| "lpu.ddp_launch(fn, args) or config gpu.strategy=ddp via the " | |
| "training session, not build_strategy" | |
| ) | |
| raise StrategyError(f"unknown strategy {strategy!r}") | |