Spaces:
Running
Running
Download lpu/memory/offloading.py from iparaco/Virtual-LPU: direct link, hf CLI and curl.
- Browser
- Download file 3.97 kB
-
https://huggingface.co/spaces/iparaco/Virtual-LPU/resolve/main/lpu/memory/offloading.py
- Command line
-
hf download hf://spaces/iparaco/Virtual-LPU/lpu/memory/offloading.py
-
curl -L -o offloading.py https://huggingface.co/spaces/iparaco/Virtual-LPU/resolve/main/lpu/memory/offloading.py
3.97 kB
| """Adaptive offloading (OPT-IN, disabled by default, experimental). | |
| GPU_FIRST is the default memory policy (spec section 7.1). This module only | |
| runs when memory.policy=adaptive_offloading AND memory.adaptive_offloading | |
| are both set. It moves LPU-CACHE-OWNED tensors (never user model parameters | |
| unless explicitly registered via register()) between VRAM and pinned host | |
| memory when a device is under pressure. | |
| Honest performance statement: for SR training, PCIe round-trips are usually | |
| more expensive than recomputation, and results must be measured per case. | |
| This subsystem reports every offload/restore so the user can judge. | |
| """ | |
| from __future__ import annotations | |
| import time | |
| from typing import Dict, Optional | |
| from ..compat import require_torch | |
| from .memory_manager import MemoryManager | |
| class OffloadManager: | |
| """Moves registered tensors between GPU and pinned host memory.""" | |
| def __init__(self, memory: MemoryManager, pressure_pct: float = 92.0, log_dir=None): | |
| from ..monitoring.logger import get_logger | |
| self.memory = memory | |
| self.pressure_pct = pressure_pct | |
| self._log = get_logger("offload", log_dir=log_dir, console=False) | |
| self._registered: Dict[str, dict] = {} | |
| self._transfers: list = [] | |
| def register(self, name: str, tensor, allow_offload: bool = True) -> None: | |
| """Explicit user opt-in for a tensor it owns (e.g. a val-set bank).""" | |
| self._registered[name] = { | |
| "tensor": tensor, | |
| "allow": allow_offload, | |
| "location": str(getattr(tensor, "device", "?")), | |
| } | |
| def maybe_offload(self, device_index: int) -> Dict[str, int]: | |
| """When device under pressure, move eligible tensors to pinned RAM.""" | |
| torch = require_torch() | |
| moved = {"tensors": 0, "mb": 0.0} | |
| pressure = self.memory.is_under_pressure(device_index, self.pressure_pct) | |
| if pressure is not True: | |
| return moved | |
| for name, rec in self._registered.items(): | |
| t = rec["tensor"] | |
| if not rec["allow"] or not getattr(t, "is_cuda", False): | |
| continue | |
| if t.device.index != device_index: | |
| continue | |
| try: | |
| host = t.cpu().pin_memory() | |
| rec["tensor"] = host | |
| rec["location"] = "cpu(pinned)" | |
| mb = t.numel() * t.element_size() / (1024 * 1024) | |
| moved["tensors"] += 1 | |
| moved["mb"] += mb | |
| self._transfers.append( | |
| {"ts": time.time(), "name": name, "dir": "to_host", "mb": mb, | |
| "device": device_index} | |
| ) | |
| except Exception as exc: # noqa: BLE001 | |
| self._log.warning("offload of %s failed: %s", name, exc) | |
| if moved["tensors"]: | |
| self._log.info("offloaded %d tensors (%.1f MB) from cuda:%d", | |
| moved["tensors"], moved["mb"], device_index) | |
| return moved | |
| def restore(self, name: str, device_index: int): | |
| """Bring one registered tensor back to the device when needed.""" | |
| torch = require_torch() | |
| rec = self._registered.get(name) | |
| if rec is None: | |
| return None | |
| t = rec["tensor"] | |
| if not getattr(t, "is_cuda", False): | |
| t = t.to(f"cuda:{device_index}", non_blocking=True) | |
| rec["tensor"] = t | |
| rec["location"] = str(t.device) | |
| self._transfers.append( | |
| {"ts": time.time(), "name": name, "dir": "to_device", | |
| "mb": t.numel() * t.element_size() / (1024 * 1024), | |
| "device": device_index} | |
| ) | |
| return t | |
| def report(self) -> dict: | |
| return { | |
| "policy": "adaptive_offloading", | |
| "registered": len(self._registered), | |
| "transfers": list(self._transfers[-50:]), | |
| "note": "offloading is opt-in; GPU_FIRST is the default policy", | |
| } | |