Virtual-LPU / lpu /memory /offloading.py
iparaco's picture
Upload 118 files
e59bd9d verified
Raw History Blame Contribute Delete
3.97 kB
"""Adaptive offloading (OPT-IN, disabled by default, experimental).
GPU_FIRST is the default memory policy (spec section 7.1). This module only
runs when memory.policy=adaptive_offloading AND memory.adaptive_offloading
are both set. It moves LPU-CACHE-OWNED tensors (never user model parameters
unless explicitly registered via register()) between VRAM and pinned host
memory when a device is under pressure.
Honest performance statement: for SR training, PCIe round-trips are usually
more expensive than recomputation, and results must be measured per case.
This subsystem reports every offload/restore so the user can judge.
"""
from __future__ import annotations
import time
from typing import Dict, Optional
from ..compat import require_torch
from .memory_manager import MemoryManager
class OffloadManager:
"""Moves registered tensors between GPU and pinned host memory."""
def __init__(self, memory: MemoryManager, pressure_pct: float = 92.0, log_dir=None):
from ..monitoring.logger import get_logger
self.memory = memory
self.pressure_pct = pressure_pct
self._log = get_logger("offload", log_dir=log_dir, console=False)
self._registered: Dict[str, dict] = {}
self._transfers: list = []
def register(self, name: str, tensor, allow_offload: bool = True) -> None:
"""Explicit user opt-in for a tensor it owns (e.g. a val-set bank)."""
self._registered[name] = {
"tensor": tensor,
"allow": allow_offload,
"location": str(getattr(tensor, "device", "?")),
}
def maybe_offload(self, device_index: int) -> Dict[str, int]:
"""When device under pressure, move eligible tensors to pinned RAM."""
torch = require_torch()
moved = {"tensors": 0, "mb": 0.0}
pressure = self.memory.is_under_pressure(device_index, self.pressure_pct)
if pressure is not True:
return moved
for name, rec in self._registered.items():
t = rec["tensor"]
if not rec["allow"] or not getattr(t, "is_cuda", False):
continue
if t.device.index != device_index:
continue
try:
host = t.cpu().pin_memory()
rec["tensor"] = host
rec["location"] = "cpu(pinned)"
mb = t.numel() * t.element_size() / (1024 * 1024)
moved["tensors"] += 1
moved["mb"] += mb
self._transfers.append(
{"ts": time.time(), "name": name, "dir": "to_host", "mb": mb,
"device": device_index}
)
except Exception as exc: # noqa: BLE001
self._log.warning("offload of %s failed: %s", name, exc)
if moved["tensors"]:
self._log.info("offloaded %d tensors (%.1f MB) from cuda:%d",
moved["tensors"], moved["mb"], device_index)
return moved
def restore(self, name: str, device_index: int):
"""Bring one registered tensor back to the device when needed."""
torch = require_torch()
rec = self._registered.get(name)
if rec is None:
return None
t = rec["tensor"]
if not getattr(t, "is_cuda", False):
t = t.to(f"cuda:{device_index}", non_blocking=True)
rec["tensor"] = t
rec["location"] = str(t.device)
self._transfers.append(
{"ts": time.time(), "name": name, "dir": "to_device",
"mb": t.numel() * t.element_size() / (1024 * 1024),
"device": device_index}
)
return t
def report(self) -> dict:
return {
"policy": "adaptive_offloading",
"registered": len(self._registered),
"transfers": list(self._transfers[-50:]),
"note": "offloading is opt-in; GPU_FIRST is the default policy",
}