"""Adaptive offloading (OPT-IN, disabled by default, experimental). GPU_FIRST is the default memory policy (spec section 7.1). This module only runs when memory.policy=adaptive_offloading AND memory.adaptive_offloading are both set. It moves LPU-CACHE-OWNED tensors (never user model parameters unless explicitly registered via register()) between VRAM and pinned host memory when a device is under pressure. Honest performance statement: for SR training, PCIe round-trips are usually more expensive than recomputation, and results must be measured per case. This subsystem reports every offload/restore so the user can judge. """ from __future__ import annotations import time from typing import Dict, Optional from ..compat import require_torch from .memory_manager import MemoryManager class OffloadManager: """Moves registered tensors between GPU and pinned host memory.""" def __init__(self, memory: MemoryManager, pressure_pct: float = 92.0, log_dir=None): from ..monitoring.logger import get_logger self.memory = memory self.pressure_pct = pressure_pct self._log = get_logger("offload", log_dir=log_dir, console=False) self._registered: Dict[str, dict] = {} self._transfers: list = [] def register(self, name: str, tensor, allow_offload: bool = True) -> None: """Explicit user opt-in for a tensor it owns (e.g. a val-set bank).""" self._registered[name] = { "tensor": tensor, "allow": allow_offload, "location": str(getattr(tensor, "device", "?")), } def maybe_offload(self, device_index: int) -> Dict[str, int]: """When device under pressure, move eligible tensors to pinned RAM.""" torch = require_torch() moved = {"tensors": 0, "mb": 0.0} pressure = self.memory.is_under_pressure(device_index, self.pressure_pct) if pressure is not True: return moved for name, rec in self._registered.items(): t = rec["tensor"] if not rec["allow"] or not getattr(t, "is_cuda", False): continue if t.device.index != device_index: continue try: host = t.cpu().pin_memory() rec["tensor"] = host rec["location"] = "cpu(pinned)" mb = t.numel() * t.element_size() / (1024 * 1024) moved["tensors"] += 1 moved["mb"] += mb self._transfers.append( {"ts": time.time(), "name": name, "dir": "to_host", "mb": mb, "device": device_index} ) except Exception as exc: # noqa: BLE001 self._log.warning("offload of %s failed: %s", name, exc) if moved["tensors"]: self._log.info("offloaded %d tensors (%.1f MB) from cuda:%d", moved["tensors"], moved["mb"], device_index) return moved def restore(self, name: str, device_index: int): """Bring one registered tensor back to the device when needed.""" torch = require_torch() rec = self._registered.get(name) if rec is None: return None t = rec["tensor"] if not getattr(t, "is_cuda", False): t = t.to(f"cuda:{device_index}", non_blocking=True) rec["tensor"] = t rec["location"] = str(t.device) self._transfers.append( {"ts": time.time(), "name": name, "dir": "to_device", "mb": t.numel() * t.element_size() / (1024 * 1024), "device": device_index} ) return t def report(self) -> dict: return { "policy": "adaptive_offloading", "registered": len(self._registered), "transfers": list(self._transfers[-50:]), "note": "offloading is opt-in; GPU_FIRST is the default policy", }