File size: 3,974 Bytes
e59bd9d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
"""Adaptive offloading (OPT-IN, disabled by default, experimental).

GPU_FIRST is the default memory policy (spec section 7.1). This module only
runs when memory.policy=adaptive_offloading AND memory.adaptive_offloading
are both set. It moves LPU-CACHE-OWNED tensors (never user model parameters
unless explicitly registered via register()) between VRAM and pinned host
memory when a device is under pressure.

Honest performance statement: for SR training, PCIe round-trips are usually
more expensive than recomputation, and results must be measured per case.
This subsystem reports every offload/restore so the user can judge.
"""

from __future__ import annotations

import time
from typing import Dict, Optional

from ..compat import require_torch
from .memory_manager import MemoryManager


class OffloadManager:
    """Moves registered tensors between GPU and pinned host memory."""

    def __init__(self, memory: MemoryManager, pressure_pct: float = 92.0, log_dir=None):
        from ..monitoring.logger import get_logger

        self.memory = memory
        self.pressure_pct = pressure_pct
        self._log = get_logger("offload", log_dir=log_dir, console=False)
        self._registered: Dict[str, dict] = {}
        self._transfers: list = []

    def register(self, name: str, tensor, allow_offload: bool = True) -> None:
        """Explicit user opt-in for a tensor it owns (e.g. a val-set bank)."""
        self._registered[name] = {
            "tensor": tensor,
            "allow": allow_offload,
            "location": str(getattr(tensor, "device", "?")),
        }

    def maybe_offload(self, device_index: int) -> Dict[str, int]:
        """When device under pressure, move eligible tensors to pinned RAM."""
        torch = require_torch()
        moved = {"tensors": 0, "mb": 0.0}
        pressure = self.memory.is_under_pressure(device_index, self.pressure_pct)
        if pressure is not True:
            return moved
        for name, rec in self._registered.items():
            t = rec["tensor"]
            if not rec["allow"] or not getattr(t, "is_cuda", False):
                continue
            if t.device.index != device_index:
                continue
            try:
                host = t.cpu().pin_memory()
                rec["tensor"] = host
                rec["location"] = "cpu(pinned)"
                mb = t.numel() * t.element_size() / (1024 * 1024)
                moved["tensors"] += 1
                moved["mb"] += mb
                self._transfers.append(
                    {"ts": time.time(), "name": name, "dir": "to_host", "mb": mb,
                     "device": device_index}
                )
            except Exception as exc:  # noqa: BLE001
                self._log.warning("offload of %s failed: %s", name, exc)
        if moved["tensors"]:
            self._log.info("offloaded %d tensors (%.1f MB) from cuda:%d",
                           moved["tensors"], moved["mb"], device_index)
        return moved

    def restore(self, name: str, device_index: int):
        """Bring one registered tensor back to the device when needed."""
        torch = require_torch()
        rec = self._registered.get(name)
        if rec is None:
            return None
        t = rec["tensor"]
        if not getattr(t, "is_cuda", False):
            t = t.to(f"cuda:{device_index}", non_blocking=True)
            rec["tensor"] = t
            rec["location"] = str(t.device)
            self._transfers.append(
                {"ts": time.time(), "name": name, "dir": "to_device",
                 "mb": t.numel() * t.element_size() / (1024 * 1024),
                 "device": device_index}
            )
        return t

    def report(self) -> dict:
        return {
            "policy": "adaptive_offloading",
            "registered": len(self._registered),
            "transfers": list(self._transfers[-50:]),
            "note": "offloading is opt-in; GPU_FIRST is the default policy",
        }