File size: 3,783 Bytes
e59bd9d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
"""Named tensor cache with ownership tags and per-device accounting.

Only LPU-owned tensors live here (probe results, benchmark copies, prefetch
buffers). The cache NEVER touches user model tensors unless the user
explicitly registers them. Eviction is explicit and reversible by design:
entries are (name -> {tensor, device, created, last_used, bytes}).
"""

from __future__ import annotations

import threading
import time
from typing import Dict, List, Optional

from ..compat import require_torch


class TensorCache:
    def __init__(self, owner: str = "lpu", max_entries: int = 256):
        self.owner = owner
        self.max_entries = max_entries
        self._entries: Dict[str, dict] = {}
        self._lock = threading.RLock()
        self._hits = 0
        self._misses = 0
        self._evictions = 0

    def put(self, name: str, tensor, note: str = "") -> None:
        torch = require_torch()
        with self._lock:
            self._entries[name] = {
                "tensor": tensor,
                "device": str(getattr(tensor, "device", "?")),
                "created": time.time(),
                "last_used": time.time(),
                "bytes": (
                    tensor.numel() * tensor.element_size()
                    if hasattr(tensor, "element_size")
                    else 0
                ),
                "note": note,
            }
            self._evict_locked()

    def get(self, name: str):
        with self._lock:
            e = self._entries.get(name)
            if e is None:
                self._misses += 1
                return None
            e["last_used"] = time.time()
            self._hits += 1
            return e["tensor"]

    def contains(self, name: str) -> bool:
        with self._lock:
            return name in self._entries

    def evict(self, name: str) -> bool:
        with self._lock:
            return self._entries.pop(name, None) is not None

    def evict_device(self, device_str: str) -> int:
        """Drop LPU-owned entries on one physical device. Returns count."""
        with self._lock:
            names = [n for n, e in self._entries.items() if device_str in e["device"]]
            for n in names:
                self._entries.pop(n, None)
                self._evictions += 1
            return len(names)

    def _evict_locked(self) -> None:
        if len(self._entries) <= self.max_entries:
            return
        ordered = sorted(self._entries.items(), key=lambda kv: kv[1]["last_used"])
        for name, _ in ordered[: max(0, len(self._entries) - self.max_entries)]:
            self._entries.pop(name, None)
            self._evictions += 1

    def device_bytes(self) -> Dict[str, float]:
        """Cached bytes per device string (e.g. 'cuda:0')."""
        out: Dict[str, float] = {}
        with self._lock:
            for e in self._entries.values():
                out[e["device"]] = out.get(e["device"], 0.0) + e["bytes"]
        return {k: v / (1024 * 1024) for k, v in out.items()}

    def stats(self) -> dict:
        with self._lock:
            return {
                "entries": len(self._entries),
                "max_entries": self.max_entries,
                "hits": self._hits,
                "misses": self._misses,
                "evictions": self._evictions,
                "mb_per_device": self.device_bytes(),
            }

    def clear(self) -> int:
        with self._lock:
            n = len(self._entries)
            self._entries.clear()
            return n


def _safe_deepcopy(model):
    import copy

    try:
        return copy.deepcopy(model)
    except Exception as exc:  # noqa: BLE001
        raise RuntimeError(
            f"model is not deep-copyable (needed for safe VRAM probing): {exc}"
        ) from exc