Virtual-LPU / lpu /memory /tensor_cache.py
iparaco's picture
Upload 118 files
e59bd9d verified
Raw History Blame
3.78 kB
"""Named tensor cache with ownership tags and per-device accounting.
Only LPU-owned tensors live here (probe results, benchmark copies, prefetch
buffers). The cache NEVER touches user model tensors unless the user
explicitly registers them. Eviction is explicit and reversible by design:
entries are (name -> {tensor, device, created, last_used, bytes}).
"""
from __future__ import annotations
import threading
import time
from typing import Dict, List, Optional
from ..compat import require_torch
class TensorCache:
def __init__(self, owner: str = "lpu", max_entries: int = 256):
self.owner = owner
self.max_entries = max_entries
self._entries: Dict[str, dict] = {}
self._lock = threading.RLock()
self._hits = 0
self._misses = 0
self._evictions = 0
def put(self, name: str, tensor, note: str = "") -> None:
torch = require_torch()
with self._lock:
self._entries[name] = {
"tensor": tensor,
"device": str(getattr(tensor, "device", "?")),
"created": time.time(),
"last_used": time.time(),
"bytes": (
tensor.numel() * tensor.element_size()
if hasattr(tensor, "element_size")
else 0
),
"note": note,
}
self._evict_locked()
def get(self, name: str):
with self._lock:
e = self._entries.get(name)
if e is None:
self._misses += 1
return None
e["last_used"] = time.time()
self._hits += 1
return e["tensor"]
def contains(self, name: str) -> bool:
with self._lock:
return name in self._entries
def evict(self, name: str) -> bool:
with self._lock:
return self._entries.pop(name, None) is not None
def evict_device(self, device_str: str) -> int:
"""Drop LPU-owned entries on one physical device. Returns count."""
with self._lock:
names = [n for n, e in self._entries.items() if device_str in e["device"]]
for n in names:
self._entries.pop(n, None)
self._evictions += 1
return len(names)
def _evict_locked(self) -> None:
if len(self._entries) <= self.max_entries:
return
ordered = sorted(self._entries.items(), key=lambda kv: kv[1]["last_used"])
for name, _ in ordered[: max(0, len(self._entries) - self.max_entries)]:
self._entries.pop(name, None)
self._evictions += 1
def device_bytes(self) -> Dict[str, float]:
"""Cached bytes per device string (e.g. 'cuda:0')."""
out: Dict[str, float] = {}
with self._lock:
for e in self._entries.values():
out[e["device"]] = out.get(e["device"], 0.0) + e["bytes"]
return {k: v / (1024 * 1024) for k, v in out.items()}
def stats(self) -> dict:
with self._lock:
return {
"entries": len(self._entries),
"max_entries": self.max_entries,
"hits": self._hits,
"misses": self._misses,
"evictions": self._evictions,
"mb_per_device": self.device_bytes(),
}
def clear(self) -> int:
with self._lock:
n = len(self._entries)
self._entries.clear()
return n
def _safe_deepcopy(model):
import copy
try:
return copy.deepcopy(model)
except Exception as exc: # noqa: BLE001
raise RuntimeError(
f"model is not deep-copyable (needed for safe VRAM probing): {exc}"
) from exc