"""Named tensor cache with ownership tags and per-device accounting. Only LPU-owned tensors live here (probe results, benchmark copies, prefetch buffers). The cache NEVER touches user model tensors unless the user explicitly registers them. Eviction is explicit and reversible by design: entries are (name -> {tensor, device, created, last_used, bytes}). """ from __future__ import annotations import threading import time from typing import Dict, List, Optional from ..compat import require_torch class TensorCache: def __init__(self, owner: str = "lpu", max_entries: int = 256): self.owner = owner self.max_entries = max_entries self._entries: Dict[str, dict] = {} self._lock = threading.RLock() self._hits = 0 self._misses = 0 self._evictions = 0 def put(self, name: str, tensor, note: str = "") -> None: torch = require_torch() with self._lock: self._entries[name] = { "tensor": tensor, "device": str(getattr(tensor, "device", "?")), "created": time.time(), "last_used": time.time(), "bytes": ( tensor.numel() * tensor.element_size() if hasattr(tensor, "element_size") else 0 ), "note": note, } self._evict_locked() def get(self, name: str): with self._lock: e = self._entries.get(name) if e is None: self._misses += 1 return None e["last_used"] = time.time() self._hits += 1 return e["tensor"] def contains(self, name: str) -> bool: with self._lock: return name in self._entries def evict(self, name: str) -> bool: with self._lock: return self._entries.pop(name, None) is not None def evict_device(self, device_str: str) -> int: """Drop LPU-owned entries on one physical device. Returns count.""" with self._lock: names = [n for n, e in self._entries.items() if device_str in e["device"]] for n in names: self._entries.pop(n, None) self._evictions += 1 return len(names) def _evict_locked(self) -> None: if len(self._entries) <= self.max_entries: return ordered = sorted(self._entries.items(), key=lambda kv: kv[1]["last_used"]) for name, _ in ordered[: max(0, len(self._entries) - self.max_entries)]: self._entries.pop(name, None) self._evictions += 1 def device_bytes(self) -> Dict[str, float]: """Cached bytes per device string (e.g. 'cuda:0').""" out: Dict[str, float] = {} with self._lock: for e in self._entries.values(): out[e["device"]] = out.get(e["device"], 0.0) + e["bytes"] return {k: v / (1024 * 1024) for k, v in out.items()} def stats(self) -> dict: with self._lock: return { "entries": len(self._entries), "max_entries": self.max_entries, "hits": self._hits, "misses": self._misses, "evictions": self._evictions, "mb_per_device": self.device_bytes(), } def clear(self) -> int: with self._lock: n = len(self._entries) self._entries.clear() return n def _safe_deepcopy(model): import copy try: return copy.deepcopy(model) except Exception as exc: # noqa: BLE001 raise RuntimeError( f"model is not deep-copyable (needed for safe VRAM probing): {exc}" ) from exc