Spaces:
Running
Running
Download lpu/memory/tensor_cache.py from iparaco/Virtual-LPU: direct link, hf CLI and curl.
- Browser
- Download file 3.78 kB
-
https://huggingface.co/spaces/iparaco/Virtual-LPU/resolve/refs%2Fpr%2F2/lpu/memory/tensor_cache.py
- Command line
-
hf download hf://spaces/iparaco/Virtual-LPU@refs/pr/2/lpu/memory/tensor_cache.py
-
curl -L -o tensor_cache.py https://huggingface.co/spaces/iparaco/Virtual-LPU/resolve/refs%2Fpr%2F2/lpu/memory/tensor_cache.py
3.78 kB
| """Named tensor cache with ownership tags and per-device accounting. | |
| Only LPU-owned tensors live here (probe results, benchmark copies, prefetch | |
| buffers). The cache NEVER touches user model tensors unless the user | |
| explicitly registers them. Eviction is explicit and reversible by design: | |
| entries are (name -> {tensor, device, created, last_used, bytes}). | |
| """ | |
| from __future__ import annotations | |
| import threading | |
| import time | |
| from typing import Dict, List, Optional | |
| from ..compat import require_torch | |
| class TensorCache: | |
| def __init__(self, owner: str = "lpu", max_entries: int = 256): | |
| self.owner = owner | |
| self.max_entries = max_entries | |
| self._entries: Dict[str, dict] = {} | |
| self._lock = threading.RLock() | |
| self._hits = 0 | |
| self._misses = 0 | |
| self._evictions = 0 | |
| def put(self, name: str, tensor, note: str = "") -> None: | |
| torch = require_torch() | |
| with self._lock: | |
| self._entries[name] = { | |
| "tensor": tensor, | |
| "device": str(getattr(tensor, "device", "?")), | |
| "created": time.time(), | |
| "last_used": time.time(), | |
| "bytes": ( | |
| tensor.numel() * tensor.element_size() | |
| if hasattr(tensor, "element_size") | |
| else 0 | |
| ), | |
| "note": note, | |
| } | |
| self._evict_locked() | |
| def get(self, name: str): | |
| with self._lock: | |
| e = self._entries.get(name) | |
| if e is None: | |
| self._misses += 1 | |
| return None | |
| e["last_used"] = time.time() | |
| self._hits += 1 | |
| return e["tensor"] | |
| def contains(self, name: str) -> bool: | |
| with self._lock: | |
| return name in self._entries | |
| def evict(self, name: str) -> bool: | |
| with self._lock: | |
| return self._entries.pop(name, None) is not None | |
| def evict_device(self, device_str: str) -> int: | |
| """Drop LPU-owned entries on one physical device. Returns count.""" | |
| with self._lock: | |
| names = [n for n, e in self._entries.items() if device_str in e["device"]] | |
| for n in names: | |
| self._entries.pop(n, None) | |
| self._evictions += 1 | |
| return len(names) | |
| def _evict_locked(self) -> None: | |
| if len(self._entries) <= self.max_entries: | |
| return | |
| ordered = sorted(self._entries.items(), key=lambda kv: kv[1]["last_used"]) | |
| for name, _ in ordered[: max(0, len(self._entries) - self.max_entries)]: | |
| self._entries.pop(name, None) | |
| self._evictions += 1 | |
| def device_bytes(self) -> Dict[str, float]: | |
| """Cached bytes per device string (e.g. 'cuda:0').""" | |
| out: Dict[str, float] = {} | |
| with self._lock: | |
| for e in self._entries.values(): | |
| out[e["device"]] = out.get(e["device"], 0.0) + e["bytes"] | |
| return {k: v / (1024 * 1024) for k, v in out.items()} | |
| def stats(self) -> dict: | |
| with self._lock: | |
| return { | |
| "entries": len(self._entries), | |
| "max_entries": self.max_entries, | |
| "hits": self._hits, | |
| "misses": self._misses, | |
| "evictions": self._evictions, | |
| "mb_per_device": self.device_bytes(), | |
| } | |
| def clear(self) -> int: | |
| with self._lock: | |
| n = len(self._entries) | |
| self._entries.clear() | |
| return n | |
| def _safe_deepcopy(model): | |
| import copy | |
| try: | |
| return copy.deepcopy(model) | |
| except Exception as exc: # noqa: BLE001 | |
| raise RuntimeError( | |
| f"model is not deep-copyable (needed for safe VRAM probing): {exc}" | |
| ) from exc | |