Create modelManager.py
Browse files- modelManager.py +73 -0
modelManager.py
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# modelManager.py - Model Cache Manager
|
| 2 |
+
# PRODUCTION HARDENED
|
| 3 |
+
|
| 4 |
+
import time
|
| 5 |
+
import gc
|
| 6 |
+
import psutil
|
| 7 |
+
from typing import Dict, Optional, Any
|
| 8 |
+
|
| 9 |
+
class ModelManager:
|
| 10 |
+
"""Memory-aware model management with LRU cache."""
|
| 11 |
+
|
| 12 |
+
def __init__(self, max_models: int = 3, memory_limit_mb: int = 6000):
|
| 13 |
+
self.models: Dict[str, Any] = {}
|
| 14 |
+
self.load_times: Dict[str, float] = {}
|
| 15 |
+
self.use_counts: Dict[str, int] = {}
|
| 16 |
+
self.max_models = max_models
|
| 17 |
+
self.memory_limit_mb = memory_limit_mb
|
| 18 |
+
|
| 19 |
+
def get_model(self, name: str, loader_func) -> Optional[Any]:
|
| 20 |
+
"""Get or load a model with caching."""
|
| 21 |
+
# Check if already loaded
|
| 22 |
+
if name in self.models:
|
| 23 |
+
self.use_counts[name] = self.use_counts.get(name, 0) + 1
|
| 24 |
+
return self.models[name]
|
| 25 |
+
|
| 26 |
+
# Check memory before loading
|
| 27 |
+
try:
|
| 28 |
+
mem = psutil.virtual_memory()
|
| 29 |
+
if mem.percent > 85:
|
| 30 |
+
self._evict_models()
|
| 31 |
+
except:
|
| 32 |
+
pass
|
| 33 |
+
|
| 34 |
+
# Load model
|
| 35 |
+
start = time.time()
|
| 36 |
+
try:
|
| 37 |
+
session = loader_func(name)
|
| 38 |
+
load_time = time.time() - start
|
| 39 |
+
self.models[name] = session
|
| 40 |
+
self.load_times[name] = load_time
|
| 41 |
+
self.use_counts[name] = 1
|
| 42 |
+
|
| 43 |
+
# Evict if over limit
|
| 44 |
+
if len(self.models) > self.max_models:
|
| 45 |
+
self._evict_models()
|
| 46 |
+
|
| 47 |
+
return session
|
| 48 |
+
except Exception as e:
|
| 49 |
+
return None
|
| 50 |
+
|
| 51 |
+
def _evict_models(self):
|
| 52 |
+
"""Evict least recently used models."""
|
| 53 |
+
if len(self.models) <= 1:
|
| 54 |
+
return
|
| 55 |
+
|
| 56 |
+
sorted_models = sorted(
|
| 57 |
+
self.models.keys(),
|
| 58 |
+
key=lambda x: self.use_counts.get(x, 0)
|
| 59 |
+
)
|
| 60 |
+
|
| 61 |
+
to_remove = sorted_models[0]
|
| 62 |
+
del self.models[to_remove]
|
| 63 |
+
if to_remove in self.use_counts:
|
| 64 |
+
del self.use_counts[to_remove]
|
| 65 |
+
gc.collect()
|
| 66 |
+
|
| 67 |
+
def get_stats(self) -> Dict:
|
| 68 |
+
return {
|
| 69 |
+
'loaded_models': list(self.models.keys()),
|
| 70 |
+
'model_count': len(self.models),
|
| 71 |
+
'use_counts': self.use_counts,
|
| 72 |
+
'load_times': self.load_times
|
| 73 |
+
}
|