andevs commited on
Commit
0e757f3
·
verified ·
1 Parent(s): c5c0b4b

Create modelManager.py

Browse files
Files changed (1) hide show
  1. modelManager.py +73 -0
modelManager.py ADDED
@@ -0,0 +1,73 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # modelManager.py - Model Cache Manager
2
+ # PRODUCTION HARDENED
3
+
4
+ import time
5
+ import gc
6
+ import psutil
7
+ from typing import Dict, Optional, Any
8
+
9
+ class ModelManager:
10
+ """Memory-aware model management with LRU cache."""
11
+
12
+ def __init__(self, max_models: int = 3, memory_limit_mb: int = 6000):
13
+ self.models: Dict[str, Any] = {}
14
+ self.load_times: Dict[str, float] = {}
15
+ self.use_counts: Dict[str, int] = {}
16
+ self.max_models = max_models
17
+ self.memory_limit_mb = memory_limit_mb
18
+
19
+ def get_model(self, name: str, loader_func) -> Optional[Any]:
20
+ """Get or load a model with caching."""
21
+ # Check if already loaded
22
+ if name in self.models:
23
+ self.use_counts[name] = self.use_counts.get(name, 0) + 1
24
+ return self.models[name]
25
+
26
+ # Check memory before loading
27
+ try:
28
+ mem = psutil.virtual_memory()
29
+ if mem.percent > 85:
30
+ self._evict_models()
31
+ except:
32
+ pass
33
+
34
+ # Load model
35
+ start = time.time()
36
+ try:
37
+ session = loader_func(name)
38
+ load_time = time.time() - start
39
+ self.models[name] = session
40
+ self.load_times[name] = load_time
41
+ self.use_counts[name] = 1
42
+
43
+ # Evict if over limit
44
+ if len(self.models) > self.max_models:
45
+ self._evict_models()
46
+
47
+ return session
48
+ except Exception as e:
49
+ return None
50
+
51
+ def _evict_models(self):
52
+ """Evict least recently used models."""
53
+ if len(self.models) <= 1:
54
+ return
55
+
56
+ sorted_models = sorted(
57
+ self.models.keys(),
58
+ key=lambda x: self.use_counts.get(x, 0)
59
+ )
60
+
61
+ to_remove = sorted_models[0]
62
+ del self.models[to_remove]
63
+ if to_remove in self.use_counts:
64
+ del self.use_counts[to_remove]
65
+ gc.collect()
66
+
67
+ def get_stats(self) -> Dict:
68
+ return {
69
+ 'loaded_models': list(self.models.keys()),
70
+ 'model_count': len(self.models),
71
+ 'use_counts': self.use_counts,
72
+ 'load_times': self.load_times
73
+ }