Spaces:
Running on Zero
Running on Zero
File size: 7,340 Bytes
11fe22d 6240e9b 11fe22d 6240e9b 11fe22d 6240e9b 11fe22d 6240e9b 11fe22d 6240e9b 11fe22d 6240e9b 11fe22d 6240e9b 11fe22d 6240e9b 11fe22d 6240e9b 11fe22d 6240e9b 11fe22d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 | """
Semantic cache module for Allcited.
In-process LRU cache using sentence embedding cosine similarity with a 0.96 threshold.
"""
import logging
import threading
import time
from collections import OrderedDict
from dataclasses import dataclass
from typing import Any, Dict, List, Optional
import numpy as np
from app.config import get_settings
from app.pipeline.embed import get_embedding_manager
logger = logging.getLogger(__name__)
@dataclass
class CacheEntry:
query: str
vector: np.ndarray # Normalized 1D float vector
answer: str
citations: List[Dict[str, Any]]
model: str
refused: bool
similarity: float = 1.0
created_at: float = 0.0
class SemanticCache:
"""
In-process LRU semantic cache.
Stores query embeddings and answers scoped by session_id.
Matches queries with cosine similarity >= threshold (default 0.96).
"""
def __init__(self, max_entries: int = 500, default_threshold: Optional[float] = None):
settings = get_settings()
self.max_entries = max_entries
self.default_threshold = (
default_threshold
if default_threshold is not None
else getattr(settings, "CACHE_THRESHOLD", 0.96)
)
# Mapping: session_id -> OrderedDict[query, CacheEntry]
self._stores: Dict[str, OrderedDict[str, CacheEntry]] = {}
# Mapping: filename -> set[session_id]
self._filename_to_sessions: Dict[str, set[str]] = {}
self._lock = threading.RLock()
self._hits = 0
self._misses = 0
def get(
self,
session_id: str,
query: str,
threshold: Optional[float] = None,
model: Optional[str] = None,
) -> Optional[CacheEntry]:
"""
Check if a semantically equivalent query exists in cache for the session.
If model is specified, only matches entries generated by the same model.
"""
with self._lock:
thresh = threshold if threshold is not None else self.default_threshold
session_cache = self._stores.get(session_id)
if not session_cache:
self._misses += 1
return None
# Embed incoming query
embed_mgr = get_embedding_manager()
query_vec = embed_mgr.embed_query(query)[0]
norm = float(np.linalg.norm(query_vec))
if norm > 0:
query_vec = query_vec / norm
best_entry: Optional[CacheEntry] = None
best_sim: float = -1.0
best_key: Optional[str] = None
for key, entry in session_cache.items():
if model and entry.model != model:
continue
sim = float(np.dot(query_vec, entry.vector))
if sim > best_sim:
best_sim = sim
best_entry = entry
best_key = key
if best_sim >= thresh and best_entry is not None and best_key is not None:
# Move to end (most recently used in LRU)
session_cache.move_to_end(best_key)
self._hits += 1
logger.info(
"Semantic cache HIT (similarity=%.4f >= %.4f) for query: '%s'",
best_sim,
thresh,
query,
)
return CacheEntry(
query=best_entry.query,
vector=best_entry.vector,
answer=best_entry.answer,
citations=best_entry.citations,
model=best_entry.model,
refused=best_entry.refused,
similarity=best_sim,
created_at=best_entry.created_at,
)
self._misses += 1
return None
def set(
self,
session_id: str,
query: str,
answer: str,
citations: List[Dict[str, Any]],
model: str = "",
refused: bool = False,
filename: Optional[str] = None,
) -> None:
"""
Store query and answer in the session cache.
"""
with self._lock:
if session_id not in self._stores:
self._stores[session_id] = OrderedDict()
if filename:
if filename not in self._filename_to_sessions:
self._filename_to_sessions[filename] = set()
self._filename_to_sessions[filename].add(session_id)
session_cache = self._stores[session_id]
embed_mgr = get_embedding_manager()
query_vec = embed_mgr.embed_query(query)[0]
norm = float(np.linalg.norm(query_vec))
if norm > 0:
query_vec = query_vec / norm
entry = CacheEntry(
query=query,
vector=query_vec,
answer=answer,
citations=citations,
model=model,
refused=refused,
similarity=1.0,
created_at=time.time(),
)
session_cache[query] = entry
session_cache.move_to_end(query)
# Evict LRU if total entries exceed max_entries
total_entries = sum(len(c) for c in self._stores.values())
if total_entries > self.max_entries:
for sc in self._stores.values():
if len(sc) > 0:
sc.popitem(last=False)
break
def invalidate(self, session_id: str) -> int:
"""Invalidate all cached entries for a session."""
with self._lock:
if session_id in self._stores:
count = len(self._stores[session_id])
del self._stores[session_id]
logger.info("Invalidated semantic cache for session %s (%d entries removed)", session_id, count)
return count
return 0
def invalidate_filename(self, filename: str) -> int:
"""Invalidate all cached entries for sessions matching a filename."""
with self._lock:
sids = self._filename_to_sessions.pop(filename, set())
total = 0
for sid in sids:
total += self.invalidate(sid)
return total
def clear(self) -> None:
"""Clear the entire cache."""
with self._lock:
self._stores.clear()
self._filename_to_sessions.clear()
self._hits = 0
self._misses = 0
def get_stats(self) -> Dict[str, Any]:
"""Return cache hit rate metrics and entry counts."""
with self._lock:
total_requests = self._hits + self._misses
hit_rate = (self._hits / total_requests) if total_requests > 0 else 0.0
total_entries = sum(len(c) for c in self._stores.values())
return {
"hits": self._hits,
"misses": self._misses,
"hit_rate": round(hit_rate, 4),
"total_entries": total_entries,
}
_SEMANTIC_CACHE: Optional[SemanticCache] = None
def get_semantic_cache() -> SemanticCache:
global _SEMANTIC_CACHE
if _SEMANTIC_CACHE is None:
_SEMANTIC_CACHE = SemanticCache()
return _SEMANTIC_CACHE
|