allcited-api / app /cache.py
suvendu-2006
feat(answer): add conversational greeting handling and sync latest app logic
6240e9b
Raw History Blame Contribute Delete
7.34 kB
"""
Semantic cache module for Allcited.
In-process LRU cache using sentence embedding cosine similarity with a 0.96 threshold.
"""
import logging
import threading
import time
from collections import OrderedDict
from dataclasses import dataclass
from typing import Any, Dict, List, Optional
import numpy as np
from app.config import get_settings
from app.pipeline.embed import get_embedding_manager
logger = logging.getLogger(__name__)
@dataclass
class CacheEntry:
query: str
vector: np.ndarray # Normalized 1D float vector
answer: str
citations: List[Dict[str, Any]]
model: str
refused: bool
similarity: float = 1.0
created_at: float = 0.0
class SemanticCache:
"""
In-process LRU semantic cache.
Stores query embeddings and answers scoped by session_id.
Matches queries with cosine similarity >= threshold (default 0.96).
"""
def __init__(self, max_entries: int = 500, default_threshold: Optional[float] = None):
settings = get_settings()
self.max_entries = max_entries
self.default_threshold = (
default_threshold
if default_threshold is not None
else getattr(settings, "CACHE_THRESHOLD", 0.96)
)
# Mapping: session_id -> OrderedDict[query, CacheEntry]
self._stores: Dict[str, OrderedDict[str, CacheEntry]] = {}
# Mapping: filename -> set[session_id]
self._filename_to_sessions: Dict[str, set[str]] = {}
self._lock = threading.RLock()
self._hits = 0
self._misses = 0
def get(
self,
session_id: str,
query: str,
threshold: Optional[float] = None,
model: Optional[str] = None,
) -> Optional[CacheEntry]:
"""
Check if a semantically equivalent query exists in cache for the session.
If model is specified, only matches entries generated by the same model.
"""
with self._lock:
thresh = threshold if threshold is not None else self.default_threshold
session_cache = self._stores.get(session_id)
if not session_cache:
self._misses += 1
return None
# Embed incoming query
embed_mgr = get_embedding_manager()
query_vec = embed_mgr.embed_query(query)[0]
norm = float(np.linalg.norm(query_vec))
if norm > 0:
query_vec = query_vec / norm
best_entry: Optional[CacheEntry] = None
best_sim: float = -1.0
best_key: Optional[str] = None
for key, entry in session_cache.items():
if model and entry.model != model:
continue
sim = float(np.dot(query_vec, entry.vector))
if sim > best_sim:
best_sim = sim
best_entry = entry
best_key = key
if best_sim >= thresh and best_entry is not None and best_key is not None:
# Move to end (most recently used in LRU)
session_cache.move_to_end(best_key)
self._hits += 1
logger.info(
"Semantic cache HIT (similarity=%.4f >= %.4f) for query: '%s'",
best_sim,
thresh,
query,
)
return CacheEntry(
query=best_entry.query,
vector=best_entry.vector,
answer=best_entry.answer,
citations=best_entry.citations,
model=best_entry.model,
refused=best_entry.refused,
similarity=best_sim,
created_at=best_entry.created_at,
)
self._misses += 1
return None
def set(
self,
session_id: str,
query: str,
answer: str,
citations: List[Dict[str, Any]],
model: str = "",
refused: bool = False,
filename: Optional[str] = None,
) -> None:
"""
Store query and answer in the session cache.
"""
with self._lock:
if session_id not in self._stores:
self._stores[session_id] = OrderedDict()
if filename:
if filename not in self._filename_to_sessions:
self._filename_to_sessions[filename] = set()
self._filename_to_sessions[filename].add(session_id)
session_cache = self._stores[session_id]
embed_mgr = get_embedding_manager()
query_vec = embed_mgr.embed_query(query)[0]
norm = float(np.linalg.norm(query_vec))
if norm > 0:
query_vec = query_vec / norm
entry = CacheEntry(
query=query,
vector=query_vec,
answer=answer,
citations=citations,
model=model,
refused=refused,
similarity=1.0,
created_at=time.time(),
)
session_cache[query] = entry
session_cache.move_to_end(query)
# Evict LRU if total entries exceed max_entries
total_entries = sum(len(c) for c in self._stores.values())
if total_entries > self.max_entries:
for sc in self._stores.values():
if len(sc) > 0:
sc.popitem(last=False)
break
def invalidate(self, session_id: str) -> int:
"""Invalidate all cached entries for a session."""
with self._lock:
if session_id in self._stores:
count = len(self._stores[session_id])
del self._stores[session_id]
logger.info("Invalidated semantic cache for session %s (%d entries removed)", session_id, count)
return count
return 0
def invalidate_filename(self, filename: str) -> int:
"""Invalidate all cached entries for sessions matching a filename."""
with self._lock:
sids = self._filename_to_sessions.pop(filename, set())
total = 0
for sid in sids:
total += self.invalidate(sid)
return total
def clear(self) -> None:
"""Clear the entire cache."""
with self._lock:
self._stores.clear()
self._filename_to_sessions.clear()
self._hits = 0
self._misses = 0
def get_stats(self) -> Dict[str, Any]:
"""Return cache hit rate metrics and entry counts."""
with self._lock:
total_requests = self._hits + self._misses
hit_rate = (self._hits / total_requests) if total_requests > 0 else 0.0
total_entries = sum(len(c) for c in self._stores.values())
return {
"hits": self._hits,
"misses": self._misses,
"hit_rate": round(hit_rate, 4),
"total_entries": total_entries,
}
_SEMANTIC_CACHE: Optional[SemanticCache] = None
def get_semantic_cache() -> SemanticCache:
global _SEMANTIC_CACHE
if _SEMANTIC_CACHE is None:
_SEMANTIC_CACHE = SemanticCache()
return _SEMANTIC_CACHE