""" Semantic cache module for Allcited. In-process LRU cache using sentence embedding cosine similarity with a 0.96 threshold. """ import logging import threading import time from collections import OrderedDict from dataclasses import dataclass from typing import Any, Dict, List, Optional import numpy as np from app.config import get_settings from app.pipeline.embed import get_embedding_manager logger = logging.getLogger(__name__) @dataclass class CacheEntry: query: str vector: np.ndarray # Normalized 1D float vector answer: str citations: List[Dict[str, Any]] model: str refused: bool similarity: float = 1.0 created_at: float = 0.0 class SemanticCache: """ In-process LRU semantic cache. Stores query embeddings and answers scoped by session_id. Matches queries with cosine similarity >= threshold (default 0.96). """ def __init__(self, max_entries: int = 500, default_threshold: Optional[float] = None): settings = get_settings() self.max_entries = max_entries self.default_threshold = ( default_threshold if default_threshold is not None else getattr(settings, "CACHE_THRESHOLD", 0.96) ) # Mapping: session_id -> OrderedDict[query, CacheEntry] self._stores: Dict[str, OrderedDict[str, CacheEntry]] = {} # Mapping: filename -> set[session_id] self._filename_to_sessions: Dict[str, set[str]] = {} self._lock = threading.RLock() self._hits = 0 self._misses = 0 def get( self, session_id: str, query: str, threshold: Optional[float] = None, model: Optional[str] = None, ) -> Optional[CacheEntry]: """ Check if a semantically equivalent query exists in cache for the session. If model is specified, only matches entries generated by the same model. """ with self._lock: thresh = threshold if threshold is not None else self.default_threshold session_cache = self._stores.get(session_id) if not session_cache: self._misses += 1 return None # Embed incoming query embed_mgr = get_embedding_manager() query_vec = embed_mgr.embed_query(query)[0] norm = float(np.linalg.norm(query_vec)) if norm > 0: query_vec = query_vec / norm best_entry: Optional[CacheEntry] = None best_sim: float = -1.0 best_key: Optional[str] = None for key, entry in session_cache.items(): if model and entry.model != model: continue sim = float(np.dot(query_vec, entry.vector)) if sim > best_sim: best_sim = sim best_entry = entry best_key = key if best_sim >= thresh and best_entry is not None and best_key is not None: # Move to end (most recently used in LRU) session_cache.move_to_end(best_key) self._hits += 1 logger.info( "Semantic cache HIT (similarity=%.4f >= %.4f) for query: '%s'", best_sim, thresh, query, ) return CacheEntry( query=best_entry.query, vector=best_entry.vector, answer=best_entry.answer, citations=best_entry.citations, model=best_entry.model, refused=best_entry.refused, similarity=best_sim, created_at=best_entry.created_at, ) self._misses += 1 return None def set( self, session_id: str, query: str, answer: str, citations: List[Dict[str, Any]], model: str = "", refused: bool = False, filename: Optional[str] = None, ) -> None: """ Store query and answer in the session cache. """ with self._lock: if session_id not in self._stores: self._stores[session_id] = OrderedDict() if filename: if filename not in self._filename_to_sessions: self._filename_to_sessions[filename] = set() self._filename_to_sessions[filename].add(session_id) session_cache = self._stores[session_id] embed_mgr = get_embedding_manager() query_vec = embed_mgr.embed_query(query)[0] norm = float(np.linalg.norm(query_vec)) if norm > 0: query_vec = query_vec / norm entry = CacheEntry( query=query, vector=query_vec, answer=answer, citations=citations, model=model, refused=refused, similarity=1.0, created_at=time.time(), ) session_cache[query] = entry session_cache.move_to_end(query) # Evict LRU if total entries exceed max_entries total_entries = sum(len(c) for c in self._stores.values()) if total_entries > self.max_entries: for sc in self._stores.values(): if len(sc) > 0: sc.popitem(last=False) break def invalidate(self, session_id: str) -> int: """Invalidate all cached entries for a session.""" with self._lock: if session_id in self._stores: count = len(self._stores[session_id]) del self._stores[session_id] logger.info("Invalidated semantic cache for session %s (%d entries removed)", session_id, count) return count return 0 def invalidate_filename(self, filename: str) -> int: """Invalidate all cached entries for sessions matching a filename.""" with self._lock: sids = self._filename_to_sessions.pop(filename, set()) total = 0 for sid in sids: total += self.invalidate(sid) return total def clear(self) -> None: """Clear the entire cache.""" with self._lock: self._stores.clear() self._filename_to_sessions.clear() self._hits = 0 self._misses = 0 def get_stats(self) -> Dict[str, Any]: """Return cache hit rate metrics and entry counts.""" with self._lock: total_requests = self._hits + self._misses hit_rate = (self._hits / total_requests) if total_requests > 0 else 0.0 total_entries = sum(len(c) for c in self._stores.values()) return { "hits": self._hits, "misses": self._misses, "hit_rate": round(hit_rate, 4), "total_entries": total_entries, } _SEMANTIC_CACHE: Optional[SemanticCache] = None def get_semantic_cache() -> SemanticCache: global _SEMANTIC_CACHE if _SEMANTIC_CACHE is None: _SEMANTIC_CACHE = SemanticCache() return _SEMANTIC_CACHE