Spaces:
Running on Zero
Running on Zero
Download app/cache.py from subh-007/allcited-api: direct link, hf CLI and curl.
- Browser
- Download file 7.34 kB
-
https://huggingface.co/spaces/subh-007/allcited-api/resolve/main/app/cache.py
- Command line
-
hf download hf://spaces/subh-007/allcited-api/app/cache.py
-
curl -L -o cache.py https://huggingface.co/spaces/subh-007/allcited-api/resolve/main/app/cache.py
7.34 kB
| """ | |
| Semantic cache module for Allcited. | |
| In-process LRU cache using sentence embedding cosine similarity with a 0.96 threshold. | |
| """ | |
| import logging | |
| import threading | |
| import time | |
| from collections import OrderedDict | |
| from dataclasses import dataclass | |
| from typing import Any, Dict, List, Optional | |
| import numpy as np | |
| from app.config import get_settings | |
| from app.pipeline.embed import get_embedding_manager | |
| logger = logging.getLogger(__name__) | |
| class CacheEntry: | |
| query: str | |
| vector: np.ndarray # Normalized 1D float vector | |
| answer: str | |
| citations: List[Dict[str, Any]] | |
| model: str | |
| refused: bool | |
| similarity: float = 1.0 | |
| created_at: float = 0.0 | |
| class SemanticCache: | |
| """ | |
| In-process LRU semantic cache. | |
| Stores query embeddings and answers scoped by session_id. | |
| Matches queries with cosine similarity >= threshold (default 0.96). | |
| """ | |
| def __init__(self, max_entries: int = 500, default_threshold: Optional[float] = None): | |
| settings = get_settings() | |
| self.max_entries = max_entries | |
| self.default_threshold = ( | |
| default_threshold | |
| if default_threshold is not None | |
| else getattr(settings, "CACHE_THRESHOLD", 0.96) | |
| ) | |
| # Mapping: session_id -> OrderedDict[query, CacheEntry] | |
| self._stores: Dict[str, OrderedDict[str, CacheEntry]] = {} | |
| # Mapping: filename -> set[session_id] | |
| self._filename_to_sessions: Dict[str, set[str]] = {} | |
| self._lock = threading.RLock() | |
| self._hits = 0 | |
| self._misses = 0 | |
| def get( | |
| self, | |
| session_id: str, | |
| query: str, | |
| threshold: Optional[float] = None, | |
| model: Optional[str] = None, | |
| ) -> Optional[CacheEntry]: | |
| """ | |
| Check if a semantically equivalent query exists in cache for the session. | |
| If model is specified, only matches entries generated by the same model. | |
| """ | |
| with self._lock: | |
| thresh = threshold if threshold is not None else self.default_threshold | |
| session_cache = self._stores.get(session_id) | |
| if not session_cache: | |
| self._misses += 1 | |
| return None | |
| # Embed incoming query | |
| embed_mgr = get_embedding_manager() | |
| query_vec = embed_mgr.embed_query(query)[0] | |
| norm = float(np.linalg.norm(query_vec)) | |
| if norm > 0: | |
| query_vec = query_vec / norm | |
| best_entry: Optional[CacheEntry] = None | |
| best_sim: float = -1.0 | |
| best_key: Optional[str] = None | |
| for key, entry in session_cache.items(): | |
| if model and entry.model != model: | |
| continue | |
| sim = float(np.dot(query_vec, entry.vector)) | |
| if sim > best_sim: | |
| best_sim = sim | |
| best_entry = entry | |
| best_key = key | |
| if best_sim >= thresh and best_entry is not None and best_key is not None: | |
| # Move to end (most recently used in LRU) | |
| session_cache.move_to_end(best_key) | |
| self._hits += 1 | |
| logger.info( | |
| "Semantic cache HIT (similarity=%.4f >= %.4f) for query: '%s'", | |
| best_sim, | |
| thresh, | |
| query, | |
| ) | |
| return CacheEntry( | |
| query=best_entry.query, | |
| vector=best_entry.vector, | |
| answer=best_entry.answer, | |
| citations=best_entry.citations, | |
| model=best_entry.model, | |
| refused=best_entry.refused, | |
| similarity=best_sim, | |
| created_at=best_entry.created_at, | |
| ) | |
| self._misses += 1 | |
| return None | |
| def set( | |
| self, | |
| session_id: str, | |
| query: str, | |
| answer: str, | |
| citations: List[Dict[str, Any]], | |
| model: str = "", | |
| refused: bool = False, | |
| filename: Optional[str] = None, | |
| ) -> None: | |
| """ | |
| Store query and answer in the session cache. | |
| """ | |
| with self._lock: | |
| if session_id not in self._stores: | |
| self._stores[session_id] = OrderedDict() | |
| if filename: | |
| if filename not in self._filename_to_sessions: | |
| self._filename_to_sessions[filename] = set() | |
| self._filename_to_sessions[filename].add(session_id) | |
| session_cache = self._stores[session_id] | |
| embed_mgr = get_embedding_manager() | |
| query_vec = embed_mgr.embed_query(query)[0] | |
| norm = float(np.linalg.norm(query_vec)) | |
| if norm > 0: | |
| query_vec = query_vec / norm | |
| entry = CacheEntry( | |
| query=query, | |
| vector=query_vec, | |
| answer=answer, | |
| citations=citations, | |
| model=model, | |
| refused=refused, | |
| similarity=1.0, | |
| created_at=time.time(), | |
| ) | |
| session_cache[query] = entry | |
| session_cache.move_to_end(query) | |
| # Evict LRU if total entries exceed max_entries | |
| total_entries = sum(len(c) for c in self._stores.values()) | |
| if total_entries > self.max_entries: | |
| for sc in self._stores.values(): | |
| if len(sc) > 0: | |
| sc.popitem(last=False) | |
| break | |
| def invalidate(self, session_id: str) -> int: | |
| """Invalidate all cached entries for a session.""" | |
| with self._lock: | |
| if session_id in self._stores: | |
| count = len(self._stores[session_id]) | |
| del self._stores[session_id] | |
| logger.info("Invalidated semantic cache for session %s (%d entries removed)", session_id, count) | |
| return count | |
| return 0 | |
| def invalidate_filename(self, filename: str) -> int: | |
| """Invalidate all cached entries for sessions matching a filename.""" | |
| with self._lock: | |
| sids = self._filename_to_sessions.pop(filename, set()) | |
| total = 0 | |
| for sid in sids: | |
| total += self.invalidate(sid) | |
| return total | |
| def clear(self) -> None: | |
| """Clear the entire cache.""" | |
| with self._lock: | |
| self._stores.clear() | |
| self._filename_to_sessions.clear() | |
| self._hits = 0 | |
| self._misses = 0 | |
| def get_stats(self) -> Dict[str, Any]: | |
| """Return cache hit rate metrics and entry counts.""" | |
| with self._lock: | |
| total_requests = self._hits + self._misses | |
| hit_rate = (self._hits / total_requests) if total_requests > 0 else 0.0 | |
| total_entries = sum(len(c) for c in self._stores.values()) | |
| return { | |
| "hits": self._hits, | |
| "misses": self._misses, | |
| "hit_rate": round(hit_rate, 4), | |
| "total_entries": total_entries, | |
| } | |
| _SEMANTIC_CACHE: Optional[SemanticCache] = None | |
| def get_semantic_cache() -> SemanticCache: | |
| global _SEMANTIC_CACHE | |
| if _SEMANTIC_CACHE is None: | |
| _SEMANTIC_CACHE = SemanticCache() | |
| return _SEMANTIC_CACHE | |