File size: 7,340 Bytes
11fe22d
 
 
 
 
 
6240e9b
11fe22d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6240e9b
11fe22d
 
 
 
 
 
 
 
 
 
 
 
 
 
6240e9b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
11fe22d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6240e9b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
11fe22d
6240e9b
 
11fe22d
6240e9b
 
 
 
 
 
 
11fe22d
 
 
6240e9b
 
 
 
 
 
 
11fe22d
 
 
6240e9b
 
 
 
 
 
11fe22d
 
 
6240e9b
 
 
 
 
11fe22d
 
 
6240e9b
 
 
 
 
 
 
 
 
 
11fe22d
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
"""
Semantic cache module for Allcited.
In-process LRU cache using sentence embedding cosine similarity with a 0.96 threshold.
"""

import logging
import threading
import time
from collections import OrderedDict
from dataclasses import dataclass
from typing import Any, Dict, List, Optional

import numpy as np

from app.config import get_settings
from app.pipeline.embed import get_embedding_manager

logger = logging.getLogger(__name__)


@dataclass
class CacheEntry:
    query: str
    vector: np.ndarray  # Normalized 1D float vector
    answer: str
    citations: List[Dict[str, Any]]
    model: str
    refused: bool
    similarity: float = 1.0
    created_at: float = 0.0


class SemanticCache:
    """
    In-process LRU semantic cache.
    Stores query embeddings and answers scoped by session_id.
    Matches queries with cosine similarity >= threshold (default 0.96).
    """

    def __init__(self, max_entries: int = 500, default_threshold: Optional[float] = None):
        settings = get_settings()
        self.max_entries = max_entries
        self.default_threshold = (
            default_threshold
            if default_threshold is not None
            else getattr(settings, "CACHE_THRESHOLD", 0.96)
        )
        # Mapping: session_id -> OrderedDict[query, CacheEntry]
        self._stores: Dict[str, OrderedDict[str, CacheEntry]] = {}
        # Mapping: filename -> set[session_id]
        self._filename_to_sessions: Dict[str, set[str]] = {}
        self._lock = threading.RLock()
        self._hits = 0
        self._misses = 0

    def get(
        self,
        session_id: str,
        query: str,
        threshold: Optional[float] = None,
        model: Optional[str] = None,
    ) -> Optional[CacheEntry]:
        """
        Check if a semantically equivalent query exists in cache for the session.
        If model is specified, only matches entries generated by the same model.
        """
        with self._lock:
            thresh = threshold if threshold is not None else self.default_threshold
            session_cache = self._stores.get(session_id)
            if not session_cache:
                self._misses += 1
                return None

            # Embed incoming query
            embed_mgr = get_embedding_manager()
            query_vec = embed_mgr.embed_query(query)[0]
            norm = float(np.linalg.norm(query_vec))
            if norm > 0:
                query_vec = query_vec / norm

            best_entry: Optional[CacheEntry] = None
            best_sim: float = -1.0
            best_key: Optional[str] = None

            for key, entry in session_cache.items():
                if model and entry.model != model:
                    continue
                sim = float(np.dot(query_vec, entry.vector))
                if sim > best_sim:
                    best_sim = sim
                    best_entry = entry
                    best_key = key

            if best_sim >= thresh and best_entry is not None and best_key is not None:
                # Move to end (most recently used in LRU)
                session_cache.move_to_end(best_key)
                self._hits += 1
                logger.info(
                    "Semantic cache HIT (similarity=%.4f >= %.4f) for query: '%s'",
                    best_sim,
                    thresh,
                    query,
                )
                return CacheEntry(
                    query=best_entry.query,
                    vector=best_entry.vector,
                    answer=best_entry.answer,
                    citations=best_entry.citations,
                    model=best_entry.model,
                    refused=best_entry.refused,
                    similarity=best_sim,
                    created_at=best_entry.created_at,
                )

            self._misses += 1
            return None

    def set(
        self,
        session_id: str,
        query: str,
        answer: str,
        citations: List[Dict[str, Any]],
        model: str = "",
        refused: bool = False,
        filename: Optional[str] = None,
    ) -> None:
        """
        Store query and answer in the session cache.
        """
        with self._lock:
            if session_id not in self._stores:
                self._stores[session_id] = OrderedDict()

            if filename:
                if filename not in self._filename_to_sessions:
                    self._filename_to_sessions[filename] = set()
                self._filename_to_sessions[filename].add(session_id)

            session_cache = self._stores[session_id]

            embed_mgr = get_embedding_manager()
            query_vec = embed_mgr.embed_query(query)[0]
            norm = float(np.linalg.norm(query_vec))
            if norm > 0:
                query_vec = query_vec / norm

            entry = CacheEntry(
                query=query,
                vector=query_vec,
                answer=answer,
                citations=citations,
                model=model,
                refused=refused,
                similarity=1.0,
                created_at=time.time(),
            )

            session_cache[query] = entry
            session_cache.move_to_end(query)

            # Evict LRU if total entries exceed max_entries
            total_entries = sum(len(c) for c in self._stores.values())
            if total_entries > self.max_entries:
                for sc in self._stores.values():
                    if len(sc) > 0:
                        sc.popitem(last=False)
                        break

    def invalidate(self, session_id: str) -> int:
        """Invalidate all cached entries for a session."""
        with self._lock:
            if session_id in self._stores:
                count = len(self._stores[session_id])
                del self._stores[session_id]
                logger.info("Invalidated semantic cache for session %s (%d entries removed)", session_id, count)
                return count
            return 0

    def invalidate_filename(self, filename: str) -> int:
        """Invalidate all cached entries for sessions matching a filename."""
        with self._lock:
            sids = self._filename_to_sessions.pop(filename, set())
            total = 0
            for sid in sids:
                total += self.invalidate(sid)
            return total

    def clear(self) -> None:
        """Clear the entire cache."""
        with self._lock:
            self._stores.clear()
            self._filename_to_sessions.clear()
            self._hits = 0
            self._misses = 0

    def get_stats(self) -> Dict[str, Any]:
        """Return cache hit rate metrics and entry counts."""
        with self._lock:
            total_requests = self._hits + self._misses
            hit_rate = (self._hits / total_requests) if total_requests > 0 else 0.0
            total_entries = sum(len(c) for c in self._stores.values())
            return {
                "hits": self._hits,
                "misses": self._misses,
                "hit_rate": round(hit_rate, 4),
                "total_entries": total_entries,
            }


_SEMANTIC_CACHE: Optional[SemanticCache] = None


def get_semantic_cache() -> SemanticCache:
    global _SEMANTIC_CACHE
    if _SEMANTIC_CACHE is None:
        _SEMANTIC_CACHE = SemanticCache()
    return _SEMANTIC_CACHE