Anna Zhou commited on
Commit
76a3d04
·
1 Parent(s): 5517c1f

Use lightweight hosted retrieval by default

Browse files
Files changed (1) hide show
  1. rag/retriever.py +54 -0
rag/retriever.py CHANGED
@@ -10,7 +10,10 @@ Three retrieval modes:
10
  retrieve_for_case() — all chunks for a specific case study ID
11
  """
12
  import json
 
 
13
  import sys
 
14
  from pathlib import Path
15
 
16
  import numpy as np
@@ -19,6 +22,11 @@ sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
19
 
20
  EMBED_MODEL = "all-MiniLM-L6-v2"
21
  PRECOMPUTED_PATH = Path("data/extracted/precomputed_embeddings.json")
 
 
 
 
 
22
 
23
  # Module-level cache — loaded once per process
24
  _store = None
@@ -55,6 +63,14 @@ def _cosine_scores(query_vec):
55
  return _embeddings_np @ q
56
 
57
 
 
 
 
 
 
 
 
 
58
  class MRRetriever:
59
  def __init__(self, n_results=8):
60
  self.model = None
@@ -72,6 +88,9 @@ class MRRetriever:
72
  if not store.get("ids"):
73
  return []
74
 
 
 
 
75
  q_vec = self._get_model().encode(query, convert_to_numpy=True)
76
  scores = _cosine_scores(q_vec)
77
  ranked = np.argsort(-scores)
@@ -90,6 +109,41 @@ class MRRetriever:
90
  break
91
  return output
92
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
93
  def retrieve_all_overviews(self):
94
  """Return one overview/summary chunk per case for broad summary queries."""
95
  store = _load_store()
 
10
  retrieve_for_case() — all chunks for a specific case study ID
11
  """
12
  import json
13
+ import os
14
+ import re
15
  import sys
16
+ from collections import Counter
17
  from pathlib import Path
18
 
19
  import numpy as np
 
22
 
23
  EMBED_MODEL = "all-MiniLM-L6-v2"
24
  PRECOMPUTED_PATH = Path("data/extracted/precomputed_embeddings.json")
25
+ STOPWORDS = {
26
+ "a", "an", "and", "are", "as", "at", "be", "by", "for", "from", "has", "have",
27
+ "how", "in", "is", "it", "of", "on", "or", "that", "the", "their", "this",
28
+ "to", "what", "when", "where", "which", "who", "why", "with", "worked",
29
+ }
30
 
31
  # Module-level cache — loaded once per process
32
  _store = None
 
63
  return _embeddings_np @ q
64
 
65
 
66
+ def _tokens(text):
67
+ return [
68
+ token
69
+ for token in re.findall(r"[a-z0-9]{3,}", text.lower())
70
+ if token not in STOPWORDS
71
+ ]
72
+
73
+
74
  class MRRetriever:
75
  def __init__(self, n_results=8):
76
  self.model = None
 
88
  if not store.get("ids"):
89
  return []
90
 
91
+ if os.getenv("RAG_RETRIEVAL_MODE", "lexical").lower() != "semantic":
92
+ return self._retrieve_lexical(query, filters=filters, max_per_case=max_per_case)
93
+
94
  q_vec = self._get_model().encode(query, convert_to_numpy=True)
95
  scores = _cosine_scores(q_vec)
96
  ranked = np.argsort(-scores)
 
109
  break
110
  return output
111
 
112
+ def _retrieve_lexical(self, query, filters=None, max_per_case=2):
113
+ store = _load_store()
114
+ query_terms = _tokens(query)
115
+ if not query_terms:
116
+ return []
117
+
118
+ query_counts = Counter(query_terms)
119
+ ranked = []
120
+ for idx, (text, metadata) in enumerate(zip(store.get("texts", []), store.get("metadatas", []))):
121
+ if filters and not all(metadata.get(k) == str(v) for k, v in filters.items()):
122
+ continue
123
+ text_l = text.lower()
124
+ text_terms = Counter(_tokens(text_l))
125
+ score = 0.0
126
+ for term, count in query_counts.items():
127
+ if term in text_terms:
128
+ score += min(text_terms[term], 4) * count
129
+ if term in text_l:
130
+ score += 0.75
131
+ if score:
132
+ ranked.append((score, idx))
133
+
134
+ ranked.sort(reverse=True)
135
+ seen = {}
136
+ output = []
137
+ for _, idx in ranked:
138
+ metadata = store["metadatas"][idx]
139
+ cid = metadata.get("case_id", "unknown")
140
+ if seen.get(cid, 0) < max_per_case:
141
+ seen[cid] = seen.get(cid, 0) + 1
142
+ output.append({"text": store["texts"][idx], "metadata": metadata})
143
+ if len(output) >= self.n_results:
144
+ break
145
+ return output
146
+
147
  def retrieve_all_overviews(self):
148
  """Return one overview/summary chunk per case for broad summary queries."""
149
  store = _load_store()