Spaces:
Sleeping
Sleeping
Anna Zhou commited on
Commit ·
76a3d04
1
Parent(s): 5517c1f
Use lightweight hosted retrieval by default
Browse files- rag/retriever.py +54 -0
rag/retriever.py
CHANGED
|
@@ -10,7 +10,10 @@ Three retrieval modes:
|
|
| 10 |
retrieve_for_case() — all chunks for a specific case study ID
|
| 11 |
"""
|
| 12 |
import json
|
|
|
|
|
|
|
| 13 |
import sys
|
|
|
|
| 14 |
from pathlib import Path
|
| 15 |
|
| 16 |
import numpy as np
|
|
@@ -19,6 +22,11 @@ sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
| 19 |
|
| 20 |
EMBED_MODEL = "all-MiniLM-L6-v2"
|
| 21 |
PRECOMPUTED_PATH = Path("data/extracted/precomputed_embeddings.json")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 22 |
|
| 23 |
# Module-level cache — loaded once per process
|
| 24 |
_store = None
|
|
@@ -55,6 +63,14 @@ def _cosine_scores(query_vec):
|
|
| 55 |
return _embeddings_np @ q
|
| 56 |
|
| 57 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 58 |
class MRRetriever:
|
| 59 |
def __init__(self, n_results=8):
|
| 60 |
self.model = None
|
|
@@ -72,6 +88,9 @@ class MRRetriever:
|
|
| 72 |
if not store.get("ids"):
|
| 73 |
return []
|
| 74 |
|
|
|
|
|
|
|
|
|
|
| 75 |
q_vec = self._get_model().encode(query, convert_to_numpy=True)
|
| 76 |
scores = _cosine_scores(q_vec)
|
| 77 |
ranked = np.argsort(-scores)
|
|
@@ -90,6 +109,41 @@ class MRRetriever:
|
|
| 90 |
break
|
| 91 |
return output
|
| 92 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 93 |
def retrieve_all_overviews(self):
|
| 94 |
"""Return one overview/summary chunk per case for broad summary queries."""
|
| 95 |
store = _load_store()
|
|
|
|
| 10 |
retrieve_for_case() — all chunks for a specific case study ID
|
| 11 |
"""
|
| 12 |
import json
|
| 13 |
+
import os
|
| 14 |
+
import re
|
| 15 |
import sys
|
| 16 |
+
from collections import Counter
|
| 17 |
from pathlib import Path
|
| 18 |
|
| 19 |
import numpy as np
|
|
|
|
| 22 |
|
| 23 |
EMBED_MODEL = "all-MiniLM-L6-v2"
|
| 24 |
PRECOMPUTED_PATH = Path("data/extracted/precomputed_embeddings.json")
|
| 25 |
+
STOPWORDS = {
|
| 26 |
+
"a", "an", "and", "are", "as", "at", "be", "by", "for", "from", "has", "have",
|
| 27 |
+
"how", "in", "is", "it", "of", "on", "or", "that", "the", "their", "this",
|
| 28 |
+
"to", "what", "when", "where", "which", "who", "why", "with", "worked",
|
| 29 |
+
}
|
| 30 |
|
| 31 |
# Module-level cache — loaded once per process
|
| 32 |
_store = None
|
|
|
|
| 63 |
return _embeddings_np @ q
|
| 64 |
|
| 65 |
|
| 66 |
+
def _tokens(text):
|
| 67 |
+
return [
|
| 68 |
+
token
|
| 69 |
+
for token in re.findall(r"[a-z0-9]{3,}", text.lower())
|
| 70 |
+
if token not in STOPWORDS
|
| 71 |
+
]
|
| 72 |
+
|
| 73 |
+
|
| 74 |
class MRRetriever:
|
| 75 |
def __init__(self, n_results=8):
|
| 76 |
self.model = None
|
|
|
|
| 88 |
if not store.get("ids"):
|
| 89 |
return []
|
| 90 |
|
| 91 |
+
if os.getenv("RAG_RETRIEVAL_MODE", "lexical").lower() != "semantic":
|
| 92 |
+
return self._retrieve_lexical(query, filters=filters, max_per_case=max_per_case)
|
| 93 |
+
|
| 94 |
q_vec = self._get_model().encode(query, convert_to_numpy=True)
|
| 95 |
scores = _cosine_scores(q_vec)
|
| 96 |
ranked = np.argsort(-scores)
|
|
|
|
| 109 |
break
|
| 110 |
return output
|
| 111 |
|
| 112 |
+
def _retrieve_lexical(self, query, filters=None, max_per_case=2):
|
| 113 |
+
store = _load_store()
|
| 114 |
+
query_terms = _tokens(query)
|
| 115 |
+
if not query_terms:
|
| 116 |
+
return []
|
| 117 |
+
|
| 118 |
+
query_counts = Counter(query_terms)
|
| 119 |
+
ranked = []
|
| 120 |
+
for idx, (text, metadata) in enumerate(zip(store.get("texts", []), store.get("metadatas", []))):
|
| 121 |
+
if filters and not all(metadata.get(k) == str(v) for k, v in filters.items()):
|
| 122 |
+
continue
|
| 123 |
+
text_l = text.lower()
|
| 124 |
+
text_terms = Counter(_tokens(text_l))
|
| 125 |
+
score = 0.0
|
| 126 |
+
for term, count in query_counts.items():
|
| 127 |
+
if term in text_terms:
|
| 128 |
+
score += min(text_terms[term], 4) * count
|
| 129 |
+
if term in text_l:
|
| 130 |
+
score += 0.75
|
| 131 |
+
if score:
|
| 132 |
+
ranked.append((score, idx))
|
| 133 |
+
|
| 134 |
+
ranked.sort(reverse=True)
|
| 135 |
+
seen = {}
|
| 136 |
+
output = []
|
| 137 |
+
for _, idx in ranked:
|
| 138 |
+
metadata = store["metadatas"][idx]
|
| 139 |
+
cid = metadata.get("case_id", "unknown")
|
| 140 |
+
if seen.get(cid, 0) < max_per_case:
|
| 141 |
+
seen[cid] = seen.get(cid, 0) + 1
|
| 142 |
+
output.append({"text": store["texts"][idx], "metadata": metadata})
|
| 143 |
+
if len(output) >= self.n_results:
|
| 144 |
+
break
|
| 145 |
+
return output
|
| 146 |
+
|
| 147 |
def retrieve_all_overviews(self):
|
| 148 |
"""Return one overview/summary chunk per case for broad summary queries."""
|
| 149 |
store = _load_store()
|