File size: 3,619 Bytes
fbb4123 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 | """Vector index + lexical retriever.
v2 has no pretrained transformers on the target device, so the vector index is a
deterministic hash embedding (no download required). Retrieval ranks beliefs by a
weighted blend of cosine(vector) and BM25(lexical overlap) so "price of item X"
matches by subject words, not by flaky embedding similarity.
"""
import hashlib
import re
import numpy as np
TOKEN_RE = re.compile(r"[a-z0-9]+")
STOPWORDS = {
"what", "is", "are", "was", "were", "the", "a", "an", "of", "in", "to",
"for", "and", "or", "it", "this", "that", "my", "your", "by", "on", "at"
}
# Stable per-process hash (avoid Python's salted hash). Use a small vector dim.
DIM = 384
def tokenize(text: str):
words = TOKEN_RE.findall(text.lower())
filtered = [w for w in words if w not in STOPWORDS]
return filtered if filtered else words
def _h(word: str, dim: int = DIM) -> int:
return int(hashlib.md5(word.encode("utf-8")).hexdigest(), 16) % dim
def embed(text: str, dim: int = DIM) -> np.ndarray:
"""Deterministic dense vector from word hashes; zero vector for empty text."""
vec = np.zeros(dim, dtype=np.float32)
ids = tokenize(text)
if not ids:
return vec
counts = {}
for w in ids:
counts[w] = counts.get(w, 0) + 1
n = float(len(ids))
for w, c in counts.items():
d = _h(w)
vec[d] += c / (n ** 0.5) # length-normalise-ish
return vec
class BM25:
"""Standard BM25 over a small corpus (pure numpy, no extra deps)."""
def __init__(self, docs):
self.docs = docs # list of (id, text)
self.df = {}
self.avgdl = 0.0
self.N = len(docs)
for _, text in self.docs:
for w in set(tokenize(text)):
self.df[w] = self.df.get(w, 0) + 1
self.avgdl = sum(len(tokenize(t)) for _, t in self.docs) / max(1, self.N)
self.idf = {w: np.log((self.N - df + 0.5) / (df + 0.5) + 1.0) for w, df in self.df.items()}
def scores(self, query: str, k: int = 8):
qf = {}
for w in tokenize(query):
qf[w] = qf.get(w, 0) + 1
if not qf:
return np.zeros(self.N, dtype=np.float64)
scores = np.zeros(self.N, dtype=np.float64)
for i, (_, text) in enumerate(self.docs):
freq = {}
for w in tokenize(text):
freq[w] = freq.get(w, 0) + 1
dl = len(tokenize(text))
score = 0.0
for w, q in qf.items():
if w not in self.df:
continue
f = freq.get(w, 0)
if f == 0:
continue
x = (f * (self.k1 + 1)) / (f + self.k1 * (1 - self.k1 + self.k2 * dl / self.avgdl))
score += self.idf[w] * x * q
scores[i] = score
return scores
k1 = 1.5
k2 = 0.75
def cosine(a: np.ndarray, b: np.ndarray) -> float:
denom = max(1e-9, np.linalg.norm(a) * np.linalg.norm(b))
return float(a @ b / denom)
class VectorIndex:
"""Brute-force cosine over a matrix of embeddings (fine under ~50k docs)."""
def __init__(self, vectors: np.ndarray):
self.vectors = vectors.astype(np.float32)
self.row_norms = np.linalg.norm(self.vectors, axis=1, keepdims=True)
self.row_norms[self.row_norms == 0] = 1.0
def search(self, query_vec: np.ndarray, k: int = 8):
q = query_vec / (np.linalg.norm(query_vec) + 1e-9)
dots = self.vectors @ q
n = self.row_norms.ravel()
cos = dots / n
idx = np.argsort(cos)[::-1][:k]
return idx, cos[idx]
|