"""Vector index + lexical retriever. v2 has no pretrained transformers on the target device, so the vector index is a deterministic hash embedding (no download required). Retrieval ranks beliefs by a weighted blend of cosine(vector) and BM25(lexical overlap) so "price of item X" matches by subject words, not by flaky embedding similarity. """ import hashlib import re import numpy as np TOKEN_RE = re.compile(r"[a-z0-9]+") STOPWORDS = { "what", "is", "are", "was", "were", "the", "a", "an", "of", "in", "to", "for", "and", "or", "it", "this", "that", "my", "your", "by", "on", "at" } # Stable per-process hash (avoid Python's salted hash). Use a small vector dim. DIM = 384 def tokenize(text: str): words = TOKEN_RE.findall(text.lower()) filtered = [w for w in words if w not in STOPWORDS] return filtered if filtered else words def _h(word: str, dim: int = DIM) -> int: return int(hashlib.md5(word.encode("utf-8")).hexdigest(), 16) % dim def embed(text: str, dim: int = DIM) -> np.ndarray: """Deterministic dense vector from word hashes; zero vector for empty text.""" vec = np.zeros(dim, dtype=np.float32) ids = tokenize(text) if not ids: return vec counts = {} for w in ids: counts[w] = counts.get(w, 0) + 1 n = float(len(ids)) for w, c in counts.items(): d = _h(w) vec[d] += c / (n ** 0.5) # length-normalise-ish return vec class BM25: """Standard BM25 over a small corpus (pure numpy, no extra deps).""" def __init__(self, docs): self.docs = docs # list of (id, text) self.df = {} self.avgdl = 0.0 self.N = len(docs) for _, text in self.docs: for w in set(tokenize(text)): self.df[w] = self.df.get(w, 0) + 1 self.avgdl = sum(len(tokenize(t)) for _, t in self.docs) / max(1, self.N) self.idf = {w: np.log((self.N - df + 0.5) / (df + 0.5) + 1.0) for w, df in self.df.items()} def scores(self, query: str, k: int = 8): qf = {} for w in tokenize(query): qf[w] = qf.get(w, 0) + 1 if not qf: return np.zeros(self.N, dtype=np.float64) scores = np.zeros(self.N, dtype=np.float64) for i, (_, text) in enumerate(self.docs): freq = {} for w in tokenize(text): freq[w] = freq.get(w, 0) + 1 dl = len(tokenize(text)) score = 0.0 for w, q in qf.items(): if w not in self.df: continue f = freq.get(w, 0) if f == 0: continue x = (f * (self.k1 + 1)) / (f + self.k1 * (1 - self.k1 + self.k2 * dl / self.avgdl)) score += self.idf[w] * x * q scores[i] = score return scores k1 = 1.5 k2 = 0.75 def cosine(a: np.ndarray, b: np.ndarray) -> float: denom = max(1e-9, np.linalg.norm(a) * np.linalg.norm(b)) return float(a @ b / denom) class VectorIndex: """Brute-force cosine over a matrix of embeddings (fine under ~50k docs).""" def __init__(self, vectors: np.ndarray): self.vectors = vectors.astype(np.float32) self.row_norms = np.linalg.norm(self.vectors, axis=1, keepdims=True) self.row_norms[self.row_norms == 0] = 1.0 def search(self, query_vec: np.ndarray, k: int = 8): q = query_vec / (np.linalg.norm(query_vec) + 1e-9) dots = self.vectors @ q n = self.row_norms.ravel() cos = dots / n idx = np.argsort(cos)[::-1][:k] return idx, cos[idx]