HumpBack / humpback2 /embedding.py
gregrsea's picture
v2: implement belief memory, rule-core emulator, API and tests; delete v1
fbb4123
Raw History Blame Contribute Delete
3.62 kB
"""Vector index + lexical retriever.
v2 has no pretrained transformers on the target device, so the vector index is a
deterministic hash embedding (no download required). Retrieval ranks beliefs by a
weighted blend of cosine(vector) and BM25(lexical overlap) so "price of item X"
matches by subject words, not by flaky embedding similarity.
"""
import hashlib
import re
import numpy as np
TOKEN_RE = re.compile(r"[a-z0-9]+")
STOPWORDS = {
"what", "is", "are", "was", "were", "the", "a", "an", "of", "in", "to",
"for", "and", "or", "it", "this", "that", "my", "your", "by", "on", "at"
}
# Stable per-process hash (avoid Python's salted hash). Use a small vector dim.
DIM = 384
def tokenize(text: str):
words = TOKEN_RE.findall(text.lower())
filtered = [w for w in words if w not in STOPWORDS]
return filtered if filtered else words
def _h(word: str, dim: int = DIM) -> int:
return int(hashlib.md5(word.encode("utf-8")).hexdigest(), 16) % dim
def embed(text: str, dim: int = DIM) -> np.ndarray:
"""Deterministic dense vector from word hashes; zero vector for empty text."""
vec = np.zeros(dim, dtype=np.float32)
ids = tokenize(text)
if not ids:
return vec
counts = {}
for w in ids:
counts[w] = counts.get(w, 0) + 1
n = float(len(ids))
for w, c in counts.items():
d = _h(w)
vec[d] += c / (n ** 0.5) # length-normalise-ish
return vec
class BM25:
"""Standard BM25 over a small corpus (pure numpy, no extra deps)."""
def __init__(self, docs):
self.docs = docs # list of (id, text)
self.df = {}
self.avgdl = 0.0
self.N = len(docs)
for _, text in self.docs:
for w in set(tokenize(text)):
self.df[w] = self.df.get(w, 0) + 1
self.avgdl = sum(len(tokenize(t)) for _, t in self.docs) / max(1, self.N)
self.idf = {w: np.log((self.N - df + 0.5) / (df + 0.5) + 1.0) for w, df in self.df.items()}
def scores(self, query: str, k: int = 8):
qf = {}
for w in tokenize(query):
qf[w] = qf.get(w, 0) + 1
if not qf:
return np.zeros(self.N, dtype=np.float64)
scores = np.zeros(self.N, dtype=np.float64)
for i, (_, text) in enumerate(self.docs):
freq = {}
for w in tokenize(text):
freq[w] = freq.get(w, 0) + 1
dl = len(tokenize(text))
score = 0.0
for w, q in qf.items():
if w not in self.df:
continue
f = freq.get(w, 0)
if f == 0:
continue
x = (f * (self.k1 + 1)) / (f + self.k1 * (1 - self.k1 + self.k2 * dl / self.avgdl))
score += self.idf[w] * x * q
scores[i] = score
return scores
k1 = 1.5
k2 = 0.75
def cosine(a: np.ndarray, b: np.ndarray) -> float:
denom = max(1e-9, np.linalg.norm(a) * np.linalg.norm(b))
return float(a @ b / denom)
class VectorIndex:
"""Brute-force cosine over a matrix of embeddings (fine under ~50k docs)."""
def __init__(self, vectors: np.ndarray):
self.vectors = vectors.astype(np.float32)
self.row_norms = np.linalg.norm(self.vectors, axis=1, keepdims=True)
self.row_norms[self.row_norms == 0] = 1.0
def search(self, query_vec: np.ndarray, k: int = 8):
q = query_vec / (np.linalg.norm(query_vec) + 1e-9)
dots = self.vectors @ q
n = self.row_norms.ravel()
cos = dots / n
idx = np.argsort(cos)[::-1][:k]
return idx, cos[idx]