Download humpback2/embedding.py from gregrsea/HumpBack: direct link, hf CLI and curl.
- Browser
- Download file 3.62 kB
-
https://huggingface.co/gregrsea/HumpBack/resolve/main/humpback2/embedding.py
- Command line
-
hf download hf://gregrsea/HumpBack/humpback2/embedding.py
-
curl -L -o embedding.py https://huggingface.co/gregrsea/HumpBack/resolve/main/humpback2/embedding.py
3.62 kB
| """Vector index + lexical retriever. | |
| v2 has no pretrained transformers on the target device, so the vector index is a | |
| deterministic hash embedding (no download required). Retrieval ranks beliefs by a | |
| weighted blend of cosine(vector) and BM25(lexical overlap) so "price of item X" | |
| matches by subject words, not by flaky embedding similarity. | |
| """ | |
| import hashlib | |
| import re | |
| import numpy as np | |
| TOKEN_RE = re.compile(r"[a-z0-9]+") | |
| STOPWORDS = { | |
| "what", "is", "are", "was", "were", "the", "a", "an", "of", "in", "to", | |
| "for", "and", "or", "it", "this", "that", "my", "your", "by", "on", "at" | |
| } | |
| # Stable per-process hash (avoid Python's salted hash). Use a small vector dim. | |
| DIM = 384 | |
| def tokenize(text: str): | |
| words = TOKEN_RE.findall(text.lower()) | |
| filtered = [w for w in words if w not in STOPWORDS] | |
| return filtered if filtered else words | |
| def _h(word: str, dim: int = DIM) -> int: | |
| return int(hashlib.md5(word.encode("utf-8")).hexdigest(), 16) % dim | |
| def embed(text: str, dim: int = DIM) -> np.ndarray: | |
| """Deterministic dense vector from word hashes; zero vector for empty text.""" | |
| vec = np.zeros(dim, dtype=np.float32) | |
| ids = tokenize(text) | |
| if not ids: | |
| return vec | |
| counts = {} | |
| for w in ids: | |
| counts[w] = counts.get(w, 0) + 1 | |
| n = float(len(ids)) | |
| for w, c in counts.items(): | |
| d = _h(w) | |
| vec[d] += c / (n ** 0.5) # length-normalise-ish | |
| return vec | |
| class BM25: | |
| """Standard BM25 over a small corpus (pure numpy, no extra deps).""" | |
| def __init__(self, docs): | |
| self.docs = docs # list of (id, text) | |
| self.df = {} | |
| self.avgdl = 0.0 | |
| self.N = len(docs) | |
| for _, text in self.docs: | |
| for w in set(tokenize(text)): | |
| self.df[w] = self.df.get(w, 0) + 1 | |
| self.avgdl = sum(len(tokenize(t)) for _, t in self.docs) / max(1, self.N) | |
| self.idf = {w: np.log((self.N - df + 0.5) / (df + 0.5) + 1.0) for w, df in self.df.items()} | |
| def scores(self, query: str, k: int = 8): | |
| qf = {} | |
| for w in tokenize(query): | |
| qf[w] = qf.get(w, 0) + 1 | |
| if not qf: | |
| return np.zeros(self.N, dtype=np.float64) | |
| scores = np.zeros(self.N, dtype=np.float64) | |
| for i, (_, text) in enumerate(self.docs): | |
| freq = {} | |
| for w in tokenize(text): | |
| freq[w] = freq.get(w, 0) + 1 | |
| dl = len(tokenize(text)) | |
| score = 0.0 | |
| for w, q in qf.items(): | |
| if w not in self.df: | |
| continue | |
| f = freq.get(w, 0) | |
| if f == 0: | |
| continue | |
| x = (f * (self.k1 + 1)) / (f + self.k1 * (1 - self.k1 + self.k2 * dl / self.avgdl)) | |
| score += self.idf[w] * x * q | |
| scores[i] = score | |
| return scores | |
| k1 = 1.5 | |
| k2 = 0.75 | |
| def cosine(a: np.ndarray, b: np.ndarray) -> float: | |
| denom = max(1e-9, np.linalg.norm(a) * np.linalg.norm(b)) | |
| return float(a @ b / denom) | |
| class VectorIndex: | |
| """Brute-force cosine over a matrix of embeddings (fine under ~50k docs).""" | |
| def __init__(self, vectors: np.ndarray): | |
| self.vectors = vectors.astype(np.float32) | |
| self.row_norms = np.linalg.norm(self.vectors, axis=1, keepdims=True) | |
| self.row_norms[self.row_norms == 0] = 1.0 | |
| def search(self, query_vec: np.ndarray, k: int = 8): | |
| q = query_vec / (np.linalg.norm(query_vec) + 1e-9) | |
| dots = self.vectors @ q | |
| n = self.row_norms.ravel() | |
| cos = dots / n | |
| idx = np.argsort(cos)[::-1][:k] | |
| return idx, cos[idx] | |