File size: 3,619 Bytes
fbb4123
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
"""Vector index + lexical retriever.

v2 has no pretrained transformers on the target device, so the vector index is a
deterministic hash embedding (no download required). Retrieval ranks beliefs by a
weighted blend of cosine(vector) and BM25(lexical overlap) so "price of item X"
matches by subject words, not by flaky embedding similarity.
"""

import hashlib
import re
import numpy as np

TOKEN_RE = re.compile(r"[a-z0-9]+")
STOPWORDS = {
    "what", "is", "are", "was", "were", "the", "a", "an", "of", "in", "to",
    "for", "and", "or", "it", "this", "that", "my", "your", "by", "on", "at"
}

# Stable per-process hash (avoid Python's salted hash). Use a small vector dim.
DIM = 384


def tokenize(text: str):
    words = TOKEN_RE.findall(text.lower())
    filtered = [w for w in words if w not in STOPWORDS]
    return filtered if filtered else words


def _h(word: str, dim: int = DIM) -> int:
    return int(hashlib.md5(word.encode("utf-8")).hexdigest(), 16) % dim


def embed(text: str, dim: int = DIM) -> np.ndarray:
    """Deterministic dense vector from word hashes; zero vector for empty text."""
    vec = np.zeros(dim, dtype=np.float32)
    ids = tokenize(text)
    if not ids:
        return vec
    counts = {}
    for w in ids:
        counts[w] = counts.get(w, 0) + 1
    n = float(len(ids))
    for w, c in counts.items():
        d = _h(w)
        vec[d] += c / (n ** 0.5)  # length-normalise-ish
    return vec


class BM25:
    """Standard BM25 over a small corpus (pure numpy, no extra deps)."""

    def __init__(self, docs):
        self.docs = docs  # list of (id, text)
        self.df = {}
        self.avgdl = 0.0
        self.N = len(docs)
        for _, text in self.docs:
            for w in set(tokenize(text)):
                self.df[w] = self.df.get(w, 0) + 1
        self.avgdl = sum(len(tokenize(t)) for _, t in self.docs) / max(1, self.N)
        self.idf = {w: np.log((self.N - df + 0.5) / (df + 0.5) + 1.0) for w, df in self.df.items()}

    def scores(self, query: str, k: int = 8):
        qf = {}
        for w in tokenize(query):
            qf[w] = qf.get(w, 0) + 1
        if not qf:
            return np.zeros(self.N, dtype=np.float64)
        scores = np.zeros(self.N, dtype=np.float64)
        for i, (_, text) in enumerate(self.docs):
            freq = {}
            for w in tokenize(text):
                freq[w] = freq.get(w, 0) + 1
            dl = len(tokenize(text))
            score = 0.0
            for w, q in qf.items():
                if w not in self.df:
                    continue
                f = freq.get(w, 0)
                if f == 0:
                    continue
                x = (f * (self.k1 + 1)) / (f + self.k1 * (1 - self.k1 + self.k2 * dl / self.avgdl))
                score += self.idf[w] * x * q
            scores[i] = score
        return scores

    k1 = 1.5
    k2 = 0.75


def cosine(a: np.ndarray, b: np.ndarray) -> float:
    denom = max(1e-9, np.linalg.norm(a) * np.linalg.norm(b))
    return float(a @ b / denom)


class VectorIndex:
    """Brute-force cosine over a matrix of embeddings (fine under ~50k docs)."""

    def __init__(self, vectors: np.ndarray):
        self.vectors = vectors.astype(np.float32)
        self.row_norms = np.linalg.norm(self.vectors, axis=1, keepdims=True)
        self.row_norms[self.row_norms == 0] = 1.0

    def search(self, query_vec: np.ndarray, k: int = 8):
        q = query_vec / (np.linalg.norm(query_vec) + 1e-9)
        dots = self.vectors @ q
        n = self.row_norms.ravel()
        cos = dots / n
        idx = np.argsort(cos)[::-1][:k]
        return idx, cos[idx]