Spaces:
Sleeping
Sleeping
File size: 4,991 Bytes
fa1fead | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 | # embeddings/sparse_embedder.py
# WHY THIS EXISTS: BM25 sparse index for exact-match retrieval.
# Dense vectors miss exact keyword matches (section numbers, names).
# BM25 catches them. Combined with dense = hybrid retrieval. LAW 5.
# Indic normalization MUST run before BM25 indexing for Hindi text.
import re
import numpy as np
from rank_bm25 import BM25Okapi
from langdetect import detect, LangDetectException
from ingestion.indic_preprocessing import (
indic_preprocessing_pipeline,
ALL_INDIC_LANGUAGES,
)
def tokenize_for_bm25(text: str, lang_code: str = "en") -> list[str]:
"""
Tokenizes text for BM25 indexing.
For Indic languages: runs full normalization pipeline first.
For Latin: standard whitespace + punctuation tokenization.
This is why BM25 works for Hindi in SmartDocs —
indic_normalize runs before tokenization, ensuring
Unicode variants of the same word map to the same token.
Args:
text: Text to tokenize
lang_code: ISO language code
Returns:
List of tokens
"""
if not text:
return []
# For Indic languages — normalize first, then tokenize
if lang_code in ALL_INDIC_LANGUAGES:
result = indic_preprocessing_pipeline(
text=text,
lang_code=lang_code,
return_sentences=False,
)
normalized_text = result["cleaned_text"]
else:
normalized_text = text.lower()
# Remove punctuation and split on whitespace
normalized_text = re.sub(r"[^\w\s]", " ", normalized_text)
tokens = normalized_text.split()
# Remove empty tokens and single characters (except Indic scripts)
tokens = [
t for t in tokens
if len(t) > 1 or any(
"\u0900" <= c <= "\u0d7f" for c in t
)
]
return tokens
class SparseEmbedder:
"""
BM25 sparse index for a single document's chunks.
One SparseEmbedder instance per document per ingestion session.
Rebuilt from stored chunks on retrieval.
Works alongside DenseEmbedder in hybrid retrieval.
Dense weight: 0.7 | Sparse weight: 0.3 (from retrieval_config.yaml)
"""
def __init__(self, lang_code: str = "en"):
self.lang_code = lang_code
self.bm25: BM25Okapi | None = None
self.corpus_chunks: list[str] = []
self.tokenized_corpus: list[list[str]] = []
def build_index(self, chunks: list[str]) -> None:
"""
Builds BM25 index from list of text chunks.
Called during ingestion after chunking and indic preprocessing.
Indic normalization runs inside tokenize_for_bm25.
Args:
chunks: List of preprocessed text chunks
"""
if not chunks:
return
self.corpus_chunks = chunks
self.tokenized_corpus = [
tokenize_for_bm25(chunk, self.lang_code)
for chunk in chunks
]
# Filter out empty tokenized chunks
valid_indices = [
i for i, tokens in enumerate(self.tokenized_corpus)
if tokens
]
if not valid_indices:
return
valid_corpus = [self.tokenized_corpus[i] for i in valid_indices]
self.bm25 = BM25Okapi(valid_corpus)
self._valid_indices = valid_indices
def get_scores(self, query: str) -> np.ndarray:
"""
Returns BM25 scores for all chunks against query.
Query normalized using same pipeline as corpus.
Args:
query: User query text
Returns:
numpy array of BM25 scores, one per chunk
"""
if self.bm25 is None or not self.corpus_chunks:
return np.zeros(len(self.corpus_chunks))
query_tokens = tokenize_for_bm25(query, self.lang_code)
if not query_tokens:
return np.zeros(len(self.corpus_chunks))
raw_scores = self.bm25.get_scores(query_tokens)
# Map back to full corpus size
full_scores = np.zeros(len(self.corpus_chunks))
for score_idx, corpus_idx in enumerate(self._valid_indices):
if score_idx < len(raw_scores):
full_scores[corpus_idx] = raw_scores[score_idx]
return full_scores
def get_top_n(
self,
query: str,
n: int = 20,
) -> list[tuple[int, float, str]]:
"""
Returns top-N chunks by BM25 score.
Args:
query: User query
n: Number of results to return
Returns:
List of (chunk_index, score, chunk_text) tuples
"""
scores = self.get_scores(query)
if len(scores) == 0:
return []
top_indices = np.argsort(scores)[::-1][:n]
results = []
for idx in top_indices:
if scores[idx] > 0:
results.append((
int(idx),
float(scores[idx]),
self.corpus_chunks[idx],
))
return results |