smartdocs-api / embeddings /sparse_embedder.py
Sahilalaknur's picture
Clean HF API deploy (no binary history)
fa1fead
Raw History Blame Contribute Delete
4.99 kB
# embeddings/sparse_embedder.py
# WHY THIS EXISTS: BM25 sparse index for exact-match retrieval.
# Dense vectors miss exact keyword matches (section numbers, names).
# BM25 catches them. Combined with dense = hybrid retrieval. LAW 5.
# Indic normalization MUST run before BM25 indexing for Hindi text.
import re
import numpy as np
from rank_bm25 import BM25Okapi
from langdetect import detect, LangDetectException
from ingestion.indic_preprocessing import (
indic_preprocessing_pipeline,
ALL_INDIC_LANGUAGES,
)
def tokenize_for_bm25(text: str, lang_code: str = "en") -> list[str]:
"""
Tokenizes text for BM25 indexing.
For Indic languages: runs full normalization pipeline first.
For Latin: standard whitespace + punctuation tokenization.
This is why BM25 works for Hindi in SmartDocs —
indic_normalize runs before tokenization, ensuring
Unicode variants of the same word map to the same token.
Args:
text: Text to tokenize
lang_code: ISO language code
Returns:
List of tokens
"""
if not text:
return []
# For Indic languages — normalize first, then tokenize
if lang_code in ALL_INDIC_LANGUAGES:
result = indic_preprocessing_pipeline(
text=text,
lang_code=lang_code,
return_sentences=False,
)
normalized_text = result["cleaned_text"]
else:
normalized_text = text.lower()
# Remove punctuation and split on whitespace
normalized_text = re.sub(r"[^\w\s]", " ", normalized_text)
tokens = normalized_text.split()
# Remove empty tokens and single characters (except Indic scripts)
tokens = [
t for t in tokens
if len(t) > 1 or any(
"\u0900" <= c <= "\u0d7f" for c in t
)
]
return tokens
class SparseEmbedder:
"""
BM25 sparse index for a single document's chunks.
One SparseEmbedder instance per document per ingestion session.
Rebuilt from stored chunks on retrieval.
Works alongside DenseEmbedder in hybrid retrieval.
Dense weight: 0.7 | Sparse weight: 0.3 (from retrieval_config.yaml)
"""
def __init__(self, lang_code: str = "en"):
self.lang_code = lang_code
self.bm25: BM25Okapi | None = None
self.corpus_chunks: list[str] = []
self.tokenized_corpus: list[list[str]] = []
def build_index(self, chunks: list[str]) -> None:
"""
Builds BM25 index from list of text chunks.
Called during ingestion after chunking and indic preprocessing.
Indic normalization runs inside tokenize_for_bm25.
Args:
chunks: List of preprocessed text chunks
"""
if not chunks:
return
self.corpus_chunks = chunks
self.tokenized_corpus = [
tokenize_for_bm25(chunk, self.lang_code)
for chunk in chunks
]
# Filter out empty tokenized chunks
valid_indices = [
i for i, tokens in enumerate(self.tokenized_corpus)
if tokens
]
if not valid_indices:
return
valid_corpus = [self.tokenized_corpus[i] for i in valid_indices]
self.bm25 = BM25Okapi(valid_corpus)
self._valid_indices = valid_indices
def get_scores(self, query: str) -> np.ndarray:
"""
Returns BM25 scores for all chunks against query.
Query normalized using same pipeline as corpus.
Args:
query: User query text
Returns:
numpy array of BM25 scores, one per chunk
"""
if self.bm25 is None or not self.corpus_chunks:
return np.zeros(len(self.corpus_chunks))
query_tokens = tokenize_for_bm25(query, self.lang_code)
if not query_tokens:
return np.zeros(len(self.corpus_chunks))
raw_scores = self.bm25.get_scores(query_tokens)
# Map back to full corpus size
full_scores = np.zeros(len(self.corpus_chunks))
for score_idx, corpus_idx in enumerate(self._valid_indices):
if score_idx < len(raw_scores):
full_scores[corpus_idx] = raw_scores[score_idx]
return full_scores
def get_top_n(
self,
query: str,
n: int = 20,
) -> list[tuple[int, float, str]]:
"""
Returns top-N chunks by BM25 score.
Args:
query: User query
n: Number of results to return
Returns:
List of (chunk_index, score, chunk_text) tuples
"""
scores = self.get_scores(query)
if len(scores) == 0:
return []
top_indices = np.argsort(scores)[::-1][:n]
results = []
for idx in top_indices:
if scores[idx] > 0:
results.append((
int(idx),
float(scores[idx]),
self.corpus_chunks[idx],
))
return results