Spaces:
Sleeping
Sleeping
Download embeddings/sparse_embedder.py from Sahilalaknur/smartdocs-api: direct link, hf CLI and curl.
- Browser
- Download file 4.99 kB
-
https://huggingface.co/spaces/Sahilalaknur/smartdocs-api/resolve/main/embeddings/sparse_embedder.py
- Command line
-
hf download hf://spaces/Sahilalaknur/smartdocs-api/embeddings/sparse_embedder.py
-
curl -L -o sparse_embedder.py https://huggingface.co/spaces/Sahilalaknur/smartdocs-api/resolve/main/embeddings/sparse_embedder.py
4.99 kB
| # embeddings/sparse_embedder.py | |
| # WHY THIS EXISTS: BM25 sparse index for exact-match retrieval. | |
| # Dense vectors miss exact keyword matches (section numbers, names). | |
| # BM25 catches them. Combined with dense = hybrid retrieval. LAW 5. | |
| # Indic normalization MUST run before BM25 indexing for Hindi text. | |
| import re | |
| import numpy as np | |
| from rank_bm25 import BM25Okapi | |
| from langdetect import detect, LangDetectException | |
| from ingestion.indic_preprocessing import ( | |
| indic_preprocessing_pipeline, | |
| ALL_INDIC_LANGUAGES, | |
| ) | |
| def tokenize_for_bm25(text: str, lang_code: str = "en") -> list[str]: | |
| """ | |
| Tokenizes text for BM25 indexing. | |
| For Indic languages: runs full normalization pipeline first. | |
| For Latin: standard whitespace + punctuation tokenization. | |
| This is why BM25 works for Hindi in SmartDocs — | |
| indic_normalize runs before tokenization, ensuring | |
| Unicode variants of the same word map to the same token. | |
| Args: | |
| text: Text to tokenize | |
| lang_code: ISO language code | |
| Returns: | |
| List of tokens | |
| """ | |
| if not text: | |
| return [] | |
| # For Indic languages — normalize first, then tokenize | |
| if lang_code in ALL_INDIC_LANGUAGES: | |
| result = indic_preprocessing_pipeline( | |
| text=text, | |
| lang_code=lang_code, | |
| return_sentences=False, | |
| ) | |
| normalized_text = result["cleaned_text"] | |
| else: | |
| normalized_text = text.lower() | |
| # Remove punctuation and split on whitespace | |
| normalized_text = re.sub(r"[^\w\s]", " ", normalized_text) | |
| tokens = normalized_text.split() | |
| # Remove empty tokens and single characters (except Indic scripts) | |
| tokens = [ | |
| t for t in tokens | |
| if len(t) > 1 or any( | |
| "\u0900" <= c <= "\u0d7f" for c in t | |
| ) | |
| ] | |
| return tokens | |
| class SparseEmbedder: | |
| """ | |
| BM25 sparse index for a single document's chunks. | |
| One SparseEmbedder instance per document per ingestion session. | |
| Rebuilt from stored chunks on retrieval. | |
| Works alongside DenseEmbedder in hybrid retrieval. | |
| Dense weight: 0.7 | Sparse weight: 0.3 (from retrieval_config.yaml) | |
| """ | |
| def __init__(self, lang_code: str = "en"): | |
| self.lang_code = lang_code | |
| self.bm25: BM25Okapi | None = None | |
| self.corpus_chunks: list[str] = [] | |
| self.tokenized_corpus: list[list[str]] = [] | |
| def build_index(self, chunks: list[str]) -> None: | |
| """ | |
| Builds BM25 index from list of text chunks. | |
| Called during ingestion after chunking and indic preprocessing. | |
| Indic normalization runs inside tokenize_for_bm25. | |
| Args: | |
| chunks: List of preprocessed text chunks | |
| """ | |
| if not chunks: | |
| return | |
| self.corpus_chunks = chunks | |
| self.tokenized_corpus = [ | |
| tokenize_for_bm25(chunk, self.lang_code) | |
| for chunk in chunks | |
| ] | |
| # Filter out empty tokenized chunks | |
| valid_indices = [ | |
| i for i, tokens in enumerate(self.tokenized_corpus) | |
| if tokens | |
| ] | |
| if not valid_indices: | |
| return | |
| valid_corpus = [self.tokenized_corpus[i] for i in valid_indices] | |
| self.bm25 = BM25Okapi(valid_corpus) | |
| self._valid_indices = valid_indices | |
| def get_scores(self, query: str) -> np.ndarray: | |
| """ | |
| Returns BM25 scores for all chunks against query. | |
| Query normalized using same pipeline as corpus. | |
| Args: | |
| query: User query text | |
| Returns: | |
| numpy array of BM25 scores, one per chunk | |
| """ | |
| if self.bm25 is None or not self.corpus_chunks: | |
| return np.zeros(len(self.corpus_chunks)) | |
| query_tokens = tokenize_for_bm25(query, self.lang_code) | |
| if not query_tokens: | |
| return np.zeros(len(self.corpus_chunks)) | |
| raw_scores = self.bm25.get_scores(query_tokens) | |
| # Map back to full corpus size | |
| full_scores = np.zeros(len(self.corpus_chunks)) | |
| for score_idx, corpus_idx in enumerate(self._valid_indices): | |
| if score_idx < len(raw_scores): | |
| full_scores[corpus_idx] = raw_scores[score_idx] | |
| return full_scores | |
| def get_top_n( | |
| self, | |
| query: str, | |
| n: int = 20, | |
| ) -> list[tuple[int, float, str]]: | |
| """ | |
| Returns top-N chunks by BM25 score. | |
| Args: | |
| query: User query | |
| n: Number of results to return | |
| Returns: | |
| List of (chunk_index, score, chunk_text) tuples | |
| """ | |
| scores = self.get_scores(query) | |
| if len(scores) == 0: | |
| return [] | |
| top_indices = np.argsort(scores)[::-1][:n] | |
| results = [] | |
| for idx in top_indices: | |
| if scores[idx] > 0: | |
| results.append(( | |
| int(idx), | |
| float(scores[idx]), | |
| self.corpus_chunks[idx], | |
| )) | |
| return results |