Spaces:
Sleeping
Sleeping
Download ingestion/indic_preprocessing.py from Sahilalaknur/smartdocs-api: direct link, hf CLI and curl.
- Browser
- Download file 12.2 kB
-
https://huggingface.co/spaces/Sahilalaknur/smartdocs-api/resolve/main/ingestion/indic_preprocessing.py
- Command line
-
hf download hf://spaces/Sahilalaknur/smartdocs-api/ingestion/indic_preprocessing.py
-
curl -L -o indic_preprocessing.py https://huggingface.co/spaces/Sahilalaknur/smartdocs-api/resolve/main/ingestion/indic_preprocessing.py
12.2 kB
| # ingestion/indic_preprocessing.py | |
| # WHY THIS EXISTS: Full 5-step Indic text normalization pipeline. | |
| # Every Hindi/Indic chunk passes through this before any downstream | |
| # processing. Skipping this = silent BM25 failures on Hindi text. | |
| # LAW 1 β non-negotiable. | |
| import re | |
| import os | |
| import unicodedata | |
| from pathlib import Path | |
| from langdetect import detect, LangDetectException | |
| # Set indic_nlp_resources path before importing | |
| INDIC_NLP_RESOURCES = str( | |
| Path(__file__).parent.parent / "indic_nlp_resources" | |
| ) | |
| os.environ["INDIC_RESOURCES_PATH"] = INDIC_NLP_RESOURCES | |
| from indicnlp import common | |
| common.set_resources_path(INDIC_NLP_RESOURCES) | |
| from indicnlp.normalize.indic_normalize import IndicNormalizerFactory | |
| from indicnlp.tokenize import sentence_tokenize | |
| from indicnlp.transliterate.unicode_transliterate import UnicodeIndicTransliterator | |
| # ββ Constants ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| DEVANAGARI_LANGUAGES = { | |
| "hi": "hi", # Hindi | |
| "mr": "mr", # Marathi | |
| "ne": "ne", # Nepali | |
| "sa": "sa", # Sanskrit | |
| "kok": "kok", # Konkani | |
| } | |
| DRAVIDIAN_LANGUAGES = { | |
| "ta": "ta", # Tamil | |
| "te": "te", # Telugu | |
| "kn": "kn", # Kannada | |
| "ml": "ml", # Malayalam | |
| } | |
| ALL_INDIC_LANGUAGES = {**DEVANAGARI_LANGUAGES, **DRAVIDIAN_LANGUAGES} | |
| # Zero-width characters that corrupt BM25 tokenization | |
| ZERO_WIDTH_CHARS = [ | |
| "\u200b", # Zero Width Space | |
| "\u200c", # Zero Width Non-Joiner | |
| "\u200d", # Zero Width Joiner | |
| "\u2060", # Word Joiner | |
| "\ufeff", # Byte Order Mark | |
| "\u00ad", # Soft Hyphen | |
| ] | |
| # Quality detection thresholds | |
| MIN_MEANINGFUL_CHARS_RATIO = 0.3 | |
| MIN_TEXT_LENGTH = 10 | |
| # ββ Normalizer factory cache ββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| _normalizer_cache: dict = {} | |
| def _get_normalizer(lang_code: str): | |
| """Returns cached normalizer for language code.""" | |
| if lang_code not in _normalizer_cache: | |
| factory = IndicNormalizerFactory() | |
| _normalizer_cache[lang_code] = factory.get_normalizer(lang_code) | |
| return _normalizer_cache[lang_code] | |
| # ββ Step 1: Indic Normalize βββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def indic_normalize(text: str, lang_code: str = "hi") -> str: | |
| """ | |
| Step 1 of Indic preprocessing pipeline. | |
| Normalizes Unicode representations of Indic scripts. | |
| Hindi has multiple Unicode representations for the same character. | |
| Without normalization, BM25 treats them as different tokens. | |
| Args: | |
| text: Raw text from PDF | |
| lang_code: ISO language code (default: "hi" for Hindi) | |
| Returns: | |
| Normalized text | |
| """ | |
| if not text or not text.strip(): | |
| return text | |
| # Only normalize Indic languages | |
| if lang_code not in ALL_INDIC_LANGUAGES: | |
| return unicodedata.normalize("NFC", text) | |
| try: | |
| normalizer = _get_normalizer(lang_code) | |
| normalized = normalizer.normalize(text) | |
| return normalized | |
| except Exception: | |
| # Fallback to Unicode NFC normalization | |
| return unicodedata.normalize("NFC", text) | |
| # ββ Step 2: Remove Zero Width Joiners ββββββββββββββββββββββββββββββββββββββββ | |
| def remove_zero_width_joiners(text: str) -> str: | |
| """ | |
| Step 2 of Indic preprocessing pipeline. | |
| Removes invisible Unicode characters that corrupt BM25 tokenization. | |
| PDFs generated from Hindi word processors are full of these. | |
| They are invisible in any text editor but break exact-match retrieval. | |
| Args: | |
| text: Text after indic_normalize | |
| Returns: | |
| Text with all zero-width characters removed | |
| """ | |
| if not text: | |
| return text | |
| for char in ZERO_WIDTH_CHARS: | |
| text = text.replace(char, "") | |
| return text | |
| # ββ Step 3: Normalize Devanagari Spaces ββββββββββββββββββββββββββββββββββββββ | |
| def normalize_devanagari_spaces(text: str) -> str: | |
| """ | |
| Step 3 of Indic preprocessing pipeline. | |
| Normalizes various space characters in Devanagari text. | |
| Hindi PDFs often contain non-standard space characters that | |
| prevent word boundary detection in BM25. | |
| Args: | |
| text: Text after remove_zero_width_joiners | |
| Returns: | |
| Text with normalized spaces | |
| """ | |
| if not text: | |
| return text | |
| # Normalize various Unicode space characters to standard space | |
| space_variants = [ | |
| "\u00a0", # Non-breaking space | |
| "\u1680", # Ogham space mark | |
| "\u2000", # En quad | |
| "\u2001", # Em quad | |
| "\u2002", # En space | |
| "\u2003", # Em space | |
| "\u2004", # Three-per-em space | |
| "\u2005", # Four-per-em space | |
| "\u2006", # Six-per-em space | |
| "\u2007", # Figure space | |
| "\u2008", # Punctuation space | |
| "\u2009", # Thin space | |
| "\u200a", # Hair space | |
| "\u202f", # Narrow no-break space | |
| "\u205f", # Medium mathematical space | |
| "\u3000", # Ideographic space | |
| ] | |
| for space in space_variants: | |
| text = text.replace(space, " ") | |
| # Collapse multiple spaces into single space | |
| text = re.sub(r" {2,}", " ", text) | |
| # Remove spaces before Devanagari punctuation | |
| text = re.sub(r" ([ΰ₯€ΰ₯₯,])", r"\1", text) | |
| return text.strip() | |
| # ββ Step 4: Detect and Flag Low Quality ββββββββββββββββββββββββββββββββββββββ | |
| def detect_and_flag_low_quality(text: str) -> tuple[str, bool]: | |
| """ | |
| Step 4 of Indic preprocessing pipeline. | |
| Detects low-quality text from PDF extraction β garbled characters, | |
| encoding errors, or mostly-whitespace chunks. | |
| Flags them so they can be excluded from embedding but still stored | |
| for audit purposes. | |
| Args: | |
| text: Text after normalize_devanagari_spaces | |
| Returns: | |
| tuple: (text, is_low_quality) | |
| is_low_quality=True means this chunk should not be embedded | |
| """ | |
| if not text or len(text.strip()) < MIN_TEXT_LENGTH: | |
| return text, True | |
| # Check ratio of meaningful characters | |
| total_chars = len(text) | |
| meaningful_chars = len(re.sub(r"[\s\n\t\r]", "", text)) | |
| if total_chars > 0: | |
| ratio = meaningful_chars / total_chars | |
| if ratio < MIN_MEANINGFUL_CHARS_RATIO: | |
| return text, True | |
| # Check for excessive replacement characters (encoding errors) | |
| replacement_count = text.count("\ufffd") | |
| if replacement_count > 5: | |
| return text, True | |
| # Check for garbled text (too many non-printable characters) | |
| non_printable = sum( | |
| 1 for c in text | |
| if unicodedata.category(c) in ("Cc", "Cf") and c not in "\n\t\r" | |
| ) | |
| if non_printable > len(text) * 0.1: | |
| return text, True | |
| return text, False | |
| # ββ Step 5: Sentence Tokenize Indic ββββββββββββββββββββββββββββββββββββββββββ | |
| def sentence_tokenize_indic(text: str, lang_code: str = "hi") -> list[str]: | |
| """ | |
| Step 5 of Indic preprocessing pipeline. | |
| Splits Indic text into clean sentences BEFORE chunking. | |
| Critical for Devanagari β standard sentence splitters miss | |
| Hindi sentence boundaries (ΰ₯€) and produce chunks that cut | |
| mid-sentence, destroying context for retrieval. | |
| Args: | |
| text: Text after detect_and_flag_low_quality | |
| lang_code: ISO language code | |
| Returns: | |
| List of clean sentences | |
| """ | |
| if not text or not text.strip(): | |
| return [] | |
| if lang_code not in ALL_INDIC_LANGUAGES: | |
| # Fallback for non-Indic text | |
| sentences = re.split(r"(?<=[.!?])\s+", text) | |
| return [s.strip() for s in sentences if s.strip()] | |
| try: | |
| sentences = sentence_tokenize.sentence_split(text, lang=lang_code) | |
| return [s.strip() for s in sentences if s.strip()] | |
| except Exception: | |
| # Fallback: split on Devanagari danda and standard punctuation | |
| sentences = re.split(r"[ΰ₯€ΰ₯₯.!?]+", text) | |
| return [s.strip() for s in sentences if s.strip()] | |
| # ββ Master Pipeline βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def indic_preprocessing_pipeline( | |
| text: str, | |
| lang_code: str = "hi", | |
| return_sentences: bool = False | |
| ) -> dict: | |
| """ | |
| Master pipeline β runs all 5 steps in order. | |
| Import this function into every ingestion function. | |
| Never call individual steps directly in production code. | |
| Pipeline: | |
| raw_text | |
| β indic_normalize(text) | |
| β remove_zero_width_joiners(text) | |
| β normalize_devanagari_spaces(text) | |
| β detect_and_flag_low_quality(text) | |
| β sentence_tokenize_indic(text) | |
| β cleaned_text | |
| Args: | |
| text: Raw text from PDF extraction | |
| lang_code: ISO language code (auto-detected if not provided) | |
| return_sentences: If True, returns list of sentences | |
| Returns: | |
| dict with keys: | |
| - cleaned_text: fully preprocessed text | |
| - sentences: list of sentences (if return_sentences=True) | |
| - lang_code: language code used | |
| - is_low_quality: True if chunk should not be embedded | |
| - steps_applied: list of steps that ran | |
| """ | |
| if not text: | |
| return { | |
| "cleaned_text": "", | |
| "sentences": [], | |
| "lang_code": lang_code, | |
| "is_low_quality": True, | |
| "steps_applied": [], | |
| } | |
| steps_applied = [] | |
| # Auto-detect language if not Indic β still run pipeline for safety | |
| detected_lang = lang_code | |
| try: | |
| detected_lang = detect(text[:500]) | |
| except LangDetectException: | |
| detected_lang = "en" | |
| # Use provided lang_code if it's Indic, otherwise use detected | |
| if lang_code not in ALL_INDIC_LANGUAGES: | |
| lang_code = detected_lang | |
| # Step 1 | |
| text = indic_normalize(text, lang_code) | |
| steps_applied.append("indic_normalize") | |
| # Step 2 | |
| text = remove_zero_width_joiners(text) | |
| steps_applied.append("remove_zero_width_joiners") | |
| # Step 3 | |
| text = normalize_devanagari_spaces(text) | |
| steps_applied.append("normalize_devanagari_spaces") | |
| # Step 4 | |
| text, is_low_quality = detect_and_flag_low_quality(text) | |
| steps_applied.append("detect_and_flag_low_quality") | |
| # Step 5 | |
| sentences = [] | |
| if return_sentences or lang_code in ALL_INDIC_LANGUAGES: | |
| sentences = sentence_tokenize_indic(text, lang_code) | |
| steps_applied.append("sentence_tokenize_indic") | |
| return { | |
| "cleaned_text": text, | |
| "sentences": sentences, | |
| "lang_code": lang_code, | |
| "is_low_quality": is_low_quality, | |
| "steps_applied": steps_applied, | |
| } | |
| def detect_script_type(text: str) -> str: | |
| """ | |
| Detects the script type of a text chunk. | |
| Used to set chunk metadata.script_type field. | |
| Returns: | |
| "devanagari" | "latin" | "dravidian" | "mixed" | |
| """ | |
| if not text: | |
| return "latin" | |
| devanagari_count = sum( | |
| 1 for c in text if "\u0900" <= c <= "\u097f" | |
| ) | |
| latin_count = sum( | |
| 1 for c in text if "\u0041" <= c <= "\u007a" | |
| ) | |
| dravidian_count = sum( | |
| 1 for c in text | |
| if ("\u0b80" <= c <= "\u0bff") # Tamil | |
| or ("\u0c00" <= c <= "\u0c7f") # Telugu | |
| or ("\u0c80" <= c <= "\u0cff") # Kannada | |
| or ("\u0d00" <= c <= "\u0d7f") # Malayalam | |
| ) | |
| total = max(devanagari_count + latin_count + dravidian_count, 1) | |
| if devanagari_count / total > 0.6: | |
| return "devanagari" | |
| elif dravidian_count / total > 0.6: | |
| return "dravidian" | |
| elif latin_count / total > 0.6: | |
| return "latin" | |
| else: | |
| return "mixed" |