Spaces:
Running on Zero
Running on Zero
| import re | |
| from typing import List, Tuple | |
| from .language_detector import LanguageDetector | |
| from .arabic_preprocessor import ArabicPreprocessor | |
| from .english_preprocessor import EnglishPreprocessor | |
| class BilingualTokenizer: | |
| """ | |
| Unified Bilingual Tokenizer handling both Arabic and English sentence | |
| and word level segmentation. | |
| """ | |
| # Protected English abbreviations to prevent erroneous sentence splitting | |
| ABBREVIATIONS = r'(?:Dr|Mr|Mrs|Ms|Prof|Inc|Ltd|Corp|Co|e\.g|i\.e|vs|etc|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Oct|Nov|Dec)' | |
| def __init__(self): | |
| self.lang_detector = LanguageDetector() | |
| self.arabic_preprocessor = ArabicPreprocessor() | |
| self.english_preprocessor = EnglishPreprocessor() | |
| def split_sentences(self, text: str, lang: str = None) -> List[str]: | |
| """ | |
| Splits raw text into individual sentences while preserving | |
| meaningful punctuation and handling abbreviations. | |
| """ | |
| if not text or not text.strip(): | |
| return [] | |
| if lang is None: | |
| lang = self.lang_detector.detect_language(text) | |
| # Standardize paragraph breaks | |
| cleaned_text = re.sub(r'\r\n|\r', '\n', text) | |
| cleaned_text = re.sub(r'\n+', '\n', cleaned_text) | |
| # Handle sentence boundaries | |
| if lang == 'ar': | |
| # Arabic sentence delimiters: . ! ؟ ؛ \n | |
| # Use regex looking for delimiters followed by space or newline or end of text | |
| pattern = r'(?<=[\.\!\؟\؛\n])\s+' | |
| raw_sentences = re.split(pattern, cleaned_text) | |
| else: | |
| # English sentence segmentation protecting abbreviations | |
| # Protect abbreviations with temporary placeholder | |
| temp_text = cleaned_text | |
| abbr_matches = re.findall(rf'\b{self.ABBREVIATIONS}\.', temp_text, flags=re.IGNORECASE) | |
| placeholder_map = {} | |
| for i, abbr in enumerate(set(abbr_matches)): | |
| placeholder = f"__ABBR_{i}__" | |
| placeholder_map[placeholder] = abbr | |
| temp_text = temp_text.replace(abbr, placeholder) | |
| # English sentence delimiters: . ! ? \n | |
| pattern = r'(?<=[\.\!\?\n])\s+' | |
| raw_sentences = re.split(pattern, temp_text) | |
| # Restore abbreviations | |
| restored_sentences = [] | |
| for s in raw_sentences: | |
| for placeholder, orig in placeholder_map.items(): | |
| s = s.replace(placeholder, orig) | |
| restored_sentences.append(s) | |
| raw_sentences = restored_sentences | |
| # Final filtering of empty or whitespace-only sentences | |
| sentences = [s.strip() for s in raw_sentences if s.strip() and len(s.strip()) > 5] | |
| return sentences | |
| def tokenize_words(self, text: str, lang: str = None, remove_stopwords: bool = False, stem: bool = False) -> List[str]: | |
| """ | |
| Tokenizes words with appropriate preprocessing for the detected or specified language. | |
| """ | |
| if lang is None: | |
| lang = self.lang_detector.detect_language(text) | |
| if lang == 'ar': | |
| return self.arabic_preprocessor.tokenize_words(text, remove_stopwords=remove_stopwords, stem=stem) | |
| else: | |
| return self.english_preprocessor.tokenize_words(text, remove_stopwords=remove_stopwords, stem=stem) | |
| def preprocess_sentence(self, sentence: str, lang: str = None, remove_stopwords: bool = True, stem: bool = False) -> str: | |
| """ | |
| Cleans and pre-processes a sentence string for vectorization/feature extraction. | |
| """ | |
| if lang is None: | |
| lang = self.lang_detector.detect_language(sentence) | |
| if lang == 'ar': | |
| return self.arabic_preprocessor.preprocess_text(sentence, remove_stopwords=remove_stopwords, stem=stem) | |
| else: | |
| return self.english_preprocessor.preprocess_text(sentence, remove_stopwords=remove_stopwords, stem=stem) | |