Spaces:
Sleeping
Sleeping
File size: 3,964 Bytes
3d9ba5b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 | import re
from typing import List, Tuple
from .language_detector import LanguageDetector
from .arabic_preprocessor import ArabicPreprocessor
from .english_preprocessor import EnglishPreprocessor
class BilingualTokenizer:
"""
Unified Bilingual Tokenizer handling both Arabic and English sentence
and word level segmentation.
"""
# Protected English abbreviations to prevent erroneous sentence splitting
ABBREVIATIONS = r'(?:Dr|Mr|Mrs|Ms|Prof|Inc|Ltd|Corp|Co|e\.g|i\.e|vs|etc|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Oct|Nov|Dec)'
def __init__(self):
self.lang_detector = LanguageDetector()
self.arabic_preprocessor = ArabicPreprocessor()
self.english_preprocessor = EnglishPreprocessor()
def split_sentences(self, text: str, lang: str = None) -> List[str]:
"""
Splits raw text into individual sentences while preserving
meaningful punctuation and handling abbreviations.
"""
if not text or not text.strip():
return []
if lang is None:
lang = self.lang_detector.detect_language(text)
# Standardize paragraph breaks
cleaned_text = re.sub(r'\r\n|\r', '\n', text)
cleaned_text = re.sub(r'\n+', '\n', cleaned_text)
# Handle sentence boundaries
if lang == 'ar':
# Arabic sentence delimiters: . ! ؟ ؛ \n
# Use regex looking for delimiters followed by space or newline or end of text
pattern = r'(?<=[\.\!\؟\؛\n])\s+'
raw_sentences = re.split(pattern, cleaned_text)
else:
# English sentence segmentation protecting abbreviations
# Protect abbreviations with temporary placeholder
temp_text = cleaned_text
abbr_matches = re.findall(rf'\b{self.ABBREVIATIONS}\.', temp_text, flags=re.IGNORECASE)
placeholder_map = {}
for i, abbr in enumerate(set(abbr_matches)):
placeholder = f"__ABBR_{i}__"
placeholder_map[placeholder] = abbr
temp_text = temp_text.replace(abbr, placeholder)
# English sentence delimiters: . ! ? \n
pattern = r'(?<=[\.\!\?\n])\s+'
raw_sentences = re.split(pattern, temp_text)
# Restore abbreviations
restored_sentences = []
for s in raw_sentences:
for placeholder, orig in placeholder_map.items():
s = s.replace(placeholder, orig)
restored_sentences.append(s)
raw_sentences = restored_sentences
# Final filtering of empty or whitespace-only sentences
sentences = [s.strip() for s in raw_sentences if s.strip() and len(s.strip()) > 5]
return sentences
def tokenize_words(self, text: str, lang: str = None, remove_stopwords: bool = False, stem: bool = False) -> List[str]:
"""
Tokenizes words with appropriate preprocessing for the detected or specified language.
"""
if lang is None:
lang = self.lang_detector.detect_language(text)
if lang == 'ar':
return self.arabic_preprocessor.tokenize_words(text, remove_stopwords=remove_stopwords, stem=stem)
else:
return self.english_preprocessor.tokenize_words(text, remove_stopwords=remove_stopwords, stem=stem)
def preprocess_sentence(self, sentence: str, lang: str = None, remove_stopwords: bool = True, stem: bool = False) -> str:
"""
Cleans and pre-processes a sentence string for vectorization/feature extraction.
"""
if lang is None:
lang = self.lang_detector.detect_language(sentence)
if lang == 'ar':
return self.arabic_preprocessor.preprocess_text(sentence, remove_stopwords=remove_stopwords, stem=stem)
else:
return self.english_preprocessor.preprocess_text(sentence, remove_stopwords=remove_stopwords, stem=stem)
|