Spaces:
Sleeping
Sleeping
| import os | |
| import re | |
| from typing import List, Set, Dict | |
| class EnglishPreprocessor: | |
| """ | |
| English NLP Preprocessor with contraction expansion, lowercasing, | |
| punctuation cleaning, rule-based stemming, and stop-word filtering. | |
| """ | |
| CONTRACTIONS: Dict[str, str] = { | |
| "ain't": "am not", "aren't": "are not", "can't": "cannot", "can't've": "cannot have", | |
| "'cause": "because", "could've": "could have", "couldn't": "could not", | |
| "couldn't've": "could not have", "didn't": "did not", "doesn't": "does not", | |
| "don't": "do not", "hadn't": "had not", "hadn't've": "had not have", | |
| "hasn't": "has not", "haven't": "have not", "he'd": "he would", | |
| "he'd've": "he would have", "he'll": "he will", "he'll've": "he will have", | |
| "he's": "he is", "how'd": "how did", "how'd'y": "how do you", | |
| "how'll": "how will", "how's": "how is", "i'd": "i would", | |
| "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have", | |
| "i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would", | |
| "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have", | |
| "it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", | |
| "might've": "might have", "mightn't": "might not", "must've": "must have", | |
| "mustn't": "must not", "needn't": "need not", "o'clock": "of the clock", | |
| "oughtn't": "ought not", "shan't": "shall not", "sha'n't": "shall not", | |
| "she'd": "she would", "she'd've": "she would have", "she'll": "she will", | |
| "she's": "she is", "should've": "should have", "shouldn't": "should not", | |
| "that's": "that is", "there's": "there is", "they'd": "they would", | |
| "they'll": "they will", "they're": "they are", "they've": "they have", | |
| "wasn't": "was not", "we'd": "we would", "we'll": "we will", | |
| "we're": "we are", "we've": "we have", "weren't": "were not", | |
| "what'll": "what will", "what're": "what are", "what's": "what is", | |
| "what've": "what have", "when's": "when is", "where'd": "where did", | |
| "where's": "where is", "where've": "where have", "who'll": "who will", | |
| "who's": "who is", "who've": "who have", "why's": "why is", | |
| "won't": "will not", "would've": "would have", "wouldn't": "would not", | |
| "you'd": "you would", "you'll": "you will", "you're": "you are", | |
| "you've": "you have" | |
| } | |
| PUNCTUATION_REGEX = re.compile(r'[^a-zA-Z0-9\s]') | |
| def __init__(self, stopwords_file: str = None): | |
| self.stopwords: Set[str] = set() | |
| if stopwords_file and os.path.exists(stopwords_file): | |
| self.load_stopwords(stopwords_file) | |
| else: | |
| default_path = os.path.join(os.path.dirname(__file__), '..', 'data', 'english_stopwords.txt') | |
| if os.path.exists(default_path): | |
| self.load_stopwords(default_path) | |
| def load_stopwords(self, filepath: str): | |
| with open(filepath, 'r', encoding='utf-8') as f: | |
| for line in f: | |
| word = line.strip().lower() | |
| if word: | |
| self.stopwords.add(word) | |
| def expand_contractions(self, text: str) -> str: | |
| """Expands conversational contractions like don't -> do not.""" | |
| pattern = re.compile(r'\b(' + '|'.join(re.escape(k) for k in self.CONTRACTIONS.keys()) + r')\b', re.IGNORECASE) | |
| def replace(match): | |
| val = match.group(0).lower() | |
| return self.CONTRACTIONS.get(val, val) | |
| return pattern.sub(replace, text) | |
| def clean_punctuation(self, text: str) -> str: | |
| """Removes punctuation and special characters.""" | |
| return self.PUNCTUATION_REGEX.sub(' ', text) | |
| def simple_stem(self, word: str) -> str: | |
| """Rule-based suffix stripping for English words.""" | |
| word = word.lower() | |
| if len(word) <= 3: | |
| return word | |
| suffixes = [ | |
| ('sses', 'ss'), ('ies', 'i'), ('ational', 'ate'), | |
| ('tional', 'tion'), ('enci', 'ence'), ('anci', 'ance'), | |
| ('izer', 'ize'), ('bli', 'ble'), ('alli', 'al'), | |
| ('entli', 'ent'), ('eli', 'e'), ('ousli', 'ous'), | |
| ('ization', 'ize'), ('ation', 'ate'), ('ator', 'ate'), | |
| ('alism', 'al'), ('iveness', 'ive'), ('fulness', 'ful'), | |
| ('ousness', 'ous'), ('aliti', 'al'), ('iviti', 'ive'), | |
| ('biliti', 'ble'), ('logi', 'log'), ('ing', ''), | |
| ('ed', ''), ('es', ''), ('s', ''), ('ly', '') | |
| ] | |
| for suff, rep in suffixes: | |
| if word.endswith(suff) and len(word) - len(suff) >= 3: | |
| return word[:-len(suff)] + rep | |
| return word | |
| def preprocess_text(self, text: str, remove_stopwords: bool = True, stem: bool = False) -> str: | |
| """Full cleaning pipeline for English text.""" | |
| if not text: | |
| return "" | |
| text = self.expand_contractions(text) | |
| text = text.lower() | |
| text = self.clean_punctuation(text) | |
| words = text.split() | |
| processed_words = [] | |
| for w in words: | |
| if remove_stopwords and w in self.stopwords: | |
| continue | |
| if stem: | |
| w = self.simple_stem(w) | |
| if len(w) > 0: | |
| processed_words.append(w) | |
| return ' '.join(processed_words) | |
| def tokenize_words(self, text: str, remove_stopwords: bool = False, stem: bool = False) -> List[str]: | |
| """Tokenizes text into a list of words.""" | |
| cleaned = self.preprocess_text(text, remove_stopwords=remove_stopwords, stem=stem) | |
| return cleaned.split() | |