Spaces:
Sleeping
Sleeping
| import re | |
| import nltk | |
| from nltk.corpus import stopwords | |
| from nltk.stem import WordNetLemmatizer | |
| def _ensure_nltk_data(): | |
| try: | |
| stopwords.words("english") | |
| except LookupError: | |
| nltk.download("stopwords", quiet=True) | |
| try: | |
| nltk.data.find("corpora/wordnet") | |
| except LookupError: | |
| nltk.download("wordnet", quiet=True) | |
| _ensure_nltk_data() | |
| stop_words = set(stopwords.words("english")) | |
| lemmatizer = WordNetLemmatizer() | |
| def preprocess_text(text): | |
| text = text.lower() | |
| text = re.sub(r"[^a-z\s]", " ", text) | |
| tokens = text.split() | |
| cleaned_tokens = [ | |
| lemmatizer.lemmatize(token) | |
| for token in tokens | |
| if token not in stop_words and len(token) > 2 | |
| ] | |
| return " ".join(cleaned_tokens) | |