Spaces:
Sleeping
Sleeping
File size: 767 Bytes
81f3119 6142b3f 81f3119 6142b3f 81f3119 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 | import re
import nltk
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
def _ensure_nltk_data():
try:
stopwords.words("english")
except LookupError:
nltk.download("stopwords", quiet=True)
try:
nltk.data.find("corpora/wordnet")
except LookupError:
nltk.download("wordnet", quiet=True)
_ensure_nltk_data()
stop_words = set(stopwords.words("english"))
lemmatizer = WordNetLemmatizer()
def preprocess_text(text):
text = text.lower()
text = re.sub(r"[^a-z\s]", " ", text)
tokens = text.split()
cleaned_tokens = [
lemmatizer.lemmatize(token)
for token in tokens
if token not in stop_words and len(token) > 2
]
return " ".join(cleaned_tokens)
|