Spaces:
Sleeping
Sleeping
File size: 652 Bytes
03f844d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 | import nltk
from nltk.corpus import stopwords
from nltk.stem.porter import PorterStemmer
import string
def textPreprocessor(text):
stemmer = PorterStemmer()
# convert to lower case
text = text.lower()
# tokenize the text to individual words
text = nltk.word_tokenize(text)
# remove special characters and convert
lst = []
for word in text:
# stopwords : which helps in formation of sentences and has no special meaning
if word.isalnum() and word not in stopwords.words("english") and word not in string.punctuation:
lst.append(stemmer.stem(word))
text = " ".join(lst)
return text
|