File size: 230 Bytes
2db7cb0 | 1 2 3 4 5 6 7 8 | from nltk.stem import WordNetLemmatizer
from nltk.tokenize import word_tokenize
lemmatizer = WordNetLemmatizer()
def preprocess(text):
tokens = word_tokenize(text.lower())
return [lemmatizer.lemmatize(t) for t in tokens] |