File size: 772 Bytes
1f4627a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 | """Preprocessing for the keyword intent classifier.
Ship this module alongside keyword_intent_model.pkl. The pickled
TfidfVectorizer references `intent_preprocess.preprocess_text`, so this
module must be importable (on sys.path) before joblib.load().
"""
import re
import string
def preprocess_text(text):
"""Must match the preprocessing used at training time."""
if text is None:
return ""
text = str(text).lower().strip()
text = re.sub(r"\s+", " ", text)
# keep '+', '#', '.' (e.g. "c++", "c#", ".net"); strip other punctuation
punctuation_to_remove = (
string.punctuation.replace("+", "").replace("#", "").replace(".", "")
)
text = text.translate(str.maketrans("", "", punctuation_to_remove))
return text.strip()
|