| """Preprocessing for the keyword intent classifier. | |
| Ship this module alongside keyword_intent_model.pkl. The pickled | |
| TfidfVectorizer references `intent_preprocess.preprocess_text`, so this | |
| module must be importable (on sys.path) before joblib.load(). | |
| """ | |
| import re | |
| import string | |
| def preprocess_text(text): | |
| """Must match the preprocessing used at training time.""" | |
| if text is None: | |
| return "" | |
| text = str(text).lower().strip() | |
| text = re.sub(r"\s+", " ", text) | |
| # keep '+', '#', '.' (e.g. "c++", "c#", ".net"); strip other punctuation | |
| punctuation_to_remove = ( | |
| string.punctuation.replace("+", "").replace("#", "").replace(".", "") | |
| ) | |
| text = text.translate(str.maketrans("", "", punctuation_to_remove)) | |
| return text.strip() | |