"""Input normalisation shared by training, evaluation and the runtime. Every text the student ever sees (and every text the teacher is asked to label) goes through `normalize`: * NFKD, accents stripped, lower-cased * apostrophes inside words removed ("don't" -> "dont") * everything that is not [a-z0-9] -> space (punctuation, emoji, symbols, ...) * at most MAX_WORDS words kept - the *last* ones, because the words nearest the cursor carry the intent when someone is typing * each word capped at MAX_WORD_LEN characters """ from __future__ import annotations import re import unicodedata MAX_WORDS = 4 MAX_WORD_LEN = 20 _APOS = str.maketrans({"’": "'", "‘": "'", "´": "'", "`": "'"}) _INNER_APOS = re.compile(r"(?<=[a-z0-9])'(?=[a-z0-9])") _NON_ALNUM = re.compile(r"[^a-z0-9]+") def words_of(text: str) -> list[str]: """Normalised words of `text`, *without* the max-words truncation.""" t = unicodedata.normalize("NFKD", text.translate(_APOS)).lower() t = "".join(c for c in t if not unicodedata.combining(c)) t = _INNER_APOS.sub("", t) return [w[:MAX_WORD_LEN] for w in _NON_ALNUM.sub(" ", t).split()] def normalize(text: str, max_words: int = MAX_WORDS) -> str: return " ".join(words_of(text)[-max_words:])