File size: 1,270 Bytes
c394b65 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 | """Input normalisation shared by training, evaluation and the runtime.
Every text the student ever sees (and every text the teacher is asked to label)
goes through `normalize`:
* NFKD, accents stripped, lower-cased
* apostrophes inside words removed ("don't" -> "dont")
* everything that is not [a-z0-9] -> space (punctuation, emoji, symbols, ...)
* at most MAX_WORDS words kept - the *last* ones, because the words nearest
the cursor carry the intent when someone is typing
* each word capped at MAX_WORD_LEN characters
"""
from __future__ import annotations
import re
import unicodedata
MAX_WORDS = 4
MAX_WORD_LEN = 20
_APOS = str.maketrans({"’": "'", "‘": "'", "´": "'", "`": "'"})
_INNER_APOS = re.compile(r"(?<=[a-z0-9])'(?=[a-z0-9])")
_NON_ALNUM = re.compile(r"[^a-z0-9]+")
def words_of(text: str) -> list[str]:
"""Normalised words of `text`, *without* the max-words truncation."""
t = unicodedata.normalize("NFKD", text.translate(_APOS)).lower()
t = "".join(c for c in t if not unicodedata.combining(c))
t = _INNER_APOS.sub("", t)
return [w[:MAX_WORD_LEN] for w in _NON_ALNUM.sub(" ", t).split()]
def normalize(text: str, max_words: int = MAX_WORDS) -> str:
return " ".join(words_of(text)[-max_words:])
|