tinyemo / normalize.py
anudit's picture
Release tinyemo model and inference code
c394b65 verified
Raw History Blame Contribute Delete
1.27 kB
"""Input normalisation shared by training, evaluation and the runtime.
Every text the student ever sees (and every text the teacher is asked to label)
goes through `normalize`:
* NFKD, accents stripped, lower-cased
* apostrophes inside words removed ("don't" -> "dont")
* everything that is not [a-z0-9] -> space (punctuation, emoji, symbols, ...)
* at most MAX_WORDS words kept - the *last* ones, because the words nearest
the cursor carry the intent when someone is typing
* each word capped at MAX_WORD_LEN characters
"""
from __future__ import annotations
import re
import unicodedata
MAX_WORDS = 4
MAX_WORD_LEN = 20
_APOS = str.maketrans({"’": "'", "‘": "'", "´": "'", "`": "'"})
_INNER_APOS = re.compile(r"(?<=[a-z0-9])'(?=[a-z0-9])")
_NON_ALNUM = re.compile(r"[^a-z0-9]+")
def words_of(text: str) -> list[str]:
"""Normalised words of `text`, *without* the max-words truncation."""
t = unicodedata.normalize("NFKD", text.translate(_APOS)).lower()
t = "".join(c for c in t if not unicodedata.combining(c))
t = _INNER_APOS.sub("", t)
return [w[:MAX_WORD_LEN] for w in _NON_ALNUM.sub(" ", t).split()]
def normalize(text: str, max_words: int = MAX_WORDS) -> str:
return " ".join(words_of(text)[-max_words:])