Download normalize.py from anudit/tinyemo: direct link, hf CLI and curl.
- Browser
- Download file 1.27 kB
-
https://huggingface.co/anudit/tinyemo/resolve/main/normalize.py
- Command line
-
hf download hf://anudit/tinyemo/normalize.py
-
curl -L -o normalize.py https://huggingface.co/anudit/tinyemo/resolve/main/normalize.py
1.27 kB
| """Input normalisation shared by training, evaluation and the runtime. | |
| Every text the student ever sees (and every text the teacher is asked to label) | |
| goes through `normalize`: | |
| * NFKD, accents stripped, lower-cased | |
| * apostrophes inside words removed ("don't" -> "dont") | |
| * everything that is not [a-z0-9] -> space (punctuation, emoji, symbols, ...) | |
| * at most MAX_WORDS words kept - the *last* ones, because the words nearest | |
| the cursor carry the intent when someone is typing | |
| * each word capped at MAX_WORD_LEN characters | |
| """ | |
| from __future__ import annotations | |
| import re | |
| import unicodedata | |
| MAX_WORDS = 4 | |
| MAX_WORD_LEN = 20 | |
| _APOS = str.maketrans({"’": "'", "‘": "'", "´": "'", "`": "'"}) | |
| _INNER_APOS = re.compile(r"(?<=[a-z0-9])'(?=[a-z0-9])") | |
| _NON_ALNUM = re.compile(r"[^a-z0-9]+") | |
| def words_of(text: str) -> list[str]: | |
| """Normalised words of `text`, *without* the max-words truncation.""" | |
| t = unicodedata.normalize("NFKD", text.translate(_APOS)).lower() | |
| t = "".join(c for c in t if not unicodedata.combining(c)) | |
| t = _INNER_APOS.sub("", t) | |
| return [w[:MAX_WORD_LEN] for w in _NON_ALNUM.sub(" ", t).split()] | |
| def normalize(text: str, max_words: int = MAX_WORDS) -> str: | |
| return " ".join(words_of(text)[-max_words:]) | |