File size: 1,270 Bytes
c394b65
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
"""Input normalisation shared by training, evaluation and the runtime.

Every text the student ever sees (and every text the teacher is asked to label)
goes through `normalize`:

  * NFKD, accents stripped, lower-cased
  * apostrophes inside words removed        ("don't" -> "dont")
  * everything that is not [a-z0-9] -> space (punctuation, emoji, symbols, ...)
  * at most MAX_WORDS words kept - the *last* ones, because the words nearest
    the cursor carry the intent when someone is typing
  * each word capped at MAX_WORD_LEN characters
"""

from __future__ import annotations

import re
import unicodedata

MAX_WORDS = 4
MAX_WORD_LEN = 20

_APOS = str.maketrans({"’": "'", "‘": "'", "´": "'", "`": "'"})
_INNER_APOS = re.compile(r"(?<=[a-z0-9])'(?=[a-z0-9])")
_NON_ALNUM = re.compile(r"[^a-z0-9]+")


def words_of(text: str) -> list[str]:
    """Normalised words of `text`, *without* the max-words truncation."""
    t = unicodedata.normalize("NFKD", text.translate(_APOS)).lower()
    t = "".join(c for c in t if not unicodedata.combining(c))
    t = _INNER_APOS.sub("", t)
    return [w[:MAX_WORD_LEN] for w in _NON_ALNUM.sub(" ", t).split()]


def normalize(text: str, max_words: int = MAX_WORDS) -> str:
    return " ".join(words_of(text)[-max_words:])