File size: 1,950 Bytes
ed46c5d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 | """Transliteration normalisation shared by the recognition target and the translation input.
Rundata-style editorial markup is stripped: section labels, word separators, lacunae and
Latin text in braces become token boundaries; brackets, parentheses, bind-rune carets and
uncertainty marks are deleted inside words. What remains is a lowercase string over the
transliteration alphabet, with R (yr) kept distinct from r.
"""
import re
import unicodedata
ALPHABET = set("abcdefghijklmnopqrstuvwxyzþðæøąïŋöåäR")
_BRACES = re.compile(r"\{[^}]*\}")
_SECTION = re.compile(r"§\s?[A-Za-z]{1,3}\b|\b[Ss]ide\s+[A-Za-z]\b|\$|¶|‖|\|")
_DELETE = re.compile(r"[\[\]()<>^?´`ˈ]")
_BOUNDARY = re.compile(r"[·•:;×÷+*'\"/\\=\-–—…!,.#°†¤_᛫᛬⁝│~0-9]")
def normalize(text):
if not isinstance(text, str):
return ""
s = unicodedata.normalize("NFC", text)
s = _BRACES.sub(" ", s)
s = _SECTION.sub(" ", s)
s = s.replace("ʀ", "R").replace("ᴀ", "a")
s = _DELETE.sub("", s)
s = _BOUNDARY.sub(" ", s)
tokens = []
for tok in s.split():
tok = "".join(ch if ch == "R" else ch.lower() for ch in tok)
if all(ch in ALPHABET for ch in tok):
tokens.append(tok)
return " ".join(tokens)
_T_CODES = re.compile(r"†\s?[$A-Z]?|§\s?[A-Z]{1,2}\b|[$¶|{}*]")
_T_UNCERTAIN = re.compile(r"\s?\(\?\)")
_T_BRACKETS = re.compile(r"[\[\]<>]")
_T_ELLIPSIS = re.compile(r"(?:\.\.\.|…)(?:\s*(?:\.\.\.|…))*")
def clean_translation(text):
"""Strip Rundata editorial markup from a translation, keeping its wording."""
if not isinstance(text, str):
return ""
s = unicodedata.normalize("NFC", text)
s = _T_CODES.sub(" ", s)
s = _T_UNCERTAIN.sub("", s)
s = _T_BRACKETS.sub("", s)
s = _T_ELLIPSIS.sub("...", s)
s = re.sub(r"\s+", " ", s).strip()
s = re.sub(r"\s+([,;:!?])", r"\1", s)
s = re.sub(r"\s+\.(?!\.)", ".", s)
return s
|