| """Transliteration normalisation shared by the recognition target and the translation input. |
| |
| Rundata-style editorial markup is stripped: section labels, word separators, lacunae and |
| Latin text in braces become token boundaries; brackets, parentheses, bind-rune carets and |
| uncertainty marks are deleted inside words. What remains is a lowercase string over the |
| transliteration alphabet, with R (yr) kept distinct from r. |
| """ |
| import re |
| import unicodedata |
|
|
| ALPHABET = set("abcdefghijklmnopqrstuvwxyzþðæøąïŋöåäR") |
|
|
| _BRACES = re.compile(r"\{[^}]*\}") |
| _SECTION = re.compile(r"§\s?[A-Za-z]{1,3}\b|\b[Ss]ide\s+[A-Za-z]\b|\$|¶|‖|\|") |
| _DELETE = re.compile(r"[\[\]()<>^?´`ˈ]") |
| _BOUNDARY = re.compile(r"[·•:;×÷+*'\"/\\=\-–—…!,.#°†¤_᛫᛬⁝│~0-9]") |
|
|
|
|
| def normalize(text): |
| if not isinstance(text, str): |
| return "" |
| s = unicodedata.normalize("NFC", text) |
| s = _BRACES.sub(" ", s) |
| s = _SECTION.sub(" ", s) |
| s = s.replace("ʀ", "R").replace("ᴀ", "a") |
| s = _DELETE.sub("", s) |
| s = _BOUNDARY.sub(" ", s) |
| tokens = [] |
| for tok in s.split(): |
| tok = "".join(ch if ch == "R" else ch.lower() for ch in tok) |
| if all(ch in ALPHABET for ch in tok): |
| tokens.append(tok) |
| return " ".join(tokens) |
|
|
|
|
| _T_CODES = re.compile(r"†\s?[$A-Z]?|§\s?[A-Z]{1,2}\b|[$¶|{}*]") |
| _T_UNCERTAIN = re.compile(r"\s?\(\?\)") |
| _T_BRACKETS = re.compile(r"[\[\]<>]") |
| _T_ELLIPSIS = re.compile(r"(?:\.\.\.|…)(?:\s*(?:\.\.\.|…))*") |
|
|
|
|
| def clean_translation(text): |
| """Strip Rundata editorial markup from a translation, keeping its wording.""" |
| if not isinstance(text, str): |
| return "" |
| s = unicodedata.normalize("NFC", text) |
| s = _T_CODES.sub(" ", s) |
| s = _T_UNCERTAIN.sub("", s) |
| s = _T_BRACKETS.sub("", s) |
| s = _T_ELLIPSIS.sub("...", s) |
| s = re.sub(r"\s+", " ", s).strip() |
| s = re.sub(r"\s+([,;:!?])", r"\1", s) |
| s = re.sub(r"\s+\.(?!\.)", ".", s) |
| return s |
|
|
