"""Text normalisation for Maya3 (numbers, currency, dates, OTPs, URLs, ... -> spoken words, per language). from maya3.text import normalise_text normalise_text("Your OTP is 482913 [laughs]", "en") # 'Your OTP is four, eight, two, nine, one, three [laughs]' Same behaviour as the normalise_text wrapper the release evals used: * inline [tags] (e.g. [laughs], [sighs]) are protected: the text between them is normalised piece by piece; * ALL-CAPS Latin words that look like names (>= 5 letters with >= 2 vowels, or 4 letters with >= 2 vowels) are title-cased first, so the normaliser and the model read them as words; short / vowel-poor tokens (OTP, NEFT, HDFC, KYC) stay acronyms; * each piece goes through the bundled normaliser maya3.text.tn (pure Python, standard library only); * if the normaliser raises on a piece, that piece is kept unchanged (the eval wrapper's second fallback, a separate benchmark pipeline, is not part of the release); * whitespace is collapsed and the result stripped; * `rules=True` (default) adds the targeted pre/post rules of maya3.text.rules (two-digit-year dates, card expiry, FY ranges, spaced card/phone digits, PIN codes, version strings, rupees+paise in every language, the comma before an inline [tag]). `rules=False` is the exact behaviour of the release evals before these rules were added. `lang` is a Maya3 language code (as bn en gu hi kn ml mr or pa ta te); for an unknown code the normaliser raises, so the text comes back with only the caps fold and whitespace collapse applied. """ from __future__ import annotations import logging import re from maya3.text import lexicon as _lexicon from maya3.text import rules as _rules __all__ = ["normalise_text", "normalize_text", "SOURCE"] SOURCE = "Maya Research text normaliser, upstream commit fe5e6ec" _TAG = re.compile(r"(\[[a-z ]+\])") _CAPS = re.compile(r"\b[A-Z]{4,}\b") _VOWELS = set("AEIOU") _TN = None log = logging.getLogger("maya3.text") def _caps_fold(t): def f(m): w = m.group(0) if len(w) >= 5 and sum(ch in _VOWELS for ch in w) >= 2 or (len(w) == 4 and sum(ch in _VOWELS for ch in w) >= 2): return w.title() return w return _CAPS.sub(f, t) def _load_tn(): global _TN if _TN is None: from maya3.text.tn.normalize import normalize _TN = normalize return _TN def normalise_text(text: str, lang: str, rules: bool = True) -> str: tn = _load_tn() _rules.install(rules) _lexicon.install(rules) out = [] segs = _TAG.split(text) for i, seg in enumerate(segs): if not seg or _TAG.fullmatch(seg): out.append(seg) continue s = _caps_fold(seg) try: s2 = tn(_rules.pre(s, lang) if rules else s, lang) if rules: s2 = _rules.post(s2, seg, lang, before_tag=any(_TAG.fullmatch(x) for x in segs[i + 1:i + 2])) except Exception as exc: # never fail a request on normalisation: keep this piece as written log.debug("normaliser fallback (%s: %s)", type(exc).__name__, exc) s2 = s lead = seg[:len(seg) - len(seg.lstrip())] trail = seg[len(seg.rstrip()):] out.append(lead + s2.strip() + trail) return re.sub(r"\s+", " ", "".join(out)).strip() normalize_text = normalise_text