"""Targeted pre/post rules around the vendored normaliser (maya3.text.tn), derived from the eval rows that still failed with text normalisation on. The vendored copy stays byte-identical. Each rule rewrites one input shape that the strict normaliser either rejects (it then falls back to reading the span character by character: "15-08-25" -> "one five minus zero eight minus two five") or reads in a way listeners do not expect (a PIN code as "five lakh sixty thousand thirty eight"): R1 date_2y DD-MM-YY, DD/MM/YY, DD.MM.YY -> four-digit year (pivot: 00-50 -> 20xx, 51-99 -> 19xx), then the normaliser's own date reading R2 expiry MM/YY or MM/YYYY next to an expiry / validity / date word or a CVV -> " " R3 fy_range 2023-24 (second part = next year) -> " " R4 digit_groups 4111 2233 4455 6677, 98765 43210, 1800 123 4567 (>= 10 digits in space-separated groups) -> one digit run, which the normaliser reads digit by digit R5 pin_keyword PIN / पिन / ... followed by 4-6 digits -> digit by digit R6 pincode a bare 6-digit number (no currency or unit next to it, no grouping commas) -> digit by digit R7 version v4.5.2 -> "v four point five point two" (language's decimal word) R8 ta_date_year Tamil "ஆகஸ்ட் 15, 2025" -> "15 ஆகஸ்ட் 2025" (the normaliser keeps the year) R10 pct_range 3.5%-4% -> 3.5-4% (the normaliser's own percent range) R11 per_price ₹500/kg -> " K G" R12 per_annum 8.5% p.a. -> "P A" (en: "per annum") R9 paise rupee amounts with two decimals read " rupees paise" in every language (the tables of bn en gu kn ml mr pa ta had no paise word, so they read "point five zero rupees") post: P1 tag_comma a piece that ends in "," before an inline [tag] keeps its comma (the normaliser turns a trailing comma into a full stop, and the model then ends the sentence at the tag) P2 uri_dash "-" inside an e-mail / URL is spoken in the request language, not as the English word "dash" P3 uri_acronym lowercase acronym labels in an e-mail / URL (sbi, kyc, hdfc in hdfcbank, sbi in onlinesbi) are spelled letter by letter in the request language; other labels keep their English spelling Language-table corrections (Assamese lexicon, "per" units, sub-domain hosts) live in maya3.text.lexicon. """ from __future__ import annotations import re __all__ = ["pre", "post", "install", "RULES"] RULES = ("date_2y", "expiry", "fy_range", "digit_groups", "pin_keyword", "pincode", "version", "ta_date_year", "paise", "pct_range", "per_price", "per_annum", "tag_comma", "uri_dash", "uri_acronym") PAISE = {"en": "paise", "bn": "পয়সা", "gu": "પૈસા", "kn": "ಪೈಸೆ", "ml": "പൈസ", "mr": "पैसे", "pa": "ਪੈਸੇ", "ta": "பைசா"} # hi as or te already have one DASH = {"hi": "डैश", "mr": "डॅश", "bn": "ড্যাশ", "as": "ডেশ্ব", "gu": "ડૅશ", "pa": "ਡੈਸ਼", "or": "ଡ୍ୟାସ୍", "ta": "டாஷ்", "te": "డాష్", "kn": "ಡ್ಯಾಶ್", "ml": "ഡാഷ്"} # expiry / validity / date words (any language; matched case-insensitively in the 40 characters before the span) _EXPIRY_WORDS = [ "expiry", "expires", "expire", "exp", "valid", "validity", "thru", "expiration", "एक्सपायरी", "समाप्ति", "वैधता", "मुदत", "समाप्ती", "মেয়াদ", "মেয়াদ", "এক্সপায়ারি", "এক্সপায়ারি", "এক্সপায়াৰী", "ম্যাদ", "એક્સપાયરી", "સમાપ્તિ", "માન્યતા", "ਮਿਆਦ", "ਐਕਸਪਾਇਰੀ", "காலாவதி", "எக்ஸ்பைரி", "గడువు", "ఎక్స్‌పైరీ", "ఎక్స్పైరీ", "ಮುಕ್ತಾಯ", "ಅಂತಿಮ ದಿನಾಂಕ", "ಎಕ್ಸ್‌ಪೈರಿ", "കാലാവധി", "എക്സ്പയറി", "ମିଆଦ", "ଏକ୍ସପାୟାରୀ", "ସମାପ୍ତି", ] _DATE_WORDS = ["date", "तारीख", "तिथि", "दिनांक", "डेट", "তারিখ", "তাৰিখ", "તારીખ", "ડેટ", "ਤਾਰੀਖ਼", "ਤਾਰੀਖ", "ਡੇਟ", "தேதி", "டேட்", "తేదీ", "డేట్", "ದಿನಾಂಕ", "ಡೇಟ್", "തീയതി", "ഡേറ്റ്", "ତାରିଖ", "ଡେଟ୍"] _alt = lambda ws: "|".join(sorted(map(re.escape, ws), key=len, reverse=True)) _EXPIRY_RE = re.compile(_alt(_EXPIRY_WORDS), re.I) _DATE_RE = re.compile(_alt(_DATE_WORDS), re.I) _PIN_WORDS = ["pin", "पिन", "पिनकोड", "ಪಿನ್", "పిన్", "பின்", "പിൻ", "পিন", "ପିନ", "ਪਿੰਨ", "ਪਿਨ", "પિન"] _PIN_RE = re.compile(r"(?]*)?") # P3: lowercase acronym labels inside e-mail / URL hosts and paths are spelled letter by letter (sbi.co.in -> # "S B I dot co dot in", hdfcbank.com -> "H D F C bank dot com"); other labels keep their English spelling. _URI_ACRONYMS = {"sbi", "hdfc", "icici", "kyc", "irctc", "pnb", "rbi", "idbi", "npci", "uidai", "epfo", "mf", "lic", "upi", "gst", "sms", "atm", "emi", "bob", "boi", "uco", "iob", "nsdl", "cdsl", "pnr", "otp", "ifsc"} _URI_ACR_PREFIX = ("icici", "irctc", "uidai", "hdfc", "idbi", "npci", "epfo", "sbi", "pnb", "kyc", "rbi") _URI_ACR_SUFFIX = ("sbi", "mf") _SPELL_UNITS = {"kg", "km", "gm", "g", "ml", "ltr", "l"} # R11: read letter by letter (as the normaliser reads "15 kg") _installed = False def install(on: bool = True): """R9: add the paise word to the language tables that lack it (bn en gu kn ml mr pa ta), so the normaliser's own main+sub-unit path reads ₹95.41 as " rupees paise" (with its native scale words and case suffixes). The tables are module-level; install(False) removes the added words again (used only by rules=False, the eval-parity mode; do not mix rules=False with concurrent rules=True calls).""" global _installed if on == _installed: return from maya3.text.tn.normalize import language_table for code, word in PAISE.items(): sym = language_table(code).SYM if on: sym.setdefault("paisa", word) elif sym.get("paisa") == word: del sym["paisa"] _installed = on def _table(lang): from maya3.text.tn.normalize import language_table return language_table(lang) def _tn_id_context(L, before): """True when the normaliser itself will read the next number digit by digit (an ID keyword such as PIN / OTP / number in the three tokens before it); the rule then leaves the span alone so the normaliser's postposition sandhi ("...ಒಂದರಲ್ಲಿ") is kept.""" from maya3.text.tn import normalize as _N # the function; its module holds the keyword sets import sys mod = sys.modules["maya3.text.tn.normalize"] kws = mod._ID_KEYWORDS_EN | mod._OTP_INPUT_WORDS | {w.lower() for w in L.ID_KEYWORDS} toks = re.findall(r"[\w#\u0900-\u0DFF']+", before.replace("\u200c", "").lower())[-3:] return any(t in kws for t in toks) def _digits(L, s): return ", ".join(L.digit(int(c)) for c in s) def _year4(yy): y = int(yy) return 2000 + y if y <= 50 else 1900 + y def _decimal_word(L): w = L.decimal(1, "2") a, b = L.cardinal(1), L.digit(2) return w[len(a):len(w) - len(b)].strip() if w.startswith(a) and w.endswith(b) else "point" def pre(text: str, lang: str) -> str: """Apply R1-R8 outside e-mail / URL spans (those are identifiers, spelled by the normaliser).""" try: L = _table(lang) except Exception: return text out, pos = [], 0 for m in _URI_RE.finditer(text): out.append(_pre(text[pos:m.start()], lang, L)); out.append(m.group(0)); pos = m.end() out.append(_pre(text[pos:], lang, L)) return "".join(out) def _pre(text, lang, L): t = text # R10 percent ranges written with the sign twice: 3.5%-4% (the normaliser read "...percentminus four percent") t = re.sub(r"(?= 10 digits in total) def r4(m): groups = m.group(0).split() return "".join(groups) if sum(len(g) for g in groups) >= 10 else m.group(0) t = re.sub(r"(? 12) # second part > 12: not DD/MM, so MM/YY if not (strong or weak): return m.group(0) year = int(y) if len(y) == 4 else 2000 + int(y) return f"{L.month(mo)} {L.year(year)}" t = re.sub(r"(?]\s*$", before) # ID suffix (CUST-567890), #ref, maths or re.match(r"\s*[÷×*/+=<>^]", t[m.end():]) or _tn_id_context(L, re.split(r"[.!?;।॥\n]", t[max(0, m.start() - 60):m.start()])[-1])): return m.group(0) return _digits(L, m.group(0)) t = re.sub(r"(? "D Month YYYY" if lang == "ta": months = "|".join(_TA_MONTHS) t = re.sub(rf"(? str: # P1 keep a trailing comma that precedes an inline [tag] (disabled, see TAG_COMMA) if TAG_COMMA and before_tag and seg.rstrip().endswith(",") and out.rstrip().endswith("."): out = out.rstrip()[:-1] + "," # P2 spoken hyphen inside e-mail / URL in the request language uris = [m.group(0) for m in _URI_RE.finditer(seg)] if lang in DASH and uris: out = re.sub(r"(?<=\s)dash(?=\s)", DASH[lang], out) # P3 acronym labels inside e-mail / URL labels = {w for u in uris for w in re.findall(r"[a-z]+", re.sub(r"^https?://", "", u)) if w in _URI_ACRONYMS or w.startswith(_URI_ACR_PREFIX) or w.endswith(_URI_ACR_SUFFIX)} if labels: try: L = _table(lang) except Exception: return out spell = lambda w: " ".join(L.letter(c.upper()) for c in w) def p3(m): w = m.group(0) if w in _URI_ACRONYMS: return spell(w) for a in _URI_ACR_PREFIX: if w.startswith(a) and len(w) - len(a) >= 3: return spell(a) + " " + w[len(a):] for a in _URI_ACR_SUFFIX: if w.endswith(a) and len(w) - len(a) >= 4: return w[:-len(a)] + " " + spell(a) return w out = re.sub(r"(?