"""normalize() from indic_stt/text.py, copied verbatim so the MLX runner needs no PyTorch: NFC, lower case, no punctuation, single spaces. The same function scored every reported number. Keep it equal to indic_stt/text.py (export_mlx.py verify checks).""" from __future__ import annotations import re import unicodedata # Danda, double danda, ASCII and common Unicode punctuation. Hyphens become spaces; apostrophes # inside words (don't, it's) survive via _INNER_APOS, all others become spaces. _PUNCT = re.compile(r"[।॥!\"#$%&'()*+,\-./:;<=>?@\[\\\]^_`{|}~‘’“”–—… ]") _SPACE = re.compile(r"\s+") _INNER_APOS = re.compile(r"(?<=\w)['’](?=\w)") _KEEP = "" # private-use placeholder for a protected apostrophe _NUKTA = "़" # Precomposed nukta letters -> base letter (क़ ख़ ग़ ज़ ड़ ढ़ फ़ य़) _NUKTA_FOLD = str.maketrans({"क़": "क", "ख़": "ख", "ग़": "ग", "ज़": "ज", "ड़": "ड", "ढ़": "ढ", "फ़": "फ", "य़": "य"}) def normalize(text: str, fold_nukta: bool = False) -> str: """Training target / scoring normalization: NFC, no punctuation, single spaces. fold_nukta=True also maps nukta letters to their base letter (क़ -> क). Use it for a "lenient" WER that ignores the nukta spelling variation common in Hindi transcripts. """ text = unicodedata.normalize("NFC", text).lower() # lower() only changes cased scripts (English) text = _INNER_APOS.sub(_KEEP, text) text = _PUNCT.sub(" ", text).replace(_KEEP, "'") if fold_nukta: text = unicodedata.normalize("NFD", text).replace(_NUKTA, "") text = unicodedata.normalize("NFC", text).translate(_NUKTA_FOLD) return _SPACE.sub(" ", text).strip()