junafinity's picture
claude opus 5.5
VeriSafe-Tanglish-1-STT (STT-2)
ec01e2d
Raw History Blame Contribute Delete
1.76 kB
"""normalize() from indic_stt/text.py, copied verbatim so the MLX runner needs no PyTorch: NFC, lower case, no punctuation,
single spaces. The same function scored every reported number. Keep it equal to indic_stt/text.py (export_mlx.py verify checks)."""
from __future__ import annotations
import re
import unicodedata
# Danda, double danda, ASCII and common Unicode punctuation. Hyphens become spaces; apostrophes
# inside words (don't, it's) survive via _INNER_APOS, all others become spaces.
_PUNCT = re.compile(r"[।॥!\"#$%&'()*+,\-./:;<=>?@\[\\\]^_`{|}~‘’“”–—… ]")
_SPACE = re.compile(r"\s+")
_INNER_APOS = re.compile(r"(?<=\w)['’](?=\w)")
_KEEP = "" # private-use placeholder for a protected apostrophe
_NUKTA = "़"
# Precomposed nukta letters -> base letter (क़ ख़ ग़ ज़ ड़ ढ़ फ़ य़)
_NUKTA_FOLD = str.maketrans({"क़": "क", "ख़": "ख", "ग़": "ग", "ज़": "ज",
"ड़": "ड", "ढ़": "ढ", "फ़": "फ", "य़": "य"})
def normalize(text: str, fold_nukta: bool = False) -> str:
"""Training target / scoring normalization: NFC, no punctuation, single spaces.
fold_nukta=True also maps nukta letters to their base letter (क़ -> क). Use it for a
"lenient" WER that ignores the nukta spelling variation common in Hindi transcripts.
"""
text = unicodedata.normalize("NFC", text).lower() # lower() only changes cased scripts (English)
text = _INNER_APOS.sub(_KEEP, text)
text = _PUNCT.sub(" ", text).replace(_KEEP, "'")
if fold_nukta:
text = unicodedata.normalize("NFD", text).replace(_NUKTA, "")
text = unicodedata.normalize("NFC", text).translate(_NUKTA_FOLD)
return _SPACE.sub(" ", text).strip()