Private review: measured five-language release, Hindi sentence mode and custom GGUFs
a449848 verified | """Text normalisation, Unicode validation and tokenisation shared by data prep, baselines and eval.""" | |
| import re | |
| import unicodedata | |
| ZW = "\u200c\u200d\u200b\ufeff\u00ad" # ZWNJ, ZWJ, ZWSP, BOM, soft hyphen | |
| _ZW_RE = re.compile(f"[{ZW}]") | |
| _WS_RE = re.compile(r"\s+") | |
| # Devanagari character classes | |
| CONS = set(chr(c) for c in range(0x0915, 0x093A)) | set(chr(c) for c in range(0x0958, 0x0960)) | set("ॹॺॻॼॽॾॿ") | |
| INDEP_V = set(chr(c) for c in range(0x0904, 0x0915)) | set("ॠॡॲॳॴॵॶॷ") | |
| MATRA = set(chr(c) for c in range(0x093E, 0x094D)) | set("ॢॣॏॕॖॗ\u093a\u093b\u094e") | |
| VIRAMA = "\u094d" | |
| NUKTA = "\u093c" | |
| NASAL = set("\u0901\u0902") # chandrabindu, anusvara | |
| VISARGA = "\u0903" | |
| DEVA_DIGITS = "०१२३४५६७८९" | |
| DEVA_RANGE = re.compile(r"[\u0900-\u097F]") | |
| DANDA = "।॥" | |
| def nfc(s: str) -> str: | |
| return unicodedata.normalize("NFC", s) | |
| def is_malformed(s: str) -> bool: | |
| """True if the string contains characters we treat as malformed/garbage: | |
| replacement char, private use, surrogates, unassigned, or control chars other than tab/newline.""" | |
| for ch in s: | |
| if ch == "\ufffd": | |
| return True | |
| cat = unicodedata.category(ch) | |
| if cat in ("Co", "Cs", "Cn"): | |
| return True | |
| if cat == "Cc" and ch not in "\t\n": | |
| return True | |
| return False | |
| def clean(s: str) -> str: | |
| """NFC + strip zero-width/format chars + collapse whitespace.""" | |
| s = nfc(s) | |
| s = _ZW_RE.sub("", s) | |
| s = "".join(ch for ch in s if unicodedata.category(ch) != "Cf") | |
| return _WS_RE.sub(" ", s).strip() | |
| def valid_devanagari_word(w: str) -> bool: | |
| """Orthographic sanity check for a single Devanagari word (letters only). | |
| Rejects: non-Devanagari chars, leading combining marks, matra/virama/nukta after a non-consonant, | |
| two dependent vowel signs in a row, nukta not on a consonant, dangling virama+matra.""" | |
| if not w: | |
| return False | |
| prev = None | |
| for i, ch in enumerate(w): | |
| if not ("\u0900" <= ch <= "\u097F"): | |
| return False | |
| if ch in DEVA_DIGITS or ch in DANDA: | |
| return False | |
| if ch in MATRA or ch == VIRAMA or ch == NUKTA: | |
| if prev is None: | |
| return False | |
| if ch == NUKTA and prev not in CONS: | |
| return False | |
| if ch in MATRA and not (prev in CONS or prev == NUKTA): | |
| return False | |
| if ch == VIRAMA and not (prev in CONS or prev == NUKTA): | |
| return False | |
| if ch in NASAL or ch == VISARGA: | |
| if prev is None or prev in NASAL or prev == VIRAMA: | |
| return False | |
| prev = ch | |
| return True | |
| def valid_roman_word(w: str) -> bool: | |
| return bool(re.fullmatch(r"[a-z]+", w)) | |
| # --- tokenisation ----------------------------------------------------------- | |
| # Words (Latin or Devanagari letters incl. marks), numbers, URLs/emails/handles, punctuation. | |
| TOKEN_RE = re.compile( | |
| r"(https?://\S+|www\.\S+|[\w.+-]+@[\w-]+\.[\w.]+|[@#][\w_]+" | |
| r"|[A-Za-z]+(?:'[A-Za-z]+)?" | |
| r"|[\u0900-\u0963\u0971-\u097F]+" | |
| r"|[0-9\u0966-\u096F]+(?:[.,:][0-9\u0966-\u096F]+)*" | |
| r"|[\u0964\u0965]|[^\sA-Za-z0-9\u0900-\u097F])" | |
| ) | |
| def tokenize(s: str): | |
| return TOKEN_RE.findall(s) | |
| PUNCT_RE = re.compile(r"[^\w\u0900-\u097F]+", re.UNICODE) | |
| # --- scoring normalisation -------------------------------------------------- | |
| _DIGIT_TR = str.maketrans(DEVA_DIGITS, "0123456789") | |
| def norm_for_score(s: str) -> str: | |
| """'norm' scoring view: NFC, no zero-width chars, Devanagari digits -> ASCII, punctuation removed | |
| (incl. danda), whitespace collapsed. Spelling is otherwise untouched (strict on letters).""" | |
| s = clean(s).translate(_DIGIT_TR) | |
| s = re.sub(r"[^\w\s\u0900-\u097F]|[\u0964\u0965]", " ", s) | |
| return _WS_RE.sub(" ", s).strip() | |
| _FOLD_MAP = { | |
| "\u0901": "\u0902", # chandrabindu -> anusvara (हूँ ~ हूं) | |
| NUKTA: "", # drop nukta (ज़ ~ ज) | |
| "\u0945": "\u0947", # candra e (ॅ) -> e | |
| "\u0949": "\u094b", # candra o (ॉ) -> o (ऑफिस ~ ओफिस) | |
| "\u0911": "\u0913", # independent candra O -> O | |
| "\u090d": "\u090f", | |
| } | |
| def fold(s: str) -> str: | |
| """'fold' scoring view (variant-tolerant, reported separately): norm_for_score + nukta dropped, | |
| chandrabindu->anusvara, candra vowels -> plain. Never used as the primary metric.""" | |
| s = unicodedata.normalize("NFD", norm_for_score(s)) # decompose precomposed nukta letters | |
| s = "".join(_FOLD_MAP.get(ch, ch) for ch in s) | |
| return unicodedata.normalize("NFC", s) | |